acidvegas/eris
acidvegas
/
eris
1
Fork 0
Code Issues Pull Requests Activity

Certstream ingestion improved, still need to test stripping nulls and refactor the index mapping so we only store whats needed

This commit is contained in:
Dionysus 2024-03-06 14:38:34 -05:00
parent 90d6260b99
commit fd617e3c9d
Signed by: acidvegas
GPG Key ID: EF4B922DB85DC9DE
1 changed files with 6 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
ingestors/ingest_certs.py
View File

@ -101,26 +101,28 @@ async def process_data(file_path: str = None):
while True:
try:
async with websockets.connect('wss://certstream.calidog.io/') as websocket:
while True:
line = await websocket.recv()
if line == '~eof': # Sentinel value to indicate the end of a process (Used with --watch with FIFO)
break
try:
record = json.loads(line)
except json.decoder.JSONDecodeError:
logging.error(f'Failed to parse JSON record from Certstream! ({line})')
input('Press Enter to continue...') # Pause the script to allow the user to read the error message
input('Press Enter to continue...')
continue
yield record
except websockets.ConnectionClosed:
logging.error('Connection to Certstream was closed. Attempting to reconnect...')
await asyncio.sleep(10)
await asyncio.sleep(15)
except Exception as e:
logging.error(f'An error occurred while processing Certstream records! ({e})')
await asyncio.sleep(10)
await asyncio.sleep(15)
async def strip_struct_empty(data: dict) -> dict: