acidvegas/eris
acidvegas/eris
1
Fork 0
Code Issues Pull Requests Activity

Certstream ingestion improved, still need to test stripping nulls and refactor the index mapping so we only store whats needed

Browse Source
This commit is contained in:
Dionysus 2024-03-06 14:38:34 -05:00
parent 90d6260b99
commit fd617e3c9d
Signed by: acidvegas
GPG Key ID: EF4B922DB85DC9DE
1 changed files with 6 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
ingestors/ingest_certs.py
View File

@ -101,26 +101,28 @@ async def process_data(file_path: str = None):
while True:
try:
async with websockets.connect('wss://certstream.calidog.io/') as websocket:
while True:
line = await websocket.recv()
if line == '~eof': # Sentinel value to indicate the end of a process (Used with --watch with FIFO)
break
try:
record = json.loads(line)
except json.decoder.JSONDecodeError:
logging.error(f'Failed to parse JSON record from Certstream! ({line})')
input('Press Enter to continue...') # Pause the script to allow the user to read the error message
input('Press Enter to continue...')
continue
yield record
except websockets.ConnectionClosed:
logging.error('Connection to Certstream was closed. Attempting to reconnect...')
await asyncio.sleep(10)
await asyncio.sleep(15)
except Exception as e:
logging.error(f'An error occurred while processing Certstream records! ({e})')
await asyncio.sleep(10)
await asyncio.sleep(15)
async def strip_struct_empty(data: dict) -> dict: