diff --git a/async_dev/eris.py b/async_dev/eris.py index e2ecfbe..a3d37cf 100644 --- a/async_dev/eris.py +++ b/async_dev/eris.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # Elasticsearch Recon Ingestion Scripts (ERIS) - Developed by Acidvegas (https://git.acid.vegas/eris) -# eris.py [asyncronous developement] +# eris.py import asyncio import argparse diff --git a/async_dev/ingestors/ingest_certs.py b/async_dev/ingestors/ingest_certs.py index 40404d4..36dccd4 100644 --- a/async_dev/ingestors/ingest_certs.py +++ b/async_dev/ingestors/ingest_certs.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # Elasticsearch Recon Ingestion Scripts (ERIS) - Developed by Acidvegas (https://git.acid.vegas/eris) -# ingest_certstream.py +# ingest_certs.py import asyncio import json @@ -123,6 +123,35 @@ async def process_data(file_path: str = None): await asyncio.sleep(10) +async def strip_struct_empty(data: dict) -> dict: + ''' + Recursively remove empty values from a nested dictionary or list. + + :param data: The dictionary or list to clean. + ''' + + empties = [None, '', [], {}] + + if isinstance(data, dict): + for key, value in list(data.items()): + if value in empties: + del data[key] + else: + cleaned_value = strip_struct_empty(value) + if cleaned_value in empties: + del data[key] + else: + data[key] = cleaned_value + + return data + + elif isinstance(data, list): + return [strip_struct_empty(item) for item in data if item not in empties and strip_struct_empty(item) not in empties] + + else: + return data + + ''' Example record: