acidvegas/eris
acidvegas
/
eris
1
Fork 0
Code Issues Pull Requests Activity

Added zone field too better querying on specific tlds/zones

This commit is contained in:
Dionysus 2024-03-23 14:31:39 -04:00
parent 78d7556f09
commit a53541c328
Signed by: acidvegas
GPG Key ID: EF4B922DB85DC9DE
1 changed files with 5 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
ingestors/ingest_zone.py
View File

@ -29,6 +29,7 @@ def construct_map() -> dict:
'mappings': {
'properties': {
'domain' : keyword_mapping,
'zone' : { 'type': 'keyword' },
'records' : { 'type': 'nested', 'properties': {} },
'source' : { 'type': 'keyword' },
'seen' : { 'type': 'date' }
@ -62,6 +63,9 @@ async def process_data(file_path: str):
# Initialize the cache
last = None
# Determine the zone name from the file path (e.g., /path/to/zones/com.eu.txt -> com.eu zone)
zone = '.'.join(file_path.split('/')[-1].split('.')[:-1])
# Read the input file line by line
async for line in input_file:
line = line.strip()
@ -129,6 +133,7 @@ async def process_data(file_path: str):
'_index' : default_index,
'_doc' : {
'domain' : domain,
'zone' : zone,
'records' : {record_type: [{'data': data, 'ttl': ttl}]},
'source' : 'czds',
'seen' : time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()) # Zone files do not contain a timestamp, so we use the current time