acidvegas/eris
acidvegas/eris
1
Fork 0
Code Issues Pull Requests Activity

Records stored as a nested type for better querying, added source field so we can identify where the zone data derived (icann, axfr, breach, etc)

Browse Source
This commit is contained in:
Dionysus 2024-03-23 13:46:38 -04:00
parent 603d005a47
commit 78d7556f09
Signed by: acidvegas
GPG Key ID: EF4B922DB85DC9DE
1 changed files with 9 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
ingestors/ingest_zone.py
View File

@ -12,7 +12,7 @@ except ImportError:
# Set a default elasticsearch index if one is not provided # Set a default elasticsearch index if one is not provided
default_index = 'dns-zones' default_index = 'eris-zones'
# Known DNS record types found in zone files # Known DNS record types found in zone files
record_types = ('a','aaaa','caa','cdnskey','cds','cname','dnskey','ds','mx','naptr','ns','nsec','nsec3','nsec3param','ptr','rrsig','rp','sshfp','soa','srv','txt','type65534') record_types = ('a','aaaa','caa','cdnskey','cds','cname','dnskey','ds','mx','naptr','ns','nsec','nsec3','nsec3param','ptr','rrsig','rp','sshfp','soa','srv','txt','type65534')
@ -29,7 +29,8 @@ def construct_map() -> dict:
'mappings': { 'mappings': {
'properties': { 'properties': {
'domain' : keyword_mapping, 'domain' : keyword_mapping,
'records' : { 'properties': {} }, 'records' : { 'type': 'nested', 'properties': {} },
'source' : { 'type': 'keyword' },
'seen' : { 'type': 'date' } 'seen' : { 'type': 'date' }
} }
} }
@ -39,6 +40,7 @@ def construct_map() -> dict:
for record_type in record_types: for record_type in record_types:
if record_type in ('a','aaaa'): if record_type in ('a','aaaa'):
mapping['mappings']['properties']['records']['properties'][record_type] = { mapping['mappings']['properties']['records']['properties'][record_type] = {
'type' : 'nested',
'properties' : { 'properties' : {
'data' : { 'type': 'ip' if record_type in ('a','aaaa') else keyword_mapping }, 'data' : { 'type': 'ip' if record_type in ('a','aaaa') else keyword_mapping },
'ttl' : { 'type': 'integer' } 'ttl' : { 'type': 'integer' }
@ -127,7 +129,8 @@ async def process_data(file_path: str):
'_index' : default_index, '_index' : default_index,
'_doc' : { '_doc' : {
'domain' : domain, 'domain' : domain,
'records' : {record_type: [{'ttl': ttl, 'data': data}]}, 'records' : {record_type: [{'data': data, 'ttl': ttl}]},
'source' : 'czds',
'seen' : time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()) # Zone files do not contain a timestamp, so we use the current time 'seen' : time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()) # Zone files do not contain a timestamp, so we use the current time
}, },
'doc_as_upsert' : True # This will create the document if it does not exist 'doc_as_upsert' : True # This will create the document if it does not exist