Updated massdns ingestion script with sentinal value checking and using the ip address as the document id

ingestors/ingest_massdns.py

@@ -2,6 +2,7 @@
 # Elasticsearch Recon Ingestion Scripts (ERIS) - Developed by Acidvegas (https://git.acid.vegas/eris)
 # ingest_massdns.py
 
+import logging
 import time
 
 try:
@@ -41,6 +42,9 @@ async def process_data(file_path: str):
         async for line in input_file:
             line = line.strip()
 
+            if line == '~eof': # Sentinel value to indicate the end of a process (Used with --watch with FIFO)
+                break
+
             if not line:
                 continue
 
@@ -49,30 +53,29 @@ async def process_data(file_path: str):
             if len(parts) < 3:
                 raise ValueError(f'Invalid PTR record: {line}')
             
-            name, record_type, data = parts[0].rstrip('.'), parts[1], ' '.join(parts[2:]).rstrip('.')
+            name, record_type, record = parts[0].rstrip('.'), parts[1], ' '.join(parts[2:]).rstrip('.')
 
+            # Do we handle CNAME records returned by MassDNS?
             if record_type != 'PTR':
                 continue
 
-                #if record_type == 'CNAME':
-                #    if data.endswith('.in-addr.arpa'):
-                #        continue
-
             # Let's not index the PTR record if it's the same as the in-addr.arpa domain
-            if data == name:
+            if record == name:
+                continue
+
+            if not record: # Skip empty records
                 continue
                     
             ip = '.'.join(name.replace('.in-addr.arpa', '').split('.')[::-1])
             
             struct = {
                 'ip'     : ip,
-                'record': data,
+                'record' : record,
                 'seen'   : time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
             }
 
-            yield {'_index': default_index, '_source': struct}
+            yield {'_id': ip, '_index': default_index, '_source': struct} # Store with ip as the unique id to allow the record to be reindexed if it exists.
 
-    return None # EOF
 
 
 '''
@@ -85,8 +88,12 @@ Example PTR record:
 
 Will be indexed as:
 {
+    "_id"     : "47.229.6.0"
+    "_index"  : "ptr-records",
+    "_source" : {
         "ip"     : "47.229.6.0",
         "record" : "047-229-006-000.res.spectrum.com.",
         "seen"   : "2021-06-30T18:31:00Z"
     }
+}
 '''