acidvegas/eris
acidvegas/eris
1
Fork 0
Code Issues Pull Requests Activity

Full source commenting, uniformity in testing function, records stored as a list by default incase an IP address yields multiple PTR records

Browse Source
This commit is contained in:
Dionysus 2024-03-11 19:18:03 -04:00
parent 6c4ae3e988
commit b018da4e4d
Signed by: acidvegas
GPG Key ID: EF4B922DB85DC9DE
1 changed files with 83 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

162
ingestors/ingest_massdns.py
View File

@ -11,14 +11,17 @@ except ImportError:
raise ImportError('Missing required \'aiofiles\' library. (pip install aiofiles)') raise ImportError('Missing required \'aiofiles\' library. (pip install aiofiles)')
# Set a default elasticsearch index if one is not provided
default_index = 'eris-massdns' default_index = 'eris-massdns'
def construct_map() -> dict: def construct_map() -> dict:
'''Construct the Elasticsearch index mapping for MassDNS records''' '''Construct the Elasticsearch index mapping for records'''
# Match on exact value or full text search
keyword_mapping = { 'type': 'text', 'fields': { 'keyword': { 'type': 'keyword', 'ignore_above': 256 } } } keyword_mapping = { 'type': 'text', 'fields': { 'keyword': { 'type': 'keyword', 'ignore_above': 256 } } }
# Construct the index mapping
mapping = { mapping = {
'mappings': { 'mappings': {
'properties': { 'properties': {
@ -32,91 +35,92 @@ def construct_map() -> dict:
return mapping return mapping
async def process_data(file_path: str): async def process_data(input_path: str):
''' '''
Read and process Massdns records from the log file. Read and process the input file
:param file_path: Path to the Massdns log file :param input_path: Path to the input file
''' '''
async with aiofiles.open(file_path) as input_file: async with aiofiles.open(input_path) as input_file:
# Cache the last document to avoid creating a new one for the same IP address
last = None last = None
async for line in input_file: try:
line = line.strip() # Read the input file line by line
async for line in input_file:
line = line.strip()
# Sentinel value to indicate the end of a process (for closing out a FIFO stream) # Sentinel value to indicate the end of a process (for closing out a FIFO stream)
if line == '~eof': if line == '~eof':
yield last yield last
break break
# Skip empty lines # Skip empty lines (doubtful we will have any, but just in case)
if not line: if not line:
continue
# Split the line into its parts
parts = line.split()
# Ensure the line has at least 3 parts
if len(parts) < 3:
logging.warning(f'Invalid PTR record: {line}')
continue
# Split the PTR record into its parts
name, record_type, record = parts[0].rstrip('.'), parts[1], ' '.join(parts[2:]).rstrip('.')
# Do not index other records
if record_type != 'PTR':
continue
# Do not index PTR records that do not have a record
if not record:
continue
# Let's not index the PTR record if it's the same as the in-addr.arpa domain
if record == name:
continue
# Get the IP address from the in-addr.arpa domain
ip = '.'.join(name.replace('.in-addr.arpa', '').split('.')[::-1])
# Check if we are still processing the same IP address
if last:
if ip == last['_id']:
last_record = last['doc']['record']
if isinstance(last_record, list):
if record not in last_record:
last['doc']['record'].append(record)
else:
logging.warning(f'Duplicate PTR record: {line}')
else:
if record != last_record:
last['doc']['record'] = [last_record, record] # IP addresses with more than one PTR record will turn into a list
continue continue
else:
yield last # Return the last document and start a new one
# Cache the this document in-case we have more for the same IP address # Split the line into its parts
last = { parts = line.split()
'_op_type' : 'update',
'_id' : ip, # Ensure the line has at least 3 parts
'_index' : default_index, if len(parts) < 3:
'doc' : { logging.warning(f'Invalid PTR record: {line}')
'ip' : ip, continue
'record' : record,
'seen' : time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()) # Split the PTR record into its parts
}, name, record_type, record = parts[0].rstrip('.'), parts[1], ' '.join(parts[2:]).rstrip('.')
'doc_as_upsert' : True # This will create the document if it does not exist
} # Do not index other records
if record_type != 'PTR':
continue
# Do not index PTR records that do not have a record
if not record:
continue
# Do not index PTR records that have the same record as the in-addr.arpa domain
if record == name:
continue
# Get the IP address from the in-addr.arpa domain
ip = '.'.join(name.replace('.in-addr.arpa', '').split('.')[::-1])
# Check if we are still processing the same IP address
if last:
if ip == last['_id']: # This record is for the same IP address as the cached document
last_records = last['doc']['record']
if record not in last_records: # Do not index duplicate records
last['doc']['record'].append(record)
continue
else:
yield last # Return the last document and start a new one
# Cache the document
last = {
'_op_type' : 'update',
'_id' : ip,
'_index' : default_index,
'doc' : {
'ip' : ip,
'record' : [record], # Consider using painless script to add to list if it exists (Use 'seen' per-record and 'last_seen' for the IP address)
'seen' : time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
},
'doc_as_upsert' : True # Create the document if it does not exist
}
except Exception as e:
logging.error(f'Error processing data: {e}')
async def test(input_path: str): async def test(input_path: str):
''' '''
Test the MassDNS ingestion process Test the ingestion process
:param input_path: Path to the MassDNS log file :param input_path: Path to the input file
''' '''
async for document in process_data(input_path): async for document in process_data(input_path):
print(document) print(document)
@ -126,7 +130,7 @@ if __name__ == '__main__':
import argparse import argparse
import asyncio import asyncio
parser = argparse.ArgumentParser(description='MassDNS Ingestor for ERIS') parser = argparse.ArgumentParser(description='Ingestor for ERIS')
parser.add_argument('input_path', help='Path to the input file or directory') parser.add_argument('input_path', help='Path to the input file or directory')
args = parser.parse_args() args = parser.parse_args()
@ -139,9 +143,9 @@ Deployment:
sudo apt-get install build-essential gcc make sudo apt-get install build-essential gcc make
git clone --depth 1 https://github.com/blechschmidt/massdns.git $HOME/massdns && cd $HOME/massdns && make git clone --depth 1 https://github.com/blechschmidt/massdns.git $HOME/massdns && cd $HOME/massdns && make
curl -s https://public-dns.info/nameservers.txt | grep -v ':' > $HOME/massdns/nameservers.txt curl -s https://public-dns.info/nameservers.txt | grep -v ':' > $HOME/massdns/nameservers.txt
python3 ./scripts/ptr.py | ./bin/massdns -r $HOME/massdns/nameservers.txt -t PTR --filter NOERROR-s 500 -o S -w $HOME/massdns/fifo.json python3 ./scripts/ptr.py | ./bin/massdns -r $HOME/massdns/nameservers.txt -t PTR --filter NOERROR -o S -w $HOME/massdns/fifo.json
or... or...
while true; do python ./scripts/ptr.py | ./bin/massdns -r $HOME/massdns/nameservers.txt -t PTR --filter NOERROR -s 1000 -o S -w $HOME/massdns/fifo.json; done while true; do python ./scripts/ptr.py | ./bin/massdns -r $HOME/massdns/nameservers.txt -t PTR --filter NOERROR -o S -w $HOME/massdns/fifo.json; done
Output: Output:
0.6.229.47.in-addr.arpa. PTR 047-229-006-000.res.spectrum.com. 0.6.229.47.in-addr.arpa. PTR 047-229-006-000.res.spectrum.com.
@ -150,16 +154,16 @@ Output:
Input: Input:
{ {
"_id" : "47.229.6.0" '_id' : '47.229.6.0'
"_index" : "eris-massdns", '_index' : 'eris-massdns',
"_source" : { '_source' : {
"ip" : "47.229.6.0", 'ip' : '47.229.6.0',
"record" : "047-229-006-000.res.spectrum.com", # This will be a list if there are more than one PTR record 'record' : ['047-229-006-000.res.spectrum.com'], # We will store as a list for IP addresses with multiple PTR records
"seen" : "2021-06-30T18:31:00Z" 'seen' : '2021-06-30T18:31:00Z'
} }
} }
Notes: Notes:
Why do some IP addresses return a CNAME from a PTR request Why do some IP addresses return a A/CNAME from a PTR request
What is dns-servfail.net (Frequent CNAME response from PTR requests) What is dns-servfail.net (Frequent CNAME response from PTR requests)
''' '''