99 lines
2.9 KiB
Python
99 lines
2.9 KiB
Python
#!/usr/bin/env python
|
|
# Elasticsearch Recon Ingestion Scripts (ERIS) - Developed by Acidvegas (https://git.acid.vegas/eris)
|
|
# ingest_massdns.py
|
|
|
|
import logging
|
|
import time
|
|
|
|
try:
|
|
import aiofiles
|
|
except ImportError:
|
|
raise ImportError('Missing required \'aiofiles\' library. (pip install aiofiles)')
|
|
|
|
default_index = 'ptr-records'
|
|
|
|
def construct_map() -> dict:
|
|
'''Construct the Elasticsearch index mapping for MassDNS records'''
|
|
|
|
keyword_mapping = { 'type': 'text', 'fields': { 'keyword': { 'type': 'keyword', 'ignore_above': 256 } } }
|
|
|
|
mapping = {
|
|
'mappings': {
|
|
'properties': {
|
|
'ip' : { 'type': 'ip' },
|
|
'name' : { 'type': 'keyword' },
|
|
'record' : keyword_mapping,
|
|
'seen' : { 'type': 'date' }
|
|
}
|
|
}
|
|
}
|
|
|
|
return mapping
|
|
|
|
|
|
async def process_data(file_path: str):
|
|
'''
|
|
Read and process Massdns records from the log file.
|
|
|
|
:param file_path: Path to the Massdns log file
|
|
'''
|
|
|
|
async with aiofiles.open(file_path, mode='r') as input_file:
|
|
async for line in input_file:
|
|
line = line.strip()
|
|
|
|
if line == '~eof': # Sentinel value to indicate the end of a process (Used with --watch with FIFO)
|
|
break
|
|
|
|
if not line:
|
|
continue
|
|
|
|
parts = line.split()
|
|
|
|
if len(parts) < 3:
|
|
raise ValueError(f'Invalid PTR record: {line}')
|
|
|
|
name, record_type, record = parts[0].rstrip('.'), parts[1], ' '.join(parts[2:]).rstrip('.')
|
|
|
|
# Do we handle CNAME records returned by MassDNS?
|
|
if record_type != 'PTR':
|
|
continue
|
|
|
|
# Let's not index the PTR record if it's the same as the in-addr.arpa domain
|
|
if record == name:
|
|
continue
|
|
|
|
if not record: # Skip empty records
|
|
continue
|
|
|
|
ip = '.'.join(name.replace('.in-addr.arpa', '').split('.')[::-1])
|
|
|
|
struct = {
|
|
'ip' : ip,
|
|
'record' : record,
|
|
'seen' : time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
|
|
}
|
|
|
|
yield {'_id': ip, '_index': default_index, '_source': struct} # Store with ip as the unique id to allow the record to be reindexed if it exists.
|
|
|
|
|
|
|
|
'''
|
|
Example PTR record:
|
|
0.6.229.47.in-addr.arpa. PTR 047-229-006-000.res.spectrum.com.
|
|
0.6.228.75.in-addr.arpa. PTR 0.sub-75-228-6.myvzw.com.
|
|
0.6.207.73.in-addr.arpa. PTR c-73-207-6-0.hsd1.ga.comcast.net.
|
|
0.6.212.173.in-addr.arpa. PTR 173-212-6-0.cpe.surry.net.
|
|
0.6.201.133.in-addr.arpa. PTR flh2-133-201-6-0.tky.mesh.ad.jp.
|
|
|
|
Will be indexed as:
|
|
{
|
|
"_id" : "47.229.6.0"
|
|
"_index" : "ptr-records",
|
|
"_source" : {
|
|
"ip" : "47.229.6.0",
|
|
"record" : "047-229-006-000.res.spectrum.com.",
|
|
"seen" : "2021-06-30T18:31:00Z"
|
|
}
|
|
}
|
|
''' |