acidvegas/eris
acidvegas/eris
1
Fork 0
Code Issues Pull Requests Activity

OCD about formatting again

Browse Source
This commit is contained in:
Dionysus 2024-03-06 15:07:52 -05:00
parent fd617e3c9d
commit b15b3d8241
Signed by: acidvegas
GPG Key ID: EF4B922DB85DC9DE
4 changed files with 83 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
ingestors/ingest_httpx.py
View File

@ -19,7 +19,37 @@ def construct_map() -> dict:
mapping = {
'mappings': {
'properties': {
'change': 'me'
"timestamp" : { 'type' : 'date' },
"hash" : {
"body_md5" : { 'type': 'keyword' },
"body_mmh3" : { 'type': 'keyword' },
"body_sha256" : { 'type': 'keyword' },
"body_simhash" : { 'type': 'keyword' },
"header_md5" : { 'type': 'keyword' },
"header_mmh3" : { 'type': 'keyword' },
"header_sha256" : { 'type': 'keyword' },
"header_simhash" : { 'type': 'keyword' }
},
"port" : { 'type': 'integer' },
"url" : keyword_mapping,
"input" : keyword_mapping,
"title" : keyword_mapping,
"scheme" : { 'type': 'keyword' },
"webserver" : { 'type': 'keyword' },
"body_preview" : keyword_mapping,
"content_type" : { 'type': 'keyword' },
"method" : { 'type': 'keyword'},
"host" : { 'type': 'ip'},
"path" : keyword_mapping,
"favicon" : { 'type': 'keyword' },
"favicon_path" : keyword_mapping,
"a" : { 'type': 'ip'},
"aaaa" : { 'type': 'ip'},
"tech" : keyword_mapping,
"words" : { 'type': 'integer'},
"lines" : { 'type': 'integer'},
"status_code" : { 'type': 'integer'},
"content_length" : { 'type': 'integer'}
}
}
}
@ -46,11 +76,10 @@ async def process_data(file_path: str):
record['seen'] = record.pop('timestamp').split('.')[0] + 'Z' # Hacky solution to maintain ISO 8601 format without milliseconds or offsets
record['domain'] = record.pop('input')
del record['failed'], record['knowledgebase'], record['time']
for item in ('failed', 'knowledgebase', 'time'):
del record[item]
yield {'_index': default_index, '_source': record}
return None # EOF
yield {'_id': record['domain'], '_index': default_index, '_source': record}
@ -76,20 +105,14 @@ Example record:
"webserver" : "nginx",
"body_preview" : "SUPERNETS Home About Contact Donate Docs Network IRC Git Invidious Jitsi LibreX Mastodon Matrix Sup",
"content_type" : "text/html",
"method":"GET", # Do we need this ?
"method" : "GET", # Remove this
"host" : "51.89.151.158",
"path" : "/",
"favicon" : "-674048714",
"favicon_path" : "/i/favicon.png",
"time" : "592.907689ms", # Do we need this ?
"a":[
"6.150.220.23"
],
"tech":[
"Bootstrap:4.0.0",
"HSTS",
"Nginx"
],
"a" : ["6.150.220.23"],
"tech" : ["Bootstrap:4.0.0", "HSTS", "Nginx"],
"words" : 436, # Do we need this ?
"lines" : 79, # Do we need this ?
"status_code" : 200,