acidvegas/eris
acidvegas/eris
1
Fork 0
Code Issues Pull Requests Activity

OCD about formatting again

Browse Source
This commit is contained in:
Dionysus 2024-03-06 15:07:52 -05:00
parent fd617e3c9d
commit b15b3d8241
Signed by: acidvegas
GPG Key ID: EF4B922DB85DC9DE
4 changed files with 83 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
ingestors/ingest_certs.py
View File

@ -24,8 +24,8 @@ def construct_map() -> dict:
'data': { 'data': {
'properties': { 'properties': {
'cert_index': { 'type': 'integer' }, 'cert_index': { 'type': 'integer' },
'cert_link': { 'type': 'keyword' }, 'cert_link' : { 'type': 'keyword' },
'leaf_cert': { 'leaf_cert' : {
'properties': { 'properties': {
'all_domains': { 'type': 'keyword' }, 'all_domains': { 'type': 'keyword' },
'extensions': { 'extensions': {
@ -55,10 +55,10 @@ def construct_map() -> dict:
'emailAddress' : { 'type': 'text' } 'emailAddress' : { 'type': 'text' }
} }
}, },
'not_after': { 'type': 'integer' }, 'not_after' : { 'type': 'integer' },
'not_before': { 'type': 'integer' }, 'not_before' : { 'type': 'integer' },
'serial_number': { 'type': 'keyword' }, 'serial_number' : { 'type': 'keyword' },
'signature_algorithm': { 'type': 'text' }, 'signature_algorithm' : { 'type': 'text' },
'subject': { 'subject': {
'properties': { 'properties': {
'C' : { 'type': 'keyword' }, 'C' : { 'type': 'keyword' },

109
ingestors/ingest_httpx.py
View File

@ -19,7 +19,37 @@ def construct_map() -> dict:
mapping = { mapping = {
'mappings': { 'mappings': {
'properties': { 'properties': {
'change': 'me' "timestamp" : { 'type' : 'date' },
"hash" : {
"body_md5" : { 'type': 'keyword' },
"body_mmh3" : { 'type': 'keyword' },
"body_sha256" : { 'type': 'keyword' },
"body_simhash" : { 'type': 'keyword' },
"header_md5" : { 'type': 'keyword' },
"header_mmh3" : { 'type': 'keyword' },
"header_sha256" : { 'type': 'keyword' },
"header_simhash" : { 'type': 'keyword' }
},
"port" : { 'type': 'integer' },
"url" : keyword_mapping,
"input" : keyword_mapping,
"title" : keyword_mapping,
"scheme" : { 'type': 'keyword' },
"webserver" : { 'type': 'keyword' },
"body_preview" : keyword_mapping,
"content_type" : { 'type': 'keyword' },
"method" : { 'type': 'keyword'},
"host" : { 'type': 'ip'},
"path" : keyword_mapping,
"favicon" : { 'type': 'keyword' },
"favicon_path" : keyword_mapping,
"a" : { 'type': 'ip'},
"aaaa" : { 'type': 'ip'},
"tech" : keyword_mapping,
"words" : { 'type': 'integer'},
"lines" : { 'type': 'integer'},
"status_code" : { 'type': 'integer'},
"content_length" : { 'type': 'integer'}
} }
} }
} }
@ -46,11 +76,10 @@ async def process_data(file_path: str):
record['seen'] = record.pop('timestamp').split('.')[0] + 'Z' # Hacky solution to maintain ISO 8601 format without milliseconds or offsets record['seen'] = record.pop('timestamp').split('.')[0] + 'Z' # Hacky solution to maintain ISO 8601 format without milliseconds or offsets
record['domain'] = record.pop('input') record['domain'] = record.pop('input')
del record['failed'], record['knowledgebase'], record['time'] for item in ('failed', 'knowledgebase', 'time'):
del record[item]
yield {'_index': default_index, '_source': record} yield {'_id': record['domain'], '_index': default_index, '_source': record}
return None # EOF
@ -59,45 +88,39 @@ Example record:
{ {
"timestamp":"2024-01-14T13:08:15.117348474-05:00", # Rename to seen and remove milliseconds and offset "timestamp":"2024-01-14T13:08:15.117348474-05:00", # Rename to seen and remove milliseconds and offset
"hash": { # Do we need all of these ? "hash": { # Do we need all of these ?
"body_md5":"4ae9394eb98233b482508cbda3b33a66", "body_md5" : "4ae9394eb98233b482508cbda3b33a66",
"body_mmh3":"-4111954", "body_mmh3" : "-4111954",
"body_sha256":"89e06e8374353469c65adb227b158b265641b424fba7ddb2c67eef0c4c1280d3", "body_sha256" : "89e06e8374353469c65adb227b158b265641b424fba7ddb2c67eef0c4c1280d3",
"body_simhash":"9814303593401624250", "body_simhash" : "9814303593401624250",
"header_md5":"980366deb2b2fb5df2ad861fc63e79ce", "header_md5" : "980366deb2b2fb5df2ad861fc63e79ce",
"header_mmh3":"-813072798", "header_mmh3" : "-813072798",
"header_sha256":"39aea75ad548e38b635421861641ad1919ed3b103b17a33c41e7ad46516f736d", "header_sha256" : "39aea75ad548e38b635421861641ad1919ed3b103b17a33c41e7ad46516f736d",
"header_simhash":"10962523587435277678" "header_simhash" : "10962523587435277678"
}, },
"port":"443", "port" : "443",
"url":"https://supernets.org", # Remove this and only use the input field as "domain" maybe "url" : "https://supernets.org", # Remove this and only use the input field as "domain" maybe
"input":"supernets.org", # rename to domain "input" : "supernets.org", # rename to domain
"title":"SuperNETs", "title" : "SuperNETs",
"scheme":"https", "scheme" : "https",
"webserver":"nginx", "webserver" : "nginx",
"body_preview":"SUPERNETS Home About Contact Donate Docs Network IRC Git Invidious Jitsi LibreX Mastodon Matrix Sup", "body_preview" : "SUPERNETS Home About Contact Donate Docs Network IRC Git Invidious Jitsi LibreX Mastodon Matrix Sup",
"content_type":"text/html", "content_type" : "text/html",
"method":"GET", # Do we need this ? "method" : "GET", # Remove this
"host":"51.89.151.158", "host" : "51.89.151.158",
"path":"/", "path" : "/",
"favicon":"-674048714", "favicon" : "-674048714",
"favicon_path":"/i/favicon.png", "favicon_path" : "/i/favicon.png",
"time":"592.907689ms", # Do we need this ? "time" : "592.907689ms", # Do we need this ?
"a":[ "a" : ["6.150.220.23"],
"6.150.220.23" "tech" : ["Bootstrap:4.0.0", "HSTS", "Nginx"],
], "words" : 436, # Do we need this ?
"tech":[ "lines" : 79, # Do we need this ?
"Bootstrap:4.0.0", "status_code" : 200,
"HSTS", "content_length" : 4597,
"Nginx" "failed" : false, # Do we need this ?
], "knowledgebase" : { # Do we need this ?
"words":436, # Do we need this ? "PageType" : "nonerror",
"lines":79, # Do we need this ? "pHash" : 0
"status_code":200,
"content_length":4597,
"failed":false, # Do we need this ?
"knowledgebase":{ # Do we need this ?
"PageType":"nonerror",
"pHash":0
} }
} }
''' '''

6
ingestors/ingest_massdns.py
View File

@ -20,10 +20,10 @@ def construct_map() -> dict:
mapping = { mapping = {
'mappings': { 'mappings': {
'properties': { 'properties': {
'ip' : { 'type' : 'ip' }, 'ip' : { 'type': 'ip' },
'name' : { 'type' : 'keyword' }, 'name' : { 'type': 'keyword' },
'record' : keyword_mapping, 'record' : keyword_mapping,
'seen' : { 'type' : 'date' } 'seen' : { 'type': 'date' }
} }
} }
} }

6
ingestors/ingest_zone.py
View File

@ -22,9 +22,9 @@ def construct_map() -> dict:
mapping = { mapping = {
'mappings': { 'mappings': {
'properties': { 'properties': {
'domain': keyword_mapping, 'domain' : keyword_mapping,
'records': { 'properties': {} }, 'records' : { 'properties': {} },
'seen': {'type': 'date'} 'seen' : { 'type': 'date' }
} }
} }
} }