Updated code & readme a little

README.md

@@ -1,13 +1,30 @@
 # massclude
 
 ## Information
-This is a simple script that will generate an exclude.conf file for masscan that contains a set of IP ranges that are pointless to scan.
+This is a simple script that will generate an *exclude.conf* file for [masscan](https://github.com/robertdavidgraham/masscan) that contains a set of IP ranges that are mostly pointless to scan thus making it much faster & raising less suspicion.
 
-These include a private & reserved IP ranges, bogon IP ranges, and IP ranges that belong to root DNS servers, internet exchange points, & government agencies.
+###### These ranges include:
+- [Bogons](https://en.wikipedia.org/wiki/Bogon_filtering)
+- [Department Of Defense DNIC](https://en.wikipedia.org/wiki/List_of_assigned_/8_IPv4_address_blocks#List_of_assigned_/8_blocks_to_the_United_States_Department_of_Defense)
+- [Internet Excahnge Points](https://en.wikipedia.org/wiki/Internet_exchange_point)
+- [Root DNS Servers](https://en.wikipedia.org/wiki/Root_name_server)
+- [Reserved IP Ranges](https://en.wikipedia.org/wiki/Reserved_IP_addresses)
 
-This removes over 1.5 billion ip addresses from the scan, thus making it much faster and raising less suspicion.
 
-This repository is hosted on a VPS that updates every 6 hours with the latest IP ranges to exclude. (soon)
+## What kind of numbers?
+###### IPv4
+**Total IPv4 Addresses** : 4,294,967,296
+
+**Total After Massclude**  : 3,176,439,555
+
+This is a **26%** drop in total IP addresses...
+
+###### IPv6
+**Total IP Addresses** : 340,282,366,920,938,463,463,374,607,431,768,211,456
+
+**Total After Massclude**  : 12,551,294,199,370,633,260,152,632,202,625,108,965
+
+While this is still a huge number, total IP addresses dropped **96.31%** here...
 
 ___
 

exclude4.conf

@@ -1,7 +1,4 @@
-
 # Excludes from bogons
-# Invalid IP/range from bogons
-
 0.0.0.0/8
 10.0.0.0/8
 100.64.0.0/10
@@ -10,24 +7,18 @@
 102.192.0.0/12
 102.208.0.0/15
 102.210.0.0/16
-102.211.0.0/18
+102.211.16.0/20
-102.211.104.0/22
+102.211.32.0/21
-102.211.64.0/19
+102.211.40.0/22
-102.211.96.0/21
+102.211.55.0/24
-103.127.216.0/21
+103.203.245.0/24
-103.131.128.0/21
+103.203.246.0/23
-103.15.94.0/23
-103.20.96.0/21
-103.203.232.0/21
-103.203.240.0/21
 103.204.40.0/21
-103.21.96.0/21
 103.211.200.0/21
 103.213.216.0/21
 103.213.8.0/21
 103.217.64.0/21
 103.224.72.0/21
-103.23.88.0/21
 103.240.0.0/21
 103.241.192.0/21
 103.249.112.0/21
@@ -36,37 +27,10 @@
 103.253.16.0/21
 103.255.160.0/21
 103.255.192.0/21
-103.255.223.0/24
-103.26.24.0/21
-103.38.104.0/21
-103.38.216.0/21
-103.41.72.0/21
-103.42.240.0/21
-103.43.56.0/21
-103.55.248.0/21
-103.60.88.0/21
-103.65.120.0/21
 103.65.128.0/20
-103.67.40.0/21
 103.67.48.0/20
 103.67.64.0/19
-103.69.48.0/21
-103.69.80.0/21
-103.69.96.0/21
-103.70.112.0/21
-103.71.104.0/21
 103.71.208.0/20
-103.72.56.0/21
-103.72.64.0/21
-103.75.72.0/21
-103.75.80.0/21
-103.76.104.0/21
-103.77.176.0/21
-103.77.208.0/21
-103.77.240.0/21
-103.78.0.0/21
-103.80.224.0/21
-103.99.136.0/21
 110.172.24.0/21
 113.192.0.0/18
 113.20.132.0/22
@@ -228,7 +192,6 @@
 180.214.160.0/21
 180.94.0.0/19
 182.161.48.0/22
-189.36.240.0/22
 192.0.0.0/24
 192.0.2.0/24
 192.103.132.0/24
@@ -356,9 +319,6 @@
 192.94.200.0/24
 192.94.220.0/24
 192.94.238.0/24
-193.0.232.0/23
-194.77.158.0/24
-195.35.112.0/23
 196.1.108.0/24
 196.61.4.0/22
 198.15.16.0/20
@@ -384,7 +344,6 @@
 198.56.16.0/21
 198.56.24.0/23
 198.61.8.0/21
-200.189.68.0/22
 202.0.109.0/24
 202.0.116.0/24
 202.0.152.0/24
@@ -486,6 +445,7 @@
 202.94.71.0/24
 203.0.113.0/24
 203.0.140.0/24
+203.0.37.0/24
 203.1.109.0/24
 203.1.2.0/24
 203.1.72.0/22
@@ -598,7 +558,7 @@
 203.55.68.0/24
 203.57.43.0/24
 203.57.80.0/24
-203.57.85.0/24
+203.57.84.0/23
 203.62.138.0/24
 203.62.165.0/24
 203.62.240.0/22
@@ -679,7 +639,7 @@
 43.248.252.0/22
 43.248.56.0/22
 43.249.124.0/22
-43.249.20.0/22
+43.249.16.0/21
 43.249.88.0/22
 43.250.180.0/22
 43.252.224.0/22
@@ -701,16 +661,22 @@
 45.248.112.0/21
 45.248.176.0/22
 45.248.184.0/22
-45.249.180.0/22
+45.249.128.0/22
-45.249.224.0/22
+45.249.140.0/22
+45.249.172.0/22
+45.249.176.0/21
+45.249.224.0/21
 45.249.60.0/22
 45.251.248.0/22
+45.254.232.0/22
 45.254.44.0/22
 45.65.56.0/23
 49.143.248.0/22
 49.213.32.0/19
 85.217.216.0/22
-94.101.104.0/21
+89.207.156.0/22
+91.198.40.0/24
+91.208.67.0/24
 
 # Excludes from dns_root_servers
 198.41.0.4
@@ -2273,3 +2239,23 @@
 95.140.128.0/23
 95.140.128.0/23
 95.140.130.0/23
+
+# Excludes from private
+0.0.0.0/8
+10.0.0.0/8
+100.64.0.0/10
+127.0.0.0/8
+169.254.0.0/16
+172.16.0.0/12
+192.0.0.0/24
+192.0.0.0/29
+192.0.0.170/32
+192.0.0.171/32
+192.0.2.0/24
+192.88.99.0/24
+192.168.0.0/16
+198.18.0.0/15
+198.51.100.0/24
+203.0.113.0/24
+240.0.0.0/4
+255.255.255.255/32

massclude.py

@@ -12,21 +12,38 @@ def get_bogons(version):
     url = f'https://team-cymru.org/Services/Bogons/fullbogons-ipv{version}.txt'
     return urllib.request.urlopen(url).read().decode().split('\n')[2:]
 
+def determine_latest_db():
+	'''Determine the latest IXP database.'''
+	data = urllib.request.urlopen('https://publicdata.caida.org/datasets/ixps/').read().decode()
+	latest = time.strftime('%Y%m')
+	if f'_{latest}.jsonl' in data:
+		return latest
+	else: # TODO: This is a mess, clean it up
+		latest = str(int(latest)-1)
+		if f'_{latest}.jsonl' in data:
+			return latest
+		else:
+			latest = str(int(latest)-1)
+			if f'_{latest}.jsonl' in data:
+				return latest
+			else:
+				return None
+
 def get_ixps(version):
 	'''Returns a list of IXP IP addresses from CAIDA.'''
-	latest = time.strftime('%Y%m')
+	if (latest := determine_latest_db()):
-	try:
+		try:
-		data = urllib.request.urlopen(f'https://publicdata.caida.org/datasets/ixps/ixs_{latest}.jsonl').read().decode()
+			data = urllib.request.urlopen(f'https://publicdata.caida.org/datasets/ixps/ixs_{latest}.jsonl').read().decode()
-	except:
+		except:
-		latest = str(int(time.strftime('%Y%m'))-1)
+			latest = str(int(time.strftime('%Y%m'))-1)
-		data = urllib.request.urlopen(f'https://publicdata.caida.org/datasets/ixps/ixs_{latest}.jsonl').read().decode()
+			data = urllib.request.urlopen(f'https://publicdata.caida.org/datasets/ixps/ixs_{latest}.jsonl').read().decode()
-	decoder = json.JSONDecoder()
+		decoder = json.JSONDecoder()
-	objects = []
+		objects = []
-	for line in data.split('\n'):
+		for line in data.split('\n'):
-		if len(line) > 0 and line[0][0] != "#":
+			if len(line) > 0 and line[0][0] != "#":
-			objects.append(decoder.decode(line))
+				objects.append(decoder.decode(line))
-	json_data = json.loads(json.dumps(objects))
+		json_data = json.loads(json.dumps(objects))
-	return [ip for item in json_data if item['prefixes']['ipv'+version] for ip in item['prefixes']['ipv'+version]]
+		return [ip for item in json_data if item['prefixes']['ipv'+version] for ip in item['prefixes']['ipv'+version]]
 
 def generate_list():
 	return {
@@ -88,7 +105,46 @@ def generate_list():
 		'ixps' : {
 			'4': sorted(get_ixps('4')),
 			'6': sorted(get_ixps('6'))
-		}
+		},
+        'private' : {
+			'4': [
+				'0.0.0.0/8',         # "This" network
+				'10.0.0.0/8',        # Private networks
+				'100.64.0.0/10',     # Carrier-grade NAT - RFC 6598
+				'127.0.0.0/8',       # Host loopback
+				'169.254.0.0/16',    # Link local
+				'172.16.0.0/12',     # Private networks
+				'192.0.0.0/24',      # IETF Protocol Assignments
+				'192.0.0.0/29',      # DS-Lite
+				'192.0.0.170/32',    # NAT64
+				'192.0.0.171/32',    # DNS64
+				'192.0.2.0/24',      # Documentation (TEST-NET-1)
+				'192.88.99.0/24',    # 6to4 Relay Anycast
+				'192.168.0.0/16',    # Private networks
+				'198.18.0.0/15',     # Benchmarking
+				'198.51.100.0/24',   # Documentation (TEST-NET-2)
+				'203.0.113.0/24',    # Documentation (TEST-NET-3)
+				'240.0.0.0/4',       # Reserved
+				'255.255.255.255/32' # Limited Broadcast
+			],
+            '6': [
+				'::/128',            # Unspecified address
+				'::1/128',           # Loopback address
+				'::ffff:0:0/96',     # IPv4 mapped addresses
+				'64:ff9b::/96',      # IPv4/IPv6 translation
+				'100::/64',          # Discard prefix
+				'2001::/32',         # Teredo tunneling	\
+				'2001:10::/28',      # ORCHIDv2
+				'2001:20::/28',      # ORCHIDv2
+				'2001:2::/48',       # Benchmarking
+				'2001:db8::/32',     # Documentation
+				'2002::/16',         # 6to4
+				'fc00::/7',          # Unique local
+				'fe80::/10',         # Link local
+				'ff00::/8'           # Multicast
+                        
+			]
+		},
 	}