json dumps output for better inegstion

LICENSE

@@ -1,6 +1,6 @@
 ISC License
 
-Copyright (c) 2024, acidvegas <acid.vegas@acid.vegas>
+Copyright (c) 2025, acidvegas <acid.vegas@acid.vegas>
 
 Permission to use, copy, modify, and/or distribute this software for any
 purpose with or without fee is hereby granted, provided that the above

README.md

@@ -20,16 +20,16 @@ This library connects to a Meshtastic MQTT broker and decodes various message ty
 Install from PyPI:
 
 ```bash
-pip install meshtastic-mqtt
+pip install meshtastic-mqtt-json
 ```
 
 ## usage
 ```bash
-python meshtastic_mqtt [options]
+python meshtastic_mqtt_json [options]
 ```
 
 ```python
-from meshtastic_mqtt import MeshtasticMQTT
+from meshtastic_mqtt_json import MeshtasticMQTT
 client = MeshtasticMQTT()
 client.connect(broker='mqtt.meshtastic.org', port=1883, root='msh/US/2/e/', channel='LongFast', username='meshdev', password='large4cats', key='AQ==')
 ```
@@ -48,7 +48,7 @@ client.connect(broker='mqtt.meshtastic.org', port=1883, root='msh/US/2/e/', chan
 
 ### Filter Example
 ```bash
-python meshtastic_mqtt.py --filter "NODEINFO,POSITION,TEXT_MESSAGE"
+python meshtastic_mqtt_json.py --filter "NODEINFO,POSITION,TEXT_MESSAGE"
 ```
 
 
@@ -91,4 +91,4 @@ The library supports parsing of the following Meshtastic message types:
 
 ___
 
-###### Mirrors for this repository: [acid.vegas](https://git.acid.vegas/meshtastic_mqtt) • [SuperNETs](https://git.supernets.org/acidvegas/meshtastic_mqtt) • [GitHub](https://github.com/acidvegas/meshtastic_mqtt) • [GitLab](https://gitlab.com/acidvegas/meshtastic_mqtt) • [Codeberg](https://codeberg.org/acidvegas/meshtastic_mqtt)
+###### Mirrors for this repository: [acid.vegas](https://git.acid.vegas/meshtastic_mqtt_json) • [SuperNETs](https://git.supernets.org/acidvegas/meshtastic_mqtt_json) • [GitHub](https://github.com/acidvegas/meshtastic_mqtt_json) • [GitLab](https://gitlab.com/acidvegas/meshtastic_mqtt_json) • [Codeberg](https://codeberg.org/acidvegas/meshtastic_mqtt_json)

setup.py

@@ -1,18 +1,18 @@
 from setuptools import setup, find_packages
 
 setup(
-    name="meshtastic_mqtt",
+    name="meshtastic-mqtt-json",
-    version='0.1.0',
+    version='1.0.10',
     author='acidvegas',
     author_email='acid.vegas@acid.vegas',
     description='A lightweight Python library for parsing Meshtastic MQTT messages',
     long_description=open('README.md').read(),
     long_description_content_type='text/markdown',
-    url='https://github.com/acidvegas/meshtastic_mqtt',
+    url='https://github.com/acidvegas/meshtastic_mqtt_json',
     project_urls={
-        'Bug Tracker'   : 'https://github.com/acidvegas/meshtastic_mqtt/issues',
+        'Bug Tracker'   : 'https://github.com/acidvegas/meshtastic_mqtt_json/issues',
-        'Documentation' : 'https://github.com/acidvegas/meshtastic_mqtt#readme',
+        'Documentation' : 'https://github.com/acidvegas/meshtastic_mqtt_json#readme',
-        'Source Code'   : 'https://github.com/acidvegas/meshtastic_mqtt',
+        'Source Code'   : 'https://github.com/acidvegas/meshtastic_mqtt_json',
     },
     packages=find_packages(where='src'),
     package_dir={'': 'src'},
@@ -37,7 +37,7 @@ setup(
     ],
     entry_points={
         'console_scripts': [
-            'meshtastic-mqtt=meshtastic_mqtt.client:main',
+            'meshtastic-mqtt-json=meshtastic_mqtt_json.client:main',
         ],
     },
 ) 

src/meshtastic_mqtt_json/__init__.py

@@ -1,9 +1,9 @@
-"""
+''' 
 Meshtastic MQTT Interface - A lightweight Python library for parsing Meshtastic MQTT messages
-"""
+'''
 
 from .client import MeshtasticMQTT
 
-__version__ = "0.1.0"
+__version__ = '1.0.10'
-__author__  = "acidvegas"
+__author__  = 'acidvegas'
-__license__ = "ISC" 
+__license__ = 'ISC'

src/meshtastic_mqtt_json/client.py

@@ -1,9 +1,10 @@
 #!/usr/bin/env python
-# Meshtastic MQTT Interface - Developed by acidvegas in Python (https://acid.vegas/meshtastic)
+# Meshtastic MQTT Interface - Developed by acidvegas in Python (https://acid.vegas/meshtastic_mqtt_json)
 
 import argparse
 import base64
 import json
+import time
 
 try:
 	from cryptography.hazmat.backends           import default_backend
@@ -27,6 +28,26 @@ except ImportError:
 	raise ImportError('missing the paho-mqtt module (pip install paho-mqtt)')
 
 
+def clean_json(data) -> dict:
+	'''
+	Clean the JSON data by replacing NaN values with null
+
+	:param data: The JSON data to clean
+	'''
+	# Handle protobuf messages
+	if hasattr(data, 'DESCRIPTOR'):
+		data = json.loads(MessageToJson(data))
+
+	# Remove empty and NaN values from the JSON data
+	if isinstance(data, dict):
+		return {k: v for k, v in ((k, clean_json(v)) for k, v in data.items()) if str(v) not in ('None', 'nan', '')}
+	elif isinstance(data, list):
+		return [v for v in (clean_json(v) for v in data) if str(v) not in ('None', 'nan', '')]
+
+	# Return primitive types as-is
+	return data
+
+
 class MeshtasticMQTT(object):
 	def __init__(self):
 		'''Initialize the Meshtastic MQTT client'''
@@ -52,6 +73,7 @@ class MeshtasticMQTT(object):
 
 		# Initialize the MQTT client
 		client = mqtt.Client(mqtt.CallbackAPIVersion.VERSION2, client_id='', clean_session=True, userdata=None)
+		client.connect_timeout = 10
 
 		# Set the username and password for the MQTT broker
 		client.username_pw_set(username=username, password=password)
@@ -69,11 +91,16 @@ class MeshtasticMQTT(object):
 			raise
 
 		# Set the MQTT callbacks
-		client.on_connect = self.event_mqtt_connect
+		client.on_connect    = self.event_mqtt_connect
-		client.on_message = self.event_mqtt_recv
+		client.on_message    = self.event_mqtt_recv
+		client.on_disconnect = self.event_mqtt_disconnect
 
 		# Connect to the MQTT broker
-		client.connect(broker, port, 60)
+		try:
+			client.connect(broker, port, 60)
+		except Exception as e:
+			print(f'Error connecting to MQTT broker: {e}')
+			self.event_mqtt_disconnect(client, '', 1, None)
 
 		# Set the subscribe topic
 		self.subscribe_topic = f'{root}{channel}/#'
@@ -88,7 +115,6 @@ class MeshtasticMQTT(object):
 
 		:param mp: The message packet to decrypt
 		'''
-
 		try:
 			# Extract the nonce from the packet
 			nonce_packet_id = getattr(mp, 'id').to_bytes(8, 'little')
@@ -96,22 +122,32 @@ class MeshtasticMQTT(object):
 			nonce = nonce_packet_id + nonce_from_node
 
 			# Decrypt the message
-			cipher          = Cipher(algorithms.AES(self.key_bytes), modes.CTR(nonce), backend=default_backend())
+			cipher = Cipher(algorithms.AES(self.key_bytes), modes.CTR(nonce), backend=default_backend())
-			decryptor       = cipher.decryptor()
+			decryptor = cipher.decryptor()
 			decrypted_bytes = decryptor.update(getattr(mp, 'encrypted')) + decryptor.finalize()
 
 			# Parse the decrypted message
 			data = mesh_pb2.Data()
-			data.ParseFromString(decrypted_bytes)
+			try:
+				data.ParseFromString(decrypted_bytes)
+			except:
+				# Ignore this as the message does not need to be decrypted
+				return None
+
 			mp.decoded.CopyFrom(data)
+
 			return mp
+
 		except Exception as e:
 			print(f'Error decrypting message: {e}')
-			print(mp)
+			print(f'Message packet details:')
+			print(f'- From: {getattr(mp, "from", "unknown")}')
+			print(f'- To: {getattr(mp, "to", "unknown")}')
+			print(f'- Channel: {getattr(mp, "channel", "unknown")}')
+			print(f'- ID: {getattr(mp, "id", "unknown")}')
 			return None
 
 
-
 	def event_mqtt_connect(self, client, userdata, flags, rc, properties):
 		'''
 		Callback for when the client receives a CONNACK response from the server.
@@ -135,15 +171,20 @@ class MeshtasticMQTT(object):
 
 		:param client:   The client instance for this callback
 		:param userdata: The private user data as set in Client() or user_data_set()
-		:param msg:      An instance of MQTTMessage. This is a
+		:param msg:      An instance of MQTTMessage
 		'''
 
 		try:
 			# Define the service envelope
 			service_envelope = mqtt_pb2.ServiceEnvelope()
 
-			# Parse the message payload
+			try:
-			service_envelope.ParseFromString(msg.payload)
+				# Parse the message payload
+				service_envelope.ParseFromString(msg.payload)
+			except Exception as e:
+				print(f'Error parsing service envelope: {e}')
+				print(f'Raw payload: {msg.payload}')
+				return
 
 			# Extract the message packet from the service envelope
 			mp = service_envelope.packet
@@ -162,144 +203,188 @@ class MeshtasticMQTT(object):
 			if self.filters and portnum_name not in self.filters:
 				return
 
-			json_packet = json.loads(MessageToJson(mp))
+			# Convert to JSON and handle NaN values in one shot
+			json_packet = clean_json(mp)
+			
+			#print(f'Raw packet: {json_packet}')  # Debug print
 
 			# Process the message based on its type
 			if mp.decoded.portnum == portnums_pb2.ADMIN_APP:
 				data = mesh_pb2.Admin()
 				data.ParseFromString(mp.decoded.payload)
-				print(f'{MessageToJson(data)}')
+				msg = json.dumps(clean_json(data))
+				print(f'{msg}')
 
 			elif mp.decoded.portnum == portnums_pb2.ATAK_FORWARDER:
 				data = mesh_pb2.AtakForwarder()
 				data.ParseFromString(mp.decoded.payload)
-				print(f'{MessageToJson(data)}')
+				msg = json.dumps(clean_json(data))
+				print(f'{msg}')
 
 			elif mp.decoded.portnum == portnums_pb2.ATAK_PLUGIN:
 				data = mesh_pb2.AtakPlugin()
 				data.ParseFromString(mp.decoded.payload)
-				print(f'{MessageToJson(data)}')
+				msg = json.dumps(clean_json(data))
+				print(f'{msg}')
 
 			elif mp.decoded.portnum == portnums_pb2.AUDIO_APP:
 				data = mesh_pb2.Audio()
 				data.ParseFromString(mp.decoded.payload)
-				print(f'{MessageToJson(data)}')
+				msg = json.dumps(clean_json(data))
+				print(f'{msg}')
 
 			elif mp.decoded.portnum == portnums_pb2.DETECTION_SENSOR_APP:
 				data = mesh_pb2.DetectionSensor()
 				data.ParseFromString(mp.decoded.payload)
-				print(f'{MessageToJson(data)}')
+				msg = json.dumps(clean_json(data))
+				print(f'{msg}')
 
 			elif mp.decoded.portnum == portnums_pb2.IP_TUNNEL_APP:
 				data = mesh_pb2.IPTunnel()
 				data.ParseFromString(mp.decoded.payload)
-				print(f'{MessageToJson(data)}')
+				msg = json.dumps(clean_json(data))
+				print(f'{msg}')
+
+			elif mp.decoded.portnum == portnums_pb2.MAP_REPORT_APP:
+				map_report = mesh_pb2.MapReport()
+				map_report.ParseFromString(mp.decoded.payload)
+				json_packet['decoded']['payload'] = clean_json(map_report)
+				print(f'{json.dumps(json_packet)}')
 
 			elif mp.decoded.portnum == portnums_pb2.NEIGHBORINFO_APP:
 				neighborInfo = mesh_pb2.NeighborInfo()
 				neighborInfo.ParseFromString(mp.decoded.payload)
-				json_packet['decoded']['payload'] = json.loads(MessageToJson(neighborInfo))
+				json_packet['decoded']['payload'] = clean_json(neighborInfo)
-				print(json.dumps(json_packet))
+				print(f'{json.dumps(json_packet)}')
 
 			elif mp.decoded.portnum == portnums_pb2.NODEINFO_APP:
 				from_id = getattr(mp, 'from')
 				node_info = mesh_pb2.User()
 				node_info.ParseFromString(mp.decoded.payload)
-				json_packet['decoded']['payload'] = json.loads(MessageToJson(node_info))
+				json_packet['decoded']['payload'] = clean_json(node_info)
-				print(json.dumps(json_packet))
+				print(f'{json.dumps(json_packet)}')
 				self.names[from_id] = node_info.long_name
 
 			elif mp.decoded.portnum == portnums_pb2.PAXCOUNTER_APP:
 				data = mesh_pb2.Paxcounter()
 				data.ParseFromString(mp.decoded.payload)
-				print(f'{MessageToJson(data)}')
+				msg = json.dumps(clean_json(data))
+				print(f'{msg}')
 
 			elif mp.decoded.portnum == portnums_pb2.POSITION_APP:
 				position = mesh_pb2.Position()
 				position.ParseFromString(mp.decoded.payload)
-				json_packet['decoded']['payload'] = json.loads(MessageToJson(position))
+				json_packet['decoded']['payload'] = clean_json(position)
-				print(json.dumps(json_packet))
+				print(f'{json.dumps(json_packet)}')
 
 			elif mp.decoded.portnum == portnums_pb2.PRIVATE_APP:
 				data = mesh_pb2.Private()
 				data.ParseFromString(mp.decoded.payload)
-				print(f'{MessageToJson(data)}')
+				msg = json.dumps(clean_json(data))
+				print(f'{msg}')
 
 			elif mp.decoded.portnum == portnums_pb2.RANGE_TEST_APP:
 				data = mesh_pb2.RangeTest()
 				data.ParseFromString(mp.decoded.payload)
-				json_packet['decoded']['payload'] = json.loads(MessageToJson(data))
+				json_packet['decoded']['payload'] = clean_json(data)
-				print(json.dumps(json_packet))
+				print(f'{json.dumps(json_packet)}')
 
 			elif mp.decoded.portnum == portnums_pb2.REMOTE_HARDWARE_APP:
 				data = mesh_pb2.RemoteHardware()
 				data.ParseFromString(mp.decoded.payload)
-				print(f'{MessageToJson(data)}')
+				msg = json.dumps(clean_json(data))
+				print(f'{msg}')
 
 			elif mp.decoded.portnum == portnums_pb2.REPLY_APP:
 				data = mesh_pb2.Reply()
 				data.ParseFromString(mp.decoded.payload)
-				print(f'{MessageToJson(data)}')
+				msg = json.dumps(clean_json(data))
+				print(f'{msg}')
 
 			elif mp.decoded.portnum == portnums_pb2.ROUTING_APP:
 				routing = mesh_pb2.Routing()
 				routing.ParseFromString(mp.decoded.payload)
-				json_packet['decoded']['payload'] = json.loads(MessageToJson(routing))
+				json_packet['decoded']['payload'] = clean_json(routing)
-				print(json.dumps(json_packet))
+				print(f'{json.dumps(json_packet)}')
 
 			elif mp.decoded.portnum == portnums_pb2.SERIAL_APP:
 				data = mesh_pb2.Serial()
 				data.ParseFromString(mp.decoded.payload)
-				print(f'{MessageToJson(data)}')
+				msg = json.dumps(clean_json(data))
+				print(f'{msg}')
 
 			elif mp.decoded.portnum == portnums_pb2.SIMULATOR_APP:
 				data = mesh_pb2.Simulator()
 				data.ParseFromString(mp.decoded.payload)
-				print(f'{MessageToJson(data)}')
+				msg = json.dumps(clean_json(data))
+				print(f'{msg}')
 
 			elif mp.decoded.portnum == portnums_pb2.STORE_FORWARD_APP:
-				print(f'{MessageToJson(mp)}')
+				print(f'{clean_json(mp)}')
 				print(f'{mp.decoded.payload}')
 
 			elif mp.decoded.portnum == portnums_pb2.TELEMETRY_APP:
 				telemetry = telemetry_pb2.Telemetry()
 				telemetry.ParseFromString(mp.decoded.payload)
-				json_packet['decoded']['payload'] = json.loads(MessageToJson(telemetry))
+				json_packet['decoded']['payload'] = clean_json(telemetry)
-				print(json.dumps(json_packet))
+				print(f'{json.dumps(json_packet)}')
 
 			elif mp.decoded.portnum == portnums_pb2.TEXT_MESSAGE_APP:
 				text_payload = mp.decoded.payload.decode('utf-8')
 				json_packet['decoded']['payload'] = text_payload
-				print(json.dumps(json_packet))
+				print(f'{json.dumps(json_packet)}')
 
 			elif mp.decoded.portnum == portnums_pb2.TEXT_MESSAGE_COMPRESSED_APP:
 				data = mesh_pb2.TextMessageCompressed()
 				data.ParseFromString(mp.decoded.payload)
-				print(f'{MessageToJson(data)}')
+				msg = json.dumps(clean_json(data))
+				print(f'{msg}')
 
 			elif mp.decoded.portnum == portnums_pb2.TRACEROUTE_APP:
-					routeDiscovery = mesh_pb2.RouteDiscovery()
+				routeDiscovery = mesh_pb2.RouteDiscovery()
-					routeDiscovery.ParseFromString(mp.decoded.payload)
+				routeDiscovery.ParseFromString(mp.decoded.payload)
-					json_packet['decoded']['payload'] = json.loads(MessageToJson(routeDiscovery))
+				json_packet['decoded']['payload'] = clean_json(routeDiscovery)
-					print(json.dumps(json_packet))
+				print(f'{json.dumps(json_packet)}')
+
+			elif mp.decoded.portnum == portnums_pb2.UNKNOWN_APP:
+				print(f'{clean_json(mp)}')
 
 			elif mp.decoded.portnum == portnums_pb2.WAYPOINT_APP:
 				data = mesh_pb2.Waypoint()
 				data.ParseFromString(mp.decoded.payload)
-				print(f'{MessageToJson(data)}')
+				msg = json.dumps(clean_json(data))
+				print(f'{msg}')
 
 			elif mp.decoded.portnum == portnums_pb2.ZPS_APP:
 				data = mesh_pb2.Zps()
 				data.ParseFromString(mp.decoded.payload)
-				print(f'{MessageToJson(data)}')
+				msg = json.dumps(clean_json(data))
+				print(f'{msg}')
 
 			else:
 				print(f'UNKNOWN: Received Portnum name: {portnum_name}')
-				print(f'UNKNOWN: {MessageToJson(mp)}')
+				msg = json.dumps(clean_json(mp))
+				print(f'UNKNOWN: {msg}')
 
 		except Exception as e:
 			print(f'Error processing message: {e}')
-			print(mp)
+			print(f'Topic: {msg.topic}')
+			print(f'Payload: {msg.payload}')
+
+
+	def event_mqtt_disconnect(self, client, userdata, rc, packet_from_broker=None, properties=None, reason_code=None):
+		'''Callback for when the client disconnects from the server.'''
+		print(f'Disconnected with result code: {rc}')
+		while True:
+			print('Attempting to reconnect...')
+			try:
+				client.reconnect()
+			except Exception as e:
+				print(f'Error reconnecting to MQTT broker: {e}')
+				time.sleep(5)
+			else:
+				print('Reconnected to MQTT broker')
+				break
 
 
 def main():
@@ -324,4 +409,4 @@ def main():
 
 
 if __name__ == '__main__':
-    main() 
+    main()