diff --git a/README.md b/README.md index 877247c..a2ea3a4 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,7 @@ sudo ./venv/bin/python agent.py ws://DASHBOARD_IP:8888/ws/agent Open the dashboard in a browser, click Settings. Enter your Gotify server URL and app token, hit Test, then Save. Alert thresholds for temperature, space usage, SMART failures, and pool health are all configured from the same panel. - +ws://10.0.0.34:8888/ws/agent ## What It Monitors - Fleet overview with all connected servers, health status, storage usage, alert counts @@ -100,8 +100,6 @@ Open the dashboard in a browser, click Settings. Enter your Gotify server URL an - ZFS datasets *(used, available, referenced, compression ratio, quotas)* - Snapshots *(name, used, referenced, creation time)* - Live I/O charts *(per-disk throughput, IOPS, read/write latency, busy%)* - - ARC cache stats *(size, hit rate, MRU/MFU, L2ARC)* - - Temperature charts *(per-disk, live)* - - Disk details *(model, serial, firmware, capacity, RPM, protocol, health score 0-100, full SMART attributes, SAS error counters, grown defects)* + - Disk details *(model, serial, firmware, capacity, RPM, temperature, protocol, health score 0-100, full SMART attributes, SAS error counters, grown defects)* - SMART self-test triggering from the UI - Alerts pushed to Gotify with configurable cooldowns \ No newline at end of file diff --git a/agent/agent.py b/agent/agent.py index 1d44c1d..259e106 100644 --- a/agent/agent.py +++ b/agent/agent.py @@ -57,14 +57,11 @@ cache = { 'datasets' : [], 'snapshots' : [], 'io_rates' : {}, - 'arc' : None, 'pool_map' : {}, - 'temps' : {}, 'system_info' : {}, } _prev_diskstats = {} -_prev_arcstats = {} # ── Helpers ────────────────────────────────────────────────────────────────── @@ -97,8 +94,43 @@ def detect_capabilities(): # ── System Info ────────────────────────────────────────────────────────────── +def collect_dimm_info(): + '''Collect DIMM slot details from dmidecode.''' + + out, _, rc = run_cmd(['dmidecode', '-t', 'memory']) + if rc != 0: + return [] + dimms = [] + for block in out.split('Memory Device')[1:]: + d = {} + for line in block.splitlines(): + line = line.strip() + if ':' not in line: + continue + key, val = line.split(':', 1) + d[key.strip()] = val.strip() + if not d: + continue + size_str = d.get('Size', '') + populated = bool(size_str) and 'No Module Installed' not in size_str + dimms.append({ + 'slot' : d.get('Locator', ''), + 'size' : size_str if populated else '', + 'type' : d.get('Type', ''), + 'speed' : d.get('Speed', ''), + 'configured_speed' : d.get('Configured Memory Speed', d.get('Configured Clock Speed', '')), + 'manufacturer' : d.get('Manufacturer', ''), + 'part_number' : d.get('Part Number', '').strip(), + 'serial' : d.get('Serial Number', ''), + 'form_factor' : d.get('Form Factor', ''), + 'rank' : d.get('Rank', ''), + 'populated' : populated, + }) + return dimms + + def collect_system_info(): - '''Collect hostname, kernel, ZFS version, uptime, RAM, and CPU info from /proc and /sys.''' + '''Collect hostname, kernel, ZFS version, uptime, RAM, CPU, and DIMM info.''' info = { 'hostname' : capabilities['hostname'], @@ -109,6 +141,7 @@ def collect_system_info(): 'ram_available' : 0, 'cpu_model' : '', 'cpu_count' : os.cpu_count() or 0, + 'dimms' : [], } out, _, rc = run_cmd(['uname', '-r']) if rc == 0: @@ -144,6 +177,7 @@ def collect_system_info(): break except Exception: pass + info['dimms'] = collect_dimm_info() return info @@ -185,40 +219,6 @@ def compute_health_score(disk: dict): # ── Fast Temperature ───────────────────────────────────────────────────────── -def collect_temps_fast(): - '''Read disk temperatures from /sys/block hwmon entries without shelling out.''' - - temps = {} - try: - for name in os.listdir('/sys/block'): - if not re.match(r'^(sd[a-z]+|nvme\d+n\d+)$', name): - continue - for base in [Path(f'/sys/block/{name}/device/hwmon'), Path(f'/sys/block/{name}/device')]: - if not base.exists(): - continue - found = False - try: - entries = sorted(base.iterdir()) - except OSError: - continue - for item in entries: - if not item.name.startswith('hwmon'): - continue - tf = item / 'temp1_input' - if tf.exists(): - try: - with open(tf) as f: - temps[name] = int(f.read().strip()) // 1000 - found = True - except (ValueError, OSError): - pass - break - if found: - break - except OSError: - pass - return temps - # ── Disk & SMART Collection ───────────────────────────────────────────────── @@ -453,7 +453,7 @@ def parse_pool_status(text: str): 'read' : parts[2] if len(parts) > 2 else '0', 'write' : parts[3] if len(parts) > 3 else '0', 'cksum' : parts[4] if len(parts) > 4 else '0', - 'indent': len(line) - len(line.lstrip('\t')), + 'indent': len(line) - len(line.lstrip()), }) return ' '.join(scan_lines), vdevs, errors_summary @@ -502,7 +502,7 @@ def collect_datasets_and_snapshots(): return datasets, snapshots -# ── I/O & ARC Collection ──────────────────────────────────────────────────── +# ── I/O Collection ─────────────────────────────────────────────────────────── def collect_iostat(): '''Read /proc/diskstats and compute per-disk I/O rates from deltas.''' @@ -556,62 +556,6 @@ def collect_iostat(): return rates -def collect_arc_stats(): - '''Read /proc/spl/kstat/zfs/arcstats and compute ARC hit rates.''' - - global _prev_arcstats - raw = {} - try: - with open('/proc/spl/kstat/zfs/arcstats') as f: - for line in f: - parts = line.split() - if len(parts) >= 3: - try: - raw[parts[0]] = int(parts[2]) - except ValueError: - pass - except FileNotFoundError: - return None - - if not raw: - return None - - hits = raw.get('hits', 0) - misses = raw.get('misses', 0) - total = hits + misses - lifetime_rate = round(hits / total * 100, 2) if total > 0 else 0 - - if _prev_arcstats: - dh = hits - _prev_arcstats.get('hits', 0) - dm = misses - _prev_arcstats.get('misses', 0) - dt = dh + dm - hit_rate = round(dh / dt * 100, 2) if dt > 0 else lifetime_rate - else: - hit_rate = lifetime_rate - - _prev_arcstats = {'hits': hits, 'misses': misses} - - return { - 'size' : raw.get('size', 0), - 'max_size' : raw.get('c_max', 0), - 'min_size' : raw.get('c_min', 0), - 'target_size' : raw.get('c', 0), - 'hits' : hits, - 'misses' : misses, - 'hit_rate' : hit_rate, - 'lifetime_hit_rate' : lifetime_rate, - 'mru_size' : raw.get('mru_size', 0), - 'mfu_size' : raw.get('mfu_size', 0), - 'anon_size' : raw.get('anon_size', 0), - 'metadata_size' : raw.get('arc_meta_used', 0), - 'demand_hits' : raw.get('demand_data_hits', 0) + raw.get('demand_metadata_hits', 0), - 'prefetch_hits' : raw.get('prefetch_data_hits', 0) + raw.get('prefetch_metadata_hits', 0), - 'l2_hits' : raw.get('l2_hits', 0), - 'l2_misses' : raw.get('l2_misses', 0), - 'l2_size' : raw.get('l2_size', 0), - 'l2_asize' : raw.get('l2_asize', 0), - } - # ── Background Worker ──────────────────────────────────────────────────────── @@ -624,18 +568,10 @@ def background_worker(): while True: try: - io_rates = collect_iostat() - arc = collect_arc_stats() - fast_temps = collect_temps_fast() + io_rates = collect_iostat() with lock: - for d in cache.get('disks', []): - name = d.get('name', '') - if name not in fast_temps and d.get('temperature') is not None: - fast_temps[name] = d['temperature'] cache['io_rates'] = io_rates - cache['arc'] = arc - cache['temps'] = fast_temps if tick % (POOL_INTERVAL // IO_INTERVAL) == 0: pools = collect_pools() @@ -671,8 +607,7 @@ async def ws_sender(ws): ''' with lock: - io_msg = json.dumps({'type': 'io', 'ts': time.time(), 'rates': cache['io_rates'], 'pool_map': cache['pool_map'], 'temps': cache['temps']}) - arc_msg = json.dumps({'type': 'arc', 'ts': time.time(), 'arc': cache['arc']}) if cache['arc'] else None + io_msg = json.dumps({'type': 'io', 'ts': time.time(), 'rates': cache['io_rates'], 'pool_map': cache['pool_map']}) pools_msg = json.dumps({'type': 'pools', 'ts': time.time(), 'pools': cache['pools']}) datasets_msg = json.dumps({'type': 'datasets', 'ts': time.time(), 'datasets': cache['datasets']}) snaps_msg = json.dumps({'type': 'snapshots', 'ts': time.time(), 'snapshots': cache['snapshots']}) @@ -685,8 +620,6 @@ async def ws_sender(ws): await ws.send(snaps_msg) await ws.send(disks_msg) await ws.send(io_msg) - if arc_msg: - await ws.send(arc_msg) tick = 0 while True: @@ -694,14 +627,9 @@ async def ws_sender(ws): tick += 1 with lock: - io_msg = json.dumps({'type': 'io', 'ts': time.time(), 'rates': cache['io_rates'], 'pool_map': cache['pool_map'], 'temps': cache['temps']}) + io_msg = json.dumps({'type': 'io', 'ts': time.time(), 'rates': cache['io_rates'], 'pool_map': cache['pool_map']}) await ws.send(io_msg) - with lock: - arc = cache['arc'] - if arc: - await ws.send(json.dumps({'type': 'arc', 'ts': time.time(), 'arc': arc})) - if tick % (POOL_INTERVAL // IO_INTERVAL) == 0: with lock: pools_msg = json.dumps({'type': 'pools', 'ts': time.time(), 'pools': cache['pools']}) diff --git a/dashboard/dashboard.py b/dashboard/dashboard.py index 8f700ff..79809b3 100644 --- a/dashboard/dashboard.py +++ b/dashboard/dashboard.py @@ -66,9 +66,6 @@ class AgentState: self.history = { 'timestamps' : deque(maxlen=HISTORY_SIZE), 'io' : {}, - 'arc_size' : deque(maxlen=HISTORY_SIZE), - 'arc_hit_rate' : deque(maxlen=HISTORY_SIZE), - 'temps' : {}, } self.alerts_active = [] self.alert_log = deque(maxlen=200) @@ -266,20 +263,6 @@ def update_history(agent: 'AgentState', data: dict): h['read_iops'].append(rates.get('read_iops', 0)) h['write_iops'].append(rates.get('write_iops', 0)) - for dname, temp in data.get('temps', {}).items(): - if dname not in agent.history['temps']: - agent.history['temps'][dname] = deque(maxlen=HISTORY_SIZE) - agent.history['temps'][dname].append(temp) - - arc_msg = agent.current.get('arc', {}) - arc = arc_msg.get('arc') if isinstance(arc_msg, dict) else None - if arc: - agent.history['arc_size'].append(arc.get('size', 0)) - agent.history['arc_hit_rate'].append(arc.get('hit_rate', 0)) - - elif msg_type == 'arc': - pass - def serialize_history(h: dict): '''Convert history deques to plain lists for JSON serialization. @@ -290,9 +273,6 @@ def serialize_history(h: dict): return { 'timestamps' : list(h['timestamps']), 'io' : {dn: {k: list(v) for k, v in s.items()} for dn, s in h['io'].items()}, - 'arc_size' : list(h['arc_size']), - 'arc_hit_rate' : list(h['arc_hit_rate']), - 'temps' : {dn: list(v) for dn, v in h['temps'].items()}, } diff --git a/dashboard/setup.sh b/dashboard/setup.sh index 5c16810..b70f075 100755 --- a/dashboard/setup.sh +++ b/dashboard/setup.sh @@ -58,4 +58,4 @@ echo " Open: http://$(hostname -I | awk '{print $1}'):8888" echo " Status: systemctl status ${SERVICE_NAME}" echo " Logs: journalctl -u ${SERVICE_NAME} -f" echo " Stop: systemctl stop ${SERVICE_NAME}" -echo " Restart: systemctl restart ${SERVICE_NAME}" +echo " Restart: systemctl restart ${SERVICE_NAME}" \ No newline at end of file diff --git a/dashboard/templates/index.html b/dashboard/templates/index.html index 4ec286e..e65901a 100644 --- a/dashboard/templates/index.html +++ b/dashboard/templates/index.html @@ -30,17 +30,30 @@ section{margin-bottom:1.5rem} .section-title .badge{font-size:.62rem;padding:.1rem .4rem;border-radius:99px;font-weight:500} .section-sub{font-size:.72rem;color:var(--text2);margin-top:-.35rem;margin-bottom:.65rem} .grid{display:grid;gap:.65rem} -.g4{grid-template-columns:repeat(4,1fr)}.g3{grid-template-columns:repeat(3,1fr)}.g2{grid-template-columns:repeat(2,1fr)} +.g5{grid-template-columns:repeat(5,1fr)}.g4{grid-template-columns:repeat(4,1fr)}.g3{grid-template-columns:repeat(3,1fr)}.g2{grid-template-columns:repeat(2,1fr)} .card{background:var(--surface);border:1px solid var(--border);border-radius:var(--radius);padding:.9rem 1rem} .card-label{font-size:.65rem;color:var(--text2);text-transform:uppercase;letter-spacing:.06em;margin-bottom:.15rem} .card-value{font-size:1.4rem;font-weight:700;font-family:var(--mono);letter-spacing:-.03em} .card-sub{font-size:.7rem;color:var(--text2);margin-top:.1rem} +.overview-bar{display:flex;align-items:center;gap:1.5rem;background:var(--surface);border:1px solid var(--border);border-radius:var(--radius);padding:.6rem 1.2rem;margin-bottom:1rem;flex-wrap:wrap;font-size:.78rem;font-family:var(--mono)} +.overview-bar .ob-item{display:flex;align-items:center;gap:.35rem;white-space:nowrap} +.overview-bar .ob-label{color:var(--text2);font-size:.65rem;text-transform:uppercase;letter-spacing:.04em} +.overview-bar .ob-val{font-weight:700} +.overview-bar .ob-sep{color:var(--border);font-size:.7rem} +.overview-bar .ob-bar{width:80px;height:4px;border-radius:2px;background:var(--surface3);overflow:hidden;margin-left:.25rem} +.overview-bar .ob-bar-fill{height:100%;border-radius:2px;transition:width .5s} .stat-bar{height:4px;border-radius:2px;background:var(--surface3);margin-top:.35rem;overflow:hidden} .stat-bar-fill{height:100%;border-radius:2px;transition:width .5s} .fleet-card{cursor:pointer;transition:border-color .2s,transform .15s} .fleet-card:hover{border-color:var(--accent);transform:translateY(-2px)} .fleet-card.offline{opacity:.6} -.pool-header{display:flex;justify-content:space-between;align-items:center;margin-bottom:.6rem} +.pool-header{display:flex;justify-content:space-between;align-items:center;cursor:pointer;user-select:none} +.pool-header:hover{opacity:.8} +.pool-header .pool-toggle{font-size:.6rem;color:var(--text2);margin-right:.4rem;transition:transform .2s} +.pool-header.open .pool-toggle{transform:rotate(90deg)} +.pool-body{overflow:hidden;max-height:0;transition:max-height .3s ease} +.pool-body.open{max-height:800px} +.pool-body-inner{padding-top:.6rem} .pool-name{font-size:.95rem;font-weight:600;font-family:var(--mono)} .hb{font-size:.65rem;padding:.12rem .45rem;border-radius:99px;font-weight:600;text-transform:uppercase;letter-spacing:.04em} .h-on{background:rgba(34,197,94,.12);color:var(--green)}.h-deg{background:rgba(234,179,8,.12);color:var(--yellow)}.h-flt{background:rgba(239,68,68,.12);color:var(--red)}.h-unk{background:rgba(122,135,158,.12);color:var(--text2)} @@ -52,13 +65,24 @@ section{margin-bottom:1.5rem} .vr .zero{color:var(--surface3)}.vr .err{color:var(--red);font-weight:600} .chart-container{position:relative;height:195px;width:100%} .chart-unavail{display:flex;align-items:center;justify-content:center;height:195px;color:var(--text2);font-size:.8rem} -.disk-header{display:flex;justify-content:space-between;align-items:flex-start;margin-bottom:.5rem} -.disk-name{font-size:.95rem;font-weight:600;font-family:var(--mono)} .dtb{font-size:.62rem;padding:.1rem .4rem;border-radius:99px;font-weight:600;text-transform:uppercase;letter-spacing:.04em;background:rgba(59,130,246,.12);color:var(--accent)} .dtb-sas{background:rgba(168,85,247,.12);color:var(--purple)}.dtb-nv{background:rgba(6,182,212,.12);color:var(--cyan)} -.disk-stats{display:grid;grid-template-columns:repeat(4,1fr);gap:.35rem;margin-bottom:.5rem} -.ds{padding:.35rem .45rem;background:var(--surface2);border-radius:6px} -.ds-l{font-size:.58rem;color:var(--text2);text-transform:uppercase}.ds-v{font-size:.8rem;font-weight:600;font-family:var(--mono)} +.disk-tbl{table-layout:auto!important} +.dimm-tbl{table-layout:auto!important} +.dimm-tbl td.dimm-empty{color:var(--text2);font-style:italic} +.ds-tbl{table-layout:auto!important} +.ds-tbl tr.ds-row{cursor:pointer}.ds-tbl tr.ds-row:hover td{background:var(--surface2)}.ds-tbl tr.ds-row.expanded td{background:var(--surface2)} +.ds-detail-row td{padding:0!important;border-bottom:1px solid var(--border)} +.ds-tbl th:nth-child(2),.ds-tbl td:nth-child(2),.ds-tbl th:nth-child(3),.ds-tbl td:nth-child(3),.ds-tbl th:nth-child(4),.ds-tbl td:nth-child(4),.ds-tbl th:nth-child(5),.ds-tbl td:nth-child(5),.ds-tbl th:nth-child(6),.ds-tbl td:nth-child(6){width:1%;white-space:nowrap} +.disk-tbl th,.disk-tbl td{white-space:nowrap} +.disk-tbl th:nth-child(2),.disk-tbl td:nth-child(2),.disk-tbl th:nth-child(3),.disk-tbl td:nth-child(3){white-space:nowrap;overflow:hidden;text-overflow:ellipsis;max-width:220px} +.disk-tbl th:nth-child(5),.disk-tbl td:nth-child(5),.disk-tbl th:nth-child(6),.disk-tbl td:nth-child(6),.disk-tbl th:nth-child(7),.disk-tbl td:nth-child(7),.disk-tbl th:nth-child(8),.disk-tbl td:nth-child(8){width:1%;white-space:nowrap} +.disk-tbl th:nth-child(9),.disk-tbl td:nth-child(9),.disk-tbl th:nth-child(10),.disk-tbl td:nth-child(10),.disk-tbl th:nth-child(11),.disk-tbl td:nth-child(11){width:1%;white-space:nowrap} +.disk-tbl tr.disk-row{cursor:pointer}.disk-tbl tr.disk-row:hover td{background:var(--surface2)} +.disk-tbl tr.disk-row.expanded td{background:var(--surface2)} +.disk-tbl tr.disk-row td:first-child{font-weight:600;font-family:var(--mono)} +.disk-detail-row td{padding:0!important;border-bottom:1px solid var(--border)} +.disk-detail-inner{padding:1rem;background:var(--surface2)} .score-badge{font-size:.65rem;padding:.12rem .4rem;border-radius:99px;font-weight:700;font-family:var(--mono)} .tbl{width:100%;font-size:.75rem;border-collapse:collapse;table-layout:fixed} .tbl th{text-align:left;padding:.35rem .5rem;color:var(--text2);border-bottom:1px solid var(--border);font-weight:600;font-size:.65rem;text-transform:uppercase;letter-spacing:.04em;white-space:nowrap;overflow:hidden} @@ -82,7 +106,6 @@ section{margin-bottom:1.5rem} .toggle .slider::before{content:'';position:absolute;width:13px;height:13px;border-radius:50%;left:3px;top:3px;background:var(--text2);transition:all .2s} .toggle input:checked+.slider{background:var(--accent)}.toggle input:checked+.slider::before{transform:translateX(17px);background:#fff} .modal-actions{display:flex;gap:.5rem;margin-top:1rem}.modal-actions .btn{flex:1;text-align:center} -.smart-modal{max-width:820px}.smart-modal h2{display:flex;align-items:center;gap:.5rem;flex-wrap:wrap} .si-grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(125px,1fr));gap:.35rem;margin-bottom:.85rem} .si-item{padding:.35rem .5rem;background:var(--surface2);border-radius:6px} .si-l{font-size:.58rem;color:var(--text2);text-transform:uppercase;letter-spacing:.04em}.si-v{font-size:.8rem;font-weight:600;font-family:var(--mono)} @@ -92,8 +115,8 @@ section{margin-bottom:1.5rem} .st .attr-warn{color:var(--yellow)}.st .attr-crit{color:var(--red);font-weight:600}.st .attr-note{color:var(--text2);font-style:italic;font-size:.62rem} .empty{text-align:center;padding:1.25rem;color:var(--text2);font-size:.82rem} .tc{color:var(--green)}.tw{color:var(--yellow)}.th{color:var(--red)} -@media(max-width:1024px){.g4{grid-template-columns:repeat(2,1fr)}.g2{grid-template-columns:1fr}.pool-stats{grid-template-columns:repeat(3,1fr)}.disk-stats{grid-template-columns:repeat(2,1fr)}} -@media(max-width:640px){nav .nav-links{display:none}main{padding:.75rem}.g4,.g3{grid-template-columns:1fr 1fr}.smart-modal{max-width:98%;padding:1rem}} +@media(max-width:1024px){.g5{grid-template-columns:repeat(3,1fr)}.g4{grid-template-columns:repeat(2,1fr)}.g2{grid-template-columns:1fr}.pool-stats{grid-template-columns:repeat(3,1fr)}.disk-stats{grid-template-columns:repeat(2,1fr)}} +@media(max-width:640px){nav .nav-links{display:none}main{padding:.75rem}.g5,.g4,.g3{grid-template-columns:1fr 1fr}} @@ -104,7 +127,7 @@ section{margin-bottom:1.5rem}
@@ -117,11 +140,15 @@ section{margin-bottom:1.5rem} -
-
ARC Cache
-
Adaptive Replacement Cache — ZFS uses free RAM as a read cache. Hit rate shown is instantaneous (per sample interval).
-
-
-
Hit Rate
-
ARC Size
-
-
-
-
Disk Temperatures
-
-
-
Physical Disks
-
-
ZFS Datasets
-
NameUsedAvailReferCompRatioMount
-
Snapshots 0
NameUsedReferencedCreated
@@ -160,14 +169,19 @@ section{margin-bottom:1.5rem}
Log
+ - diff --git a/ramexample.py b/ramexample.py new file mode 100644 index 0000000..e69de29