intial commit
This commit is contained in:
+21
@@ -0,0 +1,21 @@
|
||||
# byte-compiled / optimized
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
# virtual environments
|
||||
.venv/
|
||||
venv/
|
||||
env/
|
||||
|
||||
# glassbox exported reports
|
||||
glassbox-report-*.json
|
||||
|
||||
# editor / os cruft
|
||||
.vscode/
|
||||
.idea/
|
||||
*.swp
|
||||
.DS_Store
|
||||
|
||||
# logs
|
||||
*.log
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 1002 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 513 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.2 MiB |
Binary file not shown.
|
After Width: | Height: | Size: 468 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 962 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 628 KiB |
+15
@@ -0,0 +1,15 @@
|
||||
# glassbox - live python execution profiler
|
||||
FROM python:3.12-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY . .
|
||||
|
||||
EXPOSE 7000
|
||||
|
||||
# mount the project you want to profile at /work:
|
||||
# docker run -p 7000:7000 -v /path/to/your/project:/work glassbox
|
||||
CMD ["python", "app.py"]
|
||||
@@ -0,0 +1,15 @@
|
||||
ISC License
|
||||
|
||||
Copyright (c) 2026, acidvegas <acid.vegas@acid.vegas>
|
||||
|
||||
Permission to use, copy, modify, and/or distribute this software for any
|
||||
purpose with or without fee is hereby granted, provided that the above
|
||||
copyright notice and this permission notice appear in all copies.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||
@@ -0,0 +1,95 @@
|
||||
# glassbox
|
||||
|
||||
**Live Python execution profiler and visualizer.** Point it at a script or a whole project and watch it run — which line is executing right now, where memory piles up, which functions burn the most CPU, and what's blocking your asyncio loop — all streamed in real time to your browser.
|
||||
|
||||
glassbox runs your target in-process under `sys.setprofile` plus a statistical line sampler and `tracemalloc`, and streams telemetry over websockets to an animated single-page UI.
|
||||
|
||||
---
|
||||
|
||||
## Screenshots
|
||||
|
||||
**Live** — the executing line, per-line heat, live call stack, and leveled output:
|
||||

|
||||
|
||||
**Profile** — function benchmarks (self / CPU / total), CPU timeline, hot lines, and the asyncio event-loop blocker:
|
||||

|
||||
|
||||
**Memory** — RSS / heap timeline, `tracemalloc` allocation hotspots, growth-since-baseline, and per-variable sizes:
|
||||

|
||||
|
||||
**Flame** — a time-proportional call tree, click any frame to zoom:
|
||||

|
||||
|
||||
**Flow** — an animated 3D call graph with blips streaming along the live call path:
|
||||

|
||||
|
||||
**Files** — which file calls into which, with live call blips between files:
|
||||

|
||||
|
||||
---
|
||||
|
||||
## Features
|
||||
|
||||
- **Live source view** — highlights the line executing in real time, with a per-line heat map of where time is spent, and follows execution across files.
|
||||
- **Call stack** — the current frame chain, updated live without strobing.
|
||||
- **Leveled output** — captures `print()` and the `logging` module, tagged and filterable by level (debug/info/warn/error/crit).
|
||||
- **Function benchmarks** — per-function calls, self time, CPU time, and total time, sorted and filterable. Wall − CPU exposes time spent waiting on I/O, sleep, locks or the GIL.
|
||||
- **Hot lines** — the source lines the sampler caught executing most often.
|
||||
- **Memory** — a live RSS / Python-heap timeline, `tracemalloc` allocation hotspots, per-variable deep sizes, and growth-since-baseline for leak spotting.
|
||||
- **asyncio event loop** — times every loop callback and flags the coroutines that block the loop, split into CPU vs. wait.
|
||||
- **Flame graph** — a time-proportional call tree, click to zoom.
|
||||
- **3D call flow** — an animated force-directed graph of the call structure with blips streaming along the live call path.
|
||||
- **File communication graph** — which file calls into which, with live call blips between files.
|
||||
- **Export** — download any run as a JSON report.
|
||||
|
||||
Works on a **single file** or an entire **project** (auto-detects the project root via `.git` / `pyproject.toml` / package layout and profiles every local `.py` under it).
|
||||
|
||||
---
|
||||
|
||||
## Quick start
|
||||
|
||||
```bash
|
||||
git clone <this-repo> glassbox && cd glassbox
|
||||
./setup.sh
|
||||
.venv/bin/python app.py
|
||||
```
|
||||
|
||||
Then open **http://localhost:7000**, browse to a Python file, pick **file** or **project** scope, and hit **run**.
|
||||
|
||||
To try it immediately, point it at the bundled demo `sample_project/main.py` in **project** scope — an async pipeline that fetches Wikipedia and GitHub data, indexes and analyses it, and deliberately exercises every panel (network wait, CPU hotspots, memory growth, deep recursion, and an asyncio loop blocker).
|
||||
|
||||
### Manual install
|
||||
|
||||
```bash
|
||||
pip install -r requirements.txt
|
||||
python app.py
|
||||
```
|
||||
|
||||
Requires Python 3.10+. Frontend libraries (socket.io, highlight.js, three.js, 3d-force-graph) load from a CDN, so no build step or npm is needed.
|
||||
|
||||
---
|
||||
|
||||
## Docker
|
||||
|
||||
```bash
|
||||
docker build -t glassbox .
|
||||
docker run -p 7000:7000 -v /path/to/your/project:/work glassbox
|
||||
```
|
||||
|
||||
Then browse to `/work` inside the picker to reach your mounted project.
|
||||
|
||||
---
|
||||
|
||||
## How it works
|
||||
|
||||
- **Timing** — `sys.setprofile` records call/return events (far cheaper than `settrace`), timing each function with both `perf_counter` (wall) and `thread_time` (CPU); the difference is wait time.
|
||||
- **Line heat** — a background thread samples `sys._current_frames()` every few milliseconds, so the hot-line view is statistical and stays out of the hot path.
|
||||
- **Memory** — `tracemalloc` supplies allocation hotspots and growth; variable sizes use a bounded deep-sizeof.
|
||||
- **asyncio** — `asyncio.events.Handle._run` is wrapped to time every loop callback; long ones are attributed to the coroutine blocking the loop.
|
||||
- **Streaming** — a background task pushes telemetry frames over Flask-SocketIO; heavy analysis runs off the emit path to keep the stream smooth.
|
||||
|
||||
Everything runs in-process, so glassbox sees your real objects and frames — at the cost of running untrusted code in its own process. Only profile code you trust.
|
||||
|
||||
---
|
||||
|
||||
###### Mirrors: [SuperNETs](https://git.supernets.org/acidvegas/) • [GitHub](https://github.com/acidvegas/) • [GitLab](https://gitlab.com/acidvegas/) • [Codeberg](https://codeberg.org/acidvegas/)
|
||||
@@ -0,0 +1,124 @@
|
||||
#!/usr/bin/env python3
|
||||
# glassbox - live python execution profiler and visualizer
|
||||
|
||||
import os
|
||||
|
||||
from flask import Flask, jsonify, render_template, request
|
||||
|
||||
try:
|
||||
from flask_socketio import SocketIO
|
||||
except ImportError:
|
||||
raise SystemExit('missing dependency: flask-socketio (pip install flask-socketio simple-websocket)')
|
||||
|
||||
from profiler import Profiler
|
||||
|
||||
|
||||
app = Flask(__name__)
|
||||
socketio = SocketIO(app, async_mode='threading', cors_allowed_origins='*')
|
||||
|
||||
state = {'profiler': None, 'emitter': False}
|
||||
|
||||
|
||||
def emit_loop(prof: Profiler):
|
||||
'''
|
||||
Background task that streams telemetry frames to connected clients.
|
||||
|
||||
:param prof: the active profiler whose state is polled and broadcast
|
||||
'''
|
||||
|
||||
socketio.emit('started', {'path': prof.path, 'args': prof.script_args, 'scope': prof.scope, 'root': prof.root})
|
||||
while True:
|
||||
logs = prof.drain_logs()
|
||||
if logs:
|
||||
socketio.emit('logs', {'lines': logs})
|
||||
socketio.emit('telemetry', prof.snapshot())
|
||||
if not prof.running:
|
||||
break
|
||||
socketio.sleep(0.08)
|
||||
for logs in (prof.drain_logs(),):
|
||||
if logs:
|
||||
socketio.emit('logs', {'lines': logs})
|
||||
final = prof.snapshot()
|
||||
prof.stop_tracing()
|
||||
socketio.emit('telemetry', final)
|
||||
socketio.emit('finished', {'exit_code': prof.exit_code, 'error': prof.error, 'elapsed': final['elapsed']})
|
||||
state['emitter'] = False
|
||||
|
||||
|
||||
@app.route('/')
|
||||
def index() -> str:
|
||||
'''Serve the single page application shell.'''
|
||||
|
||||
return render_template('index.html')
|
||||
|
||||
|
||||
@app.route('/api/browse')
|
||||
def browse():
|
||||
'''List directories and python files for the file picker.'''
|
||||
|
||||
path = request.args.get('path') or os.path.expanduser('~')
|
||||
path = os.path.abspath(path)
|
||||
if not os.path.isdir(path):
|
||||
path = os.path.dirname(path)
|
||||
dirs, files = [], []
|
||||
try:
|
||||
for name in sorted(os.listdir(path), key=str.lower):
|
||||
full = os.path.join(path, name)
|
||||
if name.startswith('.'):
|
||||
continue
|
||||
if os.path.isdir(full):
|
||||
dirs.append({'name': name, 'path': full})
|
||||
elif name.endswith('.py'):
|
||||
files.append({'name': name, 'path': full})
|
||||
except PermissionError:
|
||||
return jsonify({'error': 'permission denied', 'path': path, 'parent': os.path.dirname(path), 'dirs': [], 'files': []})
|
||||
return jsonify({'path': path, 'parent': os.path.dirname(path), 'dirs': dirs, 'files': files})
|
||||
|
||||
|
||||
@app.route('/api/source')
|
||||
def source():
|
||||
'''Return the raw source lines of a target file for the code viewer.'''
|
||||
|
||||
path = request.args.get('path', '')
|
||||
path = os.path.abspath(path)
|
||||
if not (os.path.isfile(path) and path.endswith('.py')):
|
||||
return jsonify({'error': 'not a python file'})
|
||||
try:
|
||||
with open(path, 'r') as fh:
|
||||
lines = fh.read().split('\n')
|
||||
except Exception as ex:
|
||||
return jsonify({'error': str(ex)})
|
||||
return jsonify({'path': path, 'lines': lines})
|
||||
|
||||
|
||||
@socketio.on('start')
|
||||
def on_start(data: dict):
|
||||
'''
|
||||
Handle a run request from the client, launching a fresh profiler.
|
||||
|
||||
:param data: payload holding the target path and optional argument string
|
||||
'''
|
||||
|
||||
prev = state.get('profiler')
|
||||
if prev is not None and prev.running:
|
||||
socketio.emit('error', {'message': 'a run is already in progress'})
|
||||
return
|
||||
path = os.path.abspath(data.get('path', ''))
|
||||
if not (os.path.isfile(path) and path.endswith('.py')):
|
||||
socketio.emit('error', {'message': f'not a python file: {path}'})
|
||||
return
|
||||
raw_args = data.get('args', '') or ''
|
||||
script_args = raw_args.split()
|
||||
scope = data.get('scope', 'project')
|
||||
root = data.get('root') or None
|
||||
prof = Profiler(path, script_args, scope=scope, root=root)
|
||||
state['profiler'] = prof
|
||||
prof.start()
|
||||
if not state['emitter']:
|
||||
state['emitter'] = True
|
||||
socketio.start_background_task(emit_loop, prof)
|
||||
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
socketio.run(app, host='0.0.0.0', port=7000, allow_unsafe_werkzeug=True)
|
||||
+751
@@ -0,0 +1,751 @@
|
||||
#!/usr/bin/env python3
|
||||
# glassbox - live python execution profiler engine
|
||||
|
||||
import asyncio
|
||||
import builtins
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import tracemalloc
|
||||
|
||||
from collections import deque
|
||||
|
||||
try:
|
||||
import psutil
|
||||
except ImportError:
|
||||
raise SystemExit('missing dependency: psutil (pip install psutil)')
|
||||
|
||||
|
||||
CONTAINER_TYPES = (list, tuple, set, frozenset, dict)
|
||||
SAMPLE_DT = 0.004 # statistical line sampler period in seconds
|
||||
BLOCK_DT = 0.012 # asyncio callback longer than this is treated as blocking the loop
|
||||
EXCLUDE_DIRS = ('/site-packages/', '/dist-packages/', '/.venv/', '/venv/', '/.tox/', '/node_modules/', '/__pycache__/')
|
||||
TOOL_FILE = os.path.realpath(__file__) # never profile glassbox's own tracing code (runs on the target thread)
|
||||
|
||||
|
||||
def deep_sizeof(obj: object, seen: set = None, depth: int = 0) -> int:
|
||||
'''
|
||||
Recursively estimate the memory footprint of an object in bytes.
|
||||
|
||||
:param obj: the object to measure
|
||||
:param seen: set of already counted object ids to avoid double counting
|
||||
:param depth: current recursion depth, bounded to keep the cost sane
|
||||
'''
|
||||
|
||||
if seen is None:
|
||||
seen = set()
|
||||
oid = id(obj)
|
||||
if oid in seen:
|
||||
return 0
|
||||
seen.add(oid)
|
||||
try:
|
||||
size = sys.getsizeof(obj)
|
||||
except Exception:
|
||||
return 0
|
||||
if depth > 3 or len(seen) > 12_000: # hard budget: keep a single variable from stalling the profiler
|
||||
return size
|
||||
try:
|
||||
if isinstance(obj, dict):
|
||||
for k, v in list(obj.items())[:512]:
|
||||
size += deep_sizeof(k, seen, depth + 1)
|
||||
size += deep_sizeof(v, seen, depth + 1)
|
||||
elif isinstance(obj, (list, tuple, set, frozenset)):
|
||||
for item in list(obj)[:512]:
|
||||
size += deep_sizeof(item, seen, depth + 1)
|
||||
elif hasattr(obj, '__dict__'):
|
||||
for value in list(vars(obj).values())[:512]:
|
||||
size += deep_sizeof(value, seen, depth + 1)
|
||||
except Exception:
|
||||
pass
|
||||
return size
|
||||
|
||||
|
||||
def project_root(path: str) -> str:
|
||||
'''
|
||||
Infer a sensible project root for a launched file: the git/repo root if present, else the top of its package.
|
||||
|
||||
:param path: path to the launched python file
|
||||
'''
|
||||
|
||||
start = os.path.dirname(os.path.abspath(path))
|
||||
probe = start
|
||||
while True:
|
||||
if any(os.path.exists(os.path.join(probe, marker)) for marker in ('.git', 'pyproject.toml', 'setup.py', 'setup.cfg')):
|
||||
return probe
|
||||
parent = os.path.dirname(probe)
|
||||
if parent == probe:
|
||||
break
|
||||
probe = parent
|
||||
root = start
|
||||
while os.path.isfile(os.path.join(root, '__init__.py')):
|
||||
parent = os.path.dirname(root)
|
||||
if parent == root:
|
||||
break
|
||||
root = parent
|
||||
return root
|
||||
|
||||
|
||||
def type_name(obj: object) -> str:
|
||||
'''
|
||||
Return a short human readable type label for an object.
|
||||
|
||||
:param obj: the object to inspect
|
||||
'''
|
||||
|
||||
return type(obj).__name__
|
||||
|
||||
|
||||
class StreamCapture(io.TextIOBase):
|
||||
'''Line buffered stdout/stderr replacement that feeds a shared queue.'''
|
||||
|
||||
def __init__(self, stream: str, sink: deque, lock: threading.Lock, mirror):
|
||||
self.stream = stream
|
||||
self.sink = sink
|
||||
self.lock = lock
|
||||
self.mirror = mirror
|
||||
self.buffer_text = ''
|
||||
|
||||
|
||||
def write(self, text: str) -> int:
|
||||
'''
|
||||
Capture written text, splitting on newlines into discrete log records.
|
||||
|
||||
:param text: chunk of text written by the running program
|
||||
'''
|
||||
|
||||
if self.mirror is not None:
|
||||
try:
|
||||
self.mirror.write(text)
|
||||
except Exception:
|
||||
pass
|
||||
self.buffer_text += text
|
||||
while '\n' in self.buffer_text:
|
||||
line, self.buffer_text = self.buffer_text.split('\n', 1)
|
||||
with self.lock:
|
||||
self.sink.append({'stream': self.stream, 'level': None, 'text': line})
|
||||
return len(text)
|
||||
|
||||
|
||||
def flush(self):
|
||||
'''Flush any partial buffered line into the sink.'''
|
||||
|
||||
if self.buffer_text:
|
||||
with self.lock:
|
||||
self.sink.append({'stream': self.stream, 'level': None, 'text': self.buffer_text})
|
||||
self.buffer_text = ''
|
||||
|
||||
|
||||
class LogSink(logging.Handler):
|
||||
'''Logging handler that funnels records into the shared log queue with their level name intact.'''
|
||||
|
||||
def __init__(self, sink: deque, sink_lock: threading.Lock, mirror):
|
||||
super().__init__()
|
||||
self.sink = sink
|
||||
self.sink_lock = sink_lock # not named self.lock - that is Handler's own reentrant lock
|
||||
self.mirror = mirror
|
||||
|
||||
|
||||
def emit(self, record):
|
||||
'''
|
||||
Append a formatted record tagged with its level so the client need not guess it.
|
||||
|
||||
:param record: the logging record to capture
|
||||
'''
|
||||
|
||||
try:
|
||||
msg = self.format(record)
|
||||
except Exception:
|
||||
msg = record.getMessage()
|
||||
if self.mirror is not None:
|
||||
try:
|
||||
self.mirror.write(f'{record.levelname} {msg}\n')
|
||||
except Exception:
|
||||
pass
|
||||
with self.sink_lock:
|
||||
self.sink.append({'stream': 'stdout', 'level': record.levelname, 'text': msg})
|
||||
|
||||
|
||||
class Profiler:
|
||||
'''Traces a target python file, collecting per line, per function and memory telemetry.'''
|
||||
|
||||
def __init__(self, path: str, script_args: list, scope: str = 'project', root: str = None):
|
||||
self.path = os.path.realpath(os.path.abspath(path))
|
||||
self.script_args = script_args
|
||||
self.scope = 'file' if scope == 'file' else 'project'
|
||||
self.root = os.path.realpath(root) if root else project_root(self.path)
|
||||
self.proc = psutil.Process()
|
||||
|
||||
self._scope_cache = {} # co_filename -> bool in scope
|
||||
self._rel_cache = {} # co_filename -> short display path
|
||||
self._abs_cache = {} # co_filename -> canonical realpath
|
||||
|
||||
self.func_stats = {} # key -> stats dict
|
||||
self.call_edges = {} # 'caller|callee' -> edge dict
|
||||
self.stats_lock = threading.Lock()
|
||||
self.call_stack = [] # timing frames
|
||||
self.start_time = 0.0
|
||||
self.end_time = 0.0
|
||||
|
||||
self.logs = deque(maxlen=5000)
|
||||
self.log_lock = threading.Lock()
|
||||
|
||||
self.line_samples = {} # abs file -> {lineno -> statistical hit count}
|
||||
self.line_lock = threading.Lock()
|
||||
self.sampler = None
|
||||
|
||||
self._hotspots = [] # cached heavy analysis, refreshed off the emit path
|
||||
self._vars = ('', '', [])
|
||||
self._growth = [] # tracemalloc size diff vs a baseline snapshot (leak detection)
|
||||
self._baseline = None
|
||||
self.analyzer = None
|
||||
|
||||
self.block_events = {} # asyncio loop blocking: location key -> stats
|
||||
self.block_lock = threading.Lock()
|
||||
self.loop_seen = False # True once an asyncio callback has run
|
||||
|
||||
self.thread = None
|
||||
self.thread_id = None
|
||||
self.running = False
|
||||
self.error = None
|
||||
self.exit_code = None
|
||||
|
||||
|
||||
def _abs(self, filename: str) -> str:
|
||||
'''
|
||||
Return the cached canonical absolute path for a code file.
|
||||
|
||||
:param filename: a frame co_filename
|
||||
'''
|
||||
|
||||
hit = self._abs_cache.get(filename)
|
||||
if hit is not None:
|
||||
return hit
|
||||
try:
|
||||
real = os.path.realpath(filename)
|
||||
except Exception:
|
||||
real = filename
|
||||
self._abs_cache[filename] = real
|
||||
return real
|
||||
|
||||
|
||||
def _in_scope(self, filename: str) -> bool:
|
||||
'''
|
||||
Decide whether a code file should be profiled under the active scope.
|
||||
|
||||
In file scope only the launched file counts; in project scope any file under the project root
|
||||
counts, minus virtualenvs, site-packages and caches.
|
||||
|
||||
:param filename: a frame co_filename
|
||||
'''
|
||||
|
||||
hit = self._scope_cache.get(filename)
|
||||
if hit is not None:
|
||||
return hit
|
||||
ok = False
|
||||
if filename and not filename.startswith('<'):
|
||||
real = self._abs(filename)
|
||||
if real == TOOL_FILE:
|
||||
ok = False
|
||||
elif self.scope == 'file':
|
||||
ok = real == self.path
|
||||
else:
|
||||
ok = (real == self.path or real.startswith(self.root + os.sep)) and not any(seg in real for seg in EXCLUDE_DIRS)
|
||||
self._scope_cache[filename] = ok
|
||||
return ok
|
||||
|
||||
|
||||
def _rel(self, filename: str) -> str:
|
||||
'''
|
||||
Return a short display path for a file, relative to the project root when possible.
|
||||
|
||||
:param filename: a frame co_filename
|
||||
'''
|
||||
|
||||
hit = self._rel_cache.get(filename)
|
||||
if hit is not None:
|
||||
return hit
|
||||
real = self._abs(filename)
|
||||
try:
|
||||
rel = os.path.relpath(real, self.root)
|
||||
if rel.startswith('..'):
|
||||
rel = os.path.basename(real)
|
||||
except Exception:
|
||||
rel = os.path.basename(real)
|
||||
self._rel_cache[filename] = rel
|
||||
return rel
|
||||
|
||||
|
||||
def _key(self, code) -> str:
|
||||
'''
|
||||
Build a stable identifier for a code object, unique across files.
|
||||
|
||||
:param code: the frame code object being profiled
|
||||
'''
|
||||
|
||||
return f'{self._rel(code.co_filename)}:{code.co_name}:{code.co_firstlineno}'
|
||||
|
||||
|
||||
def _profile(self, frame, event: str, arg):
|
||||
'''
|
||||
Profile callback, records call counts and exclusive/cumulative timing for target frames only.
|
||||
|
||||
Uses sys.setprofile so it fires on call/return without the per line overhead of settrace,
|
||||
keeping heavy workloads near native speed. The current line is sampled separately by the emitter.
|
||||
|
||||
:param frame: the current stack frame
|
||||
:param event: profile event name (call/return/c_call/c_return/c_exception)
|
||||
:param arg: event specific argument
|
||||
'''
|
||||
|
||||
if event == 'call':
|
||||
code = frame.f_code
|
||||
if not self._in_scope(code.co_filename):
|
||||
return
|
||||
key = self._key(code)
|
||||
st = self.func_stats.get(key)
|
||||
if st is None: # lock only on first sight of a function
|
||||
with self.stats_lock:
|
||||
st = self.func_stats.get(key)
|
||||
if st is None:
|
||||
body = not (code.co_flags & 0x0001) # class/module bodies lack CO_OPTIMIZED - they are not real functions
|
||||
st = {'name': code.co_name, 'file': self._rel(code.co_filename), 'line': code.co_firstlineno, 'calls': 0, 'cum': 0.0, 'own': 0.0, 'cum_cpu': 0.0, 'own_cpu': 0.0, 'body': body}
|
||||
self.func_stats[key] = st
|
||||
st['calls'] += 1
|
||||
if self.call_stack: # record the caller -> callee edge for the flow graph
|
||||
ekey = f'{self.call_stack[-1]["key"]}|{key}'
|
||||
edge = self.call_edges.get(ekey)
|
||||
if edge is None:
|
||||
with self.stats_lock:
|
||||
edge = self.call_edges.get(ekey)
|
||||
if edge is None:
|
||||
edge = {'source': self.call_stack[-1]['key'], 'target': key, 'count': 0, 'time': 0.0}
|
||||
self.call_edges[ekey] = edge
|
||||
edge['count'] += 1
|
||||
self.call_stack.append({'st': st, 'key': key, 'start': time.perf_counter(), 'cpu': time.thread_time(), 'child': 0.0, 'child_cpu': 0.0})
|
||||
elif event == 'return':
|
||||
if not self._in_scope(frame.f_code.co_filename) or not self.call_stack:
|
||||
return
|
||||
now = time.perf_counter()
|
||||
cpu_now = time.thread_time()
|
||||
top = self.call_stack.pop()
|
||||
elapsed = now - top['start'] # wall time
|
||||
cpu = cpu_now - top['cpu'] # cpu time; wall - cpu = time waiting (i/o, sleep, gil, locks)
|
||||
st = top['st']
|
||||
st['cum'] += elapsed
|
||||
st['own'] += elapsed - top['child']
|
||||
st['cum_cpu'] += cpu
|
||||
st['own_cpu'] += cpu - top['child_cpu']
|
||||
if self.call_stack:
|
||||
self.call_stack[-1]['child'] += elapsed
|
||||
self.call_stack[-1]['child_cpu'] += cpu
|
||||
edge = self.call_edges.get(f'{self.call_stack[-1]["key"]}|{top["key"]}')
|
||||
if edge is not None:
|
||||
edge['time'] += elapsed
|
||||
|
||||
|
||||
def _record_block(self, handle, wall: float, cpu: float):
|
||||
'''
|
||||
Record an asyncio loop callback that ran long enough to block the event loop.
|
||||
|
||||
:param handle: the asyncio Handle whose callback just ran
|
||||
:param wall: wall seconds the callback took
|
||||
:param cpu: cpu seconds the callback consumed (wall - cpu is time spent waiting on i/o or sleeping)
|
||||
'''
|
||||
|
||||
self.loop_seen = True
|
||||
name, file, line = '?', '', 0
|
||||
try:
|
||||
cb = handle._callback
|
||||
task = getattr(cb, '__self__', None)
|
||||
coro = task.get_coro() if task is not None and hasattr(task, 'get_coro') else None
|
||||
while coro is not None: # descend the await chain, but only through in-scope user coroutines
|
||||
nxt = getattr(coro, 'cr_await', None)
|
||||
code_nxt = getattr(nxt, 'cr_code', None) or getattr(nxt, 'gi_code', None)
|
||||
if code_nxt is None or not self._in_scope(code_nxt.co_filename):
|
||||
break
|
||||
coro = nxt
|
||||
code = getattr(coro, 'cr_code', None) or getattr(coro, 'gi_code', None) or getattr(cb, '__code__', None)
|
||||
fr = getattr(coro, 'cr_frame', None) or getattr(coro, 'gi_frame', None)
|
||||
if code is not None:
|
||||
name = code.co_name
|
||||
file = self._rel(code.co_filename)
|
||||
line = fr.f_lineno if fr is not None else code.co_firstlineno
|
||||
except Exception:
|
||||
pass
|
||||
key = f'{file}:{name}'
|
||||
with self.block_lock:
|
||||
ev = self.block_events.get(key)
|
||||
if ev is None:
|
||||
ev = {'name': name, 'file': file, 'line': line, 'total': 0.0, 'cpu': 0.0, 'count': 0, 'max': 0.0}
|
||||
self.block_events[key] = ev
|
||||
ev['total'] += wall
|
||||
ev['cpu'] += cpu
|
||||
ev['count'] += 1
|
||||
ev['line'] = line
|
||||
if wall > ev['max']:
|
||||
ev['max'] = wall
|
||||
|
||||
|
||||
def _analysis_loop(self):
|
||||
'''
|
||||
Compute the expensive telemetry (variable deep sizes, tracemalloc hotspots) off the emit path.
|
||||
|
||||
These take up to seconds on large programs, so running them here and caching the result keeps
|
||||
snapshot() cheap and the vitals/timeline stream flowing at full rate.
|
||||
'''
|
||||
|
||||
last_hot = 0.0
|
||||
while self.running:
|
||||
try:
|
||||
self._vars = self.local_vars()
|
||||
except Exception:
|
||||
pass
|
||||
now = time.perf_counter()
|
||||
if now - last_hot > 2.0 and tracemalloc.is_tracing(): # one heavy snapshot, reused for hotspots + growth
|
||||
try:
|
||||
snap = tracemalloc.take_snapshot()
|
||||
if self._baseline is None:
|
||||
self._baseline = snap
|
||||
self._hotspots = self.memory_hotspots(snap)
|
||||
self._growth = self.memory_growth(snap)
|
||||
except Exception:
|
||||
pass
|
||||
last_hot = now
|
||||
time.sleep(0.3)
|
||||
|
||||
|
||||
def _sample_loop(self):
|
||||
'''High resolution statistical sampler recording which source line is live, off the hot path.'''
|
||||
|
||||
while self.running:
|
||||
file, line = self.cur_line()
|
||||
if file and line:
|
||||
with self.line_lock:
|
||||
bucket = self.line_samples.get(file)
|
||||
if bucket is None:
|
||||
bucket = {}
|
||||
self.line_samples[file] = bucket
|
||||
bucket[line] = bucket.get(line, 0) + 1
|
||||
time.sleep(SAMPLE_DT)
|
||||
|
||||
|
||||
def _run(self):
|
||||
'''Compile and execute the target file under trace and stream redirection.'''
|
||||
|
||||
self.thread_id = threading.get_ident()
|
||||
try:
|
||||
with open(self.path, 'r') as fh:
|
||||
source = fh.read()
|
||||
except Exception as ex:
|
||||
self.error = f'could not read file: {ex}'
|
||||
self.running = False
|
||||
return
|
||||
|
||||
out = StreamCapture('stdout', self.logs, self.log_lock, sys.__stdout__)
|
||||
err = StreamCapture('stderr', self.logs, self.log_lock, sys.__stderr__)
|
||||
|
||||
old_out, old_err = sys.stdout, sys.stderr
|
||||
old_argv = sys.argv
|
||||
sys.stdout, sys.stderr = out, err
|
||||
sys.argv = [self.path, *self.script_args]
|
||||
|
||||
# emulate `python <file>` so the target's own package imports resolve
|
||||
old_path = sys.path[:]
|
||||
sys.path.insert(0, os.path.dirname(self.path))
|
||||
|
||||
# own the root logger for this run so the target's log output is captured every run, not
|
||||
# just the first - basicConfig is a no-op once handlers exist, so we install our own
|
||||
root = logging.getLogger()
|
||||
old_log_handlers = root.handlers[:]
|
||||
old_log_level = root.level
|
||||
log_handler = LogSink(self.logs, self.log_lock, sys.__stdout__)
|
||||
log_handler.setFormatter(logging.Formatter('%(message)s'))
|
||||
root.handlers = [log_handler]
|
||||
root.setLevel(logging.DEBUG)
|
||||
|
||||
module_globals = {'__name__': '__main__', '__file__': self.path, '__builtins__': builtins, '__package__': None}
|
||||
|
||||
old_switch = sys.getswitchinterval()
|
||||
sys.setswitchinterval(0.0008) # let the emit/sampler threads interleave despite the profiled target hogging the GIL
|
||||
|
||||
# time every asyncio loop callback so we can flag the ones that block the loop (only fires if the target runs a loop)
|
||||
old_handle_run = asyncio.events.Handle._run
|
||||
def _timed_run(handle):
|
||||
t0, c0 = time.perf_counter(), time.thread_time()
|
||||
old_handle_run(handle)
|
||||
wall = time.perf_counter() - t0
|
||||
if wall >= BLOCK_DT:
|
||||
self._record_block(handle, wall, time.thread_time() - c0)
|
||||
asyncio.events.Handle._run = _timed_run
|
||||
|
||||
tracemalloc.start(1)
|
||||
self.start_time = time.perf_counter()
|
||||
self.running = True
|
||||
sys.setprofile(self._profile)
|
||||
try:
|
||||
code = compile(source, self.path, 'exec')
|
||||
exec(code, module_globals)
|
||||
self.exit_code = 0
|
||||
except SystemExit as ex:
|
||||
self.exit_code = ex.code if isinstance(ex.code, int) else 0
|
||||
except BaseException as ex:
|
||||
import traceback
|
||||
self.error = ''.join(traceback.format_exception(type(ex), ex, ex.__traceback__))
|
||||
self.exit_code = 1
|
||||
with self.log_lock:
|
||||
for line in self.error.rstrip('\n').split('\n'):
|
||||
self.logs.append({'stream': 'stderr', 'level': None, 'text': line})
|
||||
finally:
|
||||
sys.setprofile(None)
|
||||
sys.setswitchinterval(old_switch)
|
||||
asyncio.events.Handle._run = old_handle_run
|
||||
self.end_time = time.perf_counter()
|
||||
out.flush()
|
||||
err.flush()
|
||||
root.handlers = old_log_handlers
|
||||
root.setLevel(old_log_level)
|
||||
sys.path[:] = old_path
|
||||
sys.stdout, sys.stderr = old_out, old_err
|
||||
sys.argv = old_argv
|
||||
self.running = False
|
||||
|
||||
|
||||
def start(self):
|
||||
'''Launch the profiled execution in a background thread.'''
|
||||
|
||||
self.running = True # set before spawn so the emitter never exits early on a startup race
|
||||
self.thread = threading.Thread(target=self._run, daemon=True)
|
||||
self.thread.start()
|
||||
self.sampler = threading.Thread(target=self._sample_loop, daemon=True)
|
||||
self.sampler.start()
|
||||
self.analyzer = threading.Thread(target=self._analysis_loop, daemon=True)
|
||||
self.analyzer.start()
|
||||
|
||||
|
||||
def drain_logs(self) -> list:
|
||||
'''Pop all pending log records for streaming to the client.'''
|
||||
|
||||
out = []
|
||||
with self.log_lock:
|
||||
while self.logs:
|
||||
out.append(self.logs.popleft())
|
||||
return out
|
||||
|
||||
|
||||
def cur_line(self) -> tuple:
|
||||
'''Sample the file and line number the innermost in scope frame is currently executing.'''
|
||||
|
||||
frame = sys._current_frames().get(self.thread_id)
|
||||
while frame is not None:
|
||||
if self._in_scope(frame.f_code.co_filename):
|
||||
return self._abs(frame.f_code.co_filename), frame.f_lineno
|
||||
frame = frame.f_back
|
||||
return None, 0
|
||||
|
||||
|
||||
def live_stack(self) -> list:
|
||||
'''Walk the running thread frames to build the current call stack view.'''
|
||||
|
||||
frames = sys._current_frames()
|
||||
frame = frames.get(self.thread_id)
|
||||
stack = []
|
||||
while frame is not None and len(stack) < 60:
|
||||
code = frame.f_code
|
||||
if self._in_scope(code.co_filename):
|
||||
stack.append({'name': code.co_name, 'line': frame.f_lineno, 'id': self._key(code), 'file': self._abs(code.co_filename), 'rel': self._rel(code.co_filename)})
|
||||
frame = frame.f_back
|
||||
stack.reverse()
|
||||
return stack
|
||||
|
||||
|
||||
def graph(self) -> dict:
|
||||
'''
|
||||
Snapshot the accumulated call graph for the flow view.
|
||||
|
||||
Synthetic frames (<module>, <genexpr>, <lambda>, comprehensions) are dropped so the flow shows
|
||||
only real named functions - they are import time / inline code, not meaningful call-graph nodes.
|
||||
'''
|
||||
|
||||
with self.stats_lock:
|
||||
items = list(self.func_stats.items())
|
||||
edges = list(self.call_edges.values())
|
||||
hidden = {k for k, st in items if st['name'].startswith('<') or st.get('body')} # drop <module>, class bodies, comprehensions
|
||||
recursive = {e['source'] for e in edges if e['source'] == e['target']} # self-calls, flagged not drawn
|
||||
nodes = [{'id': k, 'name': st['name'], 'file': st['file'], 'line': st['line'], 'calls': st['calls'], 'own': st['own'], 'cum': st['cum'], 'own_cpu': st['own_cpu'], 'recursive': k in recursive} for k, st in items if k not in hidden]
|
||||
links = [{'source': e['source'], 'target': e['target'], 'count': e['count'], 'time': e['time']} for e in edges if e['source'] != e['target'] and e['source'] not in hidden and e['target'] not in hidden]
|
||||
return {'nodes': nodes, 'links': links}
|
||||
|
||||
|
||||
def local_vars(self) -> tuple:
|
||||
'''
|
||||
Inspect a stable target frame's locals with per variable deep sizes.
|
||||
|
||||
Picks the outermost non module frame (typically main) so the view holds still and tracks
|
||||
long lived data structures, instead of strobing through whichever frame is momentarily on top.
|
||||
Returns the chosen frame name and its variable rows.
|
||||
'''
|
||||
|
||||
frames = sys._current_frames()
|
||||
frame = frames.get(self.thread_id)
|
||||
targets = []
|
||||
while frame is not None:
|
||||
if self._in_scope(frame.f_code.co_filename):
|
||||
targets.append(frame)
|
||||
frame = frame.f_back
|
||||
if not targets:
|
||||
return '', '', []
|
||||
target = next((f for f in reversed(targets) if f.f_code.co_name != '<module>'), targets[-1])
|
||||
fname = target.f_code.co_name
|
||||
frel = self._rel(target.f_code.co_filename)
|
||||
out = []
|
||||
try:
|
||||
items = list(target.f_locals.items())
|
||||
except Exception:
|
||||
return fname, frel, []
|
||||
for name, value in items:
|
||||
if name.startswith('__') and name.endswith('__'):
|
||||
continue
|
||||
try:
|
||||
size = deep_sizeof(value)
|
||||
preview = repr(value)
|
||||
if len(preview) > 80:
|
||||
preview = preview[:77] + '...'
|
||||
except Exception:
|
||||
size, preview = 0, '<unrepr>'
|
||||
length = None
|
||||
if isinstance(value, CONTAINER_TYPES) or isinstance(value, (str, bytes)):
|
||||
try:
|
||||
length = len(value)
|
||||
except Exception:
|
||||
length = None
|
||||
out.append({'name': name, 'type': type_name(value), 'size': size, 'len': length, 'preview': preview})
|
||||
out.sort(key=lambda item: item['size'], reverse=True)
|
||||
return fname, frel, out[:60]
|
||||
|
||||
|
||||
def memory_hotspots(self, snap=None) -> list:
|
||||
'''
|
||||
Return the top memory allocating source lines via tracemalloc.
|
||||
|
||||
:param snap: an existing tracemalloc snapshot to reuse (taken fresh if omitted)
|
||||
'''
|
||||
|
||||
if not tracemalloc.is_tracing():
|
||||
return []
|
||||
try:
|
||||
if snap is None:
|
||||
snap = tracemalloc.take_snapshot()
|
||||
stats = snap.statistics('lineno')
|
||||
except Exception:
|
||||
return []
|
||||
out = []
|
||||
for stat in stats[:14]:
|
||||
fr = stat.traceback[0]
|
||||
own = self._in_scope(fr.filename)
|
||||
label = self._rel(fr.filename) if own else os.path.basename(fr.filename)
|
||||
out.append({'file': label, 'line': fr.lineno, 'size': stat.size, 'count': stat.count, 'own': own})
|
||||
return out
|
||||
|
||||
|
||||
def memory_growth(self, snap) -> list:
|
||||
'''
|
||||
Return the source lines whose allocation has grown the most since the baseline snapshot.
|
||||
|
||||
Persistent positive growth is the signature of a leak; a fresh baseline is captured on the
|
||||
first analysis pass, so growth reflects allocation accumulated over the run.
|
||||
|
||||
:param snap: the current tracemalloc snapshot
|
||||
'''
|
||||
|
||||
if self._baseline is None:
|
||||
return []
|
||||
try:
|
||||
stats = snap.compare_to(self._baseline, 'lineno')
|
||||
except Exception:
|
||||
return []
|
||||
out = []
|
||||
for stat in stats[:14]:
|
||||
if stat.size_diff <= 0:
|
||||
continue
|
||||
fr = stat.traceback[0]
|
||||
own = self._in_scope(fr.filename)
|
||||
label = self._rel(fr.filename) if own else os.path.basename(fr.filename)
|
||||
out.append({'file': label, 'line': fr.lineno, 'size_diff': stat.size_diff, 'count_diff': stat.count_diff, 'size': stat.size, 'own': own})
|
||||
return out
|
||||
|
||||
|
||||
def blocking(self) -> dict:
|
||||
'''Snapshot the asyncio loop-blocking callbacks, ranked by total time spent blocking the loop.'''
|
||||
|
||||
with self.block_lock:
|
||||
rows = [dict(ev) for ev in self.block_events.values()]
|
||||
rows.sort(key=lambda ev: ev['total'], reverse=True)
|
||||
return {'detected': self.loop_seen, 'total': sum(r['total'] for r in rows), 'blockers': rows[:14]}
|
||||
|
||||
|
||||
def function_table(self) -> list:
|
||||
'''Snapshot the per function benchmark stats sorted by exclusive time.'''
|
||||
|
||||
with self.stats_lock:
|
||||
rows = [dict(st) for st in list(self.func_stats.values()) if not st.get('body')] # only real functions/methods
|
||||
rows.sort(key=lambda item: item['own'], reverse=True)
|
||||
return rows
|
||||
|
||||
|
||||
def snapshot(self) -> dict:
|
||||
'''Assemble a full telemetry frame for the client.'''
|
||||
|
||||
mem_current, mem_peak = (0, 0)
|
||||
if tracemalloc.is_tracing():
|
||||
try:
|
||||
mem_current, mem_peak = tracemalloc.get_traced_memory()
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
rss = self.proc.memory_info().rss
|
||||
cpu = self.proc.cpu_percent(interval=None)
|
||||
except Exception:
|
||||
rss, cpu = 0, 0.0
|
||||
elapsed = (self.end_time or time.perf_counter()) - self.start_time if self.start_time else 0.0
|
||||
stack = self.live_stack() if self.running else []
|
||||
cur_file, cur_line = self.cur_line() if self.running else (None, 0)
|
||||
vars_frame, vars_file, vars_rows = self._vars # computed by the analysis thread, cheap to read here
|
||||
with self.line_lock:
|
||||
line_samples = {f: dict(b) for f, b in self.line_samples.items()}
|
||||
return {
|
||||
'running' : self.running,
|
||||
'elapsed' : elapsed,
|
||||
'cur_file' : cur_file,
|
||||
'cur_line' : cur_line,
|
||||
'stack' : stack,
|
||||
'active' : [f['id'] for f in stack if not f['name'].startswith('<')],
|
||||
'line_samples': line_samples,
|
||||
'sample_dt' : SAMPLE_DT,
|
||||
'functions' : self.function_table(),
|
||||
'graph' : self.graph(),
|
||||
'vars' : vars_rows,
|
||||
'vars_frame': vars_frame,
|
||||
'vars_file' : vars_file,
|
||||
'hotspots' : self._hotspots,
|
||||
'growth' : self._growth,
|
||||
'blocking' : self.blocking(),
|
||||
'rss' : rss,
|
||||
'cpu' : cpu,
|
||||
'tm_current': mem_current,
|
||||
'tm_peak' : mem_peak,
|
||||
'error' : self.error,
|
||||
'exit_code' : self.exit_code,
|
||||
}
|
||||
|
||||
|
||||
def stop_tracing(self):
|
||||
'''Stop tracemalloc once the run is finished to release overhead.'''
|
||||
|
||||
if tracemalloc.is_tracing():
|
||||
try:
|
||||
tracemalloc.stop()
|
||||
except Exception:
|
||||
pass
|
||||
@@ -0,0 +1,7 @@
|
||||
# glassbox - live python execution profiler
|
||||
# frontend libs (socket.io, highlight.js, three, 3d-force-graph) load from CDN, no pip needed
|
||||
|
||||
Flask>=3.0
|
||||
Flask-SocketIO>=5.3
|
||||
simple-websocket>=1.0
|
||||
psutil>=5.9
|
||||
@@ -0,0 +1,2 @@
|
||||
#!/usr/bin/env python3
|
||||
# algorithms package
|
||||
@@ -0,0 +1,29 @@
|
||||
#!/usr/bin/env python3
|
||||
# deep and non primitive recursion - the cpu-bound / tall-call-stack part of the workload
|
||||
|
||||
|
||||
def ackermann(m: int, n: int) -> int:
|
||||
'''
|
||||
Ackermann function - deep non primitive recursion, keep m small.
|
||||
|
||||
:param m: first argument, keep <= 3
|
||||
:param n: second argument
|
||||
'''
|
||||
|
||||
if m == 0:
|
||||
return n + 1
|
||||
if n == 0:
|
||||
return ackermann(m - 1, 1)
|
||||
return ackermann(m - 1, ackermann(m, n - 1))
|
||||
|
||||
|
||||
def fib(n: int) -> int:
|
||||
'''
|
||||
Naive recursive fibonacci - a pure cpu hotspot with a huge call count and deep stack.
|
||||
|
||||
:param n: index to compute
|
||||
'''
|
||||
|
||||
if n < 2:
|
||||
return n
|
||||
return fib(n - 1) + fib(n - 2)
|
||||
@@ -0,0 +1,38 @@
|
||||
#!/usr/bin/env python3
|
||||
# sorting algorithms with deliberately different complexity classes
|
||||
|
||||
|
||||
def bubble_sort(values: list) -> list:
|
||||
'''
|
||||
Quadratic bubble sort - the cpu hot loop that dominates the per line heatmap.
|
||||
|
||||
:param values: numbers to copy and sort
|
||||
'''
|
||||
|
||||
data = list(values)
|
||||
n = len(data)
|
||||
for i in range(n):
|
||||
swapped = False
|
||||
for j in range(n - i - 1):
|
||||
if data[j] > data[j + 1]:
|
||||
data[j], data[j + 1] = data[j + 1], data[j]
|
||||
swapped = True
|
||||
if not swapped:
|
||||
break
|
||||
return data
|
||||
|
||||
|
||||
def quicksort(values: list) -> list:
|
||||
'''
|
||||
Recursive quicksort, producing a moderate depth call tree to contrast with bubble sort.
|
||||
|
||||
:param values: numbers to sort
|
||||
'''
|
||||
|
||||
if len(values) <= 1:
|
||||
return list(values)
|
||||
pivot = values[len(values) // 2]
|
||||
lesser = [v for v in values if v < pivot]
|
||||
equal = [v for v in values if v == pivot]
|
||||
larger = [v for v in values if v > pivot]
|
||||
return quicksort(lesser) + equal + quicksort(larger)
|
||||
@@ -0,0 +1,2 @@
|
||||
#!/usr/bin/env python3
|
||||
# analysis package
|
||||
@@ -0,0 +1,64 @@
|
||||
#!/usr/bin/env python3
|
||||
# inverted index over the fetched documents - the largest retained structure
|
||||
|
||||
from analysis.text import tokenize
|
||||
|
||||
|
||||
class InvertedIndex:
|
||||
'''Maps every token to the documents that contain it, plus per token occurrence counts.'''
|
||||
|
||||
def __init__(self):
|
||||
self.postings = {} # token -> list of doc ids
|
||||
self.freq = {} # token -> total occurrences
|
||||
self.docs = 0
|
||||
|
||||
|
||||
def add_document(self, doc_id: int, tokens: list):
|
||||
'''
|
||||
Index a single document's tokens into the postings and frequency maps.
|
||||
|
||||
:param doc_id: document identifier
|
||||
:param tokens: the document's token list
|
||||
'''
|
||||
|
||||
self.docs += 1
|
||||
for token in tokens:
|
||||
bucket = self.postings.get(token)
|
||||
if bucket is None:
|
||||
bucket = []
|
||||
self.postings[token] = bucket
|
||||
bucket.append(doc_id)
|
||||
self.freq[token] = self.freq.get(token, 0) + 1
|
||||
|
||||
|
||||
def search(self, token: str) -> list:
|
||||
'''
|
||||
Return the document ids containing a token.
|
||||
|
||||
:param token: token to look up
|
||||
'''
|
||||
|
||||
return self.postings.get(token, [])
|
||||
|
||||
|
||||
def top_tokens(self, n: int) -> list:
|
||||
'''
|
||||
Return the n most frequent tokens as (token, count) pairs.
|
||||
|
||||
:param n: how many tokens to return
|
||||
'''
|
||||
|
||||
return sorted(self.freq.items(), key=lambda kv: kv[1], reverse=True)[:n]
|
||||
|
||||
|
||||
def build_index(documents: list) -> InvertedIndex:
|
||||
'''
|
||||
Tokenise and index a list of documents, retaining a large postings map in memory.
|
||||
|
||||
:param documents: documents each carrying an 'extract' text field
|
||||
'''
|
||||
|
||||
index = InvertedIndex()
|
||||
for doc_id, doc in enumerate(documents):
|
||||
index.add_document(doc_id, tokenize(doc.get('text', '')))
|
||||
return index
|
||||
@@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env python3
|
||||
# aggregation and human readable reporting across every log level
|
||||
|
||||
import logging
|
||||
|
||||
from core.metrics import mean, percentile, stddev
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def average_similarity(matrix: list) -> float:
|
||||
'''
|
||||
Mean of the off diagonal cosine similarities in a square matrix.
|
||||
|
||||
:param matrix: pairwise similarity matrix
|
||||
'''
|
||||
|
||||
scores = [matrix[i][j] for i in range(len(matrix)) for j in range(i + 1, len(matrix))]
|
||||
return mean(scores)
|
||||
|
||||
|
||||
def log_findings(summary: dict):
|
||||
'''
|
||||
Emit the summary across every log level so the console's severity colouring is exercised.
|
||||
|
||||
:param summary: aggregated metrics from summarize()
|
||||
'''
|
||||
|
||||
log.debug(f'summary keys: {sorted(summary)}')
|
||||
log.info(f'{summary["pages"]} wikipedia articles · {summary["repos"]} github repos · {summary["tokens"]} unique tokens · {summary["pairs"]} word pairs')
|
||||
log.info(f'top words: {", ".join(summary["top_words"][:8])}')
|
||||
log.info(f'top repo: {summary["top_repo"]} · mean stars {summary["mean_stars"]:.0f} · p90 {summary["p90_stars"]:.0f} · avg similarity {summary["avg_similarity"]:.3f}')
|
||||
if summary['repos'] == 0:
|
||||
log.warning('no repositories were fetched - star metrics fell back to zero')
|
||||
try:
|
||||
_ = summary['tokens'] / (summary['pages'] - summary['pages'])
|
||||
except ZeroDivisionError:
|
||||
log.error('token density is undefined (division by zero) - skipping it')
|
||||
log.critical('analysis complete')
|
||||
|
||||
|
||||
def summarize(articles: list, repos: list, index, common: list, stars: list, matrix: list) -> dict:
|
||||
'''
|
||||
Aggregate the fetched articles, repos, index and similarity matrix into a summary dict.
|
||||
|
||||
:param articles: fetched wikipedia article records
|
||||
:param repos: fetched github repo records
|
||||
:param index: the built inverted index
|
||||
:param common: most common (word, count) pairs
|
||||
:param stars: star counts across the repos
|
||||
:param matrix: pairwise document similarity matrix
|
||||
'''
|
||||
|
||||
return {
|
||||
'pages' : len(articles),
|
||||
'repos' : len(repos),
|
||||
'tokens' : len(index.postings),
|
||||
'pairs' : sum(len(v) for v in index.postings.values()),
|
||||
'top_words' : [w for w, _ in common],
|
||||
'top_repo' : max(repos, key=lambda r: r['stars'])['name'] if repos else 'n/a',
|
||||
'mean_stars' : mean(stars),
|
||||
'p90_stars' : percentile(stars, 90),
|
||||
'stddev_stars' : stddev(stars),
|
||||
'avg_similarity': average_similarity(matrix),
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
#!/usr/bin/env python3
|
||||
# cosine similarity between document word-frequency vectors - a deep helper call chain
|
||||
|
||||
def cosine(a: dict, b: dict) -> float:
|
||||
'''
|
||||
Cosine similarity between two sparse word-frequency vectors.
|
||||
|
||||
:param a: first vector as word -> weight
|
||||
:param b: second vector as word -> weight
|
||||
'''
|
||||
|
||||
denom = magnitude(a) * magnitude(b)
|
||||
return dot(a, b) / denom if denom else 0.0
|
||||
|
||||
|
||||
def dot(a: dict, b: dict) -> float:
|
||||
'''
|
||||
Dot product of two sparse vectors, iterating the smaller one.
|
||||
|
||||
:param a: first vector
|
||||
:param b: second vector
|
||||
'''
|
||||
|
||||
if len(a) > len(b):
|
||||
a, b = b, a
|
||||
return sum(weight * b.get(word, 0) for word, weight in a.items())
|
||||
|
||||
|
||||
def magnitude(vec: dict) -> float:
|
||||
'''
|
||||
Euclidean magnitude of a sparse vector.
|
||||
|
||||
:param vec: vector as word -> weight
|
||||
'''
|
||||
|
||||
return sum(w * w for w in vec.values()) ** 0.5
|
||||
|
||||
|
||||
def similarity_matrix(vectors: list) -> list:
|
||||
'''
|
||||
Build the full pairwise cosine similarity matrix over document vectors.
|
||||
|
||||
:param vectors: list of word-frequency vectors, one per document
|
||||
'''
|
||||
|
||||
size = len(vectors)
|
||||
matrix = [[0.0] * size for _ in range(size)]
|
||||
for i in range(size):
|
||||
for j in range(i, size):
|
||||
score = cosine(vectors[i], vectors[j])
|
||||
matrix[i][j] = score
|
||||
matrix[j][i] = score
|
||||
return matrix
|
||||
@@ -0,0 +1,81 @@
|
||||
#!/usr/bin/env python3
|
||||
# tokenisation and word/phrase statistics over the fetched text
|
||||
|
||||
import re
|
||||
|
||||
|
||||
WORD_RE = re.compile(r'[a-z]{3,}')
|
||||
|
||||
|
||||
def bigrams(tokens: list) -> list:
|
||||
'''
|
||||
Build the list of adjacent word pairs from a token stream.
|
||||
|
||||
:param tokens: list of word tokens
|
||||
'''
|
||||
|
||||
return [(tokens[i], tokens[i + 1]) for i in range(len(tokens) - 1)]
|
||||
|
||||
|
||||
def cooccurrence(tokens: list, window: int) -> dict:
|
||||
'''
|
||||
Count how often word pairs appear within a sliding window - a nested-loop cpu hotspot
|
||||
whose pair map is the run's main growing data structure.
|
||||
|
||||
:param tokens: list of word tokens
|
||||
:param window: how many following tokens to pair each token with
|
||||
'''
|
||||
|
||||
counts = {}
|
||||
n = len(tokens)
|
||||
for i in range(n):
|
||||
left = tokens[i]
|
||||
for j in range(i + 1, min(i + window + 1, n)):
|
||||
right = tokens[j]
|
||||
pair = (left, right) if left < right else (right, left)
|
||||
counts[pair] = counts.get(pair, 0) + 1
|
||||
return counts
|
||||
|
||||
|
||||
def normalize(text: str) -> str:
|
||||
'''
|
||||
Lowercase text and collapse runs of whitespace before tokenising.
|
||||
|
||||
:param text: raw text
|
||||
'''
|
||||
|
||||
return ' '.join(text.lower().split())
|
||||
|
||||
|
||||
def tokenize(text: str) -> list:
|
||||
'''
|
||||
Normalise a block of text and split it into word tokens of three or more letters.
|
||||
|
||||
:param text: raw text to tokenise
|
||||
'''
|
||||
|
||||
return WORD_RE.findall(normalize(text))
|
||||
|
||||
|
||||
def top_words(freq: dict, n: int) -> list:
|
||||
'''
|
||||
Return the n most common (word, count) pairs.
|
||||
|
||||
:param freq: word frequency mapping
|
||||
:param n: how many to return
|
||||
'''
|
||||
|
||||
return sorted(freq.items(), key=lambda kv: kv[1], reverse=True)[:n]
|
||||
|
||||
|
||||
def word_frequencies(tokens: list) -> dict:
|
||||
'''
|
||||
Count how often each token appears.
|
||||
|
||||
:param tokens: list of word tokens
|
||||
'''
|
||||
|
||||
freq = {}
|
||||
for tok in tokens:
|
||||
freq[tok] = freq.get(tok, 0) + 1
|
||||
return freq
|
||||
@@ -0,0 +1,16 @@
|
||||
#!/usr/bin/env python3
|
||||
# sample_project - workload configuration and the shared seeded rng
|
||||
|
||||
import random
|
||||
|
||||
|
||||
RNG = random.Random(1337)
|
||||
WIKI_TOPICS = ['Python (programming language)', 'Machine learning', 'Web scraping', 'Operating system', 'Distributed computing', 'Cryptography', 'Compiler', 'Database', 'Concurrency (computer science)', 'Garbage collection (computer science)']
|
||||
GITHUB_QUERY = 'language:python stars:>50'
|
||||
GITHUB_LIMIT = 30
|
||||
HTTP_TIMEOUT = 10
|
||||
COOC_WINDOW = 4 # co-occurrence sliding window (a nested-loop hotspot)
|
||||
BUBBLE_LIMIT = 900 # cap the O(n^2) sort so it stays a demo, not a hang
|
||||
TOP_WORDS = 12
|
||||
FIB_N = 23
|
||||
ACKERMANN = (3, 3)
|
||||
@@ -0,0 +1,2 @@
|
||||
#!/usr/bin/env python3
|
||||
# core package
|
||||
@@ -0,0 +1,40 @@
|
||||
#!/usr/bin/env python3
|
||||
# small statistics helpers used by the reporting stage
|
||||
|
||||
|
||||
def mean(values: list) -> float:
|
||||
'''
|
||||
Arithmetic mean of a sequence.
|
||||
|
||||
:param values: numbers to average
|
||||
'''
|
||||
|
||||
return sum(values) / len(values) if values else 0.0
|
||||
|
||||
|
||||
def percentile(values: list, pct: float) -> float:
|
||||
'''
|
||||
Nearest rank percentile of a sequence.
|
||||
|
||||
:param values: numbers to rank
|
||||
:param pct: percentile in the range 0..100
|
||||
'''
|
||||
|
||||
if not values:
|
||||
return 0.0
|
||||
ordered = sorted(values)
|
||||
index = min(len(ordered) - 1, int(pct / 100 * len(ordered)))
|
||||
return ordered[index]
|
||||
|
||||
|
||||
def stddev(values: list) -> float:
|
||||
'''
|
||||
Population standard deviation of a sequence.
|
||||
|
||||
:param values: numbers to measure
|
||||
'''
|
||||
|
||||
if not values:
|
||||
return 0.0
|
||||
avg = mean(values)
|
||||
return (sum((v - avg) ** 2 for v in values) / len(values)) ** 0.5
|
||||
@@ -0,0 +1,34 @@
|
||||
#!/usr/bin/env python3
|
||||
# stage timing context manager used to structure and instrument the pipeline
|
||||
|
||||
import logging
|
||||
import time
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Stage:
|
||||
'''Times a named pipeline stage, logging its start and duration into a shared sink.'''
|
||||
|
||||
def __init__(self, name: str, sink: dict):
|
||||
self.name = name
|
||||
self.sink = sink
|
||||
self.t0 = 0.0
|
||||
|
||||
|
||||
def __enter__(self) -> 'Stage':
|
||||
'''Record the start time and announce the stage.'''
|
||||
|
||||
log.debug(f'stage "{self.name}" starting')
|
||||
self.t0 = time.perf_counter()
|
||||
return self
|
||||
|
||||
|
||||
def __exit__(self, exc_type, exc, tb) -> bool:
|
||||
'''Store the elapsed time and log it, never suppressing exceptions.'''
|
||||
|
||||
elapsed = time.perf_counter() - self.t0
|
||||
self.sink[self.name] = elapsed
|
||||
log.info(f'stage "{self.name}" finished in {elapsed:.3f}s')
|
||||
return False
|
||||
@@ -0,0 +1,137 @@
|
||||
#!/usr/bin/env python3
|
||||
# sample_project - an async data pipeline that fetches real data (wikipedia + github),
|
||||
# then indexes and analyses it. built to exercise every glassbox view at once:
|
||||
# async event loop -> the blocking panel (a coroutine that stalls the loop)
|
||||
# network fetch -> wait time (blocking i/o inside a coroutine)
|
||||
# tokenise / index -> cpu + a growing postings map (memory)
|
||||
# co-occurrence -> nested-loop hotspot + the run's largest growing structure
|
||||
# similarity -> a deep helper call chain (similarity_matrix -> cosine -> dot/magnitude)
|
||||
# bubble sort -> quadratic cpu hotspot on real data (hot lines)
|
||||
# fibonacci -> deep recursive call stack, run synchronously to block the loop with cpu
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import sys
|
||||
|
||||
import config
|
||||
|
||||
from algorithms.recursion import ackermann, fib
|
||||
from algorithms.sorting import bubble_sort, quicksort
|
||||
from analysis.indexer import build_index
|
||||
from analysis.report import log_findings, summarize
|
||||
from analysis.similarity import similarity_matrix
|
||||
from analysis.text import bigrams, cooccurrence, tokenize, top_words, word_frequencies
|
||||
from core.pipeline import Stage
|
||||
from net.github import recent_repos
|
||||
from net.wikipedia import fetch_articles
|
||||
|
||||
|
||||
logging.basicConfig(level=logging.DEBUG, format='%(levelname)s %(name)s: %(message)s', stream=sys.stdout)
|
||||
log = logging.getLogger('main')
|
||||
|
||||
FALLBACK_DOCS = [
|
||||
{'title': 'Offline sample', 'text': ('glassbox profiles python programs to reveal where time and memory actually go inside a running project across every function and line. ' * 30)},
|
||||
{'title': 'Offline sample two', 'text': ('the event loop schedules coroutines while the profiler samples the running call stack every few milliseconds to build a live map of execution. ' * 30)},
|
||||
]
|
||||
|
||||
|
||||
async def fetch_stage(state: dict) -> tuple:
|
||||
'''
|
||||
Fetch data with blocking urllib calls inside a coroutine - this stalls the event loop on
|
||||
network i/o (shows up in the blocking panel as almost pure wait).
|
||||
|
||||
:param state: shared progress state
|
||||
'''
|
||||
|
||||
state['stage'] = 'fetch'
|
||||
articles = fetch_articles(config.WIKI_TOPICS) or FALLBACK_DOCS
|
||||
repos = recent_repos(config.GITHUB_QUERY, config.GITHUB_LIMIT)
|
||||
return articles, repos
|
||||
|
||||
|
||||
async def heartbeat(state: dict):
|
||||
'''
|
||||
A well-behaved coroutine that yields to the loop each tick - it visibly stalls whenever another
|
||||
coroutine blocks the loop, which is exactly what the blocking panel flags.
|
||||
|
||||
:param state: shared progress state
|
||||
'''
|
||||
|
||||
ticks = 0
|
||||
while not state['done']:
|
||||
ticks += 1
|
||||
await asyncio.sleep(0.1)
|
||||
log.debug(f'heartbeat completed {ticks} ticks')
|
||||
|
||||
|
||||
async def process_stage(articles: list, repos: list, state: dict) -> dict:
|
||||
'''
|
||||
Run the heavy analysis synchronously inside a coroutine - this blocks the loop with cpu work
|
||||
(indexing, co-occurrence, similarity and a deep recursion), the classic async anti-pattern.
|
||||
|
||||
:param articles: fetched wikipedia article records
|
||||
:param repos: fetched github repo records
|
||||
:param state: shared progress state
|
||||
'''
|
||||
|
||||
state['stage'] = 'index'
|
||||
index = build_index(articles)
|
||||
|
||||
state['stage'] = 'analyse'
|
||||
corpus = ' '.join(a['text'] for a in articles)
|
||||
tokens = tokenize(corpus)
|
||||
freq = word_frequencies(tokens)
|
||||
common = top_words(freq, config.TOP_WORDS)
|
||||
grams = bigrams(tokens)
|
||||
cooc = cooccurrence(tokens, config.COOC_WINDOW)
|
||||
vectors = [word_frequencies(tokenize(a['text'])) for a in articles]
|
||||
matrix = similarity_matrix(vectors)
|
||||
|
||||
state['stage'] = 'rank'
|
||||
pair_counts = sorted(cooc.values(), reverse=True)[:config.BUBBLE_LIMIT]
|
||||
slow = bubble_sort(pair_counts)
|
||||
fast = quicksort(pair_counts)
|
||||
stars = sorted((r['stars'] for r in repos), reverse=True)
|
||||
|
||||
state['stage'] = 'compute'
|
||||
difficulty = fib(config.FIB_N)
|
||||
ack = ackermann(*config.ACKERMANN)
|
||||
|
||||
summary = summarize(articles, repos, index, common, [r['stars'] for r in repos], matrix)
|
||||
return {'summary': summary, 'cooc': cooc, 'bigrams': grams, 'matrix': matrix, 'tokens': tokens, 'stars': stars, 'fib': difficulty, 'ack': ack, 'sorted_ok': slow == fast}
|
||||
|
||||
|
||||
async def main():
|
||||
'''Drive the async pipeline, letting the loop breathe between the loop-blocking stages.'''
|
||||
|
||||
rounds = int(sys.argv[1]) if len(sys.argv) > 1 else 1
|
||||
timings = {}
|
||||
archive = [] # retained across rounds so memory growth is visible
|
||||
state = {'stage': 'init', 'done': False}
|
||||
beat = asyncio.create_task(heartbeat(state))
|
||||
|
||||
for rnd in range(1, rounds + 1):
|
||||
log.info(f'================ run {rnd}/{rounds} ================')
|
||||
|
||||
with Stage('fetch', timings):
|
||||
articles, repos = await fetch_stage(state)
|
||||
log.info(f'fetched {len(articles)} wikipedia articles and {len(repos)} github repos')
|
||||
await asyncio.sleep(0.05) # yield so the heartbeat can tick between blocking stages
|
||||
|
||||
with Stage('process', timings):
|
||||
result = await process_stage(articles, repos, state)
|
||||
await asyncio.sleep(0.05)
|
||||
|
||||
with Stage('report', timings):
|
||||
log_findings(result['summary'])
|
||||
|
||||
archive.append({'run': rnd, 'cooc': result['cooc'], 'bigrams': result['bigrams'], 'matrix': result['matrix']})
|
||||
log.info(f'run {rnd}: {len(result["tokens"])} tokens · {len(result["cooc"])} pairs · top stars {result["stars"][:3]} · fib={result["fib"]} ack={result["ack"]} · sorted_ok={result["sorted_ok"]}')
|
||||
|
||||
state['done'] = True
|
||||
await beat
|
||||
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,2 @@
|
||||
#!/usr/bin/env python3
|
||||
# net package
|
||||
@@ -0,0 +1,35 @@
|
||||
#!/usr/bin/env python3
|
||||
# recently updated github repositories via the public search api
|
||||
|
||||
import logging
|
||||
import urllib.parse
|
||||
|
||||
from net.http import fetch_json
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
SEARCH_URL = 'https://api.github.com/search/repositories?q={}&sort=updated&order=desc&per_page={}'
|
||||
|
||||
|
||||
def recent_repos(query: str, limit: int) -> list:
|
||||
'''
|
||||
Fetch the most recently updated repositories matching a search query.
|
||||
|
||||
:param query: github search query
|
||||
:param limit: how many repositories to return
|
||||
'''
|
||||
|
||||
url = SEARCH_URL.format(urllib.parse.quote(query), limit)
|
||||
data = fetch_json(url)
|
||||
if not data or 'items' not in data:
|
||||
log.warning('github search returned no results (offline or rate limited)')
|
||||
return []
|
||||
repos = []
|
||||
for item in data['items']:
|
||||
repos.append({
|
||||
'name' : item.get('full_name', '?'),
|
||||
'stars' : item.get('stargazers_count', 0),
|
||||
'lang' : item.get('language') or 'n/a',
|
||||
'desc' : (item.get('description') or '')[:120],
|
||||
})
|
||||
return repos
|
||||
@@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env python3
|
||||
# minimal http client over urllib - no third party dependencies, network time shows up as wait
|
||||
|
||||
import json
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
from config import HTTP_TIMEOUT
|
||||
|
||||
|
||||
HEADERS = {'User-Agent': 'glassbox-sample/1.0'}
|
||||
|
||||
|
||||
def fetch_json(url: str) -> dict:
|
||||
'''
|
||||
GET a url and parse the response as json, returning None on any failure.
|
||||
|
||||
:param url: the url to fetch
|
||||
'''
|
||||
|
||||
body = fetch_text(url)
|
||||
if body is None:
|
||||
return None
|
||||
try:
|
||||
return json.loads(body)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def fetch_text(url: str) -> str:
|
||||
'''
|
||||
GET a url and return the decoded body, or None if the request fails.
|
||||
|
||||
The urllib call blocks on the network, so its time is almost all wait, not cpu.
|
||||
|
||||
:param url: the url to fetch
|
||||
'''
|
||||
|
||||
req = urllib.request.Request(url, headers=HEADERS)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=HTTP_TIMEOUT) as resp:
|
||||
return resp.read().decode('utf-8', 'replace')
|
||||
except (urllib.error.URLError, OSError):
|
||||
return None
|
||||
@@ -0,0 +1,45 @@
|
||||
#!/usr/bin/env python3
|
||||
# wikipedia article intro fetching via the mediawiki extracts api (fuller text than the summary)
|
||||
|
||||
import logging
|
||||
import urllib.parse
|
||||
|
||||
from net.http import fetch_json
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
EXTRACT_URL = 'https://en.wikipedia.org/w/api.php?action=query&prop=extracts&exintro=1&explaintext=1&redirects=1&format=json&titles={}'
|
||||
|
||||
|
||||
def fetch_article(title: str) -> dict:
|
||||
'''
|
||||
Fetch the plain-text intro of a single wikipedia article.
|
||||
|
||||
:param title: the article title
|
||||
'''
|
||||
|
||||
data = fetch_json(EXTRACT_URL.format(urllib.parse.quote(title)))
|
||||
try:
|
||||
page = next(iter(data['query']['pages'].values()))
|
||||
text = page.get('extract', '')
|
||||
except (KeyError, StopIteration, TypeError, AttributeError):
|
||||
text = ''
|
||||
if not text:
|
||||
log.debug(f'wikipedia miss for {title!r}')
|
||||
return None
|
||||
return {'title': page.get('title', title), 'text': text}
|
||||
|
||||
|
||||
def fetch_articles(titles: list) -> list:
|
||||
'''
|
||||
Fetch the intro text of each wikipedia title, skipping any that fail.
|
||||
|
||||
:param titles: list of article titles to fetch
|
||||
'''
|
||||
|
||||
articles = []
|
||||
for title in titles:
|
||||
article = fetch_article(title)
|
||||
if article is not None:
|
||||
articles.append(article)
|
||||
return articles
|
||||
@@ -0,0 +1,19 @@
|
||||
#!/usr/bin/env bash
|
||||
# glassbox - create a virtualenv and install dependencies
|
||||
set -euo pipefail
|
||||
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
PYTHON="${PYTHON:-python3}"
|
||||
|
||||
echo "[glassbox] creating virtualenv in .venv"
|
||||
"$PYTHON" -m venv .venv
|
||||
|
||||
echo "[glassbox] installing dependencies"
|
||||
.venv/bin/pip install --upgrade pip >/dev/null
|
||||
.venv/bin/pip install -r requirements.txt
|
||||
|
||||
echo
|
||||
echo "[glassbox] ready. start it with:"
|
||||
echo " .venv/bin/python app.py"
|
||||
echo "then open http://localhost:7000"
|
||||
@@ -0,0 +1,432 @@
|
||||
/* static - Developed by acidvegas in CSS (https://github.com/acidvegas) */
|
||||
/* style.css */
|
||||
|
||||
:root {
|
||||
--bg: #05080d;
|
||||
--bg2: #070c13;
|
||||
--panel: #0c121b;
|
||||
--panel2: #111a26;
|
||||
--line: #1b2735;
|
||||
--line2: #24344799;
|
||||
--txt: #cdd8e6;
|
||||
--muted: #62748a;
|
||||
--cyan: #29e6d4;
|
||||
--green: #4ef08a;
|
||||
--amber: #ffb454;
|
||||
--red: #ff5c6c;
|
||||
--purple: #b58cff;
|
||||
--blue: #4aa8ff;
|
||||
--pink: #ff6ac1;
|
||||
--glow: 0 0 12px;
|
||||
--radius: 10px;
|
||||
}
|
||||
|
||||
* { box-sizing: border-box; margin: 0; padding: 0; }
|
||||
|
||||
html, body { height: 100%; }
|
||||
|
||||
body {
|
||||
background:
|
||||
radial-gradient(1400px 700px at 78% -12%, #12232f 0%, transparent 60%),
|
||||
radial-gradient(1000px 600px at 0% 110%, #14122b 0%, transparent 55%),
|
||||
var(--bg);
|
||||
color: var(--txt);
|
||||
font-family: 'JetBrains Mono', ui-monospace, monospace;
|
||||
font-size: 13px;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
/* ---- top bar ---- */
|
||||
.topbar {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 20px;
|
||||
padding: 10px 18px;
|
||||
background: color-mix(in srgb, var(--bg2) 88%, transparent);
|
||||
backdrop-filter: blur(10px);
|
||||
border-bottom: 1px solid var(--line);
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.brand { font-family: 'Space Grotesk', sans-serif; font-size: 21px; font-weight: 700; letter-spacing: .5px; }
|
||||
.brand .logo {
|
||||
display: inline-block;
|
||||
color: var(--bg);
|
||||
background: linear-gradient(135deg, var(--cyan), var(--green));
|
||||
padding: 2px 9px 3px;
|
||||
margin-right: 6px;
|
||||
border-radius: 7px;
|
||||
box-shadow: var(--glow) rgba(41,230,212,.45);
|
||||
}
|
||||
.brand .sub { font-family: 'JetBrains Mono', monospace; font-size: 10px; color: var(--muted); font-weight: 400; margin-left: 14px; letter-spacing: 2px; text-transform: uppercase; }
|
||||
|
||||
.controls { display: flex; gap: 8px; flex: 1; }
|
||||
.field {
|
||||
background: #05090f;
|
||||
border: 1px solid var(--line);
|
||||
color: var(--txt);
|
||||
padding: 8px 12px;
|
||||
border-radius: 7px;
|
||||
font-family: inherit;
|
||||
font-size: 13px;
|
||||
outline: none;
|
||||
transition: border-color .2s, box-shadow .2s;
|
||||
}
|
||||
.field:focus { border-color: var(--cyan); box-shadow: var(--glow) rgba(41,230,212,.25); }
|
||||
#path { flex: 1; }
|
||||
.field.args { width: 170px; }
|
||||
.field.sel { padding: 8px 10px; cursor: pointer; }
|
||||
.field.sel option { background: #0a0f17; }
|
||||
|
||||
.src-tools { display: flex; align-items: center; gap: 8px; }
|
||||
.mini-sel {
|
||||
background: #05090f; border: 1px solid var(--line); color: var(--txt);
|
||||
font-family: inherit; font-size: 11px; padding: 3px 8px; border-radius: 6px;
|
||||
max-width: 240px; outline: none; cursor: pointer;
|
||||
}
|
||||
.mini-sel:focus { border-color: var(--cyan); }
|
||||
.mini-sel option { background: #0a0f17; }
|
||||
|
||||
.btn {
|
||||
background: var(--panel2);
|
||||
border: 1px solid var(--line);
|
||||
color: var(--txt);
|
||||
padding: 8px 16px;
|
||||
border-radius: 7px;
|
||||
cursor: pointer;
|
||||
font-family: inherit;
|
||||
font-size: 13px;
|
||||
transition: all .15s;
|
||||
white-space: nowrap;
|
||||
}
|
||||
.btn:hover { border-color: var(--cyan); color: #fff; box-shadow: var(--glow) rgba(41,230,212,.15); }
|
||||
.btn.tiny { padding: 4px 11px; font-size: 11px; }
|
||||
.btn.run {
|
||||
background: linear-gradient(135deg, var(--cyan), var(--green));
|
||||
color: #04140f;
|
||||
font-weight: 700;
|
||||
border: none;
|
||||
}
|
||||
.btn.run:hover { box-shadow: var(--glow) rgba(78,240,138,.5); }
|
||||
.btn.run.busy { background: linear-gradient(135deg, var(--amber), var(--red)); color: #200; }
|
||||
|
||||
/* ---- tabs ---- */
|
||||
.tabs { display: flex; align-items: center; gap: 2px; padding: 0 14px; background: var(--bg2); border-bottom: 1px solid var(--line); flex-shrink: 0; }
|
||||
.tab {
|
||||
background: none; border: none; border-bottom: 2px solid transparent;
|
||||
color: var(--muted); font-family: inherit; font-size: 12px; letter-spacing: 1px;
|
||||
padding: 11px 18px; cursor: pointer; text-transform: uppercase; transition: all .15s;
|
||||
}
|
||||
.tab:hover { color: var(--txt); }
|
||||
.tab.active { color: var(--cyan); border-bottom-color: var(--cyan); text-shadow: var(--glow) rgba(41,230,212,.4); }
|
||||
.tabfill { flex: 1; }
|
||||
.live-badge { font-size: 11px; padding: 4px 12px; border-radius: 20px; background: var(--panel); border: 1px solid var(--line); max-width: 380px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
||||
.live-badge .exec { color: var(--green); }
|
||||
|
||||
/* ---- stage + views ---- */
|
||||
.stage { flex: 1; min-height: 0; position: relative; }
|
||||
.view { position: absolute; inset: 0; }
|
||||
.view.hidden { display: none; }
|
||||
|
||||
.live-grid, .profile-grid, .memory-grid {
|
||||
display: grid;
|
||||
gap: 10px;
|
||||
padding: 10px;
|
||||
overflow: auto;
|
||||
}
|
||||
.live-grid {
|
||||
grid-template-columns: minmax(440px, 1.7fr) minmax(300px, 1fr);
|
||||
grid-template-rows: 1.5fr 1fr;
|
||||
grid-template-areas:
|
||||
'source stack'
|
||||
'logs logs';
|
||||
}
|
||||
.profile-grid {
|
||||
grid-template-columns: minmax(420px, 1.55fr) minmax(280px, 1fr);
|
||||
grid-template-rows: minmax(150px, 1fr) minmax(150px, 1fr) minmax(120px, .9fr);
|
||||
grid-template-areas:
|
||||
'funcs cpuchart'
|
||||
'funcs hotlines'
|
||||
'funcs blocking';
|
||||
}
|
||||
.memory-grid {
|
||||
grid-template-columns: 1fr 1fr;
|
||||
grid-template-rows: minmax(130px, .7fr) minmax(150px, 1fr) minmax(150px, 1fr);
|
||||
grid-template-areas:
|
||||
'memchart memchart'
|
||||
'hotspots growth'
|
||||
'vars vars';
|
||||
}
|
||||
.source { grid-area: source; }
|
||||
.stack-panel { grid-area: stack; }
|
||||
.logs { grid-area: logs; }
|
||||
.funcs { grid-area: funcs; }
|
||||
.cpuchart { grid-area: cpuchart; }
|
||||
.hotlines { grid-area: hotlines; }
|
||||
.memchart { grid-area: memchart; }
|
||||
.hotspots { grid-area: hotspots; }
|
||||
.vars { grid-area: vars; }
|
||||
.blocking { grid-area: blocking; }
|
||||
.growth { grid-area: growth; }
|
||||
.flame-grid { display: flex; padding: 10px; }
|
||||
.flamepanel { flex: 1; }
|
||||
.files-grid { display: flex; padding: 10px; }
|
||||
.filepanel { flex: 1; }
|
||||
|
||||
/* ---- project file communication graph ---- */
|
||||
.filemap { position: relative; overflow: hidden; }
|
||||
|
||||
/* ---- flame graph ---- */
|
||||
.flame { overflow: auto; padding: 8px 8px 40px; }
|
||||
.flame-kids { display: flex; align-items: flex-start; width: 100%; }
|
||||
.flame-cell { flex-shrink: 0; min-width: 0; padding: 0 1px; box-sizing: border-box; }
|
||||
.flame-frame { width: 100%; }
|
||||
.flame-bar { height: 21px; border-radius: 2px; margin-bottom: 2px; overflow: hidden; white-space: nowrap; display: flex; align-items: center; padding: 0 5px; cursor: pointer; box-shadow: inset 0 0 0 1px rgba(0,0,0,.28); transition: filter .1s; }
|
||||
.flame-bar:hover { filter: brightness(1.22); }
|
||||
.flame-lbl { font-size: 10px; color: #07131d; font-weight: 600; overflow: hidden; text-overflow: ellipsis; letter-spacing: .2px; }
|
||||
|
||||
/* ---- filter input + sortable headers ---- */
|
||||
.mini-input { background: #05090f; border: 1px solid var(--line); color: var(--txt); font-family: inherit; font-size: 11px; padding: 3px 9px; border-radius: 6px; outline: none; width: 150px; }
|
||||
.mini-input:focus { border-color: var(--cyan); }
|
||||
.tbl.sortable thead th { cursor: pointer; user-select: none; }
|
||||
.tbl.sortable thead th:hover { color: var(--txt); }
|
||||
.tbl thead th.sorted { color: var(--cyan); }
|
||||
.tbl thead th.sorted::after { content: ' ▾'; font-size: 8px; }
|
||||
.tbl thead th.sorted[data-dir="asc"]::after { content: ' ▴'; }
|
||||
|
||||
/* ---- bottom status bar ---- */
|
||||
.statusbar {
|
||||
display: flex; align-items: center; gap: 18px;
|
||||
padding: 0 16px; height: 30px; flex-shrink: 0;
|
||||
background: color-mix(in srgb, var(--bg2) 92%, transparent);
|
||||
border-top: 1px solid var(--line);
|
||||
font-size: 11px;
|
||||
}
|
||||
.dot { width: 9px; height: 9px; border-radius: 50%; display: inline-block; flex-shrink: 0; }
|
||||
.dot.idle { background: var(--muted); }
|
||||
.dot.running { background: var(--green); box-shadow: var(--glow) var(--green); animation: pulse 1s infinite; }
|
||||
.dot.done { background: var(--blue); box-shadow: var(--glow) var(--blue); }
|
||||
.dot.error { background: var(--red); box-shadow: var(--glow) var(--red); }
|
||||
@keyframes pulse { 0%,100% { opacity: 1; } 50% { opacity: .3; } }
|
||||
#statusText { text-transform: uppercase; letter-spacing: 1px; font-size: 10px; color: var(--muted); min-width: 58px; }
|
||||
.sb-item { display: flex; align-items: baseline; gap: 6px; }
|
||||
.sb-item label { font-size: 9px; color: var(--muted); text-transform: uppercase; letter-spacing: 1px; }
|
||||
.sb-item b { color: var(--cyan); font-weight: 500; font-variant-numeric: tabular-nums; }
|
||||
.sb-root { max-width: 30vw; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; direction: rtl; text-align: left; }
|
||||
|
||||
.panel {
|
||||
background: linear-gradient(180deg, var(--panel), #0a0f17);
|
||||
border: 1px solid var(--line);
|
||||
border-radius: var(--radius);
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
overflow: hidden;
|
||||
min-height: 0;
|
||||
box-shadow: 0 2px 20px rgba(0,0,0,.35);
|
||||
}
|
||||
.panel-head {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
padding: 8px 12px;
|
||||
border-bottom: 1px solid var(--line);
|
||||
background: linear-gradient(180deg, #0f1824, #0b1119);
|
||||
flex-shrink: 0;
|
||||
}
|
||||
.panel-head h2 { font-family: 'Space Grotesk', sans-serif; font-size: 11px; text-transform: uppercase; letter-spacing: 2px; color: var(--cyan); font-weight: 700; }
|
||||
.muted { color: var(--muted); font-size: 10px; letter-spacing: 1px; }
|
||||
.panel-body { overflow: auto; flex: 1; min-height: 0; }
|
||||
|
||||
/* ---- source viewer ---- */
|
||||
.code { padding: 6px 0; font-size: 12.5px; line-height: 1.55; }
|
||||
.code .row {
|
||||
display: flex; padding: 0 12px; white-space: pre; transition: background .25s;
|
||||
border-left: 2px solid transparent;
|
||||
background-image: linear-gradient(90deg, transparent 55%, color-mix(in srgb, var(--red) calc(var(--heat, 0) * 45%), transparent));
|
||||
}
|
||||
.code .ln { color: #33455a; width: 40px; text-align: right; margin-right: 10px; user-select: none; flex-shrink: 0; }
|
||||
.code .src { color: var(--txt); }
|
||||
.code .row.hit { border-left-color: rgba(41,230,212,.3); }
|
||||
.code .row.current {
|
||||
background-image: linear-gradient(90deg, rgba(78,240,138,.22), transparent 80%);
|
||||
border-left-color: var(--green);
|
||||
box-shadow: inset 0 0 24px rgba(78,240,138,.09);
|
||||
}
|
||||
.code .row.current .ln { color: var(--green); text-shadow: var(--glow) rgba(78,240,138,.6); }
|
||||
.code .heat { width: 54px; text-align: right; margin-right: 12px; font-size: 10px; color: var(--muted); flex-shrink: 0; letter-spacing: .3px; }
|
||||
|
||||
/* ---- python syntax tokens ---- */
|
||||
.hljs-comment, .hljs-quote { color: #4a5d70; font-style: italic; }
|
||||
.hljs-keyword, .hljs-selector-tag { color: var(--pink); }
|
||||
.hljs-literal, .hljs-type { color: var(--purple); }
|
||||
.hljs-number { color: var(--amber); }
|
||||
.hljs-string, .hljs-meta .hljs-string { color: var(--green); }
|
||||
.hljs-built_in { color: var(--cyan); }
|
||||
.hljs-title, .hljs-title.function_ { color: var(--blue); }
|
||||
.hljs-title.class_ { color: var(--purple); }
|
||||
.hljs-params { color: var(--txt); }
|
||||
.hljs-attr, .hljs-attribute, .hljs-property { color: #9fd4ff; }
|
||||
.hljs-meta, .hljs-decorator, .hljs-meta .hljs-keyword { color: var(--pink); }
|
||||
.hljs-symbol, .hljs-bullet { color: var(--amber); }
|
||||
.hljs-variable, .hljs-template-variable { color: var(--txt); }
|
||||
.hljs-operator, .hljs-punctuation { color: #7f93a8; }
|
||||
|
||||
/* ---- call stack ---- */
|
||||
.stack { padding: 10px; display: flex; flex-direction: column; gap: 6px; }
|
||||
.frame {
|
||||
background: var(--panel2);
|
||||
border: 1px solid var(--line);
|
||||
border-left: 3px solid var(--purple);
|
||||
border-radius: 6px;
|
||||
padding: 7px 10px;
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
animation: slidein .2s ease;
|
||||
}
|
||||
.frame.top { border-left-color: var(--green); box-shadow: var(--glow) rgba(78,240,138,.15); }
|
||||
.frame .fn { color: var(--txt); }
|
||||
.frame .at { color: var(--muted); font-size: 11px; }
|
||||
@keyframes slidein { from { opacity: 0; transform: translateX(-8px); } to { opacity: 1; transform: none; } }
|
||||
|
||||
/* ---- timeline charts ---- */
|
||||
.chart-body { padding: 10px 12px; display: flex; flex-direction: column; gap: 6px; }
|
||||
.chart-body canvas { flex: 1; width: 100%; min-height: 0; display: block; }
|
||||
.chart-legend { display: flex; gap: 14px; font-size: 10px; color: var(--muted); }
|
||||
.chart-legend span { display: flex; align-items: center; }
|
||||
.dotlgd { display: inline-block; width: 8px; height: 8px; border-radius: 2px; margin-right: 5px; vertical-align: middle; }
|
||||
.dotlgd.rss { background: var(--amber); } .dotlgd.heap { background: var(--cyan); }
|
||||
|
||||
/* ---- hot lines ---- */
|
||||
.hotline { cursor: pointer; }
|
||||
.hotline .snippet { color: var(--muted); margin-left: 8px; font-size: 11px; }
|
||||
.hotline .hl-loc { color: var(--amber); }
|
||||
.hotline:hover { background: rgba(255,180,84,.07); }
|
||||
|
||||
/* ---- tables ---- */
|
||||
.table-wrap { padding: 0; overflow-x: hidden; }
|
||||
.tbl { width: 100%; border-collapse: collapse; font-size: 12px; table-layout: fixed; }
|
||||
.tbl col.c-num { width: 62px; }
|
||||
.tbl col.c-bar { width: 122px; }
|
||||
.tbl col.c-name { width: 24%; }
|
||||
.tbl col.c-type { width: 16%; }
|
||||
.tbl thead th {
|
||||
position: sticky; top: 0; z-index: 1;
|
||||
background: #0b1119;
|
||||
color: var(--muted);
|
||||
text-transform: uppercase;
|
||||
font-size: 9px;
|
||||
letter-spacing: 1px;
|
||||
text-align: left;
|
||||
padding: 7px 10px;
|
||||
border-bottom: 1px solid var(--line);
|
||||
font-weight: 700;
|
||||
}
|
||||
.tbl tbody td { padding: 5px 10px; border-bottom: 1px solid #101823; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }
|
||||
.tbl tbody tr:hover { background: rgba(41,230,212,.05); }
|
||||
.tbl th.num, .tbl td.num { text-align: right; }
|
||||
.tbl .fn-name { color: var(--green); }
|
||||
.tbl .own { color: var(--cyan); }
|
||||
.tbl .cum { color: var(--muted); }
|
||||
.bar-cell { position: relative; }
|
||||
.bar-track {
|
||||
position: relative; height: 13px; border-radius: 3px;
|
||||
background: rgba(120,150,180,.12); overflow: hidden;
|
||||
display: inline-block; vertical-align: middle; width: 84px;
|
||||
}
|
||||
.bar-track .cum { position: absolute; inset: 0 auto 0 0; height: 100%; background: rgba(74,168,255,.26); border-radius: 3px; transition: width .3s ease; }
|
||||
.bar-track .wait { position: absolute; inset: 0 auto 0 0; height: 100%; background: var(--amber); border-radius: 3px; transition: width .3s ease; }
|
||||
.bar-track .cpu { position: absolute; inset: 0 auto 0 0; height: 100%; background: var(--green); border-radius: 3px; transition: width .3s ease; box-shadow: var(--glow) rgba(78,240,138,.3); }
|
||||
.barkey { display: flex; align-items: center; gap: 4px; font-size: 9px; color: var(--muted); text-transform: uppercase; letter-spacing: .5px; margin-left: auto; margin-right: 8px; }
|
||||
.barkey .k { width: 8px; height: 8px; border-radius: 2px; margin: 0 2px 0 6px; }
|
||||
.barkey .k.cpu { background: var(--green); } .barkey .k.wait { background: var(--amber); } .barkey .k.child { background: rgba(74,168,255,.5); }
|
||||
.growthbar { height: 13px; border-radius: 3px; background: linear-gradient(90deg, var(--amber), var(--red)); box-shadow: var(--glow) rgba(255,92,108,.25); display: inline-block; vertical-align: middle; min-width: 2px; transition: width .3s; }
|
||||
.tbl .grow { color: var(--red); }
|
||||
.bar {
|
||||
height: 13px; border-radius: 3px;
|
||||
background: linear-gradient(90deg, var(--cyan), var(--green));
|
||||
transition: width .3s ease; min-width: 2px;
|
||||
box-shadow: var(--glow) rgba(41,230,212,.25);
|
||||
display: inline-block; vertical-align: middle;
|
||||
}
|
||||
.bar.mem { background: linear-gradient(90deg, var(--amber), var(--red)); box-shadow: var(--glow) rgba(255,180,84,.25); }
|
||||
.tbl .ttype { color: var(--purple); }
|
||||
.tbl .val { color: var(--muted); max-width: 260px; overflow: hidden; text-overflow: ellipsis; }
|
||||
.tbl tr.own-src .loc { color: var(--amber); }
|
||||
.tbl .loc { color: var(--muted); }
|
||||
|
||||
/* ---- logs + levels ---- */
|
||||
.log-tools { display: flex; align-items: center; gap: 8px; }
|
||||
.lvl-filter { display: flex; gap: 4px; }
|
||||
.lvl-chip { font-size: 9px; padding: 2px 7px; border-radius: 10px; cursor: pointer; border: 1px solid var(--line); color: var(--muted); text-transform: uppercase; letter-spacing: .5px; user-select: none; }
|
||||
.lvl-chip.off { opacity: .32; }
|
||||
.log { padding: 6px 0; font-size: 12px; line-height: 1.55; }
|
||||
/* block line with an inline-block badge: renders AND copies as a single line (flex splits on copy) */
|
||||
.log .lg { display: block; padding: 2px 12px 2px 10px; white-space: pre-wrap; word-break: break-word; border-left: 2px solid transparent; text-indent: -46px; padding-left: 58px; }
|
||||
.log .lg .badge { display: inline-block; width: 42px; text-indent: 0; font-size: 9px; text-align: center; border-radius: 4px; padding: 1px 0; margin-right: 6px; letter-spacing: .5px; }
|
||||
.log .lg .txt { text-indent: 0; }
|
||||
.log .lg.stdout .txt { color: var(--txt); }
|
||||
.log .lg.stderr .txt { color: #d7b3b8; }
|
||||
.log .lg .badge { background: #16202c; color: var(--muted); }
|
||||
.log .lg.debug { border-left-color: var(--muted); }
|
||||
.log .lg.debug .badge { background: #1a2430; color: #7d90a4; }
|
||||
.log .lg.info { border-left-color: var(--blue); }
|
||||
.log .lg.info .badge { background: rgba(74,168,255,.16); color: var(--blue); }
|
||||
.log .lg.info .txt { color: #bcd4ee; }
|
||||
.log .lg.warning { border-left-color: var(--amber); }
|
||||
.log .lg.warning .badge { background: rgba(255,180,84,.18); color: var(--amber); }
|
||||
.log .lg.warning .txt { color: #f2d3a3; }
|
||||
.log .lg.error { border-left-color: var(--red); }
|
||||
.log .lg.error .badge { background: rgba(255,92,108,.2); color: var(--red); }
|
||||
.log .lg.error .txt { color: #f4b8bf; }
|
||||
.log .lg.critical { border-left-color: var(--pink); background: rgba(255,106,193,.06); }
|
||||
.log .lg.critical .badge { background: var(--pink); color: #240; font-weight: 700; }
|
||||
.log .lg.critical .txt { color: #ffc4e8; font-weight: 500; }
|
||||
|
||||
/* ---- 3d flow ---- */
|
||||
.flow { position: relative; }
|
||||
.graph3d { position: absolute; inset: 0; }
|
||||
.flow-overlay { position: absolute; top: 16px; left: 16px; pointer-events: none; }
|
||||
.flow-card {
|
||||
pointer-events: auto;
|
||||
background: color-mix(in srgb, var(--panel) 82%, transparent);
|
||||
backdrop-filter: blur(12px);
|
||||
border: 1px solid var(--line2);
|
||||
border-radius: 12px;
|
||||
padding: 14px 16px;
|
||||
width: 280px;
|
||||
box-shadow: 0 8px 40px rgba(0,0,0,.5);
|
||||
}
|
||||
.flow-card h3 { font-family: 'Space Grotesk', sans-serif; font-size: 13px; color: var(--cyan); text-transform: uppercase; letter-spacing: 2px; margin-bottom: 6px; }
|
||||
.flow-card p { line-height: 1.5; margin-bottom: 12px; }
|
||||
.legend { display: flex; flex-wrap: wrap; gap: 10px; margin-bottom: 12px; }
|
||||
.legend span { display: flex; align-items: center; gap: 5px; font-size: 10px; color: var(--muted); }
|
||||
.legend .sw { width: 11px; height: 11px; border-radius: 3px; }
|
||||
.sw.cold { background: var(--blue); } .sw.warm { background: var(--amber); } .sw.hot { background: var(--red); } .sw.active { background: var(--green); box-shadow: var(--glow) var(--green); }
|
||||
.flow-stats { display: flex; gap: 8px; margin-bottom: 12px; }
|
||||
.flow-stats div { flex: 1; background: var(--panel2); border: 1px solid var(--line); border-radius: 8px; padding: 7px; text-align: center; }
|
||||
.flow-stats label { display: block; font-size: 8px; color: var(--muted); text-transform: uppercase; letter-spacing: 1px; margin-bottom: 3px; }
|
||||
.flow-stats b { color: var(--cyan); font-size: 13px; }
|
||||
#fHot { font-size: 11px; color: var(--green); }
|
||||
.flow-btns { display: flex; gap: 6px; }
|
||||
|
||||
/* ---- modal ---- */
|
||||
.modal { position: fixed; inset: 0; background: rgba(3,6,10,.82); display: flex; align-items: center; justify-content: center; z-index: 50; backdrop-filter: blur(4px); }
|
||||
.modal.hidden { display: none; }
|
||||
.modal-box { background: var(--panel); border: 1px solid var(--cyan); border-radius: 12px; width: 640px; max-height: 72vh; display: flex; flex-direction: column; box-shadow: var(--glow) rgba(41,230,212,.3); }
|
||||
.modal-head { display: flex; justify-content: space-between; align-items: center; padding: 12px 16px; border-bottom: 1px solid var(--line); }
|
||||
.modal-head h3 { font-size: 13px; color: var(--cyan); text-transform: uppercase; letter-spacing: 1px; }
|
||||
.pick-path { padding: 8px 16px; color: var(--muted); font-size: 11px; border-bottom: 1px solid var(--line); word-break: break-all; }
|
||||
.pick-list { overflow: auto; padding: 6px; }
|
||||
.pick-item { display: flex; align-items: center; gap: 10px; padding: 7px 12px; border-radius: 6px; cursor: pointer; transition: background .12s; }
|
||||
.pick-item:hover { background: var(--panel2); }
|
||||
.pick-item .ico { width: 16px; text-align: center; }
|
||||
.pick-item.dir .ico { color: var(--blue); }
|
||||
.pick-item.file .ico { color: var(--green); }
|
||||
.pick-item.up .ico { color: var(--amber); }
|
||||
|
||||
::-webkit-scrollbar { width: 9px; height: 9px; }
|
||||
::-webkit-scrollbar-track { background: transparent; }
|
||||
::-webkit-scrollbar-thumb { background: #1e2c3b; border-radius: 5px; }
|
||||
::-webkit-scrollbar-thumb:hover { background: #2a3d50; }
|
||||
+918
@@ -0,0 +1,918 @@
|
||||
// glassbox - client side telemetry rendering, 3d call-flow and leveled logs
|
||||
|
||||
const socket = io();
|
||||
const $ = (id) => document.getElementById(id);
|
||||
|
||||
let lastCurrent = -1;
|
||||
const sources = {}; // abs path -> { lines, hl }
|
||||
let displayedFile = null; // abs path currently shown in the viewer
|
||||
let followFile = true;
|
||||
let projectRoot = '';
|
||||
let lineSamples = {}; // abs file -> { lineno: count }
|
||||
let sampleDt = 0.004;
|
||||
let funcSort = { key: 'own', dir: -1 };
|
||||
let funcFilter = '';
|
||||
let lastFunctions = [];
|
||||
let lastTelemetry = null;
|
||||
const seenFiles = new Map(); // abs path -> display label
|
||||
const cpuHist = [];
|
||||
const rssHist = [];
|
||||
const heapHist = [];
|
||||
const sampleTimes = []; // performance.now() per sample, for wall-clock time-based scrolling
|
||||
const HIST = 200;
|
||||
const WINDOW_MS = 12000; // seconds of history shown on screen
|
||||
let chartsLive = false;
|
||||
let frozenAt = 0; // freeze the scroll clock here once the run ends
|
||||
|
||||
// ---- formatting helpers ----
|
||||
function fmtBytes(n) {
|
||||
if (n === null || n === undefined) return '—';
|
||||
const u = ['B', 'KB', 'MB', 'GB', 'TB'];
|
||||
let i = 0;
|
||||
while (n >= 1024 && i < u.length - 1) { n /= 1024; i++; }
|
||||
return (i === 0 ? n : n.toFixed(1)) + ' ' + u[i];
|
||||
}
|
||||
function fmtTime(s) {
|
||||
if (!s) return '0';
|
||||
if (s < 0.001) return (s * 1e6).toFixed(0) + 'µs';
|
||||
if (s < 1) return (s * 1000).toFixed(1) + 'ms';
|
||||
return s.toFixed(3) + 's';
|
||||
}
|
||||
function esc(t) { const d = document.createElement('div'); d.textContent = t; return d.innerHTML; }
|
||||
|
||||
// ---- source viewer ----
|
||||
function highlightPython(src) {
|
||||
// tokenize the whole file then split into lines, reopening spans that cross newlines
|
||||
let html;
|
||||
try { html = hljs.highlight(src, { language: 'python' }).value; }
|
||||
catch (e) { return src.split('\n').map(esc); }
|
||||
const lines = [];
|
||||
const open = [];
|
||||
let cur = '', i = 0;
|
||||
while (i < html.length) {
|
||||
const ch = html[i];
|
||||
if (ch === '<') {
|
||||
const close = html.indexOf('>', i);
|
||||
const tag = html.slice(i, close + 1);
|
||||
if (tag[1] === '/') open.pop();
|
||||
else if (tag[tag.length - 2] !== '/') open.push(tag);
|
||||
cur += tag;
|
||||
i = close + 1;
|
||||
} else if (ch === '\n') {
|
||||
lines.push(cur + '</span>'.repeat(open.length));
|
||||
cur = open.join('');
|
||||
i++;
|
||||
} else {
|
||||
let n = i;
|
||||
while (n < html.length && html[n] !== '<' && html[n] !== '\n') n++;
|
||||
cur += html.slice(i, n);
|
||||
i = n;
|
||||
}
|
||||
}
|
||||
lines.push(cur);
|
||||
return lines;
|
||||
}
|
||||
function relOf(abs) {
|
||||
if (!abs) return '';
|
||||
if (projectRoot && abs.startsWith(projectRoot + '/')) return abs.slice(projectRoot.length + 1);
|
||||
return abs.split('/').pop();
|
||||
}
|
||||
function addSeen(abs) {
|
||||
if (abs && !seenFiles.has(abs)) { seenFiles.set(abs, relOf(abs)); return true; }
|
||||
return false;
|
||||
}
|
||||
function updateFileDropdown() {
|
||||
const sel = $('fileSel');
|
||||
if (sel.options.length === seenFiles.size && sel.dataset.n === String(seenFiles.size)) return;
|
||||
sel.dataset.n = String(seenFiles.size);
|
||||
const entries = [...seenFiles.entries()].sort((a, b) => a[1].localeCompare(b[1]));
|
||||
sel.innerHTML = entries.map(([abs, rel]) => `<option value="${esc(abs)}">${esc(rel)}</option>`).join('') || '<option>no file loaded</option>';
|
||||
if (displayedFile) sel.value = displayedFile;
|
||||
}
|
||||
async function ensureSource(abs) {
|
||||
if (sources[abs]) return sources[abs];
|
||||
const r = await fetch('/api/source?path=' + encodeURIComponent(abs));
|
||||
const j = await r.json();
|
||||
if (j.error) { sources[abs] = { lines: ['# could not load ' + abs, '# ' + j.error], hl: null }; }
|
||||
else {
|
||||
const lines = j.lines;
|
||||
sources[abs] = { lines, hl: (typeof hljs !== 'undefined') ? highlightPython(lines.join('\n')) : lines.map(esc) };
|
||||
}
|
||||
return sources[abs];
|
||||
}
|
||||
function renderSourceLines(src) {
|
||||
const hl = src.hl || src.lines.map(esc);
|
||||
$('code').innerHTML = src.lines.map((l, i) =>
|
||||
`<div class="row" id="L${i + 1}"><span class="ln">${i + 1}</span><span class="heat" id="H${i + 1}"></span><span class="src">${hl[i] || ' '}</span></div>`
|
||||
).join('');
|
||||
lastCurrent = -1;
|
||||
}
|
||||
async function showFile(abs) {
|
||||
if (!abs || abs === displayedFile) return;
|
||||
const src = await ensureSource(abs);
|
||||
displayedFile = abs;
|
||||
addSeen(abs); updateFileDropdown();
|
||||
$('fileSel').value = abs;
|
||||
renderSourceLines(src);
|
||||
applyHeat(abs);
|
||||
}
|
||||
function applyHeat(abs) {
|
||||
const samples = lineSamples[abs];
|
||||
if (!samples) return;
|
||||
let max = 1;
|
||||
for (const k in samples) if (samples[k] > max) max = samples[k];
|
||||
for (const k in samples) {
|
||||
const row = $('L' + k);
|
||||
if (!row) continue;
|
||||
const ratio = samples[k] / max;
|
||||
row.style.setProperty('--heat', ratio.toFixed(3));
|
||||
const h = $('H' + k);
|
||||
if (h) {
|
||||
h.textContent = '~' + fmtTime(samples[k] * sampleDt);
|
||||
h.style.color = ratio > 0.15 ? heatColor(ratio) : 'var(--muted)';
|
||||
h.style.opacity = (0.35 + ratio * 0.65).toFixed(2);
|
||||
}
|
||||
}
|
||||
}
|
||||
function highlightLine(no) {
|
||||
if (no === lastCurrent) return;
|
||||
if (lastCurrent > 0) { const p = $('L' + lastCurrent); if (p) p.classList.remove('current'); }
|
||||
const row = $('L' + no);
|
||||
if (row) {
|
||||
row.classList.add('current', 'hit');
|
||||
const rect = row.getBoundingClientRect();
|
||||
const box = $('code').getBoundingClientRect();
|
||||
if (rect.top < box.top + 40 || rect.bottom > box.bottom - 40)
|
||||
row.scrollIntoView({ block: 'center', behavior: 'smooth' });
|
||||
}
|
||||
lastCurrent = no;
|
||||
}
|
||||
|
||||
// ---- call stack ----
|
||||
function frameLabel(f) { return f.name === '<module>' ? '‹' + baseName(f.rel) + '›' : f.name + '()'; }
|
||||
let lastStackKey = '';
|
||||
let maxStackDepth = 0;
|
||||
function renderStack(stack) {
|
||||
if (stack.length > maxStackDepth) maxStackDepth = stack.length;
|
||||
$('stackInfo').textContent = stack.length ? `depth ${stack.length} · max ${maxStackDepth}` : 'idle';
|
||||
const key = stack.map(f => f.id).join('>'); // only redraw when the call chain actually changes - stops the strobing
|
||||
if (key === lastStackKey) return;
|
||||
lastStackKey = key;
|
||||
const el = $('stack');
|
||||
if (!stack.length) { el.innerHTML = '<div class="muted" style="padding:8px">no active frames</div>'; return; }
|
||||
el.innerHTML = stack.map((f, i) =>
|
||||
`<div class="frame ${i === stack.length - 1 ? 'top' : ''}"><span class="fn">${esc(frameLabel(f))}</span><span class="at">${esc(f.rel || '')}:${f.line}</span></div>`
|
||||
).join('');
|
||||
const top = stack[stack.length - 1];
|
||||
$('liveFn').innerHTML = '<span class="exec">▸ ' + esc(frameLabel(top)) + '</span> ' + esc(top.rel || '') + ':' + top.line;
|
||||
}
|
||||
|
||||
// ---- function benchmark table ----
|
||||
function renderFuncs(rows) {
|
||||
lastFunctions = rows;
|
||||
const maxCum = Math.max(...rows.map(r => r.cum), 1e-9); // scale bars against the full set, not the filtered view
|
||||
const totalOwn = rows.reduce((a, r) => a + r.own, 0) || 1e-9;
|
||||
const q = funcFilter.toLowerCase();
|
||||
let view = q ? rows.filter(r => (r.name + ' ' + (r.file || '')).toLowerCase().includes(q)) : rows.slice();
|
||||
const k = funcSort.key, d = funcSort.dir;
|
||||
view.sort((a, b) => k === 'name' ? d * String(a.name).localeCompare(String(b.name)) : d * ((a[k] || 0) - (b[k] || 0)));
|
||||
document.querySelectorAll('.funcs thead th').forEach(th => {
|
||||
th.classList.toggle('sorted', th.dataset.sort === k);
|
||||
th.dataset.dir = th.dataset.sort === k ? (d < 0 ? 'desc' : 'asc') : '';
|
||||
});
|
||||
$('funcBody').innerHTML = view.map(r => {
|
||||
const cumW = (r.cum / maxCum) * 100;
|
||||
const ownW = (r.own / maxCum) * 100;
|
||||
const cpuW = ((r.own_cpu || 0) / maxCum) * 100;
|
||||
const nm = r.name === '<module>' ? baseName(r.file) : r.name;
|
||||
return `<tr title="${esc(r.file || '')}:${r.line}">
|
||||
<td class="fn-name">${esc(nm)}<span class="loc"> ${esc(baseName(r.file))}:${r.line}</span></td>
|
||||
<td class="num">${r.calls.toLocaleString()}</td>
|
||||
<td class="num own">${fmtTime(r.own)}</td>
|
||||
<td class="num" style="color:var(--green)">${fmtTime(r.own_cpu || 0)}</td>
|
||||
<td class="num cum">${fmtTime(r.cum)}</td>
|
||||
<td class="bar-cell"><span class="bar-track"><span class="cum" style="width:${cumW.toFixed(1)}%"></span><span class="wait" style="width:${ownW.toFixed(1)}%"></span><span class="cpu" style="width:${cpuW.toFixed(1)}%"></span></span></td>
|
||||
</tr>`;
|
||||
}).join('') || '<tr><td colspan="6" class="muted" style="padding:10px">no functions match</td></tr>';
|
||||
}
|
||||
|
||||
// ---- memory hotspots ----
|
||||
function renderHotspots(rows) {
|
||||
const max = Math.max(...rows.map(r => r.size), 1);
|
||||
$('hotBody').innerHTML = rows.map(r => {
|
||||
const pct = (r.size / max) * 100;
|
||||
return `<tr class="${r.own ? 'own-src' : ''}" title="${esc(r.file)}:${r.line}">
|
||||
<td class="loc">${esc(r.file)}:${r.line}</td>
|
||||
<td class="num">${r.count}</td>
|
||||
<td class="num">${fmtBytes(r.size)}</td>
|
||||
<td class="bar-cell"><div class="bar mem" style="width:${pct.toFixed(1)}%"></div></td>
|
||||
</tr>`;
|
||||
}).join('');
|
||||
}
|
||||
|
||||
// ---- memory growth (leak detection) ----
|
||||
function renderGrowth(rows) {
|
||||
if (!rows || !rows.length) { $('growthBody').innerHTML = '<tr><td colspan="4" class="muted" style="padding:10px">no growth yet — baseline still forming</td></tr>'; return; }
|
||||
const max = Math.max(...rows.map(r => r.size_diff), 1);
|
||||
$('growthBody').innerHTML = rows.map(r => {
|
||||
const pct = (r.size_diff / max) * 100;
|
||||
return `<tr class="${r.own ? 'own-src' : ''}" title="${esc(r.file)}:${r.line}">
|
||||
<td class="loc">${esc(r.file)}:${r.line}</td>
|
||||
<td class="num grow">+${r.count_diff.toLocaleString()}</td>
|
||||
<td class="num grow">+${fmtBytes(r.size_diff)}</td>
|
||||
<td class="bar-cell"><div class="growthbar" style="width:${pct.toFixed(1)}%"></div></td>
|
||||
</tr>`;
|
||||
}).join('');
|
||||
}
|
||||
|
||||
// ---- asyncio event-loop blocking ----
|
||||
function renderBlocking(b) {
|
||||
if (!b || !b.detected) { $('blockNow').textContent = 'no asyncio loop'; $('blockBody').innerHTML = '<tr><td colspan="4" class="muted" style="padding:10px">no asyncio event loop detected in this run</td></tr>'; return; }
|
||||
$('blockNow').textContent = 'blocked ' + fmtTime(b.total) + ' total';
|
||||
const rows = b.blockers || [];
|
||||
if (!rows.length) { $('blockBody').innerHTML = '<tr><td colspan="4" class="muted" style="padding:10px">loop healthy — no blocking callbacks</td></tr>'; return; }
|
||||
const max = Math.max(...rows.map(r => r.total), 1e-9);
|
||||
$('blockBody').innerHTML = rows.map(r => {
|
||||
const wpct = (r.total / max) * 100;
|
||||
const cpuFrac = r.cpu / Math.max(r.total, 1e-9);
|
||||
return `<tr title="${esc(r.file)}:${r.line}">
|
||||
<td class="fn-name">${esc(r.name)}<span class="loc"> ${esc(baseName(r.file))}:${r.line}</span></td>
|
||||
<td class="num">${r.count}</td>
|
||||
<td class="num own">${fmtTime(r.total)}</td>
|
||||
<td class="bar-cell"><span class="bar-track"><span class="wait" style="width:${wpct.toFixed(1)}%"></span><span class="cpu" style="width:${(wpct * cpuFrac).toFixed(1)}%"></span></span></td>
|
||||
</tr>`;
|
||||
}).join('');
|
||||
}
|
||||
|
||||
// ---- project file communication graph (which file calls into which) ----
|
||||
let FileGraph = null;
|
||||
const fgNodes = new Map();
|
||||
const fgLinks = new Map();
|
||||
let fileActiveEdges = new Set(); // file->file edges on the current live call path
|
||||
function fileLinkActive(l) { return fileActiveEdges.has(sid(l.source) + ' ' + sid(l.target)); }
|
||||
const DIR_COLORS = ['#4aa8ff', '#4ef08a', '#ffb454', '#b58cff', '#ff6ac1', '#29e6d4', '#ff5c6c', '#9fd4ff', '#f2d33a'];
|
||||
const dirIndex = new Map();
|
||||
function dirColor(file) {
|
||||
const dir = file.includes('/') ? file.slice(0, file.indexOf('/')) : '·';
|
||||
if (!dirIndex.has(dir)) dirIndex.set(dir, dirIndex.size);
|
||||
return DIR_COLORS[dirIndex.get(dir) % DIR_COLORS.length];
|
||||
}
|
||||
function curFileRel() { return (lastTelemetry && lastTelemetry.cur_file) ? relOf(lastTelemetry.cur_file) : null; }
|
||||
function fileNodeColor(n) { return n.id === curFileRel() ? '#4ef08a' : dirColor(n.id); }
|
||||
function fileGraphData(g) {
|
||||
// collapse the function call graph into files: a link means fileA calls a function defined in fileB
|
||||
const fileOf = new Map(g.nodes.map(n => [n.id, n.file]));
|
||||
const nodes = new Map();
|
||||
g.nodes.forEach(n => {
|
||||
let fn = nodes.get(n.file);
|
||||
if (!fn) { fn = { id: n.file, cum: 0, calls: 0 }; nodes.set(n.file, fn); }
|
||||
fn.cum += n.cum; fn.calls += n.calls;
|
||||
});
|
||||
const links = new Map();
|
||||
g.links.forEach(l => {
|
||||
const sf = fileOf.get(l.source), tf = fileOf.get(l.target);
|
||||
if (!sf || !tf || sf === tf) return; // only cross-file calls count as communication
|
||||
const k = sf + ' | ||||