#!/usr/bin/env python3 """Build and serve an interactive timeline viewer for rollout trace dumps. The viewer consumes a rollout debug dump `.pt` file, extracts per-sample trace events, rebuilds spans and point events, and writes a lightweight JSON cache plus a self-contained HTML viewer next to the source file. """ from __future__ import annotations import argparse import functools import json import pickle import socketserver import sys import time import types from dataclasses import dataclass from http.server import SimpleHTTPRequestHandler from pathlib import Path from typing import Any import torch CACHE_VERSION = 1 class _MissingPickleObject: def __setstate__(self, state: Any) -> None: if isinstance(state, dict): self.__dict__.update(state) return self.__dict__["_raw_state"] = state _MISSING_PICKLE_GLOBALS: set[tuple[str, str]] = set() def _ensure_dummy_module(module_name: str) -> types.ModuleType: module = sys.modules.get(module_name) if isinstance(module, types.ModuleType): return module module = types.ModuleType(module_name) sys.modules[module_name] = module if "." in module_name: parent_name, child_name = module_name.rsplit(".", 1) parent = _ensure_dummy_module(parent_name) setattr(parent, child_name, module) return module def _make_dummy_pickle_global(module_name: str, name: str) -> type[_MissingPickleObject]: module = _ensure_dummy_module(module_name) existing = getattr(module, name, None) if isinstance(existing, type): return existing dummy_type = type(name, (_MissingPickleObject,), {"__module__": module_name}) setattr(module, name, dummy_type) _MISSING_PICKLE_GLOBALS.add((module_name, name)) return dummy_type class _DummyFallbackUnpickler(pickle.Unpickler): def find_class(self, module: str, name: str) -> Any: try: return super().find_class(module, name) except (AttributeError, ImportError, ModuleNotFoundError): return _make_dummy_pickle_global(module, name) _DUMMY_FALLBACK_PICKLE_MODULE = types.SimpleNamespace( __name__="pickle", Unpickler=_DummyFallbackUnpickler, load=pickle.load, loads=pickle.loads, ) @dataclass class TimelinePaths: pt_path: Path cache_path: Path html_path: Path def _json_safe(value: Any) -> Any: if isinstance(value, (str, int, float, bool)) or value is None: return value if isinstance(value, dict): return {str(k): _json_safe(v) for k, v in value.items()} if isinstance(value, (list, tuple)): return [_json_safe(v) for v in value] return str(value) def _round_float(value: float | None) -> float | None: if value is None: return None return round(float(value), 6) def _compact_text(value: Any, max_len: int = 256) -> Any: value = _json_safe(value) if not isinstance(value, str): return value if len(value) <= max_len: return value return f"{value[:max_len]}..." def _safe_duration(start: float | None, end: float | None) -> float | None: if start is None or end is None: return None return max(0.0, float(end) - float(start)) def _to_sample_dict(sample: Any) -> dict[str, Any]: if hasattr(sample, "to_dict"): sample = sample.to_dict() if isinstance(sample, dict): return sample result = {} for key in ( "group_index", "index", "prompt", "response", "response_length", "reward", "metadata", "source", "status", "label", "trace", ): if hasattr(sample, key): result[key] = getattr(sample, key) return result def _infer_source(sample: dict[str, Any], metadata: dict[str, Any]) -> Any: if sample.get("source") not in (None, ""): return sample.get("source") if metadata.get("source") not in (None, ""): return metadata.get("source") if metadata.get("source_name") not in (None, ""): return metadata.get("source_name") for key, value in metadata.items(): if "source" in str(key).lower() and value not in (None, ""): return value return None def _event_timestamp(event: dict[str, Any]) -> float | None: ts = event.get("ts") if ts is None: return None try: return float(ts) except (TypeError, ValueError): return None def _normalize_trace_events(trace: dict[str, Any]) -> list[dict[str, Any]]: raw_events = trace.get("events") or [] normalized = [] active_stack: list[str] = [] for order, raw_event in enumerate(raw_events): if not isinstance(raw_event, dict): continue ts = _event_timestamp(raw_event) if ts is None: continue event = { "order": order, "ts": ts, "type": _json_safe(raw_event.get("type")), "name": _json_safe(raw_event.get("name")), "attempt": int(raw_event.get("attempt", trace.get("attempt", 0)) or 0), "sample_id": _json_safe(raw_event.get("sample_id", trace.get("sample_id"))), "group_id": _json_safe(raw_event.get("group_id", trace.get("group_id"))), "span_id": _json_safe(raw_event.get("span_id")), "parent_span_id": _json_safe(raw_event.get("parent_span_id")), "attrs": _json_safe(raw_event.get("attrs") or {}), } event["inferred_parent_span_id"] = active_stack[-1] if active_stack else None normalized.append(event) if event["type"] == "span_start" and event["span_id"]: active_stack.append(event["span_id"]) continue if event["type"] == "span_end" and event["span_id"]: for idx in range(len(active_stack) - 1, -1, -1): if active_stack[idx] == event["span_id"]: del active_stack[idx] break return normalized def _span_name(item: dict[str, Any]) -> str: return str(item.get("name") or "span") def _span_type(item: dict[str, Any]) -> str: if item["type"] == "event": return "point_event" if item["type"] == "orphan_end": return "orphan_end" return item["state"] def _compute_span_depths(spans: list[dict[str, Any]]) -> dict[str, int]: span_by_id = {span["span_id"]: span for span in spans if span.get("span_id")} cache: dict[str, int] = {} def resolve(span_id: str | None, seen: set[str]) -> int: if not span_id or span_id not in span_by_id: return 0 if span_id in cache: return cache[span_id] if span_id in seen: cache[span_id] = 0 return 0 seen.add(span_id) parent_id = span_by_id[span_id].get("parent_span_id") depth = 0 if not parent_id or parent_id not in span_by_id else resolve(parent_id, seen) + 1 cache[span_id] = depth return depth for span in spans: span_id = span.get("span_id") if span_id: resolve(span_id, set()) return cache def _build_items_from_trace(sample: dict[str, Any], sample_idx: int) -> dict[str, Any] | None: trace = sample.get("trace") if not isinstance(trace, dict): return None events = _normalize_trace_events(trace) if not events: return None open_starts: dict[str, dict[str, Any]] = {} closed_spans: list[dict[str, Any]] = [] point_events: list[dict[str, Any]] = [] orphan_ends: list[dict[str, Any]] = [] all_timestamps: list[float] = [] for event in events: all_timestamps.append(event["ts"]) event_type = event["type"] if event_type == "span_start" and event["span_id"]: open_starts[event["span_id"]] = { "type": "span", "state": "closed_span", "name": event["name"], "start_ts": event["ts"], "end_ts": None, "display_end_ts": None, "attempt": event["attempt"], "span_id": event["span_id"], "parent_span_id": event.get("parent_span_id") or event.get("inferred_parent_span_id"), "start_attrs": event.get("attrs") or {}, "end_attrs": {}, } continue if event_type == "span_end": span_id = event.get("span_id") start_record = open_starts.pop(span_id, None) if span_id else None if start_record is None: orphan_ends.append( { "type": "orphan_end", "state": "orphan_end", "name": event["name"], "ts": event["ts"], "attempt": event["attempt"], "span_id": span_id, "parent_span_id": event.get("parent_span_id") or event.get("inferred_parent_span_id"), "attrs": event.get("attrs") or {}, } ) continue start_record["end_ts"] = event["ts"] start_record["display_end_ts"] = event["ts"] start_record["end_attrs"] = event.get("attrs") or {} closed_spans.append(start_record) continue point_events.append( { "type": "event", "state": "point_event", "name": event["name"], "ts": event["ts"], "attempt": event["attempt"], "span_id": None, "parent_span_id": event.get("parent_span_id") or event.get("inferred_parent_span_id"), "attrs": event.get("attrs") or {}, } ) row_end_ts = max(all_timestamps) if all_timestamps else None open_spans = list(open_starts.values()) all_spans = closed_spans + open_spans span_depths = _compute_span_depths(all_spans) span_by_id = {span["span_id"]: span for span in all_spans if span.get("span_id")} sibling_groups: dict[str | None, list[dict[str, Any]]] = {} for span in all_spans: sibling_groups.setdefault(span.get("parent_span_id"), []).append(span) for siblings in sibling_groups.values(): siblings.sort(key=lambda item: (item["start_ts"], item["end_ts"] or float("inf"), _span_name(item))) def nearest_closed_ancestor_end(span: dict[str, Any]) -> float | None: current_parent = span.get("parent_span_id") while current_parent: parent = span_by_id.get(current_parent) if parent is None: return None if parent.get("end_ts") is not None: return float(parent["end_ts"]) current_parent = parent.get("parent_span_id") return None for span in open_spans: candidates: list[tuple[float, str]] = [] if row_end_ts is not None: candidates.append((row_end_ts, "row_end")) siblings = sibling_groups.get(span.get("parent_span_id"), []) for sibling in siblings: if sibling is span: continue sibling_start = float(sibling["start_ts"]) if sibling_start > float(span["start_ts"]): candidates.append((sibling_start, "next_sibling_start")) break ancestor_end = nearest_closed_ancestor_end(span) if ancestor_end is not None and ancestor_end > float(span["start_ts"]): candidates.append((ancestor_end, "ancestor_end")) if candidates: display_end_ts, clipped_by = min(candidates, key=lambda item: item[0]) if display_end_ts <= float(span["start_ts"]): display_end_ts = float(span["start_ts"]) clipped_by = "self" else: display_end_ts = float(span["start_ts"]) clipped_by = "self" span["state"] = "open_span" span["display_end_ts"] = display_end_ts span.setdefault("end_attrs", {}) span["end_attrs"]["clipped_by"] = clipped_by for span in all_spans: span["depth"] = span_depths.get(span.get("span_id") or "", 0) span["lane"] = span["depth"] for event in point_events: parent_span_id = event.get("parent_span_id") event["depth"] = span_depths[parent_span_id] + 1 if parent_span_id in span_depths else 0 event["lane"] = event["depth"] for item in orphan_ends: parent_span_id = item.get("parent_span_id") item["depth"] = span_depths[parent_span_id] + 1 if parent_span_id in span_depths else 0 item["lane"] = item["depth"] def parent_span_name(parent_span_id: str | None) -> str | None: if not parent_span_id: return None parent = span_by_id.get(parent_span_id) if not parent: return None return parent.get("name") all_items: list[dict[str, Any]] = [] for span in all_spans: all_items.append( { "type": "span", "state": span["state"], "name": span["name"], "start_ts": _round_float(span["start_ts"]), "end_ts": _round_float(span["end_ts"]), "display_end_ts": _round_float(span["display_end_ts"]), "attempt": span["attempt"], "span_id": span.get("span_id"), "parent_span_id": span.get("parent_span_id"), "parent_span_name": parent_span_name(span.get("parent_span_id")), "lane": span["lane"], "depth": span["depth"], "attrs": { "start_attrs": _json_safe(span.get("start_attrs") or {}), "end_attrs": _json_safe(span.get("end_attrs") or {}), }, } ) for event in point_events: all_items.append( { "type": "event", "state": "point_event", "name": event["name"], "ts": _round_float(event["ts"]), "attempt": event["attempt"], "span_id": None, "parent_span_id": event.get("parent_span_id"), "parent_span_name": parent_span_name(event.get("parent_span_id")), "lane": event["lane"], "depth": event["depth"], "attrs": _json_safe(event.get("attrs") or {}), } ) for item in orphan_ends: all_items.append( { "type": "orphan_end", "state": "orphan_end", "name": item["name"], "ts": _round_float(item["ts"]), "attempt": item["attempt"], "span_id": item.get("span_id"), "parent_span_id": item.get("parent_span_id"), "parent_span_name": parent_span_name(item.get("parent_span_id")), "lane": item["lane"], "depth": item["depth"], "attrs": _json_safe(item.get("attrs") or {}), } ) pd_lane_specs = [ ( "prefill", "P", [ "pd_prefill_bootstrap_queue_duration", "pd_prefill_bootstrap_duration", "pd_prefill_alloc_wait_duration", "pd_prefill_forward_duration", "pd_prefill_transfer_queue_duration", ], ), ( "decode", "D", [ "pd_decode_prealloc_duration", "pd_decode_bootstrap_duration", "pd_decode_alloc_wait_duration", "pd_decode_transfer_duration", "pd_decode_forward_duration", ], ), ] next_virtual_lane = max((item["lane"] for item in all_items), default=-1) for span in all_spans: if span["state"] != "closed_span" or span.get("end_ts") is None: continue if str(span.get("name") or "").startswith("sglang_pd_"): continue end_attrs = span.get("end_attrs") or {} for role, suffix, keys in pd_lane_specs: role_attrs = { key: value for key in keys if isinstance((value := end_attrs.get(key)), (int, float)) and value > 0 } if not role_attrs: continue next_virtual_lane += 1 role_attrs.update( { "timeline_pd_virtual_role": role, "timeline_pd_parent_name": span["name"], "timeline_pd_parent_duration": _round_float(_safe_duration(span["start_ts"], span["end_ts"])), } ) all_items.append( { "type": "span", "state": "closed_span", "name": f'{span["name"]} [{suffix}]', "start_ts": _round_float(span["start_ts"]), "end_ts": _round_float(span["end_ts"]), "display_end_ts": _round_float(span["display_end_ts"]), "attempt": span["attempt"], "span_id": f'{span.get("span_id") or span["name"]}:pd:{role}', "parent_span_id": span.get("span_id"), "parent_span_name": span["name"], "lane": next_virtual_lane, "depth": next_virtual_lane, "attrs": { "start_attrs": {}, "end_attrs": _json_safe(role_attrs), }, } ) all_items.sort( key=lambda item: ( item["lane"], item.get("start_ts", item.get("ts", 0.0)), item.get("display_end_ts", item.get("ts", 0.0)), item["name"], ) ) row_start = min(item.get("start_ts", item.get("ts")) for item in all_items) row_end = max(item.get("display_end_ts", item.get("ts")) for item in all_items) response_lengths = [] for item in all_items: attrs = item.get("attrs") or {} for payload in (attrs, attrs.get("start_attrs"), attrs.get("end_attrs")): if not isinstance(payload, dict): continue response_length = payload.get("response_length") if isinstance(response_length, (int, float)): response_lengths.append(int(response_length)) metadata = sample.get("metadata") or {} if not isinstance(metadata, dict): metadata = {} reward = sample.get("reward") if isinstance(reward, dict): reward = _json_safe(reward) return { "row_id": sample_idx, "sample_index": sample.get("index", sample_idx), "group_index": sample.get("group_index"), "source": _compact_text(_infer_source(sample, metadata), max_len=64), "status": _compact_text(sample.get("status"), max_len=64), "label": _compact_text(sample.get("label"), max_len=256), "reward": reward, "trace_id": _json_safe(trace.get("trace_id")), "attempt": int(trace.get("attempt", 0) or 0), "start": row_start, "end": row_end, "duration": _round_float(_safe_duration(row_start, row_end)), "lane_count": 1 + max((item["lane"] for item in all_items), default=0), "item_count": len(all_items), "closed_span_count": sum(1 for item in all_items if item["state"] == "closed_span"), "open_span_count": sum(1 for item in all_items if item["state"] == "open_span"), "point_event_count": sum(1 for item in all_items if item["state"] == "point_event"), "orphan_count": sum(1 for item in all_items if item["state"] == "orphan_end"), "total_response_length": sum(response_lengths), "max_response_length": max(response_lengths, default=0), "items": all_items, } def _build_cache_data(pt_path: Path) -> dict[str, Any]: before_missing = len(_MISSING_PICKLE_GLOBALS) data = torch.load( pt_path, map_location="cpu", weights_only=False, pickle_module=_DUMMY_FALLBACK_PICKLE_MODULE, ) if len(_MISSING_PICKLE_GLOBALS) > before_missing: missing_names = ", ".join(f"{module}.{name}" for module, name in sorted(_MISSING_PICKLE_GLOBALS)) print( f"[trace_timeline_viewer] substituted missing pickle globals with dummy classes: {missing_names}", file=sys.stderr, ) samples = data["samples"] if isinstance(data, dict) and "samples" in data else data rows: list[dict[str, Any]] = [] global_start = None global_end = None for sample_idx, raw_sample in enumerate(samples): sample = _to_sample_dict(raw_sample) row = _build_items_from_trace(sample, sample_idx) if row is None: continue rows.append(row) global_start = row["start"] if global_start is None else min(global_start, row["start"]) global_end = row["end"] if global_end is None else max(global_end, row["end"]) return { "cache_version": CACHE_VERSION, "pt_path": str(pt_path), "generated_at": time.time(), "sample_count": len(rows), "global_start": _round_float(global_start), "global_end": _round_float(global_end), "rows": rows, } def _timeline_paths(pt_path: Path) -> TimelinePaths: stem = pt_path.stem directory = pt_path.parent return TimelinePaths( pt_path=pt_path, cache_path=directory / f"{stem}.trace_timeline_cache.json", html_path=directory / f"{stem}.trace_timeline_viewer.html", ) def ensure_cache(paths: TimelinePaths, rebuild: bool = False) -> dict[str, Any]: if not rebuild and paths.cache_path.exists() and paths.cache_path.stat().st_mtime >= paths.pt_path.stat().st_mtime: with paths.cache_path.open("r", encoding="utf-8") as handle: cached = json.load(handle) if cached.get("cache_version") == CACHE_VERSION: return cached cache_data = _build_cache_data(paths.pt_path) with paths.cache_path.open("w", encoding="utf-8") as handle: json.dump(cache_data, handle, ensure_ascii=True, separators=(",", ":")) return cache_data HTML_TEMPLATE = r""" __TITLE__
Trace Timeline
drag = pan, wheel = zoom, click = set cursor and select item
""" def ensure_html(paths: TimelinePaths) -> None: title = f"{paths.pt_path.name} trace timeline" html = HTML_TEMPLATE.replace("__CACHE_FILE__", paths.cache_path.name).replace("__TITLE__", title) with paths.html_path.open("w", encoding="utf-8") as handle: handle.write(html) class QuietHandler(SimpleHTTPRequestHandler): def log_message(self, format: str, *args: Any) -> None: return def serve_directory(directory: Path, port: int) -> None: handler = functools.partial(QuietHandler, directory=str(directory)) with socketserver.TCPServer(("0.0.0.0", port), handler) as httpd: print(f"Serving http://127.0.0.1:{port}/") print("Press Ctrl+C to stop.") try: httpd.serve_forever() except KeyboardInterrupt: pass def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("pt_path", help="Path to rollout debug dump .pt file") parser.add_argument("--rebuild", action="store_true", help="Rebuild cache even if it already exists") parser.add_argument( "--serve", action=argparse.BooleanOptionalAction, default=True, help="Start a local static file server for the generated HTML", ) parser.add_argument("--port", type=int, default=9999, help="Port for --serve") return parser.parse_args() def main() -> None: args = parse_args() pt_path = Path(args.pt_path).expanduser().resolve() if not pt_path.exists(): raise SystemExit(f"pt file not found: {pt_path}") paths = _timeline_paths(pt_path) cache_data = ensure_cache(paths, rebuild=args.rebuild) ensure_html(paths) print(f"pt: {paths.pt_path}") print(f"cache: {paths.cache_path}") print(f"html: {paths.html_path}") print(f"samples: {cache_data['sample_count']}") if args.serve: serve_directory(paths.html_path.parent, args.port) if __name__ == "__main__": main()