loratop

A developer released loratop, a small terminal UI monitor for tracking LoRA training runs in Ostris AI Toolkit. The tool reads a run's loss_log.db or parses loss, step and learning-rate values from tqdm training output, rendering loss and learning-rate graphs that adapt to narrow terminals and fall back to a sparkline when the optional plotext dependency is unavailable. It is Linux-only, locating the running trainer via /proc, and can be pointed at a run folder, a training log, or a trainer PID.

| | /usr/bin/env -S uv run --script | | | SPDX-License-Identifier: MIT | | | /// script | | | dependencies = "rich", "plotext<6" plotext 6 dropped clf/plotsize/build | | | /// | | | """ | | | loratop - a tiny TUI monitor for Ostris AI Toolkit LoRA training. | | | It reads a run's loss log.db ai-toolkit writes it when logging.use ui logger | | | is true or, failing that, parses loss/step/lr from the tqdm output in a | | | training log. It renders loss and lr graphs under two lines of progress. | | | Designed to be readable on tiny screens, so it adapts to narrow terminals | | | and degrades gracefully when optional deps are missing. | | | Linux only: finding the trainer reads /proc. | | | Usage | | | ----- | | | loratop the run in this folder, else the one running trainer | | | loratop output/my lora a run folder: reads its loss log.db, live or finished | | | loratop train.log a training log python run.py cfg train.log 2 &1 | | | loratop 12345 a running trainer's PID: its loss log.db, else its log | | | Run it with uv uv run loratop.py , or chmod +x it and put it on your PATH | | | as loratop. | | | Deps plotext optional, falls back to a sparkline : | | | rich, plotext. Declared inline above, so uv installs them on first run. | | | """ | | | import argparse | | | import os | | | import re | | | import select | | | import shutil | | | import signal | | | import sqlite3 | | | import sys | | | import termios | | | import time | | | import tty | | | from collections import deque | | | ---- optional dependencies ------------------------------------------------- | | | try: | | | import plotext as plt | | | if not hasattr plt, "clf" : plotext 6+: fall back to the sparkline | | | plt = None | | | except ImportError: | | | plt = None | | | try: | | | from rich.console import Console, Group | | | from rich.live import Live | | | from rich.panel import Panel | | | from rich.table import Table | | | from rich.text import Text | | | except ImportError: | | | sys.exit "loratop needs 'rich': run it with uv, or pip install rich" | | | ---- parsing patterns tweak these if your log format differs ------------- | | | matches "loss: 0.0823", "loss=1.2e-01", "avg loss 0.05", etc. | | | RE LOSS = re.compile r"loss =:\s + 0-9 \.? 0-9 + ?: eE -+ ? 0-9 + ? ", re.I | | | matches the tqdm " 500/10000" current/total fragment | | | RE STEP = re.compile r" \d+ \s /\s \d+ " | | | matches "lr: 1.0e-04" | | | RE LR = re.compile r"lr =:\s + 0-9 \.? 0-9 + ?: eE -+ ? 0-9 + ? ", re.I | | | tqdm's own " elapsed<remaining, rate " fragment: " 01:10<2:30:00, 1.2it/s" | | | RE TQDM ELAPSED = re.compile r"\ \s ?:\d+: ?\d{1,2}:\d{2} \s <" | | | RE TQDM ETA = re.compile r"<\s ?:\d+: ?\d{1,2}:\d{2} " | | | RE TQDM RATE = re.compile r" \d. + \s it/s\|s/it " | | | SPARK = "▁▂▃▄▅▆▇█" | | | ai-toolkit's UILogger database, in the run's output folder | | | DB NAME = "loss log.db" | | | train.steps in the config.yaml ai-toolkit saves beside it; the database | | | itself doesn't record the total | | | RE CONFIG STEPS = re.compile r"^\s steps:\s \d+ \s $", re.M | | | Graph layout, after nvtop's: side by side by default, stacked once each | | | graph would get at least STACK MIN ROWS rows, loss alone when too narrow. | | | STACK MIN ROWS = 11 | | | MIN GRAPH COLS = 30 | | | --------------------------------------------------------------------------- | | | def pids : | | | """Every PID but ours, or none where there's no /proc.""" | | | try: | | | names = os.listdir "/proc" | | | except OSError: | | | return | | | me = os.getpid | | | return int n for n in names if n.isdigit and int n = me | | | def discover log from pid pid : | | | """Return the file stdout/stderr of pid is redirected to, if any.""" | | | for fd in 1, 2 : | | | link = f"/proc/{pid}/fd/{fd}" | | | try: | | | target = os.readlink link | | | except OSError: | | | continue | | | if os.path.isfile target : | | | return target | | | return None | | | def comm pid : | | | try: | | | with open f"/proc/{pid}/comm" as fh: | | | return fh.read .strip | | | except OSError: | | | return "" | | | def writes to pid, target : | | | """Yield fd, link for fds of pid open for writing on target path | | | or any fd if target is None .""" | | | fd dir = f"/proc/{pid}/fd" | | | try: | | | fds = os.listdir fd dir | | | except OSError: | | | return | | | for fd in fds: | | | try: | | | link = os.readlink f"{fd dir}/{fd}" | | | if target is not None and os.path.realpath link = target: | | | continue | | | with open f"/proc/{pid}/fdinfo/{fd}" as fh: | | | info = fh.read | | | m = re.search r"flags:\s \d+ ", info | | | if m and int m.group 1 , 8 & 0o3: O WRONLY or O RDWR | | | yield fd, link | | | except OSError: | | | continue | | | def discover pid from log log path : | | | """Find the training PID given only the log file: look for processes | | | writing to it, and if the writer is tee, hop across tee's stdin pipe to | | | the process feeding it.""" | | | real = os.path.realpath log path | | | writers = pid for pid in pids if any True for in writes to pid, real | | | direct redirect python run.py log 2 &1 : the trainer holds the file | | | for pid in writers: | | | if comm pid = "tee": | | | return pid | | | piped through tee: follow tee's stdin pipe back to its feeder | | | for tee pid in writers: | | | try: | | | pipe = os.readlink f"/proc/{tee pid}/fd/0" | | | except OSError: | | | continue | | | if not pipe.startswith "pipe:" : | | | continue | | | for pid in pids : | | | if pid == tee pid: | | | continue | | | for fd in 1, 2 : | | | try: | | | if os.readlink f"/proc/{pid}/fd/{fd}" == pipe: | | | return pid | | | except OSError: | | | continue | | | return None | | | def db of pid pid : | | | """The loss log.db pid has open, if any.""" | | | for , link in writes to pid, None : | | | if os.path.basename link == DB NAME and os.path.isfile link : | | | return link | | | return None | | | def argv pid : | | | try: | | | with open f"/proc/{pid}/cmdline", "rb" as fh: | | | return fh.read .decode errors="replace" .split "\0" | | | except OSError: | | | return | | | def find trainers : | | | """{path: pid} for running trainers: processes holding a loss log.db open, | | | or ai-toolkit's run.py with its output going to a file.""" | | | found = {} | | | for pid in pids : | | | db = db of pid pid | | | if db: | | | found.setdefault db, pid | | | elif any os.path.basename a == "run.py" for a in argv pid : | | | log = discover log from pid pid | | | if log: | | | found.setdefault log, pid | | | a trainer with a database also writes a log; show the database only | | | pids with db = {pid for path, pid in found.items if os.path.basename path == DB NAME} | | | return {path: pid for path, pid in found.items | | | if os.path.basename path == DB NAME or pid not in pids with db} | | | def resolve source target : | | | """Turn the command-line target or nothing into path, pid or None . | | | A folder means its loss log.db, a file a database or log, a number a PID.""" | | | if target is None: | | | db = os.path.join os.getcwd , DB NAME | | | if os.path.isfile db : | | | return db, None | | | found = find trainers | | | if len found == 1: | | | path, pid , = found.items | | | print f"found a running trainer: pid {pid}, {path}", file=sys.stderr | | | return path, pid | | | if found: | | | sys.exit "more than one trainer is running. pick one:\n" + "\n".join | | | f" loratop {pid} {path}" for path, pid in sorted found.items | | | sys.exit "nothing to watch: no loss log.db here and no running trainer.\n" | | | "pass a run folder, a training log, or a PID" | | | if os.path.isdir target : | | | db = os.path.join target, DB NAME | | | if os.path.isfile db : | | | return db, None | | | sys.exit f"{target} has no {DB NAME}. ai-toolkit writes one when " | | | "logging.use ui logger is true; otherwise pass the training log" | | | if os.path.isfile target : | | | return target, None | | | if target.isdigit : | | | pid = int target | | | path = db of pid pid or discover log from pid pid | | | if path: | | | print f"pid {pid}: {path}", file=sys.stderr | | | return path, pid | | | sys.exit f"pid {pid} has no {DB NAME} open and its output isn't going " | | | "to a file. pass the run folder or log instead" | | | sys.exit f"not a folder, file or PID: {target}" | | | def fmt secs s : | | | if s is None or s = s or s < 0: | | | return "--:--" | | | s = int s | | | h, rem = divmod s, 3600 | | | m, sec = divmod rem, 60 | | | return f"{h}:{m:02d}:{sec:02d}" if h else f"{m:02d}:{sec:02d}" | | | def term size : | | | """Current terminal size, straight from the tty ioctl. Ignores the | | | COLUMNS/LINES env vars, which multiplexers can leave stale.""" | | | for fh in sys. stdout , sys. stderr : | | | try: | | | return os.get terminal size fh.fileno | | | except OSError, ValueError, AttributeError : | | | continue | | | return shutil.get terminal size 80, 24 | | | def sparkline vals, width : | | | if not vals: | | | return "" | | | vals = list vals -width: | | | lo, hi = min vals , max vals | | | rng = hi - lo or 1.0 | | | return "".join SPARK min 7, int v - lo / rng 7 for v in vals | | | --------------------------------------------------------------------------- | | | class Tailer: | | | """Incrementally reads a growing file, splitting on \\r and \\n so that | | | tqdm's in-place progress bar updates become a stream of parseable lines.""" | | | def init self, path, backfill=8 1024 1024 : | | | self.path = path | | | self.fh = open path, "r", errors="replace" | | | self.inode = os.fstat self.fh.fileno .st ino | | | seed history from the tail of the existing file | | | try: | | | size = os.path.getsize path | | | self.fh.seek max 0, size - backfill | | | if size backfill: | | | self.fh.readline discard partial line | | | except OSError: | | | pass | | | self.buf = "" | | | def lines self : | | | handle log rotation / truncation | | | try: | | | if os.stat self.path .st ino = self.inode or \ | | | os.path.getsize self.path < self.fh.tell : | | | self.fh.close | | | self.fh = open self.path, "r", errors="replace" | | | self.inode = os.fstat self.fh.fileno .st ino | | | self.buf = "" | | | except OSError: | | | pass | | | chunk = self.fh.read | | | if not chunk: | | | return | | | self.buf += chunk | | | parts = re.split r" \r\n ", self.buf | | | self.buf = parts.pop keep trailing partial fragment | | | for line in parts: | | | if line.strip : | | | yield line | | | --------------------------------------------------------------------------- | | | class AggSeries: | | | """Fixed-capacity series covering the whole run: step, value points are | | | averaged into buckets of stride samples, and when the buffer fills, | | | adjacent buckets merge and the stride doubles. Memory stays bounded while | | | the x-axis always spans everything seen so far, in training steps.""" | | | def init self, cap : | | | self.cap = max 4, cap - cap % 2 even, so compaction pairs cleanly | | | self.stride = 1 | | | self.pts = | | | self. sx = self. sv = 0.0 | | | self. n = 0 | | | def append self, x, v : | | | self. sx += x | | | self. sv += v | | | self. n += 1 | | | if self. n = self.stride: | | | self.pts.append self. sx / self. n, self. sv / self. n | | | self. sx = self. sv = 0.0 | | | self. n = 0 | | | if len self.pts = self.cap: | | | self.pts = a 0 + b 0 / 2, a 1 + b 1 / 2 | | | for a, b in zip self.pts ::2 , self.pts 1::2 | | | self.stride = 2 | | | def view self : | | | """ xs, values , including the in-progress bucket so the tail stays live.""" | | | out = list self.pts | | | if self. n: | | | out.append self. sx / self. n, self. sv / self. n | | | return p 0 for p in out , p 1 for p in out | | | class State: | | | def init self, history, smooth, raw smooth=0.8 : | | | self. args = history, smooth, raw smooth | | | self.raw = AggSeries history | | | self.ema = AggSeries history | | | self.lr series = AggSeries history | | | self.smooth = smooth | | | Ostris-style smoothing on the "raw" trace: keep raw smooth of the | | | previous value, so spikes don't blow out the y-axis | | | self.raw alpha = 1.0 - raw smooth | | | self. raw ema = None | | | self. ema = None | | | self.step = self.total = None | | | self.lr = None | | | self.loss = None | | | self.min loss = None | | | self.rate pts = deque maxlen=60 t, step , distinct steps only | | | self.eta str = None tqdm's own remaining-time, straight from the log | | | self.elapsed str = None tqdm's own elapsed, so it covers the whole run | | | self. loss text = None | | | self.rate str = None tqdm's own it/s | | | self. t0 = None first wall time, for database sources | | | self.finished = False database source with no trainer writing it | | | def reset self : | | | self. init self. args | | | def add point self, x, v, lr : | | | self.loss = v | | | self. raw ema = v if self. raw ema is None else \ | | | self.raw alpha v + 1 - self.raw alpha self. raw ema | | | self.raw.append x, self. raw ema | | | self. ema = v if self. ema is None else \ | | | self.smooth v + 1 - self.smooth self. ema | | | self.ema.append x, self. ema | | | self.min loss = v if self.min loss is None else min self.min loss, v | | | if lr and lr 0: | | | self.lr series.append x, lr | | | def record self, step, wall time, loss, lr : | | | """One step from loss log.db.""" | | | self.step = step | | | if lr is not None: | | | self.lr = f"{lr:.1e}" | | | if loss is not None and loss == loss: not NaN | | | self. add point step, loss, lr | | | if wall time is not None: | | | if self. t0 is None: | | | self. t0 = wall time | | | self.rate pts.append wall time, step | | | self.elapsed str = fmt secs wall time - self. t0 | | | def feed self, line : | | | ms = RE STEP.search line | | | if ms: | | | cur, tot = int ms.group 1 , int ms.group 2 | | | the training bar is the one carrying a loss; until one shows up, | | | take any long bar. Sampling and caching bars are ignored after. | | | if tot = 1 and tot == self.total or RE LOSS.search line | | | or self.total is None and tot 50 : | | | if self.step is not None and cur < self.step: | | | self.rate pts.clear step went backwards: new run | | | tqdm redraws the same step several times set postfix etc ; | | | only distinct steps carry rate information | | | if cur = self.step: | | | self.rate pts.append time.time , cur | | | self.step, self.total = cur, tot | | | else: | | | ms = None another bar sampling, caching : ignore its timing | | | if ms: | | | tqdm computes its own smoothed remaining/rate; prefer that | | | mel = RE TQDM ELAPSED.search line | | | if mel: | | | self.elapsed str = mel.group 1 | | | me = RE TQDM ETA.search line | | | if me: | | | self.eta str = me.group 1 | | | mr = RE TQDM RATE.search line | | | if mr: | | | try: | | | v, unit = float mr.group 1 , mr.group 2 | | | self.rate str = f"{1 / v:.2f}" if unit == "s/it" and v 0 \ | | | else f"{v:.2f}" | | | except ValueError: | | | pass | | | ml = RE LR.search line | | | if ml: | | | self.lr = ml.group 1 | | | m = RE LOSS.search line | | | tqdm prints each loss twice: once when it's posted, and again on the | | | next redraw when the step counter ticks. Count it once. | | | if m and m.group 1 = self. loss text: | | | self. loss text = m.group 1 | | | try: | | | v = float m.group 1 | | | except ValueError: | | | v = None | | | if v is not None and v == v: not NaN | | | try: | | | lr = float self.lr if self.lr else None | | | except ValueError: | | | lr = None | | | self. add point self.step or 0, v, lr | | | def rate self : | | | if len self.rate pts < 2: | | | return None | | | t0, s0 , t1, s1 = self.rate pts 0 , self.rate pts -1 | | | dt = t1 - t0 | | | return s1 - s0 / dt if dt = 1.0 else None | | | def eta self : | | | r = self.rate | | | if r and self.total and self.step is not None and r 0: | | | return self.total - self.step / r | | | return None | | | class LogSource: | | | def init self, path : | | | self.tail = Tailer path | | | def poll self, state : | | | for line in self.tail.lines : | | | state.feed line | | | class DbSource: | | | """Reads new steps from ai-toolkit's loss log.db, read-only.""" | | | def init self, path : | | | self.path = path | | | self.con = None | | | self.last = 0 | | | self.total = None | | | try: | | | with open os.path.join os.path.dirname path , "config.yaml" as fh: | | | m = RE CONFIG STEPS.search fh.read | | | self.total = int m.group 1 if m else None | | | except OSError: | | | pass | | | def poll self, state : | | | try: | | | if self.con is None: | | | self.con = sqlite3.connect f"file:{self.path}?mode=ro", uri=True, timeout=1 | | | newest = self.con.execute "SELECT MAX step FROM steps" .fetchone 0 or 0 | | | if newest < self.last: | | | a resumed run deletes the steps after its checkpoint | | | state.reset | | | self.last = 0 | | | rows = self.con.execute """ | | | SELECT s.step, s.wall time, | | | MAX CASE WHEN m.key = 'loss/loss' THEN m.value real END , | | | MAX CASE WHEN m.key = 'learning rate' THEN m.value real END | | | FROM steps s LEFT JOIN metrics m ON m.step = s.step | | | WHERE s.step ? GROUP BY s.step ORDER BY s.step""", self.last, .fetchall | | | except sqlite3.Error: | | | self.con = None busy or mid-write; try again next tick | | | return | | | state.total = self.total | | | for step, wall time, loss, lr in rows: | | | state.record step, wall time, loss, lr | | | self.last = step | | | --------------------------------------------------------------------------- | | | def plot start w, h : | | | plt.clf | | | plotext clamps plotsize to its own terminal detection, which can | | | lag reality and reads 80x24 off-tty ; we size it ourselves | | | for fn in "limit size", "limitsize" : | | | if hasattr plt, fn : | | | getattr plt, fn False, False | | | break | | | plt.plotsize max 20, w , max 7, h | | | plt.theme "dark" | | | def build loss graph state, w, h : | | | raw x, raw = state.raw.view | | | ema x, ema = state.ema.view | | | if plt and len ema = 2: | | | plot start w, h | | | if len raw = 2: | | | plt.plot raw x, raw, color="red", label="raw", marker="braille" | | | plt.plot ema x, ema, color="cyan", label="ema", marker="braille" | | | plt.xlabel "step" | | | stride = state.raw.stride | | | plt.title f"loss avg of {stride} steps " if stride 1 else "loss" | | | return Text.from ansi plt.build .rstrip "\n" | | | fallback: unicode sparkline | | | if ema: | | | return Text sparkline ema, w - 4 , style="cyan" | | | return Text "waiting for loss data…", style="dim" | | | def build lr graph state, w, h : | | | xs, lrs = state.lr series.view | | | if plt and len lrs = 2: | | | plot start w, h | | | log scale: automagic-style optimizers climb orders of magnitude | | | plt.yscale "log" | | | plt.plot xs, lrs, color="orange", marker="braille" | | | plotext prints these as 0.00000600; label them in e-notation instead | | | lo, hi = min lrs , max lrs | | | n = max 2, min 5, h // 3 | | | ticks = lo hi / lo i / n - 1 for i in range n if hi lo else lo | | | plt.yticks ticks, f"{t:.1e}" for t in ticks | | | plt.xlabel "step" | | | plt.title "lr log " | | | return Text.from ansi plt.build .rstrip "\n" | | | if lrs: | | | return Text sparkline lrs, w - 4 , style="yellow" | | | return Text "waiting for lr data…", style="dim" | | | def header state, pid status : | | | """Two lines: progress on top, loss and lr below.""" | | | pct = state.step / state.total 100 if state.step and state.total else None | | | r = state.rate | | | def line pairs : | | | t = Text | | | for i, k, v in enumerate pairs : | | | if i: | | | t.append " " | | | t.append f"{k} ", style="dim" | | | t.append v, style="bold" | | | return t | | | top = line | | | "step", f"{'--' if state.step is None else state.step}/{state.total}" if state.total | | | else f"{'--' if state.step is None else state.step}" , | | | "progress", f"{pct:.1f}%" if pct is not None else "--" , | | | "it/s", state.rate str or f"{r:.2f}" if r else "--" , | | | "eta", "--" if state.finished else state.eta str or fmt secs state.eta , | | | "elapsed", state.elapsed str or "--:--" , | | | | | | bottom = line | | | "loss", f"{state.loss:.4f}" if state.loss is not None else "--" , | | | "lr", state.lr or "--" , | | | | | | if pid status: | | | bottom.append f" {pid status}", style="dim" | | | return top, bottom | | | def pid status pid : | | | if not pid: | | | return "" | | | try: | | | os.kill pid, 0 | | | return f"pid {pid}" | | | except OSError: | | | return f"pid {pid}: not running" | | | def render state, pid, console : | | | size = term size | | | cols inside the outer panel: border 2 + padding 2 | | | cols = size.columns - 4 | | | head = header state, pid status pid | | | rows for graph panels: total minus outer borders 2 and the header, | | | which wraps on narrow terminals | | | rows = size.lines - 2 - sum len t.wrap console, cols for t in head | | | each graph panel spends 2 rows and 4 cols on its border and padding | | | if rows - 4 // 2 = STACK MIN ROWS: | | | h = rows - 4 // 2 | | | graphs = Group | | | Panel build loss graph state, cols - 4, h , border style="white", padding= 0, 1 , | | | Panel build lr graph state, cols - 4, rows - 4 - h , border style="white", padding= 0, 1 , | | | | | | elif cols // 2 - 4 = MIN GRAPH COLS: | | | half = cols // 2 | | | graphs = Table.grid expand=True | | | graphs.add column ratio=1 | | | graphs.add column ratio=1 | | | graphs.add row | | | Panel build loss graph state, half - 4, rows - 2 , border style="white", padding= 0, 1 , | | | Panel build lr graph state, cols - half - 4, rows - 2 , border style="white", padding= 0, 1 , | | | | | | else: | | | graphs = Panel build loss graph state, cols - 4, rows - 2 , | | | border style="white", padding= 0, 1 | | | return Panel Group head, graphs , | | | title="loratop", subtitle="q to quit", border style="white" | | | --------------------------------------------------------------------------- | | | def main : | | | ap = argparse.ArgumentParser description="TUI monitor for Ostris AI Toolkit training" | | | ap.add argument "target", nargs="?", | | | help="run folder, training log, or PID default: the run in " | | | "this folder, else the one running trainer " | | | ap.add argument "--pid", type=int, help=argparse.SUPPRESS old spelling of a PID target | | | ap.add argument "--interval", type=float, default=1.0, help="refresh seconds" | | | ap.add argument "--history", type=int, default=600, | | | help="graph resolution: points kept before the plot " | | | "stride doubles to cover the whole run" | | | ap.add argument "--smooth", type=float, default=0.1, help="EMA factor 0-1" | | | ap.add argument "--raw-smooth", type=float, default=0.8, | | | help="Ostris-style smoothing 0-1 on the raw trace " | | | " 0 = truly raw " | | | ap.add argument "--once", action="store true", help="render one frame and exit debug " | | | args = ap.parse args | | | stale COLUMNS/LINES from a parent shell would freeze rich at the old | | | size; drop them so every layer measures the tty directly | | | os.environ.pop "COLUMNS", None | | | os.environ.pop "LINES", None | | | target = args.target if args.target is not None else \ | | | str args.pid if args.pid else None | | | path, pid = resolve source target | | | pinned = pid is not None | | | pid = pid or discover pid from log path | | | state = State args.history, args.smooth, args.raw smooth | | | is db = os.path.basename path == DB NAME | | | source = DbSource path if is db else LogSource path | | | source.poll state backfill history | | | if not is db: | | | backfilled log lines all carry "now" as their timestamp, useless | | | and wildly wrong for rate; start measuring fresh from here | | | state.rate pts.clear | | | state.finished = is db and not pid | | | console = Console | | | if args.once: | | | console.print render state, pid, console | | | return | | | def pid alive p : | | | try: | | | os.kill p, 0 | | | return True | | | except OSError: | | | return False | | | resized = False | | | try: | | | signal.signal signal.SIGWINCH, lambda : resized. setitem 0, True | | | except ValueError, AttributeError : | | | pass non-main thread or platform without SIGWINCH | | | cbreak mode so a bare 'q' quits cleanly exit 0, no SIGINT to the | | | process group, plays nice with TUI wrappers that suspend around us | | | kb fd = old attrs = None | | | if sys.stdin.isatty : | | | kb fd = sys.stdin.fileno | | | old attrs = termios.tcgetattr kb fd | | | tty.setcbreak kb fd | | | def key pressed timeout : | | | if kb fd is None: | | | time.sleep timeout | | | return None | | | r, , = select.select kb fd , , , timeout | | | return os.read kb fd, 1 if r else None | | | last disc = 0.0 | | | try: | | | with Live render state, pid, console , console=console, | | | refresh per second=4, screen=True as live: | | | while True: | | | re-discover if the trainer died/restarted unless pinned | | | if not pinned and pid is None or not pid alive pid \ | | | and time.monotonic - last disc 5: | | | last disc = time.monotonic | | | pid = discover pid from log path | | | source.poll state | | | state.finished = is db and not pid and pid alive pid | | | live.update render state, pid, console , refresh=True | | | deadline = time.monotonic + args.interval | | | while time.monotonic < deadline: | | | if resized 0 : | | | resized 0 = False | | | live.update render state, pid, console , | | | refresh=True | | | if key pressed 0.05 in b"q", b"Q" : | | | return | | | except KeyboardInterrupt: | | | pass | | | finally: | | | if old attrs is not None: | | | termios.tcsetattr kb fd, termios.TCSADRAIN, old attrs | | | if name == " main ": | | | main |