gpu_retired_pages.py -- dump retired NVIDIA GPU memory pages WITH retirement timestamps

A developer released gpu_retired_pages.py, a read-only Python script that uses NVML to dump retired GPU memory pages with retirement timestamps, a feature nvidia-smi lacks. The tool auto-detects timestamp units, warns when ECC is disabled, and is intended for vetting used datacenter GPUs, though it does not support Ampere and later GPUs, which use row remapping instead.

| /usr/bin/env python3 | | | """ | | | gpu retired pages.py -- dump retired GPU memory pages WITH retirement timestamps. | | | nvidia-smi reports retired page counts and addresses but not when each page | | | was retired. NVML's nvmlDeviceGetRetiredPages v2 does return timestamps. This | | | script calls it via ctypes against libnvidia-ml.so.1, so it needs no Python | | | packages -- only the installed NVIDIA driver. | | | Retired pages live in the GPU's InfoROM and persist for the life of the board, | | | so timestamps predating a purchase date show the memory was already degrading | | | before you owned it. That makes this useful for vetting used datacenter GPUs. | | | SCOPE -- this reads dynamic page retirement, a PRE-AMPERE mechanism: | | | Supported : Tesla K20 and higher, Quadro 5000 and higher, with framebuffer | | | ECC enabled. Covers K80, P40, P100, V100, T4 and similar. | | | Excluded : GeForce and GRID -- no page retirement at all. | | | Ampere+ : A100, H100, L40, Blackwell and later replaced page retirement | | | with ROW REMAPPING. This script reports "not supported" on them. | | | The equivalent calls are nvmlDeviceGetRemappedRows and | | | nvmlDeviceGetRowRemapperHistogram banks, not pages . | | | READ-ONLY -- this script only issues NVML query calls. It never changes ECC | | | mode, resets counters, resets the GPU, or touches the InfoROM. The sole thing | | | it writes is the optional -o output file on local disk. Safe to run against | | | production GPUs and against hardware you are evaluating but do not own. | | | CRITICAL CAVEAT -- pages are only retired while ECC is ENABLED. A card running | | | with ECC off silently accumulates memory errors and retires nothing. An empty | | | table on such a card means "nothing was being watched", NOT "healthy memory". | | | The script reports ECC mode per GPU and warns whenever it is off. | | | Usage: | | | python3 gpu retired pages.py all GPUs | | | python3 gpu retired pages.py -i 0 one GPU by index | | | python3 gpu retired pages.py -i 0 -o out.txt also write to file | | | python3 gpu retired pages.py --csv machine-readable | | | """ | | | import argparse | | | import ctypes | | | import datetime | | | import sys | | | NVML SUCCESS = 0 | | | NVML ERROR NOT SUPPORTED = 3 | | | NVML ERROR INSUFFICIENT SIZE = 7 | | | CAUSE SBE = 0 multiple single-bit ECC errors on the same address | | | CAUSE DBE = 1 a single double-bit ECC error | | | CAUSE LABEL = { | | | CAUSE SBE: "SBE 2+ single-bit ", | | | CAUSE DBE: "DBE double-bit ", | | | } | | | Sanity window for interpreting timestamps: 2000-01-01 .. 2050-01-01 | | | EPOCH MIN = 946 684 800 | | | EPOCH MAX = 2 524 608 000 | | | def load nvml : | | | for name in "libnvidia-ml.so.1", "libnvidia-ml.so" : | | | try: | | | return ctypes.CDLL name | | | except OSError: | | | continue | | | sys.exit "Could not load libnvidia-ml.so.1 -- is the NVIDIA driver installed?" | | | def err str nvml, rc : | | | nvml.nvmlErrorString.restype = ctypes.c char p | | | try: | | | return nvml.nvmlErrorString rc .decode | | | except Exception: | | | return f"NVML error {rc}" | | | def check nvml, rc, what : | | | if rc = NVML SUCCESS: | | | sys.exit f"{what} failed: {err str nvml, rc }" | | | def get string fn, handle, length=96 : | | | buf = ctypes.create string buffer length | | | rc = fn handle, buf, ctypes.c uint length | | | if rc = NVML SUCCESS: | | | return "n/a" | | | return buf.value.decode errors="replace" | | | def decode timestamp raw : | | | """NVML documents these as Unix timestamps, but the unit has varied across | | | driver versions. Auto-detect the scale that lands in a plausible date range.""" | | | if raw == 0: | | | return None, "zero" | | | for divisor, unit in 1, "s" , 1e3, "ms" , 1e6, "us" , 1e9, "ns" : | | | secs = raw / divisor | | | if EPOCH MIN < secs < EPOCH MAX: | | | return datetime.datetime.fromtimestamp secs , unit | | | return None, "unrecognized" | | | def get retired pages nvml, handle, cause : | | | """Returns list of address, raw timestamp , status string .""" | | | fn = getattr nvml, "nvmlDeviceGetRetiredPages v2", None | | | if fn is None: | | | return , "nvmlDeviceGetRetiredPages v2 unavailable in this driver" | | | capacity = 1024 spec caps the InfoROM table well below this | | | while True: | | | count = ctypes.c uint capacity | | | addrs = ctypes.c ulonglong capacity | | | stamps = ctypes.c ulonglong capacity | | | rc = fn handle, ctypes.c uint cause , ctypes.byref count , addrs, stamps | | | if rc == NVML SUCCESS: | | | return addrs i , stamps i for i in range count.value , "ok" | | | if rc == NVML ERROR INSUFFICIENT SIZE: | | | capacity = max count.value, capacity 2 | | | continue | | | if rc == NVML ERROR NOT SUPPORTED: | | | return , "page retirement not supported -- Ampere and later use " | | | "row remapping instead see nvidia-smi -q -d ROW REMAPPER " | | | return , err str nvml, rc | | | def report gpu nvml, index, csv mode, out : | | | handle = ctypes.c void p | | | rc = nvml.nvmlDeviceGetHandleByIndex v2 ctypes.c uint index , ctypes.byref handle | | | check nvml, rc, f"nvmlDeviceGetHandleByIndex v2 {index} " | | | name = get string nvml.nvmlDeviceGetName, handle, 96 | | | serial = get string nvml.nvmlDeviceGetSerial, handle, 32 | | | uuid = get string nvml.nvmlDeviceGetUUID, handle, 96 | | | Page retirement only happens while ECC is on. Without this, an empty | | | table on an ECC-disabled card reads as "healthy" when it means "blind". | | | cur, pend = ctypes.c uint , ctypes.c uint | | | if nvml.nvmlDeviceGetEccMode handle, ctypes.byref cur , | | | ctypes.byref pend == NVML SUCCESS: | | | ecc = "enabled" if cur.value else "DISABLED" | | | if cur.value = pend.value: | | | ecc += f" pending: {'enabled' if pend.value else 'disabled'} -- needs reboot " | | | else: | | | ecc = "unknown" | | | rows = | | | notes = | | | for cause in CAUSE DBE, CAUSE SBE : | | | pages, status = get retired pages nvml, handle, cause | | | if status = "ok": | | | notes.append f"{CAUSE LABEL cause }: {status}" | | | for addr, raw in pages: | | | dt, unit = decode timestamp raw | | | rows.append dt, unit, raw, addr, CAUSE LABEL cause | | | Sort oldest first; undated entries last. | | | rows.sort key=lambda r: r 0 is None, r 0 or datetime.datetime.min | | | if csv mode: | | | ecc field = ecc.replace ",", ";" | | | if not rows: | | | Still emit one row so ECC state is visible for every GPU. | | | out f"{index},{serial},{uuid},{ecc field},,,," | | | for dt, unit, raw, addr, cause in rows: | | | out f"{index},{serial},{uuid},{ecc field},{cause},0x{addr:016x}," | | | f"{raw},{dt.isoformat if dt else ''}" | | | return | | | out "" | | | out f"GPU {index}: {name}" | | | out f" Serial : {serial}" | | | out f" UUID : {uuid}" | | | out f" ECC mode : {ecc}" | | | out f" Retired pages: {len rows }" | | | for note in notes: | | | out f" Note: {note}" | | | Fires whether or not pages exist. A populated table on an ECC-disabled | | | card is the more misleading case: it looks like a complete history, but | | | records only the periods when ECC happened to be on. | | | if ecc.startswith "DISABLED" : | | | out "" | | | out " WARNING: ECC is disabled on this GPU." | | | out " No pages can be retired and no new memory errors are" | | | out " detected while ECC is off. Any history below covers" | | | out " only periods when ECC was enabled, and absence of" | | | out " entries is not evidence of healthy memory." | | | if not rows: | | | out " no retired pages " | | | return | | | out "" | | | out f" {'Retired at':<21} {'Address':<20} {'Cause':<20} {'Raw timestamp'}" | | | out f" {'-' 21} {'-' 20} {'-' 20} {'-' 20}" | | | for dt, unit, raw, addr, cause in rows: | | | when = dt.strftime "%Y-%m-%d %H:%M:%S" if dt else f"unparsed {unit} " | | | out f" {when:<21} 0x{addr:016x} {cause:<20} {raw}" | | | dated = r 0 for r in rows if r 0 is not None | | | if dated: | | | out "" | | | out f" Earliest retirement: {min dated }" | | | out f" Latest retirement : {max dated }" | | | def main : | | | ap = argparse.ArgumentParser description= doc , | | | formatter class=argparse.RawDescriptionHelpFormatter | | | ap.add argument "-i", "--index", type=int, default=None, | | | help="GPU index default: all GPUs " | | | ap.add argument "-o", "--output", help="also write output to this file" | | | ap.add argument "--csv", action="store true", | | | help="CSV output: index,serial,uuid,cause,address,raw ts,iso ts" | | | args = ap.parse args | | | lines = | | | def out line="" : | | | lines.append line | | | print line | | | nvml = load nvml | | | check nvml, nvml.nvmlInit v2 , "nvmlInit v2" | | | try: | | | dbuf = ctypes.create string buffer 80 | | | driver = dbuf.value.decode errors="replace" | | | if nvml.nvmlSystemGetDriverVersion dbuf, ctypes.c uint 80 == NVML SUCCESS | | | else "n/a" | | | if args.csv: | | | out "index,serial,uuid,ecc mode,cause,address,raw timestamp,retired at iso" | | | else: | | | out f"Collected : {datetime.datetime.now .isoformat timespec='seconds' }" | | | out f"Driver version : {driver}" | | | if args.index is not None: | | | report gpu nvml, args.index, args.csv, out | | | else: | | | count = ctypes.c uint | | | check nvml, nvml.nvmlDeviceGetCount v2 ctypes.byref count , | | | "nvmlDeviceGetCount v2" | | | for i in range count.value : | | | report gpu nvml, i, args.csv, out | | | finally: | | | nvml.nvmlShutdown | | | if args.output: | | | with open args.output, "w" as fh: | | | fh.write "\n".join lines + "\n" | | | print f"\nWritten to {args.output}", file=sys.stderr | | | if name == " main ": | | | main |