# gpu_retired_pages.py -- dump retired NVIDIA GPU memory pages WITH retirement timestamps

> Source: <https://gist.github.com/samteezy/a788dcf430deb448ae48bbe17c369241>
> Published: 2026-08-21 12:22:52+00:00

| #!/usr/bin/env python3 | |
| """ | |
| gpu_retired_pages.py -- dump retired GPU memory pages WITH retirement timestamps. | |
| nvidia-smi reports retired page *counts* and *addresses* but not *when* each page | |
| was retired. NVML's nvmlDeviceGetRetiredPages_v2 does return timestamps. This | |
| script calls it via ctypes against libnvidia-ml.so.1, so it needs no Python | |
| packages -- only the installed NVIDIA driver. | |
| Retired pages live in the GPU's InfoROM and persist for the life of the board, | |
| so timestamps predating a purchase date show the memory was already degrading | |
| before you owned it. That makes this useful for vetting used datacenter GPUs. | |
| SCOPE -- this reads dynamic page retirement, a PRE-AMPERE mechanism: | |
| Supported : Tesla K20 and higher, Quadro 5000 and higher, with framebuffer | |
| ECC enabled. Covers K80, P40, P100, V100, T4 and similar. | |
| Excluded : GeForce and GRID -- no page retirement at all. | |
| Ampere+ : A100, H100, L40, Blackwell and later replaced page retirement | |
| with ROW REMAPPING. This script reports "not supported" on them. | |
| The equivalent calls are nvmlDeviceGetRemappedRows and | |
| nvmlDeviceGetRowRemapperHistogram (banks, not pages). | |
| READ-ONLY -- this script only issues NVML query calls. It never changes ECC | |
| mode, resets counters, resets the GPU, or touches the InfoROM. The sole thing | |
| it writes is the optional -o output file on local disk. Safe to run against | |
| production GPUs and against hardware you are evaluating but do not own. | |
| CRITICAL CAVEAT -- pages are only retired while ECC is ENABLED. A card running | |
| with ECC off silently accumulates memory errors and retires nothing. An empty | |
| table on such a card means "nothing was being watched", NOT "healthy memory". | |
| The script reports ECC mode per GPU and warns whenever it is off. | |
| Usage: | |
| python3 gpu_retired_pages.py # all GPUs | |
| python3 gpu_retired_pages.py -i 0 # one GPU by index | |
| python3 gpu_retired_pages.py -i 0 -o out.txt # also write to file | |
| python3 gpu_retired_pages.py --csv # machine-readable | |
| """ | |
| import argparse | |
| import ctypes | |
| import datetime | |
| import sys | |
| NVML_SUCCESS = 0 | |
| NVML_ERROR_NOT_SUPPORTED = 3 | |
| NVML_ERROR_INSUFFICIENT_SIZE = 7 | |
| CAUSE_SBE = 0 # multiple single-bit ECC errors on the same address | |
| CAUSE_DBE = 1 # a single double-bit ECC error | |
| CAUSE_LABEL = { | |
| CAUSE_SBE: "SBE (2+ single-bit)", | |
| CAUSE_DBE: "DBE (double-bit)", | |
| } | |
| # Sanity window for interpreting timestamps: 2000-01-01 .. 2050-01-01 | |
| _EPOCH_MIN = 946_684_800 | |
| _EPOCH_MAX = 2_524_608_000 | |
| def load_nvml(): | |
| for name in ("libnvidia-ml.so.1", "libnvidia-ml.so"): | |
| try: | |
| return ctypes.CDLL(name) | |
| except OSError: | |
| continue | |
| sys.exit("Could not load libnvidia-ml.so.1 -- is the NVIDIA driver installed?") | |
| def err_str(nvml, rc): | |
| nvml.nvmlErrorString.restype = ctypes.c_char_p | |
| try: | |
| return nvml.nvmlErrorString(rc).decode() | |
| except Exception: | |
| return f"NVML error {rc}" | |
| def check(nvml, rc, what): | |
| if rc != NVML_SUCCESS: | |
| sys.exit(f"{what} failed: {err_str(nvml, rc)}") | |
| def get_string(fn, handle, length=96): | |
| buf = ctypes.create_string_buffer(length) | |
| rc = fn(handle, buf, ctypes.c_uint(length)) | |
| if rc != NVML_SUCCESS: | |
| return "n/a" | |
| return buf.value.decode(errors="replace") | |
| def decode_timestamp(raw): | |
| """NVML documents these as Unix timestamps, but the unit has varied across | |
| driver versions. Auto-detect the scale that lands in a plausible date range.""" | |
| if raw == 0: | |
| return None, "zero" | |
| for divisor, unit in ((1, "s"), (1e3, "ms"), (1e6, "us"), (1e9, "ns")): | |
| secs = raw / divisor | |
| if _EPOCH_MIN < secs < _EPOCH_MAX: | |
| return datetime.datetime.fromtimestamp(secs), unit | |
| return None, "unrecognized" | |
| def get_retired_pages(nvml, handle, cause): | |
| """Returns (list of (address, raw_timestamp), status_string).""" | |
| fn = getattr(nvml, "nvmlDeviceGetRetiredPages_v2", None) | |
| if fn is None: | |
| return [], "nvmlDeviceGetRetiredPages_v2 unavailable in this driver" | |
| capacity = 1024 # spec caps the InfoROM table well below this | |
| while True: | |
| count = ctypes.c_uint(capacity) | |
| addrs = (ctypes.c_ulonglong * capacity)() | |
| stamps = (ctypes.c_ulonglong * capacity)() | |
| rc = fn(handle, ctypes.c_uint(cause), ctypes.byref(count), addrs, stamps) | |
| if rc == NVML_SUCCESS: | |
| return [(addrs[i], stamps[i]) for i in range(count.value)], "ok" | |
| if rc == NVML_ERROR_INSUFFICIENT_SIZE: | |
| capacity = max(count.value, capacity * 2) | |
| continue | |
| if rc == NVML_ERROR_NOT_SUPPORTED: | |
| return [], ("page retirement not supported -- Ampere and later use " | |
| "row remapping instead (see nvidia-smi -q -d ROW_REMAPPER)") | |
| return [], err_str(nvml, rc) | |
| def report_gpu(nvml, index, csv_mode, out): | |
| handle = ctypes.c_void_p() | |
| rc = nvml.nvmlDeviceGetHandleByIndex_v2(ctypes.c_uint(index), ctypes.byref(handle)) | |
| check(nvml, rc, f"nvmlDeviceGetHandleByIndex_v2({index})") | |
| name = get_string(nvml.nvmlDeviceGetName, handle, 96) | |
| serial = get_string(nvml.nvmlDeviceGetSerial, handle, 32) | |
| uuid = get_string(nvml.nvmlDeviceGetUUID, handle, 96) | |
| # Page retirement only happens while ECC is on. Without this, an empty | |
| # table on an ECC-disabled card reads as "healthy" when it means "blind". | |
| cur, pend = ctypes.c_uint(), ctypes.c_uint() | |
| if nvml.nvmlDeviceGetEccMode(handle, ctypes.byref(cur), | |
| ctypes.byref(pend)) == NVML_SUCCESS: | |
| ecc = "enabled" if cur.value else "DISABLED" | |
| if cur.value != pend.value: | |
| ecc += f" (pending: {'enabled' if pend.value else 'disabled'} -- needs reboot)" | |
| else: | |
| ecc = "unknown" | |
| rows = [] | |
| notes = [] | |
| for cause in (CAUSE_DBE, CAUSE_SBE): | |
| pages, status = get_retired_pages(nvml, handle, cause) | |
| if status != "ok": | |
| notes.append(f"{CAUSE_LABEL[cause]}: {status}") | |
| for addr, raw in pages: | |
| dt, unit = decode_timestamp(raw) | |
| rows.append((dt, unit, raw, addr, CAUSE_LABEL[cause])) | |
| # Sort oldest first; undated entries last. | |
| rows.sort(key=lambda r: (r[0] is None, r[0] or datetime.datetime.min)) | |
| if csv_mode: | |
| ecc_field = ecc.replace(",", ";") | |
| if not rows: | |
| # Still emit one row so ECC state is visible for every GPU. | |
| out(f"{index},{serial},{uuid},{ecc_field},,,,") | |
| for dt, unit, raw, addr, cause in rows: | |
| out(f"{index},{serial},{uuid},{ecc_field},{cause},0x{addr:016x}," | |
| f"{raw},{dt.isoformat() if dt else ''}") | |
| return | |
| out("") | |
| out(f"GPU {index}: {name}") | |
| out(f" Serial : {serial}") | |
| out(f" UUID : {uuid}") | |
| out(f" ECC mode : {ecc}") | |
| out(f" Retired pages: {len(rows)}") | |
| for note in notes: | |
| out(f" Note: {note}") | |
| # Fires whether or not pages exist. A populated table on an ECC-disabled | |
| # card is the more misleading case: it looks like a complete history, but | |
| # records only the periods when ECC happened to be on. | |
| if ecc.startswith("DISABLED"): | |
| out("") | |
| out(" WARNING: ECC is disabled on this GPU.") | |
| out(" No pages can be retired and no new memory errors are") | |
| out(" detected while ECC is off. Any history below covers") | |
| out(" only periods when ECC was enabled, and absence of") | |
| out(" entries is not evidence of healthy memory.") | |
| if not rows: | |
| out(" (no retired pages)") | |
| return | |
| out("") | |
| out(f" {'Retired at':<21} {'Address':<20} {'Cause':<20} {'Raw timestamp'}") | |
| out(f" {'-' * 21} {'-' * 20} {'-' * 20} {'-' * 20}") | |
| for dt, unit, raw, addr, cause in rows: | |
| when = dt.strftime("%Y-%m-%d %H:%M:%S") if dt else f"unparsed ({unit})" | |
| out(f" {when:<21} 0x{addr:016x} {cause:<20} {raw}") | |
| dated = [r[0] for r in rows if r[0] is not None] | |
| if dated: | |
| out("") | |
| out(f" Earliest retirement: {min(dated)}") | |
| out(f" Latest retirement : {max(dated)}") | |
| def main(): | |
| ap = argparse.ArgumentParser(description=__doc__, | |
| formatter_class=argparse.RawDescriptionHelpFormatter) | |
| ap.add_argument("-i", "--index", type=int, default=None, | |
| help="GPU index (default: all GPUs)") | |
| ap.add_argument("-o", "--output", help="also write output to this file") | |
| ap.add_argument("--csv", action="store_true", | |
| help="CSV output: index,serial,uuid,cause,address,raw_ts,iso_ts") | |
| args = ap.parse_args() | |
| lines = [] | |
| def out(line=""): | |
| lines.append(line) | |
| print(line) | |
| nvml = load_nvml() | |
| check(nvml, nvml.nvmlInit_v2(), "nvmlInit_v2") | |
| try: | |
| dbuf = ctypes.create_string_buffer(80) | |
| driver = (dbuf.value.decode(errors="replace") | |
| if nvml.nvmlSystemGetDriverVersion(dbuf, ctypes.c_uint(80)) == NVML_SUCCESS | |
| else "n/a") | |
| if args.csv: | |
| out("index,serial,uuid,ecc_mode,cause,address,raw_timestamp,retired_at_iso") | |
| else: | |
| out(f"Collected : {datetime.datetime.now().isoformat(timespec='seconds')}") | |
| out(f"Driver version : {driver}") | |
| if args.index is not None: | |
| report_gpu(nvml, args.index, args.csv, out) | |
| else: | |
| count = ctypes.c_uint() | |
| check(nvml, nvml.nvmlDeviceGetCount_v2(ctypes.byref(count)), | |
| "nvmlDeviceGetCount_v2") | |
| for i in range(count.value): | |
| report_gpu(nvml, i, args.csv, out) | |
| finally: | |
| nvml.nvmlShutdown() | |
| if args.output: | |
| with open(args.output, "w") as fh: | |
| fh.write("\n".join(lines) + "\n") | |
| print(f"\nWritten to {args.output}", file=sys.stderr) | |
| if __name__ == "__main__": | |
| main() |