| #!/usr/bin/env python3 | | | """ | | | gpu_retired_pages.py -- dump retired GPU memory pages WITH retirement timestamps. | | | nvidia-smi reports retired page counts and addresses but not when each page | | | was retired. NVML's nvmlDeviceGetRetiredPages_v2 does return timestamps. This | | | script calls it via ctypes against libnvidia-ml.so.1, so it needs no Python | | | packages -- only the installed NVIDIA driver. | | | Retired pages live in the GPU's InfoROM and persist for the life of the board, | | | so timestamps predating a purchase date show the memory was already degrading | | | before you owned it. That makes this useful for vetting used datacenter GPUs. | | | SCOPE -- this reads dynamic page retirement, a PRE-AMPERE mechanism: | | | Supported : Tesla K20 and higher, Quadro 5000 and higher, with framebuffer | | | ECC enabled. Covers K80, P40, P100, V100, T4 and similar. | | | Excluded : GeForce and GRID -- no page retirement at all. | | | Ampere+ : A100, H100, L40, Blackwell and later replaced page retirement | | | with ROW REMAPPING. This script reports "not supported" on them. | | | The equivalent calls are nvmlDeviceGetRemappedRows and | | | nvmlDeviceGetRowRemapperHistogram (banks, not pages). | | | READ-ONLY -- this script only issues NVML query calls. It never changes ECC | | | mode, resets counters, resets the GPU, or touches the InfoROM. The sole thing | | | it writes is the optional -o output file on local disk. Safe to run against | | | production GPUs and against hardware you are evaluating but do not own. | | | CRITICAL CAVEAT -- pages are only retired while ECC is ENABLED. A card running | | | with ECC off silently accumulates memory errors and retires nothing. An empty | | | table on such a card means "nothing was being watched", NOT "healthy memory". | | | The script reports ECC mode per GPU and warns whenever it is off. | | | Usage: | | | python3 gpu_retired_pages.py # all GPUs | | | python3 gpu_retired_pages.py -i 0 # one GPU by index | | | python3 gpu_retired_pages.py -i 0 -o out.txt # also write to file | | | python3 gpu_retired_pages.py --csv # machine-readable | | | """ | | | import argparse | | | import ctypes | | | import datetime | | | import sys | | | NVML_SUCCESS = 0 | | | NVML_ERROR_NOT_SUPPORTED = 3 | | | NVML_ERROR_INSUFFICIENT_SIZE = 7 | | | CAUSE_SBE = 0 # multiple single-bit ECC errors on the same address | | | CAUSE_DBE = 1 # a single double-bit ECC error | |
| CAUSE_LABEL = { | |
| CAUSE_SBE: "SBE (2+ single-bit)", | |
| CAUSE_DBE: "DBE (double-bit)", | |
| } | | | # Sanity window for interpreting timestamps: 2000-01-01 .. 2050-01-01 | | | _EPOCH_MIN = 946_684_800 | | | _EPOCH_MAX = 2_524_608_000 | |
| def load_nvml(): | |
| for name in ("libnvidia-ml.so.1", "libnvidia-ml.so"): | |
| try: | | | return ctypes.CDLL(name) | | | except OSError: | | | continue | |
| sys.exit("Could not load libnvidia-ml.so.1 -- is the NVIDIA driver installed?") | |
| def err_str(nvml, rc): | |
| nvml.nvmlErrorString.restype = ctypes.c_char_p | | | try: | | | return nvml.nvmlErrorString(rc).decode() | | | except Exception: | |
| return f"NVML error {rc}" | |
| def check(nvml, rc, what): | |
| if rc != NVML_SUCCESS: | |
| sys.exit(f"{what} failed: {err_str(nvml, rc)}") | |
| def get_string(fn, handle, length=96): | |
| buf = ctypes.create_string_buffer(length) | |
| rc = fn(handle, buf, ctypes.c_uint(length)) | |
| if rc != NVML_SUCCESS: | |
| return "n/a" | |
| return buf.value.decode(errors="replace") | |
| def decode_timestamp(raw): | |
| """NVML documents these as Unix timestamps, but the unit has varied across | | | driver versions. Auto-detect the scale that lands in a plausible date range.""" | | | if raw == 0: | | | return None, "zero" | | | for divisor, unit in ((1, "s"), (1e3, "ms"), (1e6, "us"), (1e9, "ns")): | | | secs = raw / divisor | | | if _EPOCH_MIN < secs < _EPOCH_MAX: | | | return datetime.datetime.fromtimestamp(secs), unit | | | return None, "unrecognized" | |
| def get_retired_pages(nvml, handle, cause): | |
| """Returns (list of (address, raw_timestamp), status_string).""" | |
| fn = getattr(nvml, "nvmlDeviceGetRetiredPages_v2", None) | | | if fn is None: | | | return [], "nvmlDeviceGetRetiredPages_v2 unavailable in this driver" | | | capacity = 1024 # spec caps the InfoROM table well below this | | | while True: | |
| count = ctypes.c_uint(capacity) | |
| addrs = (ctypes.c_ulonglong * capacity)() | |
| stamps = (ctypes.c_ulonglong * capacity)() | |
| rc = fn(handle, ctypes.c_uint(cause), ctypes.byref(count), addrs, stamps) | |
| if rc == NVML_SUCCESS: | |
| return [(addrs[i], stamps[i]) for i in range(count.value)], "ok" | |
| if rc == NVML_ERROR_INSUFFICIENT_SIZE: | |
| capacity = max(count.value, capacity * 2) | |
| continue | |
| if rc == NVML_ERROR_NOT_SUPPORTED: | |
| return [], ("page retirement not supported -- Ampere and later use " | |
| "row remapping instead (see nvidia-smi -q -d ROW_REMAPPER)") | |
| return [], err_str(nvml, rc) | |
| def report_gpu(nvml, index, csv_mode, out): | |
| handle = ctypes.c_void_p() | |
| rc = nvml.nvmlDeviceGetHandleByIndex_v2(ctypes.c_uint(index), ctypes.byref(handle)) | |
| check(nvml, rc, f"nvmlDeviceGetHandleByIndex_v2({index})") | |
| name = get_string(nvml.nvmlDeviceGetName, handle, 96) | |
| serial = get_string(nvml.nvmlDeviceGetSerial, handle, 32) | | | uuid = get_string(nvml.nvmlDeviceGetUUID, handle, 96) | | | # Page retirement only happens while ECC is on. Without this, an empty | | | # table on an ECC-disabled card reads as "healthy" when it means "blind". | | | cur, pend = ctypes.c_uint(), ctypes.c_uint() | | | if nvml.nvmlDeviceGetEccMode(handle, ctypes.byref(cur), | | | ctypes.byref(pend)) == NVML_SUCCESS: | | | ecc = "enabled" if cur.value else "DISABLED" | |
| if cur.value != pend.value: | |
| ecc += f" (pending: {'enabled' if pend.value else 'disabled'} -- needs reboot)" | |
| else: | | | ecc = "unknown" | |
| rows = [] | |
| notes = [] | |
| for cause in (CAUSE_DBE, CAUSE_SBE): | |
| pages, status = get_retired_pages(nvml, handle, cause) | |
| if status != "ok": | |
| notes.append(f"{CAUSE_LABEL[cause]}: {status}") | |
| for addr, raw in pages: | |
| dt, unit = decode_timestamp(raw) | |
| rows.append((dt, unit, raw, addr, CAUSE_LABEL[cause])) | |
| # Sort oldest first; undated entries last. | | | rows.sort(key=lambda r: (r[0] is None, r[0] or datetime.datetime.min)) | | | if csv_mode: | | | ecc_field = ecc.replace(",", ";") | | | if not rows: | | | # Still emit one row so ECC state is visible for every GPU. | | | out(f"{index},{serial},{uuid},{ecc_field},,,,") | | | for dt, unit, raw, addr, cause in rows: | |
| out(f"{index},{serial},{uuid},{ecc_field},{cause},0x{addr:016x}," | |
| f"{raw},{dt.isoformat() if dt else ''}") | |
| return | |
| out("") | |
| out(f"GPU {index}: {name}") | |
| out(f" Serial : {serial}") | |
| out(f" UUID : {uuid}") | |
| out(f" ECC mode : {ecc}") | |
| out(f" Retired pages: {len(rows)}") | |
| for note in notes: | | | out(f" Note: {note}") | | | # Fires whether or not pages exist. A populated table on an ECC-disabled | | | # card is the more misleading case: it looks like a complete history, but | | | # records only the periods when ECC happened to be on. | |
| if ecc.startswith("DISABLED"): | |
| out("") | |
| out(" WARNING: ECC is disabled on this GPU.") | |
| out(" No pages can be retired and no new memory errors are") | | | out(" detected while ECC is off. Any history below covers") | | | out(" only periods when ECC was enabled, and absence of") | | | out(" entries is not evidence of healthy memory.") | | | if not rows: | | | out(" (no retired pages)") | | | return | |
| out("") | |
| out(f" {'Retired at':<21} {'Address':<20} {'Cause':<20} {'Raw timestamp'}") | |
| out(f" {'-' * 21} {'-' * 20} {'-' * 20} {'-' * 20}") | |
| for dt, unit, raw, addr, cause in rows: | |
| when = dt.strftime("%Y-%m-%d %H:%M:%S") if dt else f"unparsed ({unit})" | |
| out(f" {when:<21} 0x{addr:016x} {cause:<20} {raw}") | |
| dated = [r[0] for r in rows if r[0] is not None] | |
| if dated: | |
| out("") | |
| out(f" Earliest retirement: {min(dated)}") | |
| out(f" Latest retirement : {max(dated)}") | |
| def main(): | |
| ap = argparse.ArgumentParser(description=__doc__, | |
| formatter_class=argparse.RawDescriptionHelpFormatter) | |
| ap.add_argument("-i", "--index", type=int, default=None, | |
| help="GPU index (default: all GPUs)") | |
| ap.add_argument("-o", "--output", help="also write output to this file") | |
| ap.add_argument("--csv", action="store_true", | |
| help="CSV output: index,serial,uuid,cause,address,raw_ts,iso_ts") | |
| args = ap.parse_args() | |
| lines = [] | |
| def out(line=""): | |
| lines.append(line) | |
| print(line) | |
| nvml = load_nvml() | |
| check(nvml, nvml.nvmlInit_v2(), "nvmlInit_v2") | |
| try: | |
| dbuf = ctypes.create_string_buffer(80) | |
| driver = (dbuf.value.decode(errors="replace") | |
| if nvml.nvmlSystemGetDriverVersion(dbuf, ctypes.c_uint(80)) == NVML_SUCCESS | |
| else "n/a") | | | if args.csv: | | | out("index,serial,uuid,ecc_mode,cause,address,raw_timestamp,retired_at_iso") | | | else: | |
| out(f"Collected : {datetime.datetime.now().isoformat(timespec='seconds')}") | |
| out(f"Driver version : {driver}") | |
| if args.index is not None: | | | report_gpu(nvml, args.index, args.csv, out) | | | else: | |
| count = ctypes.c_uint() | |
| check(nvml, nvml.nvmlDeviceGetCount_v2(ctypes.byref(count)), | |
| "nvmlDeviceGetCount_v2") | | | for i in range(count.value): | | | report_gpu(nvml, i, args.csv, out) | | | finally: | | | nvml.nvmlShutdown() | | | if args.output: | |
| with open(args.output, "w") as fh: | |
| fh.write("\n".join(lines) + "\n") | |
| print(f"\nWritten to {args.output}", file=sys.stderr) | |
| if __name__ == "__main__": | |
| main() |