Custom Zig Test Runner, better ouput, timing display, and support for special "tests:beforeAll" and "tests:afterAll" tests

This article describes a custom test runner for Zig 0.16 that provides improved output formatting, timing display, and support for special "tests:beforeAll" and "tests:afterAll" setup/teardown functions. The runner tracks test pass/fail/skip/leak counts, displays the five slowest tests, and includes a custom panic handler that tracks the current test name. The code also supports filtering tests by name and handles memory leak detection through Zig's testing allocator.

- - Save karlseguin/c6bea5b35e4e8d26af6f81c22cb5d76b to your computer and use it in GitHub Desktop. Learn more about bidirectional Unicode characters https://github.co/hiddenchars | // This is for the Zig 0.16. | | | // See https://gist.github.com/karlseguin/c6bea5b35e4e8d26af6f81c22cb5d76b/eb15512d6ae49663fa9df6c7a9725b20dab43edd | | | // for a version that workson Zig 0.15.2. | | | // See https://gist.github.com/karlseguin/c6bea5b35e4e8d26af6f81c22cb5d76b/1f317ebc9cd09bc50fd5591d09c34255e15d1d85 | | | // for a version that workson Zig 0.14.1. | | | // in your build.zig, you can specify a custom test runner: | | | // const tests = b.addTest .{ | | | // .root module = $MODULE BEING TESTED, | | | // .test runner = .{ .path = b.path "test runner.zig" , .mode = .simple }, | | | // } ; | | | // in your build.zig, you can specify a custom test runner: | | | // const tests = b.addTest .{ | | | // .root module = $MODULE BEING TESTED, | | | // .test runner = .{ .path = b.path "test runner.zig" , .mode = .simple }, | | | // } ; | | | const std = @import "std" ; | | | const Io = std.Io; | | | const builtin = @import "builtin" ; | | | const Allocator = std.mem.Allocator; | | | const BORDER = "=" 80; | | | // use in custom panic handler | | | var current test: ? const u8 = null; | | | pub fn main init: std.process.Init void { | | | var mem: 8192 u8 = undefined; | | | var fba = std.heap.FixedBufferAllocator.init &mem ; | | | const allocator = fba.allocator ; | | | const env = Env.init init.environ map ; | | | std.testing.io instance = .init init.gpa, .{ | | | .argv0 = .init init.minimal.args , | | | .environ = init.minimal.environ, | | | } ; | | | defer std.testing.io instance.deinit ; | | | const io = std.testing.io; | | | var slowest = SlowTracker.init allocator, io, 5 ; | | | defer slowest.deinit ; | | | var pass: usize = 0; | | | var fail: usize = 0; | | | var skip: usize = 0; | | | var leak: usize = 0; | | | Printer.fmt "\r\x1b 0K", .{} ; // beginning of line and clear to end of line | | | for builtin.test functions |t| { | | | if isSetup t { | | | t.func catch |err| { | | | Printer.status .fail, "\nsetup \"{s}\" failed: {}\n", .{ t.name, err } ; | | | return err; | | | }; | | | } | | | } | | | for builtin.test functions |t| { | | | if isSetup t or isTeardown t { | | | continue; | | | } | | | var status = Status.pass; | | | slowest.startTiming io ; | | | const is unnamed test = isUnnamed t ; | | | if env.filter |f| { | | | if is unnamed test and std.mem.indexOf u8, t.name, f == null { | | | continue; | | | } | | | } | | | const friendly name = blk: { | | | const name = t.name; | | | var it = std.mem.splitScalar u8, name, '.' ; | | | while it.next |value| { | | | if std.mem.eql u8, value, "test" { | | | const rest = it.rest ; | | | break :blk if rest.len 0 rest else name; | | | } | | | } | | | break :blk name; | | | }; | | | current test = friendly name; | | | std.testing.allocator instance = .{}; | | | const result = t.func ; | | | current test = null; | | | const ns taken = slowest.endTiming io, friendly name ; | | | if std.testing.allocator instance.deinit == .leak { | | | leak += 1; | | | Printer.status .fail, "\n{s}\n\"{s}\" - Memory Leak\n{s}\n", .{ BORDER, friendly name, BORDER } ; | | | } | | | if result | | { | | | pass += 1; | | | } else |err| switch err { | | | error.SkipZigTest = { | | | skip += 1; | | | status = .skip; | | | }, | | | else = { | | | status = .fail; | | | fail += 1; | | | Printer.status .fail, "\n{s}\n\"{s}\" - {s}\n{s}\n", .{ BORDER, friendly name, @errorName err , BORDER } ; | | | if @errorReturnTrace |trace| { | | | std.debug.dumpErrorReturnTrace trace ; | | | } | | | if env.fail first { | | | break; | | | } | | | }, | | | } | | | if env.verbose { | | | const ms = @as f64, @floatFromInt ns taken / 1 000 000.0; | | | Printer.status status, "{s} {d:.2}ms \n", .{ friendly name, ms } ; | | | } else { | | | Printer.status status, ".", .{} ; | | | } | | | } | | | for builtin.test functions |t| { | | | if isTeardown t { | | | t.func catch |err| { | | | Printer.status .fail, "\nteardown \"{s}\" failed: {}\n", .{ t.name, err } ; | | | return err; | | | }; | | | } | | | } | | | const total tests = pass + fail; | | | const status = if fail == 0 Status.pass else Status.fail; | | | Printer.status status, "\n{d} of {d} test{s} passed\n", .{ pass, total tests, if total tests = 1 "s" else "" } ; | | | if skip 0 { | | | Printer.status .skip, "{d} test{s} skipped\n", .{ skip, if skip = 1 "s" else "" } ; | | | } | | | if leak 0 { | | | Printer.status .fail, "{d} test{s} leaked\n", .{ leak, if leak = 1 "s" else "" } ; | | | } | | | Printer.fmt "\n", .{} ; | | | try slowest.display ; | | | Printer.fmt "\n", .{} ; | | | std.process.exit if fail == 0 0 else 1 ; | | | } | | | const Printer = struct { | | | fn fmt comptime format: const u8, args: anytype void { | | | std.debug.print format, args ; | | | } | | | fn status s: Status, comptime format: const u8, args: anytype void { | | | switch s { | | | .pass = std.debug.print "\x1b 32m", .{} , | | | .fail = std.debug.print "\x1b 31m", .{} , | | | .skip = std.debug.print "\x1b 33m", .{} , | | | else = {}, | | | } | | | std.debug.print format ++ "\x1b 0m", args ; | | | } | | | }; | | | const Status = enum { | | | pass, | | | fail, | | | skip, | | | text, | | | }; | | | const SlowTracker = struct { | | | max: usize, | | | slowest: SlowestQueue, | | | start: Io.Timestamp, | | | allocator: Allocator, | | | const SlowestQueue = std.PriorityDequeue TestInfo, void, compareTiming ; | | | fn init allocator: Allocator, io: Io, count: u32 SlowTracker { | | | const timestamp = Io.Clock.awake.now io ; | | | var slowest: SlowestQueue = .empty; | | | slowest.ensureTotalCapacity allocator, count catch @panic "OOM" ; | | | return .{ | | | .max = count, | | | .start = timestamp, | | | .slowest = slowest, | | | .allocator = allocator, | | | }; | | | } | | | const TestInfo = struct { | | | ns: u64, | | | name: const u8, | | | }; | | | fn deinit self: SlowTracker void { | | | self.slowest.deinit self.allocator ; | | | } | | | fn startTiming self: SlowTracker, io: Io void { | | | self.start = Io.Clock.awake.now io ; | | | } | | | fn endTiming self: SlowTracker, io: Io, test name: const u8 u64 { | | | const timestamp = Io.Clock.awake.now io ; | | | const start = self.start; | | | self.start = timestamp; | | | const ns: u64 = @intCast start.durationTo timestamp .toNanoseconds ; | | | var slowest = &self.slowest; | | | if slowest.count < self.max { | | | // Capacity is fixed to the of slow tests we want to track | | | // If we've tracked fewer tests than this capacity, than always add | | | slowest.push self.allocator, TestInfo{ .ns = ns, .name = test name } catch @panic "failed to track test timing" ; | | | return ns; | | | } | | | { | | | // Optimization to avoid shifting the dequeue for the common case | | | // where the test isn't one of our slowest. | | | const fastest of the slow = slowest.peekMin orelse unreachable; | | | if fastest of the slow.ns ns { | | | // the test was faster than our fastest slow test, don't add | | | return ns; | | | } | | | } | | | // the previous fastest of our slow tests, has been pushed off. | | | = slowest.popMin ; | | | slowest.push self.allocator, TestInfo{ .ns = ns, .name = test name } catch @panic "failed to track test timing" ; | | | return ns; | | | } | | | fn display self: SlowTracker void { | | | var slowest = self.slowest; | | | const count = slowest.count ; | | | Printer.fmt "Slowest {d} test{s}: \n", .{ count, if count = 1 "s" else "" } ; | | | while slowest.popMin |info| { | | | const ms = @as f64, @floatFromInt info.ns / 1 000 000.0; | | | Printer.fmt " {d:.2}ms\t{s}\n", .{ ms, info.name } ; | | | } | | | } | | | fn compareTiming context: void, a: TestInfo, b: TestInfo std.math.Order { | | | = context; | | | return std.math.order a.ns, b.ns ; | | | } | | | }; | | | const Env = struct { | | | verbose: bool, | | | fail first: bool, | | | filter: ? const u8, | | | fn init map: const std.process.Environ.Map Env { | | | return .{ | | | .verbose = readEnvBool map, "TEST VERBOSE", true , | | | .fail first = readEnvBool map, "TEST FAIL FIRST", false , | | | .filter = readEnv map, "TEST FILTER" , | | | }; | | | } | | | fn readEnv map: const std.process.Environ.Map, key: const u8 ? const u8 { | | | return map.get key ; | | | } | | | fn readEnvBool map: const std.process.Environ.Map, key: const u8, deflt: bool bool { | | | const value = readEnv map, key orelse return deflt; | | | return std.ascii.eqlIgnoreCase value, "true" ; | | | } | | | }; | | | pub const panic = std.debug.FullPanic struct { | | | pub fn panicFn msg: const u8, first trace addr: ?usize noreturn { | | | if current test |ct| { | | | std.debug.print "\x1b 31m{s}\npanic running \"{s}\"\n{s}\x1b 0m\n", .{ BORDER, ct, BORDER } ; | | | } | | | std.debug.defaultPanic msg, first trace addr ; | | | } | | | }.panicFn ; | | | fn isUnnamed t: std.builtin.TestFn bool { | | | const marker = ".test "; | | | const test name = t.name; | | | const index = std.mem.indexOf u8, test name, marker orelse return false; | | | = std.fmt.parseInt u32, test name index + marker.len .. , 10 catch return false; | | | return true; | | | } | | | fn isSetup t: std.builtin.TestFn bool { | | | return std.mem.endsWith u8, t.name, "tests:beforeAll" ; | | | } | | | fn isTeardown t: std.builtin.TestFn bool { | | | return std.mem.endsWith u8, t.name, "tests:afterAll" ; | | | } | 💥 Very interesting, thanks for sharing The continue for the is unnamed test case, will discard information about such tests, no? pass , fail or leak will not be incremented, since the following block is before them all: if is unnamed test { continue; } Shouldn't a leak be considered has a failure? @oliverpool https://github.com/oliverpool I think you're right that it's best to leave that out and just report unnamed tests, with their weird names, like any other. It's been updated, cheers. Hey, very interesting, thanks for sharing For anyone reading in the future, I noticed two things in zig 0.13.0 at least: - To add the test runner you should use b.path "test runner.zig" std.debug.defaultPanic has been deprecated I could not find any mention to it anywhere 😅 , it should be replaced with std.debug.panicImpl error return trace, ret addr, msg ; Yes, having a 0.13 version is nice. For the sake of correctness, it's the opposite though: std.debug.defaultPanic hasn't been deprecated it's new. It was added about a month ago https://github.com/ziglang/zig/commit/4f8d244e7ea47a8cdb41496d51961ef4ba3ec2af diff-c439163aff3333f7964979ad02f98a36c80eadbffa7c1704af2e6945ce326b98R454 Thanks, this is exactly what I was looking for One small change I needed to make for 0.14.0-dev i haven't tested this on 0.13.0 yet . When adding the .test runner in your build.zig , you need to use a Build.LazyPath , e.g: js const lib unit tests = b.addTest .{ .root source file = b.path "src/root.zig" , .target = target, .optimize = optimize, .test runner = b.path "test runner.zig" , // use Build.LazyPath instead of string literal } ; Yup, you're right @xiy https://github.com/xiy . I've updated it. Thanks. What's the license on this? @SimonMeskens https://github.com/SimonMeskens anything you want it to be. But let's say MIT if you work for a place that needs something well-defined. That's amazing, thanks It also helps clarify this downstream project: https://gist.github.com/nurpax/4afcb6e4ef3f03f0d282f7c462005f12 https://gist.github.com/nurpax/4afcb6e4ef3f03f0d282f7c462005f12 @karlseguin https://github.com/karlseguin Wow That looks great Could you make library out of this for usage within zig zon ? Single file, seems simpler to copy and paste and adjust as needed. Zig 0.14.1 doesn't seem to have the test functions field in builtin module anymore, do you have a work aruond //... for builtin.test functions |t| { //...... @Borwe https://github.com/Borwe It still does. Are you sure you're on zig 0.14.1? You can see the built-in runner is using it using the 0.14.1 tag https://github.com/ziglang/zig/blob/7c709f920bbe8f01a25392182e4a3fd02bb95219/lib/compiler/test runner.zig L99 https://github.com/ziglang/zig/blob/7c709f920bbe8f01a25392182e4a3fd02bb95219/lib/compiler/test runner.zig L99 @Borwe It still does. Are you sure you're on zig 0.14.1?You can see the built-in runner is using it using the 0.14.1 tag https://github.com/ziglang/zig/blob/7c709f920bbe8f01a25392182e4a3fd02bb95219/lib/compiler/test runner.zig L99 Sorry, you are right, I was importing the wrong builtin module. The writergate also affects this test runner. EDIT 2025/10/04: the version above now works with zig 0.15 - you can ignore the version linked in this comment Here is a version working with zig 0.15.1: https://gist.github.com/jonathanderque/c8dbeafc68c1d45e53f629d3c78331a1 https://gist.github.com/jonathanderque/c8dbeafc68c1d45e53f629d3c78331a1 I'd be curious to see if there is a better patch: I had to modify more than I initially intended. Overall I feel like I'm still struggling with the Writer API changes. Thanks for the wonderful test runner I personally added this line to mine to avoid trying to run on files with no tests so that it doesn't clog the output. pub fn main void { if builtin.test functions.len < 1 return; // ... @karlseguin https://github.com/karlseguin I think the runner is supposed to initialize the environ of testing, so this is missing in main ? std.testing.environ = init.minimal.environ; Otherwise there is no way to access envvars from your tests in 0.16, that is .