// Source: https://github.com/ziglang/zig/blob/0.14.0/lib/compiler/test_runner.zig //! Default test runner for unit tests. const builtin = @import("builtin"); const std = @import("std"); const io = std.io; const testing = std.testing; const assert = std.debug.assert; pub const std_options: std.Options = .{ .logFn = log, }; var log_err_count: usize = 0; var fba_buffer: [8192]u8 = undefined; var fba = std.heap.FixedBufferAllocator.init(&fba_buffer); const crippled = switch(builtin.zig_backend) { .stage2_powerpc, .stage2_riscv64, => true, else => false, }; pub fn main() void { @disableInstrumentation(); if(builtin.cpu.arch.isSpirV()) { // SPIR-V needs an special test-runner return; } if(crippled) { return mainSimple() catch @panic("test failure\n"); } const args = std.process.argsAlloc(fba.allocator()) catch @panic("unable to parse command line args"); var listen = false; var opt_cache_dir: ?[]const u8 = null; for(args[1..]) |arg| { if(std.mem.eql(u8, arg, "--listen=-")) { listen = true; } else if(std.mem.startsWith(u8, arg, "--seed=")) { testing.random_seed = std.fmt.parseUnsigned(u32, arg["--seed=".len..], 0) catch @panic("unable to parse --seed command line argument"); } else if(std.mem.startsWith(u8, arg, "--cache-dir")) { opt_cache_dir = arg["--cache-dir=".len..]; } else { @panic("unrecognized command line argument"); } } fba.reset(); if(builtin.fuzz) { const cache_dir = opt_cache_dir orelse @panic("missing --cache-dir=[path] argument"); fuzzer_init(FuzzerSlice.fromSlice(cache_dir)); } if(listen) { return mainServer() catch @panic("internal test runner failure"); } else { return mainTerminal(); } } fn mainServer() !void { @disableInstrumentation(); var server = try std.zig.Server.init(.{ .gpa = fba.allocator(), .in = .stdin(), .out = .stdout(), .zig_version = builtin.zig_version_string, }); defer server.deinit(); if(builtin.fuzz) { const coverage_id = fuzzer_coverage_id(); try server.serveU64Message(.coverage_id, coverage_id); } while(true) { const hdr = try server.receiveMessage(); switch(hdr.tag) { .exit => { return std.process.exit(0); }, .query_test_metadata => { testing.allocator_instance = .{}; defer if(testing.allocator_instance.deinit() == .leak) { @panic("internal test runner memory leak"); }; var string_bytes: std.ArrayListUnmanaged(u8) = .empty; defer string_bytes.deinit(testing.allocator); try string_bytes.append(testing.allocator, 0); // Reserve 0 for null. const test_fns = builtin.test_functions; const names = try testing.allocator.alloc(u32, test_fns.len); defer testing.allocator.free(names); const expected_panic_msgs = try testing.allocator.alloc(u32, test_fns.len); defer testing.allocator.free(expected_panic_msgs); for(test_fns, names, expected_panic_msgs) |test_fn, *name, *expected_panic_msg| { name.* = @as(u32, @intCast(string_bytes.items.len)); try string_bytes.ensureUnusedCapacity(testing.allocator, test_fn.name.len + 1); string_bytes.appendSliceAssumeCapacity(test_fn.name); string_bytes.appendAssumeCapacity(0); expected_panic_msg.* = 0; } try server.serveTestMetadata(.{ .names = names, .expected_panic_msgs = expected_panic_msgs, .string_bytes = string_bytes.items, }); }, .run_test => { testing.allocator_instance = .{}; log_err_count = 0; const index = try server.receiveBody_u32(); const test_fn = builtin.test_functions[index]; var fail = false; var skip = false; is_fuzz_test = false; test_fn.func() catch |err| switch(err) { error.SkipZigTest => skip = true, else => { fail = true; if(@errorReturnTrace()) |trace| { std.debug.dumpStackTrace(trace.*); } }, }; const leak = testing.allocator_instance.deinit() == .leak; try server.serveTestResults(.{ .index = index, .flags = .{ .fail = fail, .skip = skip, .leak = leak, .fuzz = is_fuzz_test, .log_err_count = std.math.lossyCast( @FieldType(std.zig.Server.Message.TestResults.Flags, "log_err_count"), log_err_count, ), }, }); }, .start_fuzzing => { if(!builtin.fuzz) unreachable; const index = try server.receiveBody_u32(); const test_fn = builtin.test_functions[index]; const entry_addr = @intFromPtr(test_fn.func); try server.serveU64Message(.fuzz_start_addr, entry_addr); defer if(testing.allocator_instance.deinit() == .leak) std.process.exit(1); is_fuzz_test = false; fuzzer_set_name(test_fn.name.ptr, test_fn.name.len); test_fn.func() catch |err| switch(err) { error.SkipZigTest => return, else => { if(@errorReturnTrace()) |trace| { std.debug.dumpStackTrace(trace.*); } std.debug.print("failed with error.{s}\n", .{@errorName(err)}); std.process.exit(1); }, }; if(!is_fuzz_test) @panic("missed call to std.testing.fuzz"); }, else => { std.debug.print("unsupported message: {x}\n", .{@intFromEnum(hdr.tag)}); std.process.exit(1); }, } } } fn mainTerminal() void { @disableInstrumentation(); const test_fn_list = builtin.test_functions; var ok_count: usize = 0; var skip_count: usize = 0; var fail_count: usize = 0; var fuzz_count: usize = 0; const root_node = if(builtin.fuzz) std.Progress.Node.none else std.Progress.start(.{ .root_name = "Test", .estimated_total_items = test_fn_list.len, }); const doColors = std.fs.File.stderr().supportsAnsiEscapeCodes(); const reset = if(doColors) "\x1b[0m" else ""; const red = if(doColors) "\x1b[31m" else ""; const yellow = if(doColors) "\x1b[33m" else ""; const green = if(doColors) "\x1b[32m" else ""; var async_frame_buffer: []align(builtin.target.stackAlignment()) u8 = undefined; // TODO this is on the next line (using `undefined` above) because otherwise zig incorrectly // ignores the alignment of the slice. async_frame_buffer = &[_]u8{}; var leaks: usize = 0; for(test_fn_list, 0..) |test_fn, i| { testing.allocator_instance = .{}; defer { if(testing.allocator_instance.deinit() == .leak) { leaks += 1; } } testing.log_level = .warn; const test_node = root_node.start(test_fn.name, 0); std.debug.print("{d: >4}/{d: <4} {s:.<65}", .{i + 1, test_fn_list.len, test_fn.name}); is_fuzz_test = false; if(test_fn.func()) |_| { ok_count += 1; test_node.end(); std.debug.print("{s}OK{s}\n", .{green, reset}); } else |err| switch(err) { error.SkipZigTest => { skip_count += 1; std.debug.print("{s}SKIP{s}\n", .{yellow, reset}); test_node.end(); }, else => { fail_count += 1; std.debug.print("{s}FAIL{s}\n{s}:\n", .{red, reset, @errorName(err)}); if(@errorReturnTrace()) |trace| { std.debug.dumpStackTrace(trace.*); } test_node.end(); }, } fuzz_count += @intFromBool(is_fuzz_test); } root_node.end(); if(ok_count == test_fn_list.len) { std.debug.print("All {d} tests passed.\n", .{ok_count}); } else { std.debug.print("{d} passed; {d} skipped; {d} failed.\n", .{ok_count, skip_count, fail_count}); } if(log_err_count != 0) { std.debug.print("{d} errors were logged.\n", .{log_err_count}); } if(leaks != 0) { std.debug.print("{d} tests leaked memory.\n", .{leaks}); } if(fuzz_count != 0) { std.debug.print("{d} fuzz tests found.\n", .{fuzz_count}); } if(leaks != 0 or fail_count != 0) { std.process.exit(1); } } pub fn log( comptime message_level: std.log.Level, comptime scope: @Type(.enum_literal), comptime format: []const u8, args: anytype, ) void { @disableInstrumentation(); if(@intFromEnum(message_level) <= @intFromEnum(std.log.Level.err)) { log_err_count +|= 1; } if(@intFromEnum(message_level) <= @intFromEnum(testing.log_level)) { std.debug.print( "[" ++ @tagName(scope) ++ "] (" ++ @tagName(message_level) ++ "): " ++ format ++ "\n", args, ); } } /// Simpler main(), exercising fewer language features, so that /// work-in-progress backends can handle it. pub fn mainSimple() anyerror!void { @disableInstrumentation(); // is the backend capable of printing to stderr? const enable_print = switch(builtin.zig_backend) { else => false, }; // is the backend capable of using std.fmt.format to print a summary at the end? const print_summary = switch(builtin.zig_backend) { .stage2_riscv64 => true, else => false, }; var passed: u64 = 0; var skipped: u64 = 0; var failed: u64 = 0; // we don't want to bring in File and Writer if the backend doesn't support it const stderr = if(comptime enable_print) std.fs.File.stderr() else {}; for(builtin.test_functions) |test_fn| { if(test_fn.func()) |_| { if(enable_print) { stderr.writeAll(test_fn.name) catch {}; stderr.writeAll("... ") catch {}; stderr.writeAll("PASS\n") catch {}; } } else |err| { if(enable_print) { stderr.writeAll(test_fn.name) catch {}; stderr.writeAll("... ") catch {}; } if(err != error.SkipZigTest) { if(enable_print) stderr.writeAll("FAIL\n") catch {}; failed += 1; if(!enable_print) return err; continue; } if(enable_print) stderr.writeAll("SKIP\n") catch {}; skipped += 1; continue; } passed += 1; } if(enable_print and print_summary) { stderr.deprecatedWriter().print("{} passed, {} skipped, {} failed\n", .{passed, skipped, failed}) catch {}; } if(failed != 0) std.process.exit(1); } const FuzzerSlice = extern struct { ptr: [*]const u8, len: usize, /// Inline to avoid fuzzer instrumentation. inline fn toSlice(s: FuzzerSlice) []const u8 { return s.ptr[0..s.len]; } /// Inline to avoid fuzzer instrumentation. inline fn fromSlice(s: []const u8) FuzzerSlice { return .{.ptr = s.ptr, .len = s.len}; } }; var is_fuzz_test: bool = undefined; extern fn fuzzer_set_name(name_ptr: [*]const u8, name_len: usize) void; extern fn fuzzer_init(cache_dir: FuzzerSlice) void; extern fn fuzzer_init_corpus_elem(input_ptr: [*]const u8, input_len: usize) void; extern fn fuzzer_start(testOne: *const fn([*]const u8, usize) callconv(.c) void) void; extern fn fuzzer_coverage_id() u64; pub fn fuzz( context: anytype, comptime testOne: fn(context: @TypeOf(context), []const u8) anyerror!void, options: testing.FuzzInputOptions, ) anyerror!void { // Prevent this function from confusing the fuzzer by omitting its own code // coverage from being considered. @disableInstrumentation(); // Some compiler backends are not capable of handling fuzz testing yet but // we still want CI test coverage enabled. if(crippled) return; // Smoke test to ensure the test did not use conditional compilation to // contradict itself by making it not actually be a fuzz test when the test // is built in fuzz mode. is_fuzz_test = true; // Ensure no test failure occurred before starting fuzzing. if(log_err_count != 0) @panic("error logs detected"); // libfuzzer is in a separate compilation unit so that its own code can be // excluded from code coverage instrumentation. It needs a function pointer // it can call for checking exactly one input. Inside this function we do // our standard unit test checks such as memory leaks, and interaction with // error logs. const global = struct { var ctx: @TypeOf(context) = undefined; fn fuzzer_one(input_ptr: [*]const u8, input_len: usize) callconv(.c) void { @disableInstrumentation(); testing.allocator_instance = .{}; defer if(testing.allocator_instance.deinit() == .leak) std.process.exit(1); log_err_count = 0; testOne(ctx, input_ptr[0..input_len]) catch |err| switch(err) { error.SkipZigTest => return, else => { std.debug.lockStdErr(); if(@errorReturnTrace()) |trace| std.debug.dumpStackTrace(trace.*); std.debug.print("failed with error.{s}\n", .{@errorName(err)}); std.process.exit(1); }, }; if(log_err_count != 0) { std.debug.lockStdErr(); std.debug.print("error logs detected\n", .{}); std.process.exit(1); } } }; if(builtin.fuzz) { const prev_allocator_state = testing.allocator_instance; testing.allocator_instance = .{}; defer testing.allocator_instance = prev_allocator_state; for(options.corpus) |elem| fuzzer_init_corpus_elem(elem.ptr, elem.len); global.ctx = context; fuzzer_start(&global.fuzzer_one); return; } // When the unit test executable is not built in fuzz mode, only run the // provided corpus. for(options.corpus) |input| { try testOne(context, input); } // In case there is no provided corpus, also use an empty // string as a smoke test. try testOne(context, ""); }