Commit Diff

Commit:
14a9269e7dd5371e152b9d2f65f9508945f73060
From:
Sergey Bronnikov <sergeyb@tarantool.org>
Date:
Message:
test/fuzz: collect and print Lua metrics Fuzzing test for LuaJIT generates random Lua programs and executes them. We want to build a fuzzing test that will produce Lua programs that will not contain semantic errors and will trigger as much as possible components in LuaJIT. This proposed patch introduces metrics that gathered after running the test. LuaJIT metrics gathered using LuaJIT getmetrics module [1]. All gathered metrics test will output after running with a finite number of runs or finite duration of time (options `-runs` and `-max_total_time`) or after sending SIGUSR1 to a test process. ``` $ ./build/test/fuzz/luaL_loadbuffer/luaL_loadbuffer_fuzzer -runs=1000 <snipped> Done 1000 runs in 1 second(s) Total number of samples: 1000 Total number of samples with errors: 438 (43%) Total number of samples with recorded traces: 87 (8%) Total number of samples with snap restores: 30 (3%) Total number of samples with abort traces: 55 (5%) ``` 1. https://www.tarantool.io/en/doc/latest/reference/tooling/luajit_getmetrics/#getmetrics-c-api NO_CHANGELOG=testing NO_DOC=testing (cherry picked from commit 430fa6a2ae173845c4ae78fd07c5801cb261433f)
Actions:
Patch | Tree
commit - 877a42c7caa232ac69e3ef45c8d255089e5cd2ef
commit + 14a9269e7dd5371e152b9d2f65f9508945f73060
blob - f8bb4e8243ead0a86bb6efdfb1b3dd4b6d32d0d1
blob + 15ccaf9227183a0216bbef7cb0b08ff29ec59bf0
--- test/fuzz/luaL_loadbuffer/luaL_loadbuffer_fuzzer.cc
+++ test/fuzz/luaL_loadbuffer/luaL_loadbuffer_fuzzer.cc
@@ -1,8 +1,13 @@
 extern "C"
 {
+#include <signal.h>
+#include <unistd.h>
+
 #include <lua.h>
 #include <lualib.h>
 #include <lauxlib.h>
+/* luaM_metrics */
+#include <lmisclib.h>
 }
 
 #include "lua_grammar.pb.h"
@@ -11,6 +16,77 @@ extern "C"
 #include <libprotobuf-mutator/port/protobuf.h>
 #include <libprotobuf-mutator/src/libfuzzer/libfuzzer_macro.h>
 
+#define PRINT_METRIC(desc, val, total) \
+		std::cerr << (desc) << (val) \
+			  << " (" << (val) * 100 / (total) << "%)" \
+			  << std::endl
+
+struct metrics {
+	size_t total_num;
+	size_t total_num_with_errors;
+	size_t jit_snap_restore;
+	size_t jit_trace_abort;
+	size_t jit_trace_num;
+};
+
+static struct metrics metrics;
+
+static inline void
+print_metrics(struct metrics *metrics)
+{
+	if (metrics->total_num == 0)
+		return;
+
+	std::cerr << "Total number of samples: "
+		  << metrics->total_num << std::endl;
+	PRINT_METRIC("Total number of samples with errors: ",
+		     metrics->total_num_with_errors, metrics->total_num);
+	PRINT_METRIC("Total number of samples with recorded traces: ",
+		     metrics->jit_trace_num, metrics->total_num);
+	PRINT_METRIC("Total number of samples with snap restores: ",
+		     metrics->jit_snap_restore, metrics->total_num);
+	PRINT_METRIC("Total number of samples with abort traces: ",
+		     metrics->jit_trace_abort, metrics->total_num);
+}
+
+/* https://www.tarantool.io/en/doc/latest/reference/tooling/luajit_getmetrics/#getmetrics-c-api */
+static inline void
+collect_lj_metrics(struct metrics *metrics, lua_State *L)
+{
+	struct luam_Metrics lj_metrics;
+	luaM_metrics(L, &lj_metrics);
+	if (lj_metrics.jit_snap_restore != 0)
+		metrics->jit_snap_restore++;
+	if (lj_metrics.jit_trace_abort != 0)
+		metrics->jit_trace_abort++;
+	if (lj_metrics.jit_trace_num != 0)
+		metrics->jit_trace_num++;
+}
+
+void
+sig_handler(int signo, siginfo_t *info, void *context)
+{
+	print_metrics(&metrics);
+}
+
+__attribute__((constructor))
+static void
+setup(void)
+{
+	metrics = {};
+	struct sigaction act = {};
+	act.sa_flags = SA_SIGINFO;
+	act.sa_sigaction = &sig_handler;
+	sigaction(SIGUSR1, &act, NULL);
+}
+
+__attribute__((destructor))
+static void
+teardown(void)
+{
+	print_metrics(&metrics);
+}
+
 /**
  * Get an error message from the stack, and report it to std::cerr.
  * Remove the message from the stack.
@@ -18,6 +94,7 @@ extern "C"
 static inline void
 report_error(lua_State *L, const std::string &prefix)
 {
+	metrics.total_num_with_errors++;
 	const char *verbose = ::getenv("LUA_FUZZER_VERBOSE");
 	if (!verbose)
 		return;
@@ -67,6 +144,8 @@ DEFINE_PROTO_FUZZER(const lua_grammar::Block &message)
 		report_error(L, "lua_pcall()");
 
 end:
+	metrics.total_num++;
+	collect_lj_metrics(&metrics, L);
 	lua_settop(L, 0);
 	lua_close(L);
 }