commit 1b0cc057f3b06138e61a47be4309200186691200
from: Maksim Tiushev <mandesero@gmail.com>
via: Alexander Turenko <totktonada.ru@gmail.com>
date: Fri Aug 09 18:37:21 2024 UTC

serializer: remove duplicate of lua serializer

This patch fixes a bug found by the ASAN instrumentation of LuaJIT
allocator [1]. The problem is using a Lua serializer object that has
been cleaned up by GC.

The crash occurs when executing `tarantool> \set output lua`.

Failing tests:
 - ./test/app-luatest/gh_7031_configure_eos_in_lua_console_test.lua
 - ./test/app-tap/console.test.lua
 - ./test/box/push.test.lua
 - ./app-tap/console_lua.test.lua
 - ./app-luatest/varbinary_test.lua

The `serializer_lua` static member is removed from
`src/box/lua/serialize_lua.c` along with `serializer_lua_init`,
because it is not used by any function other than `serializer_lua_init`,
which is not needed now too.

[1]: Issue #10231

Closes #10177 (this issue is a duplicate of #7404)

NO_TEST=rely on existing tests
NO_CHANGELOG=codehealth
NO_DOC=codehealth

(cherry picked from commit 830f27bbaf94dadcacc66a7f543e0e5feec3988d)

commit - cfac40fbb247b1bbbb8cfa6e88c102784562fe8f
commit + 1b0cc057f3b06138e61a47be4309200186691200
blob - e447fb627f0fc75c156441138e0d70f932d86bdc
blob + 443bd9025a086285fc3c0c0f912fc2ce0629b338
--- src/box/lua/console.c
+++ src/box/lua/console.c
@@ -948,9 +948,6 @@ tarantool_lua_console_init(struct lua_State *L)
 	 */
 	lua_setfield(L, -2, "formatter_lua");
 
-	/* Output formatter in Lua mode */
-	lua_serializer_init(L);
-
 	struct session_vtab console_session_vtab = {
 		.push	= console_session_push,
 		.fd	= console_session_fd,
blob - 3869f6e1da76fdd06dd2e32f86632a8448fc5b42
blob + e4f20c792b93f5c72433c2755660ddf4b54af04d
--- src/box/lua/serialize_lua.c
+++ src/box/lua/serialize_lua.c
@@ -53,9 +53,6 @@ static_assert(DT_IVAL_TO_STRING_BUFSIZE > FPCONV_G_FMT
 	      "Buffer is too small");
 static_assert(DT_IVAL_TO_STRING_BUFSIZE > DT_TO_STRING_BUFSIZE,
 	      "Buffer is too small");
-
-/* Serializer for Lua output mode */
-static struct luaL_serializer *serializer_lua;
 
 enum {
 	NODE_NONE_BIT		= 0,
@@ -1041,35 +1038,4 @@ lua_parse_opts(lua_State *L, lua_dumper_opts_t *opts)
 	if (lua_isnumber(L, -1))
 		opts->indent_lvl = (int)lua_tonumber(L, -1);
 	lua_pop(L, 1);
-}
-
-/**
- * Initialize Lua serializer.
- */
-void
-lua_serializer_init(struct lua_State *L)
-{
-	/*
-	 * We don't export it as a module
-	 * for a while, so the library
-	 * is kept empty.
-	 */
-	static const luaL_Reg lualib[] = {
-		{
-			.name = NULL,
-		},
-	};
-
-	serializer_lua = luaL_newserializer(L, NULL, lualib);
-	serializer_lua->has_compact		= 1;
-	serializer_lua->encode_invalid_numbers	= 1;
-	serializer_lua->encode_load_metatables	= 1;
-	serializer_lua->encode_use_tostring	= 1;
-	serializer_lua->encode_invalid_as_nil	= 1;
-
-	/*
-	 * Keep a reference to this module so it
-	 * won't be unloaded.
-	 */
-	lua_setfield(L, -2, "formatter_lua");
 }
blob - 923e13be9267c6f5f3180e52edff1ef9d0ceb8b2
blob + 24bc79a01f161821966c8330e6d9cf3df9ba8534
--- src/box/lua/serialize_lua.h
+++ src/box/lua/serialize_lua.h
@@ -50,9 +50,6 @@ typedef struct {
 	bool block_mode;
 } lua_dumper_opts_t;
 
-void
-lua_serializer_init(struct lua_State *L);
-
 int
 lua_encode(lua_State *L, struct luaL_serializer *serializer,
 	   lua_dumper_opts_t *opts);