commit 45f9759f6531afc9e0eeee6d5ce0e2da24dbb38f from: Ilya Verbin via: Aleksandr Lyapunov date: Wed Nov 08 13:11:24 2023 UTC box: introduce space:stat() See the doc bot request for the description. Benchmark results: NO_WRAP $ taskset 0x2 compare.py benchmarks ./memtx.perftest.old ./memtx.perftest.new \ --benchmark_min_warmup_time=10 \ --benchmark_repetitions=30 \ --benchmark_report_aggregates_only=true \ --benchmark_filter=TreeReplaceRandomExistingKeys [...] Comparing ./memtx.perftest.old to ./memtx.perftest.new Benchmark Time CPU Time Old Time New CPU Old CPU New ------------------------------------------------------------------------------------------------------------------------------------------------ MemtxFixture/TreeReplaceRandomExistingKeys_mean +0.0097 +0.0097 1073 1084 1073 1084 MemtxFixture/TreeReplaceRandomExistingKeys_median +0.0075 +0.0075 1062 1070 1062 1070 MemtxFixture/TreeReplaceRandomExistingKeys_stddev -0.1207 -0.1208 56 49 56 49 MemtxFixture/TreeReplaceRandomExistingKeys_cv -0.1291 -0.1292 0 0 0 0 NO_WRAP Closes #6762 @TarantoolBot document Title: Document `space:stat()` Product: Tarantool Since: 3.0 Root document: https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_space/ space_object:stat() - Get statistics on the memory usage Returns a table with the cumulative statistics on the memory usage by tuples in the space. The statistics is grouped by arena types: "memtx" or "malloc". For a detailed description of each field see `tuple_object:info()`. Note: The statistics is collected only for memtx storage engine. For other types of spaces, an empty table is returned. Example: ``` tarantool> box.space.test:stat() --- - tuple: memtx: data_size: 5100699 header_size: 96 field_map_size: 40 waste_size: 143093 malloc: data_size: 18850077 header_size: 70 field_map_size: 28 waste_size: 0 ... ``` commit - ab24dfb65b45ab372ccbf4831dde80d3887e686b commit + 45f9759f6531afc9e0eeee6d5ce0e2da24dbb38f blob - /dev/null blob + ee23240a042533871fc9d61b3dcaff266f4a75b7 (mode 644) --- /dev/null +++ changelogs/unreleased/gh-6762-add-tuple-info-and-space-stat.md @@ -0,0 +1,4 @@ +## feature/core + +* Introduced new methods `tuple:info()` and `space:stat()` with the detailed + information on memory consumed by data (gh-6762). blob - fe18bc604cca438f64f2550538c9c3407c162973 blob + bba34c9e19fd94b0db0349ae31b452a9c6601511 --- src/box/lua/schema.lua +++ src/box/lua/schema.lua @@ -2709,6 +2709,7 @@ space_mt.run_triggers = function(space, yesno) builtin.space_run_triggers(s, yesno) end space_mt.frommap = box.internal.space.frommap +space_mt.stat = box.internal.space.stat space_mt.__index = space_mt box.schema.index_mt = base_index_mt blob - 0706a783b387f8c1db608df769c5b6ded9883937 blob + c9da318b6264d848a5ba3054b7acd58d3162e8f9 --- src/box/lua/space.cc +++ src/box/lua/space.cc @@ -47,6 +47,7 @@ extern "C" { #include "box/func.h" #include "box/func_def.h" #include "box/space.h" +#include "box/memtx_space.h" #include "box/schema.h" #include "box/user_def.h" #include "box/tuple.h" @@ -746,6 +747,59 @@ usage_error: return luaL_error(L, "Usage: space:frommap(map, opts)"); } +/** + * Push to Lua stack a table with the statistics on the memory usage by tuples + * of the space. + */ +static int +lbox_space_stat(struct lua_State *L) +{ + int argc = lua_gettop(L); + if (argc != 1 || !lua_istable(L, 1)) + luaL_error(L, "Usage: space:stat()"); + + lua_getfield(L, 1, "id"); + uint32_t id = (int)lua_tointeger(L, -1); + struct space *space = space_by_id(id); + if (space == NULL) { + lua_pushnil(L); + lua_pushfstring(L, "Space with id '%d' doesn't exist", id); + return 2; + } + + if (!space_is_memtx(space)) { + lua_newtable(L); + return 1; + } + struct memtx_space *memtx_space = (struct memtx_space *)space; + + lua_newtable(L); /* Result table. */ + lua_newtable(L); /* result.tuple */ + + for (int i = TUPLE_ARENA_MEMTX; i <= TUPLE_ARENA_MALLOC; i++) { + enum tuple_arena_type type = (enum tuple_arena_type)i; + struct tuple_info *stat = &memtx_space->tuple_stat[type]; + lua_newtable(L); + + lua_pushnumber(L, stat->data_size); + lua_setfield(L, -2, "data_size"); + + lua_pushnumber(L, stat->header_size); + lua_setfield(L, -2, "header_size"); + + lua_pushnumber(L, stat->field_map_size); + lua_setfield(L, -2, "field_map_size"); + + lua_pushnumber(L, stat->waste_size); + lua_setfield(L, -2, "waste_size"); + + lua_setfield(L, -2, tuple_arena_type_strs[i]); + } + lua_setfield(L, -2, "tuple"); + + return 1; +} + void box_lua_space_init(struct lua_State *L) { @@ -849,6 +903,7 @@ box_lua_space_init(struct lua_State *L) static const struct luaL_Reg space_internal_lib[] = { {"frommap", lbox_space_frommap}, + {"stat", lbox_space_stat}, {NULL, NULL} }; luaL_findtable(L, LUA_GLOBALSINDEX, "box.internal.space", 0); blob - 934da1fc806851ec1ab17eb4561d6bad230229ad blob + 94c3500ff49c391f8462718764d7e58f41ad064c --- src/box/memtx_engine.cc +++ src/box/memtx_engine.cc @@ -617,10 +617,7 @@ memtx_engine_commit(struct engine *engine, struct txn stailq_foreach_entry(stmt, &txn->stmts, next) { if (memtx_tx_manager_use_mvcc_engine) { assert(stmt->space->engine == engine); - struct memtx_space *mspace = - (struct memtx_space *)stmt->space; - size_t *bsize = &mspace->bsize; - memtx_tx_history_commit_stmt(stmt, bsize); + memtx_tx_history_commit_stmt(stmt); } if (stmt->engine_savepoint != NULL) { struct space *space = stmt->space; @@ -679,7 +676,7 @@ memtx_engine_rollback_statement(struct engine *engine, } } - memtx_space_update_bsize(space, new_tuple, old_tuple); + memtx_space_update_tuple_stat(space, new_tuple, old_tuple); if (old_tuple != NULL) tuple_ref(old_tuple); if (new_tuple != NULL) blob - efd502c117be63e0f1af70098bc0021c9a3392a0 blob + 1994d2c23ccd470566dda04428fc12dcbc984230 --- src/box/memtx_space.c +++ src/box/memtx_space.c @@ -75,21 +75,56 @@ static size_t memtx_space_bsize(struct space *space) { struct memtx_space *memtx_space = (struct memtx_space *)space; - return memtx_space->bsize; + return memtx_space->tuple_stat[TUPLE_ARENA_MEMTX].data_size + + memtx_space->tuple_stat[TUPLE_ARENA_MALLOC].data_size; } /* {{{ DML */ void -memtx_space_update_bsize(struct space *space, struct tuple *old_tuple, - struct tuple *new_tuple) +memtx_space_update_tuple_stat(struct space *space, struct tuple *old_tuple, + struct tuple *new_tuple) { assert(space->vtab->destroy == &memtx_space_destroy); struct memtx_space *memtx_space = (struct memtx_space *)space; - ssize_t old_bsize = old_tuple ? box_tuple_bsize(old_tuple) : 0; - ssize_t new_bsize = new_tuple ? box_tuple_bsize(new_tuple) : 0; - assert((ssize_t)memtx_space->bsize + new_bsize - old_bsize >= 0); - memtx_space->bsize += new_bsize - old_bsize; + struct tuple_info info, *stat; + + if (new_tuple != NULL) { + tuple_info(new_tuple, &info); + + assert(info.arena_type == TUPLE_ARENA_MEMTX || + info.arena_type == TUPLE_ARENA_MALLOC); + stat = &memtx_space->tuple_stat[info.arena_type]; + + stat->data_size += info.data_size; + stat->header_size += info.header_size; + stat->field_map_size += info.field_map_size; + stat->waste_size += info.waste_size; + } + + if (old_tuple != NULL) { + tuple_info(old_tuple, &info); + + assert(info.arena_type == TUPLE_ARENA_MEMTX || + info.arena_type == TUPLE_ARENA_MALLOC); + stat = &memtx_space->tuple_stat[info.arena_type]; + + assert(stat->data_size >= info.data_size); + assert(stat->header_size >= info.header_size); + assert(stat->field_map_size >= info.field_map_size); + + stat->data_size -= info.data_size; + stat->header_size -= info.header_size; + stat->field_map_size -= info.field_map_size; + /* + * Avoid negative values, since waste_size is calculated + * imprecisely. + */ + if (stat->waste_size > info.waste_size) + stat->waste_size -= info.waste_size; + else + stat->waste_size = 0; + } } /** @@ -142,7 +177,7 @@ memtx_space_replace_build_next(struct space *space, st } if (index_build_next(space->index[0], new_tuple) != 0) return -1; - memtx_space_update_bsize(space, NULL, new_tuple); + memtx_space_update_tuple_stat(space, NULL, new_tuple); tuple_ref(new_tuple); return 0; } @@ -161,7 +196,7 @@ memtx_space_replace_primary_key(struct space *space, s if (index_replace(space->index[0], old_tuple, new_tuple, mode, &old_tuple, &successor) != 0) return -1; - memtx_space_update_bsize(space, old_tuple, new_tuple); + memtx_space_update_tuple_stat(space, old_tuple, new_tuple); if (new_tuple != NULL) tuple_ref(new_tuple); *result = old_tuple; @@ -313,7 +348,7 @@ memtx_space_replace_all_keys(struct space *space, stru goto rollback; } - memtx_space_update_bsize(space, old_tuple, new_tuple); + memtx_space_update_tuple_stat(space, old_tuple, new_tuple); if (new_tuple != NULL) tuple_ref(new_tuple); *result = old_tuple; @@ -1366,7 +1401,7 @@ memtx_space_prepare_alter(struct space *old_space, str struct memtx_space *old_memtx_space = (struct memtx_space *)old_space; struct memtx_space *new_memtx_space = (struct memtx_space *)new_space; - if (old_memtx_space->bsize != 0 && + if (memtx_space_bsize(old_space) != 0 && space_is_data_temporary(old_space) != space_is_data_temporary(new_space)) { diag_set(ClientError, ER_ALTER_SPACE, old_space->def->name, @@ -1380,9 +1415,9 @@ memtx_space_prepare_alter(struct space *old_space, str } /** - * Copy bsize to the newly altered space from the old space. + * Copy memory usage statistics to the newly altered space from the old space. * In case of DropIndex or TruncateIndex alter operations, the new space will be - * empty, and bsize must not be copied. + * empty, and tuple statistics must not be copied. */ static void memtx_space_finish_alter(struct space *old_space, struct space *new_space) @@ -1392,8 +1427,12 @@ memtx_space_finish_alter(struct space *old_space, stru bool is_empty = new_space->index_count == 0 || index_size(new_space->index[0]) == 0; - if (!is_empty) - new_memtx_space->bsize = old_memtx_space->bsize; + if (!is_empty) { + new_memtx_space->tuple_stat[TUPLE_ARENA_MEMTX] = + old_memtx_space->tuple_stat[TUPLE_ARENA_MEMTX]; + new_memtx_space->tuple_stat[TUPLE_ARENA_MALLOC] = + old_memtx_space->tuple_stat[TUPLE_ARENA_MALLOC]; + } } /* }}} DDL */ @@ -1458,7 +1497,7 @@ memtx_space_new(struct memtx_engine *memtx, /* Format is now referenced by the space. */ tuple_format_unref(format); - memtx_space->bsize = 0; + memset(&memtx_space->tuple_stat, 0, sizeof(memtx_space->tuple_stat)); memtx_space->rowid = 0; memtx_space->replace = memtx_space_replace_no_keys; return (struct space *)memtx_space; blob - a13dae7145af91fe31dfb22cfe77c6e16a9940c9 blob + 9c0547a4d6b6b035dbf7ea0b321ee103252d0eea --- src/box/memtx_space.h +++ src/box/memtx_space.h @@ -31,6 +31,7 @@ * SUCH DAMAGE. */ #include "space.h" +#include "tuple.h" #include "memtx_engine.h" #if defined(__cplusplus) @@ -41,9 +42,14 @@ struct memtx_engine; struct memtx_space { struct space base; - /* Number of bytes used in memory by tuples in the space. */ - size_t bsize; /** + * Cumulative statistics on the memory usage by tuples in the space, + * grouped by the following arena types: + * 0 - TUPLE_ARENA_MEMTX; + * 1 - TUPLE_ARENA_MALLOC. + */ + struct tuple_info tuple_stat[2]; + /** * This counter is used to generate unique ids for * ephemeral spaces. Mostly used by SQL: values of this * var are stored as separate field to hold non-unique @@ -59,17 +65,17 @@ struct memtx_space { }; /** - * Change binary size of a space subtracting old tuple's size and - * adding new tuple's size. Used also for rollback by swaping old - * and new tuple. + * Update memory usage statistics of a space by subtracting old tuple's sizes + * and adding new tuple's sizes. Used also for rollback by swapping old and new + * tuples. * * @param space Instance of memtx space. * @param old_tuple Old tuple (replaced or deleted). * @param new_tuple New tuple (inserted). */ void -memtx_space_update_bsize(struct space *space, struct tuple *old_tuple, - struct tuple *new_tuple); +memtx_space_update_tuple_stat(struct space *space, struct tuple *old_tuple, + struct tuple *new_tuple); int memtx_space_replace_no_keys(struct space *, struct tuple *, struct tuple *, blob - fd4a49d924629089ccc183908cc1307e739a745e blob + 8ad94c96d5bfaa5aa9968f6dbc75d51b1c20c6cb --- src/box/memtx_tx.c +++ src/box/memtx_tx.c @@ -29,6 +29,7 @@ * SUCH DAMAGE. */ #include "memtx_tx.h" +#include "memtx_space.h" #include #include @@ -2722,16 +2723,19 @@ memtx_tx_prepare_finalize(struct txn *txn) } void -memtx_tx_history_commit_stmt(struct txn_stmt *stmt, size_t *bsize) +memtx_tx_history_commit_stmt(struct txn_stmt *stmt) { + struct tuple *old_tuple, *new_tuple; + old_tuple = stmt->del_story == NULL ? NULL : stmt->del_story->tuple; + new_tuple = stmt->add_story == NULL ? NULL : stmt->add_story->tuple; + memtx_space_update_tuple_stat(stmt->space, old_tuple, new_tuple); + if (stmt->add_story != NULL) { assert(stmt->add_story->add_stmt == stmt); - *bsize += tuple_bsize(stmt->add_story->tuple); memtx_tx_story_unlink_added_by(stmt->add_story, stmt); } if (stmt->del_story != NULL) { assert(stmt->del_story->del_stmt == stmt); - *bsize -= tuple_bsize(stmt->del_story->tuple); memtx_tx_story_unlink_deleted_by(stmt->del_story, stmt); } memtx_tx_story_gc(); blob - 5ab55817d5f3d633bb9856b431ce7d3297c65167 blob + 510fd731beb6063d3047f91afadcd410a4fe12dc --- src/box/memtx_tx.h +++ src/box/memtx_tx.h @@ -251,10 +251,9 @@ memtx_tx_prepare_finalize(struct txn *txn); * NB: can trigger story garbage collection. * * @param stmt current statement. - * @param bsize the space bsize. */ void -memtx_tx_history_commit_stmt(struct txn_stmt *stmt, size_t *bsize); +memtx_tx_history_commit_stmt(struct txn_stmt *stmt); /** Helper of memtx_tx_tuple_clarify */ struct tuple * blob - 3318921b4a0dd940446e4b2726c7227f69859567 blob + d96a45a8692739e60808309660208d423e2a29dd --- test/box-luatest/gh_6762_tuple_and_space_size_test.lua +++ test/box-luatest/gh_6762_tuple_and_space_size_test.lua @@ -129,3 +129,103 @@ g1.test_errors = function() box.tuple.new{0}.info, 'xxx' ) end + +local g2 = t.group('gh-6762-space-stat', + {{use_mvcc = false}, {use_mvcc = true}}) + +g2.before_all(function(cg) + local box_cfg = { + slab_alloc_factor = 1.3, + slab_alloc_granularity = 32, + memtx_max_tuple_size = 1200*1000, + memtx_use_mvcc_engine = cg.params.use_mvcc + } + cg.server = server:new{box_cfg = box_cfg} + cg.server:start() +end) +g2.after_all(after_all) +g2.after_each(after_each) + +-- Test stat() method of a memtx space. +g2.test_space_stat_memtx = function(cg) + skip_if_asan_is_enabled() + cg.server:exec(function(use_mvcc) + local s = box.schema.space.create('memtx') + s:create_index('pk', {parts = {2}}) + + s:insert{string.rep('a', 251), 0} + s:insert{string.rep('b', 252), 1} + s:insert{string.rep('c', 1100*1000), 2} + s:insert{string.rep('d', 1100*1000), 3} + + local total_data_size = 0 + for _, tuple in s:pairs() do + total_data_size = total_data_size + tuple:info().data_size + end + t.assert_equals(total_data_size, s:bsize()) + t.assert_equals(total_data_size, + s:stat().tuple.memtx.data_size + + s:stat().tuple.malloc.data_size) + + local old_stat = { tuple = { memtx = { data_size = 511, + header_size = 16, + field_map_size = 8, + waste_size = 489 }, + malloc = { data_size = 2200014, + header_size = 20, + field_map_size = 8, + waste_size = 0 } } + } + local new_stat = { tuple = { memtx = { data_size = 517, + header_size = 22, + field_map_size = 12, + waste_size = 537 }, + malloc = { data_size = 1100007, + header_size = 10, + field_map_size = 4, + waste_size = 0 } } + } + t.assert_equals(s:stat(), old_stat) + + box.begin() + s:delete{3} + s:insert{'new', 3} + t.assert_equals(s:stat(), use_mvcc and old_stat or new_stat) + box.rollback() + t.assert_equals(s:stat(), old_stat) + + box.begin() + s:delete{3} + s:insert{'new', 3} + t.assert_equals(s:stat(), use_mvcc and old_stat or new_stat) + box.commit() + t.assert_equals(s:stat(), new_stat) + end, {cg.params.use_mvcc}) +end + +-- Test stat() method of a vinyl space. +g2.test_space_stat_vinyl = function(cg) + cg.server:exec(function() + local s = box.schema.space.create('vinyl', {engine = 'vinyl'}) + s:create_index('pk', {parts = {2}}) + s:insert{string.rep('a', 251), 0} + t.assert_equals(s:stat(), {}) + end) +end + +-- Test error messages. +g2.test_errors = function(cg) + cg.server:exec(function() + local s = box.schema.space.create('memtx') + t.assert_error_msg_content_equals( + 'Usage: space:stat()', + function() s.stat() end + ) + t.assert_error_msg_content_equals( + 'Usage: space:stat()', + function() s:stat('xxx') end + ) + t.assert_equals({s.stat{id = 'xxx'}}, + {nil, "Space with id '0' doesn't exist"}) + end) +end