commit - ab24dfb65b45ab372ccbf4831dde80d3887e686b
commit + 45f9759f6531afc9e0eeee6d5ce0e2da24dbb38f
blob - /dev/null
blob + ee23240a042533871fc9d61b3dcaff266f4a75b7 (mode 644)
--- /dev/null
+++ changelogs/unreleased/gh-6762-add-tuple-info-and-space-stat.md
+## feature/core
+
+* Introduced new methods `tuple:info()` and `space:stat()` with the detailed
+ information on memory consumed by data (gh-6762).
blob - fe18bc604cca438f64f2550538c9c3407c162973
blob + bba34c9e19fd94b0db0349ae31b452a9c6601511
--- src/box/lua/schema.lua
+++ src/box/lua/schema.lua
builtin.space_run_triggers(s, yesno)
end
space_mt.frommap = box.internal.space.frommap
+space_mt.stat = box.internal.space.stat
space_mt.__index = space_mt
box.schema.index_mt = base_index_mt
blob - 0706a783b387f8c1db608df769c5b6ded9883937
blob + c9da318b6264d848a5ba3054b7acd58d3162e8f9
--- src/box/lua/space.cc
+++ src/box/lua/space.cc
#include "box/func.h"
#include "box/func_def.h"
#include "box/space.h"
+#include "box/memtx_space.h"
#include "box/schema.h"
#include "box/user_def.h"
#include "box/tuple.h"
return luaL_error(L, "Usage: space:frommap(map, opts)");
}
+/**
+ * Push to Lua stack a table with the statistics on the memory usage by tuples
+ * of the space.
+ */
+static int
+lbox_space_stat(struct lua_State *L)
+{
+ int argc = lua_gettop(L);
+ if (argc != 1 || !lua_istable(L, 1))
+ luaL_error(L, "Usage: space:stat()");
+
+ lua_getfield(L, 1, "id");
+ uint32_t id = (int)lua_tointeger(L, -1);
+ struct space *space = space_by_id(id);
+ if (space == NULL) {
+ lua_pushnil(L);
+ lua_pushfstring(L, "Space with id '%d' doesn't exist", id);
+ return 2;
+ }
+
+ if (!space_is_memtx(space)) {
+ lua_newtable(L);
+ return 1;
+ }
+ struct memtx_space *memtx_space = (struct memtx_space *)space;
+
+ lua_newtable(L); /* Result table. */
+ lua_newtable(L); /* result.tuple */
+
+ for (int i = TUPLE_ARENA_MEMTX; i <= TUPLE_ARENA_MALLOC; i++) {
+ enum tuple_arena_type type = (enum tuple_arena_type)i;
+ struct tuple_info *stat = &memtx_space->tuple_stat[type];
+ lua_newtable(L);
+
+ lua_pushnumber(L, stat->data_size);
+ lua_setfield(L, -2, "data_size");
+
+ lua_pushnumber(L, stat->header_size);
+ lua_setfield(L, -2, "header_size");
+
+ lua_pushnumber(L, stat->field_map_size);
+ lua_setfield(L, -2, "field_map_size");
+
+ lua_pushnumber(L, stat->waste_size);
+ lua_setfield(L, -2, "waste_size");
+
+ lua_setfield(L, -2, tuple_arena_type_strs[i]);
+ }
+ lua_setfield(L, -2, "tuple");
+
+ return 1;
+}
+
void
box_lua_space_init(struct lua_State *L)
{
static const struct luaL_Reg space_internal_lib[] = {
{"frommap", lbox_space_frommap},
+ {"stat", lbox_space_stat},
{NULL, NULL}
};
luaL_findtable(L, LUA_GLOBALSINDEX, "box.internal.space", 0);
blob - 934da1fc806851ec1ab17eb4561d6bad230229ad
blob + 94c3500ff49c391f8462718764d7e58f41ad064c
--- src/box/memtx_engine.cc
+++ src/box/memtx_engine.cc
stailq_foreach_entry(stmt, &txn->stmts, next) {
if (memtx_tx_manager_use_mvcc_engine) {
assert(stmt->space->engine == engine);
- struct memtx_space *mspace =
- (struct memtx_space *)stmt->space;
- size_t *bsize = &mspace->bsize;
- memtx_tx_history_commit_stmt(stmt, bsize);
+ memtx_tx_history_commit_stmt(stmt);
}
if (stmt->engine_savepoint != NULL) {
struct space *space = stmt->space;
}
}
- memtx_space_update_bsize(space, new_tuple, old_tuple);
+ memtx_space_update_tuple_stat(space, new_tuple, old_tuple);
if (old_tuple != NULL)
tuple_ref(old_tuple);
if (new_tuple != NULL)
blob - efd502c117be63e0f1af70098bc0021c9a3392a0
blob + 1994d2c23ccd470566dda04428fc12dcbc984230
--- src/box/memtx_space.c
+++ src/box/memtx_space.c
memtx_space_bsize(struct space *space)
{
struct memtx_space *memtx_space = (struct memtx_space *)space;
- return memtx_space->bsize;
+ return memtx_space->tuple_stat[TUPLE_ARENA_MEMTX].data_size +
+ memtx_space->tuple_stat[TUPLE_ARENA_MALLOC].data_size;
}
/* {{{ DML */
void
-memtx_space_update_bsize(struct space *space, struct tuple *old_tuple,
- struct tuple *new_tuple)
+memtx_space_update_tuple_stat(struct space *space, struct tuple *old_tuple,
+ struct tuple *new_tuple)
{
assert(space->vtab->destroy == &memtx_space_destroy);
struct memtx_space *memtx_space = (struct memtx_space *)space;
- ssize_t old_bsize = old_tuple ? box_tuple_bsize(old_tuple) : 0;
- ssize_t new_bsize = new_tuple ? box_tuple_bsize(new_tuple) : 0;
- assert((ssize_t)memtx_space->bsize + new_bsize - old_bsize >= 0);
- memtx_space->bsize += new_bsize - old_bsize;
+ struct tuple_info info, *stat;
+
+ if (new_tuple != NULL) {
+ tuple_info(new_tuple, &info);
+
+ assert(info.arena_type == TUPLE_ARENA_MEMTX ||
+ info.arena_type == TUPLE_ARENA_MALLOC);
+ stat = &memtx_space->tuple_stat[info.arena_type];
+
+ stat->data_size += info.data_size;
+ stat->header_size += info.header_size;
+ stat->field_map_size += info.field_map_size;
+ stat->waste_size += info.waste_size;
+ }
+
+ if (old_tuple != NULL) {
+ tuple_info(old_tuple, &info);
+
+ assert(info.arena_type == TUPLE_ARENA_MEMTX ||
+ info.arena_type == TUPLE_ARENA_MALLOC);
+ stat = &memtx_space->tuple_stat[info.arena_type];
+
+ assert(stat->data_size >= info.data_size);
+ assert(stat->header_size >= info.header_size);
+ assert(stat->field_map_size >= info.field_map_size);
+
+ stat->data_size -= info.data_size;
+ stat->header_size -= info.header_size;
+ stat->field_map_size -= info.field_map_size;
+ /*
+ * Avoid negative values, since waste_size is calculated
+ * imprecisely.
+ */
+ if (stat->waste_size > info.waste_size)
+ stat->waste_size -= info.waste_size;
+ else
+ stat->waste_size = 0;
+ }
}
/**
}
if (index_build_next(space->index[0], new_tuple) != 0)
return -1;
- memtx_space_update_bsize(space, NULL, new_tuple);
+ memtx_space_update_tuple_stat(space, NULL, new_tuple);
tuple_ref(new_tuple);
return 0;
}
if (index_replace(space->index[0], old_tuple,
new_tuple, mode, &old_tuple, &successor) != 0)
return -1;
- memtx_space_update_bsize(space, old_tuple, new_tuple);
+ memtx_space_update_tuple_stat(space, old_tuple, new_tuple);
if (new_tuple != NULL)
tuple_ref(new_tuple);
*result = old_tuple;
goto rollback;
}
- memtx_space_update_bsize(space, old_tuple, new_tuple);
+ memtx_space_update_tuple_stat(space, old_tuple, new_tuple);
if (new_tuple != NULL)
tuple_ref(new_tuple);
*result = old_tuple;
struct memtx_space *old_memtx_space = (struct memtx_space *)old_space;
struct memtx_space *new_memtx_space = (struct memtx_space *)new_space;
- if (old_memtx_space->bsize != 0 &&
+ if (memtx_space_bsize(old_space) != 0 &&
space_is_data_temporary(old_space) !=
space_is_data_temporary(new_space)) {
diag_set(ClientError, ER_ALTER_SPACE, old_space->def->name,
}
/**
- * Copy bsize to the newly altered space from the old space.
+ * Copy memory usage statistics to the newly altered space from the old space.
* In case of DropIndex or TruncateIndex alter operations, the new space will be
- * empty, and bsize must not be copied.
+ * empty, and tuple statistics must not be copied.
*/
static void
memtx_space_finish_alter(struct space *old_space, struct space *new_space)
bool is_empty = new_space->index_count == 0 ||
index_size(new_space->index[0]) == 0;
- if (!is_empty)
- new_memtx_space->bsize = old_memtx_space->bsize;
+ if (!is_empty) {
+ new_memtx_space->tuple_stat[TUPLE_ARENA_MEMTX] =
+ old_memtx_space->tuple_stat[TUPLE_ARENA_MEMTX];
+ new_memtx_space->tuple_stat[TUPLE_ARENA_MALLOC] =
+ old_memtx_space->tuple_stat[TUPLE_ARENA_MALLOC];
+ }
}
/* }}} DDL */
/* Format is now referenced by the space. */
tuple_format_unref(format);
- memtx_space->bsize = 0;
+ memset(&memtx_space->tuple_stat, 0, sizeof(memtx_space->tuple_stat));
memtx_space->rowid = 0;
memtx_space->replace = memtx_space_replace_no_keys;
return (struct space *)memtx_space;
blob - a13dae7145af91fe31dfb22cfe77c6e16a9940c9
blob + 9c0547a4d6b6b035dbf7ea0b321ee103252d0eea
--- src/box/memtx_space.h
+++ src/box/memtx_space.h
* SUCH DAMAGE.
*/
#include "space.h"
+#include "tuple.h"
#include "memtx_engine.h"
#if defined(__cplusplus)
struct memtx_space {
struct space base;
- /* Number of bytes used in memory by tuples in the space. */
- size_t bsize;
/**
+ * Cumulative statistics on the memory usage by tuples in the space,
+ * grouped by the following arena types:
+ * 0 - TUPLE_ARENA_MEMTX;
+ * 1 - TUPLE_ARENA_MALLOC.
+ */
+ struct tuple_info tuple_stat[2];
+ /**
* This counter is used to generate unique ids for
* ephemeral spaces. Mostly used by SQL: values of this
* var are stored as separate field to hold non-unique
};
/**
- * Change binary size of a space subtracting old tuple's size and
- * adding new tuple's size. Used also for rollback by swaping old
- * and new tuple.
+ * Update memory usage statistics of a space by subtracting old tuple's sizes
+ * and adding new tuple's sizes. Used also for rollback by swapping old and new
+ * tuples.
*
* @param space Instance of memtx space.
* @param old_tuple Old tuple (replaced or deleted).
* @param new_tuple New tuple (inserted).
*/
void
-memtx_space_update_bsize(struct space *space, struct tuple *old_tuple,
- struct tuple *new_tuple);
+memtx_space_update_tuple_stat(struct space *space, struct tuple *old_tuple,
+ struct tuple *new_tuple);
int
memtx_space_replace_no_keys(struct space *, struct tuple *, struct tuple *,
blob - fd4a49d924629089ccc183908cc1307e739a745e
blob + 8ad94c96d5bfaa5aa9968f6dbc75d51b1c20c6cb
--- src/box/memtx_tx.c
+++ src/box/memtx_tx.c
* SUCH DAMAGE.
*/
#include "memtx_tx.h"
+#include "memtx_space.h"
#include <assert.h>
#include <limits.h>
}
void
-memtx_tx_history_commit_stmt(struct txn_stmt *stmt, size_t *bsize)
+memtx_tx_history_commit_stmt(struct txn_stmt *stmt)
{
+ struct tuple *old_tuple, *new_tuple;
+ old_tuple = stmt->del_story == NULL ? NULL : stmt->del_story->tuple;
+ new_tuple = stmt->add_story == NULL ? NULL : stmt->add_story->tuple;
+ memtx_space_update_tuple_stat(stmt->space, old_tuple, new_tuple);
+
if (stmt->add_story != NULL) {
assert(stmt->add_story->add_stmt == stmt);
- *bsize += tuple_bsize(stmt->add_story->tuple);
memtx_tx_story_unlink_added_by(stmt->add_story, stmt);
}
if (stmt->del_story != NULL) {
assert(stmt->del_story->del_stmt == stmt);
- *bsize -= tuple_bsize(stmt->del_story->tuple);
memtx_tx_story_unlink_deleted_by(stmt->del_story, stmt);
}
memtx_tx_story_gc();
blob - 5ab55817d5f3d633bb9856b431ce7d3297c65167
blob + 510fd731beb6063d3047f91afadcd410a4fe12dc
--- src/box/memtx_tx.h
+++ src/box/memtx_tx.h
* NB: can trigger story garbage collection.
*
* @param stmt current statement.
- * @param bsize the space bsize.
*/
void
-memtx_tx_history_commit_stmt(struct txn_stmt *stmt, size_t *bsize);
+memtx_tx_history_commit_stmt(struct txn_stmt *stmt);
/** Helper of memtx_tx_tuple_clarify */
struct tuple *
blob - 3318921b4a0dd940446e4b2726c7227f69859567
blob + d96a45a8692739e60808309660208d423e2a29dd
--- test/box-luatest/gh_6762_tuple_and_space_size_test.lua
+++ test/box-luatest/gh_6762_tuple_and_space_size_test.lua
box.tuple.new{0}.info, 'xxx'
)
end
+
+local g2 = t.group('gh-6762-space-stat',
+ {{use_mvcc = false}, {use_mvcc = true}})
+
+g2.before_all(function(cg)
+ local box_cfg = {
+ slab_alloc_factor = 1.3,
+ slab_alloc_granularity = 32,
+ memtx_max_tuple_size = 1200*1000,
+ memtx_use_mvcc_engine = cg.params.use_mvcc
+ }
+ cg.server = server:new{box_cfg = box_cfg}
+ cg.server:start()
+end)
+g2.after_all(after_all)
+g2.after_each(after_each)
+
+-- Test stat() method of a memtx space.
+g2.test_space_stat_memtx = function(cg)
+ skip_if_asan_is_enabled()
+ cg.server:exec(function(use_mvcc)
+ local s = box.schema.space.create('memtx')
+ s:create_index('pk', {parts = {2}})
+
+ s:insert{string.rep('a', 251), 0}
+ s:insert{string.rep('b', 252), 1}
+ s:insert{string.rep('c', 1100*1000), 2}
+ s:insert{string.rep('d', 1100*1000), 3}
+
+ local total_data_size = 0
+ for _, tuple in s:pairs() do
+ total_data_size = total_data_size + tuple:info().data_size
+ end
+ t.assert_equals(total_data_size, s:bsize())
+ t.assert_equals(total_data_size,
+ s:stat().tuple.memtx.data_size +
+ s:stat().tuple.malloc.data_size)
+
+ local old_stat = { tuple = { memtx = { data_size = 511,
+ header_size = 16,
+ field_map_size = 8,
+ waste_size = 489 },
+ malloc = { data_size = 2200014,
+ header_size = 20,
+ field_map_size = 8,
+ waste_size = 0 } }
+ }
+ local new_stat = { tuple = { memtx = { data_size = 517,
+ header_size = 22,
+ field_map_size = 12,
+ waste_size = 537 },
+ malloc = { data_size = 1100007,
+ header_size = 10,
+ field_map_size = 4,
+ waste_size = 0 } }
+ }
+ t.assert_equals(s:stat(), old_stat)
+
+ box.begin()
+ s:delete{3}
+ s:insert{'new', 3}
+ t.assert_equals(s:stat(), use_mvcc and old_stat or new_stat)
+ box.rollback()
+ t.assert_equals(s:stat(), old_stat)
+
+ box.begin()
+ s:delete{3}
+ s:insert{'new', 3}
+ t.assert_equals(s:stat(), use_mvcc and old_stat or new_stat)
+ box.commit()
+ t.assert_equals(s:stat(), new_stat)
+ end, {cg.params.use_mvcc})
+end
+
+-- Test stat() method of a vinyl space.
+g2.test_space_stat_vinyl = function(cg)
+ cg.server:exec(function()
+ local s = box.schema.space.create('vinyl', {engine = 'vinyl'})
+ s:create_index('pk', {parts = {2}})
+ s:insert{string.rep('a', 251), 0}
+ t.assert_equals(s:stat(), {})
+ end)
+end
+
+-- Test error messages.
+g2.test_errors = function(cg)
+ cg.server:exec(function()
+ local s = box.schema.space.create('memtx')
+ t.assert_error_msg_content_equals(
+ 'Usage: space:stat()',
+ function() s.stat() end
+ )
+ t.assert_error_msg_content_equals(
+ 'Usage: space:stat()',
+ function() s:stat('xxx') end
+ )
+ t.assert_equals({s.stat{id = 'xxx'}},
+ {nil, "Space with id '0' doesn't exist"})
+ end)
+end
