commit e74c0059f78764ce7fb6c165128899820722e529 from: Sulverus date: Fri Nov 13 13:58:39 2015 UTC Merge branch '1.6' into preprocessor-remove commit - 20caa2f535d61e536143aa253d10b979326e0f83 commit + e74c0059f78764ce7fb6c165128899820722e529 blob - b79c47df76797803d9d027a7d5e02929140da049 blob + 72ded82383c28c0e72e86006e564e884324edaa6 --- .gitignore +++ .gitignore @@ -49,7 +49,7 @@ src/box/bootstrap.h src/lua/*.lua.c src/box/lua/*.lua.c src/tarantool -src/trivia/tarantool.h +src/module.h tarantool-*.tar.gz test/lib/ test/unit/*.test blob - 2060105f5213ca16f62f5d8619f9b4e0f05479e0 blob + 504f1351268dfbb811c78181622dacb61de2983e --- .travis.yml +++ .travis.yml @@ -8,6 +8,11 @@ compiler: - clang - gcc +matrix: + exclude: + - os: osx + compiler: gcc + addons: postgresql: "9.1" blob - 2faa825ca61e4c77f1384f80f5c05633239e6ca9 blob + 8437eaedf5cf19b28fb123414c824eb8213e7f23 --- Doxyfile.API.in +++ Doxyfile.API.in @@ -1,5 +1,5 @@ @INCLUDE = @PROJECT_SOURCE_DIR@/Doxyfile -INPUT = @PROJECT_BINARY_DIR@/src/trivia/tarantool.h +INPUT = @PROJECT_BINARY_DIR@/src/module.h OUTPUT_DIRECTORY = @PROJECT_BINARY_DIR@/doc/api/ ENABLED_SECTIONS = public DISABLE_INDEX = YES blob - fd3258d46fd1a09d051b363f457031505e22d468 blob + 6b33d62eebc48fbcdb7be41ad8e5cedbb1df9b89 --- FreeBSD/databases/tarantool/pkg-plist +++ FreeBSD/databases/tarantool/pkg-plist @@ -7,7 +7,7 @@ include/tarantool/lua.hpp include/tarantool/luaconf.h include/tarantool/luajit.h include/tarantool/lualib.h -include/tarantool/tarantool.h +include/tarantool/module.h man/man1/tarantool.1.gz man/man1/tarantoolctl.1.gz %%PORTDOCS%%%%DOCSDIR%%/LICENSE blob - 25cb8f4934ca5d650869ce533a2b80979f35c14e blob + 73bf2eb4e402778f8c3501f3a710e96d585d79ef --- cmake/module.cmake +++ cmake/module.cmake @@ -1,6 +1,6 @@ # A helper function to extract public API function(rebuild_module_api) - set (dstfile "${CMAKE_CURRENT_BINARY_DIR}/tarantool.h") + set (dstfile "${CMAKE_CURRENT_BINARY_DIR}/module.h") set (tmpfile "${dstfile}.new") set (errcodefile "${CMAKE_CURRENT_BINARY_DIR}/errcode.i") set (headers) @@ -20,7 +20,7 @@ function(rebuild_module_api) set (cflags ${cflags} ${CMAKE_C_SYSROOT_FLAG} ${CMAKE_OSX_SYSROOT}) endif() add_custom_command(OUTPUT ${dstfile} - COMMAND cat ${CMAKE_CURRENT_SOURCE_DIR}/tarantool_header.h > ${tmpfile} + COMMAND cat ${CMAKE_CURRENT_SOURCE_DIR}/module_header.h > ${tmpfile} COMMAND cat ${headers} | ${CMAKE_SOURCE_DIR}/extra/apigen >> ${tmpfile} COMMAND ${CMAKE_C_COMPILER} ${cflags} @@ -28,15 +28,15 @@ function(rebuild_module_api) -E ${CMAKE_SOURCE_DIR}/src/box/errcode.h > ${errcodefile} COMMAND grep "enum box_error_code" ${errcodefile} >> ${tmpfile} - COMMAND cat ${CMAKE_CURRENT_SOURCE_DIR}/tarantool_footer.h >> ${tmpfile} + COMMAND cat ${CMAKE_CURRENT_SOURCE_DIR}/module_footer.h >> ${tmpfile} COMMAND ${CMAKE_COMMAND} -E copy_if_different ${tmpfile} ${dstfile} COMMAND ${CMAKE_COMMAND} -E remove ${errcodefile} ${tmpfile} DEPENDS ${srcfiles} ${CMAKE_SOURCE_DIR}/src/box/errcode.h - ${CMAKE_CURRENT_SOURCE_DIR}/tarantool_header.h - ${CMAKE_CURRENT_SOURCE_DIR}/tarantool_footer.h + ${CMAKE_CURRENT_SOURCE_DIR}/module_header.h + ${CMAKE_CURRENT_SOURCE_DIR}/module_footer.h ) add_custom_target(api ALL DEPENDS ${srcfiles} ${dstfile}) install(FILES ${dstfile} DESTINATION ${MODULE_INCLUDEDIR}) endfunction() -set_source_files_properties("${CMAKE_CURRENT_BINARY_DIR}/tarantool.h" PROPERTIES GENERATED HEADER_FILE_ONLY) +set_source_files_properties("${CMAKE_CURRENT_BINARY_DIR}/module.h" PROPERTIES GENERATED HEADER_FILE_ONLY) blob - b4832ed084aad8d2cabba20e75bdec03221da7af blob + d04191754efcf503c0b312a00dfd32fcb169056b --- doc/sphinx/reference/capi.rst +++ doc/sphinx/reference/capi.rst @@ -2,5 +2,5 @@ Module C API ------------------------------------------------------------------------------- -.. doxygenfile:: tarantool.h +.. doxygenfile:: module.h :project: api blob - 9ee2d4bea1cbe7e79af807d0d2fd41c32abfb766 blob + 1e8ef35df230165b46213b393ab77a94ff0d4e1d --- extra/rpm/tarantool.rpm.spec.in +++ extra/rpm/tarantool.rpm.spec.in @@ -227,7 +227,7 @@ chkconfig --del tarantool "%{_includedir}/tarantool/lua.hpp" "%{_includedir}/tarantool/luajit.h" "%{_includedir}/tarantool/lualib.h" -"%{_includedir}/tarantool/tarantool.h" +"%{_includedir}/tarantool/module.h" %files common %defattr(-,root,root,-) blob - bc0680466480ecdec857fe89938495cd0164ccde blob + 1275dc8395e2e52d1d4cfd919115436bff39a639 --- src/CMakeLists.txt +++ src/CMakeLists.txt @@ -3,8 +3,6 @@ # enable_tnt_compile_flags() -add_subdirectory(trivia) - include_directories(${LIBEV_INCLUDE_DIR}) include_directories(${LIBEIO_INCLUDE_DIR}) include_directories(${LIBCORO_INCLUDE_DIR}) @@ -117,6 +115,26 @@ set (server_sources ${lua_sources} ) +set(api_headers + ${CMAKE_BINARY_DIR}/src/trivia/config.h + ${CMAKE_SOURCE_DIR}/src/say.h + ${CMAKE_SOURCE_DIR}/src/fiber.h + ${CMAKE_SOURCE_DIR}/src/coio.h + ${CMAKE_SOURCE_DIR}/src/coeio.h + ${CMAKE_SOURCE_DIR}/src/lua/utils.h + ${CMAKE_SOURCE_DIR}/src/box/txn.h + ${CMAKE_SOURCE_DIR}/src/box/tuple.h + ${CMAKE_SOURCE_DIR}/src/box/schema.h + ${CMAKE_SOURCE_DIR}/src/box/box.h + ${CMAKE_SOURCE_DIR}/src/box/index.h + ${CMAKE_SOURCE_DIR}/src/box/func.h + ${CMAKE_SOURCE_DIR}/src/box/error.h + ${CMAKE_SOURCE_DIR}/src/box/lua/call.h + ${CMAKE_SOURCE_DIR}/src/latch.h + ${CMAKE_SOURCE_DIR}/src/fiber.h +) +rebuild_module_api(${api_headers}) + if (NOT TARGET_OS_DEBIAN_FREEBSD) if (TARGET_OS_FREEBSD) set_source_files_properties( blob - 13df6ee46efc3c5c205667cacbc890aa0b785d01 blob + ae1491f477f210137358723cd5a6b738d54e2383 --- src/box/alter.cc +++ src/box/alter.cc @@ -45,6 +45,7 @@ #include #include "cluster.h" /* for cluster_set_uuid() */ #include "session.h" /* to fetch the current user. */ +#include "vclock.h" /* VCLOCK_MAX */ /** * Lock of scheme modification @@ -1977,6 +1978,8 @@ on_replace_dd_cluster(struct trigger *trigger, void *e if (cserver_id_is_reserved(server_id)) tnt_raise(ClientError, ER_SERVER_ID_IS_RESERVED, (unsigned) server_id); + if (server_id >= VCLOCK_MAX) + tnt_raise(LoggedError, ER_REPLICA_MAX, server_id); tt_uuid server_uuid = tuple_field_uuid(new_tuple, 1); if (tt_uuid_is_nil(&server_uuid)) tnt_raise(ClientError, ER_INVALID_UUID, blob - 42c5c143f18d3bd05dcf1e36137bbc7009ce3a97 blob + 7bb59ef17af0baba367ea1a81713b2cf1dfde5cb --- src/box/box.cc +++ src/box/box.cc @@ -657,9 +657,6 @@ box_on_cluster_join(const tt_uuid *server_uuid) struct tuple *tuple = it->next(it); /** Assign a new server id. */ uint32_t server_id = tuple ? tuple_field_u32(tuple, 0) + 1 : 1; - if (server_id >= VCLOCK_MAX) - tnt_raise(LoggedError, ER_REPLICA_MAX, server_id); - boxk(IPROTO_INSERT, BOX_CLUSTER_ID, "%u%s", (unsigned) server_id, tt_uuid_str(server_uuid)); } blob - d69c19133c6086d400cf0e43c3da052e340982d3 blob + 599665b91f5731fa0043e03a73facaf0d6cba497 --- src/box/cluster.cc +++ src/box/cluster.cc @@ -77,7 +77,7 @@ cluster_set_server(const tt_uuid *server_uuid, uint32_ struct recovery *r = ::recovery; /** Checked in the before-commit trigger */ assert(!tt_uuid_is_nil(server_uuid)); - assert(!cserver_id_is_reserved(server_id)); + assert(!cserver_id_is_reserved(server_id) && server_id < VCLOCK_MAX); if (r->server_id == server_id) { if (tt_uuid_is_equal(&r->server_uuid, server_uuid)) blob - 3347d0ae66ebfa222adfa71674a8eeb4b63a961e blob + 56a0bbffa84edcc861260bcc0c85e41de96c3661 --- src/box/lua/space.cc +++ src/box/lua/space.cc @@ -43,6 +43,7 @@ extern "C" { #include "box/schema.h" #include "box/tuple.h" #include "box/txn.h" +#include "box/vclock.h" /* VCLOCK_MAX */ /** * Trigger function for all spaces @@ -332,5 +333,7 @@ box_lua_space_init(struct lua_State *L) lua_setfield(L, -2, "NAME_MAX"); lua_pushnumber(L, FORMAT_ID_MAX); lua_setfield(L, -2, "FORMAT_ID_MAX"); + lua_pushnumber(L, VCLOCK_MAX); + lua_setfield(L, -2, "REPLICA_MAX"); lua_pop(L, 2); /* box, schema */ } blob - 2f3550281db3c6e61f69a9c1858a2d4408362acb blob + 72863dad5f11e5493ec48a8584512f7aaefe7eb2 --- src/box/request.cc +++ src/box/request.cc @@ -132,7 +132,7 @@ request_encode(struct request *request, struct iovec * const int MAP_LEN_MAX = 40; uint32_t key_len = request->key_end - request->key; uint32_t ops_len = request->ops_end - request->ops; - uint32_t len = MAP_LEN_MAX + key_len; + uint32_t len = MAP_LEN_MAX + key_len + ops_len; char *begin = (char *) region_alloc_xc(&fiber()->gc, len); char *pos = begin + 1; /* skip 1 byte for MP_MAP */ int map_size = 0; blob - 82a1d32aed5e2d4dff84485d5334cdd28df61b8e blob + 4d0070c64a3846933a0c3ce8f880a9f8f39fc69a --- src/box/wal.cc +++ src/box/wal.cc @@ -66,8 +66,8 @@ wal_flush_input(ev_loop * /* loop */, ev_async *watche struct cpipe *pipe = (struct cpipe *) watcher->data; cbus_lock(pipe->bus); - bool input_was_empty = STAILQ_EMPTY(&pipe->pipe); - STAILQ_CONCAT(&pipe->pipe, &pipe->input); + bool input_was_empty = stailq_empty(&pipe->pipe); + stailq_concat(&pipe->pipe, &pipe->input); cbus_unlock(pipe->bus); if (input_was_empty) @@ -90,34 +90,36 @@ wal_flush_input(ev_loop * /* loop */, ev_async *watche * call in the writer thread loop). */ static void -tx_schedule_queue(struct cmsg_fifo *queue) +tx_schedule_queue(struct stailq *queue) { /* - * Can't use STAILQ_FOREACH since fiber_call() + * Can't use stailq_foreach since fiber_call() * destroys the list entry. */ - struct cmsg *m, *tmp; - STAILQ_FOREACH_SAFE(m, queue, fifo, tmp) - fiber_call(((struct wal_request *) m)->fiber); + struct wal_request *req, *tmp; + stailq_foreach_entry_safe(req, tmp, queue, fifo) + fiber_call(req->fiber); } static void tx_fetch_output(ev_loop * /* loop */, ev_async *watcher, int /* event */) { struct wal_writer *writer = (struct wal_writer *) watcher->data; - struct cmsg_fifo commit = STAILQ_HEAD_INITIALIZER(commit); - struct cmsg_fifo rollback = STAILQ_HEAD_INITIALIZER(rollback); + struct stailq commit; + struct stailq rollback; + stailq_create(&commit); + stailq_create(&rollback); bool is_rollback; cbus_lock(&writer->tx_wal_bus); - STAILQ_CONCAT(&commit, &writer->tx_pipe.pipe); + stailq_concat(&commit, &writer->tx_pipe.pipe); is_rollback = writer->is_rollback; if (is_rollback) - STAILQ_CONCAT(&rollback, &writer->wal_pipe.pipe); + stailq_concat(&rollback, &writer->wal_pipe.pipe); writer->is_rollback = false; cbus_unlock(&writer->tx_wal_bus); if (is_rollback) - STAILQ_CONCAT(&rollback, &writer->wal_pipe.input); + stailq_concat(&rollback, &writer->wal_pipe.input); tx_schedule_queue(&commit); /* @@ -127,7 +129,7 @@ tx_fetch_output(ev_loop * /* loop */, ev_async *watche * in reverse order, performing a playback of the * in-memory database state. */ - STAILQ_REVERSE(&rollback, cmsg, fifo); + stailq_reverse(&rollback); tx_schedule_queue(&rollback); } @@ -304,8 +306,8 @@ wal_writer_pop(struct wal_writer *writer) while (! writer->is_shutdown) { if (! writer->is_rollback && - ! STAILQ_EMPTY(&writer->wal_pipe.pipe)) { - STAILQ_CONCAT(&writer->wal_pipe.output, + ! stailq_empty(&writer->wal_pipe.pipe)) { + stailq_concat(&writer->wal_pipe.output, &writer->wal_pipe.pipe); break; } @@ -315,19 +317,19 @@ wal_writer_pop(struct wal_writer *writer) static void wal_write_to_disk(struct recovery *r, struct wal_writer *writer, - struct cmsg_fifo *input, struct cmsg_fifo *commit, - struct cmsg_fifo *rollback) + struct stailq *input, struct stailq *commit, + struct stailq *rollback) { /* * Input queue can only be empty on wal writer shutdown. * In this case wal_opt_rotate can create an extra empty xlog. */ - if (unlikely(STAILQ_EMPTY(input))) + if (unlikely(stailq_empty(input))) return; /* Xlog is only rotated between queue processing */ if (wal_opt_rotate(&r->current_wal, r, &writer->vclock) != 0) { - STAILQ_SPLICE(input, STAILQ_FIRST(input), fifo, rollback); + stailq_concat(rollback, input); return; } @@ -365,9 +367,7 @@ wal_write_to_disk(struct recovery *r, struct wal_write * Iterate over requests (transactions) */ struct wal_request *req; - for (req = (struct wal_request *) STAILQ_FIRST(input); - req != NULL; - req = (struct wal_request *) STAILQ_NEXT(req, fifo)) { + stailq_foreach_entry(req, input, fifo) { /* Save relative offset of request start */ req->start_offset = batched_bytes; req->end_offset = -1; @@ -418,9 +418,9 @@ done: * `commit` queue and all other to `rollback` queue. */ struct wal_request *reqend = req; - for (req = (struct wal_request *) STAILQ_FIRST(input); + for (req = stailq_first_entry(input, struct wal_request, fifo); req != reqend; - req = (struct wal_request *) STAILQ_NEXT(req, fifo)) { + req = stailq_next_entry(req, fifo)) { /* * Check if request has been fully written to xlog. */ @@ -448,7 +448,7 @@ done: written_bytes = req->start_offset; /* Move tail to `rollback` queue. */ - STAILQ_SPLICE(input, req, fifo, rollback); + stailq_splice(input, &req->fifo, rollback); break; } @@ -464,7 +464,7 @@ done: fiber_gc(); /* Move all processed requests to `commit` queue */ - STAILQ_CONCAT(commit, input); + stailq_concat(commit, input); return; } @@ -479,8 +479,10 @@ wal_writer_f(va_list ap) cpipe_create(&writer->wal_pipe); cbus_join(&writer->tx_wal_bus, &writer->wal_pipe); - struct cmsg_fifo commit = STAILQ_HEAD_INITIALIZER(commit); - struct cmsg_fifo rollback = STAILQ_HEAD_INITIALIZER(rollback); + struct stailq commit; + struct stailq rollback; + stailq_create(&commit); + stailq_create(&rollback); cbus_lock(&writer->tx_wal_bus); while (! writer->is_shutdown) { @@ -498,8 +500,8 @@ wal_writer_f(va_list ap) tt_pthread_mutex_unlock(&writer->watchers_mutex); cbus_lock(&writer->tx_wal_bus); - STAILQ_CONCAT(&writer->tx_pipe.pipe, &commit); - if (! STAILQ_EMPTY(&rollback)) { + stailq_concat(&writer->tx_pipe.pipe, &commit); + if (! stailq_empty(&rollback)) { /* * Begin rollback: create a rollback queue * from all requests which were not @@ -507,8 +509,8 @@ wal_writer_f(va_list ap) * input queue. */ writer->is_rollback = true; - STAILQ_CONCAT(&rollback, &writer->wal_pipe.pipe); - STAILQ_CONCAT(&writer->wal_pipe.pipe, &rollback); + stailq_concat(&rollback, &writer->wal_pipe.pipe); + stailq_concat(&writer->wal_pipe.pipe, &rollback); } ev_async_send(writer->tx_pipe.consumer, &writer->tx_pipe.fetch_output); blob - 3a08d5ab003d1a8cc5c9b93c4ce0924f3a3c169c blob + 79a837ce18eceb713343ac4fea4db99435b0bea8 --- src/box/wal.h +++ src/box/wal.h @@ -30,7 +30,6 @@ * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ -#include #include #include "cbus.h" #include "small/rlist.h" blob - bdd98ab67a5df2c20de63b5054ba542401ea8e84 blob + cfec74600b3131e523cd1d250234dbe8ebfa9c55 --- src/cbus.c +++ src/cbus.c @@ -60,9 +60,9 @@ cpipe_fetch_output_cb(ev_loop *loop, struct ev_async * void cpipe_create(struct cpipe *pipe) { - STAILQ_INIT(&pipe->pipe); - STAILQ_INIT(&pipe->input); - STAILQ_INIT(&pipe->output); + stailq_create(&pipe->pipe); + stailq_create(&pipe->input); + stailq_create(&pipe->output); pipe->n_input = 0; pipe->max_input = INT_MAX; @@ -179,19 +179,19 @@ cbus_flush_cb(ev_loop *loop, struct ev_async *watcher, /* Trigger task processing when the queue becomes non-empty. */ bool pipe_was_empty; - bool peer_output_was_empty = STAILQ_EMPTY(&peer->output); + bool peer_output_was_empty = stailq_empty(&peer->output); cbus_lock(pipe->bus); pipe_was_empty = !ev_async_pending(&pipe->fetch_output); /** Flush input */ - STAILQ_CONCAT(&pipe->pipe, &pipe->input); + stailq_concat(&pipe->pipe, &pipe->input); /* * While at it, pop output. * The consumer of the output of the bound queue is the * same as the producer of input, so we can safely access it. * We can safely access queue because it's locked. */ - STAILQ_CONCAT(&peer->output, &peer->pipe); + stailq_concat(&peer->output, &peer->pipe); cbus_unlock(pipe->bus); @@ -202,14 +202,14 @@ cbus_flush_cb(ev_loop *loop, struct ev_async *watcher, ev_async_send(pipe->consumer, &pipe->fetch_output); } - if (peer_output_was_empty && !STAILQ_EMPTY(&peer->output)) + if (peer_output_was_empty && !stailq_empty(&peer->output)) ev_feed_event(peer->consumer, &peer->fetch_output, EV_CUSTOM); } struct cmsg * cpipe_peek_impl(struct cpipe *pipe) { - assert(STAILQ_EMPTY(&pipe->output)); + assert(stailq_empty(&pipe->output)); struct cpipe *peer = pipe->peer; assert(pipe->consumer == loop()); @@ -219,10 +219,10 @@ cpipe_peek_impl(struct cpipe *pipe) cbus_lock(pipe->bus); - STAILQ_CONCAT(&pipe->output, &pipe->pipe); - if (! STAILQ_EMPTY(&peer->input)) { + stailq_concat(&pipe->output, &pipe->pipe); + if (! stailq_empty(&peer->input)) { peer_pipe_was_empty = !ev_async_pending(&peer->fetch_output); - STAILQ_CONCAT(&peer->pipe, &peer->input); + stailq_concat(&peer->pipe, &peer->input); } cbus_unlock(pipe->bus); peer->n_input = 0; @@ -233,7 +233,7 @@ cpipe_peek_impl(struct cpipe *pipe) ev_async_send(peer->consumer, &peer->fetch_output); } - return STAILQ_FIRST(&pipe->output); + return stailq_first_entry(&pipe->output, struct cmsg, fifo); } @@ -301,7 +301,7 @@ cpipe_fiber_pool_cb(ev_loop *loop, struct ev_async *wa (struct cpipe_fiber_pool *) watcher->data; struct cpipe *pipe = pool->pipe; (void) cpipe_peek(pipe); - while (! STAILQ_EMPTY(&pipe->output)) { + while (! stailq_empty(&pipe->output)) { struct fiber *f; if (! rlist_empty(&pool->fiber_cache)) { f = rlist_shift_entry(&pool->fiber_cache, blob - fb27140b63d0ef960f583a47c10c067ebce267c1 blob + cf4286a1a7789bd848c059733e894968195e1513 --- src/cbus.h +++ src/cbus.h @@ -32,7 +32,7 @@ */ #include "fiber.h" #include "rmean.h" -#include "third_party/queue.h" +#include "salad/stailq.h" #if defined(__cplusplus) extern "C" { @@ -87,7 +87,7 @@ struct cmsg { * message is stuck in currently, waiting to get * delivered. */ - STAILQ_ENTRY(cmsg) fifo; + struct stailq_entry fifo; /** The message routing path. */ struct cmsg_hop *route; /** The current hop the message is at. */ @@ -105,8 +105,6 @@ cmsg_init(struct cmsg *msg, struct cmsg_hop *route) msg->hop = msg->route = route; } -STAILQ_HEAD(cmsg_fifo, cmsg); - #define CACHELINE_SIZE 64 /** A uni-directional FIFO queue from one cord to another. */ struct cpipe { @@ -121,7 +119,7 @@ struct cpipe { * output <-- owned by the producer thread */ struct { - struct cmsg_fifo pipe; + struct stailq pipe; /** Peer pipe - the other direction of the bus. */ struct cpipe *peer; struct cbus *bus; @@ -129,7 +127,7 @@ struct cpipe { /** Stuff most actively used in the producer thread. */ struct { /** Staging area for pushed messages */ - struct cmsg_fifo input; + struct stailq input; /** Counters are useful for finer-grained scheduling. */ int n_input; /** @@ -152,7 +150,7 @@ struct cpipe { /** Stuff related to the consumer. */ struct { /** Staged messages (for pop) */ - struct cmsg_fifo output; + struct stailq output; /** * Used to trigger task processing when * the pipe becomes non-empty. @@ -214,11 +212,9 @@ cpipe_pop_output(struct cpipe *pipe) { assert(loop() == pipe->consumer); - if (STAILQ_EMPTY(&pipe->output)) + if (stailq_empty(&pipe->output)) return NULL; - struct cmsg *msg = STAILQ_FIRST(&pipe->output); - STAILQ_REMOVE_HEAD(&pipe->output, fifo); - return msg; + return stailq_shift_entry(&pipe->output, struct cmsg, fifo); } struct cmsg * @@ -233,10 +229,10 @@ cpipe_peek(struct cpipe *pipe) { assert(loop() == pipe->consumer); - if (STAILQ_EMPTY(&pipe->output)) + if (stailq_empty(&pipe->output)) return cpipe_peek_impl(pipe); - return STAILQ_FIRST(&pipe->output); + return stailq_first_entry(&pipe->output, struct cmsg, fifo); } /** @@ -310,7 +306,7 @@ cpipe_push_input(struct cpipe *pipe, struct cmsg *msg) { assert(loop() == pipe->producer); - STAILQ_INSERT_TAIL(&pipe->input, msg, fifo); + stailq_add_tail_entry(&pipe->input, msg, fifo); pipe->n_input++; if (pipe->n_input >= pipe->max_input) ev_invoke(pipe->producer, &pipe->flush_input, EV_CUSTOM); blob - 5e68f406991900aeecfeb3878fbdb134398e6ca8 blob + ff5a3df63237f572ad12fe16b83d424a8863eca6 --- src/coro.c +++ src/coro.c @@ -48,7 +48,20 @@ tarantool_coro_create(struct tarantool_coro *coro, memset(coro, 0, sizeof(*coro)); /* TODO: guard pages */ +#ifdef __APPLE__ + /* + * We are on the verge of stack overflow already, and without + * guard pages it is super fragile. Especially se_metaserialize + * is wicked (srmeta meta[1024] local, sizeof meta[0] == 48). + * + * Mysteriously it only fails on osx and since no one really + * cares about Tarantool performance on osx we simply make fiber + * stacks bigger. + */ + coro->stack_size = page * 32 - slab_sizeof(); +#else coro->stack_size = page * 16 - slab_sizeof(); +#endif coro->stack = (char *) slab_get(slabc, coro->stack_size) + slab_sizeof(); blob - b15148c848f44a1daac3290ae6eefdb22e226756 blob + 01d051568808159cc404a2e6920d921ebfe62794 --- src/lib/salad/stailq.h +++ src/lib/salad/stailq.h @@ -73,6 +73,18 @@ stailq_add(struct stailq *head, struct stailq_entry *i } /** + * Pop an item from list head. + */ +inline static struct stailq_entry * +stailq_shift(struct stailq *head) +{ + struct stailq_entry *shift = head->first; + if ((head->first = head->first->next) == NULL) + head->last = &head->first; + return shift; +} + +/** * Add an item to list tail */ inline static void @@ -146,6 +158,21 @@ stailq_reverse(struct stailq *head) } } +/** Concat all members of head1 starting from elem to the end of head2. */ +static inline void +stailq_splice(struct stailq *head1, struct stailq_entry *elem, + struct stailq *head2) +{ + if (elem) { + *head2->last = elem; + head2->last = head1->last; + head1->last = &head1->first; + while (*head1->last != elem) + head1->last = &(*head1->last)->next; + *head1->last = NULL; + } +} + #define stailq_entry(item, type, member) ({ \ const typeof( ((type *)0)->member ) *__mptr = (item); \ (type *)( (char *)__mptr - ((size_t) &((type *)0)->member) ); }) @@ -173,6 +200,12 @@ stailq_reverse(struct stailq *head) item != stailq_entry(0, typeof(*item), member); \ item = stailq_next_entry(item, member)) +#define stailq_foreach_entry_safe(item, next, head, member) \ + for (item = stailq_first_entry((head), typeof(*item), member); \ + item != stailq_entry(0, typeof(*item), member) && \ + (next = stailq_next_entry(item, member), 1); \ + item = next) + /** * Remove one element from the list and return it * @pre the list is not empty blob - /dev/null blob + c5920a9f45b215cca06a73e608becd5641ba5a2c (mode 644) --- /dev/null +++ src/module_footer.h @@ -0,0 +1,5 @@ +#if defined(__cplusplus) +} /* extern "C" */ +#endif /* defined(__cplusplus) */ + +#endif /* TARANTOOL_MODULE_H_INCLUDED */ blob - /dev/null blob + 8e55e805b38d49272ad40579950f6db691f23be9 (mode 644) --- /dev/null +++ src/module_header.h @@ -0,0 +1,27 @@ +#ifndef TARANTOOL_MODULE_H_INCLUDED +#define TARANTOOL_MODULE_H_INCLUDED + +/** + * \file + */ + +#include +#include /* va_list */ +#include +#include /* strerror(3) */ +#include +#include +#include /* ssize_t */ + +/** Extern modifier for all public functions */ +#if defined(__cplusplus) +#define API_EXPORT extern "C" __attribute__ ((visibility ("default"))) +#else +#define API_EXPORT extern __attribute__ ((visibility ("default"))) +#endif + +#if defined(__cplusplus) +extern "C" { +#endif /* defined(__cplusplus) */ + +#include /* does not have extern C wrappers */ blob - 4fb055488047cdcbab21195f5f785c14de00bc78 (mode 644) blob + /dev/null --- src/trivia/CMakeLists.txt +++ /dev/null @@ -1,19 +0,0 @@ -set(api_headers - ${CMAKE_CURRENT_BINARY_DIR}/config.h - ${CMAKE_SOURCE_DIR}/src/say.h - ${CMAKE_SOURCE_DIR}/src/fiber.h - ${CMAKE_SOURCE_DIR}/src/coio.h - ${CMAKE_SOURCE_DIR}/src/coeio.h - ${CMAKE_SOURCE_DIR}/src/lua/utils.h - ${CMAKE_SOURCE_DIR}/src/box/txn.h - ${CMAKE_SOURCE_DIR}/src/box/tuple.h - ${CMAKE_SOURCE_DIR}/src/box/schema.h - ${CMAKE_SOURCE_DIR}/src/box/box.h - ${CMAKE_SOURCE_DIR}/src/box/index.h - ${CMAKE_SOURCE_DIR}/src/box/func.h - ${CMAKE_SOURCE_DIR}/src/box/error.h - ${CMAKE_SOURCE_DIR}/src/box/lua/call.h - ${CMAKE_SOURCE_DIR}/src/latch.h - ${CMAKE_SOURCE_DIR}/src/fiber.h -) -rebuild_module_api(${api_headers}) blob - c5920a9f45b215cca06a73e608becd5641ba5a2c (mode 644) blob + /dev/null --- src/trivia/tarantool_footer.h +++ /dev/null @@ -1,5 +0,0 @@ -#if defined(__cplusplus) -} /* extern "C" */ -#endif /* defined(__cplusplus) */ - -#endif /* TARANTOOL_MODULE_H_INCLUDED */ blob - 8e55e805b38d49272ad40579950f6db691f23be9 (mode 644) blob + /dev/null --- src/trivia/tarantool_header.h +++ /dev/null @@ -1,27 +0,0 @@ -#ifndef TARANTOOL_MODULE_H_INCLUDED -#define TARANTOOL_MODULE_H_INCLUDED - -/** - * \file - */ - -#include -#include /* va_list */ -#include -#include /* strerror(3) */ -#include -#include -#include /* ssize_t */ - -/** Extern modifier for all public functions */ -#if defined(__cplusplus) -#define API_EXPORT extern "C" __attribute__ ((visibility ("default"))) -#else -#define API_EXPORT extern __attribute__ ((visibility ("default"))) -#endif - -#if defined(__cplusplus) -extern "C" { -#endif /* defined(__cplusplus) */ - -#include /* does not have extern C wrappers */ blob - d1561644a8b321253ca40a3c821da0b034a71984 blob + 7427df196fea8ee29b16b17e1d223a89a3d3611f --- test/CMakeLists.txt +++ test/CMakeLists.txt @@ -1,6 +1,5 @@ enable_tnt_compile_flags() -include_directories(${CMAKE_BINARY_DIR}/src/trivia) function(build_module module files) add_library(${module} SHARED ${files}) set_target_properties(${module} PROPERTIES PREFIX "") blob - cf5af20f2331d6fd59404d2f2c4a547608809fe8 blob + f12025201d3cda1ef90cba2e6ee919ed08518394 --- test/app/module_api.c +++ test/app/module_api.c @@ -1,5 +1,5 @@ #include -#include +#include #include blob - de9e07b9d5e088d21b5520d736f6204f075a3ec3 blob + a9693c6165422dc69c97e2d63f8fe8a4c025417f --- test/box/function1.c +++ test/box/function1.c @@ -1,5 +1,5 @@ #include -#include "tarantool.h" +#include "module.h" #include blob - /dev/null blob + fb33cdf344b528d11763685bdda8d27fc4c50811 (mode 644) --- /dev/null +++ test/box/function1.skipcond @@ -0,0 +1,5 @@ +import os + +# skip test if .so is not found +if not os.path.exists('box/function1.so'): + self.skip=1 blob - b2ace6a60ab88831ddc5c4deb435221da2cef253 blob + 30fc9c0772924c1a4fde71519d8d78726a8f699e --- test/box/update.result +++ test/box/update.result @@ -798,7 +798,23 @@ s:update({0}, {{'+', '+', '+'}}) s:update({0}, {{0, 0, 0}}) --- - error: Illegal parameters, update operation name must be a string +... +-- test for https://github.com/tarantool/tarantool/issues/1142 +-- broken WAL during upsert +ops = {} +--- +... +for i = 1,10 do table.insert(ops, {'=', 2, '1234567890'}) end +--- +... +s:upsert({0}, ops, {0}) +--- ... +--#stop server default +--#start server default +s = box.space.tweedledum +--- +... s:drop() --- ... blob - 16715c747902f772e438b5b7b5902d8d3fe1b6d0 blob + cacf9b9a6926574bec101d590ff6994d372b1a13 --- test/box/update.test.lua +++ test/box/update.test.lua @@ -244,4 +244,13 @@ s:update({0}, {{'+', 0}}) s:update({0}, {{'+', '+', '+'}}) s:update({0}, {{0, 0, 0}}) +-- test for https://github.com/tarantool/tarantool/issues/1142 +-- broken WAL during upsert +ops = {} +for i = 1,10 do table.insert(ops, {'=', 2, '1234567890'}) end +s:upsert({0}, ops, {0}) +--#stop server default +--#start server default +s = box.space.tweedledum + s:drop() blob - /dev/null blob + f8ae138705d84f1abcc8b1acfcd4ed34369ae9a8 (mode 644) --- /dev/null +++ test/replication/lua/fast_replica.lua @@ -0,0 +1,29 @@ + +function join(inspector, n) + for i=1,n do + local rid = tostring(i) + os.execute('mkdir -p tmp') + os.execute('cp ../replication/replica.lua ./tmp/replica'..rid..'.lua') + os.execute('chmod +x ./tmp/replica'..rid..'.lua') + inspector:cmd("create server replica"..rid.." with rpl_master=default, script='./var/tmp/replica"..rid..".lua'") + inspector:cmd("start server replica"..rid) + end +end + + +function drop_all(inspector) + local all = box.space._cluster:select{} + for _, tuple in pairs(all) do + local id = tuple[1] + if id ~= box.info.server.id then + box.space._cluster:delete{id} + inspector:cmd('stop server replica'..tostring(id - 1)) + inspector:cmd('cleanup server replica'..tostring(id - 1)) + end + end +end + +return { + join = join; + drop_all = drop_all; +} blob - /dev/null blob + d36e74b2c68340f5e22cfe61bbd755f64c205c76 (mode 644) --- /dev/null +++ test/replication/prune.result @@ -0,0 +1,122 @@ +print '-------------------------------------------------------------' +--- +... +print 'gh-806: cant prune old replicas by deleting their server ids' +--- +... +print '-------------------------------------------------------------' +--- +... +env = require('test_run') +--- +... +test_run = env.new('127.0.0.1', 8080) +--- +... +replica_set = require('fast_replica') +--- +... +fiber = require('fiber') +--- +... +box.space._cluster:len() == 1 +--- +- true +... +#box.info.vclock == 1 +--- +- true +... +box.schema.user.grant('guest', 'read,write,execute', 'universe') +--- +... +-- Create space and fill it +space = box.schema.create_space('test') +--- +... +index = box.space.test:create_index('primary') +--- +... +for i=1,10 do space:insert{i, 'test'} end +--- +... +-- create max number of replicas and check +replica_set.join(test_run, box.schema.REPLICA_MAX - 2) +--- +... +while box.space._cluster:len() ~= box.schema.REPLICA_MAX - 1 do fiber.sleep(0.001) end +--- +... +box.space._cluster:len() == box.schema.REPLICA_MAX - 1 +--- +- true +... +#box.info.vclock == box.schema.REPLICA_MAX - 1 +--- +- true +... +-- try to add one more replica +uuid = require('uuid') +--- +... +box.space._cluster:insert{box.schema.REPLICA_MAX, uuid.str()} +--- +- error: 'Replica count limit reached: 32' +... +-- Delete all replication nodes +replica_set.drop_all(test_run) +--- +... +box.space._cluster:len() == 1 +--- +- true +... +#box.info.vclock == 1 +--- +- true +... +-- Save a snapshot without removed replicas in vclock +box.snapshot() +--- +- ok +... +-- Master is not crashed then recovering xlog with {replica_id: 0} in header +test_run:cmd('restart server default') +replica_set = require('fast_replica') +--- +... +fiber = require('fiber') +--- +... +-- Rejoin replica and check +replica_set.join(test_run, 1) +--- +... +while box.space._cluster:len() ~= 2 do fiber.sleep(0.001) end +--- +... +-- Check server ids +test_run:cmd('eval replica1 "return box.info.server.id"') +--- +- '{"result": [2]}' +... +box.space._cluster:len() == 2 +--- +- true +... +#box.info.vclock == 2 +--- +- true +... +-- Cleanup +replica_set.drop_all(test_run) +--- +... +box.space._cluster:len() == 1 +--- +- true +... +#box.info.vclock == 1 +--- +- true +... blob - /dev/null blob + a03bd72751c25ec5fb8f0569c01ab47c1893786a (mode 644) --- /dev/null +++ test/replication/prune.test.lua @@ -0,0 +1,56 @@ +print '-------------------------------------------------------------' +print 'gh-806: cant prune old replicas by deleting their server ids' +print '-------------------------------------------------------------' + +env = require('test_run') +test_run = env.new('127.0.0.1', 8080) +replica_set = require('fast_replica') +fiber = require('fiber') + +box.space._cluster:len() == 1 +#box.info.vclock == 1 + +box.schema.user.grant('guest', 'read,write,execute', 'universe') + +-- Create space and fill it +space = box.schema.create_space('test') +index = box.space.test:create_index('primary') +for i=1,10 do space:insert{i, 'test'} end + +-- create max number of replicas and check +replica_set.join(test_run, box.schema.REPLICA_MAX - 2) +while box.space._cluster:len() ~= box.schema.REPLICA_MAX - 1 do fiber.sleep(0.001) end + +box.space._cluster:len() == box.schema.REPLICA_MAX - 1 +#box.info.vclock == box.schema.REPLICA_MAX - 1 + +-- try to add one more replica +uuid = require('uuid') +box.space._cluster:insert{box.schema.REPLICA_MAX, uuid.str()} + +-- Delete all replication nodes +replica_set.drop_all(test_run) +box.space._cluster:len() == 1 +#box.info.vclock == 1 + +-- Save a snapshot without removed replicas in vclock +box.snapshot() +-- Master is not crashed then recovering xlog with {replica_id: 0} in header +test_run:cmd('restart server default') +replica_set = require('fast_replica') +fiber = require('fiber') + +-- Rejoin replica and check +replica_set.join(test_run, 1) +while box.space._cluster:len() ~= 2 do fiber.sleep(0.001) end + +-- Check server ids +test_run:cmd('eval replica1 "return box.info.server.id"') + +box.space._cluster:len() == 2 +#box.info.vclock == 2 + +-- Cleanup +replica_set.drop_all(test_run) +box.space._cluster:len() == 1 +#box.info.vclock == 1 blob - ecebdef354ea15c6808256f9ade5e644fcc86d4e blob + b0874a0eb1b7e5aaf720a48e780448e501b05ce7 --- test/replication/suite.ini +++ test/replication/suite.ini @@ -4,3 +4,4 @@ script = master.lua description = tarantool/box, replication disabled = consistent.test.lua release_disabled = catch.test.lua +lua_libs = lua/fast_replica.lua blob - 5200f3f9e154a11d382f51c21a894e478b56029d blob + eaa0f507c765777e1e9407d251fe54a7890758a9 --- test/replication-py/cluster.result +++ test/replication-py/cluster.result @@ -169,27 +169,6 @@ box.info.vclock[11] --- - null ... -------------------------------------------------------------- -gh-806: cant prune old replicas by deleting their server ids -------------------------------------------------------------- -box.space._schema:insert{'test', 1} ---- -- ['test', 1] -... -cluster_len = box.space._cluster:len() ---- -... -for id, lsn in pairs(box.info.vclock) do if id ~= box.info.server.id then box.space._cluster:delete{id} end end ---- -... -box.space._cluster:len() < cluster_len ---- -- true -... -box.snapshot() ---- -- ok -... box.schema.user.revoke('guest', 'replication') --- ... blob - eae175bff4decdd4fd3226d795c4eb0110c0fd82 blob + bd65a887646171b45079cfea4b90f8fb3780746e --- test/replication-py/cluster.test.py +++ test/replication-py/cluster.test.py @@ -188,28 +188,6 @@ replica.admin('box.info.vclock[%d]' % replica_id3) replica.stop() replica.cleanup(True) -print '-------------------------------------------------------------' -print 'gh-806: cant prune old replicas by deleting their server ids' -print '-------------------------------------------------------------' - -# Rotate xlog -master.restart() -master.admin("box.space._schema:insert{'test', 1}") - -# Prune old replicas -master.admin("cluster_len = box.space._cluster:len()") -# Delete from _cluster for replicas with lsn=0 is safe -master.admin('for id, lsn in pairs(box.info.vclock) do' - ' if id ~= box.info.server.id then box.space._cluster:delete{id} end ' - 'end'); -master.admin("box.space._cluster:len() < cluster_len") - -# Save a snapshot without removed replicas in vclock -master.admin("box.snapshot()") - -# Master is not crashed then recovering xlog with {replica_id: 0} in header -master.restart() - # Cleanup sys.stdout.pop_filter() blob - b530be3e64b55c675d17f65adee130d408286742 blob + be9573344da6a0a3a8b2c84a7c760c9f4adc9003 --- test/unit/stailq.c +++ test/unit/stailq.c @@ -3,7 +3,7 @@ #include #include "unit.h" -#define PLAN 29 +#define PLAN 37 #define ITEMS 7 @@ -47,6 +47,9 @@ main(void) is(stailq_first_entry(&head, struct test, next), &items[0], "first entry"); + for (i = 0; i < ITEMS; i++) + is(stailq_shift(&head), &items[i].next, "shift item %d", i); + ok(stailq_empty(&head), "list is empty after shift"); stailq_create(&head); ok(stailq_empty(&head), "next is empty"); blob - 09225c0adf588173134d9d488c0a85878b40d914 blob + debebf52db49893f2c197a6da6ad61ab86103acd --- test/unit/stailq.result +++ test/unit/stailq.result @@ -1,4 +1,4 @@ -1..29 +1..37 ok 1 - list is empty ok 2 - list is empty after reverse ok 3 - first item @@ -13,18 +13,26 @@ ok 11 - element (foreach) 6 ok 12 - first item ok 13 - head is not empty ok 14 - first entry -ok 15 - next is empty -ok 16 - element (foreach_entry) 6 -ok 17 - element (foreach_entry) 5 -ok 18 - element (foreach_entry) 4 -ok 19 - element (foreach_entry) 3 -ok 20 - element (foreach_entry) 2 -ok 21 - element (foreach_entry) 1 -ok 22 - element (foreach_entry) 0 -ok 23 - element (foreach_entry) 0 -ok 24 - element (foreach_entry) 1 -ok 25 - element (foreach_entry) 2 -ok 26 - element (foreach_entry) 3 -ok 27 - element (foreach_entry) 4 -ok 28 - element (foreach_entry) 5 -ok 29 - element (foreach_entry) 6 +ok 15 - shift item 0 +ok 16 - shift item 1 +ok 17 - shift item 2 +ok 18 - shift item 3 +ok 19 - shift item 4 +ok 20 - shift item 5 +ok 21 - shift item 6 +ok 22 - list is empty after shift +ok 23 - next is empty +ok 24 - element (foreach_entry) 6 +ok 25 - element (foreach_entry) 5 +ok 26 - element (foreach_entry) 4 +ok 27 - element (foreach_entry) 3 +ok 28 - element (foreach_entry) 2 +ok 29 - element (foreach_entry) 1 +ok 30 - element (foreach_entry) 0 +ok 31 - element (foreach_entry) 0 +ok 32 - element (foreach_entry) 1 +ok 33 - element (foreach_entry) 2 +ok 34 - element (foreach_entry) 3 +ok 35 - element (foreach_entry) 4 +ok 36 - element (foreach_entry) 5 +ok 37 - element (foreach_entry) 6