diff options
Diffstat (limited to 'storage/mroonga/vendor/groonga/src')
12 files changed, 1369 insertions, 618 deletions
diff --git a/storage/mroonga/vendor/groonga/src/CMakeLists.txt b/storage/mroonga/vendor/groonga/src/CMakeLists.txt index 258d1866c2b..57bded2dd1f 100644 --- a/storage/mroonga/vendor/groonga/src/CMakeLists.txt +++ b/storage/mroonga/vendor/groonga/src/CMakeLists.txt @@ -15,6 +15,7 @@ include_directories( ${MRUBY_INCLUDE_DIRS} + ${MESSAGE_PACK_INCLUDE_DIRS} ) add_subdirectory(suggest) @@ -27,6 +28,19 @@ set_source_files_properties(${GROONGA_SOURCES} target_link_libraries(groonga libgroonga) install(TARGETS groonga DESTINATION ${BIN_DIR}) +if(GRN_WITH_MRUBY) + read_file_list(${CMAKE_CURRENT_SOURCE_DIR}/grndb_sources.am GRNDB_SOURCES) + add_executable(grndb ${GRNDB_SOURCES}) + set_source_files_properties(${GRNDB_SOURCES} + PROPERTIES + COMPILE_FLAGS "${GRN_C_COMPILE_FLAGS}") + set_source_files_properties(${GRNDB_SOURCES} + PROPERTIES + COMPILE_DEFINITIONS "${MRUBY_DEFINITIONS}") + target_link_libraries(grndb libgroonga) + install(TARGETS grndb DESTINATION ${BIN_DIR}) +endif() + if(NOT WIN32) read_file_list(${CMAKE_CURRENT_SOURCE_DIR}/grnslap_sources.am GRNSLAP_SOURCES) add_executable(grnslap ${GRNSLAP_SOURCES}) diff --git a/storage/mroonga/vendor/groonga/src/Makefile.am b/storage/mroonga/vendor/groonga/src/Makefile.am index a4d57e85c44..2ee687233da 100644 --- a/storage/mroonga/vendor/groonga/src/Makefile.am +++ b/storage/mroonga/vendor/groonga/src/Makefile.am @@ -20,6 +20,10 @@ AM_CFLAGS = \ $(GRN_CFLAGS) \ $(MESSAGE_PACK_CFLAGS) \ $(MRUBY_CFLAGS) + +AM_CPPFLAGS = \ + $(MRUBY_CPPFLAGS) + DEFS += $(GRN_DEFS) AM_LDFLAGS = -no-undefined diff --git a/storage/mroonga/vendor/groonga/src/grndb.c b/storage/mroonga/vendor/groonga/src/grndb.c index d5a353e229a..6733be93fb2 100644 --- a/storage/mroonga/vendor/groonga/src/grndb.c +++ b/storage/mroonga/vendor/groonga/src/grndb.c @@ -1,6 +1,6 @@ /* -*- c-basic-offset: 2 -*- */ /* - Copyright(C) 2014 Brazil + Copyright(C) 2014-2016 Brazil This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public @@ -20,8 +20,11 @@ # define GROONGA_MAIN #endif /* WIN32 */ +#include <stdio.h> + #include <grn_mrb.h> #include <grn_ctx_impl.h> +#include <grn_ctx_impl_mrb.h> #include <mruby/variable.h> #include <mruby/array.h> @@ -119,8 +122,58 @@ int main(int argc, char **argv) { int exit_code = EXIT_SUCCESS; + const char *log_path = GRN_LOG_PATH; + const char *log_level_name = NULL; + + { + int i; + for (i = 1; i < argc; i++) { + const char *arg = argv[i]; + + if (arg[0] != '-') { + continue; + } + + if (arg[1] == '-' && arg[2] == '\0') { + break; + } + +#define log_path_prefix "--log-path" +#define log_level_prefix "--log-level" + if (strcmp(arg, log_path_prefix) == 0) { + if (i + 1 < argc) { + log_path = argv[i + 1]; + i++; + } + } else if (strncmp(arg, + log_path_prefix "=", + strlen(log_path_prefix "=")) == 0) { + log_path = arg + strlen(log_path_prefix "="); + } else if (strcmp(arg, log_level_prefix) == 0) { + if (i + 1 < argc) { + log_level_name = argv[i + 1]; + i++; + } + } else if (strncmp(arg, + log_level_prefix "=", + strlen(log_level_prefix "=")) == 0) { + log_level_name = arg + strlen(log_level_prefix "="); + } +#undef log_path_equal_prefix +#undef log_level_equal_prefix + } + } - grn_default_logger_set_path(GRN_LOG_PATH); + grn_default_logger_set_path(log_path); + if (log_level_name) { + grn_log_level log_level = GRN_LOG_DEFAULT_LEVEL; + if (!grn_log_level_parse(log_level_name, &log_level)) { + fprintf(stderr, "%s: failed to parse log level: <%s>\n", + argv[0], log_level_name); + return EXIT_FAILURE; + } + grn_default_logger_set_max_level(log_level); + } if (grn_init() != GRN_SUCCESS) { return EXIT_FAILURE; @@ -129,7 +182,12 @@ main(int argc, char **argv) { grn_ctx ctx; grn_ctx_init(&ctx, 0); - exit_code = run(&ctx, argc, argv); + grn_ctx_impl_mrb_ensure_init(&ctx); + if (ctx.rc == GRN_SUCCESS) { + exit_code = run(&ctx, argc, argv); + } else { + exit_code = EXIT_FAILURE; + } grn_ctx_fin(&ctx); } diff --git a/storage/mroonga/vendor/groonga/src/groonga.c b/storage/mroonga/vendor/groonga/src/groonga.c index 28c318664e0..9742712170d 100644 --- a/storage/mroonga/vendor/groonga/src/groonga.c +++ b/storage/mroonga/vendor/groonga/src/groonga.c @@ -1,6 +1,6 @@ /* -*- c-basic-offset: 2 -*- */ /* - Copyright(C) 2009-2015 Brazil + Copyright(C) 2009-2017 Brazil This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public @@ -21,6 +21,7 @@ #include <ctype.h> #include <fcntl.h> #include <sys/stat.h> +#include <errno.h> #ifdef WIN32 # define GROONGA_MAIN @@ -32,6 +33,7 @@ #include <grn_proc.h> #include <grn_db.h> #include <grn_util.h> +#include <grn_error.h> #ifdef HAVE_SYS_WAIT_H # include <sys/wait.h> @@ -58,13 +60,6 @@ # include <sys/uio.h> #endif /* WIN32 */ -#ifdef HAVE__STRNICMP -# ifdef strncasecmp -# undef strncasecmp -# endif /* strcasecmp */ -# define strncasecmp(s1,s2,n) _strnicmp(s1,s2,n) -#endif /* HAVE__STRNICMP */ - #ifndef USE_MSG_NOSIGNAL # ifdef MSG_NOSIGNAL # undef MSG_NOSIGNAL @@ -85,7 +80,7 @@ #define DEFAULT_HTTP_PORT 10041 #define DEFAULT_GQTP_PORT 10043 #define DEFAULT_DEST "localhost" -#define DEFAULT_MAX_NFTHREADS 8 +#define DEFAULT_MAX_N_FLOATING_THREADS 8 #define MAX_CON 0x10000 #define RLIMIT_NOFILE_MINIMUM 4096 @@ -101,16 +96,19 @@ static int (*do_client)(int argc, char **argv); static int (*do_server)(char *path); static const char *pid_file_path = NULL; static const char *input_path = NULL; +static grn_file_reader *input_reader = NULL; static FILE *output = NULL; +static grn_bool is_memcached_mode = GRN_FALSE; +static const char *memcached_column_name = NULL; static int ready_notify_pipe[2]; #define PIPE_READ 0 #define PIPE_WRITE 1 static grn_encoding encoding; -static grn_command_version default_command_version; -static int64_t default_match_escalation_threshold; -static int log_level; +static const char *windows_event_source_name = "Groonga"; +static grn_bool use_windows_event_log = GRN_FALSE; +static grn_obj http_response_server_line; static int grn_rc_to_exit_code(grn_rc rc) @@ -122,6 +120,25 @@ grn_rc_to_exit_code(grn_rc rc) } } +static void +break_accept_event_loop(grn_ctx *ctx) +{ + grn_com *client; + const char *address; + + if (strcmp(bind_address, "0.0.0.0") == 0) { + address = "127.0.0.1"; + } else if (strcmp(bind_address, "::") == 0) { + address = "::1"; + } else { + address = bind_address; + } + client = grn_com_copen(ctx, NULL, address, port); + if (client) { + grn_com_close(ctx, client); + } +} + #ifdef GRN_WITH_LIBEDIT #include <locale.h> #include <histedit.h> @@ -224,10 +241,10 @@ read_next_line(grn_ctx *ctx, grn_obj *buf) #else fprintf(stderr, "> "); fflush(stderr); - rc = grn_text_fgets(ctx, buf, stdin); + rc = grn_file_reader_read_line(ctx, input_reader, buf); #endif } else { - rc = grn_text_fgets(ctx, buf, stdin); + rc = grn_file_reader_read_line(ctx, input_reader, buf); if (rc != GRN_END_OF_DATA) { number_of_lines++; } @@ -299,6 +316,11 @@ s_output_raw(grn_ctx *ctx, int flags, FILE *stream) if (flags & GRN_CTX_TAIL) { grn_obj *command; + if (grn_ctx_get_output_type(ctx) == GRN_CONTENT_GROONGA_COMMAND_LIST && + chunk_size > 0 && + chunk[chunk_size - 1] != '\n') { + fwrite("\n", 1, 1, stream); + } fflush(stream); command = GRN_CTX_USER_DATA(ctx)->ptr; @@ -476,7 +498,283 @@ static grn_com_queue ctx_new; static grn_com_queue ctx_old; static grn_mutex q_mutex; static grn_cond q_cond; -static uint32_t nthreads = 0, nfthreads = 0, max_nfthreads; +static uint32_t n_running_threads = 0; +static uint32_t n_floating_threads = 0; +static uint32_t max_n_floating_threads; + +static uint32_t +groonga_get_thread_limit(void *data) +{ + return max_n_floating_threads; +} + +static void +groonga_set_thread_limit(uint32_t new_limit, void *data) +{ + uint32_t i; + uint32_t current_n_floating_threads; + static uint32_t n_changing_threads = 0; + uint32_t prev_n_changing_threads; + + GRN_ATOMIC_ADD_EX(&n_changing_threads, 1, prev_n_changing_threads); + + MUTEX_LOCK_ENSURE(&grn_gctx, q_mutex); + current_n_floating_threads = n_floating_threads; + max_n_floating_threads = new_limit; + MUTEX_UNLOCK(q_mutex); + + if (prev_n_changing_threads > 0) { + GRN_ATOMIC_ADD_EX(&n_changing_threads, -1, prev_n_changing_threads); + return; + } + + if (current_n_floating_threads > new_limit) { + for (i = 0; i < current_n_floating_threads; i++) { + MUTEX_LOCK_ENSURE(&grn_gctx, q_mutex); + COND_SIGNAL(q_cond); + MUTEX_UNLOCK(q_mutex); + } + } + + while (GRN_TRUE) { + grn_bool is_reduced; + MUTEX_LOCK_ENSURE(&grn_gctx, q_mutex); + is_reduced = (n_running_threads <= max_n_floating_threads); + if (!is_reduced && n_floating_threads > 0) { + COND_SIGNAL(q_cond); + } + MUTEX_UNLOCK(q_mutex); + if (is_reduced) { + break; + } + grn_nanosleep(1000000); + } + + GRN_ATOMIC_ADD_EX(&n_changing_threads, -1, prev_n_changing_threads); +} + +typedef struct { + grn_mutex mutex; + grn_ctx ctx; + grn_pat *entries; + uint64_t earliest_unix_time_msec; +} request_timer_data; +static request_timer_data the_request_timer_data; + +static void * +request_timer_register(const char *request_id, + unsigned int request_id_size, + double timeout, + void *user_data) +{ + request_timer_data *data = user_data; + grn_id id = GRN_ID_NIL; + + { + grn_ctx *ctx = &(data->ctx); + grn_bool is_first_timer; + grn_timeval tv; + uint64_t timeout_unix_time_msec; + void *value; + + MUTEX_LOCK(data->mutex); + is_first_timer = (grn_pat_size(ctx, data->entries) == 0); + grn_timeval_now(ctx, &tv); + timeout_unix_time_msec = GRN_TIMEVAL_TO_MSEC(&tv) + (timeout * 1000); + while (GRN_TRUE) { + int added; + id = grn_pat_add(ctx, data->entries, + &timeout_unix_time_msec, sizeof(uint64_t), + &value, &added); + if (added != 0) { + break; + } + timeout_unix_time_msec++; + } + grn_memcpy(value, &request_id_size, sizeof(unsigned int)); + grn_memcpy(((uint8_t *)value) + sizeof(unsigned int), + request_id, request_id_size); + if (data->earliest_unix_time_msec == 0 || + data->earliest_unix_time_msec > timeout_unix_time_msec) { + data->earliest_unix_time_msec = timeout_unix_time_msec; + } + if (is_first_timer) { + break_accept_event_loop(ctx); + } + MUTEX_UNLOCK(data->mutex); + } + + return (void *)(uint64_t)id; +} + +static void +request_timer_unregister(void *timer_id, + void *user_data) +{ + request_timer_data *data = user_data; + grn_id id = (grn_id)(uint64_t)timer_id; + + { + grn_ctx *ctx = &(data->ctx); + uint64_t timeout_unix_time_msec; + int key_size; + + MUTEX_LOCK(data->mutex); + key_size = grn_pat_get_key(ctx, + data->entries, + id, + &timeout_unix_time_msec, + sizeof(uint64_t)); + if (key_size > 0) { + grn_pat_delete_by_id(ctx, data->entries, id, NULL); + if (data->earliest_unix_time_msec >= timeout_unix_time_msec) { + data->earliest_unix_time_msec = 0; + } + } + MUTEX_UNLOCK(data->mutex); + } +} + +static void +request_timer_fin(void *user_data) +{ + request_timer_data *data = user_data; + + { + grn_ctx *ctx = &(data->ctx); + grn_pat_close(ctx, data->entries); + grn_ctx_fin(ctx); + MUTEX_FIN(data->mutex); + } +} + +static void +request_timer_init(void) +{ + static grn_request_timer timer; + request_timer_data *data = &the_request_timer_data; + grn_ctx *ctx; + + MUTEX_INIT(data->mutex); + ctx = &(data->ctx); + grn_ctx_init(ctx, 0); + data->entries = grn_pat_create(ctx, + NULL, + sizeof(uint64_t), + GRN_TABLE_MAX_KEY_SIZE, + GRN_OBJ_KEY_UINT); + data->earliest_unix_time_msec = 0; + + timer.user_data = data; + timer.register_func = request_timer_register; + timer.unregister_func = request_timer_unregister; + timer.fin_func = request_timer_fin; + + grn_request_timer_set(&timer); +} + +static grn_bool +request_timer_ensure_earliest_unix_time_msec(void) +{ + request_timer_data *data = &the_request_timer_data; + grn_ctx *ctx; + grn_pat_cursor *cursor; + + if (data->earliest_unix_time_msec > 0) { + return GRN_TRUE; + } + + ctx = &(data->ctx); + cursor = grn_pat_cursor_open(ctx, data->entries, + NULL, 0, + NULL, 0, + 0, 1, GRN_CURSOR_ASCENDING); + if (!cursor) { + return GRN_FALSE; + } + while (grn_pat_cursor_next(ctx, cursor) != GRN_ID_NIL) { + void *key; + uint64_t timeout_unix_time_msec; + + grn_pat_cursor_get_key(ctx, cursor, &key); + timeout_unix_time_msec = *(uint64_t *)key; + data->earliest_unix_time_msec = timeout_unix_time_msec; + break; + } + grn_pat_cursor_close(ctx, cursor); + + return data->earliest_unix_time_msec > 0; +} + +static int +request_timer_get_poll_timeout(void) +{ + request_timer_data *data = &the_request_timer_data; + int timeout = 1000; + grn_ctx *ctx; + grn_timeval tv; + + MUTEX_LOCK(data->mutex); + ctx = &(data->ctx); + if (grn_pat_size(ctx, data->entries) == 0) { + goto exit; + } + + if (!request_timer_ensure_earliest_unix_time_msec()) { + goto exit; + } + + grn_timeval_now(ctx, &tv); + timeout = data->earliest_unix_time_msec - GRN_TIMEVAL_TO_MSEC(&tv); + if (timeout < 0) { + timeout = 0; + } else if (timeout > 1000) { + timeout = 1000; + } + +exit : + MUTEX_UNLOCK(data->mutex); + + return timeout; +} + +static void +request_timer_process_timeout(void) +{ + request_timer_data *data = &the_request_timer_data; + grn_ctx *ctx; + grn_timeval tv; + uint64_t max; + grn_pat_cursor *cursor; + + ctx = &(data->ctx); + if (grn_pat_size(ctx, data->entries) == 0) { + return; + } + + grn_timeval_now(ctx, &tv); + max = GRN_TIMEVAL_TO_MSEC(&tv); + cursor = grn_pat_cursor_open(ctx, data->entries, + NULL, 0, + &max, sizeof(uint64_t), + 0, -1, GRN_CURSOR_ASCENDING); + if (!cursor) { + return; + } + + grn_id id; + while ((id = grn_pat_cursor_next(ctx, cursor)) != GRN_ID_NIL) { + void *value; + const char *request_id; + unsigned int request_id_size; + + grn_pat_cursor_get_value(ctx, cursor, &value); + request_id_size = *((unsigned int *)value); + request_id = (const char *)(((uint8_t *)value) + sizeof(unsigned int)); + grn_request_canceler_cancel(request_id, request_id_size); + } + grn_pat_cursor_close(ctx, cursor); +} static void reset_ready_notify_pipe(void) @@ -512,29 +810,40 @@ send_ready_notify(void) static void create_pid_file(void) { -#ifndef WIN32 FILE *pid_file = NULL; - pid_t pid; if (!pid_file_path) { return; } pid_file = fopen(pid_file_path, "w"); - pid = getpid(); - fprintf(pid_file, "%d\n", pid); + if (!pid_file) { + fprintf(stderr, + "Failed to open PID file: <%s>: <%s>\n", + pid_file_path, grn_strerror(errno)); + return; + } + + { +#ifdef WIN32 + DWORD pid; + pid = GetCurrentProcessId(); + fprintf(pid_file, "%" GRN_FMT_DWORD "\n", pid); +#else /* WIN32 */ + pid_t pid; + pid = grn_getpid(); + fprintf(pid_file, "%d\n", pid); +#endif /* WIN32 */ + } fclose(pid_file); -#endif } static void clean_pid_file(void) { -#ifndef WIN32 if (pid_file_path) { - unlink(pid_file_path); + grn_unlink(pid_file_path); } -#endif } static int @@ -572,7 +881,7 @@ daemonize(void) create_pid_file(); } else { pid_t pid; - pid = getpid(); + pid = grn_getpid(); fprintf(stderr, "%d\n", pid); } break; @@ -600,7 +909,9 @@ daemonize(void) static void run_server_loop(grn_ctx *ctx, grn_com_event *ev) { - while (!grn_com_event_poll(ctx, ev, 1000) && grn_gctx.stat != GRN_CTX_QUIT) { + request_timer_init(); + while (!grn_com_event_poll(ctx, ev, request_timer_get_poll_timeout()) && + grn_gctx.stat != GRN_CTX_QUIT) { grn_edge *edge; while ((edge = (grn_edge *)grn_com_queue_deque(ctx, &ctx_old))) { grn_obj *msg; @@ -616,11 +927,12 @@ run_server_loop(grn_ctx *ctx, grn_com_event *ev) } grn_edges_delete(ctx, edge); } + request_timer_process_timeout(); /* todo : log stat */ } for (;;) { - MUTEX_LOCK(q_mutex); - if (nthreads == nfthreads) { break; } + MUTEX_LOCK_ENSURE(ctx, q_mutex); + if (n_running_threads == n_floating_threads) { break; } MUTEX_UNLOCK(q_mutex); grn_nanosleep(1000000); } @@ -673,6 +985,8 @@ run_server(grn_ctx *ctx, grn_obj *db, grn_com_event *ev, return exit_code; } +static grn_bool memcached_init(grn_ctx *ctx); + static int start_service(grn_ctx *ctx, const char *db_path, grn_edge_dispatcher_func dispatcher, grn_handler_func handler) @@ -693,10 +1007,20 @@ start_service(grn_ctx *ctx, const char *db_path, grn_obj *db; db = (newdb || !db_path) ? grn_db_create(ctx, db_path, NULL) : grn_db_open(ctx, db_path); if (db) { - exit_code = run_server(ctx, db, &ev, dispatcher, handler); + if (is_memcached_mode) { + if (!memcached_init(ctx)) { + fprintf(stderr, "failed to initialize memcached mode: %s\n", + ctx->errbuf); + exit_code = EXIT_FAILURE; + send_ready_notify(); + } + } + if (exit_code == EXIT_SUCCESS) { + exit_code = run_server(ctx, db, &ev, dispatcher, handler); + } grn_obj_close(ctx, db); } else { - fprintf(stderr, "db open failed (%s)\n", db_path); + fprintf(stderr, "db open failed (%s): %s\n", db_path, ctx->errbuf); exit_code = EXIT_FAILURE; send_ready_notify(); } @@ -719,26 +1043,43 @@ typedef struct { } ht_context; static void -h_output_set_header(grn_ctx *ctx, grn_obj *header, - grn_rc rc, long long int content_length) +h_output_set_header(grn_ctx *ctx, + grn_obj *header, + grn_rc rc, + long long int content_length, + grn_obj *foot) { switch (rc) { case GRN_SUCCESS : GRN_TEXT_SETS(ctx, header, "HTTP/1.1 200 OK\r\n"); break; case GRN_INVALID_ARGUMENT : + case GRN_FUNCTION_NOT_IMPLEMENTED : case GRN_SYNTAX_ERROR : GRN_TEXT_SETS(ctx, header, "HTTP/1.1 400 Bad Request\r\n"); break; case GRN_NO_SUCH_FILE_OR_DIRECTORY : GRN_TEXT_SETS(ctx, header, "HTTP/1.1 404 Not Found\r\n"); break; + case GRN_CANCEL : + GRN_TEXT_SETS(ctx, header, "HTTP/1.1 408 Request Timeout\r\n"); + break; default : GRN_TEXT_SETS(ctx, header, "HTTP/1.1 500 Internal Server Error\r\n"); break; } + GRN_TEXT_PUT(ctx, header, + GRN_TEXT_VALUE(&http_response_server_line), + GRN_TEXT_LEN(&http_response_server_line)); GRN_TEXT_PUTS(ctx, header, "Content-Type: "); - GRN_TEXT_PUTS(ctx, header, grn_ctx_get_mime_type(ctx)); + if (grn_ctx_get_output_type(ctx) == GRN_CONTENT_JSON && + foot && + GRN_TEXT_LEN(foot) > 0 && + GRN_TEXT_VALUE(foot)[GRN_TEXT_LEN(foot) - 1] == ';') { + GRN_TEXT_PUTS(ctx, header, "application/javascript"); + } else { + GRN_TEXT_PUTS(ctx, header, grn_ctx_get_mime_type(ctx)); + } GRN_TEXT_PUTS(ctx, header, "\r\n"); if (content_length >= 0) { GRN_TEXT_PUTS(ctx, header, "Connection: close\r\n"); @@ -865,10 +1206,10 @@ h_output_raw(grn_ctx *ctx, int flags, ht_context *hc) if (!hc->in_body) { if (is_last_message) { - h_output_set_header(ctx, &header_, expr_rc, GRN_TEXT_LEN(&body_)); + h_output_set_header(ctx, &header_, expr_rc, GRN_TEXT_LEN(&body_), NULL); hc->is_chunked = GRN_FALSE; } else { - h_output_set_header(ctx, &header_, expr_rc, -1); + h_output_set_header(ctx, &header_, expr_rc, -1, NULL); hc->is_chunked = GRN_TRUE; } header = &header_; @@ -938,7 +1279,8 @@ h_output_typed(grn_ctx *ctx, int flags, ht_context *hc) h_output_set_header(ctx, &header, expr_rc, GRN_TEXT_LEN(&head) + GRN_TEXT_LEN(&body) + - GRN_TEXT_LEN(&foot)); + GRN_TEXT_LEN(&foot), + &foot); if (should_return_body) { h_output_send(ctx, fd, &header, &head, &body, &foot); } else { @@ -967,8 +1309,9 @@ h_output(grn_ctx *ctx, int flags, void *arg) } static void -do_htreq_get(grn_ctx *ctx, grn_msg *msg) +do_htreq_get(grn_ctx *ctx, ht_context *hc) { + grn_msg *msg = hc->msg; char *path = NULL; char *pathe = GRN_BULK_HEAD((grn_obj *)msg); char *e = GRN_BULK_CURR((grn_obj *)msg); @@ -987,7 +1330,7 @@ do_htreq_get(grn_ctx *ctx, grn_msg *msg) } } } - grn_ctx_send(ctx, path, pathe - path, 0); + grn_ctx_send(ctx, path, pathe - path, GRN_CTX_TAIL); } typedef struct { @@ -1004,7 +1347,7 @@ typedef struct { #define STRING_EQUAL_CI(string, string_length, constant_string)\ (string_length == strlen(constant_string) &&\ - strncasecmp(string, constant_string, string_length) == 0) + grn_strncasecmp(string, constant_string, string_length) == 0) static const char * do_htreq_post_parse_header_request_line(grn_ctx *ctx, @@ -1169,8 +1512,9 @@ do_htreq_post_parse_header(grn_ctx *ctx, } static void -do_htreq_post(grn_ctx *ctx, grn_msg *msg) +do_htreq_post(grn_ctx *ctx, ht_context *hc) { + grn_msg *msg = hc->msg; grn_sock fd = msg->u.fd; const char *end; h_post_header header; @@ -1189,7 +1533,7 @@ do_htreq_post(grn_ctx *ctx, grn_msg *msg) return; } - grn_ctx_send(ctx, header.path_start, header.path_length, GRN_CTX_QUIET); + grn_ctx_send(ctx, header.path_start, header.path_length, GRN_CTX_MORE); if (ctx->rc != GRN_SUCCESS) { ht_context context; context.msg = msg; @@ -1264,7 +1608,9 @@ do_htreq_post(grn_ctx *ctx, grn_msg *msg) int flags = 0; if (!(read_content_length == header.content_length && buffer_current + 1 == buffer_end)) { - flags |= GRN_CTX_QUIET; + flags |= GRN_CTX_MORE; + } else { + flags |= GRN_CTX_TAIL; } grn_ctx_send(ctx, GRN_TEXT_VALUE(&chunk_buffer), @@ -1280,13 +1626,19 @@ do_htreq_post(grn_ctx *ctx, grn_msg *msg) buffer_start, buffer_end - buffer_start); } #undef POST_BUFFER_SIZE + + if (ctx->rc != GRN_SUCCESS) { + break; + } } - if (GRN_TEXT_LEN(&chunk_buffer) > 0) { + if (ctx->rc == GRN_CANCEL) { + h_output(ctx, GRN_CTX_TAIL, hc); + } else if (ctx->rc == GRN_SUCCESS && GRN_TEXT_LEN(&chunk_buffer) > 0) { grn_ctx_send(ctx, GRN_TEXT_VALUE(&chunk_buffer), GRN_TEXT_LEN(&chunk_buffer), - 0); + GRN_CTX_TAIL); } GRN_OBJ_FIN(ctx, &chunk_buffer); @@ -1294,19 +1646,19 @@ do_htreq_post(grn_ctx *ctx, grn_msg *msg) } static void -do_htreq(grn_ctx *ctx, grn_msg *msg) +do_htreq(grn_ctx *ctx, ht_context *hc) { + grn_msg *msg = hc->msg; grn_com_header *header = &msg->header; switch (header->qtype) { case 'G' : /* GET */ case 'H' : /* HEAD */ - do_htreq_get(ctx, msg); + do_htreq_get(ctx, hc); break; case 'P' : /* POST */ - do_htreq_post(ctx, msg); + do_htreq_post(ctx, hc); break; } - grn_ctx_set_next_expr(ctx, NULL); /* if (ctx->rc != GRN_OPERATION_WOULD_BLOCK) {...} */ grn_msg_close(ctx, (grn_obj *)msg); /* if not keep alive connection */ @@ -1355,7 +1707,6 @@ enum { MBCMD_PREPENDQ = 0x1a }; -static grn_critical_section cache_lock; static grn_obj *cache_table = NULL; static grn_obj *cache_value = NULL; static grn_obj *cache_flags = NULL; @@ -1364,39 +1715,189 @@ static grn_obj *cache_cas = NULL; #define CTX_GET(name) (grn_ctx_get(ctx, (name), strlen(name))) -static grn_obj * -cache_init(grn_ctx *ctx) +static grn_bool +memcached_setup_flags_column(grn_ctx *ctx, const char *name) +{ + cache_flags = grn_obj_column(ctx, cache_table, name, strlen(name)); + if (cache_flags) { + return GRN_TRUE; + } + + cache_flags = grn_column_create(ctx, cache_table, name, strlen(name), NULL, + GRN_OBJ_COLUMN_SCALAR|GRN_OBJ_PERSISTENT, + grn_ctx_at(ctx, GRN_DB_UINT32)); + if (!cache_flags) { + return GRN_FALSE; + } + + return GRN_TRUE; +} + +static grn_bool +memcached_setup_expire_column(grn_ctx *ctx, const char *name) +{ + cache_expire = grn_obj_column(ctx, cache_table, name, strlen(name)); + if (cache_expire) { + return GRN_TRUE; + } + + cache_expire = grn_column_create(ctx, cache_table, name, strlen(name), NULL, + GRN_OBJ_COLUMN_SCALAR|GRN_OBJ_PERSISTENT, + grn_ctx_at(ctx, GRN_DB_UINT32)); + if (!cache_expire) { + return GRN_FALSE; + } + + return GRN_TRUE; +} + +static grn_bool +memcached_setup_cas_column(grn_ctx *ctx, const char *name) { - if (cache_cas) { return cache_cas; } - CRITICAL_SECTION_ENTER(cache_lock); + cache_cas = grn_obj_column(ctx, cache_table, name, strlen(name)); + if (cache_cas) { + return GRN_TRUE; + } + + cache_cas = grn_column_create(ctx, cache_table, name, strlen(name), NULL, + GRN_OBJ_COLUMN_SCALAR|GRN_OBJ_PERSISTENT, + grn_ctx_at(ctx, GRN_DB_UINT64)); if (!cache_cas) { - if ((cache_table = CTX_GET("Memcache"))) { - cache_value = CTX_GET("Memcache.value"); - cache_flags = CTX_GET("Memcache.flags"); - cache_expire = CTX_GET("Memcache.expire"); - cache_cas = CTX_GET("Memcache.cas"); - } else { + return GRN_FALSE; + } + + return GRN_TRUE; +} + +static grn_bool +memcached_init(grn_ctx *ctx) +{ + if (memcached_column_name) { + cache_value = CTX_GET(memcached_column_name); + if (!cache_value) { + ERR(GRN_INVALID_ARGUMENT, + "memcached column doesn't exist: <%s>", + memcached_column_name); + return GRN_FALSE; + } + if (!(grn_obj_is_column(ctx, cache_value) && + ((cache_value->header.flags & GRN_OBJ_COLUMN_TYPE_MASK) == + GRN_OBJ_COLUMN_SCALAR))) { + grn_obj inspected; + GRN_TEXT_INIT(&inspected, 0); + grn_inspect(ctx, &inspected, cache_value); + ERR(GRN_INVALID_ARGUMENT, + "memcached column must be scalar column: <%.*s>", + (int)GRN_TEXT_LEN(&inspected), + GRN_TEXT_VALUE(&inspected)); + GRN_OBJ_FIN(ctx, &inspected); + return GRN_FALSE; + } + if (!(GRN_DB_SHORT_TEXT <= grn_obj_get_range(ctx, cache_value) && + grn_obj_get_range(ctx, cache_value) <= GRN_DB_LONG_TEXT)) { + grn_obj inspected; + GRN_TEXT_INIT(&inspected, 0); + grn_inspect(ctx, &inspected, cache_value); + ERR(GRN_INVALID_ARGUMENT, + "memcached column must be text column: <%.*s>", + (int)GRN_TEXT_LEN(&inspected), + GRN_TEXT_VALUE(&inspected)); + GRN_OBJ_FIN(ctx, &inspected); + return GRN_FALSE; + } + + cache_table = grn_ctx_at(ctx, cache_value->header.domain); + if (cache_table->header.type == GRN_TABLE_NO_KEY) { + grn_obj inspected; + GRN_TEXT_INIT(&inspected, 0); + grn_inspect(ctx, &inspected, cache_table); + ERR(GRN_INVALID_ARGUMENT, + "memcached column's table must be HASH_KEY, PAT_KEY or DAT_KEY table: " + "<%.*s>", + (int)GRN_TEXT_LEN(&inspected), + GRN_TEXT_VALUE(&inspected)); + GRN_OBJ_FIN(ctx, &inspected); + return GRN_FALSE; + } + + { + char column_name[GRN_TABLE_MAX_KEY_SIZE]; + char value_column_name[GRN_TABLE_MAX_KEY_SIZE]; + int value_column_name_size; + + value_column_name_size = grn_column_name(ctx, cache_value, + value_column_name, + GRN_TABLE_MAX_KEY_SIZE); + grn_snprintf(column_name, + GRN_TABLE_MAX_KEY_SIZE, + GRN_TABLE_MAX_KEY_SIZE, + "%.*s_memcached_flags", + value_column_name_size, + value_column_name); + if (!memcached_setup_flags_column(ctx, column_name)) { + return GRN_FALSE; + } + grn_snprintf(column_name, + GRN_TABLE_MAX_KEY_SIZE, + GRN_TABLE_MAX_KEY_SIZE, + "%.*s_memcached_expire", + value_column_name_size, + value_column_name); + if (!memcached_setup_expire_column(ctx, column_name)) { + return GRN_FALSE; + } + grn_snprintf(column_name, + GRN_TABLE_MAX_KEY_SIZE, + GRN_TABLE_MAX_KEY_SIZE, + "%.*s_memcached_cas", + value_column_name_size, + value_column_name); + if (!memcached_setup_cas_column(ctx, column_name)) { + return GRN_FALSE; + } + } + } else { + const char *table_name = "Memcache"; + const char *value_column_name = "value"; + + cache_table = CTX_GET(table_name); + if (!cache_table) { + cache_table = grn_table_create(ctx, table_name, strlen(table_name), NULL, + GRN_OBJ_TABLE_PAT_KEY|GRN_OBJ_PERSISTENT, + grn_ctx_at(ctx, GRN_DB_SHORT_TEXT), + NULL); if (!cache_table) { - grn_obj *uint32_type = grn_ctx_at(ctx, GRN_DB_UINT32); - grn_obj *uint64_type = grn_ctx_at(ctx, GRN_DB_UINT64); - grn_obj *shorttext_type = grn_ctx_at(ctx, GRN_DB_SHORT_TEXT); - if ((cache_table = grn_table_create(ctx, "Memcache", 8, NULL, - GRN_OBJ_TABLE_PAT_KEY|GRN_OBJ_PERSISTENT, - shorttext_type, NULL))) { - cache_value = grn_column_create(ctx, cache_table, "value", 5, NULL, - GRN_OBJ_PERSISTENT, shorttext_type); - cache_flags = grn_column_create(ctx, cache_table, "flags", 5, NULL, - GRN_OBJ_PERSISTENT, uint32_type); - cache_expire = grn_column_create(ctx, cache_table, "expire", 6, NULL, - GRN_OBJ_PERSISTENT, uint32_type); - cache_cas = grn_column_create(ctx, cache_table, "cas", 3, NULL, - GRN_OBJ_PERSISTENT, uint64_type); - } + return GRN_FALSE; } } + + cache_value = grn_obj_column(ctx, cache_table, + value_column_name, + strlen(value_column_name)); + if (!cache_value) { + cache_value = grn_column_create(ctx, cache_table, + value_column_name, + strlen(value_column_name), + NULL, + GRN_OBJ_COLUMN_SCALAR|GRN_OBJ_PERSISTENT, + grn_ctx_at(ctx, GRN_DB_SHORT_TEXT)); + if (!cache_value) { + return GRN_FALSE; + } + } + + if (!memcached_setup_flags_column(ctx, "flags")) { + return GRN_FALSE; + } + if (!memcached_setup_expire_column(ctx, "expire")) { + return GRN_FALSE; + } + if (!memcached_setup_cas_column(ctx, "cas")) { + return GRN_FALSE; + } } - CRITICAL_SECTION_LEAVE(cache_lock); - return cache_cas; + + return GRN_TRUE; } #define RELATIVE_TIME_THRESH 1000000000 @@ -1439,7 +1940,6 @@ do_mbreq(grn_ctx *ctx, grn_edge *edge) grn_id rid; uint16_t keylen = ntohs(header->keylen); char *key = GRN_BULK_HEAD((grn_obj *)msg); - cache_init(ctx); rid = grn_table_get(ctx, cache_table, key, keylen); if (!rid) { GRN_MSG_MBRES({ @@ -1499,7 +1999,6 @@ do_mbreq(grn_ctx *ctx, grn_edge *edge) int f = (header->qtype == MBCMD_REPLACE || header->qtype == MBCMD_REPLACEQ) ? 0 : GRN_TABLE_ADD; GRN_ASSERT(extralen == 8); - cache_init(ctx); if (header->qtype == MBCMD_REPLACE || header->qtype == MBCMD_REPLACEQ) { rid = grn_table_get(ctx, cache_table, key, keylen); } else { @@ -1623,7 +2122,6 @@ do_mbreq(grn_ctx *ctx, grn_edge *edge) grn_id rid; uint16_t keylen = ntohs(header->keylen); char *key = GRN_BULK_HEAD((grn_obj *)msg); - cache_init(ctx); rid = grn_table_get(ctx, cache_table, key, keylen); if (!rid) { /* GRN_LOG(ctx, GRN_LOG_NOTICE, "GET k=%d not found", keylen); */ @@ -1655,7 +2153,6 @@ do_mbreq(grn_ctx *ctx, grn_edge *edge) grn_ntoh(&delta, body, 8); grn_ntoh(&init, body + 8, 8); GRN_ASSERT(header->level == 20); /* extralen */ - cache_init(ctx); if (expire == 0xffffffff) { rid = grn_table_get(ctx, cache_table, key, keylen); } else { @@ -1777,7 +2274,6 @@ do_mbreq(grn_ctx *ctx, grn_edge *edge) grn_id rid; uint16_t keylen = ntohs(header->keylen); char *key = GRN_BULK_HEAD((grn_obj *)msg); - cache_init(ctx); rid = grn_table_get(ctx, cache_table, key, keylen); if (!rid) { GRN_MSG_MBRES({ @@ -1824,7 +2320,6 @@ do_mbreq(grn_ctx *ctx, grn_edge *edge) char *key = GRN_BULK_HEAD((grn_obj *)msg); char *value = key + keylen; uint32_t valuelen = size - keylen; - cache_init(ctx); rid = grn_table_add(ctx, cache_table, key, keylen, NULL); if (!rid) { GRN_MSG_MBRES({ @@ -1845,7 +2340,7 @@ do_mbreq(grn_ctx *ctx, grn_edge *edge) break; case MBCMD_STAT : { - pid_t pid = getpid(); + pid_t pid = grn_getpid(); GRN_MSG_MBRES({ grn_bulk_write(ctx, re, "pid", 3); grn_text_itoa(ctx, re, pid); @@ -1906,31 +2401,44 @@ h_worker(void *arg) grn_ctx_init(ctx, 0); grn_ctx_use(ctx, (grn_obj *)arg); grn_ctx_recv_handler_set(ctx, h_output, &hc); - GRN_LOG(&grn_gctx, GRN_LOG_NOTICE, "thread start (%d/%d)", nfthreads, nthreads + 1); - MUTEX_LOCK(q_mutex); - do { + MUTEX_LOCK_ENSURE(ctx, q_mutex); + GRN_LOG(&grn_gctx, GRN_LOG_NOTICE, "thread start (%d/%d)", + n_floating_threads, n_running_threads); + while (n_running_threads <= max_n_floating_threads && + grn_gctx.stat != GRN_CTX_QUIT) { grn_obj *msg; - nfthreads++; + if (ctx->rc == GRN_CANCEL) { + ctx->rc = GRN_SUCCESS; + } + n_floating_threads++; while (!(msg = (grn_obj *)grn_com_queue_deque(&grn_gctx, &ctx_new))) { COND_WAIT(q_cond, q_mutex); if (grn_gctx.stat == GRN_CTX_QUIT) { - nfthreads--; + n_floating_threads--; + goto exit; + } + if (n_running_threads > max_n_floating_threads) { + n_floating_threads--; goto exit; } } - nfthreads--; + n_floating_threads--; MUTEX_UNLOCK(q_mutex); hc.msg = (grn_msg *)msg; hc.in_body = GRN_FALSE; hc.is_chunked = GRN_FALSE; - do_htreq(ctx, (grn_msg *)msg); - MUTEX_LOCK(q_mutex); - } while (nfthreads < max_nfthreads && grn_gctx.stat != GRN_CTX_QUIT); + do_htreq(ctx, &hc); + MUTEX_LOCK_ENSURE(ctx, q_mutex); + } exit : - nthreads--; - MUTEX_UNLOCK(q_mutex); - GRN_LOG(&grn_gctx, GRN_LOG_NOTICE, "thread end (%d/%d)", nfthreads, nthreads); + n_running_threads--; + GRN_LOG(&grn_gctx, GRN_LOG_NOTICE, "thread end (%d/%d)", + n_floating_threads, n_running_threads); + if (grn_gctx.stat == GRN_CTX_QUIT) { + break_accept_event_loop(ctx); + } grn_ctx_fin(ctx); + MUTEX_UNLOCK(q_mutex); return GRN_THREAD_FUNC_RETURN_VALUE; } @@ -1947,12 +2455,15 @@ h_handler(grn_ctx *ctx, grn_obj *msg) /* if not keep alive connection */ grn_com_event_del(ctx, com->ev, fd); ((grn_msg *)msg)->u.fd = fd; - MUTEX_LOCK(q_mutex); + MUTEX_LOCK_ENSURE(ctx, q_mutex); grn_com_queue_enque(ctx, &ctx_new, (grn_com_queue_entry *)msg); - if (!nfthreads && nthreads < max_nfthreads) { + if (n_floating_threads == 0 && n_running_threads < max_n_floating_threads) { grn_thread thread; - nthreads++; - if (THREAD_CREATE(thread, h_worker, arg)) { SERR("pthread_create"); } + n_running_threads++; + if (THREAD_CREATE(thread, h_worker, arg)) { + n_running_threads--; + SERR("pthread_create"); + } } COND_SIGNAL(q_cond); MUTEX_UNLOCK(q_mutex); @@ -1965,13 +2476,17 @@ h_server(char *path) int exit_code = EXIT_FAILURE; grn_ctx ctx_, *ctx = &ctx_; grn_ctx_init(ctx, 0); - MUTEX_INIT(q_mutex); - COND_INIT(q_cond); - CRITICAL_SECTION_INIT(cache_lock); GRN_COM_QUEUE_INIT(&ctx_new); GRN_COM_QUEUE_INIT(&ctx_old); check_rlimit_nofile(ctx); + GRN_TEXT_INIT(&http_response_server_line, 0); + grn_text_printf(ctx, + &http_response_server_line, + "Server: %s/%s\r\n", + grn_get_package_label(), + grn_get_version()); exit_code = start_service(ctx, path, NULL, h_handler); + GRN_OBJ_FIN(ctx, &http_response_server_line); grn_ctx_fin(ctx); return exit_code; } @@ -1979,21 +2494,27 @@ h_server(char *path) static grn_thread_func_result CALLBACK g_worker(void *arg) { - GRN_LOG(&grn_gctx, GRN_LOG_NOTICE, "thread start (%d/%d)", nfthreads, nthreads + 1); - MUTEX_LOCK(q_mutex); - do { + MUTEX_LOCK_ENSURE(NULL, q_mutex); + GRN_LOG(&grn_gctx, GRN_LOG_NOTICE, "thread start (%d/%d)", + n_floating_threads, n_running_threads); + while (n_running_threads <= max_n_floating_threads && + grn_gctx.stat != GRN_CTX_QUIT) { grn_ctx *ctx; grn_edge *edge; - nfthreads++; + n_floating_threads++; while (!(edge = (grn_edge *)grn_com_queue_deque(&grn_gctx, &ctx_new))) { COND_WAIT(q_cond, q_mutex); if (grn_gctx.stat == GRN_CTX_QUIT) { - nfthreads--; + n_floating_threads--; + goto exit; + } + if (n_running_threads > max_n_floating_threads) { + n_floating_threads--; goto exit; } } ctx = &edge->ctx; - nfthreads--; + n_floating_threads--; if (edge->stat == EDGE_DOING) { continue; } if (edge->stat == EDGE_WAIT) { edge->stat = EDGE_DOING; @@ -2012,6 +2533,9 @@ g_worker(void *arg) case GRN_COM_PROTO_GQTP : grn_ctx_send(ctx, GRN_BULK_HEAD(msg), GRN_BULK_VSIZE(msg), header->flags); ERRCLR(ctx); + if (ctx->rc == GRN_CANCEL) { + ctx->rc = GRN_SUCCESS; + } break; default : ctx->stat = GRN_CTX_QUIT; @@ -2022,7 +2546,7 @@ g_worker(void *arg) while ((msg = (grn_obj *)grn_com_queue_deque(ctx, &edge->send_old))) { grn_msg_close(ctx, msg); } - MUTEX_LOCK(q_mutex); + MUTEX_LOCK_ENSURE(ctx, q_mutex); if (ctx->stat == GRN_CTX_QUIT || edge->stat == EDGE_ABORT) { break; } } } @@ -2032,25 +2556,29 @@ g_worker(void *arg) } else { edge->stat = EDGE_IDLE; } - } while (nfthreads < max_nfthreads && grn_gctx.stat != GRN_CTX_QUIT); + }; exit : - nthreads--; + n_running_threads--; + GRN_LOG(&grn_gctx, GRN_LOG_NOTICE, "thread end (%d/%d)", + n_floating_threads, n_running_threads); MUTEX_UNLOCK(q_mutex); - GRN_LOG(&grn_gctx, GRN_LOG_NOTICE, "thread end (%d/%d)", nfthreads, nthreads); return GRN_THREAD_FUNC_RETURN_VALUE; } static void g_dispatcher(grn_ctx *ctx, grn_edge *edge) { - MUTEX_LOCK(q_mutex); + MUTEX_LOCK_ENSURE(ctx, q_mutex); if (edge->stat == EDGE_IDLE) { grn_com_queue_enque(ctx, &ctx_new, (grn_com_queue_entry *)edge); edge->stat = EDGE_WAIT; - if (!nfthreads && nthreads < max_nfthreads) { + if (n_floating_threads == 0 && n_running_threads < max_n_floating_threads) { grn_thread thread; - nthreads++; - if (THREAD_CREATE(thread, g_worker, NULL)) { SERR("pthread_create"); } + n_running_threads++; + if (THREAD_CREATE(thread, g_worker, NULL)) { + n_running_threads--; + SERR("pthread_create"); + } } COND_SIGNAL(q_cond); } @@ -2062,18 +2590,18 @@ g_output(grn_ctx *ctx, int flags, void *arg) { grn_edge *edge = arg; grn_com *com = edge->com; - grn_msg *req = edge->msg, *msg = (grn_msg *)ctx->impl->outbuf; + grn_msg *req = edge->msg, *msg = (grn_msg *)ctx->impl->output.buf; msg->edge_id = req->edge_id; msg->header.proto = req->header.proto == GRN_COM_PROTO_MBREQ ? GRN_COM_PROTO_MBRES : req->header.proto; - if (ctx->rc != GRN_SUCCESS && GRN_BULK_VSIZE(ctx->impl->outbuf) == 0) { - GRN_TEXT_PUTS(ctx, ctx->impl->outbuf, ctx->errbuf); + if (ctx->rc != GRN_SUCCESS && GRN_BULK_VSIZE(ctx->impl->output.buf) == 0) { + GRN_TEXT_PUTS(ctx, ctx->impl->output.buf, ctx->errbuf); } if (grn_msg_send(ctx, (grn_obj *)msg, (flags & GRN_CTX_MORE) ? GRN_CTX_MORE : GRN_CTX_TAIL)) { edge->stat = EDGE_ABORT; } - ctx->impl->outbuf = grn_msg_open(ctx, com, &edge->send_old); + ctx->impl->output.buf = grn_msg_open(ctx, com, &edge->send_old); } static void @@ -2084,7 +2612,7 @@ g_handler(grn_ctx *ctx, grn_obj *msg) if (ctx->rc) { if (com->has_sid) { if ((edge = com->opaque)) { - MUTEX_LOCK(q_mutex); + MUTEX_LOCK_ENSURE(ctx, q_mutex); if (edge->stat == EDGE_IDLE) { grn_com_queue_enque(ctx, &ctx_old, (grn_com_queue_entry *)edge); } @@ -2105,8 +2633,9 @@ g_handler(grn_ctx *ctx, grn_obj *msg) grn_ctx_use(&edge->ctx, (grn_obj *)com->ev->opaque); grn_ctx_recv_handler_set(&edge->ctx, g_output, edge); com->opaque = edge; - grn_obj_close(&edge->ctx, edge->ctx.impl->outbuf); - edge->ctx.impl->outbuf = grn_msg_open(&edge->ctx, com, &edge->send_old); + grn_obj_close(&edge->ctx, edge->ctx.impl->output.buf); + edge->ctx.impl->output.buf = + grn_msg_open(&edge->ctx, com, &edge->send_old); edge->com = com; edge->stat = EDGE_IDLE; edge->flags = GRN_EDGE_WORKER; @@ -2126,9 +2655,6 @@ g_server(char *path) int exit_code = EXIT_FAILURE; grn_ctx ctx_, *ctx = &ctx_; grn_ctx_init(ctx, 0); - MUTEX_INIT(q_mutex); - COND_INIT(q_cond); - CRITICAL_SECTION_INIT(cache_lock); GRN_COM_QUEUE_INIT(&ctx_new); GRN_COM_QUEUE_INIT(&ctx_old); check_rlimit_nofile(ctx); @@ -2151,6 +2677,7 @@ enum { #define FLAG_MODE_DAEMON (1 << 6) #define FLAG_MODE_SERVER (1 << 7) #define FLAG_NEW_DB (1 << 8) +#define FLAG_USE_WINDOWS_EVENT_LOG (1 << 9) static uint32_t get_core_number(void) @@ -2267,7 +2794,7 @@ config_file_parse(const char *path, const grn_str_getopt_opt *opts, char *ptr, *name, *value; size_t name_length, value_length; - while (isspace(*buf)) { + while (isspace((unsigned char)*buf)) { buf++; } @@ -2278,17 +2805,17 @@ config_file_parse(const char *path, const grn_str_getopt_opt *opts, do { *ptr-- = '\0'; - } while (ptr >= buf && isspace(*ptr)); + } while (ptr >= buf && isspace((unsigned char)*ptr)); if (!*buf) { return CONFIG_FILE_SUCCESS; } name = ptr = buf; - while (*ptr && !isspace(*ptr) && *ptr != '=') { + while (*ptr && !isspace((unsigned char)*ptr) && *ptr != '=') { ptr++; } - while (isspace(*ptr)) { + while (isspace((unsigned char)*ptr)) { *ptr++ = '\0'; } @@ -2303,7 +2830,7 @@ config_file_parse(const char *path, const grn_str_getopt_opt *opts, if (*ptr == '=') { *ptr++ = '\0'; - while (isspace(*ptr)) { + while (isspace((unsigned char)*ptr)) { ptr++; } value = ptr; @@ -2370,8 +2897,8 @@ config_file_load(const char *path, const grn_str_getopt_opt *opts, int *flags) static const int default_http_port = DEFAULT_HTTP_PORT; static const int default_gqtp_port = DEFAULT_GQTP_PORT; static grn_encoding default_encoding = GRN_ENC_DEFAULT; -static uint32_t default_max_num_threads = DEFAULT_MAX_NFTHREADS; -static const int default_log_level = GRN_LOG_DEFAULT_LEVEL; +static uint32_t default_max_n_threads = DEFAULT_MAX_N_FLOATING_THREADS; +static const grn_log_level default_log_level = GRN_LOG_DEFAULT_LEVEL; static const char * const default_protocol = "gqtp"; static const char *default_hostname = "localhost"; static const char * const default_dest = "localhost"; @@ -2383,6 +2910,7 @@ static grn_command_version default_default_command_version = GRN_COMMAND_VERSION_DEFAULT; static int64_t default_default_match_escalation_threshold = 0; static const char * const default_bind_address = "0.0.0.0"; +static double default_default_request_timeout = 0.0; static void init_default_hostname(void) @@ -2415,9 +2943,9 @@ init_default_settings(void) default_encoding = grn_encoding_parse(GRN_DEFAULT_ENCODING); { - const uint32_t num_cores = get_core_number(); - if (num_cores != 0) { - default_max_num_threads = num_cores; + const uint32_t n_cores = get_core_number(); + if (n_cores != 0) { + default_max_n_threads = n_cores; } } @@ -2436,17 +2964,19 @@ init_default_settings(void) #ifdef WIN32 { - static char win32_default_document_root[PATH_MAX]; - size_t document_root_length = strlen(grn_win32_base_dir()) + 1 + + static char windows_default_document_root[PATH_MAX]; + size_t document_root_length = strlen(grn_windows_base_dir()) + 1 + strlen(GRN_DEFAULT_RELATIVE_DOCUMENT_ROOT) + 1; if (document_root_length >= PATH_MAX) { fprintf(stderr, "can't use default root: too long path\n"); } else { - grn_strcpy(win32_default_document_root, PATH_MAX, grn_win32_base_dir()); - grn_strcat(win32_default_document_root, PATH_MAX, "/"); - grn_strcat(win32_default_document_root, PATH_MAX, + grn_strcpy(windows_default_document_root, PATH_MAX, + grn_windows_base_dir()); + grn_strcat(windows_default_document_root, PATH_MAX, + "/"); + grn_strcat(windows_default_document_root, PATH_MAX, GRN_DEFAULT_RELATIVE_DOCUMENT_ROOT); - default_document_root = win32_default_document_root; + default_document_root = windows_default_document_root; } } #else @@ -2455,7 +2985,8 @@ init_default_settings(void) default_default_command_version = grn_get_default_command_version(); default_default_match_escalation_threshold = - grn_get_default_match_escalation_threshold(); + grn_get_default_match_escalation_threshold(); + default_default_request_timeout = grn_get_default_request_timeout(); } static void @@ -2498,7 +3029,7 @@ static void show_version(void) { printf("%s %s [", - grn_get_package(), + grn_get_package_label(), grn_get_version()); /* FIXME: Should we detect host information dynamically on Windows? */ @@ -2534,6 +3065,9 @@ show_version(void) #ifdef GRN_WITH_LZ4 printf(",lz4"); #endif +#ifdef GRN_WITH_ZSTD + printf(",zstd"); +#endif #ifdef USE_KQUEUE printf(",kqueue"); #endif @@ -2575,7 +3109,7 @@ show_usage(FILE *output) " --file <path>: read commands from specified file\n" " --input-fd <FD>: read commands from specified file descriptor\n" " --file has a prioriry over --input-fd\n" - " --output-fd <FD>: output response to specifid file descriptor\n" + " --output-fd <FD>: output response to specified file descriptor\n" " -p, --port <port number>: specify server port number (client mode only)\n" " (default: %d)\n" "\n" @@ -2596,10 +3130,26 @@ show_usage(FILE *output) " specify max number of threads (default: %u)\n" " --pid-path <path>: specify file to write process ID to\n" " (daemon mode only)\n" + " --default-request-timeout <timeout>:\n" + " specify the default request timeout in seconds\n" + " (default: %f)\n" + " --cache-base-path <path>: specify the cache base path\n" + " You can make cache persistent by this option\n" + " You must specify path on memory file system\n" + " (default: none; disabled)\n" + "\n" + "Memcached options:\n" + " --memcached-column <column>:\n" + " specify column to access by memcached protocol\n" + " The column must be text type column and\n" + " its table must be not NO_KEY table\n" "\n" "Logging options:\n" " -l, --log-level <log level>:\n" - " specify log level (default: %d)\n" + " specify log level\n" + " [none|emergency|alert|critical|\n" + " error|warning|notice|info|debug|dump]\n" + " (default: %s)\n" " --log-path <path>: specify log path\n" " (default: %s)\n" " --log-rotate-threshold-size <threshold>:\n" @@ -2608,6 +3158,10 @@ show_usage(FILE *output) " log file size is larger than or\n" " equals to the threshold\n" " (default: 0; disabled)\n" +#ifdef WIN32 + " --use-windows-event-log:\n" + " report logs as Windows events\n" +#endif /* WIN32 */ " --query-log-path <path>:\n" " specify query log path\n" " (default: %s)\n" @@ -2642,8 +3196,10 @@ show_usage(FILE *output) grn_encoding_to_string(default_encoding), default_gqtp_port, default_bind_address, default_http_port, default_gqtp_port, default_hostname, default_protocol, - default_document_root, default_cache_limit, default_max_num_threads, - default_log_level, default_log_path, default_query_log_path, + default_document_root, default_cache_limit, default_max_n_threads, + default_default_request_timeout, + grn_log_level_to_string(default_log_level), + default_log_path, default_query_log_path, default_config_path, default_default_command_version, (long long int)default_default_match_escalation_threshold, default_dest); @@ -2654,7 +3210,7 @@ main(int argc, char **argv) { const char *port_arg = NULL; const char *encoding_arg = NULL; - const char *max_num_threads_arg = NULL; + const char *max_n_threads_arg = NULL; const char *log_level_arg = NULL; const char *bind_address_arg = NULL; const char *hostname_arg = NULL; @@ -2671,10 +3227,15 @@ main(int argc, char **argv) const char *output_fd_arg = NULL; const char *working_directory_arg = NULL; const char *config_path = NULL; + const char *default_request_timeout_arg = NULL; + const char *cache_base_path = NULL; int exit_code = EXIT_SUCCESS; int i; int flags = 0; uint32_t cache_limit = 0; + grn_command_version default_command_version; + int64_t default_match_escalation_threshold = 0; + double default_request_timeout = 0.0; grn_bool need_line_editor = GRN_FALSE; static grn_str_getopt_opt opts[] = { {'p', "port", NULL, 0, GETOPT_OP_NONE}, @@ -2705,11 +3266,16 @@ main(int argc, char **argv) {'\0', "input-fd", NULL, 0, GETOPT_OP_NONE}, {'\0', "output-fd", NULL, 0, GETOPT_OP_NONE}, {'\0', "working-directory", NULL, 0, GETOPT_OP_NONE}, + {'\0', "use-windows-event-log", NULL, + FLAG_USE_WINDOWS_EVENT_LOG, GETOPT_OP_ON}, + {'\0', "memcached-column", NULL, 0, GETOPT_OP_NONE}, + {'\0', "default-request-timeout", NULL, 0, GETOPT_OP_NONE}, + {'\0', "cache-base-path", NULL, 0, GETOPT_OP_NONE}, {'\0', NULL, NULL, 0, 0} }; opts[0].arg = &port_arg; opts[1].arg = &encoding_arg; - opts[2].arg = &max_num_threads_arg; + opts[2].arg = &max_n_threads_arg; opts[7].arg = &log_level_arg; opts[8].arg = &hostname_arg; opts[10].arg = &protocol_arg; @@ -2728,6 +3294,9 @@ main(int argc, char **argv) opts[25].arg = &input_fd_arg; opts[26].arg = &output_fd_arg; opts[27].arg = &working_directory_arg; + opts[29].arg = &memcached_column_name; + opts[30].arg = &default_request_timeout_arg; + opts[31].arg = &cache_base_path; reset_ready_notify_pipe(); @@ -2771,6 +3340,10 @@ main(int argc, char **argv) } } + if (cache_base_path) { + grn_set_default_cache_base_path(cache_base_path); + } + /* ignore mode option in config file */ flags = (flags == ACTION_ERROR) ? 0 : (flags & ~ACTION_MASK); @@ -2864,6 +3437,7 @@ main(int argc, char **argv) break; case 'm' : case 'M' : + is_memcached_mode = GRN_TRUE; do_client = g_client; do_server = g_server; break; @@ -2877,6 +3451,16 @@ main(int argc, char **argv) do_server = g_server; } +#ifdef WIN32 + if (flags & FLAG_USE_WINDOWS_EVENT_LOG) { + use_windows_event_log = GRN_TRUE; + } +#endif /* WIN32 */ + + if (use_windows_event_log) { + grn_windows_event_logger_set(NULL, windows_event_source_name); + } + if (log_path_arg) { grn_default_logger_set_path(log_path_arg); } @@ -2914,69 +3498,50 @@ main(int argc, char **argv) grn_default_query_logger_set_rotate_threshold_size(value); } - if (log_level_arg) { - const char * const end = log_level_arg + strlen(log_level_arg); - const char *rest = NULL; - const int value = grn_atoi(log_level_arg, end, &rest); - if (end != rest || value < 0 || value > 9) { - fprintf(stderr, "invalid log level: <%s>\n", log_level_arg); - return EXIT_FAILURE; + { + grn_log_level log_level; + + if (log_level_arg) { + grn_bool parsed; + + parsed = grn_log_level_parse(log_level_arg, &log_level); + if (!parsed) { + const char * const end = log_level_arg + strlen(log_level_arg); + const char *rest = NULL; + const int value = grn_atoi(log_level_arg, end, &rest); + if (end != rest || value < GRN_LOG_NONE || value > GRN_LOG_DUMP) { + fprintf(stderr, "invalid log level: <%s>\n", log_level_arg); + return EXIT_FAILURE; + } + log_level = value; + } + } else { + log_level = default_log_level; } - log_level = value; - } else { - log_level = default_log_level; + + grn_default_logger_set_max_level(log_level); } - grn_default_logger_set_max_level(log_level); - if (max_num_threads_arg) { - const char * const end = max_num_threads_arg + strlen(max_num_threads_arg); + if (max_n_threads_arg) { + const char * const end = max_n_threads_arg + strlen(max_n_threads_arg); const char *rest = NULL; - const uint32_t value = grn_atoui(max_num_threads_arg, end, &rest); + const uint32_t value = grn_atoui(max_n_threads_arg, end, &rest); if (end != rest || value < 1 || value > 100) { fprintf(stderr, "invalid max number of threads: <%s>\n", - max_num_threads_arg); - return EXIT_FAILURE; - } - max_nfthreads = value; - } else { - max_nfthreads = default_max_num_threads; - } - - if (input_path) { - if (!freopen(input_path, "r", stdin)) { - fprintf(stderr, "can't open input file: %s (%s)\n", - input_path, strerror(errno)); + max_n_threads_arg); return EXIT_FAILURE; } - batchmode = GRN_TRUE; + max_n_floating_threads = value; } else { - if (input_fd_arg) { - const char * const end = input_fd_arg + strlen(input_fd_arg); - const char *rest = NULL; - const int input_fd = grn_atoi(input_fd_arg, end, &rest); - if (rest != end || input_fd == 0) { - fprintf(stderr, "invalid input FD: <%s>\n", input_fd_arg); - return EXIT_FAILURE; - } - if (dup2(input_fd, STDIN_FILENO) == -1) { - fprintf(stderr, "can't open input FD: %d (%s)\n", - input_fd, strerror(errno)); - return EXIT_FAILURE; - } - batchmode = GRN_TRUE; + if (flags & FLAG_MODE_ALONE) { + max_n_floating_threads = 1; } else { - if (argc - i > 1) { - batchmode = GRN_TRUE; - } else { - batchmode = !isatty(0); - } + max_n_floating_threads = default_max_n_threads; } } - if ((flags & (FLAG_MODE_ALONE | FLAG_MODE_CLIENT)) && - !batchmode) { - need_line_editor = GRN_TRUE; - } + grn_thread_set_get_limit_func(groonga_get_thread_limit, NULL); + grn_thread_set_set_limit_func(groonga_set_thread_limit, NULL); if (output_fd_arg) { const char * const end = output_fd_arg + strlen(output_fd_arg); @@ -3036,18 +3601,7 @@ main(int argc, char **argv) default_command_version_arg); return EXIT_FAILURE; } - switch (value) { - case 1 : - default_command_version = GRN_COMMAND_VERSION_1; - break; - case 2 : - default_command_version = GRN_COMMAND_VERSION_2; - break; - default : - fprintf(stderr, "invalid command version: <%s>\n", - default_command_version_arg); - return EXIT_FAILURE; - } + default_command_version = value; } else { default_command_version = default_default_command_version; } @@ -3078,12 +3632,27 @@ main(int argc, char **argv) cache_limit = value; } -#ifdef GRN_WITH_LIBEDIT - if (need_line_editor) { - line_editor_init(argc, argv); + if (default_request_timeout_arg) { + const char * const end = + default_request_timeout_arg + strlen(default_request_timeout_arg); + char *rest = NULL; + double value; + value = strtod(default_request_timeout_arg, &rest); + if (end != rest) { + fprintf(stderr, "invalid default request timeout: <%s>\n", + default_request_timeout_arg); + return EXIT_FAILURE; + } + default_request_timeout = value; + } else { + default_request_timeout = default_default_request_timeout; + } + + grn_gctx.errbuf[0] = '\0'; + if (grn_init()) { + fprintf(stderr, "failed to initialize Groonga: %s\n", grn_gctx.errbuf); + return EXIT_FAILURE; } -#endif - if (grn_init()) { return EXIT_FAILURE; } grn_set_default_encoding(encoding); @@ -3095,6 +3664,10 @@ main(int argc, char **argv) grn_set_default_match_escalation_threshold(default_match_escalation_threshold); } + if (default_request_timeout_arg) { + grn_set_default_request_timeout(default_request_timeout); + } + grn_set_segv_handler(); grn_set_int_handler(); grn_set_term_handler(); @@ -3105,6 +3678,62 @@ main(int argc, char **argv) grn_cache_set_max_n_entries(&grn_gctx, cache, cache_limit); } + MUTEX_INIT(q_mutex); + COND_INIT(q_cond); + + if (input_path) { + input_reader = grn_file_reader_open(&grn_gctx, input_path); + if (!input_reader) { + fprintf(stderr, "can't open input file: %s (%s)\n", + input_path, strerror(errno)); + return EXIT_FAILURE; + } + batchmode = GRN_TRUE; + } else { + if (input_fd_arg) { + const char * const end = input_fd_arg + strlen(input_fd_arg); + const char *rest = NULL; + const int input_fd = grn_atoi(input_fd_arg, end, &rest); + if (rest != end || input_fd == 0) { + fprintf(stderr, "invalid input FD: <%s>\n", input_fd_arg); + return EXIT_FAILURE; + } + if (dup2(input_fd, STDIN_FILENO) == -1) { + fprintf(stderr, "can't open input FD: %d (%s)\n", + input_fd, strerror(errno)); + return EXIT_FAILURE; + } + input_reader = grn_file_reader_open(&grn_gctx, "-"); + if (!input_reader) { + fprintf(stderr, "%s", grn_gctx.errbuf); + return EXIT_FAILURE; + } + batchmode = GRN_TRUE; + } else { + input_reader = grn_file_reader_open(&grn_gctx, "-"); + if (!input_reader) { + fprintf(stderr, "%s", grn_gctx.errbuf); + return EXIT_FAILURE; + } + if (argc - i > 1) { + batchmode = GRN_TRUE; + } else { + batchmode = !grn_isatty(0); + } + } + } + + if ((flags & (FLAG_MODE_ALONE | FLAG_MODE_CLIENT)) && + !batchmode) { + need_line_editor = GRN_TRUE; + } + +#ifdef GRN_WITH_LIBEDIT + if (need_line_editor) { + line_editor_init(argc, argv); + } +#endif + newdb = (flags & FLAG_NEW_DB); is_daemon_mode = (flags & FLAG_MODE_DAEMON); if (flags & FLAG_MODE_CLIENT) { @@ -3115,6 +3744,12 @@ main(int argc, char **argv) exit_code = do_alone(argc - i, argv + i); } + COND_FIN(q_cond); + MUTEX_FIN(q_mutex); + + if (input_reader) { + grn_file_reader_close(&grn_gctx, input_reader); + } #ifdef GRN_WITH_LIBEDIT if (need_line_editor) { line_editor_fin(); diff --git a/storage/mroonga/vendor/groonga/src/groonga_benchmark.c b/storage/mroonga/vendor/groonga/src/groonga_benchmark.c index 2ebca387232..77543c2063d 100644 --- a/storage/mroonga/vendor/groonga/src/groonga_benchmark.c +++ b/storage/mroonga/vendor/groonga/src/groonga_benchmark.c @@ -146,7 +146,7 @@ struct job { long long int max; long long int min; FILE *outputlog; - FILE *inputlog; + grn_file_reader *inputlog; char logfile[BUF_LEN]; }; @@ -802,12 +802,12 @@ do_load_command(grn_ctx *ctx, char *command, int type, int task_id, } if (test_p(grntest_task[task_id].jobtype)) { grn_obj log; - FILE *input; + grn_file_reader *input; FILE *output; GRN_TEXT_INIT(&log, 0); input = grntest_job[grntest_task[task_id].job_id].inputlog; output = grntest_job[grntest_task[task_id].job_id].outputlog; - if (grn_text_fgets(ctx, &log, input) != GRN_SUCCESS) { + if (grn_file_reader_read_line(ctx, input, &log) != GRN_SUCCESS) { GRN_LOG(ctx, GRN_ERROR, "Cannot get input-log"); error_exit_in_thread(55); } @@ -888,12 +888,12 @@ do_command(grn_ctx *ctx, char *command, int type, int task_id) } if (test_p(grntest_task[task_id].jobtype)) { grn_obj log; - FILE *input; + grn_file_reader *input; FILE *output; GRN_TEXT_INIT(&log, 0); input = grntest_job[grntest_task[task_id].job_id].inputlog; output = grntest_job[grntest_task[task_id].job_id].outputlog; - if (grn_text_fgets(ctx, &log, input) != GRN_SUCCESS) { + if (grn_file_reader_read_line(ctx, input, &log) != GRN_SUCCESS) { GRN_LOG(ctx, GRN_ERROR, "Cannot get input-log"); error_exit_in_thread(55); } @@ -971,10 +971,10 @@ worker_sub(grn_ctx *ctx, grn_obj *log, int task_id) for (i = 0; i < task->ntimes; i++) { if (task->file != NULL) { - FILE *fp; + grn_file_reader *reader; grn_obj line; - fp = fopen(task->file, "r"); - if (!fp) { + reader = grn_file_reader_open(ctx, task->file); + if (!reader) { fprintf(stderr, "Cannot open %s\n",grntest_task[task_id].file); error_exit_in_thread(1); } @@ -982,7 +982,7 @@ worker_sub(grn_ctx *ctx, grn_obj *log, int task_id) load_count = 0; load_start = 0LL; GRN_TEXT_INIT(&line, 0); - while (grn_text_fgets(ctx, &line, fp) == GRN_SUCCESS) { + while (grn_file_reader_read_line(ctx, reader, &line) == GRN_SUCCESS) { if (GRN_TEXT_VALUE(&line)[GRN_TEXT_LEN(&line) - 1] == '\n') { grn_bulk_truncate(ctx, &line, GRN_TEXT_LEN(&line) - 1); } @@ -1022,7 +1022,7 @@ worker_sub(grn_ctx *ctx, grn_obj *log, int task_id) } } GRN_OBJ_FIN(ctx, &line); - fclose(fp); + grn_file_reader_close(ctx, reader); } else { int i, n_commands; grn_obj *commands; @@ -1602,7 +1602,7 @@ start_server(const char *dbpath, int r) } static int -parse_line(char *buf, int start, int end, int num) +parse_line(grn_ctx *ctx, char *buf, int start, int end, int num) { int i, j, error_flag = 0, out_or_test = 0; char tmpbuf[BUF_LEN]; @@ -1758,7 +1758,7 @@ parse_line(char *buf, int start, int end, int num) } } else { char outlog[BUF_LEN]; - grntest_job[num].inputlog = fopen(tmpbuf, "rb"); + grntest_job[num].inputlog = grn_file_reader_open(ctx, tmpbuf); if (grntest_job[num].inputlog == NULL) { fprintf(stderr, "Cannot open %s\n", tmpbuf); return 14; @@ -1840,7 +1840,7 @@ get_jobs(grn_ctx *ctx, char *buf, int line) while (i < len) { if (buf[i] == ';') { end = i; - ret = parse_line(buf, start, end, jnum); + ret = parse_line(ctx, buf, start, end, jnum); if (ret) { if (ret > 1) { fprintf(stderr, "Syntax error:line=%d:ret=%d:%s\n", line, ret, buf); @@ -1854,7 +1854,7 @@ get_jobs(grn_ctx *ctx, char *buf, int line) i++; } end = len; - ret = parse_line(buf, start, end, jnum); + ret = parse_line(ctx, buf, start, end, jnum); if (ret) { if (ret > 1) { fprintf(stderr, "Syntax error:line=%d:ret=%d:%s\n", line, ret, buf); @@ -1871,7 +1871,6 @@ make_task_table(grn_ctx *ctx, int jobnum) { int i, j; int tid = 0; - FILE *fp; grn_obj *commands = NULL; for (i = 0; i < jobnum; i++) { @@ -1886,6 +1885,7 @@ make_task_table(grn_ctx *ctx, int jobnum) } for (j = 0; j < grntest_job[i].concurrency; j++) { if (j == 0) { + grn_file_reader *reader; grn_obj line; GRN_TEXT_INIT(&line, 0); commands = grn_obj_open(ctx, GRN_PVECTOR, 0, GRN_VOID); @@ -1893,13 +1893,13 @@ make_task_table(grn_ctx *ctx, int jobnum) fprintf(stderr, "Cannot alloc commands\n"); error_exit(ctx, 1); } - fp = fopen(grntest_job[i].commandfile, "r"); - if (!fp) { + reader = grn_file_reader_open(ctx, grntest_job[i].commandfile); + if (!reader) { fprintf(stderr, "Cannot alloc commandfile:%s\n", grntest_job[i].commandfile); error_exit(ctx, 1); } - while (grn_text_fgets(ctx, &line, fp) == GRN_SUCCESS) { + while (grn_file_reader_read_line(ctx, reader, &line) == GRN_SUCCESS) { grn_obj *command; if (GRN_TEXT_VALUE(&line)[GRN_TEXT_LEN(&line) - 1] == '\n') { grn_bulk_truncate(ctx, &line, GRN_TEXT_LEN(&line) - 1); @@ -1924,6 +1924,7 @@ make_task_table(grn_ctx *ctx, int jobnum) GRN_PTR_PUT(ctx, commands, command); GRN_BULK_REWIND(&line); } + grn_file_reader_close(ctx, reader); GRN_OBJ_FIN(ctx, &line); } grntest_task[tid].file = NULL; @@ -2063,12 +2064,7 @@ printf("%d:type =%d:file=%s:con=%d:ntimes=%d\n", i, grntest_job[i].jobtype, } } if (grntest_job[i].inputlog) { - int ret; - ret = fclose(grntest_job[i].inputlog); - if (ret) { - fprintf(stderr, "Cannot close %s\n", grntest_job[i].logfile); - exit(1); - } + grn_file_reader_close(ctx, grntest_job[i].inputlog); } } return qnum; @@ -2081,17 +2077,17 @@ do_script(grn_ctx *ctx, const char *script_file_path) int n_lines = 0; int n_jobs; int n_queries, total_n_queries = 0; - FILE *script_file; + grn_file_reader *script_file; grn_obj line; - script_file = fopen(script_file_path, "r"); + script_file = grn_file_reader_open(ctx, script_file_path); if (script_file == NULL) { fprintf(stderr, "Cannot open script file: <%s>\n", script_file_path); error_exit(ctx, 1); } GRN_TEXT_INIT(&line, 0); - while (grn_text_fgets(ctx, &line, script_file) == GRN_SUCCESS) { + while (grn_file_reader_read_line(ctx, script_file, &line) == GRN_SUCCESS) { if (grntest_sigint) { break; } @@ -2126,7 +2122,7 @@ do_script(grn_ctx *ctx, const char *script_file_path) } grn_obj_unlink(ctx, &line); - fclose(script_file); + grn_file_reader_close(ctx, script_file); return total_n_queries; } @@ -2861,20 +2857,20 @@ get_token(char *line, char *token, int maxlen, char **next) static grn_bool check_script(grn_ctx *ctx, const char *script_file_path) { - FILE *script_file; + grn_file_reader *script_file; grn_obj line; char token[BUF_LEN]; char prev[BUF_LEN]; char *next = NULL; - script_file = fopen(script_file_path, "r"); + script_file = grn_file_reader_open(ctx, script_file_path); if (!script_file) { fprintf(stderr, "Cannot open script file: <%s>\n", script_file_path); return GRN_FALSE; } GRN_TEXT_INIT(&line, 0); - while (grn_text_fgets(ctx, &line, script_file) == GRN_SUCCESS) { + while (grn_file_reader_read_line(ctx, script_file, &line) == GRN_SUCCESS) { GRN_TEXT_VALUE(&line)[GRN_TEXT_LEN(&line) - 1] = '\0'; get_token(GRN_TEXT_VALUE(&line), token, BUF_LEN, &next); grn_strcpy(prev, BUF_LEN, token); @@ -2893,7 +2889,7 @@ check_script(grn_ctx *ctx, const char *script_file_path) } grn_obj_unlink(ctx, &line); - fclose(script_file); + grn_file_reader_close(ctx, script_file); return GRN_TRUE; } @@ -3008,7 +3004,7 @@ main(int argc, char **argv) FILE *pid_file; pid_file = fopen(pid_path, "w"); if (pid_file) { - fprintf(pid_file, "%d", getpid()); + fprintf(pid_file, "%d", grn_getpid()); fclose(pid_file); } else { fprintf(stderr, diff --git a/storage/mroonga/vendor/groonga/src/httpd/Makefile.am b/storage/mroonga/vendor/groonga/src/httpd/Makefile.am index 88bcc03a33c..736dd1cf939 100644 --- a/storage/mroonga/vendor/groonga/src/httpd/Makefile.am +++ b/storage/mroonga/vendor/groonga/src/httpd/Makefile.am @@ -1,6 +1,6 @@ NGINX_DIR = $(top_builddir)/vendor/nginx-$(NGINX_VERSION) -EXTRA_DIST = \ +EXTRA_DIST = \ nginx-module \ configure diff --git a/storage/mroonga/vendor/groonga/src/httpd/nginx-module/ngx_http_groonga_module.c b/storage/mroonga/vendor/groonga/src/httpd/nginx-module/ngx_http_groonga_module.c index 727e65fa468..15836a92e05 100644 --- a/storage/mroonga/vendor/groonga/src/httpd/nginx-module/ngx_http_groonga_module.c +++ b/storage/mroonga/vendor/groonga/src/httpd/nginx-module/ngx_http_groonga_module.c @@ -1,6 +1,6 @@ /* -*- c-basic-offset: 2 -*- */ /* - Copyright(C) 2012-2015 Brazil + Copyright(C) 2012-2017 Brazil This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public @@ -25,6 +25,7 @@ #include <ngx_http.h> #include <groonga.h> +#include <groonga/plugin.h> #include <sys/stat.h> @@ -47,11 +48,13 @@ typedef struct { ngx_str_t query_log_path; ngx_open_file_t *query_log_file; size_t cache_limit; + ngx_msec_t default_request_timeout_msec; char *config_file; int config_line; char *name; - grn_ctx context; + grn_obj *database; grn_cache *cache; + ngx_str_t cache_base_path; } ngx_http_groonga_loc_conf_t; typedef struct { @@ -62,7 +65,7 @@ typedef struct { typedef struct { grn_bool initialized; - grn_ctx context; + grn_rc rc; struct { grn_bool processed; grn_bool header_sent; @@ -78,21 +81,14 @@ typedef struct { } typed; } ngx_http_groonga_handler_data_t; -typedef struct { - ngx_pool_t *pool; - ngx_open_file_t *file; -} ngx_http_groonga_logger_data_t; - -typedef struct { - ngx_pool_t *pool; - ngx_open_file_t *file; - ngx_str_t *path; -} ngx_http_groonga_query_logger_data_t; - typedef void (*ngx_http_groonga_loc_conf_callback_pt)(ngx_http_groonga_loc_conf_t *conf, void *user_data); ngx_module_t ngx_http_groonga_module; +static grn_ctx ngx_http_groonga_context; +static grn_ctx *context = &ngx_http_groonga_context; +static ngx_http_groonga_loc_conf_t *ngx_http_groonga_current_location_conf = NULL; + static char * ngx_str_null_terminate(ngx_pool_t *pool, const ngx_str_t *string) { @@ -133,6 +129,29 @@ ngx_str_is_custom_path(ngx_str_t *string) return GRN_TRUE; } +static uint32_t +ngx_http_groonga_get_thread_limit(void *data) +{ + return 1; +} + +static ngx_int_t +ngx_http_groonga_grn_rc_to_http_status(grn_rc rc) +{ + switch (rc) { + case GRN_SUCCESS : + return NGX_HTTP_OK; + case GRN_INVALID_ARGUMENT : + case GRN_FUNCTION_NOT_IMPLEMENTED : + case GRN_SYNTAX_ERROR : + return NGX_HTTP_BAD_REQUEST; + case GRN_CANCEL : + return NGX_HTTP_REQUEST_TIME_OUT; + default : + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } +} + static void ngx_http_groonga_write_fd(ngx_fd_t fd, u_char *buffer, size_t buffer_size, @@ -163,62 +182,41 @@ ngx_http_groonga_logger_log(grn_ctx *ctx, grn_log_level level, const char *message, const char *location, void *user_data) { - ngx_http_groonga_logger_data_t *logger_data = user_data; - const char level_marks[] = " EACewnid-"; + ngx_open_file_t *file = user_data; + char level_marks[] = " EACewnid-"; u_char buffer[NGX_MAX_ERROR_STR]; - u_char *last; - size_t prefix_size; - size_t message_size; - size_t location_size; - size_t postfix_size; - size_t log_message_size; - -#define LOG_PREFIX_FORMAT "%s|%c|%s " - prefix_size = - strlen(timestamp) + - 1 /* | */ + - 1 /* %c */ + - 1 /* | */ + - strlen(title) + - 1 /* a space */; - message_size = strlen(message); - if (location && *location) { - location_size = 1 /* a space */ + strlen(location); - } else { - location_size = 0; + + if (!file) { + return; } - postfix_size = 1 /* \n */; - log_message_size = prefix_size + message_size + location_size + postfix_size; - if (log_message_size > NGX_MAX_ERROR_STR) { - last = ngx_slprintf(buffer, buffer + NGX_MAX_ERROR_STR, - LOG_PREFIX_FORMAT, - timestamp, *(level_marks + level), title); - ngx_write_fd(logger_data->file->fd, buffer, last - buffer); - ngx_http_groonga_write_fd(logger_data->file->fd, + ngx_http_groonga_write_fd(file->fd, + buffer, NGX_MAX_ERROR_STR, + timestamp, strlen(timestamp)); + ngx_write_fd(file->fd, "|", 1); + ngx_write_fd(file->fd, level_marks + level, 1); + ngx_write_fd(file->fd, "|", 1); + if (location && *location) { + ngx_http_groonga_write_fd(file->fd, buffer, NGX_MAX_ERROR_STR, - message, message_size); - if (location_size > 0) { - ngx_write_fd(logger_data->file->fd, " ", 1); - ngx_http_groonga_write_fd(logger_data->file->fd, + location, strlen(location)); + ngx_write_fd(file->fd, ": ", 2); + if (title && *title) { + ngx_http_groonga_write_fd(file->fd, buffer, NGX_MAX_ERROR_STR, - location, location_size); + title, strlen(title)); + ngx_write_fd(file->fd, " ", 1); } - ngx_write_fd(logger_data->file->fd, "\n", 1); } else { - if (location && *location) { - last = ngx_slprintf(buffer, buffer + NGX_MAX_ERROR_STR, - LOG_PREFIX_FORMAT " %s %s\n", - timestamp, *(level_marks + level), title, message, - location); - } else { - last = ngx_slprintf(buffer, buffer + NGX_MAX_ERROR_STR, - LOG_PREFIX_FORMAT " %s\n", - timestamp, *(level_marks + level), title, message); - } - ngx_write_fd(logger_data->file->fd, buffer, last - buffer); + ngx_http_groonga_write_fd(file->fd, + buffer, NGX_MAX_ERROR_STR, + title, strlen(title)); + ngx_write_fd(file->fd, " ", 1); } -#undef LOG_PREFIX_FORMAT + ngx_http_groonga_write_fd(file->fd, + buffer, NGX_MAX_ERROR_STR, + message, strlen(message)); + ngx_write_fd(file->fd, "\n", 1); } static void @@ -232,14 +230,11 @@ ngx_http_groonga_logger_reopen(grn_ctx *ctx, void *user_data) static void ngx_http_groonga_logger_fin(grn_ctx *ctx, void *user_data) { - ngx_http_groonga_logger_data_t *logger_data = user_data; - - ngx_pfree(logger_data->pool, logger_data); } static grn_logger ngx_http_groonga_logger = { GRN_LOG_DEFAULT_LEVEL, - GRN_LOG_TIME | GRN_LOG_MESSAGE, + GRN_LOG_TIME | GRN_LOG_MESSAGE | GRN_LOG_PID, NULL, ngx_http_groonga_logger_log, ngx_http_groonga_logger_reopen, @@ -247,28 +242,17 @@ static grn_logger ngx_http_groonga_logger = { }; static ngx_int_t -ngx_http_groonga_context_init_logger(grn_ctx *context, - ngx_http_groonga_loc_conf_t *location_conf, +ngx_http_groonga_context_init_logger(ngx_http_groonga_loc_conf_t *location_conf, ngx_pool_t *pool, ngx_log_t *log) { - ngx_http_groonga_logger_data_t *logger_data; - - if (!location_conf->log_file) { - return NGX_OK; - } - - logger_data = ngx_pcalloc(pool, sizeof(ngx_http_groonga_logger_data_t)); - if (!logger_data) { - ngx_log_error(NGX_LOG_ERR, log, 0, - "http_groonga: failed to allocate memory for logger"); - return NGX_ERROR; + if (ngx_http_groonga_current_location_conf) { + ngx_http_groonga_current_location_conf->log_level = + grn_logger_get_max_level(context); } - logger_data->pool = pool; - logger_data->file = location_conf->log_file; ngx_http_groonga_logger.max_level = location_conf->log_level; - ngx_http_groonga_logger.user_data = logger_data; + ngx_http_groonga_logger.user_data = location_conf->log_file; grn_logger_set(context, &ngx_http_groonga_logger); return NGX_OK; @@ -279,36 +263,29 @@ ngx_http_groonga_query_logger_log(grn_ctx *ctx, unsigned int flag, const char *timestamp, const char *info, const char *message, void *user_data) { - ngx_http_groonga_query_logger_data_t *data = user_data; + ngx_open_file_t *file = user_data; u_char buffer[NGX_MAX_ERROR_STR]; u_char *last; + if (!file) { + return; + } + last = ngx_slprintf(buffer, buffer + NGX_MAX_ERROR_STR, "%s|%s%s\n", timestamp, info, message); - ngx_write_fd(data->file->fd, buffer, last - buffer); + ngx_write_fd(file->fd, buffer, last - buffer); } static void ngx_http_groonga_query_logger_reopen(grn_ctx *ctx, void *user_data) { - ngx_http_groonga_query_logger_data_t *data = user_data; - - GRN_QUERY_LOG(ctx, GRN_QUERY_LOG_DESTINATION, " ", - "query log will be closed: <%.*s>", - (int)(data->path->len), data->path->data); ngx_reopen_files((ngx_cycle_t *)ngx_cycle, -1); - GRN_QUERY_LOG(ctx, GRN_QUERY_LOG_DESTINATION, " ", - "query log is opened: <%.*s>", - (int)(data->path->len), data->path->data); } static void ngx_http_groonga_query_logger_fin(grn_ctx *ctx, void *user_data) { - ngx_http_groonga_query_logger_data_t *data = user_data; - - ngx_pfree(data->pool, data); } static grn_query_logger ngx_http_groonga_query_logger = { @@ -320,71 +297,61 @@ static grn_query_logger ngx_http_groonga_query_logger = { }; static ngx_int_t -ngx_http_groonga_context_init_query_logger(grn_ctx *context, - ngx_http_groonga_loc_conf_t *location_conf, +ngx_http_groonga_context_init_query_logger(ngx_http_groonga_loc_conf_t *location_conf, ngx_pool_t *pool, ngx_log_t *log) { - ngx_http_groonga_query_logger_data_t *query_logger_data; - - if (!location_conf->query_log_file) { - return NGX_OK; - } - - query_logger_data = ngx_pcalloc(pool, - sizeof(ngx_http_groonga_query_logger_data_t)); - if (!query_logger_data) { - ngx_log_error(NGX_LOG_ERR, log, 0, - "http_groonga: failed to allocate memory for query logger"); - return NGX_ERROR; - } - - query_logger_data->pool = pool; - query_logger_data->file = location_conf->query_log_file; - query_logger_data->path = &(location_conf->query_log_path); - ngx_http_groonga_query_logger.user_data = query_logger_data; + ngx_http_groonga_query_logger.user_data = location_conf->query_log_file; grn_query_logger_set(context, &ngx_http_groonga_query_logger); return NGX_OK; } static ngx_int_t -ngx_http_groonga_context_init(grn_ctx *context, - ngx_http_groonga_loc_conf_t *location_conf, +ngx_http_groonga_context_init(ngx_http_groonga_loc_conf_t *location_conf, ngx_pool_t *pool, ngx_log_t *log) { ngx_int_t status; - grn_ctx_init(context, GRN_NO_FLAGS); + if (location_conf == ngx_http_groonga_current_location_conf) { + return NGX_OK; + } - status = ngx_http_groonga_context_init_logger(context, - location_conf, + status = ngx_http_groonga_context_init_logger(location_conf, pool, log); if (status == NGX_ERROR) { - grn_ctx_fin(context); return status; } - status = ngx_http_groonga_context_init_query_logger(context, - location_conf, + status = ngx_http_groonga_context_init_query_logger(location_conf, pool, log); if (status == NGX_ERROR) { - grn_ctx_fin(context); return status; } - if (location_conf->cache) { - grn_cache_current_set(context, location_conf->cache); + grn_ctx_use(context, location_conf->database); + grn_cache_current_set(context, location_conf->cache); + + /* TODO: It doesn't work yet. We need to implement request timeout + * handler. */ + if (location_conf->default_request_timeout_msec == NGX_CONF_UNSET_MSEC) { + grn_set_default_request_timeout(0.0); + } else { + double timeout; + timeout = location_conf->default_request_timeout_msec / 1000.0; + grn_set_default_request_timeout(timeout); } + ngx_http_groonga_current_location_conf = location_conf; + return status; } static void -ngx_http_groonga_context_log_error(ngx_log_t *log, grn_ctx *context) +ngx_http_groonga_context_log_error(ngx_log_t *log) { if (context->rc == GRN_SUCCESS) { return; @@ -394,12 +361,12 @@ ngx_http_groonga_context_log_error(ngx_log_t *log, grn_ctx *context) } static ngx_int_t -ngx_http_groonga_context_check_error(ngx_log_t *log, grn_ctx *context) +ngx_http_groonga_context_check_error(ngx_log_t *log) { if (context->rc == GRN_SUCCESS) { return NGX_OK; } else { - ngx_http_groonga_context_log_error(log, context); + ngx_http_groonga_context_log_error(log); return NGX_HTTP_BAD_REQUEST; } } @@ -426,19 +393,14 @@ static void ngx_http_groonga_handler_cleanup(void *user_data) { ngx_http_groonga_handler_data_t *data = user_data; - grn_ctx *context; if (!data->initialized) { return; } - context = &(data->context); GRN_OBJ_FIN(context, &(data->typed.head)); GRN_OBJ_FIN(context, &(data->typed.body)); GRN_OBJ_FIN(context, &(data->typed.foot)); - grn_logger_set(context, NULL); - grn_query_logger_set(context, NULL); - grn_ctx_fin(context); } static void @@ -566,7 +528,8 @@ ngx_http_groonga_context_receive_handler_typed(grn_ctx *context, context->stat |= GRN_CTX_QUIT; } else { context->rc = GRN_OPERATION_NOT_PERMITTED; - GRN_TEXT_PUTS(context, &(data->typed.body), "false"); + result = "false"; + result_size = strlen(result); context->stat &= ~GRN_CTX_QUIT; } } @@ -663,23 +626,20 @@ ngx_http_groonga_handler_create_data(ngx_http_request_t *r, ngx_http_cleanup_t *cleanup; ngx_http_groonga_handler_data_t *data; - grn_ctx *context; - location_conf = ngx_http_get_module_loc_conf(r, ngx_http_groonga_module); + rc = ngx_http_groonga_context_init(location_conf, r->pool, r->connection->log); + if (rc != NGX_OK) { + return rc; + } + cleanup = ngx_http_cleanup_add(r, sizeof(ngx_http_groonga_handler_data_t)); cleanup->handler = ngx_http_groonga_handler_cleanup; data = cleanup->data; *data_return = data; - context = &(data->context); - rc = ngx_http_groonga_context_init(context, location_conf, - r->pool, r->connection->log); - if (rc != NGX_OK) { - return rc; - } - data->initialized = GRN_TRUE; + data->rc = GRN_SUCCESS; data->raw.processed = GRN_FALSE; data->raw.header_sent = GRN_FALSE; @@ -692,8 +652,8 @@ ngx_http_groonga_handler_create_data(ngx_http_request_t *r, GRN_TEXT_INIT(&(data->typed.body), GRN_NO_FLAGS); GRN_TEXT_INIT(&(data->typed.foot), GRN_NO_FLAGS); - grn_ctx_use(context, grn_ctx_db(&(location_conf->context))); - rc = ngx_http_groonga_context_check_error(r->connection->log, context); + grn_ctx_use(context, location_conf->database); + rc = ngx_http_groonga_context_check_error(r->connection->log); if (rc != NGX_OK) { return rc; } @@ -705,32 +665,28 @@ ngx_http_groonga_handler_create_data(ngx_http_request_t *r, return NGX_OK; } -static ngx_int_t +static void ngx_http_groonga_handler_process_command_path(ngx_http_request_t *r, ngx_str_t *command_path, - ngx_http_groonga_handler_data_t *data) + ngx_http_groonga_handler_data_t *data, + int flags) { - grn_ctx *context; grn_obj uri; - context = &(data->context); GRN_TEXT_INIT(&uri, 0); GRN_TEXT_PUTS(context, &uri, "/d/"); GRN_TEXT_PUT(context, &uri, command_path->data, command_path->len); - grn_ctx_send(context, GRN_TEXT_VALUE(&uri), GRN_TEXT_LEN(&uri), - GRN_NO_FLAGS); - ngx_http_groonga_context_log_error(r->connection->log, context); + grn_ctx_send(context, GRN_TEXT_VALUE(&uri), GRN_TEXT_LEN(&uri), flags); + data->rc = context->rc; + ngx_http_groonga_context_log_error(r->connection->log); GRN_OBJ_FIN(context, &uri); - - return NGX_OK; } -static ngx_int_t +static grn_bool ngx_http_groonga_handler_validate_post_command(ngx_http_request_t *r, ngx_str_t *command_path, ngx_http_groonga_handler_data_t *data) { - grn_ctx *context; ngx_str_t command; command.data = command_path->data; @@ -740,166 +696,180 @@ ngx_http_groonga_handler_validate_post_command(ngx_http_request_t *r, command.len = command_path->len - r->args.len - strlen("?"); } if (ngx_str_equal_c_string(&command, "load")) { - return NGX_OK; + return GRN_TRUE; } - context = &(data->context); + data->rc = GRN_INVALID_ARGUMENT; ngx_http_groonga_handler_set_content_type(r, "text/plain"); GRN_TEXT_PUTS(context, &(data->typed.body), "command for POST must be <load>: <"); GRN_TEXT_PUT(context, &(data->typed.body), command.data, command.len); GRN_TEXT_PUTS(context, &(data->typed.body), ">"); - return NGX_HTTP_BAD_REQUEST; -} - -static ngx_int_t -ngx_http_groonga_send_lines(grn_ctx *context, - ngx_http_request_t *r, - u_char *current, - u_char *last) -{ - ngx_int_t rc; - - u_char *line_start; - - for (line_start = current; current < last; current++) { - if (*current != '\n') { - continue; - } - - grn_ctx_send(context, (const char *)line_start, current - line_start, - GRN_NO_FLAGS); - rc = ngx_http_groonga_context_check_error(r->connection->log, context); - if (rc != NGX_OK) { - return rc; - } - line_start = current + 1; - } - if (line_start < current) { - grn_ctx_send(context, (const char *)line_start, current - line_start, - GRN_NO_FLAGS); - rc = ngx_http_groonga_context_check_error(r->connection->log, context); - if (rc != NGX_OK) { - return rc; - } - } - - return NGX_OK; + return GRN_FALSE; } -static ngx_int_t -ngx_http_groonga_join_request_body_chain(ngx_http_request_t *r, - ngx_chain_t *chain, - u_char **out_start, - u_char **out_end) +static void +ngx_http_groonga_send_body(ngx_http_request_t *r, + ngx_http_groonga_handler_data_t *data) { - ngx_int_t rc; - - ngx_log_t *log = r->connection->log; - - ngx_chain_t *current; - u_char *out; - size_t out_size; + ngx_log_t *log; + grn_obj line_buffer; + size_t line_start_offset; + size_t line_check_start_offset; + ngx_chain_t *chain; + size_t line_buffer_chunk_size = 4096; - u_char *out_cursor; - ngx_buf_t *buffer; - size_t buffer_size; + log = r->connection->log; - out_size = 0; - for (current = chain; current; current = current->next) { - out_size += ngx_buf_size(current->buf); - } - out = ngx_palloc(r->pool, out_size); - if (!out) { - ngx_log_error(NGX_LOG_ERR, log, 0, - "http_groonga: failed to allocate memory for request body"); - return NGX_ERROR; - } + GRN_TEXT_INIT(&line_buffer, 0); + line_start_offset = 0; + line_check_start_offset = 0; + for (chain = r->request_body->bufs; chain; chain = chain->next) { + ngx_buf_t *buffer; + size_t rest_buffer_size; + off_t offset; + + buffer = chain->buf; + rest_buffer_size = ngx_buf_size(buffer); + offset = 0; + while (rest_buffer_size > 0) { + size_t current_buffer_size; + + if (rest_buffer_size > line_buffer_chunk_size) { + current_buffer_size = line_buffer_chunk_size; + } else { + current_buffer_size = rest_buffer_size; + } - out_cursor = out; - for (current = chain; current; current = current->next) { - buffer = current->buf; - buffer_size = ngx_buf_size(current->buf); + if (ngx_buf_in_memory(buffer)) { + GRN_TEXT_PUT(context, + &line_buffer, + buffer->pos + offset, + current_buffer_size); + } else { + ngx_int_t rc; + grn_bulk_reserve(context, &line_buffer, current_buffer_size); + rc = ngx_read_file(buffer->file, + (u_char *)GRN_BULK_CURR(&line_buffer), + current_buffer_size, + offset); + if (rc < 0) { + GRN_PLUGIN_ERROR(context, + GRN_INPUT_OUTPUT_ERROR, + "[nginx][post][body][read] " + "failed to read a request body from file"); + goto exit; + } + GRN_BULK_INCR_LEN(&line_buffer, current_buffer_size); + } + offset += current_buffer_size; + rest_buffer_size -= current_buffer_size; + + { + const char *line_start; + const char *line_current; + const char *line_end; + + line_start = GRN_TEXT_VALUE(&line_buffer) + line_start_offset; + line_end = GRN_TEXT_VALUE(&line_buffer) + GRN_TEXT_LEN(&line_buffer); + for (line_current = line_start + line_check_start_offset; + line_current < line_end; + line_current++) { + size_t line_length; + int flags = GRN_NO_FLAGS; + + if (*line_current != '\n') { + continue; + } + + line_length = line_current - line_start + 1; + if (line_current + 1 == line_end && + !chain->next && + rest_buffer_size == 0) { + flags |= GRN_CTX_TAIL; + } + grn_ctx_send(context, line_start, line_length, flags); + line_start_offset += line_length; + line_start += line_length; + ngx_http_groonga_context_log_error(log); + if (context->rc != GRN_SUCCESS && data->rc == GRN_SUCCESS) { + data->rc = context->rc; + } + } - if (buffer->file) { - rc = ngx_read_file(buffer->file, out_cursor, buffer_size, 0); - if (rc < 0) { - ngx_log_error(NGX_LOG_ERR, log, 0, - "http_groonga: failed to read a request body stored in a file"); - return rc; + if (line_start_offset == 0) { + line_buffer_chunk_size *= 2; + line_check_start_offset = GRN_TEXT_LEN(&line_buffer); + } else if ((size_t)GRN_TEXT_LEN(&line_buffer) == line_start_offset) { + GRN_BULK_REWIND(&line_buffer); + line_start_offset = 0; + line_check_start_offset = 0; + } else { + size_t rest_line_size; + rest_line_size = GRN_TEXT_LEN(&line_buffer) - line_start_offset; + grn_memmove(GRN_TEXT_VALUE(&line_buffer), + GRN_TEXT_VALUE(&line_buffer) + line_start_offset, + rest_line_size); + grn_bulk_truncate(context, &line_buffer, rest_line_size); + line_start_offset = 0; + line_check_start_offset = GRN_TEXT_LEN(&line_buffer); + } } - } else { - ngx_memcpy(out_cursor, buffer->pos, buffer_size); } - out_cursor += buffer_size; } - *out_start = out; - *out_end = out + out_size; + if (GRN_TEXT_LEN(&line_buffer) > 0) { + grn_ctx_send(context, + GRN_TEXT_VALUE(&line_buffer), + GRN_TEXT_LEN(&line_buffer), + GRN_CTX_TAIL); + ngx_http_groonga_context_log_error(log); + if (context->rc != GRN_SUCCESS && data->rc == GRN_SUCCESS) { + data->rc = context->rc; + } + } - return NGX_OK; +exit : + GRN_OBJ_FIN(context, &line_buffer); } -static ngx_int_t +static void ngx_http_groonga_handler_process_body(ngx_http_request_t *r, ngx_http_groonga_handler_data_t *data) { - ngx_int_t rc; - - grn_ctx *context; - ngx_buf_t *body; - u_char *body_data; - u_char *body_data_end; - - context = &(data->context); body = r->request_body->bufs->buf; if (!body) { + data->rc = GRN_INVALID_ARGUMENT; ngx_http_groonga_handler_set_content_type(r, "text/plain"); GRN_TEXT_PUTS(context, &(data->typed.body), "must send load data as body"); - return NGX_HTTP_BAD_REQUEST; - } - - rc = ngx_http_groonga_join_request_body_chain(r, - r->request_body->bufs, - &body_data, - &body_data_end); - if (rc != NGX_OK) { - return rc; + return; } - rc = ngx_http_groonga_send_lines(context, r, body_data, body_data_end); - ngx_pfree(r->pool, body_data); - - return rc; + ngx_http_groonga_send_body(r, data); } -static ngx_int_t +static void ngx_http_groonga_handler_process_load(ngx_http_request_t *r, ngx_str_t *command_path, ngx_http_groonga_handler_data_t *data) { - ngx_int_t rc; - - rc = ngx_http_groonga_handler_validate_post_command(r, command_path, data); - if (rc != NGX_OK) { - return rc; - } - - rc = ngx_http_groonga_handler_process_command_path(r, command_path, data); - if (rc != NGX_OK) { - return rc; + if (!ngx_http_groonga_handler_validate_post_command(r, command_path, data)) { + return; } - rc = ngx_http_groonga_handler_process_body(r, data); - if (rc != NGX_OK) { - return rc; + ngx_http_groonga_handler_process_command_path(r, + command_path, + data, + GRN_NO_FLAGS); + if (data->rc != GRN_SUCCESS) { + return; } - return NGX_OK; + ngx_http_groonga_handler_process_body(r, data); } static ngx_chain_t * @@ -931,7 +901,6 @@ ngx_http_groonga_handler_send_response(ngx_http_request_t *r, ngx_http_groonga_handler_data_t *data) { ngx_int_t rc; - grn_ctx *context; const char *content_type; ngx_buf_t *head_buf, *body_buf, *foot_buf; ngx_chain_t head_chain, body_chain, foot_chain; @@ -941,11 +910,16 @@ ngx_http_groonga_handler_send_response(ngx_http_request_t *r, return data->raw.rc; } - context = &(data->context); - /* set the 'Content-type' header */ if (r->headers_out.content_type.len == 0) { - content_type = grn_ctx_get_mime_type(context); + grn_obj *foot = &(data->typed.foot); + if (grn_ctx_get_output_type(context) == GRN_CONTENT_JSON && + GRN_TEXT_LEN(foot) > 0 && + GRN_TEXT_VALUE(foot)[GRN_TEXT_LEN(foot) - 1] == ';') { + content_type = "application/javascript"; + } else { + content_type = grn_ctx_get_mime_type(context); + } ngx_http_groonga_handler_set_content_type(r, content_type); } @@ -974,7 +948,7 @@ ngx_http_groonga_handler_send_response(ngx_http_request_t *r, output_chain = ngx_http_groonga_attach_chain(output_chain, &foot_chain); /* set the status line */ - r->headers_out.status = NGX_HTTP_OK; + r->headers_out.status = ngx_http_groonga_grn_rc_to_http_status(data->rc); r->headers_out.content_length_n = GRN_TEXT_LEN(&(data->typed.head)) + GRN_TEXT_LEN(&(data->typed.body)) + GRN_TEXT_LEN(&(data->typed.foot)); @@ -1012,10 +986,10 @@ ngx_http_groonga_handler_get(ngx_http_request_t *r) return rc; } - rc = ngx_http_groonga_handler_process_command_path(r, &command_path, data); - if (rc != NGX_OK) { - return rc; - } + ngx_http_groonga_handler_process_command_path(r, + &command_path, + data, + GRN_CTX_TAIL); /* discard request body, since we don't need it here */ rc = ngx_http_discard_request_body(r); @@ -1058,13 +1032,8 @@ ngx_http_groonga_handler_post(ngx_http_request_t *r) return; } - rc = ngx_http_groonga_handler_process_load(r, &command_path, data); - if (rc != NGX_OK) { - ngx_http_groonga_handler_post_send_error_response(r, rc); - return; - } - - ngx_http_groonga_handler_send_response(r, data); + ngx_http_groonga_handler_process_load(r, &command_path, data); + rc = ngx_http_groonga_handler_send_response(r, data); ngx_http_finalize_request(r, rc); } @@ -1162,29 +1131,9 @@ ngx_http_groonga_conf_set_log_level_slot(ngx_conf_t *cf, ngx_command_t *cmd, value = ngx_str_null_terminate(cf->cycle->pool, ((ngx_str_t *)cf->args->elts) + 1); - if (strcasecmp(value, "none") == 0) { - groonga_location_conf->log_level = GRN_LOG_NONE; - } else if (strcasecmp(value, "emergency") == 0) { - groonga_location_conf->log_level = GRN_LOG_EMERG; - } else if (strcasecmp(value, "alert") == 0) { - groonga_location_conf->log_level = GRN_LOG_ALERT; - } else if (strcasecmp(value, "critical") == 0) { - groonga_location_conf->log_level = GRN_LOG_CRIT; - } else if (strcasecmp(value, "error") == 0) { - groonga_location_conf->log_level = GRN_LOG_ERROR; - } else if (strcasecmp(value, "warning") == 0) { - groonga_location_conf->log_level = GRN_LOG_WARNING; - } else if (strcasecmp(value, "notice") == 0) { - groonga_location_conf->log_level = GRN_LOG_NOTICE; - } else if (strcasecmp(value, "info") == 0) { - groonga_location_conf->log_level = GRN_LOG_INFO; - } else if (strcasecmp(value, "debug") == 0) { - groonga_location_conf->log_level = GRN_LOG_DEBUG; - } else if (strcasecmp(value, "dump") == 0) { - groonga_location_conf->log_level = GRN_LOG_DUMP; - } else { + if (!grn_log_level_parse(value, &(groonga_location_conf->log_level))) { status = "must be one of 'none', 'emergency', 'alert', " - "'ciritical', 'error', 'warning', 'notice', 'info', 'debug' and 'dump'"; + "'critical', 'error', 'warning', 'notice', 'info', 'debug' and 'dump'"; } ngx_pfree(cf->cycle->pool, value); @@ -1216,7 +1165,7 @@ ngx_http_groonga_conf_set_query_log_path_slot(ngx_conf_t *cf, ngx_conf_open_file(cf->cycle, &(groonga_location_conf->query_log_path)); if (!groonga_location_conf->query_log_file) { ngx_log_error(NGX_LOG_ERR, cf->cycle->log, 0, - "http_groonga: failed to open groonga query log file: <%V>", + "http_groonga: failed to open Groonga query log file: <%V>", &(groonga_location_conf->query_log_path)); return NGX_CONF_ERROR; } @@ -1251,6 +1200,8 @@ ngx_http_groonga_create_loc_conf(ngx_conf_t *cf) conf->config_file = NULL; conf->config_line = 0; conf->cache = NULL; + conf->cache_base_path.data = NULL; + conf->cache_base_path.len = 0; return conf; } @@ -1260,6 +1211,11 @@ ngx_http_groonga_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) { ngx_http_groonga_loc_conf_t *prev = parent; ngx_http_groonga_loc_conf_t *conf = child; + ngx_flag_t enabled = 0; + + if (conf->enabled != NGX_CONF_UNSET) { + enabled = conf->enabled; + } ngx_conf_merge_str_value(conf->database_path, prev->database_path, NULL); ngx_conf_merge_value(conf->database_auto_create, @@ -1269,16 +1225,17 @@ ngx_http_groonga_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) GRN_CACHE_DEFAULT_MAX_N_ENTRIES); #ifdef NGX_HTTP_GROONGA_LOG_PATH - if (!conf->log_file) { - ngx_str_t default_log_path; - default_log_path.data = (u_char *)NGX_HTTP_GROONGA_LOG_PATH; - default_log_path.len = strlen(NGX_HTTP_GROONGA_LOG_PATH); - conf->log_file = ngx_conf_open_file(cf->cycle, &default_log_path); + ngx_conf_merge_str_value(conf->log_path, prev->log_path, + NGX_HTTP_GROONGA_LOG_PATH); + if (!conf->log_file && + ngx_str_is_custom_path(&(conf->log_path)) && + enabled) { + conf->log_file = ngx_conf_open_file(cf->cycle, &(conf->log_path)); if (!conf->log_file) { ngx_log_error(NGX_LOG_ERR, cf->cycle->log, 0, "http_groonga: " - "failed to open the default groonga log file: <%V>", - &default_log_path); + "failed to open the default Groonga log file: <%V>", + &(conf->log_path)); return NGX_CONF_ERROR; } } @@ -1288,18 +1245,22 @@ ngx_http_groonga_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) NGX_HTTP_GROONGA_QUERY_LOG_PATH); if (!conf->query_log_file && ngx_str_is_custom_path(&(conf->query_log_path)) && - conf->enabled) { + enabled) { conf->query_log_file = ngx_conf_open_file(cf->cycle, &(conf->query_log_path)); if (!conf->query_log_file) { ngx_log_error(NGX_LOG_ERR, cf->cycle->log, 0, "http_groonga: " - "failed to open the default groonga query log file: <%V>", + "failed to open the default Groonga query log file: <%V>", &(conf->query_log_path)); return NGX_CONF_ERROR; } } + ngx_conf_merge_str_value(conf->cache_base_path, + prev->cache_base_path, + NULL); + return NGX_CONF_OK; } @@ -1351,6 +1312,41 @@ ngx_http_groonga_each_loc_conf(ngx_http_conf_ctx_t *http_conf, ngx_http_groonga_each_loc_conf_in_tree(location_conf->static_locations, callback, user_data); + +#if NGX_PCRE + if (location_conf->regex_locations) { + ngx_uint_t j; + for (j = 0; location_conf->regex_locations[j]; j++) { + ngx_http_core_loc_conf_t *regex_location_conf; + + regex_location_conf = location_conf->regex_locations[j]; + if (regex_location_conf->handler == ngx_http_groonga_handler) { + callback(regex_location_conf->loc_conf[ngx_http_groonga_module.ctx_index], + user_data); + } + } + } +#endif + } +} + +static void +ngx_http_groonga_set_logger_callback(ngx_http_groonga_loc_conf_t *location_conf, + void *user_data) +{ + ngx_http_groonga_database_callback_data_t *data = user_data; + + data->rc = ngx_http_groonga_context_init_logger(location_conf, + data->pool, + data->log); + if (data->rc != NGX_OK) { + return; + } + data->rc = ngx_http_groonga_context_init_query_logger(location_conf, + data->pool, + data->log); + if (data->rc != NGX_OK) { + return; } } @@ -1387,7 +1383,6 @@ ngx_http_groonga_create_database(ngx_http_groonga_loc_conf_t *location_conf, ngx_http_groonga_database_callback_data_t *data) { const char *database_base_name; - grn_ctx *context; database_base_name = strrchr(location_conf->database_path_cstr, '/'); if (database_base_name) { @@ -1402,14 +1397,14 @@ ngx_http_groonga_create_database(ngx_http_groonga_loc_conf_t *location_conf, } } - context = &(location_conf->context); - grn_db_create(context, location_conf->database_path_cstr, NULL); + location_conf->database = + grn_db_create(context, location_conf->database_path_cstr, NULL); if (context->rc == GRN_SUCCESS) { return; } ngx_log_error(NGX_LOG_EMERG, data->log, 0, - "failed to create groonga database: %s", + "failed to create Groonga database: %s", context->errbuf); data->rc = NGX_ERROR; } @@ -1419,11 +1414,16 @@ ngx_http_groonga_open_database_callback(ngx_http_groonga_loc_conf_t *location_co void *user_data) { ngx_http_groonga_database_callback_data_t *data = user_data; - grn_ctx *context; - context = &(location_conf->context); - data->rc = ngx_http_groonga_context_init(context, location_conf, - data->pool, data->log); + data->rc = ngx_http_groonga_context_init_logger(location_conf, + data->pool, + data->log); + if (data->rc != NGX_OK) { + return; + } + data->rc = ngx_http_groonga_context_init_query_logger(location_conf, + data->pool, + data->log); if (data->rc != NGX_OK) { return; } @@ -1443,27 +1443,41 @@ ngx_http_groonga_open_database_callback(ngx_http_groonga_loc_conf_t *location_co ngx_str_null_terminate(data->pool, &(location_conf->database_path)); } - grn_db_open(context, location_conf->database_path_cstr); + location_conf->database = + grn_db_open(context, location_conf->database_path_cstr); if (context->rc != GRN_SUCCESS) { if (location_conf->database_auto_create) { ngx_http_groonga_create_database(location_conf, data); } else { ngx_log_error(NGX_LOG_EMERG, data->log, 0, - "failed to open groonga database: %s", + "failed to open Groonga database: %s", context->errbuf); data->rc = NGX_ERROR; + } + if (data->rc != NGX_OK) { return; } } - location_conf->cache = grn_cache_open(context); + if (location_conf->cache_base_path.data && + ngx_str_is_custom_path(&(location_conf->cache_base_path))) { + char cache_base_path[PATH_MAX]; + grn_memcpy(cache_base_path, + location_conf->cache_base_path.data, + location_conf->cache_base_path.len); + cache_base_path[location_conf->cache_base_path.len] = '\0'; + location_conf->cache = grn_persistent_cache_open(context, cache_base_path); + } else { + location_conf->cache = grn_cache_open(context); + } if (!location_conf->cache) { ngx_log_error(NGX_LOG_EMERG, data->log, 0, - "failed to open groonga cache: %s", + "failed to open Groonga cache: %s", context->errbuf); data->rc = NGX_ERROR; return; } + if (location_conf->cache_limit != NGX_CONF_UNSET_SIZE) { grn_cache_set_max_n_entries(context, location_conf->cache, @@ -1476,26 +1490,20 @@ ngx_http_groonga_close_database_callback(ngx_http_groonga_loc_conf_t *location_c void *user_data) { ngx_http_groonga_database_callback_data_t *data = user_data; - grn_ctx *context; - context = &(location_conf->context); - ngx_http_groonga_context_init_logger(context, - location_conf, + ngx_http_groonga_context_init_logger(location_conf, data->pool, data->log); - ngx_http_groonga_context_init_query_logger(context, - location_conf, + ngx_http_groonga_context_init_query_logger(location_conf, data->pool, data->log); grn_cache_current_set(context, location_conf->cache); - grn_obj_close(context, grn_ctx_db(context)); - ngx_http_groonga_context_log_error(data->log, context); + grn_obj_close(context, location_conf->database); + ngx_http_groonga_context_log_error(data->log); grn_cache_current_set(context, NULL); grn_cache_close(context, location_conf->cache); - - grn_ctx_fin(context); } static ngx_int_t @@ -1505,12 +1513,11 @@ ngx_http_groonga_init_process(ngx_cycle_t *cycle) ngx_http_conf_ctx_t *http_conf; ngx_http_groonga_database_callback_data_t data; - rc = grn_init(); - if (rc != GRN_SUCCESS) { - return NGX_ERROR; - } + grn_thread_set_get_limit_func(ngx_http_groonga_get_thread_limit, NULL); - grn_set_segv_handler(); +#ifdef NGX_HTTP_GROONGA_LOG_PATH + grn_default_logger_set_path(NGX_HTTP_GROONGA_LOG_PATH); +#endif http_conf = (ngx_http_conf_ctx_t *)ngx_get_conf(cycle->conf_ctx, ngx_http_module); @@ -1519,6 +1526,26 @@ ngx_http_groonga_init_process(ngx_cycle_t *cycle) data.pool = cycle->pool; data.rc = NGX_OK; ngx_http_groonga_each_loc_conf(http_conf, + ngx_http_groonga_set_logger_callback, + &data); + + if (data.rc != NGX_OK) { + return data.rc; + } + + rc = grn_init(); + if (rc != GRN_SUCCESS) { + return NGX_ERROR; + } + + grn_set_segv_handler(); + + rc = grn_ctx_init(context, GRN_NO_FLAGS); + if (rc != GRN_SUCCESS) { + return NGX_ERROR; + } + + ngx_http_groonga_each_loc_conf(http_conf, ngx_http_groonga_open_database_callback, &data); @@ -1539,6 +1566,8 @@ ngx_http_groonga_exit_process(ngx_cycle_t *cycle) ngx_http_groonga_close_database_callback, &data); + grn_ctx_fin(context); + grn_fin(); return; @@ -1602,6 +1631,20 @@ static ngx_command_t ngx_http_groonga_commands[] = { offsetof(ngx_http_groonga_loc_conf_t, cache_limit), NULL }, + { ngx_string("groonga_default_request_timeout"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, + ngx_conf_set_msec_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_groonga_loc_conf_t, default_request_timeout_msec), + NULL }, + + { ngx_string("groonga_cache_base_path"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, + ngx_conf_set_str_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_groonga_loc_conf_t, cache_base_path), + NULL }, + ngx_null_command }; diff --git a/storage/mroonga/vendor/groonga/src/suggest/CMakeLists.txt b/storage/mroonga/vendor/groonga/src/suggest/CMakeLists.txt index c0c7a9c9964..83ae26b8ce5 100644 --- a/storage/mroonga/vendor/groonga/src/suggest/CMakeLists.txt +++ b/storage/mroonga/vendor/groonga/src/suggest/CMakeLists.txt @@ -15,7 +15,8 @@ include_directories( ${CMAKE_CURRENT_SOURCE_DIR}/../../lib - ${MRUBY_INCLUDE_DIRS}) + ${MRUBY_INCLUDE_DIRS} + ${MESSAGE_PACK_INCLUDE_DIRS}) read_file_list(${CMAKE_CURRENT_SOURCE_DIR}/create_dataset_sources.am GROONGA_SUGGEST_CREATE_DATASET_SOURCES) diff --git a/storage/mroonga/vendor/groonga/src/suggest/Makefile.am b/storage/mroonga/vendor/groonga/src/suggest/Makefile.am index cecf4001e9a..91260016fa8 100644 --- a/storage/mroonga/vendor/groonga/src/suggest/Makefile.am +++ b/storage/mroonga/vendor/groonga/src/suggest/Makefile.am @@ -2,7 +2,6 @@ bin_PROGRAMS = NONEXISTENT_CXX_SOURCE = nonexistent.cpp -if !PLATFORM_WIN32 bin_PROGRAMS += \ groonga-suggest-create-dataset @@ -13,8 +12,6 @@ bin_PROGRAMS += \ noinst_LTLIBRARIES = libutil.la endif -endif - EXTRA_DIST = \ CMakeLists.txt diff --git a/storage/mroonga/vendor/groonga/src/suggest/groonga_suggest_create_dataset.c b/storage/mroonga/vendor/groonga/src/suggest/groonga_suggest_create_dataset.c index d566d24b96a..7220ca88ec5 100644 --- a/storage/mroonga/vendor/groonga/src/suggest/groonga_suggest_create_dataset.c +++ b/storage/mroonga/vendor/groonga/src/suggest/groonga_suggest_create_dataset.c @@ -1,5 +1,5 @@ /* -*- c-basic-offset: 2 -*- */ -/* Copyright(C) 2010-2013 Brazil +/* Copyright(C) 2010-2015 Brazil This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public @@ -15,14 +15,14 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ +/* For grn_str_getopt() */ +#include <grn_str.h> + #include <stdio.h> #include <stdlib.h> #include <string.h> #include <groonga.h> -/* For grn_str_getopt() */ -#include <grn_str.h> - typedef enum { MODE_NONE, MODE_USAGE @@ -151,26 +151,29 @@ main(int argc, char **argv) grn_obj text; GRN_TEXT_INIT(&text, 0); #define SEND(string) send_command(ctx, &text, string, dataset_name) - SEND("register suggest/suggest"); + SEND("plugin_register suggest/suggest"); SEND("table_create event_type TABLE_HASH_KEY ShortText"); { grn_obj query; GRN_TEXT_INIT(&query, 0); GRN_TEXT_PUTS(ctx, &query, - "table_create bigram TABLE_PAT_KEY|KEY_NORMALIZE ShortText " + "table_create bigram TABLE_PAT_KEY ShortText " "--default_tokenizer "); if (default_tokenizer) { GRN_TEXT_PUTS(ctx, &query, default_tokenizer); } else { GRN_TEXT_PUTS(ctx, &query, DEFAULT_DEFAULT_TOKENIZER); } + GRN_TEXT_PUTS(ctx, &query, " --normalizer NormalizerAuto"); GRN_TEXT_PUTC(ctx, &query, '\0'); SEND(GRN_TEXT_VALUE(&query)); GRN_OBJ_FIN(ctx, &query); } - SEND("table_create kana TABLE_PAT_KEY|KEY_NORMALIZE ShortText"); - SEND("table_create item_${DATASET} TABLE_PAT_KEY|KEY_NORMALIZE " - "ShortText --default_tokenizer TokenDelimit"); + SEND("table_create kana TABLE_PAT_KEY ShortText " + "--normalizer NormalizerAuto"); + SEND("table_create item_${DATASET} TABLE_PAT_KEY " + "ShortText --default_tokenizer TokenDelimit " + "--normalizer NormalizerAuto"); SEND("column_create bigram item_${DATASET}_key " "COLUMN_INDEX|WITH_POSITION item_${DATASET} _key"); SEND("column_create item_${DATASET} kana COLUMN_VECTOR kana"); diff --git a/storage/mroonga/vendor/groonga/src/suggest/groonga_suggest_httpd.c b/storage/mroonga/vendor/groonga/src/suggest/groonga_suggest_httpd.c index f3127288c73..d42ead2c907 100644 --- a/storage/mroonga/vendor/groonga/src/suggest/groonga_suggest_httpd.c +++ b/storage/mroonga/vendor/groonga/src/suggest/groonga_suggest_httpd.c @@ -212,7 +212,7 @@ log_send(struct evkeyvalq *output_headers, struct evbuffer *res_buf, zmq_msg_t msg; if (!zmq_msg_init_size(&msg, sbuf.size)) { memcpy((void *)zmq_msg_data(&msg), sbuf.data, sbuf.size); - if (zmq_msg_send(&msg, thd->zmq_sock, 0)) { + if (zmq_msg_send(&msg, thd->zmq_sock, 0) == -1) { print_error("zmq_msg_send() error"); } zmq_msg_close(&msg); @@ -512,7 +512,7 @@ recv_handler(grn_ctx *ctx, void *zmq_recv_sock, msgpack_zone *mempool, grn_obj * if (zmq_msg_init(&msg)) { print_error("cannot init zmq message."); } else { - if (zmq_msg_recv(&msg, zmq_recv_sock, 0)) { + if (zmq_msg_recv(&msg, zmq_recv_sock, 0) == -1) { print_error("cannot recv zmq message."); } else { msgpack_object obj; diff --git a/storage/mroonga/vendor/groonga/src/suggest/groonga_suggest_learner.c b/storage/mroonga/vendor/groonga/src/suggest/groonga_suggest_learner.c index 03d889f5b01..74465beffdd 100644 --- a/storage/mroonga/vendor/groonga/src/suggest/groonga_suggest_learner.c +++ b/storage/mroonga/vendor/groonga/src/suggest/groonga_suggest_learner.c @@ -207,7 +207,7 @@ zmq_send_to_httpd(void *zmq_send_sock, void *data, size_t size) zmq_msg_t msg; if (!zmq_msg_init_size(&msg, size)) { memcpy((void *)zmq_msg_data(&msg), data, size); - if (zmq_msg_send(&msg, zmq_send_sock, 0)) { + if (zmq_msg_send(&msg, zmq_send_sock, 0) == -1) { print_error("zmq_send() error"); return -1; } @@ -481,7 +481,7 @@ recv_event_loop(msgpack_zone *mempool, void *zmq_sock, grn_ctx *ctx) if (zmq_msg_init(&msg)) { print_error("cannot init zmq message."); } else { - if (zmq_msg_recv(&msg, zmq_sock, 0)) { + if (zmq_msg_recv(&msg, zmq_sock, 0) == -1) { print_error("cannot recv zmq message."); } else { msgpack_object obj; |