summaryrefslogtreecommitdiff
path: root/storage/tokudb/PerconaFT/src/tests/update-multiple-with-indexer-array.cc
diff options
context:
space:
mode:
Diffstat (limited to 'storage/tokudb/PerconaFT/src/tests/update-multiple-with-indexer-array.cc')
-rw-r--r--storage/tokudb/PerconaFT/src/tests/update-multiple-with-indexer-array.cc459
1 files changed, 459 insertions, 0 deletions
diff --git a/storage/tokudb/PerconaFT/src/tests/update-multiple-with-indexer-array.cc b/storage/tokudb/PerconaFT/src/tests/update-multiple-with-indexer-array.cc
new file mode 100644
index 00000000000..c79bca04d5e
--- /dev/null
+++ b/storage/tokudb/PerconaFT/src/tests/update-multiple-with-indexer-array.cc
@@ -0,0 +1,459 @@
+/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
+#ident "$Id$"
+/*======
+This file is part of PerconaFT.
+
+
+Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved.
+
+ PerconaFT is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License, version 2,
+ as published by the Free Software Foundation.
+
+ PerconaFT is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with PerconaFT. If not, see <http://www.gnu.org/licenses/>.
+
+----------------------------------------
+
+ PerconaFT is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License, version 3,
+ as published by the Free Software Foundation.
+
+ PerconaFT is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with PerconaFT. If not, see <http://www.gnu.org/licenses/>.
+======= */
+
+#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved."
+
+#include "test.h"
+
+// verify that update_multiple where we change the data in row[i] col[j] from x to x+1
+
+static const int MAX_KEYS = 3;
+
+static int
+array_size(int ndbs) {
+ return +
+ 1 + // 0 for old 1 for new
+ 1 + // ndbs
+ 2 * MAX_KEYS * (ndbs-1);
+}
+static int
+get_num_new_keys(int i, int dbnum) {
+ if (dbnum == 0) return 1;
+ if (i & (1<<4)) {
+ dbnum++; // Shift every once in a while.
+ }
+ return (i + dbnum) % MAX_KEYS; // 0, 1, or 2
+}
+
+static int
+get_old_num_keys(int i, int dbnum) {
+ if (dbnum == 0) return 1;
+ return (i + dbnum) % MAX_KEYS; // 0, 1, or 2
+}
+
+static int
+get_total_secondary_rows(int num_primary) {
+ assert(num_primary % MAX_KEYS == 0);
+ return num_primary / MAX_KEYS * (0 + 1 + 2);
+}
+
+static int
+get_old_key(int i, int dbnum, int which) {
+ assert(i < INT16_MAX / 2);
+ assert(which >= 0);
+ assert(which < 4);
+ assert(dbnum < 16);
+ if (dbnum == 0) {
+ assert(which == 0);
+ return htonl(2*i);
+ }
+ if (which >= get_old_num_keys(i, dbnum)) {
+ return htonl(-1);
+ }
+ return htonl(((2*i+0) << 16) + (dbnum<<8) + (which<<1));
+}
+
+static int
+get_new_key(int i, int dbnum, int which) {
+ assert(which >= 0);
+ assert(which < 4);
+ assert(dbnum < 16);
+
+ if (dbnum == 0) {
+ assert(which == 0);
+ return htonl(2*i);
+ }
+ if (which >= get_num_new_keys(i, dbnum)) {
+ return htonl(-1);
+ }
+ if ((i+dbnum+which) & (1<<5)) {
+ return htonl(((2*i+0) << 16) + (dbnum<<8) + (which<<1)); // no change from original
+ }
+ return htonl(((2*i+0) << 16) + (dbnum<<8) + (which<<1) + 1);
+}
+
+static void
+fill_data_2_and_later(int *v, int i, int ndbs) {
+ int index = 2;
+ for (int dbnum = 1; dbnum < ndbs; dbnum++) {
+ for (int which = 0; which < MAX_KEYS; ++which) {
+ v[index++] = get_old_key(i, dbnum, which);
+ }
+ }
+ for (int dbnum = 1; dbnum < ndbs; dbnum++) {
+ for (int which = 0; which < MAX_KEYS; ++which) {
+ v[index++] = get_new_key(i, dbnum, which);
+ }
+ }
+}
+
+
+static void
+fill_old_data(int *v, int i, int ndbs) {
+ v[0] = 0;
+ v[1] = ndbs;
+ fill_data_2_and_later(v, i, ndbs);
+}
+
+static void
+fill_new_data(int *v, int i, int ndbs) {
+ v[0] = 1;
+ v[1] = ndbs;
+ fill_data_2_and_later(v, i, ndbs);
+}
+
+
+static int
+put_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_key_arrays, DBT_ARRAY *dest_val_arrays, const DBT *src_key, const DBT *src_val) {
+ (void)src_val;
+ assert(src_db != dest_db);
+ assert(src_db);
+ int dbnum;
+ assert(dest_db->descriptor->dbt.size == sizeof dbnum);
+ memcpy(&dbnum, dest_db->descriptor->dbt.data, sizeof dbnum);
+ assert(dbnum > 0);
+
+ int pri_key = *(int *) src_key->data;
+ int* pri_val = (int*) src_val->data;
+
+ bool is_new = pri_val[0] == 1;
+ int i = (ntohl(pri_key)) / 2;
+
+ int num_keys = is_new ? get_num_new_keys(i, dbnum) : get_old_num_keys(i, dbnum);
+
+ toku_dbt_array_resize(dest_key_arrays, num_keys);
+
+ if (dest_val_arrays) {
+ toku_dbt_array_resize(dest_val_arrays, num_keys);
+ }
+
+ int ndbs = pri_val[1];
+ int index = 2 + (dbnum-1)*MAX_KEYS;
+ if (is_new) {
+ index += MAX_KEYS*(ndbs-1);
+ }
+
+ assert(src_val->size % sizeof(int) == 0);
+ assert((int)src_val->size / 4 >= index + num_keys);
+
+
+ for (int which = 0; which < num_keys; which++) {
+ DBT *dest_key = &dest_key_arrays->dbts[which];
+ DBT *dest_val = NULL;
+
+ assert(dest_key->flags == DB_DBT_REALLOC);
+ if (dest_key->ulen < sizeof(int)) {
+ dest_key->data = toku_xrealloc(dest_key->data, sizeof(int));
+ dest_key->ulen = sizeof(int);
+ }
+ dest_key->size = sizeof(int);
+ if (dest_val_arrays) {
+ dest_val = &dest_val_arrays->dbts[which];
+ assert(dest_val->flags == DB_DBT_REALLOC);
+ dest_val->size = 0;
+ }
+ int new_key = is_new ? get_new_key(i, dbnum, which) : get_old_key(i, dbnum, which);
+ assert(new_key == pri_val[index + which]);
+ *(int*)dest_key->data = new_key;
+ }
+ return 0;
+}
+
+static int
+del_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_key_arrays, const DBT *src_key, const DBT *src_data) {
+ return put_callback(dest_db, src_db, dest_key_arrays, NULL, src_key, src_data);
+}
+
+static void
+do_updates(DB_ENV *env, DB *db[], int ndbs, int nrows) {
+ assert(ndbs > 0);
+ int r;
+ DB_TXN *txn = NULL;
+ r = env->txn_begin(env, NULL, &txn, 0); assert_zero(r);
+ int narrays = 2 * ndbs;
+ DBT_ARRAY keys[narrays];
+ DBT_ARRAY vals[narrays];
+ for (int i = 0; i < narrays; i++) {
+ toku_dbt_array_init(&keys[i], 1);
+ toku_dbt_array_init(&vals[i], 1);
+ }
+
+ for (int i = 0; i < nrows; i++) {
+
+ // update the data i % ndbs col from x to x+1
+
+ int old_k = get_old_key(i, 0, 0);
+ DBT old_key; dbt_init(&old_key, &old_k, sizeof old_k);
+ int new_k = get_new_key(i, 0, 0);
+ DBT new_key; dbt_init(&new_key, &new_k, sizeof new_k);
+
+ int v[array_size(ndbs)]; fill_old_data(v, i, ndbs);
+ DBT old_data; dbt_init(&old_data, &v[0], sizeof v);
+
+ int newv[array_size(ndbs)]; fill_new_data(newv, i, ndbs);
+ DBT new_data; dbt_init(&new_data, &newv[0], sizeof newv);
+
+ uint32_t flags_array[ndbs]; memset(flags_array, 0, sizeof(flags_array));
+
+ r = env->update_multiple(env, db[0], txn, &old_key, &old_data, &new_key, &new_data, ndbs, db, flags_array, narrays, keys, narrays, vals);
+ assert_zero(r);
+ }
+ for (int i = 0; i < narrays; i++) {
+ toku_dbt_array_destroy(&keys[i]);
+ toku_dbt_array_destroy(&vals[i]);
+ }
+ r = txn->commit(txn, 0); assert_zero(r);
+}
+
+static void
+populate_primary(DB_ENV *env, DB *db, int ndbs, int nrows) {
+ int r;
+ DB_TXN *txn = NULL;
+ r = env->txn_begin(env, NULL, &txn, 0); assert_zero(r);
+
+ // populate
+ for (int i = 0; i < nrows; i++) {
+ int k = get_old_key(i, 0, 0);
+ int v[array_size(ndbs)];
+ fill_old_data(v, i, ndbs);
+ DBT key; dbt_init(&key, &k, sizeof k);
+ DBT val; dbt_init(&val, &v[0], sizeof v);
+ r = db->put(db, txn, &key, &val, 0); assert_zero(r);
+ }
+
+ r = txn->commit(txn, 0); assert_zero(r);
+}
+
+static void
+populate_secondary(DB_ENV *env, DB *db, int dbnum, int nrows) {
+ int r;
+ DB_TXN *txn = NULL;
+ r = env->txn_begin(env, NULL, &txn, 0); assert_zero(r);
+
+ // populate
+ for (int i = 0; i < nrows; i++) {
+ for (int which = 0; which < MAX_KEYS; which++) {
+ int k = get_old_key(i, dbnum, which);
+ if (k >= 0) {
+ DBT key; dbt_init(&key, &k, sizeof k);
+ DBT val; dbt_init(&val, NULL, 0);
+ r = db->put(db, txn, &key, &val, 0); assert_zero(r);
+ }
+ }
+ }
+
+ r = txn->commit(txn, 0); assert_zero(r);
+}
+
+static void
+verify_pri_seq(DB_ENV *env, DB *db, int ndbs, int nrows) {
+ const int dbnum = 0;
+ int r;
+ DB_TXN *txn = NULL;
+ r = env->txn_begin(env, NULL, &txn, 0); assert_zero(r);
+
+ DBC *cursor = NULL;
+ r = db->cursor(db, txn, &cursor, 0); assert_zero(r);
+ int i;
+ for (i = 0; ; i++) {
+ DBT key; memset(&key, 0, sizeof key);
+ DBT val; memset(&val, 0, sizeof val);
+ r = cursor->c_get(cursor, &key, &val, DB_NEXT);
+ if (r != 0)
+ break;
+ int k;
+ int expectk = get_new_key(i, dbnum, 0);
+
+ assert(key.size == sizeof k);
+ memcpy(&k, key.data, key.size);
+ assert(k == expectk);
+
+ int num_keys = array_size(ndbs);
+ assert(val.size == num_keys*sizeof(int));
+ int v[num_keys]; fill_new_data(v, i, ndbs);
+ assert(memcmp(val.data, v, val.size) == 0);
+ }
+ assert(i == nrows); // if (i != nrows) printf("%s:%d %d %d\n", __FUNCTION__, __LINE__, i, nrows); // assert(i == nrows);
+ r = cursor->c_close(cursor); assert_zero(r);
+ r = txn->commit(txn, 0); assert_zero(r);
+}
+
+static void
+verify_sec_seq(DB_ENV *env, DB *db, int dbnum, int nrows) {
+ assert(dbnum > 0);
+ int r;
+ DB_TXN *txn = NULL;
+ r = env->txn_begin(env, NULL, &txn, 0); assert_zero(r);
+
+ DBC *cursor = NULL;
+ r = db->cursor(db, txn, &cursor, 0); assert_zero(r);
+ int i;
+ int rows_found = 0;
+
+ for (i = 0; ; i++) {
+ int num_keys = get_num_new_keys(i, dbnum);
+ for (int which = 0; which < num_keys; ++which) {
+ DBT key; memset(&key, 0, sizeof key);
+ DBT val; memset(&val, 0, sizeof val);
+ r = cursor->c_get(cursor, &key, &val, DB_NEXT);
+ if (r != 0) {
+ CKERR2(r, DB_NOTFOUND);
+ goto done;
+ }
+ rows_found++;
+ int k;
+ int expectk = get_new_key(i, dbnum, which);
+
+ assert(key.size == sizeof k);
+ memcpy(&k, key.data, key.size);
+ int got_i = (ntohl(k) >> 16) / 2;
+ if (got_i < i) {
+ // Will fail. Too many old i's
+ assert(k == expectk);
+ } else if (got_i > i) {
+ // Will fail. Too few in previous i.
+ assert(k == expectk);
+ }
+
+ if (k != expectk && which < get_old_num_keys(i, dbnum) && k == get_old_key(i, dbnum, which)) {
+ // Will fail, never got updated.
+ assert(k == expectk);
+ }
+ assert(k == expectk);
+ assert(val.size == 0);
+ }
+ }
+done:
+ assert(rows_found == get_total_secondary_rows(nrows));
+ r = cursor->c_close(cursor); assert_zero(r);
+ r = txn->commit(txn, 0); assert_zero(r);
+}
+
+static void
+run_test(int ndbs, int nrows) {
+ int r;
+ DB_ENV *env = NULL;
+ r = db_env_create(&env, 0); assert_zero(r);
+
+ r = env->set_generate_row_callback_for_put(env, put_callback); assert_zero(r);
+ r = env->set_generate_row_callback_for_del(env, del_callback); assert_zero(r);
+
+ r = env->open(env, TOKU_TEST_FILENAME, DB_INIT_MPOOL|DB_CREATE|DB_THREAD |DB_INIT_LOCK|DB_INIT_LOG|DB_INIT_TXN|DB_PRIVATE, S_IRWXU+S_IRWXG+S_IRWXO); assert_zero(r);
+
+ DB *db[ndbs];
+ for (int dbnum = 0; dbnum < ndbs; dbnum++) {
+ r = db_create(&db[dbnum], env, 0); assert_zero(r);
+
+ DBT dbt_dbnum; dbt_init(&dbt_dbnum, &dbnum, sizeof dbnum);
+
+ char dbname[32]; sprintf(dbname, "%d.tdb", dbnum);
+ r = db[dbnum]->open(db[dbnum], NULL, dbname, NULL, DB_BTREE, DB_AUTO_COMMIT+DB_CREATE, S_IRWXU+S_IRWXG+S_IRWXO); assert_zero(r);
+ IN_TXN_COMMIT(env, NULL, txn_desc, 0, {
+ { int chk_r = db[dbnum]->change_descriptor(db[dbnum], txn_desc, &dbt_dbnum, 0); CKERR(chk_r); }
+ });
+ }
+
+ populate_primary(env, db[0], ndbs, nrows);
+ for (int dbnum = 1; dbnum < ndbs-1; dbnum++) {
+ populate_secondary(env, db[dbnum], dbnum, nrows);
+ }
+
+ DB_TXN *indexer_txn = NULL;
+ r = env->txn_begin(env, NULL, &indexer_txn, 0); assert_zero(r);
+
+ DB_INDEXER *indexer = NULL;
+ uint32_t db_flags = 0;
+ assert(ndbs > 2);
+ r = env->create_indexer(env, indexer_txn, &indexer, db[0], 1, &db[ndbs-1], &db_flags, 0); assert_zero(r);
+
+ do_updates(env, db, ndbs, nrows);
+
+ r = indexer->build(indexer); assert_zero(r);
+ r = indexer->close(indexer); assert_zero(r);
+
+ r = indexer_txn->commit(indexer_txn, 0); assert_zero(r);
+
+ verify_pri_seq(env, db[0], ndbs, nrows);
+ for (int dbnum = 1; dbnum < ndbs; dbnum++)
+ verify_sec_seq(env, db[dbnum], dbnum, nrows);
+ for (int dbnum = 0; dbnum < ndbs; dbnum++)
+ r = db[dbnum]->close(db[dbnum], 0); assert_zero(r);
+
+ r = env->close(env, 0); assert_zero(r);
+}
+
+int
+test_main(int argc, char * const argv[]) {
+ int r;
+ int ndbs = 10;
+ int nrows = MAX_KEYS*(1<<5)*4;
+
+ // parse_args(argc, argv);
+ for (int i = 1; i < argc; i++) {
+ char * const arg = argv[i];
+ if (strcmp(arg, "-v") == 0) {
+ verbose++;
+ continue;
+ }
+ if (strcmp(arg, "-q") == 0) {
+ verbose = 0;
+ continue;
+ }
+ if (strcmp(arg, "--ndbs") == 0 && i+1 < argc) {
+ ndbs = atoi(argv[++i]);
+ continue;
+ }
+ if (strcmp(arg, "--nrows") == 0 && i+1 < argc) {
+ nrows = atoi(argv[++i]);
+ continue;
+ }
+ }
+ while (nrows % (MAX_KEYS*(1<<5)) != 0) {
+ nrows++;
+ }
+ //Need at least one to update, and one to index
+ while (ndbs < 3) {
+ ndbs++;
+ }
+
+ toku_os_recursive_delete(TOKU_TEST_FILENAME);
+ r = toku_os_mkdir(TOKU_TEST_FILENAME, S_IRWXU+S_IRWXG+S_IRWXO); assert_zero(r);
+
+ run_test(ndbs, nrows);
+
+ return 0;
+}
+
View Lorry Controller Status