diff options
Diffstat (limited to 'src/key-value-store')
| -rw-r--r-- | src/key-value-store/crc32.c | 170 | ||||
| -rw-r--r-- | src/key-value-store/crc32.h | 43 | ||||
| -rw-r--r-- | src/key-value-store/database/kissdb.c | 1511 | ||||
| -rw-r--r-- | src/key-value-store/database/kissdb.h | 323 | ||||
| -rw-r--r-- | src/key-value-store/hashtable/md5.h | 50 | ||||
| -rw-r--r-- | src/key-value-store/hashtable/md5c.c | 296 | ||||
| -rw-r--r-- | src/key-value-store/hashtable/qhash.c | 155 | ||||
| -rw-r--r-- | src/key-value-store/hashtable/qhash.h | 64 | ||||
| -rw-r--r-- | src/key-value-store/hashtable/qhasharr.c | 869 | ||||
| -rw-r--r-- | src/key-value-store/hashtable/qhasharr.h | 136 | ||||
| -rw-r--r-- | src/key-value-store/hashtable/qlibc.h | 55 | ||||
| -rw-r--r-- | src/key-value-store/hashtable/qtype.h | 123 | ||||
| -rw-r--r-- | src/key-value-store/pers_low_level_db_access.c | 2053 |
13 files changed, 5848 insertions, 0 deletions
diff --git a/src/key-value-store/crc32.c b/src/key-value-store/crc32.c new file mode 100644 index 0000000..caa58e4 --- /dev/null +++ b/src/key-value-store/crc32.c @@ -0,0 +1,170 @@ +/*- + * COPYRIGHT (C) 1986 Gary S. Brown. You may use this program, or + * code or tables extracted from it, as desired without restriction. + * + * First, the polynomial itself and its table of feedback terms. The + * polynomial is + * X^32+X^26+X^23+X^22+X^16+X^12+X^11+X^10+X^8+X^7+X^5+X^4+X^2+X^1+X^0 + * + * Note that we take it "backwards" and put the highest-order term in + * the lowest-order bit. The X^32 term is "implied"; the LSB is the + * X^31 term, etc. The X^0 term (usually shown as "+1") results in + * the MSB being 1 + * + * Note that the usual hardware shift register implementation, which + * is what we're using (we're merely optimizing it by doing eight-bit + * chunks at a time) shifts bits into the lowest-order term. In our + * implementation, that means shifting towards the right. Why do we + * do it this way? Because the calculated CRC must be transmitted in + * order from highest-order term to lowest-order term. UARTs transmit + * characters in order from LSB to MSB. By storing the CRC this way + * we hand it to the UART in the order low-byte to high-byte; the UART + * sends each low-bit to hight-bit; and the result is transmission bit + * by bit from highest- to lowest-order term without requiring any bit + * shuffling on our part. Reception works similarly + * + * The feedback terms table consists of 256, 32-bit entries. Notes + * + * The table can be generated at runtime if desired; code to do so + * is shown later. It might not be obvious, but the feedback + * terms simply represent the results of eight shift/xor opera + * tions for all combinations of data and CRC register values + * + * The values must be right-shifted by eight bits by the "updcrc + * logic; the shift must be unsigned (bring in zeroes). On some + * hardware you could probably optimize the shift in assembler by + * using byte-swap instructions + * polynomial $edb88320 + * + * + * CRC32 code derived from work by Gary S. Brown. + */ + + +/* + * File has been copied from: + * http://www.opensource.apple.com/source/xnu/xnu-1456.1.26/bsd/libkern/crc32.c + */ + +/* + * Modified parts of this files by XS Embedded GmbH, 2014 + */ + + +#include "crc32.h" +#include <stdio.h> +#include <sys/stat.h> +#include <stdlib.h> +#include <unistd.h> + +enum crc32ConstantDefinition +{ + crc32_array_size = 255 +}; + + +static unsigned int crc32_tab[] = +{ + 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, + 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, + 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, 0x1db71064, 0x6ab020f2, + 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, + 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, + 0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, + 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, 0x35b5a8fa, 0x42b2986c, + 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, + 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, + 0xcfba9599, 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, + 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190, 0x01db7106, + 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, + 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, + 0x91646c97, 0xe6635c01, 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, + 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950, + 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, + 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, + 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, + 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, + 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, + 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, + 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, + 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, 0xe3630b12, 0x94643b84, + 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, + 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, + 0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, + 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, 0xd6d6a3e8, 0xa1d1937e, + 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, + 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, + 0x316e8eef, 0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, + 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, 0xb2bd0b28, + 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, + 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, + 0x72076785, 0x05005713, 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, + 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242, + 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, + 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, + 0x616bffd3, 0x166ccf45, 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, + 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, + 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, + 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, + 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, + 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d +}; + +unsigned int pcoCrc32(unsigned int crc, const unsigned char *buf, size_t theSize) +{ + const unsigned char *p = 0; + unsigned int rval = 0; + + p = buf; + crc = crc ^ ~0U; + + if(p != 0) + { + while(theSize--) + { + unsigned int idx = (crc ^ *p++) & 0xFF; + + if(idx < crc32_array_size) + crc = crc32_tab[idx] ^ (crc >> 8); + } + rval = crc ^ ~0U; + } + return rval; +} + + +int pcoCalcCrc32Csum(int fd, int startOffset) +{ + int rval = 1; + char* buf; + struct stat statBuf; + unsigned int crc = 0; + + fstat(fd, &statBuf); + buf = malloc((unsigned int) statBuf.st_size- startOffset); + + if (buf != 0) + { + memset(buf, 0, statBuf.st_size- startOffset); + off_t curPos = 0; + // remember the current position + curPos = lseek(fd, 0, SEEK_CUR); + // set to start offset + lseek(fd, startOffset, SEEK_SET); + //printf("FSTAT -> Filesize: %d \n", (int) statBuf.st_size); + if( read(fd, buf, statBuf.st_size- startOffset) != statBuf.st_size- startOffset) + return -1; + crc = 0; + crc = pcoCrc32(crc, (unsigned char*) buf, statBuf.st_size- startOffset); + rval = crc; + // set back to the position + lseek(fd, curPos, SEEK_SET); + free(buf); + } + else + rval = -1; + return rval; +} + + + diff --git a/src/key-value-store/crc32.h b/src/key-value-store/crc32.h new file mode 100644 index 0000000..13ffa4c --- /dev/null +++ b/src/key-value-store/crc32.h @@ -0,0 +1,43 @@ +#ifndef CRC32_H +#define CRC32_H + +/****************************************************************************** + * Project Persistence key value store + * (c) copyright 2014 + * Company XS Embedded GmbH + *****************************************************************************/ +/****************************************************************************** + * Copyright + * + * This Source Code Form is subject to the terms of the + * Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed + * with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +******************************************************************************/ + /** + * @file crc32.h + * @ingroup Persistence key value store + * @author Simon Disch + * @brief Header of crc32 checksum generation + * @see + */ + +#ifdef __cplusplus +extern "C" { +#endif + + +#define PERS_COM_CRC32_INTERFACE_VERSION (0x01000000U) +#define CHKSUMBUFSIZE 64 + +#include <string.h> +#include <stdio.h> + +unsigned int pcoCrc32(unsigned int crc, const unsigned char *buf, size_t theSize); +int pcoCalcCrc32Csum(int fd, int startOffset); + + +#ifdef __cplusplus +} +#endif + +#endif /* CRC32_H */ diff --git a/src/key-value-store/database/kissdb.c b/src/key-value-store/database/kissdb.c new file mode 100644 index 0000000..4c8e7b6 --- /dev/null +++ b/src/key-value-store/database/kissdb.c @@ -0,0 +1,1511 @@ + /****************************************************************************** + * Project Persistency + * (c) copyright 2014 + * Company XS Embedded GmbH + *****************************************************************************/ +/* (Keep It) Simple Stupid Database +* +* Written by Adam Ierymenko <adam.ierymenko@zerotier.com> +* Modified by Simon Disch <simon.disch@xse.de> +* +* KISSDB is in the public domain and is distributed with NO WARRANTY. +* +* http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* Compile with KISSDB_TEST to build as a test program. */ + +/* Note: big-endian systems will need changes to implement byte swapping +* on hash table file I/O. Or you could just use it as-is if you don't care +* that your database files will be unreadable on little-endian systems. */ + +#define _FILE_OFFSET_BITS 64 +#define TMP_BUFFER_LENGTH 128 +#define KISSDB_HEADER_SIZE sizeof(Header_s) +#define __useBackups +//#define __useFileMapping +//#define __writeThrough +#define __checkerror + +#include "./kissdb.h" +#include "../crc32.h" +#include <string.h> +#include <stdlib.h> +#include <inttypes.h> +#include <stdint.h> +#include <fcntl.h> +#include <errno.h> +#include <sys/mman.h> +#include <unistd.h> +#include <ctype.h> +#include <sys/time.h> + +#include "dlt.h" + +DLT_DECLARE_CONTEXT(persComLldbDLTCtx); + +#ifdef __showTimeMeasurements +inline long long getNsDuration(struct timespec* start, struct timespec* end) +{ + return ((end->tv_sec * SECONDS2NANO) + end->tv_nsec) - ((start->tv_sec * SECONDS2NANO) + start->tv_nsec); +} +#endif + +/* djb2 hash function */ +static uint64_t KISSDB_hash(const void *b, unsigned long len) +{ + unsigned long i; + uint64_t hash = 5381; + for (i = 0; i < len; ++i) + hash = ((hash << 5) + hash) + (uint64_t) (((const uint8_t *) b)[i]); + return hash; +} + +//returns a name for shared memory objects beginning with a slash followed by "path" (non alphanumeric chars are replaced with '_') appended with "tailing" +char * kdbGetShmName(const char *tailing, const char * path) +{ + char * result = (char *) malloc(1 + strlen(path) + strlen(tailing) + 1); //free happens at lifecycle shutdown + int i =0; + int x = 1; + + if (result != NULL) + { + result[0] = '/'; + for (i = 0; i < strlen(path); i++) + { + if (!isalnum(path[i])) + result[i + 1] = '_'; + else + result[i + 1] = path[i]; + } + for (x = 0; x < strlen(tailing); x++) + { + result[i + x + 1] = tailing[x]; + } + result[i + x + 1] = '\0'; + } + return result; +} + +//returns -1 on error and positive value for success +int kdbShmemOpen(const char * name, size_t length, Kdb_bool* shmCreator) +{ + int result; + result = shm_open(name, O_CREAT | O_EXCL | O_RDWR, S_IRUSR | S_IWUSR); + if (result < 0) + { + if (errno == EEXIST) + { + *shmCreator = Kdb_false; + result = shm_open(name, O_CREAT | O_RDWR, S_IRUSR | S_IWUSR); + if (result < 0) + return -1; + } + } + else + { + *shmCreator = Kdb_true; + if (ftruncate(result, length) < 0) + return -1; + } + return result; +} + +void Kdb_wrlock(pthread_rwlock_t * wrlock) +{ + pthread_rwlock_wrlock(wrlock); +} + +void Kdb_rdlock(pthread_rwlock_t * rdlock) +{ + pthread_rwlock_rdlock(rdlock); +} + +void Kdb_unlock(pthread_rwlock_t * lock) +{ + pthread_rwlock_unlock(lock); +} + +Kdb_bool kdbShmemClose(int shmem, const char * shmName) +{ + if( close(shmem) == -1) + return Kdb_false; + if( shm_unlink(shmName) < 0) + return Kdb_false; + return Kdb_true; +} + +void * getKdbShmemPtr(int shmem, size_t length) +{ + void* result = mmap(NULL, length, PROT_READ | PROT_WRITE, MAP_SHARED, shmem, 0); + if (result == MAP_FAILED) + return ((void *) -1); + return result; +} + +Kdb_bool freeKdbShmemPtr(void * shmem_ptr, size_t length) +{ + if(munmap(shmem_ptr, length) == 0) + return Kdb_true; + else + return Kdb_false; +} + +Kdb_bool resizeKdbShmem(int shmem, Hashtable_slot_s** shmem_ptr, size_t oldLength, size_t newLength) +{ + //unmap shm with old size + if( freeKdbShmemPtr(*shmem_ptr, oldLength) == Kdb_false) + return Kdb_false; + + if (ftruncate(shmem, newLength) < 0) + return Kdb_false; + + //get pointer to resized shm with new Length + *shmem_ptr = getKdbShmemPtr(shmem, newLength); + if(*shmem_ptr == ((void *) -1)) + return Kdb_false; + return Kdb_true; +} + +#ifdef __writeThrough +Kdb_bool remapKdbShmem(int shmem, uint64_t** shmem_ptr, size_t oldLength, size_t newLength) +{ + //unmap shm with old size + if( freeKdbShmemPtr(*shmem_ptr, oldLength) == Kdb_false ) + return Kdb_false; + //get pointer to resized shm with new Length + *shmem_ptr = getKdbShmemPtr(shmem, newLength); + if(*shmem_ptr == ((void *) -1)) + return Kdb_false; + return Kdb_true; +} +#endif + + +int KISSDB_open(KISSDB *db, const char *path, int mode, uint16_t hash_table_size, uint64_t key_size, + uint64_t value_size) +{ + Hashtable_slot_s *httmp; + Kdb_bool tmp_creator; + int ret = 0; + + //TODO check if usage of O_SYNC O_DIRECT flags is needed. If O_SYNC and O_DIrect is specified, no additional fsync calls are needed after fflush + if(mode == KISSDB_OPEN_MODE_RWCREAT) + db->fd = open(path, O_CREAT | O_RDWR , S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH ); //gets closed when db->f is closed + else + db->fd = open(path, O_RDWR , S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH ); //gets closed when db->f is closed + + if(db->fd == -1) + return KISSDB_ERROR_IO; + + if (lseek(db->fd, 0, SEEK_END) == -1) + { + close(db->fd); + return KISSDB_ERROR_IO; + } + if (lseek(db->fd, 0, SEEK_CUR) < KISSDB_HEADER_SIZE) + { + /* write header if not already present */ + if ((hash_table_size) && (key_size) && (value_size)) + { + ret = writeHeader(db, &hash_table_size, &key_size, &value_size); + if(0 != ret) + { + close(db->fd); + return ret; + } + //Seek behind header + if (lseek(db->fd, KISSDB_HEADER_SIZE, SEEK_SET) == -1) + { + close(db->fd); + return KISSDB_ERROR_IO; + } + } + else + { + close(db->fd); + return KISSDB_ERROR_INVALID_PARAMETERS; + } + } + else + { + //read existing header + ret = readHeader(db, &hash_table_size, &key_size, &value_size); + if( 0 != ret) + return ret; + + if (lseek(db->fd, KISSDB_HEADER_SIZE, SEEK_SET) == -1) + { + close(db->fd); + return KISSDB_ERROR_IO; + } //Seek behind header + } + //store non shared db information + db->hash_table_size = hash_table_size; + db->key_size = key_size; + db->value_size = value_size; + db->hash_table_size_bytes = sizeof(Hashtable_slot_s) * (hash_table_size + 1); /* [hash_table_size] == next table */ + + //DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, DLT_STRING("Hashtable size in bytes: "), DLT_UINT64(db->hash_table_size_bytes)); + + if (db->already_open == Kdb_false) //check if this instance has already opened the db before + { + db->shmem_cached_name = kdbGetShmName("-cache", path); + if(db->shmem_cached_name == NULL) + return KISSDB_ERROR_MALLOC; + db->shmem_info_name = kdbGetShmName("-shm-info", path); + if(db->shmem_info_name == NULL) + return KISSDB_ERROR_MALLOC; + db->shmem_info_fd = kdbShmemOpen(db->shmem_info_name, sizeof(Shared_Data_s), &db->shmem_creator); + if(db->shmem_info_fd < 0) + return KISSDB_ERROR_OPEN_SHM; + db->shmem_info = (Shared_Data_s *) getKdbShmemPtr(db->shmem_info_fd, sizeof(Shared_Data_s)); + if(db->shmem_info == ((void *) -1)) + return KISSDB_ERROR_MAP_SHM; + + size_t first_mapping; + if(db->shmem_info->shmem_size > db->hash_table_size_bytes ) + first_mapping = db->shmem_info->shmem_size; + else + first_mapping = db->hash_table_size_bytes; + + //open / create shared memory for first hashtable + db->shmem_ht_name = kdbGetShmName("-ht", path); + if(db->shmem_ht_name == NULL) + return KISSDB_ERROR_MALLOC; + db->shmem_ht_fd = kdbShmemOpen(db->shmem_ht_name, first_mapping, &tmp_creator); + if(db->shmem_ht_fd < 0) + return KISSDB_ERROR_OPEN_SHM; + db->hash_tables = (Hashtable_slot_s *) getKdbShmemPtr(db->shmem_ht_fd, first_mapping); + if(db->hash_tables == ((void *) -1)) + return KISSDB_ERROR_MAP_SHM; + db->old_mapped_size = first_mapping; //local information + + //if shared memory for rwlock was opened (created) with this call to KISSDB_open for the first time -> init rwlock + if (db->shmem_creator == Kdb_true) + { + //[Initialize rwlock attributes] + pthread_rwlockattr_t rwlattr, cache_rwlattr; + pthread_rwlockattr_init(&rwlattr); + pthread_rwlockattr_init(&cache_rwlattr); + pthread_rwlockattr_setpshared(&rwlattr, PTHREAD_PROCESS_SHARED); + pthread_rwlockattr_setpshared(&cache_rwlattr, PTHREAD_PROCESS_SHARED); + pthread_rwlock_init(&db->shmem_info->rwlock, &rwlattr); + pthread_rwlock_init(&db->shmem_info->cache_rwlock, &cache_rwlattr); + Kdb_wrlock(&db->shmem_info->rwlock); + +#ifdef __checkerror + //CHECK POWERLOSS FLAGS + ret = checkErrorFlags(db); + if (0 != ret) + { + close(db->fd); + Kdb_unlock(&db->shmem_info->rwlock); + return ret; + } +#endif + db->shmem_info->num_hash_tables = 0; + } + else // already initialized + Kdb_wrlock(&db->shmem_info->rwlock); + + db->already_open = Kdb_true; + } + else + Kdb_wrlock(&db->shmem_info->rwlock); + + //only read header from file into memory for first caller of KISSDB_open + if (db->shmem_creator == Kdb_true) + { + httmp = (Hashtable_slot_s*) malloc(db->hash_table_size_bytes); //read hashtable from file + if (!httmp) + { + close(db->fd); + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_MALLOC; + } + while (read(db->fd, httmp, db->hash_table_size_bytes) == db->hash_table_size_bytes) + { + Kdb_bool result = Kdb_false; + //if new size would exceed old shared memory size-> allocate additional memory page to shared memory + if (db->hash_table_size_bytes * (db->shmem_info->num_hash_tables + 1) > db->old_mapped_size) + { + Kdb_bool temp; + if (db->shmem_ht_fd <= 0) + { + db->shmem_ht_fd = kdbShmemOpen(db->shmem_ht_name, db->old_mapped_size, &temp); + if(db->shmem_ht_fd < 0) + { + free(httmp); + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_OPEN_SHM; + } + } + result = resizeKdbShmem(db->shmem_ht_fd, &db->hash_tables, db->old_mapped_size, db->old_mapped_size + db->hash_table_size_bytes); + if (result == Kdb_false) + { + free(httmp); + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_RESIZE_SHM; + } + else + { + db->shmem_info->shmem_size = db->old_mapped_size + db->hash_table_size_bytes; + db->old_mapped_size = db->old_mapped_size + db->hash_table_size_bytes; + } + } + // copy the current hashtable read from file to (htadress + (htsize * htcount)) in memory + memcpy(((uint8_t *) db->hash_tables) + (db->hash_table_size_bytes * db->shmem_info->num_hash_tables), httmp, db->hash_table_size_bytes); + ++db->shmem_info->num_hash_tables; + + //read until all hash tables have been read + if (httmp[db->hash_table_size].offsetA) //if httable[hash_table_size] contains a offset to a further hashtable + { + //ONE MORE HASHTABLE FOUND + if (lseek(db->fd, httmp[db->hash_table_size].offsetA, SEEK_SET) == -1) + { //move the filepointer to the next hashtable in the file + KISSDB_close(db); + free(httmp); + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } + } + else + break; // no further hashtables exist + } + free(httmp); + } + + //printSharedHashtable(db); + + Kdb_unlock(&db->shmem_info->rwlock); + return 0; +} + + + + + +int KISSDB_close(KISSDB *db) +{ + Kdb_wrlock(&db->shmem_info->rwlock); + + uint64_t crc = 0; + Header_s* ptr = 0; +#ifdef __showTimeMeasurements + long long KdbDuration = 0; + struct timespec mmapStart, mmapEnd; + KdbDuration = 0; +#endif + + //printSharedHashtable(db); + if (db->shmem_creator == Kdb_true) + { + //free shared hashtable + if( freeKdbShmemPtr(db->hash_tables, db->old_mapped_size) == Kdb_false) + { + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_UNMAP_SHM; + } + if( kdbShmemClose(db->shmem_ht_fd, db->shmem_ht_name) == Kdb_false) + return KISSDB_ERROR_CLOSE_SHM; + + free(db->shmem_ht_name); + Kdb_unlock(&db->shmem_info->rwlock); + pthread_rwlock_destroy(&db->shmem_info->rwlock); + pthread_rwlock_destroy(&db->shmem_info->cache_rwlock); + + // free shared information + if (freeKdbShmemPtr(db->shmem_info, sizeof(Kdb_bool)) == Kdb_false) + return KISSDB_ERROR_UNMAP_SHM; + if (kdbShmemClose(db->shmem_info_fd, db->shmem_info_name) == Kdb_false) + return KISSDB_ERROR_CLOSE_SHM; + free(db->shmem_info_name); + +#ifdef __showTimeMeasurements + clock_gettime(CLOCK_ID, &mmapStart); +#endif + + //update header (checksum and flags) + int mapFlag = PROT_WRITE | PROT_READ; + ptr = (Header_s*) mmap(NULL,KISSDB_HEADER_SIZE, mapFlag, MAP_SHARED, db->fd, 0); + if (ptr == MAP_FAILED) + { + close(db->fd); + return KISSDB_ERROR_IO; + } +#ifdef __checkerror + // generate checksum over database file (beginning at file offset [sizeof(ptr->KdbV) + sizeof(ptr->checksum)] up to EOF) + if( db->fd ) + { + crc = 0; + crc = (uint64_t) pcoCalcCrc32Csum(db->fd, sizeof(Header_s) ); + ptr->checksum = crc; + //printf("CLOSING ------ DB: %s, WITH CHECKSUM CALCULATED: %" PRIu64 " \n", db->shmem_ht_name, ptr->checksum); + } +#endif + ptr->closeFailed = 0x00; //remove closeFailed flag + ptr->closeOk = 0x01; //set closeOk flag + + //sync changes with file + if( 0 != msync(ptr, KISSDB_HEADER_SIZE, MS_SYNC | MS_INVALIDATE)) + { + close(db->fd); + return KISSDB_ERROR_IO; + } + //unmap memory + if( 0 != munmap(ptr, KISSDB_HEADER_SIZE)) + { + close(db->fd); + return KISSDB_ERROR_IO; + } +#ifdef __showTimeMeasurements + clock_gettime(CLOCK_ID, &mmapEnd); + KdbDuration += getNsDuration(&mmapStart, &mmapEnd); + printf("mmap duration for => %f ms\n", (double)((double)KdbDuration/NANO2MIL)); +#endif + fsync(db->fd); + + if( db->fd) + close(db->fd); + + db->already_open = Kdb_false; + //memset(db, 0, sizeof(KISSDB)); //todo check if necessary + } + else + //if caller is not the creator of the lock + Kdb_unlock(&db->shmem_info->rwlock); + return 0; +} + + +int KISSDB_get(KISSDB *db, const void *key, void *vbuf) +{ + Kdb_rdlock(&db->shmem_info->rwlock); + + uint8_t tmp[TMP_BUFFER_LENGTH]; + uint64_t current; + const uint8_t *kptr; + unsigned long klen, i; + long n = 0; + uint64_t checksum, backupChecksum, crc; + uint64_t hash = KISSDB_hash(key, db->key_size) % (uint64_t) db->hash_table_size; + int64_t offset, backupOffset, htoffset, checksumOffset, flagOffset; //lasthtoffset + Hashtable_slot_s *cur_hash_table; + +#ifdef __writeThrough + //if new one or more hashtables were appended, remap shared memory block to adress space + if (db->old_mapped_size < db->shmem_info->shmem_size) + { + Kdb_bool temp; + db->shmem_ht_fd = kdbShmemOpen(db->shmem_ht_name, db->old_mapped_size, &temp); + if(db->shmem_ht_fd < 0) + return KISSDB_ERROR_OPEN_SHM; + res = remapKdbShmem(db->shmem_ht_fd, &db->hash_tables, db->old_mapped_size, db->shmem_info->shmem_size); + if (res == Kdb_false) + return KISSDB_ERROR_REMAP_SHM; + db->old_mapped_size = db->shmem_info->shmem_size; + } +#endif + + htoffset = KISSDB_HEADER_SIZE; //lasthtoffset + cur_hash_table = db->hash_tables;//pointer to current hashtable in memory + for (i = 0; i < db->shmem_info->num_hash_tables; ++i) + { + offset = cur_hash_table[hash].offsetA;//get fileoffset where the data can be found in the file +#ifdef __useBackups + //get information about current valid offset to latest written data + if(cur_hash_table[hash].current == 0x00) //valid is offsetA + { + offset = cur_hash_table[hash].offsetA; + checksum = cur_hash_table[hash].checksumA; + } + else + { + offset = cur_hash_table[hash].offsetB; + checksum = cur_hash_table[hash].checksumB; + } +#endif + + if (offset >= KISSDB_HEADER_SIZE) //if a valid offset is available in the slot + { + if (lseek(db->fd, offset, SEEK_SET) == -1) //move filepointer to this offset + { + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } + kptr = (const uint8_t *) key; + klen = db->key_size; + while (klen) + { + n = (long) read(db->fd, tmp, (klen > sizeof(tmp)) ? sizeof(tmp) : klen); + if (n > 0) + { + if (memcmp(kptr, tmp, n))//if key does not match -> search in next hashtable + goto get_no_match_next_hash_table; + kptr += n; + klen -= (unsigned long) n; + } + else + { + Kdb_unlock(&db->shmem_info->rwlock); + return 1; /* not found */ + } + } + if (read(db->fd, vbuf, db->value_size) == db->value_size) //if key matches at the fileoffset -> read the value + { + //crc check for file content +#ifdef __useBackups + //only validate checksums at read if checksum of file is invalid + if (db->shmem_info->crc_invalid == Kdb_true) + { + //verify checksum of current key/value pair + crc = 0; + crc = (uint64_t) pcoCrc32(crc, (unsigned char*) vbuf, db->value_size); + if (checksum != crc) + { + //printf("KISSDB_get: WARNING: checksum invalid -> try to read from valid data block \n"); + //try to read valid data from backup + Hashtable_slot_s slot = cur_hash_table[hash]; + if (cur_hash_table[hash].current == 0x00) //current is offsetA, but Data there is corrupt--> so use offsetB as backupOffset + { + backupOffset = cur_hash_table[hash].offsetB; + backupChecksum = cur_hash_table[hash].checksumB; + checksumOffset = htoffset + (sizeof(Hashtable_slot_s) * hash + sizeof(slot.offsetA)); //offset that points to checksumA + current = 0x01; //current is offsetB + } + else + { + backupOffset = cur_hash_table[hash].offsetA; + backupChecksum = cur_hash_table[hash].checksumA; + checksumOffset = htoffset + + (sizeof(Hashtable_slot_s) * hash + sizeof(slot.offsetA) + sizeof(slot.checksumA) + + sizeof(slot.offsetB)); //offset that points to checksumB + current = 0x00; + } + flagOffset = htoffset + + (sizeof(Hashtable_slot_s) * hash + (sizeof(Hashtable_slot_s) - sizeof(slot.current))); //offset that points to currentflag + + //seek to backup data + if (lseek(db->fd, backupOffset + db->key_size, SEEK_SET) == -1) //move filepointer to data of key-value pair //TODO make checksum over key AND data ?? + { + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } + + //verify checksum of backup key/value pair + //read from backup data + if (read(db->fd, vbuf, db->value_size) == db->value_size) //read value of backup Data block + { + //generate checksum of backup + crc = 0; + crc = (uint64_t) pcoCrc32(crc, (unsigned char*) vbuf, db->value_size); + if (crc == backupChecksum) //if checksum ok + { + //printf("KISSDB_get: WARNING: OVERWRITING CORRUPT DATA \n"); + //seek to corrupt data + if (lseek(db->fd, offset + db->key_size, SEEK_SET) == -1) //move filepointer to data of corrupt key-value pair + { + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } + //overwrite corrupt data + if (write( db->fd, vbuf, db->value_size) != db->value_size ) //write value + { + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } + //seek to header slot and update checksum of corrupt data (do not modify offsets) + if (lseek(db->fd, checksumOffset, SEEK_SET) == -1) //move to checksumX in file + { + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } + if (write( db->fd, &crc, sizeof(uint64_t)) != sizeof(uint64_t) ) //write checksumX to hashtbale slot + { + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } + //update checksumX in memory + if (cur_hash_table[hash].current == 0x00) //current is offsetA, but Data there is corrupt--> so update checksumA with new checksum + cur_hash_table[hash].checksumA = crc; + else + cur_hash_table[hash].checksumB = crc; + //switch current valid to backup + + if (lseek(db->fd, flagOffset, SEEK_SET) == -1) //move to current flag in file + { + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } + if (write( db->fd, ¤t, sizeof(uint64_t)) != sizeof(uint64_t) ) //write current hashtable slot in file + { + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } + //update current valid in memory + cur_hash_table[hash].current = current; + //fsync(db->fd) + Kdb_unlock(&db->shmem_info->rwlock); + return 0; /* success */ + } + else //if checksum not valid, return NOT FOUND + { + Kdb_unlock(&db->shmem_info->rwlock); + return 1; /* not found */ + } + } + else + { + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } + } + } +#endif + Kdb_unlock(&db->shmem_info->rwlock); + return 0; /* success */ + } + else + { + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } + } + else + { + Kdb_unlock(&db->shmem_info->rwlock); + return 1; /* not found */ + } + //get_no_match_next_hash_table: cur_hash_table += db->hash_table_size + 1; + get_no_match_next_hash_table: //update lastht offset //lasthtoffset = htoffset + htoffset = cur_hash_table[db->hash_table_size].offsetA; // fileoffset to the next file-hashtable + cur_hash_table += (db->hash_table_size + 1); //pointer to the next memory-hashtable + } + Kdb_unlock(&db->shmem_info->rwlock); + return 1; /* not found */ +} + + +//TODO check current valid data to be deleted ? +int KISSDB_delete(KISSDB *db, const void *key) +{ + Kdb_wrlock(&db->shmem_info->rwlock); + + uint8_t tmp[TMP_BUFFER_LENGTH]; + //uint64_t current = 0x00; + const uint8_t *kptr; + long n; + unsigned long klen, i; + //uint64_t crc = 0x00; + uint64_t hash = KISSDB_hash(key, db->key_size) % (uint64_t) db->hash_table_size; + //int64_t empty_offset = 0; + int64_t empty_offsetB = 0; + int64_t offset = 0; + int64_t htoffset = 0; + Hashtable_slot_s *cur_hash_table; + +#ifdef __writeThrough + //if new hashtable was appended, remap shared memory block to adress space + if (db->old_mapped_size < db->shmem_info->shmem_size) + { + Kdb_bool temp; + db->shmem_ht_fd = kdbShmemOpen(db->shmem_ht_name, db->old_mapped_size, &temp); + if(db->shmem_ht_fd < 0) + return KISSDB_ERROR_OPEN_SHM; + result = remapKdbShmem(db->shmem_ht_fd, &db->hash_tables, db->old_mapped_size, db->shmem_info->shmem_size); + if (result == Kdb_false) + return KISSDB_ERROR_REMAP_SHM; + db->old_mapped_size = db->shmem_info->shmem_size; + } +#endif + + htoffset = KISSDB_HEADER_SIZE; + cur_hash_table = db->hash_tables; //pointer to current hashtable in memory + + for (i = 0; i < db->shmem_info->num_hash_tables; ++i) + { + offset = cur_hash_table[hash].offsetA; //get fileoffset where the data can be found in the file + if (offset >= KISSDB_HEADER_SIZE) + { + if (lseek(db->fd, offset, SEEK_SET) == -1) + { + //set filepointer to Key value offset in file + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } + kptr = (const uint8_t *) key; + klen = db->key_size; + while (klen) + { + n = (long) read(db->fd, tmp, (klen > sizeof(tmp)) ? sizeof(tmp) : klen); + if (n > 0) + { + if (memcmp(kptr, tmp, n))//if key does not match, search in next hashtable + goto get_no_match_next_hash_table; + kptr += n; + klen -= (unsigned long) n; + } + else + { + Kdb_unlock(&db->shmem_info->rwlock); + return 1; /* not found */ + } + } + //TODO: mmap Hashtable slot structure to avoid seeking -> align hashtables at a multiple of a pagesize +#ifdef __useFileMapping + empty_offsetB = -(offset + (db->key_size + db->value_size)); //todo check if offset is rewritten in put function ! + cur_hash_table[hash].offsetB = empty_offsetB; + cur_hash_table[hash].checksumA = 0x00; + cur_hash_table[hash].checksumB = 0x00; + cur_hash_table[hash].current = 0x00; + int testoffset= lseek(fd, 0, SEEK_CUR); //filepointer position + int myoffset = htoffset + (sizeof(Hashtable_slot_s) * hash); + + printf("Endoffset in file: %d , Offset for mmap: %d , size for mmap: %d \n", testoffset, myoffset, sizeof(Hashtable_slot_s)); + + //mmap the current hashtable slot + int mapFlag = PROT_WRITE | PROT_READ; + printf("In Delete: filedes: %d\n", db->fd); + htSlot = (Hashtable_slot_s*) mmap(NULL, sizeof(Hashtable_slot_s), mapFlag, MAP_SHARED, db->fd, htoffset + (sizeof(Hashtable_slot_s) * hash) ); //TODO offset must be a multiple of pagesize + if (htSlot == MAP_FAILED) + { + printf("MMAP ERROR !\n"); + close(db->fd); + return KISSDB_ERROR_IO; + } + //do changes to slot in file + htSlot->offsetA = empty_offset; + htSlot->checksumA = 0x00; + htSlot->offsetB = empty_offsetB; + htSlot->checksumB = 0x00; + htSlot->current = 0x00; + + //sync changes with file + if (0 != msync(htSlot, sizeof(Hashtable_slot_s), MS_SYNC | MS_INVALIDATE)) + { + close(db->fd); + return KISSDB_ERROR_IO; + } + //unmap memory + if (0 != munmap(htSlot, sizeof(Hashtable_slot_s))) + { + close(db->fd); + return KISSDB_ERROR_IO; + } +#endif + if (lseek(db->fd, htoffset + (sizeof(Hashtable_slot_s) * hash), SEEK_SET) == -1) //move Filepointer to used slot in file-hashtable. + { + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } + +#ifndef __useBackups + cur_hash_table[hash].offsetA = -offset; //negate offset in hashtable that points to the data + empty_offset = -offset; + //update hashtable slot in file header (delete existing offset information) + if (write( db->fd, &empty_offset, sizeof(int64_t)) != sizeof(int64_t) ) //mark slot in file-hashtable as deleted + { + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } +#endif + +#ifdef __useBackups + //negate offsetB, delete checksums and current flag in memory + cur_hash_table[hash].offsetA = -offset; //negate offset in hashtable that points to the data + empty_offsetB = -(offset + (db->key_size + db->value_size)); + cur_hash_table[hash].checksumA = 0x00; + cur_hash_table[hash].offsetB = empty_offsetB; + cur_hash_table[hash].checksumB = 0x00; + cur_hash_table[hash].current = 0x00; + if (write( db->fd, &cur_hash_table[hash], sizeof(Hashtable_slot_s)) != sizeof(Hashtable_slot_s) ) //write updated data in the file-hashtable slot + { + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } +#endif + //TODO currently, no synchronus Filedescriptor is used!!!! fsync after fflush is needed to do synchronus writes + //fsync(db->fd) // associating a file stream with a synchronous file descriptor means that an fsync() call is not needed on the file descriptor after the fflush() + Kdb_unlock(&db->shmem_info->rwlock); + return 0; /* success */ + } + else + { + Kdb_unlock(&db->shmem_info->rwlock); + return 1; /* not found */ //if no offset is found at hashed position in ht + } + get_no_match_next_hash_table: htoffset = cur_hash_table[db->hash_table_size].offsetA; // fileoffset to next ht in file + cur_hash_table += (db->hash_table_size + 1); //pointer to next hashtable in memory + } + Kdb_unlock(&db->shmem_info->rwlock); + return 1; /* not found */ +} + +int KISSDB_put(KISSDB *db, const void *key, const void *value) +{ + Kdb_wrlock(&db->shmem_info->rwlock); + + uint8_t tmp[TMP_BUFFER_LENGTH]; + uint64_t current = 0x00; + const uint8_t *kptr; + unsigned long klen, i; + uint64_t hash = KISSDB_hash(key, db->key_size) % (uint64_t) db->hash_table_size; + int64_t offset, endoffset, htoffset, lasthtoffset; + Hashtable_slot_s *cur_hash_table; + Kdb_bool result = Kdb_false; + Kdb_bool temp = Kdb_false; + uint64_t crc = 0x00; + long n; + char delimiter[8] = "||||||||"; + +#ifdef __writeThrough + //if new hashtable was appended, remap shared memory block to adress space + if(db->old_mapped_size < db->shmem_info->shmem_size) + { + db->shmem_ht_fd = kdbShmemOpen(db->shmem_ht_name, db->old_mapped_size, &temp); + if(db->shmem_ht_fd < 0) + return KISSDB_ERROR_OPEN_SHM; + res = remapKdbShmem(db->shmem_ht_fd, &db->hash_tables, db->old_mapped_size,db->shmem_info->shmem_size); + if (res == Kdb_false) + return KISSDB_ERROR_REMAP_SHM; + db->old_mapped_size = db->shmem_info->shmem_size; + } +#endif + lasthtoffset = htoffset = KISSDB_HEADER_SIZE; + cur_hash_table = db->hash_tables; //pointer to current hashtable in memory + + for (i = 0; i < db->shmem_info->num_hash_tables; ++i) + { + offset = cur_hash_table[hash].offsetA; //fileoffset to data in file + if (offset >= KISSDB_HEADER_SIZE || offset < 0) //if a key with same hash is already in this slot or the same key must be overwritten + { + // if slot is marked as deleted, use this slot and negate the offset in order to reuse the existing data block + if(offset < 0) + { + offset = -offset; //get original offset where data was deleted + //printf("Overwriting slot for key: [%s] which was deleted before, offsetA: %d \n",key, offset); + if (lseek(db->fd, offset, SEEK_SET) == -1) //move filepointer to fileoffset where the key can be found + { + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } + if (write( db->fd, key, db->key_size) != db->key_size ) //write key + { + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } + if (write( db->fd, value, db->value_size) != db->value_size ) //write value + { + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } + + // write same key and value again here because slot was deleted an can be reused like an initial write +#ifdef __useBackups + if (write( db->fd, key, db->key_size) != db->key_size ) //write key + { + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } + if (write( db->fd, value, db->value_size) != db->value_size ) //write value + { + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } +#endif + //seek back to hashtbale slot + if (lseek(db->fd, htoffset + (sizeof(Hashtable_slot_s) * hash), SEEK_SET) == -1) //move to beginning of hashtable slot in file + { + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } + +#ifndef __useBackups + cur_hash_table[hash].offsetA = offset; //write the offset to the data in the memory-hashtable slot + if (write( db->fd, &offset, sizeof(int64_t)) != sizeof(int64_t) ) //write the offsetA to the data in the file-hashtable slot + { + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } +#endif + +#ifdef __useBackups + crc = 0x00; + crc = (uint32_t) pcoCrc32(crc, (unsigned char*)value, db->value_size); + cur_hash_table[hash].offsetA = offset; //write the offset to the data in the memory-hashtable slot + cur_hash_table[hash].checksumA = crc; + offset += (db->key_size + db->value_size); + cur_hash_table[hash].offsetB = offset; //write the offset to the data in the memory-hashtable slot + cur_hash_table[hash].checksumB = crc; + cur_hash_table[hash].current = 0x00; + + if (write( db->fd, &cur_hash_table[hash], sizeof(Hashtable_slot_s)) != sizeof(Hashtable_slot_s) ) //write updated data in the file-hashtable slot + { + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } +#endif + //fsync(db->fd) //associating a file stream with a synchronous file descriptor means that an fsync() call is not needed on the file descriptor after the fflush() + Kdb_unlock(&db->shmem_info->rwlock); + return 0; /* success */ + } + + //overwrite existing if key matches + // if cur_hash_table[hash].current == 0x00 -> offsetA is latest so write to offsetB else offsetB is latest and write to offsetA +#ifdef __useBackups + if( cur_hash_table[hash].current == 0x00 ) + offset = cur_hash_table[hash].offsetA; //0x00 -> offsetA is latest + else + offset = cur_hash_table[hash].offsetB; //else offsetB is latest +#endif + if (lseek(db->fd, offset, SEEK_SET) == -1) //move filepointer to fileoffset where valid data can be found + { + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } + + kptr = (const uint8_t *) key; //pointer to search key + klen = db->key_size; + while (klen) + { + n = (long) read(db->fd, tmp, (klen > sizeof(tmp)) ? sizeof(tmp) : klen); + if (n > 0) + { + if (memcmp(kptr, tmp, n)) //if search key does not match with key in file + goto put_no_match_next_hash_table; + kptr += n; + klen -= (unsigned long) n; + } + } + + //if key matches -> seek to currently non valid data block for this key +#ifdef __useBackups + if( cur_hash_table[hash].current == 0x00 ) + offset = cur_hash_table[hash].offsetB; // 0x00 -> offsetA is latest so write new data to offsetB which holds old data + else + offset = cur_hash_table[hash].offsetA; // offsetB is latest so write new data to offsetA which holds old data + + if (lseek(db->fd, offset, SEEK_SET) == -1)//move filepointer to fileoffset where backup data can be found + { + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } + if (write( db->fd, key, db->key_size) != db->key_size ) //write key + { + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } +#endif + if (write( db->fd, value, db->value_size) != db->value_size ) + { + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } + // seek back to slot in header for update of checksum and flag + if (lseek(db->fd, htoffset + (sizeof(Hashtable_slot_s) * hash), SEEK_SET) == -1) //move to beginning of hashtable slot in file + { + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } + + //generate crc for value + crc = 0x00; + crc = (uint64_t) pcoCrc32(crc, (unsigned char*)value, db->value_size); + current = 0x00; + Hashtable_slot_s slot = cur_hash_table[hash]; + + // check current flag and decide what parts of hashtable slot in file must be updated + if( cur_hash_table[hash].current == 0x00 ) //offsetA is latest -> modify settings of B + { + int seek = sizeof(slot.offsetA) + sizeof(slot.checksumA) + sizeof(slot.offsetB); + lseek(db->fd, seek , SEEK_CUR); //move to checksumB in file + if( write( db->fd, &crc, sizeof(uint64_t)) != sizeof(uint64_t)) //write checksumB to file + { + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } + current = 0x01; + if( write( db->fd, ¤t, sizeof(uint64_t)) != sizeof(uint64_t)) //write current to hashtbale slot + { + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } + cur_hash_table[hash].checksumB = crc; + cur_hash_table[hash].current = current; + } + else //offsetB is latest -> modify settings of A + { + + int seek = sizeof(slot.offsetA); + lseek(db->fd, seek , SEEK_CUR); //move to checksumA in file + if( write( db->fd, &crc, sizeof(uint64_t)) != sizeof(uint64_t)) //write checksumA to file + { + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } + seek = sizeof(slot.offsetB) + sizeof(slot.checksumB);; + lseek(db->fd, seek , SEEK_CUR); //move to checksumA in file + current = 0x00; + if( write( db->fd, ¤t, sizeof(uint64_t)) != sizeof(uint64_t))//write current to hashtbale slot + { + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } + cur_hash_table[hash].checksumA = crc; + cur_hash_table[hash].current = current; + } + Kdb_unlock(&db->shmem_info->rwlock); + return 0; //success + } + else //if key is not already inserted + { + /* add new data if an empty hash table slot is discovered */ + if (lseek(db->fd, 0, SEEK_END) == -1) //filepointer to the end of the file + { + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } + endoffset = lseek(db->fd, 0, SEEK_CUR); //filepointer position + if (write( db->fd, key, db->key_size) != db->key_size ) + { + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } + if (write( db->fd, value, db->value_size) != db->value_size ) + { + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } + + // write same key and value again here --> initial write +#ifdef __useBackups + if (write( db->fd, key, db->key_size) != db->key_size ) + { + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } + if (write( db->fd, value, db->value_size) != db->value_size ) + { + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } +#endif + if (lseek(db->fd, htoffset + (sizeof(Hashtable_slot_s) * hash), SEEK_SET) == -1) //move filepointer to file-hashtable slot in file (offsetA) + { + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } +#ifndef __useBackups + if (write( db->fd, &endoffset, sizeof(int64_t)) != sizeof(int64_t) ) //write the offsetA to the data in the file-hashtable slot + { + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } + cur_hash_table[hash].offsetA = endoffset; //write the offsetA to the data in the memory-hashtable slot +#endif + +#ifdef __useBackups + crc = 0x00; + crc = (uint64_t) pcoCrc32(crc, (unsigned char*) value, db->value_size); + offset = endoffset + (db->key_size + db->value_size); + cur_hash_table[hash].offsetA = endoffset; //write the offsetA to the data in the memory-hashtable slot + cur_hash_table[hash].checksumA = crc; + cur_hash_table[hash].offsetB = offset; //write the offset to the data in the memory-hashtable slot + cur_hash_table[hash].checksumB = crc; + cur_hash_table[hash].current = 0x00; + current = 0x00; //current + + if (write( db->fd, &cur_hash_table[hash], sizeof(Hashtable_slot_s)) != sizeof(Hashtable_slot_s) ) //write updated data in the file-hashtable slot + { + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } +#endif + //fsync(db->fd) // associating a file stream with a synchronous file descriptor means that an fsync() call is not needed on the file descriptor after the fflush() + Kdb_unlock(&db->shmem_info->rwlock); + return 0; /* success */ + } + put_no_match_next_hash_table: lasthtoffset = htoffset; + htoffset = cur_hash_table[db->hash_table_size].offsetA; // fileoffset to the next file-hashtable + cur_hash_table += (db->hash_table_size + 1); //pointer to the next memory-hashtable + } + + /* if no existing slots, add a new page of hash table entries */ + if (lseek(db->fd, 0, SEEK_END) == -1) //Filepointer to the end of file + { + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } + if(db->shmem_info->num_hash_tables > 0) //only write delimiter if first hashtable has been written (first delimiter is written by open call) + { + if (write( db->fd, &delimiter, sizeof(delimiter)) != sizeof(delimiter) ) //write delimiter + { + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } + } + endoffset = lseek(db->fd, 0, SEEK_CUR); + + //if new size would exceed old shared memory size-> allocate additional memory to shared memory (+ db->hash_table_size_bytes) + if( (db->hash_table_size_bytes * (db->shmem_info->num_hash_tables + 1)) > db->shmem_info->shmem_size) + { + if (db->shmem_ht_fd <= 0) + { + db->shmem_ht_fd = kdbShmemOpen(db->shmem_ht_name, db->old_mapped_size, &temp); + if(db->shmem_ht_fd < 0) + return KISSDB_ERROR_OPEN_SHM; + } + result = resizeKdbShmem(db->shmem_ht_fd, &db->hash_tables, db->old_mapped_size, db->old_mapped_size + db->hash_table_size_bytes); + if (result == Kdb_false) + { + return KISSDB_ERROR_RESIZE_SHM; + } + else + { + db->shmem_info->shmem_size = db->old_mapped_size + db->hash_table_size_bytes; + db->old_mapped_size = db->shmem_info->shmem_size; + } + } + + //if( currentHtOffset <= db->old_mapped_size / sizeof(Hashtable_slot_s) ) + cur_hash_table = &(db->hash_tables[(db->hash_table_size + 1) * db->shmem_info->num_hash_tables]); + //else + // return KISSDB_ERROR_ACCESS_VIOLATION; + memset(cur_hash_table, 0, db->hash_table_size_bytes); //hashtable init + cur_hash_table[hash].offsetA = endoffset + db->hash_table_size_bytes; /* where new entry will go (behind the new Ht that gets written)*/ + +#ifdef __useBackups + crc = 0x00; + crc = (uint64_t) pcoCrc32(crc, (unsigned char*)value, db->value_size); + cur_hash_table[hash].checksumA = crc; + cur_hash_table[hash].checksumB = crc; + cur_hash_table[hash].offsetB = cur_hash_table[hash].offsetA + (db->key_size + db->value_size);//write the offset to the data in the memory-hashtable slot + cur_hash_table[hash].current = 0x00; +#endif + + // write new hashtable at the end of the file + if (write( db->fd, cur_hash_table, db->hash_table_size_bytes) != db->hash_table_size_bytes ) + { + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } + // write key behind new hashtable + if (write( db->fd, key, db->key_size) != db->key_size ) + { + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } + // write value behind key + if (write( db->fd, value, db->value_size) != db->value_size ) + { + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } + // write same key and value again here --> initial write +#ifdef __useBackups + if (write( db->fd, key, db->key_size) != db->key_size ) + { + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } + if (write( db->fd, value, db->value_size) != db->value_size ) + { + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } +#endif + + //if a hashtable exists, update link to new hashtable + if (db->shmem_info->num_hash_tables) + { + if (lseek(db->fd, lasthtoffset + (sizeof(Hashtable_slot_s) * db->hash_table_size), SEEK_SET) == -1) + { + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } + if (write( db->fd, &endoffset, sizeof(int64_t)) != sizeof(int64_t) ) + { + Kdb_unlock(&db->shmem_info->rwlock); + return KISSDB_ERROR_IO; + } + db->hash_tables[((db->hash_table_size + 1) * (db->shmem_info->num_hash_tables - 1)) + db->hash_table_size].offsetA = endoffset; //update link to new hashtable in old hashtable + } + ++db->shmem_info->num_hash_tables; + //fsync(db->fd) + Kdb_unlock(&db->shmem_info->rwlock); + return 0; /* success */ +} + + + +#if 0 +/* + * prints the offsets stored in the shared Hashtable + */ +void printSharedHashtable(KISSDB *db) +{ + Hashtable_slot_s *cur_hash_table; + cur_hash_table = db->hash_tables; + unsigned long k; + unsigned long x = (db->hash_table_size * db->shmem_info->num_hash_tables); + //printf("Address of SHARED HT_NUMBER: %p \n", &db->shmem_info->num_hash_tables); + printf("Address of SHARED HEADER: %p \n", &cur_hash_table); + Header_s* ptr; + printf("HT Struct sizes: %d, %d, %d, %d,%d, %d, %d, %d\n", sizeof(ptr->KdbV), sizeof(ptr->checksum), sizeof(ptr->closeFailed), sizeof(ptr->closeOk), sizeof(ptr->hash_table_size),sizeof(ptr->key_size),sizeof(ptr->value_size),sizeof(ptr->delimiter)); + printf("HEADER SIZE: %d \n", sizeof(Header_s)); + printf("Hashtable_slot_s SIZE: %d \n", sizeof(Hashtable_slot_s)); + for (k = 0; k < x; k++) + { + if (db->hash_tables[k].offsetA != 0) + { + printf("offsetA [%lu]: %" PRId64 " \n", k, db->hash_tables[k].offsetA); + printf("checksumA[%lu]: %" PRIu64 " \n", k, db->hash_tables[k].checksumA); + printf("offsetB [%lu]: %" PRId64 " \n", k, db->hash_tables[k].offsetB); + printf("checksumB[%lu]: %" PRIu64 " \n", k, db->hash_tables[k].checksumB); + printf("current [%lu]: %" PRIu64 " \n", k, db->hash_tables[k].current); + } + } +} +#endif + + +void KISSDB_Iterator_init(KISSDB *db, KISSDB_Iterator *dbi) +{ + dbi->db = db; + dbi->h_no = 0; // number of read hashtables + dbi->h_idx = 0; // index in current hashtable +} + + +int KISSDB_Iterator_next(KISSDB_Iterator *dbi, void *kbuf, void *vbuf) +{ + int64_t offset; + Kdb_rdlock(&dbi->db->shmem_info->rwlock); + + if ((dbi->h_no < (dbi->db->shmem_info->num_hash_tables)) && (dbi->h_idx < dbi->db->hash_table_size)) + { + //TODO check for currently valid data block flag and use this offset instead of offsetA + while (!(offset = dbi->db->hash_tables[((dbi->db->hash_table_size + 1) * dbi->h_no) + dbi->h_idx].offsetA)) + { + if (++dbi->h_idx >= dbi->db->hash_table_size) + { + dbi->h_idx = 0; + if (++dbi->h_no >= (dbi->db->shmem_info->num_hash_tables)) + { + Kdb_unlock(&dbi->db->shmem_info->rwlock); + return 0; + } + } + } + + if (lseek(dbi->db->fd, offset, SEEK_SET) == -1) + return KISSDB_ERROR_IO; + if (read(dbi->db->fd, kbuf, dbi->db->key_size) != dbi->db->key_size) + return KISSDB_ERROR_IO; + if (vbuf != NULL) + { + if (read(dbi->db->fd, vbuf, dbi->db->value_size) != dbi->db->value_size) + return KISSDB_ERROR_IO; + } + else + { + if (lseek(dbi->db->fd, dbi->db->value_size, SEEK_CUR) == -1) + return KISSDB_ERROR_IO; + } + + if (++dbi->h_idx >= dbi->db->hash_table_size) + { + dbi->h_idx = 0; + ++dbi->h_no; + } + Kdb_unlock(&dbi->db->shmem_info->rwlock); + return 1; + } + Kdb_unlock(&dbi->db->shmem_info->rwlock); + return 0; +} + + + +int readHeader(KISSDB* db, uint16_t* hash_table_size, uint64_t* key_size, uint64_t* value_size) +{ + //set Filepointer to the beginning of the file + if (lseek(db->fd, 0, SEEK_SET) == -1) + return KISSDB_ERROR_IO; + //mmap header from beginning of file + int mapFlag = PROT_WRITE | PROT_READ; + Header_s* ptr = 0; + ptr = (Header_s*) mmap(NULL, KISSDB_HEADER_SIZE, mapFlag, MAP_SHARED, db->fd, 0); + if (ptr == MAP_FAILED) + return KISSDB_ERROR_IO; + + if ((ptr->KdbV[0] != 'K') || (ptr->KdbV[1] != 'd') || (ptr->KdbV[2] != 'B') || (ptr->KdbV[3] != KISSDB_VERSION)) + return KISSDB_ERROR_CORRUPT_DBFILE; + + if (!ptr->hash_table_size) + return KISSDB_ERROR_CORRUPT_DBFILE; + (*hash_table_size) = (uint16_t) ptr->hash_table_size; + + if (!ptr->key_size) + return KISSDB_ERROR_CORRUPT_DBFILE; + (*key_size) = (uint64_t) ptr->key_size; + + if (!ptr->value_size) + return KISSDB_ERROR_CORRUPT_DBFILE; + (*value_size) = (uint64_t) ptr->value_size; + + //sync changes with file + if (0 != msync(ptr, KISSDB_HEADER_SIZE, MS_SYNC | MS_INVALIDATE)) + return KISSDB_ERROR_IO; + + //unmap memory + if (0 != munmap(ptr, KISSDB_HEADER_SIZE)) + return KISSDB_ERROR_IO; + return 0; +} + + + + +int writeHeader(KISSDB* db, uint16_t* hash_table_size, uint64_t* key_size, uint64_t* value_size) +{ + Header_s* ptr = 0; + int ret= 0; + + //Seek to beginning of file + if (lseek(db->fd, 0, SEEK_SET) == -1) + return KISSDB_ERROR_IO; + + //ftruncate file to needed size for header + ret = ftruncate(db->fd, KISSDB_HEADER_SIZE); + if (ret < 0) + return KISSDB_ERROR_IO; + + //mmap header from beginning of file + int mapFlag = PROT_WRITE | PROT_READ; + ptr = (Header_s*) mmap(NULL, KISSDB_HEADER_SIZE, mapFlag, MAP_SHARED, db->fd, 0); + if (ptr == MAP_FAILED) + return KISSDB_ERROR_IO; + + ptr->KdbV[0] = 'K'; + ptr->KdbV[1] = 'd'; + ptr->KdbV[2] = 'B'; + ptr->KdbV[3] = KISSDB_VERSION; + ptr->KdbV[4] = '-'; + ptr->KdbV[5] = '-'; + ptr->KdbV[6] = '-'; + ptr->KdbV[7] = '-'; + ptr->checksum = 0x00; + ptr->closeFailed = 0x00; //remove closeFailed flag + ptr->closeOk = 0x01; //set closeOk flag + ptr->hash_table_size = (uint64_t)(*hash_table_size); + ptr->key_size = (uint64_t)(*key_size); + ptr->value_size = (uint64_t)(*value_size); + memcpy(ptr->delimiter,"||||||||", 8); + + //sync changes with file + if (0 != msync(ptr, KISSDB_HEADER_SIZE, MS_SYNC | MS_INVALIDATE)) + return KISSDB_ERROR_IO; + + //unmap memory + if (0 != munmap(ptr, KISSDB_HEADER_SIZE)) + return KISSDB_ERROR_IO; + return 0; +} + + +int checkErrorFlags(KISSDB* db) +{ + //mmap header from beginning of file + int mapFlag = PROT_WRITE | PROT_READ; + Header_s* ptr = 0; + ptr = (Header_s*) mmap(NULL, KISSDB_HEADER_SIZE, mapFlag, MAP_SHARED, db->fd, 0); + if (ptr == MAP_FAILED) + return KISSDB_ERROR_IO; + //uint64_t crc = 0; + +#ifdef __checkerror + //check if closeFailed flag is set + if(ptr->closeFailed == 0x01) + { + //TODO implement verifyHashtableCS + + //if closeFailed flag is set, something went wrong at last close -> so check crc + db->shmem_info->crc_invalid = Kdb_true; //check crc for further reads + +#if 0 + DLT_LOG(persComLldbDLTCtx, DLT_LOG_WARN, DLT_STRING("OPENING DB -> closeFailed flag is set: "), DLT_UINT64(ptr->closeFailed)); + crc = (uint64_t) pcoCalcCrc32Csum(db->fd, sizeof(Header_s)); + if(ptr->checksum != 0) //do not check if database is currently in creation + { + if (crc != ptr->checksum) + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_WARN, DLT_STRING("OPENING DB: "), DLT_STRING(db->shmem_ht_name), DLT_STRING(" CHECKSUM IN HEADER : "), DLT_UINT64(ptr->checksum), DLT_STRING(" != CHECKSUM CALCULATED: "), DLT_UINT64(crc)); + //db->shmem_info->crc_invalid = Kdb_true; //check datablocks at further reads + //return KISSDB_ERROR_CORRUPT_DBFILE; //previous close failed and checksum invalid -> error state -> return error + } + else + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, DLT_STRING("OPENING DB: "), DLT_STRING(db->shmem_ht_name), DLT_STRING(" CECHKSUM IN HEADER: "), DLT_UINT64(ptr->checksum), DLT_STRING(" == CHECKSUM CALCULATED: "), DLT_UINT64(crc)); + //db->shmem_info->crc_invalid = Kdb_false; //do not check datablocks at further reads + } + } + else + DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, DLT_STRING("Do not check checksum, database in creation: "), DLT_STRING(db->shmem_ht_name)); +#endif + } + else + { + //DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, DLT_STRING("OPENING DB: closeFailed flag is not set: "), DLT_UINT64(ptr->closeFailed)); + ptr->closeFailed = 0x01; //NO: create close failed flag + } + + + //check if closeOk flag is set + if(ptr->closeOk == 0x01) + { + //DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, DLT_STRING("OPENING DB -> closeOk flag is set: "), DLT_UINT64(ptr->closeOk)); + ptr->closeOk = 0x00; + } + else + { + //if closeOK is not set , something went wrong at last close + db->shmem_info->crc_invalid = Kdb_true; //do crc check at read + +#if 0 + crc = (uint64_t) pcoCalcCrc32Csum(db->fd, sizeof(Header_s)); + if(ptr->checksum != 0) //do not check if database is currently in creation + { + if (crc != ptr->checksum) + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_WARN, DLT_STRING("OPENING DB: "), DLT_STRING(db->shmem_ht_name), DLT_STRING(" CHECKSUM IN HEADER : "), DLT_UINT64(ptr->checksum), DLT_STRING(" != CHECKSUM CALCULATED: "), DLT_UINT64(crc)); + //db->shmem_info->crc_invalid = Kdb_true; + //return KISSDB_ERROR_CORRUPT_DBFILE; //previous close failed and checksum invalid -> error state -> return error + } + else + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, DLT_STRING("OPENING DB: "), DLT_STRING(db->shmem_ht_name), DLT_STRING(" CECHKSUM IN HEADER: "), DLT_UINT64(ptr->checksum), DLT_STRING(" == CHECKSUM CALCULATED: "), DLT_UINT64(crc)); + //db->shmem_info->crc_invalid = Kdb_false; + } + } + else + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, DLT_STRING("Do not check checksum, database in creation: "), DLT_STRING(db->shmem_ht_name)); + } + + DLT_LOG(persComLldbDLTCtx, DLT_LOG_WARN, DLT_STRING("OPENING DB -> closeOk flag is not set: "), DLT_UINT64(ptr->closeOk)); +#endif + + + } +#endif + //sync changes with file + if (0 != msync(ptr, KISSDB_HEADER_SIZE, MS_SYNC | MS_INVALIDATE)) + return KISSDB_ERROR_IO; + + //unmap memory + if (0 != munmap(ptr, KISSDB_HEADER_SIZE)) + return KISSDB_ERROR_IO; + + return 0; +} diff --git a/src/key-value-store/database/kissdb.h b/src/key-value-store/database/kissdb.h new file mode 100644 index 0000000..90446ff --- /dev/null +++ b/src/key-value-store/database/kissdb.h @@ -0,0 +1,323 @@ + /****************************************************************************** + * Project Persistency + * (c) copyright 2014 + * Company XS Embedded GmbH + *****************************************************************************/ +/* (Keep It) Simple Stupid Database +* +* Written by Adam Ierymenko <adam.ierymenko@zerotier.com> +* Modified by Simon Disch <simon.disch@xse.de> +* +* KISSDB is in the public domain and is distributed with NO WARRANTY. +* +* http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* Compile with KISSDB_TEST to build as a test program. */ + +/* Note: big-endian systems will need changes to implement byte swapping +* on hash table file I/O. Or you could just use it as-is if you don't care +* that your database files will be unreadable on little-endian systems. */ + + +#ifndef ___KISSDB_H +#define ___KISSDB_H + +#include <stdio.h> +#include <stdint.h> +#include <pthread.h> +#include "../hashtable/qlibc.h" + +#ifdef __cplusplus +extern "C" { +#endif + +//#define __showTimeMeasurements + +#ifdef __showTimeMeasurements +#define SECONDS2NANO 1000000000L +#define NANO2MIL 1000000L +#define MIL2SEC 1000L +// define for the used clock: "CLOCK_MONOTONIC" or "CLOCK_REALTIME" +#define CLOCK_ID CLOCK_MONOTONIC + +#endif + + +/** + * Version: 2 + * + * This is the file format identifier, and changes any time the file + * format changes. The code version will be this dot something, and can + * be seen in tags in the git repository. + */ +#define KISSDB_VERSION 2 + +//boolean type +typedef int16_t Kdb_bool; +static const int16_t Kdb_true = -1; +static const int16_t Kdb_false = 0; + +typedef struct +{ + uint64_t shmem_size; + uint16_t num_hash_tables; + Kdb_bool cache_initialised; + Kdb_bool crc_invalid; + pthread_rwlock_t rwlock; + pthread_rwlock_t cache_rwlock; +} Shared_Data_s; + + +/** + * Header of the database file -> + */ +typedef struct +{ + char KdbV[8]; + uint64_t checksum; // checksum over database file + uint64_t closeFailed; + uint64_t closeOk; + uint64_t hash_table_size; + uint64_t key_size; + uint64_t value_size; + char delimiter[8]; +} Header_s; + + + +/** + * Hashtable slot entry -> same size for all struct members because of alignment problems on target system!! + */ +typedef struct +{ + int64_t offsetA; + uint64_t checksumA; + int64_t offsetB; + uint64_t checksumB; + uint64_t current; //flag which offset points to the current data -> (if 0x00 offsetA points to current data, if 0x01 offsetB) +} Hashtable_slot_s; + + + +/** + * KISSDB database + * + * These fields should never be changed. + */ +typedef struct { + uint16_t hash_table_size; + uint64_t key_size; + uint64_t value_size; + uint64_t hash_table_size_bytes; + uint64_t old_mapped_size; + Kdb_bool shmem_creator; + Kdb_bool already_open; + Hashtable_slot_s *hash_tables; //shared: stores the hashtables + void* shmem_cached; //shared: memory for key-value pair caching + int shmem_info_fd; + int shmem_ht_fd; + int shmem_cached_fd; + char* shmem_info_name; + char* shmem_cached_name; + char* shmem_ht_name; + Shared_Data_s* shmem_info; + qhasharr_t *tbl; //reference to cached datastructure + int fd; //local fd +} KISSDB; + + +/** + * I/O error or file not found + */ +#define KISSDB_ERROR_IO -1 + +/** + * Out of memory + */ +#define KISSDB_ERROR_MALLOC -2 + +/** + * Invalid paramters (e.g. missing _size paramters on init to create database) + */ +#define KISSDB_ERROR_INVALID_PARAMETERS -3 + +/** + * Database file appears corrupt + */ +#define KISSDB_ERROR_CORRUPT_DBFILE -4 + +/** + * Database file appears corrupt + */ +#define KISSDB_ERROR_ACCESS_VIOLATION -5 + +/** + * Unable to unmap shared memory + */ +#define KISSDB_ERROR_UNMAP_SHM -6 + +/** + * Unable to open shared memory + */ +#define KISSDB_ERROR_OPEN_SHM -7 + +/** + * Unable to remap shared memory + */ +#define KISSDB_ERROR_REMAP_SHM -8 + +/** + * Unable to map shared memory + */ +#define KISSDB_ERROR_MAP_SHM -9 + +/** + * Unable to resize shared memory + */ +#define KISSDB_ERROR_RESIZE_SHM -10 + +/** + * Unable to close shared memory + */ +#define KISSDB_ERROR_CLOSE_SHM -11 + +/** + * Open mode: read only + */ +#define KISSDB_OPEN_MODE_RDONLY 1 + +/** + * Open mode: read/write + */ +#define KISSDB_OPEN_MODE_RDWR 2 + +/** + * Open mode: read/write, create if doesn't exist + */ +#define KISSDB_OPEN_MODE_RWCREAT 3 + +/** + * Open mode: truncate database, open for reading and writing + */ +#define KISSDB_OPEN_MODE_RWREPLACE 4 + +/** + * Open database + * + * The three _size parameters must be specified if the database could + * be created or re-created. Otherwise an error will occur. If the + * database already exists, these parameters are ignored and are read + * from the database. You can check the struture afterwords to see what + * they were. + * + * @param db Database struct + * @param path Path to file + * @param mode One of the KISSDB_OPEN_MODE constants + * @param hash_table_size Size of hash table in entries (must be >0) + * @param key_size Size of keys in bytes + * @param value_size Size of values in bytes + * @return 0 on success, nonzero on error (see kissdb.h for error codes) + */ +extern int KISSDB_open( + KISSDB *db, + const char *path, + int mode, + uint16_t hash_table_size, + uint64_t key_size, + uint64_t value_size); + +/** + * Close database + * + * @param db Database struct + * @return negative on error (see kissdb.h for error codes), 0 on success + */ +extern int KISSDB_close(KISSDB *db); + +/** + * Get an entry + * + * @param db Database struct + * @param key Key (key_size bytes) + * @param vbuf Value buffer (value_size bytes capacity) + * @return negative on error (see kissdb.h for error codes), 0 on success, 1 if key not found + */ +extern int KISSDB_get(KISSDB *db,const void *key,void *vbuf); + + + +/** + * delete an entry (offset in hashtable is set to 0 and record content is set to 0 + * + * @param db Database struct + * @param key Key (key_size bytes) + * @return negative on error (see kissdb.h for error codes), 0 on success, 1 if key not found + */ +extern int KISSDB_delete(KISSDB *db,const void *key); + +/** + * Put an entry (overwriting it if it already exists) + * + * In the already-exists case the size of the database file does not + * change. + * + * @param db Database struct + * @param key Key (key_size bytes) + * @param value Value (value_size bytes) + * @return negative on error (see kissdb.h for error codes) error, 0 on success + */ +extern int KISSDB_put(KISSDB *db,const void *key,const void *value); + +/** + * Cursor used for iterating over all entries in database + */ +typedef struct { + KISSDB *db; + unsigned long h_no; + unsigned long h_idx; +} KISSDB_Iterator; + +/** + * Initialize an iterator + * + * @param db Database struct + * @param i Iterator to initialize + */ +extern void KISSDB_Iterator_init(KISSDB *db,KISSDB_Iterator *dbi); + +/** + * Get the next entry + * + * The order of entries returned by iterator is undefined. It depends on + * how keys hash. + * + * @param Database iterator + * @param kbuf Buffer to fill with next key (key_size bytes) + * @param vbuf Buffer to fill with next value (value_size bytes) + * @return 0 if there are no more entries, negative on error, positive if kbuf/vbuf have been filled + */ +extern int KISSDB_Iterator_next(KISSDB_Iterator *dbi,void *kbuf,void *vbuf); + + +extern Kdb_bool freeKdbShmemPtr(void * shmem_ptr, size_t length); +extern void * getKdbShmemPtr(int shmem, size_t length); +extern Kdb_bool kdbShmemClose(int shmem, const char * shmName); +extern int kdbShmemOpen(const char * name, size_t length, Kdb_bool* shmCreator); +extern char * kdbGetShmName(const char * format, const char * path); +extern void Kdb_wrlock(pthread_rwlock_t * wrlock); +extern void Kdb_rdlock(pthread_rwlock_t * rdlock); +extern void Kdb_unlock(pthread_rwlock_t * lock); +extern int readHeader(KISSDB* db, uint16_t* hash_table_size, uint64_t* key_size, uint64_t* value_size); +extern int writeHeader(KISSDB* db, uint16_t* hash_table_size, uint64_t* key_size, uint64_t* value_size); +extern int checkErrorFlags(KISSDB* db); + +#if 0 +extern void printSharedHashtable(KISSDB *db); +#endif + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/src/key-value-store/hashtable/md5.h b/src/key-value-store/hashtable/md5.h new file mode 100644 index 0000000..cfcf6c6 --- /dev/null +++ b/src/key-value-store/hashtable/md5.h @@ -0,0 +1,50 @@ +/* + * Copyright (C) 1991-2, RSA Data Security, Inc. Created 1991. All + * rights reserved. + * + * License to copy and use this software is granted provided that it + * is identified as the "RSA Data Security, Inc. MD5 Message-Digest + * Algorithm" in all material mentioning or referencing this software + * or this function. + * + * License is also granted to make and use derivative works provided + * that such works are identified as "derived from the RSA Data + * Security, Inc. MD5 Message-Digest Algorithm" in all material + * mentioning or referencing the derived work. + * + * RSA Data Security, Inc. makes no representations concerning either + * the merchantability of this software or the suitability of this + * software for any particular purpose. It is provided "as is" + * without express or implied warranty of any kind. + * + * These notices must be retained in any copies of any part of this + * documentation and/or software. + */ + +#ifndef _Q_MD5_H_ +#define _Q_MD5_H_ + +#define MD5_BLOCK_LENGTH 64 +#define MD5_DIGEST_LENGTH 16 +#define MD5_DIGEST_STRING_LENGTH (MD5_DIGEST_LENGTH * 2 + 1) + +/* MD5 context. */ +typedef struct MD5Context { + u_int32_t state[4]; /* state (ABCD) */ + u_int32_t count[2]; /* number of bits, modulo 2^64 (lsb first) */ + unsigned char buffer[64]; /* input buffer */ +} MD5_CTX; + +#include <sys/cdefs.h> + +__BEGIN_DECLS +void MD5Init(MD5_CTX *); +void MD5Update(MD5_CTX *, const unsigned char *, unsigned int); +void MD5Final(unsigned char[16], MD5_CTX *); +char * MD5End(MD5_CTX *, char *); +char * MD5File(const char *, char *); +char * MD5FileChunk(const char *, char *, off_t, off_t); +char * MD5Data(const unsigned char *, unsigned int, char *); +__END_DECLS + +#endif /* _Q_MD5_H_ */ diff --git a/src/key-value-store/hashtable/md5c.c b/src/key-value-store/hashtable/md5c.c new file mode 100644 index 0000000..de9e47f --- /dev/null +++ b/src/key-value-store/hashtable/md5c.c @@ -0,0 +1,296 @@ +/* + * MD5C.C - RSA Data Security, Inc., MD5 message-digest algorithm + * + * Copyright (C) 1991-2, RSA Data Security, Inc. Created 1991. All + * rights reserved. + * + * License to copy and use this software is granted provided that it + * is identified as the "RSA Data Security, Inc. MD5 Message-Digest + * Algorithm" in all material mentioning or referencing this software + * or this function. + * + * License is also granted to make and use derivative works provided + * that such works are identified as "derived from the RSA Data + * Security, Inc. MD5 Message-Digest Algorithm" in all material + * mentioning or referencing the derived work. + * + * RSA Data Security, Inc. makes no representations concerning either + * the merchantability of this software or the suitability of this + * software for any particular purpose. It is provided "as is" + * without express or implied warranty of any kind. + * + * These notices must be retained in any copies of any part of this + * documentation and/or software. + * + * This code is the same as the code published by RSA Inc. It has been + * edited for clarity and style only. + */ + +#include <sys/cdefs.h> +#include <sys/types.h> +#include <string.h> +#include "md5.h" + +static void MD5Transform( u_int32_t[4], const unsigned char[64]); + +#if (BYTE_ORDER == LITTLE_ENDIAN) +#define Encode memcpy +#define Decode memcpy +#else + +/* + * Encodes input (u_int32_t) into output (unsigned char). Assumes len is + * a multiple of 4. + */ + +static void Encode (unsigned char *output, u_int32_t *input, unsigned int len) { + unsigned int i; + u_int32_t *op = (u_int32_t *)output; + + for (i = 0; i < len / 4; i++) + op[i] = htole32(input[i]); +} + +/* + * Decodes input (unsigned char) into output (u_int32_t). Assumes len is + * a multiple of 4. + */ + +static void Decode (u_int32_t *output, const unsigned char *input, unsigned int len) { + unsigned int i; + const u_int32_t *ip = (const u_int32_t *)input; + + for (i = 0; i < len / 4; i++) + output[i] = le32toh(ip[i]); +} +#endif + +static unsigned char PADDING[64] = { 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0 }; + +/* F, G, H and I are basic MD5 functions. */ +#define F(x, y, z) (((x) & (y)) | ((~x) & (z))) +#define G(x, y, z) (((x) & (z)) | ((y) & (~z))) +#define H(x, y, z) ((x) ^ (y) ^ (z)) +#define I(x, y, z) ((y) ^ ((x) | (~z))) + +/* ROTATE_LEFT rotates x left n bits. */ +#define ROTATE_LEFT(x, n) (((x) << (n)) | ((x) >> (32-(n)))) + +/* + * FF, GG, HH, and II transformations for rounds 1, 2, 3, and 4. + * Rotation is separate from addition to prevent recomputation. + */ +#define FF(a, b, c, d, x, s, ac) { \ + (a) += F ((b), (c), (d)) + (x) + (u_int32_t)(ac); \ + (a) = ROTATE_LEFT ((a), (s)); \ + (a) += (b); \ + } +#define GG(a, b, c, d, x, s, ac) { \ + (a) += G ((b), (c), (d)) + (x) + (u_int32_t)(ac); \ + (a) = ROTATE_LEFT ((a), (s)); \ + (a) += (b); \ + } +#define HH(a, b, c, d, x, s, ac) { \ + (a) += H ((b), (c), (d)) + (x) + (u_int32_t)(ac); \ + (a) = ROTATE_LEFT ((a), (s)); \ + (a) += (b); \ + } +#define II(a, b, c, d, x, s, ac) { \ + (a) += I ((b), (c), (d)) + (x) + (u_int32_t)(ac); \ + (a) = ROTATE_LEFT ((a), (s)); \ + (a) += (b); \ + } + +/* MD5 initialization. Begins an MD5 operation, writing a new context. */ + +void MD5Init(MD5_CTX *context) { + + context->count[0] = context->count[1] = 0; + + /* Load magic initialization constants. */ + context->state[0] = 0x67452301; + context->state[1] = 0xefcdab89; + context->state[2] = 0x98badcfe; + context->state[3] = 0x10325476; +} + +/* + * MD5 block update operation. Continues an MD5 message-digest + * operation, processing another message block, and updating the + * context. + */ + +void MD5Update(MD5_CTX *context, const unsigned char *input, + unsigned int inputLen) { + unsigned int i, idx, partLen; + + /* Compute number of bytes mod 64 */ + idx = (unsigned int) ((context->count[0] >> 3) & 0x3F); + + /* Update number of bits */ + if ((context->count[0] += ((u_int32_t) inputLen << 3)) + < ((u_int32_t) inputLen << 3)) + context->count[1]++; + context->count[1] += ((u_int32_t) inputLen >> 29); + + partLen = 64 - idx; + + /* Transform as many times as possible. */ + if (inputLen >= partLen) { + memcpy((void *) &context->buffer[idx], (const void *) input, partLen); + MD5Transform(context->state, context->buffer); + + for (i = partLen; i + 63 < inputLen; i += 64) + MD5Transform(context->state, &input[i]); + + idx = 0; + } else + i = 0; + + /* Buffer remaining input */ + memcpy((void *) &context->buffer[idx], (const void *) &input[i], + inputLen - i); +} + +/* + * MD5 padding. Adds padding followed by original length. + */ + +static void MD5Pad(MD5_CTX *context) { + unsigned char bits[8]; + unsigned int idx, padLen; + + /* Save number of bits */ + Encode(bits, context->count, 8); + + /* Pad out to 56 mod 64. */ + idx = (unsigned int) ((context->count[0] >> 3) & 0x3f); + padLen = (idx < 56) ? (56 - idx) : (120 - idx); + MD5Update(context, PADDING, padLen); + + /* Append length (before padding) */ + MD5Update(context, bits, 8); +} + +/* + * MD5 finalization. Ends an MD5 message-digest operation, writing the + * the message digest and zeroizing the context. + */ + +void MD5Final(unsigned char digest[16], MD5_CTX *context) { + /* Do padding. */ + MD5Pad(context); + + /* Store state in digest */ + Encode(digest, context->state, 16); + + /* Zeroize sensitive information. */ + memset((void *) context, 0, sizeof(*context)); +} + +/* MD5 basic transformation. Transforms state based on block. */ + +static void MD5Transform(u_int32_t state[4], const unsigned char block[64]) { + u_int32_t a = state[0], b = state[1], c = state[2], d = state[3], x[16]; + + Decode(x, block, 64); + + /* Round 1 */ +#define S11 7 +#define S12 12 +#define S13 17 +#define S14 22 + FF(a, b, c, d, x[0], S11, 0xd76aa478); /* 1 */ + FF(d, a, b, c, x[1], S12, 0xe8c7b756); /* 2 */ + FF(c, d, a, b, x[2], S13, 0x242070db); /* 3 */ + FF(b, c, d, a, x[3], S14, 0xc1bdceee); /* 4 */ + FF(a, b, c, d, x[4], S11, 0xf57c0faf); /* 5 */ + FF(d, a, b, c, x[5], S12, 0x4787c62a); /* 6 */ + FF(c, d, a, b, x[6], S13, 0xa8304613); /* 7 */ + FF(b, c, d, a, x[7], S14, 0xfd469501); /* 8 */ + FF(a, b, c, d, x[8], S11, 0x698098d8); /* 9 */ + FF(d, a, b, c, x[9], S12, 0x8b44f7af); /* 10 */ + FF(c, d, a, b, x[10], S13, 0xffff5bb1); /* 11 */ + FF(b, c, d, a, x[11], S14, 0x895cd7be); /* 12 */ + FF(a, b, c, d, x[12], S11, 0x6b901122); /* 13 */ + FF(d, a, b, c, x[13], S12, 0xfd987193); /* 14 */ + FF(c, d, a, b, x[14], S13, 0xa679438e); /* 15 */ + FF(b, c, d, a, x[15], S14, 0x49b40821); /* 16 */ + + /* Round 2 */ +#define S21 5 +#define S22 9 +#define S23 14 +#define S24 20 + GG(a, b, c, d, x[1], S21, 0xf61e2562); /* 17 */ + GG(d, a, b, c, x[6], S22, 0xc040b340); /* 18 */ + GG(c, d, a, b, x[11], S23, 0x265e5a51); /* 19 */ + GG(b, c, d, a, x[0], S24, 0xe9b6c7aa); /* 20 */ + GG(a, b, c, d, x[5], S21, 0xd62f105d); /* 21 */ + GG(d, a, b, c, x[10], S22, 0x2441453); /* 22 */ + GG(c, d, a, b, x[15], S23, 0xd8a1e681); /* 23 */ + GG(b, c, d, a, x[4], S24, 0xe7d3fbc8); /* 24 */ + GG(a, b, c, d, x[9], S21, 0x21e1cde6); /* 25 */ + GG(d, a, b, c, x[14], S22, 0xc33707d6); /* 26 */ + GG(c, d, a, b, x[3], S23, 0xf4d50d87); /* 27 */ + GG(b, c, d, a, x[8], S24, 0x455a14ed); /* 28 */ + GG(a, b, c, d, x[13], S21, 0xa9e3e905); /* 29 */ + GG(d, a, b, c, x[2], S22, 0xfcefa3f8); /* 30 */ + GG(c, d, a, b, x[7], S23, 0x676f02d9); /* 31 */ + GG(b, c, d, a, x[12], S24, 0x8d2a4c8a); /* 32 */ + + /* Round 3 */ +#define S31 4 +#define S32 11 +#define S33 16 +#define S34 23 + HH(a, b, c, d, x[5], S31, 0xfffa3942); /* 33 */ + HH(d, a, b, c, x[8], S32, 0x8771f681); /* 34 */ + HH(c, d, a, b, x[11], S33, 0x6d9d6122); /* 35 */ + HH(b, c, d, a, x[14], S34, 0xfde5380c); /* 36 */ + HH(a, b, c, d, x[1], S31, 0xa4beea44); /* 37 */ + HH(d, a, b, c, x[4], S32, 0x4bdecfa9); /* 38 */ + HH(c, d, a, b, x[7], S33, 0xf6bb4b60); /* 39 */ + HH(b, c, d, a, x[10], S34, 0xbebfbc70); /* 40 */ + HH(a, b, c, d, x[13], S31, 0x289b7ec6); /* 41 */ + HH(d, a, b, c, x[0], S32, 0xeaa127fa); /* 42 */ + HH(c, d, a, b, x[3], S33, 0xd4ef3085); /* 43 */ + HH(b, c, d, a, x[6], S34, 0x4881d05); /* 44 */ + HH(a, b, c, d, x[9], S31, 0xd9d4d039); /* 45 */ + HH(d, a, b, c, x[12], S32, 0xe6db99e5); /* 46 */ + HH(c, d, a, b, x[15], S33, 0x1fa27cf8); /* 47 */ + HH(b, c, d, a, x[2], S34, 0xc4ac5665); /* 48 */ + + /* Round 4 */ +#define S41 6 +#define S42 10 +#define S43 15 +#define S44 21 + II(a, b, c, d, x[0], S41, 0xf4292244); /* 49 */ + II(d, a, b, c, x[7], S42, 0x432aff97); /* 50 */ + II(c, d, a, b, x[14], S43, 0xab9423a7); /* 51 */ + II(b, c, d, a, x[5], S44, 0xfc93a039); /* 52 */ + II(a, b, c, d, x[12], S41, 0x655b59c3); /* 53 */ + II(d, a, b, c, x[3], S42, 0x8f0ccc92); /* 54 */ + II(c, d, a, b, x[10], S43, 0xffeff47d); /* 55 */ + II(b, c, d, a, x[1], S44, 0x85845dd1); /* 56 */ + II(a, b, c, d, x[8], S41, 0x6fa87e4f); /* 57 */ + II(d, a, b, c, x[15], S42, 0xfe2ce6e0); /* 58 */ + II(c, d, a, b, x[6], S43, 0xa3014314); /* 59 */ + II(b, c, d, a, x[13], S44, 0x4e0811a1); /* 60 */ + II(a, b, c, d, x[4], S41, 0xf7537e82); /* 61 */ + II(d, a, b, c, x[11], S42, 0xbd3af235); /* 62 */ + II(c, d, a, b, x[2], S43, 0x2ad7d2bb); /* 63 */ + II(b, c, d, a, x[9], S44, 0xeb86d391); /* 64 */ + + state[0] += a; + state[1] += b; + state[2] += c; + state[3] += d; + + /* Zeroize sensitive information. */ + memset((void *) x, 0, sizeof(x)); +} diff --git a/src/key-value-store/hashtable/qhash.c b/src/key-value-store/hashtable/qhash.c new file mode 100644 index 0000000..e7709aa --- /dev/null +++ b/src/key-value-store/hashtable/qhash.c @@ -0,0 +1,155 @@ +/****************************************************************************** + * qLibc + * + * Copyright (c) 2010-2014 Seungyoung Kim. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + *****************************************************************************/ + +/** + * @file qhash.c Hash APIs. + */ +/* + * Modified parts of this file by XS Embedded GmbH, 2014 + */ + +#include <stdio.h> +#include <stdlib.h> +#include <stdbool.h> +#include <stdint.h> +#include <string.h> +#include <unistd.h> +#include <fcntl.h> +#include <errno.h> +#include <sys/types.h> +#include <sys/stat.h> +#include "md5.h" +#include "qhash.h" + +/** + * Calculate 128-bit(16-bytes) MD5 hash. + * + * @param data source object + * @param nbytes size of data + * @param retbuf user buffer. It must be at leat 16-bytes long. + * + * @return true if successful, otherwise false. + * + * @code + * // get MD5 + * unsigned char md5hash[16]; + * qhashmd5((void*)"hello", 5, md5hash); + * + * // hex encode + * char *md5ascii = qhex_encode(md5hash, 16); + * printf("Hex encoded MD5: %s\n", md5ascii); + * free(md5ascii); + * @endcode + */ +bool qhashmd5(const void *data, size_t nbytes, void *retbuf) { + if (data == NULL || retbuf == NULL) { + errno = EINVAL; + return false; + } + + MD5_CTX context; + MD5Init(&context); + MD5Update(&context, (unsigned char *) data, (unsigned int) nbytes); + MD5Final(retbuf, &context); + + return true; +} + + +/** + * Get 32-bit Murmur3 hash. + * + * @param data source data + * @param nbytes size of data + * + * @return 32-bit unsigned hash value. + * + * @code + * uint32_t hashval = qhashmurmur3_32((void*)"hello", 5); + * @endcode + * + * @code + * MurmurHash3 was created by Austin Appleby in 2008. The initial + * implementation was published in C++ and placed in the public. + * https://sites.google.com/site/murmurhash/ + * Seungyoung Kim has ported its implementation into C language + * in 2012 and published it as a part of qLibc component. + * @endcode + */ +uint32_t qhashmurmur3_32(const void *data, size_t nbytes) { + if (data == NULL || nbytes == 0) + return 0; + + const uint32_t c1 = 0xcc9e2d51; + const uint32_t c2 = 0x1b873593; + + const int nblocks = nbytes / 4; + const uint32_t *blocks = (const uint32_t *) (data); + const uint8_t *tail = (const uint8_t *) (data + (nblocks * 4)); + + uint32_t h = 0; + + int i; + uint32_t k; + for (i = 0; i < nblocks; i++) { + k = blocks[i]; + + k *= c1; + k = (k << 15) | (k >> (32 - 15)); + k *= c2; + + h ^= k; + h = (h << 13) | (h >> (32 - 13)); + h = (h * 5) + 0xe6546b64; + } + + k = 0; + switch (nbytes & 3) { + case 3: + k ^= tail[2] << 16; + case 2: + k ^= tail[1] << 8; + case 1: + k ^= tail[0]; + k *= c1; + k = (k << 15) | (k >> (32 - 15)); + k *= c2; + h ^= k; + }; + + h ^= nbytes; + + h ^= h >> 16; + h *= 0x85ebca6b; + h ^= h >> 13; + h *= 0xc2b2ae35; + h ^= h >> 16; + + return h; +} + diff --git a/src/key-value-store/hashtable/qhash.h b/src/key-value-store/hashtable/qhash.h new file mode 100644 index 0000000..b873799 --- /dev/null +++ b/src/key-value-store/hashtable/qhash.h @@ -0,0 +1,64 @@ +/******************************************************************************
+ * qLibc
+ *
+ * Copyright (c) 2010-2014 Seungyoung Kim.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *****************************************************************************/
+
+/**
+ * qhash header file.
+ *
+ * @file qhash.h
+ */
+
+/*
+ * Modified parts of this file by XS Embedded GmbH, 2014
+ */
+
+#ifndef _QHASH_H
+#define _QHASH_H
+
+#include <stdlib.h>
+#include <stdbool.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern bool qhashmd5(const void *data, size_t nbytes, void *retbuf);
+//extern bool qhashmd5_file(const char *filepath, off_t offset, ssize_t nbytes,
+// void *retbuf);
+
+//extern uint32_t qhashfnv1_32(const void *data, size_t nbytes);
+//extern uint64_t qhashfnv1_64(const void *data, size_t nbytes);
+
+extern uint32_t qhashmurmur3_32(const void *data, size_t nbytes);
+//extern bool qhashmurmur3_128(const void *data, size_t nbytes, void *retbuf);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_QHASH_H */
diff --git a/src/key-value-store/hashtable/qhasharr.c b/src/key-value-store/hashtable/qhasharr.c new file mode 100644 index 0000000..97169bf --- /dev/null +++ b/src/key-value-store/hashtable/qhasharr.c @@ -0,0 +1,869 @@ +/****************************************************************************** + * qLibc + * + * Copyright (c) 2010-2014 Seungyoung Kim. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + *****************************************************************************/ + +/** + * @file qhasharr.c Static(array) hash-table implementation. + * + * qhasharr implements a hash-table which maps keys to values and stores into + * fixed size static memory like shared-memory and memory-mapped file. + * The creator qhasharr() initializes static memory to makes small slots in it. + * The default slot size factors are defined in _Q_HASHARR_KEYSIZE and + * _Q_HASHARR_VALUESIZE. And they are applied at compile time. + * + * The value part of an element will be stored across several slots if it's size + * exceeds the slot size. But the key part of an element will be truncated if + * the size exceeds and it's length and more complex MD5 hash value will be + * stored with the key. So to look up a particular key, first we find an element + * which has same hash value. If the key was not truncated, we just do key + * comparison. But if the key was truncated because it's length exceeds, we do + * both md5 and key comparison(only stored size) to verify that the key is same. + * So please be aware of that, theoretically there is a possibility we pick + * wrong element in case a key exceeds the limit, has same length and MD5 hash + * with lookup key. But this possibility is extreamly low and almost never + * happen in practice. If you happpen to want to make sure everything, + * you set _Q_HASHARR_KEYSIZE big enough at compile time to make sure all keys + * fits in it. + * + * qhasharr hash-table does not support thread-safe. So users should handle + * race conditions on application side by raising user lock before calling + * functions which modify the table data. + * + * @code + * [Data Structure Diagram] + * + * +--[Static Flat Memory Area]-----------------------------------------------+ + * | +-[Header]---------+ +-[Slot 0]---+ +-[Slot 1]---+ +-[Slot N]---+ | + * | |Private table data| |KEY A|DATA A| |KEY B|DATA B| .... |KEY N|DATA N| | + * | +------------------+ +------------+ +------------+ +------------+ | + * +--------------------------------------------------------------------------+ + * + * Below diagram shows how a big value is stored. + * +--[Static Flat Memory Area------------------------------------------------+ + * | +--------+ +-[Slot 0]---+ +-[Slot 1]---+ +-[Slot 2]---+ +-[Slot 3]-----+ | + * | |TBL INFO| |KEY A|DATA A| |DATA A cont.| |KEY B|DATA B| |DATA A cont. | | + * | +--------+ +------------+ +------------+ +------------+ +--------------+ | + * | ^~~link~~^ ^~~~~~~~~~link~~~~~~~~~^ | + * +--------------------------------------------------------------------------+ + * @endcode + * + * @code + * // initialize hash-table. + * char memory[1000 * 10]; + * qhasharr_t *tbl = qhasharr(memory, sizeof(memory)); + * if(tbl == NULL) return; + * + * // insert elements (key duplication does not allowed) + * tbl->putstr(tbl, "e1", "a"); + * tbl->putstr(tbl, "e2", "b"); + * tbl->putstr(tbl, "e3", "c"); + * + * // debug print out + * tbl->//DEBUG(tbl, stdout); + * + * char *e2 = tbl->getstr(tbl, "e2"); + * if(e2 != NULL) { + * printf("getstr('e2') : %s\n", e2); + * free(e2); + * } + * + * // Release reference object. + * tbl->free(tbl); + * @endcode + * + * An example for using hash table over shared memory. + * + * @code + * [CREATOR SIDE] + * int maxslots = 1000; + * int memsize = qhasharr_calculate_memsize(maxslots); + * + * // create shared memory + * int shmid = qshm_init("/tmp/some_id_file", 'q', memsize, true); + * if(shmid < 0) return -1; // creation failed + * void *memory = qshm_get(shmid); + * + * // initialize hash-table + * qhasharr_t *tbl = qhasharr(memory, memsize); + * if(hasharr == NULL) return -1; + * + * (...your codes with your own locking mechanism...) + * + * // Release reference object + * tbl->free(tbl); + * + * // destroy shared memory + * qshm_free(shmid); + * + * [USER SIDE] + * int shmid = qshm_getid("/tmp/some_id_file", 'q'); + * + * // get shared memory + * void *memory = qshm_get(shmid); + * + * // map existing memory into table + * qhasharr_t *tbl = qhasharr(memory, 0); + * + * (...your codes with your own locking mechanism...) + * + * // Release reference object + * tbl->free(tbl); + * @endcode + */ + +/* + * Modified parts of this file by XS Embedded GmbH, 2014 + */ + +#include <stdio.h> +#include <stdlib.h> +#include <stdbool.h> +#include <stdint.h> +#include <inttypes.h> +#include <stdarg.h> +#include <string.h> +#include <errno.h> +#include "qhash.h" +#include "qhasharr.h" + +#ifndef _DOXYGEN_SKIP + +static bool put(qhasharr_t *tbl, const char *key, const void *value, + size_t size); + +static void *get(qhasharr_t *tbl, const char *key, size_t *size); + +static bool getnext(qhasharr_t *tbl, qnobj_t *obj, int *idx); + +static bool remove_(qhasharr_t *tbl, const char *key); + +static int size(qhasharr_t *tbl, int *maxslots, int *usedslots); + +static void free_(qhasharr_t *tbl); + +// internal usages +static int _find_empty(qhasharr_t *tbl, int startidx); +static int _get_idx(qhasharr_t *tbl, const char *key, unsigned int hash); +static void *_get_data(qhasharr_t *tbl, int idx, size_t *size); +static bool _put_data(qhasharr_t *tbl, int idx, unsigned int hash, + const char *key, const void *value, size_t size, + int count); +static bool _copy_slot(qhasharr_t *tbl, int idx1, int idx2); +static bool _remove_slot(qhasharr_t *tbl, int idx); +static bool _remove_data(qhasharr_t *tbl, int idx); + +#endif + +/** + * Get how much memory is needed for N slots. + * + * @param max a number of maximum internal slots + * + * @return memory size needed + * + * @note + * This can be used for calculating minimum memory size for N slots. + */ +size_t qhasharr_calculate_memsize(int max) { + size_t memsize = sizeof(qhasharr_data_t) + + (sizeof(qhasharr_slot_t) * (max)); + return memsize; +} + +/** + * Initialize static hash table + * + * @param memory a pointer of data memory. + * @param memsize a size of data memory, 0 for using existing data. + * + * @return qhasharr_t container pointer, otherwise returns NULL. + * @retval errno will be set in error condition. + * - EINVAL : Assigned memory is too small. It must bigger enough to allocate + * at least 1 slot. + * + * @code + * // initialize hash-table with 100 slots. + * // A single element can take several slots. + * char memory[112 * 100]; + * + * // Initialize new table. + * qhasharr_t *tbl = qhasharr(memory, sizeof(memory)); + * + * // Use existing table. + * qhasharr_t *tbl2 = qhasharr(memory, 0); + * @endcode + */ +qhasharr_t *qhasharr(void *memory, size_t memsize) { + // Structure memory. + qhasharr_data_t *data = (qhasharr_data_t *) memory; + + // Initialize data if memsize is set or use existing data. + if (memsize > 0) { + // calculate max + int maxslots = (memsize - sizeof(qhasharr_data_t)) + / sizeof(qhasharr_slot_t); + if (maxslots < 1 || memsize <= sizeof(qhasharr_t)) { + //errno = EINVAL; + return NULL; + } + + // Set memory. + //memset((void *) data, 0, memsize); //TODO check if initialisation is needed + data->maxslots = maxslots; + data->usedslots = 0; + data->num = 0; + } + + // Set data address. Shared memory returns virtul address. + data->slots = (qhasharr_slot_t *) (memory + sizeof(qhasharr_data_t)); + + // Create the table object. + qhasharr_t *tbl = (qhasharr_t *) malloc(sizeof(qhasharr_t)); + if (tbl == NULL) { + //errno = ENOMEM; + return NULL; + } + memset((void *) tbl, 0, sizeof(qhasharr_t)); + + // assign methods + tbl->put = put; + + tbl->get = get; + + tbl->getnext = getnext; + + tbl->remove = remove_; + + tbl->size = size; + + tbl->free = free_; + + tbl->data = data; + + return tbl; +} + +/** + * qhasharr->put(): Put an object into this table. + * + * @param tbl qhasharr_t container pointer. + * @param key key string + * @param value value object data + * @param size size of value + * + * @return true if successful, otherwise returns false + * @retval errno will be set in error condition. + * - ENOBUFS : Table doesn't have enough space to store the object. + * - EINVAL : Invalid argument. + * - EFAULT : Unexpected error. Data structure is not constant. + */ +static bool put(qhasharr_t *tbl, const char *key, const void *value, + size_t size) { + if (tbl == NULL || key == NULL || value == NULL) { + //errno = EINVAL; + return false; + } + + qhasharr_data_t *data = tbl->data; + + // check full + if (data->usedslots >= data->maxslots || ((data->usedslots + (size / 32)) >= data->maxslots)) { + //DEBUG("hasharr: put %s - FULL", key); + //errno = ENOBUFS; + return false; + } + + // get hash integer + unsigned int hash = qhashmurmur3_32(key, strlen(key)) % data->maxslots; + + // check, is slot empty + if (data->slots[hash].count == 0) { // empty slot + // put data + if (_put_data(tbl, hash, hash, key, value, size, 1) == false) { + //DEBUG("hasharr: FAILED put(new) %s", key); + return false; + } //DEBUG("hasharr: put(new) %s (idx=%d,hash=%u,tot=%d)", + // key, hash, hash, data->usedslots); + } else if (data->slots[hash].count > 0) { // same key or hash collision + // check same key; + int idx = _get_idx(tbl, key, hash); + if (idx >= 0) { // same key + // remove and recall + remove_(tbl, key); + //printf("overwriting existent key 2%s\n", key); + return put(tbl, key, value, size); + } else { // no same key, just hash collision + // find empty slot + int idx = _find_empty(tbl, hash); + if (idx < 0) { + //errno = ENOBUFS; + return false; + } + + // put data. -1 is used for collision resolution (idx != hash); + if (_put_data(tbl, idx, hash, key, value, size, -1) == false) { + //DEBUG("hasharr: FAILED put(col) %s", key); + return false; + } + + // increase counter from leading slot + data->slots[hash].count++; + + //DEBUG("hasharr: put(col) %s (idx=%d,hash=%u,tot=%d)", + // key, idx, hash, data->usedslots); + } + } else { + // in case of -1 or -2, move it. -1 used for collision resolution, + // -2 used for oversized value data. + + // find empty slot + int idx = _find_empty(tbl, hash + 1); + if (idx < 0) { + //errno = ENOBUFS; + return false; + } + + // move dup slot to empty + _copy_slot(tbl, idx, hash); + _remove_slot(tbl, hash); + + // in case of -2, adjust link of mother + if (data->slots[idx].count == -2) { + data->slots[data->slots[idx].hash].link = idx; + if (data->slots[idx].link != -1) { + data->slots[data->slots[idx].link].hash = idx; + } + } + + // store data + if (_put_data(tbl, hash, hash, key, value, size, 1) == false) { + //DEBUG("hasharr: FAILED put(swp) %s", key); + return false; + } + + //DEBUG("hasharr: put(swp) %s (idx=%u,hash=%u,tot=%d)", + // key, hash, hash, data->usedslots); + } + return true; +} + + + + +/** + * qhasharr->get(): Get an object from this table + * + * @param tbl qhasharr_t container pointer. + * @param key key string + * @param size if not NULL, oject size will be stored + * + * @return malloced object pointer if successful, otherwise(not found) + * returns NULL + * @retval errno will be set in error condition. + * - ENOENT : No such key found. + * - EINVAL : Invalid argument. + * - ENOMEM : Memory allocation failed. + * + * @note + * returned object must be freed after done using. + */ +static void *get(qhasharr_t *tbl, const char *key, size_t *size) { + if (tbl == NULL || key == NULL) { + //errno = EINVAL; + return NULL; + } + + qhasharr_data_t *data = tbl->data; + + // get hash integer + unsigned int hash = qhashmurmur3_32(key, strlen(key)) % data->maxslots; + + int idx = _get_idx(tbl, key, hash); + if (idx < 0) { + //errno = ENOENT; + return NULL; + } + + return _get_data(tbl, idx, size); +} + +/** + * qhasharr->getnext(): Get next element. + * + * @param tbl qhasharr_t container pointer. + * @param idx index pointer + * + * @return key name string if successful, otherwise(end of table) returns NULL + * @retval errno will be set in error condition. + * - ENOENT : No next element. + * - EINVAL : Invald argument. + * - ENOMEM : Memory allocation failed. + * + * @code + * int idx = 0; + * qnobj_t obj; + * while(tbl->getnext(tbl, &obj, &idx) == true) { + * printf("NAME=%s, DATA=%s, SIZE=%zu\n", + * obj.name, (char*)obj.data, obj.size); + * free(obj.name); + * free(obj.data); + * } + * @endcode + * + * @note + * Please be aware a key name will be returned with truncated length + * because key name is truncated when it put into the table if it's length is + * longer than _Q_HASHARR_KEYSIZE. + */ +static bool getnext(qhasharr_t *tbl, qnobj_t *obj, int *idx) { + if (tbl == NULL || obj == NULL || idx == NULL) { + //errno = EINVAL; + return NULL; + } + + qhasharr_data_t *data = tbl->data; + + for (; *idx < data->maxslots; (*idx)++) { + if (data->slots[*idx].count == 0 || data->slots[*idx].count == -2) { + continue; + } + + size_t keylen = data->slots[*idx].data.pair.keylen; + if (keylen > _Q_HASHARR_KEYSIZE) + keylen = _Q_HASHARR_KEYSIZE; + + obj->name = (char *) malloc(keylen + 1); + if (obj->name == NULL) { + //errno = ENOMEM; + return false; + } + memcpy(obj->name, data->slots[*idx].data.pair.key, keylen); + obj->name[keylen] = '\0'; + + obj->data = _get_data(tbl, *idx, &obj->size); + if (obj->data == NULL) { + free(obj->name); + //errno = ENOMEM; + return false; + } + + *idx += 1; + return true; + } + + //errno = ENOENT; + return false; +} + +/** + * qhasharr->remove(): Remove an object from this table. + * + * @param tbl qhasharr_t container pointer. + * @param key key string + * + * @return true if successful, otherwise(not found) returns false + * @retval errno will be set in error condition. + * - ENOENT : No such key found. + * - EINVAL : Invald argument. + * - EFAULT : Unexpected error. Data structure is not constant. + */ +static bool remove_(qhasharr_t *tbl, const char *key) { + if (tbl == NULL || key == NULL) { + //errno = EINVAL; + return false; + } + + qhasharr_data_t *data = tbl->data; + + // get hash integer + unsigned int hash = qhashmurmur3_32(key, strlen(key)) % data->maxslots; + + int idx = _get_idx(tbl, key, hash); + if (idx < 0) { + //DEBUG("not found %s", key); + //errno = ENOENT; + return false; + } + + if (data->slots[idx].count == 1) { + // just remove + _remove_data(tbl, idx); + //DEBUG("hasharr: rem %s (idx=%d,tot=%d)", key, idx, data->usedslots); + } else if (data->slots[idx].count > 1) { // leading slot and has dup + // find dup + int idx2; + for (idx2 = idx + 1;; idx2++) + { + if (idx2 >= data->maxslots) + idx2 = 0; + if (idx2 == idx) { + //DEBUG("hasharr: [BUG] failed to remove dup key %s.", key); + //errno = EFAULT; + return false; + } + if (data->slots[idx2].count == -1 && data->slots[idx2].hash == hash) + { + break; + } + } + + // move to leading slot + int backupcount = data->slots[idx].count; + _remove_data(tbl, idx); // remove leading data + _copy_slot(tbl, idx, idx2); // copy slot + _remove_slot(tbl, idx2); // remove moved slot + + data->slots[idx].count = backupcount - 1; // adjust collision counter + if (data->slots[idx].link != -1) { + data->slots[data->slots[idx].link].hash = idx; + } + + //DEBUG("hasharr: rem(lead) %s (idx=%d,tot=%d)", + // key, idx, data->usedslots); + } else { // in case of -1. used for collision resolution + // decrease counter from leading slot + if (data->slots[data->slots[idx].hash].count <= 1) { + //DEBUG("hasharr: [BUG] failed to remove %s. " + // "counter of leading slot mismatch.", key); + //errno = EFAULT; + return false; + } + data->slots[data->slots[idx].hash].count--; + + // remove data + _remove_data(tbl, idx); + //DEBUG("hasharr: rem(dup) %s (idx=%d,tot=%d)", key, idx, data->usedslots); + } + + return true; +} + +/** + * qhasharr->size(): Returns the number of objects in this table. + * + * @param tbl qhasharr_t container pointer. + * + * @return a number of elements stored. + */ +static int size(qhasharr_t *tbl, int *maxslots, int *usedslots) { + if (tbl == NULL) { + //errno = EINVAL; + return -1; + } + + qhasharr_data_t *data = tbl->data; + + if (maxslots != NULL) + *maxslots = data->maxslots; + if (usedslots != NULL) + *usedslots = data->usedslots; + + return data->num; +} + + +/** + * qhasharr->free(): De-allocate table reference object. + * + * @param tbl qhashtbl_t container pointer. + * + * @note + * This does not de-allocate memory but only function reference object. + * Data memory such as shared memory must be de-allocated separately. + */ +void free_(qhasharr_t *tbl) { + free(tbl); +} + +#ifndef _DOXYGEN_SKIP + +// find empty slot : return empty slow number, otherwise returns -1. +static int _find_empty(qhasharr_t *tbl, int startidx) { + qhasharr_data_t *data = tbl->data; + + if (startidx >= data->maxslots) + startidx = 0; + + int idx = startidx; + while (true) { + if (data->slots[idx].count == 0) + return idx; + + idx++; + if (idx >= data->maxslots) + idx = 0; + if (idx == startidx) + break; + } + + return -1; +} + +static int _get_idx(qhasharr_t *tbl, const char *key, unsigned int hash) { + qhasharr_data_t *data = tbl->data; + + if (data->slots[hash].count > 0) { + int count, idx; + for (count = 0, idx = hash; count < data->slots[hash].count;) { + if (data->slots[idx].hash == hash + && (data->slots[idx].count > 0 + || data->slots[idx].count == -1)) { + // same hash + count++; + + // is same key? + size_t keylen = strlen(key); + // first check key length + if (keylen == data->slots[idx].data.pair.keylen) { + if (keylen <= _Q_HASHARR_KEYSIZE) { + // original key is stored + if (!memcmp(key, data->slots[idx].data.pair.key, keylen)) + { + return idx; + } + } else { + // key is truncated, compare MD5 also. + unsigned char keymd5[16]; + qhashmd5(key, keylen, keymd5); + if (!memcmp(key, data->slots[idx].data.pair.key, _Q_HASHARR_KEYSIZE) && !memcmp(keymd5, data->slots[idx].data.pair.keymd5, 16)) + { + return idx; + } + } + } + } + + // increase idx + idx++; + if (idx >= data->maxslots) + idx = 0; + + // check loop + if (idx == hash) + break; + + continue; + } + } + + return -1; +} + +static void *_get_data(qhasharr_t *tbl, int idx, size_t *size) { + if (idx < 0) { + //errno = ENOENT; + return NULL; + } + + qhasharr_data_t *data = tbl->data; + + int newidx; + size_t valsize; + for (newidx = idx, valsize = 0;; newidx = data->slots[newidx].link) + { + valsize += data->slots[newidx].size; + if (data->slots[newidx].link == -1) + break; + } + + void *value, *vp; + value = malloc(valsize); + if (value == NULL) { + //errno = ENOMEM; + return NULL; + } + + for (newidx = idx, vp = value;; newidx = data->slots[newidx].link) { + if (data->slots[newidx].count == -2) { + // extended data block + memcpy(vp, (void *) data->slots[newidx].data.ext.value, + data->slots[newidx].size); + } else { + // key/value pair data block + memcpy(vp, (void *) data->slots[newidx].data.pair.value, + data->slots[newidx].size); + } + + vp += data->slots[newidx].size; + if (data->slots[newidx].link == -1) + break; + } + + if (size != NULL) + { + *size = valsize; + } + return value; +} + +static bool _put_data(qhasharr_t *tbl, int idx, unsigned int hash, + const char *key, const void *value, size_t size, + int count) { + qhasharr_data_t *data = tbl->data; + + // check if used + if (data->slots[idx].count != 0) { + //DEBUG("hasharr: BUG found."); + //errno = EFAULT; + return false; + } + + size_t keylen = strlen(key); + unsigned char keymd5[16]; + qhashmd5(key, keylen, keymd5); + + // store key + data->slots[idx].count = count; + data->slots[idx].hash = hash; + strncpy(data->slots[idx].data.pair.key, key, _Q_HASHARR_KEYSIZE); + memcpy((char *) data->slots[idx].data.pair.keymd5, (char *) keymd5, 16); + data->slots[idx].data.pair.keylen = keylen; + data->slots[idx].link = -1; + + // store value + int newidx; + size_t savesize; + for (newidx = idx, savesize = 0; savesize < size;) { + if (savesize > 0) { // find next empty slot + int tmpidx = _find_empty(tbl, newidx + 1); + if (tmpidx < 0) { + //DEBUG("hasharr: Can't expand slot for key %s.", key); + _remove_data(tbl, idx); + //errno = ENOBUFS; + return false; + } + + // clear & set + memset((void *) (&data->slots[tmpidx]), '\0', + sizeof(qhasharr_slot_t)); + + data->slots[tmpidx].count = -2; // extended data block + data->slots[tmpidx].hash = newidx; // prev link + data->slots[tmpidx].link = -1; // end block mark + data->slots[tmpidx].size = 0; + + data->slots[newidx].link = tmpidx; // link chain + + //DEBUG("hasharr: slot %d is linked to slot %d for key %s.", + // tmpidx, newidx, key); + newidx = tmpidx; + } + + // copy data + size_t copysize = size - savesize; + + if (data->slots[newidx].count == -2) { + // extended value + if (copysize > sizeof(struct _Q_HASHARR_SLOT_EXT)) { + copysize = sizeof(struct _Q_HASHARR_SLOT_EXT); + } + memcpy(data->slots[newidx].data.ext.value, value + savesize, + copysize); + } else { + // first slot + if (copysize > _Q_HASHARR_VALUESIZE) { + copysize = _Q_HASHARR_VALUESIZE; + } + memcpy(data->slots[newidx].data.pair.value, value + savesize, + copysize); + + // increase stored key counter + data->num++; + } + data->slots[newidx].size = copysize; + savesize += copysize; + + // increase used slot counter + data->usedslots++; + } + + return true; +} + +static bool _copy_slot(qhasharr_t *tbl, int idx1, int idx2) { + qhasharr_data_t *data = tbl->data; + + if (data->slots[idx1].count != 0 || data->slots[idx2].count == 0) { + //DEBUG("hasharr: BUG found."); + //errno = EFAULT; + return false; + } + + memcpy((void *) (&data->slots[idx1]), (void *) (&data->slots[idx2]), + sizeof(qhasharr_slot_t)); + + // increase used slot counter + data->usedslots++; + + return true; +} + +static bool _remove_slot(qhasharr_t *tbl, int idx) +{ + qhasharr_data_t *data = tbl->data; + + if (data->slots[idx].count == 0) + { + //DEBUG("hasharr: BUG found."); + //errno = EFAULT; + return false; + } + + data->slots[idx].count = 0; + + // decrease used slot counter + data->usedslots--; + + return true; +} + +static bool _remove_data(qhasharr_t *tbl, int idx) { + qhasharr_data_t *data = tbl->data; + + if (data->slots[idx].count == 0) { + //DEBUG("hasharr: BUG found."); + //errno = EFAULT; + return false; + } + + while (true) { + int link = data->slots[idx].link; + _remove_slot(tbl, idx); + + if (link == -1) + break; + + idx = link; + } + + // decrease stored key counter + data->num--; + + return true; +} + +#endif /* _DOXYGEN_SKIP */ diff --git a/src/key-value-store/hashtable/qhasharr.h b/src/key-value-store/hashtable/qhasharr.h new file mode 100644 index 0000000..37d52f3 --- /dev/null +++ b/src/key-value-store/hashtable/qhasharr.h @@ -0,0 +1,136 @@ +/******************************************************************************
+ * qLibc
+ *
+ * Copyright (c) 2010-2014 Seungyoung Kim.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *****************************************************************************/
+
+/**
+ * Static Hash Table container that works in preallocated fixed size memory.
+ *
+ * @file qhasharr.h
+ */
+
+/*
+ * Modified parts of this file by XS Embedded GmbH, 2014
+ */
+
+
+#ifndef _QHASHARR_H
+#define _QHASHARR_H
+
+#include <stdlib.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include "qtype.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* tunable knobs */
+#define _Q_HASHARR_KEYSIZE (128) /*!< knob for maximum key size. */
+#define _Q_HASHARR_VALUESIZE (32) /*!< knob for maximum data size in a slot. */
+
+#define PERS_CACHE_MAX_SLOTS 100000 /**< Max. number of slots in the cache */
+#define PERS_CACHE_MEMSIZE (sizeof(qhasharr_data_t)+ (sizeof(qhasharr_slot_t) * (PERS_CACHE_MAX_SLOTS))) //approximately 2 MB
+
+/* types */
+typedef struct qhasharr_slot_s qhasharr_slot_t;
+typedef struct qhasharr_data_s qhasharr_data_t;
+typedef struct qhasharr_s qhasharr_t;
+
+/* public functions */
+extern qhasharr_t *qhasharr(void *memory, size_t memsize);
+extern size_t qhasharr_calculate_memsize(int max);
+
+/**
+ * qhasharr internal data slot structure
+ */
+struct qhasharr_slot_s {
+ short count; /*!< hash collision counter. 0 indicates empty slot,
+ -1 is used for collision resolution, -2 is used for
+ indicating linked block */
+ uint32_t hash; /*!< key hash. we use FNV32 */
+
+ uint8_t size; /*!< value size in this slot*/
+ int link; /*!< next link */
+
+ union {
+ /*!< key/value data */
+ struct _Q_HASHARR_SLOT_KEYVAL {
+ unsigned char value[_Q_HASHARR_VALUESIZE]; /*!< value */
+
+ char key[_Q_HASHARR_KEYSIZE]; /*!< key string, can be cut */
+ uint16_t keylen; /*!< original key length */
+ unsigned char keymd5[16]; /*!< md5 hash of the key */
+ } pair;
+
+ /*!< extended data block, used only when the count value is -2 */
+ struct _Q_HASHARR_SLOT_EXT {
+ unsigned char value[sizeof(struct _Q_HASHARR_SLOT_KEYVAL)];
+ } ext;
+ } data;
+};
+
+/**
+ * qhasharr memory structure
+ */
+struct qhasharr_data_s {
+ int maxslots; /*!< number of maximum slots */
+ int usedslots; /*!< number of used slots */
+ int num; /*!< number of stored keys */
+ qhasharr_slot_t *slots; /*!< data area pointer */
+};
+
+/**
+ * qhasharr container object
+ */
+struct qhasharr_s {
+ /* encapsulated member functions */
+ bool (*put) (qhasharr_t *tbl, const char *key, const void *value,
+ size_t size);
+
+ void *(*get) (qhasharr_t *tbl, const char *key, size_t *size);
+
+ bool (*getnext) (qhasharr_t *tbl, qnobj_t *obj, int *idx);
+
+ bool (*remove) (qhasharr_t *tbl, const char *key);
+
+ int (*size) (qhasharr_t *tbl, int *maxslots, int *usedslots);
+
+ void (*clear) (qhasharr_t *tbl);
+
+ void (*free) (qhasharr_t *tbl);
+
+ /* private variables */
+ qhasharr_data_t *data;
+};
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_QHASHARR_H */
+
diff --git a/src/key-value-store/hashtable/qlibc.h b/src/key-value-store/hashtable/qlibc.h new file mode 100644 index 0000000..0e35e48 --- /dev/null +++ b/src/key-value-store/hashtable/qlibc.h @@ -0,0 +1,55 @@ + /****************************************************************************** + * Project Persistency + * (c) copyright 2014 + * Company XS Embedded GmbH + *****************************************************************************/ +/****************************************************************************** + * qLibc + * + * Copyright (c) 2010-2014 Seungyoung Kim. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + *****************************************************************************/ + +/** + * qlibc header file. + * + * @file qlibc.h + */ + +/* + * Modified parts of this file by XS Embedded GmbH, 2014 + */ + +#ifndef _QLIBC_H +#define _QLIBC_H + +/* containers */ +#include "qtype.h" +#include "qhasharr.h" + +/* utilities */ +#include "qhash.h" + +#endif /*_QLIBC_H */ + diff --git a/src/key-value-store/hashtable/qtype.h b/src/key-value-store/hashtable/qtype.h new file mode 100644 index 0000000..5da24e3 --- /dev/null +++ b/src/key-value-store/hashtable/qtype.h @@ -0,0 +1,123 @@ +/******************************************************************************
+ * qLibc
+ *
+ * Copyright (c) 2010-2014 Seungyoung Kim.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *****************************************************************************/
+
+/**
+ * Defines basic object types that are commonly used in various containers.
+ *
+ * @file qtype.h
+ */
+
+#ifndef _QTYPE_H
+#define _QTYPE_H
+
+#include <stdlib.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <pthread.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* types */
+typedef struct qmutex_s qmutex_t; /*!< qlibc pthread mutex type*/
+typedef struct qobj_s qobj_t; /*!< object type*/
+typedef struct qnobj_s qnobj_t; /*!< named-object type*/
+typedef struct qdlobj_s qdlobj_t; /*!< doubly-linked-object type*/
+typedef struct qdlnobj_s qdlnobj_t; /*!< doubly-linked-named-object type*/
+typedef struct qhnobj_s qhnobj_t; /*!< hashed-named-object type*/
+
+/**
+ * qlibc pthread mutex data structure.
+ */
+struct qmutex_s {
+ pthread_mutex_t mutex; /*!< pthread mutex */
+ pthread_t owner; /*!< mutex owner thread id */
+ int count; /*!< recursive lock counter */
+};
+
+/**
+ * object data structure.
+ */
+struct qobj_s {
+ void *data; /*!< data */
+ size_t size; /*!< data size */
+ uint8_t type; /*!< data type */
+};
+
+/**
+ * named-object data structure.
+ */
+struct qnobj_s {
+ char *name; /*!< object name */
+ void *data; /*!< data */
+ size_t size; /*!< data size */
+};
+
+/**
+ * doubly-linked-object data structure.
+ */
+struct qdlobj_s {
+ void *data; /*!< data */
+ size_t size; /*!< data size */
+
+ qdlobj_t *prev; /*!< previous link */
+ qdlobj_t *next; /*!< next link */
+};
+
+/**
+ * doubly-linked-named-object data structure.
+ */
+struct qdlnobj_s {
+ uint32_t hash; /*!< 32bit-hash value of object name */
+ char *name; /*!< object name */
+ void *data; /*!< data */
+ size_t size; /*!< data size */
+
+ qdlnobj_t *prev; /*!< previous link */
+ qdlnobj_t *next; /*!< next link */
+};
+
+/**
+ * hashed-named-object data structure.
+ */
+struct qhnobj_s {
+ uint32_t hash; /*!< 32bit-hash value of object name */
+ char *name; /*!< object name */
+ void *data; /*!< data */
+ size_t size; /*!< data size */
+
+ qhnobj_t *next; /*!< for chaining next collision object */
+};
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_QTYPE_H */
+
diff --git a/src/key-value-store/pers_low_level_db_access.c b/src/key-value-store/pers_low_level_db_access.c new file mode 100644 index 0000000..1ca2b27 --- /dev/null +++ b/src/key-value-store/pers_low_level_db_access.c @@ -0,0 +1,2053 @@ + /****************************************************************************** + * Project Persistency + * (c) copyright 2014 + * Company XS Embedded GmbH + *****************************************************************************/ +/****************************************************************************** + * This Source Code Form is subject to the terms of the + * Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed + * with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +******************************************************************************/ + /** + * @file pers_low_level_db_access.c + * @author Simon Disch + * @brief Implementation of persComDbAccess.h + * @see + */ + + +#include <pthread.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <malloc.h> +#include <sys/types.h> +#include <unistd.h> +#include "./database/kissdb.h" +#include "./hashtable/qlibc.h" +#include <inttypes.h> +#include "persComTypes.h" +#include "persComErrors.h" +#include "persComDataOrg.h" +#include "persComDbAccess.h" +#include "persComRct.h" +#include "pers_low_level_db_access_if.h" +#include "dlt.h" +#include <errno.h> +#include <sys/time.h> +/* L&T context */ +#define LT_HDR "[persComLLDB]" + +DLT_DECLARE_CONTEXT(persComLldbDLTCtx); + +/* ---------------------- local definition ---------------------------- */ +/* max number of open handlers per process */ +#define PERS_LLDB_NO_OF_STATIC_HANDLES 16 +#define PERS_LLDB_MAX_STATIC_HANDLES (PERS_LLDB_NO_OF_STATIC_HANDLES-1) + +#define PERS_STATUS_KEY_NOT_IN_CACHE -10 //!< key not in cache + +typedef struct +{ + char m_data[PERS_DB_MAX_SIZE_KEY_DATA]; + uint32_t m_dataSize; +} Data_LocalDB_s; + +typedef enum pers_lldb_cache_flag_e +{ + CachedDataDelete = 0, /* Resource-Configuration-Table */ + CachedDataWrite, /* Local/Shared DB */ +} pers_lldb_cache_flag_e; + +typedef struct +{ + pers_lldb_cache_flag_e eFlag; + int m_dataSize; + char m_data[PERS_DB_MAX_SIZE_KEY_DATA]; +} Data_Cached_s; + +typedef struct +{ + pers_lldb_cache_flag_e eFlag; + uint32_t m_dataSize; + char m_data[sizeof(PersistenceConfigurationKey_s)]; +} Data_Cached_RCT_s; + +typedef struct +{ + bool_t bIsAssigned; + sint_t dbHandler; + pers_lldb_purpose_e ePurpose; + KISSDB kissDb; + str_t dbPathname[PERS_ORG_MAX_LENGTH_PATH_FILENAME]; +} lldb_handler_s; + +typedef struct lldb_handles_list_el_s_ +{ + lldb_handler_s sHandle; + struct lldb_handles_list_el_s_ * pNext; +} lldb_handles_list_el_s; + +typedef struct +{ + lldb_handler_s asStaticHandles[PERS_LLDB_NO_OF_STATIC_HANDLES]; /* static area should be enough for most of the processes*/ + lldb_handles_list_el_s* pListHead; /* for the processes with a large number of threads which use Persistency */ +} lldb_handlers_s; + +/* ---------------------- local variables --------------------------------- */ +static const char ListItemsSeparator = '\0'; + +/* shared by all the threads within a process */ +static lldb_handlers_s g_sHandlers = { { { 0 } } }; +static pthread_mutex_t g_mutexLldb = PTHREAD_MUTEX_INITIALIZER; + +/* ---------------------- local macros --------------------------------- */ + +/* ---------------------- local functions --------------------------------- */ +static sint_t DeleteDataFromKissDB(sint_t dbHandler, pconststr_t key); +static sint_t DeleteDataFromKissRCT(sint_t dbHandler, pconststr_t key); +static sint_t GetAllKeysFromKissLocalDB(sint_t dbHandler, pstr_t buffer, sint_t size); +static sint_t GetAllKeysFromKissRCT(sint_t dbHandler, pstr_t buffer, sint_t size); +static sint_t GetKeySizeFromKissLocalDB(sint_t dbHandler, pconststr_t key); +static sint_t GetDataFromKissLocalDB(sint_t dbHandler, pconststr_t key, pstr_t buffer_out, sint_t bufSize); +static sint_t GetDataFromKissRCT(sint_t dbHandler, pconststr_t key, PersistenceConfigurationKey_s* pConfig); +static sint_t SetDataInKissLocalDB(sint_t dbHandler, pconststr_t key, pconststr_t data, sint_t dataSize); +static sint_t SetDataInKissRCT(sint_t dbHandler, pconststr_t key, PersistenceConfigurationKey_s const * pConfig); +static sint_t writeBackKissDB(KISSDB* db, lldb_handler_s* pLldbHandler); +static sint_t writeBackKissRCT(KISSDB* db, lldb_handler_s* pLldbHandler); +static sint_t getListandSize(KISSDB* db, pstr_t buffer, sint_t size, bool_t bOnlySizeNeeded, pers_lldb_purpose_e purpose); +static sint_t putToCache(KISSDB* db, sint_t dataSize, char* tmp_key, void* insert_cached_data); +static sint_t getFromCache(KISSDB* db, void* tmp_key, void* readBuffer, sint_t bufsize, bool_t sizeOnly); +static sint_t getFromDatabaseFile(KISSDB* db, void* tmp_key, void* readBuffer, pers_lldb_purpose_e purpose, sint_t bufsize, bool_t sizeOnly); + +/* access to resources shared by the threads within a process */ +static bool_t lldb_handles_Lock(void); +static bool_t lldb_handles_Unlock(void); +static lldb_handler_s* lldb_handles_FindInUseHandle(sint_t dbHandler); +static lldb_handler_s* lldb_handles_FindAvailableHandle(void); +static void lldb_handles_InitHandle(lldb_handler_s* psHandle_inout, pers_lldb_purpose_e ePurpose, str_t const * dbPathname); +static bool_t lldb_handles_DeinitHandle(sint_t dbHandler); + +static int createCache(KISSDB* db); +static int openCache(KISSDB* db); +static int closeCache(KISSDB* db); + +/** + * \open or create a key-value database + * \note : DB type is identified from dbPathname (based on extension) + * + * \param dbPathname [in] absolute path to DB + * \param ePurpose [in] see pers_lldb_purpose_e + * \param bForceCreationIfNotPresent [in] if true, the DB is created if it does not exist + * + * \return >=0 for success, negative value otherway (see pers_error_codes.h) + */ +sint_t pers_lldb_open(str_t const * dbPathname, pers_lldb_purpose_e ePurpose, bool_t bForceCreationIfNotPresent) +{ + sint_t returnValue = PERS_COM_FAILURE; + bool_t bCanContinue = true; + lldb_handler_s* pLldbHandler = NIL; + bool_t bLocked = false; + int mode = KISSDB_OPEN_MODE_RDWR; + static bool_t bFirstCall = true; + + if (bFirstCall) + { + pid_t pid = getpid(); + str_t dltContextID[16]; /* should be at most 4 characters string, but colissions occure */ + + /* set an error handler - the default one will cause the termination of the calling process */ + bFirstCall = false; + /* init DLT */ + (void) snprintf(dltContextID, sizeof(dltContextID), "Pers_%04d", pid); + DLT_REGISTER_CONTEXT(persComLldbDLTCtx, dltContextID, "PersCommonLLDB"); + DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("register context PersCommonLLDB ContextID="); DLT_STRING(dltContextID)); + } + + DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING("Begin opening:"); DLT_STRING("<<"); DLT_STRING(dbPathname); DLT_STRING(">>, "); ((PersLldbPurpose_RCT == ePurpose) ? DLT_STRING("RCT, ") : DLT_STRING("DB, ")); ((true == bForceCreationIfNotPresent) ? DLT_STRING("forced, ") : DLT_STRING("unforced, ")); DLT_STRING(" ... ")); + + if (lldb_handles_Lock()) + { + bLocked = true; + pLldbHandler = lldb_handles_FindAvailableHandle(); + if (NIL == pLldbHandler) + { + bCanContinue = false; + returnValue = PERS_COM_ERR_OUT_OF_MEMORY; + } + } + else + bCanContinue = false; + if (bCanContinue) + { + int kissdb_state = 0; + size_t datasize = + (PersLldbPurpose_RCT == ePurpose) ? sizeof(PersistenceConfigurationKey_s) : sizeof(Data_LocalDB_s); + size_t keysize = + (PersLldbPurpose_RCT == ePurpose) ? PERS_RCT_MAX_LENGTH_RESOURCE_ID : PERS_DB_MAX_LENGTH_KEY_NAME; + + if (bForceCreationIfNotPresent & (1 << 0) ) //check bit 0 + mode = KISSDB_OPEN_MODE_RWCREAT; + +#ifdef __writeThrough + if(bForceCreationIfNotPresent & (1 << 1)) //check bit 1 + printf("cached \n"); + else + printf("uncached \n"); +#endif + + kissdb_state = KISSDB_open(&pLldbHandler->kissDb, dbPathname, mode, 256, keysize, datasize); + if (kissdb_state != 0) + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_WARN, + DLT_STRING("KISSDB_open: "); DLT_STRING("<<"); DLT_STRING(dbPathname); DLT_STRING(">>, "); DLT_STRING(" retval=<"); DLT_INT(kissdb_state); DLT_STRING(">"), DLT_STRING(strerror(errno))); + bCanContinue = false; + } + if (bCanContinue) + { + lldb_handles_InitHandle(pLldbHandler, ePurpose, dbPathname); + returnValue = pLldbHandler->dbHandler; + } + else + { + /* clean up */ + returnValue = PERS_COM_FAILURE; + (void) lldb_handles_DeinitHandle(pLldbHandler->dbHandler); + } + } + + if (bLocked) + (void) lldb_handles_Unlock(); + DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING("End of open for:"); DLT_STRING("<<"); DLT_STRING(dbPathname); DLT_STRING(">>, "); ((PersLldbPurpose_RCT == ePurpose) ? DLT_STRING("RCT, ") : DLT_STRING("DB, ")); ((true == bForceCreationIfNotPresent) ? DLT_STRING("forced, ") : DLT_STRING("unforced, ")); DLT_STRING("retval=<"); DLT_INT(returnValue); DLT_STRING(">")); + + return returnValue; +} + + + + +/** + * \close a key-value database + * \note : DB type is identified from dbPathname (based on extension) + * + * \param handlerDB [in] handler obtained with pers_lldb_open + * + * \return 0 for success, negative value otherway (see pers_error_codes.h) + */ +sint_t pers_lldb_close(sint_t handlerDB) +{ + int kissdb_state = 0; + sint_t returnValue = PERS_COM_SUCCESS; + lldb_handler_s* pLldbHandler = NIL; + bool_t bLocked = false; + + DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("dbHandler="); DLT_INT(handlerDB); DLT_STRING("...")); + +#ifdef __showTimeMeasurements + long long duration = 0; + long long KdbDuration = 0; + long long writeDuration = 0; + struct timespec writeStart, writeEnd, kdbStart, kdbEnd, writebackStart, writebackEnd; + duration = 0; + KdbDuration = 0; + writeDuration = 0; + clock_gettime(CLOCK_ID, &writeStart); +#endif + + if (handlerDB >= 0) + { + if (lldb_handles_Lock()) + { + bLocked = true; + pLldbHandler = lldb_handles_FindInUseHandle(handlerDB); + + if (NIL == pLldbHandler) + returnValue = PERS_COM_FAILURE; + } + } + else + returnValue = PERS_COM_ERR_INVALID_PARAM; + + if (PERS_COM_SUCCESS == returnValue) + { + //persist cached data to flash memory + KISSDB* db = &pLldbHandler->kissDb; + + DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("Closing database =<<"); DLT_STRING(pLldbHandler->dbPathname); DLT_STRING(">>, ")); + + + Kdb_wrlock(&db->shmem_info->cache_rwlock); + + if (db->shmem_info->cache_initialised == Kdb_true) + { + if (db->shmem_creator == Kdb_true) + { + //open existing cache in existing shared memory + if (db->shmem_cached_fd <= 0) + { + if (openCache(db) != 0) + { + Kdb_unlock(&db->shmem_info->cache_rwlock); + return PERS_COM_FAILURE; + } + } +#ifdef __showTimeMeasurements + clock_gettime(CLOCK_ID, &writebackStart); +#endif + if (pLldbHandler->ePurpose == PersLldbPurpose_DB) //write back to local database + writeBackKissDB(&pLldbHandler->kissDb, pLldbHandler); + else if (pLldbHandler->ePurpose == PersLldbPurpose_RCT) //write back to RCT database + { + writeBackKissRCT(&pLldbHandler->kissDb, pLldbHandler); + } +#ifdef __showTimeMeasurements + clock_gettime(CLOCK_ID, &writebackEnd); +#endif + if (db->shmem_info->cache_initialised) + { + db->tbl->free(db->tbl); + if (closeCache(db) != 0) + { + Kdb_unlock(&db->shmem_info->cache_rwlock); + return PERS_COM_FAILURE; + } + } + } + } + Kdb_unlock(&db->shmem_info->cache_rwlock); + +#ifdef __showTimeMeasurements + clock_gettime(CLOCK_ID, &kdbStart); +#endif + + kissdb_state = KISSDB_close(&pLldbHandler->kissDb); + +#ifdef __showTimeMeasurements + clock_gettime(CLOCK_ID, &kdbEnd); +#endif + + if (kissdb_state == 0) + { + if (!lldb_handles_DeinitHandle(pLldbHandler->dbHandler)) + returnValue = PERS_COM_FAILURE; + } + else + { + switch (kissdb_state) + { + case KISSDB_ERROR_UNMAP_SHM: + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, + DLT_STRING("KISSDB_close: "); DLT_STRING("Could not unmap shared memory object, retval=<"); DLT_INT(kissdb_state); DLT_STRING(">")); + break; + } + case KISSDB_ERROR_CLOSE_SHM: + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, + DLT_STRING("KISSDB_close: "); DLT_STRING("Could not close shared memory object, retval=<"); DLT_INT(kissdb_state); DLT_STRING(">")); + break; + } + default: + break; + } + returnValue = PERS_COM_FAILURE; + } + } + if (bLocked) + (void) lldb_handles_Unlock(); + + DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("handlerDB="); DLT_INT(handlerDB); DLT_STRING(" retval=<"); DLT_INT(returnValue); DLT_STRING(">")); + +#ifdef __showTimeMeasurements + clock_gettime(CLOCK_ID, &writeEnd); + writeDuration += getNsDuration(&writebackStart, &writebackEnd); + printf("Writeback to flash duration for %s => %f ms\n", pLldbHandler->dbPathname, (double)((double)writeDuration/NANO2MIL)); + KdbDuration += getNsDuration(&kdbStart, &kdbEnd); + printf("KISSDB_close duration for %s => %f ms\n", pLldbHandler->dbPathname, (double)((double)KdbDuration/NANO2MIL)); + duration += getNsDuration(&writeStart, &writeEnd); + printf("Overall Close duration for %s => %f ms\n", pLldbHandler->dbPathname, (double)((double)duration/NANO2MIL)); +#endif + return returnValue; +} + + + + +/** + * \writeback cache of RCT key-value database + * \return 0 for success, negative value otherway (see pers_error_codes.h) + */ +static sint_t writeBackKissRCT(KISSDB* db, lldb_handler_s* pLldbHandler) +{ + int kissdb_state = 0; + int idx = 0; + sint_t returnValue = PERS_COM_SUCCESS; + //lldb_handler_s* pLldbHandler = NIL; + pers_lldb_cache_flag_e eFlag; + char* ptr; + qnobj_t obj; + char tmp_key[PERS_RCT_MAX_LENGTH_RESOURCE_ID]; + + DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("START writeback for RCT: "), DLT_STRING(db->shmem_ht_name) ); + + while (db->tbl->getnext(db->tbl, &obj, &idx) == true) + { + ptr = obj.data; + eFlag = (pers_lldb_cache_flag_e) *(int*) ptr; + ptr += 2 * (sizeof(int)); + (void) strncpy(tmp_key, obj.name, PERS_RCT_MAX_LENGTH_RESOURCE_ID); + + //check how data should be persisted + switch (eFlag) + { + case CachedDataDelete: //data must be deleted from file + { + kissdb_state = KISSDB_delete(&pLldbHandler->kissDb, tmp_key); + if (kissdb_state != 0) + { + if (kissdb_state == 1) + DLT_LOG(persComLldbDLTCtx, DLT_LOG_WARN, + DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("KISSDB_delete: RCT key=<"); DLT_STRING(tmp_key); DLT_STRING(">, "); DLT_STRING("not found in database file, retval=<"); DLT_INT(kissdb_state); DLT_STRING(">")); + else + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, + DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("KISSDB_delete: RCT key=<");DLT_STRING(tmp_key); DLT_STRING(">, "); DLT_STRING("Error with retval=<"); DLT_INT(kissdb_state); DLT_STRING(">")); + } + break; + } + case CachedDataWrite: //data must be written to file + { + kissdb_state = KISSDB_put(&pLldbHandler->kissDb, tmp_key, ptr); + if (kissdb_state != 0) + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, + DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("KISSDB_put: RCT key=<");DLT_STRING(tmp_key); DLT_STRING(">, "); DLT_STRING("Error: Writing back to file failed with retval=<"); DLT_INT(kissdb_state); DLT_STRING(">")); + break; + } + default: + break; + } + free(obj.name); + free(obj.data); + } + + DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("END writeback for RCT: "), DLT_STRING(db->shmem_ht_name) ); + return returnValue; +} + + + + + + + +/** + * \writeback cache of local DB key-value database + * \return 0 for success, negative value otherway (see pers_error_codes.h) + */ +static sint_t writeBackKissDB(KISSDB* db, lldb_handler_s* pLldbHandler) +{ + int kissdb_state = 0; + int idx = 0; + sint_t returnValue = PERS_COM_SUCCESS; + //lldb_handler_s* pLldbHandler = NIL; + pers_lldb_cache_flag_e eFlag; + char* ptr; + qnobj_t obj; + + char tmp_key[PERS_DB_MAX_LENGTH_KEY_NAME]; + Data_LocalDB_s insert = { { 0 } }; + int datasize = 0; + + DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("START writeback for DB: "), DLT_STRING(db->shmem_ht_name) ); + + while (db->tbl->getnext(db->tbl, &obj, &idx) == true) + { + //get flag and datasize + ptr = obj.data; + eFlag = (pers_lldb_cache_flag_e) *(int*) ptr; //pointer in obj.data to eflag + ptr += sizeof(int); + datasize = *(int*) ptr; //pointer in obj.data to datasize + ptr += sizeof(int); //pointer in obj.data to data + (void) strncpy(tmp_key, obj.name, PERS_DB_MAX_LENGTH_KEY_NAME); + + //check how data should be persisted + switch (eFlag) + { + case CachedDataDelete: //data must be deleted from file + { + //delete key-value pair from database file + kissdb_state = KISSDB_delete(&pLldbHandler->kissDb, tmp_key); + if (kissdb_state != 0) + { + if (kissdb_state == 1) + DLT_LOG(persComLldbDLTCtx, DLT_LOG_WARN, + DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("KISSDB_delete: key=<"); DLT_STRING(tmp_key); DLT_STRING(">, "); DLT_STRING("not found in database file, retval=<"); DLT_INT(kissdb_state); DLT_STRING(">")); + else + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, + DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("KISSDB_delete: key=<");DLT_STRING(tmp_key); DLT_STRING(">, "); DLT_STRING("Error with retval=<"); DLT_INT(kissdb_state); DLT_STRING(">"); DLT_STRING("Error Message: ");DLT_STRING(strerror(errno))); + } + break; + } + case CachedDataWrite: //data must be written to file + { + (void) memcpy(insert.m_data, ptr, datasize); + insert.m_dataSize = datasize; + + kissdb_state = KISSDB_put(&pLldbHandler->kissDb, tmp_key, &insert); //store data followed by datasize + if (kissdb_state != 0) + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, + DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("KISSDB_put: key=<");DLT_STRING(tmp_key); DLT_STRING(">, "); DLT_STRING("Error: Writing back to file failed with retval=<"); DLT_INT(kissdb_state); DLT_STRING(">"); DLT_STRING("Error Message: ");DLT_STRING(strerror(errno))); + break; + } + default: + break; + } + free(obj.name); + free(obj.data); + } + DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("END writeback for DB: "), DLT_STRING(db->shmem_ht_name) ); + return returnValue; +} + + + + + + + +/** + * \brief write a key-value pair into database + * \note : DB type is identified from dbPathname (based on extension) + * \note : DB is created if it does not exist + * + * \param handlerDB [in] handler obtained with pers_lldb_open + * \param ePurpose [in] see pers_lldb_purpose_e + * \param key [in] key's name + * \param data [in] buffer with key's data + * \param dataSize [in] size of key's data + * + * \return 0 for success, negative value otherway (see pers_error_codes.h) + */ +sint_t pers_lldb_write_key(sint_t handlerDB, pers_lldb_purpose_e ePurpose, str_t const * key, str_t const * data, + sint_t dataSize) +{ + sint_t eErrorCode = PERS_COM_SUCCESS; + + //int i =0; + //DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, + // DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("Datatest="); DLT_RAW(data,dataSize); DLT_STRING("key=<<"); DLT_STRING(key); DLT_STRING(">>"); DLT_STRING(" Data Size=<<"); DLT_INT(dataSize); DLT_STRING(">>...")); + + switch (ePurpose) + { + case PersLldbPurpose_DB: + { + eErrorCode = SetDataInKissLocalDB(handlerDB, key, data, dataSize); + break; + } + case PersLldbPurpose_RCT: + { + eErrorCode = SetDataInKissRCT(handlerDB, key, (PersistenceConfigurationKey_s const *) data); + break; + } + default: + { + eErrorCode = PERS_COM_ERR_INVALID_PARAM; + break; + } + } + return eErrorCode; +} + +/** + * \brief read a key's value from database + * \note : DB type is identified from dbPathname (based on extension) + * + * \param handlerDB [in] handler obtained with pers_lldb_open + * \param ePurpose [in] see pers_lldb_purpose_e + * \param key [in] key's name + * \param dataBuffer_out [out]buffer where to return the read data + * \param bufSize [in] size of dataBuffer_out + * + * \return read size, or negative value in case of error (see pers_error_codes.h) + */ +sint_t pers_lldb_read_key(sint_t handlerDB, pers_lldb_purpose_e ePurpose, str_t const * key, pstr_t dataBuffer_out, + sint_t bufSize) +{ + sint_t eErrorCode = PERS_COM_SUCCESS; + + switch (ePurpose) + { + case PersLldbPurpose_DB: + { + eErrorCode = GetDataFromKissLocalDB(handlerDB, key, dataBuffer_out, bufSize); + break; + } + case PersLldbPurpose_RCT: + { + eErrorCode = GetDataFromKissRCT(handlerDB, key, (PersistenceConfigurationKey_s*) dataBuffer_out); + break; + } + default: + { + eErrorCode = PERS_COM_ERR_INVALID_PARAM; + break; + } + } + return eErrorCode; +} + +/** + * \brief reads the size of a value that corresponds to a key + * \note : DB type is identified from dbPathname (based on extension) + * + * \param handlerDB [in] handler obtained with pers_lldb_open + * \param ePurpose [in] see pers_lldb_purpose_e + * \param key [in] key's name + * \return size of the value corresponding to the key, or negative value in case of error (see pers_error_codes.h) + */ +sint_t pers_lldb_get_key_size(sint_t handlerDB, pers_lldb_purpose_e ePurpose, str_t const * key) +{ + sint_t eErrorCode = PERS_COM_SUCCESS; + + switch (ePurpose) + { + case PersLldbPurpose_DB: + { + eErrorCode = GetKeySizeFromKissLocalDB(handlerDB, key); + break; + } + default: + { + eErrorCode = PERS_COM_ERR_INVALID_PARAM; + break; + } + } + + return eErrorCode; +} + +/** + * \brief delete key from database + * \note : DB type is identified from dbPathname (based on extension) + * + * \param handlerDB [in] handler obtained with pers_lldb_open + * \param ePurpose [in] see pers_lldb_purpose_e + * \param key [in] key's name + * + * \return 0 for success, negative value otherway (see pers_error_codes.h) + */ +sint_t pers_lldb_delete_key(sint_t handlerDB, pers_lldb_purpose_e ePurpose, str_t const * key) +{ + sint_t eErrorCode = PERS_COM_SUCCESS; + + switch (ePurpose) + { + case PersLldbPurpose_DB: + { + eErrorCode = DeleteDataFromKissDB(handlerDB, key); + break; + } + case PersLldbPurpose_RCT: + { + eErrorCode = DeleteDataFromKissRCT(handlerDB, key); + break; + } + default: + { + eErrorCode = PERS_COM_ERR_INVALID_PARAM; + break; + } + } + return eErrorCode; +} + +/** + * \brief Find the buffer's size needed to accomodate the listing of keys' names in database + * \note : DB type is identified from dbPathname (based on extension) + * + * \param handlerDB [in] handler obtained with pers_lldb_open + * \param ePurpose [in] see pers_lldb_purpose_e + * + * \return needed size, or negative value in case of error (see pers_error_codes.h) + */ +sint_t pers_lldb_get_size_keys_list(sint_t handlerDB, pers_lldb_purpose_e ePurpose) +{ + sint_t eErrorCode = PERS_COM_SUCCESS; + + switch (ePurpose) + { + case PersLldbPurpose_DB: + { + eErrorCode = GetAllKeysFromKissLocalDB(handlerDB, NIL, 0); + break; + } + case PersLldbPurpose_RCT: + { + eErrorCode = GetAllKeysFromKissRCT(handlerDB, NIL, 0); + break; + } + default: + { + eErrorCode = PERS_COM_ERR_INVALID_PARAM; + break; + } + } + return eErrorCode; +} + +/** + * \brief List the keys' names in database + * \note : DB type is identified from dbPathname (based on extension) + * \note : keys are separated by '\0' + * + * \param handlerDB [in] handler obtained with pers_lldb_open + * \param ePurpose [in] see pers_lldb_purpose_e + * \param listingBuffer_out [out]buffer where to return the listing + * \param bufSize [in] size of listingBuffer_out + * + * \return listing size, or negative value in case of error (see pers_error_codes.h) + */ +sint_t pers_lldb_get_keys_list(sint_t handlerDB, pers_lldb_purpose_e ePurpose, pstr_t listingBuffer_out, sint_t bufSize) +{ + sint_t eErrorCode = PERS_COM_SUCCESS; + + switch (ePurpose) + { + case PersLldbPurpose_DB: + { + eErrorCode = GetAllKeysFromKissLocalDB(handlerDB, listingBuffer_out, bufSize); + break; + } + case PersLldbPurpose_RCT: + { + eErrorCode = GetAllKeysFromKissRCT(handlerDB, listingBuffer_out, bufSize); + break; + } + default: + { + eErrorCode = PERS_COM_ERR_INVALID_PARAM; + break; + } + } + + return eErrorCode; +} + +//TODO add write through compatibility +static sint_t DeleteDataFromKissDB(sint_t dbHandler, pconststr_t key) +{ + bool_t bCanContinue = true; + sint_t delete_size = PERS_COM_FAILURE; + lldb_handler_s* pLldbHandler = NIL; + bool_t bLocked = false; + char m_data[sizeof(Data_LocalDB_s)] = {0}; + pers_lldb_cache_flag_e eFlag; + void *val; + char *ptr; + int status = PERS_COM_FAILURE; + int datasize = 0; + Kdb_bool not_found = Kdb_false; + size_t size = 0; + Data_Cached_s data_cached = { 0 }; + + DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("handlerDB="); DLT_INT(dbHandler); DLT_STRING("key=<<"); DLT_STRING(key); DLT_STRING(">>...")); + + if ((dbHandler >= 0) && (NIL != key)) + { + if (lldb_handles_Lock()) + { + bLocked = true; + pLldbHandler = lldb_handles_FindInUseHandle(dbHandler); + if (NIL == pLldbHandler) + bCanContinue = false; + } + } + else + bCanContinue = false; + + if (bCanContinue) + { + KISSDB* db = &pLldbHandler->kissDb; + + //DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, + // DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("Working on DB: "), DLT_STRING(db->shmem_ht_name) ); + + Kdb_wrlock(&db->shmem_info->cache_rwlock); + + char tmp_key[PERS_DB_MAX_LENGTH_KEY_NAME]; + (void) strncpy(tmp_key, key, PERS_DB_MAX_LENGTH_KEY_NAME); + data_cached.eFlag = CachedDataDelete; + data_cached.m_dataSize = 0; + + //if cache not already created + if (db->shmem_info->cache_initialised == Kdb_false) + { + if (createCache(db) != 0) + { + Kdb_unlock(&db->shmem_info->cache_rwlock); + return PERS_COM_FAILURE; + } + } + else //open cache + { + if (openCache(db) != 0) + { + Kdb_unlock(&db->shmem_info->cache_rwlock); + return PERS_COM_FAILURE; + } + } + val = db->tbl->get(db->tbl, tmp_key, &size); + if (NULL != val) //check if key to be deleted is in Cache + { + ptr = val; + eFlag = (pers_lldb_cache_flag_e) *(int*) ptr; + ptr += sizeof(int); + datasize = *(int*) ptr; + + //Mark data in cache as deleted + if (eFlag != CachedDataDelete) + { + if (db->tbl->put(db->tbl, tmp_key, &data_cached, sizeof(pers_lldb_cache_flag_e) + sizeof(int)) == false) //do not store any data + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, + DLT_STRING(__FUNCTION__); DLT_STRING(":");DLT_STRING("Failed to mark data in cache as deleted")); + delete_size = PERS_COM_ERR_NOT_FOUND; + not_found = Kdb_true; + } + else + delete_size = datasize; + } + } + else //check if key to be deleted is in database file + { + //get dataSize + status = KISSDB_get(&pLldbHandler->kissDb, tmp_key, m_data); + if (status == 0) + { + ptr = m_data; + ptr += PERS_DB_MAX_SIZE_KEY_DATA; + datasize = *(int*) ptr; + //put information about the key to be deleted in cache (deletion in file happens at system shutdown) + if (db->tbl->put(db->tbl, tmp_key, &data_cached, sizeof(pers_lldb_cache_flag_e) + sizeof(int)) == false) + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, + DLT_STRING(__FUNCTION__); DLT_STRING(":");DLT_STRING("Failed to mark existing data as deleted")); + delete_size = PERS_COM_ERR_NOT_FOUND; + } + else + delete_size = datasize; + } + else + { + if (status == 1) + { + not_found = Kdb_true; + DLT_LOG(persComLldbDLTCtx, DLT_LOG_WARN, + DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("KISSDB_get: key=<"); DLT_STRING(key); DLT_STRING(">, "); DLT_STRING("not found, retval=<"); DLT_INT(status); DLT_STRING(">")); + } + else + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, + DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("KISSDB_get: key=<"); DLT_STRING(tmp_key); DLT_STRING(">, "); DLT_STRING("Error with retval=<"); DLT_INT(status); DLT_STRING(">")); + } + } + } + + if (not_found == Kdb_true) //key not found, + delete_size = PERS_COM_ERR_NOT_FOUND; + Kdb_unlock(&db->shmem_info->cache_rwlock); + } + + if (bLocked) + (void) lldb_handles_Unlock(); + + DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("handlerDB="); DLT_INT(dbHandler); DLT_STRING("key=<<"); DLT_STRING(key); DLT_STRING(">>, "); DLT_STRING("retval=<"); DLT_INT(delete_size); DLT_STRING(">")); + + return delete_size; +} + + + +//TODO add write through compatibility +static sint_t DeleteDataFromKissRCT(sint_t dbHandler, pconststr_t key) +{ + bool_t bCanContinue = true; + sint_t delete_size = PERS_COM_FAILURE; + lldb_handler_s* pLldbHandler = NIL; + bool_t bLocked = false; + char m_data[sizeof(Data_LocalDB_s)] = {0}; + pers_lldb_cache_flag_e eFlag; + void *val; + char *ptr; + int status = PERS_COM_FAILURE; + int datasize = 0; + Kdb_bool not_found = Kdb_false; + size_t size = 0; + Data_Cached_s data_cached = { 0 }; + + DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("handlerDB="); DLT_INT(dbHandler); DLT_STRING("key=<<"); DLT_STRING(key); DLT_STRING(">>...")); + + if ((dbHandler >= 0) && (NIL != key)) + { + if (lldb_handles_Lock()) + { + bLocked = true; + pLldbHandler = lldb_handles_FindInUseHandle(dbHandler); + if (NIL == pLldbHandler) + bCanContinue = false; + } + } + else + bCanContinue = false; + + if (bCanContinue) + { + KISSDB* db = &pLldbHandler->kissDb; + + //DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, + // DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("Working on DB: "), DLT_STRING(db->shmem_ht_name) ); + + Kdb_wrlock(&db->shmem_info->cache_rwlock); + + char tmp_key[PERS_RCT_MAX_LENGTH_RESOURCE_ID]; + (void) strncpy(tmp_key, key, PERS_RCT_MAX_LENGTH_RESOURCE_ID); + data_cached.eFlag = CachedDataDelete; + data_cached.m_dataSize = 0; + + //if cache not already created + if (db->shmem_info->cache_initialised == Kdb_false) + { + if (createCache(db) != 0) + { + Kdb_unlock(&db->shmem_info->cache_rwlock); + return PERS_COM_FAILURE; + } + } + else //open cache + { + if (openCache(db) != 0) + { + Kdb_unlock(&db->shmem_info->cache_rwlock); + return PERS_COM_FAILURE; + } + } + //get dataSize + val = db->tbl->get(db->tbl, tmp_key, &size); + if (NULL != val) //check if key to be deleted is in Cache + { + ptr = val; + eFlag = (pers_lldb_cache_flag_e) *(int*) ptr; + ptr += sizeof(int); + datasize = *(int*) ptr; + + //Mark data in cache as deleted + if (eFlag != CachedDataDelete) + { + if (db->tbl->put(db->tbl, tmp_key, &data_cached, sizeof(pers_lldb_cache_flag_e) + sizeof(int)) == false) + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, + DLT_STRING(__FUNCTION__); DLT_STRING(":");DLT_STRING("Failed to mark RCT data in cache as deleted")); + delete_size = PERS_COM_ERR_NOT_FOUND; + not_found = Kdb_true; + } + else + delete_size = datasize; + } + } + else //check if key to be deleted is in database file + { + status = KISSDB_get(&pLldbHandler->kissDb, tmp_key, m_data); + if (status == 0) + { + //Data to be deleted is not in cache, but was found in local database + //put information about the key to be deleted in cache (deletion in file happens at system shutdown) + if (db->tbl->put(db->tbl, tmp_key, &data_cached, sizeof(pers_lldb_cache_flag_e) + sizeof(int)) == false) + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, + DLT_STRING(__FUNCTION__); DLT_STRING(":");DLT_STRING("Failed to mark existing RCT data as deleted")); + delete_size = PERS_COM_ERR_NOT_FOUND; + } + else + delete_size = sizeof(PersistenceConfigurationKey_s); + } + else + { + if (status == 1) + { + not_found = Kdb_true; + DLT_LOG(persComLldbDLTCtx, DLT_LOG_WARN, + DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("KISSDB_get: key=<"); DLT_STRING(key); DLT_STRING(">, "); DLT_STRING("not found, retval=<"); DLT_INT(status); DLT_STRING(">")); + } + else + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, + DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("KISSDB_get: key=<"); DLT_STRING(key); DLT_STRING(">, "); DLT_STRING("Error with retval=<"); DLT_INT(status); DLT_STRING(">")); + } + } + if (not_found == Kdb_true) + delete_size = PERS_COM_ERR_NOT_FOUND; + + Kdb_unlock(&db->shmem_info->cache_rwlock); + } + + if (bLocked) + (void) lldb_handles_Unlock(); + + DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("handlerDB="); DLT_INT(dbHandler); DLT_STRING("key=<<"); DLT_STRING(key); DLT_STRING(">>, "); DLT_STRING("retval=<"); DLT_INT(delete_size); DLT_STRING(">")); + + return delete_size; +} + + + +//TODO add write through compatibility +static sint_t GetAllKeysFromKissLocalDB(sint_t dbHandler, pstr_t buffer, sint_t size) +{ + bool_t bCanContinue = true; + sint_t result = 0; + bool_t bOnlySizeNeeded = (NIL == buffer); + lldb_handler_s* pLldbHandler = NIL; + bool_t bLocked = false; + + //DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, + //DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("dbHandler="); DLT_INT(dbHandler); DLT_STRING("buffer="); DLT_UINT((uint_t)buffer); DLT_STRING("size="); DLT_INT(size); DLT_STRING("...")); + + if (dbHandler >= 0) + { + if (lldb_handles_Lock()) + { + bLocked = true; + pLldbHandler = lldb_handles_FindInUseHandle(dbHandler); + if (NIL == pLldbHandler) + { + bCanContinue = false; + result = PERS_COM_ERR_INVALID_PARAM; + } + else + { + if (PersLldbPurpose_DB != pLldbHandler->ePurpose) + {/* this would be very bad */ + bCanContinue = false; + result = PERS_COM_FAILURE; + } + /* to not use DLT while mutex locked */ + } + } + } + else + { + bCanContinue = false; + result = PERS_COM_ERR_INVALID_PARAM; + } + + if (bCanContinue) + { + if ((buffer != NIL) && (size > 0)) + (void) memset(buffer, 0, (size_t) size); + + Kdb_wrlock(&pLldbHandler->kissDb.shmem_info->cache_rwlock); + result = getListandSize(&pLldbHandler->kissDb, buffer, size, bOnlySizeNeeded, PersLldbPurpose_DB); + Kdb_unlock(&pLldbHandler->kissDb.shmem_info->cache_rwlock); + if (result < 0) + result = PERS_COM_FAILURE; + } + if (bLocked) + (void) lldb_handles_Unlock(); + + //DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, + //DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("dbHandler="); DLT_INT(dbHandler); DLT_STRING("retval=<"); DLT_INT(result); DLT_STRING(">")); + return result; +} + + +//TODO add write through compatibility +static sint_t GetAllKeysFromKissRCT(sint_t dbHandler, pstr_t buffer, sint_t size) +{ + bool_t bCanContinue = true; + sint_t result = 0; + bool_t bOnlySizeNeeded = (NIL == buffer); + lldb_handler_s* pLldbHandler = NIL; + bool_t bLocked = false; + + //DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, + //DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("dbHandler="); DLT_INT(dbHandler); DLT_STRING("buffer="); DLT_UINT((uint_t)buffer); DLT_STRING("size="); DLT_INT(size); DLT_STRING("...")); + + if (dbHandler >= 0) + { + if (lldb_handles_Lock()) + { + bLocked = true; + pLldbHandler = lldb_handles_FindInUseHandle(dbHandler); + if (NIL == pLldbHandler) + { + bCanContinue = false; + result = PERS_COM_ERR_INVALID_PARAM; + } + else + { + if (PersLldbPurpose_RCT != pLldbHandler->ePurpose) + {/* this would be very bad */ + bCanContinue = false; + result = PERS_COM_FAILURE; + } + /* to not use DLT while mutex locked */ + } + } + } + else + { + bCanContinue = false; + result = PERS_COM_ERR_INVALID_PARAM; + } + + if (bCanContinue) + { + if ((buffer != NIL) && (size > 0)) + (void) memset(buffer, 0, (size_t) size); + Kdb_wrlock(&pLldbHandler->kissDb.shmem_info->cache_rwlock); + result = getListandSize(&pLldbHandler->kissDb, buffer, size, bOnlySizeNeeded, PersLldbPurpose_RCT); + Kdb_unlock(&pLldbHandler->kissDb.shmem_info->cache_rwlock); + if (result < 0) + result = PERS_COM_FAILURE; + } + if (bLocked) + (void) lldb_handles_Unlock(); + + //DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, + //DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("dbHandler="); DLT_INT(dbHandler); DLT_STRING("retval=<"); DLT_INT(result); DLT_STRING(">")); + + return result; +} + + +//TODO add write through compatibility +static sint_t SetDataInKissLocalDB(sint_t dbHandler, pconststr_t key, pconststr_t data, sint_t dataSize) +{ + bool_t bCanContinue = true; + sint_t size_written = PERS_COM_FAILURE; + lldb_handler_s* pLldbHandler = NIL; + bool_t bLocked = false; + Data_Cached_s data_cached = { 0 }; + + DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("dbHandler="); DLT_INT(dbHandler); DLT_STRING("key=<<"); DLT_STRING(key); DLT_STRING(">>, "); DLT_STRING("size<<"); DLT_INT(dataSize); DLT_STRING(">> ...")); + + if ((dbHandler >= 0) && (NIL != key) && (NIL != data) && (dataSize > 0)) + { + if (lldb_handles_Lock()) + { + bLocked = true; + pLldbHandler = lldb_handles_FindInUseHandle(dbHandler); + if (NIL == pLldbHandler) + { + bCanContinue = false; + size_written = PERS_COM_ERR_INVALID_PARAM; + } + else + { + if (PersLldbPurpose_DB != pLldbHandler->ePurpose) + {/* this would be very bad */ + bCanContinue = false; + size_written = PERS_COM_FAILURE; + } + } + } + } + else + { + bCanContinue = false; + size_written = PERS_COM_ERR_INVALID_PARAM; + } + + if (bCanContinue) + { + //DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, + // DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("Working on DB: "), DLT_STRING(pLldbHandler->dbPathname) ); + + //TODO add write through (call KissDB_put) + char tmp_key[PERS_DB_MAX_LENGTH_KEY_NAME]; + (void) strncpy(tmp_key, key, PERS_DB_MAX_LENGTH_KEY_NAME); + data_cached.eFlag = CachedDataWrite; + data_cached.m_dataSize = dataSize; + (void) memcpy(data_cached.m_data, data, (size_t) dataSize); + size_written = putToCache(&pLldbHandler->kissDb, dataSize, (char*) &tmp_key, &data_cached); + } + + if (bLocked) + (void) lldb_handles_Unlock(); + + DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("dbHandler="); DLT_INT(dbHandler); DLT_STRING("key=<<"); DLT_STRING(key); DLT_STRING(">>, "); DLT_STRING("size<<"); DLT_INT(dataSize); DLT_STRING(">>, "); DLT_STRING("retval=<"); DLT_INT(size_written); DLT_STRING(">")); + + return size_written; +} + +//TODO add write through compatibility +static sint_t SetDataInKissRCT(sint_t dbHandler, pconststr_t key, PersistenceConfigurationKey_s const * pConfig) +{ + bool_t bCanContinue = true; + sint_t size_written = PERS_COM_FAILURE; + lldb_handler_s* pLldbHandler = NIL; + bool_t bLocked = false; + Data_Cached_RCT_s data_cached = { 0 }; + + DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("dbHandler="); DLT_INT(dbHandler); DLT_STRING("key=<<"); DLT_STRING(key); DLT_STRING(">>...")); + + if ((dbHandler >= 0) && (NIL != key) && (NIL != pConfig)) + { + if (lldb_handles_Lock()) + { + bLocked = true; + pLldbHandler = lldb_handles_FindInUseHandle(dbHandler); + if (NIL == pLldbHandler) + { + bCanContinue = false; + size_written = PERS_COM_ERR_INVALID_PARAM; + } + else + { + if (PersLldbPurpose_RCT != pLldbHandler->ePurpose) + {/* this would be very bad */ + bCanContinue = false; + size_written = PERS_COM_FAILURE; + } + /* to not use DLT while mutex locked */ + } + } + } + else + { + bCanContinue = false; + size_written = PERS_COM_ERR_INVALID_PARAM; + } + if (bCanContinue) + { + //DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, + // DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("Working on DB: "), DLT_STRING(pLldbHandler->dbPathname) ); + + //TODO add RCT write through (call KissDB_put) + int dataSize = sizeof(PersistenceConfigurationKey_s); + char tmp_key[PERS_RCT_MAX_LENGTH_RESOURCE_ID]; + (void) strncpy(tmp_key, key, PERS_RCT_MAX_LENGTH_RESOURCE_ID); + data_cached.eFlag = CachedDataWrite; + data_cached.m_dataSize = dataSize; + (void) memcpy(data_cached.m_data, pConfig, (size_t) dataSize); + size_written = putToCache(&pLldbHandler->kissDb, dataSize, (char*) &tmp_key, &data_cached); + } + if (bLocked) + (void) lldb_handles_Unlock(); + + DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("dbHandler="); DLT_INT(dbHandler); DLT_STRING("key=<<"); DLT_STRING(key); DLT_STRING(">>, "); DLT_STRING("retval=<"); DLT_INT(size_written); DLT_STRING(">")); + + return size_written; +} + + +//TODO add write through compatibility +static sint_t GetKeySizeFromKissLocalDB(sint_t dbHandler, pconststr_t key) +{ + bool_t bCanContinue = true; + sint_t size_read = PERS_COM_FAILURE; + lldb_handler_s* pLldbHandler = NIL; + bool_t bLocked = false; + + DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("dbHandler="); DLT_INT(dbHandler); DLT_STRING("key=<<"); DLT_STRING(key); DLT_STRING(">> ...")); + + if ((dbHandler >= 0) && (NIL != key)) + { + if (lldb_handles_Lock()) + { + bLocked = true; + pLldbHandler = lldb_handles_FindInUseHandle(dbHandler); + if (NIL == pLldbHandler) + { + bCanContinue = false; + size_read = PERS_COM_ERR_INVALID_PARAM; + } + else + { + if (PersLldbPurpose_DB != pLldbHandler->ePurpose) + {/* this would be very bad */ + bCanContinue = false; + size_read = PERS_COM_FAILURE; + } + } + } + } + else + { + bCanContinue = false; + size_read = PERS_COM_ERR_INVALID_PARAM; + } + if (bCanContinue) + { + char tmp_key[PERS_DB_MAX_LENGTH_KEY_NAME]; + (void) strncpy(tmp_key, key, PERS_DB_MAX_LENGTH_KEY_NAME); + size_read = getFromCache(&pLldbHandler->kissDb, &tmp_key, NULL, 0, true); + if (size_read == PERS_STATUS_KEY_NOT_IN_CACHE) + size_read = getFromDatabaseFile(&pLldbHandler->kissDb, &tmp_key, NULL, PersLldbPurpose_DB, 0, true); + } + if (bLocked) + (void) lldb_handles_Unlock(); + DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("dbHandler="); DLT_INT(dbHandler); DLT_STRING("key=<<"); DLT_STRING(key); DLT_STRING(">>, "); DLT_STRING("retval=<"); DLT_INT(size_read); DLT_STRING(">")); + + return size_read; +} + + + + +//TODO add write through compatibility +/* return no of bytes read, or negative value in case of error */ +static sint_t GetDataFromKissLocalDB(sint_t dbHandler, pconststr_t key, pstr_t buffer_out, sint_t bufSize) +{ + bool_t bCanContinue = true; + sint_t size_read = PERS_COM_FAILURE; + lldb_handler_s* pLldbHandler = NIL; + bool_t bLocked = false; + + DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("dbHandler="); DLT_INT(dbHandler); DLT_STRING("key=<<"); DLT_STRING(key); DLT_STRING(">>, "); DLT_STRING("bufsize=<<"); DLT_INT(bufSize); DLT_STRING(">> ... ")); + + if ((dbHandler >= 0) && (NIL != key) && (NIL != buffer_out) && (bufSize > 0)) + { + if (lldb_handles_Lock()) + { + bLocked = true; + pLldbHandler = lldb_handles_FindInUseHandle(dbHandler); + if (NIL == pLldbHandler) + { + bCanContinue = false; + size_read = PERS_COM_ERR_INVALID_PARAM; + } + else + { + if (PersLldbPurpose_DB != pLldbHandler->ePurpose) + {/* this would be very bad */ + bCanContinue = false; + size_read = PERS_COM_FAILURE; + } + /* to not use DLT while mutex locked */ + } + } + } + else + { + bCanContinue = false; + size_read = PERS_COM_ERR_INVALID_PARAM; + } + + if (bCanContinue) + { + + //DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, + // DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("Working on DB: "), DLT_STRING(pLldbHandler->dbPathname) ); + + char tmp_key[PERS_DB_MAX_LENGTH_KEY_NAME]; + (void) strncpy(tmp_key, key, PERS_DB_MAX_LENGTH_KEY_NAME); + size_read = getFromCache(&pLldbHandler->kissDb, &tmp_key, buffer_out, bufSize, false); + //if key is not already in cache + if (size_read == PERS_STATUS_KEY_NOT_IN_CACHE) + size_read = getFromDatabaseFile(&pLldbHandler->kissDb, &tmp_key, buffer_out, PersLldbPurpose_DB, bufSize, + false); + } + if (bLocked) + (void) lldb_handles_Unlock(); + + DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("dbHandler="); DLT_INT(dbHandler); DLT_STRING("key=<<"); DLT_STRING(key); DLT_STRING(">>, "); DLT_STRING("bufsize=<<"); DLT_INT(bufSize); DLT_STRING(">>, "); DLT_STRING("retval=<"); DLT_INT(size_read); DLT_STRING(">")); + return size_read; +} + + +//TODO add write through compatibility +static sint_t GetDataFromKissRCT(sint_t dbHandler, pconststr_t key, PersistenceConfigurationKey_s* pConfig) +{ + bool_t bCanContinue = true; + sint_t size_read = PERS_COM_FAILURE; + lldb_handler_s* pLldbHandler = NIL; + bool_t bLocked = false; + + DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("dbHandler="); DLT_INT(dbHandler); DLT_STRING("key=<<"); DLT_STRING(key); DLT_STRING(">> ...")); + + if ((dbHandler >= 0) && (NIL != key) && (NIL != pConfig)) + { + if (lldb_handles_Lock()) + { + bLocked = true; + pLldbHandler = lldb_handles_FindInUseHandle(dbHandler); + if (NIL == pLldbHandler) + { + bCanContinue = false; + size_read = PERS_COM_ERR_INVALID_PARAM; + } + else + { + if (PersLldbPurpose_RCT != pLldbHandler->ePurpose) + {/* this would be very bad */ + bCanContinue = false; + size_read = PERS_COM_FAILURE; + } + /* to not use DLT while mutex locked */ + } + } + } + else + { + bCanContinue = false; + size_read = PERS_COM_ERR_INVALID_PARAM; + } + + //read RCT + if (bCanContinue) + { + //DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, + // DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("Working on DB: "), DLT_STRING(pLldbHandler->dbPathname) ); + + char tmp_key[PERS_RCT_MAX_LENGTH_RESOURCE_ID]; + (void) strncpy(tmp_key, key, PERS_RCT_MAX_LENGTH_RESOURCE_ID); + + size_read = getFromCache(&pLldbHandler->kissDb, &tmp_key, pConfig, sizeof(PersistenceConfigurationKey_s), false); + if (size_read == PERS_STATUS_KEY_NOT_IN_CACHE) + size_read = getFromDatabaseFile(&pLldbHandler->kissDb, &tmp_key, pConfig, PersLldbPurpose_RCT, + sizeof(PersistenceConfigurationKey_s), false); + } + if (bLocked) + (void) lldb_handles_Unlock(); + + DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("dbHandler="); DLT_INT(dbHandler); DLT_STRING("key=<<"); DLT_STRING(key); DLT_STRING(">>, "); DLT_STRING("retval=<"); DLT_INT(size_read); DLT_STRING(">")); + + return size_read; +} + + + +static bool_t lldb_handles_Lock(void) +{ + bool_t bEverythingOK = true; + sint_t siErr = pthread_mutex_lock(&g_mutexLldb); + if (0 != siErr) + { + bEverythingOK = false; + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("pthread_mutex_lock failed with error=<"); DLT_INT(siErr); DLT_STRING(">")); + } + return bEverythingOK; +} + +static bool_t lldb_handles_Unlock(void) +{ + bool_t bEverythingOK = true; + + sint_t siErr = pthread_mutex_unlock(&g_mutexLldb); + if (0 != siErr) + { + bEverythingOK = false; + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("pthread_mutex_unlock failed with error=<"); DLT_INT(siErr); DLT_STRING(">")); + } + return bEverythingOK; +} + +/* it is assumed dbHandler is checked by the caller */ +static lldb_handler_s* lldb_handles_FindInUseHandle(sint_t dbHandler) +{ + lldb_handler_s* pHandler = NIL; + + if (dbHandler <= PERS_LLDB_MAX_STATIC_HANDLES) + { + if (g_sHandlers.asStaticHandles[dbHandler].bIsAssigned) + { + pHandler = &g_sHandlers.asStaticHandles[dbHandler]; + } + } + else + { + lldb_handles_list_el_s* pListElemCurrent = g_sHandlers.pListHead; + while (NIL != pListElemCurrent) + { + if (dbHandler == pListElemCurrent->sHandle.dbHandler) + { + pHandler = &pListElemCurrent->sHandle; + break; + } + pListElemCurrent = pListElemCurrent->pNext; + } + } + + DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING((NIL!=pHandler) ? "Found handler <" : "ERROR can't find handler <"); DLT_INT(dbHandler); DLT_STRING(">"); DLT_STRING((NIL!=pHandler) ? (dbHandler <= PERS_LLDB_MAX_STATIC_HANDLES ? "in static area" : "in dynamic list") : "")); + + return pHandler; +} + +static lldb_handler_s* lldb_handles_FindAvailableHandle(void) +{ + bool_t bCanContinue = true; + lldb_handler_s* pHandler = NIL; + lldb_handles_list_el_s* psListElemNew = NIL; + + /* first try to find an available handle in the static area */ + sint_t siIndex = 0; + for (siIndex = 0; siIndex <= PERS_LLDB_MAX_STATIC_HANDLES; siIndex++) + { + if (!g_sHandlers.asStaticHandles[siIndex].bIsAssigned) + { + /* index setting should be done only once at the initialization of the static array + * Anyway, doing it here is more consistent */ + g_sHandlers.asStaticHandles[siIndex].dbHandler = siIndex; + pHandler = &g_sHandlers.asStaticHandles[siIndex]; + break; + } + } + + if (NIL == pHandler) + { + /* no position available in the static array => we have to use the list + * allocate memory for the new element and process the situation when the list is headless */ + + psListElemNew = (lldb_handles_list_el_s*) malloc(sizeof(lldb_handles_list_el_s)); + memset(psListElemNew, 0, sizeof(lldb_handles_list_el_s)); + if (NIL == psListElemNew) + { + bCanContinue = false; + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("malloc failed")); + } + else + { + if (NIL == g_sHandlers.pListHead) + { + /* the list not yet used/created, so use the new created element as the head */ + g_sHandlers.pListHead = psListElemNew; + g_sHandlers.pListHead->pNext = NIL; + g_sHandlers.pListHead->sHandle.dbHandler = PERS_LLDB_MAX_STATIC_HANDLES + 1; + /* the rest of the members will be set by lldb_handles_InitHandle */ + pHandler = &psListElemNew->sHandle; + } + } + } + if ((NIL == pHandler) && bCanContinue) + { + /* no position available in the static array => we have to use the list + * the memory for psListElemNew has been allocated and the list has a head + * The new element has to get the smallest index + * Now lets consider the situation when the head of the list has an index higher than (PERS_LLDB_MAX_STATIC_HANDLES + 1) + * => the list will have a new head !!! */ + if (g_sHandlers.pListHead->sHandle.dbHandler > (PERS_LLDB_MAX_STATIC_HANDLES + 1)) + { + psListElemNew->pNext = g_sHandlers.pListHead; + psListElemNew->sHandle.dbHandler = PERS_LLDB_MAX_STATIC_HANDLES + 1; + /* the rest of the members will be set by lldb_handles_InitHandle */ + g_sHandlers.pListHead = psListElemNew; + pHandler = &psListElemNew->sHandle; + } + } + + if ((NIL == pHandler) && bCanContinue) + { + /* no position available in the static array => we have to use the list + * the memory for psListElemNew has been allocated and the list has a head (with the smallest index) + * The new element has to get the smallest available index + * So will search for the first gap between two consecutive elements of the list and will introduce the new element between */ + lldb_handles_list_el_s* pListElemCurrent1 = g_sHandlers.pListHead; + lldb_handles_list_el_s* pListElemCurrent2 = pListElemCurrent1->pNext; + while (NIL != pListElemCurrent2) + { + if (pListElemCurrent2->sHandle.dbHandler - pListElemCurrent1->sHandle.dbHandler > 1) + { + /* found a gap => insert the element between and use the index next to pListElemCurrent1's */ + psListElemNew->pNext = pListElemCurrent2; + psListElemNew->sHandle.dbHandler = pListElemCurrent1->sHandle.dbHandler + 1; + pListElemCurrent1->pNext = psListElemNew; + pHandler = &psListElemNew->sHandle; + break; + } + else + { + pListElemCurrent1 = pListElemCurrent2; + pListElemCurrent2 = pListElemCurrent2->pNext; + } + } + if (NIL == pListElemCurrent2) + { + /* reached the end of the list => the list will have a new end */ + psListElemNew->pNext = NIL; + psListElemNew->sHandle.dbHandler = pListElemCurrent1->sHandle.dbHandler + 1; + pListElemCurrent1->pNext = psListElemNew; + pHandler = &psListElemNew->sHandle; + } + } + + DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING((NIL!=pHandler) ? "Found availble handler <" : "ERROR can't find available handler <"); DLT_INT((NIL!=pHandler) ? pHandler->dbHandler : (-1)); DLT_STRING(">"); DLT_STRING((NIL!=pHandler) ? (pHandler->dbHandler <= PERS_LLDB_MAX_STATIC_HANDLES ? "in static area" : "in dynamic list") : "")); + + return pHandler; +} + +static void lldb_handles_InitHandle(lldb_handler_s* psHandle_inout, pers_lldb_purpose_e ePurpose, + str_t const * dbPathname) +{ + psHandle_inout->bIsAssigned = true; + psHandle_inout->ePurpose = ePurpose; + (void) strncpy(psHandle_inout->dbPathname, dbPathname, sizeof(psHandle_inout->dbPathname)); +} + +static bool_t lldb_handles_DeinitHandle(sint_t dbHandler) +{ + bool_t bEverythingOK = true; + bool_t bHandlerFound = false; + + if (dbHandler <= PERS_LLDB_MAX_STATIC_HANDLES) + { + bHandlerFound = true; + g_sHandlers.asStaticHandles[dbHandler].bIsAssigned = false; + } + else + { + /* consider the situation when the handle is the head of the list */ + if (NIL != g_sHandlers.pListHead) + { + if (dbHandler == g_sHandlers.pListHead->sHandle.dbHandler) + { + lldb_handles_list_el_s* pListElemTemp = NIL; + bHandlerFound = true; + pListElemTemp = g_sHandlers.pListHead; + g_sHandlers.pListHead = g_sHandlers.pListHead->pNext; + free(pListElemTemp); + } + } + else + { + bEverythingOK = false; + } + } + + if (bEverythingOK && (!bHandlerFound)) + { + /* consider the situation when the handle is in the list (but not the head) */ + lldb_handles_list_el_s* pListElemCurrent1 = g_sHandlers.pListHead; + lldb_handles_list_el_s* pListElemCurrent2 = pListElemCurrent1->pNext; + while (NIL != pListElemCurrent2) + { + if (dbHandler == pListElemCurrent2->sHandle.dbHandler) + { + /* found the handle */ + bHandlerFound = true; + pListElemCurrent1->pNext = pListElemCurrent2->pNext; + free(pListElemCurrent2); + break; + } + else + { + pListElemCurrent1 = pListElemCurrent2; + pListElemCurrent2 = pListElemCurrent2->pNext; + } + } + if (NIL == pListElemCurrent2) + { + /* reached the end of the list without finding the handle */ + bEverythingOK = false; + } + } + + DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("dbHandler=<"); DLT_INT(dbHandler); DLT_STRING("> "); DLT_STRING(bEverythingOK ? (dbHandler <= PERS_LLDB_MAX_STATIC_HANDLES ? "deinit handler in static area" : "deinit handler in dynamic list") : "ERROR - handler not found")); + + return bEverythingOK; +} + + + +sint_t getFromCache(KISSDB* db, void* tmp_key, void* readBuffer, sint_t bufsize, bool_t sizeOnly) +{ + Kdb_bool cache_empty, key_deleted, key_not_found; + key_deleted = cache_empty = key_not_found = Kdb_false; + sint_t size_read = 0; + size_t size = 0; + int datasize = 0; + char* ptr; + pers_lldb_cache_flag_e eFlag; + + //if cache not already created + Kdb_wrlock(&db->shmem_info->cache_rwlock); + + if (db->shmem_info->cache_initialised == Kdb_true) + { + //open existing cache in existing shared memory + if (db->shmem_cached_fd <= 0) + { + if (openCache(db) != 0) + { + Kdb_unlock(&db->shmem_info->cache_rwlock); + return PERS_COM_FAILURE; + } + } + void* val = db->tbl->get(db->tbl, tmp_key, &size); + if (val == NULL) + { + size_read = PERS_COM_ERR_NOT_FOUND; + key_not_found = Kdb_true; + } + else + { + ptr = val; + eFlag = (pers_lldb_cache_flag_e) *(int*) ptr; + + //check if this key has already been marked as deleted + if (eFlag != CachedDataDelete) + { + //get datasize + ptr = ptr + sizeof(pers_lldb_cache_flag_e); + datasize = *(int*) ptr; + ptr = ptr + sizeof(int); //move pointer to beginning of data + size_read = datasize; + + //get data if needed + if (!sizeOnly) + { + if (bufsize < datasize) + { + Kdb_unlock(&db->shmem_info->cache_rwlock); + return PERS_COM_FAILURE; + } + else + (void) memcpy(readBuffer, ptr, datasize); + } + } + else + { + size_read = PERS_COM_ERR_NOT_FOUND; + key_deleted = Kdb_true; + } + free(val); + } + } + else + cache_empty = Kdb_true; + + //only read from file if key was not found in cache and if key was not marked as deleted in cache + if ((cache_empty == Kdb_true && key_deleted == Kdb_false) || key_not_found == Kdb_true) + { + Kdb_unlock(&db->shmem_info->cache_rwlock); + return PERS_STATUS_KEY_NOT_IN_CACHE; //key not found in cache + } + else + { + Kdb_unlock(&db->shmem_info->cache_rwlock); + return size_read; + } +} + + + +sint_t getFromDatabaseFile(KISSDB* db, void* tmp_key, void* readBuffer, pers_lldb_purpose_e purpose, sint_t bufsize, + bool_t sizeOnly) +{ + sint_t size_read = 0; + int datasize = 0; + char* ptr; + char m_data[sizeof(Data_LocalDB_s)] = { 0 }; //temporary buffer that gets filled with read in KISSDB_get + + int kissdb_status = KISSDB_get(db, tmp_key, m_data); + if (kissdb_status == 0) + { + if (purpose == PersLldbPurpose_DB) + { + ptr = m_data; + ptr += PERS_DB_MAX_SIZE_KEY_DATA; + datasize = *(int*) ptr; + if (!sizeOnly) + { + if (bufsize < datasize) + return PERS_COM_FAILURE; + else + (void) memcpy(readBuffer, m_data, datasize); + } + size_read = datasize; + } + else + { + if (!sizeOnly) + { + if (bufsize < datasize) + return PERS_COM_FAILURE; + else + (void) memcpy(readBuffer, m_data, sizeof(PersistenceConfigurationKey_s)); + } + size_read = sizeof(PersistenceConfigurationKey_s); + } + } + else + { + if (kissdb_status == 1) + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_WARN, + DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("KISSDB_get: key=<"); DLT_STRING(tmp_key); DLT_STRING(">, "); DLT_STRING("not found, retval=<"); DLT_INT(kissdb_status); DLT_STRING(">")); + size_read = PERS_COM_ERR_NOT_FOUND; + } + else + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, + DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("KISSDB_get: key=<"); DLT_STRING(tmp_key); DLT_STRING(">, "); DLT_STRING("Error with retval=<"); DLT_INT(kissdb_status); DLT_STRING(">")); + } + size_read = PERS_COM_ERR_NOT_FOUND; + } + return size_read; +} + +sint_t putToCache(KISSDB* db, sint_t dataSize, char* tmp_key, void* insert_cached_data) +{ + sint_t size_written = 0; + Kdb_wrlock(&db->shmem_info->cache_rwlock); + + //if cache not already created + if (db->shmem_info->cache_initialised == Kdb_false) + { + if (createCache(db) != 0) + { + Kdb_unlock(&db->shmem_info->cache_rwlock); + return PERS_COM_FAILURE; + } + } + else //open cache + { + if (openCache(db) != 0) + { + Kdb_unlock(&db->shmem_info->cache_rwlock); + return PERS_COM_FAILURE; + } + } + + //put in cache + if (db->tbl->put(db->tbl, tmp_key, insert_cached_data, + sizeof(pers_lldb_cache_flag_e) + sizeof(int) + (size_t) dataSize) == false) //store flag , datasize and data as value in cache + { + size_written = PERS_COM_FAILURE; + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, + DLT_STRING(__FUNCTION__); DLT_STRING(":");DLT_STRING("Error: Failed to put data into cache")); + } + else + size_written = dataSize; + + Kdb_unlock(&db->shmem_info->cache_rwlock); + return size_written; +} + + + + + +sint_t getListandSize(KISSDB* db, pstr_t buffer, sint_t size, bool_t bOnlySizeNeeded, pers_lldb_purpose_e purpose) +{ + KISSDB_Iterator dbi; + int keycount_file = 0, keycount_cache = 0, result = 0, x = 0, idx = 0, max = 0, used = 0, obj_count, keylength = 0; + char** tmplist = NULL; + qnobj_t obj; + sint_t availableSize = size; + char* ptr; + char* memory = NULL; + void* pt; + pers_lldb_cache_flag_e eFlag; + + if (purpose == PersLldbPurpose_RCT) + keylength = PERS_RCT_MAX_LENGTH_RESOURCE_ID; + else + keylength = PERS_DB_MAX_LENGTH_KEY_NAME; + + //open existing cache if present and look for keys + if (db->shmem_info->cache_initialised == Kdb_true) + { + if (openCache(db) != 0) + { + Kdb_unlock(&db->shmem_info->cache_rwlock); + return PERS_COM_FAILURE; + } + else + { + obj_count = db->tbl->size(db->tbl, &max, &used); + if (obj_count > 0) + { + tmplist = malloc(sizeof(char*) * obj_count); + if (tmplist != NULL) + { + while (db->tbl->getnext(db->tbl, &obj, &idx) == true) + { + pt = obj.data; + eFlag = (pers_lldb_cache_flag_e) *(int*) pt; + if (eFlag != CachedDataDelete) + { + tmplist[keycount_cache] = (char*) malloc(strlen(obj.name) + 1); + (void) strncpy(tmplist[keycount_cache], obj.name, strlen(obj.name)); + ptr = tmplist[keycount_cache]; + ptr[strlen(obj.name)] = '\0'; + keycount_cache++; + } + } + } + else + return PERS_COM_ERR_MALLOC; + } + } + } + + //look for keys in database file + KISSDB_Iterator_init(db, &dbi); + char kbuf[keylength]; + + //get number of keys, stored in database file + while (KISSDB_Iterator_next(&dbi, &kbuf, NULL) > 0) + keycount_file++; + + if ((keycount_cache + keycount_file) > 0) + { + int memsize = qhasharr_calculate_memsize(keycount_cache + keycount_file); + //create hashtable that stores the list of keys without duplicates + memory = malloc(memsize); + if (memory != NULL) + { + memset(memory, 0, memsize); + qhasharr_t *tbl = qhasharr(memory, memsize); + if (tbl == NULL) + return PERS_COM_ERR_MALLOC; + + //put keys in cache to a hashtable + for (x = 0; x < keycount_cache; x++) + { + if (tbl->put(tbl, tmplist[x], "0", 1) == true) + { + if (tmplist[x] != NULL) + free(tmplist[x]); + } + } + free(tmplist); + + //put keys in database file to hashtable (existing key gets overwritten -> no duplicate keys) + KISSDB_Iterator_init(db, &dbi); + memset(kbuf, 0, keylength); + while (KISSDB_Iterator_next(&dbi, &kbuf, NULL) > 0) + { + size_t keyLen = strnlen(kbuf, sizeof(kbuf)); + if (keyLen > 0) + tbl->put(tbl, kbuf, "0", 1); + } + + //count needed size for buffer / copy keys to buffer + idx = 0; + while (tbl->getnext(tbl, &obj, &idx) == true) + { + size_t keyLen = strlen(obj.name); + if (keyLen > 0) + { + if ((!bOnlySizeNeeded) && ((sint_t) keyLen < availableSize)) + { + (void) strncpy(buffer, obj.name, keyLen); + *(buffer + keyLen) = ListItemsSeparator; + buffer += (keyLen + sizeof(ListItemsSeparator)); + availableSize -= (sint_t) (keyLen + sizeof(ListItemsSeparator)); + } + } + result += (sint_t) (keyLen + sizeof(ListItemsSeparator)); + } + //release hashtable and allocated memory + tbl->free(tbl); + free(memory); + } + else + return PERS_COM_ERR_MALLOC; + } + return result; +} + + + + +int createCache(KISSDB* db) +{ + Kdb_bool shmem_creator; + int status = -1; + + db->shmem_cached_fd = kdbShmemOpen(db->shmem_cached_name, PERS_CACHE_MEMSIZE, &shmem_creator); + if (db->shmem_cached_fd != -1) + { + db->shmem_cached = (void*) getKdbShmemPtr(db->shmem_cached_fd, PERS_CACHE_MEMSIZE); + if (db->shmem_cached != ((void *) -1)) + { + db->tbl = qhasharr(db->shmem_cached, PERS_CACHE_MEMSIZE); + if (db->tbl != NULL) + { + status = 0; + db->shmem_info->cache_initialised = Kdb_true; + } + } + } + if (status != 0) + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, + DLT_STRING(__FUNCTION__); DLT_STRING(":");DLT_STRING("Error: Failed to create cache")); + return status; +} + + + +int openCache(KISSDB* db) +{ + Kdb_bool shmem_creator; + int status = -1; + + //only open shared memory again if filedescriptor is not initialised yet + if (db->shmem_cached_fd <= 0) //not shared filedescriptor + { + db->shmem_cached_fd = kdbShmemOpen(db->shmem_cached_name, PERS_CACHE_MEMSIZE, &shmem_creator); + if (db->shmem_cached_fd != -1) + { + db->shmem_cached = (void*) getKdbShmemPtr(db->shmem_cached_fd, PERS_CACHE_MEMSIZE); + if (db->shmem_cached != ((void *) -1)) + { + // use existent hash-table + db->tbl = qhasharr(db->shmem_cached, 0); + if (db->tbl != NULL) + status = 0; + } + } + } + else + status = 0; + if (status != 0) + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, + DLT_STRING(__FUNCTION__); DLT_STRING(":");DLT_STRING("Error: Failed to open cache")); + return status; +} + + +int closeCache(KISSDB* db) +{ + int status = -1; + if (kdbShmemClose(db->shmem_cached_fd, db->shmem_cached_name) != Kdb_false) + { + free(db->shmem_cached_name); //free memory for name obtained by kdbGetShmName() function + if (freeKdbShmemPtr(db->shmem_cached, PERS_CACHE_MEMSIZE) != Kdb_false) + status = 0; + } + if (status != 0) + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, + DLT_STRING(__FUNCTION__); DLT_STRING(":");DLT_STRING("Error: Failed to close cache")); + return status; +} |