From 93df8702113728341d8ad6d86ce847ea52b79c08 Mon Sep 17 00:00:00 2001 From: Magnus Ahltorp Date: Thu, 18 Feb 2016 13:56:07 +0100 Subject: Use pascal strings for keys --- c_src/permdb.c | 143 ++++++++++++++++++++++++++++--------------------------- c_src/permdb.h | 3 -- c_src/permdbpy.c | 76 ----------------------------- 3 files changed, 72 insertions(+), 150 deletions(-) diff --git a/c_src/permdb.c b/c_src/permdb.c index 5ebdbe4..50618c5 100644 --- a/c_src/permdb.c +++ b/c_src/permdb.c @@ -22,6 +22,7 @@ #include "util.h" #include "uthash.h" +#include "pstring.h" #define INDEX_COMMIT_TRAILER_SIZE (sizeof(uint64_t) + SHA256_DIGEST_SIZE + sizeof(index_commit_cookie)) @@ -46,49 +47,49 @@ static const node_object errornode = {{NODE_ENTRY_ERROR_NODE, struct nodecache { node_object value; - char *key; + ps_string *key; UT_hash_handle hh; }; static node_object -get_node_from_cache(permdb_object *state, const char *key) +get_node_from_cache(permdb_object *state, const ps_string *key) { - dprintf(CACHE, (stderr, "getting cache key %s, ", key)); + dprintf(CACHE, (stderr, "getting cache key %*s, ", PS_PRINTF(key))); struct nodecache *node; - HASH_FIND(hh, state->nodecache, key, strlen(key), node); + HASH_FIND(hh, state->nodecache, key->value, key->length, node); if (node == NULL) { dprintf(CACHE, (stderr, "found nothing in cache\n")); return errornode; } - dprintf(CACHE, (stderr, "got cache key %s: ", node->key)); + dprintf(CACHE, (stderr, "got cache key %*s: ", PS_PRINTF(node->key))); dprinthex(CACHE, &node->value, sizeof(struct node_object)); return node->value; } static node_object -get_node_from_dirtynodes(permdb_object *state, const char *key) +get_node_from_dirtynodes(permdb_object *state, const ps_string *key) { - dprintf(CACHE, (stderr, "getting key %s, ", key)); + dprintf(CACHE, (stderr, "getting key %*s, ", PS_PRINTF(key))); struct nodecache *node; - HASH_FIND(hh, state->dirtynodes, key, strlen(key), node); + HASH_FIND(hh, state->dirtynodes, key->value, key->length, node); if (node == NULL) { dprintf(CACHE, (stderr, "found nothing\n")); return errornode; } - dprintf(CACHE, (stderr, "got key %s: ", node->key)); + dprintf(CACHE, (stderr, "got key %*s: ", PS_PRINTF(node->key))); dprinthex(CACHE, &node->value, sizeof(struct node_object)); return node->value; } static void -put_node_in_cache(permdb_object *state, const char *key, node_object value) +put_node_in_cache(permdb_object *state, const ps_string *key, node_object value) { - dprintf(CACHE, (stderr, "putting cache key %s: ", key)); + dprintf(CACHE, (stderr, "putting cache key %*s: ", PS_PRINTF(key))); dprinthex(CACHE, &value, sizeof(node_object)); struct nodecache *node; - HASH_FIND(hh, state->nodecache, key, strlen(key), node); + HASH_FIND(hh, state->nodecache, key->value, key->length, node); if (node) { node->value = value; return; @@ -96,17 +97,17 @@ put_node_in_cache(permdb_object *state, const char *key, node_object value) node = malloc(sizeof(struct nodecache)); node->value = value; - node->key = strdup(key); - HASH_ADD(hh, state->nodecache, key[0], strlen(node->key), node); + node->key = ps_strdup(key); + HASH_ADD(hh, state->nodecache, key->value[0], node->key->length, node); } static void -put_node_in_dirtynodes(permdb_object *state, const char *key, node_object value) +put_node_in_dirtynodes(permdb_object *state, const ps_string *key, node_object value) { - dprintf(CACHE, (stderr, "putting key %s: ", key)); + dprintf(CACHE, (stderr, "putting key %*s: ", PS_PRINTF(key))); dprinthex(CACHE, &value, sizeof(node_object)); struct nodecache *node; - HASH_FIND(hh, state->dirtynodes, key, strlen(key), node); + HASH_FIND(hh, state->dirtynodes, key->value, key->length, node); if (node) { node->value = value; return; @@ -114,8 +115,8 @@ put_node_in_dirtynodes(permdb_object *state, const char *key, node_object value) node = malloc(sizeof(struct nodecache)); node->value = value; - node->key = strdup(key); - HASH_ADD(hh, state->dirtynodes, key[0], strlen(node->key), node); + node->key = ps_strdup(key); + HASH_ADD(hh, state->dirtynodes, key->value[0], node->key->length, node); } void @@ -175,7 +176,7 @@ datafile_add_header(buffered_file *file) void initial_node(permdb_object *state) { - put_node_in_dirtynodes(state, "", nullnode); + put_node_in_dirtynodes(state, PS_STRING(""), nullnode); } int @@ -447,15 +448,15 @@ keybits(const unsigned char *key, unsigned int level) #define KEYPART_MAX (keylen*4+1) static void -keypart(const unsigned char *key, unsigned int level, char *s) +keypart(const unsigned char *key, unsigned int level, ps_string *s) { unsigned int i; for (i = 0; i < level; i++) { unsigned char b = keybits(key, i); - s[i] = b + (unsigned char) '0'; + s->value[i] = b + (unsigned char) '0'; } - s[level] = 0; + s->length = level; } static void @@ -519,12 +520,12 @@ iserrornode(node_object node) node.data[3] == NODE_ENTRY_ERROR_NODE; } -node_object -readnode(permdb_object *state, node_offset offset, const char *cachekey) +static node_object +readnode(permdb_object *state, node_offset offset, const ps_string *cachekey) { - dprintf(READ, (stderr, "reading node: offset %llu cachekey '%s'\n", (unsigned long long) offset, cachekey ? cachekey : "none")); - if (cachekey) { + dprintf(READ, (stderr, "reading node: offset %llu cachekey '%*s'\n", (unsigned long long) offset, PS_PRINTF(cachekey))); + node_object dirtynode = get_node_from_dirtynodes(state, cachekey); if (!iserrornode(dirtynode)) { dprintf(READ, (stderr, "reading node: found node in dirty nodes\n")); @@ -532,8 +533,8 @@ readnode(permdb_object *state, node_offset offset, const char *cachekey) } if (offset == NODE_ENTRY_DIRTY_NODE) { - set_error(&state->error, "referring to dirty node at key %s, but node not in dirtynodes\n", cachekey); - dprintf(READ, (stderr, "reading node: referring to dirty node at key %s, but node not in dirtynodes\n", cachekey)); + set_error(&state->error, "referring to dirty node at key %*s, but node not in dirtynodes\n", PS_PRINTF(cachekey)); + dprintf(READ, (stderr, "reading node: referring to dirty node at key %*s, but node not in dirtynodes\n", PS_PRINTF(cachekey))); return errornode; } @@ -542,6 +543,8 @@ readnode(permdb_object *state, node_offset offset, const char *cachekey) dprintf(READ, (stderr, "reading node: found node in cache\n")); return cachednode; } + } else { + dprintf(READ, (stderr, "reading node: offset %llu no cachekey\n", (unsigned long long) offset)); } size_t length = sizeof(index_node_cookie) + sizeof(node_object); @@ -611,7 +614,7 @@ getpath(permdb_object *state, const unsigned char *key, struct nodelist *nodes) node_offset rootoffset = bf_lastcommit(state->indexfile) - (sizeof(index_node_cookie) + sizeof(node_object) + INDEX_COMMIT_TRAILER_SIZE); - node_object node = readnode(state, rootoffset, ""); + node_object node = readnode(state, rootoffset, PS_STRING("")); if (iserrornode(node)) { fprintf(stderr, "cannot find root node at offset %llu (lastcommit %llu)\n", (long long unsigned int) rootoffset, (long long unsigned int) bf_lastcommit(state->indexfile)); @@ -633,11 +636,10 @@ getpath(permdb_object *state, const unsigned char *key, struct nodelist *nodes) return (char) kb; } level++; - char kp[KEYPART_MAX]; - keypart(key, level, kp); - node = readnode(state, entryoffset(entry), kp); + ps_string kp; + keypart(key, level, &kp); + node = readnode(state, entryoffset(entry), &kp); if (iserrornode(node)) { - free(kp); dprintf(READ, (stderr, "getpath: not found\n")); return -1; } @@ -651,7 +653,7 @@ getpathlastnode(permdb_object *state, const unsigned char *key) unsigned int level = 0; node_offset rootoffset = bf_lastcommit(state->indexfile) - (sizeof(index_node_cookie) + sizeof(node_object) + INDEX_COMMIT_TRAILER_SIZE); - node_object node = readnode(state, rootoffset, ""); + node_object node = readnode(state, rootoffset, PS_STRING("")); if (iserrornode(node)) { dprintf(READ, (stderr, "getpathlastnode: no node\n")); @@ -668,9 +670,9 @@ getpathlastnode(permdb_object *state, const unsigned char *key) break; } level++; - char kp[KEYPART_MAX]; - keypart(key, level, kp); - node = readnode(state, entryoffset(entry), kp); + ps_string kp; + keypart(key, level, &kp); + node = readnode(state, entryoffset(entry), &kp); } node_entry entry = get_entry_in_node(node, kb); @@ -678,7 +680,7 @@ getpathlastnode(permdb_object *state, const unsigned char *key) } static node_offset -writenode(permdb_object *state, node_object node, const char *cachekey) +writenode(permdb_object *state, node_object node, ps_string *cachekey) { node_offset offset = bf_total_length(state->indexfile); @@ -824,17 +826,17 @@ addindex(permdb_object *state, const unsigned char *key, unsigned int keylength, addentry(&leafnode, keybits(olddatakey, level), buildentry(1, olddataoffset)); free(olddatakey); { - char cachekey[KEYPART_MAX]; - keypart(key, level, cachekey); - put_node_in_dirtynodes(state, cachekey, leafnode); + ps_string cachekey; + keypart(key, level, &cachekey); + put_node_in_dirtynodes(state, &cachekey, leafnode); } level--; while (level > foundlevel) { node_object node = nullnode; addentry(&node, keybits(key, level), NODE_ENTRY_DIRTY_NODE); - char cachekey[KEYPART_MAX]; - keypart(key, level, cachekey); - put_node_in_dirtynodes(state, cachekey, node); + ps_string cachekey; + keypart(key, level, &cachekey); + put_node_in_dirtynodes(state, &cachekey, node); level--; } overwriteentry(&lastnode, keybits(key, foundlevel), NODE_ENTRY_DIRTY_NODE); @@ -843,9 +845,9 @@ addindex(permdb_object *state, const unsigned char *key, unsigned int keylength, int level = (int) foundlevel; { - char cachekey[KEYPART_MAX]; - keypart(key, (unsigned int) level, cachekey); - put_node_in_dirtynodes(state, cachekey, lastnode); + ps_string cachekey; + keypart(key, (unsigned int) level, &cachekey); + put_node_in_dirtynodes(state, &cachekey, lastnode); } level--; @@ -857,9 +859,9 @@ addindex(permdb_object *state, const unsigned char *key, unsigned int keylength, } node_object node = nodes.nodes[level]; overwriteentry(&node, keybits(key, (unsigned int) level), NODE_ENTRY_DIRTY_NODE); - char cachekey[KEYPART_MAX]; - keypart(key, (unsigned int) level, cachekey); - put_node_in_dirtynodes(state, cachekey, node); + ps_string cachekey; + keypart(key, (unsigned int) level, &cachekey); + put_node_in_dirtynodes(state, &cachekey, node); level--; } @@ -915,17 +917,17 @@ addvalue(permdb_object *state, const unsigned char *key, unsigned int keylength, addentry(&leafnode, keybits(olddatakey, level), buildentry(1, olddataoffset)); free(olddatakey); { - char cachekey[KEYPART_MAX]; - keypart(key, level, cachekey); - put_node_in_dirtynodes(state, cachekey, leafnode); + ps_string cachekey; + keypart(key, level, &cachekey); + put_node_in_dirtynodes(state, &cachekey, leafnode); } level--; while (level > foundlevel) { node_object node = nullnode; addentry(&node, keybits(key, level), NODE_ENTRY_DIRTY_NODE); - char cachekey[KEYPART_MAX]; - keypart(key, level, cachekey); - put_node_in_dirtynodes(state, cachekey, node); + ps_string cachekey; + keypart(key, level, &cachekey); + put_node_in_dirtynodes(state, &cachekey, node); level--; } overwriteentry(&lastnode, keybits(key, foundlevel), NODE_ENTRY_DIRTY_NODE); @@ -934,9 +936,9 @@ addvalue(permdb_object *state, const unsigned char *key, unsigned int keylength, int level = (int) foundlevel; { - char cachekey[KEYPART_MAX]; - keypart(key, (unsigned int) level, cachekey); - put_node_in_dirtynodes(state, cachekey, lastnode); + ps_string cachekey; + keypart(key, (unsigned int) level, &cachekey); + put_node_in_dirtynodes(state, &cachekey, lastnode); } level--; @@ -948,9 +950,9 @@ addvalue(permdb_object *state, const unsigned char *key, unsigned int keylength, } node_object node = nodes.nodes[level]; overwriteentry(&node, keybits(key, (unsigned int) level), NODE_ENTRY_DIRTY_NODE); - char cachekey[KEYPART_MAX]; - keypart(key, (unsigned int) level, cachekey); - put_node_in_dirtynodes(state, cachekey, node); + ps_string cachekey; + keypart(key, (unsigned int) level, &cachekey); + put_node_in_dirtynodes(state, &cachekey, node); level--; } @@ -986,8 +988,8 @@ getvalue(permdb_object *state, const unsigned char *key, size_t keylength, size_ static int string_length_comparison(struct nodecache *a, struct nodecache *b) { - size_t a_len = strlen(a->key); - size_t b_len = strlen(b->key); + size_t a_len = a->key->length; + size_t b_len = b->key->length; if (b_len > a_len) { return 1; } else if (b_len == a_len) { @@ -1000,7 +1002,7 @@ string_length_comparison(struct nodecache *a, struct nodecache *b) { int committree(permdb_object *state) { - get_node_from_dirtynodes(state, ""); + get_node_from_dirtynodes(state, PS_STRING("")); if (state->dirtynodes == NULL) { return 0; } @@ -1008,7 +1010,7 @@ committree(permdb_object *state) HASH_SORT(state->dirtynodes, string_length_comparison); struct nodecache *lastobject = ELMT_FROM_HH(state->dirtynodes->hh.tbl, state->dirtynodes->hh.tbl->tail); - if (lastobject->key[0] != '\0') { + if (lastobject->key->length != 0) { fprintf(stderr, "sorted list doesn't end with root node\n"); return -1; } @@ -1021,11 +1023,10 @@ committree(permdb_object *state) assert(get_entry_in_node(node->value, 2) != NODE_ENTRY_DIRTY_NODE); assert(get_entry_in_node(node->value, 3) != NODE_ENTRY_DIRTY_NODE); node_offset offset = writenode(state, node->value, node->key); - size_t keylength = strlen(node->key); + size_t keylength = node->key->length; if (keylength != 0) { - char *parent = strdup(node->key); - parent[keylength - 1] = '\0'; - unsigned int entrynumber = (unsigned int) (node->key[keylength - 1] - '0'); + ps_string *parent = ps_resize(node->key, node->key->length - 1); + unsigned int entrynumber = (unsigned int) (node->key->value[keylength - 1] - '0'); node_object parentnode = get_node_from_dirtynodes(state, parent); overwriteentry(&parentnode, entrynumber, offset); put_node_in_dirtynodes(state, parent, parentnode); diff --git a/c_src/permdb.h b/c_src/permdb.h index 7394143..b8af13c 100644 --- a/c_src/permdb.h +++ b/c_src/permdb.h @@ -30,9 +30,6 @@ get_entry_in_node(node_object node, unsigned char n); unsigned char * read_internal_data(permdb_object *state, node_offset offset, size_t length); -node_object -readnode(permdb_object *state, node_offset offset, const char *cachekey); - node_offset datasize(permdb_object *state); diff --git a/c_src/permdbpy.c b/c_src/permdbpy.c index a98c83a..1e8c2f7 100644 --- a/c_src/permdbpy.c +++ b/c_src/permdbpy.c @@ -45,67 +45,6 @@ permdb_alloc_py(const char *dbpath) return state; } -typedef struct node_object_py { - PyObject_HEAD - node_object nodeobj; -} node_object_py; - -static void -pynode_dealloc(node_object_py *node) -{ - -} - -static PyObject* -node_getitem(PyObject *o, PyObject *key) -{ - unsigned int n = PyLong_AsUnsignedLong(key); - node_object_py *node = (node_object_py *)o; - - return PyLong_FromUnsignedLongLong(get_entry_in_node(node->nodeobj, n)); -} - -static PyMappingMethods node_as_mapping = { - 0, /* mp_length */ - node_getitem, /* mp_subscript */ - 0, /* mp_ass_subscript */ -}; - -static PyTypeObject node_type = { - PyObject_HEAD_INIT(NULL) - 0, /*ob_size*/ - "permdb.node", /*tp_name*/ - sizeof(node_object_py), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - (destructor)pynode_dealloc, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_compare*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - &node_as_mapping, /*tp_as_mapping*/ - 0, /*tp_hash */ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT, /*tp_flags*/ - "PermDB node", /*tp_doc*/ -}; - - -static node_object_py * -node_alloc(node_object data) -{ - node_object_py *node = PyObject_New(node_object_py, &node_type); - node->nodeobj = data; - return node; -} - - static PyObject * data_pread(PyObject *self, PyObject *args) { @@ -140,20 +79,6 @@ permdb_alloc_wrapper(PyObject *self, PyObject *args) } static PyObject * -readnode_wrapper(PyObject *self, PyObject *args) -{ - permdb_object_py *state; - unsigned long long offset; - const char *key = NULL; - - if (!PyArg_ParseTuple(args, "O!K|s", &permdb_type, &state, &offset, &key)) { - return NULL; - } - - return (PyObject *)node_alloc(readnode(state->permdb, offset, key)); -} - -static PyObject * datasize_wrapper(PyObject *self, PyObject *args) { permdb_object_py *state; @@ -255,7 +180,6 @@ committree_wrapper(PyObject *self, PyObject *args) static PyMethodDef UtilMethods[] = { {"data_pread", data_pread, METH_VARARGS}, {"alloc", permdb_alloc_wrapper, METH_VARARGS}, - {"readnode", readnode_wrapper, METH_VARARGS}, {"datasize", datasize_wrapper, METH_VARARGS}, {"addvalue", addvalue_wrapper, METH_VARARGS}, {"getvalue", getvalue_wrapper, METH_VARARGS}, -- cgit v1.1