diff options
| -rw-r--r-- | c_src/Makefile | 33 | ||||
| -rw-r--r-- | c_src/permdbpy.c | 271 | ||||
| -rw-r--r-- | c_src/permdbtest.py | 103 |
3 files changed, 404 insertions, 3 deletions
diff --git a/c_src/Makefile b/c_src/Makefile index d45e601..e9b0394 100644 --- a/c_src/Makefile +++ b/c_src/Makefile @@ -1,6 +1,28 @@ +UNAME = $(shell uname) + +ifeq ($(UNAME),Darwin) +# Mac OS X 10.10 +OS_CFLAGS = -arch x86_64 -I/System/Library/Frameworks/Python.framework/Versions/2.7/include/python2.7 -DHAVE_COMMON_CRYPTO +OS_SHLIB_LINKFLAGS = -Wl,-F. -bundle -undefined dynamic_lookup +OS_LDFLAGS = +PYTHONLIB = -lpython +else ifeq ($(UNAME),Linux) +# Ubuntu 12 +OS_CFLAGS = -I/usr/include/python2.7 -std=gnu99 -fPIC -DHAVE_BSD_COMPAT_LIB +OS_SHLIB_LINKFLAGS = -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-Bsymbolic-functions -Wl,-z,relro +OS_LDFLAGS = -lbsd -lrhash +PYTHONLIB = -L/usr/lib/x86_64-linux-gnu -lpython2.7 +else +error: + echo $(UNAME) not supported +endif + +LOCAL_CFLAGS = +LOCAL_LDFLAGS = + CC = gcc -CFLAGS = -Wall -Werror -std=gnu99 -LDFLAGS = -lnettle +CFLAGS = -Wall -Werror -std=gnu99 $(LOCAL_CFLAGS) $(OS_CFLAGS) +LDFLAGS = $(LOCAL_CFLAGS) -lnettle $(OS_LDFLAGS) PORTS = fsynchelper hsmhelper permdbport @@ -8,8 +30,10 @@ common_OBJS = erlport.o net_read_write.o fsynchelper_OBJS = fsynchelper.o $(common_OBJS) hsmhelper_OBJS = hsmhelper.o pkcs11.o $(common_OBJS) permdbport_OBJS = permdb.o permdbport.o arlamath.o hash.o $(common_OBJS) +permdbso_OBJS = permdb.o arlamath.o hash.o permdbpy.o $(common_OBJS) -all: $(PORTS) + +all: $(PORTS) permdb.so clean: rm -f $(fsynchelper_OBJS) $(hsmhelper_OBJS) $(permdbport_OBJS) $(PORTS) @@ -22,3 +46,6 @@ hsmhelper: $(hsmhelper_OBJS) permdbport: $(permdbport_OBJS) $(CC) -o permdbport $(permdbport_OBJS) $(LDFLAGS) + +permdb.so: $(permdbso_OBJS) + $(CC) $(LDFLAGS) $(OS_SHLIB_LINKFLAGS) $(permdbso_OBJS) -o permdb.so diff --git a/c_src/permdbpy.c b/c_src/permdbpy.c new file mode 100644 index 0000000..a98c83a --- /dev/null +++ b/c_src/permdbpy.c @@ -0,0 +1,271 @@ +#include <Python.h> +#include "permdb.h" + +typedef struct permdb_object_py { + PyObject_HEAD + struct permdb_object *permdb; +} permdb_object_py; + +static void +permdb_dealloc(permdb_object_py *state) +{ + permdb_free(state->permdb); +} + +PyTypeObject permdb_type = { + PyObject_HEAD_INIT(NULL) + 0, /*ob_size*/ + "permdb.permdb", /*tp_name*/ + sizeof(permdb_object_py), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + (destructor)permdb_dealloc,/*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash */ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT, /*tp_flags*/ + "PermDB state", /*tp_doc*/ +}; + +permdb_object_py * +permdb_alloc_py(const char *dbpath) +{ + permdb_object_py *state = PyObject_New(permdb_object_py, &permdb_type); + state->permdb = permdb_alloc(dbpath); + return state; +} + +typedef struct node_object_py { + PyObject_HEAD + node_object nodeobj; +} node_object_py; + +static void +pynode_dealloc(node_object_py *node) +{ + +} + +static PyObject* +node_getitem(PyObject *o, PyObject *key) +{ + unsigned int n = PyLong_AsUnsignedLong(key); + node_object_py *node = (node_object_py *)o; + + return PyLong_FromUnsignedLongLong(get_entry_in_node(node->nodeobj, n)); +} + +static PyMappingMethods node_as_mapping = { + 0, /* mp_length */ + node_getitem, /* mp_subscript */ + 0, /* mp_ass_subscript */ +}; + +static PyTypeObject node_type = { + PyObject_HEAD_INIT(NULL) + 0, /*ob_size*/ + "permdb.node", /*tp_name*/ + sizeof(node_object_py), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + (destructor)pynode_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + &node_as_mapping, /*tp_as_mapping*/ + 0, /*tp_hash */ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT, /*tp_flags*/ + "PermDB node", /*tp_doc*/ +}; + + +static node_object_py * +node_alloc(node_object data) +{ + node_object_py *node = PyObject_New(node_object_py, &node_type); + node->nodeobj = data; + return node; +} + + +static PyObject * +data_pread(PyObject *self, PyObject *args) +{ + unsigned long long offset; + unsigned int length; + permdb_object_py *state; + + if (!PyArg_ParseTuple(args, "O!KI", &permdb_type, &state, &offset, &length)) { + return NULL; + } + + unsigned char *result = read_internal_data(state->permdb, offset, length); + if (result == NULL) { + return NULL; + } + + PyObject* resultObj = PyString_FromStringAndSize((char *)result, length); + free(result); + return resultObj; +} + +static PyObject * +permdb_alloc_wrapper(PyObject *self, PyObject *args) +{ + const char *dbpath = NULL; + + if (!PyArg_ParseTuple(args, "s", &dbpath)) { + return NULL; + } + + return (PyObject*)permdb_alloc_py(dbpath); +} + +static PyObject * +readnode_wrapper(PyObject *self, PyObject *args) +{ + permdb_object_py *state; + unsigned long long offset; + const char *key = NULL; + + if (!PyArg_ParseTuple(args, "O!K|s", &permdb_type, &state, &offset, &key)) { + return NULL; + } + + return (PyObject *)node_alloc(readnode(state->permdb, offset, key)); +} + +static PyObject * +datasize_wrapper(PyObject *self, PyObject *args) +{ + permdb_object_py *state; + + if (!PyArg_ParseTuple(args, "O!", &permdb_type, &state)) { + return NULL; + } + + return PyInt_FromLong(datasize(state->permdb)); +} + +static PyObject * +addvalue_wrapper(PyObject *self, PyObject *args) +{ + permdb_object_py *state; + const char *key; + unsigned int keylength; + const char *data; + unsigned int datalength; + + if (!PyArg_ParseTuple(args, "O!s#s#", &permdb_type, &state, &key, &keylength, &data, &datalength)) { + return NULL; + } + + int result = addvalue(state->permdb, (unsigned char *) key, keylength, (unsigned char *) data, datalength); + + if (result < 0) { + return NULL; + } else if (result == 0) { + Py_INCREF(Py_False); + return Py_False; + } else { + Py_INCREF(Py_True); + return Py_True; + } +} + +static PyObject * +getvalue_wrapper(PyObject *self, PyObject *args) +{ + permdb_object_py *state; + + const char *key; + int keylen; + + if (!PyArg_ParseTuple(args, "O!s#", &permdb_type, &state, &key, &keylen)) { + return NULL; + } + + size_t datalen; + unsigned char *result = getvalue(state->permdb, (unsigned char *) key, keylen, &datalen); + + if (result == NULL) { + Py_INCREF(Py_None); + return Py_None; + } + + PyObject* resultObj = PyString_FromStringAndSize((char *) result, datalen); + free(result); + return resultObj; +} + +static PyObject * +clear_nodecache(PyObject *self, PyObject *args) +{ + permdb_object_py *state; + + if (!PyArg_ParseTuple(args, "O!", &permdb_type, &state)) { + return NULL; + } + + delete_all_nodes_in_cache(state->permdb); + + Py_INCREF(Py_None); + return Py_None; +} + +static PyObject * +committree_wrapper(PyObject *self, PyObject *args) +{ + permdb_object_py *state; + + fprintf(stderr, "starting commit\n"); + + if (!PyArg_ParseTuple(args, "O!", &permdb_type, &state)) { + return NULL; + } + + int result = committree(state->permdb); + + if (result < 0) { + return NULL; + } else { + Py_INCREF(Py_None); + return Py_None; + } +} + +static PyMethodDef UtilMethods[] = { + {"data_pread", data_pread, METH_VARARGS}, + {"alloc", permdb_alloc_wrapper, METH_VARARGS}, + {"readnode", readnode_wrapper, METH_VARARGS}, + {"datasize", datasize_wrapper, METH_VARARGS}, + {"addvalue", addvalue_wrapper, METH_VARARGS}, + {"getvalue", getvalue_wrapper, METH_VARARGS}, + {"committree", committree_wrapper, METH_VARARGS}, + {"clear_nodecache", clear_nodecache, METH_VARARGS}, + {NULL, NULL} +}; + +void +initpermdb() +{ + (void) Py_InitModule("permdb", UtilMethods); +} diff --git a/c_src/permdbtest.py b/c_src/permdbtest.py new file mode 100644 index 0000000..58debe7 --- /dev/null +++ b/c_src/permdbtest.py @@ -0,0 +1,103 @@ +import argparse +import struct +import sys +import os +import random +import datetime +import permdb +import hashlib + +parser = argparse.ArgumentParser(description="") +parser.add_argument('--store', help="Store", required=True) +parser.add_argument('--fsync', metavar="n", type=int, help="Fsync every n adds", required=False) +parser.add_argument('--startrand', metavar="n", type=int, default=0, help="Start position in random sequence", required=False) +parser.add_argument("--remove", action='store_true', help="Remove database before starting test") +parser.add_argument('--datasize', metavar="n", type=int, default=32, help="Start position in random sequence", required=False) +parser.add_argument('testentries', type=int, help="Number of entries to insert") +args = parser.parse_args() + +if args.remove: + os.remove(args.store) + os.remove(args.store+".idx") +permdbobj = permdb.alloc(args.store) +q = 2 + +def timing_point(timer_dict=None, name=None): + t = datetime.datetime.now() + if timer_dict: + starttime = timer_dict["lasttime"] + stoptime = t + deltatime = stoptime - starttime + deltaseconds = deltatime.seconds + float(deltatime.microseconds) / 1000000 + print name, deltaseconds + timer_dict["deltatimes"].append((name, deltaseconds)) + timer_dict["lasttime"] = t + return None + else: + timer_dict = {"deltatimes":[], "lasttime":t} + return timer_dict + +def getsize(): + return permdb.datasize(permdbobj) + + +def main(): + permdb.addvalue(permdbobj, b"\xAB\xCDABCDEFGHIJKLMNOPQRSTUVWXYZ1234", b"hej") + permdb.clear_nodecache(permdbobj) + permdb.addvalue(permdbobj, b"\xAB\xCDABCDEFGHIJKLMNOPQRSTUVWXYZ1234", b"hej") + assert permdb.getvalue(permdbobj, b"\xAB\xCDABCDEFGHIJKLMNOPQRSTUVWXYZ1234") == b"hej" + assert permdb.getvalue(permdbobj, b"\xAB\xCEABCDEFGHIJKLMNOPQRSTUVWXYZ1234") == None + permdb.addvalue(permdbobj, b"\xAB\x12ABCDEFGHIJKLMNOPQRSTUVWXYZ1234", b"hej1") + permdb.addvalue(permdbobj, b"\xAC\x52ABCDEFGHIJKLMNOPQRSTUVWXYZ1234", b"hej2") + permdb.addvalue(permdbobj, b"\x9A\x43ABCDEFGHIJKLMNOPQRSTUVWXYZ1234", b"hej3") + permdb.committree(permdbobj) + permdb.clear_nodecache(permdbobj) + assert permdb.getvalue(permdbobj, b"\xAB\xCDABCDEFGHIJKLMNOPQRSTUVWXYZ1234") == b"hej" + assert permdb.getvalue(permdbobj, b"\xAB\xCEABCDEFGHIJKLMNOPQRSTUVWXYZ1234") == None + assert permdb.getvalue(permdbobj, b"\xAB\x12ABCDEFGHIJKLMNOPQRSTUVWXYZ1234") == b"hej1" + assert permdb.getvalue(permdbobj, b"\xAC\x52ABCDEFGHIJKLMNOPQRSTUVWXYZ1234") == b"hej2" + assert permdb.getvalue(permdbobj, b"\x9A\x43ABCDEFGHIJKLMNOPQRSTUVWXYZ1234") == b"hej3" + + print "generating test data" + timing = timing_point() + testdata = [(hashlib.sha256(struct.pack(">II", i, 0)).digest(), hashlib.sha256(struct.pack(">II", i, 1)).digest()) for i in range(args.startrand, args.testentries+args.startrand)] + timing_point(timing, "gendata2") + print "inserting test data" + written_since_fsync = 0 + datamultiplier = args.datasize // 32 + if args.datasize % 32 != 0: + print "datasize", args.datasize, "not multiple of 32, truncating to", datamultiplier * 32 + for (k, v) in testdata: + permdb.addvalue(permdbobj, k, v * datamultiplier) + written_since_fsync += 1 + if args.fsync and written_since_fsync >= args.fsync: + permdb.committree(permdbobj) + written_since_fsync = 0 + permdb.committree(permdbobj) + timing_point(timing, "insert") + nentries = args.testentries + print "reading test data" + for (k, v) in testdata: + assert permdb.getvalue(permdbobj, k) == v * datamultiplier + timing_point(timing, "read1") + random.shuffle(testdata) + timing_point(timing, "shuffle") + permdb.clear_nodecache(permdbobj) + for (k, v) in testdata: + assert permdb.getvalue(permdbobj, k) == v * datamultiplier + timing_point(timing, "read2") + if getsize() > 1024*1024: + print "db size %sM" % (getsize() / (1024*1024),) + else: + print "db size", getsize() + print "db size/entry", getsize()/nentries + print "data file size %sM, entry size %d" % (os.stat(args.store).st_size / (1024*1024), datamultiplier * 32) + print "q", q, "entries", nentries, "fsync", args.fsync + print timing["deltatimes"] + timingdict = dict(timing["deltatimes"]) + print len(testdata)/timingdict["insert"], "writeops/s", "(%f microseconds)" % (timingdict["insert"]*1000000/len(testdata)) + print len(testdata)/timingdict["read1"], "cached readops/s", "(%f microseconds)" % (timingdict["read1"]*1000000/len(testdata)) + print len(testdata)/timingdict["read2"], "uncached readops/s", "(%f microseconds)" % (timingdict["read2"]*1000000/len(testdata)) + permdb.clear_nodecache(permdbobj) + +main() |