1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
|
# Copyright (c) 2016, NORDUnet A/S.
# See LICENSE for licensing information.
import argparse
import struct
import sys
import os
import random
import datetime
import permdb
import hashlib
parser = argparse.ArgumentParser(description="")
parser.add_argument('--store', help="Store", required=True)
parser.add_argument('--fsync', metavar="n", type=int, help="Fsync every n adds", required=False)
parser.add_argument('--startrand', metavar="n", type=int, default=0, help="Start position in random sequence", required=False)
parser.add_argument("--remove", action='store_true', help="Remove database before starting test")
parser.add_argument('--datasize', metavar="n", type=int, default=32, help="Start position in random sequence", required=False)
parser.add_argument('testentries', type=int, help="Number of entries to insert")
args = parser.parse_args()
if args.remove:
os.remove(args.store)
os.remove(args.store+".idx")
permdbobj = permdb.alloc(args.store)
q = 2
def timing_point(timer_dict=None, name=None):
t = datetime.datetime.now()
if timer_dict:
starttime = timer_dict["lasttime"]
stoptime = t
deltatime = stoptime - starttime
deltaseconds = deltatime.seconds + float(deltatime.microseconds) / 1000000
print name, deltaseconds
timer_dict["deltatimes"].append((name, deltaseconds))
timer_dict["lasttime"] = t
return None
else:
timer_dict = {"deltatimes":[], "lasttime":t}
return timer_dict
def getsize():
return permdb.datasize(permdbobj)
def main():
permdb.addvalue(permdbobj, b"\xAB\xCDABCDEFGHIJKLMNOPQRSTUVWXYZ1234", b"hej")
permdb.clear_nodecache(permdbobj)
permdb.addvalue(permdbobj, b"\xAB\xCDABCDEFGHIJKLMNOPQRSTUVWXYZ1234", b"hej")
assert permdb.getvalue(permdbobj, b"\xAB\xCDABCDEFGHIJKLMNOPQRSTUVWXYZ1234") == b"hej"
assert permdb.getvalue(permdbobj, b"\xAB\xCEABCDEFGHIJKLMNOPQRSTUVWXYZ1234") == None
permdb.addvalue(permdbobj, b"\xAB\x12ABCDEFGHIJKLMNOPQRSTUVWXYZ1234", b"hej1")
permdb.addvalue(permdbobj, b"\xAC\x52ABCDEFGHIJKLMNOPQRSTUVWXYZ1234", b"hej2")
permdb.addvalue(permdbobj, b"\x9A\x43ABCDEFGHIJKLMNOPQRSTUVWXYZ1234", b"hej3")
permdb.committree(permdbobj)
permdb.clear_nodecache(permdbobj)
assert permdb.getvalue(permdbobj, b"\xAB\xCDABCDEFGHIJKLMNOPQRSTUVWXYZ1234") == b"hej"
assert permdb.getvalue(permdbobj, b"\xAB\xCEABCDEFGHIJKLMNOPQRSTUVWXYZ1234") == None
assert permdb.getvalue(permdbobj, b"\xAB\x12ABCDEFGHIJKLMNOPQRSTUVWXYZ1234") == b"hej1"
assert permdb.getvalue(permdbobj, b"\xAC\x52ABCDEFGHIJKLMNOPQRSTUVWXYZ1234") == b"hej2"
assert permdb.getvalue(permdbobj, b"\x9A\x43ABCDEFGHIJKLMNOPQRSTUVWXYZ1234") == b"hej3"
print "generating test data"
timing = timing_point()
testdata = [(hashlib.sha256(struct.pack(">II", i, 0)).digest(), hashlib.sha256(struct.pack(">II", i, 1)).digest()) for i in range(args.startrand, args.testentries+args.startrand)]
timing_point(timing, "gendata2")
print "inserting test data"
written_since_fsync = 0
datamultiplier = args.datasize // 32
if args.datasize % 32 != 0:
print "datasize", args.datasize, "not multiple of 32, truncating to", datamultiplier * 32
for (k, v) in testdata:
permdb.addvalue(permdbobj, k, v * datamultiplier)
written_since_fsync += 1
if args.fsync and written_since_fsync >= args.fsync:
permdb.committree(permdbobj)
written_since_fsync = 0
permdb.committree(permdbobj)
timing_point(timing, "insert")
nentries = args.testentries
print "reading test data"
for (k, v) in testdata:
assert permdb.getvalue(permdbobj, k) == v * datamultiplier
timing_point(timing, "read1")
random.shuffle(testdata)
timing_point(timing, "shuffle")
permdb.clear_nodecache(permdbobj)
for (k, v) in testdata:
assert permdb.getvalue(permdbobj, k) == v * datamultiplier
timing_point(timing, "read2")
if getsize() > 1024*1024:
print "db size %sM" % (getsize() / (1024*1024),)
else:
print "db size", getsize()
print "db size/entry", getsize()/nentries
print "data file size %sM, entry size %d" % (os.stat(args.store).st_size / (1024*1024), datamultiplier * 32)
print "q", q, "entries", nentries, "fsync", args.fsync
print timing["deltatimes"]
timingdict = dict(timing["deltatimes"])
print len(testdata)/timingdict["insert"], "writeops/s", "(%f microseconds)" % (timingdict["insert"]*1000000/len(testdata))
print len(testdata)/timingdict["read1"], "cached readops/s", "(%f microseconds)" % (timingdict["read1"]*1000000/len(testdata))
print len(testdata)/timingdict["read2"], "uncached readops/s", "(%f microseconds)" % (timingdict["read2"]*1000000/len(testdata))
permdb.clear_nodecache(permdbobj)
main()
|
