Refactor marshalling

Add chunking changes to listpermdb.erl

1 files changed, 54 insertions, 50 deletions

diff --git a/c_src/permdb.c b/c_src/permdb.c
index 624e1da..c5187be 100644
--- a/c_src/permdb.c
+++ b/c_src/permdb.c
@@ -19,7 +19,7 @@
 #include "permdb.h"
 #include "hash.h"
 
-#define INDEX_COMMIT_TRAILER_SIZE (8 + SHA256_DIGEST_SIZE + 8)
+#define INDEX_COMMIT_TRAILER_SIZE (sizeof(uint64_t) + SHA256_DIGEST_SIZE + sizeof(index_commit_cookie))
 
 static const int bitsperlevel = 2;
 static const int keylen = 32;
@@ -256,16 +256,31 @@ indexfile_add_header(buffered_file *file)
         return length;
 }
 
+static void
+writebuffer_add_host64(buffered_file *file, uint64_t value) {
+        writebuffer_add(file, &value, sizeof(uint64_t));
+}
+
+static void
+writebuffer_add_be32(buffered_file *file, uint32_t value) {
+        uint32_t value_be = htonl(value);
+        writebuffer_add(file, &value_be, sizeof(uint32_t));
+}
+
+static void
+writebuffer_add_be16(buffered_file *file, uint16_t value) {
+        uint16_t value_be = htons(value);
+        writebuffer_add(file, &value_be, sizeof(uint16_t));
+}
+
 node_offset
 datafile_add_header(buffered_file *file)
 {
         fprintf(stderr, "adding header to %s\n", file->name);
-        uint32_t parameters[3];
         writebuffer_add(file, &data_file_cookie, sizeof(data_file_cookie));
-        parameters[0] = htonl(4096);
-        parameters[1] = htonl(2);
-        parameters[2] = htonl(32);
-        writebuffer_add(file, parameters, sizeof(parameters));
+        writebuffer_add_be32(file, 4096);
+        writebuffer_add_be32(file, 2);
+        writebuffer_add_be32(file, 32);
         uint64_t length = writebuffer_length(file);
         writebuffer_flush(file);
         return length;
@@ -340,6 +355,16 @@ validate_checksum(struct commit_info *commit, buffered_file *file)
         return -1;
 }
 
+static uint64_t
+read_host64(void *ptr)
+{
+        uint64_t data;
+
+        memcpy(&data, ptr, sizeof(data));
+
+        return data;
+}
+
 int
 verify_index_commit(buffered_file *file, node_offset offset)
 {
@@ -353,10 +378,8 @@ verify_index_commit(buffered_file *file, node_offset offset)
                 return -1;
         }
         struct commit_info commit;
-        uint64_t length;
-        memcpy(&length, data, sizeof(uint64_t));
-        commit.length = length;
-        commit.start = offset - commit.length;
+        commit.length = read_host64(data);
+        commit.start = offset + sizeof(uint64_t) - commit.length;
         memcpy(commit.checksum, data + sizeof(uint64_t), SHA256_DIGEST_SIZE);
 
         free(data);
@@ -411,7 +434,7 @@ datafile_verify_file(buffered_file *file)
 
 
 static uint32_t
-readnet32(void *ptr);
+read_be32(void *ptr);
 
 static unsigned char *
 readdatakeyandlen(permdb_object *state, node_offset offset, size_t *datalen);
@@ -428,7 +451,7 @@ read_data_commit(buffered_file *file, node_offset *offset)
         struct commit_info *commit = malloc(sizeof(struct commit_info));
         //fprintf(stderr, "read commit: %llu\n", *offset);
         //print_hex(data, sizeof(uint32_t) + SHA256_DIGEST_SIZE);
-        commit->length = readnet32(data);
+        commit->length = read_be32(data);
         commit->start = *offset - commit->length;
         memcpy(&commit->checksum, data + sizeof(uint32_t), SHA256_DIGEST_SIZE);
         *offset += SHA256_DIGEST_SIZE + sizeof(data_commit_end_cookie);
@@ -710,16 +733,6 @@ keypart(const unsigned char *key, unsigned int level)
     return (char *)result;
 }
 
-static char *
-packnode(node_object node)
-{
-        char *data = malloc(sizeof(index_node_cookie) + sizeof(node_object));
-        memcpy(data, &index_node_cookie, sizeof(index_node_cookie));
-        memcpy(data+sizeof(index_node_cookie), &node, sizeof(node_object));
-
-        return data;
-}
-
 static void
 addentry(node_object *node, unsigned int n, node_entry entry)
 {
@@ -1023,14 +1036,11 @@ writenode(permdb_object *state, node_object node, const char *cachekey)
 
         put_node_in_cache(state, cachekey, node);
 
-        char *data = packnode(node);
-
 #if DEBUG_WRITE
         fprintf(stderr, "writing node: offset %llu\n", offset);
 #endif
-        writebuffer_add(&state->indexfile, data, sizeof(index_node_cookie) + sizeof(node_object));
-
-        free(data);
+        writebuffer_add(&state->indexfile, &index_node_cookie, sizeof(index_node_cookie));
+        writebuffer_add(&state->indexfile, &node, sizeof(node_object));
 
         return offset;
 }
@@ -1078,7 +1088,7 @@ readdatakey(permdb_object *state, node_offset offset)
 }
 
 static uint32_t
-readnet32(void *ptr)
+read_be32(void *ptr)
 {
         uint32_t data;
 
@@ -1088,7 +1098,7 @@ readnet32(void *ptr)
 }
 
 static uint16_t
-readnet16(void *ptr)
+read_be16(void *ptr)
 {
         uint16_t data;
 
@@ -1110,8 +1120,8 @@ readdatakeyandlen(permdb_object *state, node_offset offset, size_t *datalen)
                 return NULL;
         }
         unsigned char *result = memsub(data, sizeof(data_entry_cookie), keylen);
-        uint16_t nchunks = readnet16(data+sizeof(data_entry_cookie)+keylen);
-        *datalen = readnet16(data+sizeof(data_entry_cookie)+keylen+sizeof(uint16_t));
+        uint16_t nchunks = read_be16(data+sizeof(data_entry_cookie)+keylen);
+        *datalen = read_be16(data+sizeof(data_entry_cookie)+keylen+sizeof(uint16_t));
         if (nchunks != 1) {
                 errx(1, "number of chunks is %d, but only one chunk is supported right now", nchunks);
         }
@@ -1132,8 +1142,6 @@ writedata(permdb_object *state, const unsigned char *key, const unsigned char *d
         if (datalength > 65535) {
                 errx(1, "data length is %zu, but only < 64K lengths are supported right now", datalength);
         }
-        uint16_t chunk_length = htons(datalength);
-        uint16_t nchunks = htons(1);
         node_offset offset = state->datafile.datasize;
 
 #if DEBUG_WRITE
@@ -1141,8 +1149,8 @@ writedata(permdb_object *state, const unsigned char *key, const unsigned char *d
 #endif
         writebuffer_add(&state->datafile, &data_entry_cookie, sizeof(data_entry_cookie));
         writebuffer_add(&state->datafile, key, keylen);
-        writebuffer_add(&state->datafile, &nchunks, sizeof(uint16_t));
-        writebuffer_add(&state->datafile, &chunk_length, sizeof(uint16_t));
+        writebuffer_add_be16(&state->datafile, 1);
+        writebuffer_add_be16(&state->datafile, datalength);
         writebuffer_add(&state->datafile, data, datalength);
 
         return offset;
@@ -1474,21 +1482,18 @@ committree(permdb_object *state)
 #endif
 
         int data_commit_padding_size = calc_padding(state->datafile.datasize, 4);
-        int data_commit_trailer_size = SHA256_DIGEST_SIZE + 8;
-        unsigned char *data_commit_trailer = malloc(data_commit_trailer_size);
         uint8_t padding[4] = {0, 0, 0, 0};
+        unsigned char data_commit_checksum[SHA256_DIGEST_SIZE];
         writebuffer_add(&state->datafile, data_commit_start_cookie, 8);
         writebuffer_add(&state->datafile, padding, data_commit_padding_size);
-        uint32_t data_commit_length = htonl(state->datafile.datasize - state->datafile.lastcommit + sizeof(uint32_t));
-        writebuffer_add(&state->datafile, &data_commit_length, sizeof(uint32_t));
-        sha256_digest(&state->datafile.commit_checksum_context, SHA256_DIGEST_SIZE, data_commit_trailer);
-        memcpy(data_commit_trailer + SHA256_DIGEST_SIZE, &data_commit_end_cookie, sizeof(data_commit_end_cookie));
-        writebuffer_add(&state->datafile, data_commit_trailer, data_commit_trailer_size);
+        writebuffer_add_be32(&state->datafile, state->datafile.datasize - state->datafile.lastcommit + sizeof(uint32_t));
+        sha256_digest(&state->datafile.commit_checksum_context, SHA256_DIGEST_SIZE, data_commit_checksum);
+        writebuffer_add(&state->datafile, data_commit_checksum, SHA256_DIGEST_SIZE);
+        writebuffer_add(&state->datafile, &data_commit_end_cookie, sizeof(data_commit_end_cookie));
 
 #if DEBUG_WRITE
         fprintf(stderr, "finished writing data commit trailer at offset %llu\n", state->datafile.datasize);
 #endif
-        free(data_commit_trailer);
 
         if (writebuffer_flush(&state->datafile) == -1) {
                 set_error(state, "data file flushing failed\n");
@@ -1501,17 +1506,16 @@ committree(permdb_object *state)
         fprintf(stderr, "writing index commit trailer at offset %llu\n", state->indexfile.datasize);
 #endif
 
-        uint64_t index_commit_length = state->indexfile.datasize - state->indexfile.lastcommit;
-        unsigned char *index_commit_trailer = malloc(INDEX_COMMIT_TRAILER_SIZE);
-        memcpy(index_commit_trailer, &index_commit_length, 8);
-        sha256_digest(&state->indexfile.commit_checksum_context, SHA256_DIGEST_SIZE, index_commit_trailer + 8);
-        memcpy(index_commit_trailer + 8 + SHA256_DIGEST_SIZE, &index_commit_cookie, 8);
-        writebuffer_add(&state->indexfile, index_commit_trailer, INDEX_COMMIT_TRAILER_SIZE);
+        uint64_t index_commit_length = state->indexfile.datasize - state->indexfile.lastcommit + sizeof(uint64_t);
+        unsigned char index_commit_checksum[SHA256_DIGEST_SIZE];
+        writebuffer_add_host64(&state->indexfile, index_commit_length);
+        sha256_digest(&state->indexfile.commit_checksum_context, SHA256_DIGEST_SIZE, index_commit_checksum);
+        writebuffer_add(&state->indexfile, index_commit_checksum, SHA256_DIGEST_SIZE);
+        writebuffer_add(&state->indexfile, &index_commit_cookie, sizeof(index_commit_cookie));
 
 #if DEBUG_WRITE
         fprintf(stderr, "finished writing index commit trailer at offset %llu\n", state->indexfile.datasize);
 #endif
-        free(index_commit_trailer);
 
         if (writebuffer_flush(&state->indexfile) == -1) {
                 set_error(state, "index file flushing failed\n");