From 8826eb502c73df3a512a2d257f4264d68a10e1c8 Mon Sep 17 00:00:00 2001
From: Magnus Ahltorp <map@kth.se>
Date: Tue, 22 Nov 2016 14:31:37 +0100
Subject: filebuffer: continue reading after short read permdb: only read small
 chunks when verifying commit

---
 c_src/filebuffer.c | 36 ++++++++++++++++++++++++++----------
 c_src/permdb.c     | 45 ++++++++++++++++++++++++++++++++++-----------
 2 files changed, 60 insertions(+), 21 deletions(-)

(limited to 'c_src')

diff --git a/c_src/filebuffer.c b/c_src/filebuffer.c
index 1e4f89d..921ba37 100644
--- a/c_src/filebuffer.c
+++ b/c_src/filebuffer.c
@@ -15,6 +15,7 @@
 #include <fcntl.h>
 #include <err.h>
 #include <sys/file.h>
+#include <errno.h>
 #include <nettle/sha2.h>
 
 #include "permdb.h"
@@ -140,8 +141,8 @@ unsigned char *
 bf_read(buffered_file *file, uint64_t offset, size_t length, char **error)
 {
     unsigned char *result = malloc(length);
-    dprintf(READ, (stderr, "reading data: offset %llu\n",
-                   (unsigned long long) offset));
+    dprintf(READ, (stderr, "reading data: offset %llu length %llu\n",
+		   (unsigned long long) offset, (unsigned long long) length));
 
     if (offset >= file->filesize) {
             uint64_t writebufferoffset = offset - file->filesize;
@@ -164,14 +165,28 @@ bf_read(buffered_file *file, uint64_t offset, size_t length, char **error)
                     return NULL;
             }
 
-            ssize_t ret = pread(file->fd, result, length, (off_t) offset);
-            if (ret != length) {
-                    free(result);
-                    set_error(error,
-                              "short pread: %zd (wanted %zu) at offset %llu\n",
-                              ret, length, (long long unsigned int) offset);
-                    return NULL;
-            }
+	    size_t bytes_read = 0;
+	    while (length > 0) {
+		    ssize_t ret = pread(file->fd, result + bytes_read, length, (off_t) offset + bytes_read);
+		    dprintf(READ, (stderr, "pread: offset %llu length %llu ret %zu\n",
+				   (unsigned long long) offset + bytes_read, (unsigned long long) length, ret));
+
+		    if (ret == 0) {
+			    free(result);
+			    set_error(error,
+				      "eof reading %zu bytes at offset %llu\n",
+				      length, (long long unsigned int) offset);
+			    return NULL;
+		    } else if (ret < 0) {
+			    free(result);
+			    set_error(error,
+				      "error %d reading %zu bytes at offset %llu\n",
+				      errno, length, (long long unsigned int) offset);
+			    return NULL;
+		    }
+		    bytes_read += ret;
+		    length -= ret;
+	    }
     }
 
     return result;
@@ -219,6 +234,7 @@ void
 bf_close(buffered_file *file)
 {
         bf_flush(file);
+	flock(file->fd, LOCK_UN);
         close(file->fd);
         free(file->writebuffer);
         free(file);
diff --git a/c_src/permdb.c b/c_src/permdb.c
index 96dc405..a33c43a 100644
--- a/c_src/permdb.c
+++ b/c_src/permdb.c
@@ -15,6 +15,7 @@
 #include <fcntl.h>
 #include <err.h>
 #include <nettle/sha2.h>
+#include <sys/param.h>
 #include "erlport.h"
 #include "permdb.h"
 #include "filebuffer.h"
@@ -207,24 +208,40 @@ validate_checksum(struct commit_info *commit, buffered_file *file)
                  (long long unsigned) commit->length,
                  (long long unsigned) commit->start));
 
-        unsigned char *checksumdata =
-                bf_read(file, commit->start, commit->length, NULL);
-        if (checksumdata == NULL) {
-                return -1;
-        }
+	char *error = NULL;
 
         uint8_t checksum[SHA256_DIGEST_SIZE];
         struct sha256_ctx commit_checksum_context;
         sha256_init(&commit_checksum_context);
-        sha256_update(&commit_checksum_context, commit->length, checksumdata);
-        sha256_digest(&commit_checksum_context, SHA256_DIGEST_SIZE, checksum);
 
-        if (memcmp(checksum, commit->checksum, SHA256_DIGEST_SIZE) == 0) {
+	node_offset offset = commit->start;
+	node_offset bytesleft = commit->length;
+	node_offset chunksize = 1024*1024;
+
+	while (bytesleft > 0) {
+		node_offset length = MIN(chunksize, bytesleft);
+		unsigned char *checksumdata =
+			bf_read(file, offset, length, &error);
+		if (checksumdata == NULL) {
+			dprintf(READ,
+				(stderr,
+				 "validate_checksum: could not read file: %s\n", error));
+			return -1;
+		}
+
+		sha256_update(&commit_checksum_context, length, checksumdata);
+
+		offset += length;
+		bytesleft -= length;
                 free(checksumdata);
+	}
+
+	sha256_digest(&commit_checksum_context, SHA256_DIGEST_SIZE, checksum);
+
+        if (memcmp(checksum, commit->checksum, SHA256_DIGEST_SIZE) == 0) {
                 return 0;
         }
 
-        free(checksumdata);
         return -1;
 }
 
@@ -294,6 +311,12 @@ datafile_verify_file(buffered_file *file)
         struct commit_info *data_commit =
                 read_data_commit_backward(file, &offset);
 
+	if (data_commit == NULL) {
+		dprintf(READ, (stderr, "did not find a commit\n"));
+	} else {
+		dprintf(READ, (stderr, "last commit: start %llu length %llu\n", (long long unsigned) data_commit->start, (long long unsigned) data_commit->length));
+	}
+
         if (data_commit == NULL || validate_checksum(data_commit, file) < 0) {
                 fprintf(stderr, "commit broken: %llu\n",
                         (long long unsigned) offset);
@@ -589,8 +612,8 @@ unsigned char *
 read_internal_data(permdb_object *state, node_offset offset, size_t length)
 {
         buffered_file *file = state->datafile;
-        dprintf(READ, (stderr, "reading data: offset %llu\n",
-                       (unsigned long long) offset));
+        dprintf(READ, (stderr, "reading data: offset %llu length %llu\n",
+                       (unsigned long long) offset, (unsigned long long) length));
 
         return bf_read(file, offset, length, &state->error);
 }
-- 
cgit v1.1