Move hashing to top layer - dedup - deduplicating backup program
HTML git clone git://bitreich.org/dedup/ git://enlrupgkhuxnvlhsf6lc3fziv5h2hhfrinws65d7roiv6bfj7d652fid.onion/dedup/
DIR Log
DIR Files
DIR Refs
DIR Tags
DIR README
DIR LICENSE
---
DIR commit 4d8c3d50ca614a40275ad223fdfd31fd897b9627
DIR parent c3a040a2db7964be918a34acc35db912431757b1
HTML Author: sin <sin@2f30.org>
Date: Tue, 14 May 2019 13:05:50 +0300
Move hashing to top layer
The hash of the block should be the hash of the plaintext block. This
is because when encryption is enabled the final block is always
different even if it contained the same data originally. This is due
to the fact that each encrypted block contains a unique nonce.
Diffstat:
M bcompress.c | 9 ---------
M bencrypt.c | 9 ---------
M block.c | 58 ++++++++++++++++++++++++++++++-
M block.h | 1 -
M bstorage.c | 79 +------------------------------
5 files changed, 58 insertions(+), 98 deletions(-)
---
DIR diff --git a/bcompress.c b/bcompress.c
@@ -36,7 +36,6 @@ static int bcput(struct bctx *bctx, void *buf, size_t n, unsigned char *md);
static int bcget(struct bctx *bctx, unsigned char *md, void *buf, size_t *n);
static int bcrm(struct bctx *bctx, unsigned char *md);
static int bcgc(struct bctx *bctx);
-static int bccheck(struct bctx *bctx, unsigned char *md);
static int bcsync(struct bctx *bctx);
static int bcclose(struct bctx *bctx);
@@ -47,7 +46,6 @@ static struct bops bops = {
.get = bcget,
.rm = bcrm,
.gc = bcgc,
- .check = bccheck,
.sync = bcsync,
.close = bcclose,
};
@@ -318,13 +316,6 @@ bcgc(struct bctx *bctx)
}
static int
-bccheck(struct bctx *bctx, unsigned char *md)
-{
- return bencryptops()->check(bctx, md);
-
-}
-
-static int
bcsync(struct bctx *bctx)
{
return bencryptops()->sync(bctx);
DIR diff --git a/bencrypt.c b/bencrypt.c
@@ -34,7 +34,6 @@ static int beput(struct bctx *bctx, void *buf, size_t n, unsigned char *md);
static int beget(struct bctx *bctx, unsigned char *md, void *buf, size_t *n);
static int berm(struct bctx *bctx, unsigned char *md);
static int begc(struct bctx *bctx);
-static int becheck(struct bctx *bctx, unsigned char *md);
static int besync(struct bctx *bctx);
static int beclose(struct bctx *bctx);
@@ -45,7 +44,6 @@ static struct bops bops = {
.get = beget,
.rm = berm,
.gc = begc,
- .check = becheck,
.sync = besync,
.close = beclose,
};
@@ -325,13 +323,6 @@ begc(struct bctx *bctx)
}
static int
-becheck(struct bctx *bctx, unsigned char *md)
-{
- return bstorageops()->check(bctx, md);
-
-}
-
-static int
besync(struct bctx *bctx)
{
return bstorageops()->sync(bctx);
DIR diff --git a/block.c b/block.c
@@ -9,9 +9,18 @@
#include <stdlib.h>
#include <string.h>
+#include <sodium.h>
+
#include "block.h"
+#include "config.h"
#include "misc.h"
+static int
+bhash(void *buf, size_t n, unsigned char *md)
+{
+ return crypto_generichash(md, MDSIZE, buf, n, NULL, 0);
+}
+
int
bcreat(char *path, int mode, struct bctx **bctx)
{
@@ -20,6 +29,11 @@ bcreat(char *path, int mode, struct bctx **bctx)
return -1;
}
+ if (sodium_init() < 0) {
+ seterr("sodium_init: failed");
+ return -1;
+ }
+
*bctx = calloc(1, sizeof(**bctx));
if (*bctx == NULL) {
seterr("calloc: %s", strerror(errno));
@@ -41,6 +55,11 @@ bopen(char *path, int flags, int mode, struct bctx **bctx)
return -1;
}
+ if (sodium_init() < 0) {
+ seterr("sodium_init: failed");
+ return -1;
+ }
+
*bctx = calloc(1, sizeof(**bctx));
if (*bctx == NULL) {
seterr("calloc: %s", strerror(errno));
@@ -62,6 +81,11 @@ bput(struct bctx *bctx, void *buf, size_t n, unsigned char *md)
return -1;
}
+ if (bhash(buf, n, md) < 0) {
+ seterr("bhash: failed");
+ return -1;
+ }
+
return bcompressops()->put(bctx, buf, n, md);
}
@@ -98,15 +122,47 @@ bgc(struct bctx *bctx)
return bcompressops()->gc(bctx);
}
+/*
+ * Lookup the block given hash and rehash it.
+ * Check that the hashes match. It returns -1
+ * on error, 0 on success and 1 if a block hash
+ * mismatch is detected.
+ */
int
bcheck(struct bctx *bctx, unsigned char *md)
{
+ unsigned char tmp[MDSIZE];
+ void *buf;
+ size_t n;
+
if (bctx == NULL || md == NULL) {
seterr("invalid params");
return -1;
}
- return bcompressops()->check(bctx, md);
+ buf = malloc(BSIZEMAX);
+ if (buf == NULL) {
+ seterr("malloc: out of memory");
+ return -1;
+ }
+ n = BSIZEMAX;
+
+ if (bcompressops()->get(bctx, md, buf, &n) < 0) {
+ free(buf);
+ return -1;
+ }
+
+ if (bhash(buf, n, tmp) < 0) {
+ free(buf);
+ return -1;
+ }
+
+ if (memcmp(tmp, md, MDSIZE) != 0) {
+ free(buf);
+ return -1;
+ }
+ free(buf);
+ return 0;
}
int
DIR diff --git a/block.h b/block.h
@@ -21,7 +21,6 @@ struct bops {
int (*get)(struct bctx *, unsigned char *, void *, size_t *);
int (*rm)(struct bctx *, unsigned char *);
int (*gc)(struct bctx *);
- int (*check)(struct bctx *, unsigned char *);
int (*sync)(struct bctx *);
int (*close)(struct bctx *);
};
DIR diff --git a/bstorage.c b/bstorage.c
@@ -22,8 +22,6 @@
#include <strings.h>
#include <unistd.h>
-#include <sodium.h>
-
#include "block.h"
#include "config.h"
#include "misc.h"
@@ -56,7 +54,6 @@ static int bsput(struct bctx *, void *, size_t, unsigned char *);
static int bsget(struct bctx *, unsigned char *, void *, size_t *);
static int bsrm(struct bctx *, unsigned char *);
static int bsgc(struct bctx *);
-static int bscheck(struct bctx *, unsigned char *);
static int bssync(struct bctx *);
static int bsclose(struct bctx *);
@@ -67,7 +64,6 @@ static struct bops bops = {
.get = bsget,
.rm = bsrm,
.gc = bsgc,
- .check = bscheck,
.sync = bssync,
.close = bsclose,
};
@@ -116,12 +112,6 @@ bd_cmp(struct bd *b1, struct bd *b2)
static RB_PROTOTYPE(bdcache, bd, rbe, bd_cmp)
static RB_GENERATE(bdcache, bd, rbe, bd_cmp)
-static int
-bhash(void *buf, size_t n, unsigned char *md)
-{
- return crypto_generichash(md, MDSIZE, buf, n, NULL, 0);
-}
-
/* Unpack block header */
static int
unpackbhdr(unsigned char *buf, struct bhdr *bhdr)
@@ -283,11 +273,6 @@ bscreat(struct bctx *bctx, char *path, int mode)
struct bhdr *bhdr;
int fd;
- if (sodium_init() < 0) {
- seterr("sodium_init: failed");
- return -1;
- }
-
fd = open(path, O_RDWR | O_CREAT | O_EXCL, mode);
if (fd < 0) {
seterr("open: %s", strerror(errno));
@@ -342,11 +327,6 @@ bsopen(struct bctx *bctx, char *path, int flags, int mode)
return -1;
}
- if (sodium_init() < 0) {
- seterr("sodium_init: failed");
- return -1;
- }
-
fd = open(path, flags, mode);
if (fd < 0) {
seterr("open: %s", strerror(errno));
@@ -409,17 +389,13 @@ bsput(struct bctx *bctx, void *buf, size_t n, unsigned char *md)
struct bd key, *bd;
off_t offs;
- if (bhash(buf, n, key.md) < 0) {
- seterr("bhash: failed");
- return -1;
- }
-
/*
* If the block is already present in the cache
* just increment the reference count and write back
* the block descriptor associated for that block.
*/
sctx = bctx->sctx;
+ memcpy(key.md, md, MDSIZE);
bd = RB_FIND(bdcache, &sctx->bdcache, &key);
if (bd != NULL) {
off_t bdoffs;
@@ -605,59 +581,6 @@ bsgc(struct bctx *bctx)
return 0;
}
-/*
- * Lookup the block given hash and rehash it.
- * Check that the hashes match. It returns -1
- * on error, 0 on success and 1 if a block hash
- * mismatch is detected.
- */
-static int
-bscheck(struct bctx *bctx, unsigned char *md)
-{
- struct sctx *sctx;
- struct bd key, *bd;
- void *buf;
-
- sctx = bctx->sctx;
- memcpy(key.md, md, MDSIZE);
- bd = RB_FIND(bdcache, &sctx->bdcache, &key);
- if (bd == NULL) {
- seterr("unknown block");
- return -1;
- }
-
- buf = malloc(bd->size);
- if (buf == NULL) {
- seterr("malloc: %s", strerror(errno));
- return -1;
- }
-
- if (lseek(sctx->fd, bd->offset, SEEK_SET) < 0) {
- free(buf);
- seterr("lseek: %s", strerror(errno));
- return -1;
- }
-
- if (xread(sctx->fd, buf, bd->size) != bd->size) {
- free(buf);
- seterr("failed to read block: %s", strerror(errno));
- return -1;
- }
-
- if (bhash(buf, bd->size, key.md) < 0) {
- free(buf);
- seterr("bhash: failed");
- return -1;
- }
-
- if (memcmp(key.md, md, MDSIZE) != 0) {
- free(buf);
- return 1;
- }
- free(buf);
- return 0;
-}
-
/* Sync block header to storage file */
static int
bssync(struct bctx *bctx)