bcompress.c - dedup - deduplicating backup program
HTML git clone git://bitreich.org/dedup/ git://enlrupgkhuxnvlhsf6lc3fziv5h2hhfrinws65d7roiv6bfj7d652fid.onion/dedup/
DIR Log
DIR Files
DIR Refs
DIR Tags
DIR README
DIR LICENSE
---
bcompress.c (6415B)
---
1 /* Compression layer implementation */
2 #include <sys/types.h>
3 #include <sys/stat.h>
4
5 #include <assert.h>
6 #include <fcntl.h>
7 #include <stdint.h>
8 #include <stdio.h>
9 #include <stdlib.h>
10 #include <string.h>
11 #include <strings.h>
12 #include <unistd.h>
13
14 #include <lz4.h>
15 #include <snappy-c.h>
16
17 #include "block.h"
18 #include "config.h"
19 #include "misc.h"
20 #include "state.h"
21
22 #define CDNONETYPE 0x200
23 #define CDSNAPPYTYPE 0x201
24 #define CDLZ4TYPE 0x202
25 #define CDSIZE (8 + 8)
26
27 extern struct param param;
28
29 extern int pack(unsigned char *, char *, ...);
30 extern int unpack(unsigned char *, char *, ...);
31
32 static int bccreat(struct bctx *bctx, char *path, int mode);
33 static int bcopen(struct bctx *bctx, char *path, int flags, int mode);
34 static int bcput(struct bctx *bctx, void *buf, size_t n, unsigned char *md);
35 static int bcget(struct bctx *bctx, unsigned char *md, void *buf, size_t *n);
36 static int bcrm(struct bctx *bctx, unsigned char *md);
37 static int bcgc(struct bctx *bctx);
38 static int bcsync(struct bctx *bctx);
39 static int bcclose(struct bctx *bctx);
40
41 static struct bops bops = {
42 .creat = bccreat,
43 .open = bcopen,
44 .put = bcput,
45 .get = bcget,
46 .rm = bcrm,
47 .gc = bcgc,
48 .sync = bcsync,
49 .close = bcclose,
50 };
51
52 /* Compression layer context */
53 struct cctx {
54 int type; /* compression algorithm type for new blocks */
55 };
56
57 /* Compression descriptor */
58 struct cd {
59 uint16_t type; /* compression algorithm type */
60 unsigned char reserved[6]; /* should be set to 0 when writing */
61 uint64_t size; /* size of compressed block */
62 };
63
64 /* Unpack compression descriptor */
65 static int
66 unpackcd(void *buf, struct cd *cd)
67 {
68 int n;
69
70 n = unpack(buf, "s'6q",
71 &cd->type,
72 cd->reserved,
73 &cd->size);
74
75 assert(n == CDSIZE);
76 return n;
77 }
78
79 /* Pack compression descriptor */
80 static int
81 packcd(void *buf, struct cd *cd)
82 {
83 int n;
84
85 n = pack(buf, "s'6q",
86 cd->type,
87 cd->reserved,
88 cd->size);
89
90 assert(n == CDSIZE);
91 return n;
92 }
93
94 static int
95 bccreat(struct bctx *bctx, char *path, int mode)
96 {
97 struct cctx *cctx;
98 int type;
99
100 if (strcasecmp(param.calgo, "none") == 0) {
101 type = CDNONETYPE;
102 } else if (strcasecmp(param.calgo, "snappy") == 0) {
103 type = CDSNAPPYTYPE;
104 } else if (strcasecmp(param.calgo, "lz4") == 0) {
105 type = CDLZ4TYPE;
106 } else {
107 seterr("invalid compression type: %s", param.calgo);
108 return -1;
109 }
110
111 bctx->cctx = calloc(1, sizeof(struct cctx));
112 if (bctx->cctx == NULL) {
113 seterr("calloc: out of memory");
114 return -1;
115 }
116 cctx = bctx->cctx;
117 cctx->type = type;
118
119 if (bencryptops()->creat(bctx, path, mode) < 0) {
120 free(cctx);
121 return -1;
122 }
123 return 0;
124 }
125
126 static int
127 bcopen(struct bctx *bctx, char *path, int flags, int mode)
128 {
129 struct cctx *cctx;
130 int type;
131
132 if (strcasecmp(param.calgo, "none") == 0) {
133 type = CDNONETYPE;
134 } else if (strcasecmp(param.calgo, "snappy") == 0) {
135 type = CDSNAPPYTYPE;
136 } else if (strcasecmp(param.calgo, "lz4") == 0) {
137 type = CDLZ4TYPE;
138 } else {
139 seterr("invalid compression type: %s", param.calgo);
140 return -1;
141 }
142
143 bctx->cctx = calloc(1, sizeof(struct cctx));
144 if (bctx->cctx == NULL) {
145 seterr("calloc: out of memory");
146 return -1;
147 }
148 cctx = bctx->cctx;
149 cctx->type = type;
150
151 if (bencryptops()->open(bctx, path, flags, mode) < 0) {
152 free(cctx);
153 return -1;
154 }
155 return 0;
156 }
157
158 static int
159 bcput(struct bctx *bctx, void *buf, size_t n, unsigned char *md)
160 {
161 struct cctx *cctx;
162 struct cd cd;
163 char *cbuf;
164 size_t cn;
165 int r;
166
167 /* Calculate compressed block size */
168 cctx = bctx->cctx;
169 switch (cctx->type) {
170 case CDNONETYPE:
171 cn = n;
172 break;
173 case CDSNAPPYTYPE:
174 cn = snappy_max_compressed_length(n);
175 break;
176 case CDLZ4TYPE:
177 cn = LZ4_compressBound(n);
178 break;
179 }
180
181 cbuf = malloc(CDSIZE + cn);
182 if (cbuf == NULL) {
183 seterr("malloc: out of memory");
184 return -1;
185 }
186
187 /* Compress block */
188 switch (cctx->type) {
189 case CDNONETYPE:
190 memcpy(&cbuf[CDSIZE], buf, cn);
191 break;
192 case CDSNAPPYTYPE:
193 if (snappy_compress(buf, n, &cbuf[CDSIZE], &cn) != SNAPPY_OK) {
194 free(cbuf);
195 seterr("snappy_compress: failed");
196 return -1;
197 }
198 break;
199 case CDLZ4TYPE:
200 r = LZ4_compress_default(buf, &cbuf[CDSIZE], n, cn);
201 if (r < 0) {
202 free(cbuf);
203 seterr("LZ4_compress_default: failed");
204 return -1;
205 }
206 cn = r;
207 break;
208 }
209
210 /* Prepare compression descriptor */
211 cd.type = cctx->type;
212 memset(cd.reserved, 0, sizeof(cd.reserved));
213 cd.size = cn;
214 /* Prepend compression descriptor */
215 packcd(cbuf, &cd);
216
217 if (bencryptops()->put(bctx, cbuf, CDSIZE + cn, md) < 0) {
218 free(cbuf);
219 return -1;
220 }
221
222 free(cbuf);
223 return cd.size;
224 }
225
226 static int
227 bcget(struct bctx *bctx, unsigned char *md, void *buf, size_t *n)
228 {
229 struct cd cd;
230 char *cbuf;
231 size_t cn, un, size;
232 int r;
233
234 /* Calculate maximum compressed block size */
235 size = *n;
236 cn = snappy_max_compressed_length(*n);
237 if (cn > size)
238 size = cn;
239 cn = LZ4_compressBound(*n);
240 if (cn > size)
241 size = cn;
242 size += CDSIZE;
243
244 cbuf = malloc(size);
245 if (cbuf == NULL) {
246 seterr("malloc: out of memory");
247 return -1;
248 }
249
250 if (bencryptops()->get(bctx, md, cbuf, &size) < 0) {
251 free(cbuf);
252 return -1;
253 }
254
255 unpackcd(cbuf, &cd);
256
257 /* Decompress block */
258 switch (cd.type) {
259 case CDNONETYPE:
260 un = cd.size;
261 if (*n < un) {
262 free(cbuf);
263 seterr("buffer too small");
264 return -1;
265 }
266 memcpy(buf, &cbuf[CDSIZE], un);
267 break;
268 case CDSNAPPYTYPE:
269 if (snappy_uncompressed_length(&cbuf[CDSIZE], cd.size,
270 &un) != SNAPPY_OK) {
271 free(cbuf);
272 seterr("snappy_uncompressed_length: failed");
273 return -1;
274 }
275
276 if (*n < un) {
277 free(cbuf);
278 seterr("buffer too small");
279 return -1;
280 }
281
282 if (snappy_uncompress(&cbuf[CDSIZE], cd.size, buf,
283 &un) != SNAPPY_OK) {
284 free(cbuf);
285 seterr("snappy_uncompress: failed");
286 return -1;
287 }
288 break;
289 case CDLZ4TYPE:
290 r = LZ4_decompress_safe(&cbuf[CDSIZE], buf, cd.size, *n);
291 if (r < 0) {
292 free(cbuf);
293 seterr("LZ4_decompress_safe: failed");
294 return -1;
295 }
296 un = r;
297 break;
298 }
299
300 free(cbuf);
301 *n = un;
302 return 0;
303 }
304
305 static int
306 bcrm(struct bctx *bctx, unsigned char *md)
307 {
308 return bencryptops()->rm(bctx, md);
309 }
310
311 static int
312 bcgc(struct bctx *bctx)
313 {
314 return bencryptops()->gc(bctx);
315 }
316
317 static int
318 bcsync(struct bctx *bctx)
319 {
320 return bencryptops()->sync(bctx);
321 }
322
323 static int
324 bcclose(struct bctx *bctx)
325 {
326 struct cctx *cctx = bctx->cctx;
327
328 free(cctx);
329 return bencryptops()->close(bctx);
330 }
331
332 struct bops *
333 bcompressops(void)
334 {
335 return &bops;
336 }