tutils.c - dedup - data deduplication program
HTML git clone git://bitreich.org/dedup/ git://hg6vgqziawt5s4dj.onion/dedup/
DIR Log
DIR Files
DIR Refs
DIR Tags
DIR README
DIR LICENSE
---
tutils.c (5534B)
---
1 #include <sys/types.h>
2
3 #include <err.h>
4 #include <stdint.h>
5 #include <stdio.h>
6 #include <stdlib.h>
7 #include <string.h>
8 #include <unistd.h>
9
10 #include "blake2.h"
11 #include "dedup.h"
12
13 static void
14 match_ver(uint64_t v)
15 {
16 uint8_t maj, min;
17
18 min = v & VER_MIN_MASK;
19 maj = (v >> VER_MAJ_SHIFT) & VER_MAJ_MASK;
20 if (maj == VER_MAJ && min == VER_MIN)
21 return;
22 errx(1, "format version mismatch: expected %u.%u but got %u.%u",
23 VER_MAJ, VER_MIN, maj, min);
24 }
25
26 void
27 str2bin(char *s, uint8_t *d)
28 {
29 size_t i, size = strlen(s) / 2;
30
31 for (i = 0; i < size; i++, s += 2)
32 sscanf(s, "%2hhx", &d[i]);
33 }
34
35 off_t
36 xlseek(int fd, off_t offset, int whence)
37 {
38 off_t ret;
39
40 ret = lseek(fd, offset, whence);
41 if (ret < 0)
42 err(1, "lseek");
43 return ret;
44 }
45
46 ssize_t
47 xread(int fd, void *buf, size_t nbytes)
48 {
49 uint8_t *bp = buf;
50 ssize_t total = 0;
51
52 while (nbytes > 0) {
53 ssize_t n;
54
55 n = read(fd, &bp[total], nbytes);
56 if (n < 0)
57 err(1, "read");
58 else if (n == 0)
59 return total;
60 total += n;
61 nbytes -= n;
62 }
63 return total;
64 }
65
66 ssize_t
67 xwrite(int fd, const void *buf, size_t nbytes)
68 {
69 const uint8_t *bp = buf;
70 ssize_t total = 0;
71
72 while (nbytes > 0) {
73 ssize_t n;
74
75 n = write(fd, &bp[total], nbytes);
76 if (n < 0)
77 err(1, "write");
78 else if (n == 0)
79 return total;
80 total += n;
81 nbytes -= n;
82 }
83 return total;
84 }
85
86 void
87 init_blk_hdr(struct blk_hdr *hdr, int compr_algo, int hash_algo)
88 {
89 hdr->flags = (VER_MAJ << VER_MAJ_SHIFT) | VER_MIN;
90 hdr->flags |= compr_algo << COMPR_ALGO_SHIFT;
91 hdr->flags |= hash_algo << HASH_ALGO_SHIFT;
92 hdr->size = BLK_HDR_SIZE;
93 }
94
95 void
96 init_snap_hdr(struct snap_hdr *hdr)
97 {
98 hdr->flags = (VER_MAJ << VER_MAJ_SHIFT) | VER_MIN;
99 hdr->size = SNAP_HDR_SIZE;
100 hdr->st.min_blk_size = UINT64_MAX;
101 }
102
103 void
104 load_blk_hdr(int fd, struct blk_hdr *hdr, int *compr_algo, int *hash_algo)
105 {
106 uint64_t v;
107
108 read_blk_hdr(fd, hdr);
109 match_ver(hdr->flags);
110
111 v = hdr->flags >> COMPR_ALGO_SHIFT;
112 v &= COMPR_ALGO_MASK;
113 *compr_algo = v;
114
115 if (*compr_algo < 0 || *compr_algo >= NR_COMPRS)
116 errx(1, "unsupported compression algorithm: %d", *compr_algo);
117
118 v = hdr->flags >> HASH_ALGO_SHIFT;
119 v &= HASH_ALGO_MASK;
120 *hash_algo = v;
121
122 if (*hash_algo < 0 || *hash_algo >= NR_HASHES)
123 errx(1, "unsupported hash algorithm: %d", *hash_algo);
124 }
125
126 void
127 load_snap_hdr(int fd, struct snap_hdr *hdr)
128 {
129 read_snap_hdr(fd, hdr);
130 match_ver(hdr->flags);
131 }
132
133 struct snap *
134 alloc_snap(void)
135 {
136 struct snap *snap;
137
138 snap = calloc(1, sizeof(*snap));
139 if (snap == NULL)
140 err(1, "%s", __func__);
141 return snap;
142 }
143
144 void
145 free_snap(struct snap *snap)
146 {
147 free(snap);
148 }
149
150 struct snap *
151 grow_snap(struct snap *snap, uint64_t nr_blk_descs)
152 {
153 size_t size;
154
155 if (nr_blk_descs > SIZE_MAX / sizeof(snap->blk_desc[0]))
156 errx(1, "%s: overflow", __func__);
157 size = nr_blk_descs * sizeof(snap->blk_desc[0]);
158
159 if (size > SIZE_MAX - sizeof(*snap))
160 errx(1, "%s: overflow", __func__);
161 size += sizeof(*snap);
162
163 snap = realloc(snap, size);
164 if (snap == NULL)
165 err(1, "%s", __func__);
166 return snap;
167 }
168
169 void
170 append_snap(int fd, struct snap_hdr *hdr, struct snap *snap)
171 {
172 if (snap->nr_blk_descs > UINT64_MAX / BLK_DESC_SIZE)
173 errx(1, "%s: overflow", __func__);
174 snap->size = snap->nr_blk_descs * BLK_DESC_SIZE;
175
176 if (snap->size > UINT64_MAX - SNAPSHOT_SIZE)
177 errx(1, "%s: overflow", __func__);
178 snap->size += SNAPSHOT_SIZE;
179
180 xlseek(fd, hdr->size, SEEK_SET);
181 write_snap(fd, snap);
182 write_snap_blk_descs(fd, snap);
183
184 if (hdr->size > UINT64_MAX - snap->size)
185 errx(1, "%s: overflow", __func__);
186 hdr->size += snap->size;
187
188 if (hdr->nr_snaps > UINT64_MAX - 1)
189 errx(1, "%s: overflow", __func__);
190 hdr->nr_snaps++;
191 }
192
193 /*
194 * The snapshot hash is calculated over the
195 * hash of its block descriptors.
196 */
197 void
198 hash_snap(struct snap *snap, uint8_t *md, int hash_algo)
199 {
200 struct hash_ctx ctx;
201 uint64_t i;
202
203 if (hash_init(&ctx, hash_algo, MD_SIZE) < 0)
204 errx(1, "hash_init failed");
205 for (i = 0; i < snap->nr_blk_descs; i++) {
206 struct blk_desc *blk_desc;
207
208 blk_desc = &snap->blk_desc[i];
209 hash_update(&ctx, blk_desc->md, sizeof(blk_desc->md));
210 }
211 hash_final(&ctx, md, MD_SIZE);
212 }
213
214 /* Walk through all snapshots and call fn() on each one */
215 void
216 walk_snap(int fd, struct snap_hdr *hdr,
217 int (*fn)(struct snap *, void *), void *arg)
218 {
219 uint64_t i;
220
221 xlseek(fd, SNAP_HDR_SIZE, SEEK_SET);
222 for (i = 0; i < hdr->nr_snaps; i++) {
223 struct snap *snap;
224 int ret;
225
226 snap = alloc_snap();
227 read_snap(fd, snap);
228 snap = grow_snap(snap, snap->nr_blk_descs);
229 read_snap_descs(fd, snap);
230
231 ret = (*fn)(snap, arg);
232 free_snap(snap);
233 if (ret == WALK_STOP)
234 break;
235 }
236 }
237
238 uint8_t *
239 alloc_buf(size_t size)
240 {
241 void *p;
242
243 p = calloc(1, size);
244 if (p == NULL)
245 err(1, "%s", __func__);
246 return p;
247 }
248
249 void
250 free_buf(uint8_t *buf)
251 {
252 free(buf);
253 }
254
255 void
256 read_blk(int fd, uint8_t *buf, struct blk_desc *blk_desc)
257 {
258 ssize_t n;
259
260 xlseek(fd, blk_desc->offset, SEEK_SET);
261 n = xread(fd, buf, blk_desc->size);
262 if (n == 0)
263 errx(1, "%s: unexpected EOF", __func__);
264 if (n != blk_desc->size)
265 errx(1, "%s: short read", __func__);
266 }
267
268 void
269 append_blk(int fd, struct blk_hdr *hdr, uint8_t *buf, struct blk_desc *blk_desc)
270 {
271 xlseek(fd, hdr->size, SEEK_SET);
272 xwrite(fd, buf, blk_desc->size);
273
274 if (hdr->size > UINT64_MAX - blk_desc->size)
275 errx(1, "%s: overflow", __func__);
276 hdr->size += blk_desc->size;
277 }
278
279 void
280 hash_blk(uint8_t *buf, size_t size, uint8_t *md, int hash_algo)
281 {
282 struct hash_ctx ctx;
283
284 if (hash_init(&ctx, hash_algo, MD_SIZE) < 0)
285 errx(1, "hash_init failed");
286 hash_update(&ctx, buf, size);
287 hash_final(&ctx, md, MD_SIZE);
288 }