tvac.c - plan9port - [fork] Plan 9 from user space
HTML git clone git://src.adamsgaard.dk/plan9port
DIR Log
DIR Files
DIR Refs
DIR README
DIR LICENSE
---
tvac.c (16320B)
---
1 #include "stdinc.h"
2 #include "vac.h"
3 #include "dat.h"
4 #include "fns.h"
5
6 // TODO: qids
7
8 void
9 usage(void)
10 {
11 fprint(2, "vac [-imqsv] [-a archive.vac] [-b bsize] [-d old.vac] [-f new.vac] [-e exclude]... [-h host] file...\n");
12 threadexitsall("usage");
13 }
14
15 enum
16 {
17 BlockSize = 8*1024,
18 CacheSize = 128<<20,
19 };
20
21 struct
22 {
23 int nfile;
24 int ndir;
25 vlong data;
26 vlong skipdata;
27 int skipfiles;
28 } stats;
29
30 int qdiff;
31 int merge;
32 int verbose;
33 char *host;
34 VtConn *z;
35 VacFs *fs;
36 char *archivefile;
37 char *vacfile;
38
39 int vacmerge(VacFile*, char*);
40 void vac(VacFile*, VacFile*, char*, Dir*);
41 void vacstdin(VacFile*, char*);
42 VacFile *recentarchive(VacFs*, char*);
43
44 static u64int unittoull(char*);
45 static void warn(char *fmt, ...);
46 static void removevacfile(void);
47
48 #ifdef PLAN9PORT
49 /*
50 * We're between a rock and a hard place here.
51 * The pw library (getpwnam, etc.) reads the
52 * password and group files into an on-stack buffer,
53 * so if you have some huge groups, you overflow
54 * the stack. Because of this, the thread library turns
55 * it off by default, so that dirstat returns "14571" instead of "rsc".
56 * But for vac we want names. So cautiously turn the pwlibrary
57 * back on (see threadmain) and make the main thread stack huge.
58 */
59 extern int _p9usepwlibrary;
60 int mainstacksize = 4*1024*1024;
61
62 #endif
63 void
64 threadmain(int argc, char **argv)
65 {
66 int i, j, fd, n, printstats;
67 Dir *d;
68 char *s;
69 uvlong u;
70 VacFile *f, *fdiff;
71 VacFs *fsdiff;
72 int blocksize;
73 int outfd;
74 char *stdinname;
75 char *diffvac;
76 uvlong qid;
77
78 #ifdef PLAN9PORT
79 /* see comment above */
80 _p9usepwlibrary = 1;
81 #endif
82
83 fmtinstall('F', vtfcallfmt);
84 fmtinstall('H', encodefmt);
85 fmtinstall('V', vtscorefmt);
86
87 blocksize = BlockSize;
88 stdinname = nil;
89 printstats = 0;
90 fsdiff = nil;
91 diffvac = nil;
92
93 ARGBEGIN{
94 case 'V':
95 chattyventi++;
96 break;
97 case 'a':
98 archivefile = EARGF(usage());
99 break;
100 case 'b':
101 u = unittoull(EARGF(usage()));
102 if(u < 512)
103 u = 512;
104 blocksize = u;
105 break;
106 case 'd':
107 diffvac = EARGF(usage());
108 break;
109 case 'e':
110 excludepattern(EARGF(usage()));
111 break;
112 case 'f':
113 vacfile = EARGF(usage());
114 break;
115 case 'h':
116 host = EARGF(usage());
117 break;
118 case 'i':
119 stdinname = EARGF(usage());
120 break;
121 case 'm':
122 merge++;
123 break;
124 case 'q':
125 qdiff++;
126 break;
127 case 's':
128 printstats++;
129 break;
130 case 'v':
131 verbose++;
132 break;
133 case 'x':
134 loadexcludefile(EARGF(usage()));
135 break;
136 default:
137 usage();
138 }ARGEND
139
140 if(argc == 0 && !stdinname)
141 usage();
142
143 if(archivefile && (vacfile || diffvac)){
144 fprint(2, "cannot use -a with -f, -d\n");
145 usage();
146 }
147
148 z = vtdial(host);
149 if(z == nil)
150 sysfatal("could not connect to server: %r");
151 if(vtconnect(z) < 0)
152 sysfatal("vtconnect: %r");
153
154 // Setup:
155 // fs is the output vac file system
156 // f is directory in output vac to write new files
157 // fdiff is corresponding directory in existing vac
158 if(archivefile){
159 VacFile *fp;
160 char yyyy[5];
161 char mmdd[10];
162 char oldpath[40];
163 Tm tm;
164
165 fdiff = nil;
166 if((outfd = open(archivefile, ORDWR)) < 0){
167 if(access(archivefile, 0) >= 0)
168 sysfatal("open %s: %r", archivefile);
169 if((outfd = create(archivefile, OWRITE, 0666)) < 0)
170 sysfatal("create %s: %r", archivefile);
171 atexit(removevacfile); // because it is new
172 if((fs = vacfscreate(z, blocksize, CacheSize)) == nil)
173 sysfatal("vacfscreate: %r");
174 }else{
175 if((fs = vacfsopen(z, archivefile, VtORDWR, CacheSize)) == nil)
176 sysfatal("vacfsopen %s: %r", archivefile);
177 if((fdiff = recentarchive(fs, oldpath)) != nil){
178 if(verbose)
179 fprint(2, "diff %s\n", oldpath);
180 }else
181 if(verbose)
182 fprint(2, "no recent archive to diff against\n");
183 }
184
185 // Create yyyy/mmdd.
186 tm = *localtime(time(0));
187 snprint(yyyy, sizeof yyyy, "%04d", tm.year+1900);
188 fp = vacfsgetroot(fs);
189 if((f = vacfilewalk(fp, yyyy)) == nil
190 && (f = vacfilecreate(fp, yyyy, ModeDir|0555)) == nil)
191 sysfatal("vacfscreate %s: %r", yyyy);
192 vacfiledecref(fp);
193 fp = f;
194
195 snprint(mmdd, sizeof mmdd, "%02d%02d", tm.mon+1, tm.mday);
196 n = 0;
197 while((f = vacfilewalk(fp, mmdd)) != nil){
198 vacfiledecref(f);
199 n++;
200 snprint(mmdd+4, sizeof mmdd-4, ".%d", n);
201 }
202 f = vacfilecreate(fp, mmdd, ModeDir|0555);
203 if(f == nil)
204 sysfatal("vacfscreate %s/%s: %r", yyyy, mmdd);
205 vacfiledecref(fp);
206
207 if(verbose)
208 fprint(2, "archive %s/%s\n", yyyy, mmdd);
209 }else{
210 if(vacfile == nil)
211 outfd = 1;
212 else if((outfd = create(vacfile, OWRITE, 0666)) < 0)
213 sysfatal("create %s: %r", vacfile);
214 atexit(removevacfile);
215 if((fs = vacfscreate(z, blocksize, CacheSize)) == nil)
216 sysfatal("vacfscreate: %r");
217 f = vacfsgetroot(fs);
218
219 fdiff = nil;
220 if(diffvac){
221 if((fsdiff = vacfsopen(z, diffvac, VtOREAD, CacheSize)) == nil)
222 warn("vacfsopen %s: %r", diffvac);
223 else
224 fdiff = vacfsgetroot(fsdiff);
225 }
226 }
227
228 if(stdinname)
229 vacstdin(f, stdinname);
230 for(i=0; i<argc; i++){
231 // We can't use / and . and .. and ../.. as valid archive
232 // names, so expand to the list of files in the directory.
233 if(argv[i][0] == 0){
234 warn("empty string given as command-line argument");
235 continue;
236 }
237 cleanname(argv[i]);
238 if(strcmp(argv[i], "/") == 0
239 || strcmp(argv[i], ".") == 0
240 || strcmp(argv[i], "..") == 0
241 || (strlen(argv[i]) > 3 && strcmp(argv[i]+strlen(argv[i])-3, "/..") == 0)){
242 if((fd = open(argv[i], OREAD)) < 0){
243 warn("open %s: %r", argv[i]);
244 continue;
245 }
246 while((n = dirread(fd, &d)) > 0){
247 for(j=0; j<n; j++){
248 s = vtmalloc(strlen(argv[i])+1+strlen(d[j].name)+1);
249 strcpy(s, argv[i]);
250 strcat(s, "/");
251 strcat(s, d[j].name);
252 cleanname(s);
253 vac(f, fdiff, s, &d[j]);
254 }
255 free(d);
256 }
257 close(fd);
258 continue;
259 }
260 if((d = dirstat(argv[i])) == nil){
261 warn("stat %s: %r", argv[i]);
262 continue;
263 }
264 vac(f, fdiff, argv[i], d);
265 free(d);
266 }
267 if(fdiff)
268 vacfiledecref(fdiff);
269
270 /*
271 * Record the maximum qid so that vacs can be merged
272 * without introducing overlapping qids. Older versions
273 * of vac arranged that the root would have the largest
274 * qid in the file system, but we can't do that anymore
275 * (the root gets created first!).
276 */
277 if(_vacfsnextqid(fs, &qid) >= 0)
278 vacfilesetqidspace(f, 0, qid);
279 vacfiledecref(f);
280
281 /*
282 * Copy fsdiff's root block score into fs's slot for that,
283 * so that vacfssync will copy it into root.prev for us.
284 * Just nice documentation, no effect.
285 */
286 if(fsdiff)
287 memmove(fs->score, fsdiff->score, VtScoreSize);
288 if(vacfssync(fs) < 0)
289 fprint(2, "vacfssync: %r\n");
290
291 fprint(outfd, "vac:%V\n", fs->score);
292 atexitdont(removevacfile);
293 vacfsclose(fs);
294 vthangup(z);
295
296 if(printstats){
297 fprint(2,
298 "%d files, %d files skipped, %d directories\n"
299 "%lld data bytes written, %lld data bytes skipped\n",
300 stats.nfile, stats.skipfiles, stats.ndir, stats.data, stats.skipdata);
301 dup(2, 1);
302 packetstats();
303 }
304 threadexitsall(0);
305 }
306
307 VacFile*
308 recentarchive(VacFs *fs, char *path)
309 {
310 VacFile *fp, *f;
311 VacDirEnum *de;
312 VacDir vd;
313 char buf[10];
314 int year, mmdd, nn, n, n1;
315 char *p;
316
317 fp = vacfsgetroot(fs);
318 de = vdeopen(fp);
319 year = 0;
320 if(de){
321 for(; vderead(de, &vd) > 0; vdcleanup(&vd)){
322 if(strlen(vd.elem) != 4)
323 continue;
324 if((n = strtol(vd.elem, &p, 10)) < 1900 || *p != 0)
325 continue;
326 if(year < n)
327 year = n;
328 }
329 }
330 vdeclose(de);
331 if(year == 0){
332 vacfiledecref(fp);
333 return nil;
334 }
335 snprint(buf, sizeof buf, "%04d", year);
336 if((f = vacfilewalk(fp, buf)) == nil){
337 fprint(2, "warning: dirread %s but cannot walk", buf);
338 vacfiledecref(fp);
339 return nil;
340 }
341 fp = f;
342
343 de = vdeopen(fp);
344 mmdd = 0;
345 nn = 0;
346 if(de){
347 for(; vderead(de, &vd) > 0; vdcleanup(&vd)){
348 if(strlen(vd.elem) < 4)
349 continue;
350 if((n = strtol(vd.elem, &p, 10)) < 100 || n > 1231 || p != vd.elem+4)
351 continue;
352 if(*p == '.'){
353 if(p[1] == '0' || (n1 = strtol(p+1, &p, 10)) == 0 || *p != 0)
354 continue;
355 }else{
356 if(*p != 0)
357 continue;
358 n1 = 0;
359 }
360 if(n < mmdd || (n == mmdd && n1 < nn))
361 continue;
362 mmdd = n;
363 nn = n1;
364 }
365 }
366 vdeclose(de);
367 if(mmdd == 0){
368 vacfiledecref(fp);
369 return nil;
370 }
371 if(nn == 0)
372 snprint(buf, sizeof buf, "%04d", mmdd);
373 else
374 snprint(buf, sizeof buf, "%04d.%d", mmdd, nn);
375 if((f = vacfilewalk(fp, buf)) == nil){
376 fprint(2, "warning: dirread %s but cannot walk", buf);
377 vacfiledecref(fp);
378 return nil;
379 }
380 vacfiledecref(fp);
381
382 sprint(path, "%04d/%s", year, buf);
383 return f;
384 }
385
386 static void
387 removevacfile(void)
388 {
389 if(vacfile)
390 remove(vacfile);
391 }
392
393 void
394 plan9tovacdir(VacDir *vd, Dir *dir)
395 {
396 memset(vd, 0, sizeof *vd);
397
398 vd->elem = dir->name;
399 vd->uid = dir->uid;
400 vd->gid = dir->gid;
401 vd->mid = dir->muid;
402 if(vd->mid == nil)
403 vd->mid = "";
404 vd->mtime = dir->mtime;
405 vd->mcount = 0;
406 vd->ctime = dir->mtime; /* ctime: not available on plan 9 */
407 vd->atime = dir->atime;
408 vd->size = dir->length;
409
410 vd->mode = dir->mode & 0777;
411 if(dir->mode & DMDIR)
412 vd->mode |= ModeDir;
413 if(dir->mode & DMAPPEND)
414 vd->mode |= ModeAppend;
415 if(dir->mode & DMEXCL)
416 vd->mode |= ModeExclusive;
417 #ifdef PLAN9PORT
418 if(dir->mode & DMDEVICE)
419 vd->mode |= ModeDevice;
420 if(dir->mode & DMNAMEDPIPE)
421 vd->mode |= ModeNamedPipe;
422 if(dir->mode & DMSYMLINK)
423 vd->mode |= ModeLink;
424 #endif
425
426 vd->plan9 = 1;
427 vd->p9path = dir->qid.path;
428 vd->p9version = dir->qid.vers;
429 }
430
431 #ifdef PLAN9PORT
432 enum {
433 Special =
434 DMSOCKET |
435 DMSYMLINK |
436 DMNAMEDPIPE |
437 DMDEVICE
438 };
439 #endif
440
441 /*
442 * Archive the file named name, which has stat info d,
443 * into the vac directory fp (p = parent).
444 *
445 * If we're doing a vac -d against another archive, the
446 * equivalent directory to fp in that archive is diffp.
447 */
448 void
449 vac(VacFile *fp, VacFile *diffp, char *name, Dir *d)
450 {
451 char *elem, *s;
452 static char *buf;
453 int fd, i, n, bsize;
454 vlong off;
455 Dir *dk; // kids
456 VacDir vd, vddiff;
457 VacFile *f, *fdiff;
458 VtEntry e;
459
460 if(!includefile(name)){
461 warn("excluding %s%s", name, (d->mode&DMDIR) ? "/" : "");
462 return;
463 }
464
465 if(d->mode&DMDIR)
466 stats.ndir++;
467 else
468 stats.nfile++;
469
470 if(merge && vacmerge(fp, name) >= 0)
471 return;
472
473 if(verbose)
474 fprint(2, "%s%s\n", name, (d->mode&DMDIR) ? "/" : "");
475
476 #ifdef PLAN9PORT
477 if(d->mode&Special)
478 fd = -1;
479 else
480 #endif
481 if((fd = open(name, OREAD)) < 0){
482 warn("open %s: %r", name);
483 return;
484 }
485
486 elem = strrchr(name, '/');
487 if(elem)
488 elem++;
489 else
490 elem = name;
491
492 plan9tovacdir(&vd, d);
493 if((f = vacfilecreate(fp, elem, vd.mode)) == nil){
494 warn("vacfilecreate %s: %r", name);
495 return;
496 }
497 if(diffp)
498 fdiff = vacfilewalk(diffp, elem);
499 else
500 fdiff = nil;
501
502 if(vacfilesetdir(f, &vd) < 0)
503 warn("vacfilesetdir %s: %r", name);
504
505 bsize = fs->bsize;
506 if(buf == nil)
507 buf = vtmallocz(bsize);
508
509 #ifdef PLAN9PORT
510 if(d->mode&(DMSOCKET|DMNAMEDPIPE)){
511 /* don't write anything */
512 }
513 else if(d->mode&DMSYMLINK){
514 n = readlink(name, buf, sizeof buf);
515 if(n > 0 && vacfilewrite(f, buf, n, 0) < 0){
516 warn("venti write %s: %r", name);
517 goto Out;
518 }
519 stats.data += n;
520 }else if(d->mode&DMDEVICE){
521 snprint(buf, sizeof buf, "%c %d %d",
522 (char)((d->qid.path >> 16) & 0xFF),
523 (int)(d->qid.path & 0xFF),
524 (int)((d->qid.path >> 8) & 0xFF));
525 if(vacfilewrite(f, buf, strlen(buf), 0) < 0){
526 warn("venti write %s: %r", name);
527 goto Out;
528 }
529 stats.data += strlen(buf);
530 }else
531 #endif
532 if(d->mode&DMDIR){
533 while((n = dirread(fd, &dk)) > 0){
534 for(i=0; i<n; i++){
535 s = vtmalloc(strlen(name)+1+strlen(dk[i].name)+1);
536 strcpy(s, name);
537 strcat(s, "/");
538 strcat(s, dk[i].name);
539 vac(f, fdiff, s, &dk[i]);
540 free(s);
541 }
542 free(dk);
543 }
544 }else{
545 off = 0;
546 if(fdiff){
547 /*
548 * Copy fdiff's contents into f by moving the score.
549 * We'll diff and update below.
550 */
551 if(vacfilegetentries(fdiff, &e, nil) >= 0)
552 if(vacfilesetentries(f, &e, nil) >= 0){
553 bsize = e.dsize;
554
555 /*
556 * Or if -q is set, and the metadata looks the same,
557 * don't even bother reading the file.
558 */
559 if(qdiff && vacfilegetdir(fdiff, &vddiff) >= 0){
560 if(vddiff.mtime == vd.mtime)
561 if(vddiff.size == vd.size)
562 if(!vddiff.plan9 || (/* vddiff.p9path == vd.p9path && */ vddiff.p9version == vd.p9version)){
563 stats.skipfiles++;
564 stats.nfile--;
565 vdcleanup(&vddiff);
566 goto Out;
567 }
568
569 /*
570 * Skip over presumably-unchanged prefix
571 * of an append-only file.
572 */
573 if(vd.mode&ModeAppend)
574 if(vddiff.size < vd.size)
575 if(vddiff.plan9 && vd.plan9)
576 if(vddiff.p9path == vd.p9path){
577 off = vd.size/bsize*bsize;
578 if(seek(fd, off, 0) >= 0)
579 stats.skipdata += off;
580 else{
581 seek(fd, 0, 0); // paranoia
582 off = 0;
583 }
584 }
585
586 vdcleanup(&vddiff);
587 // XXX different verbose chatty prints for kaminsky?
588 }
589 }
590 }
591 if(qdiff && verbose)
592 fprint(2, "+%s\n", name);
593 while((n = readn(fd, buf, bsize)) > 0){
594 if(fdiff && sha1matches(f, off/bsize, (uchar*)buf, n)){
595 off += n;
596 stats.skipdata += n;
597 continue;
598 }
599 if(vacfilewrite(f, buf, n, off) < 0){
600 warn("venti write %s: %r", name);
601 goto Out;
602 }
603 stats.data += n;
604 off += n;
605 }
606 /*
607 * Since we started with fdiff's contents,
608 * set the size in case fdiff was bigger.
609 */
610 if(fdiff && vacfilesetsize(f, off) < 0)
611 warn("vtfilesetsize %s: %r", name);
612 }
613
614 Out:
615 vacfileflush(f, 1);
616 vacfiledecref(f);
617 if(fdiff)
618 vacfiledecref(fdiff);
619 close(fd);
620 }
621
622 void
623 vacstdin(VacFile *fp, char *name)
624 {
625 vlong off;
626 VacFile *f;
627 static char buf[8192];
628 int n;
629
630 if((f = vacfilecreate(fp, name, 0666)) == nil){
631 warn("vacfilecreate %s: %r", name);
632 return;
633 }
634
635 off = 0;
636 while((n = read(0, buf, sizeof buf)) > 0){
637 if(vacfilewrite(f, buf, n, off) < 0){
638 warn("venti write %s: %r", name);
639 vacfiledecref(f);
640 return;
641 }
642 off += n;
643 }
644 vacfileflush(f, 1);
645 vacfiledecref(f);
646 }
647
648 /*
649 * fp is the directory we're writing.
650 * mp is the directory whose contents we're merging in.
651 * d is the directory entry of the file from mp that we want to add to fp.
652 * vacfile is the name of the .vac file, for error messages.
653 * offset is the qid that qid==0 in mp should correspond to.
654 * max is the maximum qid we expect to see (not really needed).
655 */
656 int
657 vacmergefile(VacFile *fp, VacFile *mp, VacDir *d, char *vacfile,
658 vlong offset, vlong max)
659 {
660 VtEntry ed, em;
661 VacFile *mf;
662 VacFile *f;
663
664 mf = vacfilewalk(mp, d->elem);
665 if(mf == nil){
666 warn("could not walk %s in %s", d->elem, vacfile);
667 return -1;
668 }
669 if(vacfilegetentries(mf, &ed, &em) < 0){
670 warn("could not get entries for %s in %s", d->elem, vacfile);
671 vacfiledecref(mf);
672 return -1;
673 }
674
675 if((f = vacfilecreate(fp, d->elem, d->mode)) == nil){
676 warn("vacfilecreate %s: %r", d->elem);
677 vacfiledecref(mf);
678 return -1;
679 }
680 if(d->qidspace){
681 d->qidoffset += offset;
682 d->qidmax += offset;
683 }else{
684 d->qidspace = 1;
685 d->qidoffset = offset;
686 d->qidmax = max;
687 }
688 if(vacfilesetdir(f, d) < 0
689 || vacfilesetentries(f, &ed, &em) < 0
690 || vacfilesetqidspace(f, d->qidoffset, d->qidmax) < 0){
691 warn("vacmergefile %s: %r", d->elem);
692 vacfiledecref(mf);
693 vacfiledecref(f);
694 return -1;
695 }
696
697 vacfiledecref(mf);
698 vacfiledecref(f);
699 return 0;
700 }
701
702 int
703 vacmerge(VacFile *fp, char *name)
704 {
705 VacFs *mfs;
706 VacDir vd;
707 VacDirEnum *de;
708 VacFile *mp;
709 uvlong maxqid, offset;
710
711 if(strlen(name) < 4 || strcmp(name+strlen(name)-4, ".vac") != 0)
712 return -1;
713 if((mfs = vacfsopen(z, name, VtOREAD, CacheSize)) == nil)
714 return -1;
715 if(verbose)
716 fprint(2, "merging %s\n", name);
717
718 mp = vacfsgetroot(mfs);
719 de = vdeopen(mp);
720 if(de){
721 offset = 0;
722 if(vacfsgetmaxqid(mfs, &maxqid) >= 0){
723 _vacfsnextqid(fs, &offset);
724 vacfsjumpqid(fs, maxqid+1);
725 }
726 while(vderead(de, &vd) > 0){
727 if(vd.qid > maxqid){
728 warn("vacmerge %s: maxqid=%lld but %s has %lld",
729 name, maxqid, vd.elem, vd.qid);
730 vacfsjumpqid(fs, vd.qid - maxqid);
731 maxqid = vd.qid;
732 }
733 vacmergefile(fp, mp, &vd, name,
734 offset, maxqid);
735 vdcleanup(&vd);
736 }
737 vdeclose(de);
738 }
739 vacfiledecref(mp);
740 vacfsclose(mfs);
741 return 0;
742 }
743
744 #define TWID64 ((u64int)~(u64int)0)
745
746 static u64int
747 unittoull(char *s)
748 {
749 char *es;
750 u64int n;
751
752 if(s == nil)
753 return TWID64;
754 n = strtoul(s, &es, 0);
755 if(*es == 'k' || *es == 'K'){
756 n *= 1024;
757 es++;
758 }else if(*es == 'm' || *es == 'M'){
759 n *= 1024*1024;
760 es++;
761 }else if(*es == 'g' || *es == 'G'){
762 n *= 1024*1024*1024;
763 es++;
764 }
765 if(*es != '\0')
766 return TWID64;
767 return n;
768 }
769
770 static void
771 warn(char *fmt, ...)
772 {
773 va_list arg;
774
775 va_start(arg, fmt);
776 fprint(2, "vac: ");
777 vfprint(2, fmt, arg);
778 fprint(2, "\n");
779 va_end(arg);
780 }