tFixes; allow to check only individual files - lumia - Archive checksum manager
HTML git clone git://lumidify.org/git/lumia.git
DIR Log
DIR Files
DIR Refs
DIR README
DIR LICENSE
---
DIR commit a964a391d65fce7963a55c9964bbbe6a5520a2ff
DIR parent 3d69b3355b084a09aaa2a64a4b7d47e9296d4b01
HTML Author: lumidify <nobody@lumidify.org>
Date: Mon, 23 Mar 2020 15:33:23 +0100
Fixes; allow to check only individual files
Diffstat:
M lumia.pl | 143 +++++++++++++++++--------------
1 file changed, 78 insertions(+), 65 deletions(-)
---
DIR diff --git a/lumia.pl b/lumia.pl
t@@ -1,25 +1,16 @@
#!/usr/bin/env perl
-# FIXME: add ckold to check what files don't exist anymore and remove them from cksums
-# FIXME: write cksums as we go along during check_add_new, not just at end
# FIXME: some way to avoid writing .lumidify* in dirs but still index them? e.g. Code/CMSG
# FIXME: cksum don't create malformed line if permission denied
-# FIXME: dirs in ignore still created when extracting
# FIXME: make generic function to traverse dirs and call other function on each dir
-# FIXME: extract - actually copy files, don't write again and regen cksums.cksum
-# FIXME: check cksums.cksum when reading files
-# FIXME: check if .lumidify* are writable first so cksums aren't generated uselessly
# FIXME: handle rm, etc. on .lumidify* files
-# FIXME: handle files with newline?
# FIXME: ignore all except for a certain file/folder
-# FIXME: store modified date and recksum filed with changed date
+# FIXME: store modified date and checksum filed with changed date
# FIXME: sha256 works with leading zeroes, maybe switch?
# FIXME: allow different hash types
# FIXME: don't write anything if cksum fails (will create malformed line)
# FIXME: add option to just check dir structure or maybe check if everything exists
-# WARNING: convert regenerates cksums.cksum
# FIXME: add option to compare cksums of two dirs
-# FIXME: add option to check only individual files
use strict;
use warnings;
t@@ -34,12 +25,21 @@ use Data::Dumper;
use Scalar::Util qw(looks_like_number);
use Getopt::Long;
+# the file used to store checksums for files
+my $CKSUM_FILE = ".lumidify_archive_cksums";
+# the file used to store directory names
+my $DIR_FILE = ".lumidify_archive_dirs";
+# the file read to ignore files or directories
+my $IGNORE_FILE = ".lumidify_archive_ignore";
+# the file containing checksums of $CKSUM_FILE and $DIR_FILE
+my $DOUBLE_CKSUM_FILE = ".lumidify_archive_cksums.cksum";
+
my $CKSUM_CMD = 'cksum -q';
my %SPECIAL_FILES = (
- ".lumidify_archive_cksums" => 1,
- ".lumidify_archive_cksums.cksum" => 1,
- ".lumidify_archive_ignore" => 1,
- ".lumidify_archive_dirs" => 1
+ $CKSUM_FILE => 1,
+ $DIR_FILE => 1,
+ $IGNORE_FILE => 1,
+ $DOUBLE_CKSUM_FILE => 1
);
# escape a filename for writing into the checksum files
t@@ -86,18 +86,14 @@ sub make_file_iter_basic {
}, @files;
}
-# make a basic directory iterator, which only returns all directories
-sub make_dir_iter {
- make_file_iter_basic sub {-d $_[0]}, @_;
-}
-
# make an interator that only returns the directories which are present
-# in the .lumidify_archive_dirs files
+# in the $DIR_FILE files, in addition to the files and directories that
+# were originally passed as arguments
# note: this returns nonexistent directories if those are still
# specified in the lumia files
sub make_lumia_iter {
make_file_iter sub {1}, sub {
- my $path = "$_[0]/.lumidify_archive_dirs";
+ my $path = "$_[0]/$DIR_FILE";
return [] if !-f $path;
my $dirs = read_file($path, {});
return if !defined $dirs;
t@@ -117,10 +113,7 @@ sub make_lumia_iter {
# remove all special lumia files from the given directory
sub clean_files {
my $dir = shift;
- my $match_lumia_files = sub {
- exists $SPECIAL_FILES{basename $_[0]};
- };
- my $iter = make_file_iter $match_lumia_files, $dir;
+ my $iter = make_file_iter_basic sub {exists $SPECIAL_FILES{basename $_[0]};}, $dir;
while (my $file = $iter->()) {
if (!unlink $file) {
warn "WARNING: Unable to remove file \"$file\"!";
t@@ -202,9 +195,9 @@ sub read_cksum_file {
# read the checksums and directory names in $dir
sub read_cksums {
my $dir = shift;
- my $cksums = read_cksum_file("$dir/.lumidify_archive_cksums", {});
+ my $cksums = read_cksum_file("$dir/$CKSUM_FILE", {});
return undef if !defined $cksums;
- $cksums = read_file("$dir/.lumidify_archive_dirs", $cksums);
+ $cksums = read_file("$dir/$DIR_FILE", $cksums);
return undef if !defined $cksums;
return $cksums;
}
t@@ -246,19 +239,40 @@ sub check_cksums {
# check the checksums of all files in $top_dir
sub check_files {
- my $top_dir = shift;
- my $iter = make_lumia_iter $top_dir;
- while (my $dir = $iter->()) {
- check_cksums $dir, ".lumidify_archive_cksums.cksum";
- check_cksums $dir, ".lumidify_archive_cksums";
+ my $iter = make_lumia_iter @_;
+ while (my $file = $iter->()) {
+ if (-d $file) {
+ check_cksums $file, "$DOUBLE_CKSUM_FILE";
+ check_cksums $file, "$CKSUM_FILE";
+ } else {
+ my $dir = dirname $file;
+ my $base = basename $file;
+ if (exists $SPECIAL_FILES{$base}) {
+ warn "ERROR: File is reserved for lumia.pl: $file\n";
+ next;
+ }
+ my $cksums = read_cksum_file("$dir/$CKSUM_FILE");
+ next if !defined $cksums;
+ if (!exists $cksums->{$base}) {
+ warn "ERROR: File doesn't exist in checksums: $file\n";
+ next;
+ }
+ my $output = get_cksum "$file";
+ next if !defined $output;
+ if ($output eq $cksums->{$base}) {
+ print "OK $file\n";
+ } else {
+ print "FAILED $file\n";
+ }
+ }
}
}
# write the checksums of the special lumia files given as arguments
-# to ".lumidify_archive_cksums.cksum" in $dir
+# to "$DOUBLE_CKSUM_FILE" in $dir
sub write_special_cksums {
my ($dir, @files) = @_;
- my $cksum_file = "$dir/.lumidify_archive_cksums.cksum";
+ my $cksum_file = "$dir/$DOUBLE_CKSUM_FILE";
my $cksums = {};
if (-f $cksum_file) {
$cksums = read_cksum_file $cksum_file, {};
t@@ -276,7 +290,7 @@ sub write_special_cksums {
# - if $file_func is set, it is called for each new file
# - if $before_dir_func is set, it is called before processing the
# files in each directory that has new files OR if a directory
-# is entirely new (well, it only checks if ".lumidify_archive_cksums.cksum" exists)
+# is entirely new (well, it only checks if "$DOUBLE_CKSUM_FILE" exists)
# - if $after_dir_func is set, it is called after processing the
# files in each directory that has new files
sub check_new_files {
t@@ -294,9 +308,9 @@ sub check_new_files {
}
return {};
};
- my $ignore = $read_file_noerror->("$dir/.lumidify_archive_ignore", \&read_file);
- my $lumia_dirs = $read_file_noerror->("$dir/.lumidify_archive_dirs", \&read_file);
- my $lumia_files = $read_file_noerror->("$dir/.lumidify_archive_cksums", \&read_cksum_file);
+ my $ignore = $read_file_noerror->("$dir/$IGNORE_FILE", \&read_file);
+ my $lumia_dirs = $read_file_noerror->("$dir/$DIR_FILE", \&read_file);
+ my $lumia_files = $read_file_noerror->("$dir/$CKSUM_FILE", \&read_cksum_file);
my @dirs;
my $found = 0;
while (my $file = readdir $dh) {
t@@ -317,7 +331,7 @@ sub check_new_files {
}
closedir $dh;
# also call $before_dir_func if the directory has not been initialized yet
- if (!$found && !-f "$dir/.lumidify_archive_cksums.cksum" && defined $before_dir_func) {
+ if (!$found && !-f "$dir/$DOUBLE_CKSUM_FILE" && defined $before_dir_func) {
$before_dir_func->($dir);
}
if ($found && defined $after_dir_func) {
t@@ -338,7 +352,7 @@ sub check_add_new_files {
my ($dir, $file) = @_;
my $fullpath = "$dir/$file";
if (-d $fullpath) {
- my $dir_file = "$dir/.lumidify_archive_dirs";
+ my $dir_file = "$dir/$DIR_FILE";
my $fh;
if (!open $fh, ">>", $dir_file) {
warn "ERROR: Unable to append to file \"$dir_file\"!";
t@@ -350,7 +364,7 @@ sub check_add_new_files {
} else {
my $cksum_output = get_cksum $fullpath;
return if !defined $cksum_output;
- my $cksum_file = "$dir/.lumidify_archive_cksums";
+ my $cksum_file = "$dir/$CKSUM_FILE";
my $fh;
if (!open $fh, ">>", $cksum_file) {
warn "ERROR: Unable to append to file \"$cksum_file\"!";
t@@ -362,8 +376,8 @@ sub check_add_new_files {
}
print "Added \"$fullpath\"\n";
}, sub {
- if (-f "$_[0]/.lumidify_archive_cksums.cksum") {
- if (!check_cksums $_[0], ".lumidify_archive_cksums.cksum", 1) {
+ if (-f "$_[0]/$DOUBLE_CKSUM_FILE") {
+ if (!check_cksums $_[0], "$DOUBLE_CKSUM_FILE", 1) {
warn "Checksum files corrupt in \"$_[0]\", not adding new checksums!\n";
return 0;
}
t@@ -373,11 +387,11 @@ sub check_add_new_files {
return 1;
}, sub {
if ($changed_dirs) {
- write_special_cksums $_[0], ".lumidify_archive_dirs";
+ write_special_cksums $_[0], "$DIR_FILE";
$changed_dirs = 0;
}
if ($changed_files) {
- write_special_cksums $_[0], ".lumidify_archive_cksums";
+ write_special_cksums $_[0], "$CKSUM_FILE";
$changed_files = 0;
}
};
t@@ -417,13 +431,13 @@ sub write_cksums {
# No, this isn't efficient...
if ($files_modified) {
my %file_cksums = map {$_ => $contents->{$_}} grep({defined $contents->{$_}} keys %$contents);
- write_cksum_file("$dir/.lumidify_archive_cksums", \%file_cksums);
- write_special_cksums $dir, ".lumidify_archive_cksums";
+ write_cksum_file("$dir/$CKSUM_FILE", \%file_cksums);
+ write_special_cksums $dir, "$CKSUM_FILE";
}
if ($dirs_modified) {
my %dir_cksums = map {$_ => undef} grep({!defined $contents->{$_}} keys %$contents);
- write_file "$dir/.lumidify_archive_dirs", \%dir_cksums;
- write_special_cksums $dir, ".lumidify_archive_dirs";
+ write_file "$dir/$DIR_FILE", \%dir_cksums;
+ write_special_cksums $dir, "$DIR_FILE";
}
}
t@@ -434,7 +448,7 @@ sub check_old_files {
while (my $dir = $iter->()) {
# if $dir doesn't exist, the iterator already issued a warning
if (-e $dir) {
- my $cksums = read_cksum_file("$dir/.lumidify_archive_cksums", {}) // {};
+ my $cksums = read_cksum_file("$dir/$CKSUM_FILE", {}) // {};
foreach my $file (keys %$cksums) {
if (!-e "$dir/$file") {
warn "Nonexistent file: \"$dir/$file\"!\n";
t@@ -453,26 +467,26 @@ sub remove_old_files {
if (!-e $dir) {
my $parent = dirname $dir;
my $child = basename $dir;
- my $lumia_dirs = read_file("$parent/.lumidify_archive_dirs", {}) // {};
+ my $lumia_dirs = read_file("$parent/$DIR_FILE", {}) // {};
if (exists $lumia_dirs->{$child}) {
delete $lumia_dirs->{$child};
- write_file "$parent/.lumidify_archive_dirs", $lumia_dirs;
- print "Removed \"$dir\" from \"$parent/.lumidify_archive_dirs\"\n";
- write_special_cksums $parent, ".lumidify_archive_dirs";
+ write_file "$parent/$DIR_FILE", $lumia_dirs;
+ print "Removed \"$dir\" from \"$parent/$DIR_FILE\"\n";
+ write_special_cksums $parent, "$DIR_FILE";
}
} else {
- my $cksums = read_cksum_file("$dir/.lumidify_archive_cksums", {}) // {};
+ my $cksums = read_cksum_file("$dir/$CKSUM_FILE", {}) // {};
my $found = 0;
foreach my $file (keys %$cksums) {
if (!-e "$dir/$file") {
delete $cksums->{$file};
- print "Removed \"$dir/$file\" from \"$dir/.lumidify_archive_cksums\"\n";
+ print "Removed \"$dir/$file\" from \"$dir/$CKSUM_FILE\"\n";
$found = 1;
}
}
if ($found) {
- write_cksum_file "$dir/.lumidify_archive_cksums", $cksums;
- write_special_cksums $dir, ".lumidify_archive_cksums";
+ write_cksum_file "$dir/$CKSUM_FILE", $cksums;
+ write_special_cksums $dir, "$CKSUM_FILE";
}
}
}
t@@ -775,13 +789,13 @@ sub make_dirs {
push(@{$dirs{$parent}}, basename($dir));
}
foreach my $parent (keys %dirs) {
- my $parent_dirs = read_file "$parent/.lumidify_archive_dirs", {};
+ my $parent_dirs = read_file "$parent/$DIR_FILE", {};
next if !defined $parent_dirs;
foreach my $dir (@{$dirs{$parent}}) {
$parent_dirs->{$dir} = "";
}
- write_file "$parent/.lumidify_archive_dirs", $parent_dirs;
- write_special_cksums $parent, ".lumidify_archive_dirs";
+ write_file "$parent/$DIR_FILE", $parent_dirs;
+ write_special_cksums $parent, "$DIR_FILE";
}
}
t@@ -842,11 +856,10 @@ if ($ARGV[0] eq "mv") {
}
remove_old_files $dir;
} elsif ($ARGV[0] eq "check") {
- my $dir = ".";
- if ($#ARGV > 0) {
- $dir = $ARGV[1];
+ if ($#ARGV < 1) {
+ die "At least one path required";
}
- check_files $dir;
+ check_files @ARGV[1..$#ARGV];
} elsif ($ARGV[0] eq "clean") {
my $dir = ".";
if ($#ARGV > 0) {