tMore fixes; add some documentation - lumia - Archive checksum manager
HTML git clone git://lumidify.org/git/lumia.git
DIR Log
DIR Files
DIR Refs
DIR README
DIR LICENSE
---
DIR commit 3d69b3355b084a09aaa2a64a4b7d47e9296d4b01
DIR parent 9c5afc6b5d09f269a33c30bb8567157e1967533a
HTML Author: lumidify <nobody@lumidify.org>
Date: Mon, 23 Mar 2020 14:24:40 +0100
More fixes; add some documentation
Diffstat:
M lumia.pl | 116 ++++++++++++++++++++++++++-----
M test/.lumidify_archive_cksums.cksum | 2 +-
M test/.lumidify_archive_dirs | 1 -
M test/dir2/.lumidify_archive_cksums… | 2 +-
M test/dir2/.lumidify_archive_dirs | 1 +
R test/dir/.lumidify_archive_cksums … | 0
R test/dir/.lumidify_archive_cksums.… | 0
R test/dir/.lumidify_archive_dirs ->… | 0
R test/dir/meh -> test/dir2/dir/meh | 0
9 files changed, 100 insertions(+), 22 deletions(-)
---
DIR diff --git a/lumia.pl b/lumia.pl
t@@ -28,18 +28,32 @@ use File::Basename qw(basename dirname);
use File::Copy qw(move copy);
use File::Path qw(remove_tree make_path);
use String::ShellQuote;
-use Cwd qw(getcwd);
+use Cwd qw(realpath);
use POSIX qw(SIGINT);
use Data::Dumper;
use Scalar::Util qw(looks_like_number);
use Getopt::Long;
+my $CKSUM_CMD = 'cksum -q';
+my %SPECIAL_FILES = (
+ ".lumidify_archive_cksums" => 1,
+ ".lumidify_archive_cksums.cksum" => 1,
+ ".lumidify_archive_ignore" => 1,
+ ".lumidify_archive_dirs" => 1
+);
+
+# escape a filename for writing into the checksum files
sub escape_filename {
my $file = shift;
$file =~ s/\\/\\\\/g;
$file =~ s/"/\\"/g;
return $file;
}
+
+# make a generic file iterator
+# $file_func determines whether a file should be returned by the iterator
+# $dir_func is called for each directory and returns all files that
+# should be added to the queue
sub make_file_iter {
my ($file_func, $dir_func, @queue) = @_;
return sub {
t@@ -56,6 +70,8 @@ sub make_file_iter {
};
}
+# make a basic filename iterator, which simply returns all files
+# for which $file_func returns a true value
sub make_file_iter_basic {
my ($file_func, @files) = @_;
make_file_iter $file_func, sub {
t@@ -70,10 +86,15 @@ sub make_file_iter_basic {
}, @files;
}
+# make a basic directory iterator, which only returns all directories
sub make_dir_iter {
make_file_iter_basic sub {-d $_[0]}, @_;
}
+# make an interator that only returns the directories which are present
+# in the .lumidify_archive_dirs files
+# note: this returns nonexistent directories if those are still
+# specified in the lumia files
sub make_lumia_iter {
make_file_iter sub {1}, sub {
my $path = "$_[0]/.lumidify_archive_dirs";
t@@ -93,14 +114,7 @@ sub make_lumia_iter {
}, @_;
}
-my $CKSUM_CMD = 'cksum -q';
-my %SPECIAL_FILES = (
- ".lumidify_archive_cksums" => 1,
- ".lumidify_archive_cksums.cksum" => 1,
- ".lumidify_archive_ignore" => 1,
- ".lumidify_archive_dirs" => 1
-);
-
+# remove all special lumia files from the given directory
sub clean_files {
my $dir = shift;
my $match_lumia_files = sub {
t@@ -116,6 +130,11 @@ sub clean_files {
}
}
+# read a file, processing each line with $handle_cksum_func if set
+# and writing the results into $cksums
+# $handle_cksum_func must return two values, the checksum of the
+# argument and the rest of the string (that is then parsed for
+# the filename); if it returns undef, this function also returns undef
sub read_file {
my ($file, $cksums, $handle_cksum_func) = @_;
my $fh;
t@@ -164,6 +183,7 @@ sub read_file {
return $cksums;
}
+# read a single checksum file, writing the checksums into the hash $cksums and returning it
sub read_cksum_file {
my ($file, $cksums) = @_;
return read_file $file, $cksums, sub {
t@@ -179,6 +199,7 @@ sub read_cksum_file {
};
}
+# read the checksums and directory names in $dir
sub read_cksums {
my $dir = shift;
my $cksums = read_cksum_file("$dir/.lumidify_archive_cksums", {});
t@@ -188,6 +209,8 @@ sub read_cksums {
return $cksums;
}
+# get the checksum output for $path
+# returns undef if $CKSUM_CMD returns an error
sub get_cksum {
my $path = shift;
my $path_esc = shell_quote $path;
t@@ -200,6 +223,8 @@ sub get_cksum {
return $cksum_output;
}
+# check the checksums in $dir/$cksum_file
+# if $quiet is set, only print failed files
sub check_cksums {
my ($dir, $cksum_file, $quiet) = @_;
my $cksums = read_cksum_file("$dir/$cksum_file", {});
t@@ -219,6 +244,7 @@ sub check_cksums {
return $failed;
}
+# check the checksums of all files in $top_dir
sub check_files {
my $top_dir = shift;
my $iter = make_lumia_iter $top_dir;
t@@ -228,6 +254,8 @@ sub check_files {
}
}
+# write the checksums of the special lumia files given as arguments
+# to ".lumidify_archive_cksums.cksum" in $dir
sub write_special_cksums {
my ($dir, @files) = @_;
my $cksum_file = "$dir/.lumidify_archive_cksums.cksum";
t@@ -244,6 +272,13 @@ sub write_special_cksums {
write_file($cksum_file, $cksums, 1);
}
+# search for new files that aren't present in the checksum files
+# - if $file_func is set, it is called for each new file
+# - if $before_dir_func is set, it is called before processing the
+# files in each directory that has new files OR if a directory
+# is entirely new (well, it only checks if ".lumidify_archive_cksums.cksum" exists)
+# - if $after_dir_func is set, it is called after processing the
+# files in each directory that has new files
sub check_new_files {
my ($dir, $file_func, $before_dir_func, $after_dir_func) = @_;
my $iter = make_file_iter sub {-d $_[0]}, sub {
t@@ -294,6 +329,7 @@ sub check_new_files {
while ($iter->()) {}
}
+# add all new files in $top_dir to the checksum files
sub check_add_new_files {
my $top_dir = shift;
my $changed_dirs = 0;
t@@ -347,6 +383,9 @@ sub check_add_new_files {
};
}
+# write the "checksums" in $contents to $path
+# if $is_cksum_file is set, the value each of the keys in $contents points
+# to is written before the key
sub write_file {
my ($path, $contents, $is_cksum_file) = @_;
my $fh;
t@@ -363,11 +402,16 @@ sub write_file {
close $fh;
}
+# write the checksums in $contents to the file at $path
sub write_cksum_file {
my ($path, $contents) = @_;
write_file $path, $contents, 1;
}
+# write the checksums in $contents to $dir
+# any keys that point to undef are taken to be directories and vice versa
+# $files_modified and $dirs_modified control which of the special lumia
+# files actually get written
sub write_cksums {
my ($dir, $contents, $files_modified, $dirs_modified) = @_;
# No, this isn't efficient...
t@@ -383,6 +427,7 @@ sub write_cksums {
}
}
+# show all files that are present in the checksum files but don't exist on the filesystem anymore
sub check_old_files {
my $top_dir = shift;
my $iter = make_lumia_iter $top_dir;
t@@ -399,6 +444,8 @@ sub check_old_files {
}
}
+# clean up the lumia checksum files, removing any files that aren't present
+# on the filesystem anymore
sub remove_old_files {
my $top_dir = shift;
my $iter = make_lumia_iter $top_dir;
t@@ -431,6 +478,10 @@ sub remove_old_files {
}
}
+# sort the given paths into hash based on the dirname
+# returns: a hash with the keys being the dirnames of the given paths and
+# each one pointing to an array containing the basenames of all paths
+# that had this dirname
sub sort_by_dir {
my %sorted_files;
foreach my $file (@_) {
t@@ -447,6 +498,7 @@ sub sort_by_dir {
return \%sorted_files;
}
+# copies the $src files to $dst and updates the checksums in $dst
# $src: list of source paths
# $dst: destination directory or file (in latter case only one src is allowed)
sub copy_files {
t@@ -494,14 +546,15 @@ sub copy_files {
write_cksums $dst_dir, $dst_cksums, $files_touched, $dirs_touched;
}
+# return whether the two paths are the same
sub cmp_path {
my ($src, $dst) = @_;
- # remove trailing slash so compare works
- $src =~ s/\/$//;
- $dst =~ s/\/$//;
- return $src eq $dst;
+ my $src_real = realpath $src;
+ return defined $src_real && $src_real eq realpath $dst;
}
+# move a file from $src to $dst, prompting for confirmation if $dst already exists
+# automatically appends the basename of $src to $dst if $dst is a directory
sub move_file {
my ($src, $dst) = @_;
if (-d $dst) {
t@@ -519,6 +572,9 @@ sub move_file {
return 0;
}
+# move all files/directories in $src_files from $src_dir to $dst_dir ($src_files
+# only contains the basenames of the files), removing them from the checksum files
+# in $src_dir and adding them to $dst_cksums
sub move_from_same_dir {
my ($src_dir, $src_files, $dst_cksums, $dst_dir) = @_;
my $src_cksums = read_cksums $src_dir;
t@@ -531,15 +587,21 @@ sub move_from_same_dir {
warn "ERROR: can't move \"$fullpath\" into \"$dst_dir\" (same dir)\n";
next;
}
+ my $tmp_dirs_touched = 0;
+ my $tmp_files_touched = 0;
+ if (-d $fullpath) {
+ $tmp_dirs_touched = 1;
+ } else {
+ $tmp_files_touched = 1;
+ }
if (my $err = move_file($fullpath, $dst_dir)) {
warn "$err\n";
next;
}
- if (-d $fullpath) {
- $dirs_touched = 1;
- } else {
- $files_touched = 1;
- }
+ # need to be able to check if the path is a directory
+ # before actually moving it
+ $dirs_touched ||= $tmp_dirs_touched;
+ $files_touched ||= $tmp_files_touched;
if (exists $src_cksums->{$src_file}) {
$dst_cksums->{$src_file} = $src_cksums->{$src_file};
delete $src_cksums->{$src_file};
t@@ -551,6 +613,7 @@ sub move_from_same_dir {
return ($files_touched, $dirs_touched);
}
+# rename a single file or directory from $src to $dst
sub move_rename {
my ($src, $dst) = @_;
my $src_dir = dirname $src;
t@@ -594,6 +657,12 @@ sub move_rename {
}
}
+# move all files and directories in $src to $dst
+# - if $dst does not exist, $src is only allowed to contain one path, which is
+# renamed to $dst
+# - if $dst is a file, $src is only allowed to contain a single path (which
+# must be a file), which is renamed to $dst
+# - otherwise, all files and directories in $src are moved to $dst
# $src: list of source paths
# $dst: destination directory or file (in latter case only one src is allowed)
sub move_files {
t@@ -631,6 +700,7 @@ sub move_files {
write_cksums $dst, $dst_cksums, $files_touched, $dirs_touched;
}
+# remove a file or directory from the filesystem
sub remove_file_dir {
my $path = shift;
if (-d $path) {
t@@ -641,6 +711,9 @@ sub remove_file_dir {
return 0;
}
+# remove all files in one directory, updating the checksum files in the process
+# note: the files are only allowed to be basenames, i.e., they must be the
+# actual filenames present in the checksum files
sub remove_from_same_dir {
my ($dir, @files) = @_;
my $cksums = read_cksums $dir;
t@@ -670,6 +743,8 @@ sub remove_from_same_dir {
write_cksums $dir, $cksums, $files_touched, $dirs_touched;
}
+# remove all given files and directories, updating the appropriate checksum
+# files in the process
sub remove_files {
my $sorted_files = sort_by_dir(@_);
foreach my $dir (keys %$sorted_files) {
t@@ -677,6 +752,9 @@ sub remove_files {
}
}
+# create the given directories, initializing them with empty checksum files
+# note: does not work like "mkdir -p", i.e., the new directories have to
+# be located inside already existing directories
sub make_dirs {
my @created_dirs;
foreach (@_) {
t@@ -726,7 +804,7 @@ sub extract {
}
if ($#ARGV < 0) {
- die("USAGE: test.pl {init|check|clean|checknew|addnew|cp|mv|rm|mkdir}\n");
+ die("USAGE: test.pl {init|check|clean|checknew|addnew|checkold|rmold|extract|cp|mv|rm|mkdir}\n");
}
if ($ARGV[0] eq "mv") {
if ($#ARGV < 2) {
DIR diff --git a/test/.lumidify_archive_cksums.cksum b/test/.lumidify_archive_cksums.cksum
t@@ -1,2 +1,2 @@
2507213385 41 ".lumidify_archive_cksums"
-1201997706 27 ".lumidify_archive_dirs"
+3971863640 21 ".lumidify_archive_dirs"
DIR diff --git a/test/.lumidify_archive_dirs b/test/.lumidify_archive_dirs
t@@ -1,4 +1,3 @@
-"dir"
"dir2"
"dir3"
"dir4"
DIR diff --git a/test/dir2/.lumidify_archive_cksums.cksum b/test/dir2/.lumidify_archive_cksums.cksum
t@@ -1,2 +1,2 @@
4294967295 0 ".lumidify_archive_cksums"
-4294967295 0 ".lumidify_archive_dirs"
+137730780 6 ".lumidify_archive_dirs"
DIR diff --git a/test/dir2/.lumidify_archive_dirs b/test/dir2/.lumidify_archive_dirs
t@@ -0,0 +1 @@
+"dir"
DIR diff --git a/test/dir/.lumidify_archive_cksums b/test/dir2/dir/.lumidify_archive_cksums
DIR diff --git a/test/dir/.lumidify_archive_cksums.cksum b/test/dir2/dir/.lumidify_archive_cksums.cksum
DIR diff --git a/test/dir/.lumidify_archive_dirs b/test/dir2/dir/.lumidify_archive_dirs
DIR diff --git a/test/dir/meh b/test/dir2/dir/meh