463 lines
18 KiB
Bash
Executable File
463 lines
18 KiB
Bash
Executable File
#!/bin/bash
|
|
# Copyright © 2022 Matthias Quintern.
|
|
# This software comes with no warranty.
|
|
# This software is licensed under the GPL3
|
|
|
|
# ABOUT
|
|
# Synchronizes files from a sender to a receiver using rsync
|
|
# The files can be encrypted before or decrypted after the transfer using the mkrypt script
|
|
|
|
# SETTINGS
|
|
# The directory that is searched for config files, use ~/.config if $XDG_CONFIG_HOME is not defined
|
|
CONFIG_DIR=~/.config/msynk
|
|
[[ -n $XDG_CONFIG_HOME ]] && CONFIG_DIR=$XDG_CONFIG_HOME/msynk
|
|
[[ -n $MSYNK_CONFIG_HOME ]] && CONFIG_DIR=$MSYNK_CONFIG_HOME
|
|
|
|
# YOU CAN OVERRIDE THESE IN THE CONFIG FILES
|
|
# When using encrypt/decrypt, the source is en/decrypted to TMP_DIR and then rsynced to receiver from there Path must contain 'msynk' TMP_DIR=/tmp/msynk/ When using --delete, this file is used to determine the files that will be deleted TMP_FILE=/tmp/msynk_file Path to the mkrypt script mkrypt=/usr/bin/mkrypt
|
|
# Path must contain 'msynk'
|
|
TMP_DIR=/tmp/msynk/
|
|
# When using --delete, this file is used to determine the files that will be deleted
|
|
TMP_FILE=/tmp/msynk_file
|
|
# Path to the mkrypt script
|
|
mkrypt=/usr/bin/mkrypt
|
|
|
|
# Additional flags for mkrypt
|
|
mkrypt_flags=()
|
|
|
|
rsync_flags=(-ruh)
|
|
# rsync_flags+=(--rsh="ssh -p 22")
|
|
# r - recursive
|
|
# v - verbose
|
|
# Ut - preserve modification&access times
|
|
# u - update, skip files that are newer on the receiver
|
|
# P - show progress and keep partially transferred files
|
|
# h - human readable
|
|
# see "man rsync" for more
|
|
|
|
# expand * to hidden files as well
|
|
shopt -s dotglob
|
|
|
|
# UTILITY
|
|
NAME="\e[1;34mmsynk"
|
|
FMT_MESSAGE="$NAME: \e[0;34m%s\e[0m %s\n"
|
|
FMT_ERROR="\e[1;31m$NAME: \e[1;31mERROR: \e[0m%s\n"
|
|
# exit 1: error, exit 2: rsync exited non 0, exit 3: mkrypt exited non 0
|
|
FMT_SYNC="$NAME: \e[1;32mSyncing: \e[0m%s\n"
|
|
FMT_CMD="$NAME: \e[1;33mRunning: \e[0m%s\n"
|
|
FMT_CONFIG="\e[34m%s\e:\t\e[1;33m%s\e[0m\n"
|
|
|
|
# FUNCTIONS
|
|
# silence commands
|
|
quiet() { "$@" > /dev/null 2>&1; }
|
|
|
|
# check if a passed parameter is contained in file
|
|
# path must be set before calling
|
|
check_path_in_args()
|
|
{
|
|
if [ $all = 1 ]; then
|
|
return 0
|
|
fi
|
|
for string in ${paths_2[@]}; do
|
|
if [[ $path == *$string* ]]; then
|
|
if [[ $exclude = 1 ]]; then
|
|
return 1
|
|
else
|
|
return 0
|
|
fi
|
|
fi
|
|
done
|
|
if [[ $exclude = 1 ]]; then
|
|
return 0
|
|
else
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
|
|
_sync_set_dest() {
|
|
if [[ -d "$path" ]]; then
|
|
if [[ -n "$relative" ]]; then
|
|
dest="$receiver"
|
|
else
|
|
dest="$receiver"$(basename "$path")
|
|
fi
|
|
elif [[ -f $path ]]; then
|
|
dest="$receiver"
|
|
elif [[ "$sender" == *:* ]]; then # if sender is remote, assume the path is a dir if it has a slash and a file otherwise
|
|
if [[ "$path" == *\/ ]]; then
|
|
dest="$receiver"$(basename "$path")
|
|
else
|
|
dest="$receiver"
|
|
fi
|
|
else
|
|
printf "$FMT_ERROR" "Invalid path: $path"; exit 1
|
|
fi
|
|
}
|
|
|
|
|
|
# SYNC SENDER TO RECEIVER
|
|
sync_sender_to_receiver()
|
|
{
|
|
# perform a dry run to see which files would be deleted
|
|
if [[ -n $delete && -z $skip_dryrun ]]; then
|
|
[[ $v -ge 1 ]] && printf "$FMT_MESSAGE" "Performing dryrun" "to generate list of files that will be deleted from $receiver..."
|
|
echo "" > $TMP_FILE
|
|
for path in "${filtered_paths[@]}"; do
|
|
_sync_set_dest
|
|
# if [ -n $relative ]; then
|
|
# dest_subdir="$receiver"
|
|
# else
|
|
# dest_subdir="$receiver"$(basename "$path")
|
|
# fi
|
|
[[ $v -ge 3 ]] && printf "$FMT_CMD" "rsync ${rsync_flags[*]} -v --dry-run --delete $path $dest | grep deleting | sed \"s(deleting (${BACKUP_SUBDIR}/(\" >> $TMP_FILE"
|
|
rsync "${rsync_flags[@]}" -v --dry-run --delete $path $dest | grep deleting | sed "s(deleting (${BACKUP_SUBDIR}/(" >> $TMP_FILE
|
|
done
|
|
# print files that will be deleted and ask if continue
|
|
# grep deleting $TMP_FILE | sed "s(deleting (${receiver}/(" | less
|
|
less $TMP_FILE
|
|
DONE=0
|
|
while [[ $DONE == 0 ]]; do
|
|
printf "$FMT_MESSAGE" "The listed files will be deleted from $receiver."
|
|
printf "Proceed?\e[34m ([y]es, [s]how again, [*] no)\e[0m: "
|
|
# printf "Proceed?\e[34m ([y]es, [s]how again, [l]ist all files, [*] no)\e[0m: "
|
|
read answer
|
|
case $answer in
|
|
y|Y)
|
|
DONE=1
|
|
;;
|
|
s|S)
|
|
less $TMP_FILE
|
|
;;
|
|
# l|L)
|
|
# less $TMP_FILE
|
|
# ;;
|
|
*)
|
|
printf "$FMT_MESSAGE" "Cancelled."
|
|
exit 0 ;;
|
|
esac
|
|
done
|
|
fi
|
|
|
|
# actual syncing
|
|
for path in "${filtered_paths[@]}"; do
|
|
[[ $v -ge 1 ]] && printf "$FMT_SYNC" "$path"
|
|
_sync_set_dest
|
|
#if [[ -d "$path" ]]; then
|
|
# if [[ -n "$relative" ]]; then
|
|
# dest="$receiver"
|
|
# else
|
|
# dest="$receiver"$(basename "$path")
|
|
# fi
|
|
#elif [[ -f $path ]]; then
|
|
# dest="$receiver"
|
|
#elif [[ "$sender" == *:* ]]; then # if sender is remote, assume the path is a dir if it has a slash and a file otherwise
|
|
# #[[ "$path" == *"/" ]] && dest="$receiver"$(basename "$path") || dest="$receiver"
|
|
# dest=$receiver
|
|
#else
|
|
# printf "$FMT_ERROR" "Invalid path: $path"; exit 1
|
|
#fi
|
|
if [[ -n $encrypt || -n $decrypt ]]; then
|
|
mkdir -p "$TMP_DIR"
|
|
tmp_source="$TMP_DIR"
|
|
|
|
if [[ -n $encrypt ]]; then
|
|
# check if TMP_DIR has enough space
|
|
tmp_free=$(df --output=avail $TMP_DIR | grep -E "[[:digit:]]+")
|
|
path_size=$(du -s "$path" | awk '{print $1}' | sed -e "s/[^0-9]//g")
|
|
if [[ $tmp_free -le $path_size ]]; then
|
|
printf "$FMT_ERROR" "Not enough space on $TMP_DIR to copy $path! $TMP_DIR $tmp_free free - Size $path: $path_size"
|
|
printf "$FMT_MESSAGE" "" "If using a tmpfs, you can increase its size by running: 'sudo mount -o remount,size=XXG,noatime /tmp' to increase the size to XXG (You will need a large enough swap space)"
|
|
exit 1
|
|
fi
|
|
|
|
[[ -f "$path" ]] && tmp_source="$TMP_DIR"$(basename "$path").gpg
|
|
[[ $v -ge 3 ]] && printf "$FMT_CMD" "bash $mkrypt --encrypt $path --output $TMP_DIR ${mkrypt_flags[*]}"
|
|
bash $mkrypt --encrypt "$path" --output "$TMP_DIR" ${mkrypt_flags[@]} || exit 3
|
|
[[ $v -ge 3 ]] && printf "$FMT_CMD" "rsync ${rsync_flags[*]} $tmp_source $dest"
|
|
rsync "${rsync_flags[@]}" "$tmp_source" "$dest" || { printf "$FMT_ERROR" "rsync exited with exit code $?"; exit 2; }
|
|
else
|
|
[[ -f $path ]] && tmp_source=$TMP_DIR$(basename $path)
|
|
[[ $v -ge 3 ]] && printf "$FMT_CMD" "rsync ${rsync_flags[*]} $path $TMP_DIR"
|
|
rsync "${rsync_flags[@]}" "$path" "$TMP_DIR" || { printf "$FMT_ERROR" "rsync exited with exit code $?"; exit 2; }
|
|
[[ $v -ge 3 ]] && printf "$FMT_CMD" "bash $mkrypt --decrypt $tmp_source --output $(readlink -f $dest) ${mkrypt_flags[*]}"
|
|
bash $mkrypt --decrypt "$tmp_source" --output "$(readlink -f "$dest")" ${mkrypt_flags[@]} || exit 3
|
|
fi
|
|
else
|
|
[[ $v -ge 3 ]] && printf "$FMT_CMD" "rsync ${rsync_flags[*]} $delete $path $dest"
|
|
rsync "${rsync_flags[@]}" $delete "$path" "$dest" || { printf "$FMT_ERROR" "rsync exited with exit code $?"; exit 2; }
|
|
fi
|
|
[[ "$TMP_DIR" == *msynk* ]] && {
|
|
rm -rf "$TMP_DIR"/* || { printf "$FMT_ERROR" "Can not delete contents of \$TMP_DIR: $TMP_DIR"; exit 1; }
|
|
} || { printf "$FMT_ERROR" "Will not delete \$TMP_DIR because it might be dangerous (path does not contain 'msynk'): $TMP_DIR"; exit 1; }
|
|
done
|
|
|
|
# put todays date in the config
|
|
if [[ -f $CONFIG_DIR/$config ]]; then
|
|
if grep -xq "date=.*" $CONFIG_DIR/$config; then
|
|
[[ $v -ge 2 ]] && printf "$FMT_MESSAGE" "Updating" "date in $CONFIG_DIR/$config."
|
|
sed "s/date=.*/date=\"$(date --iso=sec)\"/" $CONFIG_DIR/$config -i
|
|
else
|
|
[[ $v -ge 2 ]] && printf "$FMT_MESSAGE" "Writing" "current date to $CONFIG_DIR/$config."
|
|
echo "date=\"$(date --iso=sec)\"" >> $CONFIG_DIR/$config
|
|
fi
|
|
fi
|
|
|
|
[[ $v -ge 2 ]] && printf "$FMT_MESSAGE" "Running:" "'sync' to write cached writes to persistent storage"
|
|
sync
|
|
[[ $v -ge 1 ]] && printf "$FMT_MESSAGE" "Done!"
|
|
}
|
|
|
|
|
|
# HELP
|
|
show_help()
|
|
{
|
|
printf "\e[33mArgument Short Action:\e[0m
|
|
--help -h Show this.
|
|
--settings -s Show current settings.
|
|
|
|
--config [name] -c [name] Use sender, receiver, paths from config file with [name].
|
|
--show-config [name] Print variables defined by a config with [name].
|
|
|
|
--sender [path] -s [path] Sender directory, with trailing slash! Defaults to the current working directory.
|
|
--receiver [path] -r [path] Receiver directory, with trailing slash!.
|
|
--reverse Swap receiver and sender.
|
|
|
|
--encrypt Encrypt files before syncing.
|
|
--decrypt Decrypt files before syncing.
|
|
--check-date Only process files that have been modified since the program was last run. Needs -c and (--encrypt/--decrypt).
|
|
--mkrypt-flag [arg] Additional argument for mkrypt.
|
|
|
|
--delete -d Delete files that exist on receiver, but not on sender. Does not work with --encrypt/--decrypt.
|
|
--skip-dryrun Delete without asking first.
|
|
--rsync-flag [arg] Additional argument for rsync.
|
|
|
|
--verbose -v Increase verbosity.
|
|
--silent Decrease verbosity.
|
|
--debug Maximum verbosity.
|
|
|
|
--exclude Interpret positional arguments as blacklist
|
|
|
|
Positional arguments are:
|
|
- if using a config: a string that must be in the configs paths: eg. 'foo' will include ~/foo but not ~/bar
|
|
- if not using a config: paths to sync (must be relative to --sender)
|
|
|
|
See the manpage for more information.
|
|
"
|
|
}
|
|
|
|
|
|
# SETTINGS
|
|
show_settings()
|
|
{
|
|
printf "\e[1mThe current settings are:\e[0m\n"
|
|
printf "$FMT_CONFIG" "\$TMP_DIR " "$TMP_DIR"
|
|
printf "$FMT_CONFIG" "\$TMP_FILE " "$TMP_FILE"
|
|
printf "$FMT_CONFIG" "\$CONFIG_DIR " "$CONFIG_DIR"
|
|
[[ -f $mkrypt ]] || mkrypt_installed="(file not found)"
|
|
printf "$FMT_CONFIG" "\$mkrypt " "$mkrypt $mkrypt_installed"
|
|
printf "$FMT_CONFIG" "\$rsync_flags " "${rsync_flags[*]}"
|
|
}
|
|
|
|
show_config()
|
|
{
|
|
[[ ! -f $CONFIG_DIR/$config ]] && { printf "$FMT_ERROR" "Invalid path: $CONFIG_DIR/$config"; exit 1; }
|
|
source $CONFIG_DIR/$config
|
|
printf "\e[1m$CONFIG_DIR/$config:\e[0m\n"
|
|
printf "$FMT_CONFIG" "\$sender " "$sender"
|
|
printf "$FMT_CONFIG" "\$receiver " "$receiver"
|
|
printf "$FMT_CONFIG" "\$paths " "${paths[*]}"
|
|
printf "$FMT_CONFIG" "\$use_encryption " "$use_encryption"
|
|
printf "$FMT_CONFIG" "Last run: $date"
|
|
}
|
|
|
|
# PARSE ARGS
|
|
if [ -z $1 ]; then
|
|
show_help
|
|
exit 0
|
|
fi
|
|
|
|
all=1
|
|
paths_2=()
|
|
v=1 # verbosity
|
|
while (( "$#" )); do
|
|
case "$1" in
|
|
-h|--help)
|
|
show_help
|
|
exit 0 ;;
|
|
--settings)
|
|
show_settings
|
|
exit 0 ;;
|
|
-c|--config)
|
|
if [[ -n $2 && ${2:0:1} != "-" ]]; then
|
|
config=$2
|
|
shift 2
|
|
else printf "$FMT_ERROR" "Missing argument for $1"; exit 1; fi ;;
|
|
--show-config)
|
|
if [[ -n $2 && ${2:0:1} != "-" ]]; then
|
|
config=$2
|
|
show_config
|
|
exit 0
|
|
else printf "$FMT_ERROR" "Missing argument for $1"; exit 1; fi ;;
|
|
-s|--sender)
|
|
if [[ -n $2 && ${2:0:1} != "-" ]]; then
|
|
sender_2=$2
|
|
shift 2
|
|
else printf "$FMT_ERROR" "Missing argument for $1"; exit 1; fi ;;
|
|
-r|--receiver)
|
|
if [[ -n $2 && ${2:0:1} != "-" ]]; then
|
|
receiver_2=$2
|
|
shift 2
|
|
else printf "$FMT_ERROR" "Missing argument for $1"; exit 1; fi ;;
|
|
--reverse)
|
|
reverse=1
|
|
shift ;;
|
|
--encrypt)
|
|
encrypt=1
|
|
use_encryption=1
|
|
shift ;;
|
|
--decrypt)
|
|
decrypt=1
|
|
use_encryption=1
|
|
shift;;
|
|
-d|--delete)
|
|
delete=--delete
|
|
shift ;;
|
|
--skip-dryrun)
|
|
skip_dryrun=1
|
|
shift ;;
|
|
--rsync-flag)
|
|
if [[ -n ${2} ]]; then # && "${2:0:1}" != "-" ]]; then # possibly problematic if $2 is an msynk flag
|
|
rsync_flags+=("${2}")
|
|
shift 2
|
|
else printf "$FMT_ERROR" "Missing argument for $1"; exit 1; fi ;;
|
|
--check-date)
|
|
check_date=1
|
|
shift ;;
|
|
--mkrypt-flag)
|
|
if [[ -n ${2} ]]; then # && "${2:0:1}" != "-" ]]; then # possibly problematic if $2 is an msynk flag
|
|
mkrypt_flags+=("${2}")
|
|
shift 2
|
|
else printf "$FMT_ERROR" "Missing argument for $1"; exit 1; fi ;;
|
|
-v|--verbose)
|
|
v=2
|
|
rsync_flags+=(-v)
|
|
mkrypt_flags+=(-v)
|
|
shift ;;
|
|
--silent)
|
|
v=0
|
|
mkrypt_flags+=(--silent)
|
|
shift ;;
|
|
--debug)
|
|
v=3
|
|
rsync_flags+=(-v)
|
|
mkrypt_flags+=(-v)
|
|
shift;;
|
|
--exclude)
|
|
exclude=1
|
|
shift ;;
|
|
-R|--relative)
|
|
relative=1
|
|
shift ;;
|
|
-*|--*=) # unsupported flags
|
|
printf "$FMT_ERROR" "Unsupported flag $1" >&2
|
|
exit 1 ;;
|
|
*) # everything that does not have a - is interpreted as filepath
|
|
all=0
|
|
paths_2+=($1)
|
|
shift ;;
|
|
esac
|
|
done
|
|
|
|
# PREPARATION & CHECKS
|
|
# clear $TMP_DIR: might be non-empty if previous msynk failed
|
|
[[ $TMP_DIR == *msynk* ]] && {
|
|
rm -rf $TMP_DIR/* || { printf "$FMT_ERROR" "Can not contents of \$TMP_DIR: $TMP_DIR"; exit 1; }
|
|
} || { printf "$FMT_ERROR" "Will not delete \$TMP_DIR because it might be dangerous (path does not contain 'msynk'): $TMP_DIR"; exit 1; }
|
|
|
|
# check if rsync is installed
|
|
if ! command -v rsync &> /dev/null; then
|
|
printf "$FMT_ERROR" "rsync is not installed."
|
|
exit 1
|
|
fi
|
|
|
|
# if using encryption, check if mkrypt is installed
|
|
[[ $use_encryption = 1 && ! -f $mkrypt ]] && { printf "$FMT_ERROR" "Can not use encryption: mkrypt is not installed at given path: $mkrypt."; exit 1; }
|
|
|
|
|
|
# if using a config
|
|
if [[ -n $config ]]; then
|
|
source $CONFIG_DIR/$config || { printf "$FMT_ERROR" "Error running the config file: $CONFIG_DIR/$config"; exit 1; }
|
|
if [[ $use_encryption = 1 ]]; then
|
|
[[ -z $reverse ]] && encrypt=1 || decrypt=1
|
|
fi
|
|
[[ $v -ge 3 ]] && printf "$FMT_MESSAGE" "Loaded config:" "$CONFIG_DIR/$config: date:$date sender:$sender receiver:$receiver use_encryption:$use_encryption encrypt:$encrypt decrypt:$decrypt paths:${paths[*]}"
|
|
fi
|
|
|
|
# overwrite stuff from the config if anything else was given / set variables of no config was given
|
|
[[ -n $sender_2 ]] && sender=$sender_2
|
|
[[ -n $receiver_2 ]] && receiver=$receiver_2
|
|
[[ -z $sender ]] && {
|
|
[[ $v -ge 1 ]] && printf "$FMT_MESSAGE" "Missing sender:" "Using working directory: $(pwd)"
|
|
sender="$(pwd)/"
|
|
}
|
|
|
|
# reverse: swap sender and reveiver
|
|
if [[ -n $reverse ]]; then
|
|
tmp_sender=$sender
|
|
sender=$receiver
|
|
receiver=$tmp_sender
|
|
fi
|
|
|
|
# make sure receiver has trailing slash
|
|
[[ $receiver != *"/" ]] && receiver=$receiver"/"
|
|
# create receiver dir if its local
|
|
[[ ! $receiver == *:* ]] && { mkdir -p $receiver || { printf "$FMT_ERROR" "Can not create receiver directory: $receiver"; exit 1; }; }
|
|
|
|
[[ $v -ge 3 ]] && printf "$FMT_MESSAGE" "Variables:" "sender:$sender receiver:$receiver paths:${paths[*]} paths_2:${paths_2[*]}"
|
|
|
|
# filter paths:
|
|
# - if using config: all paths from config that have a pos.arg in them
|
|
# - else: pos.args, must be relative to specified --sender
|
|
filtered_paths=()
|
|
ifs=$IFS
|
|
IFS=$'\n'
|
|
if [[ -n $config ]]; then
|
|
[[ $sender != *"/" ]] && sender=$sender"/"
|
|
for path in "${paths[@]}"; do
|
|
if check_path_in_args; then
|
|
# if reverse and encryption (use_encryption might be overwritten by config when --decrypt is used) and file, it will have a .gpg extension
|
|
[[ -n $reverse && ($use_encryption = 1 || $decrypt = 1) && "$path" != *"/" ]] && path="$path".gpg
|
|
filtered_paths+=("$sender$path")
|
|
fi
|
|
done
|
|
else
|
|
for path in ${paths_2[@]}; do # add slash if directory
|
|
[[ -d "$path" ]] && [[ "$path" != *"/" ]] && path="$path/"
|
|
filtered_paths+=("$sender$path")
|
|
done
|
|
fi
|
|
[[ $v -ge 3 ]] && printf "$FMT_MESSAGE" "Filtered paths:" "${filtered_paths[*]}"
|
|
IFS=$ifs
|
|
|
|
# apply correct base flags
|
|
if [[ -n $relative ]]; then
|
|
rsync_flags+=(--relative)
|
|
fi
|
|
|
|
# sanity checks
|
|
[[ -z $filtered_paths ]] && { printf "$FMT_ERROR" "Missing valid paths."; exit 1; }
|
|
[[ -z $receiver ]] && { printf "$FMT_ERROR" "Missing receiver. Specifiy with --receiver"; exit 1; }
|
|
[[ -n $delete && ( -n $decrypt || -n $encrypt ) ]] && { printf "$FMT_ERROR" "Can not use --delete and encryption at the same time."; exit 1; }
|
|
|
|
# if using --check-date
|
|
if [[ -n $check_date && -z $date ]]; then # if --check-date but date is not set, use 0 (unix time)
|
|
[[ $v -ge 1 ]] && printf "$FMT_MESSAGE" "--check-date:" "passed but date not set in config, using 0 (unix time)"
|
|
date=$(date --iso=sec -d @0)
|
|
fi
|
|
[[ -n $check_date ]] && { mkrypt_flags+=(--check-date "${date}"); rsync_flags+=(--times); }
|
|
|
|
# RUN!
|
|
sync_sender_to_receiver
|
|
exit 0
|