#!/bin/bash # Copyright © 2022 Matthias Quintern. # This software comes with no warranty. # This software is licensed under the GPL3 # ABOUT # Synchronizes files from a sender to a receiver using rsync # The files can be encrypted before or decrypted after the transfer using the mkrypt script # SETTINGS # The directory that is searched for config files, use ~/.config if $XDG_CONFIG_HOME is not defined CONFIG_DIR=~/.config/msynk [[ -n $XDG_CONFIG_HOME ]] && CONFIG_DIR=$XDG_CONFIG_HOME/msynk [[ -n $MSYNK_CONFIG_HOME ]] && CONFIG_DIR=$MSYNK_CONFIG_HOME # YOU CAN OVERRIDE THESE IN THE CONFIG FILES # When using encrypt/decrypt, the source is en/decrypted to TMP_DIR and then rsynced to receiver from there Path must contain 'msynk' TMP_DIR=/tmp/msynk/ When using --delete, this file is used to determine the files that will be deleted TMP_FILE=/tmp/msynk_file Path to the mkrypt script mkrypt=/usr/bin/mkrypt # Path must contain 'msynk' TMP_DIR=/tmp/msynk/ # When using --delete, this file is used to determine the files that will be deleted TMP_FILE=/tmp/msynk_file # Path to the mkrypt script mkrypt=/usr/bin/mkrypt # Additional flags for mkrypt mkrypt_flags=() rsync_flags=(-ruh) # rsync_flags+=(--rsh="ssh -p 42") # r - relative # v - verbose # Ut - preserve modification&access times # u - update, skip files that are newer on the receiver # P - show progress and keep partially transferred files # h - human readable # see "man rsync" for more # expand * to hidden files as well shopt -s dotglob # UTILITY NAME="\e[1;34mmsynk" FMT_MESSAGE="$NAME: \e[0;34m%s\e[0m %s\n" FMT_ERROR="\e[1;31m$NAME: \e[1;31mERROR: \e[0m%s\n" # exit 1: error, exit 2: rsync exited non 0, exit 3: mkrypt exited non 0 FMT_SYNC="$NAME: \e[1;32mSyncing: \e[0m%s\n" FMT_CMD="$NAME: \e[1;33mRunning: \e[0m%s\n" FMT_CONFIG="\e[34m%s\e:\t\e[1;33m%s\e[0m\n" # FUNCTIONS # silence commands quiet() { "$@" > /dev/null 2>&1; } # check if a passed parameter is contained in file # path must be set before calling check_path_in_args() { if [ $all = 1 ]; then return 0 fi for string in ${paths_2[@]}; do if [[ $path == *$string* ]]; then if [[ $exclude = 1 ]]; then return 1 else return 0 fi fi done if [[ $exclude = 1 ]]; then return 0 else return 1 fi } # SYNC SENDER TO RECEIVER sync_sender_to_receiver() { # perform a dry run to see which files would be deleted if [[ -n $delete && -z $skip_dryrun ]]; then [[ $v -ge 1 ]] && printf "$FMT_MESSAGE" "Performing dryrun" "to generate list of files that will be deleted from $receiver..." echo "" > $TMP_FILE for path in "${filtered_paths[@]}"; do dest_subdir=$receiver$(basename $path) rsync "${rsync_flags[@]}" -v --dry-run --delete $path $dest_subdir | grep deleting | sed "s(deleting (${BACKUP_SUBDIR}/(" >> $TMP_FILE done # print files that will be deleted and ask if continue # grep deleting $TMP_FILE | sed "s(deleting (${receiver}/(" | less less $TMP_FILE DONE=0 while [[ $DONE == 0 ]]; do printf "$FMT_MESSAGE" "The listed files will be deleted from $receiver." printf "Proceed?\e[34m ([y]es, [s]how again, [*] no)\e[0m: " # printf "Proceed?\e[34m ([y]es, [s]how again, [l]ist all files, [*] no)\e[0m: " read answer case $answer in y|Y) DONE=1 ;; s|S) less $TMP_FILE ;; # l|L) # less $TMP_FILE # ;; *) printf "$FMT_MESSAGE" "Cancelled." exit 0 ;; esac done fi # actual syncing for path in "${filtered_paths[@]}"; do [[ $v -ge 1 ]] && printf "$FMT_SYNC" "$path" if [[ -d "$path" ]]; then dest="$receiver"$(basename "$path") elif [[ -f $path ]]; then dest="$receiver" elif [[ "$sender" == *:* ]]; then # if sender is remote, assume the path is a dir if it has a slash and a file otherwise [[ "$path" == *"/" ]] && dest="$receiver"$(basename "$path") || dest="$receiver" else printf "$FMT_ERROR" "Invalid path: $path"; exit 1 fi if [[ -n $encrypt || -n $decrypt ]]; then mkdir -p "$TMP_DIR" tmp_source="$TMP_DIR" if [[ -n $encrypt ]]; then # check if TMP_DIR has enough space tmp_free=$(df --output=avail $TMP_DIR | grep -E "[[:digit:]]+") path_size=$(du -s "$path" | awk '{print $1}' | sed -e "s/[^0-9]//g") if [[ $tmp_free -le $path_size ]]; then printf "$FMT_ERROR" "Not enough space on $TMP_DIR to copy $path! $TMP_DIR $tmp_free free - Size $path: $path_size" printf "$FMT_MESSAGE" "" "If using a tmpfs, you can increase its size by running: 'sudo mount -o remount,size=XXG,noatime /tmp' to increase the size to XXG (You will need a large enough swap space)" exit 1 fi [[ -f "$path" ]] && tmp_source="$TMP_DIR"$(basename "$path").gpg [[ $v -ge 3 ]] && printf "$FMT_CMD" "bash $mkrypt --encrypt $path --output $TMP_DIR ${mkrypt_flags[*]}" bash $mkrypt --encrypt "$path" --output "$TMP_DIR" ${mkrypt_flags[@]} || exit 3 [[ $v -ge 3 ]] && printf "$FMT_CMD" "rsync ${rsync_flags[*]} $tmp_source $dest" rsync "${rsync_flags[@]}" "$tmp_source" "$dest" || { printf "$FMT_ERROR" "rsync exited with exit code $?"; exit 2; } else [[ -f $path ]] && tmp_source=$TMP_DIR$(basename $path) [[ $v -ge 3 ]] && printf "$FMT_CMD" "rsync ${rsync_flags[*]} $path $TMP_DIR" rsync "${rsync_flags[@]}" "$path" "$TMP_DIR" || { printf "$FMT_ERROR" "rsync exited with exit code $?"; exit 2; } [[ $v -ge 3 ]] && printf "$FMT_CMD" "bash $mkrypt --decrypt $tmp_source --output $(readlink -f $dest) ${mkrypt_flags[*]}" bash $mkrypt --decrypt "$tmp_source" --output "$(readlink -f "$dest")" ${mkrypt_flags[@]} || exit 3 fi else [[ $v -ge 3 ]] && printf "$FMT_CMD" "rsync ${rsync_flags[*]} $delete $path $dest" rsync "${rsync_flags[@]}" $delete "$path" "$dest" || { printf "$FMT_ERROR" "rsync exited with exit code $?"; exit 2; } fi [[ "$TMP_DIR" == *msynk* ]] && { rm -rf "$TMP_DIR"/* || { printf "$FMT_ERROR" "Can not delete contents of \$TMP_DIR: $TMP_DIR"; exit 1; } } || { printf "$FMT_ERROR" "Will not delete \$TMP_DIR because it might be dangerous (path does not contain 'msynk'): $TMP_DIR"; exit 1; } done # put todays date in the config if [[ -f $CONFIG_DIR/$config ]]; then if grep -xq "date=.*" $CONFIG_DIR/$config; then [[ $v -ge 2 ]] && printf "$FMT_MESSAGE" "Updating" "date in $CONFIG_DIR/$config." sed "s/date=.*/date=\"$(date --iso=sec)\"/" $CONFIG_DIR/$config -i else [[ $v -ge 2 ]] && printf "$FMT_MESSAGE" "Writing" "current date to $CONFIG_DIR/$config." echo "date=\"$(date --iso=sec)\"" >> $CONFIG_DIR/$config fi fi [[ $v -ge 2 ]] && printf "$FMT_MESSAGE" "Running:" "'sync' to write cached writes to persistent storage" sync [[ $v -ge 1 ]] && printf "$FMT_MESSAGE" "Done!" } # HELP show_help() { printf "\e[33mArgument Short Action:\e[0m --help -h Show this. --settings -s Show current settings. --config [name] -c [name] Use sender, receiver, paths from config file with [name]. --show-config [name] Print variables defined by a config with [name]. --sender [path] -s [path] Sender directory, with trailing slash! Defaults to the current working directory. --receiver [path] -r [path] Receiver directory, with trailing slash!. --reverse Swap receiver and sender. --encrypt Encrypt files before syncing. --decrypt Decrypt files before syncing. --check-date Only process files that have been modified since the program was last run. Needs -c and (--encrypt/--decrypt). --mkrypt-flag [arg] Additional argument for mkrypt. --delete -d Delete files that exist on receiver, but not on sender. Does not work with --encrypt/--decrypt. --skip-dryrun Delete without asking first. --rsync-flag [arg] Additional argument for rsync. --verbose -v Increase verbosity. --silent Decrease verbosity. --debug Maximum verbosity. --exclude Interpret positional arguments as blacklist Positional arguments are: - if using a config: a string that must be in the configs paths: eg. 'foo' will include ~/foo but not ~/bar - if not using a config: paths to sync (must be relative to --sender) See the manpage for more information. " } # SETTINGS show_settings() { printf "\e[1mThe current settings are:\e[0m\n" printf "$FMT_CONFIG" "\$TMP_DIR " "$TMP_DIR" printf "$FMT_CONFIG" "\$TMP_FILE " "$TMP_FILE" printf "$FMT_CONFIG" "\$CONFIG_DIR " "$CONFIG_DIR" [[ -f $mkrypt ]] || mkrypt_installed="(file not found)" printf "$FMT_CONFIG" "\$mkrypt " "$mkrypt $mkrypt_installed" printf "$FMT_CONFIG" "\$rsync_flags " "${rsync_flags[*]}" } show_config() { [[ ! -f $CONFIG_DIR/$config ]] && { printf "$FMT_ERROR" "Invalid path: $CONFIG_DIR/$config"; exit 1; } source $CONFIG_DIR/$config printf "\e[1m$CONFIG_DIR/$config:\e[0m\n" printf "$FMT_CONFIG" "\$sender " "$sender" printf "$FMT_CONFIG" "\$receiver " "$receiver" printf "$FMT_CONFIG" "\$paths " "${paths[*]}" printf "$FMT_CONFIG" "\$use_encryption " "$use_encryption" printf "$FMT_CONFIG" "Last run: $date" } # PARSE ARGS if [ -z $1 ]; then show_help exit 0 fi all=1 paths_2=() v=1 # verbosity while (( "$#" )); do case "$1" in -h|--help) show_help exit 0 ;; --settings) show_settings exit 0 ;; -c|--config) if [[ -n $2 && ${2:0:1} != "-" ]]; then config=$2 shift 2 else printf "$FMT_ERROR" "Missing argument for $1"; exit 1; fi ;; --show-config) if [[ -n $2 && ${2:0:1} != "-" ]]; then config=$2 show_config exit 0 else printf "$FMT_ERROR" "Missing argument for $1"; exit 1; fi ;; -s|--sender) if [[ -n $2 && ${2:0:1} != "-" ]]; then sender_2=$2 shift 2 else printf "$FMT_ERROR" "Missing argument for $1"; exit 1; fi ;; -r|--receiver) if [[ -n $2 && ${2:0:1} != "-" ]]; then receiver_2=$2 shift 2 else printf "$FMT_ERROR" "Missing argument for $1"; exit 1; fi ;; --reverse) reverse=1 shift ;; --encrypt) encrypt=1 use_encryption=1 shift ;; --decrypt) decrypt=1 use_encryption=1 shift;; -d|--delete) delete=--delete shift ;; --skip-dryrun) skip_dryrun=1 shift ;; --rsync-flag) if [[ -n ${2} ]]; then # && "${2:0:1}" != "-" ]]; then # possibly problematic if $2 is an msynk flag rsync_flags+=("${2}") shift 2 else printf "$FMT_ERROR" "Missing argument for $1"; exit 1; fi ;; --check-date) check_date=1 shift ;; --mkrypt-flag) if [[ -n ${2} ]]; then # && "${2:0:1}" != "-" ]]; then # possibly problematic if $2 is an msynk flag mkrypt_flags+=("${2}") shift 2 else printf "$FMT_ERROR" "Missing argument for $1"; exit 1; fi ;; -v|--verbose) v=2 rsync_flags+=(-v) mkrypt_flags+=(-v) shift ;; --silent) v=0 mkrypt_flags+=(--silent) shift ;; --debug) v=3 rsync_flags+=(-v) mkrypt_flags+=(-v) shift;; --exclude) exclude=1 shift ;; -*|--*=) # unsupported flags printf "$FMT_ERROR" "Unsupported flag $1" >&2 exit 1 ;; *) # everything that does not have a - is interpreted as filepath all=0 paths_2+=($1) shift ;; esac done # PREPARATION & CHECKS # clear $TMP_DIR: might be non-empty if previous msynk failed [[ $TMP_DIR == *msynk* ]] && { rm -rf $TMP_DIR/* || { printf "$FMT_ERROR" "Can not contents of \$TMP_DIR: $TMP_DIR"; exit 1; } } || { printf "$FMT_ERROR" "Will not delete \$TMP_DIR because it might be dangerous (path does not contain 'msynk'): $TMP_DIR"; exit 1; } # check if rsync is installed if ! command -v rsync &> /dev/null; then printf "$FMT_ERROR" "rsync is not installed." exit 1 fi # if using encryption, check if mkrypt is installed [[ $use_encryption = 1 && ! -f $mkrypt ]] && { printf "$FMT_ERROR" "Can not use encryption: mkrypt is not installed at given path: $mkrypt."; exit 1; } # if using a config if [[ -n $config ]]; then source $CONFIG_DIR/$config || { printf "$FMT_ERROR" "Error running the config file: $CONFIG_DIR/$config"; exit 1; } if [[ $use_encryption = 1 ]]; then [[ -z $reverse ]] && encrypt=1 || decrypt=1 fi [[ $v -ge 3 ]] && printf "$FMT_MESSAGE" "Loaded config:" "$CONFIG_DIR/$config: date:$date sender:$sender receiver:$receiver use_encryption:$use_encryption encrypt:$encrypt decrypt:$decrypt paths:${paths[*]}" fi # overwrite stuff from the config if anything else was given / set variables of no config was given [[ -n $sender_2 ]] && sender=$sender_2 [[ -n $receiver_2 ]] && receiver=$receiver_2 [[ -z $sender ]] && { [[ $v -ge 1 ]] && printf "$FMT_MESSAGE" "Missing sender:" "Using working directory: $(pwd)" sender="$(pwd)/" } # reverse: swap sender and reveiver if [[ -n $reverse ]]; then tmp_sender=$sender sender=$receiver receiver=$tmp_sender fi # make sure receiver has trailing slash [[ $receiver != *"/" ]] && receiver=$receiver"/" # create receiver dir if its local [[ ! $receiver == *:* ]] && { mkdir -p $receiver || { printf "$FMT_ERROR" "Can not create receiver directory: $receiver"; exit 1; }; } [[ $v -ge 3 ]] && printf "$FMT_MESSAGE" "Variables:" "sender:$sender receiver:$receiver paths:${paths[*]} paths_2:${paths_2[*]}" # filter paths: # - if using config: all paths from config that have a pos.arg in them # - else: pos.args, must be relative to specified --sender filtered_paths=() ifs=$IFS IFS=$'\n' if [[ -n $config ]]; then [[ $sender != *"/" ]] && sender=$sender"/" for path in "${paths[@]}"; do if check_path_in_args; then # if reverse and encryption (use_encryption might be overwritten by config when --decrypt is used) and file, it will have a .gpg extension [[ -n $reverse && ($use_encryption = 1 || $decrypt = 1) && "$path" != *"/" ]] && path="$path".gpg filtered_paths+=("$sender$path") fi done else for path in ${paths_2[@]}; do # add slash if directory [[ -d "$path" ]] && [[ "$path" != *"/" ]] && path="$path/" filtered_paths+=("$sender$path") done fi [[ $v -ge 3 ]] && printf "$FMT_MESSAGE" "Filtered paths:" "${filtered_paths[*]}" IFS=$ifs # sanity checks [[ -z $filtered_paths ]] && { printf "$FMT_ERROR" "Missing valid paths."; exit 1; } [[ -z $receiver ]] && { printf "$FMT_ERROR" "Missing receiver. Specifiy with --receiver"; exit 1; } [[ -n $delete && ( -n $decrypt || -n $encrypt ) ]] && { printf "$FMT_ERROR" "Can not use --delete and encryption at the same time."; exit 1; } # if using --check-date if [[ -n $check_date && -z $date ]]; then # if --check-date but date is not set, use 0 (unix time) [[ $v -ge 1 ]] && printf "$FMT_MESSAGE" "--check-date:" "passed but date not set in config, using 0 (unix time)" date=$(date --iso=sec -d @0) fi [[ -n $check_date ]] && { mkrypt_flags+=(--check-date "${date}"); rsync_flags+=(--times); } # RUN! sync_sender_to_receiver exit 0