From dee16cf699f626bb0f193a8d3cdf6a1dc9c8fbf2 Mon Sep 17 00:00:00 2001 From: "matthias@arch" Date: Fri, 11 Mar 2022 21:07:56 +0100 Subject: [PATCH] Initial commit --- PKGBUILD | 21 ++++ README.md | 99 ++++++++++++++++ msynk.1 | 196 ++++++++++++++++++++++++++++++++ msynk.sh | 331 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 647 insertions(+) create mode 100644 PKGBUILD create mode 100644 README.md create mode 100644 msynk.1 create mode 100755 msynk.sh diff --git a/PKGBUILD b/PKGBUILD new file mode 100644 index 0000000..3c8d99c --- /dev/null +++ b/PKGBUILD @@ -0,0 +1,21 @@ +# Maintainer: Matthias Quintern +pkgname=msynk +pkgver=1.0 +pkgrel=1 +pkgdesc="rsync helper that supports encryption and presets" +arch=('any') +url="https:/github.com/MatthiasQuintern/msynk" +license=('GPL3') +depends=('rsync') +optdepends=('mkrypt: for encryption') +source=(msynk.sh) +md5sums=(866221e5be1842d4b0de2c9f313b06cb) + +package() { + mkdir -p "${pkgdir}/usr/bin" + cp "${srcdir}/msynk.sh" "${pkgdir}/usr/bin/msynk" + chmod +x "${pkgdir}/usr/bin/msynk" + + mkdir -p "${pkgdir}/usr/share/local/man/man1/" + cp "msynk.1" "${pkgdir}/usr/share/local/man/man1/msynk.1" +} diff --git a/README.md b/README.md new file mode 100644 index 0000000..536082e --- /dev/null +++ b/README.md @@ -0,0 +1,99 @@ +% MSYNK(1) msynk 1.0 +% Matthias Quintern +% March 2022 + +# NAME +msynk - sync files locally or between devices + +# SYNOPSIS +| Local: +| **msynk** [OPTION...] -r DEST_DIR -s SOURCE_DIR PATHS... + +| Via remote shell: +| Push: +| **msynk** \[OPTION...] -r [USER@]HOST:DEST_DIR -s SOURCE_DIR PATHS... +| Pull: +| **msynk** [OPTION...] -r DEST_DIR -s [USER@]HOST:SOURCE_DIR PATHS... + +| Using a configuration file: +| **msynk** [OPTION...] -c CONFIG NAME [SELECTIONS...] + +# DESCRIPTION +**msynk** uses *rsync* to synchronise files or directories locally or between devices. +In general, you should always add a trailing '/' to directories. + +## Encryption +Files can also be encrypted or decrypted using *mkrypt*. If encryption is used, the files are encrypted to a temporary directory and then synced with *rsync*. +If you sync encrypted files to your system, the encrypted files are transferred with *rsync* to a temporary directory and then decrypted to your filesystem. + +## Configuration Files +**msynk** can read from config files usually stored in ~/.config/msynk. +The config files are shellscripts run with bash to set certain environent variables. These are:\ +- paths: array containing all paths (relative to sender) that should be synced\ +- sender: the parent directory of the paths\ +- receiver: the path where the files should be synced to\ +- rsync_flags: array containing all the flags for rsync. If you just want to add a flag, use ´rsync_flags+=(-yourflag)\ +- use_encryption: 0 for no, 1 for yes + +### Example +To sync the directories "/home/user/foo" "/home/user/bar" and the file "/home/user/file.txt" to a remote server "user@foobar.com:~/backup" which is available with ssh at port 42, you could have a configuration file "my_backup":\ + sender=/home/user/\ + receiver=user@foobar.com:~/backup/\ + paths=(foo/ bar/ file.txt)\ + rsync_flags+=(--rsh="ssh -p 42")\ +You can then run this with: "msynk -c my_backup"\, or if you only want to sync the "foo" directory and ".txt" files: "msynk -c my_backup foo .txt" + +# OPTIONS +**-h**, **--help** +: Show a list of arguments. + +**--settings** +: Show the current settings. + +**-c**, **--config** config-name +: Retrieve settings from config file ~/.config/msynk/*config-name* + +**-s**, **--sender** path +: Set sender to directory. This can also be a remote, like user@domain.com:/dir. Defaults to to the current working directory. + +**-r**, **--receiver** path +: Set the receiver directory. All files and directories will be placed inside this directory. This can also be a remote, like user@domain.com:/dir. This option always needs to be set. + +**--reverse** +: Swaps sender and receiver. Useful when you want to reverse a config file. + +**--encrypt** +: Encrypt files with *mkrypt* before sending them to the receiver. +: The files are encrypted to $TMP_DIR and then synced to the receiver. Does not work with --delete. + +**--decrypt** +: Decrypt files with *mkrypt* after receiving them from the sender. Assumes that all files are encrypted and fails if that is not the case. +: The files are synced to $TMP_DIR and then decrypted to the receiver directory. Does not work with --delete. + +**--mkrypt-flags** flags +: Additional flags for *mkrypt*. You can use this to set the key for the encryption. + +**-d**, **--delete** +: *rsync* --delete option. Deletes all files on the receiver that do not exist on the sender. *msynk* will open *less* with a list of all files that will be deleted and prompt you wether you want continue (unless --skip-dryrun is set) + +**--skip-dryrun** +: Can be used with --delete. This skips listing the files that will be deleted and the prompt. + +**--rsync-flags** flags +: Additional flags for *rsync*. You can use this to set the port for the connection. + +**-v**, **--verbose** +: Increase verbosity: Passes -v to rsync and mkrypt. + +**--silent** +: Decrease verbosity: Print only error messages and passes --silent to mkrypt. + +**--debug** +: Maximum verbosity: -v but also print out rsync and mkrypt commands. + +**positional arguments** +: If you are using a config file: pos. args. are strings that have to be contained in a path in order for it to be synced. If no pos. args. are given, all files are synced. +: If you are not using a config file: pos. args. are filepaths relative to the sender directory. + +# COPYRIGHT +Alles meins!!! diff --git a/msynk.1 b/msynk.1 new file mode 100644 index 0000000..d5d6e3a --- /dev/null +++ b/msynk.1 @@ -0,0 +1,196 @@ +.\" Automatically generated by Pandoc 2.14.2 +.\" +.TH "MSYNK" "1" "March 2022" "msynk 1.0" "" +.hy +.SH NAME +.PP +msynk - sync files locally or between devices +.SH SYNOPSIS +.PP +Local: +.PD 0 +.P +.PD +\ \ \ \f[B]msynk\f[R] [OPTION\&...] -r DEST_DIR -s SOURCE_DIR PATHS\&... +.PP +Via remote shell: +.PD 0 +.P +.PD +\ \ \ \ Push: +.PD 0 +.P +.PD +\ \ \ \ \ \ \ \ \f[B]msynk\f[R] [OPTION\&...] -r +[USER\[at]]HOST:DEST_DIR -s SOURCE_DIR PATHS\&... +.PD 0 +.P +.PD +\ \ \ \ Pull: +.PD 0 +.P +.PD +\ \ \ \ \ \ \ \ \f[B]msynk\f[R] [OPTION\&...] -r DEST_DIR -s +[USER\[at]]HOST:SOURCE_DIR PATHS\&... +.PP +Using a configuration file: +.PD 0 +.P +.PD +\ \ \ \ \f[B]msynk\f[R] [OPTION\&...] -c CONFIG NAME [SELECTIONS\&...] +.SH DESCRIPTION +.PP +\f[B]msynk\f[R] uses \f[I]rsync\f[R] to synchronise files or directories +locally or between devices. +In general, you should always add a trailing `/' to directories. +.SS Encryption +.PP +Files can also be encrypted or decrypted using \f[I]mkrypt\f[R]. +If encryption is used, the files are encrypted to a temporary directory +and then synced with \f[I]rsync\f[R]. +If you sync encrypted files to your system, the encrypted files are +transferred with \f[I]rsync\f[R] to a temporary directory and then +decrypted to your filesystem. +.SS Configuration Files +.PP +\f[B]msynk\f[R] can read from config files usually stored in +\[ti]/.config/msynk. +The config files are shellscripts run with bash to set certain +environent variables. +These are: +.PD 0 +.P +.PD +- paths: array containing all paths (relative to sender) that should be +synced +.PD 0 +.P +.PD +- sender: the parent directory of the paths +.PD 0 +.P +.PD +- receiver: the path where the files should be synced to +.PD 0 +.P +.PD +- rsync_flags: array containing all the flags for rsync. +If you just want to add a flag, use \[aa]rsync_flags+=(-yourflag) +.PD 0 +.P +.PD +- use_encryption: 0 for no, 1 for yes +.SS Example +.PP +To sync the directories \[lq]/home/user/foo\[rq] +\[lq]/home/user/bar\[rq] and the file \[lq]/home/user/file.txt\[rq] to a +remote server \[lq]user\[at]foobar.com:\[ti]/backup\[rq] which is +available with ssh at port 42, you could have a configuration file +\[lq]my_backup\[rq]: +.PD 0 +.P +.PD +sender=/home/user/ +.PD 0 +.P +.PD +receiver=user\[at]foobar.com:\[ti]/backup/ +.PD 0 +.P +.PD +paths=(foo/ bar/ file.txt) +.PD 0 +.P +.PD +rsync_flags+=(\[em]-rsh=\[lq]ssh -p 42\[rq]) +.PD 0 +.P +.PD +You can then run this with: \[lq]msynk -c my_backup\[rq], or if you only +want to sync the \[lq]foo\[rq] directory and \[lq].txt\[rq] files: +\[lq]msynk -c my_backup foo .txt\[rq] +.SH OPTIONS +.TP +\f[B]-h\f[R], \f[B]\[em]-help\f[R] +Show a list of arguments. +.TP +\f[B]\[em]-settings\f[R] +Show the current settings. +.TP +\f[B]-c\f[R], \f[B]\[em]-config\f[R] config-name +Retrieve settings from config file +\[ti]/.config/msynk/\f[I]config-name\f[R] +.TP +\f[B]-s\f[R], \f[B]\[em]-sender\f[R] path +Set sender to directory. +This can also be a remote, like user\[at]domain.com:/dir. +Defaults to to the current working directory. +.TP +\f[B]-r\f[R], \f[B]\[em]-receiver\f[R] path +Set the receiver directory. +All files and directories will be placed inside this directory. +This can also be a remote, like user\[at]domain.com:/dir. +This option always needs to be set. +.TP +\f[B]\[em]-reverse\f[R] +Swaps sender and receiver. +Useful when you want to reverse a config file. +.TP +\f[B]\[em]-encrypt\f[R] +Encrypt files with \f[I]mkrypt\f[R] before sending them to the receiver. +The files are encrypted to $TMP_DIR and then synced to the receiver. +Does not work with \[em]-delete. +.TP +\f[B]\[em]-decrypt\f[R] +Decrypt files with \f[I]mkrypt\f[R] after receiving them from the +sender. +Assumes that all files are encrypted and fails if that is not the case. +The files are synced to $TMP_DIR and then decrypted to the receiver +directory. +Does not work with \[em]-delete. +.TP +\f[B]\[em]-mkrypt-flags\f[R] flags +Additional flags for \f[I]mkrypt\f[R]. +You can use this to set the key for the encryption. +.TP +\f[B]-d\f[R], \f[B]\[em]-delete\f[R] +\f[I]rsync\f[R] \[em]-delete option. +Deletes all files on the receiver that do not exist on the sender. +\f[I]msynk\f[R] will open \f[I]less\f[R] with a list of all files that +will be deleted and prompt you wether you want continue (unless +\[en]skip-dryrun is set) +.TP +\f[B]\[em]-skip-dryrun\f[R] +Can be used with \[em]-delete. +This skips listing the files that will be deleted and the prompt. +.TP +\f[B]\[em]-rsync-flags\f[R] flags +Additional flags for \f[I]rsync\f[R]. +You can use this to set the port for the connection. +.TP +\f[B]-v\f[R], \f[B]\[em]-verbose\f[R] +Increase verbosity: Passes -v to rsync and mkrypt. +.TP +\f[B]\[em]-silent\f[R] +Decrease verbosity: Print only error messages and passes \[em]-silent to +mkrypt. +.TP +\f[B]\[em]-debug\f[R] +Maximum verbosity: -v but also print out rsync and mkrypt commands. +.TP +\f[B]positional arguments\f[R] +If you are using a config file: pos. +args. +are strings that have to be contained in a path in order for it to be +synced. +If no pos. +args. +are given, all files are synced. +If you are not using a config file: pos. +args. +are filepaths relative to the sender directory. +.SH COPYRIGHT +.PP +Alles meins!!! +.SH AUTHORS +Matthias Quintern. diff --git a/msynk.sh b/msynk.sh new file mode 100755 index 0000000..0e204c0 --- /dev/null +++ b/msynk.sh @@ -0,0 +1,331 @@ +#!/bin/bash +# Made by Matthias Quintern +# 02/2022 +# This software comes with no warranty. + +# ABOUT +# Synchronizes files from a sender to a receiver using rsync +# The files can be encrypted before or decrypted after the transfer using the mkrypt script + +# +# SETTINGS +# +# The directory that is searched for config files +CONFIG_DIR=~/.config/msynk/ +# When using encrypt/decrypt, the source is en/decrypted to TMP_DIR and then rsynced to receiver from there +# Path must contain 'msynk' +TMP_DIR=/tmp/msynk/ +# When using --delete, this file is used to determine the files that will be deleted +TMP_FILE=/tmp/msynk_file +# Path to the mkrypt script +mkrypt=/usr/bin/mkrypt +# Additional flags for mkrypt +mkrypt_flags= + +rsync_flags=(-ruvh) +# rsync_flags+=(--rsh="ssh -p 69") +# r - relative +# v - verbose +# Ut - preserve modification&access times +# u - update, skip files that are newer on the receiver +# P - show progress and keep partially transferred files +# h - human readable +# see "man rsync" for more + + +# +# UTILITY +# +FMT_MESSAGE="\e[1;34m%s\e[0m\n" +FMT_ERROR="\e[1;31mERROR: \e[0m%s\n" +FMT_SYNC="\e[1;32m:Syncing\e[0m %s\n" +# FMT_UPDATE="\e[1;33mUpdating:\e[0m %s\n" +FMT_CONFIG="\e[34m%s\e:\t\e[1;33m%s\e[0m\n" + +quiet() { "$@" > /dev/null 2>&1; } + +# check if a passed parameter is contained in file +check_path_in_args() +{ + if [ $all = 1 ]; then + return 0 + fi + for string in ${FILES[@]}; do + if [[ $path == *$string* ]]; then + return 0 + fi + done + return 1 +} + + +# +# sync sender to receiver +# +sync_sender_to_receiver() +{ + # perform a dry run to see which files would be deleted + if [[ -n $delete && -z $skip_dryrun ]]; then + [[ $v -ge 1 ]] && printf "$FMT_MESSAGE" "Performing dryrun to generate list of files that will be deleted from $receiver..." + echo "" > $TMP_FILE + for path in "${filtered_paths[@]}"; do + dest_subdir=$receiver$(basename $path) + rsync $rsync_flags -v --dry-run --delete $path $dest_subdir | grep deleting | sed "s(deleting (${BACKUP_SUBDIR}/(" >> $TMP_FILE + done + # print files that will be deleted and ask if continue + # grep deleting $TMP_FILE | sed "s(deleting (${receiver}/(" | less + less $TMP_FILE + + printf "$FMT_MESSAGE" "The listed files will be deleted from $receiver." + # FMT_UPDATE="\e[1;33mUpdating:\e[0m %s\n" + printf "Proceed?\e[34m (y/n)\e[0m: " + read answer + case $answer in + y|Y) + # printf "$FMT_MESSAGE" "Backing up to $receiver..." + ;; + *) + printf "$FMT_MESSAGE" "Cancelled." + exit 0 ;; + esac + fi + + # actual syncing + for path in "${filtered_paths[@]}"; do + [[ $v -ge 1 ]] && printf "$FMT_SYNC" "$path" + if [[ -d $path ]]; then + dest=$receiver$(basename $path) + elif [[ -f $path ]]; then + dest=$receiver + elif [[ $sender == *@*:* ]]; then # if sender is remote, assume the path is a dir if it has a slash and a file otherwise + [[ $path == *"/" ]] && dest=$receiver$(basename $path) || dest=$receiver + else + printf "$FMT_ERROR" "Invalid path: $path"; exit 1 + fi + if [[ -n $encrypt || -n $decrypt ]]; then + [[ -d $path ]] && tmp_source=$TMP_DIR + + if [[ -n $encrypt ]]; then + [[ -f $path ]] && tmp_source=$TMP_DIR$(basename $path).gpg + [[ $v -ge 3 ]] && printf "$FMT_CMD" "bash $mkrypt --encrypt $path --output $TMP_DIR $mkrypt_flags" + bash $mkrypt --encrypt $path --output $TMP_DIR $mkrypt_flags || exit 1 + [[ $v -ge 3 ]] && printf "$FMT_CMD" "rsync ${rsync_flags[@]} $tmp_source $dest" + rsync "${rsync_flags[@]}" $tmp_source $dest + else + [[ -f $path ]] && tmp_source=$TMP_DIR$(basename $path) + [[ $v -ge 3 ]] && printf "$FMT_CMD" "rsync ${rsync_flags[@]} $path $TMP_DIR" + rsync "${rsync_flags[@]}" $path $TMP_DIR + [[ $v -ge 3 ]] && printf "$FMT_CMD" "bash $mkrypt --decrypt $tmp_source --output $(readlink -f $dest) $mkrypt_flags" + bash $mkrypt --decrypt $tmp_source --output $(readlink -f $dest) $mkrypt_flags || exit 1 + fi + else + printf "$FMT_CMD" "rsync ${rsync_flags[@]} $delete $path $dest" + rsync "${rsync_flags[@]}" $delete $path $dest + fi + [[ $TMP_DIR == *msynk* ]] && { + rm -rf $TMP_DIR/* || { printf "$FMT_ERROR" "Can not delete \$TMP_DIR: $TMP_DIR"; exit 1; } + } || { printf "$FMT_ERROR" "Will not delete \$TMP_DIR because it might be dangerous (path does not contain 'msynk'): $TMP_DIR"; exit 1; } + done + + [[ $v -ge 2 ]] && printf "$FMT_MESSAGE" "Running 'sync' to write cached writes to persistent storage" + sync + [[ $v -ge 1 ]] && printf "$FMT_MESSAGE" "Done!" +} + + +# +# HELP +# +show_help() +{ + printf "\e[33mArgument Short Action:\e[0m +--help -h show this +--settings -s show current settings + +--config [name] -c use sender, receiver, paths from config file with [name] + +--sender [path] -s sender directory, with trailing slash! Defaults to the current working directory +--receiver [path] -r receiver directory, with trailing slash! +--reverse swap receiver and sender + +--encrypt encrypt files before syncing +--decrypt decrypt files before syncing +--mkrypt-flags [flags] additional flags for mkrypt + +--delete -d delete files that exist on receiver, but not on sender +--skip-dryrun delete without asking first +--rsync-flags [flags] additional flags for rsync + +--verbose -v increase verbosity +--silent decrease verbosity +--debug maximum verbosity + +Positional arguments are: +- if using a config: a string that must be in the configs paths: eg. 'foo' will include ~/foo but nor ~/bar +- if not using a config: paths to sync (must be relative to --sender) + +See the manpage for more information. +" +} + + +show_settings() +{ + printf "\e[1mThe current settings are:\e[0m\n" + printf "$FMT_CONFIG" "\$TMP_FILE " "$TMP_FILE" + printf "$FMT_CONFIG" "\$rsync_flags " "${rsync_flags[*]}" +} + + +# +# PARSE ARGS +# +if [ -z $1 ]; then + show_help + exit 0 +fi + + + +all=0 +BACKUP=1 # use if the script gets more functions later +# all command line args with no "-" are interpreted as part of filepaths. +paths_2=() +while (( "$#" )); do + case "$1" in + -h|--help) + show_help + exit 0 ;; + --settings) + show_settings + exit 0 ;; + -c|--config) + if [[ -n $2 && ${2:0:1} != "-" ]]; then + config=$2 + shift 2 + else printf "$FMT_ERROR" "Missing argument for $1"; exit 1; fi ;; + -s|--sender) + if [[ -n $2 && ${2:0:1} != "-" ]]; then + sender_2=$2 + shift 2 + else printf "$FMT_ERROR" "Missing argument for $1"; exit 1; fi ;; + -r|--receiver) + if [[ -n $2 && ${2:0:1} != "-" ]]; then + receiver_2=$2 + shift 2 + else printf "$FMT_ERROR" "Missing argument for $1"; exit 1; fi ;; + --reverse) + reverse=1 + shift ;; + --encrypt) + encrypt=1 + shift ;; + --decrypt) + decrypt=1 + shift;; + -d|--delete) + delete=--delete + shift ;; + --skip-dryrun) + skip_dryrun=1 + shift ;; + --rsync-flags) + if [[ -n ${2} ]]; then # && "${2:0:1}" != "-" ]]; then # possibly problematic if $2 is an msynk flag + rsync_flags+=("${2}") + shift 2 + else printf "$FMT_ERROR" "Missing argument for $1"; exit 1; fi ;; + --msynk-flags) + if [[ -n ${2} ]]; then # && "${2:0:1}" != "-" ]]; then # possibly problematic if $2 is an msynk flag + mkrypt_flags+=("${2}") + shift 2 + else printf "$FMT_ERROR" "Missing argument for $1"; exit 1; fi ;; + -v|--verbose) + v=2 + rsync_flags+=(-v) + mkrypt_flags+=(-v) + shift ;; + --silent) + v=0 + mkrypt_flags+=(--silent) + shift ;; + --debug) + v=3 + rsync_flags+=(-v) + mkrypt_flags+=(-v) + shift;; + -*|--*=) # unsupported flags + printf "$FMT_ERROR" "Unsupported flag $1" >&2 + exit 1 ;; + *) # everything that does not have a - is interpreted as filepath + all= + paths_2+=($1) + shift ;; + esac +done + +# check if rsync is installed +if ! command -v rsync &> /dev/null; then + printf "$FMT_ERROR" "rsync is not installed." +fi + +# if using a config +if [[ -n $config ]]; then + bash $CONFIG_DIR/$config || { printf "$FMT_ERROR" "Error running the config file: $CONFIG_DIR/$config"; exit 1; } +fi + +# overwrite stuff from the config if anything else was given / set variables of no config was given +[[ -n $sender_2 ]] && sender=$sender_2 +[[ -n $receiver_2 ]] && receiver=$receiver_2 +[[ -z $sender ]] && { + [[ $v -ge 1 ]] && printf "$FMT_MESSAGE" "Missing sender. Using working directory: $(pwd)"; + sender="$(pwd)/" +} + +# reverse: swap sender and reveiver +if [[ -n $reverse ]]; then + tmp_sender=$sender + sender=$receiver + receiver=$tmp_sender +fi + +# make sure sender is absolute with slash +wd=$PWD +# quiet cd $sender && sender="$PWD/" || { printf "$FMT_ERROR" "Sender directory does not exist: $sender"; exit 1; } +# make sure reveiver is absolute +# cd $wd +# mkdir -p $receiver || { printf "$FMT_ERROR" "Can not create config dir: $receiver"; exit 1; } +# quiet cd $receiver && receiver="$PWD/" || { printf "$FMT_ERROR" "Receiver directory does not exist: $receiver"; exit 1; } +# make sure receiver has trailing slash +[[ $receiver != *"/" ]] && receiver=$receiver"/" + +echo "sender:$sender rec:$receiver paths=$paths paths2=$paths_2" + +# filter paths: +# - if using config: all paths from config that have a pos.arg in them +# - else: pos.args, must be relative to specified --sender +filtered_paths=() +if [[ -n $config ]]; then + for path in ${paths[@]}; do + if check_path_in_args; then + filtered_paths+=($sender$path) + fi + done +else + for path in ${paths_2[@]}; do # add slash if directory + # quiet cd $sender + [[ -d $path ]] && [[ $path != *"/" ]] && path=$path"/" + # quiet cd $sender && quiet cd $path && filtered_paths+=($PWD/) || { + # quiet cd $sender && ls -d $path && filtered_paths+=($PWD/$path) + # } + filtered_paths+=($sender$path) + done +fi +cd $wd +echo "FPaths: ${filtered_paths[@]}" + +# sanity checks +[[ -z $filtered_paths ]] && { printf "$FMT_ERROR" "Missing valid paths."; exit 1; } +[[ -z $receiver ]] && { printf "$FMT_ERROR" "Missing receiver. Specifiy with --receiver"; exit 1; } + +sync_sender_to_receiver +exit 0