Put files in directory structure

usr/local/sbin/restic_backup.sh

@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+# Make backup my system with restic to Backblaze B2.
+# This script is typically run by: /etc/systemd/system/restic-backup.{service,timer}
+
+# Exit on failure, pipe failure
+set -e -o pipefail
+
+# Redirect stdout ( > ) into a named pipe ( >() ) running "tee" to a file, so we can observe the status by simply tailing the log file.
+me=$(basename "$0")
+now=$(date +%F_%R)
+log_dir=/var/local/log/restic
+log_file="${log_dir}/${now}_${me}.$$.log"
+test -d $log_dir || mkdir -p $log_dir
+exec > >(tee -i $log_file)
+exec 2>&1
+
+# Clean up lock if we are killed.
+# If killed by systemd, like $(systemctl stop restic), then it kills the whole cgroup and all it's subprocesses.
+# However if we kill this script ourselves, we need this trap that kills all subprocesses manually.
+exit_hook() {
+	echo "In exit_hook(), being killed" >&2
+	jobs -p | xargs kill
+	restic unlock
+}
+trap exit_hook INT TERM
+
+RETENTION_DAYS=7
+RETENTION_WEEKS=12
+RETENTION_MONTHS=18
+RETENTION_YEARS=4
+
+BACKUP_PATHS="/ /boot /home /mnt/media"
+BACKUP_EXCLUDES="--exclude-file /.rsync_exclude --exclude-file /mnt/media/.rsync_exclude --exclude-file /home/erikw/.rsync_exclude"
+BACKUP_TAG=systemd.timer
+
+# Set all environment variables like
+# B2_ACCOUNT_ID, B2_ACCOUNT_KEY, RESTIC_REPOSITORY etc.
+source /etc/restic/b2_env.sh
+
+
+# NOTE start all commands in background and wait for them to finish.
+# Reason: bash ignores any signals while child process is executing and thus my trap exit hook is not triggered.
+# However if put in subprocesses, wait(1) waits until the process finishes OR signal is received.
+# Reference: https://unix.stackexchange.com/questions/146756/forward-sigterm-to-child-in-bash
+
+# Remove locks from other stale processes to keep the automated backup running.
+restic unlock &
+wait $!
+
+# See restic-backup(1) or http://restic.readthedocs.io/en/latest/040_backup.html
+#restic backup --tag $BACKUP_TAG --one-file-system $BACKUP_EXCLUDES $BACKUP_PATHS &
+#wait $!
+
+# Until 
+# https://github.com/restic/restic/issues/1557
+# is fixed with the PR
+# https://github.com/restic/restic/pull/1494
+# we have to use a work-around and skip the --one-file-system and explicitly black-list the paths we don't want, as described here
+# https://forum.restic.net/t/full-system-restore/126/8?u=fd0
+restic backup \
+	--tag $BACKUP_TAG \
+	--exclude-file /.restic-excludes \
+	$BACKUP_EXCLUDES \
+	/ &
+wait $!
+
+# See restic-forget(1) or http://restic.readthedocs.io/en/latest/060_forget.html
+restic forget \
+	--tag $BACKUP_TAG \
+	--keep-daily $RETENTION_DAYS \
+	--keep-weekly $RETENTION_WEEKS \
+	--keep-monthly $RETENTION_MONTHS \
+	--keep-yearly $RETENTION_YEARS &
+wait $!
+
+# Remove old data not linked anymore.
+# See restic-prune(1) or http://restic.readthedocs.io/en/latest/060_forget.html
+restic prune &
+wait $!
+
+
+# Check repository for errors.
+# NOTE this takes much time (and data transfer from remote repo?), do this in a separate systemd.timer which is run less often.
+#restic check &
+#wait $!

usr/local/sbin/restic_check.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+# Check my backup with  restic to Backblaze B2 for errors.
+# This script is typically run by: /etc/systemd/system/restic-check.{service,timer}
+
+# Exit on failure, pipe failure
+set -e -o pipefail
+
+# Redirect stdout ( > ) into a named pipe ( >() ) running "tee" to a file, so we can observe the status by simply tailing the log file.
+me=$(basename "$0")
+now=$(date +%F_%R)
+log_dir=/var/local/log/restic
+log_file="${log_dir}/${now}_${me}.$$.log"
+test -d $log_dir || mkdir -p $log_dir
+exec > >(tee -i $log_file)
+exec 2>&1
+
+# Clean up lock if we are killed.
+# If killed by systemd, like $(systemctl stop restic), then it kills the whole cgroup and all it's subprocesses.
+# However if we kill this script ourselves, we need this trap that kills all subprocesses manually.
+exit_hook() {
+	echo "In exit_hook(), being killed" >&2
+	jobs -p | xargs kill
+	restic unlock
+}
+trap exit_hook INT TERM
+
+
+
+source /etc/restic/b2_env.sh
+
+# Remove locks from other stale processes to keep the automated backup running.
+# NOTE nope, dont' unlock liek restic_backup.sh. restic_backup.sh should take preceedance over this script.
+#restic unlock &
+#wait $!
+
+# Check repository for errors.
+restic check &
+wait $!

usr/local/sbin/systemd-email

@@ -0,0 +1,61 @@
+#!/usr/bin/env sh
+# Send email notification from systemd.
+# Source: https://serverfault.com/questions/876233/how-to-send-an-email-if-a-systemd-service-is-restarted
+# Source: https://wiki.archlinux.org/index.php/Systemd/Timers#MAILTO
+# Usage: systemd-email <recipinent-email> <failed-systemd-unit-name>
+
+
+# According to 
+# http://www.flashissue.com/blog/gmail-sending-limits/
+# Gmail blocks your account if you send more than 500 emails per day, which is one email every
+# (24 * 60 * 60) / 500 = 172.8 second => choose a min wait time which is significantly longer than this to be on the safe time to not exceed 500 emails per day.
+# However this source
+# https://group-mail.com/sending-email/email-send-limits-and-options/
+# says the limit when not using the Gmail webinterface but going directly to the SMTP server is 100-150 per day, which yelds maximum one email every
+# (24 * 60 * 60) / 100 = 864 second 
+# One option that I used with my old Axis cameras it to use my gmx.com accunt for sending emails instead, as there are (no?) higher limits there.
+MIN_WAIT_TIME_S=900
+SCRIPT_NAME=$(basename $0)
+LAST_RUN_FILE="/tmp/${SCRIPT_NAME}_last_run.txt"
+
+last_touch() {
+	stat -c %Y $1
+}
+
+waited_long_enough() {
+	retval=1
+	if [ -e $LAST_RUN_FILE ]; then
+		now=$(date +%s)
+		last=$(last_touch $LAST_RUN_FILE)
+		wait_s=$(expr $now - $last)
+		if [ "$wait_s" -gt "$MIN_WAIT_TIME_S" ]; then
+			retval=0
+		fi
+	else
+		retval=0
+	fi
+
+	[ $retval -eq 0 ] && touch $LAST_RUN_FILE
+	return $retval
+}
+
+
+# Make sure that my Gmail account dont' get shut down because of sending too many emails!
+if ! waited_long_enough; then
+	echo "Systemd email was not sent, as it's less than ${MIN_WAIT_TIME_S} seconds since the last one was sent."
+	exit 1
+fi
+
+
+recipinent=$1
+system_unit=$2
+
+sendmail -t <<ERRMAIL
+To: $recipinent
+From: systemd <root@$HOSTNAME>
+Subject: [systemd-email] ${system_unit}
+Content-Transfer-Encoding: 8bit
+Content-Type: text/plain; charset=UTF-8
+
+$(systemctl status --full "$system_unit")
+ERRMAIL