From aa7fdd0df9ef79ed9f0c2807203d02458654804f Mon Sep 17 00:00:00 2001 From: Erik Westrup Date: Fri, 13 Apr 2018 20:36:20 +0200 Subject: [PATCH] Initial commit --- README.md | 44 ++++++++++++++++++++ b2_env.sh | 8 ++++ b2_pw.txt | 1 + restic-backup.service | 10 +++++ restic-backup.timer | 9 ++++ restic-check.service | 9 ++++ restic-check.timer | 9 ++++ restic_backup.sh | 85 ++++++++++++++++++++++++++++++++++++++ restic_check.sh | 38 +++++++++++++++++ status-email-user@.service | 11 +++++ systemd-email | 61 +++++++++++++++++++++++++++ 11 files changed, 285 insertions(+) create mode 100644 README.md create mode 100644 b2_env.sh create mode 100644 b2_pw.txt create mode 100644 restic-backup.service create mode 100644 restic-backup.timer create mode 100644 restic-check.service create mode 100644 restic-check.timer create mode 100644 restic_backup.sh create mode 100644 restic_check.sh create mode 100644 status-email-user@.service create mode 100644 systemd-email diff --git a/README.md b/README.md new file mode 100644 index 0000000..720f298 --- /dev/null +++ b/README.md @@ -0,0 +1,44 @@ +# Automatic restic backups using systemd services and timers + +## Restic + +[restic](https://restic.net/) is a command-line tool for making backups, the right way. Check the official website for a feature explanation. As a storage backend, I recommend [Backblaze B2](https://www.backblaze.com/b2/cloud-storage.html) as restic works well with it, and it is (at the time of writing) very affordable for the hobbyist hacker! + +First, see this official Backblaze [tutorial](https://help.backblaze.com/hc/en-us/articles/115002880514-How-to-configure-Backblaze-B2-with-Restic-on-Linux) on restic, on how to setup your B2 bucket. + +## Automatic scheduled backups +Unfortunately restic does not come per-configured with a way to run automated backups, say every day. However it's possible to set this up yourself using. This example also features email notifications when a backup fails to complete. + +Put this file in `/etc/restic/`: +* `b2_env.sh`: Fill this file out with your B2 bucket settings etc. The reason for putting these in a separeate file is that it can be used also for you to simply source, when you want to issue some restic commands. For example: +```bash +$ source /etc/restic/b2_env.sh +$ restic snapshots # You don't have to supply all paramters like --repo, as they are now in your envionment! +```` +* `b2_pw.txt`: Put your b2 password in this file. + +Put these files in `/usr/local/sbin`: +* `restic_backup.sh`: A script that defines how to run the backup. Edit this file to respect your needs in terms of backup which paths to backup, retention (number of bakcups to save), etc. +* `systemd-email`: Sends email using sendmail. You must set up your computer so it can send mail, for example using [postfix and Gmail](https://easyengine.io/tutorials/linux/ubuntu-postfix-gmail-smtp/). This script also features time-out for not spamming Gmail servers. Edit the email target address in this file. + + +Put these files in `/etc/systemd/system/`: +* `restic-backup.service`: A service that calls the script above. +* `restic-backup.timer`: A timer (systemd's cronjobs) that starts the backup every day. +* `status-email-user@.service`: A service that can notify you via email when a systemd service fails. + + +Now simply enable the timer with: +```bash +$ systemctl enable restic-backup.timer +```` +and enjoy your computer being backed up every day! + +You can see when your next backup will be schedued +```bash +$ systemctl list-timers | grep restic +``` + +## Automatic backup checks + +Furthermore there are some `*-check*`-files in this repo too. Install these too if you want to run restic-check once in a while to verify that your remote backup is not corrupt. diff --git a/b2_env.sh b/b2_env.sh new file mode 100644 index 0000000..a2b6393 --- /dev/null +++ b/b2_env.sh @@ -0,0 +1,8 @@ +# B2 credentials. +# Extracted settings so both systemd timers and user can just source this when want to work on my B2 backup. +# See https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html + +export RESTIC_REPOSITORY="b2:" +export RESTIC_PASSWORD_FILE="/etc/restic/b2_pw.txt" +export B2_ACCOUNT_ID="" +export B2_ACCOUNT_KEY="" diff --git a/b2_pw.txt b/b2_pw.txt new file mode 100644 index 0000000..978dd50 --- /dev/null +++ b/b2_pw.txt @@ -0,0 +1 @@ + diff --git a/restic-backup.service b/restic-backup.service new file mode 100644 index 0000000..060be03 --- /dev/null +++ b/restic-backup.service @@ -0,0 +1,10 @@ +[Unit] +Description=Backup with restic to Backblaze B2 +OnFailure=status-email-user@%n.service + +[Service] +Type=simple +Nice=10 +ExecStart=/usr/local/sbin/restic_backup.sh +# $HOME or $XDG_CACHE_HOME must be set for restic to find /root/.cache/restic/ +Environment="HOME=/root" \ No newline at end of file diff --git a/restic-backup.timer b/restic-backup.timer new file mode 100644 index 0000000..c492c33 --- /dev/null +++ b/restic-backup.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Backup with restic on schedule + +[Timer] +OnCalendar=daily +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/restic-check.service b/restic-check.service new file mode 100644 index 0000000..7f90e78 --- /dev/null +++ b/restic-check.service @@ -0,0 +1,9 @@ +[Unit] +Description=Check restic backup Backblaze B2 for errors +OnFailure=status-email-user@%n.service +Conflicts=restic.service + +[Service] +Type=simple +Nice=10 +ExecStart=/usr/local/sbin/restic_check.sh diff --git a/restic-check.timer b/restic-check.timer new file mode 100644 index 0000000..b637948 --- /dev/null +++ b/restic-check.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Check restic backup Backblaze B2 for errors on a schedule + +[Timer] +OnCalendar=monthly +Persistent=true + +[Install] +WantedBy=timers.target \ No newline at end of file diff --git a/restic_backup.sh b/restic_backup.sh new file mode 100644 index 0000000..5bdbc92 --- /dev/null +++ b/restic_backup.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# Make backup my system with restic to Backblaze B2. +# This script is typically run by: /etc/systemd/system/restic-backup.{service,timer} + +# Exit on failure, pipe failure +set -e -o pipefail + +# Redirect stdout ( > ) into a named pipe ( >() ) running "tee" to a file, so we can observe the status by simply tailing the log file. +me=$(basename "$0") +now=$(date +%F_%R) +log_dir=/var/local/log/restic +log_file="${log_dir}/${now}_${me}.$$.log" +test -d $log_dir || mkdir -p $log_dir +exec > >(tee -i $log_file) +exec 2>&1 + +# Clean up lock if we are killed. +# If killed by systemd, like $(systemctl stop restic), then it kills the whole cgroup and all it's subprocesses. +# However if we kill this script ourselves, we need this trap that kills all subprocesses manually. +exit_hook() { + echo "In exit_hook(), being killed" >&2 + jobs -p | xargs kill + restic unlock +} +trap exit_hook INT TERM + +RETENTION_DAYS=7 +RETENTION_WEEKS=12 +RETENTION_MONTHS=18 +RETENTION_YEARS=4 + +BACKUP_PATHS="/ /boot /home /mnt/media" +BACKUP_EXCLUDES="--exclude-file /.rsync_exclude --exclude-file /mnt/media/.rsync_exclude --exclude-file /home/erikw/.rsync_exclude" +BACKUP_TAG=systemd.timer + +# Set all environment variables like +# B2_ACCOUNT_ID, B2_ACCOUNT_KEY, RESTIC_REPOSITORY etc. +source /etc/restic/b2_env.sh + + +# NOTE start all commands in background and wait for them to finish. +# Reason: bash ignores any signals while child process is executing and thus my trap exit hook is not triggered. +# However if put in subprocesses, wait(1) waits until the process finishes OR signal is received. +# Reference: https://unix.stackexchange.com/questions/146756/forward-sigterm-to-child-in-bash + +# Remove locks from other stale processes to keep the automated backup running. +restic unlock & +wait $! + +# See restic-backup(1) or http://restic.readthedocs.io/en/latest/040_backup.html +#restic backup --tag $BACKUP_TAG --one-file-system $BACKUP_EXCLUDES $BACKUP_PATHS & +#wait $! + +# Until +# https://github.com/restic/restic/issues/1557 +# is fixed with the PR +# https://github.com/restic/restic/pull/1494 +# we have to use a work-around and skip the --one-file-system and explicitly black-list the paths we don't want, as described here +# https://forum.restic.net/t/full-system-restore/126/8?u=fd0 +restic backup \ + --tag $BACKUP_TAG \ + --exclude-file /.restic-excludes \ + $BACKUP_EXCLUDES \ + / & +wait $! + +# See restic-forget(1) or http://restic.readthedocs.io/en/latest/060_forget.html +restic forget \ + --tag $BACKUP_TAG \ + --keep-daily $RETENTION_DAYS \ + --keep-weekly $RETENTION_WEEKS \ + --keep-monthly $RETENTION_MONTHS \ + --keep-yearly $RETENTION_YEARS & +wait $! + +# Remove old data not linked anymore. +# See restic-prune(1) or http://restic.readthedocs.io/en/latest/060_forget.html +restic prune & +wait $! + + +# Check repository for errors. +# NOTE this takes much time (and data transfer from remote repo?), do this in a separate systemd.timer which is run less often. +#restic check & +#wait $! diff --git a/restic_check.sh b/restic_check.sh new file mode 100644 index 0000000..346682d --- /dev/null +++ b/restic_check.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Check my backup with restic to Backblaze B2 for errors. +# This script is typically run by: /etc/systemd/system/restic-check.{service,timer} + +# Exit on failure, pipe failure +set -e -o pipefail + +# Redirect stdout ( > ) into a named pipe ( >() ) running "tee" to a file, so we can observe the status by simply tailing the log file. +me=$(basename "$0") +now=$(date +%F_%R) +log_dir=/var/local/log/restic +log_file="${log_dir}/${now}_${me}.$$.log" +test -d $log_dir || mkdir -p $log_dir +exec > >(tee -i $log_file) +exec 2>&1 + +# Clean up lock if we are killed. +# If killed by systemd, like $(systemctl stop restic), then it kills the whole cgroup and all it's subprocesses. +# However if we kill this script ourselves, we need this trap that kills all subprocesses manually. +exit_hook() { + echo "In exit_hook(), being killed" >&2 + jobs -p | xargs kill + restic unlock +} +trap exit_hook INT TERM + + + +source /etc/restic/b2_env.sh + +# Remove locks from other stale processes to keep the automated backup running. +# NOTE nope, dont' unlock liek restic_backup.sh. restic_backup.sh should take preceedance over this script. +#restic unlock & +#wait $! + +# Check repository for errors. +restic check & +wait $! diff --git a/status-email-user@.service b/status-email-user@.service new file mode 100644 index 0000000..a2897ed --- /dev/null +++ b/status-email-user@.service @@ -0,0 +1,11 @@ +# Source: https://serverfault.com/questions/876233/how-to-send-an-email-if-a-systemd-service-is-restarted +# Source: https://wiki.archlinux.org/index.php/Systemd/Timers#MAILTO + +[Unit] +Description=Send status email for %i to user + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/systemd-email abc@gmail.com %i +User=root +Group=systemd-journal diff --git a/systemd-email b/systemd-email new file mode 100644 index 0000000..a96dca7 --- /dev/null +++ b/systemd-email @@ -0,0 +1,61 @@ +#!/usr/bin/env sh +# Send email notification from systemd. +# Source: https://serverfault.com/questions/876233/how-to-send-an-email-if-a-systemd-service-is-restarted +# Source: https://wiki.archlinux.org/index.php/Systemd/Timers#MAILTO +# Usage: systemd-email + + +# According to +# http://www.flashissue.com/blog/gmail-sending-limits/ +# Gmail blocks your account if you send more than 500 emails per day, which is one email every +# (24 * 60 * 60) / 500 = 172.8 second => choose a min wait time which is significantly longer than this to be on the safe time to not exceed 500 emails per day. +# However this source +# https://group-mail.com/sending-email/email-send-limits-and-options/ +# says the limit when not using the Gmail webinterface but going directly to the SMTP server is 100-150 per day, which yelds maximum one email every +# (24 * 60 * 60) / 100 = 864 second +# One option that I used with my old Axis cameras it to use my gmx.com accunt for sending emails instead, as there are (no?) higher limits there. +MIN_WAIT_TIME_S=900 +SCRIPT_NAME=$(basename $0) +LAST_RUN_FILE="/tmp/${SCRIPT_NAME}_last_run.txt" + +last_touch() { + stat -c %Y $1 +} + +waited_long_enough() { + retval=1 + if [ -e $LAST_RUN_FILE ]; then + now=$(date +%s) + last=$(last_touch $LAST_RUN_FILE) + wait_s=$(expr $now - $last) + if [ "$wait_s" -gt "$MIN_WAIT_TIME_S" ]; then + retval=0 + fi + else + retval=0 + fi + + [ $retval -eq 0 ] && touch $LAST_RUN_FILE + return $retval +} + + +# Make sure that my Gmail account dont' get shut down because of sending too many emails! +if ! waited_long_enough; then + echo "Systemd email was not sent, as it's less than ${MIN_WAIT_TIME_S} seconds since the last one was sent." + exit 1 +fi + + +recipinent=$1 +system_unit=$2 + +sendmail -t < +Subject: [systemd-email] ${system_unit} +Content-Transfer-Encoding: 8bit +Content-Type: text/plain; charset=UTF-8 + +$(systemctl status --full "$system_unit") +ERRMAIL