From 79976f5019b8e15b95655dcc19f150eeea87cb50 Mon Sep 17 00:00:00 2001 From: Kevin Woley Date: Wed, 8 Sep 2021 16:08:38 -0700 Subject: [PATCH 1/4] update version of restic to 12.1 --- install.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install.ps1 b/install.ps1 index a9d7865..39327b2 100644 --- a/install.ps1 +++ b/install.ps1 @@ -5,10 +5,10 @@ if(-not (Test-Path $ResticExe)) { $url = $null if([Environment]::Is64BitOperatingSystem){ - $url = "https://github.com/restic/restic/releases/download/v0.12.0/restic_0.12.0_windows_amd64.zip" + $url = "https://github.com/restic/restic/releases/download/v0.12.1/restic_0.12.1_windows_386.zip" } else { - $url = "https://github.com/restic/restic/releases/download/v0.12.0/restic_0.12.0_windows_386.zip" + $url = "https://github.com/restic/restic/releases/download/v0.12.1/restic_0.12.1_windows_amd64.zip" } $output = Join-Path $InstallPath "restic.zip" Invoke-WebRequest -Uri $url -OutFile $output From d448db94c0e390b2978a8b4bb6a369cd00c674d9 Mon Sep 17 00:00:00 2001 From: Kevin Woley Date: Wed, 8 Sep 2021 16:35:59 -0700 Subject: [PATCH 2/4] add support for external, removable drive backup - select backup source by drive label, device name, or serial number - fix forget policy to be safe for multiple drives (group by host,tags) - tag each backup source with drive/source name --- backup.ps1 | 83 ++++++++++++++++++++++++++++++++++++++++++++++++------ config.ps1 | 5 +++- 2 files changed, 79 insertions(+), 9 deletions(-) diff --git a/backup.ps1 b/backup.ps1 index 4243f53..5558a7e 100644 --- a/backup.ps1 +++ b/backup.ps1 @@ -17,6 +17,51 @@ $Script:ResticStateRepositoryInitialized = $null $Script:ResticStateLastMaintenance = $null $Script:ResticStateLastDeepMaintenance = $null $Script:ResticStateMaintenanceCounter = $null + +# Returns all drive letters which exactly match the serial number, drive label, or drive name of +# the input parameter. Returns all drives if no input parameter is provided. +# inspiration: https://stackoverflow.com/questions/31088930/combine-get-disk-info-and-logicaldisk-info-in-powershell +function Get-Drives { + Param($ID) + + foreach($disk in Get-CimInstance Win32_Diskdrive) { + $diskMetadata = Get-Disk | Where-Object { $_.Number -eq $disk.Index } | Select-Object -First 1 + $partitions = Get-CimAssociatedInstance -ResultClassName Win32_DiskPartition -InputObject $disk + + foreach($partition in $partitions) { + + $drives = Get-CimAssociatedInstance -ResultClassName Win32_LogicalDisk -InputObject $partition + + foreach($drive in $drives) { + + $volume = Get-Volume | + Where-Object { $_.DriveLetter -eq $drive.DeviceID.Trim(":") } | + Select-Object -First 1 + + if(($diskMetadata.SerialNumber.trim() -eq $ID) -or + ($disk.Caption -eq $ID) -or + ($volume.FileSystemLabel -eq $ID) -or + ($null -eq $ID)) { + + [PSCustomObject] @{ + DriveLetter = $drive.DeviceID + Number = $disk.Index + Label = $volume.FileSystemLabel + Manufacturer = $diskMetadata.Manufacturer + Model = $diskMetadata.Model + SerialNumber = $diskMetadata.SerialNumber.trim() + Name = $disk.Caption + FileSystem = $volume.FileSystem + PartitionKind = $diskMetadata.PartitionStyle + Drive = $drive + Partition = $partition + Disk = $disk + } + } + } + } + } +} # restore backup state from disk function Get-BackupState { @@ -123,19 +168,39 @@ function Invoke-Backup { Write-Output "[[Backup]] Start $(Get-Date)" | Tee-Object -Append $SuccessLog $return_value = $true - $drive_count = $BackupSources.Count $starting_location = Get-Location ForEach ($item in $BackupSources.GetEnumerator()) { - # Get the source drive letter and set as the root path + # Get the source drive letter or identifier and set as the root path $root_path = $item.Key + $tag = $item.Key - # Avoid storing the drive letter in the backup path if only backing up a single drive - # FIXME: this doesn't really work. "C:\" still gets stored - if($drive_count -eq 1) { - Set-Location $root_path - $root_path = "." + $vss_option = "--use-fs-snapshot" + + # Test if root path is a valid path, if not assume it is an external drive identifier + if(-not (Test-Path $root_path)) { + # attempt to find a drive letter associated with the identifier provided + $drives = Get-Drives $root_path + if($drives.Count -gt 1) { + Write-Output "[[Backup]] Fatal error - external drives with more than one partition are not currently supported." | Tee-Object -Append $SuccessLog | Tee-Object -Append $ErrorLog + return $false + } + elseif ($drives.Count -eq 0) { + # TODO: Silently fails if an entire drive is missing. This is good for occasionally mounted external drives, but bad for + # drives that are always expected to be here. May want to make this an optional error. + Write-Output "[[Backup]] Warning - backup path $root_path not found." | Tee-Object -Append $SuccessLog #| Tee-Object -Append $ErrorLog + # $return_value = $false + continue + } + + $root_path = Join-Path $drives[0].DriveLetter "" + + # disable VSS / file system snapshot for external drives + # TODO: would be best to just test for VSS compatibility on the drive, rather than assume it won't work + $vss_option = $null } + + Write-Output "[[Backup]] Start $(Get-Date) [$tag]" | Tee-Object -Append $SuccessLog # Build the new list of folders from settings (if there are any) $folder_list = New-Object System.Collections.Generic.List[System.Object] @@ -150,11 +215,13 @@ function Invoke-Backup { } # Launch Restic - & $ResticExe backup $folder_list --use-fs-snapshot --exclude-file=$WindowsExcludeFile --exclude-file=$LocalExcludeFile 3>&1 2>> $ErrorLog | Tee-Object -Append $SuccessLog + & $ResticExe backup $folder_list $vss_option --tag "$tag" --exclude-file=$WindowsExcludeFile --exclude-file=$LocalExcludeFile 3>&1 2>> $ErrorLog | Tee-Object -Append $SuccessLog if(-not $?) { Write-Output "[[Backup]] Completed with errors" | Tee-Object -Append $ErrorLog | Tee-Object -Append $SuccessLog $return_value = $false } + + Write-Output "[[Backup]] End $(Get-Date) [$tag]" | Tee-Object -Append $SuccessLog } Set-Location $starting_location diff --git a/config.ps1 b/config.ps1 index 7afa23d..e4f3f52 100644 --- a/config.ps1 +++ b/config.ps1 @@ -12,7 +12,7 @@ $GlobalRetryAttempts = 4 # maintenance configuration $SnapshotMaintenanceEnabled = $true -$SnapshotRetentionPolicy = @("--group-by", "host", "--keep-daily", "30", "--keep-weekly", "52", "--keep-monthly", "24", "--keep-yearly", "10") +$SnapshotRetentionPolicy = @("--group-by", "host,tags", "--keep-daily", "30", "--keep-weekly", "52", "--keep-monthly", "24", "--keep-yearly", "10") $SnapshotPrunePolicy = @("--max-unused", "1%") $SnapshotMaintenanceInterval = 7 $SnapshotMaintenanceDays = 30 @@ -30,3 +30,6 @@ $BackupSources["C:\"] = @( #$BackupSources["D:\"] = @( # 'Software' #) +#$BackupSources["DRIVE_LABEL_NAME_OR_SERIAL_NUMBER"] = @( +# 'FolderName' +#) From fac24629815cd28f3069c22cdd942ec70599773c Mon Sep 17 00:00:00 2001 From: Kevin Woley Date: Fri, 10 Sep 2021 21:51:01 -0700 Subject: [PATCH 3/4] Add the ability to $IgnoreMissingBackupSources - when $true, missing external drives or folders don't produce errors - when $null or $false, missing drives and/or folders result in an error - Default is set ot $false as not to silently fail backing up a source --- backup.ps1 | 72 +++++++++++++++++++++++++++++++++++++++++------------- config.ps1 | 1 + 2 files changed, 56 insertions(+), 17 deletions(-) diff --git a/backup.ps1 b/backup.ps1 index 5558a7e..fc50318 100644 --- a/backup.ps1 +++ b/backup.ps1 @@ -186,10 +186,15 @@ function Invoke-Backup { return $false } elseif ($drives.Count -eq 0) { - # TODO: Silently fails if an entire drive is missing. This is good for occasionally mounted external drives, but bad for - # drives that are always expected to be here. May want to make this an optional error. - Write-Output "[[Backup]] Warning - backup path $root_path not found." | Tee-Object -Append $SuccessLog #| Tee-Object -Append $ErrorLog - # $return_value = $false + $ignore_error = ($null -ne $IgnoreMissingBackupSources) -and $IgnoreMissingBackupSources + $warning_message = {Write-Output "[[Backup]] Warning - backup path $root_path not found."} + if($ignore_error) { + & $warning_message | Tee-Object -Append $SuccessLog + } + else { + & $warning_message | Tee-Object -Append $SuccessLog | Tee-Object -Append $ErrorLog + $return_value = $false + } continue } @@ -202,23 +207,56 @@ function Invoke-Backup { Write-Output "[[Backup]] Start $(Get-Date) [$tag]" | Tee-Object -Append $SuccessLog - # Build the new list of folders from settings (if there are any) + # build the list of folders to backup $folder_list = New-Object System.Collections.Generic.List[System.Object] - ForEach ($path in $item.Value) { - $p = '"{0}"' -f ((Join-Path $root_path $path) -replace "\\$") - $folder_list.Add($p) - } - - # backup everything in the root if no folders are provided - if (-not $folder_list) { + if ($item.Value.Count -eq 0) { + # backup everything in the root if no folders are provided $folder_list.Add($root_path) } + else { + # Build the list of folders from settings + ForEach ($path in $item.Value) { + $p = '"{0}"' -f ((Join-Path $root_path $path) -replace "\\$") + + if(Test-Path ($p -replace '"')) { + # add the folder if it exists + $folder_list.Add($p) + } + else { + # if the folder doesn't exist, log a warning/error + $ignore_error = ($null -ne $IgnoreMissingBackupSources) -and $IgnoreMissingBackupSources + $warning_message = {Write-Output "[[Backup]] Warning - backup path $p not found."} + if($ignore_error) { + & $warning_message | Tee-Object -Append $SuccessLog + } + else { + & $warning_message | Tee-Object -Append $SuccessLog | Tee-Object -Append $ErrorLog + $return_value = $false + } + } + } - # Launch Restic - & $ResticExe backup $folder_list $vss_option --tag "$tag" --exclude-file=$WindowsExcludeFile --exclude-file=$LocalExcludeFile 3>&1 2>> $ErrorLog | Tee-Object -Append $SuccessLog - if(-not $?) { - Write-Output "[[Backup]] Completed with errors" | Tee-Object -Append $ErrorLog | Tee-Object -Append $SuccessLog - $return_value = $false + } + + if(-not $folder_list) { + # there are no folders to backup + $ignore_error = ($null -ne $IgnoreMissingBackupSources) -and $IgnoreMissingBackupSources + $warning_message = {Write-Output "[[Backup]] Warning - no folders to back up!"} + if($ignore_error) { + & $warning_message | Tee-Object -Append $SuccessLog + } + else { + & $warning_message | Tee-Object -Append $SuccessLog | Tee-Object -Append $ErrorLog + $return_value = $false + } + } + else { + # Launch Restic + & $ResticExe backup $folder_list $vss_option --tag "$tag" --exclude-file=$WindowsExcludeFile --exclude-file=$LocalExcludeFile 3>&1 2>> $ErrorLog | Tee-Object -Append $SuccessLog + if(-not $?) { + Write-Output "[[Backup]] Completed with errors" | Tee-Object -Append $ErrorLog | Tee-Object -Append $SuccessLog + $return_value = $false + } } Write-Output "[[Backup]] End $(Get-Date) [$tag]" | Tee-Object -Append $SuccessLog diff --git a/config.ps1 b/config.ps1 index e4f3f52..0dee974 100644 --- a/config.ps1 +++ b/config.ps1 @@ -9,6 +9,7 @@ $LogPath = Join-Path $InstallPath "logs" $LogRetentionDays = 30 $InternetTestAttempts = 10 $GlobalRetryAttempts = 4 +$IgnoreMissingBackupSources = $false # maintenance configuration $SnapshotMaintenanceEnabled = $true From 48d0ab73b2ebe22666739a0be100001097017092 Mon Sep 17 00:00:00 2001 From: Kevin Woley Date: Sat, 11 Sep 2021 05:25:19 -0700 Subject: [PATCH 4/4] update changelog for 1.5 release --- CHANGELOG.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 988ef47..25daa4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ # Changelog +## [1.5](https://github.com/kmwoley/restic-windows-backup/tree/1.5) (2021-09-11) +[Full Changelog](https://github.com/kmwoley/restic-windows-backup/compare/1.4.1...1.5) + +Added support for backing up removable drives (i.e. external USB disks). It's now possible to define a backup source by it's Volume lable, device Serial Number, or the hardware Name. + +**WARNING** If you have been previously backing up multiple drives, the default `forget` policy was likely pruning backup sets too agressively and could lead to data loss. You **must** update your `$SnapshotRetentionPolicy` to include `@("--group-by", "host,tags", ...` to avoid pruning an entire drive's contents inadvertently! + +## Fixes +- Updated default snapshot forget/prune retention policy to group by "host,tags" to prevent major data loss. Only configurations with multiple `$BackupSources` are impacted by this change. +- Added tags to each backup source to support grouping by tags. For existing backup sets, this change will result in a slightly longer backup the first time this updated script is run. + +## Enhancements +- External, removalbe disk drives (i.e. USB hard drives) can be identified by their Volume Label, Serial Number, or Device Name. For example, if you have an external device with the Volume Lable "MY BOOK", you can define a backup source as `$BackupSources["MY BOOK"]`. I would recommend using the device serial number to identify external drives to backup, which you can find using the Powershell `get-disk` command. +- Add the ability to $IgnoreMissingBackupSources. To make sure that errors are not thrown if the device is not present, there is now an option to ignore error reporting when a folder and entire backup source are missing. When `$true`, missing external drives or folders don't produce errors. When `$null` or `$false`, missing drives and/or folders result in an error. The default is set to `$false` as not to silently fail backing up a source. +- Updated install script to download Restic 12.1 + ## [1.4.1](https://github.com/kmwoley/restic-windows-backup/tree/1.4.1) (2021-05-29) [Full Changelog](https://github.com/kmwoley/restic-windows-backup/compare/1.4...1.4.1)