From 387390759dc53242a5851294e3b1c18357d8c4ef Mon Sep 17 00:00:00 2001 From: Kevin Woley Date: Wed, 24 Feb 2021 19:47:35 -0800 Subject: [PATCH 1/3] removed snapshot code, added `--use-fs-snapshot` Related changes - removed folder enumeration and instead point to the root filesystem: snapshot handling will be better now that changing folders won't group sets default exlusion list updates: C:\$Recycle.Bin C:\$WINDOWS.~BT C:\$WinREAgent --- backup.ps1 | 46 ++++++++++++---------------------------------- windows.exclude | 40 +++++++++++++++++++++------------------- 2 files changed, 33 insertions(+), 53 deletions(-) diff --git a/backup.ps1 b/backup.ps1 index bd4ff19..06d0159 100644 --- a/backup.ps1 +++ b/backup.ps1 @@ -127,29 +127,17 @@ function Invoke-Backup { $starting_location = Get-Location ForEach ($item in $BackupSources.GetEnumerator()) { - $ShadowPath = Join-Path $item.Key 'resticVSS' + # Get the source drive letter and set as the root path + $root_path = $item.Key - # check for existance of previous, orphaned VSS directory (and remove it) before creating the shadow copy - if(Test-Path $ShadowPath) { - Write-Output "[[Backup]] VSS directory exists: '$ShadowPath' - removing. Past script failure?" | Tee-Object -Append $ErrorLog | Tee-Object -Append $SuccessLog - cmd /c rmdir $ShadowPath - } - - # Create the shadow copy - $s1 = (Get-WmiObject -List Win32_ShadowCopy).Create($item.Key, "ClientAccessible") - $s2 = Get-WmiObject -Class Win32_ShadowCopy | Where-Object { $_.ID -eq $s1.ShadowID } - - # Create a symbolic link to the shadow copy - $device = $s2.DeviceObject + "\" - cmd /c mklink /d $ShadowPath "$device" 3>&1 2>> $ErrorLog | Tee-Object -Append $SuccessLog - - # Build the new list of folders - $root_path = $ShadowPath + # Avoid storing the drive letter in the backup path if only backing up a single drive + # FIXME: this doesn't really work. "C:\" still gets stored if($drive_count -eq 1) { + Set-Location $root_path $root_path = "." - Set-Location $ShadowPath } - + + # Build the new list of folders from settings (if there are any) $folder_list = New-Object System.Collections.Generic.List[System.Object] ForEach ($path in $item.Value) { $p = Join-Path $root_path $path @@ -157,30 +145,20 @@ function Invoke-Backup { } # backup everything in the root if no folders are provided - # note this won't select items with hidden attributes (a good thing to avoid) if (-not $folder_list) { - ForEach ($path in Get-ChildItem $ShadowPath) { - $p = Join-Path $root_path $path - $folder_list.Add($p) - } + $folder_list.Add($root_path) } # Launch Restic - & $ResticExe backup $folder_list --exclude-file=$WindowsExcludeFile --exclude-file=$LocalExcludeFile 3>&1 2>> $ErrorLog | Tee-Object -Append $SuccessLog + & $ResticExe backup $folder_list --use-fs-snapshot --exclude-file=$WindowsExcludeFile --exclude-file=$LocalExcludeFile 3>&1 2>> $ErrorLog | Tee-Object -Append $SuccessLog if(-not $?) { Write-Output "[[Backup]] Completed with errors" | Tee-Object -Append $ErrorLog | Tee-Object -Append $SuccessLog $return_value = $false } - - # Delete the shadow copy and remove the symbolic link - if($drive_count -eq 1) { - Set-Location $starting_location - } - $s2.Delete() - cmd /c rmdir $ShadowPath - - Write-Output "[[Backup]] End $(Get-Date)" | Tee-Object -Append $SuccessLog } + + Set-Location $starting_location + Write-Output "[[Backup]] End $(Get-Date)" | Tee-Object -Append $SuccessLog return $return_value } diff --git a/windows.exclude b/windows.exclude index 87b0c98..8cc555f 100644 --- a/windows.exclude +++ b/windows.exclude @@ -1,23 +1,25 @@ # default excludes # examples https://github.com/duplicati/duplicati/blob/master/Duplicati/Library/Utility/FilterGroups.cs -# note, because we're using a VSS directory, we can use that as the root dir for exclude rules (i.e. resticVSS) -resticVSS\hiberfil.sys -resticVSS\pagefile.sys -resticVSS\swapfile.sys -resticVSS\$Recycle.Bin -resticVSS\autoexec.bat -resticVSS\Config.Msi -resticVSS\Documents and Settings -resticVSS\Recycled -resticVSS\Recycler -resticVSS\System Volume Information -resticVSS\Recovery -resticVSS\Program Files -resticVSS\Program Files (x86) -resticVSS\ProgramData -resticVSS\PerfLogs -resticVSS\Windows -resticVSS\Windows.old +C:\hiberfil.sys +C:\pagefile.sys +C:\swapfile.sys +C:\$Recycle.Bin +C:\autoexec.bat +C:\Config.Msi +C:\Documents and Settings +C:\Recycled +C:\Recycler +C:\$$Recycle.Bin +C:\System Volume Information +C:\Recovery +C:\Program Files +C:\Program Files (x86) +C:\ProgramData +C:\PerfLogs +C:\Windows +C:\Windows.old +C:\$$WINDOWS.~BT +C:\$$WinREAgent Microsoft\Windows\Recent Microsoft\**\RecoveryStore* Microsoft\**\Windows\*.edb @@ -32,7 +34,7 @@ UsrClass.dat Dropbox AppData\Local\Google\Drive Google Drive\.tmp.drivedownload -resticVSS\OneDriveTemp +C:\OneDriveTemp # browsers Google\Chrome From a840f5ae04e9ffa350271badd4bc71111adb0704 Mon Sep 17 00:00:00 2001 From: Kevin Woley Date: Wed, 24 Feb 2021 19:59:40 -0800 Subject: [PATCH 2/3] default retention polciy to group-by host Fixes the issues from previous backup's having changing directory names, which cause snapshot retention inflation. --- config.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.ps1 b/config.ps1 index 72d3e35..05193a4 100644 --- a/config.ps1 +++ b/config.ps1 @@ -12,7 +12,7 @@ $GlobalRetryAttempts = 4 # maintenance configuration $SnapshotMaintenanceEnabled = $true -$SnapshotRetentionPolicy = @("--keep-daily", "30", "--keep-weekly", "52", "--keep-monthly", "24", "--keep-yearly", "10") +$SnapshotRetentionPolicy = @("--group-by", "host", "--keep-daily", "30", "--keep-weekly", "52", "--keep-monthly", "24", "--keep-yearly", "10") $SnapshotMaintenanceInterval = 7 $SnapshotMaintenanceDays = 30 $SnapshotDeepMaintenanceDays = 90; From 01a38f893ae414c1fe6fffa535314ab6a8c2889f Mon Sep 17 00:00:00 2001 From: Kevin Woley Date: Wed, 24 Feb 2021 21:02:41 -0800 Subject: [PATCH 3/3] add the ability to set a pruning policy defaulted to --max-unused 1% in configuration. --- backup.ps1 | 2 +- config.ps1 | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/backup.ps1 b/backup.ps1 index 06d0159..b98dc2e 100644 --- a/backup.ps1 +++ b/backup.ps1 @@ -76,7 +76,7 @@ function Invoke-Maintenance { # prune (remove) data from the backup step. Running this separate from `forget` because # `forget` only prunes when it detects removed snapshots upon invocation, not previously removed Write-Output "[[Maintenance]] Start pruning..." | Tee-Object -Append $SuccessLog - & $ResticExe prune 3>&1 2>> $ErrorLog | Tee-Object -Append $SuccessLog + & $ResticExe prune $SnapshotPrunePolicy 3>&1 2>> $ErrorLog | Tee-Object -Append $SuccessLog if(-not $?) { Write-Output "[[Maintenance]] Prune operation completed with errors" | Tee-Object -Append $ErrorLog | Tee-Object -Append $SuccessLog $maintenance_success = $false diff --git a/config.ps1 b/config.ps1 index 05193a4..7afa23d 100644 --- a/config.ps1 +++ b/config.ps1 @@ -13,6 +13,7 @@ $GlobalRetryAttempts = 4 # maintenance configuration $SnapshotMaintenanceEnabled = $true $SnapshotRetentionPolicy = @("--group-by", "host", "--keep-daily", "30", "--keep-weekly", "52", "--keep-monthly", "24", "--keep-yearly", "10") +$SnapshotPrunePolicy = @("--max-unused", "1%") $SnapshotMaintenanceInterval = 7 $SnapshotMaintenanceDays = 30 $SnapshotDeepMaintenanceDays = 90;