windows/backup.ps1

#
# Restic Windows Backup Script
#

# =========== start configuration =========== # 

# set restic configuration parmeters (destination, passwords, etc.)
$SecretsScript = Join-Path $PSScriptRoot "secrets.ps1"

# backup configuration variables
$ConfigScript = Join-Path $PSScriptRoot "config.ps1"

# =========== end configuration =========== #

# globals for state storage
$Script:ResticStateRepositoryInitialized = $null
$Script:ResticStateLastMaintenance = $null
$Script:ResticStateLastDeepMaintenance = $null
$Script:ResticStateMaintenanceCounter = $null

# restore backup state from disk
function Get-BackupState {
    if(Test-Path $StateFile) {
        Import-Clixml $StateFile | ForEach-Object{ Set-Variable -Scope Script $_.Name $_.Value }
    }
}
function Set-BackupState {
    Get-Variable ResticState* | Export-Clixml $StateFile
}

# unlock the repository if need be
function Invoke-Unlock {
    Param($SuccessLog, $ErrorLog)

    $locks = & $ResticExe list locks --no-lock -q 3>&1 2>> $ErrorLog
    if($locks.Length -gt 0) {
        # unlock the repository (assumes this machine is the only one that will ever use it)
        & $ResticExe unlock 3>&1 2>> $ErrorLog | Tee-Object -Append $SuccessLog
        Write-Output "[[Unlock]] Repository was locked. Unlocking. Past script failure?" | Tee-Object -Append $ErrorLog | Tee-Object -Append $SuccessLog
        Start-Sleep 120 
    }
}

# run maintenance on the backup set
function Invoke-Maintenance {
    Param($SuccessLog, $ErrorLog)
    
    # skip maintenance if disabled
    if($SnapshotMaintenanceEnabled -eq $false) {
        Write-Output "[[Maintenance]] Skipped - maintenance disabled" | Tee-Object -Append $SuccessLog
        return
    }

    # skip maintenance if it's been done recently
    if(($null -ne $ResticStateLastMaintenance) -and ($null -ne $ResticStateMaintenanceCounter)) {
        $Script:ResticStateMaintenanceCounter += 1
        $delta = New-TimeSpan -Start $ResticStateLastMaintenance -End $(Get-Date)
        if(($delta.Days -lt $SnapshotMaintenanceDays) -and ($ResticStateMaintenanceCounter -lt $SnapshotMaintenanceInterval)) {
            Write-Output "[[Maintenance]] Skipped - last maintenance $ResticStateLastMaintenance ($($delta.Days) days, $ResticStateMaintenanceCounter backups ago)" | Tee-Object -Append $SuccessLog
            return
        }
    }

    Write-Output "[[Maintenance]] Start $(Get-Date)" | Tee-Object -Append $SuccessLog
    $maintenance_success = $true
    Start-Sleep 120

    # forget snapshots based upon the retention policy
    Write-Output "[[Maintenance]] Start forgetting..." | Tee-Object -Append $SuccessLog
    & $ResticExe --verbose -q forget $SnapshotRetentionPolicy 3>&1 2>> $ErrorLog | Tee-Object -Append $SuccessLog
    if(-not $?) {
        Write-Output "[[Maintenance]] Forget operation completed with errors" | Tee-Object -Append $ErrorLog | Tee-Object -Append $SuccessLog
        $maintenance_success = $false
    }

    # prune (remove) data from the backup step. Running this separate from `forget` because
    #   `forget` only prunes when it detects removed snapshots upon invocation, not previously removed
    Write-Output "[[Maintenance]] Start pruning..." | Tee-Object -Append $SuccessLog
    & $ResticExe --verbose -q prune 3>&1 2>> $ErrorLog | Tee-Object -Append $SuccessLog
    if(-not $?) {
        Write-Output "[[Maintenance]] Prune operation completed with errors" | Tee-Object -Append $ErrorLog | Tee-Object -Append $SuccessLog
        $maintenance_success = $false
    }

    # check data to ensure consistency
    Write-Output "[[Maintenance]] Start checking..." | Tee-Object -Append $SuccessLog

    # check to determine if we want to do a full data check or not
    $data_check = @()
    if($null -ne $ResticStateLastDeepMaintenance) {
        $delta = New-TimeSpan -Start $ResticStateLastDeepMaintenance -End $(Get-Date)
        if($delta.Days -ge $SnapshotDeepMaintenanceDays) {
            Write-Output "[[Maintenance]] Performing full data check - deep '--read-data' check last ran $ResticStateLastDeepMaintenance ($($delta.Days) days ago)" | Tee-Object -Append $SuccessLog
            $data_check = @("--read-data")
            $Script:ResticStateLastDeepMaintenance = Get-Date
        }
        else {
            Write-Output "[[Maintenance]] Performing fast data check - deep '--read-data' check last ran $ResticStateLastDeepMaintenance ($($delta.Days) days ago)" | Tee-Object -Append $SuccessLog
        }
    }
    else {
        # set the date, but don't do a deep check if we've never done a full data read
        $Script:ResticStateLastDeepMaintenance = Get-Date
    }

    & $ResticExe --verbose -q check @data_check 3>&1 2>> $ErrorLog | Tee-Object -Append $SuccessLog
    if(-not $?) {
        Write-Output "[[Maintenance]] Check completed with errors" | Tee-Object -Append $ErrorLog | Tee-Object -Append $SuccessLog
        $maintenance_success = $false
    }

    Write-Output "[[Maintenance]] End $(Get-Date)" | Tee-Object -Append $SuccessLog
    
    if($maintenance_success -eq $true) {
        $Script:ResticStateLastMaintenance = Get-Date
        $Script:ResticStateMaintenanceCounter = 0;
    }
}

# Run restic backup 
function Invoke-Backup {
    Param($SuccessLog, $ErrorLog)

    Write-Output "[[Backup]] Start $(Get-Date)" | Tee-Object -Append $SuccessLog
    $return_value = $true

	# Launch Restic
	& $ResticExe --verbose backup --use-fs-snapshot --files-from=$LocalIncludeFile --iexclude-file=$WindowsExcludeFile --iexclude-file=$LocalExcludeFile 3>&1 2>> $ErrorLog | Tee-Object -Append $SuccessLog
	Switch ($LastExitCode)
	{
		1 {
		Write-Output "[[Backup]] Failed with errors, see log for more information" | Tee-Object -Append $ErrorLog | Tee-Object -Append $SuccessLog
		$return_value = $false
		}
		3 {
			if ($FailOnIncomplete) {
				Write-Output "[[Backup]] Failed with errors, see log for more information" | Tee-Object -Append $ErrorLog | Tee-Object -Append $SuccessLog
				$return_value = $false
			} else {
				Write-Output "[[Backup]] Completed with errors, see log for more information" | Tee-Object -Append $ErrorLog | Tee-Object -Append $SuccessLog
			}
		}
	}
	Write-Output "[[Backup]] End $(Get-Date)" | Tee-Object -Append $SuccessLog

    return $return_value
}

function Send-Email {
    Param($SuccessLog, $ErrorLog)
    $password = ConvertTo-SecureString $ResticEmailPassword -AsPlainText -Force
    $credentials = New-Object System.Management.Automation.PSCredential ($ResticEmailUsername, $password)

    $status = "SUCCESS"
    $success_after_failure = $false
    $body = ""
    if (($null -ne $SuccessLog) -and (Test-Path $SuccessLog) -and (Get-Item $SuccessLog).Length -gt 0) {
        $body = $(Get-Content -Raw $SuccessLog)
        # if previous run contained an error, send the success email confirming that the error has been resolved
        # (i.e. get previous error log, if it's not empty, trigger the send of the success-after-failure email)
        $previous_error_log = Get-ChildItem $LogPath -Filter '*err.txt' | Sort-Object -Descending LastWriteTime | Select-Object -Skip 1 | Select-Object -First 1
        if(($null -ne $previous_error_log) -and ($previous_error_log.Length -gt 0)){
            $success_after_failure = $true
        }
    }
    else {
        $body = "Critical Error! Restic backup log is empty or missing. Check log file path."
        $status = "ERROR"
    }
    $attachments = @{}
    if (($null -ne $ErrorLog) -and (Test-Path $ErrorLog) -and (Get-Item $ErrorLog).Length -gt 0) {
        $attachments = @{Attachments = $ErrorLog}
        $status = "ERROR"
    }
    if((($status -eq "SUCCESS") -and ($SendEmailOnSuccess -ne $false)) -or ((($status -eq "ERROR") -or $success_after_failure) -and ($SendEmailOnError -ne $false))) {
        $subject = "$env:COMPUTERNAME Restic Backup Report [$status]"
        Send-MailMessage @ResticEmailConfig -From $ResticEmailFrom -To $ResticEmailTo -Credential $credentials -Subject $subject -Body $body @attachments
    }
}

function Invoke-ConnectivityCheck {
    Param($SuccessLog, $ErrorLog)
    # skip the internet connectivity check for local repos
    if(Test-Path $env:RESTIC_REPOSITORY) {
        Write-Output "[[Internet]] Skipping internet connectivity check." | Tee-Object -Append $SuccessLog    
        return $true
    }

    $repository_host = ''

    # use generic internet service for non-specific repo types (e.g. swift:, rclone:, etc. )
    if(($env:RESTIC_REPOSITORY -match "^swift:") -or 
        ($env:RESTIC_REPOSITORY -match "^rclone:")) {
        $repository_host = "cloudflare.com"
    }
    elseif($env:RESTIC_REPOSITORY -match "^b2:") {
        $repository_host = "api.backblazeb2.com"
    }
    elseif($env:RESTIC_REPOSITORY -match "^azure:") {
        $repository_host = "azure.microsoft.com"
    }
    elseif($env:RESTIC_REPOSITORY -match "^gs:") {
        $repository_host = "storage.googleapis.com"
    }
    else {
        # parse connection string for hostname
        #   Uri parser doesn't handle leading connection type info (s3:, sftp:, rest:)
        $connection_string = $env:RESTIC_REPOSITORY -replace "^s3:" -replace "^sftp:" -replace "^rest:"
        $repository_host = ([System.Uri]$connection_string).host
    }

    if([string]::IsNullOrEmpty($repository_host)) {
        Write-Output "[[Internet]] Repository string could not be parsed." | Tee-Object -Append $SuccessLog | Tee-Object -Append $ErrorLog
        return $false
    }

    # test for internet connectivity
    $connections = 0
    $sleep_count = $InternetTestAttempts
    if (Test-NetMetered) {
	if ($BackupoverMetered) {
		    Write-Output "[[Internet]] Current connection is metered. Change config to DISALLOW backup over metered connection." | Tee-Object -Append $SuccessLog | Tee-Object -Append $Successlog
	    }
	    else {
		    Write-Output "[[Internet]] Current connection is metered. Change config to allow backup over metered." | Tee-Object -Append $SuccessLog | Tee-Object -Append $ErrorLog
		    return $false
	    }
    }
        
    while($true) {
        $connections = Get-NetRoute | Where-Object DestinationPrefix -eq '0.0.0.0/0' | Get-NetIPInterface | Where-Object ConnectionState -eq 'Connected' | Measure-Object | ForEach-Object{$_.Count}
        if($sleep_count -le 0) {
            Write-Output "[[Internet]] Connection to repository ($repository_host) could not be established." | Tee-Object -Append $SuccessLog | Tee-Object -Append $ErrorLog
            return $false
        }
        if(($null -eq $connections) -or ($connections -eq 0)) {
            Write-Output "[[Internet]] Waiting for internet connectivity... $sleep_count" | Tee-Object -Append $SuccessLog
            Start-Sleep 30
        }
        elseif(!(Test-Connection -Server $repository_host -Quiet)) {
            Write-Output "[[Internet]] Waiting for connection to repository ($repository_host)... $sleep_count" | Tee-Object -Append $SuccessLog
            Start-Sleep 30
        }
        else {
            return $true
        }
        $sleep_count--
    }
}

# check previous logs
function Invoke-HistoryCheck {
    Param($SuccessLog, $ErrorLog)
    $logs = Get-ChildItem $LogPath -Filter '*err.txt' | ForEach-Object{$_.Length -gt 0}
    $logs_with_success = ($logs | Where-Object {($_ -eq $false)}).Count
    if($logs.Count -gt 0) {
        Write-Output "[[History]] Backup success rate: $logs_with_success / $($logs.Count) ($(($logs_with_success / $logs.Count).tostring("P")))" | Tee-Object -Append $SuccessLog
    }
}

#
#   .SYNOPSIS
#   Returns if current connection is a metered connection or not.
#
#   .DESCRIPTION
#   This cmdlet checks if connection is metered or not.
#
#   This is based on the example in https://msdn.microsoft.com/en-us/library/windows/apps/xaml/jj835821.aspx
#
#   .EXAMPLE
#   Check if connected to a metered connection
#
#   If(Test-NetMetered) { Write-Host "Metered" }

function Test-NetMetered
{
    [void][Windows.Networking.Connectivity.NetworkInformation, Windows, ContentType = WindowsRuntime]
    $networkprofile = [Windows.Networking.Connectivity.NetworkInformation]::GetInternetConnectionProfile()

    if ($networkprofile -eq $null)
    {
        Write-Warning "Can't find any internet connections!"
        return $false
    }

    $cost = $networkprofile.GetConnectionCost()


    if ($cost -eq $null)
    {
        Write-Warning "Can't find any internet connections with a cost!"
        return $false
    }

    if ($cost.Roaming -or $cost.OverDataLimit)
    {
        return $true
    }

    if ($cost.NetworkCostType -eq [Windows.Networking.Connectivity.NetworkCostType]::Fixed -or
    $cost.NetworkCostType -eq [Windows.Networking.Connectivity.NetworkCostType]::Variable)
    {
        return $true
    }

    if ($cost.NetworkCostType -eq [Windows.Networking.Connectivity.NetworkCostType]::Unrestricted)
    {
        return $false
    }

    throw "Network cost type is unknown!"
}

# main function
function Invoke-Main {
    
    # check for elevation, required for creation of shadow copy (VSS)
    if (-not (New-Object Security.Principal.WindowsPrincipal([Security.Principal.WindowsIdentity]::GetCurrent())).IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator))
    {
        Write-Error "[[Backup]] Elevation required (run as administrator). Exiting."
        exit
    }

    # initialize secrets
    . $SecretsScript
    
    # initialize config
    . $ConfigScript
    
    Get-BackupState

    if(!(Test-Path $LogPath)) {
        Write-Error "[[Backup]] Log file directory $LogPath does not exist. Exiting."
        Send-Email
        exit
    }

    $error_count = 0;
    $attempt_count = $GlobalRetryAttempts
    while ($attempt_count -gt 0) {
        # setup logfiles
        $timestamp = Get-Date -Format FileDateTime
        $success_log = Join-Path $LogPath ($timestamp + ".log.txt")
        $error_log = Join-Path $LogPath ($timestamp + ".err.txt")
        
        $internet_available = Invoke-ConnectivityCheck $success_log $error_log
        if($internet_available -eq $true) { 
            Invoke-Unlock $success_log $error_log
            $backup_success = Invoke-Backup $success_log $error_log
            if($backup_success) {
                Invoke-Maintenance $success_log $error_log
            }

            if (!(Test-Path $error_log) -or ((Get-Item $error_log).Length -eq 0)) {
                # successful with no errors; end
                $total_attempts = $GlobalRetryAttempts - $attempt_count + 1
                Write-Output "Succeeded after $total_attempts attempt(s)" | Tee-Object -Append $success_log
                Invoke-HistoryCheck $success_log $error_log
                Send-Email $success_log $error_log
                break;
            }
        }

        Write-Warning "Errors found! Error Log: $error_log"
        $error_count++
        
        $attempt_count--
        if($attempt_count -gt 0) {
            Write-Output "Sleeping for 15 min and then retrying..." | Tee-Object -Append $success_log
        }
        else {
            Write-Output "Retry limit has been reached. No more attempts to backup will be made." | Tee-Object -Append $success_log
        }
        if($internet_available -eq $true) {
            Invoke-HistoryCheck $success_log $error_log
            Send-Email $success_log $error_log
        }
        if($attempt_count -gt 0) {
            Start-Sleep (15*60)
        }
    }

    Set-BackupState

    # cleanup older log files
    Get-ChildItem $LogPath | Where-Object {$_.CreationTime -lt $(Get-Date).AddDays(-$LogRetentionDays)} | Remove-Item

    exit $error_count
}

Invoke-Main