Adds a server-wide backup capability beyond the existing ops_dashboard pg_dump flow: - Daily systemd timer (03:30) runs pg_dumpall + Forgejo dump, then restic to a local NAS repo and an offsite Backblaze B2 repo with Object Lock. Phase-based script with single-instance flock, structured statusfile, systemd hardening, and live-datadir excludes (Postgres / Forgejo) so the dumps stay authoritative. - Ops-agent gets nine new read-only/trigger commands (snapshots, stats, status, logs, plus two triggers) backed by sudoers-whitelisted wrapper scripts that source /etc/restic-backup.env so the agent never sees the restic password or B2 keys. - Two new flows (server_backup_full, server_backup_restore_test) drive the dashboard's "Backup now" and "Restore test" buttons. - /settings/backups gains a Server backup section with overall + per-phase status, NAS / B2 snapshot tables, restore-size / raw-data / dedup-ratio stats, and the last restore-test result. The existing pg_dump section is preserved unchanged. - Runbook docs/runbooks/server-backup.md follows the tailscale-setup pattern (plan + addendum) and covers B2 Object Lock + scoped keys, Forgejo subplan with isolated restore-test stack, the off-server maintenance flow for B2 prune, and the integrity-check schedule. Code-only change — installation on scrum4me-srv follows the runbook. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
191 lines
5.6 KiB
TypeScript
191 lines
5.6 KiB
TypeScript
import type {
|
|
BackupPhase,
|
|
BackupStatus,
|
|
BackupStatusEnvelope,
|
|
OverallStatus,
|
|
PhaseStatus,
|
|
ResticSnapshot,
|
|
ResticStats,
|
|
RestoreTestAssertion,
|
|
RestoreTestStatus,
|
|
} from './types'
|
|
|
|
const PHASE_ORDER = [
|
|
'postgres_dump',
|
|
'forgejo_dump',
|
|
'forgejo_db_dump',
|
|
'restic_nas',
|
|
'restic_b2',
|
|
'forget_nas',
|
|
'check_nas',
|
|
'check_b2',
|
|
] as const
|
|
|
|
function isRecord(v: unknown): v is Record<string, unknown> {
|
|
return typeof v === 'object' && v !== null && !Array.isArray(v)
|
|
}
|
|
|
|
function asString(v: unknown): string | null {
|
|
return typeof v === 'string' ? v : null
|
|
}
|
|
|
|
function asNumber(v: unknown): number | null {
|
|
return typeof v === 'number' && Number.isFinite(v) ? v : null
|
|
}
|
|
|
|
function asPhaseStatus(v: unknown): PhaseStatus {
|
|
if (
|
|
v === 'success' ||
|
|
v === 'skipped' ||
|
|
v === 'degraded' ||
|
|
v === 'failed' ||
|
|
v === 'pending'
|
|
) {
|
|
return v
|
|
}
|
|
return 'pending'
|
|
}
|
|
|
|
function asOverallStatus(v: unknown): OverallStatus {
|
|
if (v === 'success' || v === 'partial_failure' || v === 'failed') return v
|
|
return 'unknown'
|
|
}
|
|
|
|
function parsePhase(name: string, raw: unknown): BackupPhase {
|
|
if (!isRecord(raw)) {
|
|
return {
|
|
name,
|
|
status: 'pending',
|
|
exitCode: null,
|
|
startedAt: null,
|
|
completedAt: null,
|
|
error: null,
|
|
}
|
|
}
|
|
return {
|
|
name,
|
|
status: asPhaseStatus(raw.status),
|
|
exitCode: asNumber(raw.exit_code),
|
|
startedAt: asString(raw.started_at),
|
|
completedAt: asString(raw.completed_at),
|
|
error: asString(raw.error),
|
|
snapshotId: asString(raw.snapshot_id) ?? undefined,
|
|
filesNew: asNumber(raw.files_new),
|
|
dataAddedBytes: asNumber(raw.data_added_bytes),
|
|
outputFile: asString(raw.output_file) ?? undefined,
|
|
bytes: asNumber(raw.bytes),
|
|
}
|
|
}
|
|
|
|
function parseBackupStatus(raw: unknown): BackupStatus | null {
|
|
if (!isRecord(raw)) return null
|
|
const phasesRaw = isRecord(raw.phases) ? raw.phases : {}
|
|
const phases = PHASE_ORDER.map((name) => parsePhase(name, phasesRaw[name]))
|
|
return {
|
|
schemaVersion: asNumber(raw.schema_version) ?? 1,
|
|
overallStatus: asOverallStatus(raw.overall_status),
|
|
startedAt: asString(raw.started_at) ?? '',
|
|
completedAt: asString(raw.completed_at) ?? '',
|
|
durationSeconds: asNumber(raw.duration_seconds) ?? 0,
|
|
host: asString(raw.host) ?? '',
|
|
phases,
|
|
}
|
|
}
|
|
|
|
function parseRestoreTestAssertion(raw: unknown): RestoreTestAssertion | null {
|
|
if (!isRecord(raw)) return null
|
|
const status = raw.status
|
|
if (status !== 'ok' && status !== 'empty' && status !== 'missing') return null
|
|
return {
|
|
path: asString(raw.path) ?? '',
|
|
status,
|
|
bytes: asNumber(raw.bytes) ?? 0,
|
|
}
|
|
}
|
|
|
|
function parseRestoreTestStatus(raw: unknown): RestoreTestStatus | null {
|
|
if (!isRecord(raw)) return null
|
|
const assertionsRaw = Array.isArray(raw.assertions) ? raw.assertions : []
|
|
const assertions: RestoreTestAssertion[] = []
|
|
for (const a of assertionsRaw) {
|
|
const parsed = parseRestoreTestAssertion(a)
|
|
if (parsed) assertions.push(parsed)
|
|
}
|
|
return {
|
|
schemaVersion: asNumber(raw.schema_version) ?? 1,
|
|
overallStatus: asOverallStatus(raw.overall_status),
|
|
startedAt: asString(raw.started_at) ?? '',
|
|
completedAt: asString(raw.completed_at) ?? '',
|
|
durationSeconds: asNumber(raw.duration_seconds) ?? 0,
|
|
repo: asString(raw.repo) ?? '',
|
|
snapshotId: asString(raw.snapshot_id),
|
|
restoreExitCode: asNumber(raw.restore_exit_code),
|
|
target: asString(raw.target) ?? undefined,
|
|
assertions,
|
|
error: asString(raw.error) ?? undefined,
|
|
}
|
|
}
|
|
|
|
export function parseStatusEnvelope(output: string): BackupStatusEnvelope {
|
|
try {
|
|
const trimmed = output.trim()
|
|
if (!trimmed) return { lastRun: null, lastRestoreTest: null }
|
|
const parsed: unknown = JSON.parse(trimmed)
|
|
if (!isRecord(parsed)) return { lastRun: null, lastRestoreTest: null }
|
|
return {
|
|
lastRun: parseBackupStatus(parsed.last_run),
|
|
lastRestoreTest: parseRestoreTestStatus(parsed.last_restore_test),
|
|
}
|
|
} catch {
|
|
return { lastRun: null, lastRestoreTest: null }
|
|
}
|
|
}
|
|
|
|
export function parseResticSnapshots(output: string, repo: 'nas' | 'b2'): ResticSnapshot[] {
|
|
try {
|
|
const trimmed = output.trim()
|
|
if (!trimmed) return []
|
|
const parsed: unknown = JSON.parse(trimmed)
|
|
if (!Array.isArray(parsed)) return []
|
|
const result: ResticSnapshot[] = []
|
|
for (const s of parsed) {
|
|
if (!isRecord(s)) continue
|
|
const id = asString(s.id)
|
|
if (!id) continue
|
|
const shortId = asString(s.short_id) ?? id.slice(0, 8)
|
|
const time = asString(s.time) ?? ''
|
|
const hostname = asString(s.hostname) ?? ''
|
|
const tags = Array.isArray(s.tags)
|
|
? s.tags.filter((t): t is string => typeof t === 'string')
|
|
: []
|
|
const paths = Array.isArray(s.paths)
|
|
? s.paths.filter((p): p is string => typeof p === 'string')
|
|
: []
|
|
const summary = isRecord(s.summary) ? (s.summary as ResticSnapshot['summary']) : null
|
|
result.push({ id, shortId, time, hostname, tags, paths, repo, summary })
|
|
}
|
|
return result
|
|
} catch {
|
|
return []
|
|
}
|
|
}
|
|
|
|
export function parseResticStats(output: string, repo: 'nas' | 'b2'): ResticStats | null {
|
|
try {
|
|
const trimmed = output.trim()
|
|
if (!trimmed) return null
|
|
const parsed: unknown = JSON.parse(trimmed)
|
|
if (!isRecord(parsed)) return null
|
|
return {
|
|
repo,
|
|
snapshotsCount: asNumber(parsed.snapshots_count) ?? 0,
|
|
restoreSizeBytes: asNumber(parsed.restore_size_bytes),
|
|
restoreSizeFiles: asNumber(parsed.restore_size_files),
|
|
rawDataBytes: asNumber(parsed.raw_data_bytes),
|
|
rawBlobCount: asNumber(parsed.raw_blob_count),
|
|
dedupRatio: asNumber(parsed.dedup_ratio),
|
|
}
|
|
} catch {
|
|
return null
|
|
}
|
|
}
|