diff --git a/DISCOURSE_RECOVERY.md b/DISCOURSE_RECOVERY.md new file mode 100644 index 0000000..e3ffb70 --- /dev/null +++ b/DISCOURSE_RECOVERY.md @@ -0,0 +1,139 @@ +# Discourse State and Database Recovery + +This restores Discourse from the BorgBase backup configured in `discourse.nix`. +It replaces the current Discourse state directory and PostgreSQL database with +the selected Borg archive. + +The backup contains: + +- `/var/lib/discourse` +- `/var/backup/discourse/discourse.dump` + +## Requirements + +- BorgBase repo URL: + `ssh://oas17j8p@oas17j8p.repo.borgbase.com/./repo` +- Decrypted Borg SSH private key from the flash drive +- Borg repository passphrase from the flash drive +- A target server that has already been switched to this NixOS config +- A shell with `borg`, `openssh`, `postgresql`, and `rsync` + +On NixOS or another machine with Nix: + +```sh +nix-shell -p borgbackup openssh postgresql rsync +``` + +## Prepare Borg Access + +Copy the Borg SSH key into a local recovery directory: + +```sh +mkdir -p ~/borg-recovery/discourse +cp /path/to/flash/borg-private-key ~/borg-recovery/discourse/borg-private-key +chmod 600 ~/borg-recovery/discourse/borg-private-key +``` + +Set the Borg connection environment: + +```sh +export BORG_REPO='ssh://oas17j8p@oas17j8p.repo.borgbase.com/./repo' +export BORG_RSH='ssh -i ~/borg-recovery/discourse/borg-private-key -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new' +``` + +Read the Borg passphrase without showing it on screen: + +```sh +read -rsp 'Borg passphrase: ' BORG_PASSPHRASE +export BORG_PASSPHRASE +echo +``` + +## Extract a Backup + +List available archives: + +```sh +borg list +``` + +Choose an archive name from the list, then extract only the Discourse state and +database dump into a temporary directory. Do not extract directly into `/`. + +```sh +export ARCHIVE='ARCHIVE_NAME_FROM_BORG_LIST' + +mkdir -p ~/borg-recovery/discourse/extract +cd ~/borg-recovery/discourse/extract + +borg extract ::$ARCHIVE var/lib/discourse var/backup/discourse/discourse.dump +``` + +## Restore on the Server + +Run these commands from `~/borg-recovery/discourse/extract` on the target +server. They assume the NixOS config has already created the `discourse` user, +`postgresql.service`, and `discourse-postgresql.service`. + +Capture absolute tool paths so they still work through `sudo`: + +```sh +RSYNC="$(command -v rsync)" +PSQL="$(command -v psql)" +DROPDB="$(command -v dropdb)" +PG_RESTORE="$(command -v pg_restore)" +``` + +Stop Discourse and make sure PostgreSQL is running: + +```sh +sudo systemctl stop discourse.service +sudo systemctl start postgresql.service +``` + +Restore `/var/lib/discourse`: + +```sh +sudo "$RSYNC" -a --delete ./var/lib/discourse/ /var/lib/discourse/ +sudo chown -R discourse:discourse /var/lib/discourse +``` + +Replace the `discourse` PostgreSQL database: + +```sh +sudo -u postgres "$PSQL" -d postgres -v ON_ERROR_STOP=1 \ + -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = 'discourse';" + +sudo -u postgres "$DROPDB" --if-exists discourse +sudo systemctl restart discourse-postgresql.service + +sudo -u postgres "$PG_RESTORE" \ + --exit-on-error \ + --no-owner \ + --role=discourse \ + --dbname=discourse \ + ./var/backup/discourse/discourse.dump + +sudo systemctl restart discourse-postgresql.service +``` + +Start Discourse: + +```sh +sudo systemctl start discourse.service +``` + +## Verify + +Check the service and recent logs: + +```sh +sudo systemctl status discourse.service --no-pager +sudo journalctl -u discourse.service -b --no-pager -n 100 +``` + +If networking and DNS are already restored, open: + +```text +https://discourse.torahimderecheretz.com +``` diff --git a/discourse.nix b/discourse.nix index 352326f..e8304ba 100644 --- a/discourse.nix +++ b/discourse.nix @@ -36,6 +36,75 @@ }; }; + services.borgbackup.jobs.discourse = { + paths = [ + "/var/lib/discourse" + "/var/backup/discourse/discourse.dump" + ]; + repo = "ssh://oas17j8p@oas17j8p.repo.borgbase.com/./repo"; + user = "root"; + group = "root"; + encryption = { + mode = "repokey-blake2"; + passCommand = "${pkgs.coreutils}/bin/cat ${config.sops.secrets."borg-passphrase".path}"; + }; + doInit = true; + compression = "auto,zstd"; + startAt = "*-*-* 04:15:00"; + persistentTimer = true; + prune.keep = { + daily = 7; + weekly = 4; + monthly = 6; + }; + readWritePaths = [ "/var/backup/discourse" ]; + environment.BORG_RSH = "ssh -i ${config.sops.secrets."borg-private-key".path} -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.config/borg/known_hosts"; + preHook = '' + discourseWasActive=/root/.cache/borg/discourse-was-active + dumpFile=/var/backup/discourse/discourse.dump + + if ${pkgs.systemd}/bin/systemctl is-active --quiet discourse.service; then + touch "$discourseWasActive" + ${pkgs.systemd}/bin/systemctl stop discourse.service + else + rm -f "$discourseWasActive" + fi + + rm -f "$dumpFile" + umask 077 + ${lib.getExe' pkgs.su "su"} -s ${pkgs.runtimeShell} ${config.services.postgresql.superUser} -c '${config.services.postgresql.package}/bin/pg_dump --format=custom --dbname=${config.services.discourse.database.name}' > "$dumpFile" + ''; + postHook = '' + discourseWasActive=/root/.cache/borg/discourse-was-active + dumpFile=/var/backup/discourse/discourse.dump + + rm -f "$dumpFile" + + if [ -e "$discourseWasActive" ]; then + ${pkgs.systemd}/bin/systemctl start discourse.service + rm -f "$discourseWasActive" + fi + ''; + }; + + systemd.tmpfiles.rules = [ + "d /var/backup/discourse 0700 root root - -" + ]; + + systemd.services.borgbackup-job-discourse = { + after = [ + "sops-install-secrets.service" + "network-online.target" + "postgresql.target" + "discourse-postgresql.service" + ]; + wants = [ + "network-online.target" + "postgresql.target" + "discourse-postgresql.service" + ]; + }; + sops.secrets = { "discourse-admin-password" = { sopsFile = ./secrets/discourseAdminPassword.yaml;