set backup time to 7:15 Israel Time

DISCOURSE_RECOVERY.md

@@ -0,0 +1,139 @@
+# Discourse State and Database Recovery
+
+This restores Discourse from the BorgBase backup configured in `discourse.nix`.
+It replaces the current Discourse state directory and PostgreSQL database with
+the selected Borg archive.
+
+The backup contains:
+
+- `/var/lib/discourse`
+- `/var/backup/discourse/discourse.dump`
+
+## Requirements
+
+- BorgBase repo URL:
+  `ssh://oas17j8p@oas17j8p.repo.borgbase.com/./repo`
+- Decrypted Borg SSH private key from the flash drive
+- Borg repository passphrase from the flash drive
+- A target server that has already been switched to this NixOS config
+- A shell with `borg`, `openssh`, `postgresql`, and `rsync`
+
+On NixOS or another machine with Nix:
+
+```sh
+nix-shell -p borgbackup openssh postgresql rsync
+```
+
+## Prepare Borg Access
+
+Copy the Borg SSH key into a local recovery directory:
+
+```sh
+mkdir -p ~/borg-recovery/discourse
+cp /path/to/flash/borg-private-key ~/borg-recovery/discourse/borg-private-key
+chmod 600 ~/borg-recovery/discourse/borg-private-key
+```
+
+Set the Borg connection environment:
+
+```sh
+export BORG_REPO='ssh://oas17j8p@oas17j8p.repo.borgbase.com/./repo'
+export BORG_RSH='ssh -i ~/borg-recovery/discourse/borg-private-key -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new'
+```
+
+Read the Borg passphrase without showing it on screen:
+
+```sh
+read -rsp 'Borg passphrase: ' BORG_PASSPHRASE
+export BORG_PASSPHRASE
+echo
+```
+
+## Extract a Backup
+
+List available archives:
+
+```sh
+borg list
+```
+
+Choose an archive name from the list, then extract only the Discourse state and
+database dump into a temporary directory. Do not extract directly into `/`.
+
+```sh
+export ARCHIVE='ARCHIVE_NAME_FROM_BORG_LIST'
+
+mkdir -p ~/borg-recovery/discourse/extract
+cd ~/borg-recovery/discourse/extract
+
+borg extract ::$ARCHIVE var/lib/discourse var/backup/discourse/discourse.dump
+```
+
+## Restore on the Server
+
+Run these commands from `~/borg-recovery/discourse/extract` on the target
+server. They assume the NixOS config has already created the `discourse` user,
+`postgresql.service`, and `discourse-postgresql.service`.
+
+Capture absolute tool paths so they still work through `sudo`:
+
+```sh
+RSYNC="$(command -v rsync)"
+PSQL="$(command -v psql)"
+DROPDB="$(command -v dropdb)"
+PG_RESTORE="$(command -v pg_restore)"
+```
+
+Stop Discourse and make sure PostgreSQL is running:
+
+```sh
+sudo systemctl stop discourse.service
+sudo systemctl start postgresql.service
+```
+
+Restore `/var/lib/discourse`:
+
+```sh
+sudo "$RSYNC" -a --delete ./var/lib/discourse/ /var/lib/discourse/
+sudo chown -R discourse:discourse /var/lib/discourse
+```
+
+Replace the `discourse` PostgreSQL database:
+
+```sh
+sudo -u postgres "$PSQL" -d postgres -v ON_ERROR_STOP=1 \
+  -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = 'discourse';"
+
+sudo -u postgres "$DROPDB" --if-exists discourse
+sudo systemctl restart discourse-postgresql.service
+
+sudo -u postgres "$PG_RESTORE" \
+  --exit-on-error \
+  --no-owner \
+  --role=discourse \
+  --dbname=discourse \
+  ./var/backup/discourse/discourse.dump
+
+sudo systemctl restart discourse-postgresql.service
+```
+
+Start Discourse:
+
+```sh
+sudo systemctl start discourse.service
+```
+
+## Verify
+
+Check the service and recent logs:
+
+```sh
+sudo systemctl status discourse.service --no-pager
+sudo journalctl -u discourse.service -b --no-pager -n 100
+```
+
+If networking and DNS are already restored, open:
+
+```text
+https://discourse.torahimderecheretz.com
+```

borgbackup.nix

@@ -0,0 +1,17 @@
+{
+  sops.secrets."borg-private-key" = {
+    sopsFile = ./secrets/borgbackup.yaml;
+    mode = "0400";
+    key = "private-key";
+    owner = "root";
+    group = "root";
+  };
+
+  sops.secrets."borg-passphrase" = {
+    sopsFile = ./secrets/borgbackup.yaml;
+    mode = "0400";
+    key = "passphrase";
+    owner = "root";
+    group = "root";
+  };
+}

configuration.nix

@@ -9,6 +9,7 @@
   imports =
     [ # Include the results of the hardware scan.
       ./hardware-configuration.nix
+      ./borgbackup.nix
       ./forgejo.nix
       ./boot.nix
     ];

discourse.nix

@@ -36,6 +36,75 @@
     };
   };
 
+  services.borgbackup.jobs.discourse = {
+    paths = [
+      "/var/lib/discourse"
+      "/var/backup/discourse/discourse.dump"
+    ];
+    repo = "ssh://oas17j8p@oas17j8p.repo.borgbase.com/./repo";
+    user = "root";
+    group = "root";
+    encryption = {
+      mode = "repokey-blake2";
+      passCommand = "${pkgs.coreutils}/bin/cat ${config.sops.secrets."borg-passphrase".path}";
+    };
+    doInit = true;
+    compression = "auto,zstd";
+    startAt = "*-*-* 07:15:00";
+    persistentTimer = true;
+    prune.keep = {
+      daily = 7;
+      weekly = 4;
+      monthly = 6;
+    };
+    readWritePaths = [ "/var/backup/discourse" ];
+    environment.BORG_RSH = "ssh -i ${config.sops.secrets."borg-private-key".path} -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.config/borg/known_hosts";
+    preHook = ''
+      discourseWasActive=/root/.cache/borg/discourse-was-active
+      dumpFile=/var/backup/discourse/discourse.dump
+
+      if ${pkgs.systemd}/bin/systemctl is-active --quiet discourse.service; then
+        touch "$discourseWasActive"
+        ${pkgs.systemd}/bin/systemctl stop discourse.service
+      else
+        rm -f "$discourseWasActive"
+      fi
+
+      rm -f "$dumpFile"
+      umask 077
+      ${lib.getExe' pkgs.su "su"} -s ${pkgs.runtimeShell} ${config.services.postgresql.superUser} -c '${config.services.postgresql.package}/bin/pg_dump --format=custom --dbname=${config.services.discourse.database.name}' > "$dumpFile"
+    '';
+    postHook = ''
+      discourseWasActive=/root/.cache/borg/discourse-was-active
+      dumpFile=/var/backup/discourse/discourse.dump
+
+      rm -f "$dumpFile"
+
+      if [ -e "$discourseWasActive" ]; then
+        ${pkgs.systemd}/bin/systemctl start discourse.service
+        rm -f "$discourseWasActive"
+      fi
+    '';
+  };
+
+  systemd.tmpfiles.rules = [
+    "d /var/backup/discourse 0700 root root - -"
+  ];
+
+  systemd.services.borgbackup-job-discourse = {
+    after = [
+      "sops-install-secrets.service"
+      "network-online.target"
+      "postgresql.target"
+      "discourse-postgresql.service"
+    ];
+    wants = [
+      "network-online.target"
+      "postgresql.target"
+      "discourse-postgresql.service"
+    ];
+  };
+
   sops.secrets = {
     "discourse-admin-password" = {
       sopsFile = ./secrets/discourseAdminPassword.yaml;

forgejo.nix

@@ -104,18 +104,4 @@
     sopsFile = ./secrets/forgejo.yaml;
     mode = "0400";
   };
-  sops.secrets."borg-private-key" = {
-    sopsFile = ./secrets/borgbackup.yaml;
-    mode = "0400";
-    key = "private-key";
-    owner = "root";
-    group = "root";
-  };
-  sops.secrets."borg-passphrase" = {
-    sopsFile = ./secrets/borgbackup.yaml;
-    mode = "0400";
-    key = "passphrase";
-    owner = "root";
-    group = "root";
-  };
 }