Merge branch 'borg-backups'

README.md

@@ -82,6 +82,10 @@ RABBI_GERZI_INITIAL_ADMIN_PASSWORD=replace-with-a-long-random-password
 - `RABBI_GERZI_INITIAL_ADMIN_PASSWORD` is used only to create that admin if
   it does not already exist.
 
+The module also requires paths to a Borg repository passphrase file and SSH
+private key through `services.rabbi-gerzi.secretFiles`. Keep both files
+outside the Nix store and readable only by root.
+
 Generate an app key inside the dev shell:
 
 ```sh
@@ -148,6 +152,34 @@ cd frontend/rabbi_gerzi
 printf '%s\n' 'VITE_API_BASE_URL=http://127.0.0.1:8000' > .env.local
 ```
 
+## Production Backups
+
+The NixOS module creates an encrypted Borg backup in:
+
+```text
+ssh://mgjjruz9@mgjjruz9.repo.borgbase.com/./repo
+```
+
+Each archive contains the complete application state directory and a
+custom-format PostgreSQL dump. This covers database records, users, sessions,
+and every uploaded icon and PDF. The Nix store, runtime secrets, and
+`/var/cache/rabbi-gerzi` are reproducible or recovered separately and are not
+archived.
+
+The persistent timer runs daily at 05:15 in the server's local timezone and
+keeps 7 daily, 4 weekly, and 6 monthly archives. PHP-FPM drains active requests
+and remains stopped while the dump and archive are created, so database rows
+and uploaded files come from the same write-free window. The static frontend
+remains available during that window.
+
+Run an immediate backup with:
+
+```sh
+sudo systemctl start borgbackup-job-rabbi-gerzi.service
+```
+
+See [RECOVERY.md](./RECOVERY.md) for archive verification and full restoration.
+
 ## Local Development
 
 Enter the Nix dev shell before running local development commands:
@@ -364,6 +396,8 @@ Example host configuration:
                   "app-key" = { };
                   "admin-email" = { };
                   "admin-password" = { };
+                  "borg-passphrase" = { };
+                  "borg-private-key" = { };
                 };
 
                 templates."rabbi-gerzi.env".content = ''
@@ -376,6 +410,13 @@ Example host configuration:
               services.rabbi-gerzi = {
                 enable = true;
 
+                secretFiles = {
+                  borgPassphrase =
+                    config.sops.secrets."borg-passphrase".path;
+                  borgPrivateKey =
+                    config.sops.secrets."borg-private-key".path;
+                };
+
                 frontend.hostName = "rabbigerzi.com";
 
                 backend = {
@@ -412,6 +453,11 @@ Add these values in the editor opened by `sops`:
 app-key: base64:replace-with-laravel-app-key
 admin-email: admin@example.com
 admin-password: replace-with-a-long-random-password
+borg-passphrase: replace-with-the-repository-passphrase
+borg-private-key: |
+  -----BEGIN OPENSSH PRIVATE KEY-----
+  replace-with-the-private-key
+  -----END OPENSSH PRIVATE KEY-----
 ```
 
 Commit only the encrypted file written by `sops`. At activation time,

RECOVERY.md

@@ -0,0 +1,185 @@
+# Rabbi Gerzi State and Database Recovery
+
+This procedure restores Rabbi Gerzi from its BorgBase backup. It replaces the
+current application state directory and PostgreSQL database with the selected
+archive.
+
+The backup contains:
+
+- `/var/lib/rabbi-gerzi`
+- `/var/backup/rabbi-gerzi/rabbi-gerzi.dump`
+
+## Requirements
+
+- BorgBase repository URL:
+  `ssh://mgjjruz9@mgjjruz9.repo.borgbase.com/./repo`
+- Decrypted Borg SSH private key and repository passphrase
+- The original Rabbi Gerzi production environment, including `APP_KEY`
+- A target server already switched to the Rabbi Gerzi NixOS module
+- A shell with `borg`, `openssh`, `postgresql`, and `rsync`
+
+On NixOS or another machine with Nix:
+
+```sh
+nix-shell -p borgbackup openssh postgresql rsync
+```
+
+The commands below assume the default state directory, database, user, and
+PHP-FPM pool names. Substitute the configured values if the NixOS module uses
+non-default names.
+
+## Prepare Borg Access
+
+Copy the Borg SSH key into a local recovery directory:
+
+```sh
+mkdir -p ~/borg-recovery/rabbi-gerzi
+cp /path/to/borg-private-key \
+  ~/borg-recovery/rabbi-gerzi/borg-private-key
+chmod 600 ~/borg-recovery/rabbi-gerzi/borg-private-key
+```
+
+Set the Borg connection environment:
+
+```sh
+export BORG_REPO='ssh://mgjjruz9@mgjjruz9.repo.borgbase.com/./repo'
+export BORG_RSH='ssh'\
+' -i ~/borg-recovery/rabbi-gerzi/borg-private-key'\
+' -o IdentitiesOnly=yes'\
+' -o StrictHostKeyChecking=accept-new'
+```
+
+Read the Borg passphrase without displaying it:
+
+```sh
+read -rsp 'Borg passphrase: ' BORG_PASSPHRASE
+export BORG_PASSPHRASE
+echo
+```
+
+## Extract an Archive
+
+List the available archives:
+
+```sh
+borg list
+```
+
+Choose an archive, then extract it into a temporary working directory. Do not
+extract directly into `/`.
+
+```sh
+export ARCHIVE='ARCHIVE_NAME_FROM_BORG_LIST'
+
+mkdir -p ~/borg-recovery/rabbi-gerzi/extract
+cd ~/borg-recovery/rabbi-gerzi/extract
+
+borg extract ::"$ARCHIVE" var/lib/rabbi-gerzi \
+  var/backup/rabbi-gerzi/rabbi-gerzi.dump
+```
+
+## Verify Without Replacing Production
+
+Restore the extracted dump into a temporary database and inspect its core
+table counts:
+
+```sh
+sudo -u postgres dropdb --if-exists rabbi-gerzi-backup-verify
+sudo -u postgres createdb \
+  --owner=rabbi-gerzi rabbi-gerzi-backup-verify
+sudo -u postgres pg_restore \
+  --exit-on-error \
+  --no-owner \
+  --role=rabbi-gerzi \
+  --dbname=rabbi-gerzi-backup-verify \
+  ./var/backup/rabbi-gerzi/rabbi-gerzi.dump
+sudo -u postgres psql -d rabbi-gerzi-backup-verify <<'SQL'
+SELECT 'users' AS table_name, count(*) FROM users
+UNION ALL
+SELECT 'sets', count(*) FROM sets
+UNION ALL
+SELECT 'elements', count(*) FROM elements;
+SQL
+```
+
+List the extracted managed files and confirm expected uploads are present:
+
+```sh
+find ./var/lib/rabbi-gerzi/storage/app/public -type f -print
+```
+
+Remove only the temporary verification database when the checks pass:
+
+```sh
+sudo -u postgres dropdb rabbi-gerzi-backup-verify
+```
+
+## Restore Production
+
+Run these commands from `~/borg-recovery/rabbi-gerzi/extract` on the target
+server. Capture absolute tool paths so they remain available through `sudo`:
+
+```sh
+RSYNC="$(command -v rsync)"
+PSQL="$(command -v psql)"
+DROPDB="$(command -v dropdb)"
+CREATEDB="$(command -v createdb)"
+PG_RESTORE="$(command -v pg_restore)"
+```
+
+Stop the API and ensure PostgreSQL is running:
+
+```sh
+sudo systemctl stop phpfpm-rabbi-gerzi.service
+sudo systemctl start postgresql.service
+```
+
+Replace the application state and repair ownership:
+
+```sh
+sudo "$RSYNC" -a --delete \
+  ./var/lib/rabbi-gerzi/ /var/lib/rabbi-gerzi/
+sudo chown -R rabbi-gerzi:rabbi-gerzi /var/lib/rabbi-gerzi
+```
+
+Replace the PostgreSQL database:
+
+```sh
+sudo -u postgres "$PSQL" -d postgres -v ON_ERROR_STOP=1 <<'SQL'
+SELECT pg_terminate_backend(pid)
+FROM pg_stat_activity
+WHERE datname = 'rabbi-gerzi'
+  AND pid <> pg_backend_pid();
+SQL
+
+sudo -u postgres "$DROPDB" --if-exists rabbi-gerzi
+sudo -u postgres "$CREATEDB" --owner=rabbi-gerzi rabbi-gerzi
+sudo -u postgres "$PG_RESTORE" \
+  --exit-on-error \
+  --no-owner \
+  --role=rabbi-gerzi \
+  --dbname=rabbi-gerzi \
+  ./var/backup/rabbi-gerzi/rabbi-gerzi.dump
+```
+
+Run current migrations and restart the API:
+
+```sh
+set -e
+sudo systemctl restart rabbi-gerzi-setup.service
+sudo systemctl start phpfpm-rabbi-gerzi.service
+```
+
+## Verify Production
+
+Check service state and recent logs:
+
+```sh
+sudo systemctl status rabbi-gerzi-setup.service --no-pager
+sudo systemctl status phpfpm-rabbi-gerzi.service --no-pager
+sudo journalctl -u rabbi-gerzi-setup.service -b --no-pager -n 100
+sudo journalctl -u phpfpm-rabbi-gerzi.service -b --no-pager -n 100
+```
+
+Open the public site and confirm that sets, elements, uploaded images, PDFs,
+and administrator login all work.

nix/checks/module-eval.nix

@@ -19,6 +19,10 @@ let
 
           services.rabbi-gerzi = {
             enable = true;
+            secretFiles = {
+              borgPassphrase = "/run/secrets/borg-passphrase";
+              borgPrivateKey = "/run/secrets/borg-private-key";
+            };
             frontend.hostName = "www.example.test";
             backend = {
               hostName = "api.example.test";
@@ -35,6 +39,14 @@ let
 
   phpOptions = evaluatedConfig.services.phpfpm.pools.rabbi-gerzi.phpOptions;
 
+  backupConfig = evaluatedConfig.services.borgbackup.jobs.rabbi-gerzi;
+
+  backupService = evaluatedConfig.systemd.services.borgbackup-job-rabbi-gerzi;
+
+  phpFpmService = evaluatedConfig.systemd.services.phpfpm-rabbi-gerzi;
+
+  stateDir = evaluatedConfig.services.rabbi-gerzi.stateDir;
+
   expectedUploadMaxFilesize = "upload_max_filesize = ${uploadLimits.phpUploadLimit}";
 
   expectedPostMaxSize = "post_max_size = ${uploadLimits.phpUploadLimit}";
@@ -54,6 +66,81 @@ let
       passed = lib.hasInfix expectedNginxClientMaxBodySize backendVirtualHost.extraConfig;
       message = "nginx client_max_body_size is not 6m";
     }
+    {
+      passed = backupConfig.repo == "ssh://mgjjruz9@mgjjruz9.repo.borgbase.com/./repo";
+      message = "Borg repository is not the Rabbi Gerzi repository";
+    }
+    {
+      passed = builtins.elem stateDir backupConfig.paths;
+      message = "Borg backup does not include the application state";
+    }
+    {
+      passed = builtins.elem "/var/backup/rabbi-gerzi/rabbi-gerzi.dump" backupConfig.paths;
+      message = "Borg backup does not include the database dump";
+    }
+    {
+      passed = backupConfig.startAt == "*-*-* 05:15:00";
+      message = "Borg backup does not run at 05:15";
+    }
+    {
+      passed = backupConfig.persistentTimer;
+      message = "Borg timer is not persistent";
+    }
+    {
+      passed =
+        backupConfig.prune.keep.daily == 7
+        && backupConfig.prune.keep.weekly == 4
+        && backupConfig.prune.keep.monthly == 6;
+      message = "Borg retention policy is incorrect";
+    }
+    {
+      passed = backupConfig.encryption.mode == "repokey-blake2";
+      message = "Borg encryption mode is incorrect";
+    }
+    {
+      passed = lib.hasInfix "/run/secrets/borg-passphrase" (backupConfig.encryption.passCommand);
+      message = "Borg passphrase secret is not wired";
+    }
+    {
+      passed = lib.hasInfix "/run/secrets/borg-private-key" (backupConfig.environment.BORG_RSH);
+      message = "Borg SSH private key is not wired";
+    }
+    {
+      passed = backupConfig.compression == "auto,zstd";
+      message = "Borg compression is incorrect";
+    }
+    {
+      passed = backupConfig.doInit;
+      message = "Borg repository initialization is disabled";
+    }
+    {
+      passed = builtins.elem "/var/backup/rabbi-gerzi" backupConfig.readWritePaths;
+      message = "Borg job cannot write the database dump";
+    }
+    {
+      passed = lib.hasInfix "phpfpm-rabbi-gerzi.service" (backupConfig.preHook);
+      message = "Borg pre-hook does not stop PHP-FPM";
+    }
+    {
+      passed = lib.hasInfix "--format=custom" backupConfig.preHook;
+      message = "Borg pre-hook does not create a custom database dump";
+    }
+    {
+      passed = lib.hasInfix "phpfpm-rabbi-gerzi.service" (backupConfig.postHook);
+      message = "Borg post-hook does not restore PHP-FPM";
+    }
+    {
+      passed = phpFpmService.serviceConfig.KillSignal == "SIGQUIT";
+      message = "PHP-FPM does not drain requests before stopping";
+    }
+    {
+      passed = builtins.elem "sops-install-secrets.service" (backupService.after);
+      message = "Borg job is not ordered after secrets";
+    }
+    {
+      passed = builtins.elem "postgresql.service" backupService.after;
+      message = "Borg job is not ordered after PostgreSQL";
+    }
   ];
 
   failedAssertions = lib.filter (assertion: !assertion.passed) assertions;

nix/nixos-module.nix

@@ -16,9 +16,25 @@ let
   uploadLimits = import ./upload-limits.nix;
   storagePath = "${cfg.stateDir}/storage";
   cachePath = "${cfg.cacheDir}/bootstrap-cache";
+  backupDir = "/var/backup/rabbi-gerzi";
+  databaseDumpPath = "${backupDir}/rabbi-gerzi.dump";
+  phpFpmWasActivePath = "/root/.cache/borg/rabbi-gerzi-phpfpm-was-active";
   setupService = "rabbi-gerzi-setup";
   phpfpmService = "phpfpm-${poolName}";
 
+  databaseDumpCommand = lib.concatStringsSep " " [
+    "${config.services.postgresql.package}/bin/pg_dump"
+    "--format=custom"
+    "--dbname=${cfg.database.name}"
+  ];
+
+  secretFileOption =
+    description:
+    lib.mkOption {
+      type = lib.types.path;
+      inherit description;
+    };
+
   appEnvironment = {
     APP_ENV = "production";
     APP_DEBUG = "false";
@@ -55,6 +71,11 @@ in
   options.services.rabbi-gerzi = {
     enable = lib.mkEnableOption "the Rabbi Gerzi application";
 
+    secretFiles = {
+      borgPassphrase = secretFileOption "Borg repository passphrase file.";
+      borgPrivateKey = secretFileOption "Borg repository SSH private key file.";
+    };
+
     user = lib.mkOption {
       type = lib.types.str;
       default = "rabbi-gerzi";
@@ -221,21 +242,85 @@ in
       ];
     };
 
-    systemd.tmpfiles.rules = map makeDirectoryRule [
+    services.borgbackup.jobs.rabbi-gerzi = {
-      cfg.stateDir
+      paths = [
-      cfg.cacheDir
+        cfg.stateDir
-      storagePath
+        databaseDumpPath
-      "${storagePath}/app"
+      ];
-      "${storagePath}/app/private"
+      repo = "ssh://mgjjruz9@mgjjruz9.repo.borgbase.com/./repo";
-      "${storagePath}/app/public"
+      user = "root";
-      "${storagePath}/framework"
+      group = "root";
-      "${storagePath}/framework/cache"
+      encryption = {
-      "${storagePath}/framework/cache/data"
+        mode = "repokey-blake2";
-      "${storagePath}/framework/sessions"
+        passCommand = "${pkgs.coreutils}/bin/cat ${cfg.secretFiles.borgPassphrase}";
-      "${storagePath}/framework/views"
+      };
-      "${storagePath}/logs"
+      doInit = true;
-      cachePath
+      compression = "auto,zstd";
-    ];
+      startAt = "*-*-* 05:15:00";
+      persistentTimer = true;
+      prune.keep = {
+        daily = 7;
+        weekly = 4;
+        monthly = 6;
+      };
+      readWritePaths = [ backupDir ];
+      environment.BORG_RSH = lib.concatStringsSep " " [
+        "ssh"
+        "-i ${cfg.secretFiles.borgPrivateKey}"
+        "-o IdentitiesOnly=yes"
+        "-o BatchMode=yes"
+        "-o StrictHostKeyChecking=accept-new"
+        "-o UserKnownHostsFile=/root/.config/borg/known_hosts"
+      ];
+      preHook = ''
+        phpFpmWasActive=${phpFpmWasActivePath}
+        dumpFile=${databaseDumpPath}
+
+        if ${pkgs.systemd}/bin/systemctl is-active --quiet ${phpfpmService}.service; then
+          touch "$phpFpmWasActive"
+          ${pkgs.systemd}/bin/systemctl stop ${phpfpmService}.service
+        else
+          rm -f "$phpFpmWasActive"
+        fi
+
+        rm -f "$dumpFile"
+        umask 077
+        ${lib.getExe' pkgs.su "su"} \
+          -s ${pkgs.runtimeShell} \
+          ${config.services.postgresql.superUser} \
+          -c '${databaseDumpCommand}' \
+          > "$dumpFile"
+      '';
+      postHook = ''
+        phpFpmWasActive=${phpFpmWasActivePath}
+        dumpFile=${databaseDumpPath}
+
+        rm -f "$dumpFile"
+
+        if [ -e "$phpFpmWasActive" ]; then
+          ${pkgs.systemd}/bin/systemctl start ${phpfpmService}.service
+          rm -f "$phpFpmWasActive"
+        fi
+      '';
+    };
+
+    systemd.tmpfiles.rules =
+      (map makeDirectoryRule [
+        cfg.stateDir
+        cfg.cacheDir
+        storagePath
+        "${storagePath}/app"
+        "${storagePath}/app/private"
+        "${storagePath}/app/public"
+        "${storagePath}/framework"
+        "${storagePath}/framework/cache"
+        "${storagePath}/framework/cache/data"
+        "${storagePath}/framework/sessions"
+        "${storagePath}/framework/views"
+        "${storagePath}/logs"
+        cachePath
+      ])
+      ++ [ "d ${backupDir} 0700 root root - -" ];
 
     services.phpfpm.pools.${poolName} = {
       inherit (cfg) user group;
@@ -289,6 +374,7 @@ in
       environment = appEnvironment;
       serviceConfig = {
         EnvironmentFile = cfg.backend.environmentFile;
+        KillSignal = "SIGQUIT";
         ReadWritePaths = [
           cfg.stateDir
           cfg.cacheDir
@@ -296,6 +382,20 @@ in
       };
     };
 
+    systemd.services.borgbackup-job-rabbi-gerzi = {
+      after = [
+        "sops-install-secrets.service"
+        "network-online.target"
+        "postgresql.service"
+        "${setupService}.service"
+      ];
+      wants = [
+        "network-online.target"
+        "postgresql.service"
+        "${setupService}.service"
+      ];
+    };
+
     services.nginx = {
       enable = lib.mkDefault true;
       recommendedGzipSettings = lib.mkDefault true;