From 22a01e0c0417b33ca2e5343645adede05a84ca7c Mon Sep 17 00:00:00 2001 From: Yisroel Baum Date: Mon, 17 Aug 2026 20:47:36 +0300 Subject: [PATCH 1/3] test borg backup configuration --- nix/checks/module-eval.nix | 87 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/nix/checks/module-eval.nix b/nix/checks/module-eval.nix index 1de9137..f9f2a97 100644 --- a/nix/checks/module-eval.nix +++ b/nix/checks/module-eval.nix @@ -19,6 +19,10 @@ let services.rabbi-gerzi = { enable = true; + secretFiles = { + borgPassphrase = "/run/secrets/borg-passphrase"; + borgPrivateKey = "/run/secrets/borg-private-key"; + }; frontend.hostName = "www.example.test"; backend = { hostName = "api.example.test"; @@ -35,6 +39,14 @@ let phpOptions = evaluatedConfig.services.phpfpm.pools.rabbi-gerzi.phpOptions; + backupConfig = evaluatedConfig.services.borgbackup.jobs.rabbi-gerzi; + + backupService = evaluatedConfig.systemd.services.borgbackup-job-rabbi-gerzi; + + phpFpmService = evaluatedConfig.systemd.services.phpfpm-rabbi-gerzi; + + stateDir = evaluatedConfig.services.rabbi-gerzi.stateDir; + expectedUploadMaxFilesize = "upload_max_filesize = ${uploadLimits.phpUploadLimit}"; expectedPostMaxSize = "post_max_size = ${uploadLimits.phpUploadLimit}"; @@ -54,6 +66,81 @@ let passed = lib.hasInfix expectedNginxClientMaxBodySize backendVirtualHost.extraConfig; message = "nginx client_max_body_size is not 6m"; } + { + passed = backupConfig.repo == "ssh://mgjjruz9@mgjjruz9.repo.borgbase.com/./repo"; + message = "Borg repository is not the Rabbi Gerzi repository"; + } + { + passed = builtins.elem stateDir backupConfig.paths; + message = "Borg backup does not include the application state"; + } + { + passed = builtins.elem "/var/backup/rabbi-gerzi/rabbi-gerzi.dump" backupConfig.paths; + message = "Borg backup does not include the database dump"; + } + { + passed = backupConfig.startAt == "*-*-* 05:15:00"; + message = "Borg backup does not run at 05:15"; + } + { + passed = backupConfig.persistentTimer; + message = "Borg timer is not persistent"; + } + { + passed = + backupConfig.prune.keep.daily == 7 + && backupConfig.prune.keep.weekly == 4 + && backupConfig.prune.keep.monthly == 6; + message = "Borg retention policy is incorrect"; + } + { + passed = backupConfig.encryption.mode == "repokey-blake2"; + message = "Borg encryption mode is incorrect"; + } + { + passed = lib.hasInfix "/run/secrets/borg-passphrase" (backupConfig.encryption.passCommand); + message = "Borg passphrase secret is not wired"; + } + { + passed = lib.hasInfix "/run/secrets/borg-private-key" (backupConfig.environment.BORG_RSH); + message = "Borg SSH private key is not wired"; + } + { + passed = backupConfig.compression == "auto,zstd"; + message = "Borg compression is incorrect"; + } + { + passed = backupConfig.doInit; + message = "Borg repository initialization is disabled"; + } + { + passed = builtins.elem "/var/backup/rabbi-gerzi" backupConfig.readWritePaths; + message = "Borg job cannot write the database dump"; + } + { + passed = lib.hasInfix "phpfpm-rabbi-gerzi.service" (backupConfig.preHook); + message = "Borg pre-hook does not stop PHP-FPM"; + } + { + passed = lib.hasInfix "--format=custom" backupConfig.preHook; + message = "Borg pre-hook does not create a custom database dump"; + } + { + passed = lib.hasInfix "phpfpm-rabbi-gerzi.service" (backupConfig.postHook); + message = "Borg post-hook does not restore PHP-FPM"; + } + { + passed = phpFpmService.serviceConfig.KillSignal == "SIGQUIT"; + message = "PHP-FPM does not drain requests before stopping"; + } + { + passed = builtins.elem "sops-install-secrets.service" (backupService.after); + message = "Borg job is not ordered after secrets"; + } + { + passed = builtins.elem "postgresql.service" backupService.after; + message = "Borg job is not ordered after PostgreSQL"; + } ]; failedAssertions = lib.filter (assertion: !assertion.passed) assertions; From 9cb10a96fd14400871e71aff8b30c6220c4ef369 Mon Sep 17 00:00:00 2001 From: Yisroel Baum Date: Mon, 17 Aug 2026 20:49:30 +0300 Subject: [PATCH 2/3] configure borg production backups --- nix/nixos-module.nix | 130 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 115 insertions(+), 15 deletions(-) diff --git a/nix/nixos-module.nix b/nix/nixos-module.nix index 8bfb03a..30f8ddf 100644 --- a/nix/nixos-module.nix +++ b/nix/nixos-module.nix @@ -16,9 +16,25 @@ let uploadLimits = import ./upload-limits.nix; storagePath = "${cfg.stateDir}/storage"; cachePath = "${cfg.cacheDir}/bootstrap-cache"; + backupDir = "/var/backup/rabbi-gerzi"; + databaseDumpPath = "${backupDir}/rabbi-gerzi.dump"; + phpFpmWasActivePath = "/root/.cache/borg/rabbi-gerzi-phpfpm-was-active"; setupService = "rabbi-gerzi-setup"; phpfpmService = "phpfpm-${poolName}"; + databaseDumpCommand = lib.concatStringsSep " " [ + "${config.services.postgresql.package}/bin/pg_dump" + "--format=custom" + "--dbname=${cfg.database.name}" + ]; + + secretFileOption = + description: + lib.mkOption { + type = lib.types.path; + inherit description; + }; + appEnvironment = { APP_ENV = "production"; APP_DEBUG = "false"; @@ -55,6 +71,11 @@ in options.services.rabbi-gerzi = { enable = lib.mkEnableOption "the Rabbi Gerzi application"; + secretFiles = { + borgPassphrase = secretFileOption "Borg repository passphrase file."; + borgPrivateKey = secretFileOption "Borg repository SSH private key file."; + }; + user = lib.mkOption { type = lib.types.str; default = "rabbi-gerzi"; @@ -221,21 +242,85 @@ in ]; }; - systemd.tmpfiles.rules = map makeDirectoryRule [ - cfg.stateDir - cfg.cacheDir - storagePath - "${storagePath}/app" - "${storagePath}/app/private" - "${storagePath}/app/public" - "${storagePath}/framework" - "${storagePath}/framework/cache" - "${storagePath}/framework/cache/data" - "${storagePath}/framework/sessions" - "${storagePath}/framework/views" - "${storagePath}/logs" - cachePath - ]; + services.borgbackup.jobs.rabbi-gerzi = { + paths = [ + cfg.stateDir + databaseDumpPath + ]; + repo = "ssh://mgjjruz9@mgjjruz9.repo.borgbase.com/./repo"; + user = "root"; + group = "root"; + encryption = { + mode = "repokey-blake2"; + passCommand = "${pkgs.coreutils}/bin/cat ${cfg.secretFiles.borgPassphrase}"; + }; + doInit = true; + compression = "auto,zstd"; + startAt = "*-*-* 05:15:00"; + persistentTimer = true; + prune.keep = { + daily = 7; + weekly = 4; + monthly = 6; + }; + readWritePaths = [ backupDir ]; + environment.BORG_RSH = lib.concatStringsSep " " [ + "ssh" + "-i ${cfg.secretFiles.borgPrivateKey}" + "-o IdentitiesOnly=yes" + "-o BatchMode=yes" + "-o StrictHostKeyChecking=accept-new" + "-o UserKnownHostsFile=/root/.config/borg/known_hosts" + ]; + preHook = '' + phpFpmWasActive=${phpFpmWasActivePath} + dumpFile=${databaseDumpPath} + + if ${pkgs.systemd}/bin/systemctl is-active --quiet ${phpfpmService}.service; then + touch "$phpFpmWasActive" + ${pkgs.systemd}/bin/systemctl stop ${phpfpmService}.service + else + rm -f "$phpFpmWasActive" + fi + + rm -f "$dumpFile" + umask 077 + ${lib.getExe' pkgs.su "su"} \ + -s ${pkgs.runtimeShell} \ + ${config.services.postgresql.superUser} \ + -c '${databaseDumpCommand}' \ + > "$dumpFile" + ''; + postHook = '' + phpFpmWasActive=${phpFpmWasActivePath} + dumpFile=${databaseDumpPath} + + rm -f "$dumpFile" + + if [ -e "$phpFpmWasActive" ]; then + ${pkgs.systemd}/bin/systemctl start ${phpfpmService}.service + rm -f "$phpFpmWasActive" + fi + ''; + }; + + systemd.tmpfiles.rules = + (map makeDirectoryRule [ + cfg.stateDir + cfg.cacheDir + storagePath + "${storagePath}/app" + "${storagePath}/app/private" + "${storagePath}/app/public" + "${storagePath}/framework" + "${storagePath}/framework/cache" + "${storagePath}/framework/cache/data" + "${storagePath}/framework/sessions" + "${storagePath}/framework/views" + "${storagePath}/logs" + cachePath + ]) + ++ [ "d ${backupDir} 0700 root root - -" ]; services.phpfpm.pools.${poolName} = { inherit (cfg) user group; @@ -289,6 +374,7 @@ in environment = appEnvironment; serviceConfig = { EnvironmentFile = cfg.backend.environmentFile; + KillSignal = "SIGQUIT"; ReadWritePaths = [ cfg.stateDir cfg.cacheDir @@ -296,6 +382,20 @@ in }; }; + systemd.services.borgbackup-job-rabbi-gerzi = { + after = [ + "sops-install-secrets.service" + "network-online.target" + "postgresql.service" + "${setupService}.service" + ]; + wants = [ + "network-online.target" + "postgresql.service" + "${setupService}.service" + ]; + }; + services.nginx = { enable = lib.mkDefault true; recommendedGzipSettings = lib.mkDefault true; From cee2d18ab2fbdbd373f9d9a051791e4c9cfef51f Mon Sep 17 00:00:00 2001 From: Yisroel Baum Date: Mon, 17 Aug 2026 20:52:30 +0300 Subject: [PATCH 3/3] document borg backup recovery --- README.md | 46 +++++++++++++ RECOVERY.md | 185 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 231 insertions(+) create mode 100644 RECOVERY.md diff --git a/README.md b/README.md index cfc6dad..2360599 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,10 @@ RABBI_GERZI_INITIAL_ADMIN_PASSWORD=replace-with-a-long-random-password - `RABBI_GERZI_INITIAL_ADMIN_PASSWORD` is used only to create that admin if it does not already exist. +The module also requires paths to a Borg repository passphrase file and SSH +private key through `services.rabbi-gerzi.secretFiles`. Keep both files +outside the Nix store and readable only by root. + Generate an app key inside the dev shell: ```sh @@ -148,6 +152,34 @@ cd frontend/rabbi_gerzi printf '%s\n' 'VITE_API_BASE_URL=http://127.0.0.1:8000' > .env.local ``` +## Production Backups + +The NixOS module creates an encrypted Borg backup in: + +```text +ssh://mgjjruz9@mgjjruz9.repo.borgbase.com/./repo +``` + +Each archive contains the complete application state directory and a +custom-format PostgreSQL dump. This covers database records, users, sessions, +and every uploaded icon and PDF. The Nix store, runtime secrets, and +`/var/cache/rabbi-gerzi` are reproducible or recovered separately and are not +archived. + +The persistent timer runs daily at 05:15 in the server's local timezone and +keeps 7 daily, 4 weekly, and 6 monthly archives. PHP-FPM drains active requests +and remains stopped while the dump and archive are created, so database rows +and uploaded files come from the same write-free window. The static frontend +remains available during that window. + +Run an immediate backup with: + +```sh +sudo systemctl start borgbackup-job-rabbi-gerzi.service +``` + +See [RECOVERY.md](./RECOVERY.md) for archive verification and full restoration. + ## Local Development Enter the Nix dev shell before running local development commands: @@ -364,6 +396,8 @@ Example host configuration: "app-key" = { }; "admin-email" = { }; "admin-password" = { }; + "borg-passphrase" = { }; + "borg-private-key" = { }; }; templates."rabbi-gerzi.env".content = '' @@ -376,6 +410,13 @@ Example host configuration: services.rabbi-gerzi = { enable = true; + secretFiles = { + borgPassphrase = + config.sops.secrets."borg-passphrase".path; + borgPrivateKey = + config.sops.secrets."borg-private-key".path; + }; + frontend.hostName = "rabbigerzi.com"; backend = { @@ -412,6 +453,11 @@ Add these values in the editor opened by `sops`: app-key: base64:replace-with-laravel-app-key admin-email: admin@example.com admin-password: replace-with-a-long-random-password +borg-passphrase: replace-with-the-repository-passphrase +borg-private-key: | + -----BEGIN OPENSSH PRIVATE KEY----- + replace-with-the-private-key + -----END OPENSSH PRIVATE KEY----- ``` Commit only the encrypted file written by `sops`. At activation time, diff --git a/RECOVERY.md b/RECOVERY.md new file mode 100644 index 0000000..84903b8 --- /dev/null +++ b/RECOVERY.md @@ -0,0 +1,185 @@ +# Rabbi Gerzi State and Database Recovery + +This procedure restores Rabbi Gerzi from its BorgBase backup. It replaces the +current application state directory and PostgreSQL database with the selected +archive. + +The backup contains: + +- `/var/lib/rabbi-gerzi` +- `/var/backup/rabbi-gerzi/rabbi-gerzi.dump` + +## Requirements + +- BorgBase repository URL: + `ssh://mgjjruz9@mgjjruz9.repo.borgbase.com/./repo` +- Decrypted Borg SSH private key and repository passphrase +- The original Rabbi Gerzi production environment, including `APP_KEY` +- A target server already switched to the Rabbi Gerzi NixOS module +- A shell with `borg`, `openssh`, `postgresql`, and `rsync` + +On NixOS or another machine with Nix: + +```sh +nix-shell -p borgbackup openssh postgresql rsync +``` + +The commands below assume the default state directory, database, user, and +PHP-FPM pool names. Substitute the configured values if the NixOS module uses +non-default names. + +## Prepare Borg Access + +Copy the Borg SSH key into a local recovery directory: + +```sh +mkdir -p ~/borg-recovery/rabbi-gerzi +cp /path/to/borg-private-key \ + ~/borg-recovery/rabbi-gerzi/borg-private-key +chmod 600 ~/borg-recovery/rabbi-gerzi/borg-private-key +``` + +Set the Borg connection environment: + +```sh +export BORG_REPO='ssh://mgjjruz9@mgjjruz9.repo.borgbase.com/./repo' +export BORG_RSH='ssh'\ +' -i ~/borg-recovery/rabbi-gerzi/borg-private-key'\ +' -o IdentitiesOnly=yes'\ +' -o StrictHostKeyChecking=accept-new' +``` + +Read the Borg passphrase without displaying it: + +```sh +read -rsp 'Borg passphrase: ' BORG_PASSPHRASE +export BORG_PASSPHRASE +echo +``` + +## Extract an Archive + +List the available archives: + +```sh +borg list +``` + +Choose an archive, then extract it into a temporary working directory. Do not +extract directly into `/`. + +```sh +export ARCHIVE='ARCHIVE_NAME_FROM_BORG_LIST' + +mkdir -p ~/borg-recovery/rabbi-gerzi/extract +cd ~/borg-recovery/rabbi-gerzi/extract + +borg extract ::"$ARCHIVE" var/lib/rabbi-gerzi \ + var/backup/rabbi-gerzi/rabbi-gerzi.dump +``` + +## Verify Without Replacing Production + +Restore the extracted dump into a temporary database and inspect its core +table counts: + +```sh +sudo -u postgres dropdb --if-exists rabbi-gerzi-backup-verify +sudo -u postgres createdb \ + --owner=rabbi-gerzi rabbi-gerzi-backup-verify +sudo -u postgres pg_restore \ + --exit-on-error \ + --no-owner \ + --role=rabbi-gerzi \ + --dbname=rabbi-gerzi-backup-verify \ + ./var/backup/rabbi-gerzi/rabbi-gerzi.dump +sudo -u postgres psql -d rabbi-gerzi-backup-verify <<'SQL' +SELECT 'users' AS table_name, count(*) FROM users +UNION ALL +SELECT 'sets', count(*) FROM sets +UNION ALL +SELECT 'elements', count(*) FROM elements; +SQL +``` + +List the extracted managed files and confirm expected uploads are present: + +```sh +find ./var/lib/rabbi-gerzi/storage/app/public -type f -print +``` + +Remove only the temporary verification database when the checks pass: + +```sh +sudo -u postgres dropdb rabbi-gerzi-backup-verify +``` + +## Restore Production + +Run these commands from `~/borg-recovery/rabbi-gerzi/extract` on the target +server. Capture absolute tool paths so they remain available through `sudo`: + +```sh +RSYNC="$(command -v rsync)" +PSQL="$(command -v psql)" +DROPDB="$(command -v dropdb)" +CREATEDB="$(command -v createdb)" +PG_RESTORE="$(command -v pg_restore)" +``` + +Stop the API and ensure PostgreSQL is running: + +```sh +sudo systemctl stop phpfpm-rabbi-gerzi.service +sudo systemctl start postgresql.service +``` + +Replace the application state and repair ownership: + +```sh +sudo "$RSYNC" -a --delete \ + ./var/lib/rabbi-gerzi/ /var/lib/rabbi-gerzi/ +sudo chown -R rabbi-gerzi:rabbi-gerzi /var/lib/rabbi-gerzi +``` + +Replace the PostgreSQL database: + +```sh +sudo -u postgres "$PSQL" -d postgres -v ON_ERROR_STOP=1 <<'SQL' +SELECT pg_terminate_backend(pid) +FROM pg_stat_activity +WHERE datname = 'rabbi-gerzi' + AND pid <> pg_backend_pid(); +SQL + +sudo -u postgres "$DROPDB" --if-exists rabbi-gerzi +sudo -u postgres "$CREATEDB" --owner=rabbi-gerzi rabbi-gerzi +sudo -u postgres "$PG_RESTORE" \ + --exit-on-error \ + --no-owner \ + --role=rabbi-gerzi \ + --dbname=rabbi-gerzi \ + ./var/backup/rabbi-gerzi/rabbi-gerzi.dump +``` + +Run current migrations and restart the API: + +```sh +set -e +sudo systemctl restart rabbi-gerzi-setup.service +sudo systemctl start phpfpm-rabbi-gerzi.service +``` + +## Verify Production + +Check service state and recent logs: + +```sh +sudo systemctl status rabbi-gerzi-setup.service --no-pager +sudo systemctl status phpfpm-rabbi-gerzi.service --no-pager +sudo journalctl -u rabbi-gerzi-setup.service -b --no-pager -n 100 +sudo journalctl -u phpfpm-rabbi-gerzi.service -b --no-pager -n 100 +``` + +Open the public site and confirm that sets, elements, uploaded images, PDFs, +and administrator login all work.