diff --git a/README.md b/README.md index 2360599..cfc6dad 100644 --- a/README.md +++ b/README.md @@ -82,10 +82,6 @@ RABBI_GERZI_INITIAL_ADMIN_PASSWORD=replace-with-a-long-random-password - `RABBI_GERZI_INITIAL_ADMIN_PASSWORD` is used only to create that admin if it does not already exist. -The module also requires paths to a Borg repository passphrase file and SSH -private key through `services.rabbi-gerzi.secretFiles`. Keep both files -outside the Nix store and readable only by root. - Generate an app key inside the dev shell: ```sh @@ -152,34 +148,6 @@ cd frontend/rabbi_gerzi printf '%s\n' 'VITE_API_BASE_URL=http://127.0.0.1:8000' > .env.local ``` -## Production Backups - -The NixOS module creates an encrypted Borg backup in: - -```text -ssh://mgjjruz9@mgjjruz9.repo.borgbase.com/./repo -``` - -Each archive contains the complete application state directory and a -custom-format PostgreSQL dump. This covers database records, users, sessions, -and every uploaded icon and PDF. The Nix store, runtime secrets, and -`/var/cache/rabbi-gerzi` are reproducible or recovered separately and are not -archived. - -The persistent timer runs daily at 05:15 in the server's local timezone and -keeps 7 daily, 4 weekly, and 6 monthly archives. PHP-FPM drains active requests -and remains stopped while the dump and archive are created, so database rows -and uploaded files come from the same write-free window. The static frontend -remains available during that window. - -Run an immediate backup with: - -```sh -sudo systemctl start borgbackup-job-rabbi-gerzi.service -``` - -See [RECOVERY.md](./RECOVERY.md) for archive verification and full restoration. - ## Local Development Enter the Nix dev shell before running local development commands: @@ -396,8 +364,6 @@ Example host configuration: "app-key" = { }; "admin-email" = { }; "admin-password" = { }; - "borg-passphrase" = { }; - "borg-private-key" = { }; }; templates."rabbi-gerzi.env".content = '' @@ -410,13 +376,6 @@ Example host configuration: services.rabbi-gerzi = { enable = true; - secretFiles = { - borgPassphrase = - config.sops.secrets."borg-passphrase".path; - borgPrivateKey = - config.sops.secrets."borg-private-key".path; - }; - frontend.hostName = "rabbigerzi.com"; backend = { @@ -453,11 +412,6 @@ Add these values in the editor opened by `sops`: app-key: base64:replace-with-laravel-app-key admin-email: admin@example.com admin-password: replace-with-a-long-random-password -borg-passphrase: replace-with-the-repository-passphrase -borg-private-key: | - -----BEGIN OPENSSH PRIVATE KEY----- - replace-with-the-private-key - -----END OPENSSH PRIVATE KEY----- ``` Commit only the encrypted file written by `sops`. At activation time, diff --git a/RECOVERY.md b/RECOVERY.md deleted file mode 100644 index 84903b8..0000000 --- a/RECOVERY.md +++ /dev/null @@ -1,185 +0,0 @@ -# Rabbi Gerzi State and Database Recovery - -This procedure restores Rabbi Gerzi from its BorgBase backup. It replaces the -current application state directory and PostgreSQL database with the selected -archive. - -The backup contains: - -- `/var/lib/rabbi-gerzi` -- `/var/backup/rabbi-gerzi/rabbi-gerzi.dump` - -## Requirements - -- BorgBase repository URL: - `ssh://mgjjruz9@mgjjruz9.repo.borgbase.com/./repo` -- Decrypted Borg SSH private key and repository passphrase -- The original Rabbi Gerzi production environment, including `APP_KEY` -- A target server already switched to the Rabbi Gerzi NixOS module -- A shell with `borg`, `openssh`, `postgresql`, and `rsync` - -On NixOS or another machine with Nix: - -```sh -nix-shell -p borgbackup openssh postgresql rsync -``` - -The commands below assume the default state directory, database, user, and -PHP-FPM pool names. Substitute the configured values if the NixOS module uses -non-default names. - -## Prepare Borg Access - -Copy the Borg SSH key into a local recovery directory: - -```sh -mkdir -p ~/borg-recovery/rabbi-gerzi -cp /path/to/borg-private-key \ - ~/borg-recovery/rabbi-gerzi/borg-private-key -chmod 600 ~/borg-recovery/rabbi-gerzi/borg-private-key -``` - -Set the Borg connection environment: - -```sh -export BORG_REPO='ssh://mgjjruz9@mgjjruz9.repo.borgbase.com/./repo' -export BORG_RSH='ssh'\ -' -i ~/borg-recovery/rabbi-gerzi/borg-private-key'\ -' -o IdentitiesOnly=yes'\ -' -o StrictHostKeyChecking=accept-new' -``` - -Read the Borg passphrase without displaying it: - -```sh -read -rsp 'Borg passphrase: ' BORG_PASSPHRASE -export BORG_PASSPHRASE -echo -``` - -## Extract an Archive - -List the available archives: - -```sh -borg list -``` - -Choose an archive, then extract it into a temporary working directory. Do not -extract directly into `/`. - -```sh -export ARCHIVE='ARCHIVE_NAME_FROM_BORG_LIST' - -mkdir -p ~/borg-recovery/rabbi-gerzi/extract -cd ~/borg-recovery/rabbi-gerzi/extract - -borg extract ::"$ARCHIVE" var/lib/rabbi-gerzi \ - var/backup/rabbi-gerzi/rabbi-gerzi.dump -``` - -## Verify Without Replacing Production - -Restore the extracted dump into a temporary database and inspect its core -table counts: - -```sh -sudo -u postgres dropdb --if-exists rabbi-gerzi-backup-verify -sudo -u postgres createdb \ - --owner=rabbi-gerzi rabbi-gerzi-backup-verify -sudo -u postgres pg_restore \ - --exit-on-error \ - --no-owner \ - --role=rabbi-gerzi \ - --dbname=rabbi-gerzi-backup-verify \ - ./var/backup/rabbi-gerzi/rabbi-gerzi.dump -sudo -u postgres psql -d rabbi-gerzi-backup-verify <<'SQL' -SELECT 'users' AS table_name, count(*) FROM users -UNION ALL -SELECT 'sets', count(*) FROM sets -UNION ALL -SELECT 'elements', count(*) FROM elements; -SQL -``` - -List the extracted managed files and confirm expected uploads are present: - -```sh -find ./var/lib/rabbi-gerzi/storage/app/public -type f -print -``` - -Remove only the temporary verification database when the checks pass: - -```sh -sudo -u postgres dropdb rabbi-gerzi-backup-verify -``` - -## Restore Production - -Run these commands from `~/borg-recovery/rabbi-gerzi/extract` on the target -server. Capture absolute tool paths so they remain available through `sudo`: - -```sh -RSYNC="$(command -v rsync)" -PSQL="$(command -v psql)" -DROPDB="$(command -v dropdb)" -CREATEDB="$(command -v createdb)" -PG_RESTORE="$(command -v pg_restore)" -``` - -Stop the API and ensure PostgreSQL is running: - -```sh -sudo systemctl stop phpfpm-rabbi-gerzi.service -sudo systemctl start postgresql.service -``` - -Replace the application state and repair ownership: - -```sh -sudo "$RSYNC" -a --delete \ - ./var/lib/rabbi-gerzi/ /var/lib/rabbi-gerzi/ -sudo chown -R rabbi-gerzi:rabbi-gerzi /var/lib/rabbi-gerzi -``` - -Replace the PostgreSQL database: - -```sh -sudo -u postgres "$PSQL" -d postgres -v ON_ERROR_STOP=1 <<'SQL' -SELECT pg_terminate_backend(pid) -FROM pg_stat_activity -WHERE datname = 'rabbi-gerzi' - AND pid <> pg_backend_pid(); -SQL - -sudo -u postgres "$DROPDB" --if-exists rabbi-gerzi -sudo -u postgres "$CREATEDB" --owner=rabbi-gerzi rabbi-gerzi -sudo -u postgres "$PG_RESTORE" \ - --exit-on-error \ - --no-owner \ - --role=rabbi-gerzi \ - --dbname=rabbi-gerzi \ - ./var/backup/rabbi-gerzi/rabbi-gerzi.dump -``` - -Run current migrations and restart the API: - -```sh -set -e -sudo systemctl restart rabbi-gerzi-setup.service -sudo systemctl start phpfpm-rabbi-gerzi.service -``` - -## Verify Production - -Check service state and recent logs: - -```sh -sudo systemctl status rabbi-gerzi-setup.service --no-pager -sudo systemctl status phpfpm-rabbi-gerzi.service --no-pager -sudo journalctl -u rabbi-gerzi-setup.service -b --no-pager -n 100 -sudo journalctl -u phpfpm-rabbi-gerzi.service -b --no-pager -n 100 -``` - -Open the public site and confirm that sets, elements, uploaded images, PDFs, -and administrator login all work. diff --git a/nix/checks/module-eval.nix b/nix/checks/module-eval.nix index f9f2a97..1de9137 100644 --- a/nix/checks/module-eval.nix +++ b/nix/checks/module-eval.nix @@ -19,10 +19,6 @@ let services.rabbi-gerzi = { enable = true; - secretFiles = { - borgPassphrase = "/run/secrets/borg-passphrase"; - borgPrivateKey = "/run/secrets/borg-private-key"; - }; frontend.hostName = "www.example.test"; backend = { hostName = "api.example.test"; @@ -39,14 +35,6 @@ let phpOptions = evaluatedConfig.services.phpfpm.pools.rabbi-gerzi.phpOptions; - backupConfig = evaluatedConfig.services.borgbackup.jobs.rabbi-gerzi; - - backupService = evaluatedConfig.systemd.services.borgbackup-job-rabbi-gerzi; - - phpFpmService = evaluatedConfig.systemd.services.phpfpm-rabbi-gerzi; - - stateDir = evaluatedConfig.services.rabbi-gerzi.stateDir; - expectedUploadMaxFilesize = "upload_max_filesize = ${uploadLimits.phpUploadLimit}"; expectedPostMaxSize = "post_max_size = ${uploadLimits.phpUploadLimit}"; @@ -66,81 +54,6 @@ let passed = lib.hasInfix expectedNginxClientMaxBodySize backendVirtualHost.extraConfig; message = "nginx client_max_body_size is not 6m"; } - { - passed = backupConfig.repo == "ssh://mgjjruz9@mgjjruz9.repo.borgbase.com/./repo"; - message = "Borg repository is not the Rabbi Gerzi repository"; - } - { - passed = builtins.elem stateDir backupConfig.paths; - message = "Borg backup does not include the application state"; - } - { - passed = builtins.elem "/var/backup/rabbi-gerzi/rabbi-gerzi.dump" backupConfig.paths; - message = "Borg backup does not include the database dump"; - } - { - passed = backupConfig.startAt == "*-*-* 05:15:00"; - message = "Borg backup does not run at 05:15"; - } - { - passed = backupConfig.persistentTimer; - message = "Borg timer is not persistent"; - } - { - passed = - backupConfig.prune.keep.daily == 7 - && backupConfig.prune.keep.weekly == 4 - && backupConfig.prune.keep.monthly == 6; - message = "Borg retention policy is incorrect"; - } - { - passed = backupConfig.encryption.mode == "repokey-blake2"; - message = "Borg encryption mode is incorrect"; - } - { - passed = lib.hasInfix "/run/secrets/borg-passphrase" (backupConfig.encryption.passCommand); - message = "Borg passphrase secret is not wired"; - } - { - passed = lib.hasInfix "/run/secrets/borg-private-key" (backupConfig.environment.BORG_RSH); - message = "Borg SSH private key is not wired"; - } - { - passed = backupConfig.compression == "auto,zstd"; - message = "Borg compression is incorrect"; - } - { - passed = backupConfig.doInit; - message = "Borg repository initialization is disabled"; - } - { - passed = builtins.elem "/var/backup/rabbi-gerzi" backupConfig.readWritePaths; - message = "Borg job cannot write the database dump"; - } - { - passed = lib.hasInfix "phpfpm-rabbi-gerzi.service" (backupConfig.preHook); - message = "Borg pre-hook does not stop PHP-FPM"; - } - { - passed = lib.hasInfix "--format=custom" backupConfig.preHook; - message = "Borg pre-hook does not create a custom database dump"; - } - { - passed = lib.hasInfix "phpfpm-rabbi-gerzi.service" (backupConfig.postHook); - message = "Borg post-hook does not restore PHP-FPM"; - } - { - passed = phpFpmService.serviceConfig.KillSignal == "SIGQUIT"; - message = "PHP-FPM does not drain requests before stopping"; - } - { - passed = builtins.elem "sops-install-secrets.service" (backupService.after); - message = "Borg job is not ordered after secrets"; - } - { - passed = builtins.elem "postgresql.service" backupService.after; - message = "Borg job is not ordered after PostgreSQL"; - } ]; failedAssertions = lib.filter (assertion: !assertion.passed) assertions; diff --git a/nix/nixos-module.nix b/nix/nixos-module.nix index 30f8ddf..8bfb03a 100644 --- a/nix/nixos-module.nix +++ b/nix/nixos-module.nix @@ -16,25 +16,9 @@ let uploadLimits = import ./upload-limits.nix; storagePath = "${cfg.stateDir}/storage"; cachePath = "${cfg.cacheDir}/bootstrap-cache"; - backupDir = "/var/backup/rabbi-gerzi"; - databaseDumpPath = "${backupDir}/rabbi-gerzi.dump"; - phpFpmWasActivePath = "/root/.cache/borg/rabbi-gerzi-phpfpm-was-active"; setupService = "rabbi-gerzi-setup"; phpfpmService = "phpfpm-${poolName}"; - databaseDumpCommand = lib.concatStringsSep " " [ - "${config.services.postgresql.package}/bin/pg_dump" - "--format=custom" - "--dbname=${cfg.database.name}" - ]; - - secretFileOption = - description: - lib.mkOption { - type = lib.types.path; - inherit description; - }; - appEnvironment = { APP_ENV = "production"; APP_DEBUG = "false"; @@ -71,11 +55,6 @@ in options.services.rabbi-gerzi = { enable = lib.mkEnableOption "the Rabbi Gerzi application"; - secretFiles = { - borgPassphrase = secretFileOption "Borg repository passphrase file."; - borgPrivateKey = secretFileOption "Borg repository SSH private key file."; - }; - user = lib.mkOption { type = lib.types.str; default = "rabbi-gerzi"; @@ -242,85 +221,21 @@ in ]; }; - services.borgbackup.jobs.rabbi-gerzi = { - paths = [ - cfg.stateDir - databaseDumpPath - ]; - repo = "ssh://mgjjruz9@mgjjruz9.repo.borgbase.com/./repo"; - user = "root"; - group = "root"; - encryption = { - mode = "repokey-blake2"; - passCommand = "${pkgs.coreutils}/bin/cat ${cfg.secretFiles.borgPassphrase}"; - }; - doInit = true; - compression = "auto,zstd"; - startAt = "*-*-* 05:15:00"; - persistentTimer = true; - prune.keep = { - daily = 7; - weekly = 4; - monthly = 6; - }; - readWritePaths = [ backupDir ]; - environment.BORG_RSH = lib.concatStringsSep " " [ - "ssh" - "-i ${cfg.secretFiles.borgPrivateKey}" - "-o IdentitiesOnly=yes" - "-o BatchMode=yes" - "-o StrictHostKeyChecking=accept-new" - "-o UserKnownHostsFile=/root/.config/borg/known_hosts" - ]; - preHook = '' - phpFpmWasActive=${phpFpmWasActivePath} - dumpFile=${databaseDumpPath} - - if ${pkgs.systemd}/bin/systemctl is-active --quiet ${phpfpmService}.service; then - touch "$phpFpmWasActive" - ${pkgs.systemd}/bin/systemctl stop ${phpfpmService}.service - else - rm -f "$phpFpmWasActive" - fi - - rm -f "$dumpFile" - umask 077 - ${lib.getExe' pkgs.su "su"} \ - -s ${pkgs.runtimeShell} \ - ${config.services.postgresql.superUser} \ - -c '${databaseDumpCommand}' \ - > "$dumpFile" - ''; - postHook = '' - phpFpmWasActive=${phpFpmWasActivePath} - dumpFile=${databaseDumpPath} - - rm -f "$dumpFile" - - if [ -e "$phpFpmWasActive" ]; then - ${pkgs.systemd}/bin/systemctl start ${phpfpmService}.service - rm -f "$phpFpmWasActive" - fi - ''; - }; - - systemd.tmpfiles.rules = - (map makeDirectoryRule [ - cfg.stateDir - cfg.cacheDir - storagePath - "${storagePath}/app" - "${storagePath}/app/private" - "${storagePath}/app/public" - "${storagePath}/framework" - "${storagePath}/framework/cache" - "${storagePath}/framework/cache/data" - "${storagePath}/framework/sessions" - "${storagePath}/framework/views" - "${storagePath}/logs" - cachePath - ]) - ++ [ "d ${backupDir} 0700 root root - -" ]; + systemd.tmpfiles.rules = map makeDirectoryRule [ + cfg.stateDir + cfg.cacheDir + storagePath + "${storagePath}/app" + "${storagePath}/app/private" + "${storagePath}/app/public" + "${storagePath}/framework" + "${storagePath}/framework/cache" + "${storagePath}/framework/cache/data" + "${storagePath}/framework/sessions" + "${storagePath}/framework/views" + "${storagePath}/logs" + cachePath + ]; services.phpfpm.pools.${poolName} = { inherit (cfg) user group; @@ -374,7 +289,6 @@ in environment = appEnvironment; serviceConfig = { EnvironmentFile = cfg.backend.environmentFile; - KillSignal = "SIGQUIT"; ReadWritePaths = [ cfg.stateDir cfg.cacheDir @@ -382,20 +296,6 @@ in }; }; - systemd.services.borgbackup-job-rabbi-gerzi = { - after = [ - "sops-install-secrets.service" - "network-online.target" - "postgresql.service" - "${setupService}.service" - ]; - wants = [ - "network-online.target" - "postgresql.service" - "${setupService}.service" - ]; - }; - services.nginx = { enable = lib.mkDefault true; recommendedGzipSettings = lib.mkDefault true;