From 4ff9a48398f1e3ebf8e6ea315b326f66bfd6ae0e Mon Sep 17 00:00:00 2001 From: Notkea Date: Sun, 11 Aug 2019 19:09:42 +0200 Subject: [PATCH] nixos/postgresql-wal-receiver: add module (#63799) --- nixos/modules/module-list.nix | 1 + .../backup/postgresql-wal-receiver.nix | 203 ++++++++++++++++++ nixos/tests/all-tests.nix | 1 + nixos/tests/postgresql-wal-receiver.nix | 86 ++++++++ 4 files changed, 291 insertions(+) create mode 100644 nixos/modules/services/backup/postgresql-wal-receiver.nix create mode 100644 nixos/tests/postgresql-wal-receiver.nix diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index 75e513b76c67..c775345ba4c0 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -214,6 +214,7 @@ ./services/backup/duplicity.nix ./services/backup/mysql-backup.nix ./services/backup/postgresql-backup.nix + ./services/backup/postgresql-wal-receiver.nix ./services/backup/restic.nix ./services/backup/restic-rest-server.nix ./services/backup/rsnapshot.nix diff --git a/nixos/modules/services/backup/postgresql-wal-receiver.nix b/nixos/modules/services/backup/postgresql-wal-receiver.nix new file mode 100644 index 000000000000..d9a37037992e --- /dev/null +++ b/nixos/modules/services/backup/postgresql-wal-receiver.nix @@ -0,0 +1,203 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + receiverSubmodule = { + options = { + postgresqlPackage = mkOption { + type = types.package; + example = literalExample "pkgs.postgresql_11"; + description = '' + PostgreSQL package to use. + ''; + }; + + directory = mkOption { + type = types.path; + example = literalExample "/mnt/pg_wal/main/"; + description = '' + Directory to write the output to. + ''; + }; + + statusInterval = mkOption { + type = types.int; + default = 10; + description = '' + Specifies the number of seconds between status packets sent back to the server. + This allows for easier monitoring of the progress from server. + A value of zero disables the periodic status updates completely, + although an update will still be sent when requested by the server, to avoid timeout disconnect. + ''; + }; + + slot = mkOption { + type = types.str; + default = ""; + example = "some_slot_name"; + description = '' + Require pg_receivewal to use an existing replication slot (see + Section 26.2.6 of the PostgreSQL manual). + When this option is used, pg_receivewal will report a flush position to the server, + indicating when each segment has been synchronized to disk so that the server can remove that segment if it is not otherwise needed. + + When the replication client of pg_receivewal is configured on the server as a synchronous standby, + then using a replication slot will report the flush position to the server, but only when a WAL file is closed. + Therefore, that configuration will cause transactions on the primary to wait for a long time and effectively not work satisfactorily. + The option must be specified in addition to make this work correctly. + ''; + }; + + synchronous = mkOption { + type = types.bool; + default = false; + description = '' + Flush the WAL data to disk immediately after it has been received. + Also send a status packet back to the server immediately after flushing, regardless of . + + This option should be specified if the replication client of pg_receivewal is configured on the server as a synchronous standby, + to ensure that timely feedback is sent to the server. + ''; + }; + + compress = mkOption { + type = types.ints.between 0 9; + default = 0; + description = '' + Enables gzip compression of write-ahead logs, and specifies the compression level + (0 through 9, 0 being no compression and 9 being best compression). + The suffix .gz will automatically be added to all filenames. + + This option requires PostgreSQL >= 10. + ''; + }; + + connection = mkOption { + type = types.str; + example = "postgresql://user@somehost"; + description = '' + Specifies parameters used to connect to the server, as a connection string. + See Section 34.1.1 of the PostgreSQL manual for more information. + + Because pg_receivewal doesn't connect to any particular database in the cluster, + database name in the connection string will be ignored. + ''; + }; + + extraArgs = mkOption { + type = with types; listOf str; + default = [ ]; + example = literalExample '' + [ + "--no-sync" + ] + ''; + description = '' + A list of extra arguments to pass to the pg_receivewal command. + ''; + }; + + environment = mkOption { + type = with types; attrsOf str; + default = { }; + example = literalExample '' + { + PGPASSFILE = "/private/passfile"; + PGSSLMODE = "require"; + } + ''; + description = '' + Environment variables passed to the service. + Usable parameters are listed in Section 34.14 of the PostgreSQL manual. + ''; + }; + }; + }; + +in { + options = { + services.postgresqlWalReceiver = { + receivers = mkOption { + type = with types; attrsOf (submodule receiverSubmodule); + default = { }; + example = literalExample '' + { + main = { + postgresqlPackage = pkgs.postgresql_11; + directory = /mnt/pg_wal/main/; + slot = "main_wal_receiver"; + connection = "postgresql://user@somehost"; + }; + } + ''; + description = '' + PostgreSQL WAL receivers. + Stream write-ahead logs from a PostgreSQL server using pg_receivewal (formerly pg_receivexlog). + See the man page for more information. + ''; + }; + }; + }; + + config = let + receivers = config.services.postgresqlWalReceiver.receivers; + in mkIf (receivers != { }) { + users = { + users.postgres = { + uid = config.ids.uids.postgres; + group = "postgres"; + description = "PostgreSQL server user"; + }; + + groups.postgres = { + gid = config.ids.gids.postgres; + }; + }; + + assertions = concatLists (attrsets.mapAttrsToList (name: config: [ + { + assertion = config.compress > 0 -> versionAtLeast config.postgresqlPackage.version "10"; + message = "Invalid configuration for WAL receiver \"${name}\": compress requires PostgreSQL version >= 10."; + } + ]) receivers); + + systemd.tmpfiles.rules = mapAttrsToList (name: config: '' + d ${escapeShellArg config.directory} 0750 postgres postgres - - + '') receivers; + + systemd.services = with attrsets; mapAttrs' (name: config: nameValuePair "postgresql-wal-receiver-${name}" { + description = "PostgreSQL WAL receiver (${name})"; + wantedBy = [ "multi-user.target" ]; + + serviceConfig = { + User = "postgres"; + Group = "postgres"; + KillSignal = "SIGINT"; + Restart = "always"; + RestartSec = 30; + }; + + inherit (config) environment; + + script = let + receiverCommand = postgresqlPackage: + if (versionAtLeast postgresqlPackage.version "10") + then "${postgresqlPackage}/bin/pg_receivewal" + else "${postgresqlPackage}/bin/pg_receivexlog"; + in '' + ${receiverCommand config.postgresqlPackage} \ + --no-password \ + --directory=${escapeShellArg config.directory} \ + --status-interval=${toString config.statusInterval} \ + --dbname=${escapeShellArg config.connection} \ + ${optionalString (config.compress > 0) "--compress=${toString config.compress}"} \ + ${optionalString (config.slot != "") "--slot=${escapeShellArg config.slot}"} \ + ${optionalString config.synchronous "--synchronous"} \ + ${concatStringsSep " " config.extraArgs} + ''; + }) receivers; + }; + + meta.maintainers = with maintainers; [ pacien ]; +} diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 4a802158752c..c24c8ae61a58 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -210,6 +210,7 @@ in plotinus = handleTest ./plotinus.nix {}; postgis = handleTest ./postgis.nix {}; postgresql = handleTest ./postgresql.nix {}; + postgresql-wal-receiver = handleTest ./postgresql-wal-receiver.nix {}; powerdns = handleTest ./powerdns.nix {}; predictable-interface-names = handleTest ./predictable-interface-names.nix {}; printing = handleTest ./printing.nix {}; diff --git a/nixos/tests/postgresql-wal-receiver.nix b/nixos/tests/postgresql-wal-receiver.nix new file mode 100644 index 000000000000..791b041ba95b --- /dev/null +++ b/nixos/tests/postgresql-wal-receiver.nix @@ -0,0 +1,86 @@ +{ system ? builtins.currentSystem +, config ? { } +, pkgs ? import ../.. { inherit system config; } }: + +with import ../lib/testing.nix { inherit system pkgs; }; +with pkgs.lib; + +let + postgresqlDataDir = "/var/db/postgresql/test"; + replicationUser = "wal_receiver_user"; + replicationSlot = "wal_receiver_slot"; + replicationConn = "postgresql://${replicationUser}@localhost"; + baseBackupDir = "/tmp/pg_basebackup"; + walBackupDir = "/tmp/pg_wal"; + recoveryConf = pkgs.writeText "recovery.conf" '' + restore_command = 'cp ${walBackupDir}/%f %p' + ''; + + makePostgresqlWalReceiverTest = subTestName: postgresqlPackage: makeTest { + name = "postgresql-wal-receiver-${subTestName}"; + meta.maintainers = with maintainers; [ pacien ]; + + machine = { ... }: { + services.postgresql = { + package = postgresqlPackage; + enable = true; + dataDir = postgresqlDataDir; + extraConfig = '' + wal_level = archive # alias for replica on pg >= 9.6 + max_wal_senders = 10 + max_replication_slots = 10 + ''; + authentication = '' + host replication ${replicationUser} all trust + ''; + initialScript = pkgs.writeText "init.sql" '' + create user ${replicationUser} replication; + select * from pg_create_physical_replication_slot('${replicationSlot}'); + ''; + }; + + services.postgresqlWalReceiver.receivers.main = { + inherit postgresqlPackage; + connection = replicationConn; + slot = replicationSlot; + directory = walBackupDir; + }; + }; + + testScript = '' + # make an initial base backup + $machine->waitForUnit('postgresql'); + $machine->waitForUnit('postgresql-wal-receiver-main'); + # WAL receiver healthchecks PG every 5 seconds, so let's be sure they have connected each other + # required only for 9.4 + $machine->sleep(5); + $machine->succeed('${postgresqlPackage}/bin/pg_basebackup --dbname=${replicationConn} --pgdata=${baseBackupDir}'); + + # create a dummy table with 100 records + $machine->succeed('sudo -u postgres psql --command="create table dummy as select * from generate_series(1, 100) as val;"'); + + # stop postgres and destroy data + $machine->systemctl('stop postgresql'); + $machine->systemctl('stop postgresql-wal-receiver-main'); + $machine->succeed('rm -r ${postgresqlDataDir}/{base,global,pg_*}'); + + # restore the base backup + $machine->succeed('cp -r ${baseBackupDir}/* ${postgresqlDataDir} && chown postgres:postgres -R ${postgresqlDataDir}'); + + # prepare WAL and recovery + $machine->succeed('chmod a+rX -R ${walBackupDir}'); + $machine->execute('for part in ${walBackupDir}/*.partial; do mv $part ''${part%%.*}; done'); # make use of partial segments too + $machine->succeed('cp ${recoveryConf} ${postgresqlDataDir}/recovery.conf && chmod 666 ${postgresqlDataDir}/recovery.conf'); + + # replay WAL + $machine->systemctl('start postgresql'); + $machine->waitForFile('${postgresqlDataDir}/recovery.done'); + $machine->systemctl('restart postgresql'); + $machine->waitForUnit('postgresql'); + + # check that our records have been restored + $machine->succeed('test $(sudo -u postgres psql --pset="pager=off" --tuples-only --command="select count(distinct val) from dummy;") -eq 100'); + ''; + }; + +in mapAttrs makePostgresqlWalReceiverTest (import ../../pkgs/servers/sql/postgresql pkgs)