nixos/btrbk: add module and test
This commit is contained in:
Guillaume Girol
parent
d8079260a3
commit
72894352b8
4 changed files with 332 additions and 0 deletions
|
|
@ -260,6 +260,7 @@
|
|||
./services/backup/bacula.nix
|
||||
./services/backup/borgbackup.nix
|
||||
./services/backup/borgmatic.nix
|
||||
./services/backup/btrbk.nix
|
||||
./services/backup/duplicati.nix
|
||||
./services/backup/duplicity.nix
|
||||
./services/backup/mysql-backup.nix
|
||||
|
|
|
|||
220
nixos/modules/services/backup/btrbk.nix
Normal file
220
nixos/modules/services/backup/btrbk.nix
Normal file
|
|
@ -0,0 +1,220 @@
|
|||
{ config, pkgs, lib, ... }:
|
||||
let
|
||||
cfg = config.services.btrbk;
|
||||
sshEnabled = cfg.sshAccess != [ ];
|
||||
serviceEnabled = cfg.instances != { };
|
||||
attr2Lines = attr:
|
||||
let
|
||||
pairs = lib.attrsets.mapAttrsToList (name: value: { inherit name value; }) attr;
|
||||
isSubsection = value:
|
||||
if builtins.isAttrs value then true
|
||||
else if builtins.isString value then false
|
||||
else throw "invalid type in btrbk config ${builtins.typeOf value}";
|
||||
sortedPairs = lib.lists.partition (x: isSubsection x.value) pairs;
|
||||
in
|
||||
lib.flatten (
|
||||
# non subsections go first
|
||||
(
|
||||
map (pair: [ "${pair.name} ${pair.value}" ]) sortedPairs.wrong
|
||||
)
|
||||
++ # subsections go last
|
||||
(
|
||||
map
|
||||
(
|
||||
pair:
|
||||
lib.mapAttrsToList
|
||||
(
|
||||
childname: value:
|
||||
[ "${pair.name} ${childname}" ] ++ (map (x: " " + x) (attr2Lines value))
|
||||
)
|
||||
pair.value
|
||||
)
|
||||
sortedPairs.right
|
||||
)
|
||||
)
|
||||
;
|
||||
addDefaults = settings: { backend = "btrfs-progs-sudo"; } // settings;
|
||||
mkConfigFile = settings: lib.concatStringsSep "\n" (attr2Lines (addDefaults settings));
|
||||
mkTestedConfigFile = name: settings:
|
||||
let
|
||||
configFile = pkgs.writeText "btrbk-${name}.conf" (mkConfigFile settings);
|
||||
in
|
||||
pkgs.runCommand "btrbk-${name}-tested.conf" { } ''
|
||||
mkdir foo
|
||||
cp ${configFile} $out
|
||||
if (set +o pipefail; ${pkgs.btrbk}/bin/btrbk -c $out ls foo 2>&1 | grep $out);
|
||||
then
|
||||
echo btrbk configuration is invalid
|
||||
cat $out
|
||||
exit 1
|
||||
fi;
|
||||
'';
|
||||
in
|
||||
{
|
||||
options = {
|
||||
services.btrbk = {
|
||||
extraPackages = lib.mkOption {
|
||||
description = "Extra packages for btrbk, like compression utilities for <literal>stream_compress</literal>";
|
||||
type = lib.types.listOf lib.types.package;
|
||||
default = [ ];
|
||||
example = lib.literalExample "[ pkgs.xz ]";
|
||||
};
|
||||
niceness = lib.mkOption {
|
||||
description = "Niceness for local instances of btrbk. Also applies to remote ones connecting via ssh when positive.";
|
||||
type = lib.types.ints.between (-20) 19;
|
||||
default = 10;
|
||||
};
|
||||
ioSchedulingClass = lib.mkOption {
|
||||
description = "IO scheduling class for btrbk (see ionice(1) for a quick description). Applies to local instances, and remote ones connecting by ssh if set to idle.";
|
||||
type = lib.types.enum [ "idle" "best-effort" "realtime" ];
|
||||
default = "best-effort";
|
||||
};
|
||||
instances = lib.mkOption {
|
||||
description = "Set of btrbk instances. The instance named <literal>btrbk</literal> is the default one.";
|
||||
type = with lib.types;
|
||||
attrsOf (
|
||||
submodule {
|
||||
options = {
|
||||
onCalendar = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "daily";
|
||||
description = "How often this btrbk instance is started. See systemd.time(7) for more information about the format.";
|
||||
};
|
||||
settings = lib.mkOption {
|
||||
type = let t = lib.types.attrsOf (lib.types.either lib.types.str (t // { description = "instances of this type recursively"; })); in t;
|
||||
default = { };
|
||||
example = {
|
||||
snapshot_preserve_min = "2d";
|
||||
snapshot_preserve = "14d";
|
||||
volume = {
|
||||
"/mnt/btr_pool" = {
|
||||
target = "/mnt/btr_backup/mylaptop";
|
||||
subvolume = {
|
||||
"rootfs" = { };
|
||||
"home" = { snapshot_create = "always"; };
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
description = "configuration options for btrbk. Nested attrsets translate to subsections.";
|
||||
};
|
||||
};
|
||||
}
|
||||
);
|
||||
default = { };
|
||||
};
|
||||
sshAccess = lib.mkOption {
|
||||
description = "SSH keys that should be able to make or push snapshots on this system remotely with btrbk";
|
||||
type = with lib.types; listOf (
|
||||
submodule {
|
||||
options = {
|
||||
key = lib.mkOption {
|
||||
type = str;
|
||||
description = "SSH public key allowed to login as user <literal>btrbk</literal> to run remote backups.";
|
||||
};
|
||||
roles = lib.mkOption {
|
||||
type = listOf (enum [ "info" "source" "target" "delete" "snapshot" "send" "receive" ]);
|
||||
example = [ "source" "info" "send" ];
|
||||
description = "What actions can be performed with this SSH key. See ssh_filter_btrbk(1) for details";
|
||||
};
|
||||
};
|
||||
}
|
||||
);
|
||||
default = [ ];
|
||||
};
|
||||
};
|
||||
|
||||
};
|
||||
config = lib.mkIf (sshEnabled || serviceEnabled) {
|
||||
environment.systemPackages = [ pkgs.btrbk ] ++ cfg.extraPackages;
|
||||
security.sudo.extraRules = [
|
||||
{
|
||||
users = [ "btrbk" ];
|
||||
commands = [
|
||||
{ command = "${pkgs.btrfs-progs}/bin/btrfs"; options = [ "NOPASSWD" ]; }
|
||||
{ command = "${pkgs.coreutils}/bin/mkdir"; options = [ "NOPASSWD" ]; }
|
||||
{ command = "${pkgs.coreutils}/bin/readlink"; options = [ "NOPASSWD" ]; }
|
||||
# for ssh, they are not the same than the one hard coded in ${pkgs.btrbk}
|
||||
{ command = "/run/current-system/bin/btrfs"; options = [ "NOPASSWD" ]; }
|
||||
{ command = "/run/current-system/sw/bin/mkdir"; options = [ "NOPASSWD" ]; }
|
||||
{ command = "/run/current-system/sw/bin/readlink"; options = [ "NOPASSWD" ]; }
|
||||
];
|
||||
}
|
||||
];
|
||||
users.users.btrbk = {
|
||||
isSystemUser = true;
|
||||
# ssh needs a home directory
|
||||
home = "/var/lib/btrbk";
|
||||
createHome = true;
|
||||
shell = "${pkgs.bash}/bin/bash";
|
||||
group = "btrbk";
|
||||
openssh.authorizedKeys.keys = map
|
||||
(
|
||||
v:
|
||||
let
|
||||
options = lib.concatMapStringsSep " " (x: "--" + x) v.roles;
|
||||
ioniceClass = {
|
||||
"idle" = 3;
|
||||
"best-effort" = 2;
|
||||
"realtime" = 1;
|
||||
}.${cfg.ioSchedulingClass};
|
||||
in
|
||||
''command="${pkgs.util-linux}/bin/ionice -t -c ${toString ioniceClass} ${lib.optionalString (cfg.niceness >= 1) "${pkgs.coreutils}/bin/nice -n ${toString cfg.niceness}"} ${pkgs.btrbk}/share/btrbk/scripts/ssh_filter_btrbk.sh --sudo ${options}" ${v.key}''
|
||||
)
|
||||
cfg.sshAccess;
|
||||
};
|
||||
users.groups.btrbk = { };
|
||||
systemd.tmpfiles.rules = [
|
||||
"d /var/lib/btrbk 0750 btrbk btrbk"
|
||||
"d /var/lib/btrbk/.ssh 0700 btrbk btrbk"
|
||||
"f /var/lib/btrbk/.ssh/config 0700 btrbk btrbk - StrictHostKeyChecking=accept-new"
|
||||
];
|
||||
environment.etc = lib.mapAttrs'
|
||||
(
|
||||
name: instance: {
|
||||
name = "btrbk/${name}.conf";
|
||||
value.source = mkTestedConfigFile name instance.settings;
|
||||
}
|
||||
)
|
||||
cfg.instances;
|
||||
systemd.services = lib.mapAttrs'
|
||||
(
|
||||
name: _: {
|
||||
name = "btrbk-${name}";
|
||||
value = {
|
||||
description = "Takes BTRFS snapshots and maintains retention policies.";
|
||||
unitConfig.Documentation = "man:btrbk(1)";
|
||||
path = [ "/run/wrappers" ] ++ cfg.extraPackages;
|
||||
serviceConfig = {
|
||||
User = "btrbk";
|
||||
Group = "btrbk";
|
||||
Type = "oneshot";
|
||||
ExecStart = "${pkgs.btrbk}/bin/btrbk -c /etc/btrbk/${name}.conf run";
|
||||
Nice = cfg.niceness;
|
||||
IOSchedulingClass = cfg.ioSchedulingClass;
|
||||
StateDirectory = "btrbk";
|
||||
};
|
||||
};
|
||||
}
|
||||
)
|
||||
cfg.instances;
|
||||
|
||||
systemd.timers = lib.mapAttrs'
|
||||
(
|
||||
name: instance: {
|
||||
name = "btrbk-${name}";
|
||||
value = {
|
||||
description = "Timer to take BTRFS snapshots and maintain retention policies.";
|
||||
wantedBy = [ "timers.target" ];
|
||||
timerConfig = {
|
||||
OnCalendar = instance.onCalendar;
|
||||
AccuracySec = "10min";
|
||||
Persistent = true;
|
||||
};
|
||||
};
|
||||
}
|
||||
)
|
||||
cfg.instances;
|
||||
};
|
||||
|
||||
}
|
||||
|
|
@ -48,6 +48,7 @@ in
|
|||
boot-stage1 = handleTest ./boot-stage1.nix {};
|
||||
borgbackup = handleTest ./borgbackup.nix {};
|
||||
botamusique = handleTest ./botamusique.nix {};
|
||||
btrbk = handleTest ./btrbk.nix {};
|
||||
buildbot = handleTest ./buildbot.nix {};
|
||||
buildkite-agents = handleTest ./buildkite-agents.nix {};
|
||||
caddy = handleTest ./caddy.nix {};
|
||||
|
|
|
|||
110
nixos/tests/btrbk.nix
Normal file
110
nixos/tests/btrbk.nix
Normal file
|
|
@ -0,0 +1,110 @@
|
|||
import ./make-test-python.nix ({ pkgs, ... }:
|
||||
|
||||
let
|
||||
privateKey = ''
|
||||
-----BEGIN OPENSSH PRIVATE KEY-----
|
||||
b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
|
||||
QyNTUxOQAAACBx8UB04Q6Q/fwDFjakHq904PYFzG9pU2TJ9KXpaPMcrwAAAJB+cF5HfnBe
|
||||
RwAAAAtzc2gtZWQyNTUxOQAAACBx8UB04Q6Q/fwDFjakHq904PYFzG9pU2TJ9KXpaPMcrw
|
||||
AAAEBN75NsJZSpt63faCuaD75Unko0JjlSDxMhYHAPJk2/xXHxQHThDpD9/AMWNqQer3Tg
|
||||
9gXMb2lTZMn0pelo8xyvAAAADXJzY2h1ZXR6QGt1cnQ=
|
||||
-----END OPENSSH PRIVATE KEY-----
|
||||
'';
|
||||
publicKey = ''
|
||||
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHxQHThDpD9/AMWNqQer3Tg9gXMb2lTZMn0pelo8xyv
|
||||
'';
|
||||
in
|
||||
{
|
||||
name = "btrbk";
|
||||
meta = with pkgs.lib; {
|
||||
maintainers = with maintainers; [ symphorien ];
|
||||
};
|
||||
|
||||
nodes = {
|
||||
archive = { ... }: {
|
||||
environment.systemPackages = with pkgs; [ btrfs-progs ];
|
||||
# note: this makes the privateKey world readable.
|
||||
# don't do it with real ssh keys.
|
||||
environment.etc."btrbk_key".text = privateKey;
|
||||
services.btrbk = {
|
||||
extraPackages = [ pkgs.lz4 ];
|
||||
instances = {
|
||||
remote = {
|
||||
onCalendar = "minutely";
|
||||
settings = {
|
||||
ssh_identity = "/etc/btrbk_key";
|
||||
ssh_user = "btrbk";
|
||||
stream_compress = "lz4";
|
||||
volume = {
|
||||
"ssh://main/mnt" = {
|
||||
target = "/mnt";
|
||||
snapshot_dir = "btrbk/remote";
|
||||
subvolume = "to_backup";
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
main = { ... }: {
|
||||
environment.systemPackages = with pkgs; [ btrfs-progs ];
|
||||
services.openssh = {
|
||||
enable = true;
|
||||
passwordAuthentication = false;
|
||||
challengeResponseAuthentication = false;
|
||||
};
|
||||
services.btrbk = {
|
||||
extraPackages = [ pkgs.lz4 ];
|
||||
sshAccess = [
|
||||
{
|
||||
key = publicKey;
|
||||
roles = [ "source" "send" "info" "delete" ];
|
||||
}
|
||||
];
|
||||
instances = {
|
||||
local = {
|
||||
onCalendar = "minutely";
|
||||
settings = {
|
||||
volume = {
|
||||
"/mnt" = {
|
||||
snapshot_dir = "btrbk/local";
|
||||
subvolume = "to_backup";
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
testScript = ''
|
||||
start_all()
|
||||
|
||||
# create btrfs partition at /mnt
|
||||
for machine in (archive, main):
|
||||
machine.succeed("dd if=/dev/zero of=/data_fs bs=120M count=1")
|
||||
machine.succeed("mkfs.btrfs /data_fs")
|
||||
machine.succeed("mkdir /mnt")
|
||||
machine.succeed("mount /data_fs /mnt")
|
||||
|
||||
# what to backup and where
|
||||
main.succeed("btrfs subvolume create /mnt/to_backup")
|
||||
main.succeed("mkdir -p /mnt/btrbk/{local,remote}")
|
||||
|
||||
# check that local snapshots work
|
||||
with subtest("local"):
|
||||
main.succeed("echo foo > /mnt/to_backup/bar")
|
||||
main.wait_until_succeeds("cat /mnt/btrbk/local/*/bar | grep foo")
|
||||
main.succeed("echo bar > /mnt/to_backup/bar")
|
||||
main.succeed("cat /mnt/btrbk/local/*/bar | grep foo")
|
||||
|
||||
# check that btrfs send/receive works and ssh access works
|
||||
with subtest("remote"):
|
||||
archive.wait_until_succeeds("cat /mnt/*/bar | grep bar")
|
||||
main.succeed("echo baz > /mnt/to_backup/bar")
|
||||
archive.succeed("cat /mnt/*/bar | grep bar")
|
||||
'';
|
||||
})
|
||||
Loading…
Reference in a new issue