From e7947f3efb269cac7bcb075986e88d28b80a3831 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Sat, 24 Apr 2021 08:58:47 +0200 Subject: [PATCH 1/4] prometheus-unbound-exporter: init at unstable-2021-03-17 --- .../prometheus/unbound-exporter.nix | 26 +++++++++++++++++++ pkgs/top-level/all-packages.nix | 1 + 2 files changed, 27 insertions(+) create mode 100644 pkgs/servers/monitoring/prometheus/unbound-exporter.nix diff --git a/pkgs/servers/monitoring/prometheus/unbound-exporter.nix b/pkgs/servers/monitoring/prometheus/unbound-exporter.nix new file mode 100644 index 000000000000..c085226d9091 --- /dev/null +++ b/pkgs/servers/monitoring/prometheus/unbound-exporter.nix @@ -0,0 +1,26 @@ +{ lib, rustPlatform, fetchFromGitHub, openssl, pkg-config }: + +rustPlatform.buildRustPackage rec { + pname = "unbound-telemetry"; + version = "unstable-2021-03-17"; + + src = fetchFromGitHub { + owner = "svartalf"; + repo = pname; + rev = "7f1b6d4e9e4b6a3216a78c23df745bcf8fc84021"; + sha256 = "xCelL6WGaTRhDJkkUdpdwj1zcKKAU2dyUv3mHeI4oAw="; + }; + + cargoSha256 = "P3nAtYOuwNSLMP7q1L5zKTsZ6rJA/qL1mhVHzP3szi4="; + + nativeBuildInputs = [ pkg-config ]; + + buildInputs = [ openssl ]; + + meta = with lib; { + description = "Prometheus exporter for Unbound DNS resolver"; + homepage = "https://github.com/svartalf/unbound-telemetry"; + license = licenses.mit; + maintainers = with maintainers; [ SuperSandro2000 ]; + }; +} diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 6915f6bcf846..b6ecedd2f33f 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -19110,6 +19110,7 @@ in prometheus-tor-exporter = callPackage ../servers/monitoring/prometheus/tor-exporter.nix { }; prometheus-statsd-exporter = callPackage ../servers/monitoring/prometheus/statsd-exporter.nix { }; prometheus-surfboard-exporter = callPackage ../servers/monitoring/prometheus/surfboard-exporter.nix { }; + prometheus-unbound-exporter = callPackage ../servers/monitoring/prometheus/unbound-exporter.nix { }; prometheus-unifi-exporter = callPackage ../servers/monitoring/prometheus/unifi-exporter { }; prometheus-varnish-exporter = callPackage ../servers/monitoring/prometheus/varnish-exporter.nix { }; prometheus-jmx-httpserver = callPackage ../servers/monitoring/prometheus/jmx-httpserver.nix { }; From ba13dc0652697b6bb05d6977d94271ea0d110de9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Sat, 24 Apr 2021 08:59:32 +0200 Subject: [PATCH 2/4] nixos/prometheus: add unbound exporter --- .../monitoring/prometheus/exporters.nix | 1 + .../prometheus/exporters/unbound.nix | 59 +++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 nixos/modules/services/monitoring/prometheus/exporters/unbound.nix diff --git a/nixos/modules/services/monitoring/prometheus/exporters.nix b/nixos/modules/services/monitoring/prometheus/exporters.nix index e0c5ceccfcc9..ce7c215fd149 100644 --- a/nixos/modules/services/monitoring/prometheus/exporters.nix +++ b/nixos/modules/services/monitoring/prometheus/exporters.nix @@ -59,6 +59,7 @@ let "surfboard" "systemd" "tor" + "unbound" "unifi" "unifi-poller" "varnish" diff --git a/nixos/modules/services/monitoring/prometheus/exporters/unbound.nix b/nixos/modules/services/monitoring/prometheus/exporters/unbound.nix new file mode 100644 index 000000000000..56a559531c14 --- /dev/null +++ b/nixos/modules/services/monitoring/prometheus/exporters/unbound.nix @@ -0,0 +1,59 @@ +{ config, lib, pkgs, options }: + +with lib; + +let + cfg = config.services.prometheus.exporters.unbound; +in +{ + port = 9167; + extraOpts = { + fetchType = mkOption { + # TODO: add shm when upstream implemented it + type = types.enum [ "tcp" "uds" ]; + default = "uds"; + description = '' + Which methods the exporter uses to get the information from unbound. + ''; + }; + + telemetryPath = mkOption { + type = types.str; + default = "/metrics"; + description = '' + Path under which to expose metrics. + ''; + }; + + controlInterface = mkOption { + type = types.nullOr types.str; + default = null; + example = "/run/unbound/unbound.socket"; + description = '' + Path to the unbound socket for uds mode or the control interface port for tcp mode. + + Example: + uds-mode: /run/unbound/unbound.socket + tcp-mode: 127.0.0.1:8953 + ''; + }; + }; + + serviceOpts = mkMerge ([{ + serviceConfig = { + ExecStart = '' + ${pkgs.prometheus-unbound-exporter}/bin/unbound-telemetry \ + ${cfg.fetchType} \ + --bind ${cfg.listenAddress}:${toString cfg.port} \ + --path ${cfg.telemetryPath} \ + ${optionalString (cfg.controlInterface != null) "--control-interface ${cfg.controlInterface}"} \ + ${toString cfg.extraFlags} + ''; + }; + }] ++ [ + (mkIf config.services.unbound.enable { + after = [ "unbound.service" ]; + requires = [ "unbound.service" ]; + }) + ]); +} From da858b16b88cad1ed67662b0fa6f483bb665d498 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Thu, 29 Apr 2021 06:00:45 +0200 Subject: [PATCH 3/4] nixos/tests/prometheus-exporters: add unbound test Author: WilliButz --- nixos/tests/prometheus-exporters.nix | 23 +++++++++++++++++++ .../prometheus/unbound-exporter.nix | 6 ++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/nixos/tests/prometheus-exporters.nix b/nixos/tests/prometheus-exporters.nix index 62c0080dd516..55fe7db4db3d 100644 --- a/nixos/tests/prometheus-exporters.nix +++ b/nixos/tests/prometheus-exporters.nix @@ -1000,6 +1000,29 @@ let ''; }; + unbound = { + exporterConfig = { + enable = true; + fetchType = "uds"; + controlInterface = "/run/unbound/unbound.ctl"; + }; + metricProvider = { + services.unbound = { + enable = true; + localControlSocketPath = "/run/unbound/unbound.ctl"; + }; + systemd.services.prometheus-unbound-exporter.serviceConfig = { + SupplementaryGroups = [ "unbound" ]; + }; + }; + exporterTest = '' + wait_for_unit("unbound.service") + wait_for_unit("prometheus-unbound-exporter.service") + wait_for_open_port(9167) + succeed("curl -sSf localhost:9167/metrics | grep -q 'unbound_up 1'") + ''; + }; + varnish = { exporterConfig = { enable = true; diff --git a/pkgs/servers/monitoring/prometheus/unbound-exporter.nix b/pkgs/servers/monitoring/prometheus/unbound-exporter.nix index c085226d9091..6b26379bf267 100644 --- a/pkgs/servers/monitoring/prometheus/unbound-exporter.nix +++ b/pkgs/servers/monitoring/prometheus/unbound-exporter.nix @@ -1,4 +1,4 @@ -{ lib, rustPlatform, fetchFromGitHub, openssl, pkg-config }: +{ lib, rustPlatform, fetchFromGitHub, openssl, pkg-config, nixosTests }: rustPlatform.buildRustPackage rec { pname = "unbound-telemetry"; @@ -17,6 +17,10 @@ rustPlatform.buildRustPackage rec { buildInputs = [ openssl ]; + passthru.tests = { + inherit (nixosTests.prometheus-exporters) unbound; + }; + meta = with lib; { description = "Prometheus exporter for Unbound DNS resolver"; homepage = "https://github.com/svartalf/unbound-telemetry"; From d3fe53a8a6a3a27f79d8e5916563b841e5e002db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Thu, 29 Apr 2021 06:01:12 +0200 Subject: [PATCH 4/4] nixos/tests/prometheus-exporters: nixpkgs-fmt --- nixos/tests/prometheus-exporters.nix | 308 ++++++++++++++------------- 1 file changed, 160 insertions(+), 148 deletions(-) diff --git a/nixos/tests/prometheus-exporters.nix b/nixos/tests/prometheus-exporters.nix index 55fe7db4db3d..0b6f80b5dc13 100644 --- a/nixos/tests/prometheus-exporters.nix +++ b/nixos/tests/prometheus-exporters.nix @@ -1,65 +1,65 @@ { system ? builtins.currentSystem -, config ? {} +, config ? { } , pkgs ? import ../.. { inherit system config; } }: let inherit (import ../lib/testing-python.nix { inherit system pkgs; }) makeTest; inherit (pkgs.lib) concatStringsSep maintainers mapAttrs mkMerge - removeSuffix replaceChars singleton splitString; + removeSuffix replaceChars singleton splitString; -/* - * The attrset `exporterTests` contains one attribute - * for each exporter test. Each of these attributes - * is expected to be an attrset containing: - * - * `exporterConfig`: - * this attribute set contains config for the exporter itself - * - * `exporterTest` - * this attribute set contains test instructions - * - * `metricProvider` (optional) - * this attribute contains additional machine config - * - * `nodeName` (optional) - * override an incompatible testnode name - * - * Example: - * exporterTests. = { - * exporterConfig = { - * enable = true; - * }; - * metricProvider = { - * services..enable = true; - * }; - * exporterTest = '' - * wait_for_unit("prometheus--exporter.service") - * wait_for_open_port("1234") - * succeed("curl -sSf 'localhost:1234/metrics'") - * ''; - * }; - * - * # this would generate the following test config: - * - * nodes. = { - * services.prometheus. = { - * enable = true; - * }; - * services..enable = true; - * }; - * - * testScript = '' - * .start() - * .wait_for_unit("prometheus--exporter.service") - * .wait_for_open_port("1234") - * .succeed("curl -sSf 'localhost:1234/metrics'") - * .shutdown() - * ''; - */ + /* + * The attrset `exporterTests` contains one attribute + * for each exporter test. Each of these attributes + * is expected to be an attrset containing: + * + * `exporterConfig`: + * this attribute set contains config for the exporter itself + * + * `exporterTest` + * this attribute set contains test instructions + * + * `metricProvider` (optional) + * this attribute contains additional machine config + * + * `nodeName` (optional) + * override an incompatible testnode name + * + * Example: + * exporterTests. = { + * exporterConfig = { + * enable = true; + * }; + * metricProvider = { + * services..enable = true; + * }; + * exporterTest = '' + * wait_for_unit("prometheus--exporter.service") + * wait_for_open_port("1234") + * succeed("curl -sSf 'localhost:1234/metrics'") + * ''; + * }; + * + * # this would generate the following test config: + * + * nodes. = { + * services.prometheus. = { + * enable = true; + * }; + * services..enable = true; + * }; + * + * testScript = '' + * .start() + * .wait_for_unit("prometheus--exporter.service") + * .wait_for_open_port("1234") + * .succeed("curl -sSf 'localhost:1234/metrics'") + * .shutdown() + * ''; + */ exporterTests = { - apcupsd = { + apcupsd = { exporterConfig = { enable = true; }; @@ -188,20 +188,21 @@ let "plugin":"testplugin", "time":DATE }] - ''; in '' - wait_for_unit("prometheus-collectd-exporter.service") - wait_for_open_port(9103) - succeed( - 'echo \'${postData}\'> /tmp/data.json' - ) - succeed('sed -ie "s DATE $(date +%s) " /tmp/data.json') - succeed( - "curl -sSfH 'Content-Type: application/json' -X POST --data @/tmp/data.json localhost:9103/collectd" - ) - succeed( - "curl -sSf localhost:9103/metrics | grep -q 'collectd_testplugin_gauge{instance=\"testhost\"} 23'" - ) - ''; + ''; in + '' + wait_for_unit("prometheus-collectd-exporter.service") + wait_for_open_port(9103) + succeed( + 'echo \'${postData}\'> /tmp/data.json' + ) + succeed('sed -ie "s DATE $(date +%s) " /tmp/data.json') + succeed( + "curl -sSfH 'Content-Type: application/json' -X POST --data @/tmp/data.json localhost:9103/collectd" + ) + succeed( + "curl -sSf localhost:9103/metrics | grep -q 'collectd_testplugin_gauge{instance=\"testhost\"} 23'" + ) + ''; }; dnsmasq = { @@ -254,7 +255,8 @@ let ''; }; - fritzbox = { # TODO add proper test case + fritzbox = { + # TODO add proper test case exporterConfig = { enable = true; }; @@ -373,19 +375,19 @@ let ''; systemd.services.lnd = { serviceConfig.ExecStart = '' - ${pkgs.lnd}/bin/lnd \ - --datadir=/var/lib/lnd \ - --tlscertpath=/var/lib/lnd/tls.cert \ - --tlskeypath=/var/lib/lnd/tls.key \ - --logdir=/var/log/lnd \ - --bitcoin.active \ - --bitcoin.mainnet \ - --bitcoin.node=bitcoind \ - --bitcoind.rpcuser=bitcoinrpc \ - --bitcoind.rpcpass=hunter2 \ - --bitcoind.zmqpubrawblock=tcp://127.0.0.1:28332 \ - --bitcoind.zmqpubrawtx=tcp://127.0.0.1:28333 \ - --readonlymacaroonpath=/var/lib/lnd/readonly.macaroon + ${pkgs.lnd}/bin/lnd \ + --datadir=/var/lib/lnd \ + --tlscertpath=/var/lib/lnd/tls.cert \ + --tlskeypath=/var/lib/lnd/tls.key \ + --logdir=/var/log/lnd \ + --bitcoin.active \ + --bitcoin.mainnet \ + --bitcoin.node=bitcoind \ + --bitcoind.rpcuser=bitcoinrpc \ + --bitcoind.rpcpass=hunter2 \ + --bitcoind.zmqpubrawblock=tcp://127.0.0.1:28332 \ + --bitcoind.zmqpubrawtx=tcp://127.0.0.1:28333 \ + --readonlymacaroonpath=/var/lib/lnd/readonly.macaroon ''; serviceConfig.StateDirectory = "lnd"; wantedBy = [ "multi-user.target" ]; @@ -407,14 +409,14 @@ let configuration = { monitoringInterval = "2s"; mailCheckTimeout = "10s"; - servers = [ { + servers = [{ name = "testserver"; server = "localhost"; port = 25; from = "mail-exporter@localhost"; to = "mail-exporter@localhost"; detectionDir = "/var/spool/mail/mail-exporter/new"; - } ]; + }]; }; }; metricProvider = { @@ -516,15 +518,17 @@ let url = "http://localhost"; }; metricProvider = { - systemd.services.nc-pwfile = let - passfile = (pkgs.writeText "pwfile" "snakeoilpw"); - in { - requiredBy = [ "prometheus-nextcloud-exporter.service" ]; - before = [ "prometheus-nextcloud-exporter.service" ]; - serviceConfig.ExecStart = '' - ${pkgs.coreutils}/bin/install -o nextcloud-exporter -m 0400 ${passfile} /var/nextcloud-pwfile - ''; - }; + systemd.services.nc-pwfile = + let + passfile = (pkgs.writeText "pwfile" "snakeoilpw"); + in + { + requiredBy = [ "prometheus-nextcloud-exporter.service" ]; + before = [ "prometheus-nextcloud-exporter.service" ]; + serviceConfig.ExecStart = '' + ${pkgs.coreutils}/bin/install -o nextcloud-exporter -m 0400 ${passfile} /var/nextcloud-pwfile + ''; + }; services.nginx = { enable = true; virtualHosts."localhost" = { @@ -581,7 +585,7 @@ let syslog = { listen_address = "udp://127.0.0.1:10000"; format = "rfc3164"; - tags = ["nginx"]; + tags = [ "nginx" ]; }; }; } @@ -701,10 +705,10 @@ let exporterConfig = { enable = true; group = "openvpn"; - statusPaths = ["/run/openvpn-test"]; + statusPaths = [ "/run/openvpn-test" ]; }; metricProvider = { - users.groups.openvpn = {}; + users.groups.openvpn = { }; services.openvpn.servers.test = { config = '' dev tun @@ -824,19 +828,21 @@ let }; metricProvider = { # Mock rtl_433 binary to return a dummy metric stream. - nixpkgs.overlays = [ (self: super: { - rtl_433 = self.runCommand "rtl_433" {} '' - mkdir -p "$out/bin" - cat < "$out/bin/rtl_433" - #!/bin/sh - while true; do - printf '{"time" : "2020-04-26 13:37:42", "model" : "zopieux", "id" : 55, "channel" : 3, "temperature_C" : 18.000}\n' - sleep 4 - done - EOF - chmod +x "$out/bin/rtl_433" - ''; - }) ]; + nixpkgs.overlays = [ + (self: super: { + rtl_433 = self.runCommand "rtl_433" { } '' + mkdir -p "$out/bin" + cat < "$out/bin/rtl_433" + #!/bin/sh + while true; do + printf '{"time" : "2020-04-26 13:37:42", "model" : "zopieux", "id" : 55, "channel" : 3, "temperature_C" : 18.000}\n' + sleep 4 + done + EOF + chmod +x "$out/bin/rtl_433" + ''; + }) + ]; }; exporterTest = '' wait_for_unit("prometheus-rtl_433-exporter.service") @@ -852,7 +858,7 @@ let smokeping = { exporterConfig = { enable = true; - hosts = ["127.0.0.1"]; + hosts = [ "127.0.0.1" ]; }; exporterTest = '' wait_for_unit("prometheus-smokeping-exporter.service") @@ -990,7 +996,7 @@ let unifi-poller = { nodeName = "unifi_poller"; exporterConfig.enable = true; - exporterConfig.controllers = [ { } ]; + exporterConfig.controllers = [{ }]; exporterTest = '' wait_for_unit("prometheus-unifi-poller-exporter.service") wait_for_open_port(9130) @@ -1052,54 +1058,60 @@ let ''; }; - wireguard = let snakeoil = import ./wireguard/snakeoil-keys.nix; in { - exporterConfig.enable = true; - metricProvider = { - networking.wireguard.interfaces.wg0 = { - ips = [ "10.23.42.1/32" "fc00::1/128" ]; - listenPort = 23542; + wireguard = let snakeoil = import ./wireguard/snakeoil-keys.nix; in + { + exporterConfig.enable = true; + metricProvider = { + networking.wireguard.interfaces.wg0 = { + ips = [ "10.23.42.1/32" "fc00::1/128" ]; + listenPort = 23542; - inherit (snakeoil.peer0) privateKey; + inherit (snakeoil.peer0) privateKey; - peers = singleton { - allowedIPs = [ "10.23.42.2/32" "fc00::2/128" ]; + peers = singleton { + allowedIPs = [ "10.23.42.2/32" "fc00::2/128" ]; - inherit (snakeoil.peer1) publicKey; + inherit (snakeoil.peer1) publicKey; + }; }; + systemd.services.prometheus-wireguard-exporter.after = [ "wireguard-wg0.service" ]; }; - systemd.services.prometheus-wireguard-exporter.after = [ "wireguard-wg0.service" ]; + exporterTest = '' + wait_for_unit("prometheus-wireguard-exporter.service") + wait_for_open_port(9586) + wait_until_succeeds( + "curl -sSf http://localhost:9586/metrics | grep '${snakeoil.peer1.publicKey}'" + ) + ''; }; - exporterTest = '' - wait_for_unit("prometheus-wireguard-exporter.service") - wait_for_open_port(9586) - wait_until_succeeds( - "curl -sSf http://localhost:9586/metrics | grep '${snakeoil.peer1.publicKey}'" - ) - ''; - }; }; in -mapAttrs (exporter: testConfig: (makeTest (let - nodeName = testConfig.nodeName or exporter; +mapAttrs + (exporter: testConfig: (makeTest ( + let + nodeName = testConfig.nodeName or exporter; -in { - name = "prometheus-${exporter}-exporter"; + in + { + name = "prometheus-${exporter}-exporter"; - nodes.${nodeName} = mkMerge [{ - services.prometheus.exporters.${exporter} = testConfig.exporterConfig; - } testConfig.metricProvider or {}]; + nodes.${nodeName} = mkMerge [{ + services.prometheus.exporters.${exporter} = testConfig.exporterConfig; + } testConfig.metricProvider or { }]; - testScript = '' - ${nodeName}.start() - ${concatStringsSep "\n" (map (line: - if (builtins.substring 0 1 line == " " || builtins.substring 0 1 line == ")") - then line - else "${nodeName}.${line}" - ) (splitString "\n" (removeSuffix "\n" testConfig.exporterTest)))} - ${nodeName}.shutdown() - ''; + testScript = '' + ${nodeName}.start() + ${concatStringsSep "\n" (map (line: + if (builtins.substring 0 1 line == " " || builtins.substring 0 1 line == ")") + then line + else "${nodeName}.${line}" + ) (splitString "\n" (removeSuffix "\n" testConfig.exporterTest)))} + ${nodeName}.shutdown() + ''; - meta = with maintainers; { - maintainers = [ willibutz elseym ]; - }; -}))) exporterTests + meta = with maintainers; { + maintainers = [ willibutz elseym ]; + }; + } + ))) + exporterTests