Merge pull request #58255 from jbgi/prometheus2
Add Prometheus 2 service in parallel with 1.x version (continuation)
This commit is contained in:
Bas van Dijk
GitHub
commit
2f2e2971d6
4 changed files with 352 additions and 32 deletions
|
|
@ -37,7 +37,52 @@
|
|||
|
||||
<itemizedlist>
|
||||
<listitem>
|
||||
<para />
|
||||
<para>
|
||||
Besides the existing <option>services.prometheus</option> module which
|
||||
targets Prometheus-1 a new <option>services.prometheus2</option> module
|
||||
has been added which targets Prometheus-2.
|
||||
</para>
|
||||
<para>
|
||||
Both modules can be enabled at the same time. In fact
|
||||
<link xlink:href="https://prometheus.io/docs/prometheus/latest/migration/#storage">
|
||||
this is needed for upgrading existing Prometheus-1 data to Prometheus-2
|
||||
</link>.
|
||||
</para>
|
||||
</listitem>
|
||||
</itemizedlist>
|
||||
</section>
|
||||
|
||||
<section xmlns="http://docbook.org/ns/docbook"
|
||||
xmlns:xlink="http://www.w3.org/1999/xlink"
|
||||
xmlns:xi="http://www.w3.org/2001/XInclude"
|
||||
version="5.0"
|
||||
xml:id="sec-release-19.09-incompatibilities">
|
||||
<title>Backward Incompatibilities</title>
|
||||
|
||||
<para>
|
||||
When upgrading from a previous release, please be aware of the following
|
||||
incompatible changes:
|
||||
</para>
|
||||
|
||||
<itemizedlist>
|
||||
<listitem>
|
||||
<para>
|
||||
The directory where Prometheus will store its metric data is now
|
||||
managed by systemd's StateDirectory mechanism. It still defaults
|
||||
to <literal>/var/lib/prometheus</literal>.
|
||||
</para>
|
||||
<para>
|
||||
Its location can be specified by the new
|
||||
<option>services.prometheus.stateDir</option> option which
|
||||
defaults to <literal>prometheus</literal>. Note that this should
|
||||
be a directory relative to <literal>/var/lib/</literal>.
|
||||
</para>
|
||||
<para>
|
||||
The option <option>services.prometheus.dataDir</option> has been
|
||||
deprecated. You can still set it but it's now required to have
|
||||
<literal>/var/lib/</literal> as a prefix and you can't set
|
||||
<option>services.prometheus.stateDir</option> at the same time.
|
||||
</para>
|
||||
</listitem>
|
||||
</itemizedlist>
|
||||
</section>
|
||||
|
|
|
|||
|
|
@ -4,9 +4,24 @@ with lib;
|
|||
|
||||
let
|
||||
cfg = config.services.prometheus;
|
||||
cfg2 = config.services.prometheus2;
|
||||
promUser = "prometheus";
|
||||
promGroup = "prometheus";
|
||||
|
||||
stateDir =
|
||||
if cfg.stateDir != null
|
||||
then cfg.stateDir
|
||||
else
|
||||
if cfg.dataDir != null
|
||||
then
|
||||
# This assumes /var/lib/ is a prefix of cfg.dataDir.
|
||||
# This is checked as an assertion below.
|
||||
removePrefix stateDirBase cfg.dataDir
|
||||
else "prometheus";
|
||||
stateDirBase = "/var/lib/";
|
||||
workingDir = stateDirBase + stateDir;
|
||||
workingDir2 = stateDirBase + cfg2.stateDir;
|
||||
|
||||
# Get a submodule without any embedded metadata:
|
||||
_filter = x: filterAttrs (k: v: k != "_module") x;
|
||||
|
||||
|
|
@ -17,13 +32,23 @@ let
|
|||
promtool ${what} $out
|
||||
'';
|
||||
|
||||
# a wrapper that verifies that the configuration is valid for
|
||||
# prometheus 2
|
||||
prom2toolCheck = what: name: file:
|
||||
pkgs.runCommand
|
||||
"${name}-${replaceStrings [" "] [""] what}-checked"
|
||||
{ buildInputs = [ cfg2.package ]; } ''
|
||||
ln -s ${file} $out
|
||||
promtool ${what} $out
|
||||
'';
|
||||
|
||||
# Pretty-print JSON to a file
|
||||
writePrettyJSON = name: x:
|
||||
pkgs.runCommand name { preferLocalBuild = true; } ''
|
||||
echo '${builtins.toJSON x}' | ${pkgs.jq}/bin/jq . > $out
|
||||
'';
|
||||
|
||||
# This becomes the main config file
|
||||
# This becomes the main config file for Prometheus 1
|
||||
promConfig = {
|
||||
global = cfg.globalConfig;
|
||||
rule_files = map (promtoolCheck "check-rules" "rules") (cfg.ruleFiles ++ [
|
||||
|
|
@ -35,20 +60,53 @@ let
|
|||
generatedPrometheusYml = writePrettyJSON "prometheus.yml" promConfig;
|
||||
|
||||
prometheusYml = let
|
||||
yml = if cfg.configText != null then
|
||||
yml = if cfg.configText != null then
|
||||
pkgs.writeText "prometheus.yml" cfg.configText
|
||||
else generatedPrometheusYml;
|
||||
in promtoolCheck "check-config" "prometheus.yml" yml;
|
||||
|
||||
cmdlineArgs = cfg.extraFlags ++ [
|
||||
"-storage.local.path=${cfg.dataDir}/metrics"
|
||||
"-storage.local.path=${workingDir}/metrics"
|
||||
"-config.file=${prometheusYml}"
|
||||
"-web.listen-address=${cfg.listenAddress}"
|
||||
"-alertmanager.notification-queue-capacity=${toString cfg.alertmanagerNotificationQueueCapacity}"
|
||||
"-alertmanager.timeout=${toString cfg.alertmanagerTimeout}s"
|
||||
(optionalString (cfg.alertmanagerURL != []) "-alertmanager.url=${concatStringsSep "," cfg.alertmanagerURL}")
|
||||
(optionalString (cfg.webExternalUrl != null) "-web.external-url=${cfg.webExternalUrl}")
|
||||
];
|
||||
] ++
|
||||
optional (cfg.alertmanagerURL != []) "-alertmanager.url=${concatStringsSep "," cfg.alertmanagerURL}" ++
|
||||
optional (cfg.webExternalUrl != null) "-web.external-url=${cfg.webExternalUrl}";
|
||||
|
||||
# This becomes the main config file for Prometheus 2
|
||||
promConfig2 = {
|
||||
global = cfg2.globalConfig;
|
||||
rule_files = map (prom2toolCheck "check rules" "rules") (cfg2.ruleFiles ++ [
|
||||
(pkgs.writeText "prometheus.rules" (concatStringsSep "\n" cfg2.rules))
|
||||
]);
|
||||
scrape_configs = cfg2.scrapeConfigs;
|
||||
alerting = optionalAttrs (cfg2.alertmanagerURL != []) {
|
||||
alertmanagers = [{
|
||||
static_configs = [{
|
||||
targets = cfg2.alertmanagerURL;
|
||||
}];
|
||||
}];
|
||||
};
|
||||
};
|
||||
|
||||
generatedPrometheus2Yml = writePrettyJSON "prometheus.yml" promConfig2;
|
||||
|
||||
prometheus2Yml = let
|
||||
yml = if cfg2.configText != null then
|
||||
pkgs.writeText "prometheus.yml" cfg2.configText
|
||||
else generatedPrometheus2Yml;
|
||||
in prom2toolCheck "check config" "prometheus.yml" yml;
|
||||
|
||||
cmdlineArgs2 = cfg2.extraFlags ++ [
|
||||
"--storage.tsdb.path=${workingDir2}/data/"
|
||||
"--config.file=${prometheus2Yml}"
|
||||
"--web.listen-address=${cfg2.listenAddress}"
|
||||
"--alertmanager.notification-queue-capacity=${toString cfg2.alertmanagerNotificationQueueCapacity}"
|
||||
"--alertmanager.timeout=${toString cfg2.alertmanagerTimeout}s"
|
||||
] ++
|
||||
optional (cfg2.webExternalUrl != null) "--web.external-url=${cfg2.webExternalUrl}";
|
||||
|
||||
promTypes.globalConfig = types.submodule {
|
||||
options = {
|
||||
|
|
@ -403,10 +461,21 @@ in {
|
|||
};
|
||||
|
||||
dataDir = mkOption {
|
||||
type = types.path;
|
||||
default = "/var/lib/prometheus";
|
||||
type = types.nullOr types.path;
|
||||
default = null;
|
||||
description = ''
|
||||
Directory to store Prometheus metrics data.
|
||||
This option is deprecated, please use <option>services.prometheus.stateDir</option>.
|
||||
'';
|
||||
};
|
||||
|
||||
stateDir = mkOption {
|
||||
type = types.nullOr types.str;
|
||||
default = null;
|
||||
description = ''
|
||||
Directory below <literal>${stateDirBase}</literal> to store Prometheus metrics data.
|
||||
This directory will be created automatically using systemd's StateDirectory mechanism.
|
||||
Defaults to <literal>prometheus</literal>.
|
||||
'';
|
||||
};
|
||||
|
||||
|
|
@ -497,30 +566,201 @@ in {
|
|||
'';
|
||||
};
|
||||
};
|
||||
};
|
||||
services.prometheus2 = {
|
||||
|
||||
config = mkIf cfg.enable {
|
||||
users.groups.${promGroup}.gid = config.ids.gids.prometheus;
|
||||
users.users.${promUser} = {
|
||||
description = "Prometheus daemon user";
|
||||
uid = config.ids.uids.prometheus;
|
||||
group = promGroup;
|
||||
home = cfg.dataDir;
|
||||
createHome = true;
|
||||
};
|
||||
systemd.services.prometheus = {
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "network.target" ];
|
||||
script = ''
|
||||
#!/bin/sh
|
||||
exec ${cfg.package}/bin/prometheus \
|
||||
${concatStringsSep " \\\n " cmdlineArgs}
|
||||
'';
|
||||
serviceConfig = {
|
||||
User = promUser;
|
||||
Restart = "always";
|
||||
WorkingDirectory = cfg.dataDir;
|
||||
enable = mkOption {
|
||||
type = types.bool;
|
||||
default = false;
|
||||
description = ''
|
||||
Enable the Prometheus 2 monitoring daemon.
|
||||
'';
|
||||
};
|
||||
|
||||
package = mkOption {
|
||||
type = types.package;
|
||||
default = pkgs.prometheus_2;
|
||||
defaultText = "pkgs.prometheus_2";
|
||||
description = ''
|
||||
The prometheus2 package that should be used.
|
||||
'';
|
||||
};
|
||||
|
||||
listenAddress = mkOption {
|
||||
type = types.str;
|
||||
default = "0.0.0.0:9090";
|
||||
description = ''
|
||||
Address to listen on for the web interface, API, and telemetry.
|
||||
'';
|
||||
};
|
||||
|
||||
stateDir = mkOption {
|
||||
type = types.str;
|
||||
default = "prometheus2";
|
||||
description = ''
|
||||
Directory below <literal>${stateDirBase}</literal> to store Prometheus metrics data.
|
||||
This directory will be created automatically using systemd's StateDirectory mechanism.
|
||||
Defaults to <literal>prometheus2</literal>.
|
||||
'';
|
||||
};
|
||||
|
||||
extraFlags = mkOption {
|
||||
type = types.listOf types.str;
|
||||
default = [];
|
||||
description = ''
|
||||
Extra commandline options when launching Prometheus 2.
|
||||
'';
|
||||
};
|
||||
|
||||
configText = mkOption {
|
||||
type = types.nullOr types.lines;
|
||||
default = null;
|
||||
description = ''
|
||||
If non-null, this option defines the text that is written to
|
||||
prometheus.yml. If null, the contents of prometheus.yml is generated
|
||||
from the structured config options.
|
||||
'';
|
||||
};
|
||||
|
||||
globalConfig = mkOption {
|
||||
type = promTypes.globalConfig;
|
||||
default = {};
|
||||
apply = _filter;
|
||||
description = ''
|
||||
Parameters that are valid in all configuration contexts. They
|
||||
also serve as defaults for other configuration sections
|
||||
'';
|
||||
};
|
||||
|
||||
rules = mkOption {
|
||||
type = types.listOf types.str;
|
||||
default = [];
|
||||
description = ''
|
||||
Alerting and/or Recording rules to evaluate at runtime.
|
||||
'';
|
||||
};
|
||||
|
||||
ruleFiles = mkOption {
|
||||
type = types.listOf types.path;
|
||||
default = [];
|
||||
description = ''
|
||||
Any additional rules files to include in this configuration.
|
||||
'';
|
||||
};
|
||||
|
||||
scrapeConfigs = mkOption {
|
||||
type = types.listOf promTypes.scrape_config;
|
||||
default = [];
|
||||
apply = x: map _filter x;
|
||||
description = ''
|
||||
A list of scrape configurations.
|
||||
'';
|
||||
};
|
||||
|
||||
alertmanagerURL = mkOption {
|
||||
type = types.listOf types.str;
|
||||
default = [];
|
||||
description = ''
|
||||
List of Alertmanager URLs to send notifications to.
|
||||
'';
|
||||
};
|
||||
|
||||
alertmanagerNotificationQueueCapacity = mkOption {
|
||||
type = types.int;
|
||||
default = 10000;
|
||||
description = ''
|
||||
The capacity of the queue for pending alert manager notifications.
|
||||
'';
|
||||
};
|
||||
|
||||
alertmanagerTimeout = mkOption {
|
||||
type = types.int;
|
||||
default = 10;
|
||||
description = ''
|
||||
Alert manager HTTP API timeout (in seconds).
|
||||
'';
|
||||
};
|
||||
|
||||
webExternalUrl = mkOption {
|
||||
type = types.nullOr types.str;
|
||||
default = null;
|
||||
example = "https://example.com/";
|
||||
description = ''
|
||||
The URL under which Prometheus is externally reachable (for example,
|
||||
if Prometheus is served via a reverse proxy).
|
||||
'';
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
config = mkMerge [
|
||||
(mkIf (cfg.enable || cfg2.enable) {
|
||||
users.groups.${promGroup}.gid = config.ids.gids.prometheus;
|
||||
users.users.${promUser} = {
|
||||
description = "Prometheus daemon user";
|
||||
uid = config.ids.uids.prometheus;
|
||||
group = promGroup;
|
||||
};
|
||||
})
|
||||
(mkIf cfg.enable {
|
||||
warnings =
|
||||
optional (cfg.dataDir != null) ''
|
||||
The option services.prometheus.dataDir is deprecated, please use
|
||||
services.prometheus.stateDir.
|
||||
'';
|
||||
assertions = [
|
||||
{
|
||||
assertion = !(cfg.dataDir != null && cfg.stateDir != null);
|
||||
message =
|
||||
"The options services.prometheus.dataDir and services.prometheus.stateDir" +
|
||||
" can't both be set at the same time! It's recommended to only set the latter" +
|
||||
" since the former is deprecated.";
|
||||
}
|
||||
{
|
||||
assertion = cfg.dataDir != null -> hasPrefix stateDirBase cfg.dataDir;
|
||||
message =
|
||||
"The option services.prometheus.dataDir should have ${stateDirBase} as a prefix!";
|
||||
}
|
||||
{
|
||||
assertion = cfg.stateDir != null -> !hasPrefix "/" cfg.stateDir;
|
||||
message =
|
||||
"The option services.prometheus.stateDir shouldn't be an absolute directory." +
|
||||
" It should be a directory relative to ${stateDirBase}.";
|
||||
}
|
||||
{
|
||||
assertion = cfg2.stateDir != null -> !hasPrefix "/" cfg2.stateDir;
|
||||
message =
|
||||
"The option services.prometheus2.stateDir shouldn't be an absolute directory." +
|
||||
" It should be a directory relative to ${stateDirBase}.";
|
||||
}
|
||||
];
|
||||
systemd.services.prometheus = {
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "network.target" ];
|
||||
serviceConfig = {
|
||||
ExecStart = "${cfg.package}/bin/prometheus" +
|
||||
optionalString (length cmdlineArgs != 0) (" \\\n " +
|
||||
concatStringsSep " \\\n " cmdlineArgs);
|
||||
User = promUser;
|
||||
Restart = "always";
|
||||
WorkingDirectory = workingDir;
|
||||
StateDirectory = stateDir;
|
||||
};
|
||||
};
|
||||
})
|
||||
(mkIf cfg2.enable {
|
||||
systemd.services.prometheus2 = {
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "network.target" ];
|
||||
serviceConfig = {
|
||||
ExecStart = "${cfg2.package}/bin/prometheus" +
|
||||
optionalString (length cmdlineArgs2 != 0) (" \\\n " +
|
||||
concatStringsSep " \\\n " cmdlineArgs2);
|
||||
User = promUser;
|
||||
Restart = "always";
|
||||
WorkingDirectory = workingDir2;
|
||||
StateDirectory = cfg2.stateDir;
|
||||
};
|
||||
};
|
||||
})
|
||||
];
|
||||
}
|
||||
|
|
|
|||
|
|
@ -194,6 +194,7 @@ in
|
|||
predictable-interface-names = handleTest ./predictable-interface-names.nix {};
|
||||
printing = handleTest ./printing.nix {};
|
||||
prometheus = handleTest ./prometheus.nix {};
|
||||
prometheus2 = handleTest ./prometheus-2.nix {};
|
||||
prometheus-exporters = handleTest ./prometheus-exporters.nix {};
|
||||
prosody = handleTest ./prosody.nix {};
|
||||
proxy = handleTest ./proxy.nix {};
|
||||
|
|
|
|||
34
nixos/tests/prometheus-2.nix
Normal file
34
nixos/tests/prometheus-2.nix
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
import ./make-test.nix {
|
||||
name = "prometheus-2";
|
||||
|
||||
nodes = {
|
||||
one = { pkgs, ... }: {
|
||||
services.prometheus2 = {
|
||||
enable = true;
|
||||
scrapeConfigs = [{
|
||||
job_name = "prometheus";
|
||||
static_configs = [{
|
||||
targets = [ "127.0.0.1:9090" ];
|
||||
labels = { instance = "localhost"; };
|
||||
}];
|
||||
}];
|
||||
rules = [
|
||||
''
|
||||
groups:
|
||||
- name: test
|
||||
rules:
|
||||
- record: testrule
|
||||
expr: count(up{job="prometheus"})
|
||||
''
|
||||
];
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
testScript = ''
|
||||
startAll;
|
||||
$one->waitForUnit("prometheus2.service");
|
||||
$one->waitForOpenPort(9090);
|
||||
$one->succeed("curl -s http://127.0.0.1:9090/metrics");
|
||||
'';
|
||||
}
|
||||
Loading…
Reference in a new issue