nixos/grafana: refactor dashboards for RFC42

This commit refactors `services.grafana.provision.dashboards` towards
the RFC42 style. To preserve backwards compatibility, we have to jump
through a ton of hoops, introducing esoteric type signatures and bizarre
structs. The Grafana module definition should hopefully become a lot
cleaner after a release cycle or two once the old configuration style is
completely deprecated.

nixos/doc/manual/from_md/release-notes/rl-2211.section.xml

@@ -833,6 +833,15 @@
           has been hardened.
         </para>
       </listitem>
+      <listitem>
+        <para>
+          The <literal>services.grafana.provision.dashboards</literal>
+          option was converted to a
+          <link xlink:href="https://github.com/NixOS/rfcs/blob/master/rfcs/0042-config-option.md">RFC
+          0042</link> configuration. It also now supports specifying the
+          provisioning YAML file with <literal>path</literal> option.
+        </para>
+      </listitem>
       <listitem>
         <para>
           Matrix Synapse now requires entries in the

nixos/doc/manual/release-notes/rl-2211.section.md

@@ -272,6 +272,8 @@ Available as [services.patroni](options.html#opt-services.patroni.enable).
 
 - The `services.matrix-synapse` systemd unit has been hardened.
 
+- The `services.grafana.provision.dashboards` option was converted to a [RFC 0042](https://github.com/NixOS/rfcs/blob/master/rfcs/0042-config-option.md) configuration. It also now supports specifying the provisioning YAML file with `path` option.
+
 - Matrix Synapse now requires entries in the `state_group_edges` table to be unique, in order to prevent accidentally introducing duplicate information (for example, because a database backup was restored multiple times). If your Synapse database already has duplicate rows in this table, this could fail with an error and require manual remediation.
 
 - The `diamond` package has been update from 0.8.36 to 2.0.15. See the [upstream release notes](https://github.com/bbuchfink/diamond/releases) for more details.

nixos/modules/services/monitoring/grafana.nix

@@ -5,6 +5,7 @@ with lib;
 let
   cfg = config.services.grafana;
   opt = options.services.grafana;
+  provisioningSettingsFormat = pkgs.formats.yaml {};
   declarativePlugins = pkgs.linkFarm "grafana-plugins" (builtins.map (pkg: { name = pkg.pname; path = pkg; }) cfg.declarativePlugins);
   useMysql = cfg.database.type == "mysql";
   usePostgresql = cfg.database.type == "postgres";
@@ -84,7 +85,8 @@ let
     providers = cfg.provision.dashboards;
   };
 
-  dashboardFile = pkgs.writeText "dashboard.yaml" (builtins.toJSON dashboardConfiguration);
+  dashboardFileNew = if (cfg.provision.dashboards.path == null) then provisioningSettingsFormat.generate "dashboard.yaml" cfg.provision.dashboards.settings else cfg.provision.dashboards.path;
+  dashboardFile = if (builtins.isList cfg.provision.dashboards) then provisioningSettingsFormat.generate "dashboard.yaml" dashboardConfiguration else dashboardFileNew;
 
   notifierConfiguration = {
     apiVersion = 1;
@@ -198,47 +200,22 @@ let
 
   # http://docs.grafana.org/administration/provisioning/#dashboards
   grafanaTypes.dashboardConfig = types.submodule {
+    freeformType = provisioningSettingsFormat.type;
+
     options = {
       name = mkOption {
         type = types.str;
         default = "default";
-        description = lib.mdDoc "Provider name.";
-      };
-      orgId = mkOption {
-        type = types.int;
-        default = 1;
-        description = lib.mdDoc "Organization ID.";
-      };
-      folder = mkOption {
-        type = types.str;
-        default = "";
-        description = lib.mdDoc "Add dashboards to the specified folder.";
+        description = lib.mdDoc "A unique provider name.";
       };
       type = mkOption {
         type = types.str;
         default = "file";
         description = lib.mdDoc "Dashboard provider type.";
       };
-      disableDeletion = mkOption {
-        type = types.bool;
-        default = false;
-        description = lib.mdDoc "Disable deletion when JSON file is removed.";
-      };
-      updateIntervalSeconds = mkOption {
-        type = types.int;
-        default = 10;
-        description = lib.mdDoc "How often Grafana will scan for changed dashboards.";
-      };
-      options = {
-        path = mkOption {
-          type = types.path;
-          description = lib.mdDoc "Path grafana will watch for dashboards.";
-        };
-        foldersFromFilesStructure = mkOption {
-          type = types.bool;
-          default = false;
-          description = lib.mdDoc "Use folder names from filesystem to create folders in Grafana.";
-        };
+      options.path = mkOption {
+        type = types.path;
+        description = lib.mdDoc "Path grafana will watch for dashboards. Required when using the 'file' type.";
       };
     };
   };
@@ -446,18 +423,69 @@ in {
 
     provision = {
       enable = mkEnableOption (lib.mdDoc "provision");
+
       datasources = mkOption {
         description = lib.mdDoc "Grafana datasources configuration.";
         default = [];
         type = types.listOf grafanaTypes.datasourceConfig;
         apply = x: map _filter x;
       };
+
+
       dashboards = mkOption {
-        description = lib.mdDoc "Grafana dashboard configuration.";
+        description = lib.mdDoc ''
+          Deprecated option for Grafana dashboard configuration. Use either
+          `services.grafana.provision.dashboards.settings` or
+          `services.grafana.provision.dashboards.path` instead.
+        '';
         default = [];
-        type = types.listOf grafanaTypes.dashboardConfig;
-        apply = x: map _filter x;
+        apply = x: if (builtins.isList x) then map _filter x else x;
+        type = with types; either (listOf grafanaTypes.dashboardConfig) (submodule {
+          options.settings = mkOption {
+            description = lib.mdDoc ''
+              Grafana dashboard configuration in Nix. Can't be used with
+              `services.grafana.provision.dashboards.path` simultaneously. See
+              <link xlink:href="https://grafana.com/docs/grafana/latest/administration/provisioning/#dashboards"/>
+              for supported options.
+            '';
+            default = null;
+            type = types.nullOr (types.submodule {
+              options.apiVersion = mkOption {
+                description = lib.mdDoc "Config file version.";
+                default = 1;
+                type = types.int;
+              };
+
+              options.providers = mkOption {
+                description = lib.mdDoc "List of dashboards to insert/update.";
+                default = [];
+                type = types.listOf grafanaTypes.dashboardConfig;
+              };
+            });
+            example = literalExpression ''
+              {
+                apiVersion = 1;
+
+                providers = [{
+                    name = "default";
+                    options.path = "/var/lib/grafana/dashboards";
+                }];
+              }
+            '';
+          };
+
+          options.path = mkOption {
+            description = lib.mdDoc ''
+              Path to YAML dashboard configuration. Can't be used with
+              `services.grafana.provision.dashboards.settings` simultaneously.
+            '';
+            default = null;
+            type = types.nullOr types.path;
+          };
+        });
       };
+
+
       notifiers = mkOption {
         description = lib.mdDoc "Grafana notifier configuration.";
         default = [];
@@ -699,6 +727,13 @@ in {
       (optional (
         any (x: x.secure_settings != null) cfg.provision.notifiers
       ) "Notifier secure settings will be stored as plaintext in the Nix store!")
+      (optional (
+        builtins.isList cfg.provision.dashboards
+      ) ''
+          Provisioning Grafana dashboards with options has been deprecated.
+          Use `services.grafana.provision.dashboards.settings` or
+          `services.grafana.provision.dashboards.path` instead.
+        '')
     ];
 
     environment.systemPackages = [ cfg.package ];
@@ -726,6 +761,10 @@ in {
           cfg.provision.datasources;
         message = "For datasources of type `prometheus`, the `direct` access mode is not supported anymore (since Grafana 9.2.0)";
       }
+      {
+        assertion = if (builtins.isList cfg.provision.dashboards) then true else cfg.provision.dashboards.settings == null || cfg.provision.dashboards.path == null;
+        message = "Cannot set both dashboards settings and dashboards path";
+      }
     ];
 
     systemd.services.grafana = {

nixos/tests/all-tests.nix

@@ -231,7 +231,7 @@ in {
   gollum = handleTest ./gollum.nix {};
   google-oslogin = handleTest ./google-oslogin {};
   gotify-server = handleTest ./gotify-server.nix {};
-  grafana = handleTest ./grafana.nix {};
+  grafana = handleTest ./grafana {};
   grafana-agent = handleTest ./grafana-agent.nix {};
   graphite = handleTest ./graphite.nix {};
   graylog = handleTest ./graylog.nix {};

nixos/tests/grafana/basic.nix

@@ -1,4 +1,6 @@
-import ./make-test-python.nix ({ lib, pkgs, ... }:
+args@{ pkgs, ... }:
+
+(import ../make-test-python.nix ({ lib, pkgs, ... }:
 
 let
   inherit (lib) mkMerge nameValuePair maintainers;
@@ -59,7 +61,7 @@ let
   ])) [ "sqlite" "declarativePlugins" "postgresql" "mysql" ]);
 
 in {
-  name = "grafana";
+  name = "grafana-basic";
 
   meta = with maintainers; {
     maintainers = [ willibutz ];
@@ -109,4 +111,4 @@ in {
         )
         mysql.shutdown()
   '';
-})
+})) args

nixos/tests/grafana/default.nix

@@ -0,0 +1,9 @@
+{ system ? builtins.currentSystem
+, config ? { }
+, pkgs ? import ../../.. { inherit system config; }
+}:
+
+{
+  basic = import ./basic.nix { inherit system pkgs; };
+  provision-dashboards = import ./provision-dashboards { inherit system pkgs; };
+}

nixos/tests/grafana/provision-dashboards/default.nix

@@ -0,0 +1,92 @@
+args@{ pkgs, ... }:
+
+(import ../../make-test-python.nix ({ lib, pkgs, ... }:
+
+let
+  inherit (lib) mkMerge nameValuePair maintainers;
+
+  baseGrafanaConf = {
+    services.grafana = {
+      enable = true;
+      addr = "localhost";
+      analytics.reporting.enable = false;
+      domain = "localhost";
+      security = {
+        adminUser = "testadmin";
+        adminPassword = "snakeoilpwd";
+      };
+      provision.enable = true;
+    };
+
+    systemd.tmpfiles.rules = [
+      "L /var/lib/grafana/dashboards/test.json 0700 grafana grafana - ${pkgs.writeText "test.json" (builtins.readFile ./test_dashboard.json)}"
+    ];
+  };
+
+  extraNodeConfs = {
+    provisionDashboardOld = {
+      services.grafana.provision = {
+        dashboards = [{ options.path = "/var/lib/grafana/dashboards"; }];
+      };
+    };
+
+    provisionDashboardNix = {
+      services.grafana.provision = {
+        dashboards.settings = {
+          apiVersion = 1;
+          providers = [{
+            name = "default";
+            options.path = "/var/lib/grafana/dashboards";
+          }];
+        };
+      };
+    };
+
+    provisionDashboardYaml = {
+      services.grafana.provision.dashboards.path = ./provision-dashboards.yaml;
+    };
+  };
+
+  nodes = builtins.listToAttrs (map (provisionType:
+    nameValuePair provisionType (mkMerge [
+    baseGrafanaConf
+    (extraNodeConfs.${provisionType} or {})
+  ])) [ "provisionDashboardOld" "provisionDashboardNix" "provisionDashboardYaml" ]);
+
+in {
+  name = "grafana-provision-dashboards";
+
+  meta = with maintainers; {
+    maintainers = [ kfears willibutz ];
+  };
+
+  inherit nodes;
+
+  testScript = ''
+    start_all()
+
+    with subtest("Successful dashboard provision with Nix (old format)"):
+        provisionDashboardOld.wait_for_unit("grafana.service")
+        provisionDashboardOld.wait_for_open_port(3000)
+        provisionDashboardOld.succeed(
+            "curl -sSfN -u testadmin:snakeoilpwd http://127.0.0.1:3000/api/dashboards/uid/test_dashboard | grep Test\ Dashboard"
+        )
+        provisionDashboardOld.shutdown()
+
+    with subtest("Successful dashboard provision with Nix (new format)"):
+        provisionDashboardNix.wait_for_unit("grafana.service")
+        provisionDashboardNix.wait_for_open_port(3000)
+        provisionDashboardNix.succeed(
+            "curl -sSfN -u testadmin:snakeoilpwd http://127.0.0.1:3000/api/dashboards/uid/test_dashboard | grep Test\ Dashboard"
+        )
+        provisionDashboardNix.shutdown()
+
+    with subtest("Successful dashboard provision with YAML"):
+        provisionDashboardYaml.wait_for_unit("grafana.service")
+        provisionDashboardYaml.wait_for_open_port(3000)
+        provisionDashboardYaml.succeed(
+            "curl -sSfN -u testadmin:snakeoilpwd http://127.0.0.1:3000/api/dashboards/uid/test_dashboard | grep Test\ Dashboard"
+        )
+        provisionDashboardYaml.shutdown()
+  '';
+})) args

nixos/tests/grafana/provision-dashboards/provision-dashboards.yaml

@@ -0,0 +1,6 @@
+apiVersion: 1
+
+providers:
+  - name: 'default'
+    options:
+      path: /var/lib/grafana/dashboards

nixos/tests/grafana/provision-dashboards/test_dashboard.json

@@ -0,0 +1,47 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": 28,
+  "links": [],
+  "liveNow": false,
+  "panels": [],
+  "schemaVersion": 37,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "Test Dashboard",
+  "uid": "test_dashboard",
+  "version": 1,
+  "weekStart": ""
+}