From cce415c7434f0a295916eb699a2ac98f0c4da0fe Mon Sep 17 00:00:00 2001 From: rnhmjoj Date: Tue, 27 Oct 2020 13:55:12 +0100 Subject: [PATCH 1/7] nixos/searx: declarative configuration --- nixos/modules/services/networking/searx.nix | 156 +++++++++++++++----- 1 file changed, 122 insertions(+), 34 deletions(-) diff --git a/nixos/modules/services/networking/searx.nix b/nixos/modules/services/networking/searx.nix index 60fb3d5d6d44..5b0836e67518 100644 --- a/nixos/modules/services/networking/searx.nix +++ b/nixos/modules/services/networking/searx.nix @@ -3,32 +3,130 @@ with lib; let - + dataDir = "/var/lib/searx"; cfg = config.services.searx; - configFile = cfg.configFile; + hasEngines = + builtins.hasAttr "engines" cfg.settings && + cfg.settings.engines != { }; + + # Script to merge NixOS settings with + # the default settings.yml bundled in searx. + mergeConfig = '' + cd ${dataDir} + # find the default settings.yml + default=$(find '${cfg.package}/' -name settings.yml) + + # write NixOS settings as JSON + cat <<'EOF' > settings.json + ${builtins.toJSON cfg.settings} + EOF + + ${optionalString hasEngines '' + # extract and convert the default engines array to an object + ${pkgs.yq-go}/bin/yq r "$default" engines -j | \ + ${pkgs.jq}/bin/jq 'reduce .[] as $e ({}; .[$e.name] = $e)' \ + > engines.json + + # merge and update the NixOS engines with the newly created object + cp settings.json temp.json + ${pkgs.jq}/bin/jq -s '. as [$s, $e] | $s | .engines |= + ($e * . | to_entries | map (.value))' \ + temp.json engines.json > settings.json + + # clean up temporary files + rm {engines,temp}.json + ''} + + # merge the default and NixOS settings + ${pkgs.yq-go}/bin/yq m -P settings.json "$default" > settings.yml + rm settings.json + + # substitute environment variables + env -0 | while IFS='=' read -r -d ''' n v; do + sed "s#@$n@#$v#g" -i settings.yml + done + ''; in { + imports = [ + (mkRenamedOptionModule + [ "services" "searx" "configFile" ] + [ "services" "searx" "settingsFile" ]) + ]; + ###### interface options = { services.searx = { - enable = mkEnableOption - "the searx server. See https://github.com/asciimoo/searx"; + enable = mkOption { + type = types.bool; + default = false; + relatedPackages = [ "searx" ]; + description = "Whether to enable Searx, the meta search engine."; + }; - configFile = mkOption { + environmentFile = mkOption { type = types.nullOr types.path; default = null; - description = " - The path of the Searx server configuration file. If no file - is specified, a default file is used (default config file has - debug mode enabled). - "; + description = '' + Environment file (see systemd.exec(5) + "EnvironmentFile=" section for the syntax) to define variables for + Searx. This option can be used to safely include secret keys into the + Searx configuration. + ''; + }; + + settings = mkOption { + type = types.attrs; + default = { }; + example = literalExample '' + { server.port = 8080; + server.bind_address = "0.0.0.0"; + server.secret_key = "@SEARX_SECRET_KEY@"; + + engines.wolframalpha = + { shortcut = "wa"; + api_key = "@WOLFRAM_API_KEY@"; + engine = "wolframalpha_api"; + }; + } + ''; + description = '' + Searx settings. These will be merged with (taking precedence over) + the default configuration. It's also possible to refer to + environment variables + (defined in ) + using the syntax @VARIABLE_NAME@. + + + For available settings, see the Searx + docs. + + + ''; + }; + + settingsFile = mkOption { + type = types.path; + default = "${dataDir}/settings.yml"; + description = '' + The path of the Searx server settings.yml file. If no file is + specified, a default file is used (default config file has debug mode + enabled). Note: setting this options overrides + . + + + This file, along with any secret key it contains, will be copied + into the world-readable Nix store. + + + ''; }; package = mkOption { @@ -46,30 +144,20 @@ in ###### implementation config = mkIf config.services.searx.enable { - - users.users.searx = - { uid = config.ids.uids.searx; - description = "Searx user"; - createHome = true; - home = "/var/lib/searx"; - }; - - users.groups.searx = - { gid = config.ids.gids.searx; - }; - - systemd.services.searx = - { - description = "Searx server, the meta search engine."; - after = [ "network.target" ]; - wantedBy = [ "multi-user.target" ]; - serviceConfig = { - User = "searx"; - ExecStart = "${cfg.package}/bin/searx-run"; - }; - } // (optionalAttrs (configFile != null) { - environment.SEARX_SETTINGS_PATH = configFile; - }); + systemd.services.searx = { + description = "Searx server, the meta search engine."; + after = [ "network.target" ]; + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + User = "searx"; + DynamicUser = true; + ExecStart = "${cfg.package}/bin/searx-run"; + StateDirectory = "searx"; + } // optionalAttrs (cfg.environmentFile != null) + { EnvironmentFile = builtins.toPath cfg.environmentFile; }; + environment.SEARX_SETTINGS_PATH = cfg.settingsFile; + preStart = mergeConfig; + }; environment.systemPackages = [ cfg.package ]; From 7ec85073ddfbedd1d0272bddd077a02798b1a4d4 Mon Sep 17 00:00:00 2001 From: rnhmjoj Date: Tue, 27 Oct 2020 14:10:33 +0100 Subject: [PATCH 2/7] nixos/ids: remove reserved searx ids --- nixos/modules/misc/ids.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nixos/modules/misc/ids.nix b/nixos/modules/misc/ids.nix index cf0198d7b93d..feb9c68301d5 100644 --- a/nixos/modules/misc/ids.nix +++ b/nixos/modules/misc/ids.nix @@ -143,7 +143,7 @@ in nix-ssh = 104; dictd = 105; couchdb = 106; - searx = 107; + #searx = 107; # dynamically allocated as of 2020-10-27 kippo = 108; jenkins = 109; systemd-journal-gateway = 110; @@ -457,7 +457,7 @@ in #nix-ssh = 104; # unused dictd = 105; couchdb = 106; - searx = 107; + #searx = 107; # dynamically allocated as of 2020-10-27 kippo = 108; jenkins = 109; systemd-journal-gateway = 110; From b7ca2d144893b44fa5baacfdcf019682070d72a3 Mon Sep 17 00:00:00 2001 From: rnhmjoj Date: Mon, 2 Nov 2020 01:34:53 +0100 Subject: [PATCH 3/7] nixos/tests: add searx test --- nixos/tests/all-tests.nix | 1 + nixos/tests/searx.nix | 62 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 nixos/tests/searx.nix diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index d53c6f6511e3..7d83b952f948 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -342,6 +342,7 @@ in sbt-extras = handleTest ./sbt-extras.nix {}; scala = handleTest ./scala.nix {}; sddm = handleTest ./sddm.nix {}; + searx = handleTest ./searx.nix {}; service-runner = handleTest ./service-runner.nix {}; shadow = handleTest ./shadow.nix {}; shadowsocks = handleTest ./shadowsocks {}; diff --git a/nixos/tests/searx.nix b/nixos/tests/searx.nix new file mode 100644 index 000000000000..128c2a9381b2 --- /dev/null +++ b/nixos/tests/searx.nix @@ -0,0 +1,62 @@ +import ./make-test-python.nix ({ pkgs, ...} : + +{ + name = "searx"; + meta = with pkgs.stdenv.lib.maintainers; { + maintainers = [ rnhmjoj ]; + }; + + machine = { ... }: { + imports = [ ../modules/profiles/minimal.nix ]; + + services.searx = { + enable = true; + environmentFile = pkgs.writeText "secrets" '' + WOLFRAM_API_KEY = sometoken + SEARX_SECRET_KEY = somesecret + ''; + + settings.server = + { port = 8080; + bind_address = "0.0.0.0"; + secret_key = "@SEARX_SECRET_KEY@"; + }; + + settings.engines = { + wolframalpha = + { api_key = "@WOLFRAM_API_KEY@"; + engine = "wolframalpha_api"; + }; + startpage.shortcut = "start"; + }; + + }; + }; + + testScript = + '' + start_all() + + with subtest("Settings have been merged"): + machine.wait_for_unit("searx") + output = machine.succeed( + "${pkgs.yq-go}/bin/yq r /var/lib/searx/settings.yml" + " 'engines.(name==startpage).shortcut'" + ).strip() + assert output == "start", "Settings not merged" + + with subtest("Environment variables have been substituted"): + machine.succeed("grep -q somesecret /var/lib/searx/settings.yml") + machine.succeed("grep -q sometoken /var/lib/searx/settings.yml") + + with subtest("Searx service is running"): + machine.wait_for_open_port(8080) + machine.succeed( + "${pkgs.curl}/bin/curl --fail http://localhost:8080" + ) + + machine.copy_from_vm("/var/lib/searx/settings.yml") + machine.shutdown() + ''; +}) + From 92d55d57ebb3a186ea23b66f3587a36bee662785 Mon Sep 17 00:00:00 2001 From: rnhmjoj Date: Wed, 4 Nov 2020 19:17:34 +0100 Subject: [PATCH 4/7] searx: make package friendly towards nginx/uWSGI - Link the /static directory to make it easier to serve - Export searx module for use in uWSGI (just import the module) --- pkgs/servers/web-apps/searx/default.nix | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pkgs/servers/web-apps/searx/default.nix b/pkgs/servers/web-apps/searx/default.nix index b56e430d9951..e83ea892529f 100644 --- a/pkgs/servers/web-apps/searx/default.nix +++ b/pkgs/servers/web-apps/searx/default.nix @@ -1,8 +1,8 @@ -{ lib, python3Packages, fetchFromGitHub, fetchpatch }: +{ lib, python3, python3Packages, fetchFromGitHub, fetchpatch }: with python3Packages; -buildPythonApplication rec { +toPythonModule (buildPythonApplication rec { pname = "searx"; version = "0.17.0"; @@ -34,10 +34,16 @@ buildPythonApplication rec { rm tests/test_robot.py # A variable that is imported is commented out ''; + postInstall = '' + # Create a symlink for easier access to static data + mkdir -p $out/share + ln -s ../${python3.sitePackages}/searx/static $out/share/ + ''; + meta = with lib; { homepage = "https://github.com/asciimoo/searx"; description = "A privacy-respecting, hackable metasearch engine"; license = licenses.agpl3Plus; maintainers = with maintainers; [ matejc fpletz globin danielfullmer ]; }; -} +}) From 623664e84f3d3db8c32d65df4cab1a174edff71e Mon Sep 17 00:00:00 2001 From: rnhmjoj Date: Fri, 6 Nov 2020 10:12:20 +0100 Subject: [PATCH 5/7] nixos/searx: add support for running in uWSGI --- nixos/modules/services/networking/searx.nix | 102 +++++++++++++++++--- nixos/tests/searx.nix | 77 ++++++++++++--- 2 files changed, 152 insertions(+), 27 deletions(-) diff --git a/nixos/modules/services/networking/searx.nix b/nixos/modules/services/networking/searx.nix index 5b0836e67518..85696beeba4a 100644 --- a/nixos/modules/services/networking/searx.nix +++ b/nixos/modules/services/networking/searx.nix @@ -3,7 +3,7 @@ with lib; let - dataDir = "/var/lib/searx"; + runDir = "/run/searx"; cfg = config.services.searx; hasEngines = @@ -13,7 +13,7 @@ let # Script to merge NixOS settings with # the default settings.yml bundled in searx. mergeConfig = '' - cd ${dataDir} + cd ${runDir} # find the default settings.yml default=$(find '${cfg.package}/' -name settings.yml) @@ -46,6 +46,9 @@ let env -0 | while IFS='=' read -r -d ''' n v; do sed "s#@$n@#$v#g" -i settings.yml done + + # set strict permissions + chmod 400 settings.yml ''; in @@ -114,7 +117,7 @@ in settingsFile = mkOption { type = types.path; - default = "${dataDir}/settings.yml"; + default = "${runDir}/settings.yml"; description = '' The path of the Searx server settings.yml file. If no file is specified, a default file is used (default config file has debug mode @@ -136,6 +139,38 @@ in description = "searx package to use."; }; + runInUwsgi = mkOption { + type = types.bool; + default = false; + description = '' + Whether to run searx in uWSGI as a "vassal", instead of using its + built-in HTTP server. This is the recommended mode for public or + large instances, but is unecessary for LAN or local-only use. + + + The built-in HTTP server logs all queries by default. + + + ''; + }; + + uwsgiConfig = mkOption { + type = types.attrs; + default = { http = ":8080"; }; + example = lib.literalExample '' + { + disable-logging = true; + http = ":8080"; # serve via HTTP... + socket = "/run/searx/searx.sock"; # ...or UNIX socket + } + ''; + description = '' + Additional configuration of the uWSGI vassal running searx. It + should notably specify on which interfaces and ports the vassal + should listen. + ''; + }; + }; }; @@ -143,23 +178,66 @@ in ###### implementation - config = mkIf config.services.searx.enable { - systemd.services.searx = { - description = "Searx server, the meta search engine."; - after = [ "network.target" ]; - wantedBy = [ "multi-user.target" ]; + config = mkIf cfg.enable { + environment.systemPackages = [ cfg.package ]; + + users.users.searx = + { description = "Searx daemon user"; + group = "searx"; + isSystemUser = true; + }; + + users.groups.searx = { }; + + systemd.services.searx-init = { + description = "Initialise Searx settings"; serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; User = "searx"; - DynamicUser = true; + RuntimeDirectory = "searx"; + RuntimeDirectoryMode = "750"; + } // optionalAttrs (cfg.environmentFile != null) + { EnvironmentFile = builtins.toPath cfg.environmentFile; }; + script = mergeConfig; + }; + + systemd.services.searx = mkIf (!cfg.runInUwsgi) { + description = "Searx server, the meta search engine."; + wantedBy = [ "network.target" "multi-user.target" ]; + requires = [ "searx-init.service" ]; + after = [ "searx-init.service" ]; + serviceConfig = { + User = "searx"; + Group = "searx"; ExecStart = "${cfg.package}/bin/searx-run"; - StateDirectory = "searx"; } // optionalAttrs (cfg.environmentFile != null) { EnvironmentFile = builtins.toPath cfg.environmentFile; }; environment.SEARX_SETTINGS_PATH = cfg.settingsFile; - preStart = mergeConfig; }; - environment.systemPackages = [ cfg.package ]; + systemd.services.uwsgi = mkIf (cfg.runInUwsgi) + { requires = [ "searx-init.service" ]; + after = [ "searx-init.service" ]; + }; + + services.uwsgi = mkIf (cfg.runInUwsgi) { + enable = true; + plugins = [ "python3" ]; + + instance.type = "emperor"; + instance.vassals.searx = { + type = "normal"; + strict = true; + immediate-uid = "searx"; + immediate-gid = "searx"; + lazy-apps = true; + enable-threads = true; + module = "searx.webapp"; + env = [ "SEARX_SETTINGS_PATH=${cfg.settingsFile}" ]; + pythonPackages = self: [ cfg.package ]; + } // cfg.uwsgiConfig; + }; }; diff --git a/nixos/tests/searx.nix b/nixos/tests/searx.nix index 128c2a9381b2..e5fee3466bfa 100644 --- a/nixos/tests/searx.nix +++ b/nixos/tests/searx.nix @@ -6,7 +6,8 @@ import ./make-test-python.nix ({ pkgs, ...} : maintainers = [ rnhmjoj ]; }; - machine = { ... }: { + # basic setup: searx running the built-in webserver + nodes.base = { ... }: { imports = [ ../modules/profiles/minimal.nix ]; services.searx = { @@ -17,11 +18,10 @@ import ./make-test-python.nix ({ pkgs, ...} : ''; settings.server = - { port = 8080; + { port = "8080"; bind_address = "0.0.0.0"; secret_key = "@SEARX_SECRET_KEY@"; }; - settings.engines = { wolframalpha = { api_key = "@WOLFRAM_API_KEY@"; @@ -29,34 +29,81 @@ import ./make-test-python.nix ({ pkgs, ...} : }; startpage.shortcut = "start"; }; - }; + + }; + + # fancy setup: run in uWSGI and use nginx as proxy + nodes.fancy = { ... }: { + imports = [ ../modules/profiles/minimal.nix ]; + + services.searx = { + enable = true; + runInUwsgi = true; + uwsgiConfig = { + # serve using the uwsgi protocol + socket = "/run/searx/uwsgi.sock"; + chmod-socket = "660"; + + # use /searx as url "mountpoint" + mount = "/searx=searx.webapp:application"; + module = ""; + manage-script-name = true; + }; + }; + + # use nginx as reverse proxy + services.nginx.enable = true; + services.nginx.virtualHosts.localhost = { + locations."/searx".extraConfig = + '' + include ${pkgs.nginx}/conf/uwsgi_params; + uwsgi_pass unix:/run/searx/uwsgi.sock; + ''; + locations."/searx/static/".alias = "${pkgs.searx}/share/static/"; + }; + + # allow nginx access to the searx socket + users.users.nginx.extraGroups = [ "searx" ]; + }; testScript = '' - start_all() + base.start() with subtest("Settings have been merged"): - machine.wait_for_unit("searx") - output = machine.succeed( - "${pkgs.yq-go}/bin/yq r /var/lib/searx/settings.yml" + base.wait_for_unit("searx-init") + base.wait_for_file("/run/searx/settings.yml") + output = base.succeed( + "${pkgs.yq-go}/bin/yq r /run/searx/settings.yml" " 'engines.(name==startpage).shortcut'" ).strip() assert output == "start", "Settings not merged" with subtest("Environment variables have been substituted"): - machine.succeed("grep -q somesecret /var/lib/searx/settings.yml") - machine.succeed("grep -q sometoken /var/lib/searx/settings.yml") + base.succeed("grep -q somesecret /run/searx/settings.yml") + base.succeed("grep -q sometoken /run/searx/settings.yml") + base.copy_from_vm("/run/searx/settings.yml") - with subtest("Searx service is running"): - machine.wait_for_open_port(8080) - machine.succeed( + with subtest("Basic setup is working"): + base.wait_for_open_port(8080) + base.wait_for_unit("searx") + base.succeed( "${pkgs.curl}/bin/curl --fail http://localhost:8080" ) + base.shutdown() - machine.copy_from_vm("/var/lib/searx/settings.yml") - machine.shutdown() + with subtest("Nginx+uWSGI setup is working"): + fancy.start() + fancy.wait_for_open_port(80) + fancy.wait_for_unit("uwsgi") + fancy.succeed( + "${pkgs.curl}/bin/curl --fail http://localhost/searx >&2" + ) + fancy.succeed( + "${pkgs.curl}/bin/curl --fail http://localhost/searx/static/js/bootstrap.min.js >&2" + ) ''; }) From 88f71722ea5d7c02946cc7773621bbe4c8844d70 Mon Sep 17 00:00:00 2001 From: rnhmjoj Date: Sat, 7 Nov 2020 02:10:39 +0100 Subject: [PATCH 6/7] nixos/doc: add searx changes to the relase notes --- nixos/doc/manual/release-notes/rl-2103.xml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/nixos/doc/manual/release-notes/rl-2103.xml b/nixos/doc/manual/release-notes/rl-2103.xml index 38bf69afa8b6..a3543aae1ea3 100644 --- a/nixos/doc/manual/release-notes/rl-2103.xml +++ b/nixos/doc/manual/release-notes/rl-2103.xml @@ -402,6 +402,18 @@ http://some.json-exporter.host:7979/probe?target=https://example.com/some/json/e SDK licenses if your project requires it. See the androidenv documentation for more details. + + + The Searx module has been updated with the ability to configure the + service declaratively and uWSGI integration. + The option services.searx.configFile has been renamed + to for consistency with + the new . In addition, the + searx uid and gid reservations have been removed + since they were not necessary: the service is now running with a + dynamically allocated uid. + + From 2a458003d93bb18e621138b3a76aad29c04e78e4 Mon Sep 17 00:00:00 2001 From: rnhmjoj Date: Mon, 11 Jan 2021 08:39:19 +0100 Subject: [PATCH 7/7] searx: add NixOS test to passthu.tests --- pkgs/servers/web-apps/searx/default.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkgs/servers/web-apps/searx/default.nix b/pkgs/servers/web-apps/searx/default.nix index e83ea892529f..41654a2f0bd9 100644 --- a/pkgs/servers/web-apps/searx/default.nix +++ b/pkgs/servers/web-apps/searx/default.nix @@ -1,4 +1,4 @@ -{ lib, python3, python3Packages, fetchFromGitHub, fetchpatch }: +{ lib, nixosTests, python3, python3Packages, fetchFromGitHub, fetchpatch }: with python3Packages; @@ -40,6 +40,8 @@ toPythonModule (buildPythonApplication rec { ln -s ../${python3.sitePackages}/searx/static $out/share/ ''; + passthru.tests = { inherit (nixosTests) searx; }; + meta = with lib; { homepage = "https://github.com/asciimoo/searx"; description = "A privacy-respecting, hackable metasearch engine";