Merge pull request #168288 from erikarvstedt/paperless-ngx

2022-04-12 22:10:52 +02:00 · 2022-04-12 22:10:52 +02:00 · 2731137d12
commit 2731137d12
parent 65cc198539 0977227b86
9 changed files with 94 additions and 101 deletions
--- a/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml
+++ b/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml
@ -557,6 +557,17 @@
          work.
        </para>
      </listitem>
+      <listitem>
+        <para>
+          <literal>services.paperless-ng</literal> was renamed to
+          <literal>services.paperless</literal>. Accordingly, the
+          <literal>paperless-ng-manage</literal> script (located in
+          <literal>dataDir</literal>) was renamed to
+          <literal>paperless-manage</literal>.
+          <literal>services.paperless</literal> now uses
+          <literal>paperless-ngx</literal>.
+        </para>
+      </listitem>
      <listitem>
        <para>
          The <literal>matrix-synapse</literal> service
--- a/nixos/doc/manual/release-notes/rl-2205.section.md
+++ b/nixos/doc/manual/release-notes/rl-2205.section.md
@ -184,6 +184,8 @@ In addition to numerous new and upgraded packages, this release has the followin

 - `services.ipfs.extraFlags` is now escaped with `utils.escapeSystemdExecArgs`. If you rely on systemd interpolating `extraFlags` in the service `ExecStart`, this will no longer work.

+- `services.paperless-ng` was renamed to `services.paperless`. Accordingly, the `paperless-ng-manage` script (located in `dataDir`) was renamed to `paperless-manage`. `services.paperless` now uses `paperless-ngx`.
+
 - The `matrix-synapse` service (`services.matrix-synapse`) has been converted to use the `settings` option defined in RFC42.
  This means that options that are part of your `homeserver.yaml` configuration, and that were specified at the top-level of the
  module (`services.matrix-synapse`) now need to be moved into `services.matrix-synapse.settings`. And while not all options you
--- a/nixos/modules/module-list.nix
+++ b/nixos/modules/module-list.nix
@ -597,7 +597,7 @@
  ./services/misc/osrm.nix
  ./services/misc/owncast.nix
  ./services/misc/packagekit.nix
-  ./services/misc/paperless-ng.nix
+  ./services/misc/paperless.nix
  ./services/misc/parsoid.nix
  ./services/misc/plex.nix
  ./services/misc/plikd.nix
--- a/nixos/modules/services/misc/paperless-ng.nix
+++ b/nixos/modules/services/misc/paperless-ng.nix
@ -2,11 +2,13 @@

 with lib;
 let
-  cfg = config.services.paperless-ng;
+  cfg = config.services.paperless;

  defaultUser = "paperless";

-  hasCustomRedis = hasAttr "PAPERLESS_REDIS" cfg.extraConfig;
+  # Don't start a redis instance if the user sets a custom redis connection
+  enableRedis = !hasAttr "PAPERLESS_REDIS" cfg.extraConfig;
+  redisServer = config.services.redis.servers.paperless;

  env = {
    PAPERLESS_DATA_DIR = cfg.dataDir;
@ -15,15 +17,15 @@ let
    GUNICORN_CMD_ARGS = "--bind=${cfg.address}:${toString cfg.port}";
  } // (
    lib.mapAttrs (_: toString) cfg.extraConfig
-  ) // (optionalAttrs (!hasCustomRedis) {
-    PAPERLESS_REDIS = "unix://${config.services.redis.servers.paperless-ng.unixSocket}";
+  ) // (optionalAttrs enableRedis {
+    PAPERLESS_REDIS = "unix://${redisServer.unixSocket}";
  });

  manage = let
    setupEnv = lib.concatStringsSep "\n" (mapAttrsToList (name: val: "export ${name}=\"${val}\"") env);
  in pkgs.writeShellScript "manage" ''
    ${setupEnv}
-    exec ${cfg.package}/bin/paperless-ng "$@"
+    exec ${cfg.package}/bin/paperless-ngx "$@"
  '';

  # Secure the services
@ -36,7 +38,7 @@ let
      "-/etc/hosts"
      "-/etc/localtime"
      "-/run/postgresql"
-    ] ++ (optional (!hasCustomRedis) config.services.redis.servers.paperless-ng.unixSocket);
+    ] ++ (optional enableRedis redisServer.unixSocket);
    BindPaths = [
      cfg.consumptionDir
      cfg.dataDir
@ -53,7 +55,6 @@ let
    PrivateNetwork = true;
    PrivateTmp = true;
    PrivateUsers = true;
-    ProcSubset = "pid";
    ProtectClock = true;
    # Breaks if the home dir of the user is in /home
    # Also does not add much value in combination with the TemporaryFileSystem.
@ -66,11 +67,15 @@ let
    ProtectKernelModules = true;
    ProtectKernelTunables = true;
    ProtectProc = "invisible";
+    # Don't restrict ProcSubset because django-q requires read access to /proc/stat
+    # to query CPU and memory information.
+    # Note that /proc only contains processes of user `paperless`, so this is safe.
+    # ProcSubset = "pid";
    RestrictAddressFamilies = [ "AF_UNIX" "AF_INET" "AF_INET6" ];
    RestrictNamespaces = true;
    RestrictRealtime = true;
    RestrictSUIDSGID = true;
-    SupplementaryGroups = optional (!hasCustomRedis) config.services.redis.servers.paperless-ng.user;
+    SupplementaryGroups = optional enableRedis redisServer.user;
    SystemCallArchitectures = "native";
    SystemCallFilter = [ "@system-service" "~@privileged @resources @setuid @keyring" ];
    # Does not work well with the temporary root
@ -81,26 +86,22 @@ in
  meta.maintainers = with maintainers; [ earvstedt Flakebi ];

  imports = [
-    (mkRemovedOptionModule [ "services" "paperless"] ''
-      The paperless module has been removed as the upstream project died.
-      Users should migrate to the paperless-ng module (services.paperless-ng).
-      More information can be found in the NixOS 21.11 release notes.
-    '')
+    (mkRenamedOptionModule [ "services" "paperless-ng" ] [ "services" "paperless" ])
  ];

-  options.services.paperless-ng = {
+  options.services.paperless = {
    enable = mkOption {
      type = lib.types.bool;
      default = false;
      description = ''
-        Enable Paperless-ng.
+        Enable Paperless.

        When started, the Paperless database is automatically created if it doesn't
        exist and updated if the Paperless package has changed.
        Both tasks are achieved by running a Django migration.

        A script to manage the Paperless instance (by wrapping Django's manage.py) is linked to
-        <literal>''${dataDir}/paperless-ng-manage</literal>.
+        <literal>''${dataDir}/paperless-manage</literal>.
      '';
    };

@ -133,13 +134,13 @@ in
    passwordFile = mkOption {
      type = types.nullOr types.path;
      default = null;
-      example = "/run/keys/paperless-ng-password";
+      example = "/run/keys/paperless-password";
      description = ''
        A file containing the superuser password.

        A superuser is required to access the web interface.
        If unset, you can create a superuser manually by running
-        <literal>''${dataDir}/paperless-ng-manage createsuperuser</literal>.
+        <literal>''${dataDir}/paperless-manage createsuperuser</literal>.

        The default superuser name is <literal>admin</literal>. To change it, set
        option <option>extraConfig.PAPERLESS_ADMIN_USER</option>.
@ -168,9 +169,9 @@ in
      type = types.attrs;
      default = {};
      description = ''
-        Extra paperless-ng config options.
+        Extra paperless config options.

-        See <link xlink:href="https://paperless-ng.readthedocs.io/en/latest/configuration.html">the documentation</link>
+        See <link xlink:href="https://paperless-ngx.readthedocs.io/en/latest/configuration.html">the documentation</link>
        for available options.
      '';
      example = literalExpression ''
@ -188,15 +189,14 @@ in

    package = mkOption {
      type = types.package;
-      default = pkgs.paperless-ng;
-      defaultText = literalExpression "pkgs.paperless-ng";
+      default = pkgs.paperless-ngx;
+      defaultText = literalExpression "pkgs.paperless-ngx";
      description = "The Paperless package to use.";
    };
  };

  config = mkIf cfg.enable {
-    # Enable redis if no special url is set
-    services.redis.servers.paperless-ng.enable = mkIf (!hasCustomRedis) true;
+    services.redis.servers.paperless.enable = mkIf enableRedis true;

    systemd.tmpfiles.rules = [
      "d '${cfg.dataDir}' - ${cfg.user} ${config.users.users.${cfg.user}.group} - -"
@ -208,11 +208,11 @@ in
      )
    ];

-    systemd.services.paperless-ng-server = {
-      description = "Paperless document server";
+    systemd.services.paperless-scheduler = {
+      description = "Paperless scheduler";
      serviceConfig = defaultServiceConfig // {
        User = cfg.user;
-        ExecStart = "${cfg.package}/bin/paperless-ng qcluster";
+        ExecStart = "${cfg.package}/bin/paperless-ngx qcluster";
        Restart = "on-failure";
        # The `mbind` syscall is needed for running the classifier.
        SystemCallFilter = defaultServiceConfig.SystemCallFilter ++ [ "mbind" ];
@ -221,15 +221,15 @@ in
      };
      environment = env;
      wantedBy = [ "multi-user.target" ];
-      wants = [ "paperless-ng-consumer.service" "paperless-ng-web.service" ];
+      wants = [ "paperless-consumer.service" "paperless-web.service" ];

      preStart = ''
-        ln -sf ${manage} ${cfg.dataDir}/paperless-ng-manage
+        ln -sf ${manage} ${cfg.dataDir}/paperless-manage

        # Auto-migrate on first run or if the package has changed
        versionFile="${cfg.dataDir}/src-version"
        if [[ $(cat "$versionFile" 2>/dev/null) != ${cfg.package} ]]; then
-          ${cfg.package}/bin/paperless-ng migrate
+          ${cfg.package}/bin/paperless-ngx migrate
          echo ${cfg.package} > "$versionFile"
        fi
      ''
@ -240,20 +240,18 @@ in
        superuserStateFile="${cfg.dataDir}/superuser-state"

        if [[ $(cat "$superuserStateFile" 2>/dev/null) != $superuserState ]]; then
-          ${cfg.package}/bin/paperless-ng manage_superuser
+          ${cfg.package}/bin/paperless-ngx manage_superuser
          echo "$superuserState" > "$superuserStateFile"
        fi
      '';
-    } // optionalAttrs (!hasCustomRedis) {
-      after = [ "redis-paperless-ng.service" ];
+    } // optionalAttrs enableRedis {
+      after = [ "redis-paperless.service" ];
    };

-    # Password copying can't be implemented as a privileged preStart script
-    # in 'paperless-ng-server' because 'defaultServiceConfig' limits the filesystem
-    # paths accessible by the service.
-    systemd.services.paperless-ng-copy-password = mkIf (cfg.passwordFile != null) {
-      requiredBy = [ "paperless-ng-server.service" ];
-      before = [ "paperless-ng-server.service" ];
+    # Reading the user-provided password file requires root access
+    systemd.services.paperless-copy-password = mkIf (cfg.passwordFile != null) {
+      requiredBy = [ "paperless-scheduler.service" ];
+      before = [ "paperless-scheduler.service" ];
      serviceConfig = {
        ExecStart = ''
          ${pkgs.coreutils}/bin/install --mode 600 --owner '${cfg.user}' --compare \
@ -263,27 +261,27 @@ in
      };
    };

-    systemd.services.paperless-ng-consumer = {
+    systemd.services.paperless-consumer = {
      description = "Paperless document consumer";
      serviceConfig = defaultServiceConfig // {
        User = cfg.user;
-        ExecStart = "${cfg.package}/bin/paperless-ng document_consumer";
+        ExecStart = "${cfg.package}/bin/paperless-ngx document_consumer";
        Restart = "on-failure";
      };
      environment = env;
-      # Bind to `paperless-ng-server` so that the consumer never runs
+      # Bind to `paperless-scheduler` so that the consumer never runs
      # during migrations
-      bindsTo = [ "paperless-ng-server.service" ];
-      after = [ "paperless-ng-server.service" ];
+      bindsTo = [ "paperless-scheduler.service" ];
+      after = [ "paperless-scheduler.service" ];
    };

-    systemd.services.paperless-ng-web = {
+    systemd.services.paperless-web = {
      description = "Paperless web server";
      serviceConfig = defaultServiceConfig // {
        User = cfg.user;
        ExecStart = ''
          ${pkgs.python3Packages.gunicorn}/bin/gunicorn \
-            -c ${cfg.package}/lib/paperless-ng/gunicorn.conf.py paperless.asgi:application
+            -c ${cfg.package}/lib/paperless-ngx/gunicorn.conf.py paperless.asgi:application
        '';
        Restart = "on-failure";

@ -296,15 +294,15 @@ in
      };
      environment = env // {
        PATH = mkForce cfg.package.path;
-        PYTHONPATH = "${cfg.package.pythonPath}:${cfg.package}/lib/paperless-ng/src";
+        PYTHONPATH = "${cfg.package.pythonPath}:${cfg.package}/lib/paperless-ngx/src";
      };
      # Allow the web interface to access the private /tmp directory of the server.
      # This is required to support uploading files via the web interface.
-      unitConfig.JoinsNamespaceOf = "paperless-ng-server.service";
-      # Bind to `paperless-ng-server` so that the web server never runs
+      unitConfig.JoinsNamespaceOf = "paperless-scheduler.service";
+      # Bind to `paperless-scheduler` so that the web server never runs
      # during migrations
-      bindsTo = [ "paperless-ng-server.service" ];
-      after = [ "paperless-ng-server.service" ];
+      bindsTo = [ "paperless-scheduler.service" ];
+      after = [ "paperless-scheduler.service" ];
    };

    users = optionalAttrs (cfg.user == defaultUser) {
--- a/nixos/tests/all-tests.nix
+++ b/nixos/tests/all-tests.nix
@ -402,7 +402,7 @@ in
  pam-ussh = handleTest ./pam/pam-ussh.nix {};
  pantalaimon = handleTest ./matrix/pantalaimon.nix {};
  pantheon = handleTest ./pantheon.nix {};
-  paperless-ng = handleTest ./paperless-ng.nix {};
+  paperless = handleTest ./paperless.nix {};
  parsedmarc = handleTest ./parsedmarc {};
  pdns-recursor = handleTest ./pdns-recursor.nix {};
  peerflix = handleTest ./peerflix.nix {};
--- a/nixos/tests/paperless-ng.nix
+++ b/nixos/tests/paperless-ng.nix
@ -1,30 +1,32 @@
 import ./make-test-python.nix ({ lib, ... }: {
-  name = "paperless-ng";
+  name = "paperless";
  meta.maintainers = with lib.maintainers; [ earvstedt Flakebi ];

  nodes.machine = { pkgs, ... }: {
    environment.systemPackages = with pkgs; [ imagemagick jq ];
-    services.paperless-ng = {
+    services.paperless = {
      enable = true;
      passwordFile = builtins.toFile "password" "admin";
    };
  };

  testScript = ''
-    machine.wait_for_unit("paperless-ng-consumer.service")
+    import json

-    with subtest("Create test doc"):
+    machine.wait_for_unit("paperless-consumer.service")
+
+    with subtest("Add a document via the file system"):
        machine.succeed(
            "convert -size 400x40 xc:white -font 'DejaVu-Sans' -pointsize 20 -fill black "
            "-annotate +5+20 'hello world 16-10-2005' /var/lib/paperless/consume/doc.png"
        )

    with subtest("Web interface gets ready"):
-        machine.wait_for_unit("paperless-ng-web.service")
+        machine.wait_for_unit("paperless-web.service")
        # Wait until server accepts connections
        machine.wait_until_succeeds("curl -fs localhost:28981")

-    with subtest("Create web test doc"):
+    with subtest("Add a document via the web interface"):
        machine.succeed(
            "convert -size 400x40 xc:white -font 'DejaVu-Sans' -pointsize 20 -fill black "
            "-annotate +5+20 'hello web 16-10-2005' /tmp/webdoc.png"
@ -35,11 +37,8 @@ import ./make-test-python.nix ({ lib, ... }: {
        machine.wait_until_succeeds(
            "(($(curl -u admin:admin -fs localhost:28981/api/documents/ | jq .count) == 2))"
        )
-        assert "2005-10-16" in machine.succeed(
-            "curl -u admin:admin -fs localhost:28981/api/documents/ | jq '.results | .[0] | .created'"
-        )
-        assert "2005-10-16" in machine.succeed(
-            "curl -u admin:admin -fs localhost:28981/api/documents/ | jq '.results | .[1] | .created'"
-        )
+        docs = json.loads(machine.succeed("curl -u admin:admin -fs localhost:28981/api/documents/"))['results']
+        assert "2005-10-16" in docs[0]['created']
+        assert "2005-10-16" in docs[1]['created']
  '';
 })
--- a/pkgs/applications/office/paperless-ngx/default.nix
+++ b/pkgs/applications/office/paperless-ngx/default.nix
@ -18,16 +18,6 @@ let
  py = python3.override {
    packageOverrides = self: super: {
      django = super.django_3;
-      # Avoid warning in django-q versions > 1.3.4
-      # https://github.com/jonaswinkler/paperless-ng/issues/857
-      # https://github.com/Koed00/django-q/issues/526
-      django-q = super.django-q.overridePythonAttrs (oldAttrs: rec {
-        version = "1.3.4";
-        src = oldAttrs.src.override {
-          inherit version;
-          sha256 = "Uj1U3PG2YVLBtlj5FPAO07UYo0MqnezUiYc4yo274Q8=";
-        };
-      });

      # Incompatible with aioredis 2
      aioredis = super.aioredis.overridePythonAttrs (oldAttrs: rec {
@ -43,30 +33,16 @@ let
  path = lib.makeBinPath [ ghostscript imagemagick jbig2enc optipng pngquant qpdf tesseract4 unpaper ];
 in
 py.pkgs.pythonPackages.buildPythonApplication rec {
-  pname = "paperless-ng";
-  version = "1.5.0";
+  pname = "paperless-ngx";
+  version = "1.6.0";

  src = fetchurl {
-    url = "https://github.com/jonaswinkler/paperless-ng/releases/download/ng-${version}/${pname}-${version}.tar.xz";
-    sha256 = "oVSq0AWksuWC81MF5xiZ6ZbdKKtqqphmL+xIzJLaDMw=";
+    url = "https://github.com/paperless-ngx/paperless-ngx/releases/download/ngx-${version}/${pname}-${version}.tar.xz";
+    sha256 = "07mrxbwahkm00n9nvssd6d13p80w333g84cd38bzp0l34nzim5zl";
  };

-  patches = [
-    # Fix the `slow_write_pdf` test:
-    # https://github.com/NixOS/nixpkgs/issues/136626
-    (fetchpatch {
-      url = "https://github.com/paperless-ngx/paperless-ngx/commit/4fbabe43ea12811864e9676b04d82a82b38e799d.patch";
-      sha256 = "sha256-8ULep5aeW3wJAQGy2OEAjFYybELNq1DzCC1uBrZx36I=";
-    })
-  ];
-
  format = "other";

-  # Make bind address configurable
-  postPatch = ''
-    substituteInPlace gunicorn.conf.py --replace "bind = '0.0.0.0:8000'" ""
-  '';
-
  propagatedBuildInputs = with py.pkgs.pythonPackages; [
    aioredis
    arrow
@ -161,11 +137,17 @@ py.pkgs.pythonPackages.buildPythonApplication rec {
    zope_interface
  ];

+  # Compile manually because `pythonRecompileBytecodeHook` only works for
+  # files in `python.sitePackages`
+  postBuild = ''
+    ${py.interpreter} -OO -m compileall src
+  '';
+
  installPhase = ''
    mkdir -p $out/lib
-    cp -r . $out/lib/paperless-ng
-    chmod +x $out/lib/paperless-ng/src/manage.py
-    makeWrapper $out/lib/paperless-ng/src/manage.py $out/bin/paperless-ng \
+    cp -r . $out/lib/paperless-ngx
+    chmod +x $out/lib/paperless-ngx/src/manage.py
+    makeWrapper $out/lib/paperless-ngx/src/manage.py $out/bin/paperless-ngx \
      --prefix PYTHONPATH : "$PYTHONPATH" \
      --prefix PATH : "${path}"
  '';
@ -200,13 +182,13 @@ py.pkgs.pythonPackages.buildPythonApplication rec {
    pythonPath = python3.pkgs.makePythonPath propagatedBuildInputs;
    inherit path;

-    tests = { inherit (nixosTests) paperless-ng; };
+    tests = { inherit (nixosTests) paperless; };
  };

  meta = with lib; {
    description = "A supercharged version of paperless: scan, index, and archive all of your physical documents";
-    homepage = "https://paperless-ng.readthedocs.io/en/latest/";
+    homepage = "https://paperless-ngx.readthedocs.io/en/latest/";
    license = licenses.gpl3Only;
-    maintainers = with maintainers; [ earvstedt Flakebi ];
+    maintainers = with maintainers; [ lukegb ];
  };
 }
--- a/pkgs/top-level/aliases.nix
+++ b/pkgs/top-level/aliases.nix
@ -902,7 +902,8 @@ mapAliases ({

  p11_kit = throw "'p11_kit' has been renamed to/replaced by 'p11-kit'"; # Converted to throw 2022-02-22
  packet-cli = metal-cli; # Added 2021-10-25
-  paperless = paperless-ng; # Added 2021-06-06
+  paperless = paperless-ngx; # Added 2021-06-06
+  paperless-ng = paperless-ngx; # Added 2022-04-11
  parity = openethereum; # Added 2020-08-01
  parity-ui = throw "parity-ui was removed because it was broken and unmaintained by upstream"; # Added 2022-01-10
  parquet-cpp = throw "'parquet-cpp' has been renamed to/replaced by 'arrow-cpp'"; # Converted to throw 2022-02-22
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@ -8804,7 +8804,7 @@ with pkgs;

  pantheon-tweaks = callPackage ../desktops/pantheon/third-party/pantheon-tweaks { };

-  paperless-ng = callPackage ../applications/office/paperless-ng { };
+  paperless-ngx = callPackage ../applications/office/paperless-ngx { };

  paperwork = callPackage ../applications/office/paperwork/paperwork-gtk.nix { };