Merge master into staging-next

doc/languages-frameworks/javascript.section.md

@@ -233,6 +233,37 @@ sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
 
 It returns a derivation with all `package-lock.json` dependencies downloaded into `$out/`, usable as an npm cache.
 
+#### importNpmLock {#javascript-buildNpmPackage-importNpmLock}
+
+`importNpmLock` is a Nix function that requires the following optional arguments:
+
+- `npmRoot`: Path to package directory containing the source tree
+- `package`: Parsed contents of `package.json`
+- `packageLock`: Parsed contents of `package-lock.json`
+- `pname`: Package name
+- `version`: Package version
+
+It returns a derivation with a patched `package.json` & `package-lock.json` with all dependencies resolved to Nix store paths.
+
+This function is analogous to using `fetchNpmDeps`, but instead of specifying `hash` it uses metadata from `package.json` & `package-lock.json`.
+
+Note that `npmHooks.npmConfigHook` cannot be used with `importNpmLock`. You will instead need to use `importNpmLock.npmConfigHook`:
+
+```nix
+{ buildNpmPackage, importNpmLock }:
+
+buildNpmPackage {
+  pname = "hello";
+  version = "0.1.0";
+
+  npmDeps = importNpmLock {
+    npmRoot = ./.;
+  };
+
+  npmConfigHook = importNpmLock.npmConfigHook;
+}
+```
+
 ### corepack {#javascript-corepack}
 
 This package puts the corepack wrappers for pnpm and yarn in your PATH, and they will honor the `packageManager` setting in the `package.json`.

pkgs/build-support/node/build-npm-package/default.nix

@@ -49,6 +49,12 @@
   name = "${name}-npm-deps";
   hash = npmDepsHash;
 }
+  # Custom npmConfigHook
+, npmConfigHook ? null
+  # Custom npmBuildHook
+, npmBuildHook ? null
+  # Custom npmInstallHook
+, npmInstallHook ? null
 , ...
 } @ args:
 
@@ -57,14 +63,19 @@ let
   npmHooks = buildPackages.npmHooks.override {
     inherit nodejs;
   };
-
-  inherit (npmHooks) npmConfigHook npmBuildHook npmInstallHook;
 in
 stdenv.mkDerivation (args // {
   inherit npmDeps npmBuildScript;
 
   nativeBuildInputs = nativeBuildInputs
-    ++ [ nodejs npmConfigHook npmBuildHook npmInstallHook nodejs.python ]
+    ++ [
+      nodejs
+      # Prefer passed hooks
+      (if npmConfigHook != null then npmConfigHook else npmHooks.npmConfigHook)
+      (if npmBuildHook != null then npmBuildHook else npmHooks.npmBuildHook)
+      (if npmInstallHook != null then npmInstallHook else npmHooks.npmInstallHook)
+      nodejs.python
+    ]
     ++ lib.optionals stdenv.isDarwin [ darwin.cctools ];
   buildInputs = buildInputs ++ [ nodejs ];
 

pkgs/build-support/node/import-npm-lock/default.nix

@@ -0,0 +1,134 @@
+{ lib
+, fetchurl
+, stdenv
+, callPackages
+, runCommand
+}:
+
+let
+  inherit (builtins) match elemAt toJSON removeAttrs;
+  inherit (lib) importJSON mapAttrs;
+
+  matchGitHubReference = match "github(.com)?:.+";
+  getName = package: package.name or "unknown";
+  getVersion = package: package.version or "0.0.0";
+
+  # Fetch a module from package-lock.json -> packages
+  fetchModule =
+    { module
+    , npmRoot ? null
+    }: (
+      if module ? "resolved" then
+        (
+          let
+            # Parse scheme from URL
+            mUrl = match "(.+)://(.+)" module.resolved;
+            scheme = elemAt mUrl 0;
+          in
+          (
+            if mUrl == null then
+              (
+                assert npmRoot != null; {
+                  outPath = npmRoot + "/${module.resolved}";
+                }
+              )
+            else if (scheme == "http" || scheme == "https") then
+              (
+                fetchurl {
+                  url = module.resolved;
+                  hash = module.integrity;
+                }
+              )
+            else if lib.hasPrefix "git" module.resolved then
+              (
+                builtins.fetchGit {
+                  url = module.resolved;
+                }
+              )
+            else throw "Unsupported URL scheme: ${scheme}"
+          )
+        )
+      else null
+    );
+
+  # Manage node_modules outside of the store with hooks
+  hooks = callPackages ./hooks { };
+
+in
+{
+  importNpmLock =
+    { npmRoot ? null
+    , package ? importJSON (npmRoot + "/package.json")
+    , packageLock ? importJSON (npmRoot + "/package-lock.json")
+    , pname ? getName package
+    , version ? getVersion package
+    }:
+    let
+      mapLockDependencies =
+        mapAttrs
+          (name: version: (
+            # Substitute the constraint with the version of the dependency from the top-level of package-lock.
+            if (
+              # if the version is `latest`
+              version == "latest"
+              ||
+              # Or if it's a github reference
+              matchGitHubReference version != null
+            ) then packageLock'.packages.${"node_modules/${name}"}.version
+            # But not a regular version constraint
+            else version
+          ));
+
+      packageLock' = packageLock // {
+        packages =
+          mapAttrs
+            (_: module:
+              let
+                src = fetchModule {
+                  inherit module npmRoot;
+                };
+              in
+              (removeAttrs module [
+                "link"
+                "funding"
+              ]) // lib.optionalAttrs (src != null) {
+                resolved = "file:${src}";
+              } // lib.optionalAttrs (module ? dependencies) {
+                dependencies = mapLockDependencies module.dependencies;
+              } // lib.optionalAttrs (module ? optionalDependencies) {
+                optionalDependencies = mapLockDependencies module.optionalDependencies;
+              })
+            packageLock.packages;
+      };
+
+      mapPackageDependencies = mapAttrs (name: _: packageLock'.packages.${"node_modules/${name}"}.resolved);
+
+      # Substitute dependency references in package.json with Nix store paths
+      packageJSON' = package // lib.optionalAttrs (package ? dependencies) {
+        dependencies = mapPackageDependencies package.dependencies;
+      } // lib.optionalAttrs (package ? devDependencies) {
+        devDependencies = mapPackageDependencies package.devDependencies;
+      };
+
+      pname = package.name or "unknown";
+
+    in
+    runCommand "${pname}-${version}-sources"
+      {
+        inherit pname version;
+
+        passAsFile = [ "package" "packageLock" ];
+
+        package = toJSON packageJSON';
+        packageLock = toJSON packageLock';
+      } ''
+      mkdir $out
+      cp "$packagePath" $out/package.json
+      cp "$packageLockPath" $out/package-lock.json
+    '';
+
+  inherit hooks;
+  inherit (hooks) npmConfigHook;
+
+  __functor = self: self.importNpmLock;
+}

pkgs/build-support/node/import-npm-lock/hooks/canonicalize-symlinks.js

@@ -0,0 +1,52 @@
+#!/usr/bin/env node
+const fs = require("fs");
+const path = require("path");
+
+// When installing files rewritten to the Nix store with npm
+// npm writes the symlinks relative to the build directory.
+//
+// This makes relocating node_modules tricky when refering to the store.
+// This script walks node_modules and canonicalizes symlinks.
+
+async function canonicalize(storePrefix, root) {
+  console.log(storePrefix, root)
+  const entries = await fs.promises.readdir(root);
+  const paths = entries.map((entry) => path.join(root, entry));
+
+  const stats = await Promise.all(
+    paths.map(async (path) => {
+      return {
+        path: path,
+        stat: await fs.promises.lstat(path),
+      };
+    })
+  );
+
+  const symlinks = stats.filter((stat) => stat.stat.isSymbolicLink());
+  const dirs = stats.filter((stat) => stat.stat.isDirectory());
+
+  // Canonicalize symlinks to their real path
+  await Promise.all(
+    symlinks.map(async (stat) => {
+      const target = await fs.promises.realpath(stat.path);
+      if (target.startsWith(storePrefix)) {
+        await fs.promises.unlink(stat.path);
+        await fs.promises.symlink(target, stat.path);
+      }
+    })
+  );
+
+  // Recurse into directories
+  await Promise.all(dirs.map((dir) => canonicalize(storePrefix, dir.path)));
+}
+
+async function main() {
+  const args = process.argv.slice(2);
+  const storePrefix = args[0];
+
+  if (fs.existsSync("node_modules")) {
+    await canonicalize(storePrefix, "node_modules");
+  }
+}
+
+main();

pkgs/build-support/node/import-npm-lock/hooks/default.nix

@@ -0,0 +1,13 @@
+{ callPackage, lib, makeSetupHook, srcOnly, nodejs }:
+{
+  npmConfigHook = makeSetupHook
+    {
+      name = "npm-config-hook";
+      substitutions = {
+        nodeSrc = srcOnly nodejs;
+        nodeGyp = "${nodejs}/lib/node_modules/npm/node_modules/node-gyp/bin/node-gyp.js";
+        canonicalizeSymlinksScript = ./canonicalize-symlinks.js;
+        storePrefix = builtins.storeDir;
+      };
+    } ./npm-config-hook.sh;
+}

pkgs/build-support/node/import-npm-lock/hooks/npm-config-hook.sh

@@ -0,0 +1,70 @@
+# shellcheck shell=bash
+
+npmConfigHook() {
+    echo "Executing npmConfigHook"
+
+    if [ -n "${npmRoot-}" ]; then
+      pushd "$npmRoot"
+    fi
+
+    if [ -z "${npmDeps-}" ]; then
+        echo "Error: 'npmDeps' should be set when using npmConfigHook."
+        exit 1
+    fi
+
+    echo "Configuring npm"
+
+    export HOME="$TMPDIR"
+    export npm_config_nodedir="@nodeSrc@"
+    export npm_config_node_gyp="@nodeGyp@"
+    npm config set offline true
+    npm config set progress false
+    npm config set fund false
+
+    echo "Installing patched package.json/package-lock.json"
+
+    # Save original package.json/package-lock.json for closure size reductions.
+    # The patched one contains store paths we don't want at runtime.
+    mv package.json .package.json.orig
+    if test -f package-lock.json; then # Not all packages have package-lock.json.
+        mv package-lock.json .package-lock.json.orig
+    fi
+    cp --no-preserve=mode "${npmDeps}/package.json" package.json
+    cp --no-preserve=mode "${npmDeps}/package-lock.json" package-lock.json
+
+    echo "Installing dependencies"
+
+    if ! npm install --ignore-scripts $npmInstallFlags "${npmInstallFlagsArray[@]}" $npmFlags "${npmFlagsArray[@]}"; then
+        echo
+        echo "ERROR: npm failed to install dependencies"
+        echo
+        echo "Here are a few things you can try, depending on the error:"
+        echo '1. Set `npmFlags = [ "--legacy-peer-deps" ]`'
+        echo
+
+        exit 1
+    fi
+
+    patchShebangs node_modules
+
+    npm rebuild $npmRebuildFlags "${npmRebuildFlagsArray[@]}" $npmFlags "${npmFlagsArray[@]}"
+
+    patchShebangs node_modules
+
+    # Canonicalize symlinks from relative paths to the Nix store.
+    node @canonicalizeSymlinksScript@ @storePrefix@
+
+    # Revert to pre-patched package.json/package-lock.json for closure size reductions
+    mv .package.json.orig package.json
+    if test -f ".package-lock.json.orig"; then
+        mv .package-lock.json.orig package-lock.json
+    fi
+
+    if [ -n "${npmRoot-}" ]; then
+      popd
+    fi
+
+    echo "Finished npmConfigHook"
+}
+
+postConfigureHooks+=(npmConfigHook)

pkgs/by-name/ic/icewm/package.nix

@@ -41,13 +41,13 @@
 
 stdenv.mkDerivation (finalAttrs: {
   pname = "icewm";
-  version = "3.4.5";
+  version = "3.4.6";
 
   src = fetchFromGitHub {
     owner = "ice-wm";
     repo = "icewm";
     rev = finalAttrs.version;
-    hash = "sha256-Auuu+hRYVziAF3hXH7XSOyNlDehEKg6QmSJicY+XQLk=";
+    hash = "sha256-j4o4/Q+WWuVPZM/rij2miC7ApWrBNhzve2TAPEXIU20=";
   };
 
   nativeBuildInputs = [

pkgs/development/python-modules/acquire/default.nix

@@ -17,7 +17,7 @@
 
 buildPythonPackage rec {
   pname = "acquire";
-  version = "3.12";
+  version = "3.13";
   pyproject = true;
 
   disabled = pythonOlder "3.9";
@@ -26,7 +26,7 @@ buildPythonPackage rec {
     owner = "fox-it";
     repo = "acquire";
     rev = "refs/tags/${version}";
-    hash = "sha256-anRNCnLDKHAEfOWi6m1n4R9lvFTlZgw5xxh39exvzH0=";
+    hash = "sha256-Z85bHM3MtS2MLX9BaKi8VqA13QjO9KdrgqhuyBzjILQ=";
   };
 
   nativeBuildInputs = [

pkgs/development/python-modules/ago/default.nix

@@ -0,0 +1,40 @@
+{ lib
+, buildPythonPackage
+, fetchPypi
+
+, pythonOlder
+
+, pytestCheckHook
+
+, setuptools
+}:
+
+buildPythonPackage rec {
+  pname = "ago";
+  version = "0.0.95";
+  pyproject = true;
+
+  disabled = pythonOlder "3.3";
+
+  src = fetchPypi {
+    inherit pname version;
+    hash = "sha256-0gEPXqw99UTsSOwRYQLgaFkaNFsaWA8ylz24pQX8p0Q=";
+  };
+
+  build-system = [
+    setuptools
+  ];
+
+  nativeCheckInputs = [
+    pytestCheckHook
+  ];
+
+  pythonImportsCheck = [ "ago" ];
+
+  meta = with lib; {
+    description = "Human Readable Time Deltas";
+    homepage = "https://git.unturf.com/python/ago";
+    license = licenses.publicDomain;
+    maintainers = with maintainers; [ vizid ];
+  };
+}

pkgs/development/python-modules/autopep8/default.nix

@@ -1,27 +1,33 @@
 { lib
 , buildPythonPackage
 , fetchFromGitHub
+, fetchpatch
 , glibcLocales
 , pycodestyle
 , pytestCheckHook
 , pythonOlder
+, setuptools
 , tomli
 }:
 
 buildPythonPackage rec {
   pname = "autopep8";
-  version = "2.0.4";
-  format = "setuptools";
+  version = "2.0.4-unstable-2023-10-27";
+  pyproject = true;
 
   disabled = pythonOlder "3.7";
 
   src = fetchFromGitHub {
     owner = "hhatto";
     repo = "autopep8";
-    rev = "refs/tags/v${version}";
-    hash = "sha256-TuaDnZqn9mRUsoDJkj9JK4ztvzl9JTwAk8nghIkZBvw=";
+    rev = "af7399d90926f2fe99a71f15197a08fa197f73a1";
+    hash = "sha256-psGl9rXxTQGHyXf1VskJ/I/goVH5hRRP5bUXQdaT/8M=";
   };
 
+  build-system = [
+    setuptools
+  ];
+
   propagatedBuildInputs = [
     pycodestyle
   ] ++ lib.optionals (pythonOlder "3.11") [

pkgs/development/python-modules/cohere/default.nix

@@ -14,14 +14,14 @@
 
 buildPythonPackage rec {
   pname = "cohere";
-  version = "4.51";
+  version = "4.52";
   pyproject = true;
 
   disabled = pythonOlder "3.7";
 
   src = fetchPypi {
     inherit pname version;
-    hash = "sha256-AfsJLqkDjdT7Ng77NQb60kUe0jHLZ3TjJLmTyTdKVQo=";
+    hash = "sha256-C2w447+hxsFrjLpV3wrSzptuVb1JTyppq+NUzMCU+Iw=";
   };
 
   nativeBuildInputs = [

pkgs/development/python-modules/dsl2html/default.nix

@@ -0,0 +1,30 @@
+{ lib
+, buildPythonPackage
+, fetchPypi
+
+, setuptools
+}:
+
+buildPythonPackage rec {
+  pname = "dsl2html";
+  version = "0.1.5";
+  pyproject = true;
+
+  src = fetchPypi {
+    inherit pname version;
+    hash = "sha256-LUtTMqQuwahIeMm9QvMe6rdHEw5LEjMQdyZFZvf/HRU=";
+  };
+
+  build-system = [
+    setuptools
+  ];
+
+  pythonImportsCheck = [ "dsl" ];
+
+  meta = with lib; {
+    description = "Python module for converting DSL dictionary texts into HTML";
+    homepage = "https://github.com/Crissium/python-dsl";
+    license = licenses.gpl3Only;
+    maintainers = with maintainers; [ vizid ];
+  };
+}

pkgs/development/python-modules/ezyrb/default.nix

@@ -14,7 +14,7 @@
 
 buildPythonPackage rec {
   pname = "ezyrb";
-  version = "1.3.0.post2402";
+  version = "1.3.0.post2403";
   format = "setuptools";
 
   disabled = pythonOlder "3.7";
@@ -23,7 +23,7 @@ buildPythonPackage rec {
     owner = "mathLab";
     repo = "EZyRB";
     rev = "refs/tags/v${version}";
-    hash = "sha256-MiFNTz3vrN4rMHK7e4ntE35wzgnPt6yczCv7XDcUlO8=";
+    hash = "sha256-t0Mv8Kae6N+jHeQx57ljDR5lmmbW2mqrlqygtrwGWhY=";
   };
 
   propagatedBuildInputs = [

pkgs/development/python-modules/hurry-filesize/default.nix

@@ -0,0 +1,39 @@
+{ lib
+, buildPythonPackage
+, fetchPypi
+
+, pythonOlder
+
+, setuptools
+}:
+
+buildPythonPackage rec {
+  pname = "hurry-filesize";
+  version = "0.9";
+  pyproject = true;
+
+  disabled = pythonOlder "3.3";
+
+  src = fetchPypi {
+    pname = "hurry.filesize";
+    inherit version;
+    hash = "sha256-9TaDKa2++GrM07yUkFIjQLt5JgRVromxpCwQ9jgBuaY=";
+  };
+
+  # project has no repo...
+  # fix implicit namespaces (PEP 420) warning
+  patches = [ ./use-pep-420-implicit-namespace-package.patch ];
+
+  build-system = [
+    setuptools
+  ];
+
+  pythonImportsCheck = [ "hurry.filesize" ];
+
+  meta = with lib; {
+    description = "A simple Python library for human readable file sizes (or anything sized in bytes)";
+    homepage = "https://pypi.org/project/hurry.filesize/";
+    license = licenses.zpl21;
+    maintainers = with maintainers; [ vizid ];
+  };
+}

pkgs/development/python-modules/hurry-filesize/use-pep-420-implicit-namespace-package.patch

@@ -0,0 +1,24 @@
+diff --git a/setup.py b/setup.py
+index 9ec6f2e..607b680 100644
+--- a/setup.py
++++ b/setup.py
+@@ -29,7 +29,6 @@ setup(
+     license='ZPL 2.1',
+     packages=find_packages('src'),
+     package_dir= {'':'src'},
+-    namespace_packages=['hurry'],
+     include_package_data=True,
+     zip_safe=False,
+     install_requires=[
+diff --git a/src/hurry/__init__.py b/src/hurry/__init__.py
+index 2e2033b..e69de29 100644
+--- a/src/hurry/__init__.py
++++ b/src/hurry/__init__.py
+@@ -1,7 +0,0 @@
+-# this is a namespace package
+-try:
+-    import pkg_resources
+-    pkg_resources.declare_namespace(__name__)
+-except ImportError:
+-    import pkgutil
+-    __path__ = pkgutil.extend_path(__path__, __name__)

pkgs/development/python-modules/langchain-core/default.nix

@@ -16,7 +16,7 @@
 
 buildPythonPackage rec {
   pname = "langchain-core";
-  version = "0.1.29";
+  version = "0.1.30";
   pyproject = true;
 
   disabled = pythonOlder "3.8";
@@ -24,7 +24,7 @@ buildPythonPackage rec {
   src = fetchPypi {
     pname = "langchain_core";
     inherit version;
-    hash = "sha256-ZzHav/rQO5ITraJkDVTtf072uZ/Oh63jxxR0rhVN08w=";
+    hash = "sha256-4ToBblXn8IL/Pu7aLQy1BbiaiDDjojwdE00KideHGJQ=";
   };
 
   pythonRelaxDeps = [

pkgs/development/python-modules/python-matter-server/default.nix

@@ -55,7 +55,7 @@ in
 
 buildPythonPackage rec {
   pname = "python-matter-server";
-  version = "5.8.0";
+  version = "5.8.1";
   format = "pyproject";
 
   disabled = pythonOlder "3.10";
@@ -64,7 +64,7 @@ buildPythonPackage rec {
     owner = "home-assistant-libs";
     repo = "python-matter-server";
     rev = "refs/tags/${version}";
-    hash = "sha256-bpXRay4JUujqdnscGldW732e8FTkcmfShbtwp2YJC60=";
+    hash = "sha256-iisDEopaKklLvvGDTo2fv0Fpkhuhd+7KoNnQA+QmjB8=";
   };
 
   patches = [

pkgs/development/python-modules/streamlit/default.nix

@@ -32,14 +32,14 @@
 
 buildPythonPackage rec {
   pname = "streamlit";
-  version = "1.31.1";
+  version = "1.32.0";
   pyproject = true;
 
   disabled = pythonOlder "3.8";
 
   src = fetchPypi {
     inherit pname version;
-    hash = "sha256-38Q8qFtLTDHQl8J7mDuMzJYCIq2QeGKysvtN3wTFD9w=";
+    hash = "sha256-Zb8i4ZDelzuRDAsSezPYDTHFJ17Ykcb950DeRuXgUyM=";
   };
 
   nativeBuildInputs = [

pkgs/development/python-modules/xdxf2html/default.nix

@@ -0,0 +1,30 @@
+{ lib
+, buildPythonPackage
+, fetchPypi
+
+, setuptools
+}:
+
+buildPythonPackage rec {
+  pname = "xdxf2html";
+  version = "0.1.0";
+  pyproject = true;
+
+  src = fetchPypi {
+    inherit pname version;
+    hash = "sha256-u2UaEALzD583+hbgwTItQOdGQ6GIhdVy79C2gfJwzlI=";
+  };
+
+  build-system = [
+    setuptools
+  ];
+
+  pythonImportsCheck = [ "xdxf2html" ];
+
+  meta = with lib; {
+    description = "Python module for converting XDXF dictionary texts into HTML";
+    homepage = "https://github.com/Crissium/python-xdxf2html";
+    license = licenses.gpl3Only;
+    maintainers = with maintainers; [ vizid ];
+  };
+}

pkgs/top-level/all-packages.nix

@@ -10262,6 +10262,8 @@ with pkgs;
   inherit (callPackages ../build-support/node/fetch-npm-deps { })
     fetchNpmDeps prefetch-npm-deps;
 
+  importNpmLock = callPackages ../build-support/node/import-npm-lock { };
+
   nodePackages_latest = dontRecurseIntoAttrs nodejs_latest.pkgs // { __attrsFailEvaluation = true; };
 
   nodePackages = dontRecurseIntoAttrs nodejs.pkgs // { __attrsFailEvaluation = true; };

pkgs/top-level/python-packages.nix

@@ -123,6 +123,8 @@ self: super: with self; {
 
   agent-py = callPackage ../development/python-modules/agent-py { };
 
+  ago = callPackage ../development/python-modules/ago { };
+
   aggdraw = callPackage ../development/python-modules/aggdraw { };
 
   aigpy = callPackage ../development/python-modules/aigpy { };
@@ -3566,6 +3568,8 @@ self: super: with self; {
 
   dynd = callPackage ../development/python-modules/dynd { };
 
+  dsl2html = callPackage ../development/python-modules/dsl2html { };
+
   e3-core = callPackage ../development/python-modules/e3-core { };
 
   e3-testsuite = callPackage ../development/python-modules/e3-testsuite { };
@@ -5418,6 +5422,8 @@ self: super: with self; {
 
   hupper = callPackage ../development/python-modules/hupper { };
 
+  hurry-filesize = callPackage ../development/python-modules/hurry-filesize { };
+
   huum = callPackage ../development/python-modules/huum { };
 
   hvac = callPackage ../development/python-modules/hvac { };
@@ -16687,6 +16693,8 @@ self: super: with self; {
 
   xxhash = callPackage ../development/python-modules/xxhash { };
 
+  xdxf2html = callPackage ../development/python-modules/xdxf2html { };
+
   xyzservices = callPackage ../development/python-modules/xyzservices { };
 
   y-py = callPackage ../development/python-modules/y-py { };