From 12b3cf4557d2b1a92925fc247da286f20b953510 Mon Sep 17 00:00:00 2001 From: "R. RyanTM" Date: Mon, 23 Aug 2021 01:44:56 +0000 Subject: [PATCH 01/28] pscale: 0.65.0 -> 0.68.0 --- pkgs/development/tools/pscale/default.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/development/tools/pscale/default.nix b/pkgs/development/tools/pscale/default.nix index 477827b75a14..2e7a68a698c5 100644 --- a/pkgs/development/tools/pscale/default.nix +++ b/pkgs/development/tools/pscale/default.nix @@ -2,16 +2,16 @@ buildGoModule rec { pname = "pscale"; - version = "0.65.0"; + version = "0.68.0"; src = fetchFromGitHub { owner = "planetscale"; repo = "cli"; rev = "v${version}"; - sha256 = "sha256-RIyxO2nTysJLdYQvlmhZpS8R2kkwN+XeTlk4Ocbk9C8="; + sha256 = "sha256-SAKbz33Fpi3sQcqwD2UK5wloJqNs2HohsiGMl1gkfA0="; }; - vendorSha256 = "sha256-8zgWM5e+aKggGbLoL/Fmy7AuALVlLa74eHBxNGjTSy4="; + vendorSha256 = "sha256-dEkCJe6qiyB/pNh78o2/TTRmWQDsUV2TsXiuchC1JLA="; meta = with lib; { homepage = "https://www.planetscale.com/"; From 2e8211af78a1d5226bbb6d02db482a69bffca036 Mon Sep 17 00:00:00 2001 From: ajs124 Date: Fri, 10 Sep 2021 16:40:04 +0200 Subject: [PATCH 02/28] nginxMainline: 1.21.1 -> 1.21.3 - adds openssl 3 support - disables support for export ciphers and more --- pkgs/servers/http/nginx/mainline.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/servers/http/nginx/mainline.nix b/pkgs/servers/http/nginx/mainline.nix index 80e56d1bf0f0..1b63b06f2cb4 100644 --- a/pkgs/servers/http/nginx/mainline.nix +++ b/pkgs/servers/http/nginx/mainline.nix @@ -1,6 +1,6 @@ { callPackage, ... }@args: callPackage ./generic.nix args { - version = "1.21.1"; - sha256 = "0q2m2pd9x287py54kp49ys5pwnn0j17x7jjl0cx1c5916h8h7fk8"; + version = "1.21.3"; + sha256 = "0nhps7igdqcpcy1r8677ar807rfclpylmz3y858a678m1np4lxql"; } From 69a5365ae7fb0937fee4413609e2e8be33873208 Mon Sep 17 00:00:00 2001 From: "R. RyanTM" Date: Sun, 12 Sep 2021 22:54:12 +0000 Subject: [PATCH 03/28] pangomm_2_48: 2.48.0 -> 2.48.1 --- pkgs/development/libraries/pangomm/2.48.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/development/libraries/pangomm/2.48.nix b/pkgs/development/libraries/pangomm/2.48.nix index 3c25d113d727..c2ab99395453 100644 --- a/pkgs/development/libraries/pangomm/2.48.nix +++ b/pkgs/development/libraries/pangomm/2.48.nix @@ -14,13 +14,13 @@ stdenv.mkDerivation rec { pname = "pangomm"; - version= "2.48.0"; + version= "2.48.1"; outputs = [ "out" "dev" ]; src = fetchurl { url = "mirror://gnome/sources/${pname}/${lib.versions.majorMinor version}/${pname}-${version}.tar.xz"; - sha256 = "sha256-ng7UdMM/jCACyp4rYcoNHz2OQJ4J6Z9NjBnur8z1W3g="; + sha256 = "sha256-d2rVPnkeQxBrf0D/CDS+5uTrHGrXy20hVUb3o98O3E0="; }; nativeBuildInputs = [ From bbf344539ffdfa66120340732531b111af59672f Mon Sep 17 00:00:00 2001 From: "R. RyanTM" Date: Sun, 12 Sep 2021 23:28:22 +0000 Subject: [PATCH 04/28] plexRaw: 1.24.1.4931-1a38e63c6 -> 1.24.2.4973-2b1b51db9 --- pkgs/servers/plex/raw.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/servers/plex/raw.nix b/pkgs/servers/plex/raw.nix index 62247d83ddee..3cfa3c4f028f 100644 --- a/pkgs/servers/plex/raw.nix +++ b/pkgs/servers/plex/raw.nix @@ -12,16 +12,16 @@ # server, and the FHS userenv and corresponding NixOS module should # automatically pick up the changes. stdenv.mkDerivation rec { - version = "1.24.1.4931-1a38e63c6"; + version = "1.24.2.4973-2b1b51db9"; pname = "plexmediaserver"; # Fetch the source src = if stdenv.hostPlatform.system == "aarch64-linux" then fetchurl { url = "https://downloads.plex.tv/plex-media-server-new/${version}/debian/plexmediaserver_${version}_arm64.deb"; - sha256 = "1vsg90rlhynfk8wlbf080fv9wah7w8244pl878hjbi6yrjmz2s7g"; + sha256 = "12vs0d38z5w0vvs00jdgdsj3a4fsn1b0iclfaxsakr2hn0j8zmm0"; } else fetchurl { url = "https://downloads.plex.tv/plex-media-server-new/${version}/debian/plexmediaserver_${version}_amd64.deb"; - sha256 = "08xai0jcpmj1hwkkkgc87v9xwszd5bvwhn36kp6v73jnv1l5cmqb"; + sha256 = "0m9kb5a2zgfglv07f6pw76kj8k39sswby3i48g4pkxzhs1v5qskw"; }; outputs = [ "out" "basedb" ]; From a280f014c57bbe951fb6e9722f8a7d476b355084 Mon Sep 17 00:00:00 2001 From: "R. RyanTM" Date: Sun, 12 Sep 2021 23:46:06 +0000 Subject: [PATCH 05/28] punes: unstable-2021-07-19 -> unstable-2021-09-11 --- pkgs/misc/emulators/punes/default.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/misc/emulators/punes/default.nix b/pkgs/misc/emulators/punes/default.nix index d3ad2f681997..b4bdb6d28922 100644 --- a/pkgs/misc/emulators/punes/default.nix +++ b/pkgs/misc/emulators/punes/default.nix @@ -19,13 +19,13 @@ mkDerivation rec { pname = "punes"; - version = "unstable-2021-07-19"; + version = "unstable-2021-09-11"; src = fetchFromGitHub { owner = "punesemu"; repo = "puNES"; - rev = "15ab85dabb220889419df0c249c06f3db2b09dc0"; - sha256 = "1w0c5lfdl9ha4sxxva6hcpcaa444px6x25471q37l69n71rmjpy8"; + rev = "60ca36fcb066c41d0b3f2b550ca94dc7d12d84d6"; + sha256 = "JOi6AE1bpAc/wj9fQqHrUNc6vceeUyP0phT2f9kcJTY="; }; postPatch = '' From 765dd20a6c29a9ce7d0691d25c0770986dcfc2df Mon Sep 17 00:00:00 2001 From: Emily Date: Mon, 13 Sep 2021 04:20:37 +0100 Subject: [PATCH 06/28] pv: 1.6.6 -> 1.6.20 --- pkgs/tools/misc/pv/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/tools/misc/pv/default.nix b/pkgs/tools/misc/pv/default.nix index 013ddb8e3dab..7b5403eefe11 100644 --- a/pkgs/tools/misc/pv/default.nix +++ b/pkgs/tools/misc/pv/default.nix @@ -1,11 +1,11 @@ { lib, stdenv, fetchurl } : stdenv.mkDerivation rec { - name = "pv-1.6.6"; + name = "pv-1.6.20"; src = fetchurl { url = "https://www.ivarch.com/programs/sources/${name}.tar.bz2"; - sha256 = "1wbk14xh9rfypiwyy68ssl8dliyji30ly70qki1y2xx3ywszk3k0"; + sha256 = "00y6zla8h653sn4axgqz7rr0x79vfwl62a7gn6lzn607zwg9acg8"; }; meta = { From a48dff941256205c5667d7c9f80a7aee7a221174 Mon Sep 17 00:00:00 2001 From: Emily Date: Mon, 13 Sep 2021 03:36:41 +0100 Subject: [PATCH 07/28] pv: fix on aarch64-darwin Homebrew carries a patch that has been sent upstream. Since it's designed to be suitable for inclusion as-is and strictly improves correctness, we just apply it unconditionally on all platforms. --- pkgs/tools/misc/pv/default.nix | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/pkgs/tools/misc/pv/default.nix b/pkgs/tools/misc/pv/default.nix index 7b5403eefe11..5e58b128d8e3 100644 --- a/pkgs/tools/misc/pv/default.nix +++ b/pkgs/tools/misc/pv/default.nix @@ -1,4 +1,8 @@ -{ lib, stdenv, fetchurl } : +{ lib +, stdenv +, fetchurl +, fetchpatch +}: stdenv.mkDerivation rec { name = "pv-1.6.20"; @@ -8,6 +12,14 @@ stdenv.mkDerivation rec { sha256 = "00y6zla8h653sn4axgqz7rr0x79vfwl62a7gn6lzn607zwg9acg8"; }; + patches = [ + # Fix build on aarch64-darwin using patch from Homebrew + (fetchpatch { + url = "https://raw.githubusercontent.com/Homebrew/homebrew-core/0780f1df9fdbd8914ff50ac24eb0ec0d3561c1b7/Formula/pv.rb"; + sha256 = "001xayskfprri4s2gd3bqwajw6nz6nv0ggb0835par7q7bsd0dzr"; + }) + ]; + meta = { homepage = "http://www.ivarch.com/programs/pv"; description = "Tool for monitoring the progress of data through a pipeline"; From 0ce7438bd6ffa161bccb10fe693b86a18a579a85 Mon Sep 17 00:00:00 2001 From: Emily Date: Mon, 13 Sep 2021 04:28:22 +0100 Subject: [PATCH 08/28] mosh: fix on aarch64-darwin The build failure here is not due to Apple Silicon per se but instead an artifact of the aarch64-darwin toolchain using a newer version of the Xcode SDK as a basis. This causes issues building on case-insensitive filesystems due to the collision between the standard header and a mosh source file called VERSION. Fix with a MacPorts backport of an upstream mosh commit, via Homebrew. --- pkgs/tools/networking/mosh/default.nix | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkgs/tools/networking/mosh/default.nix b/pkgs/tools/networking/mosh/default.nix index 92c241967ae0..00746a7afbd2 100644 --- a/pkgs/tools/networking/mosh/default.nix +++ b/pkgs/tools/networking/mosh/default.nix @@ -31,6 +31,14 @@ stdenv.mkDerivation rec { ./bash_completion_datadir.patch ]; postPatch = '' + # Fix build with Xcode 12.5 toolchain/case-insensitive filesystems + # Backport of https://github.com/mobile-shell/mosh/commit/12199114fe4234f791ef4c306163901643b40538; + # remove on next upstream release. + patch -p0 < ${fetchpatch { + url = "https://raw.githubusercontent.com/macports/macports-ports/70ca3f65e622c17582fd938602d800157ed951c3/net/mosh/files/patch-version-subdir.diff"; + sha256 = "1yyh6d07y9zbdx4fb0r56zkq9nd9knwzj22v4dfi55k4k42qxapd"; + }} + substituteInPlace scripts/mosh.pl \ --subst-var-by ssh "${openssh}/bin/ssh" substituteInPlace scripts/mosh.pl \ From 3a7e5f6c25155e09f7fe4b9ccb13dde339007377 Mon Sep 17 00:00:00 2001 From: Arthur Gautier Date: Mon, 13 Sep 2021 01:03:07 +0000 Subject: [PATCH 09/28] ipxe: allow to pass options with values For example BANNER_TIMEOUT can be overriden like: ``` ipxe.overrideArgs(old: { enableOptions = old.enableOptions ++ [ "BANNER_TIMEOUT 100" ]; }); ``` Signed-off-by: Arthur Gautier --- pkgs/tools/misc/ipxe/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/tools/misc/ipxe/default.nix b/pkgs/tools/misc/ipxe/default.nix index 4be0b3d417e4..70b1ae48ae9c 100644 --- a/pkgs/tools/misc/ipxe/default.nix +++ b/pkgs/tools/misc/ipxe/default.nix @@ -61,7 +61,7 @@ stdenv.mkDerivation rec { configurePhase = '' runHook preConfigure - for opt in $enabledOptions; do echo "#define $opt" >> src/config/general.h; done + for opt in ${lib.escapeShellArgs enabledOptions}; do echo "#define $opt" >> src/config/general.h; done sed -i '/cp \''${ISOLINUX_BIN}/s/$/ --no-preserve=mode/' src/util/geniso substituteInPlace src/Makefile.housekeeping --replace '/bin/echo' echo runHook postConfigure From 3fbb55710d31b27c8fa5897e4e7714ea54ce06e5 Mon Sep 17 00:00:00 2001 From: Pol Dellaiera Date: Mon, 13 Sep 2021 08:51:59 +0200 Subject: [PATCH 10/28] php74: 7.4.21 -> 7.4.23 --- pkgs/development/interpreters/php/7.4.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/development/interpreters/php/7.4.nix b/pkgs/development/interpreters/php/7.4.nix index 1e72f7b48b4a..d098aeb57e1b 100644 --- a/pkgs/development/interpreters/php/7.4.nix +++ b/pkgs/development/interpreters/php/7.4.nix @@ -2,8 +2,8 @@ let base = callPackage ./generic.nix (_args // { - version = "7.4.21"; - sha256 = "0al2697d5hwq0f39rgncl1pwfxzzpc0afmr0fjvw5qjpww163v1n"; + version = "7.4.23"; + sha256 = "d1e094fe6e4f832e0a64be9c69464ba5d593fb216f914efa8bbb084e0a7a5727"; }); in From cb5b2f7de7d0d5eae309e8ea1589b0a9b3a344c1 Mon Sep 17 00:00:00 2001 From: Jonathan Ringer Date: Sat, 11 Sep 2021 18:22:30 -0700 Subject: [PATCH 11/28] python3Packages.azure-mgmt-iothubprovisioningservices: 0.2.0 -> 1.0.0 --- .../azure-mgmt-iothubprovisioningservices/default.nix | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkgs/development/python-modules/azure-mgmt-iothubprovisioningservices/default.nix b/pkgs/development/python-modules/azure-mgmt-iothubprovisioningservices/default.nix index 3401ed898c8f..aab6f96edc24 100644 --- a/pkgs/development/python-modules/azure-mgmt-iothubprovisioningservices/default.nix +++ b/pkgs/development/python-modules/azure-mgmt-iothubprovisioningservices/default.nix @@ -2,27 +2,28 @@ , buildPythonPackage , fetchPypi , python -, isPy3k , msrest , msrestazure , azure-common +, azure-mgmt-core , azure-mgmt-nspkg }: buildPythonPackage rec { pname = "azure-mgmt-iothubprovisioningservices"; - version = "0.2.0"; + version = "1.0.0"; src = fetchPypi { inherit pname version; extension = "zip"; - sha256 = "8c37acfd1c33aba845f2e0302ef7266cad31cba503cc990a48684659acb7b91d"; + sha256 = "sha256-5YcbA0iLWubfxEHNvaQMs5wABjXuV8UTBTeSs8FYJqk="; }; propagatedBuildInputs = [ msrest msrestazure azure-common + azure-mgmt-core azure-mgmt-nspkg ]; From f883f707fa0dc0d8ad9708d33d03332d733bd2f0 Mon Sep 17 00:00:00 2001 From: Jonathan Ringer Date: Sat, 11 Sep 2021 18:38:31 -0700 Subject: [PATCH 12/28] python3Packages.azure-synapse-managedprivateendpoints: init at 0.4.0 --- .../default.nix | 33 +++++++++++++++++++ pkgs/top-level/python-packages.nix | 2 ++ 2 files changed, 35 insertions(+) create mode 100644 pkgs/development/python-modules/azure-synapse-managedprivateendpoints/default.nix diff --git a/pkgs/development/python-modules/azure-synapse-managedprivateendpoints/default.nix b/pkgs/development/python-modules/azure-synapse-managedprivateendpoints/default.nix new file mode 100644 index 000000000000..4a8b98be01ba --- /dev/null +++ b/pkgs/development/python-modules/azure-synapse-managedprivateendpoints/default.nix @@ -0,0 +1,33 @@ +{ lib, buildPythonPackage, fetchPypi +, azure-common +, azure-core +, msrest +}: + +buildPythonPackage rec { + pname = "azure-synapse-managedprivateendpoints"; + version = "0.4.0"; + + src = fetchPypi { + inherit pname version; + extension = "zip"; + sha256 = "900eaeaccffdcd01012b248a7d049008c92807b749edd1c9074ca9248554c17e"; + }; + + propagatedBuildInputs = [ + azure-common + azure-core + msrest + ]; + + pythonNamespaces = [ "azure.synapse" ]; + + pythonImportsCheck = [ "azure.synapse.managedprivateendpoints" ]; + + meta = with lib; { + description = "Microsoft Azure Synapse Managed Private Endpoints Client Library for Python"; + homepage = "https://github.com/Azure/azure-sdk-for-python"; + license = licenses.mit; + maintainers = with maintainers; [ jonringer ]; + }; +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index d0dd5eb8a1de..8d00d957f7ce 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -961,6 +961,8 @@ in { azure-synapse-artifacts = callPackage ../development/python-modules/azure-synapse-artifacts { }; + azure-synapse-managedprivateendpoints = callPackage ../development/python-modules/azure-synapse-managedprivateendpoints { }; + azure-synapse-spark = callPackage ../development/python-modules/azure-synapse-spark { }; b2sdk = callPackage ../development/python-modules/b2sdk { }; From b1321c326426ad5552e758e31b91107122bd2eec Mon Sep 17 00:00:00 2001 From: Jonathan Ringer Date: Sat, 11 Sep 2021 18:39:07 -0700 Subject: [PATCH 13/28] python3Packages.azure-mgmt-batchai: add missing dep when being overriden --- .../python-modules/azure-mgmt-batchai/default.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/development/python-modules/azure-mgmt-batchai/default.nix b/pkgs/development/python-modules/azure-mgmt-batchai/default.nix index 6f7aedacfae2..0c005c3ea3c2 100644 --- a/pkgs/development/python-modules/azure-mgmt-batchai/default.nix +++ b/pkgs/development/python-modules/azure-mgmt-batchai/default.nix @@ -2,10 +2,9 @@ , buildPythonPackage , fetchPypi , azure-common +, azure-mgmt-core , azure-mgmt-nspkg , msrestazure -, python -, isPy3k }: buildPythonPackage rec { @@ -21,8 +20,9 @@ buildPythonPackage rec { propagatedBuildInputs = [ msrestazure azure-common + azure-mgmt-core azure-mgmt-nspkg - ]; + ]; pythonNamespaces = [ "azure.mgmt" ]; From 2a1f30c936041ed28340d5ee3afe56ba969711a3 Mon Sep 17 00:00:00 2001 From: Jonathan Ringer Date: Sat, 11 Sep 2021 18:39:25 -0700 Subject: [PATCH 14/28] python3Packages.azure-mgmt-sqlvirtualmachine: add missing dep when being overriden --- .../azure-mgmt-sqlvirtualmachine/default.nix | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pkgs/development/python-modules/azure-mgmt-sqlvirtualmachine/default.nix b/pkgs/development/python-modules/azure-mgmt-sqlvirtualmachine/default.nix index a34ebae2b812..76f0d7a2b39c 100644 --- a/pkgs/development/python-modules/azure-mgmt-sqlvirtualmachine/default.nix +++ b/pkgs/development/python-modules/azure-mgmt-sqlvirtualmachine/default.nix @@ -1,5 +1,6 @@ { lib, buildPythonPackage, fetchPypi, isPy27 , azure-common +, azure-mgmt-core , msrest , msrestazure }: @@ -15,7 +16,12 @@ buildPythonPackage rec { extension = "zip"; }; - propagatedBuildInputs = [ azure-common msrest msrestazure ]; + propagatedBuildInputs = [ + azure-common + azure-mgmt-core + msrest + msrestazure + ]; # no tests included doCheck = false; From c6df6f51cc64f6d86d76767a20ccaa60788e0bc0 Mon Sep 17 00:00:00 2001 From: Jonathan Ringer Date: Sat, 11 Sep 2021 18:43:54 -0700 Subject: [PATCH 15/28] azure-cli: 2.26.1 -> 2.28.0 --- pkgs/tools/admin/azure-cli/default.nix | 6 +- .../tools/admin/azure-cli/python-packages.nix | 120 ++++++++++-------- 2 files changed, 72 insertions(+), 54 deletions(-) diff --git a/pkgs/tools/admin/azure-cli/default.nix b/pkgs/tools/admin/azure-cli/default.nix index b93ab0e867b3..f9fe92c3f366 100644 --- a/pkgs/tools/admin/azure-cli/default.nix +++ b/pkgs/tools/admin/azure-cli/default.nix @@ -1,7 +1,7 @@ { stdenv, lib, python3, fetchFromGitHub, installShellFiles }: let - version = "2.26.1"; + version = "2.28.0"; srcName = "azure-cli-${version}-src"; src = fetchFromGitHub { @@ -9,7 +9,7 @@ let owner = "Azure"; repo = "azure-cli"; rev = "azure-cli-${version}"; - sha256 = "sha256-AwchP0o3I2T37dLPNw51wldwYUmcRuWptyzrhOocEaQ="; + sha256 = "sha256-fTt6tIcc96f8t3NUaR0RG0JQ+LPgqSHuJSmVx4a6X7M="; }; # put packages that needs to be overriden in the py package scope @@ -125,9 +125,11 @@ py.pkgs.toPythonApplication (py.pkgs.buildAzureCliPackage { azure-storage-blob azure-synapse-accesscontrol azure-synapse-artifacts + azure-synapse-managedprivateendpoints azure-synapse-spark colorama cryptography + distro Fabric jsmin knack diff --git a/pkgs/tools/admin/azure-cli/python-packages.nix b/pkgs/tools/admin/azure-cli/python-packages.nix index 2cde5a1da579..4740e9fd5cb1 100644 --- a/pkgs/tools/admin/azure-cli/python-packages.nix +++ b/pkgs/tools/admin/azure-cli/python-packages.nix @@ -91,14 +91,14 @@ let ''; doCheck = stdenv.isLinux; - # ignore tests that does network call + # ignore tests that does network call, or assume powershell checkPhase = '' rm azure/{,cli/}__init__.py python -c 'import azure.common; print(azure.common)' PYTHONPATH=$PWD:$PYTHONPATH HOME=$TMPDIR pytest \ --ignore=azure/cli/core/tests/test_profile.py \ --ignore=azure/cli/core/tests/test_generic_update.py \ - -k 'not metadata_url' + -k 'not metadata_url and not test_format_styled_text_legacy_powershell' ''; pythonImportsCheck = [ @@ -135,14 +135,17 @@ let azure-appconfiguration = overrideAzureMgmtPackage super.azure-appconfiguration "1.1.1" "zip" "sha256-uDzSy2PZMiXehOJ6u/wFkhL43id2b0xY3Tq7g53/C+Q="; - azure-batch = overrideAzureMgmtPackage super.azure-batch "10.0.0" "zip" - "83d7a2b0be42ca456ac2b56fa3dc6ce704c130e888d37d924072c1d3718f32d0"; + azure-batch = overrideAzureMgmtPackage super.azure-batch "11.0.0" "zip" + "83d7a2b0be42ca456ac2b56fa3dc6ce704c130e888d37d924072c1d3718f32da"; azure-mgmt-apimanagement = overrideAzureMgmtPackage super.azure-mgmt-apimanagement "0.2.0" "zip" "0whx3s8ri9939r3pdvjf8iqcslas1xy6cnccidmp10r5ng0023vr"; - azure-mgmt-batch = overrideAzureMgmtPackage super.azure-mgmt-batch "15.0.0" "zip" - "9b793bb31a0d4dc8c29186db61db24d83795851a75846aadb187cf95bf853ccb"; + azure-mgmt-batch = overrideAzureMgmtPackage super.azure-mgmt-batch "16.0.0" "zip" + "1b3cecd6f16813879c6ac1a1bb01f9a6f2752cd1f9157eb04d5e41e4a89f3c34"; + + azure-mgmt-batchai = overrideAzureMgmtPackage super.azure-mgmt-batchai "7.0.0b1" "zip" + "sha256-mT6vvjWbq0RWQidugR229E8JeVEiobPD3XA/nDM3I6Y="; azure-mgmt-billing = overrideAzureMgmtPackage super.azure-mgmt-billing "6.0.0" "zip" "d4f5c5a4188a456fe1eb32b6c45f55ca2069c74be41eb76921840b39f2f5c07f"; @@ -150,20 +153,20 @@ let azure-mgmt-botservice = overrideAzureMgmtPackage super.azure-mgmt-botservice "0.3.0" "zip" "f8318878a66a0685a01bf27b7d1409c44eb90eb72b0a616c1a2455c72330f2f1"; - azure-mgmt-policyinsights = overrideAzureMgmtPackage super.azure-mgmt-policyinsights "0.5.0" "zip" - "1wxh7mgrknnhqyafdd7sbwx8plx0zga2af21vs6yhxy48lw9w8pd"; + azure-mgmt-policyinsights = overrideAzureMgmtPackage super.azure-mgmt-policyinsights "1.0.0" "zip" + "75103fb4541aeae30bb687dee1fedd9ca65530e6b97b2d9ea87f74816905202a"; - azure-mgmt-rdbms = overrideAzureMgmtPackage super.azure-mgmt-rdbms "8.1.0b4" "zip" - "sha256-39msNYlZeZdn8cJ4LjZw9oxzy0YrNSPVEIN21wnkMKs="; + azure-mgmt-rdbms = overrideAzureMgmtPackage super.azure-mgmt-rdbms "9.1.0b1" "zip" + "sha256-O/6dE6lUnowYTWwQLWt3u1dwV4jBof+Jok0PUhFEEs0="; - azure-mgmt-recoveryservices = overrideAzureMgmtPackage super.azure-mgmt-recoveryservices "1.0.0" "zip" - "0v0ycyjnnx09jqf958hj2q6zfpsn80bxxm98jf59y8rj09v99rza"; + azure-mgmt-recoveryservices = overrideAzureMgmtPackage super.azure-mgmt-recoveryservices "2.0.0" "zip" + "sha256-p9MTfVxGD1CsLUQGHWCnC08nedTKhEt3QZtXJeZeCb4="; - azure-mgmt-recoveryservicesbackup = overrideAzureMgmtPackage super.azure-mgmt-recoveryservicesbackup "0.12.0" "zip" - "sha256-mrw1Gs3LMOPFiL5mSSrm+g1/dxihplA6ItYHbwljJbM="; + azure-mgmt-recoveryservicesbackup = overrideAzureMgmtPackage super.azure-mgmt-recoveryservicesbackup "0.14.0" "zip" + "sha256-sXGUQEO6tba3m1l+0Iir/eptTt+XRXyTr8FxbDzr6I0="; - azure-mgmt-resource = overrideAzureMgmtPackage super.azure-mgmt-resource "18.0.0" "zip" - "551036e592f409ef477d30937ea7cc4dda5126576965d9c816fdb8401bbd774c"; + azure-mgmt-resource = overrideAzureMgmtPackage super.azure-mgmt-resource "19.0.0" "zip" + "bbb60bb9419633c2339569d4e097908638c7944e782b5aef0f5d9535085a9100"; azure-mgmt-appconfiguration = overrideAzureMgmtPackage super.azure-mgmt-appconfiguration "2.0.0" "zip" "b58bbe82a7429ba589292024896b58d96fe9fa732c578569cac349928dc2ca5f"; @@ -171,23 +174,23 @@ let azure-mgmt-cognitiveservices = overrideAzureMgmtPackage super.azure-mgmt-cognitiveservices "12.0.0" "zip" "73054bd19866577e7e327518afc8f47e1639a11aea29a7466354b81804f4a676"; - azure-mgmt-compute = overrideAzureMgmtPackage super.azure-mgmt-compute "21.0.0" "zip" - "sha256-kifDdVpwvDd9cONeH8bx+rEJ3YiXT9LHEphnzKoQ+BM="; + azure-mgmt-compute = overrideAzureMgmtPackage super.azure-mgmt-compute "22.1.0" "zip" + "sha256-Kq1BSEOu4PVEJ/iH91NsxRVdaFJyjUTf7vYzrFITX9w="; azure-mgmt-consumption = overrideAzureMgmtPackage super.azure-mgmt-consumption "2.0.0" "zip" "12ai4qps73ivawh0yzvgb148ksx02r30pqlvfihx497j62gsi1cs"; - azure-mgmt-containerinstance = overrideAzureMgmtPackage super.azure-mgmt-containerinstance "1.4.0" "zip" - "1qw6228bia5pimcijr755npli2l33jyfka1s2bzgl1w4h3prsji7"; + azure-mgmt-containerinstance = overrideAzureMgmtPackage super.azure-mgmt-containerinstance "8.0.0" "zip" + "7aeb380af71fc35a71d6752fa25eb5b95fdb2a0027fa32e6f50bce87e2622916"; - azure-mgmt-containerservice = overrideAzureMgmtPackage super.azure-mgmt-containerservice "16.0.0" "zip" - "d6aa95951d32fe2cb390b3d8ae4f6459746de51bbaad94b5d1842dd35c4d0c11"; + azure-mgmt-containerservice = overrideAzureMgmtPackage super.azure-mgmt-containerservice "16.1.0" "zip" + "sha256-NlTIrOK4ho0OqcTHjHT1HobiMzDH2KY20TIlN0fm8/Q="; azure-mgmt-cosmosdb = overrideAzureMgmtPackage super.azure-mgmt-cosmosdb "6.4.0" "zip" "fb6b8ab80ab97214b94ae9e462ba1c459b68a3af296ffc26317ebd3ff500e00b"; - azure-mgmt-databoxedge = overrideAzureMgmtPackage super.azure-mgmt-databoxedge "0.2.0" "zip" - "sha256-g8BtUpIGOse8Jrws48gQ/o7sgymlgX0XIxl1ThHS3XA="; + azure-mgmt-databoxedge = overrideAzureMgmtPackage super.azure-mgmt-databoxedge "1.0.0" "zip" + "04090062bc1e8f00c2f45315a3bceb0fb3b3479ec1474d71b88342e13499b087"; azure-mgmt-deploymentmanager = overrideAzureMgmtPackage super.azure-mgmt-deploymentmanager "0.2.0" "zip" "0c6pyr36n9snx879vas5r6l25db6nlp2z96xn759mz4kg4i45qs6"; @@ -198,11 +201,14 @@ let azure-mgmt-imagebuilder = overrideAzureMgmtPackage super.azure-mgmt-imagebuilder "0.4.0" "zip" "0cqpjnkpid6a34ifd4vk4fn1h57pa1bg3r756wv082xl2szr34jc"; - azure-mgmt-iothub = overrideAzureMgmtPackage super.azure-mgmt-iothub "2.0.0" "zip" - "653a765f0beb6af0c9ecbd290b4101e1b5e0f6450405faf28ab8234c15d8b38b"; + azure-mgmt-iothub = overrideAzureMgmtPackage super.azure-mgmt-iothub "2.1.0" "zip" + "2724f48cadb1be7ee96fc26c7bfa178f82cea5d325e785e91d9f26965fa8e46f"; - azure-mgmt-iotcentral = overrideAzureMgmtPackage super.azure-mgmt-iotcentral "4.1.0" "zip" - "e6d4810f454c0d63a5e816eaa7e54a073a3f70b2256162ff1c234cfe91783ae6"; + azure-mgmt-iothubprovisioningservices = overrideAzureMgmtPackage super.azure-mgmt-iothubprovisioningservices "0.3.0" "zip" + "sha256-0Bt3JfP2jFpv8CGEqb3ajHdYiK9mN43YMUkD0KRuMrk="; + + azure-mgmt-iotcentral = overrideAzureMgmtPackage super.azure-mgmt-iotcentral "9.0.0b1" "zip" + "sha256-WEF5HuiaUTnka/2w0cVX/VwRt8/GD0+5fqGkn1BVx4I="; azure-mgmt-kusto = overrideAzureMgmtPackage super.azure-mgmt-kusto "0.3.0" "zip" "1pmcdgimd66h964a3d5m2j2fbydshcwhrk87wblhwhfl3xwbgf4y"; @@ -216,14 +222,14 @@ let azure-mgmt-dns = overrideAzureMgmtPackage super.azure-mgmt-dns "8.0.0" "zip" "407c2dacb33513ffbe9ca4be5addb5e9d4bae0cb7efa613c3f7d531ef7bf8de8"; - azure-mgmt-loganalytics = overrideAzureMgmtPackage super.azure-mgmt-loganalytics "8.0.0" "zip" - "3e7a93186594c328a6f34f0e0d9209a05021228baa85aa4c1c4ffdbf8005a45f"; + azure-mgmt-loganalytics = overrideAzureMgmtPackage super.azure-mgmt-loganalytics "11.0.0" "zip" + "41671fc6e95180fb6147cb40567410c34b85fb69bb0a9b3e09feae1ff370ee9d"; azure-mgmt-network = overrideAzureMgmtPackage super.azure-mgmt-network "19.0.0" "zip" "5e39a26ae81fa58c13c02029700f8c7b22c3fd832a294c543e3156a91b9459e8"; - azure-mgmt-maps = overrideAzureMgmtPackage super.azure-mgmt-maps "0.1.0" "zip" - "sha256-wSDiELthdo2ineJNKLgvjUKuJOUjlutlabSZcJ4i8AY="; + azure-mgmt-maps = overrideAzureMgmtPackage super.azure-mgmt-maps "2.0.0" "zip" + "384e17f76a68b700a4f988478945c3a9721711c0400725afdfcb63cf84e85f0e"; azure-mgmt-managedservices = overrideAzureMgmtPackage super.azure-mgmt-managedservices "1.0.0" "zip" "sha256-/tg5n8Z3Oq2jfB0ElqRvWUENd8lJTQyllnxTHDN2rRk="; @@ -234,8 +240,8 @@ let azure-mgmt-marketplaceordering = overrideAzureMgmtPackage super.azure-mgmt-marketplaceordering "1.1.0" "zip" "68b381f52a4df4435dacad5a97e1c59ac4c981f667dcca8f9d04453417d60ad8"; - azure-mgmt-media = overrideAzureMgmtPackage super.azure-mgmt-media "3.0.0" "zip" - "sha256-iUR3VyXFJTYU0ldXbYQe5or6NPVwsFwJJKf3Px2yiiQ="; + azure-mgmt-media = overrideAzureMgmtPackage super.azure-mgmt-media "7.0.0" "zip" + "sha256-tF6CpZTtkc1ap6XNXQHwOLesPPEiM+e6K+qqNHeQDo4="; azure-mgmt-msi = overrideAzureMgmtPackage super.azure-mgmt-msi "0.2.0" "zip" "0rvik03njz940x2hvqg6iiq8k0d88gyygsr86w8s0sa12sdbq8l6"; @@ -246,11 +252,11 @@ let azure-mgmt-web = overrideAzureMgmtPackage super.azure-mgmt-web "2.0.0" "zip" "0040e1c9c795f7bebe43647ff30b62cb0db7175175df5cbfa1e554a6a277b81e"; - azure-mgmt-redhatopenshift = overrideAzureMgmtPackage super.azure-mgmt-redhatopenshift "0.1.0" "zip" - "1g65lbia1i1jw6qkyjz2ldyl3p90rbr78l8kfryg70sj7z3gnnjn"; + azure-mgmt-redhatopenshift = overrideAzureMgmtPackage super.azure-mgmt-redhatopenshift "1.0.0" "zip" + "94cd41f1ebd82e40620fd3e6d88f666b5c19ac7cf8b4e8edadb9721bd7c80980"; - azure-mgmt-redis = overrideAzureMgmtPackage super.azure-mgmt-redis "7.0.0rc1" "zip" - "086wk31wsl8dx14qpd0g1bly8i9a8fix007djlj9cybva2f2bk6k"; + azure-mgmt-redis = overrideAzureMgmtPackage super.azure-mgmt-redis "13.0.0" "zip" + "283f776afe329472c20490b1f2c21c66895058cb06fb941eccda42cc247217f1"; azure-mgmt-reservations = overrideAzureMgmtPackage super.azure-mgmt-reservations "0.6.0" "zip" "16ycni3cjl9c0mv419gy5rgbrlg8zp0vnr6aj8z8p2ypdw6sgac3"; @@ -258,23 +264,23 @@ let azure-mgmt-search = overrideAzureMgmtPackage super.azure-mgmt-search "8.0.0" "zip" "a96d50c88507233a293e757202deead980c67808f432b8e897c4df1ca088da7e"; - azure-mgmt-security = overrideAzureMgmtPackage super.azure-mgmt-security "0.6.0" "zip" - "9f37d0151d730801222af111f0830905634795dbfd59ad1b89c35197421e74d3"; + azure-mgmt-security = overrideAzureMgmtPackage super.azure-mgmt-security "1.0.0" "zip" + "ae1cff598dfe80e93406e524c55c3f2cbffced9f9b7a5577e3375008a4c3bcad"; azure-mgmt-signalr = overrideAzureMgmtPackage super.azure-mgmt-signalr "1.0.0b2" "zip" "sha256-FTxY8qoihHG4OZuKT3sRRlKfORbIoqDqug9Ko+6S9dw="; - azure-mgmt-sql = overrideAzureMgmtPackage super.azure-mgmt-sql "0.29.0" "zip" - "sha256-a8epWPoGI4SA01DkjToaHmTx8lSPgwMB7dutI/zaoAA="; + azure-mgmt-sql = overrideAzureMgmtPackage super.azure-mgmt-sql "3.0.1" "zip" + "129042cc011225e27aee6ef2697d585fa5722e5d1aeb0038af6ad2451a285457"; - azure-mgmt-sqlvirtualmachine = overrideAzureMgmtPackage super.azure-mgmt-sqlvirtualmachine "0.5.0" "zip" - "1b9am8raa17hxnz7d5pk2ix0309wsnhnchq1mi22icd728sl5adm"; + azure-mgmt-sqlvirtualmachine = overrideAzureMgmtPackage super.azure-mgmt-sqlvirtualmachine "1.0.0b1" "zip" + "sha256-SrFTvU+67U3CpMLPZMawXuRdSIbTsfav2jFZIsZWPmw="; azure-mgmt-synapse = overrideAzureMgmtPackage super.azure-mgmt-synapse "2.0.0" "zip" "bec6bdfaeb55b4fdd159f2055e8875bf50a720bb0fce80a816e92a2359b898c8"; - azure-mgmt-datamigration = overrideAzureMgmtPackage super.azure-mgmt-datamigration "4.1.0" "zip" - "c33d1deb0ee173a15c8ec21a1e714ba544fe5f4895d3b1d8b0581f3c1b2e8ce4"; + azure-mgmt-datamigration = overrideAzureMgmtPackage super.azure-mgmt-datamigration "9.0.0" "zip" + "sha256-cDc9vrNad2ikc0G7O1cMVZGXvBujb8j4vxUTnkyLrXA="; azure-mgmt-relay = overrideAzureMgmtPackage super.azure-mgmt-relay "0.1.0" "zip" "1jss6qhvif8l5s0lblqw3qzijjf0h88agciiydaa7f4q577qgyfr"; @@ -282,14 +288,14 @@ let azure-mgmt-eventhub = overrideAzureMgmtPackage super.azure-mgmt-eventhub "4.1.0" "zip" "186g70slb259ybrr69zr2ibbmqgplnpncwxzg0nxp6rd7pml7d85"; - azure-mgmt-keyvault = overrideAzureMgmtPackage super.azure-mgmt-keyvault "9.0.0" "zip" - "2890c489289b8a0bf833852014f2f494eb96873834896910ddfa58cfa97b90da"; + azure-mgmt-keyvault = overrideAzureMgmtPackage super.azure-mgmt-keyvault "9.1.0" "zip" + "sha256-zTXoHEo8+BKt5L3PH3zPS1t4qAHvlnNAASpqyf5h3tI="; azure-mgmt-cdn = overrideAzureMgmtPackage super.azure-mgmt-cdn "11.0.0" "zip" "28e7070001e7208cdb6c2ad253ec78851abdd73be482230d2c0874eed5bc0907"; - azure-mgmt-containerregistry = overrideAzureMgmtPackage super.azure-mgmt-containerregistry "8.0.0" "zip" - "480eb5d1bb910da4e87a5d7047e2f3e426083b14911ac64c48badd0218e9e756"; + azure-mgmt-containerregistry = overrideAzureMgmtPackage super.azure-mgmt-containerregistry "8.1.0" "zip" + "62efbb03275d920894d79879ad0ed59605163abd32177dcf24e90c1862ebccbd"; azure-mgmt-monitor = overrideAzureMgmtPackage super.azure-mgmt-monitor "2.0.0" "zip" "e7f7943fe8f0efe98b3b1996cdec47c709765257a6e09e7940f7838a0f829e82"; @@ -309,8 +315,8 @@ let azure-mgmt-servicebus = overrideAzureMgmtPackage super.azure-mgmt-servicebus "6.0.0" "zip" "f6c64ed97d22d0c03c4ca5fc7594bd0f3d4147659c10110160009b93f541298e"; - azure-mgmt-servicefabric = overrideAzureMgmtPackage super.azure-mgmt-servicefabric "0.5.0" "zip" - "0x6wxb9zrvcayg3yw0nm99p10vvgc0x3zwk9amzs5m682r2z4wap"; + azure-mgmt-servicefabric = overrideAzureMgmtPackage super.azure-mgmt-servicefabric "1.0.0" "zip" + "de35e117912832c1a9e93109a8d24cab94f55703a9087b2eb1c5b0655b3b1913"; azure-mgmt-hdinsight = overrideAzureMgmtPackage super.azure-mgmt-hdinsight "8.0.0" "zip" "2c43f1a62e5b83304392b0ad7cfdaeef2ef2f47cb3fdfa2577b703b6ea126000"; @@ -367,6 +373,16 @@ let }; }); + azure-synapse-managedprivateendpoints = super.azure-synapse-managedprivateendpoints.overrideAttrs(oldAttrs: rec { + version = "0.3.0"; + src = super.fetchPypi { + inherit (oldAttrs) pname; + inherit version; + sha256 = "sha256-fN1IuZ9fjxgRZv6qh9gg6v6KYpnKlXfnoLqfZCDXoRY="; + extension = "zip"; + }; + }); + azure-synapse-spark = super.azure-synapse-spark.overrideAttrs(oldAttrs: rec { version = "0.2.0"; src = super.fetchPypi { From 6d488e823899d2dc1c2ce746a900b3267e93c2f7 Mon Sep 17 00:00:00 2001 From: Jonathan Ringer Date: Sat, 11 Sep 2021 19:02:15 -0700 Subject: [PATCH 16/28] azure-cli: allow for use with all versions of scp scp didn't have any breaking changes according to changelog: https://github.com/jbardin/scp.py/blob/master/CHANGELOG.md --- pkgs/tools/admin/azure-cli/default.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/pkgs/tools/admin/azure-cli/default.nix b/pkgs/tools/admin/azure-cli/default.nix index f9fe92c3f366..f2fec6fe6be7 100644 --- a/pkgs/tools/admin/azure-cli/default.nix +++ b/pkgs/tools/admin/azure-cli/default.nix @@ -28,6 +28,7 @@ py.pkgs.toPythonApplication (py.pkgs.buildAzureCliPackage { substituteInPlace setup.py \ --replace "javaproperties==0.5.1" "javaproperties" \ --replace "pytz==2019.1" "pytz" \ + --replace "scp~=0.13.2" "scp" \ --replace "jsondiff==1.2.0" "jsondiff~=1.2" \ --replace "antlr4-python3-runtime~=4.7.2" "antlr4-python3-runtime~=4.7" \ --replace "mock~=4.0" "mock" From a37eb5fe78f5e98f471e769e4f8dd174cf373f45 Mon Sep 17 00:00:00 2001 From: "R. RyanTM" Date: Mon, 13 Sep 2021 08:18:33 +0000 Subject: [PATCH 17/28] python38Packages.google-cloud-audit-log: 0.1.0 -> 0.1.1 --- .../python-modules/google-cloud-audit-log/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/development/python-modules/google-cloud-audit-log/default.nix b/pkgs/development/python-modules/google-cloud-audit-log/default.nix index 7d101c32284e..88e2378aaf74 100644 --- a/pkgs/development/python-modules/google-cloud-audit-log/default.nix +++ b/pkgs/development/python-modules/google-cloud-audit-log/default.nix @@ -2,11 +2,11 @@ buildPythonPackage rec { pname = "google-cloud-audit-log"; - version = "0.1.0"; + version = "0.1.1"; src = fetchPypi { inherit pname version; - sha256 = "5bf5a53c641b13828154ab21fb209669be69d71cd462f5d6456bf87722fc0eeb"; + sha256 = "a87fdf3c393b830b35c8f7db09094790d0d7babb35068736bea64e1618d286fe"; }; propagatedBuildInputs = [ googleapis-common-protos protobuf ]; From ab8017a2dd84af08ebfb4b9e43964013b3351f65 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Mon, 13 Sep 2021 10:49:16 +0200 Subject: [PATCH 18/28] php80: 8.0.8 -> 8.0.10 ChangeLog: https://www.php.net/ChangeLog-8.php#8.0.10 --- pkgs/development/interpreters/php/8.0.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/development/interpreters/php/8.0.nix b/pkgs/development/interpreters/php/8.0.nix index 82d78d95314e..624fe1c338e7 100644 --- a/pkgs/development/interpreters/php/8.0.nix +++ b/pkgs/development/interpreters/php/8.0.nix @@ -2,8 +2,8 @@ let base = callPackage ./generic.nix (_args // { - version = "8.0.8"; - sha256 = "0vyi9hhy7yl4l589dniwb3gq29sp3giq7ni4nca3x54q3bbpgg8l"; + version = "8.0.10"; + sha256 = "sha256-yUVHJxQQkAhFsITsK8s0Zq82PuypLLJL1hHcvcJvFYc="; }); in From 0f279e2e0c1ba0ad5bfbd79f055b9a2a9773f080 Mon Sep 17 00:00:00 2001 From: Alyssa Ross Date: Mon, 13 Sep 2021 10:45:17 +0000 Subject: [PATCH 19/28] nixos/top-level: fix cross When cross-compiling, we can't run the runtime shell to check syntax if it's e.g. for a different architecture. We have two options here. We can disable syntax checking when cross compiling, but that risks letting errors through. Or, we can do what I've done here, and change the syntax check to use stdenv's shell instead of the runtime shell. This requires the stdenv shell and runtime shell to be broadly compatible, but I think that's so ingrained in Nixpkgs anyway that it's fine. And this way we avoid conditionals that check for cross. --- nixos/modules/system/activation/top-level.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nixos/modules/system/activation/top-level.nix b/nixos/modules/system/activation/top-level.nix index 616e1422aa8c..dad9acba91af 100644 --- a/nixos/modules/system/activation/top-level.nix +++ b/nixos/modules/system/activation/top-level.nix @@ -61,8 +61,8 @@ let substituteInPlace $out/dry-activate --subst-var out chmod u+x $out/activate $out/dry-activate unset activationScript dryActivationScript - ${pkgs.runtimeShell} -n $out/activate - ${pkgs.runtimeShell} -n $out/dry-activate + ${pkgs.stdenv.shell} -n $out/activate + ${pkgs.stdenv.shell} -n $out/dry-activate cp ${config.system.build.bootStage2} $out/init substituteInPlace $out/init --subst-var-by systemConfig $out From a286dc9ef2fcc0748796c460f4a3ca169801b160 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Kr=C3=BCger?= Date: Mon, 13 Sep 2021 13:50:35 +0200 Subject: [PATCH 20/28] rPackages.RMySQL: fix package --- pkgs/development/r-modules/default.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkgs/development/r-modules/default.nix b/pkgs/development/r-modules/default.nix index 40ffae5e80fd..b1b1cd7d6058 100644 --- a/pkgs/development/r-modules/default.nix +++ b/pkgs/development/r-modules/default.nix @@ -786,7 +786,9 @@ let }); RMySQL = old.RMySQL.overrideDerivation (attrs: { - MYSQL_DIR="${pkgs.libmysqlclient}"; + MYSQL_DIR = "${pkgs.libmysqlclient}"; + PKGCONFIG_CFLAGS = "-I${pkgs.libmysqlclient.dev}/include/mysql"; + NIX_CFLAGS_LINK = "-L${pkgs.libmysqlclient}/lib/mysql -lmysqlclient"; preConfigure = '' patchShebangs configure ''; From ccb70a892af119fe3933ffaee852020ace79e7c9 Mon Sep 17 00:00:00 2001 From: talyz Date: Wed, 2 Jun 2021 14:25:02 +0200 Subject: [PATCH 21/28] python3Packages.mailsuite: Init at 1.6.1 --- .../python-modules/mailsuite/default.nix | 39 +++++++++++++++++++ pkgs/top-level/python-packages.nix | 2 + 2 files changed, 41 insertions(+) create mode 100644 pkgs/development/python-modules/mailsuite/default.nix diff --git a/pkgs/development/python-modules/mailsuite/default.nix b/pkgs/development/python-modules/mailsuite/default.nix new file mode 100644 index 000000000000..23167af45b0e --- /dev/null +++ b/pkgs/development/python-modules/mailsuite/default.nix @@ -0,0 +1,39 @@ +{ buildPythonPackage +, fetchPypi +, pythonOlder +, lib + +# pythonPackages +, dnspython +, html2text +, mail-parser +, IMAPClient +}: + +buildPythonPackage rec { + pname = "mailsuite"; + version = "1.6.1"; + + disabled = pythonOlder "3.6"; + + src = fetchPypi { + inherit pname version; + sha256 = "17bsnfjjzv8hx5h397p5pa92l6cqc53i0zjjz2p7bjj3xqzhs45a"; + }; + + propagatedBuildInputs = [ + dnspython + html2text + mail-parser + IMAPClient + ]; + + pythonImportsCheck = [ "mailsuite" ]; + + meta = { + description = "A Python package to simplify receiving, parsing, and sending email"; + homepage = "https://seanthegeek.github.io/mailsuite/"; + maintainers = with lib.maintainers; [ talyz ]; + license = lib.licenses.asl20; + }; +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 8d00d957f7ce..d2978ce0d824 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -4397,6 +4397,8 @@ in { makefun = callPackage ../development/python-modules/makefun { }; + mailsuite = callPackage ../development/python-modules/mailsuite { }; + Mako = callPackage ../development/python-modules/Mako { }; malduck= callPackage ../development/python-modules/malduck { }; From 5c61b9f3cb6ea038efc1390bd26f1f65f135c717 Mon Sep 17 00:00:00 2001 From: talyz Date: Wed, 2 Jun 2021 14:29:36 +0200 Subject: [PATCH 22/28] python3Packages.parsedmarc: Init at 7.0.1 --- .../python-modules/parsedmarc/default.nix | 72 +++++++++++++++++++ pkgs/top-level/python-packages.nix | 2 + 2 files changed, 74 insertions(+) create mode 100644 pkgs/development/python-modules/parsedmarc/default.nix diff --git a/pkgs/development/python-modules/parsedmarc/default.nix b/pkgs/development/python-modules/parsedmarc/default.nix new file mode 100644 index 000000000000..a19c00b004a6 --- /dev/null +++ b/pkgs/development/python-modules/parsedmarc/default.nix @@ -0,0 +1,72 @@ +{ buildPythonPackage +, fetchPypi +, fetchurl +, pythonOlder +, lib + +# pythonPackages +, tqdm +, dnspython +, expiringdict +, urllib3 +, requests +, publicsuffix2 +, xmltodict +, geoip2 +, IMAPClient +, dateparser +, elasticsearch-dsl +, kafka-python +, mailsuite +, lxml +, boto3 +}: + +let + dashboard = fetchurl { + url = "https://raw.githubusercontent.com/domainaware/parsedmarc/77331b55c54cb3269205295bd57d0ab680638964/grafana/Grafana-DMARC_Reports.json"; + sha256 = "0wbihyqbb4ndjg79qs8088zgrcg88km8khjhv2474y7nzjzkf43i"; + }; +in +buildPythonPackage rec { + pname = "parsedmarc"; + version = "7.0.1"; + + disabled = pythonOlder "3.7"; + + src = fetchPypi { + inherit pname version; + sha256 = "1mi4hx410y7ikpfy1582lm252si0c3yryj0idqgqbx417fm21jjc"; + }; + + propagatedBuildInputs = [ + tqdm + dnspython + expiringdict + urllib3 + requests + publicsuffix2 + xmltodict + geoip2 + IMAPClient + dateparser + elasticsearch-dsl + kafka-python + mailsuite + lxml + boto3 + ]; + + pythonImportsCheck = [ "parsedmarc" ]; + + passthru = { + inherit dashboard; + }; + + meta = { + description = "Python module and CLI utility for parsing DMARC reports"; + homepage = "https://domainaware.github.io/parsedmarc/"; + maintainers = with lib.maintainers; [ talyz ]; + license = lib.licenses.asl20; + }; +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index d2978ce0d824..05a811e60f6a 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -5255,6 +5255,8 @@ in { parsedatetime = callPackage ../development/python-modules/parsedatetime { }; + parsedmarc = callPackage ../development/python-modules/parsedmarc { }; + parsel = callPackage ../development/python-modules/parsel { }; parse-type = callPackage ../development/python-modules/parse-type { }; From 3a1e1f062432fbc867502187972b37f7a6f9fe63 Mon Sep 17 00:00:00 2001 From: talyz Date: Wed, 2 Jun 2021 14:33:13 +0200 Subject: [PATCH 23/28] nixos/elasticsearch: Wait for elasticsearch to start up properly Other services that depend on elasticsearch should be started after it, but since the versions we're packaging have to run as "Type=simple", they're started as soon as the elasticsearch binary has been executed, likely winning the race against it. This makes sure elasticsearch is up and running, responding to a simple query, before dependents are started. --- nixos/modules/services/search/elasticsearch.nix | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/nixos/modules/services/search/elasticsearch.nix b/nixos/modules/services/search/elasticsearch.nix index 1d7a28d5d245..3f676db27523 100644 --- a/nixos/modules/services/search/elasticsearch.nix +++ b/nixos/modules/services/search/elasticsearch.nix @@ -201,6 +201,13 @@ in if [ "$(id -u)" = 0 ]; then chown -R elasticsearch:elasticsearch ${cfg.dataDir}; fi ''; + postStart = '' + # Make sure elasticsearch is up and running before dependents + # are started + while ! ${pkgs.curl}/bin/curl -sS -f http://localhost:${toString cfg.port} 2>/dev/null; do + sleep 1 + done + ''; }; environment.systemPackages = [ cfg.package ]; From 98d9617705a2b04b12b25adc7037e798b225c1b6 Mon Sep 17 00:00:00 2001 From: talyz Date: Wed, 2 Jun 2021 14:32:20 +0200 Subject: [PATCH 24/28] nixos/parsedmarc: Add NixOS module --- nixos/modules/module-list.nix | 1 + .../services/monitoring/parsedmarc.nix | 532 ++++++++++++++++++ 2 files changed, 533 insertions(+) create mode 100644 nixos/modules/services/monitoring/parsedmarc.nix diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index fed212c0332c..c01c55b26700 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -621,6 +621,7 @@ ./services/monitoring/munin.nix ./services/monitoring/nagios.nix ./services/monitoring/netdata.nix + ./services/monitoring/parsedmarc.nix ./services/monitoring/prometheus/default.nix ./services/monitoring/prometheus/alertmanager.nix ./services/monitoring/prometheus/exporters.nix diff --git a/nixos/modules/services/monitoring/parsedmarc.nix b/nixos/modules/services/monitoring/parsedmarc.nix new file mode 100644 index 000000000000..58137a74d395 --- /dev/null +++ b/nixos/modules/services/monitoring/parsedmarc.nix @@ -0,0 +1,532 @@ +{ config, lib, pkgs, ... }: + +let + cfg = config.services.parsedmarc; + ini = pkgs.formats.ini {}; +in +{ + options.services.parsedmarc = { + + enable = lib.mkEnableOption '' + parsedmarc, a DMARC report monitoring service + ''; + + provision = { + localMail = { + enable = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Whether Postfix and Dovecot should be set up to receive + mail locally. parsedmarc will be configured to watch the + local inbox as the automatically created user specified in + + ''; + }; + + recipientName = lib.mkOption { + type = lib.types.str; + default = "dmarc"; + description = '' + The DMARC mail recipient name, i.e. the name part of the + email address which receives DMARC reports. + + A local user with this name will be set up and assigned a + randomized password on service start. + ''; + }; + + hostname = lib.mkOption { + type = lib.types.str; + default = config.networking.fqdn; + defaultText = "config.networking.fqdn"; + example = "monitoring.example.com"; + description = '' + The hostname to use when configuring Postfix. + + Should correspond to the host's fully qualified domain + name and the domain part of the email address which + receives DMARC reports. You also have to set up an MX record + pointing to this domain name. + ''; + }; + }; + + geoIp = lib.mkOption { + type = lib.types.bool; + default = true; + description = '' + Whether to enable and configure the geoipupdate + service to automatically fetch GeoIP databases. Not crucial, + but recommended for full functionality. + + To finish the setup, you need to manually set the and + + options. + ''; + }; + + elasticsearch = lib.mkOption { + type = lib.types.bool; + default = true; + description = '' + Whether to set up and use a local instance of Elasticsearch. + ''; + }; + + grafana = { + datasource = lib.mkOption { + type = lib.types.bool; + default = cfg.provision.elasticsearch && config.services.grafana.enable; + apply = x: x && cfg.provision.elasticsearch; + description = '' + Whether the automatically provisioned Elasticsearch + instance should be added as a grafana datasource. Has no + effect unless + + is also enabled. + ''; + }; + + dashboard = lib.mkOption { + type = lib.types.bool; + default = config.services.grafana.enable; + description = '' + Whether the official parsedmarc grafana dashboard should + be provisioned to the local grafana instance. + ''; + }; + }; + }; + + settings = lib.mkOption { + description = '' + Configuration parameters to set in + parsedmarc.ini. For a full list of + available parameters, see + . + ''; + + type = lib.types.submodule { + freeformType = ini.type; + + options = { + general = { + save_aggregate = lib.mkOption { + type = lib.types.bool; + default = true; + description = '' + Save aggregate report data to Elasticsearch and/or Splunk. + ''; + }; + + save_forensic = lib.mkOption { + type = lib.types.bool; + default = true; + description = '' + Save forensic report data to Elasticsearch and/or Splunk. + ''; + }; + }; + + imap = { + host = lib.mkOption { + type = lib.types.str; + default = "localhost"; + description = '' + The IMAP server hostname or IP address. + ''; + }; + + port = lib.mkOption { + type = lib.types.port; + default = 993; + description = '' + The IMAP server port. + ''; + }; + + ssl = lib.mkOption { + type = lib.types.bool; + default = true; + description = '' + Use an encrypted SSL/TLS connection. + ''; + }; + + user = lib.mkOption { + type = with lib.types; nullOr str; + default = null; + description = '' + The IMAP server username. + ''; + }; + + password = lib.mkOption { + type = with lib.types; nullOr path; + default = null; + description = '' + The path to a file containing the IMAP server password. + ''; + }; + + watch = lib.mkOption { + type = lib.types.bool; + default = true; + description = '' + Use the IMAP IDLE command to process messages as they arrive. + ''; + }; + + delete = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Delete messages after processing them, instead of archiving them. + ''; + }; + }; + + smtp = { + host = lib.mkOption { + type = with lib.types; nullOr str; + default = null; + description = '' + The SMTP server hostname or IP address. + ''; + }; + + port = lib.mkOption { + type = with lib.types; nullOr port; + default = null; + description = '' + The SMTP server port. + ''; + }; + + ssl = lib.mkOption { + type = with lib.types; nullOr bool; + default = null; + description = '' + Use an encrypted SSL/TLS connection. + ''; + }; + + user = lib.mkOption { + type = with lib.types; nullOr str; + default = null; + description = '' + The SMTP server username. + ''; + }; + + password = lib.mkOption { + type = with lib.types; nullOr path; + default = null; + description = '' + The path to a file containing the SMTP server password. + ''; + }; + + from = lib.mkOption { + type = with lib.types; nullOr str; + default = null; + description = '' + The From address to use for the + outgoing mail. + ''; + }; + + to = lib.mkOption { + type = with lib.types; nullOr (listOf str); + default = null; + description = '' + The addresses to send outgoing mail to. + ''; + }; + }; + + elasticsearch = { + hosts = lib.mkOption { + default = []; + type = with lib.types; listOf str; + apply = x: if x == [] then null else lib.concatStringsSep "," x; + description = '' + A list of Elasticsearch hosts to push parsed reports + to. + ''; + }; + + user = lib.mkOption { + type = with lib.types; nullOr str; + default = null; + description = '' + Username to use when connecting to Elasticsearch, if + required. + ''; + }; + + password = lib.mkOption { + type = with lib.types; nullOr path; + default = null; + description = '' + The path to a file containing the password to use when + connecting to Elasticsearch, if required. + ''; + }; + + ssl = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Whether to use an encrypted SSL/TLS connection. + ''; + }; + + cert_path = lib.mkOption { + type = lib.types.path; + default = "/etc/ssl/certs/ca-certificates.crt"; + description = '' + The path to a TLS certificate bundle used to verify + the server's certificate. + ''; + }; + }; + + kafka = { + hosts = lib.mkOption { + default = []; + type = with lib.types; listOf str; + apply = x: if x == [] then null else lib.concatStringsSep "," x; + description = '' + A list of Apache Kafka hosts to publish parsed reports + to. + ''; + }; + + user = lib.mkOption { + type = with lib.types; nullOr str; + default = null; + description = '' + Username to use when connecting to Kafka, if + required. + ''; + }; + + password = lib.mkOption { + type = with lib.types; nullOr path; + default = null; + description = '' + The path to a file containing the password to use when + connecting to Kafka, if required. + ''; + }; + + ssl = lib.mkOption { + type = with lib.types; nullOr bool; + default = null; + description = '' + Whether to use an encrypted SSL/TLS connection. + ''; + }; + + aggregate_topic = lib.mkOption { + type = with lib.types; nullOr str; + default = null; + example = "aggregate"; + description = '' + The Kafka topic to publish aggregate reports on. + ''; + }; + + forensic_topic = lib.mkOption { + type = with lib.types; nullOr str; + default = null; + example = "forensic"; + description = '' + The Kafka topic to publish forensic reports on. + ''; + }; + }; + + }; + + }; + }; + + }; + + config = lib.mkIf cfg.enable { + + services.elasticsearch.enable = lib.mkDefault cfg.provision.elasticsearch; + + services.geoipupdate = lib.mkIf cfg.provision.geoIp { + enable = true; + settings = { + EditionIDs = [ + "GeoLite2-ASN" + "GeoLite2-City" + "GeoLite2-Country" + ]; + DatabaseDirectory = "/var/lib/GeoIP"; + }; + }; + + services.dovecot2 = lib.mkIf cfg.provision.localMail.enable { + enable = true; + protocols = [ "imap" ]; + }; + + services.postfix = lib.mkIf cfg.provision.localMail.enable { + enable = true; + origin = cfg.provision.localMail.hostname; + config = { + myhostname = cfg.provision.localMail.hostname; + mydestination = cfg.provision.localMail.hostname; + }; + }; + + services.grafana = { + declarativePlugins = with pkgs.grafanaPlugins; + lib.mkIf cfg.provision.grafana.dashboard [ + grafana-worldmap-panel + grafana-piechart-panel + ]; + + provision = { + enable = cfg.provision.grafana.datasource || cfg.provision.grafana.dashboard; + datasources = + let + pkgVer = lib.getVersion config.services.elasticsearch.package; + esVersion = + if lib.versionOlder pkgVer "7" then + "60" + else if lib.versionOlder pkgVer "8" then + "70" + else + throw "When provisioning parsedmarc grafana datasources: unknown Elasticsearch version."; + in + lib.mkIf cfg.provision.grafana.datasource [ + { + name = "dmarc-ag"; + type = "elasticsearch"; + access = "proxy"; + url = "localhost:9200"; + jsonData = { + timeField = "date_range"; + inherit esVersion; + }; + } + { + name = "dmarc-fo"; + type = "elasticsearch"; + access = "proxy"; + url = "localhost:9200"; + jsonData = { + timeField = "date_range"; + inherit esVersion; + }; + } + ]; + dashboards = lib.mkIf cfg.provision.grafana.dashboard [{ + name = "parsedmarc"; + options.path = "${pkgs.python3Packages.parsedmarc.dashboard}"; + }]; + }; + }; + + services.parsedmarc.settings = lib.mkMerge [ + (lib.mkIf cfg.provision.elasticsearch { + elasticsearch = { + hosts = [ "localhost:9200" ]; + ssl = false; + }; + }) + (lib.mkIf cfg.provision.localMail.enable { + imap = { + host = "localhost"; + port = 143; + ssl = false; + user = cfg.provision.localMail.recipientName; + password = "${pkgs.writeText "imap-password" "@imap-password@"}"; + watch = true; + }; + }) + ]; + + systemd.services.parsedmarc = + let + # Remove any empty attributes from the config, i.e. empty + # lists, empty attrsets and null. This makes it possible to + # list interesting options in `settings` without them always + # ending up in the resulting config. + filteredConfig = lib.converge (lib.filterAttrsRecursive (_: v: ! builtins.elem v [ null [] {} ])) cfg.settings; + parsedmarcConfig = ini.generate "parsedmarc.ini" filteredConfig; + mkSecretReplacement = file: + lib.optionalString (file != null) '' + replace-secret '${file}' '${file}' /run/parsedmarc/parsedmarc.ini + ''; + in + { + wantedBy = [ "multi-user.target" ]; + after = [ "postfix.service" "dovecot2.service" "elasticsearch.service" ]; + path = with pkgs; [ replace-secret openssl shadow ]; + serviceConfig = { + ExecStartPre = let + startPreFullPrivileges = '' + set -o errexit -o pipefail -o nounset -o errtrace + shopt -s inherit_errexit + + umask u=rwx,g=,o= + cp ${parsedmarcConfig} /run/parsedmarc/parsedmarc.ini + chown parsedmarc:parsedmarc /run/parsedmarc/parsedmarc.ini + ${mkSecretReplacement cfg.settings.smtp.password} + ${mkSecretReplacement cfg.settings.imap.password} + ${mkSecretReplacement cfg.settings.elasticsearch.password} + ${mkSecretReplacement cfg.settings.kafka.password} + '' + lib.optionalString cfg.provision.localMail.enable '' + openssl rand -hex 64 >/run/parsedmarc/dmarc_user_passwd + replace-secret '@imap-password@' '/run/parsedmarc/dmarc_user_passwd' /run/parsedmarc/parsedmarc.ini + echo "Setting new randomized password for user '${cfg.provision.localMail.recipientName}'." + cat <(echo -n "${cfg.provision.localMail.recipientName}:") /run/parsedmarc/dmarc_user_passwd | chpasswd + ''; + in + "+${pkgs.writeShellScript "parsedmarc-start-pre-full-privileges" startPreFullPrivileges}"; + Type = "simple"; + User = "parsedmarc"; + Group = "parsedmarc"; + DynamicUser = true; + RuntimeDirectory = "parsedmarc"; + RuntimeDirectoryMode = 0700; + CapabilityBoundingSet = ""; + PrivateDevices = true; + PrivateMounts = true; + PrivateUsers = true; + ProtectClock = true; + ProtectControlGroups = true; + ProtectHome = true; + ProtectHostname = true; + ProtectKernelLogs = true; + ProtectKernelModules = true; + ProtectKernelTunables = true; + ProtectProc = "invisible"; + ProcSubset = "pid"; + SystemCallFilter = [ "@system-service" "~@privileged" "~@resources" ]; + RestrictAddressFamilies = [ "AF_UNIX" "AF_INET" "AF_INET6" ]; + RestrictRealtime = true; + RestrictNamespaces = true; + MemoryDenyWriteExecute = true; + LockPersonality = true; + SystemCallArchitectures = "native"; + ExecStart = "${pkgs.python3Packages.parsedmarc}/bin/parsedmarc -c /run/parsedmarc/parsedmarc.ini"; + }; + }; + + users.users.${cfg.provision.localMail.recipientName} = lib.mkIf cfg.provision.localMail.enable { + isNormalUser = true; + description = "DMARC mail recipient"; + }; + }; +} From d46e78fd76f9b27bc3b8203dc16aa21371ea1bbd Mon Sep 17 00:00:00 2001 From: talyz Date: Wed, 2 Jun 2021 18:19:37 +0200 Subject: [PATCH 25/28] nixos/parsedmarc: Add test --- nixos/tests/all-tests.nix | 1 + nixos/tests/parsedmarc/default.nix | 224 ++++++++++++++++++ .../python-modules/parsedmarc/default.nix | 2 + 3 files changed, 227 insertions(+) create mode 100644 nixos/tests/parsedmarc/default.nix diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 6baa986b2bda..dab99dd8d043 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -336,6 +336,7 @@ in pam-u2f = handleTest ./pam-u2f.nix {}; pantheon = handleTest ./pantheon.nix {}; paperless-ng = handleTest ./paperless-ng.nix {}; + parsedmarc = handleTest ./parsedmarc {}; pdns-recursor = handleTest ./pdns-recursor.nix {}; peerflix = handleTest ./peerflix.nix {}; pgjwt = handleTest ./pgjwt.nix {}; diff --git a/nixos/tests/parsedmarc/default.nix b/nixos/tests/parsedmarc/default.nix new file mode 100644 index 000000000000..d838d3b6a39c --- /dev/null +++ b/nixos/tests/parsedmarc/default.nix @@ -0,0 +1,224 @@ +# This tests parsedmarc by sending a report to its monitored email +# address and reading the results out of Elasticsearch. + +{ pkgs, ... }@args: +let + inherit (import ../../lib/testing-python.nix args) makeTest; + + dmarcTestReport = builtins.fetchurl { + name = "dmarc-test-report"; + url = "https://github.com/domainaware/parsedmarc/raw/f45ab94e0608088e0433557608d9f4e9517d3afe/samples/aggregate/estadocuenta1.infonacot.gob.mx!example.com!1536853302!1536939702!2940.xml.zip"; + sha256 = "0dq64cj49711kbja27pjl2hy0d3azrjxg91kqrh40x46fkn1dwkx"; + }; + + sendEmail = address: + pkgs.writeScriptBin "send-email" '' + #!${pkgs.python3.interpreter} + import smtplib + from email import encoders + from email.mime.base import MIMEBase + from email.mime.multipart import MIMEMultipart + from email.mime.text import MIMEText + + sender_email = "dmarc_tester@fake.domain" + receiver_email = "${address}" + + message = MIMEMultipart() + message["From"] = sender_email + message["To"] = receiver_email + message["Subject"] = "DMARC test" + + message.attach(MIMEText("Testing parsedmarc", "plain")) + + attachment = MIMEBase("application", "zip") + + with open("${dmarcTestReport}", "rb") as report: + attachment.set_payload(report.read()) + + encoders.encode_base64(attachment) + + attachment.add_header( + "Content-Disposition", + "attachment; filename= estadocuenta1.infonacot.gob.mx!example.com!1536853302!1536939702!2940.xml.zip", + ) + + message.attach(attachment) + text = message.as_string() + + with smtplib.SMTP('localhost') as server: + server.sendmail(sender_email, receiver_email, text) + server.quit() + ''; +in +{ + localMail = makeTest + { + name = "parsedmarc-local-mail"; + meta = with pkgs.lib.maintainers; { + maintainers = [ talyz ]; + }; + + nodes.parsedmarc = + { nodes, ... }: + { + virtualisation.memorySize = 2048; + + services.postfix = { + enableSubmission = true; + enableSubmissions = true; + submissionsOptions = { + smtpd_sasl_auth_enable = "yes"; + smtpd_client_restrictions = "permit"; + }; + }; + + services.parsedmarc = { + enable = true; + provision = { + geoIp = false; + localMail = { + enable = true; + hostname = "localhost"; + }; + }; + }; + + services.elasticsearch.package = pkgs.elasticsearch7-oss; + + environment.systemPackages = [ + (sendEmail "dmarc@localhost") + pkgs.jq + ]; + }; + + testScript = { nodes }: + let + esPort = toString nodes.parsedmarc.config.services.elasticsearch.port; + in '' + parsedmarc.start() + parsedmarc.wait_for_unit("postfix.service") + parsedmarc.wait_for_unit("dovecot2.service") + parsedmarc.wait_for_unit("parsedmarc.service") + parsedmarc.wait_until_succeeds( + "curl -sS -f http://localhost:${esPort}" + ) + + parsedmarc.fail( + "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940 | jq -e 'if .hits.total.value > 0 then true else null end'" + ) + parsedmarc.succeed("send-email") + parsedmarc.wait_until_succeeds( + "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940 | jq -e 'if .hits.total.value > 0 then true else null end'" + ) + ''; + }; + + externalMail = + let + certs = import ../common/acme/server/snakeoil-certs.nix; + mailDomain = certs.domain; + parsedmarcDomain = "parsedmarc.fake.domain"; + in + makeTest { + name = "parsedmarc-external-mail"; + meta = with pkgs.lib.maintainers; { + maintainers = [ talyz ]; + }; + + nodes = { + parsedmarc = + { nodes, ... }: + { + virtualisation.memorySize = 2048; + + security.pki.certificateFiles = [ + certs.ca.cert + ]; + + networking.extraHosts = '' + 127.0.0.1 ${parsedmarcDomain} + ${nodes.mail.config.networking.primaryIPAddress} ${mailDomain} + ''; + + services.parsedmarc = { + enable = true; + provision.geoIp = false; + settings.imap = { + host = mailDomain; + port = 993; + ssl = true; + user = "alice"; + password = "${pkgs.writeText "imap-password" "foobar"}"; + watch = true; + }; + }; + + services.elasticsearch.package = pkgs.elasticsearch7-oss; + + environment.systemPackages = [ + pkgs.jq + ]; + }; + + mail = + { nodes, ... }: + { + imports = [ ../common/user-account.nix ]; + + networking.extraHosts = '' + 127.0.0.1 ${mailDomain} + ${nodes.parsedmarc.config.networking.primaryIPAddress} ${parsedmarcDomain} + ''; + + services.dovecot2 = { + enable = true; + protocols = [ "imap" ]; + sslCACert = "${certs.ca.cert}"; + sslServerCert = "${certs.${mailDomain}.cert}"; + sslServerKey = "${certs.${mailDomain}.key}"; + }; + + services.postfix = { + enable = true; + origin = mailDomain; + config = { + myhostname = mailDomain; + mydestination = mailDomain; + }; + enableSubmission = true; + enableSubmissions = true; + submissionsOptions = { + smtpd_sasl_auth_enable = "yes"; + smtpd_client_restrictions = "permit"; + }; + }; + environment.systemPackages = [ (sendEmail "alice@${mailDomain}") ]; + + networking.firewall.allowedTCPPorts = [ 993 ]; + }; + }; + + testScript = { nodes }: + let + esPort = toString nodes.parsedmarc.config.services.elasticsearch.port; + in '' + mail.start() + mail.wait_for_unit("postfix.service") + mail.wait_for_unit("dovecot2.service") + + parsedmarc.start() + parsedmarc.wait_for_unit("parsedmarc.service") + parsedmarc.wait_until_succeeds( + "curl -sS -f http://localhost:${esPort}" + ) + + parsedmarc.fail( + "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940 | jq -e 'if .hits.total.value > 0 then true else null end'" + ) + mail.succeed("send-email") + parsedmarc.wait_until_succeeds( + "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940 | jq -e 'if .hits.total.value > 0 then true else null end'" + ) + ''; + }; +} diff --git a/pkgs/development/python-modules/parsedmarc/default.nix b/pkgs/development/python-modules/parsedmarc/default.nix index a19c00b004a6..b9fb35016c3b 100644 --- a/pkgs/development/python-modules/parsedmarc/default.nix +++ b/pkgs/development/python-modules/parsedmarc/default.nix @@ -3,6 +3,7 @@ , fetchurl , pythonOlder , lib +, nixosTests # pythonPackages , tqdm @@ -61,6 +62,7 @@ buildPythonPackage rec { passthru = { inherit dashboard; + tests = nixosTests.parsedmarc; }; meta = { From 6496902fb2b1496c7d5aba78afd4571584cd751e Mon Sep 17 00:00:00 2001 From: talyz Date: Thu, 3 Jun 2021 18:31:15 +0200 Subject: [PATCH 26/28] nixos/parsedmarc: Add manual entry and release note --- .../from_md/release-notes/rl-2111.section.xml | 11 ++ .../manual/release-notes/rl-2111.section.md | 5 + .../modules/services/monitoring/parsedmarc.md | 113 ++++++++++++++++ .../services/monitoring/parsedmarc.nix | 5 + .../services/monitoring/parsedmarc.xml | 125 ++++++++++++++++++ 5 files changed, 259 insertions(+) create mode 100644 nixos/modules/services/monitoring/parsedmarc.md create mode 100644 nixos/modules/services/monitoring/parsedmarc.xml diff --git a/nixos/doc/manual/from_md/release-notes/rl-2111.section.xml b/nixos/doc/manual/from_md/release-notes/rl-2111.section.xml index 57cd98db5894..6eaba9111a2b 100644 --- a/nixos/doc/manual/from_md/release-notes/rl-2111.section.xml +++ b/nixos/doc/manual/from_md/release-notes/rl-2111.section.xml @@ -239,6 +239,17 @@ programs.git. + + + parsedmarc, + a service which parses incoming + DMARC reports and + stores or sends them to a downstream service for further + analysis. Documented in + its manual + entry. + +
diff --git a/nixos/doc/manual/release-notes/rl-2111.section.md b/nixos/doc/manual/release-notes/rl-2111.section.md index 440d063abf28..b77bd30ea17c 100644 --- a/nixos/doc/manual/release-notes/rl-2111.section.md +++ b/nixos/doc/manual/release-notes/rl-2111.section.md @@ -73,6 +73,11 @@ subsonic-compatible api. Available as [navidrome](#opt-services.navidrome.enable - [git](https://git-scm.com), a distributed version control system. Available as [programs.git](options.html#opt-programs.git.enable). +- [parsedmarc](https://domainaware.github.io/parsedmarc/), a service + which parses incoming [DMARC](https://dmarc.org/) reports and stores + or sends them to a downstream service for further analysis. + Documented in [its manual entry](#module-services-parsedmarc). + ## Backward Incompatibilities {#sec-release-21.11-incompatibilities} diff --git a/nixos/modules/services/monitoring/parsedmarc.md b/nixos/modules/services/monitoring/parsedmarc.md new file mode 100644 index 000000000000..d93134a4cc76 --- /dev/null +++ b/nixos/modules/services/monitoring/parsedmarc.md @@ -0,0 +1,113 @@ +# parsedmarc {#module-services-parsedmarc} +[parsedmarc](https://domainaware.github.io/parsedmarc/) is a service +which parses incoming [DMARC](https://dmarc.org/) reports and stores +or sends them to a downstream service for further analysis. In +combination with Elasticsearch, Grafana and the included Grafana +dashboard, it provides a handy overview of DMARC reports over time. + +## Basic usage {#module-services-parsedmarc-basic-usage} +A very minimal setup which reads incoming reports from an external +email address and saves them to a local Elasticsearch instance looks +like this: + +```nix +services.parsedmarc = { + enable = true; + settings.imap = { + host = "imap.example.com"; + user = "alice@example.com"; + password = "/path/to/imap_password_file"; + watch = true; + }; + provision.geoIp = false; # Not recommended! +}; +``` + +Note that GeoIP provisioning is disabled in the example for +simplicity, but should be turned on for fully functional reports. + +## Local mail +Instead of watching an external inbox, a local inbox can be +automatically provisioned. The recipient's name is by default set to +`dmarc`, but can be configured in +[services.parsedmarc.provision.localMail.recipientName](options.html#opt-services.parsedmarc.provision.localMail.recipientName). You +need to add an MX record pointing to the host. More concretely: for +the example to work, an MX record needs to be set up for +`monitoring.example.com` and the complete email address that should be +configured in the domain's dmarc policy is +`dmarc@monitoring.example.com`. + +```nix +services.parsedmarc = { + enable = true; + provision = { + localMail = { + enable = true; + hostname = monitoring.example.com; + }; + geoIp = false; # Not recommended! + }; +}; +``` + +## Grafana and GeoIP +The reports can be visualized and summarized with parsedmarc's +official Grafana dashboard. For all views to work, and for the data to +be complete, GeoIP databases are also required. The following example +shows a basic deployment where the provisioned Elasticsearch instance +is automatically added as a Grafana datasource, and the dashboard is +added to Grafana as well. + +```nix +services.parsedmarc = { + enable = true; + provision = { + localMail = { + enable = true; + hostname = url; + }; + grafana = { + datasource = true; + dashboard = true; + }; + }; +}; + +# Not required, but recommended for full functionality +services.geoipupdate = { + settings = { + AccountID = 000000; + LicenseKey = "/path/to/license_key_file"; + }; +}; + +services.grafana = { + enable = true; + addr = "0.0.0.0"; + domain = url; + rootUrl = "https://" + url; + protocol = "socket"; + security = { + adminUser = "admin"; + adminPasswordFile = "/path/to/admin_password_file"; + secretKeyFile = "/path/to/secret_key_file"; + }; +}; + +services.nginx = { + enable = true; + recommendedTlsSettings = true; + recommendedOptimisation = true; + recommendedGzipSettings = true; + recommendedProxySettings = true; + upstreams.grafana.servers."unix:/${config.services.grafana.socket}" = {}; + virtualHosts.${url} = { + root = config.services.grafana.staticRootPath; + enableACME = true; + forceSSL = true; + locations."/".tryFiles = "$uri @grafana"; + locations."@grafana".proxyPass = "http://grafana"; + }; +}; +users.users.nginx.extraGroups = [ "grafana" ]; +``` diff --git a/nixos/modules/services/monitoring/parsedmarc.nix b/nixos/modules/services/monitoring/parsedmarc.nix index 58137a74d395..e6a72dea0260 100644 --- a/nixos/modules/services/monitoring/parsedmarc.nix +++ b/nixos/modules/services/monitoring/parsedmarc.nix @@ -529,4 +529,9 @@ in description = "DMARC mail recipient"; }; }; + + # Don't edit the docbook xml directly, edit the md and generate it: + # `pandoc parsedmarc.md -t docbook --top-level-division=chapter --extract-media=media -f markdown+smart > parsedmarc.xml` + meta.doc = ./parsedmarc.xml; + meta.maintainers = [ lib.maintainers.talyz ]; } diff --git a/nixos/modules/services/monitoring/parsedmarc.xml b/nixos/modules/services/monitoring/parsedmarc.xml new file mode 100644 index 000000000000..7167b52d0357 --- /dev/null +++ b/nixos/modules/services/monitoring/parsedmarc.xml @@ -0,0 +1,125 @@ + + parsedmarc + + parsedmarc + is a service which parses incoming + DMARC reports and + stores or sends them to a downstream service for further analysis. + In combination with Elasticsearch, Grafana and the included Grafana + dashboard, it provides a handy overview of DMARC reports over time. + +
+ Basic usage + + A very minimal setup which reads incoming reports from an external + email address and saves them to a local Elasticsearch instance + looks like this: + + +services.parsedmarc = { + enable = true; + settings.imap = { + host = "imap.example.com"; + user = "alice@example.com"; + password = "/path/to/imap_password_file"; + watch = true; + }; + provision.geoIp = false; # Not recommended! +}; + + + Note that GeoIP provisioning is disabled in the example for + simplicity, but should be turned on for fully functional reports. + +
+
+ Local mail + + Instead of watching an external inbox, a local inbox can be + automatically provisioned. The recipient’s name is by default set + to dmarc, but can be configured in + services.parsedmarc.provision.localMail.recipientName. + You need to add an MX record pointing to the host. More + concretely: for the example to work, an MX record needs to be set + up for monitoring.example.com and the complete + email address that should be configured in the domain’s dmarc + policy is dmarc@monitoring.example.com. + + +services.parsedmarc = { + enable = true; + provision = { + localMail = { + enable = true; + hostname = monitoring.example.com; + }; + geoIp = false; # Not recommended! + }; +}; + +
+
+ Grafana and GeoIP + + The reports can be visualized and summarized with parsedmarc’s + official Grafana dashboard. For all views to work, and for the + data to be complete, GeoIP databases are also required. The + following example shows a basic deployment where the provisioned + Elasticsearch instance is automatically added as a Grafana + datasource, and the dashboard is added to Grafana as well. + + +services.parsedmarc = { + enable = true; + provision = { + localMail = { + enable = true; + hostname = url; + }; + grafana = { + datasource = true; + dashboard = true; + }; + }; +}; + +# Not required, but recommended for full functionality +services.geoipupdate = { + settings = { + AccountID = 000000; + LicenseKey = "/path/to/license_key_file"; + }; +}; + +services.grafana = { + enable = true; + addr = "0.0.0.0"; + domain = url; + rootUrl = "https://" + url; + protocol = "socket"; + security = { + adminUser = "admin"; + adminPasswordFile = "/path/to/admin_password_file"; + secretKeyFile = "/path/to/secret_key_file"; + }; +}; + +services.nginx = { + enable = true; + recommendedTlsSettings = true; + recommendedOptimisation = true; + recommendedGzipSettings = true; + recommendedProxySettings = true; + upstreams.grafana.servers."unix:/${config.services.grafana.socket}" = {}; + virtualHosts.${url} = { + root = config.services.grafana.staticRootPath; + enableACME = true; + forceSSL = true; + locations."/".tryFiles = "$uri @grafana"; + locations."@grafana".proxyPass = "http://grafana"; + }; +}; +users.users.nginx.extraGroups = [ "grafana" ]; + +
+
From 1d67a676d4150227a057100c710984dd44470f40 Mon Sep 17 00:00:00 2001 From: sternenseemann <0rpkxez4ksa01gb3typccl0i@systemli.org> Date: Wed, 1 Sep 2021 15:05:14 +0200 Subject: [PATCH 27/28] inspircd: 3.10.0 -> 3.11.0 https://docs.inspircd.org/3/change-log/#inspircd-3110 --- pkgs/servers/irc/inspircd/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/servers/irc/inspircd/default.nix b/pkgs/servers/irc/inspircd/default.nix index 59fb4d4e769c..7addb8ab3a1e 100644 --- a/pkgs/servers/irc/inspircd/default.nix +++ b/pkgs/servers/irc/inspircd/default.nix @@ -142,13 +142,13 @@ in stdenv.mkDerivation rec { pname = "inspircd"; - version = "3.10.0"; + version = "3.11.0"; src = fetchFromGitHub { owner = pname; repo = pname; rev = "v${version}"; - sha256 = "1817gmxk4v7k5398d2fb6qkwadg0fd980gqmr80wdnppx450ikn7"; + sha256 = "083fp69fi4nhrw9v1dan5m3mgb19a2gpqnap356xs9nnqy01sgv7"; }; outputs = [ "bin" "lib" "man" "doc" "out" ]; From 98a9aba0c07b3213a85109ff97589bab4ab0b387 Mon Sep 17 00:00:00 2001 From: Pablo Ovelleiro Corral Date: Mon, 13 Sep 2021 15:27:36 +0200 Subject: [PATCH 28/28] Fix typo in key ID --- maintainers/maintainer-list.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maintainers/maintainer-list.nix b/maintainers/maintainer-list.nix index 628a42ce1f00..aeca70d7fe43 100644 --- a/maintainers/maintainer-list.nix +++ b/maintainers/maintainer-list.nix @@ -8582,7 +8582,7 @@ githubId = 1719781; name = "Pablo Ovelleiro Corral"; keys = [{ - longkeyid = "sa4096/0x823A6154426408D3"; + longkeyid = "rsa4096/0x823A6154426408D3"; fingerprint = "D03B 218C AE77 1F77 D7F9 20D9 823A 6154 4264 08D3"; }]; };