diff --git a/maintainers/maintainer-list.nix b/maintainers/maintainer-list.nix
index a62b19f413c3..57224c438daf 100644
--- a/maintainers/maintainer-list.nix
+++ b/maintainers/maintainer-list.nix
@@ -1080,6 +1080,12 @@
     githubId = 354741;
     name = "Austin Butler";
   };
+  autophagy = {
+    email = "mail@autophagy.io";
+    github = "autophagy";
+    githubId = 12958979;
+    name = "Mika Naylor";
+  };
   avaq = {
     email = "nixpkgs@account.avaq.it";
     github = "avaq";
diff --git a/nixos/modules/services/cluster/hadoop/conf.nix b/nixos/modules/services/cluster/hadoop/conf.nix
index 38db10406b9a..69472408cabe 100644
--- a/nixos/modules/services/cluster/hadoop/conf.nix
+++ b/nixos/modules/services/cluster/hadoop/conf.nix
@@ -1,4 +1,4 @@
-{ hadoop, pkgs }:
+{ cfg, pkgs, lib }:
 let
   propertyXml = name: value: ''
     <property>
@@ -13,19 +13,31 @@ let
       ${builtins.concatStringsSep "\n" (pkgs.lib.mapAttrsToList propertyXml properties)}
     </configuration>
   '';
+  cfgLine = name: value: ''
+    ${name}=${builtins.toString value}
+  '';
+  cfgFile = fileName: properties: pkgs.writeTextDir fileName ''
+    # generated by NixOS
+    ${builtins.concatStringsSep "" (pkgs.lib.mapAttrsToList cfgLine properties)}
+  '';
   userFunctions = ''
     hadoop_verify_logdir() {
       echo Skipping verification of log directory
     }
   '';
+  hadoopEnv = ''
+    export HADOOP_LOG_DIR=/tmp/hadoop/$USER
+  '';
 in
-pkgs.buildEnv {
-  name = "hadoop-conf";
-  paths = [
-    (siteXml "core-site.xml" hadoop.coreSite)
-    (siteXml "hdfs-site.xml" hadoop.hdfsSite)
-    (siteXml "mapred-site.xml" hadoop.mapredSite)
-    (siteXml "yarn-site.xml" hadoop.yarnSite)
-    (pkgs.writeTextDir "hadoop-user-functions.sh" userFunctions)
-  ];
-}
+pkgs.runCommand "hadoop-conf" {} ''
+  mkdir -p $out/
+  cp ${siteXml "core-site.xml" cfg.coreSite}/* $out/
+  cp ${siteXml "hdfs-site.xml" cfg.hdfsSite}/* $out/
+  cp ${siteXml "mapred-site.xml" cfg.mapredSite}/* $out/
+  cp ${siteXml "yarn-site.xml" cfg.yarnSite}/* $out/
+  cp ${cfgFile "container-executor.cfg" cfg.containerExecutorCfg}/* $out/
+  cp ${pkgs.writeTextDir "hadoop-user-functions.sh" userFunctions}/* $out/
+  cp ${pkgs.writeTextDir "hadoop-env.sh" hadoopEnv}/* $out/
+  cp ${cfg.log4jProperties} $out/log4j.properties
+  ${lib.concatMapStringsSep "\n" (dir: "cp -r ${dir}/* $out/") cfg.extraConfDirs}
+''
diff --git a/nixos/modules/services/cluster/hadoop/default.nix b/nixos/modules/services/cluster/hadoop/default.nix
index a165f619dc0c..da3e47b95d4d 100644
--- a/nixos/modules/services/cluster/hadoop/default.nix
+++ b/nixos/modules/services/cluster/hadoop/default.nix
@@ -1,5 +1,7 @@
 { config, lib, pkgs, ...}:
-
+let
+  cfg = config.services.hadoop;
+in
 with lib;
 {
   imports = [ ./yarn.nix ./hdfs.nix ];
@@ -17,7 +19,9 @@ with lib;
     };
 
     hdfsSite = mkOption {
-      default = {};
+      default = {
+        "dfs.namenode.rpc-bind-host" = "0.0.0.0";
+      };
       type = types.attrsOf types.anything;
       example = literalExpression ''
         {
@@ -28,27 +32,81 @@ with lib;
     };
 
     mapredSite = mkOption {
-      default = {};
+      default = {
+        "mapreduce.framework.name" = "yarn";
+        "yarn.app.mapreduce.am.env" = "HADOOP_MAPRED_HOME=${cfg.package}/lib/${cfg.package.untarDir}";
+        "mapreduce.map.env" = "HADOOP_MAPRED_HOME=${cfg.package}/lib/${cfg.package.untarDir}";
+        "mapreduce.reduce.env" = "HADOOP_MAPRED_HOME=${cfg.package}/lib/${cfg.package.untarDir}";
+      };
       type = types.attrsOf types.anything;
       example = literalExpression ''
-        {
-          "mapreduce.map.cpu.vcores" = "1";
+        options.services.hadoop.mapredSite.default // {
+          "mapreduce.map.java.opts" = "-Xmx900m -XX:+UseParallelGC";
         }
       '';
       description = "Hadoop mapred-site.xml definition";
     };
 
     yarnSite = mkOption {
-      default = {};
+      default = {
+        "yarn.nodemanager.admin-env" = "PATH=$PATH";
+        "yarn.nodemanager.aux-services" = "mapreduce_shuffle";
+        "yarn.nodemanager.aux-services.mapreduce_shuffle.class" = "org.apache.hadoop.mapred.ShuffleHandler";
+        "yarn.nodemanager.bind-host" = "0.0.0.0";
+        "yarn.nodemanager.container-executor.class" = "org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor";
+        "yarn.nodemanager.env-whitelist" = "JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,LANG,TZ";
+        "yarn.nodemanager.linux-container-executor.group" = "hadoop";
+        "yarn.nodemanager.linux-container-executor.path" = "/run/wrappers/yarn-nodemanager/bin/container-executor";
+        "yarn.nodemanager.log-dirs" = "/var/log/hadoop/yarn/nodemanager";
+        "yarn.resourcemanager.bind-host" = "0.0.0.0";
+        "yarn.resourcemanager.scheduler.class" = "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler";
+      };
       type = types.attrsOf types.anything;
       example = literalExpression ''
-        {
-          "yarn.resourcemanager.ha.id" = "resourcemanager1";
+        options.services.hadoop.yarnSite.default // {
+          "yarn.resourcemanager.hostname" = "''${config.networking.hostName}";
         }
       '';
       description = "Hadoop yarn-site.xml definition";
     };
 
+    log4jProperties = mkOption {
+      default = "${cfg.package}/lib/${cfg.package.untarDir}/etc/hadoop/log4j.properties";
+      type = types.path;
+      example = literalExpression ''
+        "''${pkgs.hadoop}/lib/''${pkgs.hadoop.untarDir}/etc/hadoop/log4j.properties";
+      '';
+      description = "log4j.properties file added to HADOOP_CONF_DIR";
+    };
+
+    containerExecutorCfg = mkOption {
+      default = {
+        # must be the same as yarn.nodemanager.linux-container-executor.group in yarnSite
+        "yarn.nodemanager.linux-container-executor.group"="hadoop";
+        "min.user.id"=1000;
+        "feature.terminal.enabled"=1;
+      };
+      type = types.attrsOf types.anything;
+      example = literalExpression ''
+        options.services.hadoop.containerExecutorCfg.default // {
+          "feature.terminal.enabled" = 0;
+        }
+      '';
+      description = "Yarn container-executor.cfg definition";
+    };
+
+    extraConfDirs = mkOption {
+      default = [];
+      type = types.listOf types.path;
+      example = literalExpression ''
+        [
+          ./extraHDFSConfs
+          ./extraYARNConfs
+        ]
+      '';
+      description = "Directories containing additional config files to be added to HADOOP_CONF_DIR";
+    };
+
     package = mkOption {
       type = types.package;
       default = pkgs.hadoop;
@@ -64,6 +122,12 @@ with lib;
       users.groups.hadoop = {
         gid = config.ids.gids.hadoop;
       };
+      environment = {
+        systemPackages = [ cfg.package ];
+        etc."hadoop-conf".source = let
+          hadoopConf = "${import ./conf.nix { inherit cfg pkgs lib; }}/";
+        in "${hadoopConf}";
+      };
     })
 
   ];
diff --git a/nixos/modules/services/cluster/hadoop/hdfs.nix b/nixos/modules/services/cluster/hadoop/hdfs.nix
index 4f4b0a92108f..e347b682b902 100644
--- a/nixos/modules/services/cluster/hadoop/hdfs.nix
+++ b/nixos/modules/services/cluster/hadoop/hdfs.nix
@@ -1,24 +1,54 @@
 { config, lib, pkgs, ...}:
+with lib;
 let
   cfg = config.services.hadoop;
-  hadoopConf = import ./conf.nix { hadoop = cfg; pkgs = pkgs; };
+  hadoopConf = "${import ./conf.nix { inherit cfg pkgs lib; }}/";
+  restartIfChanged  = mkOption {
+    type = types.bool;
+    description = ''
+      Automatically restart the service on config change.
+      This can be set to false to defer restarts on clusters running critical applications.
+      Please consider the security implications of inadvertently running an older version,
+      and the possibility of unexpected behavior caused by inconsistent versions across a cluster when disabling this option.
+    '';
+    default = false;
+  };
 in
-with lib;
 {
   options.services.hadoop.hdfs = {
-    namenode.enabled = mkOption {
-      type = types.bool;
-      default = false;
-      description = ''
-        Whether to run the Hadoop YARN NameNode
-      '';
+    namenode = {
+      enabled = mkOption {
+        type = types.bool;
+        default = false;
+        description = ''
+          Whether to run the HDFS NameNode
+        '';
+      };
+      inherit restartIfChanged;
+      openFirewall = mkOption {
+        type = types.bool;
+        default = true;
+        description = ''
+          Open firewall ports for namenode
+        '';
+      };
     };
-    datanode.enabled = mkOption {
-      type = types.bool;
-      default = false;
-      description = ''
-        Whether to run the Hadoop YARN DataNode
-      '';
+    datanode = {
+      enabled = mkOption {
+        type = types.bool;
+        default = false;
+        description = ''
+          Whether to run the HDFS DataNode
+        '';
+      };
+      inherit restartIfChanged;
+      openFirewall = mkOption {
+        type = types.bool;
+        default = true;
+        description = ''
+          Open firewall ports for datanode
+        '';
+      };
     };
   };
 
@@ -27,10 +57,7 @@ with lib;
       systemd.services.hdfs-namenode = {
         description = "Hadoop HDFS NameNode";
         wantedBy = [ "multi-user.target" ];
-
-        environment = {
-          HADOOP_HOME = "${cfg.package}";
-        };
+        inherit (cfg.hdfs.namenode) restartIfChanged;
 
         preStart = ''
           ${cfg.package}/bin/hdfs --config ${hadoopConf} namenode -format -nonInteractive || true
@@ -40,24 +67,34 @@ with lib;
           User = "hdfs";
           SyslogIdentifier = "hdfs-namenode";
           ExecStart = "${cfg.package}/bin/hdfs --config ${hadoopConf} namenode";
+          Restart = "always";
         };
       };
+
+      networking.firewall.allowedTCPPorts = (mkIf cfg.hdfs.namenode.openFirewall [
+        9870 # namenode.http-address
+        8020 # namenode.rpc-address
+      ]);
     })
     (mkIf cfg.hdfs.datanode.enabled {
       systemd.services.hdfs-datanode = {
         description = "Hadoop HDFS DataNode";
         wantedBy = [ "multi-user.target" ];
-
-        environment = {
-          HADOOP_HOME = "${cfg.package}";
-        };
+        inherit (cfg.hdfs.datanode) restartIfChanged;
 
         serviceConfig = {
           User = "hdfs";
           SyslogIdentifier = "hdfs-datanode";
           ExecStart = "${cfg.package}/bin/hdfs --config ${hadoopConf} datanode";
+          Restart = "always";
         };
       };
+
+      networking.firewall.allowedTCPPorts = (mkIf cfg.hdfs.datanode.openFirewall [
+        9864 # datanode.http.address
+        9866 # datanode.address
+        9867 # datanode.ipc.address
+      ]);
     })
     (mkIf (
         cfg.hdfs.namenode.enabled || cfg.hdfs.datanode.enabled
diff --git a/nixos/modules/services/cluster/hadoop/yarn.nix b/nixos/modules/services/cluster/hadoop/yarn.nix
index c92020637e47..0086a53e3b74 100644
--- a/nixos/modules/services/cluster/hadoop/yarn.nix
+++ b/nixos/modules/services/cluster/hadoop/yarn.nix
@@ -1,24 +1,62 @@
 { config, lib, pkgs, ...}:
+with lib;
 let
   cfg = config.services.hadoop;
-  hadoopConf = import ./conf.nix { hadoop = cfg; pkgs = pkgs; };
+  hadoopConf = "${import ./conf.nix { inherit cfg pkgs lib; }}/";
+  restartIfChanged  = mkOption {
+    type = types.bool;
+    description = ''
+      Automatically restart the service on config change.
+      This can be set to false to defer restarts on clusters running critical applications.
+      Please consider the security implications of inadvertently running an older version,
+      and the possibility of unexpected behavior caused by inconsistent versions across a cluster when disabling this option.
+    '';
+    default = false;
+  };
 in
-with lib;
 {
   options.services.hadoop.yarn = {
-    resourcemanager.enabled = mkOption {
-      type = types.bool;
-      default = false;
-      description = ''
-        Whether to run the Hadoop YARN ResourceManager
-      '';
+    resourcemanager = {
+      enabled = mkOption {
+        type = types.bool;
+        default = false;
+        description = ''
+          Whether to run the Hadoop YARN ResourceManager
+        '';
+      };
+      inherit restartIfChanged;
+      openFirewall = mkOption {
+        type = types.bool;
+        default = true;
+        description = ''
+          Open firewall ports for resourcemanager
+        '';
+      };
     };
-    nodemanager.enabled = mkOption {
-      type = types.bool;
-      default = false;
-      description = ''
-        Whether to run the Hadoop YARN NodeManager
-      '';
+    nodemanager = {
+      enabled = mkOption {
+        type = types.bool;
+        default = false;
+        description = ''
+          Whether to run the Hadoop YARN NodeManager
+        '';
+      };
+      inherit restartIfChanged;
+      addBinBash = mkOption {
+        type = types.bool;
+        default = true;
+        description = ''
+          Add /bin/bash. This is needed by the linux container executor's launch script.
+        '';
+      };
+      openFirewall = mkOption {
+        type = types.bool;
+        default = true;
+        description = ''
+          Open firewall ports for nodemanager.
+          Because containers can listen on any ephemeral port, TCP ports 1024–65535 will be opened.
+        '';
+      };
     };
   };
 
@@ -38,36 +76,63 @@ with lib;
       systemd.services.yarn-resourcemanager = {
         description = "Hadoop YARN ResourceManager";
         wantedBy = [ "multi-user.target" ];
-
-        environment = {
-          HADOOP_HOME = "${cfg.package}";
-        };
+        inherit (cfg.yarn.resourcemanager) restartIfChanged;
 
         serviceConfig = {
           User = "yarn";
           SyslogIdentifier = "yarn-resourcemanager";
           ExecStart = "${cfg.package}/bin/yarn --config ${hadoopConf} " +
                       " resourcemanager";
+          Restart = "always";
         };
       };
+      networking.firewall.allowedTCPPorts = (mkIf cfg.yarn.resourcemanager.openFirewall [
+        8088 # resourcemanager.webapp.address
+        8030 # resourcemanager.scheduler.address
+        8031 # resourcemanager.resource-tracker.address
+        8032 # resourcemanager.address
+      ]);
     })
 
     (mkIf cfg.yarn.nodemanager.enabled {
+      # Needed because yarn hardcodes /bin/bash in container start scripts
+      # These scripts can't be patched, they are generated at runtime
+      systemd.tmpfiles.rules = [
+        (mkIf cfg.yarn.nodemanager.addBinBash "L /bin/bash - - - - /run/current-system/sw/bin/bash")
+      ];
+
       systemd.services.yarn-nodemanager = {
         description = "Hadoop YARN NodeManager";
         wantedBy = [ "multi-user.target" ];
+        inherit (cfg.yarn.nodemanager) restartIfChanged;
 
-        environment = {
-          HADOOP_HOME = "${cfg.package}";
-        };
+        preStart = ''
+          # create log dir
+          mkdir -p /var/log/hadoop/yarn/nodemanager
+          chown yarn:hadoop /var/log/hadoop/yarn/nodemanager
+
+          # set up setuid container executor binary
+          rm -rf /run/wrappers/yarn-nodemanager/ || true
+          mkdir -p /run/wrappers/yarn-nodemanager/{bin,etc/hadoop}
+          cp ${cfg.package}/lib/${cfg.package.untarDir}/bin/container-executor /run/wrappers/yarn-nodemanager/bin/
+          chgrp hadoop /run/wrappers/yarn-nodemanager/bin/container-executor
+          chmod 6050 /run/wrappers/yarn-nodemanager/bin/container-executor
+          cp ${hadoopConf}/container-executor.cfg /run/wrappers/yarn-nodemanager/etc/hadoop/
+        '';
 
         serviceConfig = {
           User = "yarn";
           SyslogIdentifier = "yarn-nodemanager";
+          PermissionsStartOnly = true;
           ExecStart = "${cfg.package}/bin/yarn --config ${hadoopConf} " +
                       " nodemanager";
+          Restart = "always";
         };
       };
+
+      networking.firewall.allowedTCPPortRanges = [
+        (mkIf (cfg.yarn.nodemanager.openFirewall) {from = 1024; to = 65535;})
+      ];
     })
 
   ];
diff --git a/nixos/modules/services/x11/desktop-managers/plasma5.nix b/nixos/modules/services/x11/desktop-managers/plasma5.nix
index e67c71d5a704..e8a62ea7576a 100644
--- a/nixos/modules/services/x11/desktop-managers/plasma5.nix
+++ b/nixos/modules/services/x11/desktop-managers/plasma5.nix
@@ -346,7 +346,7 @@ in
         ++ lib.optional config.services.pipewire.pulse.enable plasma-pa
         ++ lib.optional config.powerManagement.enable powerdevil
         ++ lib.optional config.services.colord.enable pkgs.colord-kde
-        ++ lib.optional config.services.hardware.bolt.enable pkgs.plasma-thunderbolt
+        ++ lib.optional config.services.hardware.bolt.enable pkgs.plasma5Packages.plasma-thunderbolt
         ++ lib.optionals config.services.samba.enable [ kdenetwork-filesharing pkgs.samba ]
         ++ lib.optional config.services.xserver.wacom.enable pkgs.wacomtablet;
 
diff --git a/nixos/modules/system/boot/systemd.nix b/nixos/modules/system/boot/systemd.nix
index 93ea77d1ee72..77997ac76863 100644
--- a/nixos/modules/system/boot/systemd.nix
+++ b/nixos/modules/system/boot/systemd.nix
@@ -26,6 +26,8 @@ let
       "nss-user-lookup.target"
       "time-sync.target"
       "cryptsetup.target"
+      "cryptsetup-pre.target"
+      "remote-cryptsetup.target"
       "sigpwr.target"
       "timers.target"
       "paths.target"
diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix
index 7fd1b3ec2efc..2b276851644d 100644
--- a/nixos/tests/all-tests.nix
+++ b/nixos/tests/all-tests.nix
@@ -165,6 +165,7 @@ in
   grocy = handleTest ./grocy.nix {};
   grub = handleTest ./grub.nix {};
   gvisor = handleTest ./gvisor.nix {};
+  hadoop.all = handleTestOn [ "x86_64-linux" ] ./hadoop/hadoop.nix {};
   hadoop.hdfs = handleTestOn [ "x86_64-linux" ] ./hadoop/hdfs.nix {};
   hadoop.yarn = handleTestOn [ "x86_64-linux" ] ./hadoop/yarn.nix {};
   handbrake = handleTestOn ["x86_64-linux"] ./handbrake.nix {};
@@ -416,6 +417,7 @@ in
   solr = handleTest ./solr.nix {};
   sonarr = handleTest ./sonarr.nix {};
   spacecookie = handleTest ./spacecookie.nix {};
+  spark = handleTestOn ["x86_64-linux"] ./spark {};
   spike = handleTest ./spike.nix {};
   sslh = handleTest ./sslh.nix {};
   sssd = handleTestOn ["x86_64-linux"] ./sssd.nix {};
diff --git a/nixos/tests/hadoop/hadoop.nix b/nixos/tests/hadoop/hadoop.nix
new file mode 100644
index 000000000000..46dfac26e065
--- /dev/null
+++ b/nixos/tests/hadoop/hadoop.nix
@@ -0,0 +1,70 @@
+import ../make-test-python.nix ({pkgs, ...}: {
+
+  nodes = let
+    package = pkgs.hadoop;
+    coreSite = {
+      "fs.defaultFS" = "hdfs://master";
+    };
+  in {
+    master = {pkgs, options, ...}: {
+      services.hadoop = {
+        inherit package coreSite;
+        hdfs.namenode.enabled = true;
+        yarn.resourcemanager.enabled = true;
+      };
+      virtualisation.memorySize = 1024;
+    };
+
+    worker = {pkgs, options, ...}: {
+      services.hadoop = {
+        inherit package coreSite;
+        hdfs.datanode.enabled = true;
+        yarn.nodemanager.enabled = true;
+        yarnSite = options.services.hadoop.yarnSite.default // {
+          "yarn.resourcemanager.hostname" = "master";
+        };
+      };
+      virtualisation.memorySize = 2048;
+    };
+  };
+
+  testScript = ''
+    start_all()
+
+    master.wait_for_unit("network.target")
+    master.wait_for_unit("hdfs-namenode")
+
+    master.wait_for_open_port(8020)
+    master.wait_for_open_port(9870)
+
+    worker.wait_for_unit("network.target")
+    worker.wait_for_unit("hdfs-datanode")
+    worker.wait_for_open_port(9864)
+    worker.wait_for_open_port(9866)
+    worker.wait_for_open_port(9867)
+
+    master.succeed("curl -f http://worker:9864")
+    worker.succeed("curl -f http://master:9870")
+
+    worker.succeed("sudo -u hdfs hdfs dfsadmin -safemode wait")
+
+    master.wait_for_unit("yarn-resourcemanager")
+
+    master.wait_for_open_port(8030)
+    master.wait_for_open_port(8031)
+    master.wait_for_open_port(8032)
+    master.wait_for_open_port(8088)
+    worker.succeed("curl -f http://master:8088")
+
+    worker.wait_for_unit("yarn-nodemanager")
+    worker.wait_for_open_port(8042)
+    worker.wait_for_open_port(8040)
+    master.succeed("curl -f http://worker:8042")
+
+    assert "Total Nodes:1" in worker.succeed("yarn node -list")
+
+    assert "Estimated value of Pi is" in worker.succeed("HADOOP_USER_NAME=hdfs yarn jar $(readlink $(which yarn) | sed -r 's~bin/yarn~lib/hadoop-*/share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar~g') pi 2 10")
+    assert "SUCCEEDED" in worker.succeed("yarn application -list -appStates FINISHED")
+    worker.succeed("sudo -u hdfs hdfs dfs -ls / | systemd-cat")
+  '';
+ })
diff --git a/nixos/tests/hadoop/hdfs.nix b/nixos/tests/hadoop/hdfs.nix
index f1f98ed42eb3..f5907185c039 100644
--- a/nixos/tests/hadoop/hdfs.nix
+++ b/nixos/tests/hadoop/hdfs.nix
@@ -2,7 +2,7 @@ import ../make-test-python.nix ({...}: {
   nodes = {
     namenode = {pkgs, ...}: {
       services.hadoop = {
-        package = pkgs.hadoop_3_1;
+        package = pkgs.hadoop;
         hdfs.namenode.enabled = true;
         coreSite = {
           "fs.defaultFS" = "hdfs://namenode:8020";
@@ -20,7 +20,7 @@ import ../make-test-python.nix ({...}: {
     };
     datanode = {pkgs, ...}: {
       services.hadoop = {
-        package = pkgs.hadoop_3_1;
+        package = pkgs.hadoop;
         hdfs.datanode.enabled = true;
         coreSite = {
           "fs.defaultFS" = "hdfs://namenode:8020";
diff --git a/nixos/tests/hadoop/yarn.nix b/nixos/tests/hadoop/yarn.nix
index 01077245d397..fbbb293eecd6 100644
--- a/nixos/tests/hadoop/yarn.nix
+++ b/nixos/tests/hadoop/yarn.nix
@@ -1,7 +1,7 @@
 import ../make-test-python.nix ({...}: {
   nodes = {
     resourcemanager = {pkgs, ...}: {
-      services.hadoop.package = pkgs.hadoop_3_1;
+      services.hadoop.package = pkgs.hadoop;
       services.hadoop.yarn.resourcemanager.enabled = true;
       services.hadoop.yarnSite = {
         "yarn.resourcemanager.scheduler.class" = "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler";
@@ -12,7 +12,7 @@ import ../make-test-python.nix ({...}: {
       ];
     };
     nodemanager = {pkgs, ...}: {
-      services.hadoop.package = pkgs.hadoop_3_1;
+      services.hadoop.package = pkgs.hadoop;
       services.hadoop.yarn.nodemanager.enabled = true;
       services.hadoop.yarnSite = {
         "yarn.resourcemanager.hostname" = "resourcemanager";
diff --git a/pkgs/applications/blockchains/btcpayserver/default.nix b/pkgs/applications/blockchains/btcpayserver/default.nix
index a5d9432b143b..5a4c7f4e33dc 100644
--- a/pkgs/applications/blockchains/btcpayserver/default.nix
+++ b/pkgs/applications/blockchains/btcpayserver/default.nix
@@ -1,19 +1,7 @@
-{ lib, stdenv, fetchFromGitHub, fetchurl, linkFarmFromDrvs, makeWrapper,
-  dotnetPackages, dotnetCorePackages, altcoinSupport ? false
-}:
+{ lib, buildDotnetModule, fetchFromGitHub, dotnetCorePackages
+, altcoinSupport ? false }:
 
-let
-  deps = import ./deps.nix {
-    fetchNuGet = { name, version, sha256 }: fetchurl {
-      name = "nuget-${name}-${version}.nupkg";
-      url = "https://www.nuget.org/api/v2/package/${name}/${version}";
-      inherit sha256;
-    };
-  };
-  dotnetSdk = dotnetCorePackages.sdk_3_1;
-in
-
-stdenv.mkDerivation rec {
+buildDotnetModule rec {
   pname = "btcpayserver";
   version = "1.2.4";
 
@@ -24,35 +12,29 @@ stdenv.mkDerivation rec {
     sha256 = "sha256-vjNJ08twsJ036TTFF6srOGshDpP7ZwWCGN0XjrtFT/g=";
   };
 
-  nativeBuildInputs = [ dotnetSdk dotnetPackages.Nuget makeWrapper ];
+  projectFile = "BTCPayServer/BTCPayServer.csproj";
+  nugetDeps = ./deps.nix;
 
-  buildPhase = ''
-    export HOME=$TMP/home
-    export DOTNET_CLI_TELEMETRY_OPTOUT=1
-    export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
+  dotnet-sdk = dotnetCorePackages.sdk_3_1;
+  dotnet-runtime = dotnetCorePackages.aspnetcore_3_1;
 
-    nuget sources Add -Name tmpsrc -Source $TMP/nuget
-    nuget init ${linkFarmFromDrvs "deps" deps} $TMP/nuget
-
-    dotnet restore --source $TMP/nuget ${lib.optionalString altcoinSupport ''/p:Configuration="Altcoins-Release"''} BTCPayServer/BTCPayServer.csproj
-    dotnet publish --no-restore --output $out/share/$pname ${lib.optionalString altcoinSupport "-c Altcoins-Release"} BTCPayServer/BTCPayServer.csproj
-  '';
+  dotnetFlags = lib.optionals altcoinSupport [ "/p:Configuration=Altcoins-Release" ];
 
   # btcpayserver requires the publish directory as its working dir
   # https://github.com/btcpayserver/btcpayserver/issues/1894
-  installPhase = ''
-    makeWrapper $out/share/$pname/BTCPayServer $out/bin/$pname \
-      --set DOTNET_ROOT "${dotnetSdk}" \
-      --run "cd $out/share/$pname"
+  preInstall = ''
+    makeWrapperArgs+=(--run "cd $out/lib/btcpayserver")
   '';
 
-  dontStrip = true;
+  postInstall = ''
+    mv $out/bin/{BTCPayServer,btcpayserver}
+  '';
 
   meta = with lib; {
     description = "Self-hosted, open-source cryptocurrency payment processor";
     homepage = "https://btcpayserver.org";
     maintainers = with maintainers; [ kcalvinalvin earvstedt ];
-    license = lib.licenses.mit;
-    platforms = lib.platforms.linux;
+    license = licenses.mit;
+    platforms = platforms.linux;
   };
 }
diff --git a/pkgs/applications/blockchains/nbxplorer/default.nix b/pkgs/applications/blockchains/nbxplorer/default.nix
index c55965054b7e..2a5f1aec2dc5 100644
--- a/pkgs/applications/blockchains/nbxplorer/default.nix
+++ b/pkgs/applications/blockchains/nbxplorer/default.nix
@@ -1,19 +1,6 @@
-{ lib, stdenv, fetchFromGitHub, fetchurl, linkFarmFromDrvs, makeWrapper,
-  dotnetPackages, dotnetCorePackages
-}:
+{ lib, buildDotnetModule, fetchFromGitHub, dotnetCorePackages }:
 
-let
-  deps = import ./deps.nix {
-    fetchNuGet = { name, version, sha256 }: fetchurl {
-      name = "nuget-${name}-${version}.nupkg";
-      url = "https://www.nuget.org/api/v2/package/${name}/${version}";
-      inherit sha256;
-    };
-  };
-  dotnetSdk = dotnetCorePackages.sdk_3_1;
-in
-
-stdenv.mkDerivation rec {
+buildDotnetModule rec {
   pname = "nbxplorer";
   version = "2.2.11";
 
@@ -24,31 +11,20 @@ stdenv.mkDerivation rec {
     sha256 = "sha256-ZDqzkANGMdvv3e5gWCYcacUYKLJRquXRHLr8RAzT9hY=";
   };
 
-  nativeBuildInputs = [ dotnetSdk dotnetPackages.Nuget makeWrapper ];
+  projectFile = "NBXplorer/NBXplorer.csproj";
+  nugetDeps = ./deps.nix;
 
-  buildPhase = ''
-    export HOME=$TMP/home
-    export DOTNET_CLI_TELEMETRY_OPTOUT=1
-    export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
+  dotnet-sdk = dotnetCorePackages.sdk_3_1;
+  dotnet-runtime = dotnetCorePackages.aspnetcore_3_1;
 
-    nuget sources Add -Name tmpsrc -Source $TMP/nuget
-    nuget init ${linkFarmFromDrvs "deps" deps} $TMP/nuget
-
-    dotnet restore --source $TMP/nuget NBXplorer/NBXplorer.csproj
-    dotnet publish --no-restore --output $out/share/$pname -c Release NBXplorer/NBXplorer.csproj
+  postInstall = ''
+    mv $out/bin/{NBXplorer,nbxplorer}
   '';
 
-  installPhase = ''
-    makeWrapper $out/share/$pname/NBXplorer $out/bin/$pname \
-      --set DOTNET_ROOT "${dotnetSdk}"
-  '';
-
-  dontStrip = true;
-
   meta = with lib; {
     description = "Minimalist UTXO tracker for HD Cryptocurrency Wallets";
     maintainers = with maintainers; [ kcalvinalvin earvstedt ];
-    license = lib.licenses.mit;
-    platforms = lib.platforms.linux;
+    license = licenses.mit;
+    platforms = platforms.linux;
   };
 }
diff --git a/pkgs/applications/networking/cluster/hadoop/default.nix b/pkgs/applications/networking/cluster/hadoop/default.nix
index 589ce21fe87b..0e8e652fbb30 100644
--- a/pkgs/applications/networking/cluster/hadoop/default.nix
+++ b/pkgs/applications/networking/cluster/hadoop/default.nix
@@ -1,180 +1,96 @@
-{ lib, stdenv, fetchurl, makeWrapper, pkg-config, which, maven, cmake, jre, jdk8, bash
-, coreutils, glibc, protobuf2_5, fuse, snappy, zlib, bzip2, openssl, openssl_1_0_2, fetchpatch, libtirpc
+{ lib, stdenv, fetchurl, makeWrapper, autoPatchelfHook
+, jdk8_headless, jdk11_headless
+, bash, coreutils, which
+, bzip2, cyrus_sasl , protobuf3_7, snappy, zlib, zstd
+, openssl
 }:
 
+with lib;
+
 let
-  maven-jdk8 = maven.override {
-    jdk = jdk8;
-  };
-  common = { version, sha256, dependencies-sha256, maven, tomcat, opensslPkg ? openssl }:
-    let
-      # compile the hadoop tarball from sources, it requires some patches
-      binary-distributon = stdenv.mkDerivation rec {
-        name = "hadoop-${version}-bin";
-        src = fetchurl {
-          url = "mirror://apache/hadoop/common/hadoop-${version}/hadoop-${version}-src.tar.gz";
-          inherit sha256;
-        };
-
-        postUnpack = lib.optionalString (tomcat != null) ''
-          install -D ${tomcat.src} $sourceRoot/hadoop-hdfs-project/hadoop-hdfs-httpfs/downloads/apache-tomcat-${tomcat.version}.tar.gz
-          install -D ${tomcat.src} $sourceRoot/hadoop-common-project/hadoop-kms/downloads/apache-tomcat-${tomcat.version}.tar.gz
-        '';
-
-        # perform fake build to make a fixed-output derivation of dependencies downloaded from maven central (~100Mb in ~3000 files)
-        fetched-maven-deps = stdenv.mkDerivation {
-          name = "hadoop-${version}-maven-deps";
-          inherit src postUnpack nativeBuildInputs buildInputs;
-          buildPhase = ''
-            while mvn package -Dmaven.repo.local=$out/.m2 ${mavenFlags} -Dmaven.wagon.rto=5000; [ $? = 1 ]; do
-              echo "timeout, restart maven to continue downloading"
-            done
-          '';
-          # keep only *.{pom,jar,xml,sha1,so,dll,dylib} and delete all ephemeral files with lastModified timestamps inside
-          installPhase = ''find $out/.m2 -type f -regex '.+\(\.lastUpdated\|resolver-status\.properties\|_remote\.repositories\)' -delete'';
-          outputHashAlgo = "sha256";
-          outputHashMode = "recursive";
-          outputHash = dependencies-sha256;
-        };
-
-        nativeBuildInputs = [ maven cmake pkg-config ];
-        buildInputs = [ fuse snappy zlib bzip2 opensslPkg protobuf2_5 libtirpc ];
-        NIX_CFLAGS_COMPILE = [ "-I${libtirpc.dev}/include/tirpc" ];
-        NIX_LDFLAGS = [ "-ltirpc" ];
-
-        # most of the hardcoded pathes are fixed in 2.9.x and 3.0.0, this list of patched files might be reduced when 2.7.x and 2.8.x will be deprecated
-
-        patches = [
-          (fetchpatch {
-            url = "https://patch-diff.githubusercontent.com/raw/apache/hadoop/pull/2886.patch";
-            sha256 = "1fim1d8va050za5i8a6slphmx015fzvhxkc2wi4rwg7kbj31sv0r";
-          })
-        ];
-
-        postPatch = ''
-          for file in hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java \
-                      hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java \
-                      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java \
-                      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java \
-                      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java \
-                      hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java; do
-            if [ -f "$file" ]; then
-              substituteInPlace "$file" \
-                --replace '/usr/bin/stat' 'stat' \
-                --replace '/bin/bash'     'bash' \
-                --replace '/bin/ls'       'ls'   \
-                --replace '/bin/mv'       'mv'
-            fi
-          done
-        '';
-        dontConfigure = true; # do not trigger cmake hook
-        mavenFlags = "-Drequire.snappy -Drequire.bzip2 -DskipTests -Pdist,native -e";
-        buildPhase = ''
-          # 'maven.repo.local' must be writable
-          mvn package --offline -Dmaven.repo.local=$(cp -dpR ${fetched-maven-deps}/.m2 ./ && chmod +w -R .m2 && pwd)/.m2 ${mavenFlags}
-          # remove runtime dependency on $jdk/jre/lib/amd64/server/libjvm.so
-          patchelf --set-rpath ${lib.makeLibraryPath [glibc]} hadoop-dist/target/hadoop-${version}/lib/native/libhadoop.so.1.0.0
-          patchelf --set-rpath ${lib.makeLibraryPath [glibc]} hadoop-dist/target/hadoop-${version}/lib/native/libhdfs.so.0.0.0
-        '';
-        installPhase = "mv hadoop-dist/target/hadoop-${version} $out";
-      };
-    in
-      stdenv.mkDerivation {
-        pname = "hadoop";
-        inherit version;
-
-        src = binary-distributon;
-
-        nativeBuildInputs = [ makeWrapper ];
-
-        installPhase = ''
-          mkdir -p $out/share/doc/hadoop
-          cp -dpR * $out/
-          mv $out/*.txt $out/share/doc/hadoop/
-
-          #
-          # Do not use `wrapProgram` here, script renaming may result to weird things: http://i.imgur.com/0Xee013.png
-          #
-          mkdir -p $out/bin.wrapped
-          for n in $out/bin/*; do
-            if [ -f "$n" ]; then # only regular files
-              mv $n $out/bin.wrapped/
-              makeWrapper $out/bin.wrapped/$(basename $n) $n \
-                --prefix PATH : "${lib.makeBinPath [ which jre bash coreutils ]}" \
-                --prefix JAVA_LIBRARY_PATH : "${lib.makeLibraryPath [ opensslPkg snappy zlib bzip2 ]}" \
-                --set JAVA_HOME "${jre}" \
-                --set HADOOP_PREFIX "$out"
-            fi
-          done
-        '';
-
-        meta = with lib; {
-          homepage = "https://hadoop.apache.org/";
-          description = "Framework for distributed processing of large data sets across clusters of computers";
-          license = licenses.asl20;
-
-          longDescription = ''
-            The Apache Hadoop software library is a framework that allows for
-            the distributed processing of large data sets across clusters of
-            computers using a simple programming model. It is designed to
-            scale up from single servers to thousands of machines, each
-            offering local computation and storage. Rather than rely on
-            hardware to deliver high-avaiability, the library itself is
-            designed to detect and handle failures at the application layer,
-            so delivering a highly-availabile service on top of a cluster of
-            computers, each of which may be prone to failures.
-          '';
-          maintainers = with maintainers; [ volth ];
-          platforms = [ "x86_64-linux" ];
-        };
+  common = { pname, version, untarDir ? "${pname}-${version}", sha256, jdk, openssl, nativeLibs ? [ ], libPatches ? "" }:
+    stdenv.mkDerivation rec {
+      inherit pname version jdk libPatches untarDir openssl;
+      src = fetchurl {
+        url = "mirror://apache/hadoop/common/hadoop-${version}/hadoop-${version}.tar.gz";
+        inherit sha256;
+      };
+
+      nativeBuildInputs = [ makeWrapper ]
+        ++ optional (nativeLibs != [] || libPatches != "") [ autoPatchelfHook ];
+      buildInputs = [ openssl ] ++ nativeLibs;
+
+      installPhase = ''
+        mkdir -p $out/{lib/${untarDir}/conf,bin,lib}
+        mv * $out/lib/${untarDir}
+
+        for n in $(find $out/lib/${untarDir}/bin -type f ! -name "*.*"); do
+          makeWrapper "$n" "$out/bin/$(basename $n)"\
+            --set-default JAVA_HOME ${jdk.home}\
+            --set-default HADOOP_HOME $out/lib/${untarDir}\
+            --set-default HADOOP_CONF_DIR /etc/hadoop-conf/\
+            --prefix PATH : "${makeBinPath [ bash coreutils which]}"\
+            --prefix JAVA_LIBRARY_PATH : "${makeLibraryPath buildInputs}"
+        done
+      '' + libPatches;
+
+      meta = {
+        homepage = "https://hadoop.apache.org/";
+        description = "Framework for distributed processing of large data sets across clusters of computers";
+        license = licenses.asl20;
+
+        longDescription = ''
+          The Apache Hadoop software library is a framework that allows for
+          the distributed processing of large data sets across clusters of
+          computers using a simple programming model. It is designed to
+          scale up from single servers to thousands of machines, each
+          offering local computation and storage. Rather than rely on
+          hardware to deliver high-avaiability, the library itself is
+          designed to detect and handle failures at the application layer,
+          so delivering a highly-availabile service on top of a cluster of
+          computers, each of which may be prone to failures.
+        '';
+        maintainers = with maintainers; [ volth illustris ];
+        platforms = [ "x86_64-linux" ];
       };
 
-  tomcat_6_0_48 = rec {
-    version = "6.0.48";
-    src = fetchurl {
-      # do not use "mirror://apache/" here, tomcat-6 is legacy and has been removed from the mirrors
-      url = "https://archive.apache.org/dist/tomcat/tomcat-6/v${version}/bin/apache-tomcat-${version}.tar.gz";
-      sha256 = "1w4jf28g8p25fmijixw6b02iqlagy2rvr57y3n90hvz341kb0bbc";
     };
+in
+{
+  # Different version of hadoop support different java runtime versions
+  # https://cwiki.apache.org/confluence/display/HADOOP/Hadoop+Java+Versions
+  hadoop_3_3 = common rec {
+    pname = "hadoop";
+    version = "3.3.1";
+    sha256 = "1b3v16ihysqaxw8za1r5jlnphy8dwhivdx2d0z64309w57ihlxxd";
+    untarDir = "${pname}-${version}";
+    jdk = jdk11_headless;
+    inherit openssl;
+    # TODO: Package and add Intel Storage Acceleration Library
+    nativeLibs = [ stdenv.cc.cc.lib protobuf3_7 zlib snappy ];
+    libPatches = ''
+      ln -s ${getLib cyrus_sasl}/lib/libsasl2.so $out/lib/${untarDir}/lib/native/libsasl2.so.2
+      ln -s ${getLib openssl}/lib/libcrypto.so $out/lib/${untarDir}/lib/native/
+      ln -s ${getLib zlib}/lib/libz.so.1 $out/lib/${untarDir}/lib/native/
+      ln -s ${getLib zstd}/lib/libzstd.so.1 $out/lib/${untarDir}/lib/native/
+      ln -s ${getLib bzip2}/lib/libbz2.so.1 $out/lib/${untarDir}/lib/native/
+      patchelf --add-rpath ${jdk.home}/lib/server $out/lib/${untarDir}/lib/native/libnativetask.so.1.0.0
+    '';
   };
-
-in {
-  hadoop_2_7 = common {
-    version = "2.7.7";
-    sha256 = "1ahv67f3lwak3kbjvnk1gncq56z6dksbajj872iqd0awdsj3p5rf";
-    dependencies-sha256 = "1lsr9nvrynzspxqcamb10d596zlnmnfpxhkd884gdiva0frm0b1r";
-    tomcat = tomcat_6_0_48;
-    opensslPkg = openssl_1_0_2;
-    maven = maven-jdk8;
+  hadoop_3_2 = common rec {
+    pname = "hadoop";
+    version = "3.2.2";
+    sha256 = "1hxq297cqvkfgz2yfdiwa3l28g44i2abv5921k2d6b4pqd33prwp";
+    jdk = jdk8_headless;
+    # not using native libs because of broken openssl_1_0_2 dependency
+    # can be manually overriden
+    openssl = null;
   };
-  hadoop_2_8 = common {
-    version = "2.8.4";
-    sha256 = "16c3ljhrzibkjn3y1bmjxdgf0kn60l23ay5hqpp7vpbnqx52x68w";
-    dependencies-sha256 = "1j4f461487fydgr5978nnm245ksv4xbvskfr8pbmfhcyss6b7w03";
-    tomcat = tomcat_6_0_48;
-    opensslPkg = openssl_1_0_2;
-    maven = maven-jdk8;
-  };
-  hadoop_2_9 = common {
-    version = "2.9.1";
-    sha256 = "0qgmpfbpv7f521fkjy5ldzdb4lwiblhs0hyl8qy041ws17y5x7d7";
-    dependencies-sha256 = "1d5i8jj5y746rrqb9lscycnd7acmxlkz64ydsiyqsh5cdqgy2x7x";
-    tomcat = tomcat_6_0_48;
-    opensslPkg = openssl_1_0_2;
-    maven = maven-jdk8;
-  };
-  hadoop_3_0 = common {
-    version = "3.0.3";
-    sha256 = "1vvkci0kx4b48dg0niifn2d3r4wwq8pb3c5z20wy8pqsqrqhlci5";
-    dependencies-sha256 = "1kzkna9ywacm2m1cirj9cyip66bgqjhid2xf9rrhq6g10lhr8j9m";
-    tomcat = null;
-    maven = maven-jdk8;
-  };
-  hadoop_3_1 = common {
-    version = "3.1.1";
-    sha256 = "04hhdbyd4x1hy0fpy537f8mi0864hww97zap29x7dk1smrffwabd";
-    dependencies-sha256 = "1q63jsxg3d31x0p8hvhpvbly2b07almyzsbhwphbczl3fhlqgiwn";
-    tomcat = null;
-    maven = maven-jdk8;
+  hadoop2 = common rec {
+    pname = "hadoop";
+    version = "2.10.1";
+    sha256 = "1w31x4bk9f2swnx8qxx0cgwfg8vbpm6cy5lvfnbbpl3rsjhmyg97";
+    jdk = jdk8_headless;
+    openssl = null;
   };
 }
diff --git a/pkgs/applications/version-management/git-and-tools/git/default.nix b/pkgs/applications/version-management/git-and-tools/git/default.nix
index bd6f347792ad..521e42e845f2 100644
--- a/pkgs/applications/version-management/git-and-tools/git/default.nix
+++ b/pkgs/applications/version-management/git-and-tools/git/default.nix
@@ -338,6 +338,8 @@ stdenv.mkDerivation {
     disable_test t6300-for-each-ref
     #===(   22665;1651  9/?  1/?  0/?  0/? )= =/private/tmp/nix-build-git-2.33.0.drv-2/git-2.33.0/t/../contrib/completion/git-completion.bash: line 405: compgen: command not found
     disable_test t9902-completion
+    # not ok 1 - populate workdir (with 2.33.1 on x86_64-darwin)
+    disable_test t5003-archive-zip
   '' + lib.optionalString stdenv.hostPlatform.isMusl ''
     # Test fails (as of 2.17.0, musl 1.1.19)
     disable_test t3900-i18n-commit
diff --git a/pkgs/build-support/build-dotnet-module/default.nix b/pkgs/build-support/build-dotnet-module/default.nix
index 0161c101e5bf..3701f254d0ca 100644
--- a/pkgs/build-support/build-dotnet-module/default.nix
+++ b/pkgs/build-support/build-dotnet-module/default.nix
@@ -51,6 +51,8 @@ let
   });
 
   package = stdenv.mkDerivation (args // {
+    inherit buildType;
+
     nativeBuildInputs = args.nativeBuildInputs or [] ++ [ dotnet-sdk dotnetPackages.Nuget cacert makeWrapper ];
 
     # Stripping breaks the executable
@@ -71,7 +73,7 @@ let
       mkdir -p $HOME/.nuget/NuGet
       cp $HOME/.config/NuGet/NuGet.Config $HOME/.nuget/NuGet
 
-      dotnet restore ${lib.escapeShellArg projectFile} \
+      dotnet restore "$projectFile" \
         ${lib.optionalString (!enableParallelBuilding) "--disable-parallel"} \
         -p:ContinuousIntegrationBuild=true \
         -p:Deterministic=true \
@@ -85,13 +87,13 @@ let
     buildPhase = args.buildPhase or ''
       runHook preBuild
 
-      dotnet build ${lib.escapeShellArg projectFile} \
+      dotnet build "$projectFile" \
         -maxcpucount:${if enableParallelBuilding then "$NIX_BUILD_CORES" else "1"} \
         -p:BuildInParallel=${if enableParallelBuilding then "true" else "false"} \
         -p:ContinuousIntegrationBuild=true \
         -p:Deterministic=true \
         -p:Version=${args.version} \
-        --configuration ${buildType} \
+        --configuration "$buildType" \
         --no-restore \
         "''${dotnetBuildFlags[@]}"  \
         "''${dotnetFlags[@]}"
@@ -102,17 +104,17 @@ let
     installPhase = args.installPhase or ''
       runHook preInstall
 
-      dotnet publish ${lib.escapeShellArg projectFile} \
+      dotnet publish "$projectFile" \
         -p:ContinuousIntegrationBuild=true \
         -p:Deterministic=true \
         --output $out/lib/${args.pname} \
-        --configuration ${buildType} \
+        --configuration "$buildType" \
         --no-build \
         --no-self-contained \
         "''${dotnetInstallFlags[@]}"  \
         "''${dotnetFlags[@]}"
     '' + (if executables != null then ''
-      for executable in ''${executables}; do
+      for executable in $executables; do
         execPath="$out/lib/${args.pname}/$executable"
 
         if [[ -f "$execPath" && -x "$execPath" ]]; then
@@ -120,7 +122,7 @@ let
             --set DOTNET_ROOT "${dotnet-runtime}" \
             --suffix LD_LIBRARY_PATH : "${lib.makeLibraryPath runtimeDeps}" \
             "''${gappsWrapperArgs[@]}" \
-            ''${makeWrapperArgs}
+            "''${makeWrapperArgs[@]}"
         else
           echo "Specified binary \"$executable\" is either not an executable, or does not exist!"
           exit 1
@@ -133,7 +135,7 @@ let
             --set DOTNET_ROOT "${dotnet-runtime}" \
             --suffix LD_LIBRARY_PATH : "${lib.makeLibraryPath runtimeDeps}" \
             "''${gappsWrapperArgs[@]}" \
-            ''${makeWrapperArgs}
+            "''${makeWrapperArgs[@]}"
         fi
       done
     '') + ''
diff --git a/pkgs/build-support/docker/test-dummy/hello.txt b/pkgs/build-support/docker/test-dummy/hello.txt
new file mode 100644
index 000000000000..495cc9fa8f9c
--- /dev/null
+++ b/pkgs/build-support/docker/test-dummy/hello.txt
@@ -0,0 +1 @@
+Hello there!
diff --git a/pkgs/build-support/fetchpatch/tests.nix b/pkgs/build-support/fetchpatch/tests.nix
new file mode 100644
index 000000000000..4240b325d656
--- /dev/null
+++ b/pkgs/build-support/fetchpatch/tests.nix
@@ -0,0 +1,8 @@
+{ invalidateFetcherByDrvHash, fetchpatch, ... }:
+
+{
+  simple = invalidateFetcherByDrvHash fetchpatch {
+    url = "https://github.com/facebook/zstd/pull/2724/commits/e1f85dbca3a0ed5ef06c8396912a0914db8dea6a.patch";
+    sha256 = "sha256-PuYAqnJWAE+L9bsroOnnBGJhERW8LHrGSLtIEkKU9vg=";
+  };
+}
diff --git a/pkgs/data/documentation/linux-manual/default.nix b/pkgs/data/documentation/linux-manual/default.nix
new file mode 100644
index 000000000000..27690f9b99cd
--- /dev/null
+++ b/pkgs/data/documentation/linux-manual/default.nix
@@ -0,0 +1,40 @@
+{ lib, stdenv, fetchurl, perl, linuxPackages_latest }:
+
+stdenv.mkDerivation rec {
+  pname = "linux-manual";
+  inherit (linuxPackages_latest.kernel) version src;
+
+  nativeBuildInputs = [ perl ];
+
+  dontConfigure = true;
+  dontBuild = true;
+
+  postPatch = ''
+    patchShebangs --build \
+      scripts/kernel-doc \
+      scripts/split-man.pl
+  '';
+
+  installPhase = ''
+    mandir=$out/share/man/man9
+    mkdir -p $mandir
+
+    KBUILD_BUILD_TIMESTAMP=$(stat -c %Y Makefile) \
+    grep -F -l -Z \
+      --exclude-dir Documentation \
+      --exclude-dir tools \
+      -R '/**' \
+      | xargs -0 -n 256 -P $NIX_BUILD_CORES \
+        $SHELL -c '{ scripts/kernel-doc -man "$@" || :; } \
+          | scripts/split-man.pl '$mandir kernel-doc
+
+    test -f $mandir/kmalloc.9
+  '';
+
+  meta = with lib; {
+    homepage = "https://kernel.org/";
+    description = "Linux kernel API manual pages";
+    license = licenses.gpl2Only;
+    maintainers = with maintainers; [ mvs ];
+  };
+}
diff --git a/pkgs/development/ocaml-modules/minisat/default.nix b/pkgs/development/ocaml-modules/minisat/default.nix
index f9d37d6b4faf..433431d8f9c0 100644
--- a/pkgs/development/ocaml-modules/minisat/default.nix
+++ b/pkgs/development/ocaml-modules/minisat/default.nix
@@ -2,7 +2,7 @@
 
 buildDunePackage rec {
   pname = "minisat";
-  version = "0.3";
+  version = "0.4";
 
   useDune2 = true;
 
@@ -12,7 +12,7 @@ buildDunePackage rec {
     owner  = "c-cube";
     repo   = "ocaml-minisat";
     rev    = "v${version}";
-    sha256 = "01wggbziqz5x6d7mwdl40sbf6qal7fd853b224zjf9n0kzzsnczh";
+    sha256 = "009jncrvnl9synxx6jnm6gp0cs7zlj71z22zz7bs1750b0jrfm2r";
   };
 
   meta = {
diff --git a/pkgs/development/python-modules/aiomusiccast/default.nix b/pkgs/development/python-modules/aiomusiccast/default.nix
index 55b5d4215135..e90e9ea11fd9 100644
--- a/pkgs/development/python-modules/aiomusiccast/default.nix
+++ b/pkgs/development/python-modules/aiomusiccast/default.nix
@@ -8,7 +8,7 @@
 
 buildPythonPackage rec {
   pname = "aiomusiccast";
-  version = "0.11.0";
+  version = "0.12.0";
 
   format = "pyproject";
 
@@ -18,7 +18,7 @@ buildPythonPackage rec {
     owner = "vigonotion";
     repo = "aiomusiccast";
     rev = version;
-    sha256 = "sha256-58l9bmGmSRFHqiKwHtGcAu2H+JnKjsZZg+fxbLW5xBg=";
+    sha256 = "sha256-LZaRxpjTNkrf8JAWTQLp5LcWOf+epXOnvidJl2tEEE4=";
   };
 
   nativeBuildInputs = [
@@ -32,7 +32,9 @@ buildPythonPackage rec {
   # upstream has no tests
   doCheck = false;
 
-  pythonImportsCheck = [ "aiomusiccast" ];
+  pythonImportsCheck = [
+    "aiomusiccast"
+  ];
 
   meta = with lib; {
     description = "Companion library for musiccast devices intended for the Home Assistant integration";
diff --git a/pkgs/development/python-modules/black/default.nix b/pkgs/development/python-modules/black/default.nix
index ece5707ae676..eef639fad1c6 100644
--- a/pkgs/development/python-modules/black/default.nix
+++ b/pkgs/development/python-modules/black/default.nix
@@ -80,6 +80,6 @@ buildPythonPackage rec {
     homepage = "https://github.com/psf/black";
     changelog = "https://github.com/psf/black/blob/${version}/CHANGES.md";
     license = licenses.mit;
-    maintainers = with maintainers; [ sveitser ];
+    maintainers = with maintainers; [ sveitser autophagy ];
   };
 }
diff --git a/pkgs/development/tools/misc/kibana/7.x.nix b/pkgs/development/tools/misc/kibana/7.x.nix
index 33298f68334a..a2b6f740db4c 100644
--- a/pkgs/development/tools/misc/kibana/7.x.nix
+++ b/pkgs/development/tools/misc/kibana/7.x.nix
@@ -4,14 +4,14 @@
 , stdenv
 , makeWrapper
 , fetchurl
-, nodejs-14_x
+, nodejs-10_x
 , coreutils
 , which
 }:
 
 with lib;
 let
-  nodejs = nodejs-14_x;
+  nodejs = nodejs-10_x;
   inherit (builtins) elemAt;
   info = splitString "-" stdenv.hostPlatform.system;
   arch = elemAt info 0;
diff --git a/pkgs/tools/misc/steampipe/default.nix b/pkgs/tools/misc/steampipe/default.nix
index 691351c89b08..e7d20bd8fc25 100644
--- a/pkgs/tools/misc/steampipe/default.nix
+++ b/pkgs/tools/misc/steampipe/default.nix
@@ -2,16 +2,16 @@
 
 buildGoModule rec {
   pname = "steampipe";
-  version = "0.8.5";
+  version = "0.9.0";
 
   src = fetchFromGitHub {
     owner = "turbot";
     repo = "steampipe";
     rev = "v${version}";
-    sha256 = "sha256-3vetSUJwCeaBzKj+635siskfcDPs/kkgCH954cg/REA=";
+    sha256 = "sha256-wG5KvyY40CNxIScuQHQdJ4u8fzNU+oV7iNe9VAvTQMg=";
   };
 
-  vendorSha256 = "sha256-TGDFNHWWbEy1cD7b2yPqAN7rRrLvL0ZX/R3BWGRWjjw=";
+  vendorSha256 = "sha256-3JBCiF1gxGCVn81s7abGvNIAy+eP7orAnSBOXUNImao=";
 
   # tests are failing for no obvious reasons
   doCheck = false;
diff --git a/pkgs/tools/text/patchutils/0.4.2.nix b/pkgs/tools/text/patchutils/0.4.2.nix
new file mode 100644
index 000000000000..1b2c0d6b1f3e
--- /dev/null
+++ b/pkgs/tools/text/patchutils/0.4.2.nix
@@ -0,0 +1,8 @@
+{ callPackage, python3, ... } @ args:
+
+callPackage ./generic.nix (args // {
+  version = "0.4.2";
+  sha256 = "sha256-iHWwll/jPeYriQ9s15O+f6/kGk5VLtv2QfH+1eu/Re0=";
+  # for gitdiff
+  extraBuildInputs = [ python3 ];
+})
diff --git a/pkgs/tools/text/patchutils/generic.nix b/pkgs/tools/text/patchutils/generic.nix
index d3e9da242e2e..923dd06d1826 100644
--- a/pkgs/tools/text/patchutils/generic.nix
+++ b/pkgs/tools/text/patchutils/generic.nix
@@ -1,5 +1,5 @@
-{ lib, stdenv, fetchurl, perl
-, version, sha256, patches ? []
+{ lib, stdenv, fetchurl, perl, makeWrapper
+, version, sha256, patches ? [], extraBuildInputs ? []
 , ...
 }:
 stdenv.mkDerivation rec {
@@ -11,18 +11,32 @@ stdenv.mkDerivation rec {
     inherit sha256;
   };
 
-  buildInputs = [ perl ];
+  nativeBuildInputs = [ makeWrapper ];
+  buildInputs = [ perl ] ++ extraBuildInputs;
   hardeningDisable = [ "format" ];
 
-  doCheck = false; # fails
+  postInstall = ''
+    for bin in $out/bin/{splitdiff,rediff,editdiff,dehtmldiff}; do
+      wrapProgram "$bin" \
+        --prefix PATH : "$out/bin"
+    done
+  '';
+
+  doCheck = lib.versionAtLeast version "0.3.4";
+
+  preCheck = ''
+    patchShebangs tests
+    chmod +x scripts/*
+  '' + lib.optionalString (lib.versionOlder version "0.4.2") ''
+    find tests -type f -name 'run-test' \
+      -exec sed -i '{}' -e 's|/bin/echo|echo|g' \;
+  '';
 
   meta = with lib; {
     description = "Tools to manipulate patch files";
     homepage = "http://cyberelk.net/tim/software/patchutils";
     license = licenses.gpl2Plus;
     platforms = platforms.all;
-    executables = [ "combinediff" "dehtmldiff" "editdiff" "espdiff"
-      "filterdiff" "fixcvsdiff" "flipdiff" "grepdiff" "interdiff" "lsdiff"
-      "recountdiff" "rediff" "splitdiff" "unwrapdiff" ];
+    maintainers = with maintainers; [ artturin ];
   };
 }
diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix
index 3c8e43c89240..7499b1ea3f9d 100644
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@@ -509,7 +509,10 @@ with pkgs;
 
   mht2htm = callPackage ../tools/misc/mht2htm { };
 
-  fetchpatch = callPackage ../build-support/fetchpatch { };
+  fetchpatch = callPackage ../build-support/fetchpatch { }
+    // {
+      tests = callPackages ../build-support/fetchpatch/tests.nix { };
+    };
 
   fetchs3 = callPackage ../build-support/fetchs3 { };
 
@@ -8298,6 +8301,8 @@ with pkgs;
 
   patchutils_0_3_3 = callPackage ../tools/text/patchutils/0.3.3.nix { };
 
+  patchutils_0_4_2 = callPackage ../tools/text/patchutils/0.4.2.nix { };
+
   parted = callPackage ../tools/misc/parted { };
 
   passh = callPackage ../tools/networking/passh { };
@@ -13178,15 +13183,12 @@ with pkgs;
 
   groovy = callPackage ../development/interpreters/groovy { };
 
-  inherit (callPackages ../applications/networking/cluster/hadoop {
-    jre = jre8; # TODO: remove override https://github.com/NixOS/nixpkgs/pull/89731
-  })
-    hadoop_2_7
-    hadoop_2_8
-    hadoop_2_9
-    hadoop_3_0
-    hadoop_3_1;
-  hadoop = hadoop_2_7;
+  inherit (callPackages ../applications/networking/cluster/hadoop { })
+    hadoop_3_3
+    hadoop_3_2
+    hadoop2;
+  hadoop3 = hadoop_3_3;
+  hadoop = hadoop3;
 
   io = callPackage ../development/interpreters/io { };
 
@@ -13540,7 +13542,7 @@ with pkgs;
 
   self = pkgsi686Linux.callPackage ../development/interpreters/self { };
 
-  inherit (callPackages ../applications/networking/cluster/spark { hadoop = hadoop_3_1; })
+  inherit (callPackages ../applications/networking/cluster/spark { })
     spark3
     spark2;
   spark = spark3;
@@ -23067,6 +23069,8 @@ with pkgs;
 
   line-awesome = callPackage ../data/fonts/line-awesome { };
 
+  linux-manual = callPackage ../data/documentation/linux-manual { };
+
   lmmath = callPackage ../data/fonts/lmmath {};
 
   lmodern = callPackage ../data/fonts/lmodern { };