From 39c007ce9cc3f46db70bde794323b6cbfccfe62a Mon Sep 17 00:00:00 2001
From: illustris <rharikrishnan95@gmail.com>
Date: Tue, 2 Nov 2021 11:59:58 +0530
Subject: [PATCH 1/8] nixos/hadoop: Add HA capabilities

- Add HDFS journalnode and ZKFC services
- Test failover of HDFS and YARN master services in full hadoop test
- Check if a minimal HDFS cluster works in the minimal HDFS test
---
 .../modules/services/cluster/hadoop/hdfs.nix  |  76 +++++-
 .../modules/services/cluster/hadoop/yarn.nix  |   1 +
 nixos/tests/hadoop/hadoop.nix                 | 244 +++++++++++++++---
 nixos/tests/hadoop/hdfs.nix                   |  19 +-
 nixos/tests/hadoop/yarn.nix                   |  11 +-
 5 files changed, 286 insertions(+), 65 deletions(-)

diff --git a/nixos/modules/services/cluster/hadoop/hdfs.nix b/nixos/modules/services/cluster/hadoop/hdfs.nix
index e347b682b902..961aa35a4b15 100644
--- a/nixos/modules/services/cluster/hadoop/hdfs.nix
+++ b/nixos/modules/services/cluster/hadoop/hdfs.nix
@@ -24,6 +24,15 @@ in
           Whether to run the HDFS NameNode
         '';
       };
+      formatOnInit = mkOption {
+        type = types.bool;
+        default = false;
+        description = ''
+          Format HDFS namenode on first start. This is useful for quickly spinning up ephemeral HDFS clusters with a single namenode.
+          For HA clusters, initialization involves multiple steps across multiple nodes. Follow [this guide](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/HDFSHighAvailabilityWithQJM.html)
+          to initialize an HA cluster manually.
+        '';
+      };
       inherit restartIfChanged;
       openFirewall = mkOption {
         type = types.bool;
@@ -50,6 +59,33 @@ in
         '';
       };
     };
+    journalnode = {
+      enabled = mkOption {
+        type = types.bool;
+        default = false;
+        description = ''
+          Whether to run the HDFS JournalNode
+        '';
+      };
+      inherit restartIfChanged;
+      openFirewall = mkOption {
+        type = types.bool;
+        default = true;
+        description = ''
+          Open firewall ports for journalnode
+        '';
+      };
+    };
+    zkfc = {
+      enabled = mkOption {
+        type = types.bool;
+        default = false;
+        description = ''
+          Whether to run the HDFS ZooKeeper failover controller
+        '';
+      };
+      inherit restartIfChanged;
+    };
   };
 
   config = mkMerge [
@@ -59,9 +95,9 @@ in
         wantedBy = [ "multi-user.target" ];
         inherit (cfg.hdfs.namenode) restartIfChanged;
 
-        preStart = ''
+        preStart = (mkIf cfg.hdfs.namenode.formatOnInit ''
           ${cfg.package}/bin/hdfs --config ${hadoopConf} namenode -format -nonInteractive || true
-        '';
+        '');
 
         serviceConfig = {
           User = "hdfs";
@@ -74,6 +110,7 @@ in
       networking.firewall.allowedTCPPorts = (mkIf cfg.hdfs.namenode.openFirewall [
         9870 # namenode.http-address
         8020 # namenode.rpc-address
+        8022 # namenode. servicerpc-address
       ]);
     })
     (mkIf cfg.hdfs.datanode.enabled {
@@ -96,8 +133,41 @@ in
         9867 # datanode.ipc.address
       ]);
     })
+    (mkIf cfg.hdfs.journalnode.enabled {
+      systemd.services.hdfs-journalnode = {
+        description = "Hadoop HDFS JournalNode";
+        wantedBy = [ "multi-user.target" ];
+        inherit (cfg.hdfs.journalnode) restartIfChanged;
+
+        serviceConfig = {
+          User = "hdfs";
+          SyslogIdentifier = "hdfs-journalnode";
+          ExecStart = "${cfg.package}/bin/hdfs --config ${hadoopConf} journalnode";
+          Restart = "always";
+        };
+      };
+
+      networking.firewall.allowedTCPPorts = (mkIf cfg.hdfs.datanode.openFirewall [
+        8480 # dfs.journalnode.http-address
+        8485 # dfs.journalnode.rpc-address
+      ]);
+    })
+    (mkIf cfg.hdfs.zkfc.enabled {
+      systemd.services.hdfs-zkfc = {
+        description = "Hadoop HDFS ZooKeeper failover controller";
+        wantedBy = [ "multi-user.target" ];
+        inherit (cfg.hdfs.zkfc) restartIfChanged;
+
+        serviceConfig = {
+          User = "hdfs";
+          SyslogIdentifier = "hdfs-zkfc";
+          ExecStart = "${cfg.package}/bin/hdfs --config ${hadoopConf} zkfc";
+          Restart = "always";
+        };
+      };
+    })
     (mkIf (
-        cfg.hdfs.namenode.enabled || cfg.hdfs.datanode.enabled
+        cfg.hdfs.namenode.enabled || cfg.hdfs.datanode.enabled || cfg.hdfs.journalnode.enabled || cfg.hdfs.zkfc.enabled
     ) {
       users.users.hdfs = {
         description = "Hadoop HDFS user";
diff --git a/nixos/modules/services/cluster/hadoop/yarn.nix b/nixos/modules/services/cluster/hadoop/yarn.nix
index 0086a53e3b74..3d27ad8b1099 100644
--- a/nixos/modules/services/cluster/hadoop/yarn.nix
+++ b/nixos/modules/services/cluster/hadoop/yarn.nix
@@ -91,6 +91,7 @@ in
         8030 # resourcemanager.scheduler.address
         8031 # resourcemanager.resource-tracker.address
         8032 # resourcemanager.address
+        8033 # resourcemanager.admin.address
       ]);
     })
 
diff --git a/nixos/tests/hadoop/hadoop.nix b/nixos/tests/hadoop/hadoop.nix
index 46dfac26e065..2a55c6fc3aa4 100644
--- a/nixos/tests/hadoop/hadoop.nix
+++ b/nixos/tests/hadoop/hadoop.nix
@@ -1,70 +1,230 @@
+# This test is very comprehensive. It tests whether all hadoop services work well with each other.
+# Run this when updating the Hadoop package or making significant changes to the hadoop module.
+# For a more basic test, see hdfs.nix and yarn.nix
 import ../make-test-python.nix ({pkgs, ...}: {
 
   nodes = let
     package = pkgs.hadoop;
     coreSite = {
-      "fs.defaultFS" = "hdfs://master";
+      "fs.defaultFS" = "hdfs://ns1";
+    };
+    hdfsSite = {
+      "dfs.namenode.rpc-bind-host" = "0.0.0.0";
+      "dfs.namenode.http-bind-host" = "0.0.0.0";
+      "dfs.namenode.servicerpc-bind-host" = "0.0.0.0";
+
+      # HA Quorum Journal Manager configuration
+      "dfs.nameservices" = "ns1";
+      "dfs.ha.namenodes.ns1" = "nn1,nn2";
+      "dfs.namenode.shared.edits.dir.ns1.nn1" = "qjournal://jn1:8485;jn2:8485;jn3:8485/ns1";
+      "dfs.namenode.shared.edits.dir.ns1.nn2" = "qjournal://jn1:8485;jn2:8485;jn3:8485/ns1";
+      "dfs.namenode.rpc-address.ns1.nn1" = "nn1:8020";
+      "dfs.namenode.rpc-address.ns1.nn2" = "nn2:8020";
+      "dfs.namenode.servicerpc-address.ns1.nn1" = "nn1:8022";
+      "dfs.namenode.servicerpc-address.ns1.nn2" = "nn2:8022";
+      "dfs.namenode.http-address.ns1.nn1" = "nn1:9870";
+      "dfs.namenode.http-address.ns1.nn2" = "nn2:9870";
+
+      # Automatic failover configuration
+      "dfs.client.failover.proxy.provider.ns1" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider";
+      "dfs.ha.automatic-failover.enabled.ns1" = "true";
+      "dfs.ha.fencing.methods" = "shell(true)";
+      "ha.zookeeper.quorum" = "zk1:2181";
+    };
+    yarnSiteHA = {
+      "yarn.resourcemanager.zk-address" = "zk1:2181";
+      "yarn.resourcemanager.ha.enabled" = "true";
+      "yarn.resourcemanager.ha.rm-ids" = "rm1,rm2";
+      "yarn.resourcemanager.hostname.rm1" = "rm1";
+      "yarn.resourcemanager.hostname.rm2" = "rm2";
+      "yarn.resourcemanager.ha.automatic-failover.enabled" = "true";
+      "yarn.resourcemanager.cluster-id" = "cluster1";
+      # yarn.resourcemanager.webapp.address needs to be defined even though yarn.resourcemanager.hostname is set. This shouldn't be necessary, but there's a bug in
+      # hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java:70
+      # that causes AM containers to fail otherwise.
+      "yarn.resourcemanager.webapp.address.rm1" = "rm1:8088";
+      "yarn.resourcemanager.webapp.address.rm2" = "rm2:8088";
     };
   in {
-    master = {pkgs, options, ...}: {
-      services.hadoop = {
-        inherit package coreSite;
-        hdfs.namenode.enabled = true;
-        yarn.resourcemanager.enabled = true;
-      };
-      virtualisation.memorySize = 1024;
+    zk1 = { ... }: {
+      services.zookeeper.enable = true;
+      networking.firewall.allowedTCPPorts = [ 2181 ];
     };
 
-    worker = {pkgs, options, ...}: {
+    # HDFS cluster
+    nn1 = {pkgs, options, ...}: {
       services.hadoop = {
-        inherit package coreSite;
-        hdfs.datanode.enabled = true;
-        yarn.nodemanager.enabled = true;
-        yarnSite = options.services.hadoop.yarnSite.default // {
-          "yarn.resourcemanager.hostname" = "master";
-        };
+        inherit package coreSite hdfsSite;
+        hdfs.namenode.enabled = true;
+        hdfs.zkfc.enabled = true;
       };
+    };
+    nn2 = {pkgs, options, ...}: {
+      services.hadoop = {
+        inherit package coreSite hdfsSite;
+        hdfs.namenode.enabled = true;
+        hdfs.zkfc.enabled = true;
+      };
+    };
+
+    jn1 = {pkgs, options, ...}: {
+      services.hadoop = {
+        inherit package coreSite hdfsSite;
+        hdfs.journalnode.enabled = true;
+      };
+    };
+    jn2 = {pkgs, options, ...}: {
+      services.hadoop = {
+        inherit package coreSite hdfsSite;
+        hdfs.journalnode.enabled = true;
+      };
+    };
+    jn3 = {pkgs, options, ...}: {
+      services.hadoop = {
+        inherit package coreSite hdfsSite;
+        hdfs.journalnode.enabled = true;
+      };
+    };
+
+    dn1 = {pkgs, options, ...}: {
+      services.hadoop = {
+        inherit package coreSite hdfsSite;
+        hdfs.datanode.enabled = true;
+      };
+    };
+
+    # YARN cluster
+    rm1 = {pkgs, options, ...}: {
+      virtualisation.memorySize = 1024;
+      services.hadoop = {
+        inherit package coreSite hdfsSite;
+        yarnSite = options.services.hadoop.yarnSite.default // yarnSiteHA;
+        yarn.resourcemanager.enabled = true;
+      };
+    };
+    rm2 = {pkgs, options, ...}: {
+      virtualisation.memorySize = 1024;
+      services.hadoop = {
+        inherit package coreSite hdfsSite;
+        yarnSite = options.services.hadoop.yarnSite.default // yarnSiteHA;
+        yarn.resourcemanager.enabled = true;
+      };
+    };
+    nm1 = {pkgs, options, ...}: {
       virtualisation.memorySize = 2048;
+      services.hadoop = {
+        inherit package coreSite hdfsSite;
+        yarnSite = options.services.hadoop.yarnSite.default // yarnSiteHA;
+        yarn.nodemanager.enabled = true;
+      };
     };
   };
 
   testScript = ''
     start_all()
 
-    master.wait_for_unit("network.target")
-    master.wait_for_unit("hdfs-namenode")
+    #### HDFS tests ####
 
-    master.wait_for_open_port(8020)
-    master.wait_for_open_port(9870)
+    zk1.wait_for_unit("network.target")
+    jn1.wait_for_unit("network.target")
+    jn2.wait_for_unit("network.target")
+    jn3.wait_for_unit("network.target")
+    nn1.wait_for_unit("network.target")
+    nn2.wait_for_unit("network.target")
+    dn1.wait_for_unit("network.target")
 
-    worker.wait_for_unit("network.target")
-    worker.wait_for_unit("hdfs-datanode")
-    worker.wait_for_open_port(9864)
-    worker.wait_for_open_port(9866)
-    worker.wait_for_open_port(9867)
+    zk1.wait_for_unit("zookeeper")
+    jn1.wait_for_unit("hdfs-journalnode")
+    jn2.wait_for_unit("hdfs-journalnode")
+    jn3.wait_for_unit("hdfs-journalnode")
 
-    master.succeed("curl -f http://worker:9864")
-    worker.succeed("curl -f http://master:9870")
+    zk1.wait_for_open_port(2181)
+    jn1.wait_for_open_port(8480)
+    jn1.wait_for_open_port(8485)
+    jn2.wait_for_open_port(8480)
+    jn2.wait_for_open_port(8485)
 
-    worker.succeed("sudo -u hdfs hdfs dfsadmin -safemode wait")
+    # Namenodes must be stopped before initializing the cluster
+    nn1.succeed("systemctl stop hdfs-namenode")
+    nn2.succeed("systemctl stop hdfs-namenode")
+    nn1.succeed("systemctl stop hdfs-zkfc")
+    nn2.succeed("systemctl stop hdfs-zkfc")
 
-    master.wait_for_unit("yarn-resourcemanager")
+    # Initialize zookeeper for failover controller
+    nn1.succeed("sudo -u hdfs hdfs zkfc -formatZK 2>&1 | systemd-cat")
 
-    master.wait_for_open_port(8030)
-    master.wait_for_open_port(8031)
-    master.wait_for_open_port(8032)
-    master.wait_for_open_port(8088)
-    worker.succeed("curl -f http://master:8088")
+    # Format NN1 and start it
+    nn1.succeed("sudo -u hdfs hadoop namenode -format 2>&1 | systemd-cat")
+    nn1.succeed("systemctl start hdfs-namenode")
+    nn1.wait_for_open_port(9870)
+    nn1.wait_for_open_port(8022)
+    nn1.wait_for_open_port(8020)
 
-    worker.wait_for_unit("yarn-nodemanager")
-    worker.wait_for_open_port(8042)
-    worker.wait_for_open_port(8040)
-    master.succeed("curl -f http://worker:8042")
+    # Bootstrap NN2 from NN1 and start it
+    nn2.succeed("sudo -u hdfs hdfs namenode -bootstrapStandby 2>&1 | systemd-cat")
+    nn2.succeed("systemctl start hdfs-namenode")
+    nn2.wait_for_open_port(9870)
+    nn2.wait_for_open_port(8022)
+    nn2.wait_for_open_port(8020)
+    nn1.succeed("netstat -tulpne | systemd-cat")
 
-    assert "Total Nodes:1" in worker.succeed("yarn node -list")
+    # Start failover controllers
+    nn1.succeed("systemctl start hdfs-zkfc")
+    nn2.succeed("systemctl start hdfs-zkfc")
 
-    assert "Estimated value of Pi is" in worker.succeed("HADOOP_USER_NAME=hdfs yarn jar $(readlink $(which yarn) | sed -r 's~bin/yarn~lib/hadoop-*/share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar~g') pi 2 10")
-    assert "SUCCEEDED" in worker.succeed("yarn application -list -appStates FINISHED")
-    worker.succeed("sudo -u hdfs hdfs dfs -ls / | systemd-cat")
+    # DN should have started by now, but confirm anyway
+    dn1.wait_for_unit("hdfs-datanode")
+    # Print states of namenodes
+    dn1.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState | systemd-cat")
+    # Wait for cluster to exit safemode
+    dn1.succeed("sudo -u hdfs hdfs dfsadmin -safemode wait")
+    dn1.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState | systemd-cat")
+    # test R/W
+    dn1.succeed("echo testfilecontents | sudo -u hdfs hdfs dfs -put - /testfile")
+    assert "testfilecontents" in dn1.succeed("sudo -u hdfs hdfs dfs -cat /testfile")
+
+    # Test NN failover
+    nn1.succeed("systemctl stop hdfs-namenode")
+    assert "active" in dn1.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState")
+    dn1.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState | systemd-cat")
+    assert "testfilecontents" in dn1.succeed("sudo -u hdfs hdfs dfs -cat /testfile")
+
+    nn1.succeed("systemctl start hdfs-namenode")
+    nn1.wait_for_open_port(9870)
+    nn1.wait_for_open_port(8022)
+    nn1.wait_for_open_port(8020)
+    assert "standby" in dn1.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState")
+    dn1.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState | systemd-cat")
+
+    #### YARN tests ####
+
+    rm1.wait_for_unit("network.target")
+    rm2.wait_for_unit("network.target")
+    nm1.wait_for_unit("network.target")
+
+    rm1.wait_for_unit("yarn-resourcemanager")
+    rm1.wait_for_open_port(8088)
+    rm2.wait_for_unit("yarn-resourcemanager")
+    rm2.wait_for_open_port(8088)
+
+    nm1.wait_for_unit("yarn-nodemanager")
+    nm1.wait_for_open_port(8042)
+    nm1.wait_for_open_port(8040)
+    nm1.wait_until_succeeds("yarn node -list | grep Nodes:1")
+    nm1.succeed("sudo -u yarn yarn rmadmin -getAllServiceState | systemd-cat")
+    nm1.succeed("sudo -u yarn yarn node -list | systemd-cat")
+
+    # Test RM failover
+    rm1.succeed("systemctl stop yarn-resourcemanager")
+    assert "standby" not in nm1.succeed("sudo -u yarn yarn rmadmin -getAllServiceState")
+    nm1.succeed("sudo -u yarn yarn rmadmin -getAllServiceState | systemd-cat")
+    rm1.succeed("systemctl start yarn-resourcemanager")
+    rm1.wait_for_unit("yarn-resourcemanager")
+    rm1.wait_for_open_port(8088)
+    assert "standby" in nm1.succeed("sudo -u yarn yarn rmadmin -getAllServiceState")
+    nm1.succeed("sudo -u yarn yarn rmadmin -getAllServiceState | systemd-cat")
+
+    assert "Estimated value of Pi is" in nm1.succeed("HADOOP_USER_NAME=hdfs yarn jar $(readlink $(which yarn) | sed -r 's~bin/yarn~lib/hadoop-*/share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar~g') pi 2 10")
+    assert "SUCCEEDED" in nm1.succeed("yarn application -list -appStates FINISHED")
   '';
- })
+})
diff --git a/nixos/tests/hadoop/hdfs.nix b/nixos/tests/hadoop/hdfs.nix
index f5907185c039..9bc32cc7f8ad 100644
--- a/nixos/tests/hadoop/hdfs.nix
+++ b/nixos/tests/hadoop/hdfs.nix
@@ -1,9 +1,13 @@
+# Test a minimal HDFS cluster with no HA
 import ../make-test-python.nix ({...}: {
   nodes = {
     namenode = {pkgs, ...}: {
       services.hadoop = {
         package = pkgs.hadoop;
-        hdfs.namenode.enabled = true;
+        hdfs.namenode = {
+          enabled = true;
+          formatOnInit = true;
+        };
         coreSite = {
           "fs.defaultFS" = "hdfs://namenode:8020";
         };
@@ -13,10 +17,6 @@ import ../make-test-python.nix ({...}: {
           "dfs.namenode.http-bind-host" = "0.0.0.0";
         };
       };
-      networking.firewall.allowedTCPPorts = [
-        9870 # namenode.http-address
-        8020 # namenode.rpc-address
-      ];
     };
     datanode = {pkgs, ...}: {
       services.hadoop = {
@@ -26,11 +26,6 @@ import ../make-test-python.nix ({...}: {
           "fs.defaultFS" = "hdfs://namenode:8020";
         };
       };
-      networking.firewall.allowedTCPPorts = [
-        9864 # datanode.http.address
-        9866 # datanode.address
-        9867 # datanode.ipc.address
-      ];
     };
   };
 
@@ -50,5 +45,9 @@ import ../make-test-python.nix ({...}: {
 
     namenode.succeed("curl -f http://namenode:9870")
     datanode.succeed("curl -f http://datanode:9864")
+
+    datanode.succeed("sudo -u hdfs hdfs dfsadmin -safemode wait")
+    datanode.succeed("echo testfilecontents | sudo -u hdfs hdfs dfs -put - /testfile")
+    assert "testfilecontents" in datanode.succeed("sudo -u hdfs hdfs dfs -cat /testfile")
   '';
 })
diff --git a/nixos/tests/hadoop/yarn.nix b/nixos/tests/hadoop/yarn.nix
index fbbb293eecd6..70367dfec183 100644
--- a/nixos/tests/hadoop/yarn.nix
+++ b/nixos/tests/hadoop/yarn.nix
@@ -1,3 +1,4 @@
+# This only tests if YARN is able to start its services
 import ../make-test-python.nix ({...}: {
   nodes = {
     resourcemanager = {pkgs, ...}: {
@@ -6,10 +7,6 @@ import ../make-test-python.nix ({...}: {
       services.hadoop.yarnSite = {
         "yarn.resourcemanager.scheduler.class" = "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler";
       };
-      networking.firewall.allowedTCPPorts = [
-        8088 # resourcemanager.webapp.address
-        8031 # resourcemanager.resource-tracker.address
-      ];
     };
     nodemanager = {pkgs, ...}: {
       services.hadoop.package = pkgs.hadoop;
@@ -17,12 +14,7 @@ import ../make-test-python.nix ({...}: {
       services.hadoop.yarnSite = {
         "yarn.resourcemanager.hostname" = "resourcemanager";
         "yarn.nodemanager.log-dirs" = "/tmp/userlogs";
-        "yarn.nodemanager.address" = "0.0.0.0:8041";
       };
-      networking.firewall.allowedTCPPorts = [
-        8042 # nodemanager.webapp.address
-        8041 # nodemanager.address
-      ];
     };
 
   };
@@ -38,7 +30,6 @@ import ../make-test-python.nix ({...}: {
     nodemanager.wait_for_unit("yarn-nodemanager")
     nodemanager.wait_for_unit("network.target")
     nodemanager.wait_for_open_port(8042)
-    nodemanager.wait_for_open_port(8041)
 
     resourcemanager.succeed("curl -f http://localhost:8088")
     nodemanager.succeed("curl -f http://localhost:8042")

From 9ca4363191f05147d0bd23ffb61838d9c1ee890e Mon Sep 17 00:00:00 2001
From: illustris <rharikrishnan95@gmail.com>
Date: Tue, 2 Nov 2021 18:34:04 +0530
Subject: [PATCH 2/8] nixos/hadoop: add HTTPFS

---
 .../modules/services/cluster/hadoop/conf.nix  |  1 +
 .../services/cluster/hadoop/default.nix       | 14 +++++-
 .../modules/services/cluster/hadoop/hdfs.nix  | 50 ++++++++++++++++++-
 nixos/tests/hadoop/hdfs.nix                   | 23 ++++++---
 4 files changed, 78 insertions(+), 10 deletions(-)

diff --git a/nixos/modules/services/cluster/hadoop/conf.nix b/nixos/modules/services/cluster/hadoop/conf.nix
index 69472408cabe..0caec5cfc203 100644
--- a/nixos/modules/services/cluster/hadoop/conf.nix
+++ b/nixos/modules/services/cluster/hadoop/conf.nix
@@ -35,6 +35,7 @@ pkgs.runCommand "hadoop-conf" {} ''
   cp ${siteXml "hdfs-site.xml" cfg.hdfsSite}/* $out/
   cp ${siteXml "mapred-site.xml" cfg.mapredSite}/* $out/
   cp ${siteXml "yarn-site.xml" cfg.yarnSite}/* $out/
+  cp ${siteXml "httpfs-site.xml" cfg.httpfsSite}/* $out/
   cp ${cfgFile "container-executor.cfg" cfg.containerExecutorCfg}/* $out/
   cp ${pkgs.writeTextDir "hadoop-user-functions.sh" userFunctions}/* $out/
   cp ${pkgs.writeTextDir "hadoop-env.sh" hadoopEnv}/* $out/
diff --git a/nixos/modules/services/cluster/hadoop/default.nix b/nixos/modules/services/cluster/hadoop/default.nix
index da3e47b95d4d..e4dcfde80fba 100644
--- a/nixos/modules/services/cluster/hadoop/default.nix
+++ b/nixos/modules/services/cluster/hadoop/default.nix
@@ -70,6 +70,17 @@ with lib;
       description = "Hadoop yarn-site.xml definition";
     };
 
+    httpfsSite = mkOption {
+      default = { };
+      type = types.attrsOf types.anything;
+      example = literalExpression ''
+        {
+          "hadoop.http.max.threads" = 500;
+        }
+      '';
+      description = "Hadoop httpfs-site.xml definition";
+    };
+
     log4jProperties = mkOption {
       default = "${cfg.package}/lib/${cfg.package.untarDir}/etc/hadoop/log4j.properties";
       type = types.path;
@@ -118,7 +129,8 @@ with lib;
 
   config = mkMerge [
     (mkIf (builtins.hasAttr "yarn" config.users.users ||
-           builtins.hasAttr "hdfs" config.users.users) {
+           builtins.hasAttr "hdfs" config.users.users ||
+           builtins.hasAttr "httpfs" config.users.users) {
       users.groups.hadoop = {
         gid = config.ids.gids.hadoop;
       };
diff --git a/nixos/modules/services/cluster/hadoop/hdfs.nix b/nixos/modules/services/cluster/hadoop/hdfs.nix
index 961aa35a4b15..11b855b0c710 100644
--- a/nixos/modules/services/cluster/hadoop/hdfs.nix
+++ b/nixos/modules/services/cluster/hadoop/hdfs.nix
@@ -86,6 +86,23 @@ in
       };
       inherit restartIfChanged;
     };
+    httpfs = {
+      enabled = mkOption {
+        type = types.bool;
+        default = false;
+        description = ''
+          Whether to run the HDFS httpfs failover controller
+        '';
+      };
+      tempPath = mkOption {
+        type = types.path;
+        default = "/tmp/hadoop/httpfs";
+        description = ''
+          HTTPFS_TEMP path used by HTTPFS
+        '';
+      };
+      inherit restartIfChanged;
+    };
   };
 
   config = mkMerge [
@@ -166,6 +183,31 @@ in
         };
       };
     })
+    (mkIf cfg.hdfs.httpfs.enabled {
+      systemd.services.hdfs-httpfs = {
+        description = "Hadoop httpfs";
+        wantedBy = [ "multi-user.target" ];
+        inherit (cfg.hdfs.httpfs) restartIfChanged;
+
+        environment = {
+          HTTPFS_TEMP = cfg.hdfs.httpfs.tempPath;
+        };
+
+        preStart = ''
+          mkdir -p $HTTPFS_TEMP
+        '';
+
+        serviceConfig = {
+          User = "httpfs";
+          SyslogIdentifier = "hdfs-httpfs";
+          ExecStart = "${cfg.package}/bin/hdfs --config ${hadoopConf} httpfs";
+          Restart = "always";
+        };
+      };
+      networking.firewall.allowedTCPPorts = (mkIf cfg.hdfs.datanode.openFirewall [
+        14000 # httpfs.http.port
+      ]);
+    })
     (mkIf (
         cfg.hdfs.namenode.enabled || cfg.hdfs.datanode.enabled || cfg.hdfs.journalnode.enabled || cfg.hdfs.zkfc.enabled
     ) {
@@ -175,6 +217,12 @@ in
         uid = config.ids.uids.hdfs;
       };
     })
-
+    (mkIf cfg.hdfs.httpfs.enabled {
+      users.users.httpfs = {
+        description = "Hadoop HTTPFS user";
+        group = "hadoop";
+        isSystemUser = true;
+      };
+    })
   ];
 }
diff --git a/nixos/tests/hadoop/hdfs.nix b/nixos/tests/hadoop/hdfs.nix
index 9bc32cc7f8ad..e2cef32d05dd 100644
--- a/nixos/tests/hadoop/hdfs.nix
+++ b/nixos/tests/hadoop/hdfs.nix
@@ -2,19 +2,20 @@
 import ../make-test-python.nix ({...}: {
   nodes = {
     namenode = {pkgs, ...}: {
+      virtualisation.memorySize = 1024;
       services.hadoop = {
         package = pkgs.hadoop;
-        hdfs.namenode = {
-          enabled = true;
-          formatOnInit = true;
+        hdfs = {
+          namenode = {
+            enabled = true;
+            formatOnInit = true;
+          };
+          httpfs.enabled = true;
         };
         coreSite = {
           "fs.defaultFS" = "hdfs://namenode:8020";
-        };
-        hdfsSite = {
-          "dfs.replication" = 1;
-          "dfs.namenode.rpc-bind-host" = "0.0.0.0";
-          "dfs.namenode.http-bind-host" = "0.0.0.0";
+          "hadoop.proxyuser.httpfs.groups" = "*";
+          "hadoop.proxyuser.httpfs.hosts" = "*";
         };
       };
     };
@@ -24,6 +25,8 @@ import ../make-test-python.nix ({...}: {
         hdfs.datanode.enabled = true;
         coreSite = {
           "fs.defaultFS" = "hdfs://namenode:8020";
+          "hadoop.proxyuser.httpfs.groups" = "*";
+          "hadoop.proxyuser.httpfs.hosts" = "*";
         };
       };
     };
@@ -49,5 +52,9 @@ import ../make-test-python.nix ({...}: {
     datanode.succeed("sudo -u hdfs hdfs dfsadmin -safemode wait")
     datanode.succeed("echo testfilecontents | sudo -u hdfs hdfs dfs -put - /testfile")
     assert "testfilecontents" in datanode.succeed("sudo -u hdfs hdfs dfs -cat /testfile")
+
+    namenode.wait_for_unit("hdfs-httpfs")
+    namenode.wait_for_open_port(14000)
+    assert "testfilecontents" in datanode.succeed("curl -f \"http://namenode:14000/webhdfs/v1/testfile?user.name=hdfs&op=OPEN\" 2>&1")
   '';
 })

From c3d147f507e81455ae467f8aa589972af3449f77 Mon Sep 17 00:00:00 2001
From: illustris <rharikrishnan95@gmail.com>
Date: Tue, 2 Nov 2021 21:46:48 +0530
Subject: [PATCH 3/8] nixos/hadoop: replace "enabled" options with "enable"
 options

The module has been using "enabled" in place of enable since init ( 0c10b2baa6bf61c8ddaed7cdb6c2f2dbaab42662 )
---
 .../modules/services/cluster/hadoop/hdfs.nix  | 24 +++++++++----------
 .../modules/services/cluster/hadoop/yarn.nix  | 10 ++++----
 nixos/tests/hadoop/hadoop.nix                 | 22 ++++++++---------
 nixos/tests/hadoop/hdfs.nix                   |  6 ++---
 nixos/tests/hadoop/yarn.nix                   |  4 ++--
 5 files changed, 33 insertions(+), 33 deletions(-)

diff --git a/nixos/modules/services/cluster/hadoop/hdfs.nix b/nixos/modules/services/cluster/hadoop/hdfs.nix
index 11b855b0c710..e2cd8bc6fbaa 100644
--- a/nixos/modules/services/cluster/hadoop/hdfs.nix
+++ b/nixos/modules/services/cluster/hadoop/hdfs.nix
@@ -17,7 +17,7 @@ in
 {
   options.services.hadoop.hdfs = {
     namenode = {
-      enabled = mkOption {
+      enable = mkOption {
         type = types.bool;
         default = false;
         description = ''
@@ -43,7 +43,7 @@ in
       };
     };
     datanode = {
-      enabled = mkOption {
+      enable = mkOption {
         type = types.bool;
         default = false;
         description = ''
@@ -60,7 +60,7 @@ in
       };
     };
     journalnode = {
-      enabled = mkOption {
+      enable = mkOption {
         type = types.bool;
         default = false;
         description = ''
@@ -77,7 +77,7 @@ in
       };
     };
     zkfc = {
-      enabled = mkOption {
+      enable = mkOption {
         type = types.bool;
         default = false;
         description = ''
@@ -87,7 +87,7 @@ in
       inherit restartIfChanged;
     };
     httpfs = {
-      enabled = mkOption {
+      enable = mkOption {
         type = types.bool;
         default = false;
         description = ''
@@ -106,7 +106,7 @@ in
   };
 
   config = mkMerge [
-    (mkIf cfg.hdfs.namenode.enabled {
+    (mkIf cfg.hdfs.namenode.enable {
       systemd.services.hdfs-namenode = {
         description = "Hadoop HDFS NameNode";
         wantedBy = [ "multi-user.target" ];
@@ -130,7 +130,7 @@ in
         8022 # namenode. servicerpc-address
       ]);
     })
-    (mkIf cfg.hdfs.datanode.enabled {
+    (mkIf cfg.hdfs.datanode.enable {
       systemd.services.hdfs-datanode = {
         description = "Hadoop HDFS DataNode";
         wantedBy = [ "multi-user.target" ];
@@ -150,7 +150,7 @@ in
         9867 # datanode.ipc.address
       ]);
     })
-    (mkIf cfg.hdfs.journalnode.enabled {
+    (mkIf cfg.hdfs.journalnode.enable {
       systemd.services.hdfs-journalnode = {
         description = "Hadoop HDFS JournalNode";
         wantedBy = [ "multi-user.target" ];
@@ -169,7 +169,7 @@ in
         8485 # dfs.journalnode.rpc-address
       ]);
     })
-    (mkIf cfg.hdfs.zkfc.enabled {
+    (mkIf cfg.hdfs.zkfc.enable {
       systemd.services.hdfs-zkfc = {
         description = "Hadoop HDFS ZooKeeper failover controller";
         wantedBy = [ "multi-user.target" ];
@@ -183,7 +183,7 @@ in
         };
       };
     })
-    (mkIf cfg.hdfs.httpfs.enabled {
+    (mkIf cfg.hdfs.httpfs.enable {
       systemd.services.hdfs-httpfs = {
         description = "Hadoop httpfs";
         wantedBy = [ "multi-user.target" ];
@@ -209,7 +209,7 @@ in
       ]);
     })
     (mkIf (
-        cfg.hdfs.namenode.enabled || cfg.hdfs.datanode.enabled || cfg.hdfs.journalnode.enabled || cfg.hdfs.zkfc.enabled
+        cfg.hdfs.namenode.enable || cfg.hdfs.datanode.enable || cfg.hdfs.journalnode.enable || cfg.hdfs.zkfc.enable
     ) {
       users.users.hdfs = {
         description = "Hadoop HDFS user";
@@ -217,7 +217,7 @@ in
         uid = config.ids.uids.hdfs;
       };
     })
-    (mkIf cfg.hdfs.httpfs.enabled {
+    (mkIf cfg.hdfs.httpfs.enable {
       users.users.httpfs = {
         description = "Hadoop HTTPFS user";
         group = "hadoop";
diff --git a/nixos/modules/services/cluster/hadoop/yarn.nix b/nixos/modules/services/cluster/hadoop/yarn.nix
index 3d27ad8b1099..b6d4dbe25c03 100644
--- a/nixos/modules/services/cluster/hadoop/yarn.nix
+++ b/nixos/modules/services/cluster/hadoop/yarn.nix
@@ -17,7 +17,7 @@ in
 {
   options.services.hadoop.yarn = {
     resourcemanager = {
-      enabled = mkOption {
+      enable = mkOption {
         type = types.bool;
         default = false;
         description = ''
@@ -34,7 +34,7 @@ in
       };
     };
     nodemanager = {
-      enabled = mkOption {
+      enable = mkOption {
         type = types.bool;
         default = false;
         description = ''
@@ -62,7 +62,7 @@ in
 
   config = mkMerge [
     (mkIf (
-        cfg.yarn.resourcemanager.enabled || cfg.yarn.nodemanager.enabled
+        cfg.yarn.resourcemanager.enable || cfg.yarn.nodemanager.enable
     ) {
 
       users.users.yarn = {
@@ -72,7 +72,7 @@ in
       };
     })
 
-    (mkIf cfg.yarn.resourcemanager.enabled {
+    (mkIf cfg.yarn.resourcemanager.enable {
       systemd.services.yarn-resourcemanager = {
         description = "Hadoop YARN ResourceManager";
         wantedBy = [ "multi-user.target" ];
@@ -95,7 +95,7 @@ in
       ]);
     })
 
-    (mkIf cfg.yarn.nodemanager.enabled {
+    (mkIf cfg.yarn.nodemanager.enable {
       # Needed because yarn hardcodes /bin/bash in container start scripts
       # These scripts can't be patched, they are generated at runtime
       systemd.tmpfiles.rules = [
diff --git a/nixos/tests/hadoop/hadoop.nix b/nixos/tests/hadoop/hadoop.nix
index 2a55c6fc3aa4..b4ed0e17a852 100644
--- a/nixos/tests/hadoop/hadoop.nix
+++ b/nixos/tests/hadoop/hadoop.nix
@@ -55,41 +55,41 @@ import ../make-test-python.nix ({pkgs, ...}: {
     nn1 = {pkgs, options, ...}: {
       services.hadoop = {
         inherit package coreSite hdfsSite;
-        hdfs.namenode.enabled = true;
-        hdfs.zkfc.enabled = true;
+        hdfs.namenode.enable = true;
+        hdfs.zkfc.enable = true;
       };
     };
     nn2 = {pkgs, options, ...}: {
       services.hadoop = {
         inherit package coreSite hdfsSite;
-        hdfs.namenode.enabled = true;
-        hdfs.zkfc.enabled = true;
+        hdfs.namenode.enable = true;
+        hdfs.zkfc.enable = true;
       };
     };
 
     jn1 = {pkgs, options, ...}: {
       services.hadoop = {
         inherit package coreSite hdfsSite;
-        hdfs.journalnode.enabled = true;
+        hdfs.journalnode.enable = true;
       };
     };
     jn2 = {pkgs, options, ...}: {
       services.hadoop = {
         inherit package coreSite hdfsSite;
-        hdfs.journalnode.enabled = true;
+        hdfs.journalnode.enable = true;
       };
     };
     jn3 = {pkgs, options, ...}: {
       services.hadoop = {
         inherit package coreSite hdfsSite;
-        hdfs.journalnode.enabled = true;
+        hdfs.journalnode.enable = true;
       };
     };
 
     dn1 = {pkgs, options, ...}: {
       services.hadoop = {
         inherit package coreSite hdfsSite;
-        hdfs.datanode.enabled = true;
+        hdfs.datanode.enable = true;
       };
     };
 
@@ -99,7 +99,7 @@ import ../make-test-python.nix ({pkgs, ...}: {
       services.hadoop = {
         inherit package coreSite hdfsSite;
         yarnSite = options.services.hadoop.yarnSite.default // yarnSiteHA;
-        yarn.resourcemanager.enabled = true;
+        yarn.resourcemanager.enable = true;
       };
     };
     rm2 = {pkgs, options, ...}: {
@@ -107,7 +107,7 @@ import ../make-test-python.nix ({pkgs, ...}: {
       services.hadoop = {
         inherit package coreSite hdfsSite;
         yarnSite = options.services.hadoop.yarnSite.default // yarnSiteHA;
-        yarn.resourcemanager.enabled = true;
+        yarn.resourcemanager.enable = true;
       };
     };
     nm1 = {pkgs, options, ...}: {
@@ -115,7 +115,7 @@ import ../make-test-python.nix ({pkgs, ...}: {
       services.hadoop = {
         inherit package coreSite hdfsSite;
         yarnSite = options.services.hadoop.yarnSite.default // yarnSiteHA;
-        yarn.nodemanager.enabled = true;
+        yarn.nodemanager.enable = true;
       };
     };
   };
diff --git a/nixos/tests/hadoop/hdfs.nix b/nixos/tests/hadoop/hdfs.nix
index e2cef32d05dd..360dbd60ed27 100644
--- a/nixos/tests/hadoop/hdfs.nix
+++ b/nixos/tests/hadoop/hdfs.nix
@@ -7,10 +7,10 @@ import ../make-test-python.nix ({...}: {
         package = pkgs.hadoop;
         hdfs = {
           namenode = {
-            enabled = true;
+            enable = true;
             formatOnInit = true;
           };
-          httpfs.enabled = true;
+          httpfs.enable = true;
         };
         coreSite = {
           "fs.defaultFS" = "hdfs://namenode:8020";
@@ -22,7 +22,7 @@ import ../make-test-python.nix ({...}: {
     datanode = {pkgs, ...}: {
       services.hadoop = {
         package = pkgs.hadoop;
-        hdfs.datanode.enabled = true;
+        hdfs.datanode.enable = true;
         coreSite = {
           "fs.defaultFS" = "hdfs://namenode:8020";
           "hadoop.proxyuser.httpfs.groups" = "*";
diff --git a/nixos/tests/hadoop/yarn.nix b/nixos/tests/hadoop/yarn.nix
index 70367dfec183..09bdb35791c7 100644
--- a/nixos/tests/hadoop/yarn.nix
+++ b/nixos/tests/hadoop/yarn.nix
@@ -3,14 +3,14 @@ import ../make-test-python.nix ({...}: {
   nodes = {
     resourcemanager = {pkgs, ...}: {
       services.hadoop.package = pkgs.hadoop;
-      services.hadoop.yarn.resourcemanager.enabled = true;
+      services.hadoop.yarn.resourcemanager.enable = true;
       services.hadoop.yarnSite = {
         "yarn.resourcemanager.scheduler.class" = "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler";
       };
     };
     nodemanager = {pkgs, ...}: {
       services.hadoop.package = pkgs.hadoop;
-      services.hadoop.yarn.nodemanager.enabled = true;
+      services.hadoop.yarn.nodemanager.enable = true;
       services.hadoop.yarnSite = {
         "yarn.resourcemanager.hostname" = "resourcemanager";
         "yarn.nodemanager.log-dirs" = "/tmp/userlogs";

From 42e14ff69ff52c74f5ccbb37b7954ef4dea7bf5a Mon Sep 17 00:00:00 2001
From: illustris <rharikrishnan95@gmail.com>
Date: Wed, 3 Nov 2021 19:36:28 +0530
Subject: [PATCH 4/8] nixos/hadoop: replace enable = mkoption bools with
 mkEnableOption

---
 .../modules/services/cluster/hadoop/hdfs.nix  | 40 +++----------------
 .../modules/services/cluster/hadoop/yarn.nix  | 16 +-------
 2 files changed, 7 insertions(+), 49 deletions(-)

diff --git a/nixos/modules/services/cluster/hadoop/hdfs.nix b/nixos/modules/services/cluster/hadoop/hdfs.nix
index e2cd8bc6fbaa..52f951a6c5f4 100644
--- a/nixos/modules/services/cluster/hadoop/hdfs.nix
+++ b/nixos/modules/services/cluster/hadoop/hdfs.nix
@@ -17,13 +17,7 @@ in
 {
   options.services.hadoop.hdfs = {
     namenode = {
-      enable = mkOption {
-        type = types.bool;
-        default = false;
-        description = ''
-          Whether to run the HDFS NameNode
-        '';
-      };
+      enable = mkEnableOption "Whether to run the HDFS NameNode";
       formatOnInit = mkOption {
         type = types.bool;
         default = false;
@@ -43,13 +37,7 @@ in
       };
     };
     datanode = {
-      enable = mkOption {
-        type = types.bool;
-        default = false;
-        description = ''
-          Whether to run the HDFS DataNode
-        '';
-      };
+      enable = mkEnableOption "Whether to run the HDFS DataNode";
       inherit restartIfChanged;
       openFirewall = mkOption {
         type = types.bool;
@@ -60,13 +48,7 @@ in
       };
     };
     journalnode = {
-      enable = mkOption {
-        type = types.bool;
-        default = false;
-        description = ''
-          Whether to run the HDFS JournalNode
-        '';
-      };
+      enable = mkEnableOption "Whether to run the HDFS JournalNode";
       inherit restartIfChanged;
       openFirewall = mkOption {
         type = types.bool;
@@ -77,23 +59,11 @@ in
       };
     };
     zkfc = {
-      enable = mkOption {
-        type = types.bool;
-        default = false;
-        description = ''
-          Whether to run the HDFS ZooKeeper failover controller
-        '';
-      };
+      enable = mkEnableOption "Whether to run the HDFS ZooKeeper failover controller";
       inherit restartIfChanged;
     };
     httpfs = {
-      enable = mkOption {
-        type = types.bool;
-        default = false;
-        description = ''
-          Whether to run the HDFS httpfs failover controller
-        '';
-      };
+      enable = mkEnableOption "Whether to run the HDFS HTTPfs server";
       tempPath = mkOption {
         type = types.path;
         default = "/tmp/hadoop/httpfs";
diff --git a/nixos/modules/services/cluster/hadoop/yarn.nix b/nixos/modules/services/cluster/hadoop/yarn.nix
index b6d4dbe25c03..37c26ea10f76 100644
--- a/nixos/modules/services/cluster/hadoop/yarn.nix
+++ b/nixos/modules/services/cluster/hadoop/yarn.nix
@@ -17,13 +17,7 @@ in
 {
   options.services.hadoop.yarn = {
     resourcemanager = {
-      enable = mkOption {
-        type = types.bool;
-        default = false;
-        description = ''
-          Whether to run the Hadoop YARN ResourceManager
-        '';
-      };
+      enable = mkEnableOption "Whether to run the Hadoop YARN ResourceManager";
       inherit restartIfChanged;
       openFirewall = mkOption {
         type = types.bool;
@@ -34,13 +28,7 @@ in
       };
     };
     nodemanager = {
-      enable = mkOption {
-        type = types.bool;
-        default = false;
-        description = ''
-          Whether to run the Hadoop YARN NodeManager
-        '';
-      };
+      enable = mkEnableOption "Whether to run the Hadoop YARN NodeManager";
       inherit restartIfChanged;
       addBinBash = mkOption {
         type = types.bool;

From 0bd4b60a42a84dab321ee6ca61ff3bcf2d27e9fd Mon Sep 17 00:00:00 2001
From: illustris <rharikrishnan95@gmail.com>
Date: Wed, 3 Nov 2021 22:59:17 +0530
Subject: [PATCH 5/8] nixos/hadoop: release notes

---
 .../from_md/release-notes/rl-2111.section.xml | 23 +++++++++++++++++++
 .../manual/release-notes/rl-2111.section.md   |  5 ++++
 2 files changed, 28 insertions(+)

diff --git a/nixos/doc/manual/from_md/release-notes/rl-2111.section.xml b/nixos/doc/manual/from_md/release-notes/rl-2111.section.xml
index 4a134e42a0f9..2692415174c5 100644
--- a/nixos/doc/manual/from_md/release-notes/rl-2111.section.xml
+++ b/nixos/doc/manual/from_md/release-notes/rl-2111.section.xml
@@ -50,6 +50,29 @@
           guide</link> is available.
         </para>
       </listitem>
+      <listitem>
+        <para>
+          Improvements have been made to the Hadoop module and package:
+        </para>
+        <itemizedlist spacing="compact">
+          <listitem>
+            <para>
+              HDFS and YARN now support production-ready highly
+              available deployments with automatic failover.
+            </para>
+          </listitem>
+          <listitem>
+            <para>
+              Hadoop now defaults to Hadoop 3, updated from 2.
+            </para>
+          </listitem>
+          <listitem>
+            <para>
+              JournalNode, ZKFS and HTTPFS services have been added.
+            </para>
+          </listitem>
+        </itemizedlist>
+      </listitem>
       <listitem>
         <para>
           Activation scripts can now opt int to be run when running
diff --git a/nixos/doc/manual/release-notes/rl-2111.section.md b/nixos/doc/manual/release-notes/rl-2111.section.md
index 1e815616f201..04566980c30d 100644
--- a/nixos/doc/manual/release-notes/rl-2111.section.md
+++ b/nixos/doc/manual/release-notes/rl-2111.section.md
@@ -18,6 +18,11 @@ In addition to numerous new and upgraded packages, this release has the followin
 
 - spark now defaults to spark 3, updated from 2. A [migration guide](https://spark.apache.org/docs/latest/core-migration-guide.html#upgrading-from-core-24-to-30) is available.
 
+- Improvements have been made to the Hadoop module and package:
+  - HDFS and YARN now support production-ready highly available deployments with automatic failover.
+  - Hadoop now defaults to Hadoop 3, updated from 2.
+  - JournalNode, ZKFS and HTTPFS services have been added.
+
 - Activation scripts can now opt int to be run when running `nixos-rebuild dry-activate` and detect the dry activation by reading `$NIXOS_ACTION`.
   This allows activation scripts to output what they would change if the activation was really run.
   The users/modules activation script supports this and outputs some of is actions.

From c8df915e0e4b19364038a03d2a5203bf4805c9ff Mon Sep 17 00:00:00 2001
From: illustris <rharikrishnan95@gmail.com>
Date: Thu, 4 Nov 2021 11:07:32 +0530
Subject: [PATCH 6/8] nixos/hadoop: add links for config files

---
 .../services/cluster/hadoop/default.nix       | 30 +++++++++++++++----
 1 file changed, 24 insertions(+), 6 deletions(-)

diff --git a/nixos/modules/services/cluster/hadoop/default.nix b/nixos/modules/services/cluster/hadoop/default.nix
index e4dcfde80fba..90f22c48e055 100644
--- a/nixos/modules/services/cluster/hadoop/default.nix
+++ b/nixos/modules/services/cluster/hadoop/default.nix
@@ -15,7 +15,10 @@ with lib;
           "fs.defaultFS" = "hdfs://localhost";
         }
       '';
-      description = "Hadoop core-site.xml definition";
+      description = ''
+        Hadoop core-site.xml definition
+        <link xlink:href="https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/core-default.xml"/>
+      '';
     };
 
     hdfsSite = mkOption {
@@ -28,7 +31,10 @@ with lib;
           "dfs.nameservices" = "namenode1";
         }
       '';
-      description = "Hadoop hdfs-site.xml definition";
+      description = ''
+        Hadoop hdfs-site.xml definition
+        <link xlink:href="https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-hdfs/hdfs-default.xml"/>
+      '';
     };
 
     mapredSite = mkOption {
@@ -44,7 +50,10 @@ with lib;
           "mapreduce.map.java.opts" = "-Xmx900m -XX:+UseParallelGC";
         }
       '';
-      description = "Hadoop mapred-site.xml definition";
+      description = ''
+        Hadoop mapred-site.xml definition
+        <link xlink:href="https://hadoop.apache.org/docs/current/hadoop-mapreduce-client/hadoop-mapreduce-client-core/mapred-default.xml"/>
+      '';
     };
 
     yarnSite = mkOption {
@@ -67,7 +76,10 @@ with lib;
           "yarn.resourcemanager.hostname" = "''${config.networking.hostName}";
         }
       '';
-      description = "Hadoop yarn-site.xml definition";
+      description = ''
+        Hadoop yarn-site.xml definition
+        <link xlink:href="https://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-common/yarn-default.xml"/>
+      '';
     };
 
     httpfsSite = mkOption {
@@ -78,7 +90,10 @@ with lib;
           "hadoop.http.max.threads" = 500;
         }
       '';
-      description = "Hadoop httpfs-site.xml definition";
+      description = ''
+        Hadoop httpfs-site.xml definition
+        <link xlink:href="https://hadoop.apache.org/docs/current/hadoop-hdfs-httpfs/httpfs-default.html"/>
+      '';
     };
 
     log4jProperties = mkOption {
@@ -103,7 +118,10 @@ with lib;
           "feature.terminal.enabled" = 0;
         }
       '';
-      description = "Yarn container-executor.cfg definition";
+      description = ''
+        Yarn container-executor.cfg definition
+        <link xlink:href="https://hadoop.apache.org/docs/r2.7.2/hadoop-yarn/hadoop-yarn-site/SecureContainer.html"/>
+      '';
     };
 
     extraConfDirs = mkOption {

From 8331b56701d6df0140345e17a8ae4ac8240f33f9 Mon Sep 17 00:00:00 2001
From: illustris <rharikrishnan95@gmail.com>
Date: Thu, 4 Nov 2021 11:15:21 +0530
Subject: [PATCH 7/8] nixos/hadoop: correct openFirewall options

---
 nixos/modules/services/cluster/hadoop/hdfs.nix | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/nixos/modules/services/cluster/hadoop/hdfs.nix b/nixos/modules/services/cluster/hadoop/hdfs.nix
index 52f951a6c5f4..2b8c29b04f86 100644
--- a/nixos/modules/services/cluster/hadoop/hdfs.nix
+++ b/nixos/modules/services/cluster/hadoop/hdfs.nix
@@ -134,7 +134,7 @@ in
         };
       };
 
-      networking.firewall.allowedTCPPorts = (mkIf cfg.hdfs.datanode.openFirewall [
+      networking.firewall.allowedTCPPorts = (mkIf cfg.hdfs.journalnode.openFirewall [
         8480 # dfs.journalnode.http-address
         8485 # dfs.journalnode.rpc-address
       ]);
@@ -174,7 +174,7 @@ in
           Restart = "always";
         };
       };
-      networking.firewall.allowedTCPPorts = (mkIf cfg.hdfs.datanode.openFirewall [
+      networking.firewall.allowedTCPPorts = (mkIf cfg.hdfs.httpfs.openFirewall [
         14000 # httpfs.http.port
       ]);
     })

From 57225575dfa9f3630ba495a9fa9c57b576a23dff Mon Sep 17 00:00:00 2001
From: illustris <rharikrishnan95@gmail.com>
Date: Thu, 4 Nov 2021 11:26:32 +0530
Subject: [PATCH 8/8] nixos/hadoop: fix errors in HTTPFS

---
 nixos/modules/services/cluster/hadoop/hdfs.nix | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/nixos/modules/services/cluster/hadoop/hdfs.nix b/nixos/modules/services/cluster/hadoop/hdfs.nix
index 2b8c29b04f86..be667aa82d8a 100644
--- a/nixos/modules/services/cluster/hadoop/hdfs.nix
+++ b/nixos/modules/services/cluster/hadoop/hdfs.nix
@@ -72,6 +72,13 @@ in
         '';
       };
       inherit restartIfChanged;
+      openFirewall = mkOption {
+        type = types.bool;
+        default = true;
+        description = ''
+          Open firewall ports for HTTPFS
+        '';
+      };
     };
   };
 
@@ -159,9 +166,7 @@ in
         wantedBy = [ "multi-user.target" ];
         inherit (cfg.hdfs.httpfs) restartIfChanged;
 
-        environment = {
-          HTTPFS_TEMP = cfg.hdfs.httpfs.tempPath;
-        };
+        environment.HTTPFS_TEMP = cfg.hdfs.httpfs.tempPath;
 
         preStart = ''
           mkdir -p $HTTPFS_TEMP