nixpkgs-suyu/nixos/tests/kubernetes/rbac.nix

{ system ? builtins.currentSystem }:

with import ../../lib/testing.nix { inherit system; };
with import ../../lib/qemu-flags.nix;
with pkgs.lib;

let
  servers.master = "192.168.1.1";
  servers.one = "192.168.1.10";

  certs = import ./certs.nix { inherit servers; };

  roServiceAccount = pkgs.writeText "ro-service-account.json" (builtins.toJSON {
    kind = "ServiceAccount";
    apiVersion = "v1";
    metadata = {
      name = "read-only";
      namespace = "default";
    };
  });

  roRoleBinding = pkgs.writeText "ro-role-binding.json" (builtins.toJSON {
    "apiVersion" = "rbac.authorization.k8s.io/v1beta1";
    "kind" = "RoleBinding";
    "metadata" = {
      "name" = "read-pods";
      "namespace" = "default";
    };
    "roleRef" = {
      "apiGroup" = "rbac.authorization.k8s.io";
      "kind" = "Role";
      "name" = "pod-reader";
    };
    "subjects" = [{
      "kind" = "ServiceAccount";
      "name" = "read-only";
      "namespace" = "default";
    }];
  });

  roRole = pkgs.writeText "ro-role.json" (builtins.toJSON {
    "apiVersion" = "rbac.authorization.k8s.io/v1beta1";
    "kind" = "Role";
    "metadata" = {
      "name" = "pod-reader";
      "namespace" = "default";
    };
    "rules" = [{
      "apiGroups" = [""];
      "resources" = ["pods"];
      "verbs" = ["get" "list" "watch"];
    }];
  });

  kubectlPod = pkgs.writeText "kubectl-pod.json" (builtins.toJSON {
    kind = "Pod";
    apiVersion = "v1";
    metadata.name = "kubectl";
    metadata.namespace = "default";
    metadata.labels.name = "kubectl";
    spec.serviceAccountName = "read-only";
    spec.containers = [{
      name = "kubectl";
      image = "kubectl:latest";
      command = ["${pkgs.busybox}/bin/tail" "-f"];
      imagePullPolicy = "Never";
      tty = true;
    }];
  });

  kubectlPod2 = pkgs.writeTextDir "kubectl-pod-2.json" (builtins.toJSON {
    kind = "Pod";
    apiVersion = "v1";
    metadata.name = "kubectl-2";
    metadata.namespace = "default";
    metadata.labels.name = "kubectl-2";
    spec.serviceAccountName = "read-only";
    spec.containers = [{
      name = "kubectl-2";
      image = "kubectl:latest";
      command = ["${pkgs.busybox}/bin/tail" "-f"];
      imagePullPolicy = "Never";
      tty = true;
    }];
  });

  kubectlImage = pkgs.dockerTools.buildImage {
    name = "kubectl";
    tag = "latest";
    contents = [ pkgs.kubernetes pkgs.busybox kubectlPod2 ];  # certs kubeconfig
    config.Entrypoint = "${pkgs.busybox}/bin/sh";
  };

  test = ''
    $master->waitUntilSucceeds("kubectl get node master.nixos.xyz | grep Ready");
    $master->waitUntilSucceeds("kubectl get node one.nixos.xyz | grep Ready");

    $one->execute("docker load < ${kubectlImage}");

    $master->waitUntilSucceeds("kubectl apply -f ${roServiceAccount}");
    $master->waitUntilSucceeds("kubectl apply -f ${roRole}");
    $master->waitUntilSucceeds("kubectl apply -f ${roRoleBinding}");
    $master->waitUntilSucceeds("kubectl create -f ${kubectlPod} || kubectl apply -f ${kubectlPod}");

    $master->waitUntilSucceeds("kubectl get pod kubectl | grep Running");

    $master->succeed("kubectl exec -ti kubectl -- kubectl get pods");
    $master->fail("kubectl exec -ti kubectl -- kubectl create -f /kubectl-pod-2.json");
    $master->fail("kubectl exec -ti kubectl -- kubectl delete pods -l name=kubectl");
  '';

in makeTest {
  name = "kubernetes-multinode-rbac";

  nodes = {
    master =
      { config, pkgs, lib, nodes, ... }:
        mkMerge [
          {
            virtualisation.memorySize = 768;
            virtualisation.diskSize = 4096;
            networking.interfaces.eth1.ip4 = mkForce [{address = servers.master; prefixLength = 24;}];
            networking.primaryIPAddress = mkForce servers.master;
            services.kubernetes.apiserver.authorizationMode = mkForce ["ABAC" "RBAC"];
            services.kubernetes.apiserver.authorizationPolicy = mkForce [
              {
                apiVersion = "abac.authorization.kubernetes.io/v1beta1";
                kind = "Policy";
                spec = {
                  user  = "kubecfg";
                  namespace = "*";
                  resource = "*";
                  apiGroup = "*";
                  nonResourcePath = "*";
                };
              }
              {
                apiVersion = "abac.authorization.kubernetes.io/v1beta1";
                kind = "Policy";
                spec = {
                  user  = "kubelet";
                  namespace = "*";
                  resource = "*";
                  apiGroup = "*";
                  nonResourcePath = "*";
                };
              }
              {
                apiVersion = "abac.authorization.kubernetes.io/v1beta1";
                kind = "Policy";
                spec = {
                  user  = "kube-worker";
                  namespace = "*";
                  resource = "*";
                  apiGroup = "*";
                  nonResourcePath = "*";
                };
              }
              {
                apiVersion = "abac.authorization.kubernetes.io/v1beta1";
                kind = "Policy";
                spec = {
                  user  = "kube_proxy";
                  namespace = "*";
                  resource = "*";
                  apiGroup = "*";
                  nonResourcePath = "*";
                };
              }
              {
                apiVersion = "abac.authorization.kubernetes.io/v1beta1";
                kind = "Policy";
                spec = {
                  user  = "client";
                  namespace = "*";
                  resource = "*";
                  apiGroup = "*";
                  nonResourcePath = "*";
                };
              }
            ];
          }
          (import ./kubernetes-common.nix { inherit pkgs config certs servers; })
          (import ./kubernetes-master.nix { inherit pkgs config certs; })
        ];

    one =
      { config, pkgs, lib, nodes, ... }:
        mkMerge [
          {
            virtualisation.memorySize = 768;
            virtualisation.diskSize = 4096;
            networking.interfaces.eth1.ip4 = mkForce [{address = servers.one; prefixLength = 24;}];
            networking.primaryIPAddress = mkForce servers.one;
            services.kubernetes.roles = ["node"];
          }
          (import ./kubernetes-common.nix { inherit pkgs config certs servers; })
        ];
  };

  testScript = ''
    startAll;

    ${test}
  '';
}