709b6f664e
* Fix reference CNI plugins * The plugins were split out of the upstream cni repo around version 0.6.0 * Fix RBAC and DNS tests * Fix broken apiVersion fields * Change plugin linking to look in ${package}/bin rather than ${package.plugins} * Initial work towards a working e2e test * Test still fails, but at least the expression evaluates now Continues @srhb's work in #37199 Fixes #37199
1147 lines
38 KiB
Nix
1147 lines
38 KiB
Nix
{ config, lib, pkgs, ... }:
|
|
|
|
with lib;
|
|
|
|
let
|
|
cfg = config.services.kubernetes;
|
|
|
|
skipAttrs = attrs: map (filterAttrs (k: v: k != "enable"))
|
|
(filter (v: !(hasAttr "enable" v) || v.enable) attrs);
|
|
|
|
infraContainer = pkgs.dockerTools.buildImage {
|
|
name = "pause";
|
|
tag = "latest";
|
|
contents = cfg.package.pause;
|
|
config.Cmd = "/bin/pause";
|
|
};
|
|
|
|
mkKubeConfig = name: cfg: pkgs.writeText "${name}-kubeconfig" (builtins.toJSON {
|
|
apiVersion = "v1";
|
|
kind = "Config";
|
|
clusters = [{
|
|
name = "local";
|
|
cluster.certificate-authority = cfg.caFile;
|
|
cluster.server = cfg.server;
|
|
}];
|
|
users = [{
|
|
name = "kubelet";
|
|
user = {
|
|
client-certificate = cfg.certFile;
|
|
client-key = cfg.keyFile;
|
|
};
|
|
}];
|
|
contexts = [{
|
|
context = {
|
|
cluster = "local";
|
|
user = "kubelet";
|
|
};
|
|
current-context = "kubelet-context";
|
|
}];
|
|
});
|
|
|
|
mkKubeConfigOptions = prefix: {
|
|
server = mkOption {
|
|
description = "${prefix} kube-apiserver server address.";
|
|
default = "http://${cfg.apiserver.address}:${toString cfg.apiserver.port}";
|
|
type = types.str;
|
|
};
|
|
|
|
caFile = mkOption {
|
|
description = "${prefix} certificate authrority file used to connect to kube-apiserver.";
|
|
type = types.nullOr types.path;
|
|
default = cfg.caFile;
|
|
};
|
|
|
|
certFile = mkOption {
|
|
description = "${prefix} client certificate file used to connect to kube-apiserver.";
|
|
type = types.nullOr types.path;
|
|
default = null;
|
|
};
|
|
|
|
keyFile = mkOption {
|
|
description = "${prefix} client key file used to connect to kube-apiserver.";
|
|
type = types.nullOr types.path;
|
|
default = null;
|
|
};
|
|
};
|
|
|
|
kubeConfigDefaults = {
|
|
server = mkDefault cfg.kubeconfig.server;
|
|
caFile = mkDefault cfg.kubeconfig.caFile;
|
|
certFile = mkDefault cfg.kubeconfig.certFile;
|
|
keyFile = mkDefault cfg.kubeconfig.keyFile;
|
|
};
|
|
|
|
cniConfig = pkgs.buildEnv {
|
|
name = "kubernetes-cni-config";
|
|
paths = imap (i: entry:
|
|
pkgs.writeTextDir "${toString (10+i)}-${entry.type}.conf" (builtins.toJSON entry)
|
|
) cfg.kubelet.cni.config;
|
|
};
|
|
|
|
manifests = pkgs.buildEnv {
|
|
name = "kubernetes-manifests";
|
|
paths = mapAttrsToList (name: manifest:
|
|
pkgs.writeTextDir "${name}.json" (builtins.toJSON manifest)
|
|
) cfg.kubelet.manifests;
|
|
};
|
|
|
|
addons = pkgs.runCommand "kubernetes-addons" { } ''
|
|
mkdir -p $out
|
|
# since we are mounting the addons to the addon manager, they need to be copied
|
|
${concatMapStringsSep ";" (a: "cp -v ${a}/* $out/") (mapAttrsToList (name: addon:
|
|
pkgs.writeTextDir "${name}.json" (builtins.toJSON addon)
|
|
) (cfg.addonManager.addons))}
|
|
'';
|
|
|
|
taintOptions = { name, ... }: {
|
|
options = {
|
|
key = mkOption {
|
|
description = "Key of taint.";
|
|
default = name;
|
|
type = types.str;
|
|
};
|
|
value = mkOption {
|
|
description = "Value of taint.";
|
|
type = types.str;
|
|
};
|
|
effect = mkOption {
|
|
description = "Effect of taint.";
|
|
example = "NoSchedule";
|
|
type = types.enum ["NoSchedule" "PreferNoSchedule" "NoExecute"];
|
|
};
|
|
};
|
|
};
|
|
|
|
taints = concatMapStringsSep "," (v: "${v.key}=${v.value}:${v.effect}") (mapAttrsToList (n: v: v) cfg.kubelet.taints);
|
|
|
|
# needed for flannel to pass options to docker
|
|
mkDockerOpts = pkgs.runCommand "mk-docker-opts" {
|
|
buildInputs = [ pkgs.makeWrapper ];
|
|
} ''
|
|
mkdir -p $out
|
|
cp ${pkgs.kubernetes.src}/cluster/centos/node/bin/mk-docker-opts.sh $out/mk-docker-opts.sh
|
|
|
|
# bashInteractive needed for `compgen`
|
|
makeWrapper ${pkgs.bashInteractive}/bin/bash $out/mk-docker-opts --add-flags "$out/mk-docker-opts.sh"
|
|
'';
|
|
in {
|
|
|
|
###### interface
|
|
|
|
options.services.kubernetes = {
|
|
roles = mkOption {
|
|
description = ''
|
|
Kubernetes role that this machine should take.
|
|
|
|
Master role will enable etcd, apiserver, scheduler and controller manager
|
|
services. Node role will enable etcd, docker, kubelet and proxy services.
|
|
'';
|
|
default = [];
|
|
type = types.listOf (types.enum ["master" "node"]);
|
|
};
|
|
|
|
package = mkOption {
|
|
description = "Kubernetes package to use.";
|
|
type = types.package;
|
|
default = pkgs.kubernetes;
|
|
defaultText = "pkgs.kubernetes";
|
|
};
|
|
|
|
verbose = mkOption {
|
|
description = "Kubernetes enable verbose mode for debugging.";
|
|
default = false;
|
|
type = types.bool;
|
|
};
|
|
|
|
etcd = {
|
|
servers = mkOption {
|
|
description = "List of etcd servers. By default etcd is started, except if this option is changed.";
|
|
default = ["http://127.0.0.1:2379"];
|
|
type = types.listOf types.str;
|
|
};
|
|
|
|
keyFile = mkOption {
|
|
description = "Etcd key file.";
|
|
default = null;
|
|
type = types.nullOr types.path;
|
|
};
|
|
|
|
certFile = mkOption {
|
|
description = "Etcd cert file.";
|
|
default = null;
|
|
type = types.nullOr types.path;
|
|
};
|
|
|
|
caFile = mkOption {
|
|
description = "Etcd ca file.";
|
|
default = cfg.caFile;
|
|
type = types.nullOr types.path;
|
|
};
|
|
};
|
|
|
|
kubeconfig = mkKubeConfigOptions "Default kubeconfig";
|
|
|
|
caFile = mkOption {
|
|
description = "Default kubernetes certificate authority";
|
|
type = types.nullOr types.path;
|
|
default = null;
|
|
};
|
|
|
|
dataDir = mkOption {
|
|
description = "Kubernetes root directory for managing kubelet files.";
|
|
default = "/var/lib/kubernetes";
|
|
type = types.path;
|
|
};
|
|
|
|
featureGates = mkOption {
|
|
description = "List set of feature gates";
|
|
default = [];
|
|
type = types.listOf types.str;
|
|
};
|
|
|
|
apiserver = {
|
|
enable = mkOption {
|
|
description = "Whether to enable Kubernetes apiserver.";
|
|
default = false;
|
|
type = types.bool;
|
|
};
|
|
|
|
featureGates = mkOption {
|
|
description = "List set of feature gates";
|
|
default = cfg.featureGates;
|
|
type = types.listOf types.str;
|
|
};
|
|
|
|
address = mkOption {
|
|
description = "Kubernetes apiserver listening address.";
|
|
default = "127.0.0.1";
|
|
type = types.str;
|
|
};
|
|
|
|
publicAddress = mkOption {
|
|
description = ''
|
|
Kubernetes apiserver public listening address used for read only and
|
|
secure port.
|
|
'';
|
|
default = cfg.apiserver.address;
|
|
type = types.str;
|
|
};
|
|
|
|
advertiseAddress = mkOption {
|
|
description = ''
|
|
Kubernetes apiserver IP address on which to advertise the apiserver
|
|
to members of the cluster. This address must be reachable by the rest
|
|
of the cluster.
|
|
'';
|
|
default = null;
|
|
type = types.nullOr types.str;
|
|
};
|
|
|
|
storageBackend = mkOption {
|
|
description = ''
|
|
Kubernetes apiserver storage backend.
|
|
'';
|
|
default = "etcd3";
|
|
type = types.enum ["etcd2" "etcd3"];
|
|
};
|
|
|
|
port = mkOption {
|
|
description = "Kubernetes apiserver listening port.";
|
|
default = 8080;
|
|
type = types.int;
|
|
};
|
|
|
|
securePort = mkOption {
|
|
description = "Kubernetes apiserver secure port.";
|
|
default = 443;
|
|
type = types.int;
|
|
};
|
|
|
|
tlsCertFile = mkOption {
|
|
description = "Kubernetes apiserver certificate file.";
|
|
default = null;
|
|
type = types.nullOr types.path;
|
|
};
|
|
|
|
tlsKeyFile = mkOption {
|
|
description = "Kubernetes apiserver private key file.";
|
|
default = null;
|
|
type = types.nullOr types.path;
|
|
};
|
|
|
|
clientCaFile = mkOption {
|
|
description = "Kubernetes apiserver CA file for client auth.";
|
|
default = cfg.caFile;
|
|
type = types.nullOr types.path;
|
|
};
|
|
|
|
tokenAuthFile = mkOption {
|
|
description = ''
|
|
Kubernetes apiserver token authentication file. See
|
|
<link xlink:href="https://kubernetes.io/docs/admin/authentication.html"/>
|
|
'';
|
|
default = null;
|
|
type = types.nullOr types.path;
|
|
};
|
|
|
|
basicAuthFile = mkOption {
|
|
description = ''
|
|
Kubernetes apiserver basic authentication file. See
|
|
<link xlink:href="https://kubernetes.io/docs/admin/authentication.html"/>
|
|
'';
|
|
default = pkgs.writeText "users" ''
|
|
kubernetes,admin,0
|
|
'';
|
|
type = types.nullOr types.path;
|
|
};
|
|
|
|
authorizationMode = mkOption {
|
|
description = ''
|
|
Kubernetes apiserver authorization mode (AlwaysAllow/AlwaysDeny/ABAC/RBAC). See
|
|
<link xlink:href="https://kubernetes.io/docs/admin/authorization.html"/>
|
|
'';
|
|
default = ["RBAC" "Node"];
|
|
type = types.listOf (types.enum ["AlwaysAllow" "AlwaysDeny" "ABAC" "RBAC" "Node"]);
|
|
};
|
|
|
|
authorizationPolicy = mkOption {
|
|
description = ''
|
|
Kubernetes apiserver authorization policy file. See
|
|
<link xlink:href="https://kubernetes.io/docs/admin/authorization.html"/>
|
|
'';
|
|
default = [];
|
|
type = types.listOf types.attrs;
|
|
};
|
|
|
|
allowPrivileged = mkOption {
|
|
description = "Whether to allow privileged containers on Kubernetes.";
|
|
default = true;
|
|
type = types.bool;
|
|
};
|
|
|
|
serviceClusterIpRange = mkOption {
|
|
description = ''
|
|
A CIDR notation IP range from which to assign service cluster IPs.
|
|
This must not overlap with any IP ranges assigned to nodes for pods.
|
|
'';
|
|
default = "10.0.0.0/24";
|
|
type = types.str;
|
|
};
|
|
|
|
runtimeConfig = mkOption {
|
|
description = ''
|
|
Api runtime configuration. See
|
|
<link xlink:href="https://kubernetes.io/docs/admin/cluster-management.html"/>
|
|
'';
|
|
default = "authentication.k8s.io/v1beta1=true";
|
|
example = "api/all=false,api/v1=true";
|
|
type = types.str;
|
|
};
|
|
|
|
admissionControl = mkOption {
|
|
description = ''
|
|
Kubernetes admission control plugins to use. See
|
|
<link xlink:href="https://kubernetes.io/docs/admin/admission-controllers/"/>
|
|
'';
|
|
default = ["NamespaceLifecycle" "LimitRanger" "ServiceAccount" "ResourceQuota" "DefaultStorageClass" "DefaultTolerationSeconds" "NodeRestriction"];
|
|
example = [
|
|
"NamespaceLifecycle" "NamespaceExists" "LimitRanger"
|
|
"SecurityContextDeny" "ServiceAccount" "ResourceQuota"
|
|
"PodSecurityPolicy" "NodeRestriction" "DefaultStorageClass"
|
|
];
|
|
type = types.listOf types.str;
|
|
};
|
|
|
|
serviceAccountKeyFile = mkOption {
|
|
description = ''
|
|
Kubernetes apiserver PEM-encoded x509 RSA private or public key file,
|
|
used to verify ServiceAccount tokens. By default tls private key file
|
|
is used.
|
|
'';
|
|
default = null;
|
|
type = types.nullOr types.path;
|
|
};
|
|
|
|
kubeletClientCaFile = mkOption {
|
|
description = "Path to a cert file for connecting to kubelet.";
|
|
default = cfg.caFile;
|
|
type = types.nullOr types.path;
|
|
};
|
|
|
|
kubeletClientCertFile = mkOption {
|
|
description = "Client certificate to use for connections to kubelet.";
|
|
default = null;
|
|
type = types.nullOr types.path;
|
|
};
|
|
|
|
kubeletClientKeyFile = mkOption {
|
|
description = "Key to use for connections to kubelet.";
|
|
default = null;
|
|
type = types.nullOr types.path;
|
|
};
|
|
|
|
kubeletHttps = mkOption {
|
|
description = "Whether to use https for connections to kubelet.";
|
|
default = true;
|
|
type = types.bool;
|
|
};
|
|
|
|
extraOpts = mkOption {
|
|
description = "Kubernetes apiserver extra command line options.";
|
|
default = "";
|
|
type = types.str;
|
|
};
|
|
};
|
|
|
|
scheduler = {
|
|
enable = mkOption {
|
|
description = "Whether to enable Kubernetes scheduler.";
|
|
default = false;
|
|
type = types.bool;
|
|
};
|
|
|
|
featureGates = mkOption {
|
|
description = "List set of feature gates";
|
|
default = cfg.featureGates;
|
|
type = types.listOf types.str;
|
|
};
|
|
|
|
address = mkOption {
|
|
description = "Kubernetes scheduler listening address.";
|
|
default = "127.0.0.1";
|
|
type = types.str;
|
|
};
|
|
|
|
port = mkOption {
|
|
description = "Kubernetes scheduler listening port.";
|
|
default = 10251;
|
|
type = types.int;
|
|
};
|
|
|
|
leaderElect = mkOption {
|
|
description = "Whether to start leader election before executing main loop.";
|
|
type = types.bool;
|
|
default = true;
|
|
};
|
|
|
|
kubeconfig = mkKubeConfigOptions "Kubernetes scheduler";
|
|
|
|
extraOpts = mkOption {
|
|
description = "Kubernetes scheduler extra command line options.";
|
|
default = "";
|
|
type = types.str;
|
|
};
|
|
};
|
|
|
|
controllerManager = {
|
|
enable = mkOption {
|
|
description = "Whether to enable Kubernetes controller manager.";
|
|
default = false;
|
|
type = types.bool;
|
|
};
|
|
|
|
featureGates = mkOption {
|
|
description = "List set of feature gates";
|
|
default = cfg.featureGates;
|
|
type = types.listOf types.str;
|
|
};
|
|
|
|
address = mkOption {
|
|
description = "Kubernetes controller manager listening address.";
|
|
default = "127.0.0.1";
|
|
type = types.str;
|
|
};
|
|
|
|
port = mkOption {
|
|
description = "Kubernetes controller manager listening port.";
|
|
default = 10252;
|
|
type = types.int;
|
|
};
|
|
|
|
leaderElect = mkOption {
|
|
description = "Whether to start leader election before executing main loop.";
|
|
type = types.bool;
|
|
default = true;
|
|
};
|
|
|
|
serviceAccountKeyFile = mkOption {
|
|
description = ''
|
|
Kubernetes controller manager PEM-encoded private RSA key file used to
|
|
sign service account tokens
|
|
'';
|
|
default = null;
|
|
type = types.nullOr types.path;
|
|
};
|
|
|
|
rootCaFile = mkOption {
|
|
description = ''
|
|
Kubernetes controller manager certificate authority file included in
|
|
service account's token secret.
|
|
'';
|
|
default = cfg.caFile;
|
|
type = types.nullOr types.path;
|
|
};
|
|
|
|
kubeconfig = mkKubeConfigOptions "Kubernetes controller manager";
|
|
|
|
extraOpts = mkOption {
|
|
description = "Kubernetes controller manager extra command line options.";
|
|
default = "";
|
|
type = types.str;
|
|
};
|
|
};
|
|
|
|
kubelet = {
|
|
enable = mkOption {
|
|
description = "Whether to enable Kubernetes kubelet.";
|
|
default = false;
|
|
type = types.bool;
|
|
};
|
|
|
|
featureGates = mkOption {
|
|
description = "List set of feature gates";
|
|
default = cfg.featureGates;
|
|
type = types.listOf types.str;
|
|
};
|
|
|
|
seedDockerImages = mkOption {
|
|
description = "List of docker images to preload on system";
|
|
default = [];
|
|
type = types.listOf types.package;
|
|
};
|
|
|
|
registerNode = mkOption {
|
|
description = "Whether to auto register kubelet with API server.";
|
|
default = true;
|
|
type = types.bool;
|
|
};
|
|
|
|
address = mkOption {
|
|
description = "Kubernetes kubelet info server listening address.";
|
|
default = "0.0.0.0";
|
|
type = types.str;
|
|
};
|
|
|
|
port = mkOption {
|
|
description = "Kubernetes kubelet info server listening port.";
|
|
default = 10250;
|
|
type = types.int;
|
|
};
|
|
|
|
tlsCertFile = mkOption {
|
|
description = "File containing x509 Certificate for HTTPS.";
|
|
default = null;
|
|
type = types.nullOr types.path;
|
|
};
|
|
|
|
tlsKeyFile = mkOption {
|
|
description = "File containing x509 private key matching tlsCertFile.";
|
|
default = null;
|
|
type = types.nullOr types.path;
|
|
};
|
|
|
|
clientCaFile = mkOption {
|
|
description = "Kubernetes apiserver CA file for client authentication.";
|
|
default = cfg.caFile;
|
|
type = types.nullOr types.path;
|
|
};
|
|
|
|
healthz = {
|
|
bind = mkOption {
|
|
description = "Kubernetes kubelet healthz listening address.";
|
|
default = "127.0.0.1";
|
|
type = types.str;
|
|
};
|
|
|
|
port = mkOption {
|
|
description = "Kubernetes kubelet healthz port.";
|
|
default = 10248;
|
|
type = types.int;
|
|
};
|
|
};
|
|
|
|
hostname = mkOption {
|
|
description = "Kubernetes kubelet hostname override.";
|
|
default = config.networking.hostName;
|
|
type = types.str;
|
|
};
|
|
|
|
allowPrivileged = mkOption {
|
|
description = "Whether to allow Kubernetes containers to request privileged mode.";
|
|
default = true;
|
|
type = types.bool;
|
|
};
|
|
|
|
cadvisorPort = mkOption {
|
|
description = "Kubernetes kubelet local cadvisor port.";
|
|
default = 4194;
|
|
type = types.int;
|
|
};
|
|
|
|
clusterDns = mkOption {
|
|
description = "Use alternative DNS.";
|
|
default = "10.1.0.1";
|
|
type = types.str;
|
|
};
|
|
|
|
clusterDomain = mkOption {
|
|
description = "Use alternative domain.";
|
|
default = config.services.kubernetes.addons.dns.clusterDomain;
|
|
type = types.str;
|
|
};
|
|
|
|
networkPlugin = mkOption {
|
|
description = "Network plugin to use by Kubernetes.";
|
|
type = types.nullOr (types.enum ["cni" "kubenet"]);
|
|
default = "kubenet";
|
|
};
|
|
|
|
cni = {
|
|
packages = mkOption {
|
|
description = "List of network plugin packages to install.";
|
|
type = types.listOf types.package;
|
|
default = [];
|
|
};
|
|
|
|
config = mkOption {
|
|
description = "Kubernetes CNI configuration.";
|
|
type = types.listOf types.attrs;
|
|
default = [];
|
|
example = literalExample ''
|
|
[{
|
|
"cniVersion": "0.2.0",
|
|
"name": "mynet",
|
|
"type": "bridge",
|
|
"bridge": "cni0",
|
|
"isGateway": true,
|
|
"ipMasq": true,
|
|
"ipam": {
|
|
"type": "host-local",
|
|
"subnet": "10.22.0.0/16",
|
|
"routes": [
|
|
{ "dst": "0.0.0.0/0" }
|
|
]
|
|
}
|
|
} {
|
|
"cniVersion": "0.2.0",
|
|
"type": "loopback"
|
|
}]
|
|
'';
|
|
};
|
|
};
|
|
|
|
manifests = mkOption {
|
|
description = "List of manifests to bootstrap with kubelet (only pods can be created as manifest entry)";
|
|
type = types.attrsOf types.attrs;
|
|
default = {};
|
|
};
|
|
|
|
applyManifests = mkOption {
|
|
description = "Whether to apply manifests (this is true for master node).";
|
|
default = false;
|
|
type = types.bool;
|
|
};
|
|
|
|
unschedulable = mkOption {
|
|
description = "Whether to set node taint to unschedulable=true as it is the case of node that has only master role.";
|
|
default = false;
|
|
type = types.bool;
|
|
};
|
|
|
|
taints = mkOption {
|
|
description = "Node taints (https://kubernetes.io/docs/concepts/configuration/assign-pod-node/).";
|
|
default = {};
|
|
type = types.attrsOf (types.submodule [ taintOptions ]);
|
|
};
|
|
|
|
nodeIp = mkOption {
|
|
description = "IP address of the node. If set, kubelet will use this IP address for the node.";
|
|
default = null;
|
|
type = types.nullOr types.str;
|
|
};
|
|
|
|
kubeconfig = mkKubeConfigOptions "Kubelet";
|
|
|
|
extraOpts = mkOption {
|
|
description = "Kubernetes kubelet extra command line options.";
|
|
default = "";
|
|
type = types.str;
|
|
};
|
|
};
|
|
|
|
proxy = {
|
|
enable = mkOption {
|
|
description = "Whether to enable Kubernetes proxy.";
|
|
default = false;
|
|
type = types.bool;
|
|
};
|
|
|
|
featureGates = mkOption {
|
|
description = "List set of feature gates";
|
|
default = cfg.featureGates;
|
|
type = types.listOf types.str;
|
|
};
|
|
|
|
address = mkOption {
|
|
description = "Kubernetes proxy listening address.";
|
|
default = "0.0.0.0";
|
|
type = types.str;
|
|
};
|
|
|
|
kubeconfig = mkKubeConfigOptions "Kubernetes proxy";
|
|
|
|
extraOpts = mkOption {
|
|
description = "Kubernetes proxy extra command line options.";
|
|
default = "";
|
|
type = types.str;
|
|
};
|
|
};
|
|
|
|
addonManager = {
|
|
enable = mkOption {
|
|
description = "Whether to enable Kubernetes addon manager.";
|
|
default = false;
|
|
type = types.bool;
|
|
};
|
|
|
|
addons = mkOption {
|
|
description = "Kubernetes addons (any kind of Kubernetes resource can be an addon).";
|
|
default = { };
|
|
type = types.attrsOf (types.either types.attrs (types.listOf types.attrs));
|
|
example = literalExample ''
|
|
{
|
|
"my-service" = {
|
|
"apiVersion" = "v1";
|
|
"kind" = "Service";
|
|
"metadata" = {
|
|
"name" = "my-service";
|
|
"namespace" = "default";
|
|
};
|
|
"spec" = { ... };
|
|
};
|
|
}
|
|
// import <nixpkgs/nixos/modules/services/cluster/kubernetes/dashboard.nix> { cfg = config.services.kubernetes; };
|
|
'';
|
|
};
|
|
};
|
|
|
|
path = mkOption {
|
|
description = "Packages added to the services' PATH environment variable. Both the bin and sbin subdirectories of each package are added.";
|
|
type = types.listOf types.package;
|
|
default = [];
|
|
};
|
|
|
|
clusterCidr = mkOption {
|
|
description = "Kubernetes controller manager and proxy CIDR Range for Pods in cluster.";
|
|
default = "10.1.0.0/16";
|
|
type = types.str;
|
|
};
|
|
|
|
flannel.enable = mkOption {
|
|
description = "Whether to enable flannel networking";
|
|
default = false;
|
|
type = types.bool;
|
|
};
|
|
|
|
};
|
|
|
|
###### implementation
|
|
|
|
config = mkMerge [
|
|
(mkIf cfg.kubelet.enable {
|
|
services.kubernetes.kubelet.seedDockerImages = [infraContainer];
|
|
|
|
systemd.services.kubelet-bootstrap = {
|
|
description = "Boostrap Kubelet";
|
|
wantedBy = ["kubernetes.target"];
|
|
after = ["docker.service" "network.target"];
|
|
path = with pkgs; [ docker ];
|
|
script = ''
|
|
${concatMapStrings (img: ''
|
|
echo "Seeding docker image: ${img}"
|
|
docker load <${img}
|
|
'') cfg.kubelet.seedDockerImages}
|
|
|
|
rm /opt/cni/bin/* || true
|
|
${concatMapStrings (package: ''
|
|
echo "Linking cni package: ${package}"
|
|
ln -fs ${package}/bin/* /opt/cni/bin
|
|
'') cfg.kubelet.cni.packages}
|
|
'';
|
|
serviceConfig = {
|
|
Slice = "kubernetes.slice";
|
|
Type = "oneshot";
|
|
};
|
|
};
|
|
|
|
systemd.services.kubelet = {
|
|
description = "Kubernetes Kubelet Service";
|
|
wantedBy = [ "kubernetes.target" ];
|
|
after = [ "network.target" "docker.service" "kube-apiserver.service" "kubelet-bootstrap.service" ];
|
|
path = with pkgs; [ gitMinimal openssh docker utillinux iproute ethtool thin-provisioning-tools iptables socat ] ++ cfg.path;
|
|
serviceConfig = {
|
|
Slice = "kubernetes.slice";
|
|
ExecStart = ''${cfg.package}/bin/kubelet \
|
|
${optionalString cfg.kubelet.applyManifests
|
|
"--pod-manifest-path=${manifests}"} \
|
|
${optionalString (taints != "")
|
|
"--register-with-taints=${taints}"} \
|
|
--kubeconfig=${mkKubeConfig "kubelet" cfg.kubelet.kubeconfig} \
|
|
--require-kubeconfig \
|
|
--address=${cfg.kubelet.address} \
|
|
--port=${toString cfg.kubelet.port} \
|
|
--register-node=${boolToString cfg.kubelet.registerNode} \
|
|
${optionalString (cfg.kubelet.tlsCertFile != null)
|
|
"--tls-cert-file=${cfg.kubelet.tlsCertFile}"} \
|
|
${optionalString (cfg.kubelet.tlsKeyFile != null)
|
|
"--tls-private-key-file=${cfg.kubelet.tlsKeyFile}"} \
|
|
${optionalString (cfg.kubelet.clientCaFile != null)
|
|
"--client-ca-file=${cfg.kubelet.clientCaFile}"} \
|
|
--authentication-token-webhook \
|
|
--authentication-token-webhook-cache-ttl="10s" \
|
|
--authorization-mode=Webhook \
|
|
--healthz-bind-address=${cfg.kubelet.healthz.bind} \
|
|
--healthz-port=${toString cfg.kubelet.healthz.port} \
|
|
--hostname-override=${cfg.kubelet.hostname} \
|
|
--allow-privileged=${boolToString cfg.kubelet.allowPrivileged} \
|
|
--root-dir=${cfg.dataDir} \
|
|
--cadvisor_port=${toString cfg.kubelet.cadvisorPort} \
|
|
${optionalString (cfg.kubelet.clusterDns != "")
|
|
"--cluster-dns=${cfg.kubelet.clusterDns}"} \
|
|
${optionalString (cfg.kubelet.clusterDomain != "")
|
|
"--cluster-domain=${cfg.kubelet.clusterDomain}"} \
|
|
--pod-infra-container-image=pause \
|
|
${optionalString (cfg.kubelet.networkPlugin != null)
|
|
"--network-plugin=${cfg.kubelet.networkPlugin}"} \
|
|
--cni-conf-dir=${cniConfig} \
|
|
--hairpin-mode=hairpin-veth \
|
|
${optionalString (cfg.kubelet.nodeIp != null)
|
|
"--node-ip=${cfg.kubelet.nodeIp}"} \
|
|
${optionalString (cfg.kubelet.featureGates != [])
|
|
"--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.kubelet.featureGates}"} \
|
|
${optionalString cfg.verbose "--v=6 --log_flush_frequency=1s"} \
|
|
${cfg.kubelet.extraOpts}
|
|
'';
|
|
WorkingDirectory = cfg.dataDir;
|
|
};
|
|
};
|
|
|
|
# Allways include cni plugins
|
|
services.kubernetes.kubelet.cni.packages = [pkgs.cni-plugins];
|
|
|
|
boot.kernelModules = ["br_netfilter"];
|
|
|
|
services.kubernetes.kubelet.kubeconfig = kubeConfigDefaults;
|
|
})
|
|
|
|
(mkIf (cfg.kubelet.applyManifests && cfg.kubelet.enable) {
|
|
environment.etc = mapAttrs' (name: manifest:
|
|
nameValuePair "kubernetes/manifests/${name}.json" {
|
|
text = builtins.toJSON manifest;
|
|
mode = "0755";
|
|
}
|
|
) cfg.kubelet.manifests;
|
|
})
|
|
|
|
(mkIf (cfg.kubelet.unschedulable && cfg.kubelet.enable) {
|
|
services.kubernetes.kubelet.taints.unschedulable = {
|
|
value = "true";
|
|
effect = "NoSchedule";
|
|
};
|
|
})
|
|
|
|
(mkIf cfg.apiserver.enable {
|
|
systemd.services.kube-apiserver = {
|
|
description = "Kubernetes Kubelet Service";
|
|
wantedBy = [ "kubernetes.target" ];
|
|
after = [ "network.target" "docker.service" ];
|
|
serviceConfig = {
|
|
Slice = "kubernetes.slice";
|
|
ExecStart = ''${cfg.package}/bin/kube-apiserver \
|
|
--etcd-servers=${concatStringsSep "," cfg.etcd.servers} \
|
|
${optionalString (cfg.etcd.caFile != null)
|
|
"--etcd-cafile=${cfg.etcd.caFile}"} \
|
|
${optionalString (cfg.etcd.certFile != null)
|
|
"--etcd-certfile=${cfg.etcd.certFile}"} \
|
|
${optionalString (cfg.etcd.keyFile != null)
|
|
"--etcd-keyfile=${cfg.etcd.keyFile}"} \
|
|
--insecure-port=${toString cfg.apiserver.port} \
|
|
--bind-address=0.0.0.0 \
|
|
${optionalString (cfg.apiserver.advertiseAddress != null)
|
|
"--advertise-address=${cfg.apiserver.advertiseAddress}"} \
|
|
--allow-privileged=${boolToString cfg.apiserver.allowPrivileged}\
|
|
${optionalString (cfg.apiserver.tlsCertFile != null)
|
|
"--tls-cert-file=${cfg.apiserver.tlsCertFile}"} \
|
|
${optionalString (cfg.apiserver.tlsKeyFile != null)
|
|
"--tls-private-key-file=${cfg.apiserver.tlsKeyFile}"} \
|
|
${optionalString (cfg.apiserver.tokenAuthFile != null)
|
|
"--token-auth-file=${cfg.apiserver.tokenAuthFile}"} \
|
|
${optionalString (cfg.apiserver.basicAuthFile != null)
|
|
"--basic-auth-file=${cfg.apiserver.basicAuthFile}"} \
|
|
--kubelet-https=${if cfg.apiserver.kubeletHttps then "true" else "false"} \
|
|
${optionalString (cfg.apiserver.kubeletClientCaFile != null)
|
|
"--kubelet-certificate-authority=${cfg.apiserver.kubeletClientCaFile}"} \
|
|
${optionalString (cfg.apiserver.kubeletClientCertFile != null)
|
|
"--kubelet-client-certificate=${cfg.apiserver.kubeletClientCertFile}"} \
|
|
${optionalString (cfg.apiserver.kubeletClientKeyFile != null)
|
|
"--kubelet-client-key=${cfg.apiserver.kubeletClientKeyFile}"} \
|
|
${optionalString (cfg.apiserver.clientCaFile != null)
|
|
"--client-ca-file=${cfg.apiserver.clientCaFile}"} \
|
|
--authorization-mode=${concatStringsSep "," cfg.apiserver.authorizationMode} \
|
|
${optionalString (elem "ABAC" cfg.apiserver.authorizationMode)
|
|
"--authorization-policy-file=${
|
|
pkgs.writeText "kube-auth-policy.jsonl"
|
|
(concatMapStringsSep "\n" (l: builtins.toJSON l) cfg.apiserver.authorizationPolicy)
|
|
}"
|
|
} \
|
|
--secure-port=${toString cfg.apiserver.securePort} \
|
|
--service-cluster-ip-range=${cfg.apiserver.serviceClusterIpRange} \
|
|
${optionalString (cfg.apiserver.runtimeConfig != "")
|
|
"--runtime-config=${cfg.apiserver.runtimeConfig}"} \
|
|
--admission_control=${concatStringsSep "," cfg.apiserver.admissionControl} \
|
|
${optionalString (cfg.apiserver.serviceAccountKeyFile!=null)
|
|
"--service-account-key-file=${cfg.apiserver.serviceAccountKeyFile}"} \
|
|
${optionalString cfg.verbose "--v=6"} \
|
|
${optionalString cfg.verbose "--log-flush-frequency=1s"} \
|
|
--storage-backend=${cfg.apiserver.storageBackend} \
|
|
${optionalString (cfg.kubelet.featureGates != [])
|
|
"--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.kubelet.featureGates}"} \
|
|
${cfg.apiserver.extraOpts}
|
|
'';
|
|
WorkingDirectory = cfg.dataDir;
|
|
User = "kubernetes";
|
|
Group = "kubernetes";
|
|
AmbientCapabilities = "cap_net_bind_service";
|
|
Restart = "on-failure";
|
|
RestartSec = 5;
|
|
};
|
|
};
|
|
})
|
|
|
|
(mkIf cfg.scheduler.enable {
|
|
systemd.services.kube-scheduler = {
|
|
description = "Kubernetes Scheduler Service";
|
|
wantedBy = [ "kubernetes.target" ];
|
|
after = [ "kube-apiserver.service" ];
|
|
serviceConfig = {
|
|
Slice = "kubernetes.slice";
|
|
ExecStart = ''${cfg.package}/bin/kube-scheduler \
|
|
--address=${cfg.scheduler.address} \
|
|
--port=${toString cfg.scheduler.port} \
|
|
--leader-elect=${boolToString cfg.scheduler.leaderElect} \
|
|
--kubeconfig=${mkKubeConfig "kube-scheduler" cfg.scheduler.kubeconfig} \
|
|
${optionalString cfg.verbose "--v=6"} \
|
|
${optionalString cfg.verbose "--log-flush-frequency=1s"} \
|
|
${optionalString (cfg.scheduler.featureGates != [])
|
|
"--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.scheduler.featureGates}"} \
|
|
${cfg.scheduler.extraOpts}
|
|
'';
|
|
WorkingDirectory = cfg.dataDir;
|
|
User = "kubernetes";
|
|
Group = "kubernetes";
|
|
};
|
|
};
|
|
|
|
services.kubernetes.scheduler.kubeconfig = kubeConfigDefaults;
|
|
})
|
|
|
|
(mkIf cfg.controllerManager.enable {
|
|
systemd.services.kube-controller-manager = {
|
|
description = "Kubernetes Controller Manager Service";
|
|
wantedBy = [ "kubernetes.target" ];
|
|
after = [ "kube-apiserver.service" ];
|
|
serviceConfig = {
|
|
RestartSec = "30s";
|
|
Restart = "on-failure";
|
|
Slice = "kubernetes.slice";
|
|
ExecStart = ''${cfg.package}/bin/kube-controller-manager \
|
|
--address=${cfg.controllerManager.address} \
|
|
--port=${toString cfg.controllerManager.port} \
|
|
--kubeconfig=${mkKubeConfig "kube-controller-manager" cfg.controllerManager.kubeconfig} \
|
|
--leader-elect=${boolToString cfg.controllerManager.leaderElect} \
|
|
${if (cfg.controllerManager.serviceAccountKeyFile!=null)
|
|
then "--service-account-private-key-file=${cfg.controllerManager.serviceAccountKeyFile}"
|
|
else "--service-account-private-key-file=/var/run/kubernetes/apiserver.key"} \
|
|
${if (cfg.controllerManager.rootCaFile!=null)
|
|
then "--root-ca-file=${cfg.controllerManager.rootCaFile}"
|
|
else "--root-ca-file=/var/run/kubernetes/apiserver.crt"} \
|
|
${optionalString (cfg.clusterCidr!=null)
|
|
"--cluster-cidr=${cfg.clusterCidr}"} \
|
|
--allocate-node-cidrs=true \
|
|
${optionalString (cfg.controllerManager.featureGates != [])
|
|
"--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.controllerManager.featureGates}"} \
|
|
${optionalString cfg.verbose "--v=6"} \
|
|
${optionalString cfg.verbose "--log-flush-frequency=1s"} \
|
|
${cfg.controllerManager.extraOpts}
|
|
'';
|
|
WorkingDirectory = cfg.dataDir;
|
|
User = "kubernetes";
|
|
Group = "kubernetes";
|
|
};
|
|
path = cfg.path;
|
|
};
|
|
|
|
services.kubernetes.controllerManager.kubeconfig = kubeConfigDefaults;
|
|
})
|
|
|
|
(mkIf cfg.proxy.enable {
|
|
systemd.services.kube-proxy = {
|
|
description = "Kubernetes Proxy Service";
|
|
wantedBy = [ "kubernetes.target" ];
|
|
after = [ "kube-apiserver.service" ];
|
|
path = [pkgs.iptables pkgs.conntrack_tools];
|
|
serviceConfig = {
|
|
Slice = "kubernetes.slice";
|
|
ExecStart = ''${cfg.package}/bin/kube-proxy \
|
|
--kubeconfig=${mkKubeConfig "kube-proxy" cfg.proxy.kubeconfig} \
|
|
--bind-address=${cfg.proxy.address} \
|
|
${optionalString (cfg.proxy.featureGates != [])
|
|
"--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.proxy.featureGates}"} \
|
|
${optionalString cfg.verbose "--v=6"} \
|
|
${optionalString cfg.verbose "--log-flush-frequency=1s"} \
|
|
${optionalString (cfg.clusterCidr!=null)
|
|
"--cluster-cidr=${cfg.clusterCidr}"} \
|
|
${cfg.proxy.extraOpts}
|
|
'';
|
|
WorkingDirectory = cfg.dataDir;
|
|
};
|
|
};
|
|
|
|
# kube-proxy needs iptables
|
|
networking.firewall.enable = mkDefault true;
|
|
|
|
services.kubernetes.proxy.kubeconfig = kubeConfigDefaults;
|
|
})
|
|
|
|
(mkIf (any (el: el == "master") cfg.roles) {
|
|
virtualisation.docker.enable = mkDefault true;
|
|
services.kubernetes.kubelet.enable = mkDefault true;
|
|
services.kubernetes.kubelet.allowPrivileged = mkDefault true;
|
|
services.kubernetes.kubelet.applyManifests = mkDefault true;
|
|
services.kubernetes.apiserver.enable = mkDefault true;
|
|
services.kubernetes.scheduler.enable = mkDefault true;
|
|
services.kubernetes.controllerManager.enable = mkDefault true;
|
|
services.etcd.enable = mkDefault (cfg.etcd.servers == ["http://127.0.0.1:2379"]);
|
|
services.kubernetes.addonManager.enable = mkDefault true;
|
|
services.kubernetes.proxy.enable = mkDefault true;
|
|
})
|
|
|
|
# if this node is only a master make it unschedulable by default
|
|
(mkIf (all (el: el == "master") cfg.roles) {
|
|
services.kubernetes.kubelet.unschedulable = mkDefault true;
|
|
})
|
|
|
|
(mkIf (any (el: el == "node") cfg.roles) {
|
|
virtualisation.docker = {
|
|
enable = mkDefault true;
|
|
|
|
# kubernetes needs access to logs
|
|
logDriver = mkDefault "json-file";
|
|
|
|
# iptables must be disabled for kubernetes
|
|
extraOptions = "--iptables=false --ip-masq=false";
|
|
};
|
|
|
|
services.kubernetes.kubelet.enable = mkDefault true;
|
|
services.kubernetes.proxy.enable = mkDefault true;
|
|
})
|
|
|
|
(mkIf cfg.addonManager.enable {
|
|
environment.etc."kubernetes/addons".source = "${addons}/";
|
|
|
|
systemd.services.kube-addon-manager = {
|
|
description = "Kubernetes addon manager";
|
|
wantedBy = [ "kubernetes.target" ];
|
|
after = [ "kube-apiserver.service" ];
|
|
environment.ADDON_PATH = "/etc/kubernetes/addons/";
|
|
serviceConfig = {
|
|
Slice = "kubernetes.slice";
|
|
ExecStart = "${cfg.package}/bin/kube-addons";
|
|
WorkingDirectory = cfg.dataDir;
|
|
User = "kubernetes";
|
|
Group = "kubernetes";
|
|
};
|
|
};
|
|
})
|
|
|
|
(mkIf (
|
|
cfg.apiserver.enable ||
|
|
cfg.scheduler.enable ||
|
|
cfg.controllerManager.enable ||
|
|
cfg.kubelet.enable ||
|
|
cfg.proxy.enable
|
|
) {
|
|
systemd.targets.kubernetes = {
|
|
description = "Kubernetes";
|
|
wantedBy = [ "multi-user.target" ];
|
|
};
|
|
|
|
systemd.tmpfiles.rules = [
|
|
"d /opt/cni/bin 0755 root root -"
|
|
"d /var/run/kubernetes 0755 kubernetes kubernetes -"
|
|
"d /var/lib/kubernetes 0755 kubernetes kubernetes -"
|
|
];
|
|
|
|
environment.systemPackages = [ cfg.package ];
|
|
users.extraUsers = singleton {
|
|
name = "kubernetes";
|
|
uid = config.ids.uids.kubernetes;
|
|
description = "Kubernetes user";
|
|
extraGroups = [ "docker" ];
|
|
group = "kubernetes";
|
|
home = cfg.dataDir;
|
|
createHome = true;
|
|
};
|
|
users.extraGroups.kubernetes.gid = config.ids.gids.kubernetes;
|
|
|
|
# dns addon is enabled by default
|
|
services.kubernetes.addons.dns.enable = mkDefault true;
|
|
})
|
|
|
|
(mkIf cfg.flannel.enable {
|
|
services.flannel = {
|
|
enable = mkDefault true;
|
|
network = mkDefault cfg.clusterCidr;
|
|
etcd = mkDefault {
|
|
endpoints = cfg.etcd.servers;
|
|
inherit (cfg.etcd) caFile certFile keyFile;
|
|
};
|
|
};
|
|
|
|
services.kubernetes.kubelet = {
|
|
networkPlugin = mkDefault "cni";
|
|
cni.config = mkDefault [{
|
|
name = "mynet";
|
|
type = "flannel";
|
|
delegate = {
|
|
isDefaultGateway = true;
|
|
bridge = "docker0";
|
|
};
|
|
}];
|
|
};
|
|
|
|
systemd.services."mk-docker-opts" = {
|
|
description = "Pre-Docker Actions";
|
|
wantedBy = [ "flannel.service" ];
|
|
before = [ "docker.service" ];
|
|
after = [ "flannel.service" ];
|
|
path = [ pkgs.gawk pkgs.gnugrep ];
|
|
script = ''
|
|
mkdir -p /run/flannel
|
|
${mkDockerOpts}/mk-docker-opts -d /run/flannel/docker
|
|
'';
|
|
serviceConfig.Type = "oneshot";
|
|
};
|
|
systemd.services.docker.serviceConfig.EnvironmentFile = "/run/flannel/docker";
|
|
|
|
# read environment variables generated by mk-docker-opts
|
|
virtualisation.docker.extraOptions = "$DOCKER_OPTS";
|
|
|
|
networking.firewall.allowedUDPPorts = [
|
|
8285 # flannel udp
|
|
8472 # flannel vxlan
|
|
];
|
|
})
|
|
];
|
|
}
|