kubernetes module: flannel support, minor fixes
- add flannel support - remove deprecated authorizationRBACSuperAdmin option - rename from deprecated poratalNet to serviceClusterIpRange - add nodeIp option for kubelet - kubelet, add br_netfilter to kernelModules - enable firewall by default - enable dns by default on node and on master - disable iptables for docker by default on nodes - dns, restart on failure - update tests and other minor changes
This commit is contained in:
Jaka Hudoklin
Robin Gloster
parent
8e14e978c8
commit
7dfeac88ac
5 changed files with 133 additions and 134 deletions
|
|
@ -162,6 +162,17 @@ let
|
|||
};
|
||||
}
|
||||
]);
|
||||
|
||||
# needed for flannel to pass options to docker
|
||||
mkDockerOpts = pkgs.runCommand "mk-docker-opts" {
|
||||
buildInputs = [ pkgs.makeWrapper ];
|
||||
} ''
|
||||
mkdir -p $out
|
||||
cp ${pkgs.kubernetes.src}/cluster/centos/node/bin/mk-docker-opts.sh $out/mk-docker-opts.sh
|
||||
|
||||
# bashInteractive needed for `compgen`
|
||||
makeWrapper ${pkgs.bashInteractive}/bin/bash $out/mk-docker-opts --add-flags "$out/mk-docker-opts.sh"
|
||||
'';
|
||||
in {
|
||||
|
||||
###### interface
|
||||
|
|
@ -357,20 +368,17 @@ in {
|
|||
type = types.listOf types.attrs;
|
||||
};
|
||||
|
||||
authorizationRBACSuperAdmin = mkOption {
|
||||
description = "Role based authorization super admin.";
|
||||
default = "admin";
|
||||
type = types.str;
|
||||
};
|
||||
|
||||
allowPrivileged = mkOption {
|
||||
description = "Whether to allow privileged containers on Kubernetes.";
|
||||
default = true;
|
||||
type = types.bool;
|
||||
};
|
||||
|
||||
portalNet = mkOption {
|
||||
description = "Kubernetes CIDR notation IP range from which to assign portal IPs.";
|
||||
serviceClusterIpRange = mkOption {
|
||||
description = ''
|
||||
A CIDR notation IP range from which to assign service cluster IPs.
|
||||
This must not overlap with any IP ranges assigned to nodes for pods.
|
||||
'';
|
||||
default = "10.10.10.10/24";
|
||||
type = types.str;
|
||||
};
|
||||
|
|
@ -666,6 +674,12 @@ in {
|
|||
type = types.attrsOf (types.submodule [ taintOptions ]);
|
||||
};
|
||||
|
||||
nodeIp = mkOption {
|
||||
description = "IP address of the node. If set, kubelet will use this IP address for the node.";
|
||||
default = null;
|
||||
type = types.nullOr types.str;
|
||||
};
|
||||
|
||||
extraOpts = mkOption {
|
||||
description = "Kubernetes kubelet extra command line options.";
|
||||
default = "";
|
||||
|
|
@ -761,6 +775,12 @@ in {
|
|||
type = types.str;
|
||||
};
|
||||
|
||||
flannel.enable = mkOption {
|
||||
description = "Whether to enable flannel networking";
|
||||
default = false;
|
||||
type = types.bool;
|
||||
};
|
||||
|
||||
};
|
||||
|
||||
###### implementation
|
||||
|
|
@ -808,6 +828,8 @@ in {
|
|||
"--network-plugin=${cfg.kubelet.networkPlugin}"} \
|
||||
--cni-conf-dir=${cniConfig} \
|
||||
--hairpin-mode=hairpin-veth \
|
||||
${optionalString (cfg.kubelet.nodeIp != null)
|
||||
"--node-ip=${cfg.kubelet.nodeIp}"} \
|
||||
${optionalString cfg.verbose "--v=6 --log_flush_frequency=1s"} \
|
||||
${cfg.kubelet.extraOpts}
|
||||
'';
|
||||
|
|
@ -817,6 +839,8 @@ in {
|
|||
|
||||
# Allways include cni plugins
|
||||
services.kubernetes.kubelet.cni.packages = [pkgs.cni];
|
||||
|
||||
boot.kernelModules = ["br_netfilter"];
|
||||
})
|
||||
|
||||
(mkIf (cfg.kubelet.applyManifests && cfg.kubelet.enable) {
|
||||
|
|
@ -879,10 +903,8 @@ in {
|
|||
(concatMapStringsSep "\n" (l: builtins.toJSON l) cfg.apiserver.authorizationPolicy)
|
||||
}"
|
||||
} \
|
||||
${optionalString (elem "RBAC" cfg.apiserver.authorizationMode)
|
||||
"--authorization-rbac-super-user=${cfg.apiserver.authorizationRBACSuperAdmin}"} \
|
||||
--secure-port=${toString cfg.apiserver.securePort} \
|
||||
--service-cluster-ip-range=${cfg.apiserver.portalNet} \
|
||||
--service-cluster-ip-range=${cfg.apiserver.serviceClusterIpRange} \
|
||||
${optionalString (cfg.apiserver.runtimeConfig != "")
|
||||
"--runtime-config=${cfg.apiserver.runtimeConfig}"} \
|
||||
--admission_control=${concatStringsSep "," cfg.apiserver.admissionControl} \
|
||||
|
|
@ -981,10 +1003,9 @@ in {
|
|||
WorkingDirectory = cfg.dataDir;
|
||||
};
|
||||
};
|
||||
})
|
||||
|
||||
(mkIf cfg.kubelet.enable {
|
||||
boot.kernelModules = ["br_netfilter"];
|
||||
# kube-proxy needs iptables
|
||||
networking.firewall.enable = mkDefault true;
|
||||
})
|
||||
|
||||
(mkIf (any (el: el == "master") cfg.roles) {
|
||||
|
|
@ -999,15 +1020,25 @@ in {
|
|||
services.etcd.enable = mkDefault (cfg.etcd.servers == ["http://127.0.0.1:2379"]);
|
||||
})
|
||||
|
||||
# if this node is only a master make it unschedulable by default
|
||||
(mkIf (all (el: el == "master") cfg.roles) {
|
||||
services.kubernetes.kubelet.unschedulable = mkDefault true;
|
||||
})
|
||||
|
||||
(mkIf (any (el: el == "node") cfg.roles) {
|
||||
virtualisation.docker.enable = mkDefault true;
|
||||
virtualisation.docker.logDriver = mkDefault "json-file";
|
||||
virtualisation.docker = {
|
||||
enable = mkDefault true;
|
||||
|
||||
# kubernetes needs access to logs
|
||||
logDriver = mkDefault "json-file";
|
||||
|
||||
# iptables must be disabled for kubernetes
|
||||
extraOptions = "--iptables=false --ip-masq=false";
|
||||
};
|
||||
|
||||
services.kubernetes.kubelet.enable = mkDefault true;
|
||||
services.kubernetes.proxy.enable = mkDefault true;
|
||||
services.kubernetes.dns.enable = mkDefault true;
|
||||
})
|
||||
|
||||
(mkIf cfg.addonManager.enable {
|
||||
|
|
@ -1035,8 +1066,17 @@ in {
|
|||
Group = "kubernetes";
|
||||
AmbientCapabilities = "cap_net_bind_service";
|
||||
SendSIGHUP = true;
|
||||
RestartSec = "30s";
|
||||
Restart = "always";
|
||||
StartLimitInterval = "1m";
|
||||
};
|
||||
};
|
||||
|
||||
networking.firewall.extraCommands = ''
|
||||
# allow container to host communication for DNS traffic
|
||||
${pkgs.iptables}/bin/iptables -I nixos-fw -p tcp -m tcp -d ${cfg.clusterCidr} --dport 53 -j nixos-fw-accept
|
||||
${pkgs.iptables}/bin/iptables -I nixos-fw -p udp -m udp -d ${cfg.clusterCidr} --dport 53 -j nixos-fw-accept
|
||||
'';
|
||||
})
|
||||
|
||||
(mkIf (
|
||||
|
|
@ -1070,5 +1110,50 @@ in {
|
|||
};
|
||||
users.extraGroups.kubernetes.gid = config.ids.gids.kubernetes;
|
||||
})
|
||||
|
||||
(mkIf cfg.flannel.enable {
|
||||
services.flannel = {
|
||||
enable = mkDefault true;
|
||||
network = mkDefault cfg.clusterCidr;
|
||||
etcd = mkDefault {
|
||||
endpoints = cfg.etcd.servers;
|
||||
inherit (cfg.etcd) caFile certFile keyFile;
|
||||
};
|
||||
};
|
||||
|
||||
services.kubernetes.kubelet = {
|
||||
networkPlugin = mkDefault "cni";
|
||||
cni.config = mkDefault [{
|
||||
name = "mynet";
|
||||
type = "flannel";
|
||||
delegate = {
|
||||
isDefaultGateway = true;
|
||||
bridge = "docker0";
|
||||
};
|
||||
}];
|
||||
};
|
||||
|
||||
systemd.services."mk-docker-opts" = {
|
||||
description = "Pre-Docker Actions";
|
||||
wantedBy = [ "flannel.service" ];
|
||||
before = [ "docker.service" ];
|
||||
after = [ "flannel.service" ];
|
||||
path = [ pkgs.gawk pkgs.gnugrep ];
|
||||
script = ''
|
||||
mkdir -p /run/flannel
|
||||
${mkDockerOpts}/mk-docker-opts -d /run/flannel/docker
|
||||
'';
|
||||
serviceConfig.Type = "oneshot";
|
||||
};
|
||||
systemd.services.docker.serviceConfig.EnvironmentFile = "/run/flannel/docker";
|
||||
|
||||
# read environment variables generated by mk-docker-opts
|
||||
virtualisation.docker.extraOptions = "$DOCKER_OPTS";
|
||||
|
||||
networking.firewall.allowedUDPPorts = [
|
||||
8285 # flannel udp
|
||||
8472 # flannel vxlan
|
||||
];
|
||||
})
|
||||
];
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
{ config, pkgs, certs, servers }:
|
||||
|
||||
let
|
||||
etcd_key = "${certs}/etcd-key.pem";
|
||||
etcd_cert = "${certs}/etcd.pem";
|
||||
|
|
@ -9,8 +10,6 @@ let
|
|||
worker_key = "${certs}/worker-key.pem";
|
||||
worker_cert = "${certs}/worker.pem";
|
||||
|
||||
mkDockerOpts = "${pkgs.kubernetes.src}/cluster/centos/node/bin/mk-docker-opts.sh";
|
||||
|
||||
rootCaFile = pkgs.writeScript "rootCaFile.pem" ''
|
||||
${pkgs.lib.readFile "${certs}/ca.pem"}
|
||||
|
||||
|
|
@ -26,16 +25,9 @@ in
|
|||
environment.systemPackages = with pkgs; [ netcat bind etcd.bin ];
|
||||
|
||||
networking = {
|
||||
firewall = {
|
||||
enable = true;
|
||||
allowedTCPPorts = [
|
||||
10250 80 443
|
||||
];
|
||||
allowedUDPPorts = [
|
||||
8285 # flannel udp
|
||||
8472 # flannel vxlan
|
||||
];
|
||||
};
|
||||
firewall.allowedTCPPorts = [
|
||||
10250 # kubelet
|
||||
];
|
||||
extraHosts = ''
|
||||
# register "external" domains
|
||||
${servers.master} etcd.kubernetes.nixos.xyz
|
||||
|
|
@ -43,42 +35,7 @@ in
|
|||
${mkHosts}
|
||||
'';
|
||||
};
|
||||
virtualisation.docker.extraOptions = ''
|
||||
--iptables=false $DOCKER_OPTS
|
||||
'';
|
||||
|
||||
# lets create environment file for docker startup - network stuff
|
||||
systemd.services."pre-docker" = {
|
||||
description = "Pre-Docker Actions";
|
||||
wantedBy = [ "flannel.service" ];
|
||||
before = [ "docker.service" ];
|
||||
after = [ "flannel.service" ];
|
||||
path = [ pkgs.gawk pkgs.gnugrep ];
|
||||
script = ''
|
||||
mkdir -p /run/flannel
|
||||
# bashInteractive needed for `compgen`
|
||||
${pkgs.bashInteractive}/bin/bash ${mkDockerOpts} -d /run/flannel/docker
|
||||
cat /run/flannel/docker # just for debugging
|
||||
|
||||
# allow container to host communication for DNS traffic
|
||||
${pkgs.iptables}/bin/iptables -I nixos-fw -p tcp -m tcp -i docker0 --dport 53 -j nixos-fw-accept
|
||||
${pkgs.iptables}/bin/iptables -I nixos-fw -p udp -m udp -i docker0 --dport 53 -j nixos-fw-accept
|
||||
'';
|
||||
serviceConfig.Type = "simple";
|
||||
};
|
||||
systemd.services.docker.serviceConfig.EnvironmentFile = "/run/flannel/docker";
|
||||
|
||||
services.flannel = {
|
||||
enable = true;
|
||||
network = "10.2.0.0/16";
|
||||
iface = "eth1";
|
||||
etcd = {
|
||||
endpoints = ["https://etcd.kubernetes.nixos.xyz:2379"];
|
||||
keyFile = etcd_client_key;
|
||||
certFile = etcd_client_cert;
|
||||
caFile = ca_pem;
|
||||
};
|
||||
};
|
||||
services.flannel.iface = "eth1";
|
||||
environment.variables = {
|
||||
ETCDCTL_CERT_FILE = "${etcd_client_cert}";
|
||||
ETCDCTL_KEY_FILE = "${etcd_client_key}";
|
||||
|
|
@ -88,20 +45,10 @@ in
|
|||
|
||||
services.kubernetes = {
|
||||
kubelet = {
|
||||
networkPlugin = "cni";
|
||||
cni.config = [{
|
||||
name = "mynet";
|
||||
type = "flannel";
|
||||
delegate = {
|
||||
isDefaultGateway = true;
|
||||
bridge = "docker0";
|
||||
};
|
||||
}];
|
||||
tlsKeyFile = worker_key;
|
||||
tlsCertFile = worker_cert;
|
||||
hostname = "${config.networking.hostName}.nixos.xyz";
|
||||
extraOpts = "--node-ip ${config.networking.primaryIPAddress}";
|
||||
clusterDns = config.networking.primaryIPAddress;
|
||||
nodeIp = config.networking.primaryIPAddress;
|
||||
};
|
||||
etcd = {
|
||||
servers = ["https://etcd.kubernetes.nixos.xyz:2379"];
|
||||
|
|
@ -110,22 +57,16 @@ in
|
|||
caFile = ca_pem;
|
||||
};
|
||||
kubeconfig = {
|
||||
server = "https://kubernetes.nixos.xyz:4443";
|
||||
server = "https://kubernetes.nixos.xyz";
|
||||
caFile = rootCaFile;
|
||||
certFile = worker_cert;
|
||||
keyFile = worker_key;
|
||||
};
|
||||
flannel.enable = true;
|
||||
|
||||
# make sure you cover kubernetes.apiserver.portalNet and flannel networks
|
||||
clusterCidr = "10.0.0.0/8";
|
||||
|
||||
dns.enable = true;
|
||||
dns.port = 4453;
|
||||
};
|
||||
|
||||
services.dnsmasq.enable = true;
|
||||
services.dnsmasq.servers = ["/${config.services.kubernetes.dns.domain}/127.0.0.1#4453"];
|
||||
|
||||
virtualisation.docker.enable = true;
|
||||
virtualisation.docker.storageDriver = "overlay";
|
||||
}
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ in
|
|||
allowPing = true;
|
||||
allowedTCPPorts = [
|
||||
2379 2380 # etcd
|
||||
4443 # kubernetes
|
||||
443 # kubernetes apiserver
|
||||
];
|
||||
};
|
||||
};
|
||||
|
|
@ -43,14 +43,13 @@ in
|
|||
initialCluster = ["master=https://etcd.kubernetes.nixos.xyz:2380"];
|
||||
initialAdvertisePeerUrls = ["https://etcd.kubernetes.nixos.xyz:2380"];
|
||||
};
|
||||
|
||||
services.kubernetes = {
|
||||
roles = ["master"];
|
||||
scheduler.leaderElect = true;
|
||||
controllerManager.leaderElect = true;
|
||||
controllerManager.rootCaFile = rootCaFile;
|
||||
controllerManager.serviceAccountKeyFile = apiserver_key;
|
||||
apiserver = {
|
||||
securePort = 4443;
|
||||
publicAddress = "192.168.1.1";
|
||||
advertiseAddress = "192.168.1.1";
|
||||
tlsKeyFile = apiserver_key;
|
||||
|
|
@ -59,9 +58,6 @@ in
|
|||
kubeletClientCaFile = rootCaFile;
|
||||
kubeletClientKeyFile = worker_key;
|
||||
kubeletClientCertFile = worker_cert;
|
||||
portalNet = "10.1.10.0/24"; # --service-cluster-ip-range
|
||||
runtimeConfig = "";
|
||||
/*extraOpts = "--v=2";*/
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@ let
|
|||
servers.master = "192.168.1.1";
|
||||
servers.one = "192.168.1.10";
|
||||
servers.two = "192.168.1.20";
|
||||
servers.three = "192.168.1.30";
|
||||
|
||||
certs = import ./certs.nix { inherit servers; };
|
||||
|
||||
|
|
@ -39,7 +38,7 @@ let
|
|||
clusters = [{
|
||||
name = "local";
|
||||
cluster.certificate-authority = "/ca.pem";
|
||||
cluster.server = "https://${servers.master}:4443/";
|
||||
cluster.server = "https://${servers.master}";
|
||||
}];
|
||||
users = [{
|
||||
name = "kubelet";
|
||||
|
|
@ -61,11 +60,9 @@ let
|
|||
$master->waitUntilSucceeds("kubectl get node master.nixos.xyz | grep Ready");
|
||||
$master->waitUntilSucceeds("kubectl get node one.nixos.xyz | grep Ready");
|
||||
$master->waitUntilSucceeds("kubectl get node two.nixos.xyz | grep Ready");
|
||||
$master->waitUntilSucceeds("kubectl get node three.nixos.xyz | grep Ready");
|
||||
|
||||
$one->execute("docker load < ${kubectlImage}");
|
||||
$two->execute("docker load < ${kubectlImage}");
|
||||
$three->execute("docker load < ${kubectlImage}");
|
||||
|
||||
$master->waitUntilSucceeds("kubectl create -f ${kubectlPod} || kubectl apply -f ${kubectlPod}");
|
||||
|
||||
|
|
@ -116,19 +113,6 @@ in makeTest {
|
|||
}
|
||||
(import ./kubernetes-common.nix { inherit pkgs config certs servers; })
|
||||
];
|
||||
|
||||
three =
|
||||
{ config, pkgs, lib, nodes, ... }:
|
||||
mkMerge [
|
||||
{
|
||||
virtualisation.memorySize = 768;
|
||||
virtualisation.diskSize = 4096;
|
||||
networking.interfaces.eth1.ip4 = mkForce [{address = servers.three; prefixLength = 24;}];
|
||||
networking.primaryIPAddress = mkForce servers.three;
|
||||
services.kubernetes.roles = ["node"];
|
||||
}
|
||||
(import ./kubernetes-common.nix { inherit pkgs config certs servers; })
|
||||
];
|
||||
};
|
||||
|
||||
testScript = ''
|
||||
|
|
|
|||
|
|
@ -46,37 +46,30 @@ let
|
|||
$kubernetes->waitUntilSucceeds("kubectl get pod redis | grep Running");
|
||||
$kubernetes->succeed("nc -z \$\(dig redis.default.svc.cluster.local +short\) 6379");
|
||||
'';
|
||||
in {
|
||||
# This test runs kubernetes on a single node
|
||||
singlenode = makeTest {
|
||||
name = "kubernetes-singlenode";
|
||||
in makeTest {
|
||||
name = "kubernetes-singlenode";
|
||||
|
||||
nodes = {
|
||||
kubernetes =
|
||||
{ config, pkgs, lib, nodes, ... }:
|
||||
{
|
||||
virtualisation.memorySize = 768;
|
||||
virtualisation.diskSize = 2048;
|
||||
nodes = {
|
||||
kubernetes =
|
||||
{ config, pkgs, lib, nodes, ... }:
|
||||
{
|
||||
virtualisation.memorySize = 768;
|
||||
virtualisation.diskSize = 2048;
|
||||
|
||||
programs.bash.enableCompletion = true;
|
||||
environment.systemPackages = with pkgs; [ netcat bind ];
|
||||
programs.bash.enableCompletion = true;
|
||||
environment.systemPackages = with pkgs; [ netcat bind ];
|
||||
|
||||
services.kubernetes.roles = ["master" "node"];
|
||||
services.kubernetes.dns.port = 4453;
|
||||
virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false -b cbr0";
|
||||
services.kubernetes.roles = ["master" "node"];
|
||||
services.kubernetes.dns.port = 4453;
|
||||
|
||||
networking.bridges.cbr0.interfaces = [];
|
||||
networking.interfaces.cbr0 = {};
|
||||
|
||||
services.dnsmasq.enable = true;
|
||||
services.dnsmasq.servers = ["/${config.services.kubernetes.dns.domain}/127.0.0.1#4453"];
|
||||
};
|
||||
};
|
||||
|
||||
testScript = ''
|
||||
startAll;
|
||||
|
||||
${testSimplePod}
|
||||
'';
|
||||
services.dnsmasq.enable = true;
|
||||
services.dnsmasq.servers = ["/${config.services.kubernetes.dns.domain}/127.0.0.1#4453"];
|
||||
};
|
||||
};
|
||||
|
||||
testScript = ''
|
||||
startAll;
|
||||
|
||||
${testSimplePod}
|
||||
'';
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue