Installing Kubernetes on LXC
This example shows an installation over LXC, while for any more or less serious use this is completely unacceptable and requires using KVM with additional security measures.
And besides, installation on KVM will be much, much easier.
LXC does not provide adequate security because, for proper operation, it requires mounting the /sys and /proc partitions from the host, which are accessible to all LXC instances.
Furthermore, this example requires using a PRIVILEGED virtual machine.
However, LXC allows relatively easy setting up a testing infrastructure without the overhead associated with KVM.
So some aspects in this example will only be related to the setup in LXC.
On the HOST system
First, let’s make preparations on the host system.
nano /etc/modules
1 2 3 4 5 6 7 8 9 10 11 | overlay nf_nat br_netfilter xt_conntrack rbd fuse ip_vs ip_vs_rr ip_vs_wrr ip_vs_sh iptable_nat |
nano /etc/sysctl.d/35-lxc-kubernetes.conf
1 2 3 4 5 6 7 8 9 10 11 | kernel.dmesg_restrict = 0 net.ipv4.ip_forward = 1 net.ipv6.conf.all.forwarding = 1 net.bridge.bridge-nf-call-iptables = 1 # --conntrack-max-per-core Default: 32768 * N net.netfilter.nf_conntrack_max=131072 # net.bridge.bridge-nf-call-arptables # kernel.pid_max=100000 # user.max_user_namespaces=15000 vm.compact_memory = 1 vm.overcommit_memory = 1 |
During installation, we’ll disable swap; you can turn it back on later after setting up all the virtual machines.
swapoff -a
Setup lxc
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 | LANG=C MIRROR=https://deb.debian.org/debian lxc-create \ -n kubemaster -t debian -- -r trixie nano /var/lib/lxc/kubemaster/config lxc.net.0.type = veth lxc.net.0.flags = up lxc.net.0.link = lxcbr11 lxc.net.0.name = eth121 lxc.net.0.veth.pair = veth121 lxc.net.0.ipv4.address = 10.11.11.121/24 lxc.net.0.hwaddr = 00:16:a9:7d:99:33 lxc.net.0.ipv4.gateway = 10.11.11.1 lxc.apparmor.profile = unconfined lxc.apparmor.allow_nesting = 1 lxc.cap.drop = lxc.cgroup.devices.allow = a lxc.mount.auto = proc:rw sys:rw lxc.cgroup2.devices.allow = c 1:11 rwm lxc.mount.entry = /dev/kmsg dev/kmsg none defaults,bind,create=file lxc.cgroup2.devices.allow = c 10:200 rwm limits.memory.swap = true linux.kernel_modules = ip_tables,ip6_tables,nf_nat,overlay,br_netfilter lxc.mount.entry = /proc/sys/vm/overcommit_memory proc/sys/vm/overcommit_memory none defaults,bind,create=file lxc.mount.entry = /proc/sys/kernel/panic proc/sys/kernel/panic none defaults,bind,create=file lxc.mount.entry = /proc/sys/kernel/panic_on_oops proc/sys/kernel/panic_on_oops none defaults,bind,create=file lxc.rootfs.path = dir:/var/lib/lxc/kubemaster/rootfs # Common configuration lxc.include = /usr/share/lxc/config/debian.common.conf lxc.include = /usr/share/lxc/config/debian.userns.conf # Container specific configuration lxc.tty.max = 4 lxc.uts.name = kubemaster lxc.arch = amd64 lxc.pty.max = 1024 lxc.start.order = 5 lxc.start.auto = 1 lxc.start.delay = 25 security.nesting = true security.privileged = true |
iptables
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 | # lxc kubemaster iptables -t nat -A POSTROUTING -s 10.11.11.121/32 ! -d 10.11.11.0/24 -o eth0 -j MASQUERADE iptables -A FORWARD -s 10.11.11.121/32 -i lxcbr1 -o eth0 -m state --state NEW,RELATED,ESTABLISHED -j ACCEPT iptables -A FORWARD -d 10.11.11.121/32 -i eth0 -o lxcbr1 -m state --state RELATED,ESTABLISHED -j ACCEPT iptables -A FORWARD -s 10.11.11.121/32 -i lxcbr1 -o lxcbr1 -m state --state NEW,RELATED,ESTABLISHED -j ACCEPT iptables -A FORWARD -d 10.11.11.121/32 -o lxcbr1 -i lxcbr1 -m state --state RELATED,ESTABLISHED -j ACCEPT # lxc kubenode1 iptables -t nat -A POSTROUTING -s 10.11.11.125/32 ! -d 10.11.11.0/24 -o eth0 -j MASQUERADE iptables -A FORWARD -s 10.11.11.125/32 -i lxcbr1 -o eth0 -m state --state NEW,RELATED,ESTABLISHED -j ACCEPT iptables -A FORWARD -d 10.11.11.125/32 -i eth0 -o lxcbr1 -m state --state RELATED,ESTABLISHED -j ACCEPT iptables -A FORWARD -s 10.11.11.125/32 -i lxcbr1 -o lxcbr1 -m state --state NEW,RELATED,ESTABLISHED -j ACCEPT iptables -A FORWARD -d 10.11.11.125/32 -o lxcbr1 -i lxcbr1 -m state --state RELATED,ESTABLISHED -j ACCEPT # lxc kubenode2 iptables -t nat -A POSTROUTING -s 10.11.11.129/32 ! -d 10.11.11.0/24 -o eth0 -j MASQUERADE iptables -A FORWARD -s 10.11.11.129/32 -i lxcbr1 -o eth0 -m state --state NEW,RELATED,ESTABLISHED -j ACCEPT iptables -A FORWARD -d 10.11.11.129/32 -i eth0 -o lxcbr1 -m state --state RELATED,ESTABLISHED -j ACCEPT iptables -A FORWARD -s 10.11.11.129/32 -i lxcbr1 -o lxcbr1 -m state --state NEW,RELATED,ESTABLISHED -j ACCEPT iptables -A FORWARD -d 10.11.11.129/32 -o lxcbr1 -i lxcbr1 -m state --state RELATED,ESTABLISHED -j ACCEPT |
lxc-start -n kubemaster
Inside kubemaster
Now let’s switch to lxc.
lxc-attach -n kubemaster
nano /etc/apt/sources.list
1 2 3 4 5 | # trixie deb https://deb.debian.org/debian/ trixie main contrib non-free non-free-firmware deb http://security.debian.org/debian-security/ trixie-security main contrib non-free non-free-firmware deb https://deb.debian.org/debian/ trixie-updates main contrib non-free non-free-firmware |
apt update
1 2 3 4 | apt --no-install-recommends --no-install-suggests install \ iputils-ping net-tools htop mc nano curl wget \ ca-certificates apt-transport-https gpg locales \ binutils bash-completion dirmngr kmod ebtables ethtool |
dpkg-reconfigure locales
Setting up hostname & hosts
printf 'kubemaster.web.webart4.me\n' > /etc/hostname
nano /etc/hosts
1 2 3 4 5 | 127.0.0.1 localhost 127.0.1.1 kubemaster 10.11.11.121 kubemaster.web.webart4.me 10.11.11.125 kubenode1.web.webart4.me 10.11.11.129 kubenode2.web.webart4.me |
Install containerd
1 2 3 4 5 6 7 | cd /usr/local wget https://github.com/containerd/containerd/releases/download/v2.2.0/containerd-2.2.0-linux-amd64.tar.gz wget https://github.com/containerd/containerd/releases/download/v2.2.0/containerd-2.2.0-linux-amd64.tar.gz.sha256sum sha256sum --check containerd-2.2.0-linux-amd64.tar.gz.sha256sum tar -xzv -f containerd-2.2.0-linux-amd64.tar.gz wget https://raw.githubusercontent.com/containerd/containerd/main/containerd.service |
Since we already have overlay module loaded at the host level, we disable its loading in systemd.
nano containerd.service
1 2 | [Service] # ExecStartPre=-/sbin/modprobe overlay |
1 2 3 4 | mkdir -p /usr/local/lib/systemd/system/ mv containerd.service /usr/local/lib/systemd/system/ systemctl daemon-reload systemctl enable --now containerd |
Install runc - spawning and running tool
Download:
1 2 3 4 5 6 7 8 9 | wget https://github.com/opencontainers/runc/releases/download/v1.3.3/runc.amd64 wget https://github.com/opencontainers/runc/releases/download/v1.3.3/runc.amd64.asc wget https://github.com/opencontainers/runc/releases/download/v1.3.3/runc.sha256sum gpg --verify runc.amd64.asc gpg: Signature made Ср 05 ноя 2025 09:12:15 UTC gpg: using EDDSA key B64E4955B29FA3D463F2A9062897FAD2B7E9446F gpg: Can't check signature: No public key |
Import the Certificate:
1 2 3 4 5 6 7 8 9 | gpg --keyserver keys.gnupg.net \ --search-keys B64E4955B29FA3D463F2A9062897FAD2B7E9446F gpg: data source: http://185.125.188.26:11371 (1) Aleksa Sarai <cyphar@cyphar.com> 263 bit EDDSA key 34401015D1D2D386, created: 2019-06-21 Keys 1-1 of 1 for "B64E4955B29FA3D463F2A9062897FAD2B7E9446F". Enter number(s), N)ext, or Q)uit > 1 gpg: Total number processed: 1 gpg: imported: 1 |
Verify:
1 2 3 4 5 6 7 8 9 10 11 | gpg --verify runc.amd64.asc gpg: assuming signed data in 'runc.amd64' gpg: Signature made Ср 05 ноя 2025 09:12:15 UTC gpg: using EDDSA key B64E4955B29FA3D463F2A9062897FAD2B7E9446F gpg: Good signature from "Aleksa Sarai <cyphar@cyphar.com>" [unknown] gpg: Signature notation: manu=2,2.5+1.11,2,2 gpg: WARNING: This key is not certified with a trusted signature! gpg: There is no indication that the signature belongs to the owner. Primary key fingerprint: C9C3 70B2 46B0 9F6D BCFC 744C 3440 1015 D1D2 D386 Subkey fingerprint: B64E 4955 B29F A3D4 63F2 A906 2897 FAD2 B7E9 446F |
Check and install:
sha256sum --check runc.sha256sum
runc.amd64: OK
install -m 755 runc.amd64 /usr/local/sbin/runc
Install CNI plugins
1 2 3 4 5 6 7 | wget https://github.com/containernetworking/plugins/releases/download/v1.8.0/cni-plugins-linux-amd64-v1.8.0.tgz wget https://github.com/containernetworking/plugins/releases/download/v1.8.0/cni-plugins-linux-amd64-v1.8.0.tgz.sha512 sha512sum --check cni-plugins-linux-amd64-v1.8.0.tgz.sha512 cni-plugins-linux-amd64-v1.8.0.tgz: OK mkdir -p /opt/cni/bin tar -xzv -f cni-plugins-linux-amd64-v1.8.0.tgz -C /opt/cni/bin |
containerd config
systemctl status containerd
mkdir -p /etc/containerd
containerd config default | tee /etc/containerd/config.toml
Open nano /etc/containerd/config.toml file
and change the following two lines:
change line SystemdCgroup = true
1 2 3 4 5 6 7 8 9 10 11 | [plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.runc.options] BinaryName = '' CriuImagePath = '' CriuWorkPath = '' IoGid = 0 IoUid = 0 NoNewKeyring = false Root = '' ShimCgroup = '' # SystemdCgroup = false SystemdCgroup = true |
add line sandbox_image = 'registry.k8s.io/pause:3.10'
1 2 3 4 5 6 7 | [plugins.'io.containerd.grpc.v1.cri'] disable_tcp_service = true stream_server_address = '127.0.0.1' stream_server_port = '0' stream_idle_timeout = '4h0m0s' enable_tls_streaming = false sandbox_image = 'registry.k8s.io/pause:3.10' |
systemctl restart containerd
systemctl status containerd
Install kubernetes: kubelet kubeadm kubectl
1 2 3 4 5 6 | curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.34/deb/Release.key | \ gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] \ https://pkgs.k8s.io/core:/stable:/v1.34/deb/ /" | \ tee /etc/apt/sources.list.d/kubernetes.list |
apt update
apt install kubelet kubeadm kubectl
apt-mark hold kubelet kubeadm kubectl
Temporarily switch to HOST system and copy kernel config file:
1 2 | cp /boot/config-`uname -r` \ /var/lib/lxc/kubemaster/rootfs/boot/config-`uname -r` |
Now back to lxc kubemaster.
nano /etc/default/kubelet
1 | KUBELET_EXTRA_ARGS=--fail-swap-on=false |
nano /etc/rc.local
1 2 3 | #!/bin/sh -e mount --make-rshared / |
chmod +x /etc/rc.local
Remove temporary files and clear the cache
rm -f /usr/local/cni-plugins-linux-amd64-v1.8.0.tgz
rm -f /usr/local/cni-plugins-linux-amd64-v1.8.0.tgz.sha512
rm -f /usr/local/containerd-2.2.0-linux-amd64.tar.gz
rm -f /usr/local/containerd-2.2.0-linux-amd64.tar.gz.sha256sum
rm -f /usr/local/runc.amd64
rm -f /usr/local/runc.amd64.asc
rm -f /usr/local/runc.sha256sum
apt clean
Host: lxc-stop -n kubemaster
Clone and Backup
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 | cd /var/lib/lxc time /usr/bin/tar \ --use-compress-program="/usr/bin/pzstd -15 \ --keep --processes 3" \ -pcvf kubemaster_clean.zstd kubemaster # 254M kubemaster_clean.zstd mkdir kubenode1 mkdir kubenode2 chmod ug=rwX,o-rwXx kubenode1 kubenode2 cd /var/lib/lxc/kubemaster find ./ -xdev -print0 | cpio -pa0V /var/lib/lxc/kubenode1 find ./ -xdev -print0 | cpio -pa0V /var/lib/lxc/kubenode2 |
nano /var/lib/lxc/kubenode1/config
An example kubenode1 config is at the bottom of the article.
nano /var/lib/lxc/kubenode2/config
An example kubenode2 config is at the bottom of the article.
kubenode1
lxc-attach -n kubenode1
1 2 3 4 5 6 7 8 9 | printf 'kubenode1.web.webart4.me\n' > /etc/hostname nano /etc/hosts 127.0.0.1 localhost 127.0.1.1 kubenode1 10.11.11.121 kubemaster.web.webart4.me 10.11.11.125 kubenode1.web.webart4.me 10.11.11.129 kubenode2.web.webart4.me |
kubenode2
lxc-attach -n kubenode2
1 2 3 4 5 6 7 8 9 | printf 'kubenode2.web.webart4.me\n' > /etc/hostname nano /etc/hosts 127.0.0.1 localhost 127.0.1.1 kubenode2 10.11.11.121 kubemaster.web.webart4.me 10.11.11.125 kubenode1.web.webart4.me 10.11.11.129 kubenode2.web.webart4.me |
Restart
lxc-stop --reboot --name kubemaster
lxc-stop --reboot --name kubenode1
lxc-stop --reboot --name kubenode2
[--nowait | --kill]
Initialize the Cluster in kubemaster
lxc-attach -n kubemaster
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 | kubeadm init --control-plane-endpoint=10.11.11.121:6443 --apiserver-advertise-address=10.11.11.121 --pod-network-cidr=10.244.0.0/16 Your Kubernetes control-plane has initialized successfully! To start using your cluster, you need to run the following as a regular user: mkdir -p $HOME/.kube sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config sudo chown $(id -u):$(id -g) $HOME/.kube/config Alternatively, if you are the root user, you can run: export KUBECONFIG=/etc/kubernetes/admin.conf You should now deploy a pod network to the cluster. Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at: https://kubernetes.io/docs/concepts/cluster-administration/addons/ You can now join any number of control-plane nodes by copying certificate authorities and service account keys on each node and then running the following as root: kubeadm join 10.11.11.121:6443 --token qejrqw.uwer98qy6v7xt599 \ --discovery-token-ca-cert-hash sha256:52d72e6e39aee3127d9a649aeff25fff17c6ff5be2c3f43f6bc126631718bf1e \ --control-plane Then you can join any number of worker nodes by running the following on each as root: kubeadm join 10.11.11.121:6443 --token qejrqw.uwer98qy6v7xt599 \ --discovery-token-ca-cert-hash sha256:52d72e6e39aee3127d9a649aeff25fff17c6ff5be2c3f43f6bc126631718bf1e |
Disable the swap error and configure the ability to use it.
Again, this makes sense when configured in lxc because all VMs get the host’s swap state.
And I wouldn’t want to disable it on the host.
nano /var/lib/kubelet/config.yaml
1 2 3 4 5 6 | failSwapOn: false featureGates: NodeSwap: true memorySwap: swapBehavior: LimitedSwap # memorySwap: {} |
mkdir -p $HOME/.kube
cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
chown $(id -u):$(id -g) -R $HOME/.kube
chmod -R ug=rx,o-rwxX $HOME/.kube
Install Pod Network - Flannel
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 | cd /usr/src/ wget https://github.com/flannel-io/flannel/releases/latest/download/kube-flannel.yml nano kube-flannel.yml # add KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT variables under the POD_NAMESPACE - name: POD_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - name: KUBERNETES_SERVICE_HOST value: '10.11.11.121' - name: KUBERNETES_SERVICE_PORT value: '6443' kubectl apply -f kube-flannel.yml namespace/kube-flannel created serviceaccount/flannel created clusterrole.rbac.authorization.k8s.io/flannel created clusterrolebinding.rbac.authorization.k8s.io/flannel created configmap/kube-flannel-cfg created daemonset.apps/kube-flannel-ds created |
systemctl enable --now kubelet
systemctl restart kubelet
systemctl status kubelet
kubectl get nodes
1 2 | NAME STATUS ROLES AGE VERSION kubemaster.web.webart4.me Ready control-plane 7m35s v1.34.2 |
ifconfig -a
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 | eth121: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500 inet 10.11.11.121 netmask 255.255.255.0 broadcast 0.0.0.255 inet6 fe80::216:3eff:fe7d:ba33 prefixlen 64 scopeid 0x20<link> ether 00:16:3e:7d:ba:33 txqueuelen 1000 (Ethernet) RX packets 28035 bytes 63635564 (60.6 MiB) RX errors 0 dropped 0 overruns 0 frame 0 TX packets 14864 bytes 1100501 (1.0 MiB) TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0 flannel.1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1450 inet 10.244.0.0 netmask 255.255.255.255 broadcast 0.0.0.0 inet6 fe80::f46a:59ff:feee:f2a0 prefixlen 64 scopeid 0x20<link> ether f6:6a:59:ee:f2:a0 txqueuelen 0 (Ethernet) RX packets 0 bytes 0 (0.0 B) RX errors 0 dropped 0 overruns 0 frame 0 TX packets 0 bytes 0 (0.0 B) TX errors 0 dropped 12 overruns 0 carrier 0 collisions 0 lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536 inet 127.0.0.1 netmask 255.0.0.0 inet6 ::1 prefixlen 128 scopeid 0x10<host> loop txqueuelen 1000 (Local Loopback) RX packets 293302 bytes 68241882 (65.0 MiB) RX errors 0 dropped 0 overruns 0 frame 0 TX packets 293302 bytes 68241882 (65.0 MiB) TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0 |
kubenode1
lxc-attach -n kubenode1
ping 10.11.11.121
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 | kubeadm join 10.11.11.121:6443 \ --token qejrqw.uwer98qy6v7xt599 \ --discovery-token-ca-cert-hash sha256:52d72e6e39aee3127d9a649aeff25fff17c6ff5be2c3f43f6bc126631718bf1e [preflight] Running pre-flight checks [preflight] Reading configuration from the "kubeadm-config" ConfigMap in namespace "kube-system"... [preflight] Use 'kubeadm init phase upload-config kubeadm --config your-config-file' to re-upload it. [kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/instance-config.yaml" [patches] Applied patch of type "application/strategic-merge-patch+json" to target "kubeletconfiguration" [kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml" [kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env" [kubelet-start] Starting the kubelet [kubelet-check] Waiting for a healthy kubelet at http://127.0.0.1:10248/healthz. This can take up to 4m0s [kubelet-check] The kubelet is healthy after 501.132212ms [kubelet-start] Waiting for the kubelet to perform the TLS Bootstrap This node has joined the cluster: * Certificate signing request was sent to apiserver and a response was received. * The Kubelet was informed of the new secure connection details. Run 'kubectl get nodes' on the control-plane to see this node join the cluster. |
Open file nano /var/lib/kubelet/config.yaml.
And add the same as for kubemaster.
systemctl enable --now kubelet
systemctl restart kubelet
systemctl status kubelet
kubenode2
lxc-attach -n kubenode2
ping 10.11.11.121
1 2 3 | kubeadm join 10.11.11.121:6443 \ --token qejrqw.uwer98qy6v7xt599 \ --discovery-token-ca-cert-hash sha256:52d72e6e39aee3127d9a649aeff25fff17c6ff5be2c3f43f6bc126631718bf1e |
It’s the same here.
nano /var/lib/kubelet/config.yaml
Add the same lines from kubemaster.
systemctl enable --now kubelet
systemctl restart kubelet
systemctl status kubelet
Checking connected nodes and kubemaster cluster status
kubectl get nodes
1 2 3 4 | NAME STATUS ROLES AGE VERSION kubemaster.web.webart4.me Ready control-plane 54m v1.34.2 kubenode1.web.webart4.me Ready <none> 9m55s v1.34.2 kubenode2.web.webart4.me Ready <none> 6m46s v1.34.2 |
kubectl get pods --all-namespaces -o wide
1 2 3 4 5 6 7 8 9 10 11 12 13 | NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES kube-flannel kube-flannel-ds-nvrht 1/1 Running 10 (24m ago) 105m 10.11.11.125 kubenode1.web.webart4.me <none> <none> kube-flannel kube-flannel-ds-t95hr 1/1 Running 1 (14m ago) 17m 10.11.11.129 kubenode2.web.webart4.me <none> <none> kube-flannel kube-flannel-ds-zdml5 1/1 Running 4 (22m ago) 130m 10.11.11.121 kubemaster.web.webart4.me <none> <none> kube-system coredns-66bc5c9577-qkjpk 1/1 Running 8 (24m ago) 132m 10.244.1.18 kubenode1.web.webart4.me <none> <none> kube-system coredns-66bc5c9577-w2fh6 1/1 Running 8 (24m ago) 132m 10.244.1.19 kubenode1.web.webart4.me <none> <none> kube-system etcd-kubemaster.web.webart4.me 1/1 Running 11 (22m ago) 132m 10.11.11.121 kubemaster.web.webart4.me <none> <none> kube-system kube-apiserver-kubemaster.web.webart4.me 1/1 Running 11 (22m ago) 132m 10.11.11.121 kubemaster.web.webart4.me <none> <none> kube-system kube-controller-manager-kubemaster.web.webart4.me 1/1 Running 11 (22m ago) 132m 10.11.11.121 kubemaster.web.webart4.me <none> <none> kube-system kube-proxy-76qnk 1/1 Running 1 (14m ago) 17m 10.11.11.129 kubenode2.web.webart4.me <none> <none> kube-system kube-proxy-9mcc4 1/1 Running 47 (24m ago) 105m 10.11.11.125 kubenode1.web.webart4.me <none> <none> kube-system kube-proxy-p8pkc 1/1 Running 39 (22m ago) 132m 10.11.11.121 kubemaster.web.webart4.me <none> <none> kube-system kube-scheduler-kubemaster.web.webart4.me 1/1 Running 11 (22m ago) 132m 10.11.11.121 kubemaster.web.webart4.me <none> <none> |
Testing Kubernetes cluster
Cluster taints
kubectl describe node | grep -E "NoSchedule|Taints|Name\:"
1 2 3 4 5 6 7 | Name: kubemaster.web.webart4.me
Taints: node-role.kubernetes.io/control-plane:NoSchedule
node.kubernetes.io/disk-pressure:NoSchedule
Name: kubenode1.web.webart4.me
Taints: node.kubernetes.io/disk-pressure:NoSchedule
Name: kubenode2.web.webart4.me
Taints: node.kubernetes.io/disk-pressure:NoSchedule
|
nano /var/lib/kubelet/config.yaml
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 | evictionSoft: memory.available: "100Mi" nodefs.available: "100Mi" nodefs.inodesFree: "1%" imagefs.available: "100Mi" imagefs.inodesFree: "1%" evictionHard: memory.available: "100Mi" nodefs.available: "100Mi" nodefs.inodesFree: "1%" imagefs.available: "100Mi" imagefs.inodesFree: "1%" evictionSoftGracePeriod: memory.available: 2m nodefs.available: 2m nodefs.inodesFree: 2m imagefs.available: 2m imagefs.inodesFree: 2m |
systemctl restart kubelet
Correct cluster state
kubectl describe node | grep -E "NoSchedule|Taints|Name\:"
1 2 3 4 5 6 | Name: kubemaster.web.webart4.me Taints: node-role.kubernetes.io/control-plane:NoSchedule Name: kubenode1.web.webart4.me Taints: <none> Name: kubenode2.web.webart4.me Taints: <none> |
First deployment on a new cluster
nano nginx-deployment.yaml
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 | apiVersion: apps/v1 kind: Deployment metadata: name: nginx-deployment labels: app: nginx spec: replicas: 1 selector: matchLabels: app: nginx template: metadata: labels: app: nginx spec: containers: - name: nginx image: nginx:1.14.2 ports: - containerPort: 80 |
kubectl apply -f nginx-deployment.yaml
kubectl rollout status deployment/nginx-deployment
1 | deployment "nginx-deployment" successfully rolled out
|
kubectl get deployments
1 2 | NAME READY UP-TO-DATE AVAILABLE AGE nginx-deployment 1/1 1 1 20s |
kubectl get pods --all-namespaces -o wide
1 2 3 4 5 6 7 8 9 10 11 12 13 14 | NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES default nginx-deployment-bf744486c-5zfqz 1/1 Running 0 14m 10.244.2.2 kubenode2.web.webart4.me <none> <none> kube-flannel kube-flannel-ds-nvrht 1/1 Running 11 (3h2m ago) 5h44m 10.11.11.125 kubenode1.web.webart4.me <none> <none> kube-flannel kube-flannel-ds-t95hr 1/1 Running 2 (3h2m ago) 4h16m 10.11.11.129 kubenode2.web.webart4.me <none> <none> kube-flannel kube-flannel-ds-zdml5 1/1 Running 5 (3h2m ago) 6h9m 10.11.11.121 kubemaster.web.webart4.me <none> <none> kube-system coredns-66bc5c9577-cmtz5 1/1 Running 0 29m 10.244.1.23 kubenode1.web.webart4.me <none> <none> kube-system coredns-66bc5c9577-j5l49 1/1 Running 0 29m 10.244.1.22 kubenode1.web.webart4.me <none> <none> kube-system etcd-kubemaster.web.webart4.me 1/1 Running 12 (3h2m ago) 6h11m 10.11.11.121 kubemaster.web.webart4.me <none> <none> kube-system kube-apiserver-kubemaster.web.webart4.me 1/1 Running 12 (3h2m ago) 6h11m 10.11.11.121 kubemaster.web.webart4.me <none> <none> kube-system kube-controller-manager-kubemaster.web.webart4.me 1/1 Running 12 (3h2m ago) 6h11m 10.11.11.121 kubemaster.web.webart4.me <none> <none> kube-system kube-proxy-76qnk 1/1 Running 2 (3h2m ago) 4h16m 10.11.11.129 kubenode2.web.webart4.me <none> <none> kube-system kube-proxy-9mcc4 1/1 Running 48 (3h2m ago) 5h44m 10.11.11.125 kubenode1.web.webart4.me <none> <none> kube-system kube-proxy-p8pkc 1/1 Running 40 (3h2m ago) 6h10m 10.11.11.121 kubemaster.web.webart4.me <none> <none> kube-system kube-scheduler-kubemaster.web.webart4.me 1/1 Running 12 (3h2m ago) 6h11m 10.11.11.121 kubemaster.web.webart4.me <none> <none> |
Delete test deployment
kubectl delete deployment nginx-deployment --namespace default
The icing on the cake - Helm
lxc-attach -n kubemaster
1 2 3 4 5 6 | cd /usr/src/ wget https://get.helm.sh/helm-v4.0.1-linux-amd64.tar.gz sha256sum -c <(printf 'e0365548f01ed52a58a1181ad310b604a3244f59257425bb1739499372bdff60 helm-v4.0.1-linux-amd64.tar.gz') tar -xzv -f helm-v4.0.1-linux-amd64.tar.gz install -m 755 linux-amd64/helm /usr/local/bin/helm helm version |
A few more commands
kubectl get nodes
kubectl get svc --all-namespaces
kubectl get deploy --all-namespaces
kubectl get rs --all-namespaces
kubectl get pods --all-namespaces
kubectl config view
kubectl config current-context
kubeadm token list
kubeadm token create
kubeadm token create --print-join-command
How to get a discovery-token-ca-cert-hash string
1 2 3 4 | cat /etc/kubernetes/pki/ca.crt | \ openssl x509 -pubkey | \ openssl rsa -pubin -outform der 2>/dev/null | \ openssl dgst -sha256 -hex | sed 's/^.* //' |
kubenode1/config
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 | lxc.net.0.type = veth lxc.net.0.flags = up lxc.net.0.link = lxcbr11 lxc.net.0.name = eth125 lxc.net.0.veth.pair = veth125 lxc.net.0.ipv4.address = 10.11.11.125/24 lxc.net.0.hwaddr = 00:16:a9:7d:99:25 lxc.net.0.ipv4.gateway = 10.11.11.1 lxc.apparmor.profile = unconfined lxc.apparmor.allow_nesting = 1 lxc.cap.drop = lxc.cgroup.devices.allow = a lxc.mount.auto = proc:rw sys:rw lxc.cgroup2.devices.allow = c 1:11 rwm lxc.mount.entry = /dev/kmsg dev/kmsg none defaults,bind,create=file lxc.cgroup2.devices.allow = c 10:200 rwm limits.memory.swap = true linux.kernel_modules = ip_tables,ip6_tables,nf_nat,overlay,br_netfilter lxc.mount.entry = /proc/sys/vm/overcommit_memory proc/sys/vm/overcommit_memory none defaults,bind,create=file lxc.mount.entry = /proc/sys/kernel/panic proc/sys/kernel/panic none defaults,bind,create=file lxc.mount.entry = /proc/sys/kernel/panic_on_oops proc/sys/kernel/panic_on_oops none defaults,bind,create=file lxc.rootfs.path = dir:/var/lib/lxc/kubenode1/rootfs # Common configuration lxc.include = /usr/share/lxc/config/debian.common.conf lxc.include = /usr/share/lxc/config/debian.userns.conf # Container specific configuration lxc.tty.max = 4 lxc.uts.name = kubenode1 lxc.arch = amd64 lxc.pty.max = 1024 lxc.start.order = 5 lxc.start.auto = 1 lxc.start.delay = 25 security.nesting = true security.privileged = true |
kubenode2/config
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 | lxc.net.0.type = veth lxc.net.0.flags = up lxc.net.0.link = lxcbr11 lxc.net.0.name = eth129 lxc.net.0.veth.pair = veth129 lxc.net.0.ipv4.address = 10.11.11.129/24 lxc.net.0.hwaddr = 00:16:a9:7d:99:29 lxc.net.0.ipv4.gateway = 10.11.11.1 lxc.apparmor.profile = unconfined lxc.apparmor.allow_nesting = 1 lxc.cap.drop = lxc.cgroup.devices.allow = a lxc.mount.auto = proc:rw sys:rw lxc.cgroup2.devices.allow = c 1:11 rwm lxc.mount.entry = /dev/kmsg dev/kmsg none defaults,bind,create=file lxc.cgroup2.devices.allow = c 10:200 rwm limits.memory.swap = true linux.kernel_modules = ip_tables,ip6_tables,nf_nat,overlay,br_netfilter lxc.mount.entry = /proc/sys/vm/overcommit_memory proc/sys/vm/overcommit_memory none defaults,bind,create=file lxc.mount.entry = /proc/sys/kernel/panic proc/sys/kernel/panic none defaults,bind,create=file lxc.mount.entry = /proc/sys/kernel/panic_on_oops proc/sys/kernel/panic_on_oops none defaults,bind,create=file lxc.rootfs.path = dir:/var/lib/lxc/kubenode2/rootfs # Common configuration lxc.include = /usr/share/lxc/config/debian.common.conf lxc.include = /usr/share/lxc/config/debian.userns.conf # Container specific configuration lxc.tty.max = 4 lxc.uts.name = kubenode2 lxc.arch = amd64 lxc.pty.max = 1024 lxc.start.order = 5 lxc.start.auto = 1 lxc.start.delay = 25 security.nesting = true security.privileged = true |