kubeadm方式安装最新版k8s-1.29.1（containerd容器运行时）
containerd-1.6.27 + k8s-1.29.1（最新）（kubeadm方式）
| 主机名 | IP地址 | 资源 |
|---|---|---|
| k8s-master | 192.168.139.30 | 2c/3g-80g |
| k8s-node1 | 192.168.139.31 | 2c/3g-80g |
| k8s-node2 | 192.168.139.32 | 2c/3g-80g |
| k8s-node3 | 192.168.139.33 | 2c/3g-80g |
前期服务器配置
-
四台服务器都要配置
`修改主机名`
hostnamectl set-hostname [主机名] && su -
`添加hosts解析`
cat >> /etc/hosts << EOF
192.168.139.30 k8s-master
192.168.139.31 k8s-node1
192.168.139.32 k8s-node2
192.168.139.33 k8s-node3
EOF
`同步时间`
yum -y install ntp
systemctl enable ntpd --now
`永久关闭seLinux(需重启系统生效)`
setenforce 0
sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
`永久关闭swap(需重启系统生效)`
swapoff -a # 临时关闭
sed -i 's/.*swap.*/#&/g' /etc/fstab # 永久关闭
`更新阿里源`
wget -O /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-7.repo
wget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo
`升级内核为5.4版本(需重启系统生效)`
# https://elrepo.org/tiki/kernel-lt
# https://elrepo.org/linux/kernel/el7/x86_64/RPMS/
rpm -Uvh http://www.elrepo.org/elrepo-release-7.0-6.el7.elrepo.noarch.rpm
yum --disablerepo="*" --enablerepo="elrepo-kernel" list available
yum --enablerepo=elrepo-kernel install -y kernel-lt
reboot
`关闭防火墙、清空iptables规则`
systemctl disable --now firewalld
iptables -F && iptables -t nat -F && iptables -t mangle -F && iptables -X && iptables -P FORWARD ACCEPT
`关闭 NetworkManager`
systemctl disable --now NetworkManager
`加载IPVS模块`
yum -y install ipset ipvsadm
mkdir -p /etc/sysconfig/modules
cat > /etc/sysconfig/modules/ipvs.modules <<EOF
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack
EOF
modprobe -- nf_conntrack
chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep -E 'ip_vs|nf_conntrack'
`开启br_netfilter、ipv4 路由转发`
cat <<EOF | tee /etc/modules-load.d/k8s.conf
overlay
br_netfilter
EOF
modprobe overlay
modprobe br_netfilter
# 设置所需的 sysctl 参数，参数在重新启动后保持不变
cat <<EOF | tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward = 1
EOF
# 应用 sysctl 参数而不重新启动
sysctl -p /etc/sysctl.d/k8s.conf
# 查看是否生效
lsmod | grep br_netfilter
lsmod | grep overlay
sysctl net.bridge.bridge-nf-call-iptables net.bridge.bridge-nf-call-ip6tables net.ipv4.ip_forward
`内核调优`
cat > /etc/sysctl.d/99-sysctl.conf << 'EOF'
net.ipv4.conf.default.rp_filter = 1
net.ipv4.conf.default.accept_source_route = 0
kernel.core_uses_pid = 1
net.ipv4.tcp_syncookies = 1
kernel.msgmnb = 65536
kernel.msgmax = 65536
net.ipv4.conf.all.promote_secondaries = 1
net.ipv4.conf.default.promote_secondaries = 1
net.ipv6.neigh.default.gc_thresh3 = 4096
kernel.sysrq = 1
net.ipv6.conf.all.disable_ipv6=0
net.ipv6.conf.default.disable_ipv6=0
net.ipv6.conf.lo.disable_ipv6=0
kernel.numa_balancing = 0
kernel.shmmax = 68719476736
kernel.printk = 5
net.core.rps_sock_flow_entries=8192
net.bridge.bridge-nf-call-ip6tables=1
net.ipv4.ip_local_reserved_ports=60001,60002
net.core.rmem_max=16777216
fs.inotify.max_user_watches=524288
kernel.core_pattern=core
net.core.dev_weight_tx_bias=1
net.ipv4.tcp_max_orphans=32768
kernel.pid_max=4194304
kernel.softlockup_panic=1
fs.file-max=3355443
net.core.bpf_jit_harden=1
net.ipv4.tcp_max_tw_buckets=32768
fs.inotify.max_user_instances=8192
net.core.bpf_jit_kallsyms=1
vm.max_map_count=262144
kernel.threads-max=262144
net.core.bpf_jit_enable=1
net.ipv4.tcp_keepalive_time=600
net.ipv4.tcp_wmem=4096 12582912 16777216
net.core.wmem_max=16777216
net.ipv4.neigh.default.gc_thresh1=2048
net.core.somaxconn=32768
net.ipv4.neigh.default.gc_thresh3=8192
net.ipv4.ip_forward=1
net.ipv4.neigh.default.gc_thresh2=4096
net.ipv4.tcp_max_syn_backlog=8096
net.bridge.bridge-nf-call-iptables=1
net.ipv4.tcp_rmem=4096 12582912 16777216
EOF
# 应用 sysctl 参数而不重新启动
sysctl --system
`设置资源配置文件`
cat >> /etc/security/limits.conf << 'EOF'
* soft nofile 100001
* hard nofile 100002
root soft nofile 100001
root hard nofile 100002
* soft memlock unlimited
* hard memlock unlimited
* soft nproc 254554
* hard nproc 254554
* soft sigpending 254554
* hard sigpending 254554
EOF
部署1.29.1
安装containerd-1.6.27（官方源）
-
四台服务器都要配置
wget -O /etc/yum.repos.d/docker-ce.repo https://download.docker.com/linux/centos/docker-ce.repo
yum makecache
yum list containerd.io --showduplicates | sort -r
yum -y install containerd.io-1.6.27
containerd config default | sudo tee /etc/containerd/config.toml
# 修改cgroup Driver为systemd
sed -ri 's#SystemdCgroup = false#SystemdCgroup = true#' /etc/containerd/config.toml
# 更改sandbox_image(替换为阿里云镜像)
sed -ri 's#registry.k8s.io\/pause:3.6#registry.aliyuncs.com\/google_containers\/pause:3.9#' /etc/containerd/config.toml
添加镜像加速
docker.io 镜像加速
`指定配置文件目录`
sed -i 's/config_path = ""/config_path = "\/etc\/containerd\/certs.d\/"/g' /etc/containerd/config.toml
`配置加速`
mkdir -p /etc/containerd/certs.d/docker.io
cat > /etc/containerd/certs.d/docker.io/hosts.toml << 'EOF'
server = "https://docker.io" # 源镜像地址
[host."https://xk9ak4u9.mirror.aliyuncs.com"] # 阿里-镜像加速地址
capabilities = ["pull","resolve"]
[host."https://docker.m.daocloud.io"] # 道客-镜像加速地址
capabilities = ["pull","resolve"]
[host."https://dockerproxy.com"] # 镜像加速地址
capabilities = ["pull", "resolve"]
[host."https://docker.mirrors.sjtug.sjtu.edu.cn"] # 上海交大-镜像加速地址
capabilities = ["pull","resolve"]
[host."https://docker.mirrors.ustc.edu.cn"] # 中科大-镜像加速地址
capabilities = ["pull","resolve"]
[host."https://docker.nju.edu.cn"] # 南京大学-镜像加速地址
capabilities = ["pull","resolve"]
[host."https://registry-1.docker.io"]
capabilities = ["pull","resolve","push"]
EOF
registry.k8s.io 镜像加速
mkdir -p /etc/containerd/certs.d/registry.k8s.io
cat > /etc/containerd/certs.d/registry.k8s.io/hosts.toml << 'EOF'
server = "https://registry.k8s.io"
[host."https://k8s.m.daocloud.io"]
capabilities = ["pull", "resolve", "push"]
EOF
quay.io 镜像加速
mkdir -p /etc/containerd/certs.d/quay.io
cat > /etc/containerd/certs.d/quay.io/hosts.toml << 'EOF'
server = "https://quay.io"
[host."https://quay.m.daocloud.io"]
capabilities = ["pull", "resolve", "push"]
EOF
docker.elastic.co镜像加速
mkdir -p /etc/containerd/certs.d/docker.elastic.co
tee /etc/containerd/certs.d/docker.elastic.co/hosts.toml << 'EOF'
server = "https://docker.elastic.co"
[host."https://elastic.m.daocloud.io"]
capabilities = ["pull", "resolve", "push"]
EOF
重新加载配置
systemctl daemon-reload
systemctl enable --now containerd
systemctl restart containerd
systemctl is-active containerd
镜像加速配置无需重启服务，即可生效
设置crictl
cat << EOF >> /etc/crictl.yaml
runtime-endpoint: unix:///var/run/containerd/containerd.sock
image-endpoint: unix:///var/run/containerd/containerd.sock
timeout: 10
debug: false
EOF
安装k8s（kubeadm-1.29.1、kubelet-1.29.1、kubectl-1.29.1）（官方源）(centos-7.9 )
-
四台服务器都要配置
配置源
cat > /etc/yum.repos.d/kubernetes.repo << 'EOF'
[kubernetes]
name=Kubernetes
baseurl=https://pkgs.k8s.io/core:/stable:/v1.29/rpm/
enabled=1
gpgcheck=0
EOF
yum makecache
下载kubeadm-1.29.1、kubelet-1.29.1、kubectl-1.29.1
yum -y install kubeadm-1.29.1 kubelet-1.29.1 kubectl-1.29.1
systemctl enable --now kubelet
初始化 k8s-1.29.1 集群
-
master节点操作
查看所需镜像列表
[root@k8s-master ~]# kubeadm config images list --kubernetes-version v1.29.1
registry.k8s.io/kube-apiserver:v1.29.1
registry.k8s.io/kube-controller-manager:v1.29.1
registry.k8s.io/kube-scheduler:v1.29.1
registry.k8s.io/kube-proxy:v1.29.1
registry.k8s.io/coredns/coredns:v1.11.1
registry.k8s.io/pause:3.9
registry.k8s.io/etcd:3.5.10-0
拉取镜像
crictl pull registry.cn-hangzhou.aliyuncs.com/google_containers/kube-apiserver:v1.29.1
crictl pull registry.cn-hangzhou.aliyuncs.com/google_containers/kube-controller-manager:v1.29.1
crictl pull registry.cn-hangzhou.aliyuncs.com/google_containers/kube-scheduler:v1.29.1
crictl pull registry.cn-hangzhou.aliyuncs.com/google_containers/kube-proxy:v1.29.1
crictl pull registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.9
crictl pull registry.cn-hangzhou.aliyuncs.com/google_containers/etcd:3.5.10-0
crictl pull registry.cn-hangzhou.aliyuncs.com/google_containers/coredns:v1.11.1
初始化
[root@k8s-master ~]# kubeadm init \
--apiserver-advertise-address=192.168.139.30 \
--apiserver-bind-port=6443 \
--cri-socket=unix:///var/run/containerd/containerd.sock \
--kubernetes-version=v1.29.1 \
--pod-network-cidr=10.100.0.0/16 \
--service-cidr=10.200.0.0/16 \
--service-dns-domain=stangjedu.local \
--image-repository=registry.cn-hangzhou.aliyuncs.com/google_containers \
--ignore-preflight-errors=swap
#-------------------------------保存下列信息---------------------------
Your Kubernetes control-plane has initialized successfully!
To start using your cluster, you need to run the following as a regular user:
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
Alternatively, if you are the root user, you can run:
export KUBECONFIG=/etc/kubernetes/admin.conf
You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
https://kubernetes.io/docs/concepts/cluster-administration/addons/
Then you can join any number of worker nodes by running the following on each as root:
kubeadm join 192.168.139.30:6443 --token f8s9d0.v2k17432216ya8yc \
--discovery-token-ca-cert-hash sha256:f487e29e6848b8a4ca29422c6b52ebd67cba93f93c2183c08288f78b84fd7187
#---------------------------------------------------------------
配置 kubectl
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
安装 k8s 集群网络（calico）
-
master节点操作
-
查看calico与k8s的版本对应关系:https://docs.tigera.io/calico/3.27/getting-started/kubernetes/requirements
这里k8s-1.29.1，所以使用calico-v3.27.0版本（版本对应很关键）
下载calico.yaml
[root@k8s-master ~]# mkdir -p ~/calico-yml
[root@k8s-master ~]# cd ~/calico-yml
[root@k8s-master calico-yml]# wget https://github.com/projectcalico/calico/raw/v3.27.0/manifests/calico.yaml
修改calico.yaml
[root@k8s-master calico-yml]# vim calico.yaml
.......
- name: CALICO_IPV4POOL_CIDR
value: "10.100.0.0/16" # pod的地址段
.......
- name: CLUSTER_TYPE
value: "k8s,bgp" # 在这两行下面添加
- name: IP_AUTODETECTION_METHOD
value: "interface=eth0" # eth.*为本地网卡名字（自己机器啥网卡就改啥）也可以eth.*表示通配
.......
kind: DaemonSet
apiVersion: apps/v1
metadata:
name: calico-node
........
k8s-app: calico-node # 在这一行下面添加下面两行
annotations:
unsupported.operator.tigera.io/ignore: "true"
......
应用calico.yaml
[root@k8s-master calico-yml]# kubectl apply -f calico.yaml
验证
[root@k8s-master calico-yml]# kubectl get pod -A
NAMESPACE NAME READY STATUS RESTARTS AGE
kube-system calico-kube-controllers-5fc7d6cf67-85ncd 1/1 Running 0 2m12s
kube-system calico-node-8gdg6 1/1 Running 0 2m12s
kube-system coredns-5f98f8d567-4zksc 1/1 Running 0 32m
kube-system coredns-5f98f8d567-nl5fc 1/1 Running 0 32m
kube-system etcd-k8s-master 1/1 Running 0 32m
kube-system kube-apiserver-k8s-master 1/1 Running 0 32m
kube-system kube-controller-manager-k8s-master 1/1 Running 0 32m
kube-system kube-proxy-7bfl2 1/1 Running 0 32m
kube-system kube-scheduler-k8s-master 1/1 Running 0 32m
增加node节点
-
在node节点执行
[root@k8s-node1 ~]# kubeadm join 192.168.139.30:6443 --token f8s9d0.v2k17432216ya8yc --discovery-token-ca-cert-hash sha256:f487e29e6848b8a4ca29422c6b52ebd67cba93f93c2183c08288f78b84fd7187
`master节点验证`
[root@k8s-master ~]# kubectl get node
NAME STATUS ROLES AGE VERSION
k8s-master Ready control-plane 41m v1.29.1
k8s-node1 NotReady <none> 2m13s v1.29.1 # 这个是NotReady是因为calico网络主键还没拉取下来，需要等一会
[root@k8s-node2 ~]# kubeadm join 192.168.139.30:6443 --token f8s9d0.v2k17432216ya8yc --discovery-token-ca-cert-hash sha256:f487e29e6848b8a4ca29422c6b52ebd67cba93f93c2183c08288f78b84fd7187
[root@k8s-node3 ~]# kubeadm join 192.168.139.30:6443 --token f8s9d0.v2k17432216ya8yc --discovery-token-ca-cert-hash sha256:f487e29e6848b8a4ca29422c6b52ebd67cba93f93c2183c08288f78b84fd7187
-
验证
[root@k8s-master ~]# kubectl get node
NAME STATUS ROLES AGE VERSION
k8s-master Ready control-plane 49m v1.29.1
k8s-node1 Ready <none> 10m v1.29.1
k8s-node2 Ready <none> 6m13s v1.29.1
k8s-node3 Ready <none> 3m27s v1.29.1
验证网络通信
[root@k8s-master ~]# kubectl run test-pod1 --image=alpine sleep 400000
[root@k8s-master ~]# kubectl run test-pod2 --image=alpine sleep 400000
[root@k8s-master ~]# kubectl run test-pod3 --image=alpine sleep 400000
[root@k8s-master ~]# kubectl get pod -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
test-pod1 1/1 Running 0 83s 10.100.36.65 k8s-node1 <none> <none>
test-pod2 1/1 Running 0 78s 10.100.107.193 k8s-node3 <none> <none>
test-pod3 1/1 Running 0 73s 10.100.169.129 k8s-node2 <none> <none>
[root@k8s-master ~]# kubectl exec -it test-pod1 sh
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
/ # ping 10.100.107.193
PING 10.100.107.193 (10.100.107.193): 56 data bytes
64 bytes from 10.100.107.193: seq=0 ttl=62 time=0.488 ms
/ # ping 10.100.169.129
PING 10.100.169.129 (10.100.169.129): 56 data bytes
64 bytes from 10.100.169.129: seq=0 ttl=62 time=0.579 ms
coredns 解析测试是否正常
/ # ping www.baidu.com
PING www.baidu.com (110.242.68.4): 56 data bytes
64 bytes from 110.242.68.4: seq=0 ttl=127 time=71.823 ms
64 bytes from 110.242.68.4: seq=1 ttl=127 time=73.022 ms
64 bytes from 110.242.68.4: seq=2 ttl=127 time=67.093 ms
64 bytes from 110.242.68.4: seq=3 ttl=127 time=69.796 ms
/ # nslookup kubernetes
Server: 10.200.0.10
Address: 10.200.0.10:53
Name: kubernetes.default.svc.stangjedu.local
Address: 10.200.0.1
** server can't find kubernetes.svc.stangjedu.local: NXDOMAIN
** server can't find kubernetes.stangjedu.local: NXDOMAIN
** server can't find kubernetes.stangjedu.local: NXDOMAIN
** server can't find kubernetes.svc.stangjedu.local: NXDOMAIN
查看token
# 列出现有的token
kubeadm token list
# 创建新的token
kubeadm token create --print-join-command
文章评论