Ansible-binary部署k8s集群-1.31.2
架构
| 主机名 | IP | VIP | 域名 |
|---|---|---|---|
| master-01 | 192.168.139.121 | ||
| master-02 | 192.168.139.122 | ||
| master-03 | 192.168.139.123 | ||
| node-01 | 192.168.139.124 | ||
| node-02 | 192.168.139.125 | ||
| haproxy(部署服务器) | 192.168.139.36 | 192.168.139.248 | k8s-api.stangj.io |
| harbor | 192.168.139.100 | harbor.stangj.com |
1.）前期准备
1.1）所有节点执行
~# sed -i 's/^\/swap.*$/#&/g' /etc/fstab
`列出swap设备`
~# systemctl --type swap
...
UNIT LOAD ACTIVE SUB DESCRIPTION
swap.img.swap loaded active active /swap.img
...
`关闭swap设备`
~# systemctl mask swap.img.swap
~# swapoff -a
~# modprobe br_netfilter
~# echo "br_netfilter" | sudo tee /etc/modules-load.d/br_netfilter.conf
~# echo -e "net.bridge.bridge-nf-call-iptables = 1\nnet.ipv4.ip_forward = 1" >> /etc/sysctl.conf
~# sysctl -p
_# echo -e '192.168.139.248 k8s-api.stangj.io\n192.168.139.100 harbor.stangj.com' >> /etc/hosts
1.2）时间同步（选做）
"时间同步服务器我这里用的 master-01"
~# apt install chrony -y
~# cat /etc/chrony/chrony.conf | grep -vE "^#|^$"
confdir /etc/chrony/conf.d
server ntp1.aliyun.com iburst
server ntp2.aliyun.com iburst
server ntp3.aliyun.com iburst
allow 192.168.139.0/24
local stratum 10
sourcedir /run/chrony-dhcp
sourcedir /etc/chrony/sources.d
keyfile /etc/chrony/chrony.keys
driftfile /var/lib/chrony/chrony.drift
ntsdumpdir /var/lib/chrony
logdir /var/log/chrony
maxupdateskew 100.0
rtcsync
makestep 1 3
`启动服务`
~# systemctl enable chrony && systemctl restart chrony
`验证`
~# chronyc sources
210 Number of sources = 2
MS Name/IP address Stratum Poll Reach LastRx Last sample
===============================================================================
^- 120.25.115.20 2 6 35 48 +866us[ +866us] +/- 22ms
^* 203.107.6.88 2 6 17 49 -4324us[-9570us] +/- 21ms
"其他节点配置（集群涉及到的节点都要配置--我演示一个）"
~# apt install chrony -y
~# vim /etc/chrony/chrony.conf
#server 0.centos.pool.ntp.org iburst
#server 1.centos.pool.ntp.org iburst
#server 2.centos.pool.ntp.org iburst
#server 3.centos.pool.ntp.org iburst
server 192.168.139.121 iburst # 添加这条信息指向controller1
r~# systemctl restart chrony &&systemctl enable --now chrony
~# chronyc sources
210 Number of sources = 1
MS Name/IP address Stratum Poll Reach LastRx Last sample
===============================================================================
^* 192.168.139.31 3 6 37 60 -2089ns[ -943us] +/- 16ms
1.3）haproxy准备
harbor下载地址：https://github.com/goharbor/harbor/releases/tag/v2.5.4
root@haproxy:~# apt install haproxy
root@haproxy:~# apt -y install keepalived
`配置keepalived`
root@haproxy:~# vim /etc/keepalived/keepalived.conf
global_defs {
smtp_connect_timeout 30
router_id LVS_DEVEL
vrrp_skip_check_adv_addr
vrrp_iptables
vrrp_garp_interval 0
vrrp_gna_interval 0
}
vrrp_instance VI_1 {
state MASTER
interface eth0
virtual_router_id 51
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.139.248 dev eth0 label eth0:0
}
}
root@haproxy:~# systemctl enable --now keepalived.service
root@haproxy:~# systemctl restart keepalived.service
root@haproxy:~# echo -e 'net.ipv4.ip_nonlocal_bind = 1\nnet.ipv4.ip_forward = 1' >> /etc/sysctl.conf
root@haproxy:~# sysctl -p
root@haproxy:~# cat /etc/haproxy/haproxy.cfg
......# 最后添加下面 6 行
listen k8s-api-node
bind 192.168.139.248:6443
mode tcp
server 192.168.139.121 192.168.139.121:6443 check inter 3s fall 3 rise 5
server 192.168.139.122 192.168.139.122:6443 check inter 3s fall 3 rise 5
server 192.168.139.123 192.168.139.123:6443 check inter 3s fall 3 rise 5
root@haproxy:~# systemctl enable --now haproxy.service
root@haproxy:~# systemctl restart haproxy.service
1.4）harbor仓库准备
安装仓库
harbor下载地址：https://github.com/goharbor/harbor/releases/tag/v2.5.6
root@harbor:~# docker --version
Docker version 27.0.3, build 7d4bcd8
root@harbor:~# docker-compose version
Docker Compose version v2.22.0
root@harbor:~# mkdir /apps
root@harbor:~# tar xf harbor-offline-installer-v2.5.6.tgz -C /apps/
root@harbor:~# cd /apps/harbor/
root@harbor:/apps/harbor# cp harbor.yml.tmpl harbor.yml
root@harbor:/apps/harbor# grep -Ev '^#|^$|#' harbor.yml
......
hostname: harbor.stangj.com
http:
port: 80
harbor_admin_password: 123456
database:
password: root123
max_idle_conns: 100
max_open_conns: 900
data_volume: /data
.....
`部署安装`
root@harbor:/apps/harbor# ./install.sh --with-trivy --with-chartmuseum
http://harbor.stangj.com/
登录验证仓库
root@harbor:/apps/harbor# docker login harbor.stangj.com
Username: admin
Password:
Error response from daemon: Get "https://harbor.stangj.com/v2/": tls: failed to verify certificate: x509: certificate is valid for www.stangj.com, stangj.com, not harbor.stangj.com
# 登录出错 因为没有lts证书 不安全
`docker配置不安全仓库认证`
root@harbor:/apps/harbor# vim /etc/docker/daemon.json
{
"registry-mirrors": [
"https://docker.m.daocloud.io",
"https://dockerhub.timeweb.cloud",
"https://huecker.io",
"https://gigi8vhp.mirror.aliyuncs.com"
],
"insecure-registries": [
"harbor.stangj.com" # 将不需要tls认证的域名放这里
],
"exec-opts": ["native.cgroupdriver=systemd"],
"log-driver": "json-file",
"log-opts": {
"max-size": "200m"
},
"storage-driver": "overlay2"
}
root@harbor:/apps/harbor# systemctl restart docker
`Containerd配置不安全仓库认证`
[plugins."io.containerd.grpc.v1.cri".registry]
[plugins."io.containerd.grpc.v1.cri".registry.mirrors]
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."harbor.stangj.com"]
endpoint = ["https://harbor.stangj.com"]
[plugins."io.containerd.grpc.v1.cri".registry.configs]
[plugins."io.containerd.grpc.v1.cri".registry.configs."harbor.stangj.com".tls] # 这里配置对于域名
insecure_skip_verify = true
[plugins."io.containerd.grpc.v1.cri".registry.configs."harbor.stangj.com".auth]
username = "admin"
password = "123456"
~# systemctl daemon-reload
~# systemctl restart containerd
`登录`
root@harbor:/apps/harbor# docker login harbor.stangj.com
Username: admin
Password:
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credential-stores
Login Succeeded
注意：生产环境是要对harbor仓库配置 tls 证书，我们可以在前面用 nginx 反向代理到 harbor ,证书统一在nginx侧管理
2.）部署节点初始化
部署节点配置
1root@haproxy:/usr/local/src# echo '192.168.139.100 harbor.stangj.com' >> /etc/hosts
root@haproxy:/usr/local/src# bash runtime-install.sh docker
root@haproxy:/usr/local/src# docker login harbor.stangj.com
Username: admin
Password:
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store
Login Succeeded
root@haproxy:/usr/local/src# docker pull registry.cn-zhangjiakou.aliyuncs.com/tanghao429/tanghao:demoapp:v1.0
root@openstack-haproxy:~# docker push harbor.stangj.com/baseimage/demoapp:v1.0
The push refers to repository [harbor.stangj.com/baseimage/demoapp]
5a4de022ffa3: Pushed
cc619f457978: Pushed
318b2baca508: Pushed
5216338b40a7: Pushed
v1.0: digest: sha256:6698b205eb18fb0171398927f3a35fe27676c6bf5757ef57a35a4b055badf2c3 size: 1159
免秘钥登录配置
root@haproxy:~# apt install ansible sshpass
root@haproxy:~# ssh-keygen -t rsa-sha2-512 -b 4096
root@haproxy:~#vim key-scp.sh
IP="
192.168.139.121
192.168.139.122
192.168.139.123
192.168.139.124
192.168.139.125
"
REMOTE_PORT="22"
REMOTE_USER="root"
REMOTE_PASS="123456"
for REMOTE_HOST in ${IP};do
REMOTE_CMD="echo ${REMOTE_HOST} is successfully!"
#添加⽬标远程主机的公钥
ssh-keyscan -p "${REMOTE_PORT}" "${REMOTE_HOST}" >> ~/.ssh/known_hosts
#通过sshpass配置免秘钥登录、并创建python3软连接
sshpass -p "${REMOTE_PASS}" ssh-copy-id "${REMOTE_USER}@${REMOTE_HOST}"
ssh ${REMOTE_HOST} ln -sv /usr/bin/python3 /usr/bin/python
echo ${REMOTE_HOST} 免秘钥配置完成!
done
root@haproxy:~# bash key-scp.sh
部署kubeasz
https://github.com/easzlab/kubeasz
root@haproxy:~# apt install git
root@haproxy:~# export release=3.6.5
root@haproxy:~# wget https://github.com/easzlab/kubeasz/releases/download/${release}/ezdown
root@haproxy:~# vim ezdown #可选⾃定义下载组件版本（按需修改可以使用默认配置）
root@haproxy:~# chmod +x ezdown
root@haproxy:~# ./ezdown -D # 其实就是下载一些包为了加速可先用执行 docker pull easzlab/kubeasz:3.6.5（这里可能会报错但是不用担心，这一步的目的是下载一些docker镜像和二进制包，报错可以继续往下做）
root@haproxy:~# ll /etc/kubeasz/
total 140
drwxrwxr-x 12 root root 4096 Dec 26 12:20 ./
drwxr-xr-x 113 root root 4096 Dec 26 12:20 ../
-rw-rw-r-- 1 root root 20304 Nov 12 21:08 ansible.cfg
drwxr-xr-x 5 root root 4096 Dec 26 12:20 bin/
drwxrwxr-x 8 root root 4096 Nov 12 21:16 docs/
drwxr-xr-x 3 root root 4096 Dec 26 12:36 down/
drwxrwxr-x 2 root root 4096 Nov 12 21:16 example/
-rwxrwxr-x 1 root root 26770 Nov 12 21:08 ezctl*
-rwxrwxr-x 1 root root 32666 Nov 12 21:08 ezdown*
drwxrwxr-x 4 root root 4096 Nov 12 21:16 .github/
-rw-rw-r-- 1 root root 301 Nov 12 21:08 .gitignore
drwxrwxr-x 10 root root 4096 Nov 12 21:16 manifests/
drwxrwxr-x 2 root root 4096 Nov 12 21:16 pics/
drwxrwxr-x 2 root root 4096 Nov 12 21:16 playbooks/
-rw-rw-r-- 1 root root 6397 Nov 12 21:08 README.md
drwxrwxr-x 22 root root 4096 Nov 12 21:16 roles/
drwxrwxr-x 2 root root 4096 Nov 12 21:16 tools/
root@haproxy:/etc/kubeasz# vim clusters/k8s-cluster1/config.yml
...
# [.]添加信任的私有仓库
INSECURE_REG:
- "http://easzlab.io.local:5000"
- "http://harbor.stangj.com"
- "https://reg.yourcompany.com"
# [.]基础容器镜像
SANDBOX_IMAGE: "harbor.stangj.com/baseimage/pause:3.10"
...
生成并⾃定义hosts⽂件
`⽣产并⾃定义hosts⽂件`
root@haproxy:~# cd /etc/kubeasz/
root@haproxy:/etc/kubeasz# docker tag easzlab/pause:3.10 harbor.stangj.com/baseimage/pause:3.10
root@haproxy:/etc/kubeasz# docker push harbor.stangj.com/baseimage/pause:3.10
root@haproxy:/etc/kubeasz# ./ezctl new k8s-cluster1
root@haproxy:/etc/kubeasz# cat clusters/k8s-cluster1/hosts
# 'etcd' cluster should have odd member(s) (1,3,5,...)
[etcd]
192.168.139.121
192.168.139.122
192.168.139.123
# master node(s), set unique 'k8s_nodename' for each node
# CAUTION: 'k8s_nodename' must consist of lower case alphanumeric characters, '-' or '.',
# and must start and end with an alphanumeric character
[kube_master]
192.168.139.121 k8s_nodename='master-01'
192.168.139.122 k8s_nodename='master-02'
#192.168.139.123 k8s_nodename='master-03'
# work node(s), set unique 'k8s_nodename' for each node
# CAUTION: 'k8s_nodename' must consist of lower case alphanumeric characters, '-' or '.',
# and must start and end with an alphanumeric character
[kube_node]
192.168.139.124 k8s_nodename='node-01'
#192.168.139.125 k8s_nodename='node-02'
# [optional] harbor server, a private docker registry
# 'NEW_INSTALL': 'true' to install a harbor server; 'false' to integrate with existed one
[harbor]
#192.168.1.8 NEW_INSTALL=false
# [optional] loadbalance for accessing k8s from outside
[ex_lb]
#192.168.1.6 LB_ROLE=backup EX_APISERVER_VIP=192.168.1.250 EX_APISERVER_PORT=8443
#192.168.139.36 LB_ROLE=master EX_APISERVER_VIP=192.168.139.248 EX_APISERVER_PORT=6443
# [optional] ntp server for the cluster
[chrony]
#192.168.1.1
[all:vars]
# --------- Main Variables ---------------
# Secure port for apiservers
SECURE_PORT="6443"
# Cluster container-runtime supported: docker, containerd
# if k8s version >= 1.24, docker is not supported
CONTAINER_RUNTIME="containerd"
# Network plugins supported: calico, flannel, kube-router, cilium, kube-ovn
CLUSTER_NETWORK="calico"
# Service proxy mode of kube-proxy: 'iptables' or 'ipvs'
PROXY_MODE="ipvs"
# K8S Service CIDR, not overlap with node(host) networking
SERVICE_CIDR="10.100.0.0/16"
# Cluster CIDR (Pod CIDR), not overlap with node(host) networking
CLUSTER_CIDR="10.200.0.0/16"
# NodePort Range
NODE_PORT_RANGE="30000-62767"
# Cluster DNS Domain
CLUSTER_DNS_DOMAIN="cluster.local"
# -------- Additional Variables (don't change the default value right now) ---
# Binaries Directory
bin_dir="/usr/local/bin" # 这个比较重要
# Deploy Directory (kubeasz workspace)
base_dir="/etc/kubeasz"
# Directory for a specific cluster
cluster_dir="{{ base_dir }}/clusters/k8s-cluster1"
# CA and other components cert/key Directory
ca_dir="/etc/kubernetes/ssl"
# Default 'k8s_nodename' is empty
k8s_nodename=''
# Default python interpreter
ansible_python_interpreter=/usr/bin/python3
编辑cluster config.yml⽂件
root@haproxy:/etc/kubeasz# cat clusters/k8s-cluster1/config.yml
############################
# prepare
############################
# 可选离线安装系统软件包 (offline|online)
INSTALL_SOURCE: "online"
# 可选进行系统安全加固 github.com/dev-sec/ansible-collection-hardening
# (deprecated) 未更新上游项目，未验证最新k8s集群安装，不建议启用
OS_HARDEN: false
############################
# role:deploy
############################
# default: ca will expire in 100 years
# default: certs issued by the ca will expire in 50 years
CA_EXPIRY: "876000h"
CERT_EXPIRY: "438000h"
# force to recreate CA and other certs, not suggested to set 'true'
CHANGE_CA: false
# kubeconfig 配置参数
CLUSTER_NAME: "cluster1"
CONTEXT_NAME: "context-{{ CLUSTER_NAME }}"
# k8s version
K8S_VER: "1.31.2"
# set unique 'k8s_nodename' for each node, if not set(default:'') ip add will be used
# CAUTION: 'k8s_nodename' must consist of lower case alphanumeric characters, '-' or '.',
# and must start and end with an alphanumeric character (e.g. 'example.com'),
# regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*'
K8S_NODENAME: "{%- if k8s_nodename != '' -%} \
{{ k8s_nodename|replace('_', '-')|lower }} \
{%- else -%} \
k8s-{{ inventory_hostname|replace('.', '-') }} \
{%- endif -%}"
# use 'K8S_NODENAME' to set hostname
ENABLE_SETTING_HOSTNAME: true
############################
# role:etcd
############################
# 设置不同的wal目录，可以避免磁盘io竞争，提高性能
ETCD_DATA_DIR: "/var/lib/etcd"
ETCD_WAL_DIR: ""
############################
# role:runtime [containerd,docker]
############################
# [.]启用拉取加速镜像仓库
ENABLE_MIRROR_REGISTRY: true
# [.]添加信任的私有仓库
# 必须按照如下示例格式，协议头'http://'和'https://'不能省略
INSECURE_REG:
- "http://easzlab.io.local:5000"
- "http://harbor.stangj.com"
- "https://reg.yourcompany.com"
# [.]基础容器镜像
SANDBOX_IMAGE: "harbor.stangj.com/baseimage/pause:3.10"
# [containerd]容器持久化存储目录
CONTAINERD_STORAGE_DIR: "/var/lib/containerd"
# [docker]容器存储目录
DOCKER_STORAGE_DIR: "/var/lib/docker"
# [docker]开启Restful API
DOCKER_ENABLE_REMOTE_API: false
############################
# role:kube-master
############################
# k8s 集群 master 节点证书配置，可以添加多个ip和域名（比如增加公网ip和域名）
MASTER_CERT_HOSTS:
- "192.168.139.248"
- "k8s-api.stangj.io"
# node 节点上 pod 网段掩码长度（决定每个节点最多能分配的pod ip地址）
# 如果flannel 使用 --kube-subnet-mgr 参数，那么它将读取该设置为每个节点分配pod网段
# https://github.com/coreos/flannel/issues/847
NODE_CIDR_LEN: 24
############################
# role:kube-node
############################
# Kubelet 根目录
KUBELET_ROOT_DIR: "/var/lib/kubelet"
# node节点最大pod 数
MAX_PODS: 200
# 配置为kube组件（kubelet,kube-proxy,dockerd等）预留的资源量
# 数值设置详见templates/kubelet-config.yaml.j2
KUBE_RESERVED_ENABLED: "no"
# k8s 官方不建议草率开启 system-reserved, 除非你基于长期监控，了解系统的资源占用状况；
# 并且随着系统运行时间，需要适当增加资源预留，数值设置详见templates/kubelet-config.yaml.j2
# 系统预留设置基于 4c/8g 虚机，最小化安装系统服务，如果使用高性能物理机可以适当增加预留
# 另外，集群安装时候apiserver等资源占用会短时较大，建议至少预留1g内存
SYS_RESERVED_ENABLED: "no"
############################
# role:network [flannel,calico,cilium,kube-ovn,kube-router]
############################
# ------------------------------------------- flannel
# [flannel]设置flannel 后端"host-gw","vxlan"等
FLANNEL_BACKEND: "vxlan"
DIRECT_ROUTING: false
# [flannel]
flannel_ver: "v0.26.0"
# ------------------------------------------- calico
# [calico] IPIP隧道模式可选项有: [Always, CrossSubnet, Never],跨子网可以配置为Always与CrossSubnet(公有云建议使用always比较省事，其他的话需要修改各自公有云的网络配置，具体可以参考各个公有云说明)
# 其次CrossSubnet为隧道+BGP路由混合模式可以提升网络性能，同子网配置为Never即可.
CALICO_IPV4POOL_IPIP: "Always"
# [calico]设置 calico-node使用的host IP，bgp邻居通过该地址建立，可手工指定也可以自动发现
IP_AUTODETECTION_METHOD: "can-reach={{ groups['kube_master'][0] }}"
# [calico]设置calico 网络 backend: bird, vxlan, none
CALICO_NETWORKING_BACKEND: "bird"
# [calico]设置calico 是否使用route reflectors
# 如果集群规模超过50个节点，建议启用该特性
CALICO_RR_ENABLED: false
# CALICO_RR_NODES 配置route reflectors的节点，如果未设置默认使用集群master节点
# CALICO_RR_NODES: ["192.168.1.1", "192.168.1.2"]
CALICO_RR_NODES: []
# [calico]更新支持calico 版本: ["3.19", "3.23"]
calico_ver: "v3.28.2"
# [calico]calico 主版本
calico_ver_main: "{{ calico_ver.split('.')[0] }}.{{ calico_ver.split('.')[1] }}"
# ------------------------------------------- cilium
# [cilium]镜像版本
cilium_ver: "1.16.3"
cilium_connectivity_check: true
cilium_hubble_enabled: false
cilium_hubble_ui_enabled: false
# ------------------------------------------- kube-ovn
# [kube-ovn]离线镜像tar包
kube_ovn_ver: "v1.11.5"
# ------------------------------------------- kube-router
# [kube-router]公有云上存在限制，一般需要始终开启 ipinip；自有环境可以设置为 "subnet"
OVERLAY_TYPE: "full"
# [kube-router]NetworkPolicy 支持开关
FIREWALL_ENABLE: true
# [kube-router]kube-router 镜像版本
kube_router_ver: "v1.5.4"
############################
# role:cluster-addon
############################
# coredns 自动安装
dns_install: "no"
corednsVer: "1.11.3"
ENABLE_LOCAL_DNS_CACHE: false
dnsNodeCacheVer: "1.23.1"
# 设置 local dns cache 地址
LOCAL_DNS_CACHE: "169.254.20.10"
# metric server 自动安装
metricsserver_install: "no"
metricsVer: "v0.7.2"
# dashboard 自动安装
dashboard_install: "no"
dashboardVer: "v2.7.0"
dashboardMetricsScraperVer: "v1.0.8"
# prometheus 自动安装
prom_install: "no"
prom_namespace: "monitor"
prom_chart_ver: "45.23.0"
# kubeapps 自动安装，如果选择安装，默认同时安装local-storage（提供storageClass: "local-path"）
kubeapps_install: "no"
kubeapps_install_namespace: "kubeapps"
kubeapps_working_namespace: "default"
kubeapps_storage_class: "local-path"
kubeapps_chart_ver: "12.4.3"
# local-storage (local-path-provisioner) 自动安装
local_path_provisioner_install: "no"
local_path_provisioner_ver: "v0.0.26"
# 设置默认本地存储路径
local_path_provisioner_dir: "/opt/local-path-provisioner"
# nfs-provisioner 自动安装
nfs_provisioner_install: "no"
nfs_provisioner_namespace: "kube-system"
nfs_provisioner_ver: "v4.0.2"
nfs_storage_class: "managed-nfs-storage"
nfs_server: "192.168.1.10"
nfs_path: "/data/nfs"
# network-check 自动安装
network_check_enabled: false
network_check_schedule: "*/5 * * * *"
############################
# role:harbor
############################
# harbor version，完整版本号
HARBOR_VER: "v2.11.1"
HARBOR_DOMAIN: "harbor.easzlab.io.local"
HARBOR_PATH: /var/data
HARBOR_TLS_PORT: 8443
HARBOR_REGISTRY: "{{ HARBOR_DOMAIN }}:{{ HARBOR_TLS_PORT }}"
# if set 'false', you need to put certs named harbor.pem and harbor-key.pem in directory 'down'
HARBOR_SELF_SIGNED_CERT: false
# install extra component
HARBOR_WITH_TRIVY: false
3.）ezctl 部署k8s集群
环境初始化
root@haproxy:/etc/kubeasz# ./ezctl --help
Usage: ezctl COMMAND [args]
-------------------------------------------------------------------------------------
Cluster setups:
list to list all of the managed clusters
checkout <cluster> to switch default kubeconfig of the cluster
new <cluster> to start a new k8s deploy with name 'cluster'
setup <cluster> <step> to setup a cluster, also supporting a step-by-step way
start <cluster> to start all of the k8s services stopped by 'ezctl stop'
stop <cluster> to stop all of the k8s services temporarily
upgrade <cluster> to upgrade the k8s cluster
destroy <cluster> to destroy the k8s cluster
backup <cluster> to backup the cluster state (etcd snapshot)
restore <cluster> to restore the cluster state from backups
start-aio to quickly setup an all-in-one cluster with default settings
Cluster ops:
add-etcd <cluster> <ip> to add a etcd-node to the etcd cluster
add-master <cluster> <ip> to add a master node to the k8s cluster
add-node <cluster> <ip> to add a work node to the k8s cluster
del-etcd <cluster> <ip> to delete a etcd-node from the etcd cluster
del-master <cluster> <ip> to delete a master node from the k8s cluster
del-node <cluster> <ip> to delete a work node from the k8s cluster
Extra operation:
kca-renew <cluster> to force renew CA certs and all the other certs (with caution)
kcfg-adm <cluster> <args> to manage client kubeconfig of the k8s cluster
Use "ezctl help <command>" for more information about a given command.
root@haproxy:/etc/kubeasz# ./ezctl setup k8s-cluster1 --help
Usage: ezctl setup <cluster> <step>
available steps:
01 prepare to prepare CA/certs & kubeconfig & other system settings
02 etcd to setup the etcd cluster
03 container-runtime to setup the container runtime(docker or containerd)
04 kube-master to setup the master nodes
05 kube-node to setup the worker nodes
06 network to setup the network plugin
07 cluster-addon to setup other useful plugins
90 all to run 01~07 all at once
10 ex-lb to install external loadbalance for accessing k8s from outside
11 harbor to install a new harbor server or to integrate with an existed one
examples: ./ezctl setup test-k8s 01 (or ./ezctl setup test-k8s prepare)
./ezctl setup test-k8s 02 (or ./ezctl setup test-k8s etcd)
./ezctl setup test-k8s all
./ezctl setup test-k8s 04 -t restart_master
`初始化集群`
root@haproxy:/etc/kubeasz# ./ezctl setup k8s-cluster1 01
部署etcd集群
root@haproxy:/etc/kubeasz# ./ezctl setup k8s-cluster1 02 #部署etcd集群
验证集群
root@master-01:~# export NODE_IPS="192.168.139.121 192.168.139.122 192.168.139.123"
root@master-01:~# for ip in ${NODE_IPS}; do ETCDCTL_API=3 /usr/local/bin/etcdctl --endpoints=https://${ip}:2379 --cacert=/etc/kubernetes/ssl/ca.pem --cert=/etc/kubernetes/ssl/etcd.pem --key=/etc/kubernetes/ssl/etcd-key.pem endpoint health; done
https://192.168.139.121:2379 is healthy: successfully committed proposal: took = 8.660445ms
https://192.168.139.122:2379 is healthy: successfully committed proposal: took = 9.191232ms
https://192.168.139.123:2379 is healthy: successfully committed proposal: took = 9.962348ms
部署容器运⾏时containerd
root@haproxy:/etc/kubeasz# grep 'pause' ./clusters/k8s-cluster1/config.yml
SANDBOX_IMAGE: "harbor.stangj.com/baseimage/pause:3.10"
`配置harbor镜像仓库域名解析-公司有DNS服务器进⾏域名解析`
root@haproxy:/etc/kubeasz# vim roles/containerd/tasks/main.yml
# 在 34，35行添加下面两行内容
- name: 添加域名解析
shell: "echo '192.168.139.100 harbor.stangj.com' >> /etc/hosts"
############################################################################################################
###############################下面内容不配置 nerdctl 可以不用执行#############################################
`可选⾃定义containerd配置⽂件（可选）`
root@k8s-deploy:/etc/kubeasz# vim roles/containerd/templates/config.toml.j2
`配置nerdctl客户端方便使用（可选）nerdctl 工具可到github下载 https://github.com/containerd/nerdctl`
root@haproxy:/usr/local/src# tar xf nerdctl-2.0.1-linux-amd64.tar.gz -C /etc/kubeasz/bin/containerd-bin/
root@haproxy:/etc/kubeasz# vim roles/containerd/tasks/main.yml
# 行
5 - block:
6 - name: 准备containerd相关目录
7 file: name={{ item }} state=directory
8 with_items:
9 - "{{ bin_dir }}/containerd-bin"
10 - "/etc/containerd"
11 - "/etc/nerdctl/" # 配置文件目录
16 - name: 下载 containerd 二进制文件
17 copy: src={{ item }} dest={{ bin_dir }}/containerd-bin/ mode=0755
18 with_fileglob:
19 - "{{ base_dir }}/bin/containerd-bin/*"
20 - nerdctl # 添加分发nerdctl
21 tags: upgrade
37 - name: 创建 nerdctl 配置文件
38 template: src=nerdctl.toml.j2 dest=/etc/nerdctl/nerdctl.toml #分发nerdctl配置⽂件
39 tags: upgrade
`nerdctl配置⽂件`
root@haproxy:/etc/kubeasz# vim roles/containerd/templates/nerdctl.toml.j2
namespace = "k8s.io"
debug = false
debug_full = false
insecure_registry = true
执行部署运行时
root@haproxy:/etc/kubeasz# ./ezctl setup k8s-cluster1 03
root@master-01:~# nerdctl -v
nerdctl version 2.0.1
部署kubernetes master节点
root@haproxy:/etc/kubeasz# vim roles/kube-master/tasks/main.yml # 可自定义配置
root@haproxy:/etc/kubeasz# ./ezctl setup k8s-cluster1 04
`将 config 拷贝到 master 节点`
root@haproxy:/etc/kubeasz# export NODE_IPS="192.168.139.121 192.168.139.122 192.168.139.123"
root@haproxy:/etc/kubeasz# for IP in $NODE_IPS ;do scp /root/.kube/config ${IP}:/root/.kube/ ;done
root@master-01:~# kubectl get node
NAME STATUS ROLES AGE VERSION
master-01 Ready,SchedulingDisabled master 15m v1.31.2
master-02 Ready,SchedulingDisabled master 15m v1.31.2
部署kubernetes node节点
root@haproxy:/etc/kubeasz# ./ezctl setup k8s-cluster1 05
root@master-01:~# kubectl get node
NAME STATUS ROLES AGE VERSION
master-01 Ready,SchedulingDisabled master 15m v1.31.2
master-02 Ready,SchedulingDisabled master 15m v1.31.2
node-01 Ready node 6m18s v1.31.2
部署⽹络服务calico
更改calico的镜像地址及各种配置信息（可选）
root@haproxy:/etc/kubeasz# vim ./clusters/k8s-cluster1/config.yml # 可自定义修改
....
# [calico]更新支持calico 版本: ["3.19", "3.23"]
calico_ver: "v3.28.2"
.....
`查看镜像`
root@haproxy:/etc/kubeasz# grep "image:" roles/calico/templates/calico-v3.28.yaml.j2
image: easzlab.io.local:5000/calico/cni:{{ calico_ver }}
image: easzlab.io.local:5000/calico/node:{{ calico_ver }}
image: easzlab.io.local:5000/calico/node:{{ calico_ver }}
image: easzlab.io.local:5000/calico/kube-controllers:{{ calico_ver }}
`修改镜像`
root@haproxy:/etc/kubeasz# docker images | grep calico | grep easzlab
easzlab.io.local:5000/calico/kube-controllers v3.28.2 b5eafd94250c 3 months ago 79MB
easzlab.io.local:5000/calico/cni v3.28.2 8db78ebb3152 3 months ago 209MB
easzlab.io.local:5000/calico/node v3.28.2 f3bc59d46b56 3 months ago 372MB
root@haproxy:/etc/kubeasz# docker tag easzlab.io.local:5000/calico/kube-controllers:v3.28.2 harbor.stangj.com/baseimage/kube-controllers:v3.28.2
root@haproxy:/etc/kubeasz# docker push harbor.stangj.com/baseimage/kube-controllers:v3.28.2
root@haproxy:/etc/kubeasz# docker tag easzlab.io.local:5000/calico/cni:v3.28.2 harbor.stangj.com/baseimage/cni:v3.28.2
root@haproxy:/etc/kubeasz# docker push harbor.stangj.com/baseimage/cni:v3.28.2
root@haproxy:/etc/kubeasz# docker tag easzlab.io.local:5000/calico/node:v3.28.2 harbor.stangj.com/baseimage/node:v3.28.2
root@haproxy:/etc/kubeasz# docker push harbor.stangj.com/baseimage/node:v3.28.2
`修改yaml⽂件中的镜像地址`
root@haproxy:/etc/kubeasz# vim roles/calico/templates/calico-v3.28.yaml.j2
root@haproxy:/etc/kubeasz# grep "image:" roles/calico/templates/calico-v3.28.yaml.j2
image: harbor.stangj.com/baseimage/cni:v3.28.2
image: harbor.stangj.com/baseimage/node:v3.28.2
image: harbor.stangj.com/baseimage/node:v3.28.2
image: harbor.stangj.com/baseimage/kube-controllers:v3.28.2
部署calico
root@haproxy:/etc/kubeasz# ./ezctl setup k8s-cluster1 06
验证calico
root@master-01:~# calicoctl node status
Calico process is running.
IPv4 BGP status
+-----------------+-------------------+-------+----------+-------------+
| PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO |
+-----------------+-------------------+-------+----------+-------------+
| 192.168.139.122 | node-to-node mesh | up | 09:23:34 | Established |
| 192.168.139.124 | node-to-node mesh | up | 09:23:46 | Established |
+-----------------+-------------------+-------+----------+-------------+
验证pod 通信
root@master-01:~# kubectl run client-1 --image=harbor.stangj.com/baseimage/demoapp:v1.0
root@master-01:~# kubectl run client-2 --image=harbor.stangj.com/baseimage/demoapp:v1.0
root@master-01:~# kubectl get pod -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
client-1 1/1 Running 0 3m8s 10.200.190.6 node-01 <none> <none>
client-2 1/1 Running 0 2m59s 10.200.190.7 node-01 <none> <none>
root@master-01:~# kubectl exec -it client-1 -- sh
[root@client-1 /]# ping 10.200.190.7
PING 10.200.190.7 (10.200.190.7): 56 data bytes
64 bytes from 10.200.190.7: seq=0 ttl=63 time=0.198 ms
部署coredns
coredns1.11.3 的yaml文件
coredns版本对照表
https://github.com/coredns/deployment/blob/master/kubernetes/CoreDNS-k8s_version.md
root@haproxy:/etc/kubeasz# docker tag easzlab.io.local:5000/coredns/coredns:1.11.3 harbor.stangj.com/baseimage/coredns:1.11.3
root@haproxy:/etc/kubeasz# docker push harbor.stangj.com/baseimage/coredns:1.11.3
root@master-01:~# cat coredns-1.11.3.yaml
# __MACHINE_GENERATED_WARNING__
apiVersion: v1
kind: ServiceAccount
metadata:
name: coredns
namespace: kube-system
labels:
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
kubernetes.io/bootstrapping: rbac-defaults
addonmanager.kubernetes.io/mode: Reconcile
name: system:coredns
rules:
- apiGroups:
- ""
resources:
- endpoints
- services
- pods
- namespaces
verbs:
- list
- watch
- apiGroups:
- discovery.k8s.io
resources:
- endpointslices
verbs:
- list
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
annotations:
rbac.authorization.kubernetes.io/autoupdate: "true"
labels:
kubernetes.io/bootstrapping: rbac-defaults
addonmanager.kubernetes.io/mode: EnsureExists
name: system:coredns
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:coredns
subjects:
- kind: ServiceAccount
name: coredns
namespace: kube-system
---
apiVersion: v1
kind: ConfigMap
metadata:
name: coredns
namespace: kube-system
labels:
addonmanager.kubernetes.io/mode: EnsureExists
data:
Corefile: |
.:53 {
errors
health {
lameduck 5s
}
ready
kubernetes cluster.local in-addr.arpa ip6.arpa { # 修改点
pods insecure
fallthrough in-addr.arpa ip6.arpa
ttl 30
}
prometheus :9153
#forward . /etc/resolv.conf {
forward . 223.6.6.6 { # 修改点
max_concurrent 1000
}
cache 600 # 修改点
loop
reload
loadbalance
}
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: coredns
namespace: kube-system
labels:
k8s-app: kube-dns
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
kubernetes.io/name: "CoreDNS"
spec:
# replicas: not specified here:
# 1. In order to make Addon Manager do not reconcile this replicas parameter.
# 2. Default is 1.
# 3. Will be tuned in real time if DNS horizontal auto-scaling is turned on.
strategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 1
selector:
matchLabels:
k8s-app: kube-dns
template:
metadata:
labels:
k8s-app: kube-dns
spec:
securityContext:
seccompProfile:
type: RuntimeDefault
priorityClassName: system-cluster-critical
serviceAccountName: coredns
affinity:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchExpressions:
- key: k8s-app
operator: In
values: ["kube-dns"]
topologyKey: kubernetes.io/hostname
tolerations:
- key: "CriticalAddonsOnly"
operator: "Exists"
nodeSelector:
kubernetes.io/os: linux
containers:
- name: coredns
image: harbor.stangj.com/baseimage/coredns:1.11.3
imagePullPolicy: IfNotPresent
resources:
limits: # 修改点
memory: 256Mi
cpu: 200m
requests:
cpu: 100m
memory: 70Mi
args: [ "-conf", "/etc/coredns/Corefile" ]
volumeMounts:
- name: config-volume
mountPath: /etc/coredns
readOnly: true
ports:
- containerPort: 53
name: dns
protocol: UDP
- containerPort: 53
name: dns-tcp
protocol: TCP
- containerPort: 9153
name: metrics
protocol: TCP
livenessProbe:
httpGet:
path: /health
port: 8080
scheme: HTTP
initialDelaySeconds: 60
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 5
readinessProbe:
httpGet:
path: /ready
port: 8181
scheme: HTTP
securityContext:
allowPrivilegeEscalation: false
capabilities:
add:
- NET_BIND_SERVICE
drop:
- ALL
readOnlyRootFilesystem: true
dnsPolicy: Default
volumes:
- name: config-volume
configMap:
name: coredns
items:
- key: Corefile
path: Corefile
---
apiVersion: v1
kind: Service
metadata:
name: kube-dns
namespace: kube-system
annotations:
prometheus.io/port: "9153"
prometheus.io/scrape: "true"
labels:
k8s-app: kube-dns
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
kubernetes.io/name: "CoreDNS"
spec:
selector:
k8s-app: kube-dns
clusterIP: 10.100.0.2 # /etc/kubeasz# grep 'SERVICE_CIDR' clusters/k8s-cluster1/hosts 集群service IP # 修改点
ports: # SERVICE_CIDR="10.100.0.0/16"
- name: dns
port: 53
protocol: UDP
- name: dns-tcp
port: 53
protocol: TCP
- name: metrics
port: 9153
protocol: TCP
验证服务
root@master-01:~# kubectl get po -A
NAMESPACE NAME READY STATUS RESTARTS AGE
default client-1 1/1 Running 0 52m
default client-2 1/1 Running 0 52m
kube-system calico-kube-controllers-7c54ddc76b-x9wbl 1/1 Running 0 65m
kube-system calico-node-g2t8r 1/1 Running 0 65m
kube-system calico-node-n879l 1/1 Running 0 65m
kube-system calico-node-tvj9q 1/1 Running 0 65m
kube-system coredns-5c6d57bd94-k59j5 1/1 Running 0 4m34s
验证pod内的域名通信
root@master-01:~# kubectl create deployment demoapp --image=harbor.stangj.com/baseimage/demoapp:v1.0 --replicas=1
root@master-01:~# kubectl create service nodeport demoapp --tcp=80:80
root@master-01:~# kubectl run client-$RANDOM --image=registry.cn-zhangjiakou.aliyuncs.com/tanghao429/tanghao:admin-box.v-1.2 -it --command -- /bin/sh
root@client-10711 # ping www.baidu.com
PING www.baidu.com (110.242.68.4): 56 data bytes
64 bytes from 110.242.68.4: seq=0 ttl=127 time=12.037 ms
64 bytes from 110.242.68.4: seq=1 ttl=127 time=11.329 ms
root@client-10711 # while true; do curl demoapp.default.svc; sleep 1; done
iKubernetes demoapp v1.0 !! ClientIP: 10.200.190.15, ServerName: demoapp-69f7596784-km8wl, ServerIP: 10.200.190.14!
iKubernetes demoapp v1.0 !! ClientIP: 10.200.190.15, ServerName: demoapp-69f7596784-km8wl, ServerIP: 10.200.190.14!
iKubernetes demoapp v1.0 !! ClientIP: 10.200.190.15, ServerName: demoapp-69f7596784-km8wl, ServerIP: 10.200.190.14!
到此 K8S 集群已经搭建好了
4.）ezctl 附加功能
添加一个master节点
root@haproxy:/etc/kubeasz# ./ezctl add-master --help
root@haproxy:/etc/kubeasz# ./ezctl add-master k8s-cluster1 192.168.139.123
root@haproxy:/etc/kubeasz# kubectl get node
NAME STATUS ROLES AGE VERSION
k8s-192-168-139-123 Ready,SchedulingDisabled master 90s v1.31.2
master-01 Ready,SchedulingDisabled master 131m v1.31.2
master-02 Ready,SchedulingDisabled master 131m v1.31.2
node-01 Ready node 122m v1.31.2
删除一个master节点
root@haproxy:/etc/kubeasz# ./ezctl del-master k8s-cluster1 192.168.139.123
root@master-01:~# kubectl get node
NAME STATUS ROLES AGE VERSION
master-01 Ready,SchedulingDisabled master 134m v1.31.2
master-02 Ready,SchedulingDisabled master 134m v1.31.2
node-01 Ready node 125m v1.31.2
添加一个node节点
root@haproxy:/etc/kubeasz# ./ezctl add-node k8s-cluster1 192.168.139.125
root@master-01:~# kubectl get node
NAME STATUS ROLES AGE VERSION
k8s-192-168-139-125 Ready node 2m23s v1.31.2
master-01 Ready,SchedulingDisabled master 4h28m v1.31.2
master-02 Ready,SchedulingDisabled master 4h28m v1.31.2
node-01 Ready node 4h18m v1.31.2
验证服务
root@master-01:~# kubectl create deployment demoapp --image=harbor.stangj.com/baseimage/demoapp:v1.0 --replicas=2
root@master-01:~# kubectl create service nodeport demoapp --tcp=80:80
root@master-01:~# kubectl get pod -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
client-10711 1/1 Running 1 (154m ago) 161m 10.200.190.11 node-01 <none> <none>
demoapp-69f7596784-xrtzx 1/1 Running 0 100s 10.200.190.16 node-01 <none> <none>
demoapp-69f7596784-xzpwf 1/1 Running 0 100s 10.200.208.65 k8s-192-168-139-125 <none> <none>
root@master-01:~# kubectl run client-$RANDOM --image=registry.cn-zhangjiakou.aliyuncs.com/tanghao429/tanghao:admin-box.v-1.2 -it --command -- /bin/sh
root@client-19584 # while true; do curl demoapp.default.svc; sleep 1; done
iKubernetes demoapp v1.0 !! ClientIP: 10.200.208.66, ServerName: demoapp-69f7596784-xzpwf, ServerIP: 10.200.208.65!
iKubernetes demoapp v1.0 !! ClientIP: 10.200.208.66, ServerName: demoapp-69f7596784-xrtzx, ServerIP: 10.200.190.16!
iKubernetes demoapp v1.0 !! ClientIP: 10.200.208.66, ServerName: demoapp-69f7596784-xzpwf, ServerIP: 10.200.208.65!
root@client-19584 # ping jd.com
PING jd.com (211.144.24.218): 56 data bytes
64 bytes from 211.144.24.218: seq=0 ttl=127 time=5.443 ms
删除一个node节点
root@haproxy:/etc/kubeasz# ./ezctl del-node k8s-cluster1 192.168.139.124
root@master-01:~# kubectl get node
NAME STATUS ROLES AGE VERSION
k8s-192-168-139-123 Ready,SchedulingDisabled master 7m37s v1.31.2
k8s-192-168-139-125 Ready node 34m v1.31.2
master-01 Ready,SchedulingDisabled master 5h v1.31.2
master-02 Ready,SchedulingDisabled master 5h v1.31.2
5.）部署附件
部署 dashboard
镜像准备
`上传镜像`
root@haproxy:/etc/kubeasz# docker images | grep dashboard
kubernetesui/dashboard v2.7.0 07655ddf2eeb 2 years ago 246MB
easzlab.io.local:5000/kubernetesui/dashboard v2.7.0 07655ddf2eeb 2 years ago 246MB
root@haproxy:/etc/kubeasz# docker tag kubernetesui/dashboard:v2.7.0 harbor.stangj.com/baseimage/dashboard:v2.7.0
root@haproxy:/etc/kubeasz# docker push harbor.stangj.com/baseimage/dashboard:v2.7.0
root@haproxy:/etc/kubeasz# docker tag kubernetesui/metrics-scraper:v1.0.8 harbor.stangj.com/baseimage/metrics-scraper:v1.0.8
root@haproxy:/etc/kubeasz# docker push harbor.stangj.com/baseimage/metrics-scraper:v1.0.8
资源文件准备
`编写资源yaml文件`
root@master-01:~# cat admin-secret.yaml
apiVersion: v1
kind: Secret
type: kubernetes.io/service-account-token
metadata:
name: dashboard-admin-user
namespace: kubernetes-dashboard
annotations:
kubernetes.io/service-account.name: "admin-user"
root@master-01:~# cat admin-user.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: admin-user
namespace: kubernetes-dashboard
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: admin-user
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-admin
subjects:
- kind: ServiceAccount
name: admin-user
namespace: kubernetes-dashboard
root@master-01:~# cat dashboard-v2.7.0.yaml
# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
apiVersion: v1
kind: Namespace
metadata:
name: kubernetes-dashboard
---
apiVersion: v1
kind: ServiceAccount
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard
namespace: kubernetes-dashboard
---
kind: Service
apiVersion: v1
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard
namespace: kubernetes-dashboard
spec:
type: NodePort
ports:
- port: 443
targetPort: 8443
nodePort: 30000
selector:
k8s-app: kubernetes-dashboard
---
apiVersion: v1
kind: Secret
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard-certs
namespace: kubernetes-dashboard
type: Opaque
---
apiVersion: v1
kind: Secret
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard-csrf
namespace: kubernetes-dashboard
type: Opaque
data:
csrf: ""
---
apiVersion: v1
kind: Secret
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard-key-holder
namespace: kubernetes-dashboard
type: Opaque
---
kind: ConfigMap
apiVersion: v1
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard-settings
namespace: kubernetes-dashboard
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard
namespace: kubernetes-dashboard
rules:
# Allow Dashboard to get, update and delete Dashboard exclusive secrets.
- apiGroups: [""]
resources: ["secrets"]
resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs", "kubernetes-dashboard-csrf"]
verbs: ["get", "update", "delete"]
# Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map.
- apiGroups: [""]
resources: ["configmaps"]
resourceNames: ["kubernetes-dashboard-settings"]
verbs: ["get", "update"]
# Allow Dashboard to get metrics.
- apiGroups: [""]
resources: ["services"]
resourceNames: ["heapster", "dashboard-metrics-scraper"]
verbs: ["proxy"]
- apiGroups: [""]
resources: ["services/proxy"]
resourceNames: ["heapster", "http:heapster:", "https:heapster:", "dashboard-metrics-scraper", "http:dashboard-metrics-scraper"]
verbs: ["get"]
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard
rules:
# Allow Metrics Scraper to get metrics from the Metrics server
- apiGroups: ["metrics.k8s.io"]
resources: ["pods", "nodes"]
verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard
namespace: kubernetes-dashboard
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: kubernetes-dashboard
subjects:
- kind: ServiceAccount
name: kubernetes-dashboard
namespace: kubernetes-dashboard
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: kubernetes-dashboard
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: kubernetes-dashboard
subjects:
- kind: ServiceAccount
name: kubernetes-dashboard
namespace: kubernetes-dashboard
---
kind: Deployment
apiVersion: apps/v1
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard
namespace: kubernetes-dashboard
spec:
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
k8s-app: kubernetes-dashboard
template:
metadata:
labels:
k8s-app: kubernetes-dashboard
spec:
securityContext:
seccompProfile:
type: RuntimeDefault
containers:
- name: kubernetes-dashboard
image: harbor.stangj.com/baseimage/dashboard:v2.7.0
imagePullPolicy: Always
ports:
- containerPort: 8443
protocol: TCP
args:
- --auto-generate-certificates
- --namespace=kubernetes-dashboard
- --token-ttl=3600
# Uncomment the following line to manually specify Kubernetes API server Host
# If not specified, Dashboard will attempt to auto discover the API server and connect
# to it. Uncomment only if the default does not work.
# - --apiserver-host=http://my-address:port
volumeMounts:
- name: kubernetes-dashboard-certs
mountPath: /certs
# Create on-disk volume to store exec logs
- mountPath: /tmp
name: tmp-volume
livenessProbe:
httpGet:
scheme: HTTPS
path: /
port: 8443
initialDelaySeconds: 30
timeoutSeconds: 30
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
runAsUser: 1001
runAsGroup: 2001
volumes:
- name: kubernetes-dashboard-certs
secret:
secretName: kubernetes-dashboard-certs
- name: tmp-volume
emptyDir: {}
serviceAccountName: kubernetes-dashboard
nodeSelector:
"kubernetes.io/os": linux
# Comment the following tolerations if Dashboard must not be deployed on master
tolerations:
- key: node-role.kubernetes.io/master
effect: NoSchedule
---
kind: Service
apiVersion: v1
metadata:
labels:
k8s-app: dashboard-metrics-scraper
name: dashboard-metrics-scraper
namespace: kubernetes-dashboard
spec:
ports:
- port: 8000
targetPort: 8000
selector:
k8s-app: dashboard-metrics-scraper
---
kind: Deployment
apiVersion: apps/v1
metadata:
labels:
k8s-app: dashboard-metrics-scraper
name: dashboard-metrics-scraper
namespace: kubernetes-dashboard
spec:
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
k8s-app: dashboard-metrics-scraper
template:
metadata:
labels:
k8s-app: dashboard-metrics-scraper
spec:
securityContext:
seccompProfile:
type: RuntimeDefault
containers:
- name: dashboard-metrics-scraper
image: harbor.stangj.com/baseimage/metrics-scraper:v1.0.8
ports:
- containerPort: 8000
protocol: TCP
livenessProbe:
httpGet:
scheme: HTTP
path: /
port: 8000
initialDelaySeconds: 30
timeoutSeconds: 30
volumeMounts:
- mountPath: /tmp
name: tmp-volume
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
runAsUser: 1001
runAsGroup: 2001
serviceAccountName: kubernetes-dashboard
nodeSelector:
"kubernetes.io/os": linux
# Comment the following tolerations if Dashboard must not be deployed on master
tolerations:
- key: node-role.kubernetes.io/master
effect: NoSchedule
volumes:
- name: tmp-volume
emptyDir: {}
部署服务
root@master-01:~# kubectl create ns kubernetes-dashboard
root@master-01:~# kubectl apply -f admin-user.yaml
root@master-01:~# kubectl apply -f admin-secret.yaml
root@master-01:~# kubectl apply -f dashboard-v2.7.0.yaml
root@master-01:~# kubectl get pod -n kubernetes-dashboard
NAME READY STATUS RESTARTS AGE
dashboard-metrics-scraper-5b49cb7f4c-cggml 1/1 Running 0 26s
kubernetes-dashboard-56bc75885-ckzd8 1/1 Running 0 26s
获取token并登录dashboard
root@master-01:~# kubectl get secret -A | grep admin
kubernetes-dashboard dashboard-admin-user kubernetes.io/service-account-token 3 103s
root@master-01:~# kubectl -n kubernetes-dashboard describe secret dashboard-admin-user
Name: dashboard-admin-user
Namespace: kubernetes-dashboard
Labels: <none>
Annotations: kubernetes.io/service-account.name: admin-user
kubernetes.io/service-account.uid: 55fa17b0-edcf-4244-906a-1a10ce2a7fbb
Type: kubernetes.io/service-account-token
Data
====
namespace: 20 bytes
token: eyJhbGciOiJSUzI1NiIsImtpZCI6IkFKY3UzQXlJem9Sdm1tZ3dyZHNMcnRRZkNSS2ctTWU5ZGUtdHhTWEU1VEUifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlcm5ldGVzLWRhc2hib2FyZCIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJkYXNoYm9hcmQtYWRtaW4tdXNlciIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50Lm5hbWUiOiJhZG1pbi11c2VyIiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQudWlkIjoiNTVmYTE3YjAtZWRjZi00MjQ0LTkwNmEtMWExMGNlMmE3ZmJiIiwic3ViIjoic3lzdGVtOnNlcnZpY2VhY2NvdW50Omt1YmVybmV0ZXMtZGFzaGJvYXJkOmFkbWluLXVzZXIifQ.BoHEbTatvvH95je_7MBexzGsQBqlNLZ4SzSOWX1bgOg-0A4WGHWDvo-cx16Q2nJ7IiM3G-DWg0ZtbF_4ITfv-ra-V9YUYwA3mZ8HxRZ9SAqpEgfmk3kTXlugFV09a4sYrFhvCd24RregT1IK1zU3CfjRfpSzeFZ6RkpwHJX4Xy-rFK_Wgft_b-LYb-EEALP7xgKetu6vVvWMCEHOI6_f592_yQvcxVj55Tpt6z7jCEzpZQI3gTOAA5-2rp2B0qYDXp-qGw3kzT8YoUZvoQUTUDsmFpq1emK6Bq0WUeI3SOmmGe59ZKjjwztImGR3fAW_8lP-4z69rNKx4i63xXftPg
ca.crt: 1310 bytes
root@master-01:~# kubectl get svc -n kubernetes-dashboard
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
dashboard-metrics-scraper ClusterIP 10.100.208.50 <none> 8000/TCP 2m34s
kubernetes-dashboard NodePort 10.100.254.152 <none> 443:30000/TCP 2m35s
设置token登录会话保持时间
root@master-01:~# vim dashboard-v2.7.0.yaml
...........
spec:
securityContext:
seccompProfile:
type: RuntimeDefault
containers:
- name: kubernetes-dashboard
image: harbor.stangj.com/baseimage/dashboard:v2.7.0
imagePullPolicy: Always
ports:
- containerPort: 8443
protocol: TCP
args:
- --auto-generate-certificates
- --namespace=kubernetes-dashboard
- --token-ttl=3600 # 可以通过这个参数修改会话保持时间
部署 metrics-server
镜像准备
root@haproxy:/etc/kubeasz# docker images | grep metrics-server
easzlab/metrics-server v0.7.2 83375c676b80 4 months ago 67.1MB
easzlab.io.local:5000/easzlab/metrics-server v0.7.2 83375c676b80 4 months ago 67.1MB
root@haproxy:/etc/kubeasz# docker tag easzlab/metrics-server:v0.7.2 harbor.stangj.com/baseimage/metrics-server:v0.7.2
root@haproxy:/etc/kubeasz# docker push harbor.stangj.com/baseimage/metrics-server:v0.7.2
部署服务
root@master-01:~# wget https://github.com/kubernetes-sigs/metrics-server/releases/download/v0.7.2/components.yaml
root@master-01:~# mv components.yaml metrics-server.yaml
root@master-01:~# vim metrics-server.yaml
....
spec:
containers:
- args:
- --cert-dir=/tmp
- --secure-port=10250
- --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
- --kubelet-use-node-status-port
- --metric-resolution=15s
- --kubelet-insecure-tls # 添加这一行
image: harbor.stangj.com/baseimage/metrics-server:v0.7.2 # 修改镜像地址
imagePullPolicy: IfNotPresent
....
root@master-01:~# kubectl apply -f metrics-server.yaml
root@master-01:~# kubectl get pod -n kube-system
NAME READY STATUS RESTARTS AGE
calico-kube-controllers-7c54ddc76b-m58k6 1/1 Running 0 49m
calico-node-4wzxw 1/1 Running 0 41m
calico-node-8vb5t 1/1 Running 0 41m
calico-node-hdcmh 1/1 Running 0 42m
calico-node-q2plr 1/1 Running 0 41m
coredns-5c6d57bd94-dbbfj 1/1 Running 0 49m
metrics-server-59c9f779c9-dbj2f 1/1 Running 0 47s
root@master-01:~# kubectl top node
NAME CPU(cores) CPU% MEMORY(bytes) MEMORY%
k8s-192-168-139-123 52m 2% 1143Mi 43%
k8s-192-168-139-125 47m 2% 1436Mi 54%
master-01 71m 3% 2251Mi 62%
master-02 81m 4% 1720Mi 65%
