K8s结合pv/pvc实现Zookeeper集群部署

2024年1月20日 963点热度 0人点赞 0条评论

k8s之zookeeper集群

基于PV和PVC作为后端存储，实现zookeeper集群

1.1）准备NFS服务器

1.1.1）下载并配置NFS

root@harbor:~# apt install nfs-server -y
root@harbor:~# mkdir -p /data/k8sdata/{img,status}
root@harbor:~# vim /etc/exports 
# 最后一行添加
/data/k8sdata *(rw,no_root_squash)
root@harbor:/data/k8sdata# mkdir -p /data/k8sdata/zookeeper/datadir-pv-{1..3}
root@harbor:/data/k8sdata# mkdir -p /data/k8sdata/zookeeper/datalog-pv-{1..3}
root@harbor:~# systemctl restart nfs-server.service 
root@harbor:~# systemctl enable nfs-server.service

1.1.2）验证NFS

root@k8s-master01:~# apt install nfs-common
root@k8s-master01:~# showmount -e 192.168.139.234
Export list for 192.168.139.234:
/data/k8sdata *

1.2）编写namespace/pv/pvc文件

1.2.1）编写namespace的yaml文件并应用

root@k8s-master01:/stj-soft/yaml/namespaces# vim move-namespaces.yaml
apiVersion: v1
kind: Namespace
metadata:
  name: stj-mv-ns

`应用yaml文件`
root@k8s-master01:/stj-soft/yaml/namespaces# kubectl apply -f move-namespaces.yaml 
`验证yaml`
root@k8s-master01:/stj-soft/yaml/namespaces# kubectl get ns | grep stj-mv-ns
stj-mv-ns              Active   19s

1.2.2）编写pv的yaml文件并应用

root@k8s-master01:~# mkdir -p /stj-soft/yaml/pv-pvc/zookeeper/pv
root@k8s-master01:~# cd /stj-soft/yaml/pv-pvc/zookeeper/pv
root@k8s-master01:/stj-soft/yaml/pv-pvc/zookeeper/pv# vim zookeeper-data-pv.yml
apiVersion: v1
kind: PersistentVolume
metadata:
  name: zookeeper-datadir-pv-1
  namespace: stj-mv-ns  # namespace可以不加 因为pv是属于全局资源的
spec:
  capacity:
    storage: 2Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    server: 192.168.139.234
    path: /data/k8sdata/zookeeper/datadir-pv-1
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: zookeeper-datadir-pv-2
  namespace: stj-mv-ns  # namespace可以不加 因为pv是属于全局资源的
spec:
  capacity:
    storage: 2Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    server: 192.168.139.234
    path: /data/k8sdata/zookeeper/datadir-pv-2
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: zookeeper-datadir-pv-3
  namespace: stj-mv-ns  # namespace可以不加 因为pv是属于全局资源的
spec:
  capacity:
    storage: 2Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    server: 192.168.139.234
    path: /data/k8sdata/zookeeper/datadir-pv-3
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: zookeeper-datalog-pv-1
  namespace: stj-mv-ns  # namespace可以不加 因为pv是属于全局资源的
spec:
  capacity:
    storage: 2Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    server: 192.168.139.234
    path: /data/k8sdata/zookeeper/datalog-pv-1
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: zookeeper-datalog-pv-2
  namespace: stj-mv-ns  # namespace可以不加 因为pv是属于全局资源的
spec:
  capacity:
    storage: 2Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    server: 192.168.139.234
    path: /data/k8sdata/zookeeper/datalog-pv-2
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: zookeeper-datalog-pv-3
  namespace: stj-mv-ns  # namespace可以不加 因为pv是属于全局资源的
spec:
  capacity:
    storage: 2Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    server: 192.168.139.234
    path: /data/k8sdata/zookeeper/datalog-pv-3

`应用yaml文件`
root@k8s-master01:/stj-soft/yaml/pv-pvc/zookeeper/pv# kubectl apply -f zookeeper-data-pv.yml 
`验证yaml`
root@k8s-master01:/stj-soft/yaml/pv-pvc/zookeeper/pv# kubectl get pv -n stj-mv-ns
NAME                     CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS      CLAIM   STORAGECLASS   REASON   AGE
zookeeper-datadir-pv-1   2Gi        RWO            Retain           Available                                   23s
zookeeper-datadir-pv-2   2Gi        RWO            Retain           Available                                   23s
zookeeper-datadir-pv-3   2Gi        RWO            Retain           Available                                   23s
zookeeper-datalog-pv-1   2Gi        RWO            Retain           Available                                   23s
zookeeper-datalog-pv-2   2Gi        RWO            Retain           Available                                   23s
zookeeper-datalog-pv-3   2Gi        RWO            Retain           Available                                   23s

1.2.3）编写pvc的yaml文件并应用

root@k8s-master01:~# mkdir -p /stj-soft/yaml/pv-pvc/zookeeper/pvc
root@k8s-master01:~# cd /stj-soft/yaml/pv-pvc/zookeeper/pvc
root@k8s-master01:/stj-soft/yaml/pv-pvc/zookeeper/pvc# vim zookeeper-data-pvc.yml 
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: zookeeper-datadir-pvc-1
  namespace: stj-mv-ns
spec:
  accessModes:
    - ReadWriteOnce
  volumeName: zookeeper-datadir-pv-1
  resources:
    requests:
      storage: 2Gi

---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: zookeeper-datadir-pvc-2
  namespace: stj-mv-ns
spec:
  accessModes:
    - ReadWriteOnce
  volumeName: zookeeper-datadir-pv-2
  resources:
    requests:
      storage: 2Gi
 
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: zookeeper-datadir-pvc-3
  namespace: stj-mv-ns
spec:
  accessModes:
    - ReadWriteOnce
  volumeName: zookeeper-datadir-pv-3
  resources:
    requests:
      storage: 2Gi
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: zookeeper-datalog-pvc-1
  namespace: stj-mv-ns
spec:
  accessModes:
    - ReadWriteOnce
  volumeName: zookeeper-datalog-pv-1
  resources:
    requests:
      storage: 2Gi

---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: zookeeper-datalog-pvc-2
  namespace: stj-mv-ns
spec:
  accessModes:
    - ReadWriteOnce
  volumeName: zookeeper-datalog-pv-2
  resources:
    requests:
      storage: 2Gi
 
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: zookeeper-datalog-pvc-3
  namespace: stj-mv-ns
spec:
  accessModes:
    - ReadWriteOnce
  volumeName: zookeeper-datalog-pv-3
  resources:
    requests:
      storage: 2Gi

`应用yaml文件`
root@k8s-master01:/stj-soft/yaml/pv-pvc/zookeeper/pvc# kubectl apply -f zookeeper-data-pvc.yml 
`验证yaml`
root@k8s-master01:/stj-soft/yaml/pv-pvc/zookeeper/pvc# kubectl get pvc -n stj-mv-ns
NAME                      STATUS   VOLUME                   CAPACITY   ACCESS MODES   STORAGECLASS   AGE
zookeeper-datadir-pvc-1   Bound    zookeeper-datadir-pv-1   2Gi        RWO                           2m19s
zookeeper-datadir-pvc-2   Bound    zookeeper-datadir-pv-2   2Gi        RWO                           2m19s
zookeeper-datadir-pvc-3   Bound    zookeeper-datadir-pv-3   2Gi        RWO                           2m19s
zookeeper-datalog-pvc-1   Bound    zookeeper-datalog-pv-1   2Gi        RWO                           2m19s
zookeeper-datalog-pvc-2   Bound    zookeeper-datalog-pv-2   2Gi        RWO                           2m19s
zookeeper-datalog-pvc-3   Bound    zookeeper-datalog-pv-3   2Gi        RWO                           2m19s

1.3）准备zookeeper镜像

root@k8s-master01:~# docker pull zookeeper:3.6.4
root@k8s-master01:~# docker tag zookeeper:3.6.4 harbor.stangj.net/baseimages/zookeeper:3.6.4
root@k8s-master01:~# docker push harbor.stangj.net/baseimages/zookeeper:3.6.4

1.4）编写zookeeper集群的yaml文件并应用

1.4.1）编写zookeeper集群的yaml文件

root@k8s-master01:~# mkdir -p /stj-soft/yaml/pv-pvc/zookeeper/application
root@k8s-master01:~# cd /stj-soft/yaml/pv-pvc/zookeeper/application
root@k8s-master01:/stj-soft/yaml/pv-pvc/zookeeper/application# 
root@k8s-master01:/stj-soft/yaml/pv-pvc/zookeeper/application# cat zookeeper-cluster.yml
apiVersion: v1
kind: Service
metadata:
  name: zookeeper
  namespace: stj-mv-ns
spec:
  ports:
    - name: client
      port: 2181
  selector: 
    app: zookeeper
---

apiVersion: v1
kind: Service
metadata:
  name: zookeeper1
  namespace: stj-mv-ns
spec:
  type: NodePort
  ports:
    - name: client
      port: 2181
      nodePort: 42181
    - name: followers
      port: 2888
    - name: election
      port: 3888
  selector:
    app: zookeeper
    server-id: "1"
---
apiVersion: v1
kind: Service
metadata:
  name: zookeeper2
  namespace: stj-mv-ns
spec:
  type: NodePort
  ports:
    - name: client
      port: 2181
      nodePort: 42182
    - name: followers
      port: 2888
    - name: election
      port: 3888
  selector:
    app: zookeeper
    server-id: "2"
--- 
apiVersion: v1
kind: Service
metadata:
  name: zookeeper3
  namespace: stj-mv-ns
spec:
  type: NodePort
  ports:
    - name: client
      port: 2181
      nodePort: 42183
    - name: followers
      port: 2888
    - name: election
      port: 3888
  selector:
    app: zookeeper
    server-id: "3"
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: zookeeper1
  namespace: stj-mv-ns
spec:
  replicas: 1
  selector:
    matchLabels:
      app: zookeeper
  template:
    metadata:
      labels:
        app: zookeeper
        server-id: "1"
    spec:
      containers:
        - name: server
          image: harbor.stangj.net/baseimages/zookeeper:3.6.4
          imagePullPolicy: Always
          env:
            - name: ZOO_MY_ID
              value: "1"
            - name: SERVERS
              value: "zookeeper1"
            - name: ZOO_SERVERS
              value: server.1=0.0.0.0:2888:3888 server.2=zookeeper2:2888:3888 server.3=zookeeper3:2888:3888
          ports:
            - containerPort: 2181
            - containerPort: 2888
            - containerPort: 3888
          volumeMounts:
          - mountPath: "/data"
            name: zookeeper-datadir-pvc-1
          - mountPath: "/datalog"
            name: zookeeper-datalog-pvc-1
      volumes:
        - name: zookeeper-datadir-pvc-1
          persistentVolumeClaim:
            claimName: zookeeper-datadir-pvc-1
        - name: zookeeper-datalog-pvc-1
          persistentVolumeClaim:
            claimName: zookeeper-datalog-pvc-1
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: zookeeper2
  namespace: stj-mv-ns
spec:
  replicas: 1
  selector:
    matchLabels:
      app: zookeeper
  template:
    metadata:
      labels:
        app: zookeeper
        server-id: "2"
    spec:
      containers:
        - name: server
          image: harbor.stangj.net/baseimages/zookeeper:3.6.4
          imagePullPolicy: Always
          env:
            - name: ZOO_MY_ID
              value: "2"
            - name: SERVERS
              value: "zookeeper2"
            - name: ZOO_SERVERS
              value: server.1=zookeeper1:2888:3888 server.2=0.0.0.0:2888:3888 server.3=zookeeper3:2888:3888
          ports:
            - containerPort: 2181
            - containerPort: 2888
            - containerPort: 3888
          volumeMounts:
          - mountPath: "/data"
            name: zookeeper-datadir-pvc-2
          - mountPath: "/datalog"
            name: zookeeper-datalog-pvc-2
      volumes:
        - name: zookeeper-datadir-pvc-2
          persistentVolumeClaim:
            claimName: zookeeper-datadir-pvc-2
        - name: zookeeper-datalog-pvc-2
          persistentVolumeClaim:
            claimName: zookeeper-datalog-pvc-2
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: zookeeper3
  namespace: stj-mv-ns
spec:
  replicas: 1
  selector:
    matchLabels:
      app: zookeeper
  template:
    metadata:
      labels:
        app: zookeeper
        server-id: "3"
    spec:
      containers:
        - name: server
          image: harbor.stangj.net/baseimages/zookeeper:3.6.4
          imagePullPolicy: Always
          env:
            - name: ZOO_MY_ID
              value: "3"
            - name: SERVERS
              value: "zookeeper3"
            - name: ZOO_SERVERS
              value: server.1=zookeeper1:2888:3888 server.2=zookeeper2:2888:3888 server.3=0.0.0.0:2888:3888
          ports:
            - containerPort: 2181
            - containerPort: 2888
            - containerPort: 3888
          volumeMounts:
          - mountPath: "/data"
            name: zookeeper-datadir-pvc-3
          - mountPath: "/datalog"
            name: zookeeper-datalog-pvc-3
      volumes:
        - name: zookeeper-datadir-pvc-3
          persistentVolumeClaim:
            claimName: zookeeper-datadir-pvc-3
        - name: zookeeper-datalog-pvc-3
          persistentVolumeClaim:
            claimName: zookeeper-datalog-pvc-3

1.4.2）应用yaml文件

root@k8s-master01:/stj-soft/yaml/pv-pvc/zookeeper/application# kubectl apply -f zookeeper-cluster.yml

1.5）验证服务

1.5.1）验证pod

root@k8s-master01:/stj-soft/yaml/pv-pvc/zookeeper/application# kubectl get pod -n stj-mv-ns
NAME                          READY   STATUS    RESTARTS   AGE
zookeeper1-9cd577c78-sjbbk    1/1     Running   0          97s
zookeeper2-57465b45fb-jpwsl   1/1     Running   0          97s
zookeeper3-77676fbf5d-n4pgr   1/1     Running   0          97s

1.5.2）验证service

root@k8s-master01:/stj-soft/yaml/pv-pvc/zookeeper/application# kubectl get svc -n stj-mv-ns
NAME         TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)                                        AGE
zookeeper    ClusterIP   10.200.124.97    <none>        2181/TCP                                       2m39s
zookeeper1   NodePort    10.200.238.112   <none>        2181:42181/TCP,2888:40214/TCP,3888:36659/TCP   2m39s
zookeeper2   NodePort    10.200.5.87      <none>        2181:42182/TCP,2888:34991/TCP,3888:32692/TCP   2m39s
zookeeper3   NodePort    10.200.124.67    <none>        2181:42183/TCP,2888:39456/TCP,3888:35392/TCP   2m38s

1.5.3）验证集群状态

1.5.4）验证是否能进行切换主从

手动关闭zookeeper2

zookeeper1已经上位

zookeeper集群搭建成功

1.5.5）验证NodePort

1.5.6）验证NFS数据

root@harbor:~# tree /data/k8sdata/zookeeper
/data/k8sdata/zookeeper
├── datadir-pv-1
│   ├── myid
│   └── version-2
│       ├── acceptedEpoch
│       ├── currentEpoch
│       └── snapshot.300000000
├── datadir-pv-2
│   ├── myid
│   └── version-2
│       ├── acceptedEpoch
│       ├── currentEpoch
│       └── snapshot.300000000
├── datadir-pv-3
│   ├── myid
│   └── version-2
│       ├── acceptedEpoch
│       ├── currentEpoch
│       └── snapshot.300000000
├── datalog-pv-1
│   └── version-2
│       └── log.400000001
├── datalog-pv-2
│   └── version-2
│       └── log.400000001
└── datalog-pv-3
    └── version-2
        └── log.400000001