前置条件
集群机器规划
Ubuntu2404
192.168.139.211 ceph-211 (sdb)300g/(sdc)500g
192.168.139.212 ceph-212 (sdb)300g/(sdc)500g
192.168.139.213 ceph-213 (sdb)300g/(sdc)500g
软件安装
# 安装下列服务
Python 3
Systemd
Podman or Docker for running containers
Time synchronization (such as Chrony or the legacy ntpd)
LVM2 for provisioning storage devicesceph 版本
https://docs.ceph.com/en/latest/releases/#active-releases
安装使用ceph（19.2.3）
https://docs.ceph.com/en/latest/cephadm/install/#requirements
下载安装 ceph-admin
# 添加hosts文件解析 （使用机器执行）
cat >> /etc/hosts <<EOF
192.168.139.211 ceph-211
192.168.139.212 ceph-212
192.168.139.213 ceph-213
EOF
root@ceph-211:~# CEPH_RELEASE=19.2.3
root@ceph-211:~# curl --silent --remote-name --location https://download.ceph.com/rpm-${CEPH_RELEASE}/el9/noarch/cephadm
root@ceph-211:~# chmod +x cephadm
root@ceph-211:~# mv cephadm /usr/local/bin/
root@ceph-211:~# cephadm version
cephadm version 19.2.3 (c92aebb279828e9c3c1f5d24613efca272649e62) squid (stable)创建新集群
root@ceph-211:~# cephadm bootstrap --mon-ip 192.168.139.211 --cluster-network 192.168.139.0/24 --allow-fqdn-hostname
...
mgr epoch 9 is available
Generating a dashboard self-signed certificate...
Creating initial admin user...
Fetching dashboard port number...
Ceph Dashboard is now available at:
URL: https://ceph-211:8443/
User: admin
Password: h7lvflbfan
Enabling client.admin keyring and conf on hosts with "admin" label
Saving cluster configuration to /var/lib/ceph/e79570fc-a044-11f0-b3c5-000c293cec53/co
...密钥文件位置
root@ceph-211:~# ll /etc/ceph/
total 24
drwxr-xr-x 2 root root 4096 Oct 3 18:42 ./
drwxr-xr-x 115 root root 4096 Oct 3 18:42 ../
-rw------- 1 root root 151 Oct 3 18:40 ceph.client.admin.keyring
-rw-r--r-- 1 root root 181 Oct 3 18:40 ceph.conf
-rw-r--r-- 1 root root 595 Oct 3 18:38 ceph.pub
-rw-r--r-- 1 root root 92 Jun 24 10:03 rbdmap
访问ceph集群
root@ceph-211:~# cephadm add-repo --release squid # 失败不用管
root@ceph-211:~# cephadm install ceph-common
root@ceph-211:~# ceph -s
cluster:
id: e79570fc-a044-11f0-b3c5-000c293cec53
health: HEALTH_WARN
OSD count 0 < osd_pool_default_size 3
services:
mon: 1 daemons, quorum ceph-211 (age 5m)
mgr: ceph-211.volfyl(active, since 3m)
osd: 0 osds: 0 up, 0 in
data:
pools: 0 pools, 0 pgs
objects: 0 objects, 0 B
usage: 0 B used, 0 B / 0 B avail
pgs: 访问 dashboard
root@ceph-211:~# ceph mgr services
{
"dashboard": "https://192.168.139.211:8443/",
"prometheus": "http://192.168.139.211:9283/"
}
当密码忘记时，可以通过机器上的密码 无密修改（将密码修改为 root）
root@ceph-211:~# echo root | ceph dashboard set-login-credentials admin -i -
******************************************************************
*** WARNING: this command is deprecated. ***
*** Please use the ac-user-* related commands to manage users. ***
******************************************************************
Username and password updated
ceph集群添加或移除主机
添加主机
# 查看
root@ceph-211:~# ceph orch host ls
HOST ADDR LABELS STATUS
ceph-211 192.168.139.211 _admin
1 hosts in cluster
# 把秘钥放到其他服务器上
root@ceph-211:~# ssh-copy-id -f -i /etc/ceph/ceph.pub ceph-212
root@ceph-211:~# ssh-copy-id -f -i /etc/ceph/ceph.pub ceph-213
将秘钥节点加入到集群
root@ceph-211:~# ceph orch host add ceph-212 192.168.139.212
root@ceph-211:~# ceph orch host add ceph-213 192.168.139.213
root@ceph-211:~# ceph orch host ls
HOST ADDR LABELS STATUS
ceph-211 192.168.139.211 _admin
ceph-212 192.168.139.212
ceph-213 192.168.139.213 
移除主机
ceph orch host rm ceph-213 集群主机打 label
注意：
目前只有 ceph-211 节点能够管理集群，如果想要其他节点管理集群 请用下面目录达标：
root@ceph-211:~# ceph orch host label add ceph-212 _admin
这样 ceph-212 也能访问管理ceph集群
（需要在ceph-212也下载 cephadm 和 ceph ）
打 label 的本质是 将 ceph.conf ceph.client.admin.keyring 这两个文件拷贝到其他机器上
root@ceph-212:~# ll /etc/ceph/
total 20
drwxr-xr-x 2 root root 4096 Oct 3 19:18 ./
drwxr-xr-x 115 root root 4096 Oct 3 19:18 ../
-rw------- 1 root root 151 Oct 3 19:18 ceph.client.admin.keyring
-rw-r--r-- 1 root root 289 Oct 3 19:18 ceph.conf
-rw-r--r-- 1 root root 92 Jun 24 10:03 rbdmap
添加OSD设备到ceph集群
温馨提示:
如果一个设备想要加入ceph集群，要求满足2个条件:
- 1.设备未被使用;
- 2.设备的存储大小必须大于5GB;
root@ceph-211:~# ceph orch device ls
HOST PATH TYPE DEVICE ID SIZE AVAILABLE REFRESHED REJECT REASONS
ceph-211 /dev/sdb hdd 300G Yes 84s ago
ceph-211 /dev/sdc hdd 500G Yes 84s ago
ceph-211 /dev/sr0 hdd VMware_Virtual_IDE_CDROM_Drive_10000000000000000001 2627M No 84s ago Has a FileSystem, Insufficient space (<5GB)
ceph-212 /dev/sdb hdd 300G Yes 18m ago
ceph-212 /dev/sdc hdd 500G Yes 18m ago
ceph-212 /dev/sr0 hdd VMware_Virtual_IDE_CDROM_Drive_10000000000000000001 2627M No 18m ago Has a FileSystem, Insufficient space (<5GB)
ceph-213 /dev/sdb hdd 300G Yes 17m ago
ceph-213 /dev/sdc hdd 500G Yes 17m ago
ceph-213 /dev/sr0 hdd VMware_Virtual_IDE_CDROM_Drive_10000000000000000001 2627M No 17m ago Has a FileSystem, Insufficient space (<5GB)
# 添加 osd
root@ceph-211:~# ceph orch daemon add osd ceph-211:/dev/sdb
root@ceph-211:~# ceph orch daemon add osd ceph-211:/dev/sdc
root@ceph-211:~# ceph orch daemon add osd ceph-212:/dev/sdc
root@ceph-211:~# ceph orch daemon add osd ceph-212:/dev/sdb
root@ceph-211:~# ceph orch daemon add osd ceph-213:/dev/sdb
root@ceph-211:~# ceph orch daemon add osd ceph-213:/dev/sdc
注意：此步骤会在"/var/lib/ceph/<Ceph_Cluster_ID>/osd.<OSD_ID>/fsid"文件中记录对应ceph的OSD编号对应本地的磁盘设备标识
root@ceph-211:~# cat /var/lib/ceph/e79570fc-a044-11f0-b3c5-000c293cec53/osd.*/fsid
3e34e76d-511d-41f7-91be-bbcc7b66c3e0
dd770ef6-fc27-490b-9c99-65f007bbc789
root@ceph-211:~# lsblk
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINTS
sda 8:0 0 60G 0 disk
├─sda1 8:1 0 1M 0 part
├─sda2 8:2 0 2G 0 part /boot
└─sda3 8:3 0 58G 0 part
└─ubuntu--vg-ubuntu--lv 252:0 0 29G 0 lvm /
sdb 8:16 0 300G 0 disk
└─ceph--6aa2f027--518f--4487--af60--85df816836c0-osd--block--3e34e76d--511d--41f7--91be--bbcc7b66c3e0 252:1 0 300G 0 lvm
sdc 8:32 0 500G 0 disk
└─ceph--69228801--1d78--4e13--bcff--99ced27e17fe-osd--block--dd770ef6--fc27--490b--9c99--65f007bbc789 252:2 0 500G 0 lvm
sr0 11:0 1 2.6G 0 rom 查看集群的osd总容量大小
root@ceph-211:~# ceph osd tree
ID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF
-1 2.34389 root default
-3 0.78130 host ceph-211
0 hdd 0.29300 osd.0 up 1.00000 1.00000
1 hdd 0.48830 osd.1 up 1.00000 1.00000
-5 0.78130 host ceph-212
2 hdd 0.48830 osd.2 up 1.00000 1.00000
3 hdd 0.29300 osd.3 up 1.00000 1.00000
-7 0.78130 host ceph-213
4 hdd 0.29300 osd.4 up 1.00000 1.00000
5 hdd 0.48830 osd.5 up 1.00000 1.00000
root@ceph-211:~# ceph -s
cluster:
id: e79570fc-a044-11f0-b3c5-000c293cec53
health: HEALTH_OK
services:
mon: 3 daemons, quorum ceph-211,ceph-212,ceph-213 (age 26m)
mgr: ceph-211.volfyl(active, since 58m), standbys: ceph-212.alfizx
osd: 6 osds: 6 up (since 3m), 6 in (since 3m)
data:
pools: 1 pools, 1 pgs
objects: 2 objects, 449 KiB
usage: 161 MiB used, 2.3 TiB / 2.3 TiB avail
pgs: 1 active+clean
文章评论