ceph离线安装与日常简单维护
1 环境介绍主机名
系统
版本
角色
IP
ceph1
RHEL7.3_64bit
jewel
admin/mon1
10.10.64.130
ceph2
RHEL7.3_64bit
jewel
mon2
10.10.64.131
ceph3
RHEL7.3_64bit
jewel
mon3
10.10.64.132
ceph4
RHEL7.3_64bit
jewel
osd1
10.10.64.133
ceph5
RHEL7.3_64bit
jewel
osd2
10.10.64.134
ceph6
RHEL7.3_64bit
jewel
osd3
10.10.64.135
client_ceph
RHEL7.3_64bit
jewel
client_ceph
10.10.64.136
2 安装前准备
2.1 配置yum
#系统镜像源
# cat /etc/yum.repos.d/base.repo
name=Red Hat Enterprise Linux $releasever - $basearch - Source
baseurl=http://10.0.40.34/yum/rhel7.3_64bit
enabled=1
gpgcheck=0
#ceph镜像源
# cat /etc/yum.repos.d/ceph.repo
name=ceph
baseurl=http://10.10.64.120/mirrors.aliyun.com/ceph/rpm-jewel/el7/x86_64
enabled=1
gpgcheck=0
name=ceph_noarch
baseurl=http://10.10.64.120/mirrors.aliyun.com/ceph/rpm-jewel/el7/noarch/
enabled=1
gpgcheck=0
2.2 安装依赖包
应为是离线安装，好多依赖包在上面两个yum源中没有，我把一些关键的依赖包下载下来，手动安装
#flask依赖
# cd flask/
# rpm -ivh *
warning: python-babel-0.9.6-8.el7.noarch.rpm: Header V3 RSA/SHA256 Signature, key ID f4a80eb5: NOKEY
Preparing... #################################
Updating / installing...
1:python-werkzeug-0.9.1-2.el7 ################################# [ 17%]
2:python-markupsafe-0.11-10.el7 ################################# [ 33%]
3:python-itsdangerous-0.23-2.el7 ################################# [ 50%]
4:python-babel-0.9.6-8.el7 ################################# [ 67%]
5:python-jinja2-2.7.2-2.el7 ################################# [ 83%]
6:python-flask-1:0.10.1-4.el7 #################################
#selinux依赖
# cd selinux_policy/
# ls
selinux-policy-3.13.1-166.el7_4.9.noarch.rpm
selinux-policy-targeted-3.13.1-166.el7_4.9.noarch.rpm
# yum -y install selinux-policy-*
#其他依赖
# rpm -ivh userspace-rcu-0.7.16-1.el7.x86_64.rpm
# rpm -ivh lttng-ust-2.4.1-4.el7.x86_64.rpm
# rpm -ivh leveldb-1.12.0-11.el7.x86_64.rpm
# rpm -ivh libbabeltrace-1.2.4-3.el7.x86_64.rpm
# rpm -ivh fcgi-2.4.0-25.el7.x86_64.rpm
2.3 hosts解析
# cat /etc/hosts
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
10.10.64.130 ceph1
10.10.64.131 ceph2
10.10.64.132 ceph3
10.10.64.133 ceph4
10.10.64.134 ceph5
10.10.64.135 ceph6
10.10.64.136 client_ceph
2.4 更改hostname
# hostnamectl set-hostname ceph1
2.5 关闭iptables&selinux
# getenforce
Disabled
# systemctl status iptables
Unit iptables.service could not be found.
# systemctl status firewalld.service
firewalld.service - firewalld - dynamic firewall daemon
Loaded: loaded (/usr/lib/systemd/system/firewalld.service; disabled; vendor preset: enabled)
Active: inactive (dead)
Docs: man:firewalld(1)
2.6 配置NTP
# crontab -l
02 01 * * * /usr/sbin/ntpdate -u 10.0.54.54
2.7 关闭虚拟机克隆
略
2.8 更改节点主机名
根据环境介绍中的信息更改
2.9 admin节点安装ceph-deploy
# yum -y install ceph-deploy
2.10 创建公钥发布
# ssh-keygen
Generating public/private rsa key pair.
Enter file in which to save the key (/root/.ssh/id_rsa):
/root/.ssh/id_rsa already exists.
Overwrite (y/n)? y
Enter passphrase (empty for no passphrase):
Enter same passphrase again:
Your identification has been saved in /root/.ssh/id_rsa.
Your public key has been saved in /root/.ssh/id_rsa.pub.
The key fingerprint is:
fa:9b:a0:a9:35:c7:29:90:62:53:ed:b1:92:8a:fc:88 root@ceph1
The key's randomart image is:
+--[ RSA 2048]----+
| |
| . |
| . o |
|..o o |
|.ooo oS |
|+.o... o |
|.o+ * |
|. o. * o . |
|E..oo +. |
+-----------------+
# ssh-copy-id root@10.10.64.131
# ssh-copy-id root@10.10.64.132
# ssh-copy-id root@10.10.64.133
# ssh-copy-id root@10.10.64.134
# ssh-copy-id root@10.10.64.135
3 mon配置
3.1 配置新节点
# mkdir my-cluster && cd my-cluster/
# ceph-deploy new ceph{1..6}
Resolving host ceph6
Monitor ceph6 at 10.10.64.135
Monitor initial members are ['ceph1', 'ceph2', 'ceph3', 'ceph4', 'ceph5', 'ceph6']
Monitor addrs are ['10.10.64.130', '10.10.64.131', '10.10.64.132', '10.10.64.133', '10.10.64.134', '10.10.64.135']
Creating a random mon key...
Writing monitor keyring to ceph.mon.keyring...
Writing initial config to ceph.conf...
3.2 编辑ceph.conf文件
#增加网络信息
# vim ceph.conf
public network = 10.10.64.130/24
#保留如下信息，多余的删掉
mon_initial_members = ceph1, ceph2, ceph3
mon_host = 10.10.64.130,10.10.64.131,10.10.64.132
3.3 mon节点安装ceph
# ceph-deploy install ceph1 ceph2 ceph3
3.4 配置并启动mon
#初始化monitor并搜集所有秘钥
# ceph-deploy mon create-initial
keyring auth get-or-create client.bootstrap-mgr mon allow profile bootstrap-mgr
Running command: /usr/bin/ceph --connect-timeout=25 --cluster=ceph --name mon. --keyring=/var/lib/ceph/mon/ceph-ceph1/keyring auth get client.bootstrap-osd
Running command: /usr/bin/ceph --connect-timeout=25 --cluster=ceph --name mon. --keyring=/var/lib/ceph/mon/ceph-ceph1/keyring auth get client.bootstrap-rgw
Storing ceph.client.admin.keyring
Storing ceph.bootstrap-mds.keyring
Storing ceph.bootstrap-mgr.keyring
keyring 'ceph.mon.keyring' already exists
Storing ceph.bootstrap-osd.keyring
Storing ceph.bootstrap-rgw.keyring
Destroy temp directory /tmp/tmprtj_ZH
#查看是否成功
# ceph -s
cluster 59d344b8-5051-43e4-90c6-c29c2a91961f
health HEALTH_ERR
clock skew detected on mon.ceph2, mon.ceph3
64 pgs are stuck inactive for more than 300 seconds
64 pgs stuck inactive
64 pgs stuck unclean
no osds
Monitor clock skew detected
monmap e1: 3 mons at {ceph1=10.10.64.130:6789/0,ceph2=10.10.64.131:6789/0,ceph3=10.10.64.132:6789/0}
election epoch 6, quorum 0,1,2 ceph1,ceph2,ceph3
osdmap e1: 0 osds: 0 up, 0 in
flags sortbitwise,require_jewel_osds
pgmap v2: 64 pgs, 1 pools, 0 bytes data, 0 objects
0 kB used, 0 kB / 0 kB avail
64 creating
4 osd配置
4.1 添加osd节点
#安装ceph软件
ceph-deploy install ceph4 ceph5 ceph6
4.2 osd节点添加磁盘
在虚拟机中添加磁盘，添加磁盘规范如下
hostname
IP
osd数据盘
journal盘
备注
node3
10.10.64.133
/dev/sdb/c/d
/dev/sde
node4
10.10.64.134
/dev/sdb/c/d
/dev/sde
node5
10.10.64.135
/dev/sdb/c/d
/dev/sde
#3个node节点都是这种配置
# lsblk
NAME MAJ:MIN RMSIZE RO TYPE MOUNTPOINT
fd0 2:0 1 4K0 disk
sda 8:0 0 60G0 disk
├─sda1 8:1 0 1G0 part /boot
└─sda2 8:2 0 59G0 part
├─rhel-root 253:0 0 35.6G0 lvm/
├─rhel-swap 253:1 0 6G0 lvm
└─rhel-home 253:2 0 17.4G0 lvm/home
sdb 8:16 0100G0 disk
sdc 8:32 0100G0 disk
sdd 8:48 0100G0 disk
sde 8:64 0 50G0 disk
sr0 11:0 1 66.5M0 rom
4.2.1 磁盘分区
PS：
在每台osd服务器上我们需要对3块SAS硬盘分区、创建 xfs 文件系统
对1块用做journal的硬盘分3个区，每个区对应一块硬盘，不需要创建文件系统，留给 Ceph 自己处理。
编写脚本如下:
# cat parted.sh
#!/bin/bash
set -e
if [ ! -x "/sbin/parted" ]; then
echo "This script requires /sbin/parted to run!" >&2
exit 1
fi
DISKS="b c d"
for i in ${DISKS}; do
echo "Creating partitions on /dev/sd${i} ..."
parted -a optimal --script /dev/sd${i} -- mktable gpt
parted -a optimal --script /dev/sd${i} -- mkpart primary xfs 0% 100%
sleep 1
#echo "Formatting /dev/sd${i}1 ..."
mkfs.xfs -f /dev/sd${i}1 &
done
SSDS="e"
for i in ${SSDS}; do
parted -s /dev/sd${i} mklabel gpt
parted -s /dev/sd${i} mkpart primary 0% 32%
parted -s /dev/sd${i} mkpart primary 33% 65%
parted -s /dev/sd${i} mkpart primary 66% 98%
done
#分发到各个osd节点上
# scp parted.sh root@10.10.64.134:/root/
# scp parted.sh root@10.10.64.135:/root/
4.2.2 执行后结果
每个osd节点都是一样的，这里摘取ceph4展示
# lsblk
NAME MAJ:MIN RMSIZE RO TYPE MOUNTPOINT
fd0 2:0 1 4K0 disk
sda 8:0 0 60G0 disk
├─sda1 8:1 0 1G0 part /boot
└─sda2 8:2 0 59G0 part
├─rhel-root 253:0 0 35.6G0 lvm/
├─rhel-swap 253:1 0 6G0 lvm
└─rhel-home 253:2 0 17.4G0 lvm/home
sdb 8:16 0100G0 disk
└─sdb1 8:17 0100G0 part
sdc 8:32 0100G0 disk
└─sdc1 8:33 0100G0 part
sdd 8:48 0100G0 disk
└─sdd1 8:49 0100G0 part
sde 8:64 0 50G0 disk
├─sde1 8:65 0 16G0 part
├─sde2 8:66 0 16G0 part
└─sde3 8:67 0 16G0 part
sr0 11:0 1 66.5M0 rom
#通过admin节点查看
# ceph-deploy disk list ceph4
found configuration file at: /root/.cephdeploy.conf
Invoked (1.5.39): /usr/bin/ceph-deploy disk list ceph4
ceph-deploy options:
username : None
verbose : False
overwrite_conf : False
subcommand : list
quiet : False
cd_conf :
cluster : ceph
func :
ceph_conf : None
default_release : False
disk : [('ceph4', None, None)]
connected to host: ceph4
detect platform information from remote host
detect machine type
find the location of an executable
Distro info: Red Hat Enterprise Linux Server 7.3 Maipo
Listing disks on ceph4...
find the location of an executable
Running command: /usr/sbin/ceph-disk list
/dev/dm-0 other, xfs, mounted on /
/dev/dm-1 swap, swap
/dev/dm-2 other, xfs, mounted on /home
/dev/sda :
/dev/sda2 other, LVM2_member
/dev/sda1 other, xfs, mounted on /boot
/dev/sdb :
/dev/sdb1 other, xfs
/dev/sdc :
/dev/sdc1 other, xfs
/dev/sdd :
/dev/sdd1 other, xfs
/dev/sde :
/dev/sde1 other, ebd0a0a2-b9e5-4433-87c0-68b6b72699c7
/dev/sde2 other, ebd0a0a2-b9e5-4433-87c0-68b6b72699c7
/dev/sde3 other, ebd0a0a2-b9e5-4433-87c0-68b6b72699c7
/dev/sr0 other, iso9660
4.3 配置osd磁盘并启动
初始化Ceph硬盘，然后创建osd存储节点：规则存储节点:单个硬盘:对应的journal分区，要对应
#配置osd数据库和journal关系
# ceph-deploy osd prepare ceph4:/dev/sdb1:/dev/sde1 ceph4:/dev/sdc1:/dev/sde2 ceph4:/dev/sdd1:/dev/sde3
realtime =none extsz=4096 blocks=0, rtextents=0
command: Running command: /usr/sbin/restorecon /var/lib/ceph/tmp/mnt.7im5ip
populate_data_path: Preparing osd data dir /var/lib/ceph/tmp/mnt.7im5ip
command: Running command: /usr/sbin/restorecon -R /var/lib/ceph/tmp/mnt.7im5ip/ceph_fsid.15797.tmp
command: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/tmp/mnt.7im5ip/ceph_fsid.15797.tmp
command: Running command: /usr/sbin/restorecon -R /var/lib/ceph/tmp/mnt.7im5ip/fsid.15797.tmp
command: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/tmp/mnt.7im5ip/fsid.15797.tmp
command: Running command: /usr/sbin/restorecon -R /var/lib/ceph/tmp/mnt.7im5ip/magic.15797.tmp
command: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/tmp/mnt.7im5ip/magic.15797.tmp
command: Running command: /usr/sbin/restorecon -R /var/lib/ceph/tmp/mnt.7im5ip/journal_uuid.15797.tmp
command: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/tmp/mnt.7im5ip/journal_uuid.15797.tmp
adjust_symlink: Creating symlink /var/lib/ceph/tmp/mnt.7im5ip/journal -> /dev/sde3
command: Running command: /usr/sbin/restorecon -R /var/lib/ceph/tmp/mnt.7im5ip
command: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/tmp/mnt.7im5ip
unmount: Unmounting /var/lib/ceph/tmp/mnt.7im5ip
command_check_call: Running command: /bin/umount -- /var/lib/ceph/tmp/mnt.7im5ip
get_dm_uuid: get_dm_uuid /dev/sdd1 uuid path is /sys/dev/block/8:49/dm/uuid
checking OSD status...
find the location of an executable
Running command: /bin/ceph --cluster=ceph osd stat --format=json
# ceph-deploy osd prepare ceph5:/dev/sdb1:/dev/sde1 ceph5:/dev/sdc1:/dev/sde2 ceph5:/dev/sdd1:/dev/sde3
# ceph-deploy osd prepare ceph6:/dev/sdb1:/dev/sde1 ceph6:/dev/sdc1:/dev/sde2 ceph6:/dev/sdd1:/dev/sde3
#设置磁盘权限
# ssh ceph4 'chown ceph:ceph /dev/sdb1 /dev/sdc1 /dev/sdd1 /dev/sde1 /dev/sde2 /dev/sde3'
# ssh ceph5 'chown ceph:ceph /dev/sdb1 /dev/sdc1 /dev/sdd1 /dev/sde1 /dev/sde2 /dev/sde3'
# ssh ceph6 'chown ceph:ceph /dev/sdb1 /dev/sdc1 /dev/sdd1 /dev/sde1 /dev/sde2 /dev/sde3'
#启动osd
# ceph-deploy osd activate ceph4:/dev/sdb1:/dev/sde1 ceph4:/dev/sdc1:/dev/sde2 ceph4:/dev/sdd1:/dev/sde3
command: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/tmp/mnt.TJBcf5/systemd
activate: Authorizing OSD key...
command_check_call: Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring auth add osd.2 -i /var/lib/ceph/tmp/mnt.TJBcf5/keyring osd allow * mon allow profile osd
added key for osd.2
command: Running command: /usr/sbin/restorecon -R /var/lib/ceph/tmp/mnt.TJBcf5/active.16792.tmp
command: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/tmp/mnt.TJBcf5/active.16792.tmp
activate: ceph osd.2 data dir is ready at /var/lib/ceph/tmp/mnt.TJBcf5
move_mount: Moving mount to final location...
command_check_call: Running command: /bin/mount -o noatime,inode64 -- /dev/sdd1 /var/lib/ceph/osd/ceph-2
command_check_call: Running command: /bin/umount -l -- /var/lib/ceph/tmp/mnt.TJBcf5
start_daemon: Starting ceph osd.2...
command_check_call: Running command: /usr/bin/systemctl disable ceph-osd@2
command_check_call: Running command: /usr/bin/systemctl disable ceph-osd@2 --runtime
command_check_call: Running command: /usr/bin/systemctl enable ceph-osd@2
Created symlink from /etc/systemd/system/ceph-osd.target.wants/ceph-osd@2.service to /usr/lib/systemd/system/ceph-osd@.service.
command_check_call: Running command: /usr/bin/systemctl start ceph-osd@2
checking OSD status...
find the location of an executable
Running command: /bin/ceph --cluster=ceph osd stat --format=json
Running command: systemctl enable ceph.target
# ceph-deploy osd activate ceph5:/dev/sdb1:/dev/sde1 ceph5:/dev/sdc1:/dev/sde2 ceph5:/dev/sdd1:/dev/sde3
# ceph-deploy osd activate ceph6:/dev/sdb1:/dev/sde1 ceph6:/dev/sdc1:/dev/sde2 ceph6:/dev/sdd1:/dev/sde3
4.4 验证osd安装结果
#admin上查看
#其实还有时钟的问题懒得弄了，手动ntpdata一下就解决了，应为我的ntpdata是用crontab，一天执行一次。
# ceph -s
cluster 59d344b8-5051-43e4-90c6-c29c2a91961f
health HEALTH_WARN
clock skew detected on mon.ceph2, mon.ceph3
too few PGs per OSD (21 < min 30)
Monitor clock skew detected
monmap e1: 3 mons at {ceph1=10.10.64.130:6789/0,ceph2=10.10.64.131:6789/0,ceph3=10.10.64.132:6789/0}
election epoch 10, quorum 0,1,2 ceph1,ceph2,ceph3
osdmap e53: 9 osds: 9 up, 9 in
flags sortbitwise,require_jewel_osds
pgmap v176: 64 pgs, 1 pools, 0 bytes data, 0 objects
307 MB used, 899 GB / 899 GB avail
64 active+clean
#ceph4节点上查看（ceph4/5/6一样）
# ps -ef | grep osd
ceph 16287 10 10:56 ? 00:00:00 /usr/bin/ceph-osd -f --cluster ceph --id 0 --setuser ceph --setgroup ceph
ceph 16655 10 10:57 ? 00:00:00 /usr/bin/ceph-osd -f --cluster ceph --id 1 --setuser ceph --setgroup ceph
ceph 17054 10 10:57 ? 00:00:00 /usr/bin/ceph-osd -f --cluster ceph --id 2 --setuser ceph --setgroup ceph
root 1726724880 11:02 pts/0 00:00:00 grep --color=auto osd
5 设置使集群处于健康状态
注意上面的ceph 处于 HEALTH_WARN 状态
#查看副本数
# ceph osd dump | grep 'replicated size'
pool 0 'rbd' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 64 pgp_num 64 last_change 1 flags hashpspool stripe_width 0
#查看已存在的pools
# ceph osd lspools
0 rbd,
#查看rbd pool中的 pg_num 和 pgp_num 属性
# ceph osd pool get rbd pg_num
pg_num: 64
# ceph osd pool get rbd pgp_num
pgp_num: 64
#健康的 pg_num 和 pgp_num 计算方法：
#关于pgmap的数目，osd_num *100 / replica_num，向上取2的幂。比如15个osd，三备份，15 *100/3=500，得到pg_num = 512，线上重新设定这个数值时会引起数据迁移，请谨慎处理。
增加 PG 数目，根据 Total PGs = (#OSDs * 100) / pool size 公式来决定 pg_num（pgp_num 应该设成和 pg_num 一样），所以 9*100/3=300，Ceph 官方推荐取最接近2的指数倍，注意线上重新设定这个数值时会引起数据迁移，请谨慎处理。
#设置pg_num和pgp_num
# ceph osd pool set rbd pg_num 300
set pool 0 pg_num to 300
# ceph osd pool set rbd pgp_num 300
set pool 0 pgp_num to 300
#再次查看ceph集群状态
6 client端验证
6.1 client_ceph安装ceph软件
在ceph1上使用ceph-deploy 工具将ceph二进制程序安装到client_ceph上面
#给client_ceph安装ceph软件，这里不适用公钥，使用密码来进行安装
# ceph-deploy install client_ceph
The authenticity of host 'client_ceph (10.10.64.136)' can't be established.
ECDSA key fingerprint is e6:d0:0a:8e:7c:ed:26:2f:2b:3b:00:65:7e:c9:a3:7d.
Are you sure you want to continue connecting (yes/no)? yes #输入yes
Warning: Permanently added 'client_ceph,10.10.64.136' (ECDSA) to the list of known hosts.
root@client_ceph's password: #输入密码
root@client_ceph's password: #验证密码
#将ceph配置文件（ceph.conf）复制到client_ceph
# ceph-deploy config push client_ceph
found configuration file at: /root/.cephdeploy.conf
Invoked (1.5.39): /usr/bin/ceph-deploy config push client_ceph
ceph-deploy options:
username : None
verbose : False
overwrite_conf : False
subcommand : push
quiet : False
cd_conf :
cluster : ceph
client : ['client_ceph']
func :
ceph_conf : None
default_release : False
Pushing config to client_ceph
root@client_ceph's password:
root@client_ceph's password:
connected to host: client_ceph
detect platform information from remote host
detect machine type
write cluster configuration to /etc/ceph/{cluster}.conf
6.2 创建给client专用用户
客户机需要ceph密钥去访问ceph集群。ceph创建了一个默认用户 client.admin，它有足够的权限去访问ceph集群。不建议把client.admin共享到所有其他客户端节点。更好的做法是用分开的密钥创建一个新的ceph用户去访问特定的存储池。
#这里，创建了一个ceph用户 client.rbd，它拥有访问rbd存储池的权限
# ceph auth get-or-create client.rbd mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=rbd'
key = AQBYTeBaNUttDxAALwi67OOuNtaG4OUs+YcaPw==
#为ceph-cli上的client.rbd用户添加密钥
# ceph auth get-or-create client.rbd | ssh client_ceph 'tee /etc/ceph/ceph.client.rbd.keyring'
root@client_ceph's password:
key = AQBYTeBaNUttDxAALwi67OOuNtaG4OUs+YcaPw==
#至此，client_ceph 应该准备好充当ceph客户端了。通过提供用户名和密钥在client_ceph上检查集群的状态
# ssh client_ceph 'cat /etc/ceph/ceph.client.rbd.keyring >> /etc/ceph/keyring'
root@client_ceph's password:
6.3 客户机测试
# ceph -s --name client.rbd
cluster 59d344b8-5051-43e4-90c6-c29c2a91961f
health HEALTH_WARN
clock skew detected on mon.ceph2, mon.ceph3
Monitor clock skew detected
monmap e1: 3 mons at {ceph1=10.10.64.130:6789/0,ceph2=10.10.64.131:6789/0,ceph3=10.10.64.132:6789/0}
election epoch 10, quorum 0,1,2 ceph1,ceph2,ceph3
osdmap e57: 9 osds: 9 up, 9 in
flags sortbitwise,require_jewel_osds
pgmap v339: 300 pgs, 1 pools, 0 bytes data, 0 objects
322 MB used, 899 GB / 899 GB avail
300 active+clean
6.4 创建ceph块设备
#创建一个102400M 大小的RADOS 块设备，取名 rbd1
# rbd create rbd1 --size 102400 --name client.rbd --image-feature layering
#列出RBD 镜像
# rbd ls --name client.rbd
rbd1
#检查rbd 镜像的细节
# rbd --image rbd1 info --name client.rbd
rbd image 'rbd1':
size 102400 MB in 25600 objects
order 22 (4096 kB objects)
block_name_prefix: rbd_data.10ef238e1f29
format: 2
features: layering
flags:
#映射块设备并初始化
# rbd map --image rbd1 --name client.rbd
/dev/rbd0
#检查被映射的块设备
# rbd showmapped --name client.rbd
id pool image snap device
0rbdrbd1- /dev/rbd0
#要使用这个块设备，我们需要创建并挂载一个文件系统
# fdisk -l /dev/rbd0
Disk /dev/rbd0: 107.4 GB, 107374182400 bytes, 209715200 sectors
Units = sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 4194304 bytes / 4194304 bytes
#格式化文件系统
# mkfs.xfs /dev/rbd0
meta-data=/dev/rbd0 isize=512 agcount=17, agsize=1637376 blks
= sectsz=512 attr=2, projid32bit=1
= crc=1 finobt=0, sparse=0
data = bsize=4096 blocks=26214400, imaxpct=25
= sunit=1024 swidth=1024 blks
naming =version 2 bsize=4096 ascii-ci=0 ftype=1
log =internal log bsize=4096 blocks=12800, version=2
= sectsz=512 sunit=8 blks, lazy-count=1
realtime =none extsz=4096 blocks=0, rtextents=0
#挂载磁盘
# mkdir /mnt/ceph-disk1 && mount /dev/rbd0 /mnt/ceph-disk1
# df -h /mnt/ceph-disk1
Filesystem SizeUsed Avail Use% Mounted on
/dev/rbd0 100G 33M100G 1% /mnt/ceph-disk1
#测试块设备
# dd if=/dev/zero of=/mnt/ceph-disk1/file1 count=100 bs=1M
100+0 records in
100+0 records out
104857600 bytes (105 MB) copied, 0.12016 s, 873 MB/s
7 FAQ
7.1 /var/lib/ceph/tmp/mnt.: (13) Permission denied
此问题是因为磁盘权限不足导致，解决方法如下：
#在osd节点上更改有权限问题的磁盘或者是分区
chown ceph:ceph /dev/sdd1
问题延伸：
此问题本次修复后，系统重启磁盘权限会被修改回，导致osd服务无法正常启动，这个权限问题很坑，写了个for 循环，加入到rc.local,每次系统启动自动修改磁盘权限；
#跟进自己实际的磁盘场景来更改，此处只是提供范例
for i in a b c d e f g h i l j k;do chown ceph.ceph /dev/sd"$i"*;done
查找ceph资料，发现这其实是一个bug，社区暂未解决。
参考信息：
http://tracker.ceph.com/issues/13833
7.2 ceph_disk.main.FilesystemTypeError: Cannot discover filesystem type: device /dev/sdb: Line is truncated:
使用ceph-deploy osd activate ceph4:/dev/sdb:/dev/sde1 ceph4:/dev/sdc:/dev/sde2 ceph4:/dev/sdd:/dev/sde3
原因：由于ceph对磁盘进行了分区，/dev/sdb磁盘分区为/dev/sdb1
#解决方法:
ceph-deploy osd activate ceph4:/dev/sdb1:/dev/sde1 ceph4:/dev/sdc1:/dev/sde2 ceph4:/dev/sdd1:/dev/sde3
7.3 ceph.conf已存在
如果之前ceph.conf已存在，你更新过配置文件，需要添加—overwrite-conf config push
把改过的配置文件分发给集群内各主机
ceph-deploy --overwrite-conf config push node{1..3}
如果在初始化osd配置时出现ceph.conf需要使用如下命令
ceph-deploy --overwrite-conf mon create-initial
7.4 mon加入新节点
Ceph Monitor运行为一个轻量级的进程，一般情况下只会消耗很少的系统资源。大多数场景下可以选择入门级的CPU、千兆网卡即可，但是需要有足够大的磁盘空间来保存集群日志。 Ceph Monitor集群通过Paxos分布式协调算法来进行leader的选举。因为是通过法定人数选举的，所以当Ceph集群中有多个Monitor时，Monitor的数量应该是一个奇数，一般线上环境应该至少有3个。
前面在试验环境中初始化的Ceph集群只有一个MON。
192.168.61.30 c0 - admin-node, deploy-node
192.168.61.31 c1 - mon1
192.168.61.32 c2 - osd.1
192.168.61.33 c3 - osd.2
接下来我们添加两个MON，即把c2，c3也作为MON节点。
修改ceph.conf，在global章节加入：
mon_initial_members = c1,c2,c3
mon_host = 192.168.61.31,192.168.61.32,192.168.61.33
public network = 192.168.61.0/24
在admin-node上将配置文件推送到各个节点：
ceph-deploy --overwrite-conf config push c1 c2 c3
添加MON节点：
ceph-deploy mon create c2 c3
ceph -s
cluster 4873f3c0-2a5b-4868-b30e-f0c6f93b800a
health HEALTH_OK
monmap e11: 3 mons at {c1=192.168.61.31:6789/0,c2=192.168.61.32:6789/0,c3=192.168.61.33:6789/0}
election epoch 26, quorum 0,1,2 c1,c2,c3
osdmap e66: 2 osds: 2 up, 2 in
flags sortbitwise,require_jewel_osds
pgmap v2924: 192 pgs, 2 pools, 76900 kB data, 35 objects
12844 MB used, 182 GB / 195 GB avail
192 active+clean
移除MON节点：
ceph-deploy mon destroy c2 c3
8 命令手册
sudo stop ceph-all 停止所有ceph进程
ceph-deploy uninstall[{ceph-node}] 卸载所有ceph程序
ceph-deploy purge [ 删除ceph相关的包
ceph-deploy purgedata {ceph-node} [{ceph-node}] 删除ceph相关配置文件
ceph-deploy forgetkeys 删除key
#查看pool数量
# ceph osd lspools
1 pool-877be9dced4840e68b3629767389a35c,
#查看pool pg数量
# ceph osd pool get pool-877be9dced4840e68b3629767389a35c pg_num
pg_num: 9000
#查看pool副本数
# ceph osd dump | grep pool
pool 1 'pool-877be9dced4840e68b3629767389a35c' replicated size 3 min_size 1 crush_ruleset 1 object_hash rjenkins pg_num 9000 pgp_num 9000 last_change 296 flags hashpspool stripe_width 0 osd_full_ratio 0.9
#修改pool副本数
ceph osd pool set POOLNAME size NUM
#查看现有集群布局
# ceph osd tree
ID WEIGHT TYPE NAME UP/DOWN REWEIGHT PRIMARY-AFFINITY
-2 218.32196 root pool-877be9dced4840e68b3629767389a35c-root
-336.38699 host pool-877be9dced4840e68b3629767389a35c-b416f44c4eb848a7aabe755fc0dd7e0f
2 3.63899 osd.2 up1.00000 1.00000
4 3.63899 osd.4 up1.00000 1.00000
6 3.63899 osd.6 up1.00000 1.00000
1 3.63899 osd.1 up1.00000 1.00000
3 3.63899 osd.3 up1.00000 1.00000
5 3.63899 osd.5 up1.00000 1.00000
8 3.63899 osd.8 up1.00000 1.00000
7 3.63899 osd.7 up1.00000 1.00000
9 3.63899 osd.9 up1.00000 1.00000
0 3.63899 osd.0 up1.00000 1.00000
-436.38699 host pool-877be9dced4840e68b3629767389a35c-16b71b05afcf489e87b29a23aa64ff51
32 3.63899 osd.32 up1.00000 1.00000
34 3.63899 osd.34 up1.00000 1.00000
30 3.63899 osd.30 up1.00000 1.00000
31 3.63899 osd.31 up1.00000 1.00000
33 3.63899 osd.33 up1.00000 1.00000
39 3.63899 osd.39 up1.00000 1.00000
37 3.63899 osd.37 up1.00000 1.00000
36 3.63899 osd.36 up1.00000 1.00000
38 3.63899 osd.38 up1.00000 1.00000
35 3.63899 osd.35 up1.00000 1.00000
-536.38699 host pool-877be9dced4840e68b3629767389a35c-7fd856b99b944cd8a53757fff8917940
21 3.63899 osd.21 up1.00000 1.00000
25 3.63899 osd.25 up1.00000 1.00000
23 3.63899 osd.23 up1.00000 1.00000
24 3.63899 osd.24 up1.00000 1.00000
22 3.63899 osd.22 up1.00000 1.00000
26 3.63899 osd.26 up1.00000 1.00000
27 3.63899 osd.27 up1.00000 1.00000
20 3.63899 osd.20 up1.00000 1.00000
29 3.63899 osd.29 up1.00000 1.00000
28 3.63899 osd.28 up1.00000 1.00000
-636.38699 host pool-877be9dced4840e68b3629767389a35c-502b0fed30a9417a92593b6acfd5c80f
54 3.63899 osd.54 up1.00000 1.00000
50 3.63899 osd.50 up1.00000 1.00000
55 3.63899 osd.55 up1.00000 1.00000
59 3.63899 osd.59 up1.00000 1.00000
52 3.63899 osd.52 up1.00000 1.00000
57 3.63899 osd.57 up1.00000 1.00000
53 3.63899 osd.53 up1.00000 1.00000
56 3.63899 osd.56 up1.00000 1.00000
58 3.63899 osd.58 up1.00000 1.00000
51 3.63899 osd.51 up1.00000 1.00000
-736.38699 host pool-877be9dced4840e68b3629767389a35c-9303e065dff34847b9d427b58b3dc419
15 3.63899 osd.15 up1.00000 1.00000
17 3.63899 osd.17 up1.00000 1.00000
10 3.63899 osd.10 up1.00000 1.00000
11 3.63899 osd.11 up1.00000 1.00000
12 3.63899 osd.12 up1.00000 1.00000
14 3.63899 osd.14 up1.00000 1.00000
18 3.63899 osd.18 up1.00000 1.00000
13 3.63899 osd.13 up1.00000 1.00000
16 3.63899 osd.16 up1.00000 1.00000
19 3.63899 osd.19 up1.00000 1.00000
-836.38699 host pool-877be9dced4840e68b3629767389a35c-201902df4cd34666914a75ebbdfa3f07
48 3.63899 osd.48 up1.00000 1.00000
43 3.63899 osd.43 up1.00000 1.00000
44 3.63899 osd.44 up1.00000 1.00000
46 3.63899 osd.46 up1.00000 1.00000
42 3.63899 osd.42 up1.00000 1.00000
47 3.63899 osd.47 up1.00000 1.00000
49 3.63899 osd.49 up1.00000 1.00000
40 3.63899 osd.40 up1.00000 1.00000
45 3.63899 osd.45 up1.00000 1.00000
41 3.63899 osd.41 up1.00000 1.00000
-1 0 root default
#查看mon map
# ceph mon dump
dumped monmap epoch 5
epoch 5
fsid 0dbf4cdc-21cc-46ae-aa98-dc562bcf81e4
last_changed 2017-06-21 11:27:52.969705
created 2017-06-20 17:56:39.789318
0: 10.1.50.1:6789/0 mon.ceph01
1: 10.1.50.2:6789/0 mon.ceph02
2: 10.1.50.3:6789/0 mon.ceph03
#查看osd map
# ceph osd dump|more
epoch 732
fsid 0dbf4cdc-21cc-46ae-aa98-dc562bcf81e4
created 2017-06-20 17:56:40.149872
modified 2018-04-14 22:44:37.838302
flags noscrub,nodeep-scrub,sortbitwise,require_jewel_osds
pool 1 'pool-877be9dced4840e68b3629767389a35c' replicated size 3 min_size 1 crush_ruleset 1 object_hash rjenkins pg_num 9000 pgp_num 9000 last_change 296 flags hashpspool stripe_wi
dth 0 osd_full_ratio 0.9
removed_snaps
max_osd 60
#查看pg map
# ceph pg dump | more
dumped all in format plain
version 27409250
stamp 2018-05-11 14:15:06.522510
last_osdmap_epoch 732
last_pg_scan 732
full_ratio 0.97
nearfull_ratio 0.85
pg_stat objects mip degr misp unf bytes log disklog state state_stamp v reported up up_primary actingacting_primarylast_scrub scrub_stamp last_deep_scrub deep_scrub_stamp
1.fff 1050 0 0 0 0 4381890560 3011 3011 active+clean 2017-12-17 15:45:14.606042 732'2070811 732:6409846 1 1 323'28937 2017-08-24 14:58:39.470020 0'0 2017-06-20 18:40:53.526324
1.ffe 1085 0 0 0 0 4539440128 3045 3045 active+clean 2018-03-15 08:04:06.174409 732'2659595 732:4519618 45 45 325'87639 2017-08-25 11:47:08.619325 0'0 2017-06-20 18:40:53.526306
1.ffd 1067 0 0 0 0 4459056128 3008 3008 active+clean 2018-01-15 19:50:07.260339 732'936258 732:2563398 16 16 325'42078 2017-08-24 22:27:26.093606 0'0 2017-06-20 18:40:53.526299
1.ffc 1092 0 0 0 0 4565081088 3036 3036 active+clean 2017-12-11 16:43:52.140190 732'2065836 732:3347814 48 48 325'167904 2017-08-25 05:56:08.974475 0'0 2017-06-20 18:40:53.526289
1.ffb 1030 0 0 0 0 4285212672 3034 3034 active+clean 2017-09-17 06:47:04.756990 732'2010034 732:3453033 37 37 323'31693 2017-08-24 18:04:10.904974 0'0 2017-06-20 18:40:53.526281
#查看osd 使用情况
# ceph osd df |more
#要观察集群内正发生的事件
ceph –w
#检查监视器（monitor）的法定人数（mon选举状态）
# ceph quorum_status
{"election_epoch":34,"quorum":,"quorum_names":["ceph01","ceph02","ceph03"],"quorum_leader_name":"ceph01","monmap":{"epoch":5,"fsid":"0dbf4cdc-21cc-46ae-aa98-dc562bcf81e4","modified":"2017-06-21 11:27:52.969705","created":"2017-06-20 17:56:39.789318","mons":[{"rank":0,"name":"ceph01","addr":"10.1.50.1:6789\/0"},{"rank":1,"name":"ceph02","addr":"10.1.50.2:6789\/0"},{"rank":2,"name":"ceph03","addr":"10.1.50.3:6789\/0"}]}}
#查看集群详细信息
# ceph daemon mon.ceph01 config show | more
#查看log存储位置
# ceph-conf --name mon.ceph01 --show-config-value log_file
/var/log/ceph/ceph-mon.ceph01.log
#暂停osd
# ceph osd pause
#开启osd
# ceph osd unpause
#查询pg详细新
# ceph pg 1.ffe query
{
"state": "active+clean",
"snap_trimq": "[]",
"epoch": 732,
"up": [
45,
56,
32
],
"acting": [
45,
56,
32
],
#创建一个镜像
# rbd create -p pool-877be9dced4840e68b3629767389a35c --size 10000 lose
#列出images
# rbd ls pool-877be9dced4840e68b3629767389a35c
lose
test1
volume-1a3ebb0e95bc440f90bb2df4123f7593
volume-1eab1f13c6c14514bcde91e8810b5b2f
volume-2cf091aa66264429936c037eee9bd92f
volume-4f92f5d5e3394f5d9c26eb9cf6b6aae5
volume-6f2669c5ecf94db88ce6e7266880e149
volume-964ada4c6ae94c289b736f9b70eaeca9
volume-970f97af4c0b4597a0358d6952f8477c
volume-b60ab59db5d4495f85e81966f07edfe6
volume-c0c2210fe3904fd19dd0c81da9f82945
volume-d747877e622c4847ac3f6ea10bf9df7b
volume-e46033b5e0f94ef8a8d2911e439c9cba
volume-e9422388ef90405fb0562bb2a1bbbbad
volume-efb22c8b5dc54522bc678e6f23aff6d0
volume-f1d87c7556344a33b39e4bb994fefac1
#查看详细信息
# rbd info -p pool-877be9dced4840e68b3629767389a35c lose
rbd image 'lose':
size 10000 MB in 2500 objects
order 22 (4096 kB objects)
block_name_prefix: rbd_data.e580fa6b8b4567
format: 2
features: layering, deep-flatten
flags:
#调整大小
# rbd resize -p pool-877be9dced4840e68b3629767389a35c --size 20000 lose
Resizing image: 100% complete...done.
#创建快照，快照格式pool/image_name@snapshot_name #池/镜像@快照
# rbd snap create pool-877be9dced4840e68b3629767389a35c/lose@test_snapshot
#查看镜像文件快照
# rbd snap ls -p pool-877be9dced4840e68b3629767389a35c lose
SNAPID NAME SIZE
12 test_snapshot 20000 MB
#删除镜像快照
# rbd snap rm pool-877be9dced4840e68b3629767389a35c/lose@test_snapshot
#如果删除快照出错，有可能是写保护，需要把保护关掉后再删除
# rbd snap unprotect pool-877be9dced4840e68b3629767389a35c/lose@test_snapshot
#删除一个镜像下所有快照
# rbd snap purge -p pool-877be9dced4840e68b3629767389a35c lose
#删除镜像
# rbd rm -p pool-877be9dced4840e68b3629767389a35c lose
Removing image: 100% complete...done.
页:
[1]