#ceph架构 #ceph支持的三种接口： 1 Object：有原生的API，而且也兼容Swift和S3的API。 2 Block：支持精简配置、快照、克隆。 3 File：Posix接口，支持快照。
#ceph的三种类型的优缺点：
#vim /etc/sysconfig/network-scripts/ifcfg-eth0
NetName=eth0 rm -f /etc/sysconfig/network-scripts/ifcfg-$NetName nmcli con add con-name $NetName ifname $NetName autoconnect yes type ethernet \ ip4 172.16.1.131/24 ipv4.dns "114.114.114.114" ipv4.gateway "172.16.1.254" #设置完成后执行下reload网络 systemctl network restart#（可选）如要指定默认路由，只要在对应的网卡配置加上以下配置，如： #vim /etc/sysconfig/network-scripts/ifcfg-eth0
IPV4_ROUTE_METRIC=0#vim /etc/hosts
#[ceph14] 172.16.2.131 ceph131 172.16.2.132 ceph132 172.16.2.133 ceph133#系统yum源更换成阿里源，并更新yum文件缓存
wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo wget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo yum clean all && yum makecache#添加nautilus源
cat << EOM > /etc/yum.repos.d/ceph.repo [ceph-noarch] name=Ceph noarch packages baseurl=https://download.ceph.com/rpm-nautilus/el7/noarch enabled=1 gpgcheck=1 type=rpm-md gpgkey=https://download.ceph.com/keys/release.asc EOM#更新应用版本
yum update -y#个人比较喜欢用以下方式来同步时间
ntpdate ntp3.aliyun.com echo "*/3 * * * * ntpdate ntp3.aliyun.com &> /dev/null" > /tmp/crontab crontab /tmp/crontab##3# 1.3.7（可选）安装基础软件
yum install net-tools wget vim bash-completion lrzsz unzip zip -y#如果是在15版本，支持使用cephadm工具部署，ceph-deploy在14版本前都支持 #安装ceph-deploy，这里选用ceph131做为管理节点，因此ceph-deploy部署在这台上
yum install ceph-deploy -y #这里安装的是版本是 ceph-deploy-2.0.1-0.noarch#Ceph-deploy必须作为具有无密码sudo特权的用户登录到Ceph节点，因为它需要在不提示密码的情况下安装软件和配置文件 #在每个Ceph节点上创建ceph-deploy的用户，密码为 ceph.123
useradd -d /home/cephdeploy -m cephdeploy passwd cephdeploy usermod -G root cephdeploy#对于添加到每个Ceph节点的用户，确保该用户具有sudo特权。
echo "cephdeploy ALL = (root) NOPASSWD:ALL" | sudo tee /etc/sudoers.d/cephdeploy chmod 0440 /etc/sudoers.d/cephdeploy#登陆cephdeploy启用没有密码的ssh，在ceph131执行即可，三次回车！
[cephdeploy@ceph131 ~]$ ssh-keygen Generating public/private rsa key pair. Enter file in which to save the key (/root/.ssh/id_rsa): Enter passphrase (empty for no passphrase): Enter same passphrase again: Your identification has been saved in /root/.ssh/id_rsa. Your public key has been saved in /root/.ssh/id_rsa.pub. The key fingerprint is: SHA256:CsvXYKm8mRzasMFwgWVLx5LvvfnPrRc5S1wSb6kPytM root@ceph131 The key's randomart image is: +---[RSA 2048]----+ | +o. | | =oo. . | |. oo o . | | .. . . = | |. ....+ S . * | | + o.=.+ O | | + * oo.. + * | | B *o .+.E . | | o * ...++. | +----[SHA256]-----+#将密钥复制到每个Ceph节点
ssh-copy-id cephdeploy@ceph131 ssh-copy-id cephdeploy@ceph132 ssh-copy-id cephdeploy@ceph133#验证，不需要输入密码即为成功
[cephdeploy@ceph131 ~]$ ssh 'cephdeploy@ceph133' Last failed login: Thu Jul 2 11:18:06 +08 2020 from 172.16.1.131 on ssh:notty There were 2 failed login attempts since the last successful login. Last login: Thu Jul 2 10:32:51 2020 from 172.16.1.131 [cephdeploy@ceph133 ~]$ exit logout Connection to ceph133 closed.#创建集群目录，用于维护ceph-deploy为集群生成的配置文件和密钥。
su cephdeploy mkdir ~/cephcluster && cd ~/cephcluster [cephdeploy@ceph131 cephcluster]$ pwd /home/cephdeploy/cephcluster#注意点 #1 不要使用sudo调用ceph-deploy，如果您是以另一个用户登录，也不要以root身份运行它，因为它不会发出远程主机上所需的sudo命令。 #2 ceph-deploy会将文件输出到当前目录。如果在执行ceph-deploy时，一定要确保您在这个目录中。
#附：节点清理 #如果在任何时候你遇到麻烦，你想重新开始，执行以下清除Ceph包，并擦除其所有数据和配置:
ceph-deploy purge ceph131 ceph132 ceph133 ceph-deploy purgedata ceph131 ceph132 ceph133 ceph-deploy forgetkeys rm ceph.*#在cephcluster目录下执行创建集群
[cephdeploy@ceph131 cephcluster]$ ceph-deploy new ceph131 ceph132 ceph133 [ceph_deploy.conf][DEBUG ] found configuration file at: /home/cephdeploy/.cephdeploy.conf [ceph_deploy.cli][INFO ] Invoked (2.0.1): /bin/ceph-deploy new ceph131 ceph132 ceph133 [ceph_deploy.cli][INFO ] ceph-deploy options: [ceph_deploy.cli][INFO ] username : None [ceph_deploy.cli][INFO ] func : <function new at 0x7f9f54674e60> [ceph_deploy.cli][INFO ] verbose : False [ceph_deploy.cli][INFO ] overwrite_conf : False [ceph_deploy.cli][INFO ] quiet : False [ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf instance at 0x7f9f541fa9e0> [ceph_deploy.cli][INFO ] cluster : ceph [ceph_deploy.cli][INFO ] ssh_copykey : True [ceph_deploy.cli][INFO ] mon : ['ceph131', 'ceph132', 'ceph133'] [ceph_deploy.cli][INFO ] public_network : None [ceph_deploy.cli][INFO ] ceph_conf : None [ceph_deploy.cli][INFO ] cluster_network : None [ceph_deploy.cli][INFO ] default_release : False [ceph_deploy.cli][INFO ] fsid : None [ceph_deploy.new][DEBUG ] Creating new cluster named ceph [ceph_deploy.new][INFO ] making sure passwordless SSH succeeds [ceph131][DEBUG ] connection detected need for sudo [ceph131][DEBUG ] connected to host: ceph131 [ceph131][DEBUG ] detect platform information from remote host [ceph131][DEBUG ] detect machine type [ceph131][DEBUG ] find the location of an executable [ceph131][INFO ] Running command: sudo /usr/sbin/ip link show [ceph131][INFO ] Running command: sudo /usr/sbin/ip addr show [ceph131][DEBUG ] IP addresses found: [u'172.16.1.131', u'172.16.2.131', u'172.16.3.131'] [ceph_deploy.new][DEBUG ] Resolving host ceph131 [ceph_deploy.new][DEBUG ] Monitor ceph131 at 172.16.1.131 [ceph_deploy.new][INFO ] making sure passwordless SSH succeeds [ceph132][DEBUG ] connected to host: ceph131 [ceph132][INFO ] Running command: ssh -CT -o BatchMode=yes ceph132 [ceph132][DEBUG ] connection detected need for sudo [ceph132][DEBUG ] connected to host: ceph132 [ceph132][DEBUG ] detect platform information from remote host [ceph132][DEBUG ] detect machine type [ceph132][DEBUG ] find the location of an executable [ceph132][INFO ] Running command: sudo /usr/sbin/ip link show [ceph132][INFO ] Running command: sudo /usr/sbin/ip addr show [ceph132][DEBUG ] IP addresses found: [u'172.16.2.132', u'172.16.1.132', u'172.16.3.132'] [ceph_deploy.new][DEBUG ] Resolving host ceph132 [ceph_deploy.new][DEBUG ] Monitor ceph132 at 172.16.1.132 [ceph_deploy.new][INFO ] making sure passwordless SSH succeeds [ceph133][DEBUG ] connected to host: ceph131 [ceph133][INFO ] Running command: ssh -CT -o BatchMode=yes ceph133 [ceph133][DEBUG ] connection detected need for sudo [ceph133][DEBUG ] connected to host: ceph133 [ceph133][DEBUG ] detect platform information from remote host [ceph133][DEBUG ] detect machine type [ceph133][DEBUG ] find the location of an executable [ceph133][INFO ] Running command: sudo /usr/sbin/ip link show [ceph133][INFO ] Running command: sudo /usr/sbin/ip addr show [ceph133][DEBUG ] IP addresses found: [u'172.16.2.133', u'172.16.1.133', u'172.16.3.133'] [ceph_deploy.new][DEBUG ] Resolving host ceph133 [ceph_deploy.new][DEBUG ] Monitor ceph133 at 172.16.1.133 [ceph_deploy.new][DEBUG ] Monitor initial members are ['ceph131', 'ceph132', 'ceph133'] [ceph_deploy.new][DEBUG ] Monitor addrs are ['172.16.1.131', '172.16.1.132', '172.16.1.133'] [ceph_deploy.new][DEBUG ] Creating a random mon key... [ceph_deploy.new][DEBUG ] Writing monitor keyring to ceph.mon.keyring... [ceph_deploy.new][DEBUG ] Writing initial config to ceph.conf...#创建公共访问的网络 #vim /root/cephcluster/ceph.conf
[global] fsid = 76235629-6feb-4f0c-a106-4be33d485535 mon_initial_members = ceph131, ceph132, ceph133 mon_host = 172.16.1.131,172.16.1.132,172.16.1.133 auth_cluster_required = cephx auth_service_required = cephx auth_client_required = cephx public_network = 172.16.1.0/24 cluster_network = 172.16.2.0/24 #设置副本数 osd_pool_default_size = 3 #设置最小副本数 osd_pool_default_min_size = 2 #设置时钟偏移0.5s mon_clock_drift_allowed = .50#指定源安装ceph安装包，一定要在目录里执行 #输出无报错即可
ceph-deploy install --repo-url https://mirrors.tuna.tsinghua.edu.cn/ceph/rpm-nautilus/el7/ --gpg-url https://mirrors.tuna.tsinghua.edu.cn/ceph/keys/release.asc ceph131 ceph132 ceph133#初始化mon，执行过程无报错即可
ceph-deploy mon create-initial#这时候目录底下应该会多以下的key
[cephdeploy@ceph131 cephcluster]$ ll total 220 -rw------- 1 cephdeploy cephdeploy 113 Jul 2 14:16 ceph.bootstrap-mds.keyring -rw------- 1 cephdeploy cephdeploy 113 Jul 2 14:16 ceph.bootstrap-mgr.keyring -rw------- 1 cephdeploy cephdeploy 113 Jul 2 14:16 ceph.bootstrap-osd.keyring -rw------- 1 cephdeploy cephdeploy 113 Jul 2 14:16 ceph.bootstrap-rgw.keyring -rw------- 1 cephdeploy cephdeploy 151 Jul 2 14:16 ceph.client.admin.keyring -rw-rw-r-- 1 cephdeploy cephdeploy 453 Jul 2 14:05 ceph.conf -rw-rw-r-- 1 cephdeploy cephdeploy 177385 Jul 2 14:16 ceph-deploy-ceph.log -rw------- 1 cephdeploy cephdeploy 73 Jul 2 14:03 ceph.mon.keyring#使用Ceph -deploy将配置文件和管理密钥复制到您的管理节点和Ceph节点
[cephdeploy@ceph131 cephcluster]$ ceph-deploy admin ceph131 ceph132 ceph133 [ceph_deploy.conf][DEBUG ] found configuration file at: /home/cephdeploy/.cephdeploy.conf [ceph_deploy.cli][INFO ] Invoked (2.0.1): /bin/ceph-deploy admin ceph131 ceph132 ceph133 [ceph_deploy.cli][INFO ] ceph-deploy options: [ceph_deploy.cli][INFO ] username : None [ceph_deploy.cli][INFO ] verbose : False [ceph_deploy.cli][INFO ] overwrite_conf : False [ceph_deploy.cli][INFO ] quiet : False [ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf instance at 0x7ff46c014638> [ceph_deploy.cli][INFO ] cluster : ceph [ceph_deploy.cli][INFO ] client : ['ceph131', 'ceph132', 'ceph133'] [ceph_deploy.cli][INFO ] func : <function admin at 0x7ff46c8b52a8> [ceph_deploy.cli][INFO ] ceph_conf : None [ceph_deploy.cli][INFO ] default_release : False [ceph_deploy.admin][DEBUG ] Pushing admin keys and conf to ceph131 [ceph131][DEBUG ] connection detected need for sudo [ceph131][DEBUG ] connected to host: ceph131 [ceph131][DEBUG ] detect platform information from remote host [ceph131][DEBUG ] detect machine type [ceph131][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf [ceph_deploy.admin][DEBUG ] Pushing admin keys and conf to ceph132 [ceph132][DEBUG ] connection detected need for sudo [ceph132][DEBUG ] connected to host: ceph132 [ceph132][DEBUG ] detect platform information from remote host [ceph132][DEBUG ] detect machine type [ceph132][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf [ceph_deploy.admin][DEBUG ] Pushing admin keys and conf to ceph133 [ceph133][DEBUG ] connection detected need for sudo [ceph133][DEBUG ] connected to host: ceph133 [ceph133][DEBUG ] detect platform information from remote host [ceph133][DEBUG ] detect machine type [ceph133][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf#创建mgr，执行过程无报错即可
ceph-deploy mgr create ceph131 ceph132 ceph133#创建osd，执行过程无报错即可
ceph-deploy osd create --data /dev/sdb ceph131 ceph-deploy osd create --data /dev/sdb ceph132 ceph-deploy osd create --data /dev/sdb ceph133#所有的ceph-deploy操作都必须是使用cephdeploy用户并在配置文件目录底下执行！！！
[cephdeploy@ceph131 cephcluster]$ pwd /home/cephdeploy/cephcluster#一旦您添加了新的Ceph监视器，Ceph将开始同步监视器并形成一个quorum。您可以通过执行以下操作来检查仲裁状态:
ceph quorum_status --format json-pretty#Ceph Manager守护进程以活动/备用模式操作。部署其他管理器守护进程可以确保，如果一个守护进程或主机失败，另一个守护进程可以接管，而不会中断服务。
ceph-deploy mgr create ceph132 ceph133#验证
ceph -s#默认情况下，RGW实例将监听端口7480。这可以通过在运行RGW的节点上编辑ceph.conf来改变，如下所示:
[client] rgw frontends = civetweb port=80#修改完端口需要重启服务
systemctl restart ceph-radosgw.servicePG数量的预估 集群中单个池的PG数计算公式如下：PG 总数 = (OSD 数 * 100) / 最大副本数 / 池数 (结果必须舍入到最接近2的N次幂的值) #在ceph集群，其中一台执行命令即可，这里用ceph132
[root@ceph132 ~]# ceph osd pool create cephfs_data 16 pool 'cephfs_data' created [root@ceph132 ~]# ceph osd pool create cephfs_metadata 16 pool 'cephfs_metadata' created [root@ceph132 ~]# ceph fs new cephfs_storage cephfs_metadata cephfs_data new fs with metadata pool 2 and data pool 1 [root@ceph132 ~]# ceph df RAW STORAGE: CLASS SIZE AVAIL USED RAW USED %RAW USED hdd 96 GiB 93 GiB 11 MiB 3.0 GiB 3.14 TOTAL 96 GiB 93 GiB 11 MiB 3.0 GiB 3.14 POOLS: POOL ID STORED OBJECTS USED %USED MAX AVAIL cephfs_data 1 0 B 0 0 B 0 29 GiB cephfs_metadata 2 2.2 KiB 22 1.5 MiB 0 29 GiB#在客户端创建密码文件
[root@ceph133 ~]# cat /etc/ceph/ceph.client.admin.keyring [client.admin] key = AQC5e/1eXm9WExAAmlD9aZoc2dZO6jbU8UXSqg== caps mds = "allow *" caps mgr = "allow *" caps mon = "allow *" caps osd = "allow *" [root@ceph133 ~]# vim admin.secret [root@ceph133 ~]# ll total 8 -rw-r--r-- 1 root root 41 Jul 2 15:17 admin.secret#挂载cephfs文件夹，并验证
[root@ceph133 ~]# mkdir /mnt/cephfs_storage [root@ceph133 ~]# mount -t ceph 172.16.1.133:6789:/ /mnt/cephfs_storage -o name=admin,secretfile=admin.secret [root@ceph133 ~]# df -Th Filesystem Type Size Used Avail Use% Mounted on devtmpfs devtmpfs 983M 0 983M 0% /dev tmpfs tmpfs 995M 0 995M 0% /dev/shm tmpfs tmpfs 995M 8.6M 987M 1% /run tmpfs tmpfs 995M 0 995M 0% /sys/fs/cgroup /dev/sda1 xfs 20G 2.2G 18G 11% / tmpfs tmpfs 199M 0 199M 0% /run/user/0 tmpfs tmpfs 995M 52K 995M 1% /var/lib/ceph/osd/ceph-2 172.16.1.133:6789:/ ceph 30G 0 30G 0% /mnt/cephfs_storage#初始化rbd池 PG数量的预估 集群中单个池的PG数计算公式如下：PG 总数 = (OSD 数 * 100) / 最大副本数 / 池数 (结果必须舍入到最接近2的N次幂的值)
[root@ceph132 ~]# ceph osd pool create rbd_storage 16 16 replicated pool 'rbd_storage' created#创建一个块设备
[root@ceph132 ~]# rbd create --size 1024 rbd_image -p rbd_storage [root@ceph132 ~]# rbd ls rbd_storage rbd_image#删除命令
rbd rm rbd_storage/rbd_image#附rbd池的相关操作命令
#将块设备映射到系统内核
[root@ceph133 ~]# rbd map rbd_storage/rbd_image /dev/rbd0 [root@ceph133 ~]# lsblk NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT rbd0 252:0 0 1G 0 disk sdb 8:16 0 32G 0 disk └─ceph--376ebd83--adf0--4ee1--b7e4--b4b79bae048e-osd--block--4b0444fa--535e--40c7--b55a--167ab21dbf9b 253:0 0 32G 0 lvm sr0 11:0 1 4M 0 rom sda 8:0 0 20G 0 disk └─sda1 8:1 0 20G 0 part /#格式化rbd设备
[root@ceph133 ~]# mkfs.ext4 -m0 /dev/rbd/rbd_storage/rbd_image mke2fs 1.42.9 (28-Dec-2013) Discarding device blocks: done Filesystem label= OS type: Linux Block size=4096 (log=2) Fragment size=4096 (log=2) Stride=16 blocks, Stripe width=16 blocks 65536 inodes, 262144 blocks 0 blocks (0.00%) reserved for the super user First data block=0 Maximum filesystem blocks=268435456 8 block groups 32768 blocks per group, 32768 fragments per group 8192 inodes per group Superblock backups stored on blocks: 32768, 98304, 163840, 229376 Allocating group tables: done Writing inode tables: done Creating journal (8192 blocks): done Writing superblocks and filesystem accounting information: done#挂载rbd设备
[root@ceph133 ~]# mkdir /mnt/rbd_storage [root@ceph133 ~]# mount /dev/rbd/rbd_storage/rbd_image /mnt/rbd_storage [root@ceph133 ~]# df -Th Filesystem Type Size Used Avail Use% Mounted on devtmpfs devtmpfs 983M 0 983M 0% /dev tmpfs tmpfs 995M 0 995M 0% /dev/shm tmpfs tmpfs 995M 8.6M 987M 1% /run tmpfs tmpfs 995M 0 995M 0% /sys/fs/cgroup /dev/sda1 xfs 20G 2.2G 18G 11% / tmpfs tmpfs 199M 0 199M 0% /run/user/0 tmpfs tmpfs 995M 52K 995M 1% /var/lib/ceph/osd/ceph-2 172.16.1.133:6789:/ ceph 30G 0 30G 0% /mnt/cephfs_storage /dev/rbd0 ext4 976M 2.6M 958M 1% /mnt/rbd_storage#取消内核挂载
rbd unmap /dev/rbd0#启用rgw见3.4 #使用浏览器，查看 http://172.16.1.131:7480/，说明已经启用成功
#Ceph仪表板是一个内置的基于web的Ceph管理和监视应用程序，用于管理集群。 #节点默认没有安装mgr-dashboard程序，因此要先安装
yum install ceph-mgr-dashboard -y#任一节点启用dashboard，没有提示
[root@ceph132 ~]# ceph mgr module enable dashboard#配置登陆认证
[root@ceph132 ~]# ceph dashboard create-self-signed-cert Self-signed certificate created#配置登陆账户
[root@ceph132 ~]# ceph dashboard ac-user-create admin admin.123 administrator {"username": "admin", "lastUpdate": 1593679011, "name": null, "roles": ["administrator"], "password": "$2b$12$kQYtMXun1jKdTDKwjfuNj.WYyJcr3vSHLTMWfXIi.wrKkFRCmmC1.", "email": null}#测试登陆，浏览器查看：https://172.16.1.131:8443/ 用户名：admin 密码：admin.123
[root@ceph132 ~]# ceph mgr services { "dashboard": "https://ceph131:8443/" }
#创建rgw的用户
[root@ceph131 ~]# radosgw-admin user create --uid=rgwadmin --display-name=rgwadmin --system { "user_id": "rgwadmin", "display_name": "rgwadmin", "email": "", "suspended": 0, "max_buckets": 1000, "subusers": [], "keys": [ { "user": "rgwadmin", "access_key": "VHK8BZYA3F5BDWMJFUER", "secret_key": "WVBdt66ZbmGe0l6Wu3LoPKM1GlzF6V35JCpNKPJw" } ], "swift_keys": [], "caps": [], "op_mask": "read, write, delete", "system": "true", "default_placement": "", "default_storage_class": "", "placement_tags": [], "bucket_quota": { "enabled": false, "check_on_raw": false, "max_size": -1, "max_size_kb": 0, "max_objects": -1 }, "user_quota": { "enabled": false, "check_on_raw": false, "max_size": -1, "max_size_kb": 0, "max_objects": -1 }, "temp_url_keys": [], "type": "rgw", "mfa_ids": [] }#设置凭证，取创建用户生成的key
[root@ceph131 ~]# ceph dashboard set-rgw-api-access-key VHK8BZYA3F5BDWMJFUER Option RGW_API_ACCESS_KEY updated [root@ceph131 ~]# ceph dashboard set-rgw-api-secret-key WVBdt66ZbmGe0l6Wu3LoPKM1GlzF6V35JCpNKPJw Option RGW_API_SECRET_KEY updated#禁用SSL
[root@ceph131 ~]# ceph dashboard set-rgw-api-ssl-verify False Option RGW_API_SSL_VERIFY updated#启用rgw的dashboard
[root@ceph131 ~]# ceph dashboard set-rgw-api-host 172.16.1.131 Option RGW_API_HOST updated [root@ceph131 ~]# ceph dashboard set-rgw-api-port 7480 Option RGW_API_PORT updated [root@ceph131 ~]# ceph dashboard set-rgw-api-scheme http Option RGW_API_SCHEME updated [root@ceph131 ~]# ceph dashboard set-rgw-api-admin-resource admin Option RGW_API_ADMIN_RESOURCE updated [root@ceph131 ~]# ceph dashboard set-rgw-api-user-id rgwadmin Option RGW_API_USER_ID updated [root@ceph131 ~]# systemctl restart ceph-radosgw.target