User Tools

Site Tools


linux:ceph:clover_deploy

Deploying clover

Documentation
Name: Deploying clover
Description: raw commands of a live CEPH storage cluster deployment
Modification date :29/07/2018
Owner:dodger
Notify changes to:Owner
Tags:ceph, object storage
Scalate to:The_fucking_bofh

RAW

CloneWars deploy clover

for i in 01 03 05 07 09 11 13 15 17 19 ; do bash /home/bofher/scripts/CloneWars/CloneWars.sh -c nuciberterminal -h AVMLP-OSD-0${i} -i 10.20.54.1${i} -d 2048GB -m 20 -O -r 4096 -v 2 -o 2 -F ; rm -fv /etc/salt/pki/master/minions/avmlp-osd-0${i}.ciberterminal.net ; done
for i in 02 04 06 08 10 12 14 16 18 20 ; do bash /home/bofher/scripts/CloneWars/CloneWars.sh -c nuciberterminal2 -h AVMLP-OSD-0${i} -i 10.20.54.1${i} -d 2048GB -m 20 -O -r 4096 -v 2 -o 2 -F ; rm -fv /etc/salt/pki/master/minions/avmlp-osd-0${i}.ciberterminal.net ; done
for i in 01 03 ; do  bash /home/bofher/scripts/CloneWars/CloneWars.sh -c nuciberterminal -h AVMLP-OSM-0${i} -i 10.20.54.$((${i}+50)) -d 50GB -m 20 -O -r 4096 -v 2 -o 2   -F ;  rm -fv /etc/salt/pki/master/minions/avmlp-osm-0${i}.ciberterminal.net ; done 
for i in 02 04 ; do  bash /home/bofher/scripts/CloneWars/CloneWars.sh -c nuciberterminal2 -h AVMLP-OSM-0${i} -i 10.20.54.$((${i}+50)) -d 50GB -m 20 -O -r 4096 -v 2 -o 2  -F ;  rm -fv /etc/salt/pki/master/minions/avmlp-osm-0${i}.ciberterminal.net ; done
for i in 01 03 ; do  bash /home/bofher/scripts/CloneWars/CloneWars.sh -c nuciberterminal -h AVMLP-OSGW-0${i} -i 10.20.54.$((${i}+10)) -d 50GB -m 20 -O -r 4096 -v 2 -o 2  -F ;  rm -fv /etc/salt/pki/master/minions/avmlp-osgw-0${i}.ciberterminal.net ; done
for i in 02 04 ; do  bash /home/bofher/scripts/CloneWars/CloneWars.sh -c nuciberterminal2 -h AVMLP-OSGW-0${i} -i 10.20.54.$((${i}+10)) -d 50GB -m 20 -O -r 4096 -v 2 -o 2 -F ;  rm -fv /etc/salt/pki/master/minions/avmlp-osgw-0${i}.ciberterminal.net ; done
for i in 01 03 ; do  bash /home/bofher/scripts/CloneWars/CloneWars.sh -c nuciberterminal -h AVMLP-OSFS-0${i} -i 10.20.55.$((${i}+10)) -d 50GB -m 20 -O -r 4096 -v 2 -o 2  -F ;  rm -fv /etc/salt/pki/master/minions/avmlp-osfs-0${i}.ciberterminal.net ; done
for i in 02 04 ; do  bash /home/bofher/scripts/CloneWars/CloneWars.sh -c nuciberterminal2 -h AVMLP-OSFS-0${i} -i 10.20.55.$((${i}+10)) -d 50GB -m 20 -O -r 4096 -v 2 -o 2 -F ;  rm -fv /etc/salt/pki/master/minions/avmlp-osfs-0${i}.ciberterminal.net ; done

salt

export THESERVER="avmlp-os*-0*"
salt-key -A
salt "${THESERVER}" state.apply
salt "${THESERVER}" state.apply nsupdate
salt "${THESERVER}" test.ping
salt "${THESERVER}" pkg.install yum-plugin-priorities
salt "${THESERVER}" user.add ceph 1002
salt "${THESERVER}" file.write /etc/sudoers.d/ceph \
"ceph ALL = (root) NOPASSWD:ALL"
salt "${THESERVER}" cmd.run "sudo whoami" runas=ceph
salt "${THESERVER}" cmd.run \
    "ssh-keygen -q -N '' -f /home/ceph/.ssh/id_rsa" \
    runas=ceph
salt "${THESERVER}" cmd.run "cp /home/ceph/.ssh/id_rsa.pub /home/ceph/.ssh/authorized_keys"
 
export VMNAMESTART="avmlp-os"
salt "${THESERVER}" cmd.run "cat /home/ceph/.ssh/id_rsa.pub" |egrep -v "^${VMNAMESTART}" | sed 's/^[[:space:]]\{1,5\}//g' > all_cluster_nodes.txt
while read LINE ; do salt "${THESERVER}" file.append /home/ceph/.ssh/authorized_keys "${LINE}" ; done < all_cluster_nodes.txt

avmlp-osm-001

cat << EOM > ceph.repo
[Ceph]
name=Ceph packages for \$basearch
baseurl=http://download.ceph.com/rpm-nautilus/el7/\$basearch
enabled=1
gpgcheck=1
type=rpm-md
gpgkey=https://download.ceph.com/keys/release.asc
 
[Ceph-noarch]
name=Ceph noarch packages
baseurl=http://download.ceph.com/rpm-nautilus/el7/noarch
enabled=1
gpgcheck=1
type=rpm-md
gpgkey=https://download.ceph.com/keys/release.asc
 
[ceph-source]
name=Ceph source packages
baseurl=http://download.ceph.com/rpm-nautilus/el7/SRPMS
enabled=1
gpgcheck=1
type=rpm-md
gpgkey=https://download.ceph.com/keys/release.asc
EOM
 
 
export GWSERVERS="avmlp-osgw-004.ciberterminal.net avmlp-osgw-003.ciberterminal.net avmlp-osgw-001.ciberterminal.net avmlp-osgw-002.ciberterminal.net"
export OSDSERVERS="avmlp-osd-001.ciberterminal.net avmlp-osd-002.ciberterminal.net avmlp-osd-003.ciberterminal.net avmlp-osd-004.ciberterminal.net avmlp-osd-005.ciberterminal.net avmlp-osd-006.ciberterminal.net avmlp-osd-007.ciberterminal.net avmlp-osd-008.ciberterminal.net avmlp-osd-009.ciberterminal.net avmlp-osd-010.ciberterminal.net avmlp-osd-011.ciberterminal.net avmlp-osd-012.ciberterminal.net avmlp-osd-013.ciberterminal.net avmlp-osd-014.ciberterminal.net avmlp-osd-015.ciberterminal.net avmlp-osd-016.ciberterminal.net avmlp-osd-017.ciberterminal.net avmlp-osd-018.ciberterminal.net avmlp-osd-019.ciberterminal.net avmlp-osd-020.ciberterminal.net"
export MONSERVERS="avmlp-osm-002.ciberterminal.net avmlp-osm-001.ciberterminal.net avmlp-osm-004.ciberterminal.net avmlp-osm-003.ciberterminal.net avmlp-osm-005.ciberterminal.net avmlp-osm-006.ciberterminal.net"
export MDSSERVERS="avmlp-osfs-002.ciberterminal.net avmlp-osfs-001.ciberterminal.net avmlp-osfs-004.ciberterminal.net avmlp-osfs-003.ciberterminal.net"
export ALLSERVERS="${GWSERVERS} ${OSDSERVERS} ${MONSERVERS} ${MDSSERVERS}"
 
 
for i in ${ALLSERVERS} ; do scp ceph.repo ${i}:/home/ceph/ ; ssh ${i} "sudo cp -pfv /home/ceph/ceph.repo /etc/yum.repos.d/"  ; done
for i in ${ALLSERVERS} ; do ssh ${i} "sudo chown root. /etc/yum.repos.d/ceph.repo"  ; done
for i in ${ALLSERVERS} ; do ssh ${i} "sudo ls -l /etc/yum.repos.d/ceph.repo"  ; done

salt

export THESERVER="avmlp-os[mdgf]*-0*"
salt "${THESERVER}" cmd.run "yum -y install ceph-deploy"

avmlp-osm-001

ceph-deploy install ${ALLSERVERS} --repo-url https://download.ceph.com/rpm-nautilus/el7/

salt

CHECK

export THESERVER="avmlp-os[mdgf]*-0*"
salt "${THESERVER}" cmd.run "rpm -qa |egrep ceph"
for i in $(salt "${THESERVER}" test.ping| egrep "^a"|awk -F\: '{print $1}'| sort) ; do salt "${i}" cmd.run "sudo ceph --version" ; done

avmlp-osm-001

ceph-deploy new ${MONSERVERS}
ceph-deploy mon create-initial
 
for i in ${MONSERVERS} ; do ceph-deploy gatherkeys ${i} ; done
ceph-deploy admin ${MONSERVERS}
sudo ceph mon enable-msgr2
ceph-deploy mgr create  ${MONSERVERS}

osd's

for THESERVER in ${OSDSERVERS} ; do
     echo "${THESERVER}";   
     echo "################### ${THESERVER}: creating disk" ;  
     ceph-deploy osd create ${THESERVER} --data /dev/sdb;  
     echo "################### ${THESERVER}: listing (check) disk" ;    
     ceph-deploy osd list ${THESERVER};  
done

CRUSH

mkdir /home/ceph/crush/mod_20190730
cd /home/ceph/crush/mod_20190730
ceph osd getcrushmap  -o crushmap.bin
crushtool  -d crushmap.bin -o crushmap.txt
crushmap.txt
# begin crush map
tunable choose_local_tries 0
tunable choose_local_fallback_tries 0
tunable choose_total_tries 50
tunable chooseleaf_descend_once 1
tunable chooseleaf_vary_r 1
tunable chooseleaf_stable 1
tunable straw_calc_version 1
tunable allowed_bucket_algs 54
 
# devices
device 0 osd.0 class hdd
device 1 osd.1 class hdd
device 2 osd.2 class hdd
device 3 osd.3 class hdd
device 4 osd.4 class hdd
device 5 osd.5 class hdd
device 6 osd.6 class hdd
device 7 osd.7 class hdd
device 8 osd.8 class hdd
device 9 osd.9 class hdd
device 10 osd.10 class hdd
device 11 osd.11 class hdd
device 12 osd.12 class hdd
device 13 osd.13 class hdd
device 14 osd.14 class hdd
device 15 osd.15 class hdd
device 16 osd.16 class hdd
device 17 osd.17 class hdd
device 18 osd.18 class hdd
device 19 osd.19 class hdd
 
# types
type 0 osd
type 1 host
type 2 chassis
type 3 rack
type 4 row
type 5 pdu
type 6 pod
type 7 room
type 8 datacenter
type 9 zone
type 10 region
type 11 root
 
# buckets
host avmlp-osd-001 {
        id -3           # do not change unnecessarily
        id -4 class hdd         # do not change unnecessarily
        # weight 1.999
        alg straw2
        hash 0  # rjenkins1
        item osd.0 weight 1.999
}
 
host avmlp-osd-002 {
        id -5           # do not change unnecessarily
        id -6 class hdd         # do not change unnecessarily
        # weight 1.999
        alg straw2
        hash 0  # rjenkins1
        item osd.1 weight 1.999
}
host avmlp-osd-003 {
        id -7           # do not change unnecessarily
        id -8 class hdd         # do not change unnecessarily
        # weight 1.999
        alg straw2
        hash 0  # rjenkins1
        item osd.2 weight 1.999
}
host avmlp-osd-004 {
        id -9           # do not change unnecessarily
        id -10 class hdd                # do not change unnecessarily
        # weight 1.999
        alg straw2
        hash 0  # rjenkins1
        item osd.3 weight 1.999
}
host avmlp-osd-005 {
        id -11          # do not change unnecessarily
        id -12 class hdd                # do not change unnecessarily
        # weight 1.999
        alg straw2
        hash 0  # rjenkins1
        item osd.4 weight 1.999
}
host avmlp-osd-006 {
        id -13          # do not change unnecessarily
        id -14 class hdd                # do not change unnecessarily
        # weight 1.999
        alg straw2
        hash 0  # rjenkins1
        item osd.5 weight 1.999
}
host avmlp-osd-007 {
        id -15          # do not change unnecessarily
        id -16 class hdd                # do not change unnecessarily
        # weight 1.999
        alg straw2
        hash 0  # rjenkins1
        item osd.6 weight 1.999
}
host avmlp-osd-008 {
        id -17          # do not change unnecessarily
        id -18 class hdd                # do not change unnecessarily
        # weight 1.999
        alg straw2
        hash 0  # rjenkins1
        item osd.7 weight 1.999
}
host avmlp-osd-009 {
        id -19          # do not change unnecessarily
        id -20 class hdd                # do not change unnecessarily
        # weight 1.999
        alg straw2
        hash 0  # rjenkins1
        item osd.8 weight 1.999
}
host avmlp-osd-010 {
        id -21          # do not change unnecessarily
        id -22 class hdd                # do not change unnecessarily
        # weight 1.999
        alg straw2
        hash 0  # rjenkins1
        item osd.9 weight 1.999
}
host avmlp-osd-011 {
        id -23          # do not change unnecessarily
        id -24 class hdd                # do not change unnecessarily
        # weight 1.999
        alg straw2
        hash 0  # rjenkins1
        item osd.10 weight 1.999
}
 
 
host avmlp-osd-012 {
        id -25          # do not change unnecessarily
        id -26 class hdd                # do not change unnecessarily
        # weight 1.999
        alg straw2
        hash 0  # rjenkins1
        item osd.11 weight 1.999
}
host avmlp-osd-013 {
        id -27          # do not change unnecessarily
        id -28 class hdd                # do not change unnecessarily
        # weight 1.999
        alg straw2
        hash 0  # rjenkins1
        item osd.12 weight 1.999
}
host avmlp-osd-014 {
        id -29          # do not change unnecessarily
        id -30 class hdd                # do not change unnecessarily
        # weight 1.999
        alg straw2
        hash 0  # rjenkins1
        item osd.13 weight 1.999
}
host avmlp-osd-015 {
        id -31          # do not change unnecessarily
        id -32 class hdd                # do not change unnecessarily
        # weight 1.999
        alg straw2
        hash 0  # rjenkins1
        item osd.14 weight 1.999
}
host avmlp-osd-016 {
        id -33          # do not change unnecessarily
        id -34 class hdd                # do not change unnecessarily
        # weight 1.999
        alg straw2
        hash 0  # rjenkins1
        item osd.15 weight 1.999
}
host avmlp-osd-017 {
        id -35          # do not change unnecessarily
        id -36 class hdd                # do not change unnecessarily
        # weight 1.999
        alg straw2
        hash 0  # rjenkins1
        item osd.16 weight 1.999
}
host avmlp-osd-018 {
        id -37          # do not change unnecessarily
        id -38 class hdd                # do not change unnecessarily
        # weight 1.999
        alg straw2
        hash 0  # rjenkins1
        item osd.17 weight 1.999
}
host avmlp-osd-019 {
        id -39          # do not change unnecessarily
        id -40 class hdd                # do not change unnecessarily
        # weight 1.999
        alg straw2
        hash 0  # rjenkins1
        item osd.18 weight 1.999
}
host avmlp-osd-020 {
        id -41          # do not change unnecessarily
        id -42 class hdd                # do not change unnecessarily
        # weight 1.999
        alg straw2
        hash 0  # rjenkins1
        item osd.19 weight 1.999
}
 
rack nuciberterminal {
        # weight 19.990
        alg straw2
        hash 0  # rjenkins1
 
        item avmlp-osd-001 weight 1.999
        item avmlp-osd-003 weight 1.999
        item avmlp-osd-005 weight 1.999
        item avmlp-osd-007 weight 1.999
        item avmlp-osd-009 weight 1.999
        item avmlp-osd-011 weight 1.999
        item avmlp-osd-013 weight 1.999
        item avmlp-osd-015 weight 1.999
        item avmlp-osd-017 weight 1.999
        item avmlp-osd-019 weight 1.999
}
 
rack nuciberterminal2 {
        # weight 19.990
        alg straw2
        hash 0  # rjenkins1
 
        item avmlp-osd-002 weight 1.999
        item avmlp-osd-004 weight 1.999
        item avmlp-osd-006 weight 1.999
        item avmlp-osd-008 weight 1.999
        item avmlp-osd-010 weight 1.999
        item avmlp-osd-012 weight 1.999
        item avmlp-osd-014 weight 1.999
        item avmlp-osd-016 weight 1.999
        item avmlp-osd-018 weight 1.999
        item avmlp-osd-020 weight 1.999
} 
 
datacenter mediapro {
        # weight 0.000
        alg straw2
        hash 0  # rjenkins1
        item nuciberterminal2 weight 19.990
}
 
datacenter itconic {
        # weight 0.000
        alg straw2
        hash 0  # rjenkins1
        item nuciberterminal weight 19.990
}
 
root default {
        id -1           # do not change unnecessarily
        id -2 class hdd         # do not change unnecessarily
        # weight 39.980
        alg straw2
        hash 0  # rjenkins1
        item mediapro weight 19.990
        item itconic weight 19.990
}
 
# rules
rule replicated_rule {
        id 0
        type replicated
        min_size 2
        max_size 10
        step take default
        step chooseleaf firstn 0 type datacenter
        step emit
}
 
rule ciberterminalRule {
        id 1 
        type replicated
        min_size 2
        max_size 10
        # begin iterating in the "root" of the crush tree
        step take default
        step choose firstn 4 type datacenter
        step chooseleaf firstn 2 type host
        step emit
}
 
# end crush map
crushtool -c crushmap.txt -o crushmap_new.bin
crushtool --test -i crushmap_new.bin --show-utilization --rule 1 --num-rep=4
crushtool --test -i crushmap_new.bin --show-choose-tries --rule 1 --num-rep=4
avmlp-osm-001 /home/ceph/crush/mod_20190730 # ceph osd crush tree
ID  CLASS WEIGHT   TYPE NAME                      
 -1       39.97998 root default                   
-46       19.98999     datacenter itconic         
-43       19.98999         rack nuciberterminal           
 -3        1.99899             host avmlp-osd-001 
  0   hdd  1.99899                 osd.0          
 -7        1.99899             host avmlp-osd-003 
  2   hdd  1.99899                 osd.2          
-11        1.99899             host avmlp-osd-005 
  4   hdd  1.99899                 osd.4          
-15        1.99899             host avmlp-osd-007 
  6   hdd  1.99899                 osd.6          
-19        1.99899             host avmlp-osd-009 
  8   hdd  1.99899                 osd.8          
-23        1.99899             host avmlp-osd-011 
 10   hdd  1.99899                 osd.10         
-27        1.99899             host avmlp-osd-013 
 12   hdd  1.99899                 osd.12         
-31        1.99899             host avmlp-osd-015 
 14   hdd  1.99899                 osd.14         
-35        1.99899             host avmlp-osd-017 
 16   hdd  1.99899                 osd.16         
-39        1.99899             host avmlp-osd-019 
 18   hdd  1.99899                 osd.18         
-45       19.98999     datacenter mediapro        
-44       19.98999         rack nuciberterminal2          
 -5        1.99899             host avmlp-osd-002 
  1   hdd  1.99899                 osd.1          
 -9        1.99899             host avmlp-osd-004 
  3   hdd  1.99899                 osd.3          
-13        1.99899             host avmlp-osd-006 
  5   hdd  1.99899                 osd.5          
-17        1.99899             host avmlp-osd-008 
  7   hdd  1.99899                 osd.7          
-21        1.99899             host avmlp-osd-010 
  9   hdd  1.99899                 osd.9          
-25        1.99899             host avmlp-osd-012 
 11   hdd  1.99899                 osd.11         
-29        1.99899             host avmlp-osd-014 
 13   hdd  1.99899                 osd.13         
-33        1.99899             host avmlp-osd-016 
 15   hdd  1.99899                 osd.15         
-37        1.99899             host avmlp-osd-018 
 17   hdd  1.99899                 osd.17         
-41        1.99899             host avmlp-osd-020 
 19   hdd  1.99899                 osd.19         

recovery tunning

ceph tell 'osd.*' config set osd_recovery_max_single_start 1000
ceph tell 'osd.*' config set osd_max_backfills 100
ceph tell 'osd.*' config set osd_recovery_max_active 300
ceph tell 'osd.*' config set osd_recovery_max_single_start 1000
ceph tell 'osd.*' config set osd_backfill_scan_max 51200
ceph tell 'osd.*' config set osd_backfill_scan_min 6400
ceph tell 'osd.*' config set osd_recovery_sleep_hdd 0

Tunning CEPH recovery

radosgw

ceph-deploy rgw create ${GWSERVERS}

check:

avmlp-osm-001 /home/ceph # ceph -s
  cluster:
    id:     a3a799ce-f1d3-4230-a915-06e988fee767
    health: HEALTH_WARN
            Degraded data redundancy: 188/564 objects degraded (33.333%), 18 pgs degraded
            too few PGs per OSD (3 < min 30)
 
  services:
    mon: 6 daemons, quorum avmlp-osm-001,avmlp-osm-002,avmlp-osm-003,avmlp-osm-004,avmlp-osm-005,avmlp-osm-006 (age 3h)
    mgr: avmlp-osm-002.ciberterminal.net(active, since 4h), standbys: avmlp-osm-004.ciberterminal.net, avmlp-osm-003.ciberterminal.net, avmlp-osm-001.ciberterminal.net, avmlp-osm-005.ciberterminal.net, avmlp-osm-006.ciberterminal.net
    osd: 20 osds: 20 up (since 4h), 20 in (since 4h)
    rgw: 4 daemons active (avmlp-osgw-001.ciberterminal.net, avmlp-osgw-002.ciberterminal.net, avmlp-osgw-003.ciberterminal.net, avmlp-osgw-004.ciberterminal.net)
 
  data:
    pools:   4 pools, 32 pgs
    objects: 188 objects, 2.0 KiB
    usage:   20 GiB used, 40 TiB / 40 TiB avail
    pgs:     188/564 objects degraded (33.333%)
             18 active+undersized+degraded
             14 active+undersized
 
  io:
    client:   60 KiB/s rd, 0 B/s wr, 68 op/s rd, 45 op/s wr
 
avmlp-osm-001 /home/ceph # ceph osd pool ls detail
pool 1 '.rgw.root' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode warn last_change 85 flags hashpspool stripe_width 0 application rgw
pool 2 'default.rgw.control' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode warn last_change 87 flags hashpspool stripe_width 0 application rgw
pool 3 'default.rgw.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode warn last_change 90 flags hashpspool stripe_width 0 application rgw
pool 4 'default.rgw.log' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode warn last_change 91 flags hashpspool stripe_width 0 application rgw

Especial configs:

cat >> ceph.conf<<EOF
 
##########################################
# HYPERCEPH gateways config
##########################################
 
[client.rgw.avmlp-osgw-001.ciberterminal.net]
rgw_frontends = civetweb port=80 num_threads=512
rgw_num_rados_handles = 8
rgw_thread_pool_size = 512
rgw_cache_enabled = true
rgw s3 auth aws4 force boto2 compat = false
rgw resolve cname = true
rgw dns name = clover.ciberterminal.net
 
 
[client.rgw.avmlp-osgw-002.ciberterminal.net]
rgw_frontends = civetweb port=80 num_threads=512
rgw_num_rados_handles = 8
rgw_thread_pool_size = 512
rgw_cache_enabled = true
rgw s3 auth aws4 force boto2 compat = false
rgw resolve cname = true
rgw dns name = clover.ciberterminal.net
 
 
[client.rgw.avmlp-osgw-003.ciberterminal.net]
rgw_frontends = civetweb port=80 num_threads=512
rgw_num_rados_handles = 8
rgw_thread_pool_size = 512
rgw_cache_enabled = true
rgw s3 auth aws4 force boto2 compat = false
rgw resolve cname = true
rgw dns name = clover.ciberterminal.net
 
[client.rgw.avmlp-osgw-004.ciberterminal.net]
rgw_frontends = civetweb port=80 num_threads=512
rgw_num_rados_handles = 8
rgw_thread_pool_size = 512
rgw_cache_enabled = true
rgw s3 auth aws4 force boto2 compat = false
rgw resolve cname = true
rgw dns name = clover.ciberterminal.net
EOF
 
 
ceph-deploy --overwrite-conf config push ${ALLSERVERS}
 
# deploy admin
ceph-deploy admin ${GWSERVERS}

Restart gateways:

salt "avmlp-osgw-*" service.restart ceph-radosgw.target
salt "avmlp-osgw-*" cmd.run "netstat -nap|egrep radosgw|egrep LISTEN"

Increase number of pg's per osd:

# check
ceph tell 'osd.*' config get mon_max_pg_per_osd
# change
ceph tell 'osd.*' config set mon_max_pg_per_osd ${NEW_PG_PER_OSD_NUMBER}
# check again
ceph tell 'osd.*' config get mon_max_pg_per_osd

Change pool “size”:

for i in $(ceph osd pool ls) ; do ceph osd pool set ${i} size 4 ; done
for i in $(ceph osd pool ls) ; do ceph osd pool set ${i} min_size 4 ; done
for i in $(ceph osd pool ls) ; do ceph osd pool set ${i} pg_num 40 ; done
for i in $(ceph osd pool ls) ; do ceph osd pool set ${i} pgp_num 40 ; done
for i in $(ceph osd pool ls) ; do ceph osd pool set ${i} pg_autoscale_mode on ; done
for i in $(ceph osd pool ls) ; do ceph osd pool set ${i} crush_rule ciberterminalRule ; done

Create data+index pool:

export POOL_NAME="default.rgw.buckets.data"
ceph osd pool create ${POOL_NAME} 512 512 replicated ciberterminalRule
ceph osd pool set ${POOL_NAME} pg_autoscale_mode on
ceph osd pool application enable ${POOL_NAME} rgw
ceph osd pool set ${POOL_NAME} compression_algorithm snappy
ceph osd pool set ${POOL_NAME} compression_mode aggressive
ceph osd pool set ${POOL_NAME} compression_min_blob_size 10240
ceph osd pool set ${POOL_NAME} compression_max_blob_size 4194304
 
 
export POOL_NAME="default.rgw.buckets.index"
ceph osd pool create ${POOL_NAME} 128 128 replicated ciberterminalRule
ceph osd pool set ${POOL_NAME} pg_autoscale_mode on
ceph osd pool application enable ${POOL_NAME} rgw

Check CRUSH Maps

Now that there're pools with PG's, is time to check that the CRUSH map is ok:

ceph pg dump | egrep "^[0-9]" | awk '{print $17}'| less
avmlp-osm-001 /home/ceph # ceph pg dump | egrep "^[0-9]" | awk '{print $17}'| head
dumped all
[8,6,19,7]
[19,15,18,0]
[17,13,10,8]
[6,0,7,13]
[18,4,13,15]
[0,6,15,11]
[4,8,17,1]
[3,5,6,10]
[11,17,12,10]
[13,15,6,2]

The PG's must be on both CPD's (even/odd OSD's).

Deploy MDS's

for i in ${MDSSERVERS} ; do ssh ${i} "sudo rm -fv /etc/ceph/ceph.conf" ; done
ceph-deploy mds create ${MDSSERVERS}
 
export POOL_NAME="cephfs_data-ftp"
ceph osd pool create ${POOL_NAME} 256 256 replicated ciberterminalRule
ceph osd pool set ${POOL_NAME} size 4
ceph osd pool set ${POOL_NAME} compression_algorithm snappy
ceph osd pool set ${POOL_NAME} compression_mode aggressive
ceph osd pool set ${POOL_NAME} compression_min_blob_size 10240
ceph osd pool set ${POOL_NAME} compression_max_blob_size 4194304
ceph osd pool set ${POOL_NAME} pg_autoscale_mode on
 
export POOL_NAME="cephfs_metadata-ftp"
ceph osd pool create ${POOL_NAME} 60 60 replicated ciberterminalRule
ceph osd pool set ${POOL_NAME} size 4
ceph osd pool set ${POOL_NAME} pg_autoscale_mode on
 
ceph fs new cephfs cephfs_metadata-ftp cephfs_data-ftp
ceph fs ls
ceph -s
ceph mds stat

Create client:

export CLIENTNAME="avmlp-sftp"
ceph auth get-or-create client.cephfs-${CLIENTNAME} mon "allow r" mds "allow r path=/, allow rw path=/ftp" osd "allow class-read object_prefix rbd_children, allow rw pool=cephfs_data-ftp" > /etc/ceph/client.cephfs-${CLIENTNAME}.keyring
# ceph auth caps client.cephfs-avmlp-sftp mon "allow r" mds "allow r path=/, allow rw path=/ftp" osd "allow class-read object_prefix rbd_children, allow rw pool=cephfs_data-ftp"

radosgw User creation

export THEUSERNAME="dodger"
radosgw-admin user create --uid="${THEUSERNAME}" --display-name="${THEUSERNAME} user"
radosgw-admin subuser create --uid=${THEUSERNAME} --subuser=${THEUSERNAME} --access=full
radosgw-admin key create --subuser=${THEUSERNAME}:swift --key-type=swift --gen-secret
radosgw-admin user modify --uid=${THEUSERNAME} --max-buckets=1
radosgw-admin user modify --uid=${THEUSERNAME} --max-buckets=10

monitoring

export THEUSERNAME="centreon"
sudo radosgw-admin user create --uid="${THEUSERNAME}" --display-name="${THEUSERNAME} user"
sudo radosgw-admin user modify --uid=${THEUSERNAME} --max-buckets=1
export QUOTASCOPE="user"
sudo radosgw-admin quota set --quota-scope=${QUOTASCOPE} --uid=${THEUSERNAME} --max-objects=10

Added mds's to centreon:

for ((x=1; x<5; x++)); do HOSTNAME="avmlp-osfs-0$(printf "%02d" ${x})" ; HOSTIP=$(dig ${HOSTNAME}.ciberterminal.net | egrep -v "^;|^$"| awk '{print $5}') ; centreon -u ${CENTUSER} -p ${CENTPASSWORD} -o HOST -a ADD -v "${HOSTNAME};${HOSTNAME}.ciberterminal.net;${HOSTIP};ciberterminal-CEPH-osfs;Central;CEPH-PRO"  ; let x++ ; done
for ((x=2; x<5; x++)); do HOSTNAME="avmlp-osfs-0$(printf "%02d" ${x})" ; HOSTIP=$(dig ${HOSTNAME}.ciberterminal.net | egrep -v "^;|^$"| awk '{print $5}') ; centreon -u ${CENTUSER} -p ${CENTPASSWORD} -o HOST -a ADD -v "${HOSTNAME};${HOSTNAME}.ciberterminal.net;${HOSTIP};ciberterminal-CEPH-osfs;Central2;CEPH-PRO"  ; let x++ ; done
for ((x=1; x<5; x++)); do HOSTNAME="avmlp-osfs-0$(printf "%02d" ${x})" ; echo centreon -u ${CENTUSER} -p ${CENTPASSWORD} -o HOST -a applytpl -v "${HOSTNAME}" ; done

Dashboard

# only for monitors
export THESERVER="a*-osm-00*.ciberterminal.net"
salt "${THESERVER}" pkg.install ceph-mgr-dashboard
ceph mgr module enable dashboard
ceph dashboard create-self-signed-cert
ceph mgr services
ceph dashboard ac-user-create bofher **** administrator
radosgw-admin user create --uid=cephdashboard --display-name=cephdashboard --system
ceph dashboard set-rgw-api-access-key $(cat access_cephdash)
ceph dashboard set-rgw-api-secret-key $(cat secret_cephdash)
ceph dashboard set-rgw-api-host ceph_endpoint.ciberterminal.net
ceph dashboard set-rgw-api-port 80
ceph mgr module disable dashboard
ceph mgr module enable dashboard

Enable pg_autoscaler

ceph mgr module enable pg_autoscaler

Additional modules

# only for monitors
export THESERVER="a*-osm-00*.ciberterminal.net"
for i in ceph-mgr-diskprediction-cloud.noarch ceph-mgr-diskprediction-local.noarch ceph-mgr-rook.noarch ceph-mgr-ssh.noarch ceph-medic ceph-grafana-dashboards.noarch ; do salt "${THESERVER}" pkg.install ${i} ; done

Maybe you'll need to performa a hard reset of the systemd service… If you see that:

acclm-osm-001 /home/ceph/ceph-deploy # systemctl start ceph-mgr@acclm-osm-001.ciberterminal.net.service                                                                                                             
Job for ceph-mgr@acclm-osm-001.ciberterminal.net.service failed because start of the service was attempted too often. See "systemctl status ceph-mgr@acclm-osm-001.ciberterminal.net.service" and "journalctl -xe" for details.
To force a start use "systemctl reset-failed ceph-mgr@acclm-osm-001.ciberterminal.net.service" followed by "systemctl start ceph-mgr@acclm-osm-001.ciberterminal.net.service" again.

I used that line:

for i in 1 2 3 4 ; do salt "acclm-osm-00${i}*" cmd.run "systemctl reset-failed ceph-mgr@acclm-osm-00${i}.ciberterminal.net.service" ; salt "acclm-osm-00${i}*" cmd.run "systemctl start ceph-mgr@acclm-osm-00${i}.ciberterminal.net.service" ; done
linux/ceph/clover_deploy.txt · Last modified: 2022/02/11 11:36 (external edit)