====== Deploying clover ======

^  Documentation  ^|
^Name:| Deploying clover |
^Description:| raw commands of a live CEPH storage cluster deployment |
^Modification date :|29/07/2018|
^Owner:|dodger|
^Notify changes to:|Owner |
^Tags:|ceph, object storage |
^Scalate to:|The_fucking_bofh|


====== RAW  ======

===== CloneWars deploy clover =====

<code bash>
for i in 01 03 05 07 09 11 13 15 17 19 ; do bash /home/bofher/scripts/CloneWars/CloneWars.sh -c nuciberterminal -h AVMLP-OSD-0${i} -i 10.20.54.1${i} -d 2048GB -m 20 -O -r 4096 -v 2 -o 2 -F ; rm -fv /etc/salt/pki/master/minions/avmlp-osd-0${i}.ciberterminal.net ; done
for i in 02 04 06 08 10 12 14 16 18 20 ; do bash /home/bofher/scripts/CloneWars/CloneWars.sh -c nuciberterminal2 -h AVMLP-OSD-0${i} -i 10.20.54.1${i} -d 2048GB -m 20 -O -r 4096 -v 2 -o 2 -F ; rm -fv /etc/salt/pki/master/minions/avmlp-osd-0${i}.ciberterminal.net ; done
for i in 01 03 ; do  bash /home/bofher/scripts/CloneWars/CloneWars.sh -c nuciberterminal -h AVMLP-OSM-0${i} -i 10.20.54.$((${i}+50)) -d 50GB -m 20 -O -r 4096 -v 2 -o 2   -F ;  rm -fv /etc/salt/pki/master/minions/avmlp-osm-0${i}.ciberterminal.net ; done 
for i in 02 04 ; do  bash /home/bofher/scripts/CloneWars/CloneWars.sh -c nuciberterminal2 -h AVMLP-OSM-0${i} -i 10.20.54.$((${i}+50)) -d 50GB -m 20 -O -r 4096 -v 2 -o 2  -F ;  rm -fv /etc/salt/pki/master/minions/avmlp-osm-0${i}.ciberterminal.net ; done
for i in 01 03 ; do  bash /home/bofher/scripts/CloneWars/CloneWars.sh -c nuciberterminal -h AVMLP-OSGW-0${i} -i 10.20.54.$((${i}+10)) -d 50GB -m 20 -O -r 4096 -v 2 -o 2  -F ;  rm -fv /etc/salt/pki/master/minions/avmlp-osgw-0${i}.ciberterminal.net ; done
for i in 02 04 ; do  bash /home/bofher/scripts/CloneWars/CloneWars.sh -c nuciberterminal2 -h AVMLP-OSGW-0${i} -i 10.20.54.$((${i}+10)) -d 50GB -m 20 -O -r 4096 -v 2 -o 2 -F ;  rm -fv /etc/salt/pki/master/minions/avmlp-osgw-0${i}.ciberterminal.net ; done
for i in 01 03 ; do  bash /home/bofher/scripts/CloneWars/CloneWars.sh -c nuciberterminal -h AVMLP-OSFS-0${i} -i 10.20.55.$((${i}+10)) -d 50GB -m 20 -O -r 4096 -v 2 -o 2  -F ;  rm -fv /etc/salt/pki/master/minions/avmlp-osfs-0${i}.ciberterminal.net ; done
for i in 02 04 ; do  bash /home/bofher/scripts/CloneWars/CloneWars.sh -c nuciberterminal2 -h AVMLP-OSFS-0${i} -i 10.20.55.$((${i}+10)) -d 50GB -m 20 -O -r 4096 -v 2 -o 2 -F ;  rm -fv /etc/salt/pki/master/minions/avmlp-osfs-0${i}.ciberterminal.net ; done
</code>



===== salt =====

<code bash>
export THESERVER="avmlp-os*-0*"
salt-key -A
salt "${THESERVER}" state.apply
salt "${THESERVER}" state.apply nsupdate
salt "${THESERVER}" test.ping
salt "${THESERVER}" pkg.install yum-plugin-priorities
salt "${THESERVER}" user.add ceph 1002
salt "${THESERVER}" file.write /etc/sudoers.d/ceph \
"ceph ALL = (root) NOPASSWD:ALL"
salt "${THESERVER}" cmd.run "sudo whoami" runas=ceph
salt "${THESERVER}" cmd.run \
    "ssh-keygen -q -N '' -f /home/ceph/.ssh/id_rsa" \
    runas=ceph
salt "${THESERVER}" cmd.run "cp /home/ceph/.ssh/id_rsa.pub /home/ceph/.ssh/authorized_keys"

export VMNAMESTART="avmlp-os"
salt "${THESERVER}" cmd.run "cat /home/ceph/.ssh/id_rsa.pub" |egrep -v "^${VMNAMESTART}" | sed 's/^[[:space:]]\{1,5\}//g' > all_cluster_nodes.txt
while read LINE ; do salt "${THESERVER}" file.append /home/ceph/.ssh/authorized_keys "${LINE}" ; done < all_cluster_nodes.txt
</code>

===== avmlp-osm-001 =====

<code bash>
cat << EOM > ceph.repo
[Ceph]
name=Ceph packages for \$basearch
baseurl=http://download.ceph.com/rpm-nautilus/el7/\$basearch
enabled=1
gpgcheck=1
type=rpm-md
gpgkey=https://download.ceph.com/keys/release.asc

[Ceph-noarch]
name=Ceph noarch packages
baseurl=http://download.ceph.com/rpm-nautilus/el7/noarch
enabled=1
gpgcheck=1
type=rpm-md
gpgkey=https://download.ceph.com/keys/release.asc

[ceph-source]
name=Ceph source packages
baseurl=http://download.ceph.com/rpm-nautilus/el7/SRPMS
enabled=1
gpgcheck=1
type=rpm-md
gpgkey=https://download.ceph.com/keys/release.asc
EOM


export GWSERVERS="avmlp-osgw-004.ciberterminal.net avmlp-osgw-003.ciberterminal.net avmlp-osgw-001.ciberterminal.net avmlp-osgw-002.ciberterminal.net"
export OSDSERVERS="avmlp-osd-001.ciberterminal.net avmlp-osd-002.ciberterminal.net avmlp-osd-003.ciberterminal.net avmlp-osd-004.ciberterminal.net avmlp-osd-005.ciberterminal.net avmlp-osd-006.ciberterminal.net avmlp-osd-007.ciberterminal.net avmlp-osd-008.ciberterminal.net avmlp-osd-009.ciberterminal.net avmlp-osd-010.ciberterminal.net avmlp-osd-011.ciberterminal.net avmlp-osd-012.ciberterminal.net avmlp-osd-013.ciberterminal.net avmlp-osd-014.ciberterminal.net avmlp-osd-015.ciberterminal.net avmlp-osd-016.ciberterminal.net avmlp-osd-017.ciberterminal.net avmlp-osd-018.ciberterminal.net avmlp-osd-019.ciberterminal.net avmlp-osd-020.ciberterminal.net"
export MONSERVERS="avmlp-osm-002.ciberterminal.net avmlp-osm-001.ciberterminal.net avmlp-osm-004.ciberterminal.net avmlp-osm-003.ciberterminal.net avmlp-osm-005.ciberterminal.net avmlp-osm-006.ciberterminal.net"
export MDSSERVERS="avmlp-osfs-002.ciberterminal.net avmlp-osfs-001.ciberterminal.net avmlp-osfs-004.ciberterminal.net avmlp-osfs-003.ciberterminal.net"
export ALLSERVERS="${GWSERVERS} ${OSDSERVERS} ${MONSERVERS} ${MDSSERVERS}"


for i in ${ALLSERVERS} ; do scp ceph.repo ${i}:/home/ceph/ ; ssh ${i} "sudo cp -pfv /home/ceph/ceph.repo /etc/yum.repos.d/"  ; done
for i in ${ALLSERVERS} ; do ssh ${i} "sudo chown root. /etc/yum.repos.d/ceph.repo"  ; done
for i in ${ALLSERVERS} ; do ssh ${i} "sudo ls -l /etc/yum.repos.d/ceph.repo"  ; done

</code>



===== salt =====

<code bash>
export THESERVER="avmlp-os[mdgf]*-0*"
salt "${THESERVER}" cmd.run "yum -y install ceph-deploy"
</code>

===== avmlp-osm-001 =====
<code bash>
ceph-deploy install ${ALLSERVERS} --repo-url https://download.ceph.com/rpm-nautilus/el7/
</code>

===== salt =====
CHECK
<code bash>
export THESERVER="avmlp-os[mdgf]*-0*"
salt "${THESERVER}" cmd.run "rpm -qa |egrep ceph"
for i in $(salt "${THESERVER}" test.ping| egrep "^a"|awk -F\: '{print $1}'| sort) ; do salt "${i}" cmd.run "sudo ceph --version" ; done
</code>

===== avmlp-osm-001 =====
<code bash>
ceph-deploy new ${MONSERVERS}
ceph-deploy mon create-initial

for i in ${MONSERVERS} ; do ceph-deploy gatherkeys ${i} ; done
ceph-deploy admin ${MONSERVERS}
sudo ceph mon enable-msgr2
ceph-deploy mgr create  ${MONSERVERS}
</code>


osd's
<code bash>
for THESERVER in ${OSDSERVERS} ; do
     echo "${THESERVER}";   
     echo "################### ${THESERVER}: creating disk" ;  
     ceph-deploy osd create ${THESERVER} --data /dev/sdb;  
     echo "################### ${THESERVER}: listing (check) disk" ;    
     ceph-deploy osd list ${THESERVER};  
done
</code>


===== CRUSH =====
<code bash>
mkdir /home/ceph/crush/mod_20190730
cd /home/ceph/crush/mod_20190730
ceph osd getcrushmap  -o crushmap.bin
crushtool  -d crushmap.bin -o crushmap.txt
</code>

<file yaml crushmap.txt>
# begin crush map
tunable choose_local_tries 0
tunable choose_local_fallback_tries 0
tunable choose_total_tries 50
tunable chooseleaf_descend_once 1
tunable chooseleaf_vary_r 1
tunable chooseleaf_stable 1
tunable straw_calc_version 1
tunable allowed_bucket_algs 54

# devices
device 0 osd.0 class hdd
device 1 osd.1 class hdd
device 2 osd.2 class hdd
device 3 osd.3 class hdd
device 4 osd.4 class hdd
device 5 osd.5 class hdd
device 6 osd.6 class hdd
device 7 osd.7 class hdd
device 8 osd.8 class hdd
device 9 osd.9 class hdd
device 10 osd.10 class hdd
device 11 osd.11 class hdd
device 12 osd.12 class hdd
device 13 osd.13 class hdd
device 14 osd.14 class hdd
device 15 osd.15 class hdd
device 16 osd.16 class hdd
device 17 osd.17 class hdd
device 18 osd.18 class hdd
device 19 osd.19 class hdd

# types
type 0 osd
type 1 host
type 2 chassis
type 3 rack
type 4 row
type 5 pdu
type 6 pod
type 7 room
type 8 datacenter
type 9 zone
type 10 region
type 11 root

# buckets
host avmlp-osd-001 {
        id -3           # do not change unnecessarily
        id -4 class hdd         # do not change unnecessarily
        # weight 1.999
        alg straw2
        hash 0  # rjenkins1
        item osd.0 weight 1.999
}

host avmlp-osd-002 {
        id -5           # do not change unnecessarily
        id -6 class hdd         # do not change unnecessarily
        # weight 1.999
        alg straw2
        hash 0  # rjenkins1
        item osd.1 weight 1.999
}
host avmlp-osd-003 {
        id -7           # do not change unnecessarily
        id -8 class hdd         # do not change unnecessarily
        # weight 1.999
        alg straw2
        hash 0  # rjenkins1
        item osd.2 weight 1.999
}
host avmlp-osd-004 {
        id -9           # do not change unnecessarily
        id -10 class hdd                # do not change unnecessarily
        # weight 1.999
        alg straw2
        hash 0  # rjenkins1
        item osd.3 weight 1.999
}
host avmlp-osd-005 {
        id -11          # do not change unnecessarily
        id -12 class hdd                # do not change unnecessarily
        # weight 1.999
        alg straw2
        hash 0  # rjenkins1
        item osd.4 weight 1.999
}
host avmlp-osd-006 {
        id -13          # do not change unnecessarily
        id -14 class hdd                # do not change unnecessarily
        # weight 1.999
        alg straw2
        hash 0  # rjenkins1
        item osd.5 weight 1.999
}
host avmlp-osd-007 {
        id -15          # do not change unnecessarily
        id -16 class hdd                # do not change unnecessarily
        # weight 1.999
        alg straw2
        hash 0  # rjenkins1
        item osd.6 weight 1.999
}
host avmlp-osd-008 {
        id -17          # do not change unnecessarily
        id -18 class hdd                # do not change unnecessarily
        # weight 1.999
        alg straw2
        hash 0  # rjenkins1
        item osd.7 weight 1.999
}
host avmlp-osd-009 {
        id -19          # do not change unnecessarily
        id -20 class hdd                # do not change unnecessarily
        # weight 1.999
        alg straw2
        hash 0  # rjenkins1
        item osd.8 weight 1.999
}
host avmlp-osd-010 {
        id -21          # do not change unnecessarily
        id -22 class hdd                # do not change unnecessarily
        # weight 1.999
        alg straw2
        hash 0  # rjenkins1
        item osd.9 weight 1.999
}
host avmlp-osd-011 {
        id -23          # do not change unnecessarily
        id -24 class hdd                # do not change unnecessarily
        # weight 1.999
        alg straw2
        hash 0  # rjenkins1
        item osd.10 weight 1.999
}


host avmlp-osd-012 {
        id -25          # do not change unnecessarily
        id -26 class hdd                # do not change unnecessarily
        # weight 1.999
        alg straw2
        hash 0  # rjenkins1
        item osd.11 weight 1.999
}
host avmlp-osd-013 {
        id -27          # do not change unnecessarily
        id -28 class hdd                # do not change unnecessarily
        # weight 1.999
        alg straw2
        hash 0  # rjenkins1
        item osd.12 weight 1.999
}
host avmlp-osd-014 {
        id -29          # do not change unnecessarily
        id -30 class hdd                # do not change unnecessarily
        # weight 1.999
        alg straw2
        hash 0  # rjenkins1
        item osd.13 weight 1.999
}
host avmlp-osd-015 {
        id -31          # do not change unnecessarily
        id -32 class hdd                # do not change unnecessarily
        # weight 1.999
        alg straw2
        hash 0  # rjenkins1
        item osd.14 weight 1.999
}
host avmlp-osd-016 {
        id -33          # do not change unnecessarily
        id -34 class hdd                # do not change unnecessarily
        # weight 1.999
        alg straw2
        hash 0  # rjenkins1
        item osd.15 weight 1.999
}
host avmlp-osd-017 {
        id -35          # do not change unnecessarily
        id -36 class hdd                # do not change unnecessarily
        # weight 1.999
        alg straw2
        hash 0  # rjenkins1
        item osd.16 weight 1.999
}
host avmlp-osd-018 {
        id -37          # do not change unnecessarily
        id -38 class hdd                # do not change unnecessarily
        # weight 1.999
        alg straw2
        hash 0  # rjenkins1
        item osd.17 weight 1.999
}
host avmlp-osd-019 {
        id -39          # do not change unnecessarily
        id -40 class hdd                # do not change unnecessarily
        # weight 1.999
        alg straw2
        hash 0  # rjenkins1
        item osd.18 weight 1.999
}
host avmlp-osd-020 {
        id -41          # do not change unnecessarily
        id -42 class hdd                # do not change unnecessarily
        # weight 1.999
        alg straw2
        hash 0  # rjenkins1
        item osd.19 weight 1.999
}

rack nuciberterminal {
        # weight 19.990
        alg straw2
        hash 0  # rjenkins1

        item avmlp-osd-001 weight 1.999
        item avmlp-osd-003 weight 1.999
        item avmlp-osd-005 weight 1.999
        item avmlp-osd-007 weight 1.999
        item avmlp-osd-009 weight 1.999
        item avmlp-osd-011 weight 1.999
        item avmlp-osd-013 weight 1.999
        item avmlp-osd-015 weight 1.999
        item avmlp-osd-017 weight 1.999
        item avmlp-osd-019 weight 1.999
}
 
rack nuciberterminal2 {
        # weight 19.990
        alg straw2
        hash 0  # rjenkins1

        item avmlp-osd-002 weight 1.999
        item avmlp-osd-004 weight 1.999
        item avmlp-osd-006 weight 1.999
        item avmlp-osd-008 weight 1.999
        item avmlp-osd-010 weight 1.999
        item avmlp-osd-012 weight 1.999
        item avmlp-osd-014 weight 1.999
        item avmlp-osd-016 weight 1.999
        item avmlp-osd-018 weight 1.999
        item avmlp-osd-020 weight 1.999
} 

datacenter mediapro {
        # weight 0.000
        alg straw2
        hash 0  # rjenkins1
        item nuciberterminal2 weight 19.990
}

datacenter itconic {
        # weight 0.000
        alg straw2
        hash 0  # rjenkins1
        item nuciberterminal weight 19.990
}

root default {
        id -1           # do not change unnecessarily
        id -2 class hdd         # do not change unnecessarily
        # weight 39.980
        alg straw2
        hash 0  # rjenkins1
        item mediapro weight 19.990
        item itconic weight 19.990
}

# rules
rule replicated_rule {
        id 0
        type replicated
        min_size 2
        max_size 10
        step take default
        step chooseleaf firstn 0 type datacenter
        step emit
}

rule ciberterminalRule {
        id 1 
        type replicated
        min_size 2
        max_size 10
        # begin iterating in the "root" of the crush tree
        step take default
        step choose firstn 4 type datacenter
        step chooseleaf firstn 2 type host
        step emit
}

# end crush map
</file>

<code bash>
crushtool -c crushmap.txt -o crushmap_new.bin
crushtool --test -i crushmap_new.bin --show-utilization --rule 1 --num-rep=4
crushtool --test -i crushmap_new.bin --show-choose-tries --rule 1 --num-rep=4
</code>


<code bash>
avmlp-osm-001 /home/ceph/crush/mod_20190730 # ceph osd crush tree
ID  CLASS WEIGHT   TYPE NAME                      
 -1       39.97998 root default                   
-46       19.98999     datacenter itconic         
-43       19.98999         rack nuciberterminal           
 -3        1.99899             host avmlp-osd-001 
  0   hdd  1.99899                 osd.0          
 -7        1.99899             host avmlp-osd-003 
  2   hdd  1.99899                 osd.2          
-11        1.99899             host avmlp-osd-005 
  4   hdd  1.99899                 osd.4          
-15        1.99899             host avmlp-osd-007 
  6   hdd  1.99899                 osd.6          
-19        1.99899             host avmlp-osd-009 
  8   hdd  1.99899                 osd.8          
-23        1.99899             host avmlp-osd-011 
 10   hdd  1.99899                 osd.10         
-27        1.99899             host avmlp-osd-013 
 12   hdd  1.99899                 osd.12         
-31        1.99899             host avmlp-osd-015 
 14   hdd  1.99899                 osd.14         
-35        1.99899             host avmlp-osd-017 
 16   hdd  1.99899                 osd.16         
-39        1.99899             host avmlp-osd-019 
 18   hdd  1.99899                 osd.18         
-45       19.98999     datacenter mediapro        
-44       19.98999         rack nuciberterminal2          
 -5        1.99899             host avmlp-osd-002 
  1   hdd  1.99899                 osd.1          
 -9        1.99899             host avmlp-osd-004 
  3   hdd  1.99899                 osd.3          
-13        1.99899             host avmlp-osd-006 
  5   hdd  1.99899                 osd.5          
-17        1.99899             host avmlp-osd-008 
  7   hdd  1.99899                 osd.7          
-21        1.99899             host avmlp-osd-010 
  9   hdd  1.99899                 osd.9          
-25        1.99899             host avmlp-osd-012 
 11   hdd  1.99899                 osd.11         
-29        1.99899             host avmlp-osd-014 
 13   hdd  1.99899                 osd.13         
-33        1.99899             host avmlp-osd-016 
 15   hdd  1.99899                 osd.15         
-37        1.99899             host avmlp-osd-018 
 17   hdd  1.99899                 osd.17         
-41        1.99899             host avmlp-osd-020 
 19   hdd  1.99899                 osd.19         
</code>


===== recovery tunning =====

<code bash>
ceph tell 'osd.*' config set osd_recovery_max_single_start 1000
ceph tell 'osd.*' config set osd_max_backfills 100
ceph tell 'osd.*' config set osd_recovery_max_active 300
ceph tell 'osd.*' config set osd_recovery_max_single_start 1000
ceph tell 'osd.*' config set osd_backfill_scan_max 51200
ceph tell 'osd.*' config set osd_backfill_scan_min 6400
ceph tell 'osd.*' config set osd_recovery_sleep_hdd 0
</code>

[[linux:ceph:tunning_recovery|Tunning CEPH recovery]]
===== radosgw =====

<code bash>
ceph-deploy rgw create ${GWSERVERS}
</code>


check:
<code bash>
avmlp-osm-001 /home/ceph # ceph -s
  cluster:
    id:     a3a799ce-f1d3-4230-a915-06e988fee767
    health: HEALTH_WARN
            Degraded data redundancy: 188/564 objects degraded (33.333%), 18 pgs degraded
            too few PGs per OSD (3 < min 30)
 
  services:
    mon: 6 daemons, quorum avmlp-osm-001,avmlp-osm-002,avmlp-osm-003,avmlp-osm-004,avmlp-osm-005,avmlp-osm-006 (age 3h)
    mgr: avmlp-osm-002.ciberterminal.net(active, since 4h), standbys: avmlp-osm-004.ciberterminal.net, avmlp-osm-003.ciberterminal.net, avmlp-osm-001.ciberterminal.net, avmlp-osm-005.ciberterminal.net, avmlp-osm-006.ciberterminal.net
    osd: 20 osds: 20 up (since 4h), 20 in (since 4h)
    rgw: 4 daemons active (avmlp-osgw-001.ciberterminal.net, avmlp-osgw-002.ciberterminal.net, avmlp-osgw-003.ciberterminal.net, avmlp-osgw-004.ciberterminal.net)
 
  data:
    pools:   4 pools, 32 pgs
    objects: 188 objects, 2.0 KiB
    usage:   20 GiB used, 40 TiB / 40 TiB avail
    pgs:     188/564 objects degraded (33.333%)
             18 active+undersized+degraded
             14 active+undersized
 
  io:
    client:   60 KiB/s rd, 0 B/s wr, 68 op/s rd, 45 op/s wr
 
avmlp-osm-001 /home/ceph # ceph osd pool ls detail
pool 1 '.rgw.root' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode warn last_change 85 flags hashpspool stripe_width 0 application rgw
pool 2 'default.rgw.control' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode warn last_change 87 flags hashpspool stripe_width 0 application rgw
pool 3 'default.rgw.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode warn last_change 90 flags hashpspool stripe_width 0 application rgw
pool 4 'default.rgw.log' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode warn last_change 91 flags hashpspool stripe_width 0 application rgw
</code>


Especial configs:
<code bash>
cat >> ceph.conf<<EOF

##########################################
# HYPERCEPH gateways config
##########################################

[client.rgw.avmlp-osgw-001.ciberterminal.net]
rgw_frontends = civetweb port=80 num_threads=512
rgw_num_rados_handles = 8
rgw_thread_pool_size = 512
rgw_cache_enabled = true
rgw s3 auth aws4 force boto2 compat = false
rgw resolve cname = true
rgw dns name = clover.ciberterminal.net


[client.rgw.avmlp-osgw-002.ciberterminal.net]
rgw_frontends = civetweb port=80 num_threads=512
rgw_num_rados_handles = 8
rgw_thread_pool_size = 512
rgw_cache_enabled = true
rgw s3 auth aws4 force boto2 compat = false
rgw resolve cname = true
rgw dns name = clover.ciberterminal.net


[client.rgw.avmlp-osgw-003.ciberterminal.net]
rgw_frontends = civetweb port=80 num_threads=512
rgw_num_rados_handles = 8
rgw_thread_pool_size = 512
rgw_cache_enabled = true
rgw s3 auth aws4 force boto2 compat = false
rgw resolve cname = true
rgw dns name = clover.ciberterminal.net

[client.rgw.avmlp-osgw-004.ciberterminal.net]
rgw_frontends = civetweb port=80 num_threads=512
rgw_num_rados_handles = 8
rgw_thread_pool_size = 512
rgw_cache_enabled = true
rgw s3 auth aws4 force boto2 compat = false
rgw resolve cname = true
rgw dns name = clover.ciberterminal.net
EOF


ceph-deploy --overwrite-conf config push ${ALLSERVERS}

# deploy admin
ceph-deploy admin ${GWSERVERS}
</code>

Restart gateways:
<code bash>
salt "avmlp-osgw-*" service.restart ceph-radosgw.target
salt "avmlp-osgw-*" cmd.run "netstat -nap|egrep radosgw|egrep LISTEN"
</code>

Increase number of pg's per osd:
<code bash>
# check
ceph tell 'osd.*' config get mon_max_pg_per_osd
# change
ceph tell 'osd.*' config set mon_max_pg_per_osd ${NEW_PG_PER_OSD_NUMBER}
# check again
ceph tell 'osd.*' config get mon_max_pg_per_osd
</code>


Change pool "size":
<code bash>
for i in $(ceph osd pool ls) ; do ceph osd pool set ${i} size 4 ; done
for i in $(ceph osd pool ls) ; do ceph osd pool set ${i} min_size 4 ; done
for i in $(ceph osd pool ls) ; do ceph osd pool set ${i} pg_num 40 ; done
for i in $(ceph osd pool ls) ; do ceph osd pool set ${i} pgp_num 40 ; done
for i in $(ceph osd pool ls) ; do ceph osd pool set ${i} pg_autoscale_mode on ; done
for i in $(ceph osd pool ls) ; do ceph osd pool set ${i} crush_rule ciberterminalRule ; done
</code>

Create data+index pool:
<code bash>
export POOL_NAME="default.rgw.buckets.data"
ceph osd pool create ${POOL_NAME} 512 512 replicated ciberterminalRule
ceph osd pool set ${POOL_NAME} pg_autoscale_mode on
ceph osd pool application enable ${POOL_NAME} rgw
ceph osd pool set ${POOL_NAME} compression_algorithm snappy
ceph osd pool set ${POOL_NAME} compression_mode aggressive
ceph osd pool set ${POOL_NAME} compression_min_blob_size 10240
ceph osd pool set ${POOL_NAME} compression_max_blob_size 4194304


export POOL_NAME="default.rgw.buckets.index"
ceph osd pool create ${POOL_NAME} 128 128 replicated ciberterminalRule
ceph osd pool set ${POOL_NAME} pg_autoscale_mode on
ceph osd pool application enable ${POOL_NAME} rgw
</code>

===== Check CRUSH Maps =====

Now that there're pools with PG's, is time to check that the CRUSH map is ok:
<code bash>
ceph pg dump | egrep "^[0-9]" | awk '{print $17}'| less
</code>

<code bash>
avmlp-osm-001 /home/ceph # ceph pg dump | egrep "^[0-9]" | awk '{print $17}'| head
dumped all
[8,6,19,7]
[19,15,18,0]
[17,13,10,8]
[6,0,7,13]
[18,4,13,15]
[0,6,15,11]
[4,8,17,1]
[3,5,6,10]
[11,17,12,10]
[13,15,6,2]
</code>

The PG's must be on both CPD's (even/odd OSD's).

===== Deploy MDS's =====

<code bash>
for i in ${MDSSERVERS} ; do ssh ${i} "sudo rm -fv /etc/ceph/ceph.conf" ; done
ceph-deploy mds create ${MDSSERVERS}

export POOL_NAME="cephfs_data-ftp"
ceph osd pool create ${POOL_NAME} 256 256 replicated ciberterminalRule
ceph osd pool set ${POOL_NAME} size 4
ceph osd pool set ${POOL_NAME} compression_algorithm snappy
ceph osd pool set ${POOL_NAME} compression_mode aggressive
ceph osd pool set ${POOL_NAME} compression_min_blob_size 10240
ceph osd pool set ${POOL_NAME} compression_max_blob_size 4194304
ceph osd pool set ${POOL_NAME} pg_autoscale_mode on

export POOL_NAME="cephfs_metadata-ftp"
ceph osd pool create ${POOL_NAME} 60 60 replicated ciberterminalRule
ceph osd pool set ${POOL_NAME} size 4
ceph osd pool set ${POOL_NAME} pg_autoscale_mode on
 
ceph fs new cephfs cephfs_metadata-ftp cephfs_data-ftp
ceph fs ls
ceph -s
ceph mds stat
</code>

Create client:
<code bash>
export CLIENTNAME="avmlp-sftp"
ceph auth get-or-create client.cephfs-${CLIENTNAME} mon "allow r" mds "allow r path=/, allow rw path=/ftp" osd "allow class-read object_prefix rbd_children, allow rw pool=cephfs_data-ftp" > /etc/ceph/client.cephfs-${CLIENTNAME}.keyring
# ceph auth caps client.cephfs-avmlp-sftp mon "allow r" mds "allow r path=/, allow rw path=/ftp" osd "allow class-read object_prefix rbd_children, allow rw pool=cephfs_data-ftp"
</code>


===== radosgw User creation =====
<code bash>
export THEUSERNAME="dodger"
radosgw-admin user create --uid="${THEUSERNAME}" --display-name="${THEUSERNAME} user"
radosgw-admin subuser create --uid=${THEUSERNAME} --subuser=${THEUSERNAME} --access=full
radosgw-admin key create --subuser=${THEUSERNAME}:swift --key-type=swift --gen-secret
radosgw-admin user modify --uid=${THEUSERNAME} --max-buckets=1
radosgw-admin user modify --uid=${THEUSERNAME} --max-buckets=10
</code>

===== monitoring =====
<code bash>

export THEUSERNAME="centreon"
sudo radosgw-admin user create --uid="${THEUSERNAME}" --display-name="${THEUSERNAME} user"
sudo radosgw-admin user modify --uid=${THEUSERNAME} --max-buckets=1
export QUOTASCOPE="user"
sudo radosgw-admin quota set --quota-scope=${QUOTASCOPE} --uid=${THEUSERNAME} --max-objects=10
</code>

Added mds's to centreon:
<code bash>
for ((x=1; x<5; x++)); do HOSTNAME="avmlp-osfs-0$(printf "%02d" ${x})" ; HOSTIP=$(dig ${HOSTNAME}.ciberterminal.net | egrep -v "^;|^$"| awk '{print $5}') ; centreon -u ${CENTUSER} -p ${CENTPASSWORD} -o HOST -a ADD -v "${HOSTNAME};${HOSTNAME}.ciberterminal.net;${HOSTIP};ciberterminal-CEPH-osfs;Central;CEPH-PRO"  ; let x++ ; done
for ((x=2; x<5; x++)); do HOSTNAME="avmlp-osfs-0$(printf "%02d" ${x})" ; HOSTIP=$(dig ${HOSTNAME}.ciberterminal.net | egrep -v "^;|^$"| awk '{print $5}') ; centreon -u ${CENTUSER} -p ${CENTPASSWORD} -o HOST -a ADD -v "${HOSTNAME};${HOSTNAME}.ciberterminal.net;${HOSTIP};ciberterminal-CEPH-osfs;Central2;CEPH-PRO"  ; let x++ ; done
for ((x=1; x<5; x++)); do HOSTNAME="avmlp-osfs-0$(printf "%02d" ${x})" ; echo centreon -u ${CENTUSER} -p ${CENTPASSWORD} -o HOST -a applytpl -v "${HOSTNAME}" ; done
</code>

===== Dashboard =====
<code bash>
# only for monitors
export THESERVER="a*-osm-00*.ciberterminal.net"
salt "${THESERVER}" pkg.install ceph-mgr-dashboard
</code>


<code bash>
ceph mgr module enable dashboard
ceph dashboard create-self-signed-cert
ceph mgr services
ceph dashboard ac-user-create bofher **** administrator
</code>

<code bash>
radosgw-admin user create --uid=cephdashboard --display-name=cephdashboard --system
ceph dashboard set-rgw-api-access-key $(cat access_cephdash)
ceph dashboard set-rgw-api-secret-key $(cat secret_cephdash)
ceph dashboard set-rgw-api-host ceph_endpoint.ciberterminal.net
ceph dashboard set-rgw-api-port 80
ceph mgr module disable dashboard
ceph mgr module enable dashboard
</code>
===== Enable pg_autoscaler =====
<code bash>
ceph mgr module enable pg_autoscaler
</code>


===== Additional modules =====
<code bash>
# only for monitors
export THESERVER="a*-osm-00*.ciberterminal.net"
for i in ceph-mgr-diskprediction-cloud.noarch ceph-mgr-diskprediction-local.noarch ceph-mgr-rook.noarch ceph-mgr-ssh.noarch ceph-medic ceph-grafana-dashboards.noarch ; do salt "${THESERVER}" pkg.install ${i} ; done
</code>


Maybe you'll need to performa a hard reset of the systemd service... If you see that:
<code bash>
acclm-osm-001 /home/ceph/ceph-deploy # systemctl start ceph-mgr@acclm-osm-001.ciberterminal.net.service                                                                                                             
Job for ceph-mgr@acclm-osm-001.ciberterminal.net.service failed because start of the service was attempted too often. See "systemctl status ceph-mgr@acclm-osm-001.ciberterminal.net.service" and "journalctl -xe" for details.
To force a start use "systemctl reset-failed ceph-mgr@acclm-osm-001.ciberterminal.net.service" followed by "systemctl start ceph-mgr@acclm-osm-001.ciberterminal.net.service" again.
</code>

I used that line:
<code bash>
for i in 1 2 3 4 ; do salt "acclm-osm-00${i}*" cmd.run "systemctl reset-failed ceph-mgr@acclm-osm-00${i}.ciberterminal.net.service" ; salt "acclm-osm-00${i}*" cmd.run "systemctl start ceph-mgr@acclm-osm-00${i}.ciberterminal.net.service" ; done
</code>