ceph:modifying_crush_map
Differences
This shows you the differences between two versions of the page.
Next revision | Previous revisionLast revisionBoth sides next revision | ||
ceph:modifying_crush_map [2019/07/18 07:17] – external edit 127.0.0.1 | ceph:modifying_crush_map [2019/07/31 10:53] – dodger | ||
---|---|---|---|
Line 31: | Line 31: | ||
Example: | Example: | ||
< | < | ||
- | avmlp-osm-001 ~ # ceph osd crush tree | + | proceph-osm-001 ~ # ceph osd crush tree |
ID CLASS WEIGHT | ID CLASS WEIGHT | ||
| | ||
- | | + | |
0 | 0 | ||
- | | + | |
1 | 1 | ||
- | | + | |
2 | 2 | ||
- | | + | |
3 | 3 | ||
- | -11 1.99899 | + | -11 1.99899 |
4 | 4 | ||
- | -13 1.99899 | + | -13 1.99899 |
5 | 5 | ||
- | -15 1.99899 | + | -15 1.99899 |
6 | 6 | ||
- | -17 1.99899 | + | -17 1.99899 |
7 | 7 | ||
- | -19 1.99899 | + | -19 1.99899 |
8 | 8 | ||
- | -21 1.99899 | + | -21 1.99899 |
9 | 9 | ||
- | -23 1.99899 | + | -23 1.99899 |
| | ||
- | -25 1.99899 | + | -25 1.99899 |
| | ||
- | -27 1.99899 | + | -27 1.99899 |
| | ||
- | -29 1.99899 | + | -29 1.99899 |
| | ||
- | -31 1.99899 | + | -31 1.99899 |
| | ||
- | -33 1.99899 | + | -33 1.99899 |
| | ||
- | -35 1.99899 | + | -35 1.99899 |
| | ||
- | -37 1.99899 | + | -37 1.99899 |
| | ||
- | -39 1.99899 | + | -39 1.99899 |
| | ||
- | -41 1.99899 | + | -41 1.99899 |
| | ||
</ | </ | ||
Line 80: | Line 80: | ||
This command export the map from the running cluster, the file is **binary**: | This command export the map from the running cluster, the file is **binary**: | ||
<code bash> | <code bash> | ||
- | ceph osd getcrushmap | + | ceph osd getcrushmap |
</ | </ | ||
===== Convert the map to text ===== | ===== Convert the map to text ===== | ||
<code bash> | <code bash> | ||
- | crushtool | + | crushtool |
</ | </ | ||
Line 98: | Line 98: | ||
^ rack | < | ^ rack | < | ||
root default { | root default { | ||
- | | + | # |
- | id -2 class hdd # do not change unnecessarily | + | # id -2 class hdd # do not change unnecessarily |
# weight 39.980 | # weight 39.980 | ||
alg straw2 | alg straw2 | ||
hash 0 # rjenkins1 | hash 0 # rjenkins1 | ||
- | item itconic | + | item CPD1 weight 0.000 |
- | item mediacloud | + | item CPD2 weight 0.000 |
} | } | ||
</ | </ | ||
^ datacenter | < | ^ datacenter | < | ||
- | datacenter | + | datacenter |
- | id -44 # do not change unnecessarily | + | # id -44 # do not change unnecessarily |
- | id -46 class hdd # do not change unnecessarily | + | # id -46 class hdd # do not change unnecessarily |
# weight 0.000 | # weight 0.000 | ||
alg straw2 | alg straw2 | ||
hash 0 # rjenkins1 | hash 0 # rjenkins1 | ||
- | item datacenter01 | + | item ciberterminal |
} | } | ||
</ | </ | ||
Line 120: | Line 120: | ||
So each " | So each " | ||
+ | <WRAP center round important 60%> | ||
+ | Negative id's from entities **MUST BE REMOVED!!!** \\ | ||
+ | That's why I've left commented: | ||
+ | < | ||
+ | # id -44 # do not change unnecessarily | ||
+ | </ | ||
+ | </ | ||
+ | |||
+ | Sample CRUSH map: | ||
+ | <file crusmap.txt> | ||
+ | # begin crush map | ||
+ | tunable choose_local_tries 0 | ||
+ | tunable choose_local_fallback_tries 0 | ||
+ | tunable choose_total_tries 50 | ||
+ | tunable chooseleaf_descend_once 1 | ||
+ | tunable chooseleaf_vary_r 1 | ||
+ | tunable chooseleaf_stable 1 | ||
+ | tunable straw_calc_version 1 | ||
+ | tunable allowed_bucket_algs 54 | ||
+ | |||
+ | # devices | ||
+ | device 0 osd.0 class hdd | ||
+ | device 1 osd.1 class hdd | ||
+ | device 2 osd.2 class hdd | ||
+ | device 3 osd.3 class hdd | ||
+ | device 4 osd.4 class hdd | ||
+ | device 5 osd.5 class hdd | ||
+ | device 6 osd.6 class hdd | ||
+ | device 7 osd.7 class hdd | ||
+ | device 8 osd.8 class hdd | ||
+ | device 9 osd.9 class hdd | ||
+ | device 10 osd.10 class hdd | ||
+ | device 11 osd.11 class hdd | ||
+ | device 12 osd.12 class hdd | ||
+ | device 13 osd.13 class hdd | ||
+ | device 14 osd.14 class hdd | ||
+ | device 15 osd.15 class hdd | ||
+ | device 16 osd.16 class hdd | ||
+ | device 17 osd.17 class hdd | ||
+ | device 18 osd.18 class hdd | ||
+ | device 19 osd.19 class hdd | ||
+ | |||
+ | # types | ||
+ | type 0 osd | ||
+ | type 1 host | ||
+ | type 2 chassis | ||
+ | type 3 rack | ||
+ | type 4 row | ||
+ | type 5 pdu | ||
+ | type 6 pod | ||
+ | type 7 room | ||
+ | type 8 datacenter | ||
+ | type 9 zone | ||
+ | type 10 region | ||
+ | type 11 root | ||
+ | |||
+ | # buckets | ||
+ | |||
+ | host ciberterminal_cluster { | ||
+ | # weight 19.990 | ||
+ | alg straw2 | ||
+ | hash 0 # rjenkins1 | ||
+ | |||
+ | item osd.0 weight 1.999 | ||
+ | item osd.2 weight 1.999 | ||
+ | item osd.4 weight 1.999 | ||
+ | item osd.6 weight 1.999 | ||
+ | item osd.8 weight 1.999 | ||
+ | item osd.10 weight 1.999 | ||
+ | item osd.12 weight 1.999 | ||
+ | item osd.14 weight 1.999 | ||
+ | item osd.16 weight 1.999 | ||
+ | item osd.18 weight 1.999 | ||
+ | } | ||
+ | |||
+ | host ciberterminal2_cluster { | ||
+ | # weight 19.990 | ||
+ | alg straw2 | ||
+ | hash 0 # rjenkins1 | ||
+ | |||
+ | item osd.1 weight 1.999 | ||
+ | item osd.3 weight 1.999 | ||
+ | item osd.5 weight 1.999 | ||
+ | item osd.7 weight 1.999 | ||
+ | item osd.9 weight 1.999 | ||
+ | item osd.11 weight 1.999 | ||
+ | item osd.13 weight 1.999 | ||
+ | item osd.15 weight 1.999 | ||
+ | item osd.17 weight 1.999 | ||
+ | item osd.19 weight 1.999 | ||
+ | |||
+ | } | ||
+ | |||
+ | root default { | ||
+ | # weight 0.000 | ||
+ | alg straw2 | ||
+ | hash 0 # rjenkins1 | ||
+ | item ciberterminal_cluster weight 1.000 | ||
+ | item ciberterminal2_cluster weight 1.000 | ||
+ | } | ||
+ | |||
+ | # rules | ||
+ | rule replicated_rule { | ||
+ | id 0 | ||
+ | type replicated | ||
+ | min_size 1 | ||
+ | max_size 10 | ||
+ | step take default | ||
+ | step chooseleaf firstn 3 type host | ||
+ | step emit | ||
+ | } | ||
+ | rule ciberterminalRule { | ||
+ | id 1 | ||
+ | type replicated | ||
+ | min_size 1 | ||
+ | max_size 10 | ||
+ | step take default | ||
+ | step chooseleaf firstn 1 type host | ||
+ | step emit | ||
+ | } | ||
+ | |||
+ | # end crush map | ||
+ | </ | ||
+ | |||
+ | ===== Important CRUSH parameters ===== | ||
+ | |||
+ | * '' | ||
+ | * '' | ||
+ | * '' | ||
+ | * '' | ||
+ | * '' | ||
+ | * '' | ||
Line 125: | Line 257: | ||
<code bash> | <code bash> | ||
- | crushtool -c crushmap.txt -o crushmap.new | + | crushtool -c crushmap.txt -o crushmap_new.bin |
</ | </ | ||
+ | |||
+ | |||
+ | ===== Checking map before applying ===== | ||
+ | |||
+ | <WRAP center round info 60%> | ||
+ | This step is **MANDATORY**!!!! | ||
+ | </ | ||
+ | |||
+ | |||
+ | Perform a test of OSD utilization with ruleset '' | ||
+ | <code bash> | ||
+ | crushtool --test -i crushmap_new.bin --show-utilization --rule <ID> --num-rep=< | ||
+ | </ | ||
+ | |||
+ | Sample: | ||
+ | < | ||
+ | # crushtool --test -i crushmap_new.map --show-utilization --rule 1 --num-rep=4 | ||
+ | rule 1 (ciberterminalRule), | ||
+ | rule 1 (ciberterminalRule) num_rep 4 result size == 1: 1024/1024 | ||
+ | device 0: stored : 58 | ||
+ | device 1: stored : 45 | ||
+ | device 2: stored : 46 | ||
+ | device 3: stored : 62 | ||
+ | device 4: stored : 45 | ||
+ | device 5: stored : 40 | ||
+ | device 6: stored : 47 | ||
+ | device 7: stored : 53 | ||
+ | device 8: stored : 36 | ||
+ | device 9: stored : 46 | ||
+ | device 10: | ||
+ | device 11: | ||
+ | device 12: | ||
+ | device 13: | ||
+ | device 14: | ||
+ | device 15: | ||
+ | device 16: | ||
+ | device 17: | ||
+ | device 18: | ||
+ | device 19: | ||
+ | (reverse-i-search)`p': | ||
+ | </ | ||
+ | |||
+ | |||
+ | |||
+ | |||
+ | Perform a CRUSH algorithm check displaying the necessary steps to allocate any object: | ||
+ | <code bash> | ||
+ | crushtool --test -i crushmap_new.bin --show-choose-tries --rule <ID> --num-rep=< | ||
+ | </ | ||
+ | |||
+ | |||
+ | Sample (in this sample, with rule " | ||
+ | < | ||
+ | # crushtool --test -i crushmap_new.map --show-choose-tries --rule 1 --num-rep=4 | ||
+ | | ||
+ | | ||
+ | | ||
+ | | ||
+ | | ||
+ | | ||
+ | | ||
+ | | ||
+ | | ||
+ | | ||
+ | 10: 0 | ||
+ | 11: 0 | ||
+ | 12: 0 | ||
+ | 13: 0 | ||
+ | 14: 0 | ||
+ | 15: 0 | ||
+ | 16: 0 | ||
+ | 17: 0 | ||
+ | 18: 0 | ||
+ | 19: 0 | ||
+ | 20: 0 | ||
+ | 21: 0 | ||
+ | 22: 0 | ||
+ | 23: 0 | ||
+ | 24: 0 | ||
+ | 25: 0 | ||
+ | 26: 0 | ||
+ | 27: 0 | ||
+ | 28: 0 | ||
+ | 29: 0 | ||
+ | 30: 0 | ||
+ | 31: 0 | ||
+ | 32: 0 | ||
+ | 33: 0 | ||
+ | 34: 0 | ||
+ | 35: 0 | ||
+ | 36: 0 | ||
+ | 37: 0 | ||
+ | 38: 0 | ||
+ | 39: 0 | ||
+ | 40: 0 | ||
+ | 41: 0 | ||
+ | 42: 0 | ||
+ | 43: 0 | ||
+ | 44: 0 | ||
+ | 45: 0 | ||
+ | 46: 0 | ||
+ | 47: 0 | ||
+ | 48: 0 | ||
+ | 49: 0 | ||
+ | </ | ||
+ | |||
+ | Sample (in this sample, with rule " | ||
+ | < | ||
+ | proceph-osm-001 ~/ | ||
+ | | ||
+ | | ||
+ | | ||
+ | | ||
+ | | ||
+ | | ||
+ | | ||
+ | | ||
+ | | ||
+ | | ||
+ | 10: 0 | ||
+ | 11: 0 | ||
+ | 12: 1 | ||
+ | 13: 0 | ||
+ | 14: 0 | ||
+ | 15: 0 | ||
+ | 16: 0 | ||
+ | 17: 0 | ||
+ | 18: 0 | ||
+ | 19: 0 | ||
+ | 20: 0 | ||
+ | 21: 0 | ||
+ | 22: 0 | ||
+ | 23: 0 | ||
+ | 24: 0 | ||
+ | 25: 0 | ||
+ | 26: 0 | ||
+ | 27: 0 | ||
+ | 28: 0 | ||
+ | 29: 0 | ||
+ | 30: 0 | ||
+ | 31: 0 | ||
+ | 32: 0 | ||
+ | 33: 0 | ||
+ | 34: 0 | ||
+ | 35: 0 | ||
+ | 36: 0 | ||
+ | 37: 0 | ||
+ | 38: 0 | ||
+ | 39: 0 | ||
+ | 40: 0 | ||
+ | 41: 0 | ||
+ | 42: 0 | ||
+ | 43: 0 | ||
+ | 44: 0 | ||
+ | 45: 0 | ||
+ | 46: 0 | ||
+ | 47: 0 | ||
+ | 48: 0 | ||
+ | 49: 0 | ||
+ | </ | ||
===== Apply the new map ===== | ===== Apply the new map ===== | ||
Line 142: | Line 434: | ||
Sample: | Sample: | ||
< | < | ||
- | avmlp-osm-001 ~ :( # ceph osd crush tree | + | proceph-osm-001 ~/ |
- | ID CLASS WEIGHT | + | ID CLASS WEIGHT |
- | -1 | + | -3 |
- | -44 | + | -2 1.00000 host ciberterminal2_cluster |
- | -43 | + | 1 hdd 1.99899 |
- | | + | 3 hdd 1.99899 |
- | | + | 5 hdd 1.99899 |
- | | + | 7 hdd 1.99899 |
- | | + | 9 hdd 1.99899 |
- | -11 | + | 11 hdd 1.99899 |
- | | + | 13 hdd 1.99899 |
- | -15 | + | 15 hdd 1.99899 |
- | | + | 17 hdd 1.99899 |
- | -19 | + | 19 hdd 1.99899 |
- | | + | -1 1.00000 host ciberterminal_cluster |
- | -23 | + | 0 hdd 1.99899 |
- | 10 hdd 1.99899 | + | 2 hdd 1.99899 |
- | -27 | + | 4 hdd 1.99899 |
- | | + | 6 hdd 1.99899 |
- | -31 | + | 8 hdd 1.99899 |
- | | + | 10 hdd 1.99899 |
- | -35 | + | 12 hdd 1.99899 |
- | | + | 14 hdd 1.99899 |
- | -39 | + | 16 hdd 1.99899 |
- | | + | 18 hdd 1.99899 |
- | -48 | + | |
- | -47 | + | |
- | | + | |
- | | + | |
- | | + | |
- | | + | |
- | -13 | + | |
- | | + | |
- | -17 | + | |
- | | + | |
- | -21 | + | |
- | | + | |
- | -25 | + | |
- | 11 hdd 1.99899 | + | |
- | -29 | + | |
- | | + | |
- | -33 | + | |
- | | + | |
- | -37 | + | |
- | | + | |
- | -41 | + | |
- | | + | |
</ | </ | ||
+ | |||
+ | In graphic mode:\\ | ||
+ | < | ||
+ | digraph G { | ||
+ | compound=true; | ||
+ | default | ||
+ | cluster_1 [shape=rectangle, | ||
+ | osd1 [shape=cylinder, | ||
+ | osd3 [shape=cylinder, | ||
+ | osd5 [shape=cylinder, | ||
+ | osd7 [shape=cylinder, | ||
+ | osd9 [shape=cylinder, | ||
+ | cluster_2 [shape=rectangle, | ||
+ | osd0 [shape=cylinder, | ||
+ | osd2 [shape=cylinder, | ||
+ | osd4 [shape=cylinder, | ||
+ | osd6 [shape=cylinder, | ||
+ | osd8 [shape=cylinder, | ||
+ | default-> | ||
+ | default-> | ||
+ | cluster_1-> | ||
+ | cluster_1-> | ||
+ | cluster_1-> | ||
+ | cluster_1-> | ||
+ | cluster_1-> | ||
+ | cluster_2-> | ||
+ | cluster_2-> | ||
+ | cluster_2-> | ||
+ | cluster_2-> | ||
+ | cluster_2-> | ||
+ | } | ||
+ | </ | ||
+ | |||
+ | ===== Check pg location ===== | ||
+ | |||
+ | This will show the placement of each PG in '' | ||
+ | <code bash> | ||
+ | ceph pg dump | egrep " | ||
+ | </ | ||
+ | |||
+ | |||
====== External documentation ====== | ====== External documentation ====== | ||
Line 195: | Line 506: | ||
* [[http:// | * [[http:// | ||
* [[https:// | * [[https:// | ||
+ | * [[http:// | ||
+ | * [[https:// | ||
+ | * [[https:// | ||