IT. POST LIST

POST ALL LABEL
CEPH

CEPH - CRUSHMAP 으로 룰적용. (Ver. 10.2.11_jewel, OS. Centos7 ) with CRUSHTOOL

By 때찌때찌맴매 - 12월 21, 2018

CEPH - CRUSHMAP 으로 룰적용. (Ver. 10.2.11_jewel, OS. Centos7 )  with CRUSHTOOL

* 수동으로 bucket 이동 작업을 하면서  거슬리는 default bucket 을 제거 하기로 함.

* default bucket을 제거 하기 위해서는 crushmap 을 이용하여 제거가 가능.

* 해당 작업은 default bucket 제거겸!! 익숙해지면 편한 작업임을 알립니다.
[ CRUSHMAP 으로 rule set 적용 ]

i. CRUSHMAP 생성 및 수정

* crushtool을 이용하여 crushmap  추출.

[root@MGMT10:33:18:~]# ceph osd getcrushmap -o /tmp/crushmap
[root@MGMT10:33:18:~]# crushtool -d /tmp/crushmap -o /tmp/crushmap.txt
[root@MGMT10:33:18:~]# cat /tmp/crushmap.txt 

* crusnmap 내용 수정전

# begin crush map
tunable choose_local_tries 0
tunable choose_local_fallback_tries 0
tunable choose_total_tries 50
tunable chooseleaf_descend_once 1
tunable chooseleaf_vary_r 1
tunable straw_calc_version 1

# devices
device 0 osd.0
device 1 osd.1
device 2 osd.2
device 3 osd.3
device 4 osd.4
device 5 osd.5

# types
type 0 osd
type 1 host
type 2 chassis
type 3 rack
type 4 row
type 5 pdu
type 6 pod
type 7 room
type 8 datacenter
type 9 region
type 10 root

# buckets
host OSD-0 {
 id -2  # do not change unnecessarily
 # weight 14.540
 alg straw
 hash 0 # rjenkins1
 item osd.0 weight 14.540
}
host OSD-1 {
 id -3  # do not change unnecessarily
 # weight 14.540
 alg straw
 hash 0 # rjenkins1
 item osd.1 weight 14.540
}
host OSD-2 {
 id -4  # do not change unnecessarily
 # weight 14.540
 alg straw
 hash 0 # rjenkins1
 item osd.2 weight 14.540
}
host OSD-3 {
 id -5  # do not change unnecessarily
 # weight 14.540
 alg straw
 hash 0 # rjenkins1
 item osd.3 weight 14.540
}
root hdd {
 id -8  # do not change unnecessarily
 # weight 58.161
 alg straw
 hash 0 # rjenkins1
 item OSD-0 weight 14.540
 item OSD-1 weight 14.540
 item OSD-2 weight 14.540
 item OSD-3 weight 14.540
}
host OSD-20 {
 id -6  # do not change unnecessarily
 # weight 0.454
 alg straw
 hash 0 # rjenkins1
 item osd.4 weight 0.454
}
host OSD-21 {
 id -7  # do not change unnecessarily
 # weight 0.452
 alg straw
 hash 0 # rjenkins1
 item osd.5 weight 0.452
}
root ssd {
 id -9  # do not change unnecessarily
 # weight 0.906
 alg straw
 hash 0 # rjenkins1
 item OSD-20 weight 0.454
 item OSD-21 weight 0.452
}
root default {
 id -1  # do not change unnecessarily
 # weight 59.067
 alg straw
 hash 0 # rjenkins1
 item hdd weight 58.161
 item ssd weight 0.906
}

# rules
rule hdd {
 ruleset 0
 type replicated
 min_size 1
 max_size 10
 step take default
 step chooseleaf firstn 0 type host
 step emit
}
rule ssd {
 ruleset 1
 type replicated
 min_size 1
 max_size 10
 step take default
 step chooseleaf firstn 0 type host
 step emit
}

# end crush map


* 처음에는 복잡해 보이지만 몇 번 보면 규칙적으로 보기가 편해짐.
* 수정 내용은 #type 과, root default 가 삭제 되고 rules type 변경 등이 있습니다.

** CRUSHMAP 수정 후

# begin crush map
tunable choose_local_tries 0
tunable choose_local_fallback_tries 0
tunable choose_total_tries 50
tunable chooseleaf_descend_once 1
tunable chooseleaf_vary_r 1
tunable straw_calc_version 1
tunable allowed_bucket_algs 54

# devices
device 0 osd.0
device 1 osd.1
device 2 osd.2
device 3 osd.3
device 4 osd.4
device 5 osd.5

# types
type 0 osd
type 1 ssd_osd
type 2 hdd_osd
type 3 root

# buckets
hdd_osd OSD-0 {
 id -10  # do not change unnecessarily
 # weight 14.540
 alg straw
 hash 0 # rjenkins1
 item osd.0 weight 14.540
}
hdd_osd OSD-1 {
 id -11  # do not change unnecessarily
 # weight 14.540
 alg straw
 hash 0 # rjenkins1
 item osd.1 weight 14.540
}
hdd_osd OSD-2 {
 id -12  # do not change unnecessarily
 # weight 14.540
 alg straw
 hash 0 # rjenkins1
 item osd.2 weight 14.540
}
hdd_osd OSD-3 {
 id -13  # do not change unnecessarily
 # weight 14.540
 alg straw
 hash 0 # rjenkins1
 item osd.3 weight 14.540
}
root hdd {
 id -1  # do not change unnecessarily
 # weight 58.160
 alg straw
 hash 0 # rjenkins1
 item OSD-0 weight 14.540
 item OSD-1 weight 14.540
 item OSD-2 weight 14.540
 item OSD-3 weight 14.540
}
ssd_osd OSD-20 {
 id -20  # do not change unnecessarily
 # weight 0.454
 alg straw
 hash 0 # rjenkins1
 item osd.4 weight 0.454
}
ssd_osd OSD-21 {
 id -21  # do not change unnecessarily
 # weight 0.454
 alg straw
 hash 0 # rjenkins1
 item osd.5 weight 0.454
}
root ssd {
 id -2  # do not change unnecessarily
 # weight 0.908
 alg straw
 hash 0 # rjenkins1
 item OSD-20 weight 0.454
 item OSD-21 weight 0.454
}

# rules
rule hdd {
 ruleset 0
 type replicated
 min_size 2
 max_size 2
 step take hdd
 step chooseleaf firstn 0 type hdd_osd
 step emit
}
rule ssd {
 ruleset 1
 type replicated
 min_size 2
 max_size 2
 step take ssd
 step chooseleaf firstn 0 type ssd_osd
 step emit
}

# end crush map

* 수정된 crushmap 적용

[root@MGMT10:51:15:~]# crushtool -c /tmp/crushmap.txt -o /tmp/crushmap-new.bin
[root@MGMT11:13:45:~]# crushtool -c /tmp/crushmap.txt -o /tmp/crushmap.coloc
[root@MGMT11:14:19:~]# ceph osd setcrushmap -i /tmp/crushmap.coloc 
set crush map

[root@MGMT12:35:32:~]# ceph osd tree
ID  WEIGHT   TYPE NAME          UP/DOWN REWEIGHT PRIMARY-AFFINITY 
 -2  0.90799 root ssd                                             
-20  0.45399     ssd_osd OSD-20                                   
  4  0.45399         osd.4           up  1.00000          1.00000 
-21  0.45399     ssd_osd OSD-21                                   
  5  0.45399         osd.5           up  1.00000          1.00000 
 -1 58.15997 root hdd                                             
-10 14.53999     hdd_osd OSD-0                                    
  0 14.53999         osd.0           up  1.00000          1.00000 
-11 14.53999     hdd_osd OSD-1                                    
  1 14.53999         osd.1           up  1.00000          1.00000 
-12 14.53999     hdd_osd OSD-2                                    
  2 14.53999         osd.2           up  1.00000          1.00000 
-13 14.53999     hdd_osd OSD-3                                    
  3 14.53999         osd.3           up  1.00000          1.00000 

[root@MGMT12:37:11:~]# ceph -s
    cluster 427f2e6a-5722-4365-a475-8fcdc218a418
     health HEALTH_OK
     monmap e2: 4 mons at {MON-0=192.168.1.13:6789/0,MON-1=192.168.1.14:6789/0,MON-2=192.168.1.15:6789/0,MON-3=192.168.1.16:6789/0}
            election epoch 6, quorum 0,1,2,3 MON-0,MON-1,MON-2,MON-3
     osdmap e125: 6 osds: 6 up, 6 in
            flags sortbitwise,require_jewel_osds
      pgmap v424: 256 pgs, 2 pools, 0 bytes data, 0 objects
            667 MB used, 60483 GB / 60484 GB avail
                 256 active+clean

[root@MGMT12:37:20:~]# ceph osd dump | grep pool
pool 0 'hdd_pool' replicated size 2 min_size 1 crush_ruleset 0 object_hash rjenkins pg_num 128 pgp_num 128 last_change 59 flags hashpspool stripe_width 0
pool 1 'ssd_pool' replicated size 2 min_size 1 crush_ruleset 1 object_hash rjenkins pg_num 128 pgp_num 128 last_change 74 flags hashpspool stripe_width 0


* 불필요한 내용들은 트리에 재거 되어 깔끔하게 구성이 됐습니다.

  • Share:

You Might Also Like

0 개의 댓글