Hi Nathan,
Attached crushmap output.
let me know if you find any thing odd.
On Sat, Oct 24, 2020 at 6:47 PM Nathan Fish <lordcirth@xxxxxxxxx> wrote:
> Can you post your crush map? Perhaps some OSDs are in the wrong place.
>
> On Sat, Oct 24, 2020 at 8:51 AM Amudhan P <amudhan83@xxxxxxxxx> wrote:
> >
> > Hi,
> >
> > I have created a test Ceph cluster with Ceph Octopus using cephadm.
> >
> > Cluster total RAW disk capacity is 262 TB but it's allowing to use of
> only
> > 132TB.
> > I have not set quota for any of the pool. what could be the issue?
> >
> > Output from :-
> > ceph -s
> > cluster:
> > id: f8bc7682-0d11-11eb-a332-0cc47a5ec98a
> > health: HEALTH_WARN
> > clock skew detected on mon.strg-node3, mon.strg-node2
> > 2 backfillfull osd(s)
> > 4 pool(s) backfillfull
> > 1 pools have too few placement groups
> >
> > services:
> > mon: 3 daemons, quorum strg-node1,strg-node3,strg-node2 (age 7m)
> > mgr: strg-node3.jtacbn(active, since 7m), standbys: strg-node1.gtlvyv
> > mds: cephfs-strg:1 {0=cephfs-strg.strg-node1.lhmeea=up:active} 1
> > up:standby
> > osd: 48 osds: 48 up (since 7m), 48 in (since 5d)
> >
> > task status:
> > scrub status:
> > mds.cephfs-strg.strg-node1.lhmeea: idle
> >
> > data:
> > pools: 4 pools, 289 pgs
> > objects: 17.29M objects, 66 TiB
> > usage: 132 TiB used, 130 TiB / 262 TiB avail
> > pgs: 288 active+clean
> > 1 active+clean+scrubbing+deep
> >
> > mounted volume shows
> > node1:/ 67T 66T 910G 99% /mnt/cephfs
> > _______________________________________________
> > ceph-users mailing list -- ceph-users@xxxxxxx
> > To unsubscribe send an email to ceph-users-leave@xxxxxxx
> _______________________________________________
> ceph-users mailing list -- ceph-users@xxxxxxx
> To unsubscribe send an email to ceph-users-leave@xxxxxxx
>
{
"devices": [
{
"id": 0,
"name": "osd.0",
"class": "hdd"
},
{
"id": 1,
"name": "osd.1",
"class": "hdd"
},
{
"id": 2,
"name": "osd.2",
"class": "hdd"
},
{
"id": 3,
"name": "osd.3",
"class": "hdd"
},
{
"id": 4,
"name": "osd.4",
"class": "hdd"
},
{
"id": 5,
"name": "osd.5",
"class": "hdd"
},
{
"id": 6,
"name": "osd.6",
"class": "hdd"
},
{
"id": 7,
"name": "osd.7",
"class": "hdd"
},
{
"id": 8,
"name": "osd.8",
"class": "hdd"
},
{
"id": 9,
"name": "osd.9",
"class": "hdd"
},
{
"id": 10,
"name": "osd.10",
"class": "hdd"
},
{
"id": 11,
"name": "osd.11",
"class": "hdd"
},
{
"id": 12,
"name": "osd.12",
"class": "hdd"
},
{
"id": 13,
"name": "osd.13",
"class": "hdd"
},
{
"id": 14,
"name": "osd.14",
"class": "hdd"
},
{
"id": 15,
"name": "osd.15",
"class": "hdd"
},
{
"id": 16,
"name": "osd.16",
"class": "hdd"
},
{
"id": 17,
"name": "osd.17",
"class": "hdd"
},
{
"id": 18,
"name": "osd.18",
"class": "hdd"
},
{
"id": 19,
"name": "osd.19",
"class": "hdd"
},
{
"id": 20,
"name": "osd.20",
"class": "hdd"
},
{
"id": 21,
"name": "osd.21",
"class": "hdd"
},
{
"id": 22,
"name": "osd.22",
"class": "hdd"
},
{
"id": 23,
"name": "osd.23",
"class": "hdd"
},
{
"id": 24,
"name": "osd.24",
"class": "hdd"
},
{
"id": 25,
"name": "osd.25",
"class": "hdd"
},
{
"id": 26,
"name": "osd.26",
"class": "hdd"
},
{
"id": 27,
"name": "osd.27",
"class": "hdd"
},
{
"id": 28,
"name": "osd.28",
"class": "hdd"
},
{
"id": 29,
"name": "osd.29",
"class": "hdd"
},
{
"id": 30,
"name": "osd.30",
"class": "hdd"
},
{
"id": 31,
"name": "osd.31",
"class": "hdd"
},
{
"id": 32,
"name": "osd.32",
"class": "hdd"
},
{
"id": 33,
"name": "osd.33",
"class": "hdd"
},
{
"id": 34,
"name": "osd.34",
"class": "hdd"
},
{
"id": 35,
"name": "osd.35",
"class": "hdd"
},
{
"id": 36,
"name": "osd.36",
"class": "hdd"
},
{
"id": 37,
"name": "osd.37",
"class": "hdd"
},
{
"id": 38,
"name": "osd.38",
"class": "hdd"
},
{
"id": 39,
"name": "osd.39",
"class": "hdd"
},
{
"id": 40,
"name": "osd.40",
"class": "hdd"
},
{
"id": 41,
"name": "osd.41",
"class": "hdd"
},
{
"id": 42,
"name": "osd.42",
"class": "hdd"
},
{
"id": 43,
"name": "osd.43",
"class": "hdd"
},
{
"id": 44,
"name": "osd.44",
"class": "hdd"
},
{
"id": 45,
"name": "osd.45",
"class": "hdd"
},
{
"id": 46,
"name": "osd.46",
"class": "hdd"
},
{
"id": 47,
"name": "osd.47",
"class": "hdd"
}
],
"types": [
{
"type_id": 0,
"name": "osd"
},
{
"type_id": 1,
"name": "host"
},
{
"type_id": 2,
"name": "chassis"
},
{
"type_id": 3,
"name": "rack"
},
{
"type_id": 4,
"name": "row"
},
{
"type_id": 5,
"name": "pdu"
},
{
"type_id": 6,
"name": "pod"
},
{
"type_id": 7,
"name": "room"
},
{
"type_id": 8,
"name": "datacenter"
},
{
"type_id": 9,
"name": "zone"
},
{
"type_id": 10,
"name": "region"
},
{
"type_id": 11,
"name": "root"
}
],
"buckets": [
{
"id": -1,
"name": "default",
"type_id": 11,
"type_name": "root",
"weight": 17169360,
"alg": "straw2",
"hash": "rjenkins1",
"items": [
{
"id": -3,
"weight": 4292340,
"pos": 0
},
{
"id": -5,
"weight": 4292340,
"pos": 1
},
{
"id": -7,
"weight": 4292340,
"pos": 2
},
{
"id": -9,
"weight": 4292340,
"pos": 3
}
]
},
{
"id": -2,
"name": "default~hdd",
"type_id": 11,
"type_name": "root",
"weight": 17169360,
"alg": "straw2",
"hash": "rjenkins1",
"items": [
{
"id": -4,
"weight": 4292340,
"pos": 0
},
{
"id": -6,
"weight": 4292340,
"pos": 1
},
{
"id": -8,
"weight": 4292340,
"pos": 2
},
{
"id": -10,
"weight": 4292340,
"pos": 3
}
]
},
{
"id": -3,
"name": "strg-node1",
"type_id": 1,
"type_name": "host",
"weight": 4292340,
"alg": "straw2",
"hash": "rjenkins1",
"items": [
{
"id": 0,
"weight": 357695,
"pos": 0
},
{
"id": 1,
"weight": 357695,
"pos": 1
},
{
"id": 2,
"weight": 357695,
"pos": 2
},
{
"id": 3,
"weight": 357695,
"pos": 3
},
{
"id": 4,
"weight": 357695,
"pos": 4
},
{
"id": 5,
"weight": 357695,
"pos": 5
},
{
"id": 6,
"weight": 357695,
"pos": 6
},
{
"id": 7,
"weight": 357695,
"pos": 7
},
{
"id": 8,
"weight": 357695,
"pos": 8
},
{
"id": 9,
"weight": 357695,
"pos": 9
},
{
"id": 10,
"weight": 357695,
"pos": 10
},
{
"id": 11,
"weight": 357695,
"pos": 11
}
]
},
{
"id": -4,
"name": "strg-node1~hdd",
"type_id": 1,
"type_name": "host",
"weight": 4292340,
"alg": "straw2",
"hash": "rjenkins1",
"items": [
{
"id": 0,
"weight": 357695,
"pos": 0
},
{
"id": 1,
"weight": 357695,
"pos": 1
},
{
"id": 2,
"weight": 357695,
"pos": 2
},
{
"id": 3,
"weight": 357695,
"pos": 3
},
{
"id": 4,
"weight": 357695,
"pos": 4
},
{
"id": 5,
"weight": 357695,
"pos": 5
},
{
"id": 6,
"weight": 357695,
"pos": 6
},
{
"id": 7,
"weight": 357695,
"pos": 7
},
{
"id": 8,
"weight": 357695,
"pos": 8
},
{
"id": 9,
"weight": 357695,
"pos": 9
},
{
"id": 10,
"weight": 357695,
"pos": 10
},
{
"id": 11,
"weight": 357695,
"pos": 11
}
]
},
{
"id": -5,
"name": "strg-node2",
"type_id": 1,
"type_name": "host",
"weight": 4292340,
"alg": "straw2",
"hash": "rjenkins1",
"items": [
{
"id": 12,
"weight": 357695,
"pos": 0
},
{
"id": 13,
"weight": 357695,
"pos": 1
},
{
"id": 14,
"weight": 357695,
"pos": 2
},
{
"id": 15,
"weight": 357695,
"pos": 3
},
{
"id": 16,
"weight": 357695,
"pos": 4
},
{
"id": 17,
"weight": 357695,
"pos": 5
},
{
"id": 18,
"weight": 357695,
"pos": 6
},
{
"id": 19,
"weight": 357695,
"pos": 7
},
{
"id": 20,
"weight": 357695,
"pos": 8
},
{
"id": 21,
"weight": 357695,
"pos": 9
},
{
"id": 22,
"weight": 357695,
"pos": 10
},
{
"id": 23,
"weight": 357695,
"pos": 11
}
]
},
{
"id": -6,
"name": "strg-node2~hdd",
"type_id": 1,
"type_name": "host",
"weight": 4292340,
"alg": "straw2",
"hash": "rjenkins1",
"items": [
{
"id": 12,
"weight": 357695,
"pos": 0
},
{
"id": 13,
"weight": 357695,
"pos": 1
},
{
"id": 14,
"weight": 357695,
"pos": 2
},
{
"id": 15,
"weight": 357695,
"pos": 3
},
{
"id": 16,
"weight": 357695,
"pos": 4
},
{
"id": 17,
"weight": 357695,
"pos": 5
},
{
"id": 18,
"weight": 357695,
"pos": 6
},
{
"id": 19,
"weight": 357695,
"pos": 7
},
{
"id": 20,
"weight": 357695,
"pos": 8
},
{
"id": 21,
"weight": 357695,
"pos": 9
},
{
"id": 22,
"weight": 357695,
"pos": 10
},
{
"id": 23,
"weight": 357695,
"pos": 11
}
]
},
{
"id": -7,
"name": "strg-node3",
"type_id": 1,
"type_name": "host",
"weight": 4292340,
"alg": "straw2",
"hash": "rjenkins1",
"items": [
{
"id": 24,
"weight": 357695,
"pos": 0
},
{
"id": 25,
"weight": 357695,
"pos": 1
},
{
"id": 26,
"weight": 357695,
"pos": 2
},
{
"id": 27,
"weight": 357695,
"pos": 3
},
{
"id": 28,
"weight": 357695,
"pos": 4
},
{
"id": 29,
"weight": 357695,
"pos": 5
},
{
"id": 30,
"weight": 357695,
"pos": 6
},
{
"id": 31,
"weight": 357695,
"pos": 7
},
{
"id": 32,
"weight": 357695,
"pos": 8
},
{
"id": 33,
"weight": 357695,
"pos": 9
},
{
"id": 34,
"weight": 357695,
"pos": 10
},
{
"id": 35,
"weight": 357695,
"pos": 11
}
]
},
{
"id": -8,
"name": "strg-node3~hdd",
"type_id": 1,
"type_name": "host",
"weight": 4292340,
"alg": "straw2",
"hash": "rjenkins1",
"items": [
{
"id": 24,
"weight": 357695,
"pos": 0
},
{
"id": 25,
"weight": 357695,
"pos": 1
},
{
"id": 26,
"weight": 357695,
"pos": 2
},
{
"id": 27,
"weight": 357695,
"pos": 3
},
{
"id": 28,
"weight": 357695,
"pos": 4
},
{
"id": 29,
"weight": 357695,
"pos": 5
},
{
"id": 30,
"weight": 357695,
"pos": 6
},
{
"id": 31,
"weight": 357695,
"pos": 7
},
{
"id": 32,
"weight": 357695,
"pos": 8
},
{
"id": 33,
"weight": 357695,
"pos": 9
},
{
"id": 34,
"weight": 357695,
"pos": 10
},
{
"id": 35,
"weight": 357695,
"pos": 11
}
]
},
{
"id": -9,
"name": "strg-node4",
"type_id": 1,
"type_name": "host",
"weight": 4292340,
"alg": "straw2",
"hash": "rjenkins1",
"items": [
{
"id": 36,
"weight": 357695,
"pos": 0
},
{
"id": 37,
"weight": 357695,
"pos": 1
},
{
"id": 38,
"weight": 357695,
"pos": 2
},
{
"id": 39,
"weight": 357695,
"pos": 3
},
{
"id": 40,
"weight": 357695,
"pos": 4
},
{
"id": 41,
"weight": 357695,
"pos": 5
},
{
"id": 42,
"weight": 357695,
"pos": 6
},
{
"id": 43,
"weight": 357695,
"pos": 7
},
{
"id": 44,
"weight": 357695,
"pos": 8
},
{
"id": 45,
"weight": 357695,
"pos": 9
},
{
"id": 46,
"weight": 357695,
"pos": 10
},
{
"id": 47,
"weight": 357695,
"pos": 11
}
]
},
{
"id": -10,
"name": "strg-node4~hdd",
"type_id": 1,
"type_name": "host",
"weight": 4292340,
"alg": "straw2",
"hash": "rjenkins1",
"items": [
{
"id": 36,
"weight": 357695,
"pos": 0
},
{
"id": 37,
"weight": 357695,
"pos": 1
},
{
"id": 38,
"weight": 357695,
"pos": 2
},
{
"id": 39,
"weight": 357695,
"pos": 3
},
{
"id": 40,
"weight": 357695,
"pos": 4
},
{
"id": 41,
"weight": 357695,
"pos": 5
},
{
"id": 42,
"weight": 357695,
"pos": 6
},
{
"id": 43,
"weight": 357695,
"pos": 7
},
{
"id": 44,
"weight": 357695,
"pos": 8
},
{
"id": 45,
"weight": 357695,
"pos": 9
},
{
"id": 46,
"weight": 357695,
"pos": 10
},
{
"id": 47,
"weight": 357695,
"pos": 11
}
]
}
],
"rules": [
{
"rule_id": 0,
"rule_name": "replicated_rule",
"ruleset": 0,
"type": 1,
"min_size": 1,
"max_size": 10,
"steps": [
{
"op": "take",
"item": -1,
"item_name": "default"
},
{
"op": "chooseleaf_firstn",
"num": 0,
"type": "host"
},
{
"op": "emit"
}
]
},
{
"rule_id": 1,
"rule_name": "Crush_rule_manual",
"ruleset": 1,
"type": 1,
"min_size": 1,
"max_size": 10,
"steps": [
{
"op": "take",
"item": -2,
"item_name": "default~hdd"
},
{
"op": "chooseleaf_firstn",
"num": 0,
"type": "host"
},
{
"op": "emit"
}
]
}
],
"tunables": {
"choose_local_tries": 0,
"choose_local_fallback_tries": 0,
"choose_total_tries": 50,
"chooseleaf_descend_once": 1,
"chooseleaf_vary_r": 1,
"chooseleaf_stable": 1,
"straw_calc_version": 1,
"allowed_bucket_algs": 54,
"profile": "jewel",
"optimal_tunables": 1,
"legacy_tunables": 0,
"minimum_required_version": "jewel",
"require_feature_tunables": 1,
"require_feature_tunables2": 1,
"has_v2_rules": 0,
"require_feature_tunables3": 1,
"has_v3_rules": 0,
"has_v4_buckets": 1,
"require_feature_tunables5": 1,
"has_v5_rules": 0
},
"choose_args": {}
}
_______________________________________________ ceph-users mailing list -- ceph-users@xxxxxxx To unsubscribe send an email to ceph-users-leave@xxxxxxx
