Hi All,
Running an object storage cluster, originally deployed with Nautilus 14.2.1 and now running 14.2.4.
Last week I was alerted to a new warning from my object storage cluster:
[root@ceph1 ~]# ceph health detail
HEALTH_WARN 1 large omap objects
LARGE_OMAP_OBJECTS 1 large omap objects
1 large objects found in pool 'default.rgw.log'
Search the cluster log for 'Large omap object found' for more details.
I looked into this and found the object and pool in question (default.rgw.log):
[root@ceph1 /var/log/ceph]# grep -R -i 'Large omap object found' .
./ceph.log:2019-10-24 12:21:26.984802 osd.194 (osd.194) 715 : cluster [WRN] Large omap object found. Object: 5:0fbdcb32:usage::usage.17:head Key count: 702330 Size (bytes): 92881228
HEALTH_WARN 1 large omap objects
LARGE_OMAP_OBJECTS 1 large omap objects
1 large objects found in pool 'default.rgw.log'
Search the cluster log for 'Large omap object found' for more details.
I looked into this and found the object and pool in question (default.rgw.log):
[root@ceph1 /var/log/ceph]# grep -R -i 'Large omap object found' .
./ceph.log:2019-10-24 12:21:26.984802 osd.194 (osd.194) 715 : cluster [WRN] Large omap object found. Object: 5:0fbdcb32:usage::usage.17:head Key count: 702330 Size (bytes): 92881228
[root@ceph1 ~]# ceph --format=json pg ls-by-pool default.rgw.log | jq '.[]' | egrep '(pgid|num_large_omap_objects)' | grep -v '"num_large_omap_objects": 0,' | grep -B1 num_large_omap_objects
"pgid": "5.70",
"num_large_omap_objects": 1,While I was investigating, I noticed an enormous amount of entries in the RGW usage log:
223326
There are entries for over 223k buckets! This was pretty scary to see, considering we only have maybe 500 legitimate buckets in this fairly new cluster. Almost all of the entries in the usage log are bogus entries from anonymous users. It looks like someone/something was scanning, looking for vulnerabilities, etc. Here are a few example entries, notice none of the operations were successful:
<-----SNIP----->
{
"bucket": "pk1914.php",
"time": "2019-07-26 21:00:00.000000Z",
"epoch": 1564174800,
"owner": "anonymous",
"categories": [
{
"category": "post_obj",
"bytes_sent": 586,
"bytes_received": 0,
"ops": 2,
"successful_ops": 0
}
]
},
{
"bucket": "plus",
"time": "2019-07-26 21:00:00.000000Z",
"epoch": 1564174800,
"owner": "anonymous",
"categories": [
{
"category": "post_obj",
"bytes_sent": 6314,
"bytes_received": 0,
"ops": 22,
"successful_ops": 0
}
]
},
{
"bucket": "pma.php",
"time": "2019-07-26 21:00:00.000000Z",
"epoch": 1564174800,
"owner": "anonymous",
"categories": [
{
"category": "post_obj",
"bytes_sent": 580,
"bytes_received": 0,
"ops": 2,
"successful_ops": 0
}
]
"bucket": "pk1914.php",
"time": "2019-07-26 21:00:00.000000Z",
"epoch": 1564174800,
"owner": "anonymous",
"categories": [
{
"category": "post_obj",
"bytes_sent": 586,
"bytes_received": 0,
"ops": 2,
"successful_ops": 0
}
]
},
{
"bucket": "plus",
"time": "2019-07-26 21:00:00.000000Z",
"epoch": 1564174800,
"owner": "anonymous",
"categories": [
{
"category": "post_obj",
"bytes_sent": 6314,
"bytes_received": 0,
"ops": 22,
"successful_ops": 0
}
]
},
{
"bucket": "pma.php",
"time": "2019-07-26 21:00:00.000000Z",
"epoch": 1564174800,
"owner": "anonymous",
"categories": [
{
"category": "post_obj",
"bytes_sent": 580,
"bytes_received": 0,
"ops": 2,
"successful_ops": 0
}
]
<-----SNIP----->
I suspect that the large omap warning from Ceph is related to the 223k+ entries in the RGW usage log. I have two questions:
1) How can I remove these bogus bucket entries from the RGW usage log? When I issue $(radosgw-admin usage trim), it only resets the stats to 0, but does not actually remove the bogus bucket entries.
2) Is it possible to prevent Ceph from logging usage statistics for buckets that do not exist?
1) How can I remove these bogus bucket entries from the RGW usage log? When I issue $(radosgw-admin usage trim), it only resets the stats to 0, but does not actually remove the bogus bucket entries.
2) Is it possible to prevent Ceph from logging usage statistics for buckets that do not exist?
Some more output if it's useful:
##
ceph --format=json pg ls-by-pool default.rgw.log | jq '.[]'
<-----SNIP----->
{
"pgid": "5.70",
"version": "4712'5226850",
"reported_seq": "5636679",
"reported_epoch": "4712",
"state": "active+clean",
"last_fresh": "2019-10-24 14:40:59.287019",
"last_change": "2019-10-24 12:21:26.984997",
"last_active": "2019-10-24 14:40:59.287019",
"last_peered": "2019-10-24 14:40:59.287019",
"last_clean": "2019-10-24 14:40:59.287019",
"last_became_active": "2019-10-18 16:02:06.007865",
"last_became_peered": "2019-10-18 16:02:06.007865",
"last_unstale": "2019-10-24 14:40:59.287019",
"last_undegraded": "2019-10-24 14:40:59.287019",
"last_fullsized": "2019-10-24 14:40:59.287019",
"mapping_epoch": 4672,
"log_start": "4712'5223781",
"ondisk_log_start": "4712'5223781",
"created": 1105,
"last_epoch_clean": 4673,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "4712'5223077",
"last_scrub_stamp": "2019-10-24 12:21:26.984947",
"last_deep_scrub": "4712'5223077",
"last_deep_scrub_stamp": "2019-10-24 12:21:26.984947",
"last_clean_scrub_stamp": "2019-10-24 12:21:26.984947",
"log_size": 3069,
"ondisk_log_size": 3069,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 0,
"num_objects": 2,
"num_object_clones": 0,
"num_object_copies": 6,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 2,
"num_whiteouts": 0,
"num_read": 6140544,
"num_read_kb": 6769195,
"num_write": 10664318,
"num_write_kb": 9859326,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 8,
"num_bytes_recovered": 0,
"num_keys_recovered": 2666590,
"num_objects_omap": 1,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 1,
"num_objects_manifest": 0,
"num_omap_bytes": 92881228,
"num_omap_keys": 702330,
"num_objects_repaired": 0
},
"up": [
194,
79,
51
],
"acting": [
194,
79,
51
],
"avail_no_missing": [],
"object_location_counts": [],
"blocked_by": [],
"up_primary": 194,
"acting_primary": 194,
"purged_snaps": []
},I appreciate any assistance. Thank you.
- Dave Monschein
_______________________________________________ ceph-users mailing list -- ceph-users@xxxxxxx To unsubscribe send an email to ceph-users-leave@xxxxxxx
