I have been trying to build a multisite ceph rgw installation with a single realm, multiple zonegroups, and a single zone per zonegroup. This is a model around 3 locations spread over long distances. I have sucessfully create an installation with a single realm, a single zonegroup and multiple zones on that one zonegroup. I have had no luck getting my multiple zonegroup installation even setup. I have read the docs over and over but I still think that I am doing something incorrect (or possibly a bug?) I am running Pacific 16.2.7 in a containerized environment I have created a github gist of all of the commands and output show below as that may be easier to read for some. https://gist.github.com/tokenrain/4edf85b0060ce5004f2003aa8a66e67d Cluster 1 and Cluster 2 are separate ceph clusters. Cluster 1 commands were run on a node in cluster1 and Cluster 2 commands were run on a node in cluster2 All and any help is greatly appreciated. ############ # TOPOLOGY # ############ realm = acme.com zonegroup = us zone = us-west-1 zonegroup = eu zone = eu-west-1 zonegroup = as zone = as-west-1 ###################### # CLUSTER 1 COMMANDS # ###################### radosgw-admin realm create --rgw-realm=acme --default radosgw-admin zonegroup create --rgw-zonegroup=us --rgw-realm=acme --master --default --endpoints=https://ceph1dev01.acme.com:443 radosgw-admin zone create --rgw-zonegroup=us \ --rgw-zone=us-west-1 \ --master \ --default \ --endpoints=https://ceph1dev01.acme.com:443 radosgw-admin user create --uid="sync-user" --display-name="Synchronization User" --system radosgw-admin zone modify --rgw-zonegroup=us --rgw-zone=us-west-1 --access-key=<ACCESS_KEY> --secret=<SECRET> radosgw-admin period update --commit ceph orch apply -i rgw-us-west-1.yml ######################### # Cluster 1 -- RGW Spec # ######################### --- service_type: rgw service_id: us-west-1 placement: hosts: - ceph1dev01.acme.com - ceph1dev02.acme.com - ceph1dev03.acme.com spec: ssl: true rgw_realm: acme rgw_zone: us-west-1 rgw_frontend_port: 443 rgw_frontend_type: beast rgw_frontend_ssl_certificate: | ###################### # CLUSTER 2 COMMANDS # ###################### radosgw-admin realm pull --rgw-realm=acme --url=https://ceph1dev01.acme.com:443 --access-key=<ACCESS_KEY> --secret=<SECRET> --default radosgw-admin zonegroup create --rgw-realm=acme --rgw-zonegroup=eu --endpoints=https://ceph2dev01.acme.com:443 radosgw-admin zone create --rgw-zone=eu-west-1 \ --rgw-zonegroup=eu \ --default \ --master \ --endpoints=https://ceph2dev01.acme.com:443 radosgw-admin zone modify --rgw-zone=eu-west-1 --rgw-zonegroup=eu --access-key=<ACCESS_KEY> --secret=<SECRET radosgw-admin period update radosgw-admin period update --commit ceph orch apply -i rgw-eu-west1-2.yml ############################################## # CLUSTER 1 OUTPUT OF period update --commit # ############################################## { "id": "b153187a-1d91-4bbf-a674-d3cad9fd23da", "epoch": 1, "predecessor_uuid": "740d6999-ce83-47ff-81f5-615a3a441a96", "sync_status": [], "period_map": { "id": "b153187a-1d91-4bbf-a674-d3cad9fd23da", "zonegroups": [ { "id": "e39e0b42-43a8-47eb-b6cd-6c2524ff51d2", "name": "us", "api_name": "us", "is_master": "true", "endpoints": [ "https://ceph1dev01.acme.com:443"; ], "hostnames": [], "hostnames_s3website": [], "master_zone": "d7ceaa4f-06c0-4c21-bcec-efe90f55ecfd", "zones": [ { "id": "d7ceaa4f-06c0-4c21-bcec-efe90f55ecfd", "name": "us-west-1", "endpoints": [ "https://ceph1dev01.acme.com:443"; ], "log_meta": "false", "log_data": "false", "bucket_index_max_shards": 11, "read_only": "false", "tier_type": "", "sync_from_all": "true", "sync_from": [], "redirect_zone": "" } ], "placement_targets": [ { "name": "default-placement", "tags": [], "storage_classes": [ "STANDARD" ] } ], "default_placement": "default-placement", "realm_id": "1504a901-d969-463e-b43c-ce362782c9e2", "sync_policy": { "groups": [] } } ], "short_zone_ids": [ { "key": "d7ceaa4f-06c0-4c21-bcec-efe90f55ecfd", "val": 1041973059 } ] }, "master_zonegroup": "e39e0b42-43a8-47eb-b6cd-6c2524ff51d2", "master_zone": "d7ceaa4f-06c0-4c21-bcec-efe90f55ecfd", "period_config": { "bucket_quota": { "enabled": false, "check_on_raw": false, "max_size": -1, "max_size_kb": 0, "max_objects": -1 }, "user_quota": { "enabled": false, "check_on_raw": false, "max_size": -1, "max_size_kb": 0, "max_objects": -1 } }, "realm_id": "1504a901-d969-463e-b43c-ce362782c9e2", "realm_name": "acme", "realm_epoch": 2 } ##################################### # CLUSTER 2 OUTPUT OF period update # ##################################### { "id": "1504a901-d969-463e-b43c-ce362782c9e2:staging", "epoch": 1, "predecessor_uuid": "b153187a-1d91-4bbf-a674-d3cad9fd23da", "sync_status": [], "period_map": { "id": "b153187a-1d91-4bbf-a674-d3cad9fd23da", "zonegroups": [ { "id": "7e0bc1f3-a15f-449a-aeb5-ef0cb0fd6d8d", "name": "eu", "api_name": "eu", "is_master": "false", "endpoints": [ "https://ceph2dev01.acme.com:443"; ], "hostnames": [], "hostnames_s3website": [], "master_zone": "9f8a06eb-5a1c-4052-b04d-359f21c95371", "zones": [ { "id": "9f8a06eb-5a1c-4052-b04d-359f21c95371", "name": "eu-west-1", "endpoints": [ "https://ceph2dev01.acme.com:443"; ], "log_meta": "false", "log_data": "false", "bucket_index_max_shards": 11, "read_only": "false", "tier_type": "", "sync_from_all": "true", "sync_from": [], "redirect_zone": "" } ], "placement_targets": [ { "name": "default-placement", "tags": [], "storage_classes": [ "STANDARD" ] } ], "default_placement": "default-placement", "realm_id": "1504a901-d969-463e-b43c-ce362782c9e2", "sync_policy": { "groups": [] } }, { "id": "e39e0b42-43a8-47eb-b6cd-6c2524ff51d2", "name": "us", "api_name": "us", "is_master": "true", "endpoints": [ "https://ceph1dev01.acme.com:443"; ], "hostnames": [], "hostnames_s3website": [], "master_zone": "d7ceaa4f-06c0-4c21-bcec-efe90f55ecfd", "zones": [ { "id": "d7ceaa4f-06c0-4c21-bcec-efe90f55ecfd", "name": "us-west-1", "endpoints": [ "https://ceph1dev01.acme.com:443"; ], "log_meta": "false", "log_data": "false", "bucket_index_max_shards": 11, "read_only": "false", "tier_type": "", "sync_from_all": "true", "sync_from": [], "redirect_zone": "" } ], "placement_targets": [ { "name": "default-placement", "tags": [], "storage_classes": [ "STANDARD" ] } ], "default_placement": "default-placement", "realm_id": "1504a901-d969-463e-b43c-ce362782c9e2", "sync_policy": { "groups": [] } } ], "short_zone_ids": [ { "key": "9f8a06eb-5a1c-4052-b04d-359f21c95371", "val": 2999707275 }, { "key": "d7ceaa4f-06c0-4c21-bcec-efe90f55ecfd", "val": 1041973059 } ] }, "master_zonegroup": "e39e0b42-43a8-47eb-b6cd-6c2524ff51d2", "master_zone": "d7ceaa4f-06c0-4c21-bcec-efe90f55ecfd", "period_config": { "bucket_quota": { "enabled": false, "check_on_raw": false, "max_size": -1, "max_size_kb": 0, "max_objects": -1 }, "user_quota": { "enabled": false, "check_on_raw": false, "max_size": -1, "max_size_kb": 0, "max_objects": -1 } }, "realm_id": "1504a901-d969-463e-b43c-ce362782c9e2", "realm_name": "acme", "realm_epoch": 3 } ############################################## # CLUSTER 2 OUTPUT OF period update --commit # ############################################## 2022-04-16T09:16:20.345-0700 7faf98ab6380 1 Cannot find zone id=9f8a06eb-5a1c-4052-b04d-359f21c95371 (name=eu-west-1), switching to local zonegroup configuration 2022-04-16T09:16:20.345-0700 7faf98ab6380 -1 Cannot find zone id=9f8a06eb-5a1c-4052-b04d-359f21c95371 (name=eu-west-1) 2022-04-16T09:16:20.345-0700 7faf98ab6380 0 ERROR: failed to start notify service ((22) Invalid argument 2022-04-16T09:16:20.345-0700 7faf98ab6380 0 ERROR: failed to init services (ret=(22) Invalid argument) couldn't init storage provider -- Mark Selby Sr Linux Administrator, The Voleon Group mselby@xxxxxxxxxx This email is subject to important conditions and disclosures that are listed on this web page: https://voleon.com/disclaimer/.
_______________________________________________ ceph-users mailing list -- ceph-users@xxxxxxx To unsubscribe send an email to ceph-users-leave@xxxxxxx
