see comment below
On Mon, Sep 26, 2016 at 10:00 PM, John Rowe <john.rowe@xxxxxxxxxxxxxx> wrote:
> Hi Orit,
>
> Sure thing, please see below.
>
> Thanks!
>
>
> DFW (Primary)
> radosgw-admin zonegroupmap get
> {
> "zonegroups": [
> {
> "key": "235b010c-22e2-4b43-8fcc-8ae01939273e",
> "val": {
> "id": "235b010c-22e2-4b43-8fcc-8ae01939273e",
> "name": "us",
> "api_name": "us",
> "is_master": "true",
> "endpoints": [
> "http:\/\/ELB_FQDN:80"
> ],
> "hostnames": [],
> "hostnames_s3website": [],
> "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
> "zones": [
> {
> "id": "58aa3eef-fc1f-492c-a08e-9c6019e7c266",
> "name": "us-phx",
> "endpoints": [
> "http:\/\/cephrgw1-1:80"
> ],
> "log_meta": "false",
> "log_data": "true",
> "bucket_index_max_shards": 0,
> "read_only": "false"
> },
> {
> "id": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
> "name": "us-dfw",
> "endpoints": [
> "http:\/\/cephrgw1-1-dfw:80"
> ],
> "log_meta": "true",
> "log_data": "true",
> "bucket_index_max_shards": 0,
> "read_only": "false"
> }
> ],
> "placement_targets": [
> {
> "name": "default-placement",
> "tags": []
> }
> ],
> "default_placement": "default-placement",
> "realm_id": "3af93a86-916a-490f-b38f-17922b472b19"
> }
> }
> ],
> "master_zonegroup": "235b010c-22e2-4b43-8fcc-8ae01939273e",
> "bucket_quota": {
> "enabled": false,
> "max_size_kb": -1,
> "max_objects": -1
> },
> "user_quota": {
> "enabled": false,
> "max_size_kb": -1,
> "max_objects": -1
> }
> }
>
> PHX (Secondary)
> {
> "zonegroups": [
> {
> "key": "235b010c-22e2-4b43-8fcc-8ae01939273e",
> "val": {
> "id": "235b010c-22e2-4b43-8fcc-8ae01939273e",
> "name": "us",
> "api_name": "us",
> "is_master": "true",
> "endpoints": [
> "http:\/\/ELB_FQDN:80"
> ],
> "hostnames": [],
> "hostnames_s3website": [],
> "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
> "zones": [
> {
> "id": "58aa3eef-fc1f-492c-a08e-9c6019e7c266",
> "name": "us-phx",
> "endpoints": [
> "http:\/\/cephrgw1-1-phx:80"
> ],
> "log_meta": "false",
> "log_data": "true",
> "bucket_index_max_shards": 0,
> "read_only": "false"
> },
> {
> "id": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
> "name": "us-dfw",
> "endpoints": [
> "http:\/\/cephrgw1-1-dfw:80"
> ],
> "log_meta": "true",
> "log_data": "true",
> "bucket_index_max_shards": 0,
> "read_only": "false"
> }
> ],
> "placement_targets": [
> {
> "name": "default-placement",
> "tags": []
> }
> ],
> "default_placement": "default-placement",
> "realm_id": "3af93a86-916a-490f-b38f-17922b472b19"
> }
> }
> ],
> "master_zonegroup": "235b010c-22e2-4b43-8fcc-8ae01939273e",
> "bucket_quota": {
> "enabled": false,
> "max_size_kb": -1,
> "max_objects": -1
> },
> "user_quota": {
> "enabled": false,
> "max_size_kb": -1,
> "max_objects": -1
> }
> }
>
>
> Realms and Periods:
> DFW (Primary)
> rgw1-1-dfw
> radosgw-admin realm get
> {
> "id": "3af93a86-916a-490f-b38f-17922b472b19",
> "name": "my_realm",
> "current_period": "75e5f8df-5d53-4ac9-b87d-625eee5455c1",
> "epoch": 2
> }
> radosgw-admin period get
> {
> "id": "75e5f8df-5d53-4ac9-b87d-625eee5455c1",
> "epoch": 4,
> "predecessor_uuid": "ab3ffbbb-0fc9-49f7-a44f-e804f9ed7697",
> "sync_status": [
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> ""
> ],
> "period_map": {
> "id": "75e5f8df-5d53-4ac9-b87d-625eee5455c1",
> "zonegroups": [
> {
> "id": "235b010c-22e2-4b43-8fcc-8ae01939273e",
> "name": "us",
> "api_name": "us",
> "is_master": "true",
> "endpoints": [
> "http:\/\/ELB_FQDN:80"
> ],
> "hostnames": [],
> "hostnames_s3website": [],
> "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
> "zones": [
> {
> "id": "58aa3eef-fc1f-492c-a08e-9c6019e7c266",
> "name": "us-phx",
> "endpoints": [
> "http:\/\/cephrgw1-1-phx:80"
> ],
> "log_meta": "false",
> "log_data": "true",
> "bucket_index_max_shards": 0,
> "read_only": "false"
> },
> {
> "id": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
> "name": "us-dfw",
> "endpoints": [
> "http:\/\/cephrgw1-1-dfw:80"
> ],
> "log_meta": "true",
> "log_data": "true",
> "bucket_index_max_shards": 0,
> "read_only": "false"
> }
> ],
> "placement_targets": [
> {
> "name": "default-placement",
> "tags": []
> }
> ],
> "default_placement": "default-placement",
> "realm_id": "3af93a86-916a-490f-b38f-17922b472b19"
> }
> ],
> "short_zone_ids": [
> {
> "key": "58aa3eef-fc1f-492c-a08e-9c6019e7c266",
> "val": 1694436416
> },
> {
> "key": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
> "val": 2500379964
> }
> ]
> },
> "master_zonegroup": "235b010c-22e2-4b43-8fcc-8ae01939273e",
> "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
> "period_config": {
> "bucket_quota": {
> "enabled": false,
> "max_size_kb": -1,
> "max_objects": -1
> },
> "user_quota": {
> "enabled": false,
> "max_size_kb": -1,
> "max_objects": -1
> }
> },
> "realm_id": "3af93a86-916a-490f-b38f-17922b472b19",
> "realm_name": "my_realm",
> "realm_epoch": 2
> }
>
> rgw1-2-dfw
> radosgw-admin realm get
> {
> "id": "3af93a86-916a-490f-b38f-17922b472b19",
> "name": "my_realm",
> "current_period": "75e5f8df-5d53-4ac9-b87d-625eee5455c1",
> "epoch": 2
> }
>
> radosgw-admin period get
> {
> "id": "75e5f8df-5d53-4ac9-b87d-625eee5455c1",
> "epoch": 4,
> "predecessor_uuid": "ab3ffbbb-0fc9-49f7-a44f-e804f9ed7697",
> "sync_status": [
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> ""
> ],
> "period_map": {
> "id": "75e5f8df-5d53-4ac9-b87d-625eee5455c1",
> "zonegroups": [
> {
> "id": "235b010c-22e2-4b43-8fcc-8ae01939273e",
> "name": "us",
> "api_name": "us",
> "is_master": "true",
> "endpoints": [
> "http:\/\/ELB_FQDN:80"
> ],
> "hostnames": [],
> "hostnames_s3website": [],
> "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
> "zones": [
> {
> "id": "58aa3eef-fc1f-492c-a08e-9c6019e7c266",
> "name": "us-phx",
> "endpoints": [
> "http:\/\/cephrgw1-1-phx:80"
> ],
> "log_meta": "false",
> "log_data": "true",
> "bucket_index_max_shards": 0,
> "read_only": "false"
> },
> {
> "id": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
> "name": "us-dfw",
> "endpoints": [
> "http:\/\/cephrgw1-1-dfw:80"
> ],
> "log_meta": "true",
> "log_data": "true",
> "bucket_index_max_shards": 0,
> "read_only": "false"
> }
> ],
> "placement_targets": [
> {
> "name": "default-placement",
> "tags": []
> }
> ],
> "default_placement": "default-placement",
> "realm_id": "3af93a86-916a-490f-b38f-17922b472b19"
> }
> ],
> "short_zone_ids": [
> {
> "key": "58aa3eef-fc1f-492c-a08e-9c6019e7c266",
> "val": 1694436416
> },
> {
> "key": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
> "val": 2500379964
> }
> ]
> },
> "master_zonegroup": "235b010c-22e2-4b43-8fcc-8ae01939273e",
> "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
> "period_config": {
> "bucket_quota": {
> "enabled": false,
> "max_size_kb": -1,
> "max_objects": -1
> },
> "user_quota": {
> "enabled": false,
> "max_size_kb": -1,
> "max_objects": -1
> }
> },
> "realm_id": "3af93a86-916a-490f-b38f-17922b472b19",
> "realm_name": "my_realm",
> "realm_epoch": 2
> }
>
> rgw1-3-dfw
> radosgw-admin realm get
> radosgw-admin realm get
> {
> "id": "3af93a86-916a-490f-b38f-17922b472b19",
> "name": "my_realm",
> "current_period": "75e5f8df-5d53-4ac9-b87d-625eee5455c1",
> "epoch": 2
> }
>
> radosgw-admin period get
> {
> "id": "75e5f8df-5d53-4ac9-b87d-625eee5455c1",
> "epoch": 4,
> "predecessor_uuid": "ab3ffbbb-0fc9-49f7-a44f-e804f9ed7697",
> "sync_status": [
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> ""
> ],
> "period_map": {
> "id": "75e5f8df-5d53-4ac9-b87d-625eee5455c1",
> "zonegroups": [
> {
> "id": "235b010c-22e2-4b43-8fcc-8ae01939273e",
> "name": "us",
> "api_name": "us",
> "is_master": "true",
> "endpoints": [
> "http:\/\/ELB_FQDN:80"
> ],
> "hostnames": [],
> "hostnames_s3website": [],
> "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
> "zones": [
> {
> "id": "58aa3eef-fc1f-492c-a08e-9c6019e7c266",
> "name": "us-phx",
> "endpoints": [
> "http:\/\/cephrgw1-1-phx:80"
> ],
> "log_meta": "false",
> "log_data": "true",
> "bucket_index_max_shards": 0,
> "read_only": "false"
> },
> {
> "id": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
> "name": "us-dfw",
> "endpoints": [
> "http:\/\/cephrgw1-1-dfw:80"
> ],
> "log_meta": "true",
> "log_data": "true",
> "bucket_index_max_shards": 0,
> "read_only": "false"
> }
> ],
> "placement_targets": [
> {
> "name": "default-placement",
> "tags": []
> }
> ],
> "default_placement": "default-placement",
> "realm_id": "3af93a86-916a-490f-b38f-17922b472b19"
> }
> ],
> "short_zone_ids": [
> {
> "key": "58aa3eef-fc1f-492c-a08e-9c6019e7c266",
> "val": 1694436416
> },
> {
> "key": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
> "val": 2500379964
> }
> ]
> },
> "master_zonegroup": "235b010c-22e2-4b43-8fcc-8ae01939273e",
> "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
> "period_config": {
> "bucket_quota": {
> "enabled": false,
> "max_size_kb": -1,
> "max_objects": -1
> },
> "user_quota": {
> "enabled": false,
> "max_size_kb": -1,
> "max_objects": -1
> }
> },
> "realm_id": "3af93a86-916a-490f-b38f-17922b472b19",
> "realm_name": "my_realm",
> "realm_epoch": 2
> }
>
>
> PHX (secondary)
> radosgw-admin realm get
> {
> "id": "3af93a86-916a-490f-b38f-17922b472b19",
> "name": "my_realm",
> "current_period": "75e5f8df-5d53-4ac9-b87d-625eee5455c1",
> "epoch": 2
> }
> radosgw-admin period get
> {
> "id": "75e5f8df-5d53-4ac9-b87d-625eee5455c1",
> "epoch": 4,
> "predecessor_uuid": "ab3ffbbb-0fc9-49f7-a44f-e804f9ed7697",
> "sync_status": [
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> ""
> ],
> "period_map": {
> "id": "75e5f8df-5d53-4ac9-b87d-625eee5455c1",
> "zonegroups": [
> {
> "id": "235b010c-22e2-4b43-8fcc-8ae01939273e",
> "name": "us",
> "api_name": "us",
> "is_master": "true",
> "endpoints": [
> "http:\/\/ELB_FQDN:80"
> ],
> "hostnames": [],
> "hostnames_s3website": [],
> "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
> "zones": [
> {
> "id": "58aa3eef-fc1f-492c-a08e-9c6019e7c266",
> "name": "us-phx",
> "endpoints": [
> "http:\/\/cephrgw1-1-phx:80"
> ],
> "log_meta": "false",
> "log_data": "true",
> "bucket_index_max_shards": 0,
> "read_only": "false"
> },
> {
> "id": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
> "name": "us-dfw",
> "endpoints": [
> "http:\/\/cephrgw1-1-dfw:80"
> ],
> "log_meta": "true",
> "log_data": "true",
> "bucket_index_max_shards": 0,
> "read_only": "false"
> }
> ],
> "placement_targets": [
> {
> "name": "default-placement",
> "tags": []
> }
> ],
> "default_placement": "default-placement",
> "realm_id": "3af93a86-916a-490f-b38f-17922b472b19"
> }
> ],
> "short_zone_ids": [
> {
> "key": "58aa3eef-fc1f-492c-a08e-9c6019e7c266",
> "val": 1694436416
> },
> {
> "key": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
> "val": 2500379964
> }
> ]
> },
> "master_zonegroup": "235b010c-22e2-4b43-8fcc-8ae01939273e",
> "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
> "period_config": {
> "bucket_quota": {
> "enabled": false,
> "max_size_kb": -1,
> "max_objects": -1
> },
> "user_quota": {
> "enabled": false,
> "max_size_kb": -1,
> "max_objects": -1
> }
> },
> "realm_id": "3af93a86-916a-490f-b38f-17922b472b19",
> "realm_name": "my_realm",
> "realm_epoch": 2
> }
>
> cephrgw1-3-phx
> radosgw-admin realm get
> {
> "id": "3af93a86-916a-490f-b38f-17922b472b19",
> "name": "pardot",
> "current_period": "75e5f8df-5d53-4ac9-b87d-625eee5455c1",
> "epoch": 2
> }
> radosgw-admin period get
> {
> "id": "75e5f8df-5d53-4ac9-b87d-625eee5455c1",
> "epoch": 4,
> "predecessor_uuid": "ab3ffbbb-0fc9-49f7-a44f-e804f9ed7697",
> "sync_status": [
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> ""
> ],
> "period_map": {
> "id": "75e5f8df-5d53-4ac9-b87d-625eee5455c1",
> "zonegroups": [
> {
> "id": "235b010c-22e2-4b43-8fcc-8ae01939273e",
> "name": "us",
> "api_name": "us",
> "is_master": "true",
> "endpoints": [
> "http:\/\/ELB_FQDN:80"
> ],
> "hostnames": [],
> "hostnames_s3website": [],
> "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
> "zones": [
> {
> "id": "58aa3eef-fc1f-492c-a08e-9c6019e7c266",
> "name": "us-phx",
> "endpoints": [
> "http:\/\/cephrgw1-1-phx:80"
> ],
> "log_meta": "false",
> "log_data": "true",
> "bucket_index_max_shards": 0,
> "read_only": "false"
> },
> {
> "id": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
> "name": "us-dfw",
> "endpoints": [
> "http:\/\/cephrgw1-1-dfw:80"
> ],
> "log_meta": "true",
> "log_data": "true",
> "bucket_index_max_shards": 0,
> "read_only": "false"
> }
> ],
> "placement_targets": [
> {
> "name": "default-placement",
> "tags": []
> }
> ],
> "default_placement": "default-placement",
> "realm_id": "3af93a86-916a-490f-b38f-17922b472b19"
> }
> ],
> "short_zone_ids": [
> {
> "key": "58aa3eef-fc1f-492c-a08e-9c6019e7c266",
> "val": 1694436416
> },
> {
> "key": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
> "val": 2500379964
> }
> ]
> },
> "master_zonegroup": "235b010c-22e2-4b43-8fcc-8ae01939273e",
> "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
> "period_config": {
> "bucket_quota": {
> "enabled": false,
> "max_size_kb": -1,
> "max_objects": -1
> },
> "user_quota": {
> "enabled": false,
> "max_size_kb": -1,
> "max_objects": -1
> }
> },
> "realm_id": "3af93a86-916a-490f-b38f-17922b472b19",
> "realm_name": "my_realm",
> "realm_epoch": 2
> }
>
> cephrgw-1-3-phx
> radosgw-admin realm get
> {
> "id": "3af93a86-916a-490f-b38f-17922b472b19",
> "name": "pardot",
In other nodes the name is "my_realm" , maybe that is the problem.
You can try fixing it by renaming it using realm rename or using realm
set command
Orit
> "current_period": "75e5f8df-5d53-4ac9-b87d-625eee5455c1",
> "epoch": 2
> }
>
> radsogw-admin period get
> {
> "id": "75e5f8df-5d53-4ac9-b87d-625eee5455c1",
> "epoch": 4,
> "predecessor_uuid": "ab3ffbbb-0fc9-49f7-a44f-e804f9ed7697",
> "sync_status": [
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> "",
> ""
> ],
> "period_map": {
> "id": "75e5f8df-5d53-4ac9-b87d-625eee5455c1",
> "zonegroups": [
> {
> "id": "235b010c-22e2-4b43-8fcc-8ae01939273e",
> "name": "us",
> "api_name": "us",
> "is_master": "true",
> "endpoints": [
> "http:\/\/ELB_FQDN:80"
> ],
> "hostnames": [],
> "hostnames_s3website": [],
> "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
> "zones": [
> {
> "id": "58aa3eef-fc1f-492c-a08e-9c6019e7c266",
> "name": "us-phx",
> "endpoints": [
> "http:\/\/cephrgw1-1-phx:80"
> ],
> "log_meta": "false",
> "log_data": "true",
> "bucket_index_max_shards": 0,
> "read_only": "false"
> },
> {
> "id": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
> "name": "us-dfw",
> "endpoints": [
> "http:\/\/cephrgw1-1-dfw:80"
> ],
> "log_meta": "true",
> "log_data": "true",
> "bucket_index_max_shards": 0,
> "read_only": "false"
> }
> ],
> "placement_targets": [
> {
> "name": "default-placement",
> "tags": []
> }
> ],
> "default_placement": "default-placement",
> "realm_id": "3af93a86-916a-490f-b38f-17922b472b19"
> }
> ],
> "short_zone_ids": [
> {
> "key": "58aa3eef-fc1f-492c-a08e-9c6019e7c266",
> "val": 1694436416
> },
> {
> "key": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
> "val": 2500379964
> }
> ]
> },
> "master_zonegroup": "235b010c-22e2-4b43-8fcc-8ae01939273e",
> "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
> "period_config": {
> "bucket_quota": {
> "enabled": false,
> "max_size_kb": -1,
> "max_objects": -1
> },
> "user_quota": {
> "enabled": false,
> "max_size_kb": -1,
> "max_objects": -1
> }
> },
> "realm_id": "3af93a86-916a-490f-b38f-17922b472b19",
> "realm_name": "my_realm",
> "realm_epoch": 2
> }
>
>
> On Mon, Sep 26, 2016 at 5:27 AM Orit Wasserman <owasserm@xxxxxxxxxx> wrote:
>>
>> Hi John,
>> Can you provide:
>> radosgw-admin zonegroupmap get on both us-dfw and us-phx?
>> radosgw-admin realm get and radosgw-admin period get on all the gateways?
>>
>> Orit
>>
>> On Thu, Sep 22, 2016 at 4:37 PM, John Rowe <john.rowe@xxxxxxxxxxxxxx>
>> wrote:
>> > Hello Orit, thanks.
>> >
>> > I will do all 6 just in case. Also as an FYI I originally had all 6 as
>> > endpoint (3 in each zone) but have it down to just the two "1" servers
>> > talking to each other until I can get it working. Eventually I would
>> > like to
>> > have all 6 cross connecting again.
>> >
>> > rgw-primary-1:
>> > radosgw-admin zonegroup get
>> > {
>> > "id": "235b010c-22e2-4b43-8fcc-8ae01939273e",
>> > "name": "us",
>> > "api_name": "us",
>> > "is_master": "true",
>> > "endpoints": [
>> > "http:\/\/LB_FQDN:80"
>> > ],
>> > "hostnames": [],
>> > "hostnames_s3website": [],
>> > "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> > "zones": [
>> > {
>> > "id": "58aa3eef-fc1f-492c-a08e-9c6019e7c266",
>> > "name": "us-phx",
>> > "endpoints": [
>> > "http:\/\/PHX-RGW-1:80"
>> > ],
>> > "log_meta": "false",
>> > "log_data": "true",
>> > "bucket_index_max_shards": 0,
>> > "read_only": "false"
>> > },
>> > {
>> > "id": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> > "name": "us-dfw",
>> > "endpoints": [
>> > "http:\/\/DFW-RGW-1:80"
>> > ],
>> > "log_meta": "true",
>> > "log_data": "true",
>> > "bucket_index_max_shards": 0,
>> > "read_only": "false"
>> > }
>> > ],
>> > "placement_targets": [
>> > {
>> > "name": "default-placement",
>> > "tags": []
>> > }
>> > ],
>> > "default_placement": "default-placement",
>> > "realm_id": "3af93a86-916a-490f-b38f-17922b472b19"
>> > }
>> > radosgw-admin zone get
>> > {
>> > "id": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> > "name": "us-dfw",
>> > "domain_root": "us-dfw.rgw.data.root",
>> > "control_pool": "us-dfw.rgw.control",
>> > "gc_pool": "us-dfw.rgw.gc",
>> > "log_pool": "us-dfw.rgw.log",
>> > "intent_log_pool": "us-dfw.rgw.intent-log",
>> > "usage_log_pool": "us-dfw.rgw.usage",
>> > "user_keys_pool": "us-dfw.rgw.users.keys",
>> > "user_email_pool": "us-dfw.rgw.users.email",
>> > "user_swift_pool": "us-dfw.rgw.users.swift",
>> > "user_uid_pool": "us-dfw.rgw.users.uid",
>> > "system_key": {
>> > "access_key": "SYSTEM_ACCESS_KEY",
>> > "secret_key": "SYSTEM_SECRET_KEY"
>> > },
>> > "placement_pools": [
>> > {
>> > "key": "default-placement",
>> > "val": {
>> > "index_pool": "us-dfw.rgw.buckets.index",
>> > "data_pool": "us-dfw.rgw.buckets.data",
>> > "data_extra_pool": "us-dfw.rgw.buckets.non-ec",
>> > "index_type": 0
>> > }
>> > }
>> > ],
>> > "metadata_heap": "us-dfw.rgw.meta",
>> > "realm_id": "3af93a86-916a-490f-b38f-17922b472b19"
>> > }
>> >
>> > ----
>> > rgw-primary-2:
>> > radosgw-admin zonegroup get
>> > {
>> > "id": "235b010c-22e2-4b43-8fcc-8ae01939273e",
>> > "name": "us",
>> > "api_name": "us",
>> > "is_master": "true",
>> > "endpoints": [
>> > "http:\/\/LB_FQDN:80"
>> > ],
>> > "hostnames": [],
>> > "hostnames_s3website": [],
>> > "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> > "zones": [
>> > {
>> > "id": "58aa3eef-fc1f-492c-a08e-9c6019e7c266",
>> > "name": "us-phx",
>> > "endpoints": [
>> > "http:\/\/PHX-RGW-1:80"
>> > ],
>> > "log_meta": "false",
>> > "log_data": "true",
>> > "bucket_index_max_shards": 0,
>> > "read_only": "false"
>> > },
>> > {
>> > "id": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> > "name": "us-dfw",
>> > "endpoints": [
>> > "http:\/\/DFW-RGW-1:80"
>> > ],
>> > "log_meta": "true",
>> > "log_data": "true",
>> > "bucket_index_max_shards": 0,
>> > "read_only": "false"
>> > }
>> > ],
>> > "placement_targets": [
>> > {
>> > "name": "default-placement",
>> > "tags": []
>> > }
>> > ],
>> > "default_placement": "default-placement",
>> > "realm_id": "3af93a86-916a-490f-b38f-17922b472b19"
>> > }
>> >
>> > radosgw-admin zone get
>> > {
>> > "id": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> > "name": "us-dfw",
>> > "domain_root": "us-dfw.rgw.data.root",
>> > "control_pool": "us-dfw.rgw.control",
>> > "gc_pool": "us-dfw.rgw.gc",
>> > "log_pool": "us-dfw.rgw.log",
>> > "intent_log_pool": "us-dfw.rgw.intent-log",
>> > "usage_log_pool": "us-dfw.rgw.usage",
>> > "user_keys_pool": "us-dfw.rgw.users.keys",
>> > "user_email_pool": "us-dfw.rgw.users.email",
>> > "user_swift_pool": "us-dfw.rgw.users.swift",
>> > "user_uid_pool": "us-dfw.rgw.users.uid",
>> > "system_key": {
>> > "access_key": "SYSTEM_ACCESS_KEY",
>> > "secret_key": "SYSTEM_SECRET_KEY"
>> > },
>> > "placement_pools": [
>> > {
>> > "key": "default-placement",
>> > "val": {
>> > "index_pool": "us-dfw.rgw.buckets.index",
>> > "data_pool": "us-dfw.rgw.buckets.data",
>> > "data_extra_pool": "us-dfw.rgw.buckets.non-ec",
>> > "index_type": 0
>> > }
>> > }
>> > ],
>> > "metadata_heap": "us-dfw.rgw.meta",
>> > "realm_id": "3af93a86-916a-490f-b38f-17922b472b19"
>> > }
>> >
>> > ----
>> > rgw-primary-3:
>> > radosgw-admin zonegroup get
>> > {
>> > "id": "235b010c-22e2-4b43-8fcc-8ae01939273e",
>> > "name": "us",
>> > "api_name": "us",
>> > "is_master": "true",
>> > "endpoints": [
>> > "http:\/\/LB_FQDN:80"
>> > ],
>> > "hostnames": [],
>> > "hostnames_s3website": [],
>> > "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> > "zones": [
>> > {
>> > "id": "58aa3eef-fc1f-492c-a08e-9c6019e7c266",
>> > "name": "us-phx",
>> > "endpoints": [
>> > "http:\/\/PHX-RGW-1:80"
>> > ],
>> > "log_meta": "false",
>> > "log_data": "true",
>> > "bucket_index_max_shards": 0,
>> > "read_only": "false"
>> > },
>> > {
>> > "id": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> > "name": "us-dfw",
>> > "endpoints": [
>> > "http:\/\/DFW-RGW-1:80"
>> > ],
>> > "log_meta": "true",
>> > "log_data": "true",
>> > "bucket_index_max_shards": 0,
>> > "read_only": "false"
>> > }
>> > ],
>> > "placement_targets": [
>> > {
>> > "name": "default-placement",
>> > "tags": []
>> > }
>> > ],
>> > "default_placement": "default-placement",
>> > "realm_id": "3af93a86-916a-490f-b38f-17922b472b19"
>> > }
>> >
>> > sudo radosgw-admin zone get
>> > {
>> > "id": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> > "name": "us-dfw",
>> > "domain_root": "us-dfw.rgw.data.root",
>> > "control_pool": "us-dfw.rgw.control",
>> > "gc_pool": "us-dfw.rgw.gc",
>> > "log_pool": "us-dfw.rgw.log",
>> > "intent_log_pool": "us-dfw.rgw.intent-log",
>> > "usage_log_pool": "us-dfw.rgw.usage",
>> > "user_keys_pool": "us-dfw.rgw.users.keys",
>> > "user_email_pool": "us-dfw.rgw.users.email",
>> > "user_swift_pool": "us-dfw.rgw.users.swift",
>> > "user_uid_pool": "us-dfw.rgw.users.uid",
>> > "system_key": {
>> > "access_key": "SYSTEM_ACCESS_KEY",
>> > "secret_key": "SYSTEM_SECRET_KEY"
>> > },
>> > "placement_pools": [
>> > {
>> > "key": "default-placement",
>> > "val": {
>> > "index_pool": "us-dfw.rgw.buckets.index",
>> > "data_pool": "us-dfw.rgw.buckets.data",
>> > "data_extra_pool": "us-dfw.rgw.buckets.non-ec",
>> > "index_type": 0
>> > }
>> > }
>> > ],
>> > "metadata_heap": "us-dfw.rgw.meta",
>> > "realm_id": "3af93a86-916a-490f-b38f-17922b472b19"
>> > }
>> >
>> > ---
>> > rgw-secondary-1:
>> > radosgw-admin zonegroup get
>> > {
>> > "id": "235b010c-22e2-4b43-8fcc-8ae01939273e",
>> > "name": "us",
>> > "api_name": "us",
>> > "is_master": "true",
>> > "endpoints": [
>> > "http:\/\/LB_FQDN:80"
>> > ],
>> > "hostnames": [],
>> > "hostnames_s3website": [],
>> > "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> > "zones": [
>> > {
>> > "id": "58aa3eef-fc1f-492c-a08e-9c6019e7c266",
>> > "name": "us-phx",
>> > "endpoints": [
>> > "http:\/\/PHX-RGW-1:80"
>> > ],
>> > "log_meta": "false",
>> > "log_data": "true",
>> > "bucket_index_max_shards": 0,
>> > "read_only": "false"
>> > },
>> > {
>> > "id": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> > "name": "us-dfw",
>> > "endpoints": [
>> > "http:\/\/DFW-RGW-1:80"
>> > ],
>> > "log_meta": "true",
>> > "log_data": "true",
>> > "bucket_index_max_shards": 0,
>> > "read_only": "false"
>> > }
>> > ],
>> > "placement_targets": [
>> > {
>> > "name": "default-placement",
>> > "tags": []
>> > }
>> > ],
>> > "default_placement": "default-placement",
>> > "realm_id": "3af93a86-916a-490f-b38f-17922b472b19"
>> > }
>> >
>> > radosgw-admin zone get
>> > {
>> > "id": "58aa3eef-fc1f-492c-a08e-9c6019e7c266",
>> > "name": "us-phx",
>> > "domain_root": "us-phx.rgw.data.root",
>> > "control_pool": "us-phx.rgw.control",
>> > "gc_pool": "us-phx.rgw.gc",
>> > "log_pool": "us-phx.rgw.log",
>> > "intent_log_pool": "us-phx.rgw.intent-log",
>> > "usage_log_pool": "us-phx.rgw.usage",
>> > "user_keys_pool": "us-phx.rgw.users.keys",
>> > "user_email_pool": "us-phx.rgw.users.email",
>> > "user_swift_pool": "us-phx.rgw.users.swift",
>> > "user_uid_pool": "us-phx.rgw.users.uid",
>> > "system_key": {
>> > "access_key": "SYSTEM_ACCESS_KEY",
>> > "secret_key": "SYSTEM_SECRET_KEY"
>> > },
>> > "placement_pools": [
>> > {
>> > "key": "default-placement",
>> > "val": {
>> > "index_pool": "us-phx.rgw.buckets.index",
>> > "data_pool": "us-phx.rgw.buckets.data",
>> > "data_extra_pool": "us-phx.rgw.buckets.non-ec",
>> > "index_type": 0
>> > }
>> > }
>> > ],
>> > "metadata_heap": "us-phx.rgw.meta",
>> > "realm_id": "3af93a86-916a-490f-b38f-17922b472b19"
>> > }
>> >
>> > ---
>> > rgw-secondary-2:
>> > radosgw-admin zonegroup get
>> > {
>> > "id": "235b010c-22e2-4b43-8fcc-8ae01939273e",
>> > "name": "us",
>> > "api_name": "us",
>> > "is_master": "true",
>> > "endpoints": [
>> > "http:\/\/LB_FQDN:80"
>> > ],
>> > "hostnames": [],
>> > "hostnames_s3website": [],
>> > "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> > "zones": [
>> > {
>> > "id": "58aa3eef-fc1f-492c-a08e-9c6019e7c266",
>> > "name": "us-phx",
>> > "endpoints": [
>> > "http:\/\/PHX-RGW-1:80"
>> > ],
>> > "log_meta": "false",
>> > "log_data": "true",
>> > "bucket_index_max_shards": 0,
>> > "read_only": "false"
>> > },
>> > {
>> > "id": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> > "name": "us-dfw",
>> > "endpoints": [
>> > "http:\/\/DFW-RGW-1:80"
>> > ],
>> > "log_meta": "true",
>> > "log_data": "true",
>> > "bucket_index_max_shards": 0,
>> > "read_only": "false"
>> > }
>> > ],
>> > "placement_targets": [
>> > {
>> > "name": "default-placement",
>> > "tags": []
>> > }
>> > ],
>> > "default_placement": "default-placement",
>> > "realm_id": "3af93a86-916a-490f-b38f-17922b472b19"
>> > }
>> > radosgw-admin zone get
>> > {
>> > "id": "58aa3eef-fc1f-492c-a08e-9c6019e7c266",
>> > "name": "us-phx",
>> > "domain_root": "us-phx.rgw.data.root",
>> > "control_pool": "us-phx.rgw.control",
>> > "gc_pool": "us-phx.rgw.gc",
>> > "log_pool": "us-phx.rgw.log",
>> > "intent_log_pool": "us-phx.rgw.intent-log",
>> > "usage_log_pool": "us-phx.rgw.usage",
>> > "user_keys_pool": "us-phx.rgw.users.keys",
>> > "user_email_pool": "us-phx.rgw.users.email",
>> > "user_swift_pool": "us-phx.rgw.users.swift",
>> > "user_uid_pool": "us-phx.rgw.users.uid",
>> > "system_key": {
>> > "access_key": "SYSTEM_ACCESS_KEY",
>> > "secret_key": "SYSTEM_SECRET_KEY"
>> > },
>> > "placement_pools": [
>> > {
>> > "key": "default-placement",
>> > "val": {
>> > "index_pool": "us-phx.rgw.buckets.index",
>> > "data_pool": "us-phx.rgw.buckets.data",
>> > "data_extra_pool": "us-phx.rgw.buckets.non-ec",
>> > "index_type": 0
>> > }
>> > }
>> > ],
>> > "metadata_heap": "us-phx.rgw.meta",
>> > "realm_id": "3af93a86-916a-490f-b38f-17922b472b19"
>> > }
>> >
>> > ---
>> > rgw-secondary-3
>> > radosgw-admin zonegroup get
>> > {
>> > "id": "235b010c-22e2-4b43-8fcc-8ae01939273e",
>> > "name": "us",
>> > "api_name": "us",
>> > "is_master": "true",
>> > "endpoints": [
>> > "http:\/\/LB_FQDN:80"
>> > ],
>> > "hostnames": [],
>> > "hostnames_s3website": [],
>> > "master_zone": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> > "zones": [
>> > {
>> > "id": "58aa3eef-fc1f-492c-a08e-9c6019e7c266",
>> > "name": "us-phx",
>> > "endpoints": [
>> > "http:\/\/PHX-RGW-1:80"
>> > ],
>> > "log_meta": "false",
>> > "log_data": "true",
>> > "bucket_index_max_shards": 0,
>> > "read_only": "false"
>> > },
>> > {
>> > "id": "6c830b44-4e39-4e19-9bd8-03c37c2021f2",
>> > "name": "us-dfw",
>> > "endpoints": [
>> > "http:\/\/DFW-RGW-1:80"
>> > ],
>> > "log_meta": "true",
>> > "log_data": "true",
>> > "bucket_index_max_shards": 0,
>> > "read_only": "false"
>> > }
>> > ],
>> > "placement_targets": [
>> > {
>> > "name": "default-placement",
>> > "tags": []
>> > }
>> > ],
>> > "default_placement": "default-placement",
>> > "realm_id": "3af93a86-916a-490f-b38f-17922b472b19"
>> > }
>> >
>> > radosgw-admin zone get
>> > {
>> > "id": "58aa3eef-fc1f-492c-a08e-9c6019e7c266",
>> > "name": "us-phx",
>> > "domain_root": "us-phx.rgw.data.root",
>> > "control_pool": "us-phx.rgw.control",
>> > "gc_pool": "us-phx.rgw.gc",
>> > "log_pool": "us-phx.rgw.log",
>> > "intent_log_pool": "us-phx.rgw.intent-log",
>> > "usage_log_pool": "us-phx.rgw.usage",
>> > "user_keys_pool": "us-phx.rgw.users.keys",
>> > "user_email_pool": "us-phx.rgw.users.email",
>> > "user_swift_pool": "us-phx.rgw.users.swift",
>> > "user_uid_pool": "us-phx.rgw.users.uid",
>> > "system_key": {
>> > "access_key": "SYSTEM_ACCESS_KEY",
>> > "secret_key": "SYSTEM_SECRET_KEY"
>> > },
>> > "placement_pools": [
>> > {
>> > "key": "default-placement",
>> > "val": {
>> > "index_pool": "us-phx.rgw.buckets.index",
>> > "data_pool": "us-phx.rgw.buckets.data",
>> > "data_extra_pool": "us-phx.rgw.buckets.non-ec",
>> > "index_type": 0
>> > }
>> > }
>> > ],
>> > "metadata_heap": "us-phx.rgw.meta",
>> > "realm_id": "3af93a86-916a-490f-b38f-17922b472b19"
>> > }
>> >
>> >
>> > On Thu, Sep 22, 2016 at 5:18 AM Orit Wasserman <owasserm@xxxxxxxxxx>
>> > wrote:
>> >>
>> >> Hi John,
>> >> Can you provide your zonegroup and zones configurations on all 3 rgw?
>> >> (run the commands on each rgw)
>> >>
>> >> Thanks,
>> >> Orit
>> >>
>> >> On Wed, Sep 21, 2016 at 11:14 PM, John Rowe <john.rowe@xxxxxxxxxxxxxx>
>> >> wrote:
>> >> > Hello,
>> >> >
>> >> > We have 2 Ceph clusters running in two separate data centers, each
>> >> > one
>> >> > with
>> >> > 3 mons, 3 rgws, and 5 osds. I am attempting to get bi-directional
>> >> > multi-site
>> >> > replication setup as described in the ceph documentation here:
>> >> > http://docs.ceph.com/docs/jewel/radosgw/multisite/
>> >> >
>> >> > We are running Jewel v 10.2.2:
>> >> > rpm -qa | grep ceph
>> >> > ceph-base-10.2.2-0.el7.x86_64
>> >> > ceph-10.2.2-0.el7.x86_64
>> >> > ceph-radosgw-10.2.2-0.el7.x86_64
>> >> > libcephfs1-10.2.2-0.el7.x86_64
>> >> > python-cephfs-10.2.2-0.el7.x86_64
>> >> > ceph-selinux-10.2.2-0.el7.x86_64
>> >> > ceph-mon-10.2.2-0.el7.x86_64
>> >> > ceph-osd-10.2.2-0.el7.x86_64
>> >> > ceph-release-1-1.el7.noarch
>> >> > ceph-common-10.2.2-0.el7.x86_64
>> >> > ceph-mds-10.2.2-0.el7.x86_64
>> >> >
>> >> > It appears syncing is happening, however it is not able to sync the
>> >> > metadata, and therefore no users/buckets from the primary are going
>> >> > to
>> >> > the
>> >> > secondary.
>> >> > Primary sync status:
>> >> > radosgw-admin sync status
>> >> > realm 3af93a86-916a-490f-b38f-17922b472b19 (my_realm)
>> >> > zonegroup 235b010c-22e2-4b43-8fcc-8ae01939273e (us)
>> >> > zone 6c830b44-4e39-4e19-9bd8-03c37c2021f2 (us-dfw)
>> >> > metadata sync no sync (zone is master)
>> >> > data sync source: 58aa3eef-fc1f-492c-a08e-9c6019e7c266 (us-phx)
>> >> > syncing
>> >> > full sync: 0/128 shards
>> >> > incremental sync: 128/128 shards
>> >> > data is caught up with source
>> >> >
>> >> > radosgw-admin data sync status --source-zone=us-phx
>> >> > {
>> >> > "sync_status": {
>> >> > "info": {
>> >> > "status": "sync",
>> >> > "num_shards": 128
>> >> > },
>> >> > "markers": [
>> >> > ...
>> >> > }
>> >> >
>> >> > radosgw-admin metadata sync status
>> >> > {
>> >> > "sync_status": {
>> >> > "info": {
>> >> > "status": "init",
>> >> > "num_shards": 0,
>> >> > "period": "",
>> >> > "realm_epoch": 0
>> >> > },
>> >> > "markers": []
>> >> > },
>> >> > "full_sync": {
>> >> > "total": 0,
>> >> > "complete": 0
>> >> > }
>> >> > }
>> >> > Secondary sync status:
>> >> > radosgw-admin sync status
>> >> > realm 3af93a86-916a-490f-b38f-17922b472b19 (my_realm)
>> >> > zonegroup 235b010c-22e2-4b43-8fcc-8ae01939273e (us)
>>
>> >> > zone 58aa3eef-fc1f-492c-a08e-9c6019e7c266 (us-phx)
>>
>> >> > metadata sync failed to read sync status: (2) No such file or
>> >> > directory
>> >> > data sync source: 6c830b44-4e39-4e19-9bd8-03c37c2021f2 (us-dfw)
>> >> > syncing
>> >> > full sync: 0/128 shards
>> >> > incremental sync: 128/128 shards
>> >> > data is behind on 10 shards
>> >> > oldest incremental change not applied:
>> >> > 2016-09-20
>> >> > 15:00:17.0.330225s
>> >> >
>> >> > radosgw-admin data sync status --source-zone=us-dfw
>> >> > {
>> >> > "sync_status": {
>> >> > "info": {
>> >> > "status": "building-full-sync-maps",
>> >> > "num_shards": 128
>> >> > },
>> >> > ....
>> >> > }
>> >> >
>> >> > radosgw-admin metadata sync status --source-zone=us-dfw
>> >> > ERROR: sync.read_sync_status() returned ret=-2
>> >> >
>> >> >
>> >> >
>> >> > In the logs I am seeing (date stamps not in order to pick out
>> >> > non-dupes):
>> >> > Primary logs:
>> >> > 2016-09-20 15:02:44.313204 7f2a2dffb700 0 ERROR:
>> >> > client_io->complete_request() returned -5
>> >> > 2016-09-20 10:31:57.501247 7faf4bfff700 0 ERROR: failed to wait for
>> >> > op,
>> >> > ret=-11: POST
>> >> >
>>
>> >> >
>> >> > http://cephrgw1-3-phx:80/admin/realm/period?period=385c44c7-0506-4204-90d7-9d26a6cbaad2&epoch=12&rgwx-zonegroup=f46ce11b-ee5d-489b-aa30-752fc5353931
>>
>>
>> >> > 2016-09-20 10:32:03.391118 7fb12affd700 0 ERROR: failed to fetch
>> >> > datalog
>> >> > info
>> >> > 2016-09-20 10:32:03.491520 7fb12affd700 0 ERROR: lease cr failed,
>> >> > done
>> >> > early
>> >> >
>> >> > Secondary logs;
>> >> > 2016-09-20 10:28:15.290050 7faab2fed700 0 ERROR: failed to get
>> >> > bucket
>> >> > instance info for bucket
>> >> > id=BUCKET1:78301214-35bb-41df-a77f-24968ee4b3ff.104293.1
>> >> > 2016-09-20 10:28:15.290108 7faab5ff3700 0 ERROR: failed to get
>> >> > bucket
>> >> > instance info for bucket
>> >> > id=BUCKET1:78301214-35bb-41df-a77f-24968ee4b3ff.104293.1
>> >> > 2016-09-20 10:28:15.290571 7faab77f6700 0 ERROR: failed to get
>> >> > bucket
>> >> > instance info for bucket
>> >> > id=BUCKET1:78301214-35bb-41df-a77f-24968ee4b3ff.104293.1
>> >> > 2016-09-20 10:28:15.304619 7faaad7e2700 0 ERROR: failed to get
>> >> > bucket
>> >> > instance info for bucket
>> >> > id=BUCKET1:78301214-35bb-41df-a77f-24968ee4b3ff.104293.1
>> >> > 2016-09-20 10:28:38.169629 7fa98bfff700 0 ERROR: failed to
>> >> > distribute
>> >> > cache
>> >> > for .rgw.root:periods.385c44c7-0506-4204-90d7-9d26a6cbaad2.12
>> >> > 2016-09-20 10:28:38.169642 7fa98bfff700 -1 period epoch 12 is not
>> >> > newer
>> >> > than
>> >> > current epoch 12, discarding update
>> >> > 2016-09-21 03:19:01.550808 7fe10bfff700 0 rgw meta sync: ERROR:
>> >> > failed
>> >> > to
>> >> > fetch mdlog info
>> >> > 2016-09-21 15:45:09.799195 7fcd677fe700 0 ERROR: failed to fetch
>> >> > remote
>> >> > data log info: ret=-11
>> >> >
>> >> > Each of those logs are repeated several times each, and constantly.
>> >> >
>> >> > Any help would be greatly appreciated. Thanks!
>> >> >
>> >> >
>> >> >
>> >> > _______________________________________________
>> >> > ceph-users mailing list
>> >> > ceph-users@xxxxxxxxxxxxxx
>> >> > http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
>> >> >
_______________________________________________
ceph-users mailing list
ceph-users@xxxxxxxxxxxxxx
http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
