On Fri, Jul 20, 2012 at 5:41 PM, Sage Weil <sage@xxxxxxxxxxx> wrote:
> From: caleb miles <caleb.miles@xxxxxxxxxxx>
>
> The server side recently added support for tuning some magic
> crush variables. Decode these variables if they are present, or use the
> default values if they are not present.
>
> Corresponds to ceph.git commit 89af369c25f274fe62ef730e5e8aad0c54f1e5a5.
>
> Signed-off-by: caleb miles <caleb.miles@xxxxxxxxxxx>
> Reviewed-by: Sage Weil <sage@xxxxxxxxxxx>
> ---
> include/linux/ceph/ceph_features.h | 4 ++-
> include/linux/crush/crush.h | 8 +++++++
> net/ceph/crush/mapper.c | 13 ++++++-----
> net/ceph/osdmap.c | 39 ++++++++++++++++++++++++++++++++++++
> 4 files changed, 57 insertions(+), 7 deletions(-)
>
> diff --git a/include/linux/ceph/ceph_features.h b/include/linux/ceph/ceph_features.h
> index 342f93d..df25dcf 100644
> --- a/include/linux/ceph/ceph_features.h
> +++ b/include/linux/ceph/ceph_features.h
> @@ -12,12 +12,14 @@
> #define CEPH_FEATURE_MONNAMES (1<<5)
> #define CEPH_FEATURE_RECONNECT_SEQ (1<<6)
> #define CEPH_FEATURE_DIRLAYOUTHASH (1<<7)
> +#define CEPH_FEATURE_CRUSH_TUNABLES (1<<18)
any reason why this is 18 and not 8?
>
> /*
> * Features supported.
> */
> #define CEPH_FEATURES_SUPPORTED_DEFAULT \
> - (CEPH_FEATURE_NOSRCADDR)
> + (CEPH_FEATURE_NOSRCADDR | \
> + CEPH_FEATURE_CRUSH_TUNABLES)
>
> #define CEPH_FEATURES_REQUIRED_DEFAULT \
> (CEPH_FEATURE_NOSRCADDR)
> diff --git a/include/linux/crush/crush.h b/include/linux/crush/crush.h
> index 7c47508..25baa28 100644
> --- a/include/linux/crush/crush.h
> +++ b/include/linux/crush/crush.h
> @@ -154,6 +154,14 @@ struct crush_map {
> __s32 max_buckets;
> __u32 max_rules;
> __s32 max_devices;
> +
> + /* choose local retries before re-descent */
> + __u32 choose_local_tries;
> + /* choose local attempts using a fallback permutation before
> + * re-descent */
> + __u32 choose_local_fallback_tries;
> + /* choose attempts before giving up */
> + __u32 choose_total_tries;
> };
>
>
> diff --git a/net/ceph/crush/mapper.c b/net/ceph/crush/mapper.c
> index d7edc24..35fce75 100644
> --- a/net/ceph/crush/mapper.c
> +++ b/net/ceph/crush/mapper.c
> @@ -306,7 +306,6 @@ static int crush_choose(const struct crush_map *map,
> int item = 0;
> int itemtype;
> int collide, reject;
> - const unsigned int orig_tries = 5; /* attempts before we fall back to search */
>
> dprintk("CHOOSE%s bucket %d x %d outpos %d numrep %d\n", recurse_to_leaf ? "_LEAF" : "",
> bucket->id, x, outpos, numrep);
> @@ -351,8 +350,9 @@ static int crush_choose(const struct crush_map *map,
> reject = 1;
> goto reject;
> }
> - if (flocal >= (in->size>>1) &&
> - flocal > orig_tries)
> + if (map->choose_local_fallback_tries > 0 &&
> + flocal >= (in->size>>1) &&
> + flocal > map->choose_local_fallback_tries)
is flocal right here or should it be ftotal?
> item = bucket_perm_choose(in, x, r);
> else
> item = crush_bucket_choose(in, x, r);
> @@ -422,13 +422,14 @@ reject:
> ftotal++;
> flocal++;
>
> - if (collide && flocal < 3)
> + if (collide && flocal <= map->choose_local_tries)
> /* retry locally a few times */
> retry_bucket = 1;
> - else if (flocal <= in->size + orig_tries)
> + else if (map->choose_local_fallback_tries > 0 &&
> + flocal <= in->size + map->choose_local_fallback_tries)
> /* exhaustive bucket search */
> retry_bucket = 1;
> - else if (ftotal < 20)
> + else if (ftotal <= map->choose_total_tries)
> /* then retry descent */
> retry_descent = 1;
> else
> diff --git a/net/ceph/osdmap.c b/net/ceph/osdmap.c
> index 9600674..3124b71 100644
> --- a/net/ceph/osdmap.c
> +++ b/net/ceph/osdmap.c
> @@ -135,6 +135,21 @@ bad:
> return -EINVAL;
> }
>
> +static int skip_name_map(void **p, void *end)
> +{
> + int len;
> + ceph_decode_32_safe(p, end, len ,bad);
> + while (len--) {
> + int strlen;
use u32 for strlen
> + *p += sizeof(u32);
> + ceph_decode_32_safe(p, end, strlen, bad);
> + *p += strlen;
> +}
> + return 0;
> +bad:
> + return -EINVAL;
> +}
> +
> static struct crush_map *crush_decode(void *pbyval, void *end)
> {
> struct crush_map *c;
> @@ -143,6 +158,7 @@ static struct crush_map *crush_decode(void *pbyval, void *end)
> void **p = &pbyval;
> void *start = pbyval;
> u32 magic;
> + u32 num_name_maps;
>
> dout("crush_decode %p to %p len %d\n", *p, end, (int)(end - *p));
>
> @@ -150,6 +166,11 @@ static struct crush_map *crush_decode(void *pbyval, void *end)
> if (c == NULL)
> return ERR_PTR(-ENOMEM);
>
> + /* set tunables to default values */
> + c->choose_local_tries = 2;
> + c->choose_local_fallback_tries = 5;
> + c->choose_total_tries = 19;
> +
> ceph_decode_need(p, end, 4*sizeof(u32), bad);
> magic = ceph_decode_32(p);
> if (magic != CRUSH_MAGIC) {
> @@ -297,7 +318,25 @@ static struct crush_map *crush_decode(void *pbyval, void *end)
> }
>
> /* ignore trailing name maps. */
> + for (num_name_maps = 0; num_name_maps < 3; num_name_maps++) {
> + err = skip_name_map(p, end);
> + if (err < 0)
> + goto done;
> + }
> +
> + /* tunables */
> + ceph_decode_need(p, end, 3*sizeof(u32), done);
> + c->choose_local_tries = ceph_decode_32(p);
> + c->choose_local_fallback_tries = ceph_decode_32(p);
> + c->choose_total_tries = ceph_decode_32(p);
> + dout("crush decode tunable choose_local_tries = %d",
> + c->choose_local_tries);
> + dout("crush decode tunable choose_local_fallback_tries = %d",
> + c->choose_local_fallback_tries);
> + dout("crush decode tunable choose_total_tries = %d",
> + c->choose_total_tries);
>
> +done:
> dout("crush_decode success\n");
> return c;
>
> --
> 1.7.9
>
> --
> To unsubscribe from this list: send the line "unsubscribe ceph-devel" in
> the body of a message to majordomo@xxxxxxxxxxxxxxx
> More majordomo info at http://vger.kernel.org/majordomo-info.html
--
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
