On Wed, Jul 29, 2020 at 06:09 AM CEST, Andrii Nakryiko wrote:
> Add test validating that all inner maps are released properly after skeleton
> is destroyed. To ensure determinism, trigger kernel-side synchronize_rcu()
> before checking map existence by their IDs.
>
> Acked-by: Song Liu <songliubraving@xxxxxx>
> Signed-off-by: Andrii Nakryiko <andriin@xxxxxx>
> ---
> .../selftests/bpf/prog_tests/btf_map_in_map.c | 124 ++++++++++++++++--
> 1 file changed, 110 insertions(+), 14 deletions(-)
>
> diff --git a/tools/testing/selftests/bpf/prog_tests/btf_map_in_map.c b/tools/testing/selftests/bpf/prog_tests/btf_map_in_map.c
> index f7ee8fa377ad..f6eee3fb933c 100644
> --- a/tools/testing/selftests/bpf/prog_tests/btf_map_in_map.c
> +++ b/tools/testing/selftests/bpf/prog_tests/btf_map_in_map.c
> @@ -5,10 +5,60 @@
>
> #include "test_btf_map_in_map.skel.h"
>
> +static int duration;
> +
> +static __u32 bpf_map_id(struct bpf_map *map)
> +{
> + struct bpf_map_info info;
> + __u32 info_len = sizeof(info);
> + int err;
> +
> + memset(&info, 0, info_len);
> + err = bpf_obj_get_info_by_fd(bpf_map__fd(map), &info, &info_len);
> + if (err)
> + return 0;
> + return info.id;
> +}
> +
> +/*
> + * Trigger synchronize_cpu() in kernel.
Nit: synchronize_*r*cu().
> + *
> + * ARRAY_OF_MAPS/HASH_OF_MAPS lookup/update operations trigger
> + * synchronize_rcu(), if looking up/updating non-NULL element. Use this fact
> + * to trigger synchronize_cpu(): create map-in-map, create a trivial ARRAY
> + * map, update map-in-map with ARRAY inner map. Then cleanup. At the end, at
> + * least one synchronize_rcu() would be called.
> + */
That's a cool trick. I'm a bit confused by "looking up/updating non-NULL
element". It looks like you're updating an element that is NULL/unset in
the code below. What am I missing?
> +static int kern_sync_rcu(void)
> +{
> + int inner_map_fd, outer_map_fd, err, zero = 0;
> +
> + inner_map_fd = bpf_create_map(BPF_MAP_TYPE_ARRAY, 4, 4, 1, 0);
> + if (CHECK(inner_map_fd < 0, "inner_map_create", "failed %d\n", -errno))
> + return -1;
> +
> + outer_map_fd = bpf_create_map_in_map(BPF_MAP_TYPE_ARRAY_OF_MAPS, NULL,
> + sizeof(int), inner_map_fd, 1, 0);
> + if (CHECK(outer_map_fd < 0, "outer_map_create", "failed %d\n", -errno)) {
> + close(inner_map_fd);
> + return -1;
> + }
> +
> + err = bpf_map_update_elem(outer_map_fd, &zero, &inner_map_fd, 0);
> + if (err)
> + err = -errno;
> + CHECK(err, "outer_map_update", "failed %d\n", err);
> + close(inner_map_fd);
> + close(outer_map_fd);
> + return err;
> +}
> +
> void test_btf_map_in_map(void)
> {
> - int duration = 0, err, key = 0, val;
> - struct test_btf_map_in_map* skel;
> + int err, key = 0, val, i;
> + struct test_btf_map_in_map *skel;
> + int outer_arr_fd, outer_hash_fd;
> + int fd, map1_fd, map2_fd, map1_id, map2_id;
>
> skel = test_btf_map_in_map__open_and_load();
> if (CHECK(!skel, "skel_open", "failed to open&load skeleton\n"))
> @@ -18,32 +68,78 @@ void test_btf_map_in_map(void)
> if (CHECK(err, "skel_attach", "skeleton attach failed: %d\n", err))
> goto cleanup;
>
> + map1_fd = bpf_map__fd(skel->maps.inner_map1);
> + map2_fd = bpf_map__fd(skel->maps.inner_map2);
> + outer_arr_fd = bpf_map__fd(skel->maps.outer_arr);
> + outer_hash_fd = bpf_map__fd(skel->maps.outer_hash);
> +
> /* inner1 = input, inner2 = input + 1 */
> - val = bpf_map__fd(skel->maps.inner_map1);
> - bpf_map_update_elem(bpf_map__fd(skel->maps.outer_arr), &key, &val, 0);
> - val = bpf_map__fd(skel->maps.inner_map2);
> - bpf_map_update_elem(bpf_map__fd(skel->maps.outer_hash), &key, &val, 0);
> + map1_fd = bpf_map__fd(skel->maps.inner_map1);
> + bpf_map_update_elem(outer_arr_fd, &key, &map1_fd, 0);
> + map2_fd = bpf_map__fd(skel->maps.inner_map2);
> + bpf_map_update_elem(outer_hash_fd, &key, &map2_fd, 0);
> skel->bss->input = 1;
> usleep(1);
>
> - bpf_map_lookup_elem(bpf_map__fd(skel->maps.inner_map1), &key, &val);
> + bpf_map_lookup_elem(map1_fd, &key, &val);
> CHECK(val != 1, "inner1", "got %d != exp %d\n", val, 1);
> - bpf_map_lookup_elem(bpf_map__fd(skel->maps.inner_map2), &key, &val);
> + bpf_map_lookup_elem(map2_fd, &key, &val);
> CHECK(val != 2, "inner2", "got %d != exp %d\n", val, 2);
>
> /* inner1 = input + 1, inner2 = input */
> - val = bpf_map__fd(skel->maps.inner_map2);
> - bpf_map_update_elem(bpf_map__fd(skel->maps.outer_arr), &key, &val, 0);
> - val = bpf_map__fd(skel->maps.inner_map1);
> - bpf_map_update_elem(bpf_map__fd(skel->maps.outer_hash), &key, &val, 0);
> + bpf_map_update_elem(outer_arr_fd, &key, &map2_fd, 0);
> + bpf_map_update_elem(outer_hash_fd, &key, &map1_fd, 0);
> skel->bss->input = 3;
> usleep(1);
>
> - bpf_map_lookup_elem(bpf_map__fd(skel->maps.inner_map1), &key, &val);
> + bpf_map_lookup_elem(map1_fd, &key, &val);
> CHECK(val != 4, "inner1", "got %d != exp %d\n", val, 4);
> - bpf_map_lookup_elem(bpf_map__fd(skel->maps.inner_map2), &key, &val);
> + bpf_map_lookup_elem(map2_fd, &key, &val);
> CHECK(val != 3, "inner2", "got %d != exp %d\n", val, 3);
>
> + for (i = 0; i < 5; i++) {
> + val = i % 2 ? map1_fd : map2_fd;
> + err = bpf_map_update_elem(outer_hash_fd, &key, &val, 0);
> + if (CHECK_FAIL(err)) {
> + printf("failed to update hash_of_maps on iter #%d\n", i);
> + goto cleanup;
> + }
> + err = bpf_map_update_elem(outer_arr_fd, &key, &val, 0);
> + if (CHECK_FAIL(err)) {
> + printf("failed to update hash_of_maps on iter #%d\n", i);
> + goto cleanup;
> + }
> + }
> +
> + map1_id = bpf_map_id(skel->maps.inner_map1);
> + map2_id = bpf_map_id(skel->maps.inner_map2);
> + CHECK(map1_id == 0, "map1_id", "failed to get ID 1\n");
> + CHECK(map2_id == 0, "map2_id", "failed to get ID 2\n");
> +
> + test_btf_map_in_map__destroy(skel);
> + skel = NULL;
> +
> + /* we need to either wait for or force synchronize_rcu(), before
> + * checking for "still exists" condition, otherwise map could still be
> + * resolvable by ID, causing false positives.
> + *
> + * Older kernels (5.8 and earlier) freed map only after two
> + * synchronize_rcu()s, so trigger two, to be entirely sure.
> + */
> + CHECK(kern_sync_rcu(), "sync_rcu", "failed\n");
> + CHECK(kern_sync_rcu(), "sync_rcu", "failed\n");
> +
> + fd = bpf_map_get_fd_by_id(map1_id);
> + if (CHECK(fd >= 0, "map1_leak", "inner_map1 leaked!\n")) {
> + close(fd);
> + goto cleanup;
> + }
> + fd = bpf_map_get_fd_by_id(map2_id);
> + if (CHECK(fd >= 0, "map2_leak", "inner_map2 leaked!\n")) {
> + close(fd);
> + goto cleanup;
> + }
> +
> cleanup:
> test_btf_map_in_map__destroy(skel);
> }
