[QUESTION] Dying cgroups in subsystem cpu, pids and named hierarchy systemd

[chenzhou <chenzhou10@xxxxxxxxxx>]

Hi all,

When i do some tests with kernel-4.19.x，i found some dying cgroups.

There are dying cgroup in subsystem cpu/cpuacct, memory, pids and named hierarchy systemd.

The root cause of dying cgroups in memory subsystem is that some charged pages aren't gone.

I don't figure out why there are dying cgroup in cpu, pids and named hierarchy systemd.

I used to turn on/off Delegate and do operation "systemctl daemon-reload" in the machine,

but now can't reproduce it.

Is this normal and what may cause this or any suggestions about this?


The details about the machine with problem is as below:
# mount -t cgroup
cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd)
cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,pids)
cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,memory)
cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,blkio)
cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,devices)
cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,perf_event)
cgroup on /sys/fs/cgroup/rdma type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,rdma)
cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,freezer)
cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,net_cls,net_prio)
cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,cpuset)
cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,hugetlb)
memory on /cgroup/memory type cgroup (rw,relatime,seclabel,memory)
cpu,cpuacct on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,relatime,seclabel,cpu,cpuacct)

# cat /proc/cgroups
#subsys_name    hierarchy    num_cgroups    enabled
cpuset    12    109    1
cpu    10    110    1
cpuacct    10    110    1
blkio    4    109    1
memory    3    2571    1
devices    5    109    1
freezer    9    109    1
net_cls    11    1    1
perf_event    6    1    1
net_prio    11    1    1
hugetlb    13    1    1
pids    2    480    1
...

# crash vmlinux /proc/kcore
crash> list cgroup_roots
ffff000081272558
...

list -H ffff000081272558 -s cgroup_root.subsys_mask,hierarchy_id,cgrp.nr_dying_descendants -l cgroup_root.root_list
ffff8007ff67c6f8        // cpu,cpuacct
  subsys_mask = 6
  hierarchy_id = 10
  cgrp.nr_dying_descendants = 1,
...
ffff8007ff6726f8        // memory
  subsys_mask = 16
  hierarchy_id = 3
  cgrp.nr_dying_descendants = 2036,
ffff800fb579c6f8        // pids
  subsys_mask = 2048
  hierarchy_id = 2
  cgrp.nr_dying_descendants = 1,
ffff800fb57986f8        // named systemd
  subsys_mask = 0
  hierarchy_id = 1
  cgrp.nr_dying_descendants = 1,
...


For the cpu subsystem, the dying cgroup is /sys/fs/cgroup/cpu/user.slice:

crash> p root_task_group.css.cgroup
$4 = (struct cgroup *) 0xffff8007ff67c010
crash> cgroup_subsys_state ffff8007ff67c010 -ox
struct cgroup_subsys_state {
  [ffff8007ff67c010] struct cgroup *cgroup;
  [ffff8007ff67c018] struct cgroup_subsys *ss;
  [ffff8007ff67c020] struct percpu_ref refcnt;
  [ffff8007ff67c058] struct list_head sibling;
  [ffff8007ff67c068] struct list_head children;
  [ffff8007ff67c078] struct list_head rstat_css_node;
  [ffff8007ff67c088] int id;
  [ffff8007ff67c08c] unsigned int flags;
  [ffff8007ff67c090] u64 serial_nr;
  [ffff8007ff67c098] atomic_t online_cnt;
  [ffff8007ff67c0a0] struct work_struct destroy_work;
  [ffff8007ff67c0e0] struct rcu_work destroy_rwork;
  [ffff8007ff67c138] struct cgroup_subsys_state *parent;
}
SIZE: 0x130
crash> list -H ffff8007ff67c068 -s cgroup_subsys_state.cgroup -l cgroup_subsys_state.sibling
ffff800faff65848
  cgroup = 0xffff800faff65800
ffff800fa18b1048
  cgroup = 0xffff800fa18b1000
ffff800fa22c0848
  cgroup = 0xffff800fa22c0800
ffff800fa940e048
  cgroup = 0xffff800fa940e000
crash>
crash> cgroup.self.refcnt,flags,kn 0xffff800fa18b1000 -x
  self.refcnt = {
    count = {
      counter = 0x3
    },
    percpu_count_ptr = 0xfffefdf041334c83,
    release = 0xffff0000801c1ea0 <css_release>,
    confirm_switch = 0x0,
    force_atomic = 0x0,
    rcu = {
      next = 0xffff80083365da30,
      func = 0xffff0000804fb748 <percpu_ref_switch_to_atomic_rcu>
    }
  },
  flags = 0x0
  kn = 0xffff800bbdb1b110
crash>
crash> kernfs_node.name 0xffff800bbdb1b110
  name = 0xffff800fa54d9c00 "user.slice"


Thanks,

Chen Zhou