On Mon 24-06-24 17:58:56, Roman Gushchin wrote:
> Unlike the legacy cgroup v1 memory controller, cgroup v2 memory
> controller doesn't support moving charged pages between cgroups.
>
> It's a fairly large and complicated code which created a number
> of problems in the past. Let's move this code into memcontrol-v1.c.
> It shaves off 1k lines from memcontrol.c. It's also another step
> towards making the legacy memory controller code optionally compiled.
Acked-by: Michal Hocko <mhocko@xxxxxxxx>
>
> Signed-off-by: Roman Gushchin <roman.gushchin@xxxxxxxxx>
> ---
> mm/memcontrol-v1.c | 981 +++++++++++++++++++++++++++++++++++++++++++
> mm/memcontrol-v1.h | 30 ++
> mm/memcontrol.c | 1004 +-------------------------------------------
> 3 files changed, 1019 insertions(+), 996 deletions(-)
>
> diff --git a/mm/memcontrol-v1.c b/mm/memcontrol-v1.c
> index 68e2f1a718d3..f4c8bec5ae1b 100644
> --- a/mm/memcontrol-v1.c
> +++ b/mm/memcontrol-v1.c
> @@ -3,7 +3,12 @@
> #include <linux/memcontrol.h>
> #include <linux/swap.h>
> #include <linux/mm_inline.h>
> +#include <linux/pagewalk.h>
> +#include <linux/backing-dev.h>
> +#include <linux/swap_cgroup.h>
>
> +#include "internal.h"
> +#include "swap.h"
> #include "memcontrol-v1.h"
>
> /*
> @@ -30,6 +35,31 @@ static struct mem_cgroup_tree soft_limit_tree __read_mostly;
> #define MEM_CGROUP_MAX_RECLAIM_LOOPS 100
> #define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS 2
>
> +/* Stuffs for move charges at task migration. */
> +/*
> + * Types of charges to be moved.
> + */
> +#define MOVE_ANON 0x1U
> +#define MOVE_FILE 0x2U
> +#define MOVE_MASK (MOVE_ANON | MOVE_FILE)
> +
> +/* "mc" and its members are protected by cgroup_mutex */
> +static struct move_charge_struct {
> + spinlock_t lock; /* for from, to */
> + struct mm_struct *mm;
> + struct mem_cgroup *from;
> + struct mem_cgroup *to;
> + unsigned long flags;
> + unsigned long precharge;
> + unsigned long moved_charge;
> + unsigned long moved_swap;
> + struct task_struct *moving_task; /* a task moving charges */
> + wait_queue_head_t waitq; /* a waitq for other context */
> +} mc = {
> + .lock = __SPIN_LOCK_UNLOCKED(mc.lock),
> + .waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
> +};
> +
> static void __mem_cgroup_insert_exceeded(struct mem_cgroup_per_node *mz,
> struct mem_cgroup_tree_per_node *mctz,
> unsigned long new_usage_in_excess)
> @@ -325,6 +355,957 @@ unsigned long memcg1_soft_limit_reclaim(pg_data_t *pgdat, int order,
> return nr_reclaimed;
> }
>
> +/*
> + * A routine for checking "mem" is under move_account() or not.
> + *
> + * Checking a cgroup is mc.from or mc.to or under hierarchy of
> + * moving cgroups. This is for waiting at high-memory pressure
> + * caused by "move".
> + */
> +static bool mem_cgroup_under_move(struct mem_cgroup *memcg)
> +{
> + struct mem_cgroup *from;
> + struct mem_cgroup *to;
> + bool ret = false;
> + /*
> + * Unlike task_move routines, we access mc.to, mc.from not under
> + * mutual exclusion by cgroup_mutex. Here, we take spinlock instead.
> + */
> + spin_lock(&mc.lock);
> + from = mc.from;
> + to = mc.to;
> + if (!from)
> + goto unlock;
> +
> + ret = mem_cgroup_is_descendant(from, memcg) ||
> + mem_cgroup_is_descendant(to, memcg);
> +unlock:
> + spin_unlock(&mc.lock);
> + return ret;
> +}
> +
> +bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg)
> +{
> + if (mc.moving_task && current != mc.moving_task) {
> + if (mem_cgroup_under_move(memcg)) {
> + DEFINE_WAIT(wait);
> + prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
> + /* moving charge context might have finished. */
> + if (mc.moving_task)
> + schedule();
> + finish_wait(&mc.waitq, &wait);
> + return true;
> + }
> + }
> + return false;
> +}
> +
> +/**
> + * folio_memcg_lock - Bind a folio to its memcg.
> + * @folio: The folio.
> + *
> + * This function prevents unlocked LRU folios from being moved to
> + * another cgroup.
> + *
> + * It ensures lifetime of the bound memcg. The caller is responsible
> + * for the lifetime of the folio.
> + */
> +void folio_memcg_lock(struct folio *folio)
> +{
> + struct mem_cgroup *memcg;
> + unsigned long flags;
> +
> + /*
> + * The RCU lock is held throughout the transaction. The fast
> + * path can get away without acquiring the memcg->move_lock
> + * because page moving starts with an RCU grace period.
> + */
> + rcu_read_lock();
> +
> + if (mem_cgroup_disabled())
> + return;
> +again:
> + memcg = folio_memcg(folio);
> + if (unlikely(!memcg))
> + return;
> +
> +#ifdef CONFIG_PROVE_LOCKING
> + local_irq_save(flags);
> + might_lock(&memcg->move_lock);
> + local_irq_restore(flags);
> +#endif
> +
> + if (atomic_read(&memcg->moving_account) <= 0)
> + return;
> +
> + spin_lock_irqsave(&memcg->move_lock, flags);
> + if (memcg != folio_memcg(folio)) {
> + spin_unlock_irqrestore(&memcg->move_lock, flags);
> + goto again;
> + }
> +
> + /*
> + * When charge migration first begins, we can have multiple
> + * critical sections holding the fast-path RCU lock and one
> + * holding the slowpath move_lock. Track the task who has the
> + * move_lock for folio_memcg_unlock().
> + */
> + memcg->move_lock_task = current;
> + memcg->move_lock_flags = flags;
> +}
> +
> +static void __folio_memcg_unlock(struct mem_cgroup *memcg)
> +{
> + if (memcg && memcg->move_lock_task == current) {
> + unsigned long flags = memcg->move_lock_flags;
> +
> + memcg->move_lock_task = NULL;
> + memcg->move_lock_flags = 0;
> +
> + spin_unlock_irqrestore(&memcg->move_lock, flags);
> + }
> +
> + rcu_read_unlock();
> +}
> +
> +/**
> + * folio_memcg_unlock - Release the binding between a folio and its memcg.
> + * @folio: The folio.
> + *
> + * This releases the binding created by folio_memcg_lock(). This does
> + * not change the accounting of this folio to its memcg, but it does
> + * permit others to change it.
> + */
> +void folio_memcg_unlock(struct folio *folio)
> +{
> + __folio_memcg_unlock(folio_memcg(folio));
> +}
> +
> +#ifdef CONFIG_SWAP
> +/**
> + * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
> + * @entry: swap entry to be moved
> + * @from: mem_cgroup which the entry is moved from
> + * @to: mem_cgroup which the entry is moved to
> + *
> + * It succeeds only when the swap_cgroup's record for this entry is the same
> + * as the mem_cgroup's id of @from.
> + *
> + * Returns 0 on success, -EINVAL on failure.
> + *
> + * The caller must have charged to @to, IOW, called page_counter_charge() about
> + * both res and memsw, and called css_get().
> + */
> +static int mem_cgroup_move_swap_account(swp_entry_t entry,
> + struct mem_cgroup *from, struct mem_cgroup *to)
> +{
> + unsigned short old_id, new_id;
> +
> + old_id = mem_cgroup_id(from);
> + new_id = mem_cgroup_id(to);
> +
> + if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
> + mod_memcg_state(from, MEMCG_SWAP, -1);
> + mod_memcg_state(to, MEMCG_SWAP, 1);
> + return 0;
> + }
> + return -EINVAL;
> +}
> +#else
> +static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
> + struct mem_cgroup *from, struct mem_cgroup *to)
> +{
> + return -EINVAL;
> +}
> +#endif
> +
> +u64 mem_cgroup_move_charge_read(struct cgroup_subsys_state *css,
> + struct cftype *cft)
> +{
> + return mem_cgroup_from_css(css)->move_charge_at_immigrate;
> +}
> +
> +#ifdef CONFIG_MMU
> +int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
> + struct cftype *cft, u64 val)
> +{
> + struct mem_cgroup *memcg = mem_cgroup_from_css(css);
> +
> + pr_warn_once("Cgroup memory moving (move_charge_at_immigrate) is deprecated. "
> + "Please report your usecase to linux-mm@xxxxxxxxx if you "
> + "depend on this functionality.\n");
> +
> + if (val & ~MOVE_MASK)
> + return -EINVAL;
> +
> + /*
> + * No kind of locking is needed in here, because ->can_attach() will
> + * check this value once in the beginning of the process, and then carry
> + * on with stale data. This means that changes to this value will only
> + * affect task migrations starting after the change.
> + */
> + memcg->move_charge_at_immigrate = val;
> + return 0;
> +}
> +#else
> +int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
> + struct cftype *cft, u64 val)
> +{
> + return -ENOSYS;
> +}
> +#endif
> +
> +#ifdef CONFIG_MMU
> +/* Handlers for move charge at task migration. */
> +static int mem_cgroup_do_precharge(unsigned long count)
> +{
> + int ret;
> +
> + /* Try a single bulk charge without reclaim first, kswapd may wake */
> + ret = try_charge(mc.to, GFP_KERNEL & ~__GFP_DIRECT_RECLAIM, count);
> + if (!ret) {
> + mc.precharge += count;
> + return ret;
> + }
> +
> + /* Try charges one by one with reclaim, but do not retry */
> + while (count--) {
> + ret = try_charge(mc.to, GFP_KERNEL | __GFP_NORETRY, 1);
> + if (ret)
> + return ret;
> + mc.precharge++;
> + cond_resched();
> + }
> + return 0;
> +}
> +
> +union mc_target {
> + struct folio *folio;
> + swp_entry_t ent;
> +};
> +
> +enum mc_target_type {
> + MC_TARGET_NONE = 0,
> + MC_TARGET_PAGE,
> + MC_TARGET_SWAP,
> + MC_TARGET_DEVICE,
> +};
> +
> +static struct page *mc_handle_present_pte(struct vm_area_struct *vma,
> + unsigned long addr, pte_t ptent)
> +{
> + struct page *page = vm_normal_page(vma, addr, ptent);
> +
> + if (!page)
> + return NULL;
> + if (PageAnon(page)) {
> + if (!(mc.flags & MOVE_ANON))
> + return NULL;
> + } else {
> + if (!(mc.flags & MOVE_FILE))
> + return NULL;
> + }
> + get_page(page);
> +
> + return page;
> +}
> +
> +#if defined(CONFIG_SWAP) || defined(CONFIG_DEVICE_PRIVATE)
> +static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
> + pte_t ptent, swp_entry_t *entry)
> +{
> + struct page *page = NULL;
> + swp_entry_t ent = pte_to_swp_entry(ptent);
> +
> + if (!(mc.flags & MOVE_ANON))
> + return NULL;
> +
> + /*
> + * Handle device private pages that are not accessible by the CPU, but
> + * stored as special swap entries in the page table.
> + */
> + if (is_device_private_entry(ent)) {
> + page = pfn_swap_entry_to_page(ent);
> + if (!get_page_unless_zero(page))
> + return NULL;
> + return page;
> + }
> +
> + if (non_swap_entry(ent))
> + return NULL;
> +
> + /*
> + * Because swap_cache_get_folio() updates some statistics counter,
> + * we call find_get_page() with swapper_space directly.
> + */
> + page = find_get_page(swap_address_space(ent), swap_cache_index(ent));
> + entry->val = ent.val;
> +
> + return page;
> +}
> +#else
> +static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
> + pte_t ptent, swp_entry_t *entry)
> +{
> + return NULL;
> +}
> +#endif
> +
> +static struct page *mc_handle_file_pte(struct vm_area_struct *vma,
> + unsigned long addr, pte_t ptent)
> +{
> + unsigned long index;
> + struct folio *folio;
> +
> + if (!vma->vm_file) /* anonymous vma */
> + return NULL;
> + if (!(mc.flags & MOVE_FILE))
> + return NULL;
> +
> + /* folio is moved even if it's not RSS of this task(page-faulted). */
> + /* shmem/tmpfs may report page out on swap: account for that too. */
> + index = linear_page_index(vma, addr);
> + folio = filemap_get_incore_folio(vma->vm_file->f_mapping, index);
> + if (IS_ERR(folio))
> + return NULL;
> + return folio_file_page(folio, index);
> +}
> +
> +/**
> + * mem_cgroup_move_account - move account of the folio
> + * @folio: The folio.
> + * @compound: charge the page as compound or small page
> + * @from: mem_cgroup which the folio is moved from.
> + * @to: mem_cgroup which the folio is moved to. @from != @to.
> + *
> + * The folio must be locked and not on the LRU.
> + *
> + * This function doesn't do "charge" to new cgroup and doesn't do "uncharge"
> + * from old cgroup.
> + */
> +static int mem_cgroup_move_account(struct folio *folio,
> + bool compound,
> + struct mem_cgroup *from,
> + struct mem_cgroup *to)
> +{
> + struct lruvec *from_vec, *to_vec;
> + struct pglist_data *pgdat;
> + unsigned int nr_pages = compound ? folio_nr_pages(folio) : 1;
> + int nid, ret;
> +
> + VM_BUG_ON(from == to);
> + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
> + VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);
> + VM_BUG_ON(compound && !folio_test_large(folio));
> +
> + ret = -EINVAL;
> + if (folio_memcg(folio) != from)
> + goto out;
> +
> + pgdat = folio_pgdat(folio);
> + from_vec = mem_cgroup_lruvec(from, pgdat);
> + to_vec = mem_cgroup_lruvec(to, pgdat);
> +
> + folio_memcg_lock(folio);
> +
> + if (folio_test_anon(folio)) {
> + if (folio_mapped(folio)) {
> + __mod_lruvec_state(from_vec, NR_ANON_MAPPED, -nr_pages);
> + __mod_lruvec_state(to_vec, NR_ANON_MAPPED, nr_pages);
> + if (folio_test_pmd_mappable(folio)) {
> + __mod_lruvec_state(from_vec, NR_ANON_THPS,
> + -nr_pages);
> + __mod_lruvec_state(to_vec, NR_ANON_THPS,
> + nr_pages);
> + }
> + }
> + } else {
> + __mod_lruvec_state(from_vec, NR_FILE_PAGES, -nr_pages);
> + __mod_lruvec_state(to_vec, NR_FILE_PAGES, nr_pages);
> +
> + if (folio_test_swapbacked(folio)) {
> + __mod_lruvec_state(from_vec, NR_SHMEM, -nr_pages);
> + __mod_lruvec_state(to_vec, NR_SHMEM, nr_pages);
> + }
> +
> + if (folio_mapped(folio)) {
> + __mod_lruvec_state(from_vec, NR_FILE_MAPPED, -nr_pages);
> + __mod_lruvec_state(to_vec, NR_FILE_MAPPED, nr_pages);
> + }
> +
> + if (folio_test_dirty(folio)) {
> + struct address_space *mapping = folio_mapping(folio);
> +
> + if (mapping_can_writeback(mapping)) {
> + __mod_lruvec_state(from_vec, NR_FILE_DIRTY,
> + -nr_pages);
> + __mod_lruvec_state(to_vec, NR_FILE_DIRTY,
> + nr_pages);
> + }
> + }
> + }
> +
> +#ifdef CONFIG_SWAP
> + if (folio_test_swapcache(folio)) {
> + __mod_lruvec_state(from_vec, NR_SWAPCACHE, -nr_pages);
> + __mod_lruvec_state(to_vec, NR_SWAPCACHE, nr_pages);
> + }
> +#endif
> + if (folio_test_writeback(folio)) {
> + __mod_lruvec_state(from_vec, NR_WRITEBACK, -nr_pages);
> + __mod_lruvec_state(to_vec, NR_WRITEBACK, nr_pages);
> + }
> +
> + /*
> + * All state has been migrated, let's switch to the new memcg.
> + *
> + * It is safe to change page's memcg here because the page
> + * is referenced, charged, isolated, and locked: we can't race
> + * with (un)charging, migration, LRU putback, or anything else
> + * that would rely on a stable page's memory cgroup.
> + *
> + * Note that folio_memcg_lock is a memcg lock, not a page lock,
> + * to save space. As soon as we switch page's memory cgroup to a
> + * new memcg that isn't locked, the above state can change
> + * concurrently again. Make sure we're truly done with it.
> + */
> + smp_mb();
> +
> + css_get(&to->css);
> + css_put(&from->css);
> +
> + folio->memcg_data = (unsigned long)to;
> +
> + __folio_memcg_unlock(from);
> +
> + ret = 0;
> + nid = folio_nid(folio);
> +
> + local_irq_disable();
> + mem_cgroup_charge_statistics(to, nr_pages);
> + memcg_check_events(to, nid);
> + mem_cgroup_charge_statistics(from, -nr_pages);
> + memcg_check_events(from, nid);
> + local_irq_enable();
> +out:
> + return ret;
> +}
> +
> +/**
> + * get_mctgt_type - get target type of moving charge
> + * @vma: the vma the pte to be checked belongs
> + * @addr: the address corresponding to the pte to be checked
> + * @ptent: the pte to be checked
> + * @target: the pointer the target page or swap ent will be stored(can be NULL)
> + *
> + * Context: Called with pte lock held.
> + * Return:
> + * * MC_TARGET_NONE - If the pte is not a target for move charge.
> + * * MC_TARGET_PAGE - If the page corresponding to this pte is a target for
> + * move charge. If @target is not NULL, the folio is stored in target->folio
> + * with extra refcnt taken (Caller should release it).
> + * * MC_TARGET_SWAP - If the swap entry corresponding to this pte is a
> + * target for charge migration. If @target is not NULL, the entry is
> + * stored in target->ent.
> + * * MC_TARGET_DEVICE - Like MC_TARGET_PAGE but page is device memory and
> + * thus not on the lru. For now such page is charged like a regular page
> + * would be as it is just special memory taking the place of a regular page.
> + * See Documentations/vm/hmm.txt and include/linux/hmm.h
> + */
> +static enum mc_target_type get_mctgt_type(struct vm_area_struct *vma,
> + unsigned long addr, pte_t ptent, union mc_target *target)
> +{
> + struct page *page = NULL;
> + struct folio *folio;
> + enum mc_target_type ret = MC_TARGET_NONE;
> + swp_entry_t ent = { .val = 0 };
> +
> + if (pte_present(ptent))
> + page = mc_handle_present_pte(vma, addr, ptent);
> + else if (pte_none_mostly(ptent))
> + /*
> + * PTE markers should be treated as a none pte here, separated
> + * from other swap handling below.
> + */
> + page = mc_handle_file_pte(vma, addr, ptent);
> + else if (is_swap_pte(ptent))
> + page = mc_handle_swap_pte(vma, ptent, &ent);
> +
> + if (page)
> + folio = page_folio(page);
> + if (target && page) {
> + if (!folio_trylock(folio)) {
> + folio_put(folio);
> + return ret;
> + }
> + /*
> + * page_mapped() must be stable during the move. This
> + * pte is locked, so if it's present, the page cannot
> + * become unmapped. If it isn't, we have only partial
> + * control over the mapped state: the page lock will
> + * prevent new faults against pagecache and swapcache,
> + * so an unmapped page cannot become mapped. However,
> + * if the page is already mapped elsewhere, it can
> + * unmap, and there is nothing we can do about it.
> + * Alas, skip moving the page in this case.
> + */
> + if (!pte_present(ptent) && page_mapped(page)) {
> + folio_unlock(folio);
> + folio_put(folio);
> + return ret;
> + }
> + }
> +
> + if (!page && !ent.val)
> + return ret;
> + if (page) {
> + /*
> + * Do only loose check w/o serialization.
> + * mem_cgroup_move_account() checks the page is valid or
> + * not under LRU exclusion.
> + */
> + if (folio_memcg(folio) == mc.from) {
> + ret = MC_TARGET_PAGE;
> + if (folio_is_device_private(folio) ||
> + folio_is_device_coherent(folio))
> + ret = MC_TARGET_DEVICE;
> + if (target)
> + target->folio = folio;
> + }
> + if (!ret || !target) {
> + if (target)
> + folio_unlock(folio);
> + folio_put(folio);
> + }
> + }
> + /*
> + * There is a swap entry and a page doesn't exist or isn't charged.
> + * But we cannot move a tail-page in a THP.
> + */
> + if (ent.val && !ret && (!page || !PageTransCompound(page)) &&
> + mem_cgroup_id(mc.from) == lookup_swap_cgroup_id(ent)) {
> + ret = MC_TARGET_SWAP;
> + if (target)
> + target->ent = ent;
> + }
> + return ret;
> +}
> +
> +#ifdef CONFIG_TRANSPARENT_HUGEPAGE
> +/*
> + * We don't consider PMD mapped swapping or file mapped pages because THP does
> + * not support them for now.
> + * Caller should make sure that pmd_trans_huge(pmd) is true.
> + */
> +static enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
> + unsigned long addr, pmd_t pmd, union mc_target *target)
> +{
> + struct page *page = NULL;
> + struct folio *folio;
> + enum mc_target_type ret = MC_TARGET_NONE;
> +
> + if (unlikely(is_swap_pmd(pmd))) {
> + VM_BUG_ON(thp_migration_supported() &&
> + !is_pmd_migration_entry(pmd));
> + return ret;
> + }
> + page = pmd_page(pmd);
> + VM_BUG_ON_PAGE(!page || !PageHead(page), page);
> + folio = page_folio(page);
> + if (!(mc.flags & MOVE_ANON))
> + return ret;
> + if (folio_memcg(folio) == mc.from) {
> + ret = MC_TARGET_PAGE;
> + if (target) {
> + folio_get(folio);
> + if (!folio_trylock(folio)) {
> + folio_put(folio);
> + return MC_TARGET_NONE;
> + }
> + target->folio = folio;
> + }
> + }
> + return ret;
> +}
> +#else
> +static inline enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
> + unsigned long addr, pmd_t pmd, union mc_target *target)
> +{
> + return MC_TARGET_NONE;
> +}
> +#endif
> +
> +static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd,
> + unsigned long addr, unsigned long end,
> + struct mm_walk *walk)
> +{
> + struct vm_area_struct *vma = walk->vma;
> + pte_t *pte;
> + spinlock_t *ptl;
> +
> + ptl = pmd_trans_huge_lock(pmd, vma);
> + if (ptl) {
> + /*
> + * Note their can not be MC_TARGET_DEVICE for now as we do not
> + * support transparent huge page with MEMORY_DEVICE_PRIVATE but
> + * this might change.
> + */
> + if (get_mctgt_type_thp(vma, addr, *pmd, NULL) == MC_TARGET_PAGE)
> + mc.precharge += HPAGE_PMD_NR;
> + spin_unlock(ptl);
> + return 0;
> + }
> +
> + pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
> + if (!pte)
> + return 0;
> + for (; addr != end; pte++, addr += PAGE_SIZE)
> + if (get_mctgt_type(vma, addr, ptep_get(pte), NULL))
> + mc.precharge++; /* increment precharge temporarily */
> + pte_unmap_unlock(pte - 1, ptl);
> + cond_resched();
> +
> + return 0;
> +}
> +
> +static const struct mm_walk_ops precharge_walk_ops = {
> + .pmd_entry = mem_cgroup_count_precharge_pte_range,
> + .walk_lock = PGWALK_RDLOCK,
> +};
> +
> +static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm)
> +{
> + unsigned long precharge;
> +
> + mmap_read_lock(mm);
> + walk_page_range(mm, 0, ULONG_MAX, &precharge_walk_ops, NULL);
> + mmap_read_unlock(mm);
> +
> + precharge = mc.precharge;
> + mc.precharge = 0;
> +
> + return precharge;
> +}
> +
> +static int mem_cgroup_precharge_mc(struct mm_struct *mm)
> +{
> + unsigned long precharge = mem_cgroup_count_precharge(mm);
> +
> + VM_BUG_ON(mc.moving_task);
> + mc.moving_task = current;
> + return mem_cgroup_do_precharge(precharge);
> +}
> +
> +/* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */
> +static void __mem_cgroup_clear_mc(void)
> +{
> + struct mem_cgroup *from = mc.from;
> + struct mem_cgroup *to = mc.to;
> +
> + /* we must uncharge all the leftover precharges from mc.to */
> + if (mc.precharge) {
> + mem_cgroup_cancel_charge(mc.to, mc.precharge);
> + mc.precharge = 0;
> + }
> + /*
> + * we didn't uncharge from mc.from at mem_cgroup_move_account(), so
> + * we must uncharge here.
> + */
> + if (mc.moved_charge) {
> + mem_cgroup_cancel_charge(mc.from, mc.moved_charge);
> + mc.moved_charge = 0;
> + }
> + /* we must fixup refcnts and charges */
> + if (mc.moved_swap) {
> + /* uncharge swap account from the old cgroup */
> + if (!mem_cgroup_is_root(mc.from))
> + page_counter_uncharge(&mc.from->memsw, mc.moved_swap);
> +
> + mem_cgroup_id_put_many(mc.from, mc.moved_swap);
> +
> + /*
> + * we charged both to->memory and to->memsw, so we
> + * should uncharge to->memory.
> + */
> + if (!mem_cgroup_is_root(mc.to))
> + page_counter_uncharge(&mc.to->memory, mc.moved_swap);
> +
> + mc.moved_swap = 0;
> + }
> + memcg_oom_recover(from);
> + memcg_oom_recover(to);
> + wake_up_all(&mc.waitq);
> +}
> +
> +static void mem_cgroup_clear_mc(void)
> +{
> + struct mm_struct *mm = mc.mm;
> +
> + /*
> + * we must clear moving_task before waking up waiters at the end of
> + * task migration.
> + */
> + mc.moving_task = NULL;
> + __mem_cgroup_clear_mc();
> + spin_lock(&mc.lock);
> + mc.from = NULL;
> + mc.to = NULL;
> + mc.mm = NULL;
> + spin_unlock(&mc.lock);
> +
> + mmput(mm);
> +}
> +
> +int mem_cgroup_can_attach(struct cgroup_taskset *tset)
> +{
> + struct cgroup_subsys_state *css;
> + struct mem_cgroup *memcg = NULL; /* unneeded init to make gcc happy */
> + struct mem_cgroup *from;
> + struct task_struct *leader, *p;
> + struct mm_struct *mm;
> + unsigned long move_flags;
> + int ret = 0;
> +
> + /* charge immigration isn't supported on the default hierarchy */
> + if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
> + return 0;
> +
> + /*
> + * Multi-process migrations only happen on the default hierarchy
> + * where charge immigration is not used. Perform charge
> + * immigration if @tset contains a leader and whine if there are
> + * multiple.
> + */
> + p = NULL;
> + cgroup_taskset_for_each_leader(leader, css, tset) {
> + WARN_ON_ONCE(p);
> + p = leader;
> + memcg = mem_cgroup_from_css(css);
> + }
> + if (!p)
> + return 0;
> +
> + /*
> + * We are now committed to this value whatever it is. Changes in this
> + * tunable will only affect upcoming migrations, not the current one.
> + * So we need to save it, and keep it going.
> + */
> + move_flags = READ_ONCE(memcg->move_charge_at_immigrate);
> + if (!move_flags)
> + return 0;
> +
> + from = mem_cgroup_from_task(p);
> +
> + VM_BUG_ON(from == memcg);
> +
> + mm = get_task_mm(p);
> + if (!mm)
> + return 0;
> + /* We move charges only when we move a owner of the mm */
> + if (mm->owner == p) {
> + VM_BUG_ON(mc.from);
> + VM_BUG_ON(mc.to);
> + VM_BUG_ON(mc.precharge);
> + VM_BUG_ON(mc.moved_charge);
> + VM_BUG_ON(mc.moved_swap);
> +
> + spin_lock(&mc.lock);
> + mc.mm = mm;
> + mc.from = from;
> + mc.to = memcg;
> + mc.flags = move_flags;
> + spin_unlock(&mc.lock);
> + /* We set mc.moving_task later */
> +
> + ret = mem_cgroup_precharge_mc(mm);
> + if (ret)
> + mem_cgroup_clear_mc();
> + } else {
> + mmput(mm);
> + }
> + return ret;
> +}
> +
> +void mem_cgroup_cancel_attach(struct cgroup_taskset *tset)
> +{
> + if (mc.to)
> + mem_cgroup_clear_mc();
> +}
> +
> +static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
> + unsigned long addr, unsigned long end,
> + struct mm_walk *walk)
> +{
> + int ret = 0;
> + struct vm_area_struct *vma = walk->vma;
> + pte_t *pte;
> + spinlock_t *ptl;
> + enum mc_target_type target_type;
> + union mc_target target;
> + struct folio *folio;
> +
> + ptl = pmd_trans_huge_lock(pmd, vma);
> + if (ptl) {
> + if (mc.precharge < HPAGE_PMD_NR) {
> + spin_unlock(ptl);
> + return 0;
> + }
> + target_type = get_mctgt_type_thp(vma, addr, *pmd, &target);
> + if (target_type == MC_TARGET_PAGE) {
> + folio = target.folio;
> + if (folio_isolate_lru(folio)) {
> + if (!mem_cgroup_move_account(folio, true,
> + mc.from, mc.to)) {
> + mc.precharge -= HPAGE_PMD_NR;
> + mc.moved_charge += HPAGE_PMD_NR;
> + }
> + folio_putback_lru(folio);
> + }
> + folio_unlock(folio);
> + folio_put(folio);
> + } else if (target_type == MC_TARGET_DEVICE) {
> + folio = target.folio;
> + if (!mem_cgroup_move_account(folio, true,
> + mc.from, mc.to)) {
> + mc.precharge -= HPAGE_PMD_NR;
> + mc.moved_charge += HPAGE_PMD_NR;
> + }
> + folio_unlock(folio);
> + folio_put(folio);
> + }
> + spin_unlock(ptl);
> + return 0;
> + }
> +
> +retry:
> + pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
> + if (!pte)
> + return 0;
> + for (; addr != end; addr += PAGE_SIZE) {
> + pte_t ptent = ptep_get(pte++);
> + bool device = false;
> + swp_entry_t ent;
> +
> + if (!mc.precharge)
> + break;
> +
> + switch (get_mctgt_type(vma, addr, ptent, &target)) {
> + case MC_TARGET_DEVICE:
> + device = true;
> + fallthrough;
> + case MC_TARGET_PAGE:
> + folio = target.folio;
> + /*
> + * We can have a part of the split pmd here. Moving it
> + * can be done but it would be too convoluted so simply
> + * ignore such a partial THP and keep it in original
> + * memcg. There should be somebody mapping the head.
> + */
> + if (folio_test_large(folio))
> + goto put;
> + if (!device && !folio_isolate_lru(folio))
> + goto put;
> + if (!mem_cgroup_move_account(folio, false,
> + mc.from, mc.to)) {
> + mc.precharge--;
> + /* we uncharge from mc.from later. */
> + mc.moved_charge++;
> + }
> + if (!device)
> + folio_putback_lru(folio);
> +put: /* get_mctgt_type() gets & locks the page */
> + folio_unlock(folio);
> + folio_put(folio);
> + break;
> + case MC_TARGET_SWAP:
> + ent = target.ent;
> + if (!mem_cgroup_move_swap_account(ent, mc.from, mc.to)) {
> + mc.precharge--;
> + mem_cgroup_id_get_many(mc.to, 1);
> + /* we fixup other refcnts and charges later. */
> + mc.moved_swap++;
> + }
> + break;
> + default:
> + break;
> + }
> + }
> + pte_unmap_unlock(pte - 1, ptl);
> + cond_resched();
> +
> + if (addr != end) {
> + /*
> + * We have consumed all precharges we got in can_attach().
> + * We try charge one by one, but don't do any additional
> + * charges to mc.to if we have failed in charge once in attach()
> + * phase.
> + */
> + ret = mem_cgroup_do_precharge(1);
> + if (!ret)
> + goto retry;
> + }
> +
> + return ret;
> +}
> +
> +static const struct mm_walk_ops charge_walk_ops = {
> + .pmd_entry = mem_cgroup_move_charge_pte_range,
> + .walk_lock = PGWALK_RDLOCK,
> +};
> +
> +static void mem_cgroup_move_charge(void)
> +{
> + lru_add_drain_all();
> + /*
> + * Signal folio_memcg_lock() to take the memcg's move_lock
> + * while we're moving its pages to another memcg. Then wait
> + * for already started RCU-only updates to finish.
> + */
> + atomic_inc(&mc.from->moving_account);
> + synchronize_rcu();
> +retry:
> + if (unlikely(!mmap_read_trylock(mc.mm))) {
> + /*
> + * Someone who are holding the mmap_lock might be waiting in
> + * waitq. So we cancel all extra charges, wake up all waiters,
> + * and retry. Because we cancel precharges, we might not be able
> + * to move enough charges, but moving charge is a best-effort
> + * feature anyway, so it wouldn't be a big problem.
> + */
> + __mem_cgroup_clear_mc();
> + cond_resched();
> + goto retry;
> + }
> + /*
> + * When we have consumed all precharges and failed in doing
> + * additional charge, the page walk just aborts.
> + */
> + walk_page_range(mc.mm, 0, ULONG_MAX, &charge_walk_ops, NULL);
> + mmap_read_unlock(mc.mm);
> + atomic_dec(&mc.from->moving_account);
> +}
> +
> +void mem_cgroup_move_task(void)
> +{
> + if (mc.to) {
> + mem_cgroup_move_charge();
> + mem_cgroup_clear_mc();
> + }
> +}
> +
> +#else /* !CONFIG_MMU */
> +static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
> +{
> + return 0;
> +}
> +static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset)
> +{
> +}
> +static void mem_cgroup_move_task(void)
> +{
> +}
> +#endif
> +
> static int __init memcg1_init(void)
> {
> int node;
> diff --git a/mm/memcontrol-v1.h b/mm/memcontrol-v1.h
> index e37bc7e8d955..55e7c4f90c39 100644
> --- a/mm/memcontrol-v1.h
> +++ b/mm/memcontrol-v1.h
> @@ -11,4 +11,34 @@ static inline void memcg1_soft_limit_reset(struct mem_cgroup *memcg)
> WRITE_ONCE(memcg->soft_limit, PAGE_COUNTER_MAX);
> }
>
> +void mem_cgroup_charge_statistics(struct mem_cgroup *memcg, int nr_pages);
> +void memcg_check_events(struct mem_cgroup *memcg, int nid);
> +void memcg_oom_recover(struct mem_cgroup *memcg);
> +int try_charge_memcg(struct mem_cgroup *memcg, gfp_t gfp_mask,
> + unsigned int nr_pages);
> +
> +static inline int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
> + unsigned int nr_pages)
> +{
> + if (mem_cgroup_is_root(memcg))
> + return 0;
> +
> + return try_charge_memcg(memcg, gfp_mask, nr_pages);
> +}
> +
> +void mem_cgroup_id_get_many(struct mem_cgroup *memcg, unsigned int n);
> +void mem_cgroup_id_put_many(struct mem_cgroup *memcg, unsigned int n);
> +
> +bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg);
> +struct cgroup_taskset;
> +int mem_cgroup_can_attach(struct cgroup_taskset *tset);
> +void mem_cgroup_cancel_attach(struct cgroup_taskset *tset);
> +void mem_cgroup_move_task(void);
> +
> +struct cftype;
> +u64 mem_cgroup_move_charge_read(struct cgroup_subsys_state *css,
> + struct cftype *cft);
> +int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
> + struct cftype *cft, u64 val);
> +
> #endif /* __MM_MEMCONTROL_V1_H */
> diff --git a/mm/memcontrol.c b/mm/memcontrol.c
> index 3479e1af12d5..3332c89cae2e 100644
> --- a/mm/memcontrol.c
> +++ b/mm/memcontrol.c
> @@ -28,7 +28,6 @@
> #include <linux/page_counter.h>
> #include <linux/memcontrol.h>
> #include <linux/cgroup.h>
> -#include <linux/pagewalk.h>
> #include <linux/sched/mm.h>
> #include <linux/shmem_fs.h>
> #include <linux/hugetlb.h>
> @@ -45,7 +44,6 @@
> #include <linux/mutex.h>
> #include <linux/rbtree.h>
> #include <linux/slab.h>
> -#include <linux/swap.h>
> #include <linux/swapops.h>
> #include <linux/spinlock.h>
> #include <linux/eventfd.h>
> @@ -71,7 +69,6 @@
> #include <net/sock.h>
> #include <net/ip.h>
> #include "slab.h"
> -#include "swap.h"
> #include "memcontrol-v1.h"
>
> #include <linux/uaccess.h>
> @@ -158,31 +155,6 @@ struct mem_cgroup_event {
> static void mem_cgroup_threshold(struct mem_cgroup *memcg);
> static void mem_cgroup_oom_notify(struct mem_cgroup *memcg);
>
> -/* Stuffs for move charges at task migration. */
> -/*
> - * Types of charges to be moved.
> - */
> -#define MOVE_ANON 0x1U
> -#define MOVE_FILE 0x2U
> -#define MOVE_MASK (MOVE_ANON | MOVE_FILE)
> -
> -/* "mc" and its members are protected by cgroup_mutex */
> -static struct move_charge_struct {
> - spinlock_t lock; /* for from, to */
> - struct mm_struct *mm;
> - struct mem_cgroup *from;
> - struct mem_cgroup *to;
> - unsigned long flags;
> - unsigned long precharge;
> - unsigned long moved_charge;
> - unsigned long moved_swap;
> - struct task_struct *moving_task; /* a task moving charges */
> - wait_queue_head_t waitq; /* a waitq for other context */
> -} mc = {
> - .lock = __SPIN_LOCK_UNLOCKED(mc.lock),
> - .waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
> -};
> -
> /* for encoding cft->private value on file */
> enum res_type {
> _MEM,
> @@ -955,8 +927,7 @@ static unsigned long memcg_events_local(struct mem_cgroup *memcg, int event)
> return READ_ONCE(memcg->vmstats->events_local[i]);
> }
>
> -static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
> - int nr_pages)
> +void mem_cgroup_charge_statistics(struct mem_cgroup *memcg, int nr_pages)
> {
> /* pagein of a big page is an event. So, ignore page size */
> if (nr_pages > 0)
> @@ -998,7 +969,7 @@ static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,
> * Check events in order.
> *
> */
> -static void memcg_check_events(struct mem_cgroup *memcg, int nid)
> +void memcg_check_events(struct mem_cgroup *memcg, int nid)
> {
> if (IS_ENABLED(CONFIG_PREEMPT_RT))
> return;
> @@ -1467,51 +1438,6 @@ static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg)
> return margin;
> }
>
> -/*
> - * A routine for checking "mem" is under move_account() or not.
> - *
> - * Checking a cgroup is mc.from or mc.to or under hierarchy of
> - * moving cgroups. This is for waiting at high-memory pressure
> - * caused by "move".
> - */
> -static bool mem_cgroup_under_move(struct mem_cgroup *memcg)
> -{
> - struct mem_cgroup *from;
> - struct mem_cgroup *to;
> - bool ret = false;
> - /*
> - * Unlike task_move routines, we access mc.to, mc.from not under
> - * mutual exclusion by cgroup_mutex. Here, we take spinlock instead.
> - */
> - spin_lock(&mc.lock);
> - from = mc.from;
> - to = mc.to;
> - if (!from)
> - goto unlock;
> -
> - ret = mem_cgroup_is_descendant(from, memcg) ||
> - mem_cgroup_is_descendant(to, memcg);
> -unlock:
> - spin_unlock(&mc.lock);
> - return ret;
> -}
> -
> -static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg)
> -{
> - if (mc.moving_task && current != mc.moving_task) {
> - if (mem_cgroup_under_move(memcg)) {
> - DEFINE_WAIT(wait);
> - prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
> - /* moving charge context might have finished. */
> - if (mc.moving_task)
> - schedule();
> - finish_wait(&mc.waitq, &wait);
> - return true;
> - }
> - }
> - return false;
> -}
> -
> struct memory_stat {
> const char *name;
> unsigned int idx;
> @@ -1904,7 +1830,7 @@ static int memcg_oom_wake_function(wait_queue_entry_t *wait,
> return autoremove_wake_function(wait, mode, sync, arg);
> }
>
> -static void memcg_oom_recover(struct mem_cgroup *memcg)
> +void memcg_oom_recover(struct mem_cgroup *memcg)
> {
> /*
> * For the following lockless ->under_oom test, the only required
> @@ -2093,87 +2019,6 @@ void mem_cgroup_print_oom_group(struct mem_cgroup *memcg)
> pr_cont(" are going to be killed due to memory.oom.group set\n");
> }
>
> -/**
> - * folio_memcg_lock - Bind a folio to its memcg.
> - * @folio: The folio.
> - *
> - * This function prevents unlocked LRU folios from being moved to
> - * another cgroup.
> - *
> - * It ensures lifetime of the bound memcg. The caller is responsible
> - * for the lifetime of the folio.
> - */
> -void folio_memcg_lock(struct folio *folio)
> -{
> - struct mem_cgroup *memcg;
> - unsigned long flags;
> -
> - /*
> - * The RCU lock is held throughout the transaction. The fast
> - * path can get away without acquiring the memcg->move_lock
> - * because page moving starts with an RCU grace period.
> - */
> - rcu_read_lock();
> -
> - if (mem_cgroup_disabled())
> - return;
> -again:
> - memcg = folio_memcg(folio);
> - if (unlikely(!memcg))
> - return;
> -
> -#ifdef CONFIG_PROVE_LOCKING
> - local_irq_save(flags);
> - might_lock(&memcg->move_lock);
> - local_irq_restore(flags);
> -#endif
> -
> - if (atomic_read(&memcg->moving_account) <= 0)
> - return;
> -
> - spin_lock_irqsave(&memcg->move_lock, flags);
> - if (memcg != folio_memcg(folio)) {
> - spin_unlock_irqrestore(&memcg->move_lock, flags);
> - goto again;
> - }
> -
> - /*
> - * When charge migration first begins, we can have multiple
> - * critical sections holding the fast-path RCU lock and one
> - * holding the slowpath move_lock. Track the task who has the
> - * move_lock for folio_memcg_unlock().
> - */
> - memcg->move_lock_task = current;
> - memcg->move_lock_flags = flags;
> -}
> -
> -static void __folio_memcg_unlock(struct mem_cgroup *memcg)
> -{
> - if (memcg && memcg->move_lock_task == current) {
> - unsigned long flags = memcg->move_lock_flags;
> -
> - memcg->move_lock_task = NULL;
> - memcg->move_lock_flags = 0;
> -
> - spin_unlock_irqrestore(&memcg->move_lock, flags);
> - }
> -
> - rcu_read_unlock();
> -}
> -
> -/**
> - * folio_memcg_unlock - Release the binding between a folio and its memcg.
> - * @folio: The folio.
> - *
> - * This releases the binding created by folio_memcg_lock(). This does
> - * not change the accounting of this folio to its memcg, but it does
> - * permit others to change it.
> - */
> -void folio_memcg_unlock(struct folio *folio)
> -{
> - __folio_memcg_unlock(folio_memcg(folio));
> -}
> -
> struct memcg_stock_pcp {
> local_lock_t stock_lock;
> struct mem_cgroup *cached; /* this never be root cgroup */
> @@ -2653,8 +2498,8 @@ void mem_cgroup_handle_over_high(gfp_t gfp_mask)
> css_put(&memcg->css);
> }
>
> -static int try_charge_memcg(struct mem_cgroup *memcg, gfp_t gfp_mask,
> - unsigned int nr_pages)
> +int try_charge_memcg(struct mem_cgroup *memcg, gfp_t gfp_mask,
> + unsigned int nr_pages)
> {
> unsigned int batch = max(MEMCG_CHARGE_BATCH, nr_pages);
> int nr_retries = MAX_RECLAIM_RETRIES;
> @@ -2849,15 +2694,6 @@ static int try_charge_memcg(struct mem_cgroup *memcg, gfp_t gfp_mask,
> return 0;
> }
>
> -static inline int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
> - unsigned int nr_pages)
> -{
> - if (mem_cgroup_is_root(memcg))
> - return 0;
> -
> - return try_charge_memcg(memcg, gfp_mask, nr_pages);
> -}
> -
> /**
> * mem_cgroup_cancel_charge() - cancel an uncommitted try_charge() call.
> * @memcg: memcg previously charged.
> @@ -3595,43 +3431,6 @@ void split_page_memcg(struct page *head, int old_order, int new_order)
> css_get_many(&memcg->css, old_nr / new_nr - 1);
> }
>
> -#ifdef CONFIG_SWAP
> -/**
> - * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
> - * @entry: swap entry to be moved
> - * @from: mem_cgroup which the entry is moved from
> - * @to: mem_cgroup which the entry is moved to
> - *
> - * It succeeds only when the swap_cgroup's record for this entry is the same
> - * as the mem_cgroup's id of @from.
> - *
> - * Returns 0 on success, -EINVAL on failure.
> - *
> - * The caller must have charged to @to, IOW, called page_counter_charge() about
> - * both res and memsw, and called css_get().
> - */
> -static int mem_cgroup_move_swap_account(swp_entry_t entry,
> - struct mem_cgroup *from, struct mem_cgroup *to)
> -{
> - unsigned short old_id, new_id;
> -
> - old_id = mem_cgroup_id(from);
> - new_id = mem_cgroup_id(to);
> -
> - if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
> - mod_memcg_state(from, MEMCG_SWAP, -1);
> - mod_memcg_state(to, MEMCG_SWAP, 1);
> - return 0;
> - }
> - return -EINVAL;
> -}
> -#else
> -static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
> - struct mem_cgroup *from, struct mem_cgroup *to)
> -{
> - return -EINVAL;
> -}
> -#endif
>
> static DEFINE_MUTEX(memcg_max_mutex);
>
> @@ -4015,42 +3814,6 @@ static ssize_t mem_cgroup_reset(struct kernfs_open_file *of, char *buf,
> return nbytes;
> }
>
> -static u64 mem_cgroup_move_charge_read(struct cgroup_subsys_state *css,
> - struct cftype *cft)
> -{
> - return mem_cgroup_from_css(css)->move_charge_at_immigrate;
> -}
> -
> -#ifdef CONFIG_MMU
> -static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
> - struct cftype *cft, u64 val)
> -{
> - struct mem_cgroup *memcg = mem_cgroup_from_css(css);
> -
> - pr_warn_once("Cgroup memory moving (move_charge_at_immigrate) is deprecated. "
> - "Please report your usecase to linux-mm@xxxxxxxxx if you "
> - "depend on this functionality.\n");
> -
> - if (val & ~MOVE_MASK)
> - return -EINVAL;
> -
> - /*
> - * No kind of locking is needed in here, because ->can_attach() will
> - * check this value once in the beginning of the process, and then carry
> - * on with stale data. This means that changes to this value will only
> - * affect task migrations starting after the change.
> - */
> - memcg->move_charge_at_immigrate = val;
> - return 0;
> -}
> -#else
> -static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
> - struct cftype *cft, u64 val)
> -{
> - return -ENOSYS;
> -}
> -#endif
> -
> #ifdef CONFIG_NUMA
>
> #define LRU_ALL_FILE (BIT(LRU_INACTIVE_FILE) | BIT(LRU_ACTIVE_FILE))
> @@ -5261,13 +5024,13 @@ static void mem_cgroup_id_remove(struct mem_cgroup *memcg)
> }
> }
>
> -static void __maybe_unused mem_cgroup_id_get_many(struct mem_cgroup *memcg,
> - unsigned int n)
> +void __maybe_unused mem_cgroup_id_get_many(struct mem_cgroup *memcg,
> + unsigned int n)
> {
> refcount_add(n, &memcg->id.ref);
> }
>
> -static void mem_cgroup_id_put_many(struct mem_cgroup *memcg, unsigned int n)
> +void mem_cgroup_id_put_many(struct mem_cgroup *memcg, unsigned int n)
> {
> if (refcount_sub_and_test(n, &memcg->id.ref)) {
> mem_cgroup_id_remove(memcg);
> @@ -5747,757 +5510,6 @@ static void mem_cgroup_css_rstat_flush(struct cgroup_subsys_state *css, int cpu)
> atomic64_set(&memcg->vmstats->stats_updates, 0);
> }
>
> -#ifdef CONFIG_MMU
> -/* Handlers for move charge at task migration. */
> -static int mem_cgroup_do_precharge(unsigned long count)
> -{
> - int ret;
> -
> - /* Try a single bulk charge without reclaim first, kswapd may wake */
> - ret = try_charge(mc.to, GFP_KERNEL & ~__GFP_DIRECT_RECLAIM, count);
> - if (!ret) {
> - mc.precharge += count;
> - return ret;
> - }
> -
> - /* Try charges one by one with reclaim, but do not retry */
> - while (count--) {
> - ret = try_charge(mc.to, GFP_KERNEL | __GFP_NORETRY, 1);
> - if (ret)
> - return ret;
> - mc.precharge++;
> - cond_resched();
> - }
> - return 0;
> -}
> -
> -union mc_target {
> - struct folio *folio;
> - swp_entry_t ent;
> -};
> -
> -enum mc_target_type {
> - MC_TARGET_NONE = 0,
> - MC_TARGET_PAGE,
> - MC_TARGET_SWAP,
> - MC_TARGET_DEVICE,
> -};
> -
> -static struct page *mc_handle_present_pte(struct vm_area_struct *vma,
> - unsigned long addr, pte_t ptent)
> -{
> - struct page *page = vm_normal_page(vma, addr, ptent);
> -
> - if (!page)
> - return NULL;
> - if (PageAnon(page)) {
> - if (!(mc.flags & MOVE_ANON))
> - return NULL;
> - } else {
> - if (!(mc.flags & MOVE_FILE))
> - return NULL;
> - }
> - get_page(page);
> -
> - return page;
> -}
> -
> -#if defined(CONFIG_SWAP) || defined(CONFIG_DEVICE_PRIVATE)
> -static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
> - pte_t ptent, swp_entry_t *entry)
> -{
> - struct page *page = NULL;
> - swp_entry_t ent = pte_to_swp_entry(ptent);
> -
> - if (!(mc.flags & MOVE_ANON))
> - return NULL;
> -
> - /*
> - * Handle device private pages that are not accessible by the CPU, but
> - * stored as special swap entries in the page table.
> - */
> - if (is_device_private_entry(ent)) {
> - page = pfn_swap_entry_to_page(ent);
> - if (!get_page_unless_zero(page))
> - return NULL;
> - return page;
> - }
> -
> - if (non_swap_entry(ent))
> - return NULL;
> -
> - /*
> - * Because swap_cache_get_folio() updates some statistics counter,
> - * we call find_get_page() with swapper_space directly.
> - */
> - page = find_get_page(swap_address_space(ent), swap_cache_index(ent));
> - entry->val = ent.val;
> -
> - return page;
> -}
> -#else
> -static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
> - pte_t ptent, swp_entry_t *entry)
> -{
> - return NULL;
> -}
> -#endif
> -
> -static struct page *mc_handle_file_pte(struct vm_area_struct *vma,
> - unsigned long addr, pte_t ptent)
> -{
> - unsigned long index;
> - struct folio *folio;
> -
> - if (!vma->vm_file) /* anonymous vma */
> - return NULL;
> - if (!(mc.flags & MOVE_FILE))
> - return NULL;
> -
> - /* folio is moved even if it's not RSS of this task(page-faulted). */
> - /* shmem/tmpfs may report page out on swap: account for that too. */
> - index = linear_page_index(vma, addr);
> - folio = filemap_get_incore_folio(vma->vm_file->f_mapping, index);
> - if (IS_ERR(folio))
> - return NULL;
> - return folio_file_page(folio, index);
> -}
> -
> -/**
> - * mem_cgroup_move_account - move account of the folio
> - * @folio: The folio.
> - * @compound: charge the page as compound or small page
> - * @from: mem_cgroup which the folio is moved from.
> - * @to: mem_cgroup which the folio is moved to. @from != @to.
> - *
> - * The folio must be locked and not on the LRU.
> - *
> - * This function doesn't do "charge" to new cgroup and doesn't do "uncharge"
> - * from old cgroup.
> - */
> -static int mem_cgroup_move_account(struct folio *folio,
> - bool compound,
> - struct mem_cgroup *from,
> - struct mem_cgroup *to)
> -{
> - struct lruvec *from_vec, *to_vec;
> - struct pglist_data *pgdat;
> - unsigned int nr_pages = compound ? folio_nr_pages(folio) : 1;
> - int nid, ret;
> -
> - VM_BUG_ON(from == to);
> - VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
> - VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);
> - VM_BUG_ON(compound && !folio_test_large(folio));
> -
> - ret = -EINVAL;
> - if (folio_memcg(folio) != from)
> - goto out;
> -
> - pgdat = folio_pgdat(folio);
> - from_vec = mem_cgroup_lruvec(from, pgdat);
> - to_vec = mem_cgroup_lruvec(to, pgdat);
> -
> - folio_memcg_lock(folio);
> -
> - if (folio_test_anon(folio)) {
> - if (folio_mapped(folio)) {
> - __mod_lruvec_state(from_vec, NR_ANON_MAPPED, -nr_pages);
> - __mod_lruvec_state(to_vec, NR_ANON_MAPPED, nr_pages);
> - if (folio_test_pmd_mappable(folio)) {
> - __mod_lruvec_state(from_vec, NR_ANON_THPS,
> - -nr_pages);
> - __mod_lruvec_state(to_vec, NR_ANON_THPS,
> - nr_pages);
> - }
> - }
> - } else {
> - __mod_lruvec_state(from_vec, NR_FILE_PAGES, -nr_pages);
> - __mod_lruvec_state(to_vec, NR_FILE_PAGES, nr_pages);
> -
> - if (folio_test_swapbacked(folio)) {
> - __mod_lruvec_state(from_vec, NR_SHMEM, -nr_pages);
> - __mod_lruvec_state(to_vec, NR_SHMEM, nr_pages);
> - }
> -
> - if (folio_mapped(folio)) {
> - __mod_lruvec_state(from_vec, NR_FILE_MAPPED, -nr_pages);
> - __mod_lruvec_state(to_vec, NR_FILE_MAPPED, nr_pages);
> - }
> -
> - if (folio_test_dirty(folio)) {
> - struct address_space *mapping = folio_mapping(folio);
> -
> - if (mapping_can_writeback(mapping)) {
> - __mod_lruvec_state(from_vec, NR_FILE_DIRTY,
> - -nr_pages);
> - __mod_lruvec_state(to_vec, NR_FILE_DIRTY,
> - nr_pages);
> - }
> - }
> - }
> -
> -#ifdef CONFIG_SWAP
> - if (folio_test_swapcache(folio)) {
> - __mod_lruvec_state(from_vec, NR_SWAPCACHE, -nr_pages);
> - __mod_lruvec_state(to_vec, NR_SWAPCACHE, nr_pages);
> - }
> -#endif
> - if (folio_test_writeback(folio)) {
> - __mod_lruvec_state(from_vec, NR_WRITEBACK, -nr_pages);
> - __mod_lruvec_state(to_vec, NR_WRITEBACK, nr_pages);
> - }
> -
> - /*
> - * All state has been migrated, let's switch to the new memcg.
> - *
> - * It is safe to change page's memcg here because the page
> - * is referenced, charged, isolated, and locked: we can't race
> - * with (un)charging, migration, LRU putback, or anything else
> - * that would rely on a stable page's memory cgroup.
> - *
> - * Note that folio_memcg_lock is a memcg lock, not a page lock,
> - * to save space. As soon as we switch page's memory cgroup to a
> - * new memcg that isn't locked, the above state can change
> - * concurrently again. Make sure we're truly done with it.
> - */
> - smp_mb();
> -
> - css_get(&to->css);
> - css_put(&from->css);
> -
> - folio->memcg_data = (unsigned long)to;
> -
> - __folio_memcg_unlock(from);
> -
> - ret = 0;
> - nid = folio_nid(folio);
> -
> - local_irq_disable();
> - mem_cgroup_charge_statistics(to, nr_pages);
> - memcg_check_events(to, nid);
> - mem_cgroup_charge_statistics(from, -nr_pages);
> - memcg_check_events(from, nid);
> - local_irq_enable();
> -out:
> - return ret;
> -}
> -
> -/**
> - * get_mctgt_type - get target type of moving charge
> - * @vma: the vma the pte to be checked belongs
> - * @addr: the address corresponding to the pte to be checked
> - * @ptent: the pte to be checked
> - * @target: the pointer the target page or swap ent will be stored(can be NULL)
> - *
> - * Context: Called with pte lock held.
> - * Return:
> - * * MC_TARGET_NONE - If the pte is not a target for move charge.
> - * * MC_TARGET_PAGE - If the page corresponding to this pte is a target for
> - * move charge. If @target is not NULL, the folio is stored in target->folio
> - * with extra refcnt taken (Caller should release it).
> - * * MC_TARGET_SWAP - If the swap entry corresponding to this pte is a
> - * target for charge migration. If @target is not NULL, the entry is
> - * stored in target->ent.
> - * * MC_TARGET_DEVICE - Like MC_TARGET_PAGE but page is device memory and
> - * thus not on the lru. For now such page is charged like a regular page
> - * would be as it is just special memory taking the place of a regular page.
> - * See Documentations/vm/hmm.txt and include/linux/hmm.h
> - */
> -static enum mc_target_type get_mctgt_type(struct vm_area_struct *vma,
> - unsigned long addr, pte_t ptent, union mc_target *target)
> -{
> - struct page *page = NULL;
> - struct folio *folio;
> - enum mc_target_type ret = MC_TARGET_NONE;
> - swp_entry_t ent = { .val = 0 };
> -
> - if (pte_present(ptent))
> - page = mc_handle_present_pte(vma, addr, ptent);
> - else if (pte_none_mostly(ptent))
> - /*
> - * PTE markers should be treated as a none pte here, separated
> - * from other swap handling below.
> - */
> - page = mc_handle_file_pte(vma, addr, ptent);
> - else if (is_swap_pte(ptent))
> - page = mc_handle_swap_pte(vma, ptent, &ent);
> -
> - if (page)
> - folio = page_folio(page);
> - if (target && page) {
> - if (!folio_trylock(folio)) {
> - folio_put(folio);
> - return ret;
> - }
> - /*
> - * page_mapped() must be stable during the move. This
> - * pte is locked, so if it's present, the page cannot
> - * become unmapped. If it isn't, we have only partial
> - * control over the mapped state: the page lock will
> - * prevent new faults against pagecache and swapcache,
> - * so an unmapped page cannot become mapped. However,
> - * if the page is already mapped elsewhere, it can
> - * unmap, and there is nothing we can do about it.
> - * Alas, skip moving the page in this case.
> - */
> - if (!pte_present(ptent) && page_mapped(page)) {
> - folio_unlock(folio);
> - folio_put(folio);
> - return ret;
> - }
> - }
> -
> - if (!page && !ent.val)
> - return ret;
> - if (page) {
> - /*
> - * Do only loose check w/o serialization.
> - * mem_cgroup_move_account() checks the page is valid or
> - * not under LRU exclusion.
> - */
> - if (folio_memcg(folio) == mc.from) {
> - ret = MC_TARGET_PAGE;
> - if (folio_is_device_private(folio) ||
> - folio_is_device_coherent(folio))
> - ret = MC_TARGET_DEVICE;
> - if (target)
> - target->folio = folio;
> - }
> - if (!ret || !target) {
> - if (target)
> - folio_unlock(folio);
> - folio_put(folio);
> - }
> - }
> - /*
> - * There is a swap entry and a page doesn't exist or isn't charged.
> - * But we cannot move a tail-page in a THP.
> - */
> - if (ent.val && !ret && (!page || !PageTransCompound(page)) &&
> - mem_cgroup_id(mc.from) == lookup_swap_cgroup_id(ent)) {
> - ret = MC_TARGET_SWAP;
> - if (target)
> - target->ent = ent;
> - }
> - return ret;
> -}
> -
> -#ifdef CONFIG_TRANSPARENT_HUGEPAGE
> -/*
> - * We don't consider PMD mapped swapping or file mapped pages because THP does
> - * not support them for now.
> - * Caller should make sure that pmd_trans_huge(pmd) is true.
> - */
> -static enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
> - unsigned long addr, pmd_t pmd, union mc_target *target)
> -{
> - struct page *page = NULL;
> - struct folio *folio;
> - enum mc_target_type ret = MC_TARGET_NONE;
> -
> - if (unlikely(is_swap_pmd(pmd))) {
> - VM_BUG_ON(thp_migration_supported() &&
> - !is_pmd_migration_entry(pmd));
> - return ret;
> - }
> - page = pmd_page(pmd);
> - VM_BUG_ON_PAGE(!page || !PageHead(page), page);
> - folio = page_folio(page);
> - if (!(mc.flags & MOVE_ANON))
> - return ret;
> - if (folio_memcg(folio) == mc.from) {
> - ret = MC_TARGET_PAGE;
> - if (target) {
> - folio_get(folio);
> - if (!folio_trylock(folio)) {
> - folio_put(folio);
> - return MC_TARGET_NONE;
> - }
> - target->folio = folio;
> - }
> - }
> - return ret;
> -}
> -#else
> -static inline enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
> - unsigned long addr, pmd_t pmd, union mc_target *target)
> -{
> - return MC_TARGET_NONE;
> -}
> -#endif
> -
> -static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd,
> - unsigned long addr, unsigned long end,
> - struct mm_walk *walk)
> -{
> - struct vm_area_struct *vma = walk->vma;
> - pte_t *pte;
> - spinlock_t *ptl;
> -
> - ptl = pmd_trans_huge_lock(pmd, vma);
> - if (ptl) {
> - /*
> - * Note their can not be MC_TARGET_DEVICE for now as we do not
> - * support transparent huge page with MEMORY_DEVICE_PRIVATE but
> - * this might change.
> - */
> - if (get_mctgt_type_thp(vma, addr, *pmd, NULL) == MC_TARGET_PAGE)
> - mc.precharge += HPAGE_PMD_NR;
> - spin_unlock(ptl);
> - return 0;
> - }
> -
> - pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
> - if (!pte)
> - return 0;
> - for (; addr != end; pte++, addr += PAGE_SIZE)
> - if (get_mctgt_type(vma, addr, ptep_get(pte), NULL))
> - mc.precharge++; /* increment precharge temporarily */
> - pte_unmap_unlock(pte - 1, ptl);
> - cond_resched();
> -
> - return 0;
> -}
> -
> -static const struct mm_walk_ops precharge_walk_ops = {
> - .pmd_entry = mem_cgroup_count_precharge_pte_range,
> - .walk_lock = PGWALK_RDLOCK,
> -};
> -
> -static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm)
> -{
> - unsigned long precharge;
> -
> - mmap_read_lock(mm);
> - walk_page_range(mm, 0, ULONG_MAX, &precharge_walk_ops, NULL);
> - mmap_read_unlock(mm);
> -
> - precharge = mc.precharge;
> - mc.precharge = 0;
> -
> - return precharge;
> -}
> -
> -static int mem_cgroup_precharge_mc(struct mm_struct *mm)
> -{
> - unsigned long precharge = mem_cgroup_count_precharge(mm);
> -
> - VM_BUG_ON(mc.moving_task);
> - mc.moving_task = current;
> - return mem_cgroup_do_precharge(precharge);
> -}
> -
> -/* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */
> -static void __mem_cgroup_clear_mc(void)
> -{
> - struct mem_cgroup *from = mc.from;
> - struct mem_cgroup *to = mc.to;
> -
> - /* we must uncharge all the leftover precharges from mc.to */
> - if (mc.precharge) {
> - mem_cgroup_cancel_charge(mc.to, mc.precharge);
> - mc.precharge = 0;
> - }
> - /*
> - * we didn't uncharge from mc.from at mem_cgroup_move_account(), so
> - * we must uncharge here.
> - */
> - if (mc.moved_charge) {
> - mem_cgroup_cancel_charge(mc.from, mc.moved_charge);
> - mc.moved_charge = 0;
> - }
> - /* we must fixup refcnts and charges */
> - if (mc.moved_swap) {
> - /* uncharge swap account from the old cgroup */
> - if (!mem_cgroup_is_root(mc.from))
> - page_counter_uncharge(&mc.from->memsw, mc.moved_swap);
> -
> - mem_cgroup_id_put_many(mc.from, mc.moved_swap);
> -
> - /*
> - * we charged both to->memory and to->memsw, so we
> - * should uncharge to->memory.
> - */
> - if (!mem_cgroup_is_root(mc.to))
> - page_counter_uncharge(&mc.to->memory, mc.moved_swap);
> -
> - mc.moved_swap = 0;
> - }
> - memcg_oom_recover(from);
> - memcg_oom_recover(to);
> - wake_up_all(&mc.waitq);
> -}
> -
> -static void mem_cgroup_clear_mc(void)
> -{
> - struct mm_struct *mm = mc.mm;
> -
> - /*
> - * we must clear moving_task before waking up waiters at the end of
> - * task migration.
> - */
> - mc.moving_task = NULL;
> - __mem_cgroup_clear_mc();
> - spin_lock(&mc.lock);
> - mc.from = NULL;
> - mc.to = NULL;
> - mc.mm = NULL;
> - spin_unlock(&mc.lock);
> -
> - mmput(mm);
> -}
> -
> -static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
> -{
> - struct cgroup_subsys_state *css;
> - struct mem_cgroup *memcg = NULL; /* unneeded init to make gcc happy */
> - struct mem_cgroup *from;
> - struct task_struct *leader, *p;
> - struct mm_struct *mm;
> - unsigned long move_flags;
> - int ret = 0;
> -
> - /* charge immigration isn't supported on the default hierarchy */
> - if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
> - return 0;
> -
> - /*
> - * Multi-process migrations only happen on the default hierarchy
> - * where charge immigration is not used. Perform charge
> - * immigration if @tset contains a leader and whine if there are
> - * multiple.
> - */
> - p = NULL;
> - cgroup_taskset_for_each_leader(leader, css, tset) {
> - WARN_ON_ONCE(p);
> - p = leader;
> - memcg = mem_cgroup_from_css(css);
> - }
> - if (!p)
> - return 0;
> -
> - /*
> - * We are now committed to this value whatever it is. Changes in this
> - * tunable will only affect upcoming migrations, not the current one.
> - * So we need to save it, and keep it going.
> - */
> - move_flags = READ_ONCE(memcg->move_charge_at_immigrate);
> - if (!move_flags)
> - return 0;
> -
> - from = mem_cgroup_from_task(p);
> -
> - VM_BUG_ON(from == memcg);
> -
> - mm = get_task_mm(p);
> - if (!mm)
> - return 0;
> - /* We move charges only when we move a owner of the mm */
> - if (mm->owner == p) {
> - VM_BUG_ON(mc.from);
> - VM_BUG_ON(mc.to);
> - VM_BUG_ON(mc.precharge);
> - VM_BUG_ON(mc.moved_charge);
> - VM_BUG_ON(mc.moved_swap);
> -
> - spin_lock(&mc.lock);
> - mc.mm = mm;
> - mc.from = from;
> - mc.to = memcg;
> - mc.flags = move_flags;
> - spin_unlock(&mc.lock);
> - /* We set mc.moving_task later */
> -
> - ret = mem_cgroup_precharge_mc(mm);
> - if (ret)
> - mem_cgroup_clear_mc();
> - } else {
> - mmput(mm);
> - }
> - return ret;
> -}
> -
> -static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset)
> -{
> - if (mc.to)
> - mem_cgroup_clear_mc();
> -}
> -
> -static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
> - unsigned long addr, unsigned long end,
> - struct mm_walk *walk)
> -{
> - int ret = 0;
> - struct vm_area_struct *vma = walk->vma;
> - pte_t *pte;
> - spinlock_t *ptl;
> - enum mc_target_type target_type;
> - union mc_target target;
> - struct folio *folio;
> -
> - ptl = pmd_trans_huge_lock(pmd, vma);
> - if (ptl) {
> - if (mc.precharge < HPAGE_PMD_NR) {
> - spin_unlock(ptl);
> - return 0;
> - }
> - target_type = get_mctgt_type_thp(vma, addr, *pmd, &target);
> - if (target_type == MC_TARGET_PAGE) {
> - folio = target.folio;
> - if (folio_isolate_lru(folio)) {
> - if (!mem_cgroup_move_account(folio, true,
> - mc.from, mc.to)) {
> - mc.precharge -= HPAGE_PMD_NR;
> - mc.moved_charge += HPAGE_PMD_NR;
> - }
> - folio_putback_lru(folio);
> - }
> - folio_unlock(folio);
> - folio_put(folio);
> - } else if (target_type == MC_TARGET_DEVICE) {
> - folio = target.folio;
> - if (!mem_cgroup_move_account(folio, true,
> - mc.from, mc.to)) {
> - mc.precharge -= HPAGE_PMD_NR;
> - mc.moved_charge += HPAGE_PMD_NR;
> - }
> - folio_unlock(folio);
> - folio_put(folio);
> - }
> - spin_unlock(ptl);
> - return 0;
> - }
> -
> -retry:
> - pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
> - if (!pte)
> - return 0;
> - for (; addr != end; addr += PAGE_SIZE) {
> - pte_t ptent = ptep_get(pte++);
> - bool device = false;
> - swp_entry_t ent;
> -
> - if (!mc.precharge)
> - break;
> -
> - switch (get_mctgt_type(vma, addr, ptent, &target)) {
> - case MC_TARGET_DEVICE:
> - device = true;
> - fallthrough;
> - case MC_TARGET_PAGE:
> - folio = target.folio;
> - /*
> - * We can have a part of the split pmd here. Moving it
> - * can be done but it would be too convoluted so simply
> - * ignore such a partial THP and keep it in original
> - * memcg. There should be somebody mapping the head.
> - */
> - if (folio_test_large(folio))
> - goto put;
> - if (!device && !folio_isolate_lru(folio))
> - goto put;
> - if (!mem_cgroup_move_account(folio, false,
> - mc.from, mc.to)) {
> - mc.precharge--;
> - /* we uncharge from mc.from later. */
> - mc.moved_charge++;
> - }
> - if (!device)
> - folio_putback_lru(folio);
> -put: /* get_mctgt_type() gets & locks the page */
> - folio_unlock(folio);
> - folio_put(folio);
> - break;
> - case MC_TARGET_SWAP:
> - ent = target.ent;
> - if (!mem_cgroup_move_swap_account(ent, mc.from, mc.to)) {
> - mc.precharge--;
> - mem_cgroup_id_get_many(mc.to, 1);
> - /* we fixup other refcnts and charges later. */
> - mc.moved_swap++;
> - }
> - break;
> - default:
> - break;
> - }
> - }
> - pte_unmap_unlock(pte - 1, ptl);
> - cond_resched();
> -
> - if (addr != end) {
> - /*
> - * We have consumed all precharges we got in can_attach().
> - * We try charge one by one, but don't do any additional
> - * charges to mc.to if we have failed in charge once in attach()
> - * phase.
> - */
> - ret = mem_cgroup_do_precharge(1);
> - if (!ret)
> - goto retry;
> - }
> -
> - return ret;
> -}
> -
> -static const struct mm_walk_ops charge_walk_ops = {
> - .pmd_entry = mem_cgroup_move_charge_pte_range,
> - .walk_lock = PGWALK_RDLOCK,
> -};
> -
> -static void mem_cgroup_move_charge(void)
> -{
> - lru_add_drain_all();
> - /*
> - * Signal folio_memcg_lock() to take the memcg's move_lock
> - * while we're moving its pages to another memcg. Then wait
> - * for already started RCU-only updates to finish.
> - */
> - atomic_inc(&mc.from->moving_account);
> - synchronize_rcu();
> -retry:
> - if (unlikely(!mmap_read_trylock(mc.mm))) {
> - /*
> - * Someone who are holding the mmap_lock might be waiting in
> - * waitq. So we cancel all extra charges, wake up all waiters,
> - * and retry. Because we cancel precharges, we might not be able
> - * to move enough charges, but moving charge is a best-effort
> - * feature anyway, so it wouldn't be a big problem.
> - */
> - __mem_cgroup_clear_mc();
> - cond_resched();
> - goto retry;
> - }
> - /*
> - * When we have consumed all precharges and failed in doing
> - * additional charge, the page walk just aborts.
> - */
> - walk_page_range(mc.mm, 0, ULONG_MAX, &charge_walk_ops, NULL);
> - mmap_read_unlock(mc.mm);
> - atomic_dec(&mc.from->moving_account);
> -}
> -
> -static void mem_cgroup_move_task(void)
> -{
> - if (mc.to) {
> - mem_cgroup_move_charge();
> - mem_cgroup_clear_mc();
> - }
> -}
> -
> -#else /* !CONFIG_MMU */
> -static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
> -{
> - return 0;
> -}
> -static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset)
> -{
> -}
> -static void mem_cgroup_move_task(void)
> -{
> -}
> -#endif
> -
> #ifdef CONFIG_MEMCG_KMEM
> static void mem_cgroup_fork(struct task_struct *task)
> {
> --
> 2.45.2
--
Michal Hocko
SUSE Labs
