On Sun 14-04-24 19:07:31, Suren Baghdasaryan wrote: > Main goal of memory allocation profiling patchset is to provide accounting > that is cheap enough to run in production. To achieve that we inject > counters using codetags at the allocation call sites to account every time > allocation is made. This injection allows us to perform accounting > efficiently because injected counters are immediately available as opposed > to the alternative methods, such as using _RET_IP_, which would require > counter lookup and appropriate locking that makes accounting much more > expensive. This method requires all allocation functions to inject > separate counters at their call sites so that their callers can be > individually accounted. Counter injection is implemented by allocation > hooks which should wrap all allocation functions. > > Inlined functions which perform allocations but do not use allocation > hooks are directly charged for the allocations they perform. In most > cases these functions are just specialized allocation wrappers used > from multiple places to allocate objects of a specific type. It would > be more useful to do the accounting at their call sites instead. > Instrument these helpers to do accounting at the call site. Simple > inlined allocation wrappers are converted directly into macros. More > complex allocators or allocators with documentation are converted into > _noprof versions and allocation hooks are added. This allows memory > allocation profiling mechanism to charge allocations to the callers > of these functions. > > Signed-off-by: Suren Baghdasaryan <surenb@xxxxxxxxxx> Feel free to add: Acked-by: Jan Kara <jack@xxxxxxx> for the jbd2 bits. Honza > --- > Changes since v1 [1]: > - Added explicit type casts in macro replacements to force type checks, > per Matthew Wilcox > - Expanded the changelog to explain the reasons codetags are used and > this change is neeed, per Jan Kara > > [1] https://lore.kernel.org/all/20240404165404.3805498-1-surenb@xxxxxxxxxx/ > > drivers/iommu/amd/amd_iommu.h | 5 ++-- > fs/nfs/iostat.h | 5 +--- > include/acpi/platform/aclinuxex.h | 19 +++++--------- > include/asm-generic/pgalloc.h | 35 +++++++++++++++---------- > include/crypto/hash.h | 7 ++--- > include/crypto/internal/acompress.h | 5 ++-- > include/crypto/skcipher.h | 7 ++--- > include/linux/bpf.h | 33 ++++++----------------- > include/linux/bpfptr.h | 5 ++-- > include/linux/dma-fence-chain.h | 6 ++--- > include/linux/hid_bpf.h | 6 ++--- > include/linux/jbd2.h | 12 +++------ > include/linux/mm.h | 5 ++-- > include/linux/mm_types.h | 5 ++-- > include/linux/percpu.h | 3 +++ > include/linux/ptr_ring.h | 28 +++++++++++--------- > include/linux/skb_array.h | 19 ++++++++------ > include/linux/skbuff.h | 20 ++++++-------- > include/linux/skmsg.h | 8 +++--- > include/linux/slab.h | 5 ++++ > include/linux/sockptr.h | 10 ++++--- > include/net/netlabel.h | 16 ++++++----- > include/net/netlink.h | 5 ++-- > include/net/request_sock.h | 5 ++-- > include/net/tcx.h | 5 ++-- > net/sunrpc/auth_gss/auth_gss_internal.h | 6 +++-- > 26 files changed, 142 insertions(+), 143 deletions(-) > > diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h > index f482aab420f7..52575ba9a141 100644 > --- a/drivers/iommu/amd/amd_iommu.h > +++ b/drivers/iommu/amd/amd_iommu.h > @@ -134,13 +134,14 @@ static inline int get_pci_sbdf_id(struct pci_dev *pdev) > return PCI_SEG_DEVID_TO_SBDF(seg, devid); > } > > -static inline void *alloc_pgtable_page(int nid, gfp_t gfp) > +static inline void *alloc_pgtable_page_noprof(int nid, gfp_t gfp) > { > struct page *page; > > - page = alloc_pages_node(nid, gfp | __GFP_ZERO, 0); > + page = alloc_pages_node_noprof(nid, gfp | __GFP_ZERO, 0); > return page ? page_address(page) : NULL; > } > +#define alloc_pgtable_page(...) alloc_hooks(alloc_pgtable_page_noprof(__VA_ARGS__)) > > /* > * This must be called after device probe completes. During probe > diff --git a/fs/nfs/iostat.h b/fs/nfs/iostat.h > index 5aa776b5a3e7..b17a9eb9b148 100644 > --- a/fs/nfs/iostat.h > +++ b/fs/nfs/iostat.h > @@ -46,10 +46,7 @@ static inline void nfs_add_stats(const struct inode *inode, > nfs_add_server_stats(NFS_SERVER(inode), stat, addend); > } > > -static inline struct nfs_iostats __percpu *nfs_alloc_iostats(void) > -{ > - return alloc_percpu(struct nfs_iostats); > -} > +#define nfs_alloc_iostats() alloc_percpu(struct nfs_iostats) > > static inline void nfs_free_iostats(struct nfs_iostats __percpu *stats) > { > diff --git a/include/acpi/platform/aclinuxex.h b/include/acpi/platform/aclinuxex.h > index 600d4e2641da..62cac266a1c8 100644 > --- a/include/acpi/platform/aclinuxex.h > +++ b/include/acpi/platform/aclinuxex.h > @@ -47,26 +47,19 @@ acpi_status acpi_os_terminate(void); > * However, boot has (system_state != SYSTEM_RUNNING) > * to quiet __might_sleep() in kmalloc() and resume does not. > */ > -static inline void *acpi_os_allocate(acpi_size size) > -{ > - return kmalloc(size, irqs_disabled()? GFP_ATOMIC : GFP_KERNEL); > -} > +#define acpi_os_allocate(_size) \ > + kmalloc(_size, irqs_disabled() ? GFP_ATOMIC : GFP_KERNEL) > > -static inline void *acpi_os_allocate_zeroed(acpi_size size) > -{ > - return kzalloc(size, irqs_disabled()? GFP_ATOMIC : GFP_KERNEL); > -} > +#define acpi_os_allocate_zeroed(_size) \ > + kzalloc(_size, irqs_disabled() ? GFP_ATOMIC : GFP_KERNEL) > > static inline void acpi_os_free(void *memory) > { > kfree(memory); > } > > -static inline void *acpi_os_acquire_object(acpi_cache_t * cache) > -{ > - return kmem_cache_zalloc(cache, > - irqs_disabled()? GFP_ATOMIC : GFP_KERNEL); > -} > +#define acpi_os_acquire_object(_cache) \ > + kmem_cache_zalloc(_cache, irqs_disabled() ? GFP_ATOMIC : GFP_KERNEL) > > static inline acpi_thread_id acpi_os_get_thread_id(void) > { > diff --git a/include/asm-generic/pgalloc.h b/include/asm-generic/pgalloc.h > index 879e5f8aa5e9..7c48f5fbf8aa 100644 > --- a/include/asm-generic/pgalloc.h > +++ b/include/asm-generic/pgalloc.h > @@ -16,15 +16,16 @@ > * > * Return: pointer to the allocated memory or %NULL on error > */ > -static inline pte_t *__pte_alloc_one_kernel(struct mm_struct *mm) > +static inline pte_t *__pte_alloc_one_kernel_noprof(struct mm_struct *mm) > { > - struct ptdesc *ptdesc = pagetable_alloc(GFP_PGTABLE_KERNEL & > + struct ptdesc *ptdesc = pagetable_alloc_noprof(GFP_PGTABLE_KERNEL & > ~__GFP_HIGHMEM, 0); > > if (!ptdesc) > return NULL; > return ptdesc_address(ptdesc); > } > +#define __pte_alloc_one_kernel(...) alloc_hooks(__pte_alloc_one_kernel_noprof(__VA_ARGS__)) > > #ifndef __HAVE_ARCH_PTE_ALLOC_ONE_KERNEL > /** > @@ -33,10 +34,11 @@ static inline pte_t *__pte_alloc_one_kernel(struct mm_struct *mm) > * > * Return: pointer to the allocated memory or %NULL on error > */ > -static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm) > +static inline pte_t *pte_alloc_one_kernel_noprof(struct mm_struct *mm) > { > - return __pte_alloc_one_kernel(mm); > + return __pte_alloc_one_kernel_noprof(mm); > } > +#define pte_alloc_one_kernel(...) alloc_hooks(pte_alloc_one_kernel_noprof(__VA_ARGS__)) > #endif > > /** > @@ -61,11 +63,11 @@ static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) > * > * Return: `struct page` referencing the ptdesc or %NULL on error > */ > -static inline pgtable_t __pte_alloc_one(struct mm_struct *mm, gfp_t gfp) > +static inline pgtable_t __pte_alloc_one_noprof(struct mm_struct *mm, gfp_t gfp) > { > struct ptdesc *ptdesc; > > - ptdesc = pagetable_alloc(gfp, 0); > + ptdesc = pagetable_alloc_noprof(gfp, 0); > if (!ptdesc) > return NULL; > if (!pagetable_pte_ctor(ptdesc)) { > @@ -75,6 +77,7 @@ static inline pgtable_t __pte_alloc_one(struct mm_struct *mm, gfp_t gfp) > > return ptdesc_page(ptdesc); > } > +#define __pte_alloc_one(...) alloc_hooks(__pte_alloc_one_noprof(__VA_ARGS__)) > > #ifndef __HAVE_ARCH_PTE_ALLOC_ONE > /** > @@ -85,10 +88,11 @@ static inline pgtable_t __pte_alloc_one(struct mm_struct *mm, gfp_t gfp) > * > * Return: `struct page` referencing the ptdesc or %NULL on error > */ > -static inline pgtable_t pte_alloc_one(struct mm_struct *mm) > +static inline pgtable_t pte_alloc_one_noprof(struct mm_struct *mm) > { > - return __pte_alloc_one(mm, GFP_PGTABLE_USER); > + return __pte_alloc_one_noprof(mm, GFP_PGTABLE_USER); > } > +#define pte_alloc_one(...) alloc_hooks(pte_alloc_one_noprof(__VA_ARGS__)) > #endif > > /* > @@ -124,14 +128,14 @@ static inline void pte_free(struct mm_struct *mm, struct page *pte_page) > * > * Return: pointer to the allocated memory or %NULL on error > */ > -static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr) > +static inline pmd_t *pmd_alloc_one_noprof(struct mm_struct *mm, unsigned long addr) > { > struct ptdesc *ptdesc; > gfp_t gfp = GFP_PGTABLE_USER; > > if (mm == &init_mm) > gfp = GFP_PGTABLE_KERNEL; > - ptdesc = pagetable_alloc(gfp, 0); > + ptdesc = pagetable_alloc_noprof(gfp, 0); > if (!ptdesc) > return NULL; > if (!pagetable_pmd_ctor(ptdesc)) { > @@ -140,6 +144,7 @@ static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr) > } > return ptdesc_address(ptdesc); > } > +#define pmd_alloc_one(...) alloc_hooks(pmd_alloc_one_noprof(__VA_ARGS__)) > #endif > > #ifndef __HAVE_ARCH_PMD_FREE > @@ -157,7 +162,7 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) > > #if CONFIG_PGTABLE_LEVELS > 3 > > -static inline pud_t *__pud_alloc_one(struct mm_struct *mm, unsigned long addr) > +static inline pud_t *__pud_alloc_one_noprof(struct mm_struct *mm, unsigned long addr) > { > gfp_t gfp = GFP_PGTABLE_USER; > struct ptdesc *ptdesc; > @@ -166,13 +171,14 @@ static inline pud_t *__pud_alloc_one(struct mm_struct *mm, unsigned long addr) > gfp = GFP_PGTABLE_KERNEL; > gfp &= ~__GFP_HIGHMEM; > > - ptdesc = pagetable_alloc(gfp, 0); > + ptdesc = pagetable_alloc_noprof(gfp, 0); > if (!ptdesc) > return NULL; > > pagetable_pud_ctor(ptdesc); > return ptdesc_address(ptdesc); > } > +#define __pud_alloc_one(...) alloc_hooks(__pud_alloc_one_noprof(__VA_ARGS__)) > > #ifndef __HAVE_ARCH_PUD_ALLOC_ONE > /** > @@ -184,10 +190,11 @@ static inline pud_t *__pud_alloc_one(struct mm_struct *mm, unsigned long addr) > * > * Return: pointer to the allocated memory or %NULL on error > */ > -static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr) > +static inline pud_t *pud_alloc_one_noprof(struct mm_struct *mm, unsigned long addr) > { > - return __pud_alloc_one(mm, addr); > + return __pud_alloc_one_noprof(mm, addr); > } > +#define pud_alloc_one(...) alloc_hooks(pud_alloc_one_noprof(__VA_ARGS__)) > #endif > > static inline void __pud_free(struct mm_struct *mm, pud_t *pud) > diff --git a/include/crypto/hash.h b/include/crypto/hash.h > index 5d61f576cfc8..e5181cc9b7c5 100644 > --- a/include/crypto/hash.h > +++ b/include/crypto/hash.h > @@ -578,19 +578,20 @@ static inline void ahash_request_set_tfm(struct ahash_request *req, > * > * Return: allocated request handle in case of success, or NULL if out of memory > */ > -static inline struct ahash_request *ahash_request_alloc( > +static inline struct ahash_request *ahash_request_alloc_noprof( > struct crypto_ahash *tfm, gfp_t gfp) > { > struct ahash_request *req; > > - req = kmalloc(sizeof(struct ahash_request) + > - crypto_ahash_reqsize(tfm), gfp); > + req = kmalloc_noprof(sizeof(struct ahash_request) + > + crypto_ahash_reqsize(tfm), gfp); > > if (likely(req)) > ahash_request_set_tfm(req, tfm); > > return req; > } > +#define ahash_request_alloc(...) alloc_hooks(ahash_request_alloc_noprof(__VA_ARGS__)) > > /** > * ahash_request_free() - zeroize and free the request data structure > diff --git a/include/crypto/internal/acompress.h b/include/crypto/internal/acompress.h > index 4ac46bafba9d..2a67793f52ad 100644 > --- a/include/crypto/internal/acompress.h > +++ b/include/crypto/internal/acompress.h > @@ -69,15 +69,16 @@ static inline void acomp_request_complete(struct acomp_req *req, > crypto_request_complete(&req->base, err); > } > > -static inline struct acomp_req *__acomp_request_alloc(struct crypto_acomp *tfm) > +static inline struct acomp_req *__acomp_request_alloc_noprof(struct crypto_acomp *tfm) > { > struct acomp_req *req; > > - req = kzalloc(sizeof(*req) + crypto_acomp_reqsize(tfm), GFP_KERNEL); > + req = kzalloc_noprof(sizeof(*req) + crypto_acomp_reqsize(tfm), GFP_KERNEL); > if (likely(req)) > acomp_request_set_tfm(req, tfm); > return req; > } > +#define __acomp_request_alloc(...) alloc_hooks(__acomp_request_alloc_noprof(__VA_ARGS__)) > > static inline void __acomp_request_free(struct acomp_req *req) > { > diff --git a/include/crypto/skcipher.h b/include/crypto/skcipher.h > index c8857d7bdb37..6c5330e316b0 100644 > --- a/include/crypto/skcipher.h > +++ b/include/crypto/skcipher.h > @@ -861,19 +861,20 @@ static inline struct skcipher_request *skcipher_request_cast( > * > * Return: allocated request handle in case of success, or NULL if out of memory > */ > -static inline struct skcipher_request *skcipher_request_alloc( > +static inline struct skcipher_request *skcipher_request_alloc_noprof( > struct crypto_skcipher *tfm, gfp_t gfp) > { > struct skcipher_request *req; > > - req = kmalloc(sizeof(struct skcipher_request) + > - crypto_skcipher_reqsize(tfm), gfp); > + req = kmalloc_noprof(sizeof(struct skcipher_request) + > + crypto_skcipher_reqsize(tfm), gfp); > > if (likely(req)) > skcipher_request_set_tfm(req, tfm); > > return req; > } > +#define skcipher_request_alloc(...) alloc_hooks(skcipher_request_alloc_noprof(__VA_ARGS__)) > > /** > * skcipher_request_free() - zeroize and free request data structure > diff --git a/include/linux/bpf.h b/include/linux/bpf.h > index 4f20f62f9d63..a63fa48ab80d 100644 > --- a/include/linux/bpf.h > +++ b/include/linux/bpf.h > @@ -2230,31 +2230,14 @@ void *bpf_map_kvcalloc(struct bpf_map *map, size_t n, size_t size, > void __percpu *bpf_map_alloc_percpu(const struct bpf_map *map, size_t size, > size_t align, gfp_t flags); > #else > -static inline void * > -bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags, > - int node) > -{ > - return kmalloc_node(size, flags, node); > -} > - > -static inline void * > -bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags) > -{ > - return kzalloc(size, flags); > -} > - > -static inline void * > -bpf_map_kvcalloc(struct bpf_map *map, size_t n, size_t size, gfp_t flags) > -{ > - return kvcalloc(n, size, flags); > -} > - > -static inline void __percpu * > -bpf_map_alloc_percpu(const struct bpf_map *map, size_t size, size_t align, > - gfp_t flags) > -{ > - return __alloc_percpu_gfp(size, align, flags); > -} > +#define bpf_map_kmalloc_node(_map, _size, _flags, _node) \ > + kmalloc_node(_size, _flags, _node) > +#define bpf_map_kzalloc(_map, _size, _flags) \ > + kzalloc(_size, _flags) > +#define bpf_map_kvcalloc(_map, _n, _size, _flags) \ > + kvcalloc(_n, _size, _flags) > +#define bpf_map_alloc_percpu(_map, _size, _align, _flags) \ > + __alloc_percpu_gfp(_size, _align, _flags) > #endif > > static inline int > diff --git a/include/linux/bpfptr.h b/include/linux/bpfptr.h > index 79b2f78eec1a..1af241525a17 100644 > --- a/include/linux/bpfptr.h > +++ b/include/linux/bpfptr.h > @@ -65,9 +65,9 @@ static inline int copy_to_bpfptr_offset(bpfptr_t dst, size_t offset, > return copy_to_sockptr_offset((sockptr_t) dst, offset, src, size); > } > > -static inline void *kvmemdup_bpfptr(bpfptr_t src, size_t len) > +static inline void *kvmemdup_bpfptr_noprof(bpfptr_t src, size_t len) > { > - void *p = kvmalloc(len, GFP_USER | __GFP_NOWARN); > + void *p = kvmalloc_noprof(len, GFP_USER | __GFP_NOWARN); > > if (!p) > return ERR_PTR(-ENOMEM); > @@ -77,6 +77,7 @@ static inline void *kvmemdup_bpfptr(bpfptr_t src, size_t len) > } > return p; > } > +#define kvmemdup_bpfptr(...) alloc_hooks(kvmemdup_bpfptr_noprof(__VA_ARGS__)) > > static inline long strncpy_from_bpfptr(char *dst, bpfptr_t src, size_t count) > { > diff --git a/include/linux/dma-fence-chain.h b/include/linux/dma-fence-chain.h > index 4bdf0b96da28..ad9e2506c2f4 100644 > --- a/include/linux/dma-fence-chain.h > +++ b/include/linux/dma-fence-chain.h > @@ -86,10 +86,8 @@ dma_fence_chain_contained(struct dma_fence *fence) > * > * Returns a new struct dma_fence_chain object or NULL on failure. > */ > -static inline struct dma_fence_chain *dma_fence_chain_alloc(void) > -{ > - return kmalloc(sizeof(struct dma_fence_chain), GFP_KERNEL); > -}; > +#define dma_fence_chain_alloc() \ > + ((struct dma_fence_chain *)kmalloc(sizeof(struct dma_fence_chain), GFP_KERNEL)) > > /** > * dma_fence_chain_free > diff --git a/include/linux/hid_bpf.h b/include/linux/hid_bpf.h > index 7118ac28d468..ca70eeb6393e 100644 > --- a/include/linux/hid_bpf.h > +++ b/include/linux/hid_bpf.h > @@ -149,10 +149,8 @@ static inline int hid_bpf_connect_device(struct hid_device *hdev) { return 0; } > static inline void hid_bpf_disconnect_device(struct hid_device *hdev) {} > static inline void hid_bpf_destroy_device(struct hid_device *hid) {} > static inline void hid_bpf_device_init(struct hid_device *hid) {} > -static inline u8 *call_hid_bpf_rdesc_fixup(struct hid_device *hdev, u8 *rdesc, unsigned int *size) > -{ > - return kmemdup(rdesc, *size, GFP_KERNEL); > -} > +#define call_hid_bpf_rdesc_fixup(_hdev, _rdesc, _size) \ > + ((u8 *)kmemdup(_rdesc, *(_size), GFP_KERNEL)) > > #endif /* CONFIG_HID_BPF */ > > diff --git a/include/linux/jbd2.h b/include/linux/jbd2.h > index 971f3e826e15..9512fe332668 100644 > --- a/include/linux/jbd2.h > +++ b/include/linux/jbd2.h > @@ -1586,10 +1586,8 @@ void jbd2_journal_put_journal_head(struct journal_head *jh); > */ > extern struct kmem_cache *jbd2_handle_cache; > > -static inline handle_t *jbd2_alloc_handle(gfp_t gfp_flags) > -{ > - return kmem_cache_zalloc(jbd2_handle_cache, gfp_flags); > -} > +#define jbd2_alloc_handle(_gfp_flags) \ > + ((handle_t *)kmem_cache_zalloc(jbd2_handle_cache, _gfp_flags)) > > static inline void jbd2_free_handle(handle_t *handle) > { > @@ -1602,10 +1600,8 @@ static inline void jbd2_free_handle(handle_t *handle) > */ > extern struct kmem_cache *jbd2_inode_cache; > > -static inline struct jbd2_inode *jbd2_alloc_inode(gfp_t gfp_flags) > -{ > - return kmem_cache_alloc(jbd2_inode_cache, gfp_flags); > -} > +#define jbd2_alloc_inode(_gfp_flags) \ > + ((struct jbd2_inode *)kmem_cache_alloc(jbd2_inode_cache, _gfp_flags)) > > static inline void jbd2_free_inode(struct jbd2_inode *jinode) > { > diff --git a/include/linux/mm.h b/include/linux/mm.h > index 07c73451d42f..d261e45bb29b 100644 > --- a/include/linux/mm.h > +++ b/include/linux/mm.h > @@ -2899,12 +2899,13 @@ static inline bool pagetable_is_reserved(struct ptdesc *pt) > * > * Return: The ptdesc describing the allocated page tables. > */ > -static inline struct ptdesc *pagetable_alloc(gfp_t gfp, unsigned int order) > +static inline struct ptdesc *pagetable_alloc_noprof(gfp_t gfp, unsigned int order) > { > - struct page *page = alloc_pages(gfp | __GFP_COMP, order); > + struct page *page = alloc_pages_noprof(gfp | __GFP_COMP, order); > > return page_ptdesc(page); > } > +#define pagetable_alloc(...) alloc_hooks(pagetable_alloc_noprof(__VA_ARGS__)) > > /** > * pagetable_free - Free pagetables > diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h > index c432add95913..db0adf5721cc 100644 > --- a/include/linux/mm_types.h > +++ b/include/linux/mm_types.h > @@ -1167,14 +1167,15 @@ static inline void mm_init_cid(struct mm_struct *mm) > cpumask_clear(mm_cidmask(mm)); > } > > -static inline int mm_alloc_cid(struct mm_struct *mm) > +static inline int mm_alloc_cid_noprof(struct mm_struct *mm) > { > - mm->pcpu_cid = alloc_percpu(struct mm_cid); > + mm->pcpu_cid = alloc_percpu_noprof(struct mm_cid); > if (!mm->pcpu_cid) > return -ENOMEM; > mm_init_cid(mm); > return 0; > } > +#define mm_alloc_cid(...) alloc_hooks(mm_alloc_cid_noprof(__VA_ARGS__)) > > static inline void mm_destroy_cid(struct mm_struct *mm) > { > diff --git a/include/linux/percpu.h b/include/linux/percpu.h > index 13a82f11e4fd..03053de557cf 100644 > --- a/include/linux/percpu.h > +++ b/include/linux/percpu.h > @@ -151,6 +151,9 @@ extern size_t pcpu_alloc_size(void __percpu *__pdata); > #define alloc_percpu(type) \ > (typeof(type) __percpu *)__alloc_percpu(sizeof(type), \ > __alignof__(type)) > +#define alloc_percpu_noprof(type) \ > + ((typeof(type) __percpu *)pcpu_alloc_noprof(sizeof(type), \ > + __alignof__(type), false, GFP_KERNEL)) > > extern void free_percpu(void __percpu *__pdata); > > diff --git a/include/linux/ptr_ring.h b/include/linux/ptr_ring.h > index 808f9d3ee546..fd037c127bb0 100644 > --- a/include/linux/ptr_ring.h > +++ b/include/linux/ptr_ring.h > @@ -464,11 +464,11 @@ static inline int ptr_ring_consume_batched_bh(struct ptr_ring *r, > /* Not all gfp_t flags (besides GFP_KERNEL) are allowed. See > * documentation for vmalloc for which of them are legal. > */ > -static inline void **__ptr_ring_init_queue_alloc(unsigned int size, gfp_t gfp) > +static inline void **__ptr_ring_init_queue_alloc_noprof(unsigned int size, gfp_t gfp) > { > if (size > KMALLOC_MAX_SIZE / sizeof(void *)) > return NULL; > - return kvmalloc_array(size, sizeof(void *), gfp | __GFP_ZERO); > + return kvmalloc_array_noprof(size, sizeof(void *), gfp | __GFP_ZERO); > } > > static inline void __ptr_ring_set_size(struct ptr_ring *r, int size) > @@ -484,9 +484,9 @@ static inline void __ptr_ring_set_size(struct ptr_ring *r, int size) > r->batch = 1; > } > > -static inline int ptr_ring_init(struct ptr_ring *r, int size, gfp_t gfp) > +static inline int ptr_ring_init_noprof(struct ptr_ring *r, int size, gfp_t gfp) > { > - r->queue = __ptr_ring_init_queue_alloc(size, gfp); > + r->queue = __ptr_ring_init_queue_alloc_noprof(size, gfp); > if (!r->queue) > return -ENOMEM; > > @@ -497,6 +497,7 @@ static inline int ptr_ring_init(struct ptr_ring *r, int size, gfp_t gfp) > > return 0; > } > +#define ptr_ring_init(...) alloc_hooks(ptr_ring_init_noprof(__VA_ARGS__)) > > /* > * Return entries into ring. Destroy entries that don't fit. > @@ -587,11 +588,11 @@ static inline void **__ptr_ring_swap_queue(struct ptr_ring *r, void **queue, > * In particular if you consume ring in interrupt or BH context, you must > * disable interrupts/BH when doing so. > */ > -static inline int ptr_ring_resize(struct ptr_ring *r, int size, gfp_t gfp, > +static inline int ptr_ring_resize_noprof(struct ptr_ring *r, int size, gfp_t gfp, > void (*destroy)(void *)) > { > unsigned long flags; > - void **queue = __ptr_ring_init_queue_alloc(size, gfp); > + void **queue = __ptr_ring_init_queue_alloc_noprof(size, gfp); > void **old; > > if (!queue) > @@ -609,6 +610,7 @@ static inline int ptr_ring_resize(struct ptr_ring *r, int size, gfp_t gfp, > > return 0; > } > +#define ptr_ring_resize(...) alloc_hooks(ptr_ring_resize_noprof(__VA_ARGS__)) > > /* > * Note: producer lock is nested within consumer lock, so if you > @@ -616,21 +618,21 @@ static inline int ptr_ring_resize(struct ptr_ring *r, int size, gfp_t gfp, > * In particular if you consume ring in interrupt or BH context, you must > * disable interrupts/BH when doing so. > */ > -static inline int ptr_ring_resize_multiple(struct ptr_ring **rings, > - unsigned int nrings, > - int size, > - gfp_t gfp, void (*destroy)(void *)) > +static inline int ptr_ring_resize_multiple_noprof(struct ptr_ring **rings, > + unsigned int nrings, > + int size, > + gfp_t gfp, void (*destroy)(void *)) > { > unsigned long flags; > void ***queues; > int i; > > - queues = kmalloc_array(nrings, sizeof(*queues), gfp); > + queues = kmalloc_array_noprof(nrings, sizeof(*queues), gfp); > if (!queues) > goto noqueues; > > for (i = 0; i < nrings; ++i) { > - queues[i] = __ptr_ring_init_queue_alloc(size, gfp); > + queues[i] = __ptr_ring_init_queue_alloc_noprof(size, gfp); > if (!queues[i]) > goto nomem; > } > @@ -660,6 +662,8 @@ static inline int ptr_ring_resize_multiple(struct ptr_ring **rings, > noqueues: > return -ENOMEM; > } > +#define ptr_ring_resize_multiple(...) \ > + alloc_hooks(ptr_ring_resize_multiple_noprof(__VA_ARGS__)) > > static inline void ptr_ring_cleanup(struct ptr_ring *r, void (*destroy)(void *)) > { > diff --git a/include/linux/skb_array.h b/include/linux/skb_array.h > index e2d45b7cb619..926496c9cc9c 100644 > --- a/include/linux/skb_array.h > +++ b/include/linux/skb_array.h > @@ -177,10 +177,11 @@ static inline int skb_array_peek_len_any(struct skb_array *a) > return PTR_RING_PEEK_CALL_ANY(&a->ring, __skb_array_len_with_tag); > } > > -static inline int skb_array_init(struct skb_array *a, int size, gfp_t gfp) > +static inline int skb_array_init_noprof(struct skb_array *a, int size, gfp_t gfp) > { > - return ptr_ring_init(&a->ring, size, gfp); > + return ptr_ring_init_noprof(&a->ring, size, gfp); > } > +#define skb_array_init(...) alloc_hooks(skb_array_init_noprof(__VA_ARGS__)) > > static void __skb_array_destroy_skb(void *ptr) > { > @@ -198,15 +199,17 @@ static inline int skb_array_resize(struct skb_array *a, int size, gfp_t gfp) > return ptr_ring_resize(&a->ring, size, gfp, __skb_array_destroy_skb); > } > > -static inline int skb_array_resize_multiple(struct skb_array **rings, > - int nrings, unsigned int size, > - gfp_t gfp) > +static inline int skb_array_resize_multiple_noprof(struct skb_array **rings, > + int nrings, unsigned int size, > + gfp_t gfp) > { > BUILD_BUG_ON(offsetof(struct skb_array, ring)); > - return ptr_ring_resize_multiple((struct ptr_ring **)rings, > - nrings, size, gfp, > - __skb_array_destroy_skb); > + return ptr_ring_resize_multiple_noprof((struct ptr_ring **)rings, > + nrings, size, gfp, > + __skb_array_destroy_skb); > } > +#define skb_array_resize_multiple(...) \ > + alloc_hooks(skb_array_resize_multiple_noprof(__VA_ARGS__)) > > static inline void skb_array_cleanup(struct skb_array *a) > { > diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h > index 9d24aec064e8..b72920c9887b 100644 > --- a/include/linux/skbuff.h > +++ b/include/linux/skbuff.h > @@ -3371,7 +3371,7 @@ void __napi_kfree_skb(struct sk_buff *skb, enum skb_drop_reason reason); > * > * %NULL is returned if there is no free memory. > */ > -static inline struct page *__dev_alloc_pages(gfp_t gfp_mask, > +static inline struct page *__dev_alloc_pages_noprof(gfp_t gfp_mask, > unsigned int order) > { > /* This piece of code contains several assumptions. > @@ -3384,13 +3384,11 @@ static inline struct page *__dev_alloc_pages(gfp_t gfp_mask, > */ > gfp_mask |= __GFP_COMP | __GFP_MEMALLOC; > > - return alloc_pages_node(NUMA_NO_NODE, gfp_mask, order); > + return alloc_pages_node_noprof(NUMA_NO_NODE, gfp_mask, order); > } > +#define __dev_alloc_pages(...) alloc_hooks(__dev_alloc_pages_noprof(__VA_ARGS__)) > > -static inline struct page *dev_alloc_pages(unsigned int order) > -{ > - return __dev_alloc_pages(GFP_ATOMIC | __GFP_NOWARN, order); > -} > +#define dev_alloc_pages(_order) __dev_alloc_pages(GFP_ATOMIC | __GFP_NOWARN, _order) > > /** > * __dev_alloc_page - allocate a page for network Rx > @@ -3400,15 +3398,13 @@ static inline struct page *dev_alloc_pages(unsigned int order) > * > * %NULL is returned if there is no free memory. > */ > -static inline struct page *__dev_alloc_page(gfp_t gfp_mask) > +static inline struct page *__dev_alloc_page_noprof(gfp_t gfp_mask) > { > - return __dev_alloc_pages(gfp_mask, 0); > + return __dev_alloc_pages_noprof(gfp_mask, 0); > } > +#define __dev_alloc_page(...) alloc_hooks(__dev_alloc_page_noprof(__VA_ARGS__)) > > -static inline struct page *dev_alloc_page(void) > -{ > - return dev_alloc_pages(0); > -} > +#define dev_alloc_page() dev_alloc_pages(0) > > /** > * dev_page_is_reusable - check whether a page can be reused for network Rx > diff --git a/include/linux/skmsg.h b/include/linux/skmsg.h > index e65ec3fd2799..78efc5b20284 100644 > --- a/include/linux/skmsg.h > +++ b/include/linux/skmsg.h > @@ -410,11 +410,9 @@ void sk_psock_stop_verdict(struct sock *sk, struct sk_psock *psock); > int sk_psock_msg_verdict(struct sock *sk, struct sk_psock *psock, > struct sk_msg *msg); > > -static inline struct sk_psock_link *sk_psock_init_link(void) > -{ > - return kzalloc(sizeof(struct sk_psock_link), > - GFP_ATOMIC | __GFP_NOWARN); > -} > +#define sk_psock_init_link() \ > + ((struct sk_psock_link *)kzalloc(sizeof(struct sk_psock_link), \ > + GFP_ATOMIC | __GFP_NOWARN)) > > static inline void sk_psock_free_link(struct sk_psock_link *link) > { > diff --git a/include/linux/slab.h b/include/linux/slab.h > index c0be1cd03cf6..4cc37ef22aae 100644 > --- a/include/linux/slab.h > +++ b/include/linux/slab.h > @@ -744,6 +744,9 @@ void *kmalloc_node_track_caller_noprof(size_t size, gfp_t flags, int node, > */ > #define kmalloc_track_caller(...) kmalloc_node_track_caller(__VA_ARGS__, NUMA_NO_NODE) > > +#define kmalloc_track_caller_noprof(...) \ > + kmalloc_node_track_caller_noprof(__VA_ARGS__, NUMA_NO_NODE, _RET_IP_) > + > static inline __alloc_size(1, 2) void *kmalloc_array_node_noprof(size_t n, size_t size, gfp_t flags, > int node) > { > @@ -781,6 +784,7 @@ extern void *kvmalloc_node_noprof(size_t size, gfp_t flags, int node) __alloc_si > #define kvmalloc_node(...) alloc_hooks(kvmalloc_node_noprof(__VA_ARGS__)) > > #define kvmalloc(_size, _flags) kvmalloc_node(_size, _flags, NUMA_NO_NODE) > +#define kvmalloc_noprof(_size, _flags) kvmalloc_node_noprof(_size, _flags, NUMA_NO_NODE) > #define kvzalloc(_size, _flags) kvmalloc(_size, _flags|__GFP_ZERO) > > #define kvzalloc_node(_size, _flags, _node) kvmalloc_node(_size, _flags|__GFP_ZERO, _node) > @@ -797,6 +801,7 @@ static inline __alloc_size(1, 2) void *kvmalloc_array_noprof(size_t n, size_t si > > #define kvmalloc_array(...) alloc_hooks(kvmalloc_array_noprof(__VA_ARGS__)) > #define kvcalloc(_n, _size, _flags) kvmalloc_array(_n, _size, _flags|__GFP_ZERO) > +#define kvcalloc_noprof(_n, _size, _flags) kvmalloc_array_noprof(_n, _size, _flags|__GFP_ZERO) > > extern void *kvrealloc_noprof(const void *p, size_t oldsize, size_t newsize, gfp_t flags) > __realloc_size(3); > diff --git a/include/linux/sockptr.h b/include/linux/sockptr.h > index 307961b41541..ba703b22e7d8 100644 > --- a/include/linux/sockptr.h > +++ b/include/linux/sockptr.h > @@ -92,9 +92,9 @@ static inline int copy_to_sockptr(sockptr_t dst, const void *src, size_t size) > return copy_to_sockptr_offset(dst, 0, src, size); > } > > -static inline void *memdup_sockptr(sockptr_t src, size_t len) > +static inline void *memdup_sockptr_noprof(sockptr_t src, size_t len) > { > - void *p = kmalloc_track_caller(len, GFP_USER | __GFP_NOWARN); > + void *p = kmalloc_track_caller_noprof(len, GFP_USER | __GFP_NOWARN); > > if (!p) > return ERR_PTR(-ENOMEM); > @@ -104,10 +104,11 @@ static inline void *memdup_sockptr(sockptr_t src, size_t len) > } > return p; > } > +#define memdup_sockptr(...) alloc_hooks(memdup_sockptr_noprof(__VA_ARGS__)) > > -static inline void *memdup_sockptr_nul(sockptr_t src, size_t len) > +static inline void *memdup_sockptr_nul_noprof(sockptr_t src, size_t len) > { > - char *p = kmalloc_track_caller(len + 1, GFP_KERNEL); > + char *p = kmalloc_track_caller_noprof(len + 1, GFP_KERNEL); > > if (!p) > return ERR_PTR(-ENOMEM); > @@ -118,6 +119,7 @@ static inline void *memdup_sockptr_nul(sockptr_t src, size_t len) > p[len] = '\0'; > return p; > } > +#define memdup_sockptr_nul(...) alloc_hooks(memdup_sockptr_nul_noprof(__VA_ARGS__)) > > static inline long strncpy_from_sockptr(char *dst, sockptr_t src, size_t count) > { > diff --git a/include/net/netlabel.h b/include/net/netlabel.h > index f3ab0b8a4b18..c31bd96dafdb 100644 > --- a/include/net/netlabel.h > +++ b/include/net/netlabel.h > @@ -274,15 +274,17 @@ struct netlbl_calipso_ops { > * on success, NULL on failure. > * > */ > -static inline struct netlbl_lsm_cache *netlbl_secattr_cache_alloc(gfp_t flags) > +static inline struct netlbl_lsm_cache *netlbl_secattr_cache_alloc_noprof(gfp_t flags) > { > struct netlbl_lsm_cache *cache; > > - cache = kzalloc(sizeof(*cache), flags); > + cache = kzalloc_noprof(sizeof(*cache), flags); > if (cache) > refcount_set(&cache->refcount, 1); > return cache; > } > +#define netlbl_secattr_cache_alloc(...) \ > + alloc_hooks(netlbl_secattr_cache_alloc_noprof(__VA_ARGS__)) > > /** > * netlbl_secattr_cache_free - Frees a netlbl_lsm_cache struct > @@ -311,10 +313,11 @@ static inline void netlbl_secattr_cache_free(struct netlbl_lsm_cache *cache) > * on failure. > * > */ > -static inline struct netlbl_lsm_catmap *netlbl_catmap_alloc(gfp_t flags) > +static inline struct netlbl_lsm_catmap *netlbl_catmap_alloc_noprof(gfp_t flags) > { > - return kzalloc(sizeof(struct netlbl_lsm_catmap), flags); > + return kzalloc_noprof(sizeof(struct netlbl_lsm_catmap), flags); > } > +#define netlbl_catmap_alloc(...) alloc_hooks(netlbl_catmap_alloc_noprof(__VA_ARGS__)) > > /** > * netlbl_catmap_free - Free a LSM secattr catmap > @@ -376,10 +379,11 @@ static inline void netlbl_secattr_destroy(struct netlbl_lsm_secattr *secattr) > * pointer on success, or NULL on failure. > * > */ > -static inline struct netlbl_lsm_secattr *netlbl_secattr_alloc(gfp_t flags) > +static inline struct netlbl_lsm_secattr *netlbl_secattr_alloc_noprof(gfp_t flags) > { > - return kzalloc(sizeof(struct netlbl_lsm_secattr), flags); > + return kzalloc_noprof(sizeof(struct netlbl_lsm_secattr), flags); > } > +#define netlbl_secattr_alloc(...) alloc_hooks(netlbl_secattr_alloc_noprof(__VA_ARGS__)) > > /** > * netlbl_secattr_free - Frees a netlbl_lsm_secattr struct > diff --git a/include/net/netlink.h b/include/net/netlink.h > index c19ff921b661..972b5484fa6f 100644 > --- a/include/net/netlink.h > +++ b/include/net/netlink.h > @@ -1891,10 +1891,11 @@ static inline struct nla_bitfield32 nla_get_bitfield32(const struct nlattr *nla) > * @src: netlink attribute to duplicate from > * @gfp: GFP mask > */ > -static inline void *nla_memdup(const struct nlattr *src, gfp_t gfp) > +static inline void *nla_memdup_noprof(const struct nlattr *src, gfp_t gfp) > { > - return kmemdup(nla_data(src), nla_len(src), gfp); > + return kmemdup_noprof(nla_data(src), nla_len(src), gfp); > } > +#define nla_memdup(...) alloc_hooks(nla_memdup_noprof(__VA_ARGS__)) > > /** > * nla_nest_start_noflag - Start a new level of nested attributes > diff --git a/include/net/request_sock.h b/include/net/request_sock.h > index 004e651e6067..29495c331d20 100644 > --- a/include/net/request_sock.h > +++ b/include/net/request_sock.h > @@ -127,12 +127,12 @@ static inline struct sock *skb_steal_sock(struct sk_buff *skb, > } > > static inline struct request_sock * > -reqsk_alloc(const struct request_sock_ops *ops, struct sock *sk_listener, > +reqsk_alloc_noprof(const struct request_sock_ops *ops, struct sock *sk_listener, > bool attach_listener) > { > struct request_sock *req; > > - req = kmem_cache_alloc(ops->slab, GFP_ATOMIC | __GFP_NOWARN); > + req = kmem_cache_alloc_noprof(ops->slab, GFP_ATOMIC | __GFP_NOWARN); > if (!req) > return NULL; > req->rsk_listener = NULL; > @@ -157,6 +157,7 @@ reqsk_alloc(const struct request_sock_ops *ops, struct sock *sk_listener, > > return req; > } > +#define reqsk_alloc(...) alloc_hooks(reqsk_alloc_noprof(__VA_ARGS__)) > > static inline void __reqsk_free(struct request_sock *req) > { > diff --git a/include/net/tcx.h b/include/net/tcx.h > index 04be9377785d..72a3e75e539f 100644 > --- a/include/net/tcx.h > +++ b/include/net/tcx.h > @@ -75,9 +75,9 @@ tcx_entry_fetch(struct net_device *dev, bool ingress) > return rcu_dereference_rtnl(dev->tcx_egress); > } > > -static inline struct bpf_mprog_entry *tcx_entry_create(void) > +static inline struct bpf_mprog_entry *tcx_entry_create_noprof(void) > { > - struct tcx_entry *tcx = kzalloc(sizeof(*tcx), GFP_KERNEL); > + struct tcx_entry *tcx = kzalloc_noprof(sizeof(*tcx), GFP_KERNEL); > > if (tcx) { > bpf_mprog_bundle_init(&tcx->bundle); > @@ -85,6 +85,7 @@ static inline struct bpf_mprog_entry *tcx_entry_create(void) > } > return NULL; > } > +#define tcx_entry_create(...) alloc_hooks(tcx_entry_create_noprof(__VA_ARGS__)) > > static inline void tcx_entry_free(struct bpf_mprog_entry *entry) > { > diff --git a/net/sunrpc/auth_gss/auth_gss_internal.h b/net/sunrpc/auth_gss/auth_gss_internal.h > index c53b329092d4..4ebc1b7043d9 100644 > --- a/net/sunrpc/auth_gss/auth_gss_internal.h > +++ b/net/sunrpc/auth_gss/auth_gss_internal.h > @@ -23,7 +23,7 @@ simple_get_bytes(const void *p, const void *end, void *res, size_t len) > } > > static inline const void * > -simple_get_netobj(const void *p, const void *end, struct xdr_netobj *dest) > +simple_get_netobj_noprof(const void *p, const void *end, struct xdr_netobj *dest) > { > const void *q; > unsigned int len; > @@ -35,7 +35,7 @@ simple_get_netobj(const void *p, const void *end, struct xdr_netobj *dest) > if (unlikely(q > end || q < p)) > return ERR_PTR(-EFAULT); > if (len) { > - dest->data = kmemdup(p, len, GFP_KERNEL); > + dest->data = kmemdup_noprof(p, len, GFP_KERNEL); > if (unlikely(dest->data == NULL)) > return ERR_PTR(-ENOMEM); > } else > @@ -43,3 +43,5 @@ simple_get_netobj(const void *p, const void *end, struct xdr_netobj *dest) > dest->len = len; > return q; > } > + > +#define simple_get_netobj(...) alloc_hooks(simple_get_netobj_noprof(__VA_ARGS__)) > > base-commit: 3aec6b2b34e219898883d1e9ea7e911b4d3762a9 > -- > 2.44.0.683.g7961c838ac-goog > -- Jan Kara <jack@xxxxxxxx> SUSE Labs, CR
