Allocate frequently-accessed data structures from the NUMA node
associated with this device to avoid slow cross-NUMA node memory
accesses.
Only the following memory allocations are made NUMA-aware:
1. Called during probe. If called in the data path then hopefully we're
executing on a CPU in the same NUMA node as the device. If the CPU is
not in the right NUMA node then it's unclear whether forcing memory
allocations to use the device's NUMA node will increase or decrease
performance.
2. Memory will be frequently accessed from the data path. There is no
need to worry about data that is not accessed from
performance-critical code paths.
This patch adds a non-meminit alloc_pages_exact_nid() caller so I've
removed the __meminit added by commit e19318116048 ("mm/page_alloc.c:
add __meminit to alloc_pages_exact_nid()").
Cc: Fabian Frederick <fabf@xxxxxxxxx>
Cc: Andrew Morton <akpm@xxxxxxxxxxxxxxxxxxxx>
Cc: Mel Gorman <mgorman@xxxxxxx>
Signed-off-by: Stefan Hajnoczi <stefanha@xxxxxxxxxx>
---
I have included the alloc_pages_exact_nid() __meminit removal in this
patch to provide context for reviewers.
---
include/linux/gfp.h | 2 +-
drivers/virtio/virtio_ring.c | 26 +++++++++++++++++---------
mm/page_alloc.c | 2 +-
3 files changed, 19 insertions(+), 11 deletions(-)
diff --git a/include/linux/gfp.h b/include/linux/gfp.h
index 4aba4c86c626..9b69df707c7a 100644
--- a/include/linux/gfp.h
+++ b/include/linux/gfp.h
@@ -563,7 +563,7 @@ extern unsigned long get_zeroed_page(gfp_t gfp_mask);
void *alloc_pages_exact(size_t size, gfp_t gfp_mask);
void free_pages_exact(void *virt, size_t size);
-void * __meminit alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask);
+void *alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask);
#define __get_free_page(gfp_mask) \
__get_free_pages((gfp_mask), 0)
diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index 58b96baa8d48..d06b42309bed 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -276,7 +276,9 @@ static void *vring_alloc_queue(struct virtio_device *vdev, size_t size,
return dma_alloc_coherent(vdev->dev.parent, size,
dma_handle, flag);
} else {
- void *queue = alloc_pages_exact(PAGE_ALIGN(size), flag);
+ int node = dev_to_node(&vdev->dev);
+ void *queue = alloc_pages_exact_nid(node, PAGE_ALIGN(size),
+ flag);
if (queue) {
phys_addr_t phys_addr = virt_to_phys(queue);
@@ -1567,6 +1569,7 @@ static struct virtqueue *vring_create_virtqueue_packed(
struct vring_packed_desc_event *driver, *device;
dma_addr_t ring_dma_addr, driver_event_dma_addr, device_event_dma_addr;
size_t ring_size_in_bytes, event_size_in_bytes;
+ int node = dev_to_node(&vdev->dev);
unsigned int i;
ring_size_in_bytes = num * sizeof(struct vring_packed_desc);
@@ -1591,7 +1594,7 @@ static struct virtqueue *vring_create_virtqueue_packed(
if (!device)
goto err_device;
- vq = kmalloc(sizeof(*vq), GFP_KERNEL);
+ vq = kmalloc_node(sizeof(*vq), GFP_KERNEL, node);
if (!vq)
goto err_vq;
@@ -1639,9 +1642,10 @@ static struct virtqueue *vring_create_virtqueue_packed(
vq->packed.event_flags_shadow = 0;
vq->packed.avail_used_flags = 1 << VRING_PACKED_DESC_F_AVAIL;
- vq->packed.desc_state = kmalloc_array(num,
+ vq->packed.desc_state = kmalloc_array_node(num,
sizeof(struct vring_desc_state_packed),
- GFP_KERNEL);
+ GFP_KERNEL,
+ node);
if (!vq->packed.desc_state)
goto err_desc_state;
@@ -1653,9 +1657,10 @@ static struct virtqueue *vring_create_virtqueue_packed(
for (i = 0; i < num-1; i++)
vq->packed.desc_state[i].next = i + 1;
- vq->packed.desc_extra = kmalloc_array(num,
+ vq->packed.desc_extra = kmalloc_array_node(num,
sizeof(struct vring_desc_extra_packed),
- GFP_KERNEL);
+ GFP_KERNEL,
+ node);
if (!vq->packed.desc_extra)
goto err_desc_extra;
@@ -2059,13 +2064,14 @@ struct virtqueue *__vring_new_virtqueue(unsigned int index,
void (*callback)(struct virtqueue *),
const char *name)
{
+ int node = dev_to_node(&vdev->dev);
unsigned int i;
struct vring_virtqueue *vq;
if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED))
return NULL;
- vq = kmalloc(sizeof(*vq), GFP_KERNEL);
+ vq = kmalloc_node(sizeof(*vq), GFP_KERNEL, node);
if (!vq)
return NULL;
@@ -2110,8 +2116,10 @@ struct virtqueue *__vring_new_virtqueue(unsigned int index,
vq->split.avail_flags_shadow);
}
- vq->split.desc_state = kmalloc_array(vring.num,
- sizeof(struct vring_desc_state_split), GFP_KERNEL);
+ vq->split.desc_state = kmalloc_array_node(vring.num,
+ sizeof(struct vring_desc_state_split),
+ GFP_KERNEL,
+ node);
if (!vq->split.desc_state) {
kfree(vq);
return NULL;
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 13cc653122b7..2216022d8987 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -5053,7 +5053,7 @@ EXPORT_SYMBOL(alloc_pages_exact);
*
* Return: pointer to the allocated area or %NULL in case of error.
*/
-void * __meminit alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask)
+void *alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask)
{
unsigned int order = get_order(size);
struct page *p;
--
2.26.2
