On 3/3/06, Avishay Traeger <atraeger@xxxxxxxxxxxxx> wrote:
Yes. This time I include the source code. Basically the This module allocate and reserver memory as a remote storage for other machines. I allocate one page (using get_free_page() function) at a time, and all the allocated pages are indexed by an array (allocated by vmalloc() function). All the memory is allocated when the module receives a RMEM_INIT message (which is in the function rmem_recv).
I tested the module on serveral machines. It works fine on machines with 160MB physical memory. But, I always get the "0-order allocation failed" message when I test the module on machines with 120MB physical memory. Even though I only let the module allocate 20% of the avaialbe free memory, I still get the error message. I'm not sure if the vmalloc() function causes the problem or something else.
Thanks!!! Really appreciate all your help!!!
Fei
Here is the source code:
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include "client-mem.h"
#include "utils.h"
MODULE_LICENSE("GPL");
#define RMEM_PROTO 100
static struct k2k_proto *proto;
//static u8 *local_mem;
static __u16 local_id;
static __u16 server_id;
static int hard_sector;
#define INDEX(pos) ((pos) >> 3)
#define OFFSET(pos) ((pos) & 0x7)
// assume 4mb (1024 pages) to allocate
// plus 1 to cover out of array size exception
static unsigned long nr_pages = 0;
void **pg_array = NULL;
int is_allocated = -1;
int rmem_unicast(struct rmem_msg *msg) {
struct sk_buff *skb;
struct rmem_msg *buff;
struct k2k_node *nd;
__u16 node_to = msg->rid;
if ((nd = k2k_get_node(node_to)) == NULL) {
if ((nd = k2k_probe(node_to, 0)) == NULL)
return -EBADR;
}
if ((skb = k2k_alloc_unicast_skb(nd, proto, GFP_ATOMIC)) == NULL)
return -ENOMEM;
buff = (struct rmem_msg *)skb_put(skb, sizeof(struct rmem_msg));
memcpy(buff, msg, sizeof(struct rmem_msg));
k2k_unicast(skb, nd, proto, 0, 0);
k2k_node_put(nd);
return 0;
}
int rmem_recv(struct sk_buff *skb, struct k2k_node *nd) {
struct rmem_msg *msg;
__u16 addr;
void *ptr = NULL;
msg = (struct rmem_msg *)skb->data;
if (msg->type == RMEM_READ) {
unsigned long nr;
msg->type = RMEM_READ_ACK;
msg->sid = local_id;
msg->rid = server_id;
// using page array
// send the requested sector, one at a time, to server
for (nr = 1; nr < msg->nr_sct; nr++) {
msg->status = MSG_STATUS_MORE;
ptr = pg_array[INDEX(msg->start_sct)] +
OFFSET(msg->start_sct) * hard_sector;
memcpy(msg->data, ptr, hard_sector);
rmem_unicast(msg);
msg->start_sct++;
PDEBUG("send msg(RMEM_READ_ACK:MSG_STATUS_MORE)\n");
}
// send the last requested sector
msg->status = MSG_STATUS_OK;
ptr = pg_array[INDEX(msg->start_sct)] +
OFFSET(msg->start_sct) * hard_sector;
memcpy(msg->data, ptr, hard_sector);
rmem_unicast(msg);
PDEBUG("send msg(RMEM_READ_ACK:MSG_STATUS_OK)\n");
} else if (msg->type == RMEM_WRITE) {
PDEBUG("write: start_sct %ld, nr_sct: %ld, hard_sector: %d\n",
msg->start_sct, msg->nr_sct, hard_sector);
// using page array
ptr = pg_array[INDEX(msg->start_sct)] +
OFFSET(msg->start_sct) * hard_sector;
memcpy(ptr, msg->data, hard_sector);
PDEBUG("write: after memcpy(ptr, msg->data, size)\n");
} else if (msg->type == RMEM_INIT) {
unsigned long i = 0;
PDEBUG("msg->start_sct: %ld\n", msg->start_sct);
// start_sct contains the number of sectors
// a page can hold 8 sectors
//plus one to avoid array overflow
nr_pages = msg->start_sct / 8 + 1;
PDEBUG("**nr_pages: %ld\n", nr_pages);
// alloc page array
pg_array = (void *)vmalloc(nr_pages * sizeof(void *));
if (!pg_array) {
printk("<1>error vmalloc pg_array\n");
goto jump;
} else {
for (i = 0; i < nr_pages; i++) {
pg_array[i] = NULL;
}
for (i = 0; i < nr_pages; i++) {
pg_array[i] = (void *)get_free_page(GFP_ATOMIC);
//new
//pg_array[i] = page_address(alloc_page(GFP_ATOMIC));
if (!pg_array[i]) {
printk("<1>error get_free_page pg_array[%lu]\n", i);
}
}
}
jump:
PDEBUG("goto-ed here: jump\n");
hard_sector = msg->nr_sct; // size of harddisk sector
local_id = msg->rid; // get the local id for the node
server_id = msg->sid; // get the server id
PDEBUG("local_id: %d\n", (int)local_id);
PDEBUG("server_id: %d\n", (int)server_id);
// reply INIT with INITACK
PDEBUG("received: msg(INIT), replied with msg(INITACK)\n");
msg->type = RMEM_INIT_ACK;
msg->sid = local_id;
msg->rid = server_id;
msg->start_sct = -1;
rmem_unicast(msg);
PDEBUG("received: sent msg(INITACK)\n");
} else if (msg->type == RMEM_EXIT) {
unsigned long i = 0;
PDEBUG("received: msg(EXIT_RMEM), replied with msg(EXIT_RMEM_ACK)\n");
addr = msg->sid;
msg->type = RMEM_EXIT_ACK;
if (pg_array) {
for (i = 0; i < nr_pages; i++)
if (pg_array[i]) {
//__free_page(__pa(pg_array[i]));
//new
free_page((unsigned long)pg_array[i]);
pg_array[i] = NULL;
}
vfree(pg_array);
pg_array = NULL;
}
/*
if (pg_array) {
msg->start_sct = -1;
printk("<1>local node %d: mem free failed\n", (int)local_id);
} else {
msg->start_sct = 1;
printk("<1>local node %d: mem free properly\n", (int)local_id);
}*/
msg->sid = local_id;
msg->rid = addr;
rmem_unicast(msg);
//PDEBUG("received: sent msg(EXIT_RMEM_ACK)\n");
}
PDEBUG("received: msg at the end\n");
kfree_skb(skb);
PDEBUG("received: free-ed all allocated mem\n");
return 0;
}
int init_client_mem(void) {
int ret;
PDEBUG("\n");
// check version if match
if (k2k_get_version() != K2K_VERSION) {
return -EACCES;
}
// Allocate the protocol data structure and initialise it to be
// for the specified protocol.
if ((proto = k2k_alloc_proto(RMEM_PROTO, GFP_ATOMIC)) == NULL)
return -ENOMEM;
proto->ucast_rcv = rmem_recv;
proto->mcast_rcv = NULL;
proto->node_av = NULL;
proto->node_del = NULL;
proto->rts = NULL;
memcpy(proto->name, "remote-mem", 10);
// After this, packets may be delivered via the
// call-back functions.
if ((ret = k2k_register_proto(proto)) < 0) {
k2k_free_proto(proto);
return ret;
}
PDEBUG("initialised\n");
return 0;
}
void cleanup_client_mem(void) {
unsigned long i = 0;
// free page array
if (pg_array) {
for (i = 0; i < nr_pages; i++)
if (pg_array[i]) {
//__free_page(pg_array[i]);
//new
free_page((unsigned long)pg_array[i]);
pg_array[i] = NULL;
}
vfree(pg_array);
pg_array = NULL;
}
if (!pg_array) {
PDEBUG("pg_array free-ed\n");
}
// unregister k2k protocol
k2k_unregister_proto(proto);
k2k_free_proto(proto);
PDEBUG("===================================\n");
}
module_init(init_client_mem);
module_exit(cleanup_client_mem);
On Thu, 2006-03-02 at 12:00 +1300, Xiangfei Jia wrote:
> I'm writing a kernel module. Most of the time, when I insert the
> module into the kernel, I got the error messgage from the kernel
> saying:
> __alloc_pages: 0-order allocation failed (gfp=0x20/0)
>
> I think the error message says that there is not a single page to be
> used for allocation. BUT, my module doesn't use a lot memory. I
> checked the message usage after inserting the module, there are still
> more free memory available. I'm quite confused what causes the
> problem. Can I get some help to figure out the problem??
Source code?
Avishay Traeger
http://www.fsl.cs.sunysb.edu/~avishay/
Yes. This time I include the source code. Basically the This module allocate and reserver memory as a remote storage for other machines. I allocate one page (using get_free_page() function) at a time, and all the allocated pages are indexed by an array (allocated by vmalloc() function). All the memory is allocated when the module receives a RMEM_INIT message (which is in the function rmem_recv).
I tested the module on serveral machines. It works fine on machines with 160MB physical memory. But, I always get the "0-order allocation failed" message when I test the module on machines with 120MB physical memory. Even though I only let the module allocate 20% of the avaialbe free memory, I still get the error message. I'm not sure if the vmalloc() function causes the problem or something else.
Thanks!!! Really appreciate all your help!!!
Fei
Here is the source code:
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include "client-mem.h"
#include "utils.h"
MODULE_LICENSE("GPL");
#define RMEM_PROTO 100
static struct k2k_proto *proto;
//static u8 *local_mem;
static __u16 local_id;
static __u16 server_id;
static int hard_sector;
#define INDEX(pos) ((pos) >> 3)
#define OFFSET(pos) ((pos) & 0x7)
// assume 4mb (1024 pages) to allocate
// plus 1 to cover out of array size exception
static unsigned long nr_pages = 0;
void **pg_array = NULL;
int is_allocated = -1;
int rmem_unicast(struct rmem_msg *msg) {
struct sk_buff *skb;
struct rmem_msg *buff;
struct k2k_node *nd;
__u16 node_to = msg->rid;
if ((nd = k2k_get_node(node_to)) == NULL) {
if ((nd = k2k_probe(node_to, 0)) == NULL)
return -EBADR;
}
if ((skb = k2k_alloc_unicast_skb(nd, proto, GFP_ATOMIC)) == NULL)
return -ENOMEM;
buff = (struct rmem_msg *)skb_put(skb, sizeof(struct rmem_msg));
memcpy(buff, msg, sizeof(struct rmem_msg));
k2k_unicast(skb, nd, proto, 0, 0);
k2k_node_put(nd);
return 0;
}
int rmem_recv(struct sk_buff *skb, struct k2k_node *nd) {
struct rmem_msg *msg;
__u16 addr;
void *ptr = NULL;
msg = (struct rmem_msg *)skb->data;
if (msg->type == RMEM_READ) {
unsigned long nr;
msg->type = RMEM_READ_ACK;
msg->sid = local_id;
msg->rid = server_id;
// using page array
// send the requested sector, one at a time, to server
for (nr = 1; nr < msg->nr_sct; nr++) {
msg->status = MSG_STATUS_MORE;
ptr = pg_array[INDEX(msg->start_sct)] +
OFFSET(msg->start_sct) * hard_sector;
memcpy(msg->data, ptr, hard_sector);
rmem_unicast(msg);
msg->start_sct++;
PDEBUG("send msg(RMEM_READ_ACK:MSG_STATUS_MORE)\n");
}
// send the last requested sector
msg->status = MSG_STATUS_OK;
ptr = pg_array[INDEX(msg->start_sct)] +
OFFSET(msg->start_sct) * hard_sector;
memcpy(msg->data, ptr, hard_sector);
rmem_unicast(msg);
PDEBUG("send msg(RMEM_READ_ACK:MSG_STATUS_OK)\n");
} else if (msg->type == RMEM_WRITE) {
PDEBUG("write: start_sct %ld, nr_sct: %ld, hard_sector: %d\n",
msg->start_sct, msg->nr_sct, hard_sector);
// using page array
ptr = pg_array[INDEX(msg->start_sct)] +
OFFSET(msg->start_sct) * hard_sector;
memcpy(ptr, msg->data, hard_sector);
PDEBUG("write: after memcpy(ptr, msg->data, size)\n");
} else if (msg->type == RMEM_INIT) {
unsigned long i = 0;
PDEBUG("msg->start_sct: %ld\n", msg->start_sct);
// start_sct contains the number of sectors
// a page can hold 8 sectors
//plus one to avoid array overflow
nr_pages = msg->start_sct / 8 + 1;
PDEBUG("**nr_pages: %ld\n", nr_pages);
// alloc page array
pg_array = (void *)vmalloc(nr_pages * sizeof(void *));
if (!pg_array) {
printk("<1>error vmalloc pg_array\n");
goto jump;
} else {
for (i = 0; i < nr_pages; i++) {
pg_array[i] = NULL;
}
for (i = 0; i < nr_pages; i++) {
pg_array[i] = (void *)get_free_page(GFP_ATOMIC);
//new
//pg_array[i] = page_address(alloc_page(GFP_ATOMIC));
if (!pg_array[i]) {
printk("<1>error get_free_page pg_array[%lu]\n", i);
}
}
}
jump:
PDEBUG("goto-ed here: jump\n");
hard_sector = msg->nr_sct; // size of harddisk sector
local_id = msg->rid; // get the local id for the node
server_id = msg->sid; // get the server id
PDEBUG("local_id: %d\n", (int)local_id);
PDEBUG("server_id: %d\n", (int)server_id);
// reply INIT with INITACK
PDEBUG("received: msg(INIT), replied with msg(INITACK)\n");
msg->type = RMEM_INIT_ACK;
msg->sid = local_id;
msg->rid = server_id;
msg->start_sct = -1;
rmem_unicast(msg);
PDEBUG("received: sent msg(INITACK)\n");
} else if (msg->type == RMEM_EXIT) {
unsigned long i = 0;
PDEBUG("received: msg(EXIT_RMEM), replied with msg(EXIT_RMEM_ACK)\n");
addr = msg->sid;
msg->type = RMEM_EXIT_ACK;
if (pg_array) {
for (i = 0; i < nr_pages; i++)
if (pg_array[i]) {
//__free_page(__pa(pg_array[i]));
//new
free_page((unsigned long)pg_array[i]);
pg_array[i] = NULL;
}
vfree(pg_array);
pg_array = NULL;
}
/*
if (pg_array) {
msg->start_sct = -1;
printk("<1>local node %d: mem free failed\n", (int)local_id);
} else {
msg->start_sct = 1;
printk("<1>local node %d: mem free properly\n", (int)local_id);
}*/
msg->sid = local_id;
msg->rid = addr;
rmem_unicast(msg);
//PDEBUG("received: sent msg(EXIT_RMEM_ACK)\n");
}
PDEBUG("received: msg at the end\n");
kfree_skb(skb);
PDEBUG("received: free-ed all allocated mem\n");
return 0;
}
int init_client_mem(void) {
int ret;
PDEBUG("\n");
// check version if match
if (k2k_get_version() != K2K_VERSION) {
return -EACCES;
}
// Allocate the protocol data structure and initialise it to be
// for the specified protocol.
if ((proto = k2k_alloc_proto(RMEM_PROTO, GFP_ATOMIC)) == NULL)
return -ENOMEM;
proto->ucast_rcv = rmem_recv;
proto->mcast_rcv = NULL;
proto->node_av = NULL;
proto->node_del = NULL;
proto->rts = NULL;
memcpy(proto->name, "remote-mem", 10);
// After this, packets may be delivered via the
// call-back functions.
if ((ret = k2k_register_proto(proto)) < 0) {
k2k_free_proto(proto);
return ret;
}
PDEBUG("initialised\n");
return 0;
}
void cleanup_client_mem(void) {
unsigned long i = 0;
// free page array
if (pg_array) {
for (i = 0; i < nr_pages; i++)
if (pg_array[i]) {
//__free_page(pg_array[i]);
//new
free_page((unsigned long)pg_array[i]);
pg_array[i] = NULL;
}
vfree(pg_array);
pg_array = NULL;
}
if (!pg_array) {
PDEBUG("pg_array free-ed\n");
}
// unregister k2k protocol
k2k_unregister_proto(proto);
k2k_free_proto(proto);
PDEBUG("===================================\n");
}
module_init(init_client_mem);
module_exit(cleanup_client_mem);
