NFS: low read/stat performance on small files

[Vivek Trivedi <vtrivedi018@xxxxxxxxx>]

Hi,
we are facing below 2 performance issue on NFS:

1. Read speed is low for small files
==========================

[ Log on NFS Client]

$ echo 3 > /proc/sys/vm/drop_caches
$ dd if=200KBfile.txt of=/dev/null
400+0 records in
400+0 records out
204800 bytes (200.0KB) copied, 0.027074 seconds, 7.2MB/s


Read speed for 200KB file is 7.2 MB


[ Log on NFS Client]

$ echo 3 > /proc/sys/vm/drop_caches
$ dd if=100MBfile.txt of=/dev/null
204800+0 records in
204800+0 records out
104857600 bytes (100.0MB) copied, 9.351221 seconds, 10.7MB/s

Read speed for 100MB file is 10.7 MB

As you see read speed for 200KB file is only 7.2MB/sec while it is
10.7 MB/sec when we read 100MB file.
Why there is so much difference in read performance ?
Is there any way to achieve high read speed for small files ?


2. Read/stat for a directory tree is slow on NFS than local
==========================================

we have lot of *.jpg files in a directory. If we try to "stat" and
"read" from this directory,
 performannce is very slow on NFS Client compared to Local(NFS server)
"stat" and "read"


[ Log on Local (NFS Server) ]

$ echo 3 > /proc/sys/vm/drop_caches
$ ./stat_read_files_test ./lot_of_jpg_files/
 Time Taken : 9288 msec


[ Log on NFS Client]

$ echo 3 > /proc/sys/vm/drop_caches
$ ./stat_read_files_test ./lot_of_jpg_files/
 Time Taken : 19966 msec

As you see, on NFS client time taken is almost *double* than that of
local(NFS server)
We are using UDP with rsize,wsize=32k on 100MB ethernet link.
I am attaching read/stat testcase.

Is there any way to improve this performance ?


Thanks,
Vivek

#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <fcntl.h>
#include <string.h>
#include <sys/time.h>

#define BUFFERSIZE 4096
char READBUFFER[BUFFERSIZE];

#include <dirent.h>
int TraversePath(char *path)
{
	struct dirent *d = NULL;
	DIR *dir = NULL; /* pointer to directory head*/
	char buf[255]={0}; /* buffer to store the complete file/dir name*/
	struct stat statbuf; /* to obtain the statistics of file/dir */
	int retval =0; /* to hold the return value*/
	int fd = 0;

	memset(&statbuf,0,sizeof(struct stat)); 

	retval = stat(path,&statbuf);

	/* if the stat returned success and path provided is of valid directory*/
	if(S_ISDIR(statbuf.st_mode) && (retval==0))
	{
		dir = opendir(path); /* open the directory*/	
		/* reads entry one by one*/
		while((d = readdir(dir)) != NULL)
		{
			if((strcmp(d->d_name,".")!=0) && (strcmp(d->d_name,"..")!=0))
			{
				sprintf(buf,"%s/%s",path,d->d_name);
				retval = stat(buf,&statbuf);
				if(retval == 0)
				{
					if(!S_ISDIR(statbuf.st_mode))
					{		
						/* This is file - read from this, Since read ahead 
						 * will itself bring 128KB, so we can just read 4KB
						 * to start with */
						fd = open(buf,O_RDONLY,(mode_t)777);
						if(fd) {
							read(fd, READBUFFER, BUFFERSIZE);
							close(fd);	
						}
					}
					else
					{
						/*
						   This is a directory, recursive search in it
						   once all files are read
						 */
						TraversePath(buf);
					}
				}
				else
				{
					perror("stat failed\n");
				}
			}
		}
	}
	else
	{
		perror("Failed");
	}	
	return retval;
}

int main(int argc, char **argv)	
{
	struct timeval rv;
	struct timeval rv1;

	int stat_time = 0;

	if(argc < 2) {
		printf("./TraversePath <path> \n");
		return 0;
	}	

	//Traverse the complete path inside timing unit
	gettimeofday(&rv, 0);
	TraversePath(argv[1]);
	gettimeofday(&rv1, 0);

	stat_time = (rv1.tv_sec * 1000 + rv1.tv_usec / 1000) - (rv.tv_sec * 1000 + rv.tv_usec / 1000);

	printf(" Traversed Path : %s \n", argv[1]);
	printf(" Time Taken : %d msec \n",stat_time);

	return 0;
}