Joe,
I was running more tests to get more data.
To answer your questions, I always use dd to write zeroes to ramdisk( you can also see this in the script I attached). So the ramdisk is allocated. I use
dd if=/dev/zero of=/dev/ram oflag=direct bs=512 count=`blockdev --getsize /dev/ram`
I get 9000+MB/s on allocated ramdisk using aio_stress. I was running the tests on same ramdisk. The performance doesn't change if i run thin test first and then raw test.
I modified the test to divide 4G ramdisk into 2 ramdisks. ( ramdisk1=2G ramdisk2=2G). Then I create tp lun on ramdisk1 and lvm lun on ramdisk2.
Numbers I see
ramdisk 10125 MB/s
ramdisk1 9690.55
ramdisk2 9548.89
unallocated tp lun 2257.67
fully allocated tp lun 5742.14
ramdisk 10288.2
ramdisk1 9721.8
ramdisk2 9415.25
Data not written lvm lun 9258.24
Data written lvm lun 9126.04
I have attached the script I use and the output. I wonder if using 16 quad core processor and doing lot of of parallel operations is causing the issue to be reproduced.
Thanks,
Jagan.
From: Joe Thornber <thornber@xxxxxxxxxx>
To: Jagan Reddy <gjmsreddy@xxxxxxxxx>
Cc: device-mapper development <dm-devel@xxxxxxxxxx>
Sent: Sunday, January 22, 2012 12:24 PM
Subject: Re: dm-thin vs lvm performance
On Fri, Jan 20, 2012 at 01:06:08PM -0800, Jagan Reddy wrote:
> One interesting thing I notice is raw ramdisk has 9000+MB/s throughput in my tests and 5440MB/s throughput in your tests.
Yes, and if the ramdisk is unallocated the throughput tests on my machine also give ~9G/s throughput. Can you please double check that it's allocated. eg, are you running all the tests on the same ram disk? If you run the thin test first and then the raw test does the raw performance change?
> I remember you mentioning that you are running tests in a VM with 4G
> memory, while I run the rests on a standalone server with 16
> quad-core processors and 12G or ram (out of that 12G, I carve out
> 4G ramdisk). Attached are the cpu and memory information. Could
> that be causing an issue?
Well memory performance will have an effect. But I'd expect your
number to be consistent with mine.
- Joe
Attachment:
lvm_thin_compare_redhat2.sh
Description: Bourne shell script
[root@lab-dm-cn4 md]# DEVICE_PATH=/dev/ram
[root@lab-dm-cn4 md]#
[root@lab-dm-cn4 md]# rm -f $DEVICE_PATH
[root@lab-dm-cn4 md]# mknod -m 660 $DEVICE_PATH b 1 1
[root@lab-dm-cn4 md]# #check the size of ramdisk
[root@lab-dm-cn4 md]# #it should be 4G as we booted the kernel with ramdisk_size=4194304
[root@lab-dm-cn4 md]# echo checking ramdisk size
checking ramdisk size
[root@lab-dm-cn4 md]# blockdev --getsize $DEVICE_PATH
8388608
[root@lab-dm-cn4 md]# echo zeroing out ramdisk
zeroing out ramdisk
[root@lab-dm-cn4 md]# dd if=/dev/zero of=$DEVICE_PATH oflag=direct bs=512 count=`blockdev --getsize $DEVICE_PATH`
8388608+0 records in
8388608+0 records out
4294967296 bytes (4.3 GB) copied, 13.8445 s, 310 MB/s
[root@lab-dm-cn4 md]#
[root@lab-dm-cn4 md]# #let's divide the ramdisk into 2 linear ramdisks of 2G in size
[root@lab-dm-cn4 md]# let fullramdisksize=`blockdev --getsize /dev/ram`
[root@lab-dm-cn4 md]# let ramdisksize=$fullramdisksize/2
[root@lab-dm-cn4 md]# echo 0 $ramdisksize linear $DEVICE_PATH 0 | $DMSETUP create ramdisk1
[root@lab-dm-cn4 md]# echo 0 $ramdisksize linear $DEVICE_PATH $ramdisksize | $DMSETUP create ramdisk2
[root@lab-dm-cn4 md]# echo checking ramdisk1 size
checking ramdisk1 size
[root@lab-dm-cn4 md]# blockdev --getsize /dev/mapper/ramdisk1
4194304
[root@lab-dm-cn4 md]# echo checking ramdisk2 size
checking ramdisk2 size
[root@lab-dm-cn4 md]# blockdev --getsize /dev/mapper/ramdisk2
4194304
[root@lab-dm-cn4 md]#
[root@lab-dm-cn4 md]# #constants
[root@lab-dm-cn4 md]# let ONEMEG=1024*2; #translated to sectors
[root@lab-dm-cn4 md]# DEVICE_PATH=/dev/mapper/ramdisk1
[root@lab-dm-cn4 md]# #user defined
[root@lab-dm-cn4 md]# DMSETUP=/home/jagan/src/LVM2.2.02.88/tools/dmsetup
[root@lab-dm-cn4 md]# LUN=1
[root@lab-dm-cn4 md]# META_DEVICE_START_SECTOR=0 #on $DEVICE_PATH this is where meta data starts
[root@lab-dm-cn4 md]# let META_DEV_SIZE=8*$ONEMEG; #8M translated to sectors
[root@lab-dm-cn4 md]# let DATA_DEV_SIZE=1*1024*$ONEMEG; #1G translated to sectors
[root@lab-dm-cn4 md]# let PAGESIZE=64*2 #page size is 64k in sectors
[root@lab-dm-cn4 md]# let DD_BS=4*1024 # dd block size = 4k bytes
[root@lab-dm-cn4 md]#
[root@lab-dm-cn4 md]# #system defined
[root@lab-dm-cn4 md]# let DATA_DEVICE_START_SECTOR=$META_DEVICE_START_SECTOR+$META_DEV_SIZE
[root@lab-dm-cn4 md]# let PAGES=$DATA_DEV_SIZE/$PAGESIZE
[root@lab-dm-cn4 md]# let SEEK=$PAGESIZE*512/$DD_BS # how many blocks we need to seek in dd to go to next page
[root@lab-dm-cn4 md]# let BLOCKS=$DATA_DEV_SIZE*512/$DD_BS
[root@lab-dm-cn4 md]# noofruns=20
[root@lab-dm-cn4 md]# AIO_PATH=/root/aio-stress.exe
[root@lab-dm-cn4 md]#
[root@lab-dm-cn4 md]# echo $DMSETUP LUNID=$LUN DEVICE=$DEVICE_PATH DEVICE_START=$META_DEVICE_START_SECTOR META_DEV_SIZE=$META_DEV_SIZE DATA_DEV_SIZE=$DATA_DEV_SIZE PAGESIZE=$PAGESIZE DD_BS=$DD_BS DATA_DEVICE_START_SECTOR=$DATA_DEVICE_START_SECTOR PAGES=$PAGES SEEK=$SEEK BLOCKS=$BLOCKS
/home/jagan/src/LVM2.2.02.88/tools/dmsetup LUNID=1 DEVICE=/dev/mapper/ramdisk1 DEVICE_START=0 META_DEV_SIZE=16384 DATA_DEV_SIZE=2097152 PAGESIZE=128 DD_BS=4096 DATA_DEVICE_START_SECTOR=16384 PAGES=16384 SEEK=16 BLOCKS=262144
[root@lab-dm-cn4 md]# echo 0 $META_DEV_SIZE linear $DEVICE_PATH $META_DEVICE_START_SECTOR | $DMSETUP create tp_meta_disk$LUN
[root@lab-dm-cn4 md]#
[root@lab-dm-cn4 md]# #zero out first 4k of meta data disk
[root@lab-dm-cn4 md]# dd if=/dev/zero of=/dev/mapper/tp_meta_disk$LUN bs=4096 count=1
1+0 records in
1+0 records out
4096 bytes (4.1 kB) copied, 3.7874e-05 s, 108 MB/s
[root@lab-dm-cn4 md]#
[root@lab-dm-cn4 md]# #create data disk
[root@lab-dm-cn4 md]# echo 0 $DATA_DEV_SIZE linear $DEVICE_PATH $DATA_DEVICE_START_SECTOR | $DMSETUP create tp_data_disk$LUN
[root@lab-dm-cn4 md]#
[root@lab-dm-cn4 md]#
[root@lab-dm-cn4 md]# #create thin pool
[root@lab-dm-cn4 md]# $DMSETUP create pool$LUN --table "0 $DATA_DEV_SIZE thin-pool /dev/mapper/tp_meta_disk$LUN /dev/mapper/tp_data_disk$LUN $PAGESIZE 0 " #1G page with 100 with no water mark
[root@lab-dm-cn4 md]#
[root@lab-dm-cn4 md]# #verify that 0 pages are allocated
[root@lab-dm-cn4 md]# echo verify that 0 pages are allocated
verify that 0 pages are allocated
[root@lab-dm-cn4 md]# $DMSETUP status pool$LUN
0 2097152 thin-pool 0 10/2048 0/16384 -
[root@lab-dm-cn4 md]# $DMSETUP message /dev/mapper/pool$LUN 0 "create_thin 0"
[root@lab-dm-cn4 md]# $DMSETUP create thin$LUN --table "0 $DATA_DEV_SIZE thin /dev/mapper/pool$LUN 0"
[root@lab-dm-cn4 md]# echo by now no pages should be allocated
by now no pages should be allocated
[root@lab-dm-cn4 md]# $DMSETUP status pool$LUN
0 2097152 thin-pool 0 11/2048 0/16384 -
[root@lab-dm-cn4 md]#
[root@lab-dm-cn4 md]#
[root@lab-dm-cn4 md]# DEVICE_PATH=/dev/mapper/ramdisk2
[root@lab-dm-cn4 md]# let LVM_DEVICE_SIZE=$META_DEV_SIZE+$DATA_DEV_SIZE;
[root@lab-dm-cn4 md]# let BLOCKS=($DATA_DEV_SIZE-$PAGESIZE)*512/$DD_BS
[root@lab-dm-cn4 md]#
[root@lab-dm-cn4 md]# #system defined
[root@lab-dm-cn4 md]#
[root@lab-dm-cn4 md]# echo $DMSETUP LUNID=$LUN DEVICE=$DEVICE_PATH DEVICE_START=$META_DEVICE_START_SECTOR META_DEV_SIZE=$META_DEV_SIZE DATA_DEV_SIZE=$DATA_DEV_SIZE PAGESIZE=$PAGESIZE DD_BS=$DD_BS DATA_DEVICE_START_SECTOR=$DATA_DEVICE_START_SECTOR PAGES=$PAGES SEEK=$SEEK BLOCKS=$BLOCKS LVM_DEVICE_SIZE=$LVM_DEVICE_SIZE
/home/jagan/src/LVM2.2.02.88/tools/dmsetup LUNID=1 DEVICE=/dev/mapper/ramdisk2 DEVICE_START=0 META_DEV_SIZE=16384 DATA_DEV_SIZE=2097152 PAGESIZE=128 DD_BS=4096 DATA_DEVICE_START_SECTOR=16384 PAGES=16384 SEEK=16 BLOCKS=262128 LVM_DEVICE_SIZE=2113536
[root@lab-dm-cn4 md]#
[root@lab-dm-cn4 md]# pvcreate -M2 --dataalignment 4K --metadatacopies 1 --metadatasize 8M --zero y -v /dev/mapper/ramdisk2
Set up physical volume for "/dev/mapper/ramdisk2" with 4194304 available sectors
Zeroing start of device /dev/mapper/ramdisk2
Writing physical volume data to disk "/dev/mapper/ramdisk2"
Physical volume "/dev/mapper/ramdisk2" successfully created
[root@lab-dm-cn4 md]# vgcreate -l 16384 -s 64k -c n -A n 6C067CWX00132-1 /dev/mapper/ramdisk2
WARNING: This metadata update is NOT backed up
Volume group "6C067CWX00132-1" successfully created
[root@lab-dm-cn4 md]# lvcreate -A n -C n -l 100%VG -Z y -n engineering -r None 6C067CWX00132-1
Using reduced mirror region size of 128 sectors
WARNING: This metadata update is NOT backed up
WARNING: This metadata update is NOT backed up
Logical volume "engineering" created
[root@lab-dm-cn4 md]#
[root@lab-dm-cn4 md]#
[root@lab-dm-cn4 md]#
[root@lab-dm-cn4 md]#
[root@lab-dm-cn4 md]#
[root@lab-dm-cn4 md]#
[root@lab-dm-cn4 md]#
[root@lab-dm-cn4 md]# echo measuring performance of ramdisk
measuring performance of ramdisk
[root@lab-dm-cn4 md]# i=0; while [ $i -lt $noofruns ] ; do $AIO_PATH -O -o 1 -c 16 -t 16 -d 256 /dev/ram 2>&1 | grep throughput | cut -f3 -d ' ' | cut -f2 -d '(' | cut -f1 -d ' ' ; let i=$i+1; done | awk -v runs=$noofruns 'BEGIN{i=0.0} {i+=$0} END{print i/runs}'
10125
[root@lab-dm-cn4 md]#
[root@lab-dm-cn4 md]# echo measuring performance of ramdisk1
measuring performance of ramdisk1
[root@lab-dm-cn4 md]# i=0; while [ $i -lt $noofruns ] ; do $AIO_PATH -O -o 1 -c 16 -t 16 -d 256 /dev/mapper/ramdisk1 2>&1 | grep throughput | cut -f3 -d ' ' | cut -f2 -d '(' | cut -f1 -d ' ' ; let i=$i+1; done | awk -v runs=$noofruns 'BEGIN{i=0.0} {i+=$0} END{print i/runs}'
9690.55
[root@lab-dm-cn4 md]#
[root@lab-dm-cn4 md]# echo measuring performance of ramdisk2
measuring performance of ramdisk2
[root@lab-dm-cn4 md]# i=0; while [ $i -lt $noofruns ] ; do $AIO_PATH -O -o 1 -c 16 -t 16 -d 256 /dev/mapper/ramdisk2 2>&1 | grep throughput | cut -f3 -d ' ' | cut -f2 -d '(' | cut -f1 -d ' ' ; let i=$i+1; done | awk -v runs=$noofruns 'BEGIN{i=0.0} {i+=$0} END{print i/runs}'
9548.89
[root@lab-dm-cn4 md]#
[root@lab-dm-cn4 md]# echo measuring performance of un-allocated tp lun
measuring performance of un-allocated tp lun
[root@lab-dm-cn4 md]# i=0; while [ $i -lt $noofruns ] ; do $AIO_PATH -O -o 1 -c 16 -t 16 -d 256 /dev/mapper/thin$LUN 2>&1 | grep throughput | cut -f3 -d ' ' | cut -f2 -d '(' | cut -f1 -d ' ' ; let i=$i+1; done | awk -v runs=$noofruns 'BEGIN{i=0.0} {i+=$0} END{print i/runs}'
2257.67
[root@lab-dm-cn4 md]#
[root@lab-dm-cn4 md]# echo touching all pages in tp lun to make it fully allocated
touching all pages in tp lun to make it fully allocated
[root@lab-dm-cn4 md]# i=0;while [ $i -lt $PAGES ]; do let j=$i*$SEEK; echo $i; dd if=/dev/zero of=/dev/mapper/thin$LUN bs=4096 seek=$j count=1; let i=$i+1; done >&/dev/null
[root@lab-dm-cn4 md]# echo now all pages should be allocated
now all pages should be allocated
[root@lab-dm-cn4 md]# #touch all pages on thin provisoned data disk so that all pages are allocated
[root@lab-dm-cn4 md]# $DMSETUP status pool$LUN
0 2097152 thin-pool 0 147/2048 16384/16384 -
[root@lab-dm-cn4 md]#
[root@lab-dm-cn4 md]# echo measuring performance of fully-allocated tp lun
measuring performance of fully-allocated tp lun
[root@lab-dm-cn4 md]# i=0; while [ $i -lt $noofruns ] ; do $AIO_PATH -O -o 1 -c 16 -t 16 -d 256 /dev/mapper/thin$LUN 2>&1 | grep throughput | cut -f3 -d ' ' | cut -f2 -d '(' | cut -f1 -d ' ' ; let i=$i+1; done | awk -v runs=$noofruns 'BEGIN{i=0.0} {i+=$0} END{print i/runs}'
5742.14
[root@lab-dm-cn4 md]#
[root@lab-dm-cn4 md]# echo measuring performance of ramdisk
measuring performance of ramdisk
[root@lab-dm-cn4 md]# i=0; while [ $i -lt $noofruns ] ; do $AIO_PATH -O -o 1 -c 16 -t 16 -d 256 /dev/ram 2>&1 | grep throughput | cut -f3 -d ' ' | cut -f2 -d '(' | cut -f1 -d ' ' ; let i=$i+1; done | awk -v runs=$noofruns 'BEGIN{i=0.0} {i+=$0} END{print i/runs}'
10288.2
[root@lab-dm-cn4 md]#
[root@lab-dm-cn4 md]# echo measuring performance of ramdisk1
measuring performance of ramdisk1
[root@lab-dm-cn4 md]# i=0; while [ $i -lt $noofruns ] ; do $AIO_PATH -O -o 1 -c 16 -t 16 -d 256 /dev/mapper/ramdisk1 2>&1 | grep throughput | cut -f3 -d ' ' | cut -f2 -d '(' | cut -f1 -d ' ' ; let i=$i+1; done | awk -v runs=$noofruns 'BEGIN{i=0.0} {i+=$0} END{print i/runs}'
9721.8
[root@lab-dm-cn4 md]#
[root@lab-dm-cn4 md]# echo measuring performance of ramdisk2
measuring performance of ramdisk2
[root@lab-dm-cn4 md]# i=0; while [ $i -lt $noofruns ] ; do $AIO_PATH -O -o 1 -c 16 -t 16 -d 256 /dev/mapper/ramdisk2 2>&1 | grep throughput | cut -f3 -d ' ' | cut -f2 -d '(' | cut -f1 -d ' ' ; let i=$i+1; done | awk -v runs=$noofruns 'BEGIN{i=0.0} {i+=$0} END{print i/runs}'
9415.25
[root@lab-dm-cn4 md]#
[root@lab-dm-cn4 md]#
[root@lab-dm-cn4 md]# echo measuring performance of data-not-written lvm lun
measuring performance of data-not-written lvm lun
[root@lab-dm-cn4 md]# i=0; while [ $i -lt $noofruns ] ; do $AIO_PATH -O -o 1 -c 16 -t 16 -d 256 /dev/mapper/6C067CWX00132--1-engineering 2>&1 | grep throughput | cut -f3 -d ' ' | cut -f2 -d '(' | cut -f1 -d ' ' ; let i=$i+1; done | awk -v runs=$noofruns 'BEGIN{i=0.0} {i+=$0} END{print i/runs}'
9258.24
[root@lab-dm-cn4 md]# #write something
[root@lab-dm-cn4 md]# dd if=/dev/zero of=/dev/mapper/6C067CWX00132--1-engineering bs=4096 count=$BLOCKS
262128+0 records in
262128+0 records out
1073676288 bytes (1.1 GB) copied, 1.96007 s, 548 MB/s
[root@lab-dm-cn4 md]#
[root@lab-dm-cn4 md]# echo measuring performance of data-written lvm lun
measuring performance of data-written lvm lun
[root@lab-dm-cn4 md]# i=0; while [ $i -lt $noofruns ] ; do $AIO_PATH -O -o 1 -c 16 -t 16 -d 256 /dev/mapper/6C067CWX00132--1-engineering 2>&1 | grep throughput | cut -f3 -d ' ' | cut -f2 -d '(' | cut -f1 -d ' ' ; let i=$i+1; done | awk -v runs=$noofruns 'BEGIN{i=0.0} {i+=$0} END{print i/runs}'
9126.04
[root@lab-dm-cn4 md]#
-- dm-devel mailing list dm-devel@xxxxxxxxxx https://www.redhat.com/mailman/listinfo/dm-devel
