I have just test the performance with a modified src/testrados.c,
the read and write speed is very slow too:
=========================================================
root@xxxxxxxxxxxxxxxxxxxxxxxxxxxx # ./testrados
rados_ioctx_create = 0
rados_ioctx_create = 0, io_ctx = 0x7f52d00022b0
need buffer size of 42
begin pools.
pool: 'data'
pool: 'metadata'
pool: 'casdata'
pool: 'rbd'
pool: 'librbdtest'
pool: 'foo'
end pools.
rados_ioctx_pool_stat = 0, 0 KB, 0 objects
rados_ioctx_snap_create snap1 = 0
rados_ioctx_snap_list got snap 1 snap1
rados_ioctx_snap_lookup snap1 got 1, result 0
rados_ioctx_snap_remove snap1 = 0
rados_write = 1024
write time used: temp=20.400
write speed: 0.49 MB/s
^^^^^^^^^^^^
rados_read = 1024
read time used: temp=7.110
read speed: 1.41 MB/s
^^^^^^^^^^^^
rados_setxattr attr1=bar = 3
rados_getxattr attr1 = 3
rados_stat size = 10485760 mtime = 1305463463 = 0
rados_stat(does_not_exist) = -2
exec result=
read result=
size=128
a safe
b safe
rados_list_objects_open = 0, h = 0x7f52d0000c10
rados_list_objects_next got object '../b/bb_bb_bb\foo\bar'
rados_list_objects_next got object 'a'
rados_list_objects_next got object 'foo_object'
rados_stat_pool = 0, 10240 KB, 1 objects
rados_delete_pool = 0
rados_ioctx_pool_delete = 0
=========================================================
Hmm, test with testceph.cc is all right. 3 replca, write speed:
25.17 MB/s read speed: 53.46 MB/s
Attachments are the test code. FYI
Thx!
Simon
2011/5/15 Simon Tian <aixt2006@xxxxxxxxx>:
>> What is the IO size? ÂIs write_test_data synchronous?
>>
>> For simple write benchmarking you can also use
>> Â Â Â Ârados mkpool foo
>> Â Â Â Ârados -p foo bench <seconds> write -b <blocksize> -t <threads>
>>
>> and you'll see latency and throughput. ÂBlocksize defaults to 4M and
>> "threads" (parallel IOs) default to 16, IIRC.
>
>
> Hi, Sage:
>
> I just did the bench:
> rados -p rbd bench 60 write -t 64  and  Ârados -p data bench 60 write -t 64
> the avg throughput is about 46MB/s, one of the result is as follow.
> But why it's slow with rbd api from <rbd/librbd.h>?
> And I tried testlibrbdpp.cc, the result is more or less the same.
> The attachments are the test codes. Could you run it on your platform please?
>
> Âsec Cur ops  started Âfinished Âavg MB/s Âcur MB/s Âlast lat  avg lat
> Â 40 Â Â Â63 Â Â Â 482 Â Â Â 419 Â 41.8884 Â Â Â Â44 Â 2.40044 Â 2.40979
> Â 41 Â Â Â63 Â Â Â 494 Â Â Â 431 Â 42.0372 Â Â Â Â48 Â 2.11044 Â Â 2.406
> Â 42 Â Â Â64 Â Â Â 506 Â Â Â 442 Â 42.0837 Â Â Â Â44 Â 2.11266 Â 2.40229
> Â 43 Â Â Â63 Â Â Â 518 Â Â Â 455 Â 42.3139 Â Â Â Â52 Â 2.33468 Â Â2.3982
> Â 44 Â Â Â63 Â Â Â 527 Â Â Â 464 Â 42.1703 Â Â Â Â36 Â Â2.4403 Â 2.39559
> Â 45 Â Â Â63 Â Â Â 539 Â Â Â 476 Â 42.2995 Â Â Â Â48 Â 2.19768 Â 2.39413
> Â 46 Â Â Â63 Â Â Â 551 Â Â Â 488 Â 42.4232 Â Â Â Â48 Â 2.51232 Â Â2.3928
> Â 47 Â Â Â63 Â Â Â 563 Â Â Â 500 Â 42.5416 Â Â Â Â48 Â 2.18025 Â 2.38958
> Â 48 Â Â Â63 Â Â Â 572 Â Â Â 509 Â 42.4051 Â Â Â Â36 Â 2.27111 Â 2.38791
> Â 49 Â Â Â63 Â Â Â 584 Â Â Â 521 Â Â42.519 Â Â Â Â48 Â 2.41684 Â 2.38695
> Â 50 Â Â Â63 Â Â Â 596 Â Â Â 533 Â 42.6284 Â Â Â Â48 Â 2.11087 Â Â 2.384
> Â 51 Â Â Â63 Â Â Â 608 Â Â Â 545 Â 42.7335 Â Â Â Â48 Â 2.18147 Â 2.37925
> Â 52 Â Â Â63 Â Â Â 620 Â Â Â 557 Â 42.8345 Â Â Â Â48 Â 2.45287 Â 2.37787
> Â 53 Â Â Â63 Â Â Â 629 Â Â Â 566 Â 42.7054 Â Â Â Â36 Â 2.45187 Â 2.37801
> Â 54 Â Â Â63 Â Â Â 644 Â Â Â 581 Â 43.0255 Â Â Â Â60 Â 2.22403 Â 2.37477
> Â 55 Â Â Â63 Â Â Â 653 Â Â Â 590 Â 42.8976 Â Â Â Â36 Â 2.22782 Â 2.37157
> Â 56 Â Â Â63 Â Â Â 668 Â Â Â 605 Â 43.2026 Â Â Â Â60 Â 2.20638 Â 2.36597
> Â 57 Â Â Â63 Â Â Â 677 Â Â Â 614 Â 43.0761 Â Â Â Â36 Â 2.19628 Â 2.36209
> Â 58 Â Â Â63 Â Â Â 689 Â Â Â 626 Â 43.1608 Â Â Â Â48 Â 2.18262 Â 2.35762
> Â 59 Â Â Â63 Â Â Â 704 Â Â Â 641 Â 43.4459 Â Â Â Â60 Â 2.27029 Â 2.35352
> min lat: 1.87981 max lat: 5.56194 avg lat: 2.34944
> Âsec Cur ops  started Âfinished Âavg MB/s Âcur MB/s Âlast lat  avg lat
> Â 60 Â Â Â63 Â Â Â 716 Â Â Â 653 Â 43.5215 Â Â Â Â48 Â 2.27835 Â 2.34944
> Â 61 Â Â Â64 Â Â Â 717 Â Â Â 653 Â Â42.808 Â Â Â Â 0 Â Â Â Â - Â 2.34944
> Â 62 Â Â Â63 Â Â Â 717 Â Â Â 654 Â 42.1821 Â Â Â Â 2 Â 2.25694 Â 2.34929
> Total time run: Â Â Â Â62.274719
> Total writes made: Â Â 717
> Write size: Â Â Â Â Â Â4194304
> Bandwidth (MB/sec): Â Â46.054
>
> Average Latency: Â Â Â 5.453
> Max latency: Â Â Â Â Â 62.0339
> Min latency: Â Â Â Â Â 1.87981
>
// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
// vim: ts=8 sw=2 smarttab
/*
* Ceph - scalable distributed file system
*
* Copyright (C) 2004-2006 Sage Weil <sage@xxxxxxxxxxxx>
*
* This is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License version 2.1, as published by the Free Software
* Foundation. See file COPYING.
*
*/
#include <rados/librados.h>
#include <stdio.h>
#include <stdlib.h>
#include <time.h>
#define TEST_IO_SIZE 1024
#define TEST_IO_NUM 1024*10
int main(int argc, const char **argv)
{
char tmp[32];
int i, r;
rados_t cl;
if (rados_create(&cl, NULL) < 0) {
printf("error initializing\n");
exit(1);
}
if (rados_conf_read_file(cl, NULL)) {
printf("error reading configuration file\n");
exit(1);
}
// Try to set a configuration option that doesn't exist.
// This should fail.
if (!rados_conf_set(cl, "config option that doesn't exist",
"some random value")) {
printf("error: succeeded in setting nonexistent config option\n");
exit(1);
}
if (rados_conf_get(cl, "log to stderr", tmp, sizeof(tmp))) {
printf("error: failed to read log_to_stderr from config\n");
exit(1);
}
// Can we change it?
if (rados_conf_set(cl, "log to stderr", "2")) {
printf("error: error setting log_to_stderr\n");
exit(1);
}
rados_reopen_log(cl);
if (rados_conf_get(cl, "log to stderr", tmp, sizeof(tmp))) {
printf("error: failed to read log_to_stderr from config\n");
exit(1);
}
if (tmp[0] != '2') {
printf("error: new setting for log_to_stderr failed to take effect.\n");
exit(1);
}
if (rados_connect(cl)) {
printf("error connecting\n");
exit(1);
}
/* create an io_ctx */
r = rados_pool_create(cl, "foo");
printf("rados_ioctx_create = %d\n", r);
rados_ioctx_t io_ctx;
r = rados_ioctx_create(cl, "foo", &io_ctx);
printf("rados_ioctx_create = %d, io_ctx = %p\n", r, io_ctx);
/* list all pools */
{
int buf_sz = rados_pool_list(cl, NULL, 0);
printf("need buffer size of %d\n", buf_sz);
char buf[buf_sz];
int r = rados_pool_list(cl, buf, buf_sz);
if (r != buf_sz) {
printf("buffer size mismatch: got %d the first time, but %d "
"the second.\n", buf_sz, r);
exit(1);
}
const char *b = buf;
printf("begin pools.\n");
while (1) {
if (b[0] == '\0')
break;
printf(" pool: '%s'\n", b);
b += strlen(b) + 1;
};
printf("end pools.\n");
}
/* stat */
struct rados_pool_stat_t st;
r = rados_ioctx_pool_stat(io_ctx, &st);
printf("rados_ioctx_pool_stat = %d, %lld KB, %lld objects\n", r, (long long)st.num_kb, (long long)st.num_objects);
/* snapshots */
r = rados_ioctx_snap_create(io_ctx, "snap1");
printf("rados_ioctx_snap_create snap1 = %d\n", r);
rados_snap_t snaps[10];
r = rados_ioctx_snap_list(io_ctx, snaps, 10);
for (i=0; i<r; i++) {
char name[100];
rados_ioctx_snap_get_name(io_ctx, snaps[i], name, sizeof(name));
printf("rados_ioctx_snap_list got snap %lld %s\n", (long long)snaps[i], name);
}
rados_snap_t snapid;
r = rados_ioctx_snap_lookup(io_ctx, "snap1", &snapid);
printf("rados_ioctx_snap_lookup snap1 got %lld, result %d\n", (long long)snapid, r);
r = rados_ioctx_snap_remove(io_ctx, "snap1");
printf("rados_ioctx_snap_remove snap1 = %d\n", r);
/* sync io */
time_t tm;
char buf[TEST_IO_SIZE], buf2[TEST_IO_SIZE];
for(i=0; i<TEST_IO_SIZE; i++)
{
buf[i] = 0xFF&i;
}
const char *oid = "foo_object";
struct timeval tv1, tv2;
time_t t1;
gettimeofday(&tv1, NULL);
int offset = 0;
for(i=0; i<TEST_IO_NUM; i++)
{
r = rados_write(io_ctx, oid, buf, TEST_IO_SIZE, offset);
offset += TEST_IO_SIZE;
}
printf("rados_write = %d\n", r);
gettimeofday(&tv2, NULL);
t1 = tv2.tv_sec-tv1.tv_sec;
float speed=0.1, temp=1.0;
temp = (float)t1 + (tv2.tv_usec-tv1.tv_usec)/1000000.0;
printf("write time used: temp=%.3f\n",temp);
speed = 1.0*TEST_IO_SIZE*TEST_IO_NUM/temp/1024/1024;
printf("write speed: %.2f MB/s\n", speed);
gettimeofday(&tv1, NULL);
offset = 0;
for(i=0; i<TEST_IO_NUM; i++)
{
r = rados_read(io_ctx, oid, buf2, TEST_IO_SIZE, offset);
offset += TEST_IO_SIZE;
}
printf("rados_read = %d\n", r);
gettimeofday(&tv2, NULL);
t1 = tv2.tv_sec-tv1.tv_sec;
temp = (float)t1 + (tv2.tv_usec-tv1.tv_usec)/1000000.0;
printf("read time used: temp=%.3f\n",temp);
speed = 1.0*TEST_IO_SIZE*TEST_IO_NUM/temp/1024/1024;
printf("read speed: %.2f MB/s\n", speed);
if (memcmp(buf, buf2, r))
printf("*** content mismatch ***\n");
/* attrs */
r = rados_setxattr(io_ctx, oid, "attr1", "bar", 3);
printf("rados_setxattr attr1=bar = %d\n", r);
char val[10];
r = rados_getxattr(io_ctx, oid, "attr1", val, sizeof(val));
printf("rados_getxattr attr1 = %d\n", r);
if (memcmp(val, "bar", 3))
printf("*** attr value mismatch ***\n");
uint64_t size;
time_t mtime;
r = rados_stat(io_ctx, oid, &size, &mtime);
printf("rados_stat size = %lld mtime = %d = %d\n", (long long)size, (int)mtime, r);
r = rados_stat(io_ctx, "does_not_exist", NULL, NULL);
printf("rados_stat(does_not_exist) = %d\n", r);
/* exec */
rados_exec(io_ctx, oid, "crypto", "md5", buf, strlen(buf) + 1, buf, 128);
printf("exec result=%s\n", buf);
r = rados_read(io_ctx, oid, buf2, 128, 0);
printf("read result=%s\n", buf2);
printf("size=%d\n", r);
/* aio */
rados_completion_t a, b;
rados_aio_create_completion(0, 0, 0, &a);
rados_aio_create_completion(0, 0, 0, &b);
rados_aio_write(io_ctx, "a", a, buf, 100, 0);
rados_aio_write(io_ctx, "../b/bb_bb_bb\\foo\\bar", b, buf, 100, 0);
rados_aio_wait_for_safe(a);
printf("a safe\n");
rados_aio_wait_for_safe(b);
printf("b safe\n");
rados_aio_release(a);
rados_aio_release(b);
rados_read(io_ctx, "../b/bb_bb_bb\\foo\\bar", buf2, 128, 0);
/* list objects */
rados_list_ctx_t h;
r = rados_objects_list_open(io_ctx, &h);
printf("rados_list_objects_open = %d, h = %p\n", r, h);
const char *poolname;
while (rados_objects_list_next(h, &poolname) == 0)
printf("rados_list_objects_next got object '%s'\n", poolname);
rados_objects_list_close(h);
/* stat */
r = rados_ioctx_pool_stat(io_ctx, &st);
printf("rados_stat_pool = %d, %lld KB, %lld objects\n", r, (long long)st.num_kb, (long long)st.num_objects);
/* delete a pool */
printf("rados_delete_pool = %d\n", r);
rados_ioctx_destroy(io_ctx);
r = rados_pool_delete(cl, "foo");
printf("rados_ioctx_pool_delete = %d\n", r);
rados_shutdown(cl);
return 0;
}
Attachment:
testceph.cc
Description: Binary data
