On 29/09/2014 14:34, Janne Grunau wrote:
> SIMD optimized erasure code computation needs aligned memory. Buffers
> aligned to a page boundary are not needed though. The buffers used
> for the erasure code computation are typical smaller than a page.
>
> The typical alignment requirements SIMD operations are 16 bytes for
> SSE2 and NEON and 32 bytes for AVX/AVX2.
>
> Signed-off-by: Janne Grunau <j@xxxxxxxxxx>
> ---
> configure.ac | 9 +++++
> src/common/buffer.cc | 106 +++++++++++++++++++++++++++++++++++++++++++++++++++
> src/include/buffer.h | 15 ++++++++
> 3 files changed, 130 insertions(+)
>
> diff --git a/configure.ac b/configure.ac
> index cccf2d9..1bb27c4 100644
> --- a/configure.ac
> +++ b/configure.ac
> @@ -793,6 +793,15 @@ AC_MSG_RESULT([no])
> ])
>
> #
> +# Check for functions to provide aligned memory
> +#
> +AC_CHECK_HEADERS([malloc.h])
> +AC_CHECK_FUNCS([posix_memalign _aligned_malloc memalign aligned_malloc],
> + [found_memalign=yes; break])
> +
> +AS_IF([test "x$found_memalign" != "xyes"], [AC_MSG_WARN([No function for aligned memory allocation found])])
> +
> +#
> # Check for pthread spinlock (depends on ACX_PTHREAD)
> #
> saved_LIBS="$LIBS"
> diff --git a/src/common/buffer.cc b/src/common/buffer.cc
> index af298ac..dfe887e 100644
> --- a/src/common/buffer.cc
> +++ b/src/common/buffer.cc
> @@ -30,6 +30,10 @@
> #include <sys/uio.h>
> #include <limits.h>
>
> +#ifdef HAVE_MALLOC_H
> +#include <malloc.h>
> +#endif
> +
> namespace ceph {
>
> #ifdef BUFFER_DEBUG
> @@ -155,9 +159,17 @@ static simple_spinlock_t buffer_debug_lock = SIMPLE_SPINLOCK_INITIALIZER;
> virtual int zero_copy_to_fd(int fd, loff_t *offset) {
> return -ENOTSUP;
> }
> + virtual bool is_aligned(unsigned align) {
> + assert((align >= sizeof(void *)) && (align & (align - 1)) == 0);
> + return ((intptr_t)data & (align - 1)) == 0;
> + }
> virtual bool is_page_aligned() {
> return ((long)data & ~CEPH_PAGE_MASK) == 0;
> }
> + bool is_n_align_sized(unsigned align) {
> + assert((align >= sizeof(void *)) && (align & (align - 1)) == 0);
This does not seem necessary in this context ?
> + return (len % align) == 0;
> + }
> bool is_n_page_sized() {
> return (len & ~CEPH_PAGE_MASK) == 0;
> }
> @@ -209,6 +221,43 @@ static simple_spinlock_t buffer_debug_lock = SIMPLE_SPINLOCK_INITIALIZER;
> }
> };
>
> + class buffer::raw_aligned : public buffer::raw {
> + unsigned _align;
> + public:
> + raw_aligned(unsigned l, unsigned align) : raw(l) {
> + _align = align;
> + assert((align >= sizeof(void *)) && (align & (align - 1)) == 0);
> + if (len) {
> +#if HAVE_POSIX_MEMALIGN
> + if (posix_memalign((void **) &data, align, len))
> + data = 0;
> +#elif HAVE__ALIGNED_MALLOC
> + data = _aligned_malloc(len, align);
> +#elif HAVE_MEMALIGN
> + data = memalign(align, len);
> +#elif HAVE_ALIGNED_MALLOC
> + data = aligned_malloc((len + align - 1) & (align - 1), align);
> +#else
> + data = malloc(len);
> +#endif
> + if (!data)
> + throw bad_alloc();
> + } else {
> + data = 0;
> + }
> + inc_total_alloc(len);
> + bdout << "raw_aligned " << this << " alloc " << (void *)data << " " << l << " " << buffer::get_total_alloc() << bendl;
> + }
> + ~raw_aligned() {
> + free(data);
> + dec_total_alloc(len);
> + bdout << "raw_aligned " << this << " free " << (void *)data << " " << buffer::get_total_alloc() << bendl;
> + }
> + raw* clone_empty() {
> + return new raw_aligned(len, _align);
> + }
> + };
> +
> #ifndef __CYGWIN__
> class buffer::raw_mmap_pages : public buffer::raw {
> public:
> @@ -334,6 +383,10 @@ static simple_spinlock_t buffer_debug_lock = SIMPLE_SPINLOCK_INITIALIZER;
> return true;
> }
>
> + bool is_aligned() {
> + return false;
> + }
> +
> bool is_page_aligned() {
> return false;
> }
> @@ -520,6 +573,9 @@ static simple_spinlock_t buffer_debug_lock = SIMPLE_SPINLOCK_INITIALIZER;
> buffer::raw* buffer::create_static(unsigned len, char *buf) {
> return new raw_static(buf, len);
> }
> + buffer::raw* buffer::create_aligned(unsigned len, unsigned align) {
> + return new raw_aligned(len, align);
> + }
> buffer::raw* buffer::create_page_aligned(unsigned len) {
> #ifndef __CYGWIN__
> //return new raw_mmap_pages(len);
> @@ -1013,6 +1069,17 @@ static simple_spinlock_t buffer_debug_lock = SIMPLE_SPINLOCK_INITIALIZER;
> return true;
> }
>
> + bool buffer::list::is_aligned(unsigned align) const
> + {
> + assert((align >= sizeof(void *)) && (align & (align - 1)) == 0);
> + for (std::list<ptr>::const_iterator it = _buffers.begin();
> + it != _buffers.end();
> + ++it)
> + if (!it->is_aligned(align))
> + return false;
> + return true;
> + }
> +
> bool buffer::list::is_page_aligned() const
> {
> for (std::list<ptr>::const_iterator it = _buffers.begin();
> @@ -1101,6 +1168,45 @@ static simple_spinlock_t buffer_debug_lock = SIMPLE_SPINLOCK_INITIALIZER;
> _buffers.push_back(nb);
> }
>
> +void buffer::list::rebuild_aligned(unsigned align)
> +{
> + std::list<ptr>::iterator p = _buffers.begin();
> + assert((align >= sizeof(void *)) && (align & (align - 1)) == 0);
> + while (p != _buffers.end()) {
> + // keep anything that's already align sized+aligned
> + if (p->is_aligned(align) && p->is_n_align_sized(align)) {
> + /*cout << " segment " << (void*)p->c_str()
> + << " offset " << ((unsigned long)p->c_str() & (align - 1))
> + << " length " << p->length()
> + << " " << (p->length() & (align - 1)) << " ok" << std::endl;
> + */
> + ++p;
> + continue;
> + }
> +
> + // consolidate unaligned items, until we get something that is sized+aligned
> + list unaligned;
> + unsigned offset = 0;
> + do {
> + /*cout << " segment " << (void*)p->c_str()
> + << " offset " << ((unsigned long)p->c_str() & (align - 1))
> + << " length " << p->length() << " " << (p->length() & (align - 1))
> + << " overall offset " << offset << " " << (offset & (align - 1))
> + << " not ok" << std::endl;
> + */
> + offset += p->length();
> + unaligned.push_back(*p);
> + _buffers.erase(p++);
> + } while (p != _buffers.end() &&
> + (!p->is_aligned(align) ||
> + !p->is_n_align_sized(align) ||
> + (offset % align)));
> + ptr nb(buffer::create_aligned(unaligned._len, align));
> + unaligned.rebuild(nb);
> + _buffers.insert(p, unaligned._buffers.front());
> + }
> +}
> +
> void buffer::list::rebuild_page_aligned()
> {
> std::list<ptr>::iterator p = _buffers.begin();
> diff --git a/src/include/buffer.h b/src/include/buffer.h
> index e5c1b50..2a32cf4 100644
> --- a/src/include/buffer.h
> +++ b/src/include/buffer.h
> @@ -124,6 +124,7 @@ private:
> */
> class raw;
> class raw_malloc;
> + class raw_aligned;
> class raw_static;
> class raw_mmap_pages;
> class raw_posix_aligned;
> @@ -144,6 +145,7 @@ public:
> static raw* create_malloc(unsigned len);
> static raw* claim_malloc(unsigned len, char *buf);
> static raw* create_static(unsigned len, char *buf);
> + static raw* create_aligned(unsigned len, unsigned align);
> static raw* create_page_aligned(unsigned len);
> static raw* create_zero_copy(unsigned len, int fd, int64_t *offset);
>
> @@ -177,7 +179,17 @@ public:
> bool at_buffer_head() const { return _off == 0; }
> bool at_buffer_tail() const;
>
> + bool is_aligned(unsigned align) const
> + {
> + assert((align >= sizeof(void *)) && (align & (align - 1)) == 0);
> + return ((intptr_t)c_str() & (align - 1)) == 0;
> + }
> bool is_page_aligned() const { return ((long)c_str() & ~CEPH_PAGE_MASK) == 0; }
> + bool is_n_align_sized(unsigned align) const
> + {
> + assert((align >= sizeof(void *)) && (align & (align - 1)) == 0);
> + return (length() % align) == 0;
> + }
> bool is_n_page_sized() const { return (length() & ~CEPH_PAGE_MASK) == 0; }
>
> // accessors
> @@ -344,7 +356,9 @@ public:
> bool contents_equal(buffer::list& other);
>
> bool can_zero_copy() const;
> + bool is_aligned(unsigned align) const;
> bool is_page_aligned() const;
> + bool is_n_align_sized(unsigned align) const;
> bool is_n_page_sized() const;
>
> bool is_zero() const;
> @@ -382,6 +396,7 @@ public:
> bool is_contiguous();
> void rebuild();
> void rebuild(ptr& nb);
> + void rebuild_aligned(unsigned align);
> void rebuild_page_aligned();
>
> // sort-of-like-assignment-op
>
--
Loïc Dachary, Artisan Logiciel Libre
Attachment:
signature.asc
Description: OpenPGP digital signature
