From: Loic Dachary Date: Thu, 6 Feb 2014 10:28:21 +0000 (+0100) Subject: erasure-code: move test files to a dedicated directory X-Git-Tag: v0.78~218^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=refs%2Fpull%2F1194%2Fhead;p=ceph.git erasure-code: move test files to a dedicated directory Signed-off-by: Loic Dachary --- diff --git a/src/test/Makefile.am b/src/test/Makefile.am index 91af9bc6fca0..bc6767ff626a 100644 --- a/src/test/Makefile.am +++ b/src/test/Makefile.am @@ -354,74 +354,6 @@ unittest_ceph_compatset_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) unittest_ceph_compatset_CXXFLAGS = $(UNITTEST_CXXFLAGS) check_PROGRAMS += unittest_ceph_compatset -libec_example_la_SOURCES = test/osd/ErasureCodePluginExample.cc -libec_example_la_CFLAGS = ${AM_CFLAGS} -libec_example_la_CXXFLAGS= ${AM_CXXFLAGS} -libec_example_la_LIBADD = $(LIBCRUSH) $(PTHREAD_LIBS) $(EXTRALIBS) -libec_example_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*' -erasure_codelib_LTLIBRARIES += libec_example.la - -libec_missing_entry_point_la_SOURCES = test/osd/ErasureCodePluginMissingEntryPoint.cc -libec_missing_entry_point_la_CFLAGS = ${AM_CFLAGS} -libec_missing_entry_point_la_CXXFLAGS= ${AM_CXXFLAGS} -libec_missing_entry_point_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) -libec_missing_entry_point_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*' -erasure_codelib_LTLIBRARIES += libec_missing_entry_point.la - -libec_hangs_la_SOURCES = test/osd/ErasureCodePluginHangs.cc -libec_hangs_la_CFLAGS = ${AM_CFLAGS} -libec_hangs_la_CXXFLAGS= ${AM_CXXFLAGS} -libec_hangs_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) -libec_hangs_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*' -erasure_codelib_LTLIBRARIES += libec_hangs.la - -libec_fail_to_initialize_la_SOURCES = test/osd/ErasureCodePluginFailToInitialize.cc -libec_fail_to_initialize_la_CFLAGS = ${AM_CFLAGS} -libec_fail_to_initialize_la_CXXFLAGS= ${AM_CXXFLAGS} -libec_fail_to_initialize_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) -libec_fail_to_initialize_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*' -erasure_codelib_LTLIBRARIES += libec_fail_to_initialize.la - -libec_fail_to_register_la_SOURCES = test/osd/ErasureCodePluginFailToRegister.cc -libec_fail_to_register_la_CFLAGS = ${AM_CFLAGS} -libec_fail_to_register_la_CXXFLAGS= ${AM_CXXFLAGS} -libec_fail_to_register_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) -libec_fail_to_register_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*' -erasure_codelib_LTLIBRARIES += libec_fail_to_register.la - -unittest_erasure_code_plugin_SOURCES = test/osd/TestErasureCodePlugin.cc -unittest_erasure_code_plugin_CXXFLAGS = $(UNITTEST_CXXFLAGS) -unittest_erasure_code_plugin_LDADD = $(LIBOSD) $(LIBCOMMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL) -if LINUX -unittest_erasure_code_plugin_LDADD += -ldl -endif -check_PROGRAMS += unittest_erasure_code_plugin - -unittest_erasure_code_jerasure_SOURCES = \ - test/osd/TestErasureCodeJerasure.cc \ - $(libec_jerasure_la_SOURCES) -unittest_erasure_code_jerasure_CXXFLAGS = $(UNITTEST_CXXFLAGS) -unittest_erasure_code_jerasure_LDADD = $(LIBOSD) $(LIBCOMMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL) -if LINUX -unittest_erasure_code_jerasure_LDADD += -ldl -endif -check_PROGRAMS += unittest_erasure_code_jerasure - -unittest_erasure_code_plugin_jerasure_SOURCES = \ - test/osd/TestErasureCodePluginJerasure.cc -unittest_erasure_code_plugin_jerasure_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} -unittest_erasure_code_plugin_jerasure_LDADD = $(LIBOSD) $(LIBCOMMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL) -if LINUX -unittest_erasure_code_plugin_jerasure_LDADD += -ldl -endif -check_PROGRAMS += unittest_erasure_code_plugin_jerasure - -unittest_erasure_code_example_SOURCES = test/osd/TestErasureCodeExample.cc -noinst_HEADERS += test/osd/ErasureCodeExample.h -unittest_erasure_code_example_CXXFLAGS = $(UNITTEST_CXXFLAGS) -unittest_erasure_code_example_LDADD = $(LIBOSD) $(LIBCOMMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL) -check_PROGRAMS += unittest_erasure_code_example - unittest_osd_types_SOURCES = test/test_osd_types.cc unittest_osd_types_CXXFLAGS = $(UNITTEST_CXXFLAGS) unittest_osd_types_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL) diff --git a/src/test/erasure-code/ErasureCodeExample.h b/src/test/erasure-code/ErasureCodeExample.h new file mode 100644 index 000000000000..2b77d51e0944 --- /dev/null +++ b/src/test/erasure-code/ErasureCodeExample.h @@ -0,0 +1,184 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2013 Cloudwatt + * + * Author: Loic Dachary + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + */ + +#ifndef CEPH_ERASURE_CODE_EXAMPLE_H +#define CEPH_ERASURE_CODE_EXAMPLE_H + +#include +#include +#include +#include + +#include "crush/CrushWrapper.h" +#include "osd/osd_types.h" +#include "erasure-code/ErasureCodeInterface.h" + +#define FIRST_DATA_CHUNK 0 +#define SECOND_DATA_CHUNK 1 +#define DATA_CHUNKS 2u + +#define CODING_CHUNK 2 +#define CODING_CHUNKS 1u + +#define MINIMUM_TO_RECOVER 2u + +class ErasureCodeExample : public ErasureCodeInterface { +public: + virtual ~ErasureCodeExample() {} + + virtual int create_ruleset(const string &name, + CrushWrapper &crush, + ostream *ss) const { + return crush.add_simple_ruleset(name, "default", "host", + "indep", pg_pool_t::TYPE_ERASURE, ss); + } + + virtual int minimum_to_decode(const set &want_to_read, + const set &available_chunks, + set *minimum) { + if (includes(available_chunks.begin(), available_chunks.end(), + want_to_read.begin(), want_to_read.end())) { + *minimum = want_to_read; + return 0; + } else if (available_chunks.size() >= MINIMUM_TO_RECOVER) { + *minimum = available_chunks; + return 0; + } else { + return -EIO; + } + } + + virtual int minimum_to_decode_with_cost(const set &want_to_read, + const map &available, + set *minimum) { + // + // If one chunk is more expensive to fetch than the others, + // recover it instead. For instance, if the cost reflects the + // time it takes for a chunk to be retrieved from a remote + // OSD and if CPU is cheap, it could make sense to recover + // instead of fetching the chunk. + // + map c2c(available); + if (c2c.size() > DATA_CHUNKS) { + if (c2c[FIRST_DATA_CHUNK] > c2c[SECOND_DATA_CHUNK] && + c2c[FIRST_DATA_CHUNK] > c2c[CODING_CHUNK]) + c2c.erase(FIRST_DATA_CHUNK); + else if(c2c[SECOND_DATA_CHUNK] > c2c[FIRST_DATA_CHUNK] && + c2c[SECOND_DATA_CHUNK] > c2c[CODING_CHUNK]) + c2c.erase(SECOND_DATA_CHUNK); + else if(c2c[CODING_CHUNK] > c2c[FIRST_DATA_CHUNK] && + c2c[CODING_CHUNK] > c2c[SECOND_DATA_CHUNK]) + c2c.erase(CODING_CHUNK); + } + set available_chunks; + for (map::const_iterator i = c2c.begin(); + i != c2c.end(); + ++i) + available_chunks.insert(i->first); + return minimum_to_decode(want_to_read, available_chunks, minimum); + } + + virtual unsigned int get_chunk_count() const { + return DATA_CHUNKS + CODING_CHUNKS; + } + + virtual unsigned int get_data_chunk_count() const { + return DATA_CHUNKS; + } + + virtual unsigned int get_chunk_size(unsigned int object_size) const { + return ( object_size / DATA_CHUNKS ) + 1; + } + + virtual int encode(const set &want_to_encode, + const bufferlist &in, + map *encoded) { + // + // make sure all data chunks have the same length, allocating + // padding if necessary. + // + unsigned int chunk_length = get_chunk_size(in.length()); + bufferlist out(in); + unsigned int width = get_chunk_count() * get_chunk_size(in.length()); + bufferptr pad(width - in.length()); + pad.zero(0, get_data_chunk_count()); + out.push_back(pad); + // + // compute the coding chunk with first chunk ^ second chunk + // + char *p = out.c_str(); + for (unsigned i = 0; i < chunk_length; i++) + p[i + CODING_CHUNK * chunk_length] = + p[i + FIRST_DATA_CHUNK * chunk_length] ^ + p[i + SECOND_DATA_CHUNK * chunk_length]; + // + // populate the bufferlist with bufferptr pointing + // to chunk boundaries + // + const bufferptr ptr = out.buffers().front(); + for (set::iterator j = want_to_encode.begin(); + j != want_to_encode.end(); + ++j) { + bufferptr chunk(ptr, (*j) * chunk_length, chunk_length); + (*encoded)[*j].push_front(chunk); + } + return 0; + } + + virtual int decode(const set &want_to_read, + const map &chunks, + map *decoded) { + // + // All chunks have the same size + // + unsigned chunk_length = (*chunks.begin()).second.length(); + for (set::iterator i = want_to_read.begin(); + i != want_to_read.end(); + ++i) { + if (chunks.find(*i) != chunks.end()) { + // + // If the chunk is available, just copy the bufferptr pointer + // to the decoded argument. + // + (*decoded)[*i] = chunks.find(*i)->second; + } else if(chunks.size() != 2) { + // + // If a chunk is missing and there are not enough chunks + // to recover, abort. + // + return -ERANGE; + } else { + // + // No matter what the missing chunk is, XOR of the other + // two recovers it. + // + map::const_iterator k = chunks.begin(); + const char *a = k->second.buffers().front().c_str(); + ++k; + const char *b = k->second.buffers().front().c_str(); + bufferptr chunk(chunk_length); + char *c = chunk.c_str(); + for (unsigned j = 0; j < chunk_length; j++) { + c[j] = a[j] ^ b[j]; + } + (*decoded)[*i].push_front(chunk); + } + } + return 0; + } +}; + +#endif diff --git a/src/test/erasure-code/ErasureCodePluginExample.cc b/src/test/erasure-code/ErasureCodePluginExample.cc new file mode 100644 index 000000000000..c1b5b3c6f0c4 --- /dev/null +++ b/src/test/erasure-code/ErasureCodePluginExample.cc @@ -0,0 +1,36 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2013 Cloudwatt + * + * Author: Loic Dachary + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + */ + +#include + +#include "erasure-code/ErasureCodePlugin.h" +#include "ErasureCodeExample.h" + +class ErasureCodePluginExample : public ErasureCodePlugin { +public: + virtual int factory(const map ¶meters, + ErasureCodeInterfaceRef *erasure_code) + { + *erasure_code = ErasureCodeInterfaceRef(new ErasureCodeExample()); + return 0; + } +}; + +int __erasure_code_init(char *plugin_name) +{ + ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance(); + return instance.add(plugin_name, new ErasureCodePluginExample()); +} diff --git a/src/test/erasure-code/ErasureCodePluginFailToInitialize.cc b/src/test/erasure-code/ErasureCodePluginFailToInitialize.cc new file mode 100644 index 000000000000..d3043a43dc6a --- /dev/null +++ b/src/test/erasure-code/ErasureCodePluginFailToInitialize.cc @@ -0,0 +1,23 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2013 Cloudwatt + * + * Author: Loic Dachary + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + */ + +#include +#include "erasure-code/ErasureCodePlugin.h" + +int __erasure_code_init(char *plugin_name) +{ + return -ESRCH; +} diff --git a/src/test/erasure-code/ErasureCodePluginFailToRegister.cc b/src/test/erasure-code/ErasureCodePluginFailToRegister.cc new file mode 100644 index 000000000000..c26ac5b66710 --- /dev/null +++ b/src/test/erasure-code/ErasureCodePluginFailToRegister.cc @@ -0,0 +1,22 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2013 Cloudwatt + * + * Author: Loic Dachary + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + */ + +#include "erasure-code/ErasureCodePlugin.h" + +int __erasure_code_init(char *plugin_name) +{ + return 0; +} diff --git a/src/test/erasure-code/ErasureCodePluginHangs.cc b/src/test/erasure-code/ErasureCodePluginHangs.cc new file mode 100644 index 000000000000..01c2fa866d6a --- /dev/null +++ b/src/test/erasure-code/ErasureCodePluginHangs.cc @@ -0,0 +1,24 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2013 Cloudwatt + * + * Author: Loic Dachary + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + */ + +#include +#include "erasure-code/ErasureCodePlugin.h" + +int __erasure_code_init(char *plugin_name) +{ + sleep(1000); + return 0; +} diff --git a/src/test/erasure-code/ErasureCodePluginMissingEntryPoint.cc b/src/test/erasure-code/ErasureCodePluginMissingEntryPoint.cc new file mode 100644 index 000000000000..4d8f1bafd1c9 --- /dev/null +++ b/src/test/erasure-code/ErasureCodePluginMissingEntryPoint.cc @@ -0,0 +1,4 @@ +// missing int __erasure_code_init(char *plugin_name) {} + +// avoid warnings about library containing no symbols +int __this_is_an_used_variable_to_avoid_warnings; diff --git a/src/test/erasure-code/Makefile.am b/src/test/erasure-code/Makefile.am index b467eea5a2fd..dc94168262b6 100644 --- a/src/test/erasure-code/Makefile.am +++ b/src/test/erasure-code/Makefile.am @@ -14,5 +14,73 @@ ceph_erasure_code_LDADD += -ldl endif bin_DEBUGPROGRAMS += ceph_erasure_code +libec_example_la_SOURCES = test/erasure-code/ErasureCodePluginExample.cc +libec_example_la_CFLAGS = ${AM_CFLAGS} +libec_example_la_CXXFLAGS= ${AM_CXXFLAGS} +libec_example_la_LIBADD = $(LIBCRUSH) $(PTHREAD_LIBS) $(EXTRALIBS) +libec_example_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*' +erasure_codelib_LTLIBRARIES += libec_example.la + +libec_missing_entry_point_la_SOURCES = test/erasure-code/ErasureCodePluginMissingEntryPoint.cc +libec_missing_entry_point_la_CFLAGS = ${AM_CFLAGS} +libec_missing_entry_point_la_CXXFLAGS= ${AM_CXXFLAGS} +libec_missing_entry_point_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) +libec_missing_entry_point_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*' +erasure_codelib_LTLIBRARIES += libec_missing_entry_point.la + +libec_hangs_la_SOURCES = test/erasure-code/ErasureCodePluginHangs.cc +libec_hangs_la_CFLAGS = ${AM_CFLAGS} +libec_hangs_la_CXXFLAGS= ${AM_CXXFLAGS} +libec_hangs_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) +libec_hangs_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*' +erasure_codelib_LTLIBRARIES += libec_hangs.la + +libec_fail_to_initialize_la_SOURCES = test/erasure-code/ErasureCodePluginFailToInitialize.cc +libec_fail_to_initialize_la_CFLAGS = ${AM_CFLAGS} +libec_fail_to_initialize_la_CXXFLAGS= ${AM_CXXFLAGS} +libec_fail_to_initialize_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) +libec_fail_to_initialize_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*' +erasure_codelib_LTLIBRARIES += libec_fail_to_initialize.la + +libec_fail_to_register_la_SOURCES = test/erasure-code/ErasureCodePluginFailToRegister.cc +libec_fail_to_register_la_CFLAGS = ${AM_CFLAGS} +libec_fail_to_register_la_CXXFLAGS= ${AM_CXXFLAGS} +libec_fail_to_register_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) +libec_fail_to_register_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*' +erasure_codelib_LTLIBRARIES += libec_fail_to_register.la + +unittest_erasure_code_plugin_SOURCES = test/erasure-code/TestErasureCodePlugin.cc +unittest_erasure_code_plugin_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_erasure_code_plugin_LDADD = $(LIBOSD) $(LIBCOMMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL) +if LINUX +unittest_erasure_code_plugin_LDADD += -ldl +endif +check_PROGRAMS += unittest_erasure_code_plugin + +unittest_erasure_code_jerasure_SOURCES = \ + test/erasure-code/TestErasureCodeJerasure.cc \ + $(libec_jerasure_la_SOURCES) +unittest_erasure_code_jerasure_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_erasure_code_jerasure_LDADD = $(LIBOSD) $(LIBCOMMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL) +if LINUX +unittest_erasure_code_jerasure_LDADD += -ldl +endif +check_PROGRAMS += unittest_erasure_code_jerasure + +unittest_erasure_code_plugin_jerasure_SOURCES = \ + test/erasure-code/TestErasureCodePluginJerasure.cc +unittest_erasure_code_plugin_jerasure_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS} +unittest_erasure_code_plugin_jerasure_LDADD = $(LIBOSD) $(LIBCOMMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL) +if LINUX +unittest_erasure_code_plugin_jerasure_LDADD += -ldl +endif +check_PROGRAMS += unittest_erasure_code_plugin_jerasure + +unittest_erasure_code_example_SOURCES = test/erasure-code/TestErasureCodeExample.cc +noinst_HEADERS += test/erasure-code/ErasureCodeExample.h +unittest_erasure_code_example_CXXFLAGS = $(UNITTEST_CXXFLAGS) +unittest_erasure_code_example_LDADD = $(LIBOSD) $(LIBCOMMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL) +check_PROGRAMS += unittest_erasure_code_example + noinst_HEADERS += \ test/erasure-code/ceph_erasure_code_benchmark.h diff --git a/src/test/erasure-code/TestErasureCodeExample.cc b/src/test/erasure-code/TestErasureCodeExample.cc new file mode 100644 index 000000000000..4df0762419b5 --- /dev/null +++ b/src/test/erasure-code/TestErasureCodeExample.cc @@ -0,0 +1,258 @@ +// -*- mode:C; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2013 Cloudwatt + * + * Author: Loic Dachary + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + */ + +#include "include/stringify.h" +#include "global/global_init.h" +#include "ErasureCodeExample.h" +#include "common/ceph_argparse.h" +#include "global/global_context.h" +#include "gtest/gtest.h" + +TEST(ErasureCodeExample, chunk_size) +{ + ErasureCodeExample example; + EXPECT_EQ(3u, example.get_chunk_count()); + EXPECT_EQ(11u, example.get_chunk_size(20)); +} + +TEST(ErasureCodeExample, minimum_to_decode) +{ + ErasureCodeExample example; + set available_chunks; + set want_to_read; + want_to_read.insert(1); + { + set minimum; + EXPECT_EQ(-EIO, example.minimum_to_decode(want_to_read, + available_chunks, + &minimum)); + } + available_chunks.insert(0); + available_chunks.insert(2); + { + set minimum; + EXPECT_EQ(0, example.minimum_to_decode(want_to_read, + available_chunks, + &minimum)); + EXPECT_EQ(available_chunks, minimum); + EXPECT_EQ(2u, minimum.size()); + EXPECT_EQ(1u, minimum.count(0)); + EXPECT_EQ(1u, minimum.count(2)); + } + { + set minimum; + available_chunks.insert(1); + EXPECT_EQ(0, example.minimum_to_decode(want_to_read, + available_chunks, + &minimum)); + EXPECT_EQ(1u, minimum.size()); + EXPECT_EQ(1u, minimum.count(1)); + } +} + +TEST(ErasureCodeExample, minimum_to_decode_with_cost) +{ + ErasureCodeExample example; + map available; + set want_to_read; + want_to_read.insert(1); + { + set minimum; + EXPECT_EQ(-EIO, example.minimum_to_decode_with_cost(want_to_read, + available, + &minimum)); + } + available[0] = 1; + available[2] = 1; + { + set minimum; + EXPECT_EQ(0, example.minimum_to_decode_with_cost(want_to_read, + available, + &minimum)); + EXPECT_EQ(2u, minimum.size()); + EXPECT_EQ(1u, minimum.count(0)); + EXPECT_EQ(1u, minimum.count(2)); + } + { + set minimum; + available[1] = 1; + EXPECT_EQ(0, example.minimum_to_decode_with_cost(want_to_read, + available, + &minimum)); + EXPECT_EQ(1u, minimum.size()); + EXPECT_EQ(1u, minimum.count(1)); + } + { + set minimum; + available[1] = 2; + EXPECT_EQ(0, example.minimum_to_decode_with_cost(want_to_read, + available, + &minimum)); + EXPECT_EQ(2u, minimum.size()); + EXPECT_EQ(1u, minimum.count(0)); + EXPECT_EQ(1u, minimum.count(2)); + } +} + +TEST(ErasureCodeExample, encode_decode) +{ + ErasureCodeExample example; + + bufferlist in; + in.append("ABCDE"); + set want_to_encode; + for(unsigned int i = 0; i < example.get_chunk_count(); i++) + want_to_encode.insert(i); + map encoded; + EXPECT_EQ(0, example.encode(want_to_encode, in, &encoded)); + EXPECT_EQ(example.get_chunk_count(), encoded.size()); + EXPECT_EQ(example.get_chunk_size(in.length()), encoded[0].length()); + EXPECT_EQ('A', encoded[0][0]); + EXPECT_EQ('B', encoded[0][1]); + EXPECT_EQ('C', encoded[0][2]); + EXPECT_EQ('D', encoded[1][0]); + EXPECT_EQ('E', encoded[1][1]); + EXPECT_EQ('A'^'D', encoded[2][0]); + EXPECT_EQ('B'^'E', encoded[2][1]); + EXPECT_EQ('C'^0, encoded[2][2]); + + // all chunks are available + { + int want_to_decode[] = { 0, 1 }; + map decoded; + EXPECT_EQ(0, example.decode(set(want_to_decode, want_to_decode+2), + encoded, + &decoded)); + EXPECT_EQ(2u, decoded.size()); + EXPECT_EQ(3u, decoded[0].length()); + EXPECT_EQ('A', decoded[0][0]); + EXPECT_EQ('B', decoded[0][1]); + EXPECT_EQ('C', decoded[0][2]); + EXPECT_EQ('D', decoded[1][0]); + EXPECT_EQ('E', decoded[1][1]); + } + + // one chunk is missing + { + map degraded = encoded; + degraded.erase(0); + EXPECT_EQ(2u, degraded.size()); + int want_to_decode[] = { 0, 1 }; + map decoded; + EXPECT_EQ(0, example.decode(set(want_to_decode, want_to_decode+2), + degraded, + &decoded)); + EXPECT_EQ(2u, decoded.size()); + EXPECT_EQ(3u, decoded[0].length()); + EXPECT_EQ('A', decoded[0][0]); + EXPECT_EQ('B', decoded[0][1]); + EXPECT_EQ('C', decoded[0][2]); + EXPECT_EQ('D', decoded[1][0]); + EXPECT_EQ('E', decoded[1][1]); + } +} + +TEST(ErasureCodeExample, decode) +{ + ErasureCodeExample example; + +#define LARGE_ENOUGH 2048 + bufferptr in_ptr(buffer::create_page_aligned(LARGE_ENOUGH)); + in_ptr.zero(); + in_ptr.set_length(0); + const char *payload = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + in_ptr.append(payload, strlen(payload)); + bufferlist in; + in.push_front(in_ptr); + int want_to_encode[] = { 0, 1, 2 }; + map encoded; + EXPECT_EQ(0, example.encode(set(want_to_encode, want_to_encode+3), + in, + &encoded)); + EXPECT_EQ(3u, encoded.size()); + + // successfull decode + bufferlist out; + EXPECT_EQ(0, example.decode_concat(encoded, &out)); + bufferlist usable; + usable.substr_of(out, 0, in.length()); + EXPECT_TRUE(usable == in); + + // cannot recover + map degraded; + degraded[0] = encoded[0]; + EXPECT_EQ(-ERANGE, example.decode_concat(degraded, &out)); +} + +TEST(ErasureCodeExample, create_ruleset) +{ + CrushWrapper *c = new CrushWrapper; + c->create(); + c->set_type_name(2, "root"); + c->set_type_name(1, "host"); + c->set_type_name(0, "osd"); + + int rootno; + c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1, + 5, 0, NULL, NULL, &rootno); + c->set_item_name(rootno, "default"); + + map loc; + loc["root"] = "default"; + + int num_host = 2; + int num_osd = 5; + int osd = 0; + for (int h=0; hinsert_item(g_ceph_context, osd, 1.0, string("osd.") + stringify(osd), loc); + } + } + + stringstream ss; + ErasureCodeExample example; + EXPECT_EQ(0, example.create_ruleset("myrule", *c, &ss)); +} + +int main(int argc, char **argv) { + vector args; + argv_to_vec(argc, (const char **)argv, args); + + global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0); + common_init_finish(g_ceph_context); + + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +/* + * Local Variables: + * compile-command: "cd ../.. ; + * make -j4 && + * make unittest_erasure_code_example && + * valgrind --leak-check=full --tool=memcheck \ + * ./unittest_erasure_code_example --gtest_filter=*.* \ + * --log-to-stderr=true --debug-osd=20 + * " + * End: + */ + diff --git a/src/test/erasure-code/TestErasureCodeJerasure.cc b/src/test/erasure-code/TestErasureCodeJerasure.cc new file mode 100644 index 000000000000..a3d603bea540 --- /dev/null +++ b/src/test/erasure-code/TestErasureCodeJerasure.cc @@ -0,0 +1,362 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2013 Cloudwatt + * + * Author: Loic Dachary + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + */ + +#include + +#include "crush/CrushWrapper.h" +#include "include/stringify.h" +#include "global/global_init.h" +#include "erasure-code/jerasure/ErasureCodeJerasure.h" +#include "common/ceph_argparse.h" +#include "global/global_context.h" +#include "gtest/gtest.h" + +template +class ErasureCodeTest : public ::testing::Test { + public: +}; + +typedef ::testing::Types< + ErasureCodeJerasureReedSolomonVandermonde, + ErasureCodeJerasureReedSolomonRAID6, + ErasureCodeJerasureCauchyOrig, + ErasureCodeJerasureCauchyGood, + ErasureCodeJerasureLiberation, + ErasureCodeJerasureBlaumRoth, + ErasureCodeJerasureLiber8tion +> JerasureTypes; +TYPED_TEST_CASE(ErasureCodeTest, JerasureTypes); + +TYPED_TEST(ErasureCodeTest, encode_decode) +{ + TypeParam jerasure; + map parameters; + parameters["erasure-code-k"] = "2"; + parameters["erasure-code-m"] = "2"; + parameters["erasure-code-w"] = "7"; + parameters["erasure-code-packetsize"] = "8"; + jerasure.init(parameters); + +#define LARGE_ENOUGH 2048 + bufferptr in_ptr(buffer::create_page_aligned(LARGE_ENOUGH)); + in_ptr.zero(); + in_ptr.set_length(0); + const char *payload = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + in_ptr.append(payload, strlen(payload)); + bufferlist in; + in.push_front(in_ptr); + int want_to_encode[] = { 0, 1, 2, 3 }; + map encoded; + EXPECT_EQ(0, jerasure.encode(set(want_to_encode, want_to_encode+4), + in, + &encoded)); + EXPECT_EQ(4u, encoded.size()); + unsigned length = encoded[0].length(); + EXPECT_EQ(0, strncmp(encoded[0].c_str(), in.c_str(), length)); + EXPECT_EQ(0, strncmp(encoded[1].c_str(), in.c_str() + length, + in.length() - length)); + + + // all chunks are available + { + int want_to_decode[] = { 0, 1 }; + map decoded; + EXPECT_EQ(0, jerasure.decode(set(want_to_decode, want_to_decode+2), + encoded, + &decoded)); + // always decode all, regardless of want_to_decode + EXPECT_EQ(4u, decoded.size()); + EXPECT_EQ(length, decoded[0].length()); + EXPECT_EQ(0, strncmp(decoded[0].c_str(), in.c_str(), length)); + EXPECT_EQ(0, strncmp(decoded[1].c_str(), in.c_str() + length, + in.length() - length)); + } + + // two chunks are missing + { + map degraded = encoded; + degraded.erase(0); + degraded.erase(1); + EXPECT_EQ(2u, degraded.size()); + int want_to_decode[] = { 0, 1 }; + map decoded; + EXPECT_EQ(0, jerasure.decode(set(want_to_decode, want_to_decode+2), + degraded, + &decoded)); + // always decode all, regardless of want_to_decode + EXPECT_EQ(4u, decoded.size()); + EXPECT_EQ(length, decoded[0].length()); + EXPECT_EQ(0, strncmp(decoded[0].c_str(), in.c_str(), length)); + EXPECT_EQ(0, strncmp(decoded[1].c_str(), in.c_str() + length, + in.length() - length)); + } +} + +TYPED_TEST(ErasureCodeTest, minimum_to_decode) +{ + TypeParam jerasure; + map parameters; + parameters["erasure-code-k"] = "2"; + parameters["erasure-code-m"] = "2"; + parameters["erasure-code-w"] = "7"; + parameters["erasure-code-packetsize"] = "8"; + jerasure.init(parameters); + + // + // If trying to read nothing, the minimum is empty. + // + { + set want_to_read; + set available_chunks; + set minimum; + + EXPECT_EQ(0, jerasure.minimum_to_decode(want_to_read, + available_chunks, + &minimum)); + EXPECT_TRUE(minimum.empty()); + } + // + // There is no way to read a chunk if none are available. + // + { + set want_to_read; + set available_chunks; + set minimum; + + want_to_read.insert(0); + + EXPECT_EQ(-EIO, jerasure.minimum_to_decode(want_to_read, + available_chunks, + &minimum)); + } + // + // Reading a subset of the available chunks is always possible. + // + { + set want_to_read; + set available_chunks; + set minimum; + + want_to_read.insert(0); + available_chunks.insert(0); + + EXPECT_EQ(0, jerasure.minimum_to_decode(want_to_read, + available_chunks, + &minimum)); + EXPECT_EQ(want_to_read, minimum); + } + // + // There is no way to read a missing chunk if there is less than k + // chunks available. + // + { + set want_to_read; + set available_chunks; + set minimum; + + want_to_read.insert(0); + want_to_read.insert(1); + available_chunks.insert(0); + + EXPECT_EQ(-EIO, jerasure.minimum_to_decode(want_to_read, + available_chunks, + &minimum)); + } + // + // When chunks are not available, the minimum can be made of any + // chunks. For instance, to read 1 and 3 below the minimum could be + // 2 and 3 which may seem better because it contains one of the + // chunks to be read. But it won't be more efficient than retrieving + // 0 and 2 instead because, in both cases, the decode function will + // need to run the same recovery operation and use the same amount + // of CPU and memory. + // + { + set want_to_read; + set available_chunks; + set minimum; + + want_to_read.insert(1); + want_to_read.insert(3); + available_chunks.insert(0); + available_chunks.insert(2); + available_chunks.insert(3); + + EXPECT_EQ(0, jerasure.minimum_to_decode(want_to_read, + available_chunks, + &minimum)); + EXPECT_EQ(2u, minimum.size()); + EXPECT_EQ(0u, minimum.count(3)); + } +} + +TEST(ErasureCodeTest, encode) +{ + ErasureCodeJerasureReedSolomonVandermonde jerasure; + map parameters; + parameters["erasure-code-k"] = "2"; + parameters["erasure-code-m"] = "2"; + parameters["erasure-code-w"] = "8"; + jerasure.init(parameters); + + unsigned alignment = jerasure.get_alignment(); + { + // + // When the input bufferlist needs to be padded because + // it is not properly aligned, it is padded with zeros. + // + bufferlist in; + map encoded; + int want_to_encode[] = { 0, 1, 2, 3 }; + int trail_length = 10; + in.append(string(alignment + trail_length, 'X')); + EXPECT_EQ(0, jerasure.encode(set(want_to_encode, want_to_encode+4), + in, + &encoded)); + EXPECT_EQ(4u, encoded.size()); + for(int i = 0; i < 4; i++) + EXPECT_EQ(alignment, encoded[i].length()); + char *last_chunk = encoded[1].c_str(); + EXPECT_EQ('X', last_chunk[0]); + EXPECT_EQ('\0', last_chunk[trail_length]); + } + + { + // + // When only the first chunk is required, the encoded map only + // contains the first chunk. Although the jerasure encode + // internally allocated a buffer because of padding requirements + // and also computes the coding chunks, they are released before + // the return of the method, as shown when running the tests thru + // valgrind (there is no leak). + // + bufferlist in; + map encoded; + set want_to_encode; + want_to_encode.insert(0); + int trail_length = 10; + in.append(string(alignment + trail_length, 'X')); + EXPECT_EQ(0, jerasure.encode(want_to_encode, in, &encoded)); + EXPECT_EQ(1u, encoded.size()); + EXPECT_EQ(alignment, encoded[0].length()); + } +} + +TEST(ErasureCodeTest, create_ruleset) +{ + CrushWrapper *c = new CrushWrapper; + c->create(); + int root_type = 2; + c->set_type_name(root_type, "root"); + int host_type = 1; + c->set_type_name(host_type, "host"); + int osd_type = 0; + c->set_type_name(osd_type, "osd"); + + int rootno; + c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1, + root_type, 0, NULL, NULL, &rootno); + c->set_item_name(rootno, "default"); + + map loc; + loc["root"] = "default"; + + int num_host = 4; + int num_osd = 5; + int osd = 0; + for (int h=0; hinsert_item(g_ceph_context, osd, 1.0, string("osd.") + stringify(osd), loc); + } + } + + { + stringstream ss; + ErasureCodeJerasureReedSolomonVandermonde jerasure; + map parameters; + parameters["erasure-code-k"] = "2"; + parameters["erasure-code-m"] = "2"; + parameters["erasure-code-w"] = "8"; + jerasure.init(parameters); + int ruleset = jerasure.create_ruleset("myrule", *c, &ss); + EXPECT_EQ(0, ruleset); + EXPECT_EQ(-EEXIST, jerasure.create_ruleset("myrule", *c, &ss)); + // + // the minimum that is expected from the created ruleset is to + // successfully map get_chunk_count() devices from the crushmap, + // at least once. + // + vector<__u32> weight(c->get_max_devices(), 0x10000); + vector out; + int x = 0; + c->do_rule(ruleset, x, out, jerasure.get_chunk_count(), weight); + ASSERT_EQ(out.size(), jerasure.get_chunk_count()); + for (unsigned i=0; i parameters; + parameters["erasure-code-k"] = "2"; + parameters["erasure-code-m"] = "2"; + parameters["erasure-code-w"] = "8"; + parameters["erasure-code-ruleset-root"] = "BAD"; + jerasure.init(parameters); + EXPECT_EQ(-ENOENT, jerasure.create_ruleset("otherrule", *c, &ss)); + EXPECT_EQ("root item BAD does not exist", ss.str()); + } + { + stringstream ss; + ErasureCodeJerasureReedSolomonVandermonde jerasure; + map parameters; + parameters["erasure-code-k"] = "2"; + parameters["erasure-code-m"] = "2"; + parameters["erasure-code-w"] = "8"; + parameters["erasure-code-ruleset-failure-domain"] = "WORSE"; + jerasure.init(parameters); + EXPECT_EQ(-EINVAL, jerasure.create_ruleset("otherrule", *c, &ss)); + EXPECT_EQ("unknown type WORSE", ss.str()); + } +} + +int main(int argc, char **argv) +{ + vector args; + argv_to_vec(argc, (const char **)argv, args); + + global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0); + common_init_finish(g_ceph_context); + + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +/* + * Local Variables: + * compile-command: "cd ../.. ; make -j4 && + * make unittest_erasure_code_jerasure && + * valgrind --tool=memcheck --leak-check=full \ + * ./unittest_erasure_code_jerasure \ + * --gtest_filter=*.* --log-to-stderr=true --debug-osd=20" + * End: + */ diff --git a/src/test/erasure-code/TestErasureCodePlugin.cc b/src/test/erasure-code/TestErasureCodePlugin.cc new file mode 100644 index 000000000000..f97b140d5d25 --- /dev/null +++ b/src/test/erasure-code/TestErasureCodePlugin.cc @@ -0,0 +1,108 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2013 Cloudwatt + * + * Author: Loic Dachary + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + */ + +#include +#include +#include "common/Thread.h" +#include "global/global_init.h" +#include "erasure-code/ErasureCodePlugin.h" +#include "common/ceph_argparse.h" +#include "global/global_context.h" +#include "gtest/gtest.h" + +class ErasureCodePluginRegistryTest : public ::testing::Test { +protected: + + class Thread_factory : public Thread { + public: + virtual void *entry() { + map parameters; + parameters["erasure-code-directory"] = ".libs"; + ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance(); + ErasureCodeInterfaceRef erasure_code; + instance.factory("hangs", parameters, &erasure_code); + return NULL; + } + }; + +}; + +TEST_F(ErasureCodePluginRegistryTest, factory_mutex) { + ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance(); + + EXPECT_TRUE(instance.lock.TryLock()); + instance.lock.Unlock(); + + // + // Test that the loading of a plugin is protected by a mutex. + // + useconds_t delay = 0; + const useconds_t DELAY_MAX = 20 * 1000 * 1000; + Thread_factory sleep_forever; + sleep_forever.create(); + do { + cout << "Trying (1) with delay " << delay << "us\n"; + if (delay > 0) + usleep(delay); + if (!instance.loading) + delay = ( delay + 1 ) * 2; + } while(!instance.loading && delay < DELAY_MAX); + ASSERT_TRUE(delay < DELAY_MAX); + + EXPECT_FALSE(instance.lock.TryLock()); + + EXPECT_EQ(0, pthread_cancel(sleep_forever.get_thread_id())); + EXPECT_EQ(0, sleep_forever.join()); +} + +TEST_F(ErasureCodePluginRegistryTest, all) +{ + map parameters; + parameters["erasure-code-directory"] = ".libs"; + ErasureCodeInterfaceRef erasure_code; + ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance(); + EXPECT_FALSE(erasure_code); + EXPECT_EQ(-EIO, instance.factory("invalid", parameters, &erasure_code)); + EXPECT_FALSE(erasure_code); + EXPECT_EQ(-ENOENT, instance.factory("missing_entry_point", parameters, + &erasure_code)); + EXPECT_FALSE(erasure_code); + EXPECT_EQ(-ESRCH, instance.factory("fail_to_initialize", parameters, + &erasure_code)); + EXPECT_FALSE(erasure_code); + EXPECT_EQ(-EBADF, instance.factory("fail_to_register", parameters, + &erasure_code)); + EXPECT_FALSE(erasure_code); + EXPECT_EQ(0, instance.factory("example", parameters, &erasure_code)); + EXPECT_TRUE(erasure_code); + ErasureCodePlugin *plugin = 0; + EXPECT_EQ(-EEXIST, instance.load("example", parameters, &plugin)); +} + +int main(int argc, char **argv) { + vector args; + argv_to_vec(argc, (const char **)argv, args); + + global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0); + common_init_finish(g_ceph_context); + + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +// Local Variables: +// compile-command: "cd ../.. ; make -j4 && make unittest_erasure_code_plugin && valgrind --leak-check=full --tool=memcheck ./unittest_erasure_code_plugin --gtest_filter=*.* --log-to-stderr=true --debug-osd=20" +// End: diff --git a/src/test/erasure-code/TestErasureCodePluginJerasure.cc b/src/test/erasure-code/TestErasureCodePluginJerasure.cc new file mode 100644 index 000000000000..0013ce8d492b --- /dev/null +++ b/src/test/erasure-code/TestErasureCodePluginJerasure.cc @@ -0,0 +1,74 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2013 Cloudwatt + * + * Author: Loic Dachary + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + */ + +#include +#include "global/global_init.h" +#include "erasure-code/ErasureCodePlugin.h" +#include "common/ceph_argparse.h" +#include "global/global_context.h" +#include "gtest/gtest.h" + +TEST(ErasureCodePlugin, factory) +{ + ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance(); + map parameters; + parameters["erasure-code-directory"] = ".libs"; + { + ErasureCodeInterfaceRef erasure_code; + EXPECT_FALSE(erasure_code); + EXPECT_EQ(-ENOENT, instance.factory("jerasure", parameters, &erasure_code)); + EXPECT_FALSE(erasure_code); + } + const char *techniques[] = { + "reed_sol_van", + "reed_sol_r6_op", + "cauchy_orig", + "cauchy_good", + "liberation", + "blaum_roth", + "liber8tion", + 0 + }; + for(const char **technique = techniques; *technique; technique++) { + ErasureCodeInterfaceRef erasure_code; + parameters["erasure-code-technique"] = *technique; + EXPECT_FALSE(erasure_code); + EXPECT_EQ(0, instance.factory("jerasure", parameters, &erasure_code)); + EXPECT_TRUE(erasure_code); + } +} + +int main(int argc, char **argv) +{ + vector args; + argv_to_vec(argc, (const char **)argv, args); + + global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0); + common_init_finish(g_ceph_context); + + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +/* + * Local Variables: + * compile-command: "cd ../.. ; make -j4 && + * make unittest_erasure_code_plugin_jerasure && + * valgrind --tool=memcheck ./unittest_erasure_code_plugin_jerasure \ + * --gtest_filter=*.* --log-to-stderr=true --debug-osd=20" + * End: + */ + diff --git a/src/test/osd/ErasureCodeExample.h b/src/test/osd/ErasureCodeExample.h deleted file mode 100644 index 2b77d51e0944..000000000000 --- a/src/test/osd/ErasureCodeExample.h +++ /dev/null @@ -1,184 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* - * Ceph - scalable distributed file system - * - * Copyright (C) 2013 Cloudwatt - * - * Author: Loic Dachary - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - */ - -#ifndef CEPH_ERASURE_CODE_EXAMPLE_H -#define CEPH_ERASURE_CODE_EXAMPLE_H - -#include -#include -#include -#include - -#include "crush/CrushWrapper.h" -#include "osd/osd_types.h" -#include "erasure-code/ErasureCodeInterface.h" - -#define FIRST_DATA_CHUNK 0 -#define SECOND_DATA_CHUNK 1 -#define DATA_CHUNKS 2u - -#define CODING_CHUNK 2 -#define CODING_CHUNKS 1u - -#define MINIMUM_TO_RECOVER 2u - -class ErasureCodeExample : public ErasureCodeInterface { -public: - virtual ~ErasureCodeExample() {} - - virtual int create_ruleset(const string &name, - CrushWrapper &crush, - ostream *ss) const { - return crush.add_simple_ruleset(name, "default", "host", - "indep", pg_pool_t::TYPE_ERASURE, ss); - } - - virtual int minimum_to_decode(const set &want_to_read, - const set &available_chunks, - set *minimum) { - if (includes(available_chunks.begin(), available_chunks.end(), - want_to_read.begin(), want_to_read.end())) { - *minimum = want_to_read; - return 0; - } else if (available_chunks.size() >= MINIMUM_TO_RECOVER) { - *minimum = available_chunks; - return 0; - } else { - return -EIO; - } - } - - virtual int minimum_to_decode_with_cost(const set &want_to_read, - const map &available, - set *minimum) { - // - // If one chunk is more expensive to fetch than the others, - // recover it instead. For instance, if the cost reflects the - // time it takes for a chunk to be retrieved from a remote - // OSD and if CPU is cheap, it could make sense to recover - // instead of fetching the chunk. - // - map c2c(available); - if (c2c.size() > DATA_CHUNKS) { - if (c2c[FIRST_DATA_CHUNK] > c2c[SECOND_DATA_CHUNK] && - c2c[FIRST_DATA_CHUNK] > c2c[CODING_CHUNK]) - c2c.erase(FIRST_DATA_CHUNK); - else if(c2c[SECOND_DATA_CHUNK] > c2c[FIRST_DATA_CHUNK] && - c2c[SECOND_DATA_CHUNK] > c2c[CODING_CHUNK]) - c2c.erase(SECOND_DATA_CHUNK); - else if(c2c[CODING_CHUNK] > c2c[FIRST_DATA_CHUNK] && - c2c[CODING_CHUNK] > c2c[SECOND_DATA_CHUNK]) - c2c.erase(CODING_CHUNK); - } - set available_chunks; - for (map::const_iterator i = c2c.begin(); - i != c2c.end(); - ++i) - available_chunks.insert(i->first); - return minimum_to_decode(want_to_read, available_chunks, minimum); - } - - virtual unsigned int get_chunk_count() const { - return DATA_CHUNKS + CODING_CHUNKS; - } - - virtual unsigned int get_data_chunk_count() const { - return DATA_CHUNKS; - } - - virtual unsigned int get_chunk_size(unsigned int object_size) const { - return ( object_size / DATA_CHUNKS ) + 1; - } - - virtual int encode(const set &want_to_encode, - const bufferlist &in, - map *encoded) { - // - // make sure all data chunks have the same length, allocating - // padding if necessary. - // - unsigned int chunk_length = get_chunk_size(in.length()); - bufferlist out(in); - unsigned int width = get_chunk_count() * get_chunk_size(in.length()); - bufferptr pad(width - in.length()); - pad.zero(0, get_data_chunk_count()); - out.push_back(pad); - // - // compute the coding chunk with first chunk ^ second chunk - // - char *p = out.c_str(); - for (unsigned i = 0; i < chunk_length; i++) - p[i + CODING_CHUNK * chunk_length] = - p[i + FIRST_DATA_CHUNK * chunk_length] ^ - p[i + SECOND_DATA_CHUNK * chunk_length]; - // - // populate the bufferlist with bufferptr pointing - // to chunk boundaries - // - const bufferptr ptr = out.buffers().front(); - for (set::iterator j = want_to_encode.begin(); - j != want_to_encode.end(); - ++j) { - bufferptr chunk(ptr, (*j) * chunk_length, chunk_length); - (*encoded)[*j].push_front(chunk); - } - return 0; - } - - virtual int decode(const set &want_to_read, - const map &chunks, - map *decoded) { - // - // All chunks have the same size - // - unsigned chunk_length = (*chunks.begin()).second.length(); - for (set::iterator i = want_to_read.begin(); - i != want_to_read.end(); - ++i) { - if (chunks.find(*i) != chunks.end()) { - // - // If the chunk is available, just copy the bufferptr pointer - // to the decoded argument. - // - (*decoded)[*i] = chunks.find(*i)->second; - } else if(chunks.size() != 2) { - // - // If a chunk is missing and there are not enough chunks - // to recover, abort. - // - return -ERANGE; - } else { - // - // No matter what the missing chunk is, XOR of the other - // two recovers it. - // - map::const_iterator k = chunks.begin(); - const char *a = k->second.buffers().front().c_str(); - ++k; - const char *b = k->second.buffers().front().c_str(); - bufferptr chunk(chunk_length); - char *c = chunk.c_str(); - for (unsigned j = 0; j < chunk_length; j++) { - c[j] = a[j] ^ b[j]; - } - (*decoded)[*i].push_front(chunk); - } - } - return 0; - } -}; - -#endif diff --git a/src/test/osd/ErasureCodePluginExample.cc b/src/test/osd/ErasureCodePluginExample.cc deleted file mode 100644 index c1b5b3c6f0c4..000000000000 --- a/src/test/osd/ErasureCodePluginExample.cc +++ /dev/null @@ -1,36 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* - * Ceph - scalable distributed file system - * - * Copyright (C) 2013 Cloudwatt - * - * Author: Loic Dachary - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - */ - -#include - -#include "erasure-code/ErasureCodePlugin.h" -#include "ErasureCodeExample.h" - -class ErasureCodePluginExample : public ErasureCodePlugin { -public: - virtual int factory(const map ¶meters, - ErasureCodeInterfaceRef *erasure_code) - { - *erasure_code = ErasureCodeInterfaceRef(new ErasureCodeExample()); - return 0; - } -}; - -int __erasure_code_init(char *plugin_name) -{ - ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance(); - return instance.add(plugin_name, new ErasureCodePluginExample()); -} diff --git a/src/test/osd/ErasureCodePluginFailToInitialize.cc b/src/test/osd/ErasureCodePluginFailToInitialize.cc deleted file mode 100644 index d3043a43dc6a..000000000000 --- a/src/test/osd/ErasureCodePluginFailToInitialize.cc +++ /dev/null @@ -1,23 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* - * Ceph - scalable distributed file system - * - * Copyright (C) 2013 Cloudwatt - * - * Author: Loic Dachary - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - */ - -#include -#include "erasure-code/ErasureCodePlugin.h" - -int __erasure_code_init(char *plugin_name) -{ - return -ESRCH; -} diff --git a/src/test/osd/ErasureCodePluginFailToRegister.cc b/src/test/osd/ErasureCodePluginFailToRegister.cc deleted file mode 100644 index c26ac5b66710..000000000000 --- a/src/test/osd/ErasureCodePluginFailToRegister.cc +++ /dev/null @@ -1,22 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* - * Ceph - scalable distributed file system - * - * Copyright (C) 2013 Cloudwatt - * - * Author: Loic Dachary - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - */ - -#include "erasure-code/ErasureCodePlugin.h" - -int __erasure_code_init(char *plugin_name) -{ - return 0; -} diff --git a/src/test/osd/ErasureCodePluginHangs.cc b/src/test/osd/ErasureCodePluginHangs.cc deleted file mode 100644 index 01c2fa866d6a..000000000000 --- a/src/test/osd/ErasureCodePluginHangs.cc +++ /dev/null @@ -1,24 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* - * Ceph - scalable distributed file system - * - * Copyright (C) 2013 Cloudwatt - * - * Author: Loic Dachary - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - */ - -#include -#include "erasure-code/ErasureCodePlugin.h" - -int __erasure_code_init(char *plugin_name) -{ - sleep(1000); - return 0; -} diff --git a/src/test/osd/ErasureCodePluginMissingEntryPoint.cc b/src/test/osd/ErasureCodePluginMissingEntryPoint.cc deleted file mode 100644 index 4d8f1bafd1c9..000000000000 --- a/src/test/osd/ErasureCodePluginMissingEntryPoint.cc +++ /dev/null @@ -1,4 +0,0 @@ -// missing int __erasure_code_init(char *plugin_name) {} - -// avoid warnings about library containing no symbols -int __this_is_an_used_variable_to_avoid_warnings; diff --git a/src/test/osd/TestErasureCodeExample.cc b/src/test/osd/TestErasureCodeExample.cc deleted file mode 100644 index 4df0762419b5..000000000000 --- a/src/test/osd/TestErasureCodeExample.cc +++ /dev/null @@ -1,258 +0,0 @@ -// -*- mode:C; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* - * Ceph - scalable distributed file system - * - * Copyright (C) 2013 Cloudwatt - * - * Author: Loic Dachary - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - */ - -#include "include/stringify.h" -#include "global/global_init.h" -#include "ErasureCodeExample.h" -#include "common/ceph_argparse.h" -#include "global/global_context.h" -#include "gtest/gtest.h" - -TEST(ErasureCodeExample, chunk_size) -{ - ErasureCodeExample example; - EXPECT_EQ(3u, example.get_chunk_count()); - EXPECT_EQ(11u, example.get_chunk_size(20)); -} - -TEST(ErasureCodeExample, minimum_to_decode) -{ - ErasureCodeExample example; - set available_chunks; - set want_to_read; - want_to_read.insert(1); - { - set minimum; - EXPECT_EQ(-EIO, example.minimum_to_decode(want_to_read, - available_chunks, - &minimum)); - } - available_chunks.insert(0); - available_chunks.insert(2); - { - set minimum; - EXPECT_EQ(0, example.minimum_to_decode(want_to_read, - available_chunks, - &minimum)); - EXPECT_EQ(available_chunks, minimum); - EXPECT_EQ(2u, minimum.size()); - EXPECT_EQ(1u, minimum.count(0)); - EXPECT_EQ(1u, minimum.count(2)); - } - { - set minimum; - available_chunks.insert(1); - EXPECT_EQ(0, example.minimum_to_decode(want_to_read, - available_chunks, - &minimum)); - EXPECT_EQ(1u, minimum.size()); - EXPECT_EQ(1u, minimum.count(1)); - } -} - -TEST(ErasureCodeExample, minimum_to_decode_with_cost) -{ - ErasureCodeExample example; - map available; - set want_to_read; - want_to_read.insert(1); - { - set minimum; - EXPECT_EQ(-EIO, example.minimum_to_decode_with_cost(want_to_read, - available, - &minimum)); - } - available[0] = 1; - available[2] = 1; - { - set minimum; - EXPECT_EQ(0, example.minimum_to_decode_with_cost(want_to_read, - available, - &minimum)); - EXPECT_EQ(2u, minimum.size()); - EXPECT_EQ(1u, minimum.count(0)); - EXPECT_EQ(1u, minimum.count(2)); - } - { - set minimum; - available[1] = 1; - EXPECT_EQ(0, example.minimum_to_decode_with_cost(want_to_read, - available, - &minimum)); - EXPECT_EQ(1u, minimum.size()); - EXPECT_EQ(1u, minimum.count(1)); - } - { - set minimum; - available[1] = 2; - EXPECT_EQ(0, example.minimum_to_decode_with_cost(want_to_read, - available, - &minimum)); - EXPECT_EQ(2u, minimum.size()); - EXPECT_EQ(1u, minimum.count(0)); - EXPECT_EQ(1u, minimum.count(2)); - } -} - -TEST(ErasureCodeExample, encode_decode) -{ - ErasureCodeExample example; - - bufferlist in; - in.append("ABCDE"); - set want_to_encode; - for(unsigned int i = 0; i < example.get_chunk_count(); i++) - want_to_encode.insert(i); - map encoded; - EXPECT_EQ(0, example.encode(want_to_encode, in, &encoded)); - EXPECT_EQ(example.get_chunk_count(), encoded.size()); - EXPECT_EQ(example.get_chunk_size(in.length()), encoded[0].length()); - EXPECT_EQ('A', encoded[0][0]); - EXPECT_EQ('B', encoded[0][1]); - EXPECT_EQ('C', encoded[0][2]); - EXPECT_EQ('D', encoded[1][0]); - EXPECT_EQ('E', encoded[1][1]); - EXPECT_EQ('A'^'D', encoded[2][0]); - EXPECT_EQ('B'^'E', encoded[2][1]); - EXPECT_EQ('C'^0, encoded[2][2]); - - // all chunks are available - { - int want_to_decode[] = { 0, 1 }; - map decoded; - EXPECT_EQ(0, example.decode(set(want_to_decode, want_to_decode+2), - encoded, - &decoded)); - EXPECT_EQ(2u, decoded.size()); - EXPECT_EQ(3u, decoded[0].length()); - EXPECT_EQ('A', decoded[0][0]); - EXPECT_EQ('B', decoded[0][1]); - EXPECT_EQ('C', decoded[0][2]); - EXPECT_EQ('D', decoded[1][0]); - EXPECT_EQ('E', decoded[1][1]); - } - - // one chunk is missing - { - map degraded = encoded; - degraded.erase(0); - EXPECT_EQ(2u, degraded.size()); - int want_to_decode[] = { 0, 1 }; - map decoded; - EXPECT_EQ(0, example.decode(set(want_to_decode, want_to_decode+2), - degraded, - &decoded)); - EXPECT_EQ(2u, decoded.size()); - EXPECT_EQ(3u, decoded[0].length()); - EXPECT_EQ('A', decoded[0][0]); - EXPECT_EQ('B', decoded[0][1]); - EXPECT_EQ('C', decoded[0][2]); - EXPECT_EQ('D', decoded[1][0]); - EXPECT_EQ('E', decoded[1][1]); - } -} - -TEST(ErasureCodeExample, decode) -{ - ErasureCodeExample example; - -#define LARGE_ENOUGH 2048 - bufferptr in_ptr(buffer::create_page_aligned(LARGE_ENOUGH)); - in_ptr.zero(); - in_ptr.set_length(0); - const char *payload = - "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" - "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" - "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" - "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" - "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; - in_ptr.append(payload, strlen(payload)); - bufferlist in; - in.push_front(in_ptr); - int want_to_encode[] = { 0, 1, 2 }; - map encoded; - EXPECT_EQ(0, example.encode(set(want_to_encode, want_to_encode+3), - in, - &encoded)); - EXPECT_EQ(3u, encoded.size()); - - // successfull decode - bufferlist out; - EXPECT_EQ(0, example.decode_concat(encoded, &out)); - bufferlist usable; - usable.substr_of(out, 0, in.length()); - EXPECT_TRUE(usable == in); - - // cannot recover - map degraded; - degraded[0] = encoded[0]; - EXPECT_EQ(-ERANGE, example.decode_concat(degraded, &out)); -} - -TEST(ErasureCodeExample, create_ruleset) -{ - CrushWrapper *c = new CrushWrapper; - c->create(); - c->set_type_name(2, "root"); - c->set_type_name(1, "host"); - c->set_type_name(0, "osd"); - - int rootno; - c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1, - 5, 0, NULL, NULL, &rootno); - c->set_item_name(rootno, "default"); - - map loc; - loc["root"] = "default"; - - int num_host = 2; - int num_osd = 5; - int osd = 0; - for (int h=0; hinsert_item(g_ceph_context, osd, 1.0, string("osd.") + stringify(osd), loc); - } - } - - stringstream ss; - ErasureCodeExample example; - EXPECT_EQ(0, example.create_ruleset("myrule", *c, &ss)); -} - -int main(int argc, char **argv) { - vector args; - argv_to_vec(argc, (const char **)argv, args); - - global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0); - common_init_finish(g_ceph_context); - - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} - -/* - * Local Variables: - * compile-command: "cd ../.. ; - * make -j4 && - * make unittest_erasure_code_example && - * valgrind --leak-check=full --tool=memcheck \ - * ./unittest_erasure_code_example --gtest_filter=*.* \ - * --log-to-stderr=true --debug-osd=20 - * " - * End: - */ - diff --git a/src/test/osd/TestErasureCodeJerasure.cc b/src/test/osd/TestErasureCodeJerasure.cc deleted file mode 100644 index a3d603bea540..000000000000 --- a/src/test/osd/TestErasureCodeJerasure.cc +++ /dev/null @@ -1,362 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* - * Ceph - scalable distributed file system - * - * Copyright (C) 2013 Cloudwatt - * - * Author: Loic Dachary - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - */ - -#include - -#include "crush/CrushWrapper.h" -#include "include/stringify.h" -#include "global/global_init.h" -#include "erasure-code/jerasure/ErasureCodeJerasure.h" -#include "common/ceph_argparse.h" -#include "global/global_context.h" -#include "gtest/gtest.h" - -template -class ErasureCodeTest : public ::testing::Test { - public: -}; - -typedef ::testing::Types< - ErasureCodeJerasureReedSolomonVandermonde, - ErasureCodeJerasureReedSolomonRAID6, - ErasureCodeJerasureCauchyOrig, - ErasureCodeJerasureCauchyGood, - ErasureCodeJerasureLiberation, - ErasureCodeJerasureBlaumRoth, - ErasureCodeJerasureLiber8tion -> JerasureTypes; -TYPED_TEST_CASE(ErasureCodeTest, JerasureTypes); - -TYPED_TEST(ErasureCodeTest, encode_decode) -{ - TypeParam jerasure; - map parameters; - parameters["erasure-code-k"] = "2"; - parameters["erasure-code-m"] = "2"; - parameters["erasure-code-w"] = "7"; - parameters["erasure-code-packetsize"] = "8"; - jerasure.init(parameters); - -#define LARGE_ENOUGH 2048 - bufferptr in_ptr(buffer::create_page_aligned(LARGE_ENOUGH)); - in_ptr.zero(); - in_ptr.set_length(0); - const char *payload = - "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" - "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" - "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" - "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" - "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; - in_ptr.append(payload, strlen(payload)); - bufferlist in; - in.push_front(in_ptr); - int want_to_encode[] = { 0, 1, 2, 3 }; - map encoded; - EXPECT_EQ(0, jerasure.encode(set(want_to_encode, want_to_encode+4), - in, - &encoded)); - EXPECT_EQ(4u, encoded.size()); - unsigned length = encoded[0].length(); - EXPECT_EQ(0, strncmp(encoded[0].c_str(), in.c_str(), length)); - EXPECT_EQ(0, strncmp(encoded[1].c_str(), in.c_str() + length, - in.length() - length)); - - - // all chunks are available - { - int want_to_decode[] = { 0, 1 }; - map decoded; - EXPECT_EQ(0, jerasure.decode(set(want_to_decode, want_to_decode+2), - encoded, - &decoded)); - // always decode all, regardless of want_to_decode - EXPECT_EQ(4u, decoded.size()); - EXPECT_EQ(length, decoded[0].length()); - EXPECT_EQ(0, strncmp(decoded[0].c_str(), in.c_str(), length)); - EXPECT_EQ(0, strncmp(decoded[1].c_str(), in.c_str() + length, - in.length() - length)); - } - - // two chunks are missing - { - map degraded = encoded; - degraded.erase(0); - degraded.erase(1); - EXPECT_EQ(2u, degraded.size()); - int want_to_decode[] = { 0, 1 }; - map decoded; - EXPECT_EQ(0, jerasure.decode(set(want_to_decode, want_to_decode+2), - degraded, - &decoded)); - // always decode all, regardless of want_to_decode - EXPECT_EQ(4u, decoded.size()); - EXPECT_EQ(length, decoded[0].length()); - EXPECT_EQ(0, strncmp(decoded[0].c_str(), in.c_str(), length)); - EXPECT_EQ(0, strncmp(decoded[1].c_str(), in.c_str() + length, - in.length() - length)); - } -} - -TYPED_TEST(ErasureCodeTest, minimum_to_decode) -{ - TypeParam jerasure; - map parameters; - parameters["erasure-code-k"] = "2"; - parameters["erasure-code-m"] = "2"; - parameters["erasure-code-w"] = "7"; - parameters["erasure-code-packetsize"] = "8"; - jerasure.init(parameters); - - // - // If trying to read nothing, the minimum is empty. - // - { - set want_to_read; - set available_chunks; - set minimum; - - EXPECT_EQ(0, jerasure.minimum_to_decode(want_to_read, - available_chunks, - &minimum)); - EXPECT_TRUE(minimum.empty()); - } - // - // There is no way to read a chunk if none are available. - // - { - set want_to_read; - set available_chunks; - set minimum; - - want_to_read.insert(0); - - EXPECT_EQ(-EIO, jerasure.minimum_to_decode(want_to_read, - available_chunks, - &minimum)); - } - // - // Reading a subset of the available chunks is always possible. - // - { - set want_to_read; - set available_chunks; - set minimum; - - want_to_read.insert(0); - available_chunks.insert(0); - - EXPECT_EQ(0, jerasure.minimum_to_decode(want_to_read, - available_chunks, - &minimum)); - EXPECT_EQ(want_to_read, minimum); - } - // - // There is no way to read a missing chunk if there is less than k - // chunks available. - // - { - set want_to_read; - set available_chunks; - set minimum; - - want_to_read.insert(0); - want_to_read.insert(1); - available_chunks.insert(0); - - EXPECT_EQ(-EIO, jerasure.minimum_to_decode(want_to_read, - available_chunks, - &minimum)); - } - // - // When chunks are not available, the minimum can be made of any - // chunks. For instance, to read 1 and 3 below the minimum could be - // 2 and 3 which may seem better because it contains one of the - // chunks to be read. But it won't be more efficient than retrieving - // 0 and 2 instead because, in both cases, the decode function will - // need to run the same recovery operation and use the same amount - // of CPU and memory. - // - { - set want_to_read; - set available_chunks; - set minimum; - - want_to_read.insert(1); - want_to_read.insert(3); - available_chunks.insert(0); - available_chunks.insert(2); - available_chunks.insert(3); - - EXPECT_EQ(0, jerasure.minimum_to_decode(want_to_read, - available_chunks, - &minimum)); - EXPECT_EQ(2u, minimum.size()); - EXPECT_EQ(0u, minimum.count(3)); - } -} - -TEST(ErasureCodeTest, encode) -{ - ErasureCodeJerasureReedSolomonVandermonde jerasure; - map parameters; - parameters["erasure-code-k"] = "2"; - parameters["erasure-code-m"] = "2"; - parameters["erasure-code-w"] = "8"; - jerasure.init(parameters); - - unsigned alignment = jerasure.get_alignment(); - { - // - // When the input bufferlist needs to be padded because - // it is not properly aligned, it is padded with zeros. - // - bufferlist in; - map encoded; - int want_to_encode[] = { 0, 1, 2, 3 }; - int trail_length = 10; - in.append(string(alignment + trail_length, 'X')); - EXPECT_EQ(0, jerasure.encode(set(want_to_encode, want_to_encode+4), - in, - &encoded)); - EXPECT_EQ(4u, encoded.size()); - for(int i = 0; i < 4; i++) - EXPECT_EQ(alignment, encoded[i].length()); - char *last_chunk = encoded[1].c_str(); - EXPECT_EQ('X', last_chunk[0]); - EXPECT_EQ('\0', last_chunk[trail_length]); - } - - { - // - // When only the first chunk is required, the encoded map only - // contains the first chunk. Although the jerasure encode - // internally allocated a buffer because of padding requirements - // and also computes the coding chunks, they are released before - // the return of the method, as shown when running the tests thru - // valgrind (there is no leak). - // - bufferlist in; - map encoded; - set want_to_encode; - want_to_encode.insert(0); - int trail_length = 10; - in.append(string(alignment + trail_length, 'X')); - EXPECT_EQ(0, jerasure.encode(want_to_encode, in, &encoded)); - EXPECT_EQ(1u, encoded.size()); - EXPECT_EQ(alignment, encoded[0].length()); - } -} - -TEST(ErasureCodeTest, create_ruleset) -{ - CrushWrapper *c = new CrushWrapper; - c->create(); - int root_type = 2; - c->set_type_name(root_type, "root"); - int host_type = 1; - c->set_type_name(host_type, "host"); - int osd_type = 0; - c->set_type_name(osd_type, "osd"); - - int rootno; - c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1, - root_type, 0, NULL, NULL, &rootno); - c->set_item_name(rootno, "default"); - - map loc; - loc["root"] = "default"; - - int num_host = 4; - int num_osd = 5; - int osd = 0; - for (int h=0; hinsert_item(g_ceph_context, osd, 1.0, string("osd.") + stringify(osd), loc); - } - } - - { - stringstream ss; - ErasureCodeJerasureReedSolomonVandermonde jerasure; - map parameters; - parameters["erasure-code-k"] = "2"; - parameters["erasure-code-m"] = "2"; - parameters["erasure-code-w"] = "8"; - jerasure.init(parameters); - int ruleset = jerasure.create_ruleset("myrule", *c, &ss); - EXPECT_EQ(0, ruleset); - EXPECT_EQ(-EEXIST, jerasure.create_ruleset("myrule", *c, &ss)); - // - // the minimum that is expected from the created ruleset is to - // successfully map get_chunk_count() devices from the crushmap, - // at least once. - // - vector<__u32> weight(c->get_max_devices(), 0x10000); - vector out; - int x = 0; - c->do_rule(ruleset, x, out, jerasure.get_chunk_count(), weight); - ASSERT_EQ(out.size(), jerasure.get_chunk_count()); - for (unsigned i=0; i parameters; - parameters["erasure-code-k"] = "2"; - parameters["erasure-code-m"] = "2"; - parameters["erasure-code-w"] = "8"; - parameters["erasure-code-ruleset-root"] = "BAD"; - jerasure.init(parameters); - EXPECT_EQ(-ENOENT, jerasure.create_ruleset("otherrule", *c, &ss)); - EXPECT_EQ("root item BAD does not exist", ss.str()); - } - { - stringstream ss; - ErasureCodeJerasureReedSolomonVandermonde jerasure; - map parameters; - parameters["erasure-code-k"] = "2"; - parameters["erasure-code-m"] = "2"; - parameters["erasure-code-w"] = "8"; - parameters["erasure-code-ruleset-failure-domain"] = "WORSE"; - jerasure.init(parameters); - EXPECT_EQ(-EINVAL, jerasure.create_ruleset("otherrule", *c, &ss)); - EXPECT_EQ("unknown type WORSE", ss.str()); - } -} - -int main(int argc, char **argv) -{ - vector args; - argv_to_vec(argc, (const char **)argv, args); - - global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0); - common_init_finish(g_ceph_context); - - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} - -/* - * Local Variables: - * compile-command: "cd ../.. ; make -j4 && - * make unittest_erasure_code_jerasure && - * valgrind --tool=memcheck --leak-check=full \ - * ./unittest_erasure_code_jerasure \ - * --gtest_filter=*.* --log-to-stderr=true --debug-osd=20" - * End: - */ diff --git a/src/test/osd/TestErasureCodePlugin.cc b/src/test/osd/TestErasureCodePlugin.cc deleted file mode 100644 index f97b140d5d25..000000000000 --- a/src/test/osd/TestErasureCodePlugin.cc +++ /dev/null @@ -1,108 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* - * Ceph - scalable distributed file system - * - * Copyright (C) 2013 Cloudwatt - * - * Author: Loic Dachary - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - */ - -#include -#include -#include "common/Thread.h" -#include "global/global_init.h" -#include "erasure-code/ErasureCodePlugin.h" -#include "common/ceph_argparse.h" -#include "global/global_context.h" -#include "gtest/gtest.h" - -class ErasureCodePluginRegistryTest : public ::testing::Test { -protected: - - class Thread_factory : public Thread { - public: - virtual void *entry() { - map parameters; - parameters["erasure-code-directory"] = ".libs"; - ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance(); - ErasureCodeInterfaceRef erasure_code; - instance.factory("hangs", parameters, &erasure_code); - return NULL; - } - }; - -}; - -TEST_F(ErasureCodePluginRegistryTest, factory_mutex) { - ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance(); - - EXPECT_TRUE(instance.lock.TryLock()); - instance.lock.Unlock(); - - // - // Test that the loading of a plugin is protected by a mutex. - // - useconds_t delay = 0; - const useconds_t DELAY_MAX = 20 * 1000 * 1000; - Thread_factory sleep_forever; - sleep_forever.create(); - do { - cout << "Trying (1) with delay " << delay << "us\n"; - if (delay > 0) - usleep(delay); - if (!instance.loading) - delay = ( delay + 1 ) * 2; - } while(!instance.loading && delay < DELAY_MAX); - ASSERT_TRUE(delay < DELAY_MAX); - - EXPECT_FALSE(instance.lock.TryLock()); - - EXPECT_EQ(0, pthread_cancel(sleep_forever.get_thread_id())); - EXPECT_EQ(0, sleep_forever.join()); -} - -TEST_F(ErasureCodePluginRegistryTest, all) -{ - map parameters; - parameters["erasure-code-directory"] = ".libs"; - ErasureCodeInterfaceRef erasure_code; - ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance(); - EXPECT_FALSE(erasure_code); - EXPECT_EQ(-EIO, instance.factory("invalid", parameters, &erasure_code)); - EXPECT_FALSE(erasure_code); - EXPECT_EQ(-ENOENT, instance.factory("missing_entry_point", parameters, - &erasure_code)); - EXPECT_FALSE(erasure_code); - EXPECT_EQ(-ESRCH, instance.factory("fail_to_initialize", parameters, - &erasure_code)); - EXPECT_FALSE(erasure_code); - EXPECT_EQ(-EBADF, instance.factory("fail_to_register", parameters, - &erasure_code)); - EXPECT_FALSE(erasure_code); - EXPECT_EQ(0, instance.factory("example", parameters, &erasure_code)); - EXPECT_TRUE(erasure_code); - ErasureCodePlugin *plugin = 0; - EXPECT_EQ(-EEXIST, instance.load("example", parameters, &plugin)); -} - -int main(int argc, char **argv) { - vector args; - argv_to_vec(argc, (const char **)argv, args); - - global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0); - common_init_finish(g_ceph_context); - - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} - -// Local Variables: -// compile-command: "cd ../.. ; make -j4 && make unittest_erasure_code_plugin && valgrind --leak-check=full --tool=memcheck ./unittest_erasure_code_plugin --gtest_filter=*.* --log-to-stderr=true --debug-osd=20" -// End: diff --git a/src/test/osd/TestErasureCodePluginJerasure.cc b/src/test/osd/TestErasureCodePluginJerasure.cc deleted file mode 100644 index 0013ce8d492b..000000000000 --- a/src/test/osd/TestErasureCodePluginJerasure.cc +++ /dev/null @@ -1,74 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* - * Ceph - scalable distributed file system - * - * Copyright (C) 2013 Cloudwatt - * - * Author: Loic Dachary - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - */ - -#include -#include "global/global_init.h" -#include "erasure-code/ErasureCodePlugin.h" -#include "common/ceph_argparse.h" -#include "global/global_context.h" -#include "gtest/gtest.h" - -TEST(ErasureCodePlugin, factory) -{ - ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance(); - map parameters; - parameters["erasure-code-directory"] = ".libs"; - { - ErasureCodeInterfaceRef erasure_code; - EXPECT_FALSE(erasure_code); - EXPECT_EQ(-ENOENT, instance.factory("jerasure", parameters, &erasure_code)); - EXPECT_FALSE(erasure_code); - } - const char *techniques[] = { - "reed_sol_van", - "reed_sol_r6_op", - "cauchy_orig", - "cauchy_good", - "liberation", - "blaum_roth", - "liber8tion", - 0 - }; - for(const char **technique = techniques; *technique; technique++) { - ErasureCodeInterfaceRef erasure_code; - parameters["erasure-code-technique"] = *technique; - EXPECT_FALSE(erasure_code); - EXPECT_EQ(0, instance.factory("jerasure", parameters, &erasure_code)); - EXPECT_TRUE(erasure_code); - } -} - -int main(int argc, char **argv) -{ - vector args; - argv_to_vec(argc, (const char **)argv, args); - - global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0); - common_init_finish(g_ceph_context); - - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} - -/* - * Local Variables: - * compile-command: "cd ../.. ; make -j4 && - * make unittest_erasure_code_plugin_jerasure && - * valgrind --tool=memcheck ./unittest_erasure_code_plugin_jerasure \ - * --gtest_filter=*.* --log-to-stderr=true --debug-osd=20" - * End: - */ -