unittest_ceph_compatset_CXXFLAGS = $(UNITTEST_CXXFLAGS)
check_PROGRAMS += unittest_ceph_compatset
-libec_example_la_SOURCES = test/osd/ErasureCodePluginExample.cc
-libec_example_la_CFLAGS = ${AM_CFLAGS}
-libec_example_la_CXXFLAGS= ${AM_CXXFLAGS}
-libec_example_la_LIBADD = $(LIBCRUSH) $(PTHREAD_LIBS) $(EXTRALIBS)
-libec_example_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*'
-erasure_codelib_LTLIBRARIES += libec_example.la
-
-libec_missing_entry_point_la_SOURCES = test/osd/ErasureCodePluginMissingEntryPoint.cc
-libec_missing_entry_point_la_CFLAGS = ${AM_CFLAGS}
-libec_missing_entry_point_la_CXXFLAGS= ${AM_CXXFLAGS}
-libec_missing_entry_point_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
-libec_missing_entry_point_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*'
-erasure_codelib_LTLIBRARIES += libec_missing_entry_point.la
-
-libec_hangs_la_SOURCES = test/osd/ErasureCodePluginHangs.cc
-libec_hangs_la_CFLAGS = ${AM_CFLAGS}
-libec_hangs_la_CXXFLAGS= ${AM_CXXFLAGS}
-libec_hangs_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
-libec_hangs_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*'
-erasure_codelib_LTLIBRARIES += libec_hangs.la
-
-libec_fail_to_initialize_la_SOURCES = test/osd/ErasureCodePluginFailToInitialize.cc
-libec_fail_to_initialize_la_CFLAGS = ${AM_CFLAGS}
-libec_fail_to_initialize_la_CXXFLAGS= ${AM_CXXFLAGS}
-libec_fail_to_initialize_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
-libec_fail_to_initialize_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*'
-erasure_codelib_LTLIBRARIES += libec_fail_to_initialize.la
-
-libec_fail_to_register_la_SOURCES = test/osd/ErasureCodePluginFailToRegister.cc
-libec_fail_to_register_la_CFLAGS = ${AM_CFLAGS}
-libec_fail_to_register_la_CXXFLAGS= ${AM_CXXFLAGS}
-libec_fail_to_register_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
-libec_fail_to_register_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*'
-erasure_codelib_LTLIBRARIES += libec_fail_to_register.la
-
-unittest_erasure_code_plugin_SOURCES = test/osd/TestErasureCodePlugin.cc
-unittest_erasure_code_plugin_CXXFLAGS = $(UNITTEST_CXXFLAGS)
-unittest_erasure_code_plugin_LDADD = $(LIBOSD) $(LIBCOMMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
-if LINUX
-unittest_erasure_code_plugin_LDADD += -ldl
-endif
-check_PROGRAMS += unittest_erasure_code_plugin
-
-unittest_erasure_code_jerasure_SOURCES = \
- test/osd/TestErasureCodeJerasure.cc \
- $(libec_jerasure_la_SOURCES)
-unittest_erasure_code_jerasure_CXXFLAGS = $(UNITTEST_CXXFLAGS)
-unittest_erasure_code_jerasure_LDADD = $(LIBOSD) $(LIBCOMMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
-if LINUX
-unittest_erasure_code_jerasure_LDADD += -ldl
-endif
-check_PROGRAMS += unittest_erasure_code_jerasure
-
-unittest_erasure_code_plugin_jerasure_SOURCES = \
- test/osd/TestErasureCodePluginJerasure.cc
-unittest_erasure_code_plugin_jerasure_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS}
-unittest_erasure_code_plugin_jerasure_LDADD = $(LIBOSD) $(LIBCOMMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
-if LINUX
-unittest_erasure_code_plugin_jerasure_LDADD += -ldl
-endif
-check_PROGRAMS += unittest_erasure_code_plugin_jerasure
-
-unittest_erasure_code_example_SOURCES = test/osd/TestErasureCodeExample.cc
-noinst_HEADERS += test/osd/ErasureCodeExample.h
-unittest_erasure_code_example_CXXFLAGS = $(UNITTEST_CXXFLAGS)
-unittest_erasure_code_example_LDADD = $(LIBOSD) $(LIBCOMMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
-check_PROGRAMS += unittest_erasure_code_example
-
unittest_osd_types_SOURCES = test/test_osd_types.cc
unittest_osd_types_CXXFLAGS = $(UNITTEST_CXXFLAGS)
unittest_osd_types_LDADD = $(UNITTEST_LDADD) $(CEPH_GLOBAL)
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com>
+ *
+ * Author: Loic Dachary <loic@dachary.org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ */
+
+#ifndef CEPH_ERASURE_CODE_EXAMPLE_H
+#define CEPH_ERASURE_CODE_EXAMPLE_H
+
+#include <unistd.h>
+#include <errno.h>
+#include <algorithm>
+#include <sstream>
+
+#include "crush/CrushWrapper.h"
+#include "osd/osd_types.h"
+#include "erasure-code/ErasureCodeInterface.h"
+
+#define FIRST_DATA_CHUNK 0
+#define SECOND_DATA_CHUNK 1
+#define DATA_CHUNKS 2u
+
+#define CODING_CHUNK 2
+#define CODING_CHUNKS 1u
+
+#define MINIMUM_TO_RECOVER 2u
+
+class ErasureCodeExample : public ErasureCodeInterface {
+public:
+ virtual ~ErasureCodeExample() {}
+
+ virtual int create_ruleset(const string &name,
+ CrushWrapper &crush,
+ ostream *ss) const {
+ return crush.add_simple_ruleset(name, "default", "host",
+ "indep", pg_pool_t::TYPE_ERASURE, ss);
+ }
+
+ virtual int minimum_to_decode(const set<int> &want_to_read,
+ const set<int> &available_chunks,
+ set<int> *minimum) {
+ if (includes(available_chunks.begin(), available_chunks.end(),
+ want_to_read.begin(), want_to_read.end())) {
+ *minimum = want_to_read;
+ return 0;
+ } else if (available_chunks.size() >= MINIMUM_TO_RECOVER) {
+ *minimum = available_chunks;
+ return 0;
+ } else {
+ return -EIO;
+ }
+ }
+
+ virtual int minimum_to_decode_with_cost(const set<int> &want_to_read,
+ const map<int, int> &available,
+ set<int> *minimum) {
+ //
+ // If one chunk is more expensive to fetch than the others,
+ // recover it instead. For instance, if the cost reflects the
+ // time it takes for a chunk to be retrieved from a remote
+ // OSD and if CPU is cheap, it could make sense to recover
+ // instead of fetching the chunk.
+ //
+ map<int, int> c2c(available);
+ if (c2c.size() > DATA_CHUNKS) {
+ if (c2c[FIRST_DATA_CHUNK] > c2c[SECOND_DATA_CHUNK] &&
+ c2c[FIRST_DATA_CHUNK] > c2c[CODING_CHUNK])
+ c2c.erase(FIRST_DATA_CHUNK);
+ else if(c2c[SECOND_DATA_CHUNK] > c2c[FIRST_DATA_CHUNK] &&
+ c2c[SECOND_DATA_CHUNK] > c2c[CODING_CHUNK])
+ c2c.erase(SECOND_DATA_CHUNK);
+ else if(c2c[CODING_CHUNK] > c2c[FIRST_DATA_CHUNK] &&
+ c2c[CODING_CHUNK] > c2c[SECOND_DATA_CHUNK])
+ c2c.erase(CODING_CHUNK);
+ }
+ set <int> available_chunks;
+ for (map<int, int>::const_iterator i = c2c.begin();
+ i != c2c.end();
+ ++i)
+ available_chunks.insert(i->first);
+ return minimum_to_decode(want_to_read, available_chunks, minimum);
+ }
+
+ virtual unsigned int get_chunk_count() const {
+ return DATA_CHUNKS + CODING_CHUNKS;
+ }
+
+ virtual unsigned int get_data_chunk_count() const {
+ return DATA_CHUNKS;
+ }
+
+ virtual unsigned int get_chunk_size(unsigned int object_size) const {
+ return ( object_size / DATA_CHUNKS ) + 1;
+ }
+
+ virtual int encode(const set<int> &want_to_encode,
+ const bufferlist &in,
+ map<int, bufferlist> *encoded) {
+ //
+ // make sure all data chunks have the same length, allocating
+ // padding if necessary.
+ //
+ unsigned int chunk_length = get_chunk_size(in.length());
+ bufferlist out(in);
+ unsigned int width = get_chunk_count() * get_chunk_size(in.length());
+ bufferptr pad(width - in.length());
+ pad.zero(0, get_data_chunk_count());
+ out.push_back(pad);
+ //
+ // compute the coding chunk with first chunk ^ second chunk
+ //
+ char *p = out.c_str();
+ for (unsigned i = 0; i < chunk_length; i++)
+ p[i + CODING_CHUNK * chunk_length] =
+ p[i + FIRST_DATA_CHUNK * chunk_length] ^
+ p[i + SECOND_DATA_CHUNK * chunk_length];
+ //
+ // populate the bufferlist with bufferptr pointing
+ // to chunk boundaries
+ //
+ const bufferptr ptr = out.buffers().front();
+ for (set<int>::iterator j = want_to_encode.begin();
+ j != want_to_encode.end();
+ ++j) {
+ bufferptr chunk(ptr, (*j) * chunk_length, chunk_length);
+ (*encoded)[*j].push_front(chunk);
+ }
+ return 0;
+ }
+
+ virtual int decode(const set<int> &want_to_read,
+ const map<int, bufferlist> &chunks,
+ map<int, bufferlist> *decoded) {
+ //
+ // All chunks have the same size
+ //
+ unsigned chunk_length = (*chunks.begin()).second.length();
+ for (set<int>::iterator i = want_to_read.begin();
+ i != want_to_read.end();
+ ++i) {
+ if (chunks.find(*i) != chunks.end()) {
+ //
+ // If the chunk is available, just copy the bufferptr pointer
+ // to the decoded argument.
+ //
+ (*decoded)[*i] = chunks.find(*i)->second;
+ } else if(chunks.size() != 2) {
+ //
+ // If a chunk is missing and there are not enough chunks
+ // to recover, abort.
+ //
+ return -ERANGE;
+ } else {
+ //
+ // No matter what the missing chunk is, XOR of the other
+ // two recovers it.
+ //
+ map<int, bufferlist>::const_iterator k = chunks.begin();
+ const char *a = k->second.buffers().front().c_str();
+ ++k;
+ const char *b = k->second.buffers().front().c_str();
+ bufferptr chunk(chunk_length);
+ char *c = chunk.c_str();
+ for (unsigned j = 0; j < chunk_length; j++) {
+ c[j] = a[j] ^ b[j];
+ }
+ (*decoded)[*i].push_front(chunk);
+ }
+ }
+ return 0;
+ }
+};
+
+#endif
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com>
+ *
+ * Author: Loic Dachary <loic@dachary.org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ */
+
+#include <unistd.h>
+
+#include "erasure-code/ErasureCodePlugin.h"
+#include "ErasureCodeExample.h"
+
+class ErasureCodePluginExample : public ErasureCodePlugin {
+public:
+ virtual int factory(const map<std::string,std::string> ¶meters,
+ ErasureCodeInterfaceRef *erasure_code)
+ {
+ *erasure_code = ErasureCodeInterfaceRef(new ErasureCodeExample());
+ return 0;
+ }
+};
+
+int __erasure_code_init(char *plugin_name)
+{
+ ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance();
+ return instance.add(plugin_name, new ErasureCodePluginExample());
+}
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com>
+ *
+ * Author: Loic Dachary <loic@dachary.org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ */
+
+#include <errno.h>
+#include "erasure-code/ErasureCodePlugin.h"
+
+int __erasure_code_init(char *plugin_name)
+{
+ return -ESRCH;
+}
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com>
+ *
+ * Author: Loic Dachary <loic@dachary.org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ */
+
+#include "erasure-code/ErasureCodePlugin.h"
+
+int __erasure_code_init(char *plugin_name)
+{
+ return 0;
+}
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com>
+ *
+ * Author: Loic Dachary <loic@dachary.org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ */
+
+#include <unistd.h>
+#include "erasure-code/ErasureCodePlugin.h"
+
+int __erasure_code_init(char *plugin_name)
+{
+ sleep(1000);
+ return 0;
+}
--- /dev/null
+// missing int __erasure_code_init(char *plugin_name) {}
+
+// avoid warnings about library containing no symbols
+int __this_is_an_used_variable_to_avoid_warnings;
endif
bin_DEBUGPROGRAMS += ceph_erasure_code
+libec_example_la_SOURCES = test/erasure-code/ErasureCodePluginExample.cc
+libec_example_la_CFLAGS = ${AM_CFLAGS}
+libec_example_la_CXXFLAGS= ${AM_CXXFLAGS}
+libec_example_la_LIBADD = $(LIBCRUSH) $(PTHREAD_LIBS) $(EXTRALIBS)
+libec_example_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*'
+erasure_codelib_LTLIBRARIES += libec_example.la
+
+libec_missing_entry_point_la_SOURCES = test/erasure-code/ErasureCodePluginMissingEntryPoint.cc
+libec_missing_entry_point_la_CFLAGS = ${AM_CFLAGS}
+libec_missing_entry_point_la_CXXFLAGS= ${AM_CXXFLAGS}
+libec_missing_entry_point_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
+libec_missing_entry_point_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*'
+erasure_codelib_LTLIBRARIES += libec_missing_entry_point.la
+
+libec_hangs_la_SOURCES = test/erasure-code/ErasureCodePluginHangs.cc
+libec_hangs_la_CFLAGS = ${AM_CFLAGS}
+libec_hangs_la_CXXFLAGS= ${AM_CXXFLAGS}
+libec_hangs_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
+libec_hangs_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*'
+erasure_codelib_LTLIBRARIES += libec_hangs.la
+
+libec_fail_to_initialize_la_SOURCES = test/erasure-code/ErasureCodePluginFailToInitialize.cc
+libec_fail_to_initialize_la_CFLAGS = ${AM_CFLAGS}
+libec_fail_to_initialize_la_CXXFLAGS= ${AM_CXXFLAGS}
+libec_fail_to_initialize_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
+libec_fail_to_initialize_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*'
+erasure_codelib_LTLIBRARIES += libec_fail_to_initialize.la
+
+libec_fail_to_register_la_SOURCES = test/erasure-code/ErasureCodePluginFailToRegister.cc
+libec_fail_to_register_la_CFLAGS = ${AM_CFLAGS}
+libec_fail_to_register_la_CXXFLAGS= ${AM_CXXFLAGS}
+libec_fail_to_register_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
+libec_fail_to_register_la_LDFLAGS = ${AM_LDFLAGS} -export-symbols-regex '.*__erasure_code_.*'
+erasure_codelib_LTLIBRARIES += libec_fail_to_register.la
+
+unittest_erasure_code_plugin_SOURCES = test/erasure-code/TestErasureCodePlugin.cc
+unittest_erasure_code_plugin_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+unittest_erasure_code_plugin_LDADD = $(LIBOSD) $(LIBCOMMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+if LINUX
+unittest_erasure_code_plugin_LDADD += -ldl
+endif
+check_PROGRAMS += unittest_erasure_code_plugin
+
+unittest_erasure_code_jerasure_SOURCES = \
+ test/erasure-code/TestErasureCodeJerasure.cc \
+ $(libec_jerasure_la_SOURCES)
+unittest_erasure_code_jerasure_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+unittest_erasure_code_jerasure_LDADD = $(LIBOSD) $(LIBCOMMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+if LINUX
+unittest_erasure_code_jerasure_LDADD += -ldl
+endif
+check_PROGRAMS += unittest_erasure_code_jerasure
+
+unittest_erasure_code_plugin_jerasure_SOURCES = \
+ test/erasure-code/TestErasureCodePluginJerasure.cc
+unittest_erasure_code_plugin_jerasure_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS}
+unittest_erasure_code_plugin_jerasure_LDADD = $(LIBOSD) $(LIBCOMMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+if LINUX
+unittest_erasure_code_plugin_jerasure_LDADD += -ldl
+endif
+check_PROGRAMS += unittest_erasure_code_plugin_jerasure
+
+unittest_erasure_code_example_SOURCES = test/erasure-code/TestErasureCodeExample.cc
+noinst_HEADERS += test/erasure-code/ErasureCodeExample.h
+unittest_erasure_code_example_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+unittest_erasure_code_example_LDADD = $(LIBOSD) $(LIBCOMMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
+check_PROGRAMS += unittest_erasure_code_example
+
noinst_HEADERS += \
test/erasure-code/ceph_erasure_code_benchmark.h
--- /dev/null
+// -*- mode:C; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com>
+ *
+ * Author: Loic Dachary <loic@dachary.org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ */
+
+#include "include/stringify.h"
+#include "global/global_init.h"
+#include "ErasureCodeExample.h"
+#include "common/ceph_argparse.h"
+#include "global/global_context.h"
+#include "gtest/gtest.h"
+
+TEST(ErasureCodeExample, chunk_size)
+{
+ ErasureCodeExample example;
+ EXPECT_EQ(3u, example.get_chunk_count());
+ EXPECT_EQ(11u, example.get_chunk_size(20));
+}
+
+TEST(ErasureCodeExample, minimum_to_decode)
+{
+ ErasureCodeExample example;
+ set<int> available_chunks;
+ set<int> want_to_read;
+ want_to_read.insert(1);
+ {
+ set<int> minimum;
+ EXPECT_EQ(-EIO, example.minimum_to_decode(want_to_read,
+ available_chunks,
+ &minimum));
+ }
+ available_chunks.insert(0);
+ available_chunks.insert(2);
+ {
+ set<int> minimum;
+ EXPECT_EQ(0, example.minimum_to_decode(want_to_read,
+ available_chunks,
+ &minimum));
+ EXPECT_EQ(available_chunks, minimum);
+ EXPECT_EQ(2u, minimum.size());
+ EXPECT_EQ(1u, minimum.count(0));
+ EXPECT_EQ(1u, minimum.count(2));
+ }
+ {
+ set<int> minimum;
+ available_chunks.insert(1);
+ EXPECT_EQ(0, example.minimum_to_decode(want_to_read,
+ available_chunks,
+ &minimum));
+ EXPECT_EQ(1u, minimum.size());
+ EXPECT_EQ(1u, minimum.count(1));
+ }
+}
+
+TEST(ErasureCodeExample, minimum_to_decode_with_cost)
+{
+ ErasureCodeExample example;
+ map<int,int> available;
+ set<int> want_to_read;
+ want_to_read.insert(1);
+ {
+ set<int> minimum;
+ EXPECT_EQ(-EIO, example.minimum_to_decode_with_cost(want_to_read,
+ available,
+ &minimum));
+ }
+ available[0] = 1;
+ available[2] = 1;
+ {
+ set<int> minimum;
+ EXPECT_EQ(0, example.minimum_to_decode_with_cost(want_to_read,
+ available,
+ &minimum));
+ EXPECT_EQ(2u, minimum.size());
+ EXPECT_EQ(1u, minimum.count(0));
+ EXPECT_EQ(1u, minimum.count(2));
+ }
+ {
+ set<int> minimum;
+ available[1] = 1;
+ EXPECT_EQ(0, example.minimum_to_decode_with_cost(want_to_read,
+ available,
+ &minimum));
+ EXPECT_EQ(1u, minimum.size());
+ EXPECT_EQ(1u, minimum.count(1));
+ }
+ {
+ set<int> minimum;
+ available[1] = 2;
+ EXPECT_EQ(0, example.minimum_to_decode_with_cost(want_to_read,
+ available,
+ &minimum));
+ EXPECT_EQ(2u, minimum.size());
+ EXPECT_EQ(1u, minimum.count(0));
+ EXPECT_EQ(1u, minimum.count(2));
+ }
+}
+
+TEST(ErasureCodeExample, encode_decode)
+{
+ ErasureCodeExample example;
+
+ bufferlist in;
+ in.append("ABCDE");
+ set<int> want_to_encode;
+ for(unsigned int i = 0; i < example.get_chunk_count(); i++)
+ want_to_encode.insert(i);
+ map<int, bufferlist> encoded;
+ EXPECT_EQ(0, example.encode(want_to_encode, in, &encoded));
+ EXPECT_EQ(example.get_chunk_count(), encoded.size());
+ EXPECT_EQ(example.get_chunk_size(in.length()), encoded[0].length());
+ EXPECT_EQ('A', encoded[0][0]);
+ EXPECT_EQ('B', encoded[0][1]);
+ EXPECT_EQ('C', encoded[0][2]);
+ EXPECT_EQ('D', encoded[1][0]);
+ EXPECT_EQ('E', encoded[1][1]);
+ EXPECT_EQ('A'^'D', encoded[2][0]);
+ EXPECT_EQ('B'^'E', encoded[2][1]);
+ EXPECT_EQ('C'^0, encoded[2][2]);
+
+ // all chunks are available
+ {
+ int want_to_decode[] = { 0, 1 };
+ map<int, bufferlist> decoded;
+ EXPECT_EQ(0, example.decode(set<int>(want_to_decode, want_to_decode+2),
+ encoded,
+ &decoded));
+ EXPECT_EQ(2u, decoded.size());
+ EXPECT_EQ(3u, decoded[0].length());
+ EXPECT_EQ('A', decoded[0][0]);
+ EXPECT_EQ('B', decoded[0][1]);
+ EXPECT_EQ('C', decoded[0][2]);
+ EXPECT_EQ('D', decoded[1][0]);
+ EXPECT_EQ('E', decoded[1][1]);
+ }
+
+ // one chunk is missing
+ {
+ map<int, bufferlist> degraded = encoded;
+ degraded.erase(0);
+ EXPECT_EQ(2u, degraded.size());
+ int want_to_decode[] = { 0, 1 };
+ map<int, bufferlist> decoded;
+ EXPECT_EQ(0, example.decode(set<int>(want_to_decode, want_to_decode+2),
+ degraded,
+ &decoded));
+ EXPECT_EQ(2u, decoded.size());
+ EXPECT_EQ(3u, decoded[0].length());
+ EXPECT_EQ('A', decoded[0][0]);
+ EXPECT_EQ('B', decoded[0][1]);
+ EXPECT_EQ('C', decoded[0][2]);
+ EXPECT_EQ('D', decoded[1][0]);
+ EXPECT_EQ('E', decoded[1][1]);
+ }
+}
+
+TEST(ErasureCodeExample, decode)
+{
+ ErasureCodeExample example;
+
+#define LARGE_ENOUGH 2048
+ bufferptr in_ptr(buffer::create_page_aligned(LARGE_ENOUGH));
+ in_ptr.zero();
+ in_ptr.set_length(0);
+ const char *payload =
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
+ in_ptr.append(payload, strlen(payload));
+ bufferlist in;
+ in.push_front(in_ptr);
+ int want_to_encode[] = { 0, 1, 2 };
+ map<int, bufferlist> encoded;
+ EXPECT_EQ(0, example.encode(set<int>(want_to_encode, want_to_encode+3),
+ in,
+ &encoded));
+ EXPECT_EQ(3u, encoded.size());
+
+ // successfull decode
+ bufferlist out;
+ EXPECT_EQ(0, example.decode_concat(encoded, &out));
+ bufferlist usable;
+ usable.substr_of(out, 0, in.length());
+ EXPECT_TRUE(usable == in);
+
+ // cannot recover
+ map<int, bufferlist> degraded;
+ degraded[0] = encoded[0];
+ EXPECT_EQ(-ERANGE, example.decode_concat(degraded, &out));
+}
+
+TEST(ErasureCodeExample, create_ruleset)
+{
+ CrushWrapper *c = new CrushWrapper;
+ c->create();
+ c->set_type_name(2, "root");
+ c->set_type_name(1, "host");
+ c->set_type_name(0, "osd");
+
+ int rootno;
+ c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
+ 5, 0, NULL, NULL, &rootno);
+ c->set_item_name(rootno, "default");
+
+ map<string,string> loc;
+ loc["root"] = "default";
+
+ int num_host = 2;
+ int num_osd = 5;
+ int osd = 0;
+ for (int h=0; h<num_host; ++h) {
+ loc["host"] = string("host-") + stringify(h);
+ for (int o=0; o<num_osd; ++o, ++osd) {
+ c->insert_item(g_ceph_context, osd, 1.0, string("osd.") + stringify(osd), loc);
+ }
+ }
+
+ stringstream ss;
+ ErasureCodeExample example;
+ EXPECT_EQ(0, example.create_ruleset("myrule", *c, &ss));
+}
+
+int main(int argc, char **argv) {
+ vector<const char*> args;
+ argv_to_vec(argc, (const char **)argv, args);
+
+ global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+ common_init_finish(g_ceph_context);
+
+ ::testing::InitGoogleTest(&argc, argv);
+ return RUN_ALL_TESTS();
+}
+
+/*
+ * Local Variables:
+ * compile-command: "cd ../.. ;
+ * make -j4 &&
+ * make unittest_erasure_code_example &&
+ * valgrind --leak-check=full --tool=memcheck \
+ * ./unittest_erasure_code_example --gtest_filter=*.* \
+ * --log-to-stderr=true --debug-osd=20
+ * "
+ * End:
+ */
+
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com>
+ *
+ * Author: Loic Dachary <loic@dachary.org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ */
+
+#include <errno.h>
+
+#include "crush/CrushWrapper.h"
+#include "include/stringify.h"
+#include "global/global_init.h"
+#include "erasure-code/jerasure/ErasureCodeJerasure.h"
+#include "common/ceph_argparse.h"
+#include "global/global_context.h"
+#include "gtest/gtest.h"
+
+template <typename T>
+class ErasureCodeTest : public ::testing::Test {
+ public:
+};
+
+typedef ::testing::Types<
+ ErasureCodeJerasureReedSolomonVandermonde,
+ ErasureCodeJerasureReedSolomonRAID6,
+ ErasureCodeJerasureCauchyOrig,
+ ErasureCodeJerasureCauchyGood,
+ ErasureCodeJerasureLiberation,
+ ErasureCodeJerasureBlaumRoth,
+ ErasureCodeJerasureLiber8tion
+> JerasureTypes;
+TYPED_TEST_CASE(ErasureCodeTest, JerasureTypes);
+
+TYPED_TEST(ErasureCodeTest, encode_decode)
+{
+ TypeParam jerasure;
+ map<std::string,std::string> parameters;
+ parameters["erasure-code-k"] = "2";
+ parameters["erasure-code-m"] = "2";
+ parameters["erasure-code-w"] = "7";
+ parameters["erasure-code-packetsize"] = "8";
+ jerasure.init(parameters);
+
+#define LARGE_ENOUGH 2048
+ bufferptr in_ptr(buffer::create_page_aligned(LARGE_ENOUGH));
+ in_ptr.zero();
+ in_ptr.set_length(0);
+ const char *payload =
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
+ in_ptr.append(payload, strlen(payload));
+ bufferlist in;
+ in.push_front(in_ptr);
+ int want_to_encode[] = { 0, 1, 2, 3 };
+ map<int, bufferlist> encoded;
+ EXPECT_EQ(0, jerasure.encode(set<int>(want_to_encode, want_to_encode+4),
+ in,
+ &encoded));
+ EXPECT_EQ(4u, encoded.size());
+ unsigned length = encoded[0].length();
+ EXPECT_EQ(0, strncmp(encoded[0].c_str(), in.c_str(), length));
+ EXPECT_EQ(0, strncmp(encoded[1].c_str(), in.c_str() + length,
+ in.length() - length));
+
+
+ // all chunks are available
+ {
+ int want_to_decode[] = { 0, 1 };
+ map<int, bufferlist> decoded;
+ EXPECT_EQ(0, jerasure.decode(set<int>(want_to_decode, want_to_decode+2),
+ encoded,
+ &decoded));
+ // always decode all, regardless of want_to_decode
+ EXPECT_EQ(4u, decoded.size());
+ EXPECT_EQ(length, decoded[0].length());
+ EXPECT_EQ(0, strncmp(decoded[0].c_str(), in.c_str(), length));
+ EXPECT_EQ(0, strncmp(decoded[1].c_str(), in.c_str() + length,
+ in.length() - length));
+ }
+
+ // two chunks are missing
+ {
+ map<int, bufferlist> degraded = encoded;
+ degraded.erase(0);
+ degraded.erase(1);
+ EXPECT_EQ(2u, degraded.size());
+ int want_to_decode[] = { 0, 1 };
+ map<int, bufferlist> decoded;
+ EXPECT_EQ(0, jerasure.decode(set<int>(want_to_decode, want_to_decode+2),
+ degraded,
+ &decoded));
+ // always decode all, regardless of want_to_decode
+ EXPECT_EQ(4u, decoded.size());
+ EXPECT_EQ(length, decoded[0].length());
+ EXPECT_EQ(0, strncmp(decoded[0].c_str(), in.c_str(), length));
+ EXPECT_EQ(0, strncmp(decoded[1].c_str(), in.c_str() + length,
+ in.length() - length));
+ }
+}
+
+TYPED_TEST(ErasureCodeTest, minimum_to_decode)
+{
+ TypeParam jerasure;
+ map<std::string,std::string> parameters;
+ parameters["erasure-code-k"] = "2";
+ parameters["erasure-code-m"] = "2";
+ parameters["erasure-code-w"] = "7";
+ parameters["erasure-code-packetsize"] = "8";
+ jerasure.init(parameters);
+
+ //
+ // If trying to read nothing, the minimum is empty.
+ //
+ {
+ set<int> want_to_read;
+ set<int> available_chunks;
+ set<int> minimum;
+
+ EXPECT_EQ(0, jerasure.minimum_to_decode(want_to_read,
+ available_chunks,
+ &minimum));
+ EXPECT_TRUE(minimum.empty());
+ }
+ //
+ // There is no way to read a chunk if none are available.
+ //
+ {
+ set<int> want_to_read;
+ set<int> available_chunks;
+ set<int> minimum;
+
+ want_to_read.insert(0);
+
+ EXPECT_EQ(-EIO, jerasure.minimum_to_decode(want_to_read,
+ available_chunks,
+ &minimum));
+ }
+ //
+ // Reading a subset of the available chunks is always possible.
+ //
+ {
+ set<int> want_to_read;
+ set<int> available_chunks;
+ set<int> minimum;
+
+ want_to_read.insert(0);
+ available_chunks.insert(0);
+
+ EXPECT_EQ(0, jerasure.minimum_to_decode(want_to_read,
+ available_chunks,
+ &minimum));
+ EXPECT_EQ(want_to_read, minimum);
+ }
+ //
+ // There is no way to read a missing chunk if there is less than k
+ // chunks available.
+ //
+ {
+ set<int> want_to_read;
+ set<int> available_chunks;
+ set<int> minimum;
+
+ want_to_read.insert(0);
+ want_to_read.insert(1);
+ available_chunks.insert(0);
+
+ EXPECT_EQ(-EIO, jerasure.minimum_to_decode(want_to_read,
+ available_chunks,
+ &minimum));
+ }
+ //
+ // When chunks are not available, the minimum can be made of any
+ // chunks. For instance, to read 1 and 3 below the minimum could be
+ // 2 and 3 which may seem better because it contains one of the
+ // chunks to be read. But it won't be more efficient than retrieving
+ // 0 and 2 instead because, in both cases, the decode function will
+ // need to run the same recovery operation and use the same amount
+ // of CPU and memory.
+ //
+ {
+ set<int> want_to_read;
+ set<int> available_chunks;
+ set<int> minimum;
+
+ want_to_read.insert(1);
+ want_to_read.insert(3);
+ available_chunks.insert(0);
+ available_chunks.insert(2);
+ available_chunks.insert(3);
+
+ EXPECT_EQ(0, jerasure.minimum_to_decode(want_to_read,
+ available_chunks,
+ &minimum));
+ EXPECT_EQ(2u, minimum.size());
+ EXPECT_EQ(0u, minimum.count(3));
+ }
+}
+
+TEST(ErasureCodeTest, encode)
+{
+ ErasureCodeJerasureReedSolomonVandermonde jerasure;
+ map<std::string,std::string> parameters;
+ parameters["erasure-code-k"] = "2";
+ parameters["erasure-code-m"] = "2";
+ parameters["erasure-code-w"] = "8";
+ jerasure.init(parameters);
+
+ unsigned alignment = jerasure.get_alignment();
+ {
+ //
+ // When the input bufferlist needs to be padded because
+ // it is not properly aligned, it is padded with zeros.
+ //
+ bufferlist in;
+ map<int,bufferlist> encoded;
+ int want_to_encode[] = { 0, 1, 2, 3 };
+ int trail_length = 10;
+ in.append(string(alignment + trail_length, 'X'));
+ EXPECT_EQ(0, jerasure.encode(set<int>(want_to_encode, want_to_encode+4),
+ in,
+ &encoded));
+ EXPECT_EQ(4u, encoded.size());
+ for(int i = 0; i < 4; i++)
+ EXPECT_EQ(alignment, encoded[i].length());
+ char *last_chunk = encoded[1].c_str();
+ EXPECT_EQ('X', last_chunk[0]);
+ EXPECT_EQ('\0', last_chunk[trail_length]);
+ }
+
+ {
+ //
+ // When only the first chunk is required, the encoded map only
+ // contains the first chunk. Although the jerasure encode
+ // internally allocated a buffer because of padding requirements
+ // and also computes the coding chunks, they are released before
+ // the return of the method, as shown when running the tests thru
+ // valgrind (there is no leak).
+ //
+ bufferlist in;
+ map<int,bufferlist> encoded;
+ set<int> want_to_encode;
+ want_to_encode.insert(0);
+ int trail_length = 10;
+ in.append(string(alignment + trail_length, 'X'));
+ EXPECT_EQ(0, jerasure.encode(want_to_encode, in, &encoded));
+ EXPECT_EQ(1u, encoded.size());
+ EXPECT_EQ(alignment, encoded[0].length());
+ }
+}
+
+TEST(ErasureCodeTest, create_ruleset)
+{
+ CrushWrapper *c = new CrushWrapper;
+ c->create();
+ int root_type = 2;
+ c->set_type_name(root_type, "root");
+ int host_type = 1;
+ c->set_type_name(host_type, "host");
+ int osd_type = 0;
+ c->set_type_name(osd_type, "osd");
+
+ int rootno;
+ c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
+ root_type, 0, NULL, NULL, &rootno);
+ c->set_item_name(rootno, "default");
+
+ map<string,string> loc;
+ loc["root"] = "default";
+
+ int num_host = 4;
+ int num_osd = 5;
+ int osd = 0;
+ for (int h=0; h<num_host; ++h) {
+ loc["host"] = string("host-") + stringify(h);
+ for (int o=0; o<num_osd; ++o, ++osd) {
+ c->insert_item(g_ceph_context, osd, 1.0, string("osd.") + stringify(osd), loc);
+ }
+ }
+
+ {
+ stringstream ss;
+ ErasureCodeJerasureReedSolomonVandermonde jerasure;
+ map<std::string,std::string> parameters;
+ parameters["erasure-code-k"] = "2";
+ parameters["erasure-code-m"] = "2";
+ parameters["erasure-code-w"] = "8";
+ jerasure.init(parameters);
+ int ruleset = jerasure.create_ruleset("myrule", *c, &ss);
+ EXPECT_EQ(0, ruleset);
+ EXPECT_EQ(-EEXIST, jerasure.create_ruleset("myrule", *c, &ss));
+ //
+ // the minimum that is expected from the created ruleset is to
+ // successfully map get_chunk_count() devices from the crushmap,
+ // at least once.
+ //
+ vector<__u32> weight(c->get_max_devices(), 0x10000);
+ vector<int> out;
+ int x = 0;
+ c->do_rule(ruleset, x, out, jerasure.get_chunk_count(), weight);
+ ASSERT_EQ(out.size(), jerasure.get_chunk_count());
+ for (unsigned i=0; i<out.size(); ++i)
+ ASSERT_NE(CRUSH_ITEM_NONE, out[i]);
+ }
+ {
+ stringstream ss;
+ ErasureCodeJerasureReedSolomonVandermonde jerasure;
+ map<std::string,std::string> parameters;
+ parameters["erasure-code-k"] = "2";
+ parameters["erasure-code-m"] = "2";
+ parameters["erasure-code-w"] = "8";
+ parameters["erasure-code-ruleset-root"] = "BAD";
+ jerasure.init(parameters);
+ EXPECT_EQ(-ENOENT, jerasure.create_ruleset("otherrule", *c, &ss));
+ EXPECT_EQ("root item BAD does not exist", ss.str());
+ }
+ {
+ stringstream ss;
+ ErasureCodeJerasureReedSolomonVandermonde jerasure;
+ map<std::string,std::string> parameters;
+ parameters["erasure-code-k"] = "2";
+ parameters["erasure-code-m"] = "2";
+ parameters["erasure-code-w"] = "8";
+ parameters["erasure-code-ruleset-failure-domain"] = "WORSE";
+ jerasure.init(parameters);
+ EXPECT_EQ(-EINVAL, jerasure.create_ruleset("otherrule", *c, &ss));
+ EXPECT_EQ("unknown type WORSE", ss.str());
+ }
+}
+
+int main(int argc, char **argv)
+{
+ vector<const char*> args;
+ argv_to_vec(argc, (const char **)argv, args);
+
+ global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+ common_init_finish(g_ceph_context);
+
+ ::testing::InitGoogleTest(&argc, argv);
+ return RUN_ALL_TESTS();
+}
+
+/*
+ * Local Variables:
+ * compile-command: "cd ../.. ; make -j4 &&
+ * make unittest_erasure_code_jerasure &&
+ * valgrind --tool=memcheck --leak-check=full \
+ * ./unittest_erasure_code_jerasure \
+ * --gtest_filter=*.* --log-to-stderr=true --debug-osd=20"
+ * End:
+ */
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com>
+ *
+ * Author: Loic Dachary <loic@dachary.org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ */
+
+#include <errno.h>
+#include <signal.h>
+#include "common/Thread.h"
+#include "global/global_init.h"
+#include "erasure-code/ErasureCodePlugin.h"
+#include "common/ceph_argparse.h"
+#include "global/global_context.h"
+#include "gtest/gtest.h"
+
+class ErasureCodePluginRegistryTest : public ::testing::Test {
+protected:
+
+ class Thread_factory : public Thread {
+ public:
+ virtual void *entry() {
+ map<std::string,std::string> parameters;
+ parameters["erasure-code-directory"] = ".libs";
+ ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance();
+ ErasureCodeInterfaceRef erasure_code;
+ instance.factory("hangs", parameters, &erasure_code);
+ return NULL;
+ }
+ };
+
+};
+
+TEST_F(ErasureCodePluginRegistryTest, factory_mutex) {
+ ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance();
+
+ EXPECT_TRUE(instance.lock.TryLock());
+ instance.lock.Unlock();
+
+ //
+ // Test that the loading of a plugin is protected by a mutex.
+ //
+ useconds_t delay = 0;
+ const useconds_t DELAY_MAX = 20 * 1000 * 1000;
+ Thread_factory sleep_forever;
+ sleep_forever.create();
+ do {
+ cout << "Trying (1) with delay " << delay << "us\n";
+ if (delay > 0)
+ usleep(delay);
+ if (!instance.loading)
+ delay = ( delay + 1 ) * 2;
+ } while(!instance.loading && delay < DELAY_MAX);
+ ASSERT_TRUE(delay < DELAY_MAX);
+
+ EXPECT_FALSE(instance.lock.TryLock());
+
+ EXPECT_EQ(0, pthread_cancel(sleep_forever.get_thread_id()));
+ EXPECT_EQ(0, sleep_forever.join());
+}
+
+TEST_F(ErasureCodePluginRegistryTest, all)
+{
+ map<std::string,std::string> parameters;
+ parameters["erasure-code-directory"] = ".libs";
+ ErasureCodeInterfaceRef erasure_code;
+ ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance();
+ EXPECT_FALSE(erasure_code);
+ EXPECT_EQ(-EIO, instance.factory("invalid", parameters, &erasure_code));
+ EXPECT_FALSE(erasure_code);
+ EXPECT_EQ(-ENOENT, instance.factory("missing_entry_point", parameters,
+ &erasure_code));
+ EXPECT_FALSE(erasure_code);
+ EXPECT_EQ(-ESRCH, instance.factory("fail_to_initialize", parameters,
+ &erasure_code));
+ EXPECT_FALSE(erasure_code);
+ EXPECT_EQ(-EBADF, instance.factory("fail_to_register", parameters,
+ &erasure_code));
+ EXPECT_FALSE(erasure_code);
+ EXPECT_EQ(0, instance.factory("example", parameters, &erasure_code));
+ EXPECT_TRUE(erasure_code);
+ ErasureCodePlugin *plugin = 0;
+ EXPECT_EQ(-EEXIST, instance.load("example", parameters, &plugin));
+}
+
+int main(int argc, char **argv) {
+ vector<const char*> args;
+ argv_to_vec(argc, (const char **)argv, args);
+
+ global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+ common_init_finish(g_ceph_context);
+
+ ::testing::InitGoogleTest(&argc, argv);
+ return RUN_ALL_TESTS();
+}
+
+// Local Variables:
+// compile-command: "cd ../.. ; make -j4 && make unittest_erasure_code_plugin && valgrind --leak-check=full --tool=memcheck ./unittest_erasure_code_plugin --gtest_filter=*.* --log-to-stderr=true --debug-osd=20"
+// End:
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com>
+ *
+ * Author: Loic Dachary <loic@dachary.org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ */
+
+#include <errno.h>
+#include "global/global_init.h"
+#include "erasure-code/ErasureCodePlugin.h"
+#include "common/ceph_argparse.h"
+#include "global/global_context.h"
+#include "gtest/gtest.h"
+
+TEST(ErasureCodePlugin, factory)
+{
+ ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance();
+ map<std::string,std::string> parameters;
+ parameters["erasure-code-directory"] = ".libs";
+ {
+ ErasureCodeInterfaceRef erasure_code;
+ EXPECT_FALSE(erasure_code);
+ EXPECT_EQ(-ENOENT, instance.factory("jerasure", parameters, &erasure_code));
+ EXPECT_FALSE(erasure_code);
+ }
+ const char *techniques[] = {
+ "reed_sol_van",
+ "reed_sol_r6_op",
+ "cauchy_orig",
+ "cauchy_good",
+ "liberation",
+ "blaum_roth",
+ "liber8tion",
+ 0
+ };
+ for(const char **technique = techniques; *technique; technique++) {
+ ErasureCodeInterfaceRef erasure_code;
+ parameters["erasure-code-technique"] = *technique;
+ EXPECT_FALSE(erasure_code);
+ EXPECT_EQ(0, instance.factory("jerasure", parameters, &erasure_code));
+ EXPECT_TRUE(erasure_code);
+ }
+}
+
+int main(int argc, char **argv)
+{
+ vector<const char*> args;
+ argv_to_vec(argc, (const char **)argv, args);
+
+ global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+ common_init_finish(g_ceph_context);
+
+ ::testing::InitGoogleTest(&argc, argv);
+ return RUN_ALL_TESTS();
+}
+
+/*
+ * Local Variables:
+ * compile-command: "cd ../.. ; make -j4 &&
+ * make unittest_erasure_code_plugin_jerasure &&
+ * valgrind --tool=memcheck ./unittest_erasure_code_plugin_jerasure \
+ * --gtest_filter=*.* --log-to-stderr=true --debug-osd=20"
+ * End:
+ */
+
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com>
- *
- * Author: Loic Dachary <loic@dachary.org>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- */
-
-#ifndef CEPH_ERASURE_CODE_EXAMPLE_H
-#define CEPH_ERASURE_CODE_EXAMPLE_H
-
-#include <unistd.h>
-#include <errno.h>
-#include <algorithm>
-#include <sstream>
-
-#include "crush/CrushWrapper.h"
-#include "osd/osd_types.h"
-#include "erasure-code/ErasureCodeInterface.h"
-
-#define FIRST_DATA_CHUNK 0
-#define SECOND_DATA_CHUNK 1
-#define DATA_CHUNKS 2u
-
-#define CODING_CHUNK 2
-#define CODING_CHUNKS 1u
-
-#define MINIMUM_TO_RECOVER 2u
-
-class ErasureCodeExample : public ErasureCodeInterface {
-public:
- virtual ~ErasureCodeExample() {}
-
- virtual int create_ruleset(const string &name,
- CrushWrapper &crush,
- ostream *ss) const {
- return crush.add_simple_ruleset(name, "default", "host",
- "indep", pg_pool_t::TYPE_ERASURE, ss);
- }
-
- virtual int minimum_to_decode(const set<int> &want_to_read,
- const set<int> &available_chunks,
- set<int> *minimum) {
- if (includes(available_chunks.begin(), available_chunks.end(),
- want_to_read.begin(), want_to_read.end())) {
- *minimum = want_to_read;
- return 0;
- } else if (available_chunks.size() >= MINIMUM_TO_RECOVER) {
- *minimum = available_chunks;
- return 0;
- } else {
- return -EIO;
- }
- }
-
- virtual int minimum_to_decode_with_cost(const set<int> &want_to_read,
- const map<int, int> &available,
- set<int> *minimum) {
- //
- // If one chunk is more expensive to fetch than the others,
- // recover it instead. For instance, if the cost reflects the
- // time it takes for a chunk to be retrieved from a remote
- // OSD and if CPU is cheap, it could make sense to recover
- // instead of fetching the chunk.
- //
- map<int, int> c2c(available);
- if (c2c.size() > DATA_CHUNKS) {
- if (c2c[FIRST_DATA_CHUNK] > c2c[SECOND_DATA_CHUNK] &&
- c2c[FIRST_DATA_CHUNK] > c2c[CODING_CHUNK])
- c2c.erase(FIRST_DATA_CHUNK);
- else if(c2c[SECOND_DATA_CHUNK] > c2c[FIRST_DATA_CHUNK] &&
- c2c[SECOND_DATA_CHUNK] > c2c[CODING_CHUNK])
- c2c.erase(SECOND_DATA_CHUNK);
- else if(c2c[CODING_CHUNK] > c2c[FIRST_DATA_CHUNK] &&
- c2c[CODING_CHUNK] > c2c[SECOND_DATA_CHUNK])
- c2c.erase(CODING_CHUNK);
- }
- set <int> available_chunks;
- for (map<int, int>::const_iterator i = c2c.begin();
- i != c2c.end();
- ++i)
- available_chunks.insert(i->first);
- return minimum_to_decode(want_to_read, available_chunks, minimum);
- }
-
- virtual unsigned int get_chunk_count() const {
- return DATA_CHUNKS + CODING_CHUNKS;
- }
-
- virtual unsigned int get_data_chunk_count() const {
- return DATA_CHUNKS;
- }
-
- virtual unsigned int get_chunk_size(unsigned int object_size) const {
- return ( object_size / DATA_CHUNKS ) + 1;
- }
-
- virtual int encode(const set<int> &want_to_encode,
- const bufferlist &in,
- map<int, bufferlist> *encoded) {
- //
- // make sure all data chunks have the same length, allocating
- // padding if necessary.
- //
- unsigned int chunk_length = get_chunk_size(in.length());
- bufferlist out(in);
- unsigned int width = get_chunk_count() * get_chunk_size(in.length());
- bufferptr pad(width - in.length());
- pad.zero(0, get_data_chunk_count());
- out.push_back(pad);
- //
- // compute the coding chunk with first chunk ^ second chunk
- //
- char *p = out.c_str();
- for (unsigned i = 0; i < chunk_length; i++)
- p[i + CODING_CHUNK * chunk_length] =
- p[i + FIRST_DATA_CHUNK * chunk_length] ^
- p[i + SECOND_DATA_CHUNK * chunk_length];
- //
- // populate the bufferlist with bufferptr pointing
- // to chunk boundaries
- //
- const bufferptr ptr = out.buffers().front();
- for (set<int>::iterator j = want_to_encode.begin();
- j != want_to_encode.end();
- ++j) {
- bufferptr chunk(ptr, (*j) * chunk_length, chunk_length);
- (*encoded)[*j].push_front(chunk);
- }
- return 0;
- }
-
- virtual int decode(const set<int> &want_to_read,
- const map<int, bufferlist> &chunks,
- map<int, bufferlist> *decoded) {
- //
- // All chunks have the same size
- //
- unsigned chunk_length = (*chunks.begin()).second.length();
- for (set<int>::iterator i = want_to_read.begin();
- i != want_to_read.end();
- ++i) {
- if (chunks.find(*i) != chunks.end()) {
- //
- // If the chunk is available, just copy the bufferptr pointer
- // to the decoded argument.
- //
- (*decoded)[*i] = chunks.find(*i)->second;
- } else if(chunks.size() != 2) {
- //
- // If a chunk is missing and there are not enough chunks
- // to recover, abort.
- //
- return -ERANGE;
- } else {
- //
- // No matter what the missing chunk is, XOR of the other
- // two recovers it.
- //
- map<int, bufferlist>::const_iterator k = chunks.begin();
- const char *a = k->second.buffers().front().c_str();
- ++k;
- const char *b = k->second.buffers().front().c_str();
- bufferptr chunk(chunk_length);
- char *c = chunk.c_str();
- for (unsigned j = 0; j < chunk_length; j++) {
- c[j] = a[j] ^ b[j];
- }
- (*decoded)[*i].push_front(chunk);
- }
- }
- return 0;
- }
-};
-
-#endif
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com>
- *
- * Author: Loic Dachary <loic@dachary.org>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- */
-
-#include <unistd.h>
-
-#include "erasure-code/ErasureCodePlugin.h"
-#include "ErasureCodeExample.h"
-
-class ErasureCodePluginExample : public ErasureCodePlugin {
-public:
- virtual int factory(const map<std::string,std::string> ¶meters,
- ErasureCodeInterfaceRef *erasure_code)
- {
- *erasure_code = ErasureCodeInterfaceRef(new ErasureCodeExample());
- return 0;
- }
-};
-
-int __erasure_code_init(char *plugin_name)
-{
- ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance();
- return instance.add(plugin_name, new ErasureCodePluginExample());
-}
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com>
- *
- * Author: Loic Dachary <loic@dachary.org>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- */
-
-#include <errno.h>
-#include "erasure-code/ErasureCodePlugin.h"
-
-int __erasure_code_init(char *plugin_name)
-{
- return -ESRCH;
-}
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com>
- *
- * Author: Loic Dachary <loic@dachary.org>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- */
-
-#include "erasure-code/ErasureCodePlugin.h"
-
-int __erasure_code_init(char *plugin_name)
-{
- return 0;
-}
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com>
- *
- * Author: Loic Dachary <loic@dachary.org>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- */
-
-#include <unistd.h>
-#include "erasure-code/ErasureCodePlugin.h"
-
-int __erasure_code_init(char *plugin_name)
-{
- sleep(1000);
- return 0;
-}
+++ /dev/null
-// missing int __erasure_code_init(char *plugin_name) {}
-
-// avoid warnings about library containing no symbols
-int __this_is_an_used_variable_to_avoid_warnings;
+++ /dev/null
-// -*- mode:C; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com>
- *
- * Author: Loic Dachary <loic@dachary.org>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- */
-
-#include "include/stringify.h"
-#include "global/global_init.h"
-#include "ErasureCodeExample.h"
-#include "common/ceph_argparse.h"
-#include "global/global_context.h"
-#include "gtest/gtest.h"
-
-TEST(ErasureCodeExample, chunk_size)
-{
- ErasureCodeExample example;
- EXPECT_EQ(3u, example.get_chunk_count());
- EXPECT_EQ(11u, example.get_chunk_size(20));
-}
-
-TEST(ErasureCodeExample, minimum_to_decode)
-{
- ErasureCodeExample example;
- set<int> available_chunks;
- set<int> want_to_read;
- want_to_read.insert(1);
- {
- set<int> minimum;
- EXPECT_EQ(-EIO, example.minimum_to_decode(want_to_read,
- available_chunks,
- &minimum));
- }
- available_chunks.insert(0);
- available_chunks.insert(2);
- {
- set<int> minimum;
- EXPECT_EQ(0, example.minimum_to_decode(want_to_read,
- available_chunks,
- &minimum));
- EXPECT_EQ(available_chunks, minimum);
- EXPECT_EQ(2u, minimum.size());
- EXPECT_EQ(1u, minimum.count(0));
- EXPECT_EQ(1u, minimum.count(2));
- }
- {
- set<int> minimum;
- available_chunks.insert(1);
- EXPECT_EQ(0, example.minimum_to_decode(want_to_read,
- available_chunks,
- &minimum));
- EXPECT_EQ(1u, minimum.size());
- EXPECT_EQ(1u, minimum.count(1));
- }
-}
-
-TEST(ErasureCodeExample, minimum_to_decode_with_cost)
-{
- ErasureCodeExample example;
- map<int,int> available;
- set<int> want_to_read;
- want_to_read.insert(1);
- {
- set<int> minimum;
- EXPECT_EQ(-EIO, example.minimum_to_decode_with_cost(want_to_read,
- available,
- &minimum));
- }
- available[0] = 1;
- available[2] = 1;
- {
- set<int> minimum;
- EXPECT_EQ(0, example.minimum_to_decode_with_cost(want_to_read,
- available,
- &minimum));
- EXPECT_EQ(2u, minimum.size());
- EXPECT_EQ(1u, minimum.count(0));
- EXPECT_EQ(1u, minimum.count(2));
- }
- {
- set<int> minimum;
- available[1] = 1;
- EXPECT_EQ(0, example.minimum_to_decode_with_cost(want_to_read,
- available,
- &minimum));
- EXPECT_EQ(1u, minimum.size());
- EXPECT_EQ(1u, minimum.count(1));
- }
- {
- set<int> minimum;
- available[1] = 2;
- EXPECT_EQ(0, example.minimum_to_decode_with_cost(want_to_read,
- available,
- &minimum));
- EXPECT_EQ(2u, minimum.size());
- EXPECT_EQ(1u, minimum.count(0));
- EXPECT_EQ(1u, minimum.count(2));
- }
-}
-
-TEST(ErasureCodeExample, encode_decode)
-{
- ErasureCodeExample example;
-
- bufferlist in;
- in.append("ABCDE");
- set<int> want_to_encode;
- for(unsigned int i = 0; i < example.get_chunk_count(); i++)
- want_to_encode.insert(i);
- map<int, bufferlist> encoded;
- EXPECT_EQ(0, example.encode(want_to_encode, in, &encoded));
- EXPECT_EQ(example.get_chunk_count(), encoded.size());
- EXPECT_EQ(example.get_chunk_size(in.length()), encoded[0].length());
- EXPECT_EQ('A', encoded[0][0]);
- EXPECT_EQ('B', encoded[0][1]);
- EXPECT_EQ('C', encoded[0][2]);
- EXPECT_EQ('D', encoded[1][0]);
- EXPECT_EQ('E', encoded[1][1]);
- EXPECT_EQ('A'^'D', encoded[2][0]);
- EXPECT_EQ('B'^'E', encoded[2][1]);
- EXPECT_EQ('C'^0, encoded[2][2]);
-
- // all chunks are available
- {
- int want_to_decode[] = { 0, 1 };
- map<int, bufferlist> decoded;
- EXPECT_EQ(0, example.decode(set<int>(want_to_decode, want_to_decode+2),
- encoded,
- &decoded));
- EXPECT_EQ(2u, decoded.size());
- EXPECT_EQ(3u, decoded[0].length());
- EXPECT_EQ('A', decoded[0][0]);
- EXPECT_EQ('B', decoded[0][1]);
- EXPECT_EQ('C', decoded[0][2]);
- EXPECT_EQ('D', decoded[1][0]);
- EXPECT_EQ('E', decoded[1][1]);
- }
-
- // one chunk is missing
- {
- map<int, bufferlist> degraded = encoded;
- degraded.erase(0);
- EXPECT_EQ(2u, degraded.size());
- int want_to_decode[] = { 0, 1 };
- map<int, bufferlist> decoded;
- EXPECT_EQ(0, example.decode(set<int>(want_to_decode, want_to_decode+2),
- degraded,
- &decoded));
- EXPECT_EQ(2u, decoded.size());
- EXPECT_EQ(3u, decoded[0].length());
- EXPECT_EQ('A', decoded[0][0]);
- EXPECT_EQ('B', decoded[0][1]);
- EXPECT_EQ('C', decoded[0][2]);
- EXPECT_EQ('D', decoded[1][0]);
- EXPECT_EQ('E', decoded[1][1]);
- }
-}
-
-TEST(ErasureCodeExample, decode)
-{
- ErasureCodeExample example;
-
-#define LARGE_ENOUGH 2048
- bufferptr in_ptr(buffer::create_page_aligned(LARGE_ENOUGH));
- in_ptr.zero();
- in_ptr.set_length(0);
- const char *payload =
- "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
- "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
- "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
- "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
- "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
- in_ptr.append(payload, strlen(payload));
- bufferlist in;
- in.push_front(in_ptr);
- int want_to_encode[] = { 0, 1, 2 };
- map<int, bufferlist> encoded;
- EXPECT_EQ(0, example.encode(set<int>(want_to_encode, want_to_encode+3),
- in,
- &encoded));
- EXPECT_EQ(3u, encoded.size());
-
- // successfull decode
- bufferlist out;
- EXPECT_EQ(0, example.decode_concat(encoded, &out));
- bufferlist usable;
- usable.substr_of(out, 0, in.length());
- EXPECT_TRUE(usable == in);
-
- // cannot recover
- map<int, bufferlist> degraded;
- degraded[0] = encoded[0];
- EXPECT_EQ(-ERANGE, example.decode_concat(degraded, &out));
-}
-
-TEST(ErasureCodeExample, create_ruleset)
-{
- CrushWrapper *c = new CrushWrapper;
- c->create();
- c->set_type_name(2, "root");
- c->set_type_name(1, "host");
- c->set_type_name(0, "osd");
-
- int rootno;
- c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
- 5, 0, NULL, NULL, &rootno);
- c->set_item_name(rootno, "default");
-
- map<string,string> loc;
- loc["root"] = "default";
-
- int num_host = 2;
- int num_osd = 5;
- int osd = 0;
- for (int h=0; h<num_host; ++h) {
- loc["host"] = string("host-") + stringify(h);
- for (int o=0; o<num_osd; ++o, ++osd) {
- c->insert_item(g_ceph_context, osd, 1.0, string("osd.") + stringify(osd), loc);
- }
- }
-
- stringstream ss;
- ErasureCodeExample example;
- EXPECT_EQ(0, example.create_ruleset("myrule", *c, &ss));
-}
-
-int main(int argc, char **argv) {
- vector<const char*> args;
- argv_to_vec(argc, (const char **)argv, args);
-
- global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
- common_init_finish(g_ceph_context);
-
- ::testing::InitGoogleTest(&argc, argv);
- return RUN_ALL_TESTS();
-}
-
-/*
- * Local Variables:
- * compile-command: "cd ../.. ;
- * make -j4 &&
- * make unittest_erasure_code_example &&
- * valgrind --leak-check=full --tool=memcheck \
- * ./unittest_erasure_code_example --gtest_filter=*.* \
- * --log-to-stderr=true --debug-osd=20
- * "
- * End:
- */
-
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com>
- *
- * Author: Loic Dachary <loic@dachary.org>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- */
-
-#include <errno.h>
-
-#include "crush/CrushWrapper.h"
-#include "include/stringify.h"
-#include "global/global_init.h"
-#include "erasure-code/jerasure/ErasureCodeJerasure.h"
-#include "common/ceph_argparse.h"
-#include "global/global_context.h"
-#include "gtest/gtest.h"
-
-template <typename T>
-class ErasureCodeTest : public ::testing::Test {
- public:
-};
-
-typedef ::testing::Types<
- ErasureCodeJerasureReedSolomonVandermonde,
- ErasureCodeJerasureReedSolomonRAID6,
- ErasureCodeJerasureCauchyOrig,
- ErasureCodeJerasureCauchyGood,
- ErasureCodeJerasureLiberation,
- ErasureCodeJerasureBlaumRoth,
- ErasureCodeJerasureLiber8tion
-> JerasureTypes;
-TYPED_TEST_CASE(ErasureCodeTest, JerasureTypes);
-
-TYPED_TEST(ErasureCodeTest, encode_decode)
-{
- TypeParam jerasure;
- map<std::string,std::string> parameters;
- parameters["erasure-code-k"] = "2";
- parameters["erasure-code-m"] = "2";
- parameters["erasure-code-w"] = "7";
- parameters["erasure-code-packetsize"] = "8";
- jerasure.init(parameters);
-
-#define LARGE_ENOUGH 2048
- bufferptr in_ptr(buffer::create_page_aligned(LARGE_ENOUGH));
- in_ptr.zero();
- in_ptr.set_length(0);
- const char *payload =
- "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
- "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
- "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
- "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
- "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
- in_ptr.append(payload, strlen(payload));
- bufferlist in;
- in.push_front(in_ptr);
- int want_to_encode[] = { 0, 1, 2, 3 };
- map<int, bufferlist> encoded;
- EXPECT_EQ(0, jerasure.encode(set<int>(want_to_encode, want_to_encode+4),
- in,
- &encoded));
- EXPECT_EQ(4u, encoded.size());
- unsigned length = encoded[0].length();
- EXPECT_EQ(0, strncmp(encoded[0].c_str(), in.c_str(), length));
- EXPECT_EQ(0, strncmp(encoded[1].c_str(), in.c_str() + length,
- in.length() - length));
-
-
- // all chunks are available
- {
- int want_to_decode[] = { 0, 1 };
- map<int, bufferlist> decoded;
- EXPECT_EQ(0, jerasure.decode(set<int>(want_to_decode, want_to_decode+2),
- encoded,
- &decoded));
- // always decode all, regardless of want_to_decode
- EXPECT_EQ(4u, decoded.size());
- EXPECT_EQ(length, decoded[0].length());
- EXPECT_EQ(0, strncmp(decoded[0].c_str(), in.c_str(), length));
- EXPECT_EQ(0, strncmp(decoded[1].c_str(), in.c_str() + length,
- in.length() - length));
- }
-
- // two chunks are missing
- {
- map<int, bufferlist> degraded = encoded;
- degraded.erase(0);
- degraded.erase(1);
- EXPECT_EQ(2u, degraded.size());
- int want_to_decode[] = { 0, 1 };
- map<int, bufferlist> decoded;
- EXPECT_EQ(0, jerasure.decode(set<int>(want_to_decode, want_to_decode+2),
- degraded,
- &decoded));
- // always decode all, regardless of want_to_decode
- EXPECT_EQ(4u, decoded.size());
- EXPECT_EQ(length, decoded[0].length());
- EXPECT_EQ(0, strncmp(decoded[0].c_str(), in.c_str(), length));
- EXPECT_EQ(0, strncmp(decoded[1].c_str(), in.c_str() + length,
- in.length() - length));
- }
-}
-
-TYPED_TEST(ErasureCodeTest, minimum_to_decode)
-{
- TypeParam jerasure;
- map<std::string,std::string> parameters;
- parameters["erasure-code-k"] = "2";
- parameters["erasure-code-m"] = "2";
- parameters["erasure-code-w"] = "7";
- parameters["erasure-code-packetsize"] = "8";
- jerasure.init(parameters);
-
- //
- // If trying to read nothing, the minimum is empty.
- //
- {
- set<int> want_to_read;
- set<int> available_chunks;
- set<int> minimum;
-
- EXPECT_EQ(0, jerasure.minimum_to_decode(want_to_read,
- available_chunks,
- &minimum));
- EXPECT_TRUE(minimum.empty());
- }
- //
- // There is no way to read a chunk if none are available.
- //
- {
- set<int> want_to_read;
- set<int> available_chunks;
- set<int> minimum;
-
- want_to_read.insert(0);
-
- EXPECT_EQ(-EIO, jerasure.minimum_to_decode(want_to_read,
- available_chunks,
- &minimum));
- }
- //
- // Reading a subset of the available chunks is always possible.
- //
- {
- set<int> want_to_read;
- set<int> available_chunks;
- set<int> minimum;
-
- want_to_read.insert(0);
- available_chunks.insert(0);
-
- EXPECT_EQ(0, jerasure.minimum_to_decode(want_to_read,
- available_chunks,
- &minimum));
- EXPECT_EQ(want_to_read, minimum);
- }
- //
- // There is no way to read a missing chunk if there is less than k
- // chunks available.
- //
- {
- set<int> want_to_read;
- set<int> available_chunks;
- set<int> minimum;
-
- want_to_read.insert(0);
- want_to_read.insert(1);
- available_chunks.insert(0);
-
- EXPECT_EQ(-EIO, jerasure.minimum_to_decode(want_to_read,
- available_chunks,
- &minimum));
- }
- //
- // When chunks are not available, the minimum can be made of any
- // chunks. For instance, to read 1 and 3 below the minimum could be
- // 2 and 3 which may seem better because it contains one of the
- // chunks to be read. But it won't be more efficient than retrieving
- // 0 and 2 instead because, in both cases, the decode function will
- // need to run the same recovery operation and use the same amount
- // of CPU and memory.
- //
- {
- set<int> want_to_read;
- set<int> available_chunks;
- set<int> minimum;
-
- want_to_read.insert(1);
- want_to_read.insert(3);
- available_chunks.insert(0);
- available_chunks.insert(2);
- available_chunks.insert(3);
-
- EXPECT_EQ(0, jerasure.minimum_to_decode(want_to_read,
- available_chunks,
- &minimum));
- EXPECT_EQ(2u, minimum.size());
- EXPECT_EQ(0u, minimum.count(3));
- }
-}
-
-TEST(ErasureCodeTest, encode)
-{
- ErasureCodeJerasureReedSolomonVandermonde jerasure;
- map<std::string,std::string> parameters;
- parameters["erasure-code-k"] = "2";
- parameters["erasure-code-m"] = "2";
- parameters["erasure-code-w"] = "8";
- jerasure.init(parameters);
-
- unsigned alignment = jerasure.get_alignment();
- {
- //
- // When the input bufferlist needs to be padded because
- // it is not properly aligned, it is padded with zeros.
- //
- bufferlist in;
- map<int,bufferlist> encoded;
- int want_to_encode[] = { 0, 1, 2, 3 };
- int trail_length = 10;
- in.append(string(alignment + trail_length, 'X'));
- EXPECT_EQ(0, jerasure.encode(set<int>(want_to_encode, want_to_encode+4),
- in,
- &encoded));
- EXPECT_EQ(4u, encoded.size());
- for(int i = 0; i < 4; i++)
- EXPECT_EQ(alignment, encoded[i].length());
- char *last_chunk = encoded[1].c_str();
- EXPECT_EQ('X', last_chunk[0]);
- EXPECT_EQ('\0', last_chunk[trail_length]);
- }
-
- {
- //
- // When only the first chunk is required, the encoded map only
- // contains the first chunk. Although the jerasure encode
- // internally allocated a buffer because of padding requirements
- // and also computes the coding chunks, they are released before
- // the return of the method, as shown when running the tests thru
- // valgrind (there is no leak).
- //
- bufferlist in;
- map<int,bufferlist> encoded;
- set<int> want_to_encode;
- want_to_encode.insert(0);
- int trail_length = 10;
- in.append(string(alignment + trail_length, 'X'));
- EXPECT_EQ(0, jerasure.encode(want_to_encode, in, &encoded));
- EXPECT_EQ(1u, encoded.size());
- EXPECT_EQ(alignment, encoded[0].length());
- }
-}
-
-TEST(ErasureCodeTest, create_ruleset)
-{
- CrushWrapper *c = new CrushWrapper;
- c->create();
- int root_type = 2;
- c->set_type_name(root_type, "root");
- int host_type = 1;
- c->set_type_name(host_type, "host");
- int osd_type = 0;
- c->set_type_name(osd_type, "osd");
-
- int rootno;
- c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
- root_type, 0, NULL, NULL, &rootno);
- c->set_item_name(rootno, "default");
-
- map<string,string> loc;
- loc["root"] = "default";
-
- int num_host = 4;
- int num_osd = 5;
- int osd = 0;
- for (int h=0; h<num_host; ++h) {
- loc["host"] = string("host-") + stringify(h);
- for (int o=0; o<num_osd; ++o, ++osd) {
- c->insert_item(g_ceph_context, osd, 1.0, string("osd.") + stringify(osd), loc);
- }
- }
-
- {
- stringstream ss;
- ErasureCodeJerasureReedSolomonVandermonde jerasure;
- map<std::string,std::string> parameters;
- parameters["erasure-code-k"] = "2";
- parameters["erasure-code-m"] = "2";
- parameters["erasure-code-w"] = "8";
- jerasure.init(parameters);
- int ruleset = jerasure.create_ruleset("myrule", *c, &ss);
- EXPECT_EQ(0, ruleset);
- EXPECT_EQ(-EEXIST, jerasure.create_ruleset("myrule", *c, &ss));
- //
- // the minimum that is expected from the created ruleset is to
- // successfully map get_chunk_count() devices from the crushmap,
- // at least once.
- //
- vector<__u32> weight(c->get_max_devices(), 0x10000);
- vector<int> out;
- int x = 0;
- c->do_rule(ruleset, x, out, jerasure.get_chunk_count(), weight);
- ASSERT_EQ(out.size(), jerasure.get_chunk_count());
- for (unsigned i=0; i<out.size(); ++i)
- ASSERT_NE(CRUSH_ITEM_NONE, out[i]);
- }
- {
- stringstream ss;
- ErasureCodeJerasureReedSolomonVandermonde jerasure;
- map<std::string,std::string> parameters;
- parameters["erasure-code-k"] = "2";
- parameters["erasure-code-m"] = "2";
- parameters["erasure-code-w"] = "8";
- parameters["erasure-code-ruleset-root"] = "BAD";
- jerasure.init(parameters);
- EXPECT_EQ(-ENOENT, jerasure.create_ruleset("otherrule", *c, &ss));
- EXPECT_EQ("root item BAD does not exist", ss.str());
- }
- {
- stringstream ss;
- ErasureCodeJerasureReedSolomonVandermonde jerasure;
- map<std::string,std::string> parameters;
- parameters["erasure-code-k"] = "2";
- parameters["erasure-code-m"] = "2";
- parameters["erasure-code-w"] = "8";
- parameters["erasure-code-ruleset-failure-domain"] = "WORSE";
- jerasure.init(parameters);
- EXPECT_EQ(-EINVAL, jerasure.create_ruleset("otherrule", *c, &ss));
- EXPECT_EQ("unknown type WORSE", ss.str());
- }
-}
-
-int main(int argc, char **argv)
-{
- vector<const char*> args;
- argv_to_vec(argc, (const char **)argv, args);
-
- global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
- common_init_finish(g_ceph_context);
-
- ::testing::InitGoogleTest(&argc, argv);
- return RUN_ALL_TESTS();
-}
-
-/*
- * Local Variables:
- * compile-command: "cd ../.. ; make -j4 &&
- * make unittest_erasure_code_jerasure &&
- * valgrind --tool=memcheck --leak-check=full \
- * ./unittest_erasure_code_jerasure \
- * --gtest_filter=*.* --log-to-stderr=true --debug-osd=20"
- * End:
- */
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com>
- *
- * Author: Loic Dachary <loic@dachary.org>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- */
-
-#include <errno.h>
-#include <signal.h>
-#include "common/Thread.h"
-#include "global/global_init.h"
-#include "erasure-code/ErasureCodePlugin.h"
-#include "common/ceph_argparse.h"
-#include "global/global_context.h"
-#include "gtest/gtest.h"
-
-class ErasureCodePluginRegistryTest : public ::testing::Test {
-protected:
-
- class Thread_factory : public Thread {
- public:
- virtual void *entry() {
- map<std::string,std::string> parameters;
- parameters["erasure-code-directory"] = ".libs";
- ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance();
- ErasureCodeInterfaceRef erasure_code;
- instance.factory("hangs", parameters, &erasure_code);
- return NULL;
- }
- };
-
-};
-
-TEST_F(ErasureCodePluginRegistryTest, factory_mutex) {
- ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance();
-
- EXPECT_TRUE(instance.lock.TryLock());
- instance.lock.Unlock();
-
- //
- // Test that the loading of a plugin is protected by a mutex.
- //
- useconds_t delay = 0;
- const useconds_t DELAY_MAX = 20 * 1000 * 1000;
- Thread_factory sleep_forever;
- sleep_forever.create();
- do {
- cout << "Trying (1) with delay " << delay << "us\n";
- if (delay > 0)
- usleep(delay);
- if (!instance.loading)
- delay = ( delay + 1 ) * 2;
- } while(!instance.loading && delay < DELAY_MAX);
- ASSERT_TRUE(delay < DELAY_MAX);
-
- EXPECT_FALSE(instance.lock.TryLock());
-
- EXPECT_EQ(0, pthread_cancel(sleep_forever.get_thread_id()));
- EXPECT_EQ(0, sleep_forever.join());
-}
-
-TEST_F(ErasureCodePluginRegistryTest, all)
-{
- map<std::string,std::string> parameters;
- parameters["erasure-code-directory"] = ".libs";
- ErasureCodeInterfaceRef erasure_code;
- ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance();
- EXPECT_FALSE(erasure_code);
- EXPECT_EQ(-EIO, instance.factory("invalid", parameters, &erasure_code));
- EXPECT_FALSE(erasure_code);
- EXPECT_EQ(-ENOENT, instance.factory("missing_entry_point", parameters,
- &erasure_code));
- EXPECT_FALSE(erasure_code);
- EXPECT_EQ(-ESRCH, instance.factory("fail_to_initialize", parameters,
- &erasure_code));
- EXPECT_FALSE(erasure_code);
- EXPECT_EQ(-EBADF, instance.factory("fail_to_register", parameters,
- &erasure_code));
- EXPECT_FALSE(erasure_code);
- EXPECT_EQ(0, instance.factory("example", parameters, &erasure_code));
- EXPECT_TRUE(erasure_code);
- ErasureCodePlugin *plugin = 0;
- EXPECT_EQ(-EEXIST, instance.load("example", parameters, &plugin));
-}
-
-int main(int argc, char **argv) {
- vector<const char*> args;
- argv_to_vec(argc, (const char **)argv, args);
-
- global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
- common_init_finish(g_ceph_context);
-
- ::testing::InitGoogleTest(&argc, argv);
- return RUN_ALL_TESTS();
-}
-
-// Local Variables:
-// compile-command: "cd ../.. ; make -j4 && make unittest_erasure_code_plugin && valgrind --leak-check=full --tool=memcheck ./unittest_erasure_code_plugin --gtest_filter=*.* --log-to-stderr=true --debug-osd=20"
-// End:
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com>
- *
- * Author: Loic Dachary <loic@dachary.org>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- */
-
-#include <errno.h>
-#include "global/global_init.h"
-#include "erasure-code/ErasureCodePlugin.h"
-#include "common/ceph_argparse.h"
-#include "global/global_context.h"
-#include "gtest/gtest.h"
-
-TEST(ErasureCodePlugin, factory)
-{
- ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance();
- map<std::string,std::string> parameters;
- parameters["erasure-code-directory"] = ".libs";
- {
- ErasureCodeInterfaceRef erasure_code;
- EXPECT_FALSE(erasure_code);
- EXPECT_EQ(-ENOENT, instance.factory("jerasure", parameters, &erasure_code));
- EXPECT_FALSE(erasure_code);
- }
- const char *techniques[] = {
- "reed_sol_van",
- "reed_sol_r6_op",
- "cauchy_orig",
- "cauchy_good",
- "liberation",
- "blaum_roth",
- "liber8tion",
- 0
- };
- for(const char **technique = techniques; *technique; technique++) {
- ErasureCodeInterfaceRef erasure_code;
- parameters["erasure-code-technique"] = *technique;
- EXPECT_FALSE(erasure_code);
- EXPECT_EQ(0, instance.factory("jerasure", parameters, &erasure_code));
- EXPECT_TRUE(erasure_code);
- }
-}
-
-int main(int argc, char **argv)
-{
- vector<const char*> args;
- argv_to_vec(argc, (const char **)argv, args);
-
- global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
- common_init_finish(g_ceph_context);
-
- ::testing::InitGoogleTest(&argc, argv);
- return RUN_ALL_TESTS();
-}
-
-/*
- * Local Variables:
- * compile-command: "cd ../.. ; make -j4 &&
- * make unittest_erasure_code_plugin_jerasure &&
- * valgrind --tool=memcheck ./unittest_erasure_code_plugin_jerasure \
- * --gtest_filter=*.* --log-to-stderr=true --debug-osd=20"
- * End:
- */
-
projects / ceph.git / commitdiff