From: t-miyamae Date: Thu, 29 Jan 2015 13:29:13 +0000 (+0900) Subject: mSHEC r44 initial commit X-Git-Tag: v0.93~78^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=refs%2Fpull%2F3534%2Fhead;p=ceph.git mSHEC r44 initial commit Signed-off-by: Takeshi Miyamae --- diff --git a/src/erasure-code/Makefile.am b/src/erasure-code/Makefile.am index e42e1260013..3390bacfa7f 100644 --- a/src/erasure-code/Makefile.am +++ b/src/erasure-code/Makefile.am @@ -5,6 +5,7 @@ erasure_codelib_LTLIBRARIES = include erasure-code/jerasure/Makefile.am include erasure-code/lrc/Makefile.am +include erasure-code/shec/Makefile.am if WITH_BETTER_YASM_ELF64 include erasure-code/isa/Makefile.am diff --git a/src/erasure-code/shec/ErasureCodePluginShec.cc b/src/erasure-code/shec/ErasureCodePluginShec.cc new file mode 100644 index 00000000000..75a64a70046 --- /dev/null +++ b/src/erasure-code/shec/ErasureCodePluginShec.cc @@ -0,0 +1,93 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2014 FUJITSU LIMITED + * Copyright (C) 2013,2014 Cloudwatt + * Copyright (C) 2014 Red Hat + * + * Author: Takanori Nakao + * Author: Takeshi Miyamae + * Author: Loic Dachary + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + */ + +#include "ceph_ver.h" +#include "common/debug.h" +#include "erasure-code/ErasureCodePlugin.h" +#include "ErasureCodeShecTableCache.h" +#include "ErasureCodeShec.h" + +#define dout_subsys ceph_subsys_osd +#undef dout_prefix +#define dout_prefix _prefix(_dout) + +static ostream& _prefix(std::ostream* _dout) +{ + return *_dout << "ErasureCodePluginShec: "; +} + +class ErasureCodePluginShec : public ErasureCodePlugin { +public: + ErasureCodeShecTableCache tcache; + + virtual int factory(const map ¶meters, + ErasureCodeInterfaceRef *erasure_code) { + ErasureCodeShec *interface; + std::string t = "multiple"; + + if (parameters.find("technique") != parameters.end()){ + t = parameters.find("technique")->second; + } + + if (t == "single"){ + interface = new ErasureCodeShecReedSolomonVandermonde(tcache, ErasureCodeShec::SINGLE); + } else if (t == "multiple"){ + interface = new ErasureCodeShecReedSolomonVandermonde(tcache, ErasureCodeShec::MULTIPLE); + } else { + derr << "technique=" << t << " is not a valid coding technique. " + << " Choose one of the following: " + << "single, multiple" + << dendl; + return -ENOENT; + } + int err = interface->init(parameters); + if (err) { + return err; + } + *erasure_code = ErasureCodeInterfaceRef(interface); + + dout(10) << "ErasureCodePluginShec: factory() completed" << dendl; + + return 0; + } +}; + +extern "C" { +#include "jerasure/include/galois.h" + +extern gf_t *gfp_array[]; +extern int gfp_is_composite[]; +} + +const char *__erasure_code_version() { return CEPH_GIT_NICE_VER; } + +int __erasure_code_init(char *plugin_name, char *directory = (char *)"") +{ + ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance(); + int w[] = { 8, 16, 32 }; + for(int i = 0; i < 3; i++) { + int r = galois_init_default_field(w[i]); + if (r) { + derr << "failed to gf_init_easy(" << w[i] << ")" << dendl; + return -r; + } + } + return instance.add(plugin_name, new ErasureCodePluginShec()); +} diff --git a/src/erasure-code/shec/ErasureCodeShec.cc b/src/erasure-code/shec/ErasureCodeShec.cc new file mode 100644 index 00000000000..f775715ec8e --- /dev/null +++ b/src/erasure-code/shec/ErasureCodeShec.cc @@ -0,0 +1,404 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2014 FUJITSU LIMITED + * Copyright (C) 2013,2014 Cloudwatt + * Copyright (C) 2014 Red Hat + * + * Author: Takanori Nakao + * Author: Takeshi Miyamae + * Author: Loic Dachary + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + */ + +#include +#include +#include "common/debug.h" +#include "ErasureCodeShec.h" +#include "crush/CrushWrapper.h" +#include "osd/osd_types.h" +#include "shec.h" +extern "C" { +#include "jerasure/include/jerasure.h" +#include "jerasure/include/galois.h" +} + +#define dout_subsys ceph_subsys_osd +#undef dout_prefix +#define dout_prefix _prefix(_dout) + +#define talloc(type, num) (type *) malloc(sizeof(type)*(num)) + +static ostream& _prefix(std::ostream* _dout) +{ + return *_dout << "ErasureCodeShec: "; +} + +int ErasureCodeShec::create_ruleset(const string &name, + CrushWrapper &crush, + ostream *ss) const +{ + int ruleid = crush.add_simple_ruleset(name, ruleset_root, ruleset_failure_domain, + "indep", pg_pool_t::TYPE_ERASURE, ss); + if (ruleid < 0) { + return ruleid; + } else { + return crush.get_rule_mask_ruleset(ruleid); + } +} + +int ErasureCodeShec::init(const map ¶meters) +{ + dout(10) << "technique=" << technique << dendl; + map::const_iterator parameter; + parameter = parameters.find("ruleset-root"); + if (parameter != parameters.end()) + ruleset_root = parameter->second; + parameter = parameters.find("ruleset-failure-domain"); + if (parameter != parameters.end()) + ruleset_failure_domain = parameter->second; + int err = parse(parameters); + if (err) { + return err; + } + prepare(); + return 0; +} + +unsigned int ErasureCodeShec::get_chunk_size(unsigned int object_size) const +{ + unsigned alignment = get_alignment(); + unsigned tail = object_size % alignment; + unsigned padded_length = object_size + ( tail ? ( alignment - tail ) : 0 ); + + assert(padded_length % k == 0); + return padded_length / k; +} + +int ErasureCodeShec::minimum_to_decode(const set &want_to_decode, + const set &available_chunks, + set *minimum_chunks) +{ + int erased[k + m]; + int avails[k + m]; + int minimum[k + m]; + int dm_ids[k]; + + if (!minimum_chunks) return -EINVAL; + + for (set::iterator it = available_chunks.begin(); it != available_chunks.end(); it++){ + if (*it < 0 || k+m <= *it) return -EINVAL; + } + + if (includes(available_chunks.begin(), available_chunks.end(), + want_to_decode.begin(), want_to_decode.end())) { + *minimum_chunks = want_to_decode; + } else { + for (int i = 0; i < k + m; i++) { + erased[i] = 0; + if (available_chunks.find(i) == available_chunks.end()) { + if (want_to_decode.count(i) > 0) { + erased[i] = 1; + } + avails[i] = 0; + } else { + avails[i] = 1; + } + } + + if (shec_make_decoding_matrix(true, k, m, w, matrix, erased, + avails, 0, dm_ids, minimum) < 0) { + return -EIO; + } + + for (int i = 0; i < k + m; i++) { + if (minimum[i] == 1) minimum_chunks->insert(i); + } + } + + return 0; +} + +int ErasureCodeShec::minimum_to_decode_with_cost(const set &want_to_decode, + const map &available, + set *minimum_chunks) +{ + set available_chunks; + + for (map::const_iterator i = available.begin(); + i != available.end(); + ++i) + available_chunks.insert(i->first); + + return minimum_to_decode(want_to_decode, available_chunks, minimum_chunks); +} + +int ErasureCodeShec::encode(const set &want_to_encode, + const bufferlist &in, + map *encoded) +{ + unsigned int k = get_data_chunk_count(); + unsigned int m = get_chunk_count() - k; + bufferlist out; + + if (!encoded || !encoded->empty()){ + return -EINVAL; + } + + int err = encode_prepare(in, *encoded); + if (err) + return err; + encode_chunks(want_to_encode, encoded); + for (unsigned int i = 0; i < k + m; i++) { + if (want_to_encode.count(i) == 0) + encoded->erase(i); + } + return 0; +} + +int ErasureCodeShec::encode_chunks(const set &want_to_encode, + map *encoded) +{ + char *chunks[k + m]; + for (int i = 0; i < k + m; i++){ + chunks[i] = (*encoded)[i].c_str(); + } + shec_encode(&chunks[0], &chunks[k], (*encoded)[0].length()); + return 0; +} + +int ErasureCodeShec::decode(const set &want_to_read, + const map &chunks, + map *decoded) +{ + vector have; + + if (!decoded || !decoded->empty()){ + return -EINVAL; + } + + have.reserve(chunks.size()); + for (map::const_iterator i = chunks.begin(); + i != chunks.end(); + ++i) { + have.push_back(i->first); + } + if (includes( + have.begin(), have.end(), want_to_read.begin(), want_to_read.end())) { + for (set::iterator i = want_to_read.begin(); + i != want_to_read.end(); + ++i) { + (*decoded)[*i] = chunks.find(*i)->second; + } + return 0; + } + unsigned int k = get_data_chunk_count(); + unsigned int m = get_chunk_count() - k; + unsigned blocksize = (*chunks.begin()).second.length(); + for (unsigned int i = 0; i < k + m; i++) { + if (chunks.find(i) == chunks.end()) { + bufferptr ptr(buffer::create_aligned(blocksize, SIMD_ALIGN)); + (*decoded)[i].push_front(ptr); + } else { + (*decoded)[i] = chunks.find(i)->second; + (*decoded)[i].rebuild_aligned(SIMD_ALIGN); + } + } + return decode_chunks(want_to_read, chunks, decoded); +} + +int ErasureCodeShec::decode_chunks(const set &want_to_read, + const map &chunks, + map *decoded) +{ + unsigned blocksize = (*chunks.begin()).second.length(); + int erased[k + m]; + int erased_count = 0; + int avails[k + m]; + char *data[k]; + char *coding[m]; + + for (int i = 0; i < k + m; i++) { + erased[i] = 0; + if (chunks.find(i) == chunks.end()) { + if (want_to_read.count(i) > 0) { + erased[i] = 1; + erased_count++; + } + avails[i] = 0; + } else { + (*decoded)[i] = chunks.find(i)->second; + avails[i] = 1; + } + if (i < k) + data[i] = (*decoded)[i].c_str(); + else + coding[i - k] = (*decoded)[i].c_str(); + } + + if (erased_count > 0) { + return shec_decode(erased, avails, data, coding, blocksize); + } else { + return 0; + } +} + +// +// ErasureCodeShecReedSolomonVandermonde +// + +void ErasureCodeShecReedSolomonVandermonde::shec_encode(char **data, + char **coding, + int blocksize) +{ + jerasure_matrix_encode(k, m, w, matrix, data, coding, blocksize); +} + +int ErasureCodeShecReedSolomonVandermonde::shec_decode(int *erased, + int *avails, + char **data, + char **coding, + int blocksize) +{ + return shec_matrix_decode(k, m, w, matrix, + erased, avails, data, coding, blocksize); +} + +unsigned ErasureCodeShecReedSolomonVandermonde::get_alignment() const +{ + return k*w*sizeof(int); +} + +int ErasureCodeShecReedSolomonVandermonde::parse(const map ¶meters) +{ + int err = 0; + // k, m, c + if (parameters.find("k") == parameters.end() && + parameters.find("m") == parameters.end() && + parameters.find("c") == parameters.end()){ + dout(10) << "(k, m, c) default to " << "(" << DEFAULT_K + << ", " << DEFAULT_M << ", " << DEFAULT_C << ")" << dendl; + k = DEFAULT_K; m = DEFAULT_M; c = DEFAULT_C; + } else if (parameters.find("k") == parameters.end() || + parameters.find("m") == parameters.end() || + parameters.find("c") == parameters.end()){ + dout(10) << "(k, m, c) must be choosed" << dendl; + err = -EINVAL; + } else { + std::string err_k, err_m, err_c, value_k, value_m, value_c; + value_k = parameters.find("k")->second; + value_m = parameters.find("m")->second; + value_c = parameters.find("c")->second; + k = strict_strtol(value_k.c_str(), 10, &err_k); + m = strict_strtol(value_m.c_str(), 10, &err_m); + c = strict_strtol(value_c.c_str(), 10, &err_c); + + if (!err_k.empty() || !err_m.empty() || !err_c.empty()){ + if (!err_k.empty()){ + derr << "could not convert k=" << value_k << "to int" << dendl; + } else if (!err_m.empty()){ + derr << "could not convert m=" << value_m << "to int" << dendl; + } else if (!err_c.empty()){ + derr << "could not convert c=" << value_c << "to int" << dendl; + } + err = -EINVAL; + } else if (k <= 0){ + derr << "k=" << k + << " must be a positive number" << dendl; + err = -EINVAL; + } else if (m <= 0){ + derr << "m=" << m + << " must be a positive number" << dendl; + err = -EINVAL; + } else if (c <= 0){ + derr << "c=" << c + << " must be a positive number" << dendl; + err = -EINVAL; + } else if (m < c){ + derr << "c=" << c + << " must be less than or equal to m=" << m << dendl; + err = -EINVAL; + } else if (k > 12){ + derr << "k=" << k + << " must be less than or equal to 12" << dendl; + err = -EINVAL; + } else if (k+m > 20){ + derr << "k+m=" << k+m + << " must be less than or equal to 20" << dendl; + err = -EINVAL; + } else if (ksecond; + w = strict_strtol(value_w.c_str(), 10, &err_w); + + if (!err_w.empty()){ + derr << "could not convert w=" << value_w << "to int" << dendl; + dout(10) << "w default to " << DEFAULT_W << dendl; + w = DEFAULT_W; + + } else if (w != 8 && w != 16 && w != 32) { + derr << "w=" << w + << " must be one of {8, 16, 32}" << dendl; + dout(10) << "w default to " << DEFAULT_W << dendl; + w = DEFAULT_W; + + } else { + dout(10) << "w set to " << w << dendl; + } + } + return 0; +} + +void ErasureCodeShecReedSolomonVandermonde::prepare() +{ + // setup shared encoding table + int** p_enc_table = + tcache.getEncodingTable(technique, k, m, c, w); + + if (!*p_enc_table) { + dout(10) << "[ cache tables ] creating coeff for k=" << + k << " m=" << m << " c=" << c << " w=" << w << dendl; + + matrix = shec_reedsolomon_coding_matrix(k, m, c, w, technique); + + // either our new created table is stored or if it has been + // created in the meanwhile the locally allocated table will be + // freed by setEncodingTable + matrix = tcache.setEncodingTable(technique, k, m, c, w, matrix); + } else { + matrix = *p_enc_table; + } + + dout(10) << " [ technique ] = " << + ((technique == MULTIPLE) ? "multiple" : "single") << dendl; + + assert((technique == SINGLE) || (technique == MULTIPLE)); + +} diff --git a/src/erasure-code/shec/ErasureCodeShec.h b/src/erasure-code/shec/ErasureCodeShec.h new file mode 100644 index 00000000000..4e63c1bbc90 --- /dev/null +++ b/src/erasure-code/shec/ErasureCodeShec.h @@ -0,0 +1,139 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2014 FUJITSU LIMITED + * Copyright (C) 2013, 2014 Cloudwatt + * Copyright (C) 2014 Red Hat + * + * Author: Takanori Nakao + * Author: Takeshi Miyamae + * Author: Loic Dachary + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + */ + +#ifndef CEPH_ERASURE_CODE_SHEC_H +#define CEPH_ERASURE_CODE_SHEC_H + +#include "common/Mutex.h" +#include "erasure-code/ErasureCode.h" +#include "ErasureCodeShecTableCache.h" +#include + +class ErasureCodeShec : public ErasureCode { + +public: + enum { + MULTIPLE = 0, + SINGLE = 1 + }; + + ErasureCodeShecTableCache &tcache; + int k; + int DEFAULT_K; + int m; + int DEFAULT_M; + int c; + int DEFAULT_C; + int w; + int DEFAULT_W; + int technique; + string ruleset_root; + string ruleset_failure_domain; + int *matrix; + + ErasureCodeShec(const int _technique, + ErasureCodeShecTableCache &_tcache) : + tcache(_tcache), + DEFAULT_K(2), + DEFAULT_M(1), + DEFAULT_C(1), + DEFAULT_W(8), + technique(_technique), + ruleset_root("default"), + ruleset_failure_domain("host"), + matrix(0) + {} + + virtual ~ErasureCodeShec() {} + + virtual int create_ruleset(const string &name, + CrushWrapper &crush, + ostream *ss) const; + + virtual unsigned int get_chunk_count() const { + return k + m; + } + + virtual unsigned int get_data_chunk_count() const { + return k; + } + + virtual unsigned int get_chunk_size(unsigned int object_size) const; + + virtual int minimum_to_decode(const set &want_to_decode, + const set &available_chunks, + set *minimum); + + virtual int minimum_to_decode_with_cost(const set &want_to_decode, + const map &available, + set *minimum); + + virtual int encode(const set &want_to_encode, + const bufferlist &in, + map *encoded); + virtual int encode_chunks(const set &want_to_encode, + map *encoded); + + virtual int decode(const set &want_to_read, + const map &chunks, + map *decoded); + virtual int decode_chunks(const set &want_to_read, + const map &chunks, + map *decoded); + + int init(const map ¶meters); + virtual void shec_encode(char **data, + char **coding, + int blocksize) = 0; + virtual int shec_decode(int *erasures, + int *avails, + char **data, + char **coding, + int blocksize) = 0; + virtual unsigned get_alignment() const = 0; + virtual int parse(const map ¶meters) = 0; + virtual void prepare() = 0; +}; + +class ErasureCodeShecReedSolomonVandermonde : public ErasureCodeShec { +public: + + ErasureCodeShecReedSolomonVandermonde(ErasureCodeShecTableCache &_tcache, + int technique = MULTIPLE) : + ErasureCodeShec(technique, _tcache) + {} + + virtual ~ErasureCodeShecReedSolomonVandermonde() { + } + + virtual void shec_encode(char **data, + char **coding, + int blocksize); + virtual int shec_decode(int *erasures, + int *avails, + char **data, + char **coding, + int blocksize); + virtual unsigned get_alignment() const; + virtual int parse(const map ¶meters); + virtual void prepare(); +}; + +#endif diff --git a/src/erasure-code/shec/ErasureCodeShecTableCache.cc b/src/erasure-code/shec/ErasureCodeShecTableCache.cc new file mode 100644 index 00000000000..8fb64b2519c --- /dev/null +++ b/src/erasure-code/shec/ErasureCodeShecTableCache.cc @@ -0,0 +1,97 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2014 FUJITSU LIMITED + * Copyright (C) 2014 CERN (Switzerland) + * + * Author: Takanori Nakao + * Author: Takeshi Miyamae + * Author: Andreas-Joachim Peters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + */ + +// ----------------------------------------------------------------------------- +#include "ErasureCodeShecTableCache.h" +#include "ErasureCodeShec.h" +#include "common/debug.h" +// ----------------------------------------------------------------------------- + +ErasureCodeShecTableCache::~ErasureCodeShecTableCache() +{ + Mutex::Locker lock(codec_tables_guard); + + codec_technique_tables_t::const_iterator ttables_it; + codec_tables_t::const_iterator tables_it; + codec_tables_t_::const_iterator tables_it_; + codec_tables_t__::const_iterator tables_it__; + codec_table_t::const_iterator table_it; + + // clean-up all allocated tables + + for (ttables_it = encoding_table.begin(); ttables_it != encoding_table.end(); ++ttables_it) { + for (tables_it = ttables_it->second.begin(); tables_it != ttables_it->second.end(); ++tables_it) { + for (tables_it_ = tables_it->second.begin(); tables_it_ != tables_it->second.end(); ++tables_it_) { + for (tables_it__ = tables_it_->second.begin(); tables_it__ != tables_it_->second.end(); ++tables_it__) { + for (table_it = tables_it__->second.begin(); table_it != tables_it__->second.end(); ++table_it) { + if (table_it->second) { + if (*(table_it->second)) { + delete *(table_it->second); + } + delete table_it->second; + } + } + } + } + } + } +} + +int** +ErasureCodeShecTableCache::getEncodingTable(int technique, int k, int m, int c, int w) +{ + Mutex::Locker lock(codec_tables_guard); + return getEncodingTableNoLock(technique,k,m,c,w); +} + +// ----------------------------------------------------------------------------- + +int** +ErasureCodeShecTableCache::getEncodingTableNoLock(int technique, int k, int m, int c, int w) +{ + // create a pointer to store an encoding table address + if (!encoding_table[technique][k][m][c][w]) { + encoding_table[technique][k][m][c][w] = new (int*); + *encoding_table[technique][k][m][c][w] = 0; + } + return encoding_table[technique][k][m][c][w]; +} + +int* +ErasureCodeShecTableCache::setEncodingTable(int technique, int k, int m, int c, int w, int* ec_in_table) +{ + Mutex::Locker lock(codec_tables_guard); + int** ec_out_table = getEncodingTableNoLock(technique, k, m, c, w); + if (*ec_out_table) { + // somebody might have deposited this table in the meanwhile, so clean + // the input table and return the stored one + free (ec_in_table); + return *ec_out_table; + } else { + // we store the provided input table and return this one + *encoding_table[technique][k][m][c][w] = ec_in_table; + return ec_in_table; + } +} + +Mutex* +ErasureCodeShecTableCache::getLock() +{ + return &codec_tables_guard; +} diff --git a/src/erasure-code/shec/ErasureCodeShecTableCache.h b/src/erasure-code/shec/ErasureCodeShecTableCache.h new file mode 100644 index 00000000000..21f65bdfa6f --- /dev/null +++ b/src/erasure-code/shec/ErasureCodeShecTableCache.h @@ -0,0 +1,66 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2014 FUJITSU LIMITED + * Copyright (C) 2014 CERN (Switzerland) + * + * Author: Takanori Nakao + * Author: Takeshi Miyamae + * Author: Andreas-Joachim Peters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + */ + +#ifndef CEPH_ERASURE_CODE_SHEC_TABLE_CACHE_H +#define CEPH_ERASURE_CODE_SHEC_TABLE_CACHE_H + +// ----------------------------------------------------------------------------- +#include "common/Mutex.h" +#include "erasure-code/ErasureCodeInterface.h" +// ----------------------------------------------------------------------------- +#include +// ----------------------------------------------------------------------------- + +class ErasureCodeShecTableCache { + // --------------------------------------------------------------------------- + // This class implements a table cache for encoding and decoding matrices. + // Encoding matrices are shared for the same (k,m,c,w) combination. + // --------------------------------------------------------------------------- + + public: + + typedef std::pair::iterator, bufferptr> lru_entry_t; + typedef std::map< int, int** > codec_table_t; + typedef std::map< int, codec_table_t > codec_tables_t__; + typedef std::map< int, codec_tables_t__ > codec_tables_t_; + typedef std::map< int, codec_tables_t_ > codec_tables_t; + typedef std::map< int, codec_tables_t > codec_technique_tables_t; + // int** matrix = codec_technique_tables_t[technique][k][m][c][w] + + ErasureCodeShecTableCache() : + codec_tables_guard("shec-lru-cache") + { + } + + virtual ~ErasureCodeShecTableCache(); + + Mutex codec_tables_guard; // mutex used to protect modifications in encoding/decoding table maps + + int** getEncodingTable(int technique, int k, int m, int c, int w); + int** getEncodingTableNoLock(int technique, int k, int m, int c, int w); + int* setEncodingTable(int technique, int k, int m, int c, int w, int*); + + private: + codec_technique_tables_t encoding_table; // encoding coefficients accessed via table[technique][k][m] + + Mutex* getLock(); + +}; + +#endif diff --git a/src/erasure-code/shec/Makefile.am b/src/erasure-code/shec/Makefile.am new file mode 100644 index 00000000000..148f1c30d0d --- /dev/null +++ b/src/erasure-code/shec/Makefile.am @@ -0,0 +1,61 @@ +# SHEC plugin + +libec_shec_la_SOURCES = \ + erasure-code/ErasureCode.cc \ + erasure-code/shec/ErasureCodePluginShec.cc \ + erasure-code/shec/ErasureCodeShec.cc \ + erasure-code/shec/ErasureCodeShecTableCache.cc \ + erasure-code/shec/shec.cc \ + erasure-code/shec/determinant.c \ + erasure-code/jerasure/jerasure/src/cauchy.c \ + erasure-code/jerasure/jerasure/src/galois.c \ + erasure-code/jerasure/jerasure/src/jerasure.c \ + erasure-code/jerasure/jerasure/src/liberation.c \ + erasure-code/jerasure/jerasure/src/reed_sol.c \ + erasure-code/jerasure/gf-complete/src/gf_wgen.c \ + erasure-code/jerasure/gf-complete/src/gf_method.c \ + erasure-code/jerasure/gf-complete/src/gf_w16.c \ + erasure-code/jerasure/gf-complete/src/gf.c \ + erasure-code/jerasure/gf-complete/src/gf_w32.c \ + erasure-code/jerasure/gf-complete/src/gf_w64.c \ + erasure-code/jerasure/gf-complete/src/gf_w128.c \ + erasure-code/jerasure/gf-complete/src/gf_general.c \ + erasure-code/jerasure/gf-complete/src/gf_w4.c \ + erasure-code/jerasure/gf-complete/src/gf_rand.c \ + erasure-code/jerasure/gf-complete/src/gf_w8.c +noinst_HEADERS += \ + erasure-code/shec/ErasureCodeShec.h \ + erasure-code/shec/ErasureCodeShecTableCache.h \ + erasure-code/shec/shec.h \ + erasure-code/jerasure/jerasure/include/cauchy.h \ + erasure-code/jerasure/jerasure/include/galois.h \ + erasure-code/jerasure/jerasure/include/jerasure.h \ + erasure-code/jerasure/jerasure/include/liberation.h \ + erasure-code/jerasure/jerasure/include/reed_sol.h \ + erasure-code/jerasure/gf-complete/include/gf_int.h \ + erasure-code/jerasure/gf-complete/include/gf_complete.h \ + erasure-code/jerasure/gf-complete/include/gf_rand.h \ + erasure-code/jerasure/gf-complete/include/gf_method.h \ + erasure-code/jerasure/gf-complete/include/gf_general.h + +erasure-code/shec/ErasureCodePluginShec.cc: ./ceph_ver.h + +libec_shec_la_CFLAGS = ${AM_CFLAGS} \ + -I$(srcdir)/erasure-code/jerasure/jerasure/include \ + -I$(srcdir)/erasure-code/jerasure/gf-complete/include \ + -I$(srcdir)/erasure-code/jerasure \ + -I$(srcdir)/erasure-code/shec +libec_shec_la_CXXFLAGS= ${AM_CXXFLAGS} \ + -I$(srcdir)/erasure-code/jerasure/jerasure/include \ + -I$(srcdir)/erasure-code/jerasure/gf-complete/include \ + -I$(srcdir)/erasure-code/jerasure \ + -I$(srcdir)/erasure-code/shec +libec_shec_la_LIBADD = $(LIBCRUSH) $(PTHREAD_LIBS) $(EXTRALIBS) +#libec_shec_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) +#libec_shec_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 -export-symbols-regex '.*__erasure_code_.*' +libec_shec_la_LDFLAGS = ${AM_LDFLAGS} -version-info 1:0:0 +if LINUX +libec_shec_la_LDFLAGS += -export-symbols-regex '.*__erasure_code_.*' +endif + +erasure_codelib_LTLIBRARIES += libec_shec.la diff --git a/src/erasure-code/shec/determinant.c b/src/erasure-code/shec/determinant.c new file mode 100755 index 00000000000..15b62c9efcf --- /dev/null +++ b/src/erasure-code/shec/determinant.c @@ -0,0 +1,94 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2014 Fujitsu Laboratories + * + * Author: Takanori Nakao + * Author: Takeshi Miyamae + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + */ + +#include +#include +#include + +#include "jerasure/include/galois.h" + +void print_matrix(int *mat, int dim) +{ + int i, j; + + for (i=0; i + * Author: Takeshi Miyamae + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + */ + +/* Jerasure's authors: + + Revision 2.x - 2014: James S. Plank and Kevin M. Greenan + Revision 1.2 - 2008: James S. Plank, Scott Simmerman and Catherine D. Schuman. + Revision 1.0 - 2007: James S. Plank + */ + +#include +#include +#include +#include + +#include "shec.h" + +extern "C"{ +#include "jerasure/include/jerasure.h" +#include "jerasure/include/reed_sol.h" + +#define talloc(type, num) (type *) malloc(sizeof(type)*(num)) + +extern int calc_determinant(int *matrix, int dim); +} + +double shec_calc_recovery_efficiency1(int k, int m1, int m2, int c1, int c2){ + int r_eff_k[k]; + double r_e1; + int i, rr, cc, start, end; + int first_flag; + + if (m1 < c1 || m2 < c2) return -1; + if ((m1 == 0 && c1 != 0) || (m2 == 0 && c2 != 0)) return -1; + + for (i=0; i m){ + return -1; + } + + mindup = k+1; + int minc[ek]; + for (i=0; i 0) dup++; + } + if (dup < mindup) { + det = calc_determinant(tmpmat, k); + if (det != 0) { + mindup = dup; + for (int i=0; i 0 && i < k; i++) { + if (erased[i]) { + jerasure_matrix_dotprod(k, w, decoding_matrix+(i*k), + dm_ids, i, data_ptrs, coding_ptrs, size); + edd--; + } + } + + /* Re-encode any erased coding devices */ + + for (i = 0; i < m; i++) { + if (erased[k+i]) { + jerasure_matrix_dotprod(k, w, matrix+(i*k), NULL, i+k, + data_ptrs, coding_ptrs, size); + } + } + + if (decoding_matrix != NULL) free(decoding_matrix); + + return 0; +} diff --git a/src/erasure-code/shec/shec.h b/src/erasure-code/shec/shec.h new file mode 100755 index 00000000000..fe4471f08bb --- /dev/null +++ b/src/erasure-code/shec/shec.h @@ -0,0 +1,35 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2014 FUJITSU LIMITED + * Copyright (C) 2014, James S. Plank and Kevin Greenan + * + * Author: Takanori Nakao + * Author: Takeshi Miyamae + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + */ + +/* Jerasure's authors: + + Revision 2.x - 2014: James S. Plank and Kevin M. Greenan + Revision 1.2 - 2008: James S. Plank, Scott Simmerman and Catherine D. Schuman. + Revision 1.0 - 2007: James S. Plank + */ + +#ifndef SHEC_H +#define SHEC_H + +int *shec_reedsolomon_coding_matrix(int k, int m, int c, int w, int is_single); +int shec_make_decoding_matrix(bool prepare, int k, int m, int w, int *matrix, + int *erased, int *avails, int *decoding_matrix, int *dm_ids, int *minimum); +int shec_matrix_decode(int k, int m, int w, int *matrix, + int *erased, int *avails, char **data_ptrs, char **coding_ptrs, int size); + +#endif diff --git a/src/test/erasure-code/Makefile.am b/src/test/erasure-code/Makefile.am index 08fe9e965f5..62e782fbad6 100644 --- a/src/test/erasure-code/Makefile.am +++ b/src/test/erasure-code/Makefile.am @@ -192,6 +192,39 @@ unittest_erasure_code_plugin_lrc_LDADD += -ldl endif check_PROGRAMS += unittest_erasure_code_plugin_lrc +unittest_erasure_code_shec_SOURCES = \ + test/erasure-code/TestErasureCodeShec.cc \ + ${libec_shec_la_SOURCES} +unittest_erasure_code_shec_CFLAGS = ${libec_shec_la_CFLAGS} +unittest_erasure_code_shec_CXXFLAGS = ${libec_shec_la_CXXFLAGS} $(UNITTEST_CXXFLAGS) +unittest_erasure_code_shec_LDADD = $(LIBOSD) $(LIBCOMMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL) +if LINUX +unittest_erasure_code_shec_LDADD += -ldl +endif +check_PROGRAMS += unittest_erasure_code_shec + +unittest_erasure_code_shec_all_SOURCES = \ + test/erasure-code/TestErasureCodeShec_all.cc \ + ${libec_shec_la_SOURCES} +unittest_erasure_code_shec_all_CFLAGS = ${libec_shec_la_CFLAGS} +unittest_erasure_code_shec_all_CXXFLAGS = ${libec_shec_la_CXXFLAGS} $(UNITTEST_CXXFLAGS) +unittest_erasure_code_shec_all_LDADD = $(LIBOSD) $(LIBCOMMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL) +if LINUX +unittest_erasure_code_shec_all_LDADD += -ldl +endif +check_PROGRAMS += unittest_erasure_code_shec_all + +unittest_erasure_code_shec_thread_SOURCES = \ + test/erasure-code/TestErasureCodeShec_thread.cc \ + ${libec_shec_la_SOURCES} +unittest_erasure_code_shec_thread_CFLAGS = ${libec_shec_la_CFLAGS} +unittest_erasure_code_shec_thread_CXXFLAGS = ${libec_shec_la_CXXFLAGS} $(UNITTEST_CXXFLAGS) +unittest_erasure_code_shec_thread_LDADD = $(LIBOSD) $(LIBCOMMON) $(UNITTEST_LDADD) $(CEPH_GLOBAL) +if LINUX +unittest_erasure_code_shec_thread_LDADD += -ldl +endif +check_PROGRAMS += unittest_erasure_code_shec_thread + unittest_erasure_code_example_SOURCES = \ erasure-code/ErasureCode.cc \ test/erasure-code/TestErasureCodeExample.cc diff --git a/src/test/erasure-code/TestErasureCodeShec.cc b/src/test/erasure-code/TestErasureCodeShec.cc new file mode 100644 index 00000000000..617d1bc83d6 --- /dev/null +++ b/src/test/erasure-code/TestErasureCodeShec.cc @@ -0,0 +1,3512 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2014,2015 FUJITSU LIMITED + * + * Author: Shotaro Kawaguchi + * Author: Takanori Nakao + * Author: Takeshi Miyamae + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + */ + +//SUMMARY: TestErasureCodeShec + +#include +#include + +#include "crush/CrushWrapper.h" +#include "osd/osd_types.h" +#include "include/stringify.h" +#include "global/global_init.h" +#include "erasure-code/shec/ErasureCodeShec.h" +#include "erasure-code/ErasureCodePlugin.h" +#include "common/ceph_argparse.h" +#include "global/global_context.h" +#include "gtest/gtest.h" + +void* thread1(void* pParam); +void* thread2(void* pParam); +void* thread3(void* pParam); +void* thread4(void* pParam); +void* thread5(void* pParam); + +static int g_flag = 0; + +TEST(ErasureCodeShec, init_1) +{ + //all parameters are normal values + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["directory"] = "/usr/lib64/ceph/erasure-code"; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + + int r = shec->init(*parameters); + + //check parameters + EXPECT_EQ(6u, shec->k); + EXPECT_EQ(4u, shec->m); + EXPECT_EQ(3u, shec->c); + EXPECT_EQ(8u, shec->w); + EXPECT_EQ(ErasureCodeShec::MULTIPLE, shec->technique); + EXPECT_STREQ("default", shec->ruleset_root.c_str()); + EXPECT_STREQ("osd", shec->ruleset_failure_domain.c_str()); + EXPECT_TRUE(shec->matrix != NULL); + EXPECT_EQ(0, r); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, init_2) +{ + //all parameters are normal values + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-root"] = "test"; + (*parameters)["ruleset-failure-domain"] = "host"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + (*parameters)["w"] = "8"; + + int r = shec->init(*parameters); + + //check parameters + EXPECT_EQ(6u, shec->k); + EXPECT_EQ(4u, shec->m); + EXPECT_EQ(3u, shec->c); + EXPECT_EQ(8u, shec->w); + EXPECT_EQ(ErasureCodeShec::MULTIPLE, shec->technique); + EXPECT_STREQ("test", shec->ruleset_root.c_str()); + EXPECT_STREQ("host", shec->ruleset_failure_domain.c_str()); + EXPECT_TRUE(shec->matrix != NULL); + EXPECT_EQ(0, r); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, init_3) +{ + //all parameters are normal values + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + (*parameters)["w"] = "16"; + + int r = shec->init(*parameters); + + //check parameters + EXPECT_EQ(6u, shec->k); + EXPECT_EQ(4u, shec->m); + EXPECT_EQ(3u, shec->c); + EXPECT_EQ(16u, shec->w); + EXPECT_EQ(ErasureCodeShec::MULTIPLE, shec->technique); + EXPECT_STREQ("default", shec->ruleset_root.c_str()); + EXPECT_STREQ("osd", shec->ruleset_failure_domain.c_str()); + EXPECT_TRUE(shec->matrix != NULL); + EXPECT_EQ(0, r); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, init_4) +{ + //all parameters are normal values + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + (*parameters)["w"] = "32"; + + int r = shec->init(*parameters); + + //check parameters + EXPECT_EQ(6u, shec->k); + EXPECT_EQ(4u, shec->m); + EXPECT_EQ(3u, shec->c); + EXPECT_EQ(32u, shec->w); + EXPECT_EQ(ErasureCodeShec::MULTIPLE, shec->technique); + EXPECT_STREQ("default", shec->ruleset_root.c_str()); + EXPECT_STREQ("osd", shec->ruleset_failure_domain.c_str()); + EXPECT_TRUE(shec->matrix != NULL); + EXPECT_EQ(0, r); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, init_5) +{ + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + //plugin is not specified + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + + int r = shec->init(*parameters); + + EXPECT_TRUE(shec->matrix != NULL); + EXPECT_EQ(0, r); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, init_6) +{ + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "jerasure"; //unexpected value + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + + int r = shec->init(*parameters); + + EXPECT_TRUE(shec->matrix != NULL); + EXPECT_EQ(0, r); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, init_7) +{ + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "abc"; //unexpected value + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + + int r = shec->init(*parameters); + + EXPECT_TRUE(shec->matrix != NULL); + EXPECT_EQ(0, r); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, init_8) +{ + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["directory"] = "/usr/lib64/"; //unexpected value + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + + int r = shec->init(*parameters); + + EXPECT_TRUE(shec->matrix != NULL); + EXPECT_EQ(0, r); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, init_9) +{ + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-root"] = "abc"; //unexpected value + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + + int r = shec->init(*parameters); + + EXPECT_TRUE(shec->matrix != NULL); + EXPECT_EQ(0, r); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, init_10) +{ + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "abc"; //unexpected value + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + + int r = shec->init(*parameters); + + EXPECT_TRUE(shec->matrix != NULL); + EXPECT_EQ(0, r); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, init_11) +{ + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = "abc"; //unexpected value + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + + int r = shec->init(*parameters); + + EXPECT_TRUE(shec->matrix != NULL); + EXPECT_EQ(0, r); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, init_12) +{ + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "-1"; //unexpected value + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + + int r = shec->init(*parameters); + + EXPECT_EQ(-EINVAL, r); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, init_13) +{ + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "abc"; + (*parameters)["k"] = "0.1"; //unexpected value + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + + int r = shec->init(*parameters); + + EXPECT_EQ(-EINVAL, r); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, init_14) +{ + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "a"; //unexpected value + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + + int r = shec->init(*parameters); + + EXPECT_EQ(-EINVAL, r); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, init_15) +{ + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + //k is not specified + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + + int r = shec->init(*parameters); + + EXPECT_EQ(-EINVAL, r); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, init_16) +{ + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "-1"; //unexpected value + (*parameters)["c"] = "3"; + + int r = shec->init(*parameters); + + EXPECT_EQ(-EINVAL, r); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, init_17) +{ + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "0.1"; //unexpected value + (*parameters)["c"] = "3"; + + int r = shec->init(*parameters); + + EXPECT_EQ(-EINVAL, r); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, init_18) +{ + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "a"; //unexpected value + (*parameters)["c"] = "3"; + + int r = shec->init(*parameters); + + EXPECT_EQ(-EINVAL, r); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, init_19) +{ + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + //m is not specified + (*parameters)["c"] = "3"; + + int r = shec->init(*parameters); + + EXPECT_EQ(-EINVAL, r); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, init_20) +{ + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "-1"; //unexpected value + + int r = shec->init(*parameters); + + EXPECT_EQ(-EINVAL, r); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, init_21) +{ + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "0.1"; //unexpected value + + int r = shec->init(*parameters); + + EXPECT_EQ(-EINVAL, r); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, init_22) +{ + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "a"; //unexpected value + + int r = shec->init(*parameters); + + EXPECT_EQ(-EINVAL, r); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, init_23) +{ + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + //c is not specified + + int r = shec->init(*parameters); + + EXPECT_EQ(-EINVAL, r); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, init_24) +{ + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + (*parameters)["w"] = "1"; //unexpected value + + int r = shec->init(*parameters); + + EXPECT_TRUE(shec->matrix != NULL); + EXPECT_EQ(0, r); + EXPECT_EQ(6u, shec->k); + EXPECT_EQ(4u, shec->m); + EXPECT_EQ(3u, shec->c); + EXPECT_EQ(8u, shec->w); + //w is default value + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, init_25) +{ + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + (*parameters)["w"] = "-1"; //unexpected value + + int r = shec->init(*parameters); + + EXPECT_TRUE(shec->matrix != NULL); + EXPECT_EQ(0, r); + EXPECT_EQ(6u, shec->k); + EXPECT_EQ(4u, shec->m); + EXPECT_EQ(3u, shec->c); + EXPECT_EQ(8u, shec->w); + //w is default value + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, init_26) +{ + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + (*parameters)["w"] = "0.1"; //unexpected value + + int r = shec->init(*parameters); + + EXPECT_TRUE(shec->matrix != NULL); + EXPECT_EQ(0, r); + EXPECT_EQ(6u, shec->k); + EXPECT_EQ(4u, shec->m); + EXPECT_EQ(3u, shec->c); + EXPECT_EQ(8u, shec->w); + //w is default value + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, init_27) +{ + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + (*parameters)["w"] = "a"; //unexpected value + + int r = shec->init(*parameters); + + EXPECT_TRUE(shec->matrix != NULL); + EXPECT_EQ(0, r); + EXPECT_EQ(6u, shec->k); + EXPECT_EQ(4u, shec->m); + EXPECT_EQ(3u, shec->c); + EXPECT_EQ(8u, shec->w); + //w is default value + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, init_28) +{ + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "10"; //c > m + + int r = shec->init(*parameters); + + EXPECT_EQ(-EINVAL, r); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, init_29) +{ + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + //k is not specified + //m is not specified + //c is not specified + + int r = shec->init(*parameters); + + EXPECT_TRUE(shec->matrix != NULL); + EXPECT_EQ(0, r); + //k,m,c are default values + EXPECT_EQ(2u, shec->k); + EXPECT_EQ(1u, shec->m); + EXPECT_EQ(1u, shec->c); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, init_30) +{ + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "12"; + (*parameters)["m"] = "8"; + (*parameters)["c"] = "8"; + + int r = shec->init(*parameters); + + EXPECT_TRUE(shec->matrix != NULL); + EXPECT_EQ(0, r); + EXPECT_EQ(12u, shec->k); + EXPECT_EQ(8u, shec->m); + EXPECT_EQ(8u, shec->c); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, init_31) +{ + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "13"; + (*parameters)["m"] = "7"; + (*parameters)["c"] = "7"; + + int r = shec->init(*parameters); + + EXPECT_EQ(-EINVAL, r); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, init_32) +{ + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "7"; + (*parameters)["m"] = "13"; + (*parameters)["c"] = "13"; + + int r = shec->init(*parameters); + + EXPECT_EQ(-EINVAL, r); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, init_33) +{ + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "12"; + (*parameters)["m"] = "9"; + (*parameters)["c"] = "8"; + + int r = shec->init(*parameters); + + EXPECT_EQ(-EINVAL, r); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, init_34) +{ + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "8"; + (*parameters)["m"] = "12"; + (*parameters)["c"] = "12"; + + int r = shec->init(*parameters); + + EXPECT_EQ(-EINVAL, r); + + delete shec; + delete parameters; +} + +/* + * This test case cannot be executed because of environment issues, + * so this is intentionally commented out. +TEST(ErasureCodeShec, init2_1) +{ + //all parameters are normal values + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + + int r = shec->init(*parameters); + + //check parameters + EXPECT_EQ(6u, shec->k); + EXPECT_EQ(4u, shec->m); + EXPECT_EQ(3u, shec->c); + EXPECT_EQ(8u, shec->w); + EXPECT_EQ(ErasureCodeShec::MULTIPLE, shec->technique); + EXPECT_STREQ("default", shec->ruleset_root.c_str()); + EXPECT_STREQ("osd", shec->ruleset_failure_domain.c_str()); + EXPECT_TRUE(shec->matrix != NULL); + EXPECT_EQ(0, r); + + delete shec; + delete parameters; +} +*/ + +/* + * This test case cannot be executed because of environment issues, + * so this is intentionally commented out. +TEST(ErasureCodeShec, init2_2) +{ + //all parameters are normal values + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + + int r = shec->init(*parameters); + + //check parameters + EXPECT_EQ(6u, shec->k); + EXPECT_EQ(4u, shec->m); + EXPECT_EQ(3u, shec->c); + EXPECT_EQ(8u, shec->w); + EXPECT_EQ(ErasureCodeShec::MULTIPLE, shec->technique); + EXPECT_STREQ("default", shec->ruleset_root.c_str()); + EXPECT_STREQ("osd", shec->ruleset_failure_domain.c_str()); + EXPECT_TRUE(shec->matrix != NULL); + EXPECT_EQ(0, r); + + delete shec; + delete parameters; +} +*/ + +/* + * This test case cannot be executed because of environment issues, + * so this is intentionally commented out. + TEST(ErasureCodeShec, init2_3) + { + //all parameters are normal values + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + + int r = shec->init(*parameters); + + //k,m,c are default values + EXPECT_EQ(2u,shec->k); + EXPECT_EQ(1u,shec->m); + EXPECT_EQ(1u,shec->c); + EXPECT_TRUE(shec->matrix != NULL); + EXPECT_NE(0,r); + delete shec; + delete parameters; + } + */ + +TEST(ErasureCodeShec, init2_4) +{ + //all parameters are normal values + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + shec->init(*parameters); + int r = shec->init(*parameters); //init executed twice + + //check parameters + EXPECT_EQ(6u, shec->k); + EXPECT_EQ(4u, shec->m); + EXPECT_EQ(3u, shec->c); + EXPECT_EQ(8u, shec->w); + EXPECT_EQ(ErasureCodeShec::MULTIPLE, shec->technique); + EXPECT_STREQ("default", shec->ruleset_root.c_str()); + EXPECT_STREQ("osd", shec->ruleset_failure_domain.c_str()); + EXPECT_TRUE(shec->matrix != NULL); + EXPECT_EQ(0, r); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, init2_5) +{ + //all parameters are normal values + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + map < std::string, std::string > *parameters2 = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "host"; + (*parameters)["k"] = "10"; + (*parameters)["m"] = "6"; + (*parameters)["c"] = "5"; + (*parameters)["w"] = "16"; + + int r = shec->init(*parameters); + + //reexecute init + (*parameters2)["plugin"] = "shec"; + (*parameters2)["technique"] = ""; + (*parameters2)["ruleset-failure-domain"] = "osd"; + (*parameters2)["k"] = "6"; + (*parameters2)["m"] = "4"; + (*parameters2)["c"] = "3"; + shec->init(*parameters2); + + EXPECT_EQ(6u, shec->k); + EXPECT_EQ(4u, shec->m); + EXPECT_EQ(3u, shec->c); + EXPECT_EQ(8u, shec->w); + EXPECT_EQ(ErasureCodeShec::MULTIPLE, shec->technique); + EXPECT_STREQ("default", shec->ruleset_root.c_str()); + EXPECT_STREQ("osd", shec->ruleset_failure_domain.c_str()); + EXPECT_TRUE(shec->matrix != NULL); + EXPECT_EQ(0, r); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, minimum_to_decode_1) +{ + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + shec->init(*parameters); + + //minimum_to_decode + set want_to_decode; + set available_chunks; + set minimum_chunks; + + want_to_decode.insert(0); + available_chunks.insert(0); + available_chunks.insert(1); + available_chunks.insert(2); + + int r = shec->minimum_to_decode(want_to_decode, available_chunks, + &minimum_chunks); + EXPECT_TRUE(shec->matrix != NULL); + EXPECT_EQ(0, r); + EXPECT_TRUE(minimum_chunks.size()); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, minimum_to_decode_2) +{ + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + shec->init(*parameters); + + //minimum_to_decode + set want_to_decode; + set available_chunks; + set minimum_chunks; + + for (int i = 0; i < 10; i++) { + want_to_decode.insert(i); + available_chunks.insert(i); + } + + int r = shec->minimum_to_decode(want_to_decode, available_chunks, + &minimum_chunks); + EXPECT_TRUE(shec->matrix != NULL); + EXPECT_EQ(0, r); + EXPECT_TRUE(minimum_chunks.size()); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, minimum_to_decode_3) +{ + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + shec->init(*parameters); + + //minimum_to_decode + set want_to_decode; + set available_chunks; + set minimum_chunks; + + for (int i = 0; i < 32; i++) { //want_to_decode.size() > k+m + want_to_decode.insert(i); + available_chunks.insert(i); + } + + int r = shec->minimum_to_decode(want_to_decode, available_chunks, + &minimum_chunks); + EXPECT_EQ(-EINVAL, r); + EXPECT_EQ(0, minimum_chunks.size()); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, minimum_to_decode_4) +{ + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + shec->init(*parameters); + + //minimum_to_decode + set want_to_decode; + set available_chunks; + set minimum_chunks; + + for (int i = 0; i < 9; i++) { + want_to_decode.insert(i); + available_chunks.insert(i); + } + want_to_decode.insert(100); + available_chunks.insert(100); + + int r = shec->minimum_to_decode(want_to_decode, available_chunks, + &minimum_chunks); + EXPECT_EQ(-EINVAL, r); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, minimum_to_decode_5) +{ + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + shec->init(*parameters); + + //minimum_to_decode + set want_to_decode; + set available_chunks; + set minimum_chunks; + + for (int i = 0; i < 10; i++) { + want_to_decode.insert(i); + } + for (int i = 0; i < 32; i++) { //available_chunks.size() > k+m + available_chunks.insert(i); + } + + int r = shec->minimum_to_decode(want_to_decode, available_chunks, + &minimum_chunks); + EXPECT_EQ(-EINVAL, r); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, minimum_to_decode_6) +{ + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + shec->init(*parameters); + + //minimum_to_decode + set want_to_decode; + set available_chunks; + set minimum_chunks; + + for (int i = 0; i < 9; i++) { + want_to_decode.insert(i); + available_chunks.insert(i); + } + available_chunks.insert(100); + + int r = shec->minimum_to_decode(want_to_decode, available_chunks, + &minimum_chunks); + EXPECT_EQ(-EINVAL, r); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, minimum_to_decode_7) +{ + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + shec->init(*parameters); + + //minimum_to_decode + set want_to_decode; + set available_chunks; + set minimum_chunks; + + want_to_decode.insert(1); + want_to_decode.insert(3); + want_to_decode.insert(5); + available_chunks.insert(1); + available_chunks.insert(3); + available_chunks.insert(6); + + int r = shec->minimum_to_decode(want_to_decode, available_chunks, + &minimum_chunks); + EXPECT_EQ(-EIO, r); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, minimum_to_decode_8) +{ + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + shec->init(*parameters); + + //minimum_to_decode + set want_to_decode; + set available_chunks; + //minimum_chunks is NULL + + for (int i = 0; i < 10; i++) { + want_to_decode.insert(i); + available_chunks.insert(i); + } + + int r = shec->minimum_to_decode(want_to_decode, available_chunks, NULL); + EXPECT_EQ(-EINVAL, r); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, minimum_to_decode_9) +{ + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + shec->init(*parameters); + + //minimum_to_decode + set want_to_decode; + set available_chunks; + set minimum_chunks, minimum; + + for (int i = 0; i < 10; i++) { + want_to_decode.insert(i); + available_chunks.insert(i); + } + shec->minimum_to_decode(want_to_decode, available_chunks, &minimum_chunks); + minimum = minimum_chunks; //normal value + for (int i = 100; i < 120; i++) { + minimum_chunks.insert(i); //insert extra data + } + + int r = shec->minimum_to_decode(want_to_decode, available_chunks, + &minimum_chunks); + EXPECT_TRUE(shec->matrix != NULL); + EXPECT_EQ(0, r); + EXPECT_EQ(minimum, minimum_chunks); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, minimum_to_decode2_1) +{ + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + shec->init(*parameters); + + //minimum_to_decode + set want_to_decode; + set available_chunks; + set minimum_chunks; + + want_to_decode.insert(0); + available_chunks.insert(0); + available_chunks.insert(1); + available_chunks.insert(2); + + int r = shec->minimum_to_decode(want_to_decode, available_chunks, + &minimum_chunks); + EXPECT_TRUE(shec->matrix != NULL); + EXPECT_EQ(0, r); + EXPECT_TRUE(minimum_chunks.size()); + + delete shec; + delete parameters; +} + +/* + * This test case cannot be executed because of environment issues, + * so this is intentionally commented out. + TEST(ErasureCodeShec, minimum_to_decode2_2) + { + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + //init is not executed + + //minimum_to_decode + set want_to_decode; + set available_chunks; + set minimum_chunks; + + want_to_decode.insert(0); + available_chunks.insert(0); + available_chunks.insert(1); + available_chunks.insert(2); + + int r = shec->minimum_to_decode(want_to_decode,available_chunks, + &minimum_chunks); + EXPECT_NE(0,r); + + delete shec; + delete parameters; + } + */ + +TEST(ErasureCodeShec, minimum_to_decode2_3) +{ + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + shec->init(*parameters); + + //minimum_to_decode + set want_to_decode; + set available_chunks; + set minimum_chunks; + + want_to_decode.insert(0); + want_to_decode.insert(2); + available_chunks.insert(0); + available_chunks.insert(1); + available_chunks.insert(2); + available_chunks.insert(3); + + pthread_t tid; + g_flag = 0; + pthread_create(&tid, NULL, thread1, shec); + while (g_flag == 0) { + usleep(1); + } + sleep(1); + printf("*** test start ***\n"); + int r = shec->minimum_to_decode(want_to_decode, available_chunks, + &minimum_chunks); + EXPECT_TRUE(shec->matrix != NULL); + EXPECT_EQ(0, r); + EXPECT_EQ(want_to_decode, minimum_chunks); + printf("*** test end ***\n"); + g_flag = 0; + pthread_join(tid, NULL); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, minimum_to_decode_with_cost_1) +{ + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + shec->init(*parameters); + + //minimum_to_decode_with_cost + set want_to_decode; + map available_chunks; + set minimum_chunks; + + want_to_decode.insert(0); + available_chunks[0] = 0; + available_chunks[1] = 1; + available_chunks[2] = 2; + + int r = shec->minimum_to_decode_with_cost(want_to_decode, available_chunks, + &minimum_chunks); + EXPECT_TRUE(shec->matrix != NULL); + EXPECT_EQ(0, r); + EXPECT_TRUE(minimum_chunks.size()); + + delete shec; + delete parameters; +} + +/* + * This test case cannot be executed because of environment issues, + * so this is intentionally commented out. + TEST(ErasureCodeShec, minimum_to_decode_with_cost_2_2) + { + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + //init is not executed + + //minimum_to_decode_with_cost + set want_to_decode; + map available_chunks; + set minimum_chunks; + + want_to_decode.insert(0); + available_chunks[0] = 0; + available_chunks[1] = 1; + available_chunks[2] = 2; + + int r = shec->minimum_to_decode_with_cost(want_to_decode, available_chunks, + &minimum_chunks); + EXPECT_NE(0,r); + + delete shec; + delete parameters; + } + */ + +TEST(ErasureCodeShec, minimum_to_decode_with_cost_2_3) +{ + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + shec->init(*parameters); + + //minimum_to_decode_with_cost + set want_to_decode; + map available_chunks; + set minimum_chunks; + + want_to_decode.insert(0); + want_to_decode.insert(2); + available_chunks[0] = 0; + available_chunks[1] = 1; + available_chunks[2] = 2; + available_chunks[3] = 3; + + pthread_t tid; + g_flag = 0; + pthread_create(&tid, NULL, thread2, shec); + while (g_flag == 0) { + usleep(1); + } + sleep(1); + printf("*** test start ***\n"); + int r = shec->minimum_to_decode_with_cost(want_to_decode, available_chunks, + &minimum_chunks); + EXPECT_TRUE(shec->matrix != NULL); + EXPECT_EQ(0, r); + EXPECT_EQ(want_to_decode, minimum_chunks); + printf("*** test end ***\n"); + g_flag = 0; + pthread_join(tid, NULL); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, encode_1) +{ + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + shec->init(*parameters); + + //encode + bufferlist in; + set want_to_encode; + map encoded; + + in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//length = 62 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//124 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//186 + "012345"//192 + ); + for (unsigned int i = 0; i < shec->get_chunk_count(); i++) { + want_to_encode.insert(i); + } + + int r = shec->encode(want_to_encode, in, &encoded); + EXPECT_EQ(0, r); + EXPECT_EQ(shec->get_chunk_count(), encoded.size()); + EXPECT_EQ(shec->get_chunk_size(in.length()), encoded[0].length()); + + //decode + int want_to_decode[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + map decoded; + decoded.clear(); + r = shec->decode(set(want_to_decode, want_to_decode + 2), + encoded, + &decoded); + EXPECT_TRUE(shec->matrix != NULL); + EXPECT_EQ(0, r); + EXPECT_EQ(2u, decoded.size()); + EXPECT_EQ(32u, decoded[0].length()); + + bufferlist out1, out2, usable; + //out1 is "encoded" + for (unsigned int i = 0; i < encoded.size(); i++) { + out1.append(encoded[i]); + } + //out2 is "decoded" + r = shec->decode_concat(encoded, &out2); + usable.substr_of(out2, 0, in.length()); + EXPECT_FALSE(out1 == in); + EXPECT_TRUE(usable == in); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, encode_2) +{ + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + shec->init(*parameters); + + //encode + bufferlist in; + set want_to_encode; + map encoded; + + in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//length = 62 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//124 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//186 + ); + for (unsigned int i = 0; i < shec->get_chunk_count(); i++) { + want_to_encode.insert(i); + } + + int r = shec->encode(want_to_encode, in, &encoded); + EXPECT_EQ(0, r); + EXPECT_EQ(shec->get_chunk_count(), encoded.size()); + EXPECT_EQ(shec->get_chunk_size(in.length()), encoded[0].length()); + + //decode + int want_to_decode[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + map decoded; + r = shec->decode(set(want_to_decode, want_to_decode + 2), encoded, + &decoded); + EXPECT_TRUE(shec->matrix != NULL); + EXPECT_EQ(0, r); + EXPECT_EQ(2u, decoded.size()); + EXPECT_EQ(32u, decoded[0].length()); + + bufferlist out1, out2, usable; + //out1 is "encoded" + for (unsigned int i = 0; i < encoded.size(); i++) + out1.append(encoded[i]); + //out2 is "decoded" + shec->decode_concat(encoded, &out2); + usable.substr_of(out2, 0, in.length()); + EXPECT_FALSE(out1 == in); + EXPECT_TRUE(usable == in); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, encode_3) +{ + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + shec->init(*parameters); + + bufferlist in; + in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//length = 62 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//124 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//186 + ); + set want_to_encode; + for (unsigned int i = 0; i < shec->get_chunk_count(); i++) { + want_to_encode.insert(i); + } + want_to_encode.insert(10); + want_to_encode.insert(11); + map encoded; + int r = shec->encode(want_to_encode, in, &encoded); + EXPECT_EQ(0, r); + EXPECT_EQ(shec->get_chunk_count(), encoded.size()); + EXPECT_EQ(shec->get_chunk_size(in.length()), encoded[0].length()); + + //decode + int want_to_decode[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + map decoded; + r = shec->decode(set(want_to_decode, want_to_decode + 2), encoded, + &decoded); + EXPECT_TRUE(shec->matrix != NULL); + EXPECT_EQ(0, r); + EXPECT_EQ(2u, decoded.size()); + EXPECT_EQ(shec->get_chunk_size(in.length()), decoded[0].length()); + + bufferlist out1, out2, usable; + //out1 is "encoded" + for (unsigned int i = 0; i < encoded.size(); i++) { + out1.append(encoded[i]); + } + //out2 is "decoded" + shec->decode_concat(encoded, &out2); + usable.substr_of(out2, 0, in.length()); + EXPECT_FALSE(out1 == in); + EXPECT_TRUE(usable == in); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, encode_4) +{ + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + shec->init(*parameters); + + //encode + bufferlist in; + set want_to_encode; + map encoded; + + in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//length = 62 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//124 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//186 + ); + for (unsigned int i = 0; i < shec->get_chunk_count() - 1; i++) { + want_to_encode.insert(i); + } + want_to_encode.insert(100); + + int r = shec->encode(want_to_encode, in, &encoded); + EXPECT_EQ(0, r); + EXPECT_EQ(shec->get_chunk_count()-1, encoded.size()); + EXPECT_EQ(shec->get_chunk_size(in.length()), encoded[0].length()); + + //decode + int want_to_decode[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + map decoded; + r = shec->decode(set(want_to_decode, want_to_decode + 2), encoded, + &decoded); + EXPECT_TRUE(shec->matrix != NULL); + EXPECT_EQ(0, r); + EXPECT_EQ(2u, decoded.size()); + EXPECT_EQ(shec->get_chunk_size(in.length()), decoded[0].length()); + + bufferlist out1, out2, usable; + //out1 is "encoded" + for (unsigned int i = 0; i < encoded.size(); i++) { + out1.append(encoded[i]); + } + //out2 is "decoded" + shec->decode_concat(encoded, &out2); + usable.substr_of(out2, 0, in.length()); + EXPECT_FALSE(out1 == in); + EXPECT_TRUE(usable == in); + + delete shec; + delete parameters; +} + +/* + * This test case cannot be executed because of environment issues, + * so this is intentionally commented out. + TEST(ErasureCodeShec, encode_6) + { + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + shec->init(*parameters); + + //encode + set want_to_encode; + map encoded; + + for(unsigned int i = 0; i < shec->get_chunk_count(); i++) + want_to_encode.insert(i); + int r = shec->encode(want_to_encode, NULL, &encoded) //inbuf=NULL + EXPECT_EQ(0, r); + EXPECT_EQ(shec->get_chunk_count(), encoded.size()); + EXPECT_EQ(0, encoded[0].length()); + + delete shec; + delete parameters; + } + */ + +TEST(ErasureCodeShec, encode_8) +{ + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + shec->init(*parameters); + + //encode + bufferlist in; + set want_to_encode; + + in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//length = 62 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//124 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//186 + ); + for (unsigned int i = 0; i < shec->get_chunk_count(); i++) { + want_to_encode.insert(i); + } + + int r = shec->encode(want_to_encode, in, NULL); //encoded = NULL + EXPECT_EQ(-EINVAL, r); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, encode_9) +{ + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + shec->init(*parameters); + + //encode + bufferlist in; + set want_to_encode; + map encoded; + + in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//length = 62 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//124 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//186 + ); + for (unsigned int i = 0; i < shec->get_chunk_count(); i++) { + want_to_encode.insert(i); + } + for (int i = 0; i < 100; i++) { + encoded[i].append("ABCDEFGHIJKLMNOPQRSTUVWXYZ"); + } + + int r = shec->encode(want_to_encode, in, &encoded); + EXPECT_EQ(-EINVAL, r); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, encode2_1) +{ + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + shec->init(*parameters); + + //encode + bufferlist in; + set want_to_encode; + map encoded; + + in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//length = 62 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//124 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//186 + "012345"//192 + ); + for (unsigned int i = 0; i < shec->get_chunk_count(); i++) { + want_to_encode.insert(i); + } + + int r = shec->encode(want_to_encode, in, &encoded); + EXPECT_EQ(0, r); + EXPECT_EQ(shec->get_chunk_count(), encoded.size()); + EXPECT_EQ(shec->get_chunk_size(in.length()), encoded[0].length()); + + //decode + int want_to_decode[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + map decoded; + r = shec->decode(set(want_to_decode, want_to_decode + 2), encoded, + &decoded); + EXPECT_TRUE(shec->matrix != NULL); + EXPECT_EQ(0, r); + EXPECT_EQ(2u, decoded.size()); + EXPECT_EQ(32u, decoded[0].length()); + + bufferlist out1, out2, usable; + //out1 is "encoded" + for (unsigned int i = 0; i < encoded.size(); i++) { + out1.append(encoded[i]); + } + //out2 is "decoded" + shec->decode_concat(encoded, &out2); + usable.substr_of(out2, 0, in.length()); + EXPECT_FALSE(out1 == in); + EXPECT_TRUE(usable == in); + + delete shec; + delete parameters; +} + +/* + * This test case cannot be executed because of environment issues, + * so this is intentionally commented out. + TEST(ErasureCodeShec, encode2_2) + { + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + //init is not executed + + //encode + bufferlist in; + set want_to_encode; + map encoded; + + in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" //length = 62 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" //124 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" //186 + "012345" //192 + ); + for(unsigned int i = 0; i < shec->get_chunk_count(); i++) + want_to_encode.insert(i); + + int r = shec->encode(want_to_encode, in, &encoded); + EXPECT_EQ(-EINVAL, r); + + delete shec; + delete parameters; + } + */ + +TEST(ErasureCodeShec, encode2_3) +{ + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + shec->init(*parameters); + + //encode + bufferlist in; + set want_to_encode; + map encoded; + + in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//length = 62 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//124 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//186 + "012345"//192 + ); + for (unsigned int i = 0; i < shec->get_chunk_count(); i++) { + want_to_encode.insert(i); + } + + pthread_t tid; + g_flag = 0; + pthread_create(&tid, NULL, thread4, shec); + while (g_flag == 0) { + usleep(1); + } + sleep(1); + printf("*** test start ***\n"); + int r = shec->encode(want_to_encode, in, &encoded); + EXPECT_EQ(0, r); + EXPECT_EQ(shec->get_chunk_count(), encoded.size()); + EXPECT_EQ(shec->get_chunk_size(in.length()), encoded[0].length()); + printf("*** test end ***\n"); + g_flag = 0; + pthread_join(tid, NULL); + + //decode + int want_to_decode[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + map decoded; + + r = shec->decode(set(want_to_decode, want_to_decode + 2), encoded, + &decoded); + EXPECT_TRUE(shec->matrix != NULL); + EXPECT_EQ(0, r); + EXPECT_EQ(2u, decoded.size()); + EXPECT_EQ(32u, decoded[0].length()); + + bufferlist out1, out2, usable; + //out1 is "encoded" + for (unsigned int i = 0; i < encoded.size(); i++) { + out1.append(encoded[i]); + } + //out2 is "decoded" + shec->decode_concat(encoded, &out2); + usable.substr_of(out2, 0, in.length()); + EXPECT_FALSE(out1 == in); + EXPECT_TRUE(usable == in); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, decode_1) +{ + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + shec->init(*parameters); + + //encode + bufferlist in; + set want_to_encode; + map encoded; + + in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//length = 62 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//124 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//186 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//248 + ); + for (unsigned int i = 0; i < shec->get_chunk_count(); i++) { + want_to_encode.insert(i); + } + + int r = shec->encode(want_to_encode, in, &encoded); + EXPECT_EQ(0, r); + EXPECT_EQ(shec->get_chunk_count(), encoded.size()); + EXPECT_EQ(shec->get_chunk_size(in.length()), encoded[0].length()); + + // all chunks are available + //decode + int want_to_decode[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + map decoded; + + r = shec->decode(set(want_to_decode, want_to_decode + 2), encoded, + &decoded); + EXPECT_TRUE(shec->matrix != NULL); + EXPECT_EQ(0, r); + EXPECT_EQ(2u, decoded.size()); + + bufferlist out, usable; + shec->decode_concat(encoded, &out); + usable.substr_of(out, 0, in.length()); + EXPECT_TRUE(usable == in); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, decode_2) +{ + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + shec->init(*parameters); + + //encode + bufferlist in; + set want_to_encode; + map encoded; + + in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//length = 62 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//124 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//186 + "012345"//192 + ); + for (unsigned int i = 0; i < shec->get_chunk_count(); i++) { + want_to_encode.insert(i); + } + + int r = shec->encode(want_to_encode, in, &encoded); + EXPECT_EQ(0, r); + EXPECT_EQ(shec->get_chunk_count(), encoded.size()); + EXPECT_EQ(shec->get_chunk_size(in.length()), encoded[0].length()); + + // all chunks are available + //decode + int want_to_decode[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + map decoded; + + r = shec->decode(set(want_to_decode, want_to_decode + 2), encoded, + &decoded); + EXPECT_TRUE(shec->matrix != NULL); + EXPECT_EQ(0, r); + EXPECT_EQ(2u, decoded.size()); + + bufferlist out, usable; + shec->decode_concat(encoded, &out); + usable.substr_of(out, 0, in.length()); + EXPECT_TRUE(usable == in); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, decode_3) +{ + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + shec->init(*parameters); + + //encode + bufferlist in; + set want_to_encode; + map encoded; + + in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//length = 62 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//124 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//186 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//248 + ); + for (unsigned int i = 0; i < shec->get_chunk_count(); i++) { + want_to_encode.insert(i); + } + + int r = shec->encode(want_to_encode, in, &encoded); + EXPECT_EQ(0, r); + EXPECT_EQ(shec->get_chunk_count(), encoded.size()); + EXPECT_EQ(shec->get_chunk_size(in.length()), encoded[0].length()); + + // all chunks are available + //decode + int want_to_decode[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 }; //more than k+m + map decoded; + + r = shec->decode(set(want_to_decode, want_to_decode + 11), encoded, + &decoded); + EXPECT_TRUE(shec->matrix != NULL); + EXPECT_EQ(0, r); + EXPECT_EQ(10u, decoded.size()); + EXPECT_EQ(shec->get_chunk_size(in.length()), decoded[0].length()); + + bufferlist out1, out2, usable; + //out1 is "encoded" + for (unsigned int i = 0; i < encoded.size(); i++) { + out1.append(encoded[i]); + } + //out2 is "decoded" + shec->decode_concat(encoded, &out2); + usable.substr_of(out2, 0, in.length()); + EXPECT_FALSE(out1 == in); + EXPECT_TRUE(usable == in); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, decode_4) +{ + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + shec->init(*parameters); + + //encode + bufferlist in; + set want_to_encode; + map encoded; + + in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//length = 62 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//124 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//186 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//248 + ); + for (unsigned int i = 0; i < shec->get_chunk_count(); i++) { + want_to_encode.insert(i); + } + + int r = shec->encode(want_to_encode, in, &encoded); + EXPECT_EQ(0, r); + EXPECT_EQ(shec->get_chunk_count(), encoded.size()); + EXPECT_EQ(shec->get_chunk_size(in.length()), encoded[0].length()); + + // all chunks are available + //decode + int want_to_decode[] = { 0, 1, 2, 3, 4, 5, 6, 7, 100 }; + map decoded; + + r = shec->decode(set(want_to_decode, want_to_decode + 9), encoded, + &decoded); + EXPECT_TRUE(shec->matrix != NULL); + EXPECT_EQ(0, r); + EXPECT_EQ(10u, decoded.size()); + EXPECT_EQ(shec->get_chunk_size(in.length()), decoded[0].length()); + + bufferlist out1, out2, usable; + //out1 is "encoded" + for (unsigned int i = 0; i < encoded.size(); i++) { + out1.append(encoded[i]); + } + //out2 is "decoded" + shec->decode_concat(encoded, &out2); + usable.substr_of(out2, 0, in.length()); + EXPECT_FALSE(out1 == in); + EXPECT_TRUE(usable == in); + + delete shec; + delete parameters; +} + +/* + * This test case cannot be executed because of environment issues, + * so this is intentionally commented out. + TEST(ErasureCodeShec, decode_6) + { + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + shec->init(*parameters); + + //decode + int want_to_decode[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + map decoded; + + // map inchunks; + EXPECT_NE(0,shec->decode(set(want_to_decode, want_to_decode+2), NULL, + &decoded)); + + delete shec; + delete parameters; + } + */ + +TEST(ErasureCodeShec, decode_7) +{ + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + shec->init(*parameters); + + //encode + bufferlist in; + set want_to_encode; + map encoded; + + in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//length = 62 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//124 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//186 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//248 + ); + for (unsigned int i = 0; i < shec->get_chunk_count(); i++) { + want_to_encode.insert(i); + } + + int r = shec->encode(want_to_encode, in, &encoded); + EXPECT_EQ(0, r); + EXPECT_EQ(shec->get_chunk_count(), encoded.size()); + EXPECT_EQ(shec->get_chunk_size(in.length()), encoded[0].length()); + + // all chunks are available + //decode + int want_to_decode[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + map decoded; + + //extra data + bufferlist buf; + buf.append("abc"); + encoded[100] = buf; + + r = shec->decode(set(want_to_decode, want_to_decode + 2), encoded, + &decoded); + EXPECT_TRUE(shec->matrix != NULL); + EXPECT_EQ(0, r); + EXPECT_EQ(2u, decoded.size()); + EXPECT_EQ(shec->get_chunk_size(in.length()), decoded[0].length()); + + bufferlist out1, out2, usable; + //out1 is "encoded" + for (unsigned int i = 0; i < encoded.size(); i++) { + out1.append(encoded[i]); + } + //out2 is "decoded" + shec->decode_concat(encoded, &out2); + usable.substr_of(out2, 0, in.length()); + EXPECT_FALSE(out1 == in); + EXPECT_TRUE(usable == in); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, decode_8) +{ + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + shec->init(*parameters); + + //encode + bufferlist in; + set want_to_encode; + map encoded; + + in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//length = 62 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//124 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//186 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//248 + ); + for (unsigned int i = 0; i < shec->get_chunk_count(); i++) { + want_to_encode.insert(i); + } + + int r = shec->encode(want_to_encode, in, &encoded); + EXPECT_EQ(0, r); + EXPECT_EQ(shec->get_chunk_count(), encoded.size()); + EXPECT_EQ(shec->get_chunk_size(in.length()), encoded[0].length()); + + // all chunks are available + //decode + int want_to_decode[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + + //decoded = NULL + r = shec->decode(set(want_to_decode, want_to_decode + 2), encoded, + NULL); + EXPECT_NE(0, r); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, decode_9) +{ + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + shec->init(*parameters); + + //encode + bufferlist in; + set want_to_encode; + map encoded; + + in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//length = 62 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//124 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//186 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//248 + ); + for (unsigned int i = 0; i < shec->get_chunk_count(); i++) { + want_to_encode.insert(i); + } + + int r = shec->encode(want_to_encode, in, &encoded); + EXPECT_EQ(0, r); + EXPECT_EQ(shec->get_chunk_count(), encoded.size()); + EXPECT_EQ(shec->get_chunk_size(in.length()), encoded[0].length()); + + // all chunks are available + //decode + int want_to_decode[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + map decoded; + + //extra data + bufferlist buf; + buf.append("a"); + for (int i = 0; i < 100; i++) { + decoded[i] = buf; + } + + r = shec->decode(set(want_to_decode, want_to_decode + 2), encoded, + &decoded); + EXPECT_NE(0, r); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, decode2_1) +{ + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + shec->init(*parameters); + + //encode + bufferlist in; + set want_to_encode; + map encoded; + + in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//length = 62 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//124 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//186 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//248 + ); + for (unsigned int i = 0; i < shec->get_chunk_count(); i++) { + want_to_encode.insert(i); + } + + int r = shec->encode(want_to_encode, in, &encoded); + EXPECT_EQ(0, r); + EXPECT_EQ(shec->get_chunk_count(), encoded.size()); + EXPECT_EQ(shec->get_chunk_size(in.length()), encoded[0].length()); + + // all chunks are available + //decode + int want_to_decode[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + map decoded; + + r = shec->decode(set(want_to_decode, want_to_decode + 2), encoded, + &decoded); + EXPECT_TRUE(shec->matrix != NULL); + EXPECT_EQ(0, r); + EXPECT_EQ(2u, decoded.size()); + + bufferlist out; + shec->decode_concat(encoded, &out); + bufferlist usable; + usable.substr_of(out, 0, in.length()); + EXPECT_TRUE(usable == in); + + delete shec; + delete parameters; +} + +/* + * This test case cannot be executed because of environment issues, + * so this is intentionally commented out. + TEST(ErasureCodeShec, decode2_2) + { + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + // init is not executed + + //create encoded + map encoded; + bufferlist buf; + buf.append("ABCDEFGH"); + for(unsigned int i = 0; i < shec->get_chunk_count(); i++) + encoded[i] = buf; + + // all chunks are available + //decode + int want_to_decode[] = { 0 }; + map decoded; + + EXPECT_NE(0,shec->decode(set(want_to_decode, want_to_decode+2), encoded, + &decoded)); + + delete shec; + delete parameters; + } + */ + +TEST(ErasureCodeShec, decode2_3) +{ + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + shec->init(*parameters); + + //encode + bufferlist in; + set want_to_encode; + map encoded; + + in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//length = 62 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//124 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//186 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//248 + ); + for (unsigned int i = 0; i < shec->get_chunk_count(); i++) { + want_to_encode.insert(i); + } + + int r = shec->encode(want_to_encode, in, &encoded); + EXPECT_EQ(0, r); + EXPECT_EQ(shec->get_chunk_count(), encoded.size()); + EXPECT_EQ(shec->get_chunk_size(in.length()), encoded[0].length()); + + // all chunks are available + //decode + int want_to_decode[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + map decoded; + + pthread_t tid; + g_flag = 0; + pthread_create(&tid, NULL, thread4, shec); + while (g_flag == 0) { + usleep(1); + } + sleep(1); + printf("*** test start ***\n"); + r = shec->decode(set(want_to_decode, want_to_decode + 2), encoded, + &decoded); + EXPECT_TRUE(shec->matrix != NULL); + EXPECT_EQ(0, r); + EXPECT_EQ(2u, decoded.size()); + printf("*** test end ***\n"); + g_flag = 0; + pthread_join(tid, NULL); + + bufferlist out; + shec->decode_concat(encoded, &out); + bufferlist usable; + usable.substr_of(out, 0, in.length()); + EXPECT_TRUE(usable == in); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, decode2_4) +{ + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + shec->init(*parameters); + + //encode + bufferlist in; + set want_to_encode; + map encoded; + + in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//length = 62 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//124 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//186 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//248 + ); + for (unsigned int i = 0; i < shec->get_chunk_count(); i++) { + want_to_encode.insert(i); + } + + int r = shec->encode(want_to_encode, in, &encoded); + EXPECT_EQ(0, r); + EXPECT_EQ(shec->get_chunk_count(), encoded.size()); + EXPECT_EQ(shec->get_chunk_size(in.length()), encoded[0].length()); + + //decode + int want_to_decode[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + map decoded; + + // cannot recover + bufferlist out; + map degraded; + degraded[0] = encoded[0]; + + r = shec->decode(set(want_to_decode, want_to_decode + 2), degraded, + &decoded); + EXPECT_EQ(-1, r); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, create_ruleset_1_2) +{ + //create ruleset + CrushWrapper *crush = new CrushWrapper; + crush->create(); + crush->set_type_name(2, "root"); + crush->set_type_name(1, "host"); + crush->set_type_name(0, "osd"); + + int rootno; + crush->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1, 2, 0, NULL, + NULL, &rootno); + crush->set_item_name(rootno, "default"); + + map < string, string > loc; + loc["root"] = "default"; + + int num_host = 2; + int num_osd = 5; + int osd = 0; + for (int h = 0; h < num_host; ++h) { + loc["host"] = string("host-") + stringify(h); + for (int o = 0; o < num_osd; ++o, ++osd) { + crush->insert_item(g_ceph_context, osd, 1.0, + string("osd.") + stringify(osd), loc); + } + } + + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + shec->init(*parameters); + + //create_ruleset + stringstream ss; + + int r = shec->create_ruleset("myrule", *crush, &ss); + EXPECT_EQ(0, r); + EXPECT_STREQ("myrule", crush->rule_name_map[0].c_str()); + + //reexecute create_ruleset + r = shec->create_ruleset("myrule", *crush, &ss); + EXPECT_EQ(-EEXIST, r); + + delete shec; + delete parameters; + delete crush; +} + +/* + * This test case cannot be executed because of environment issues, + * so this is intentionally commented out. + TEST(ErasureCodeShec, create_ruleset_3) + { + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + shec->init(*parameters); + + //create_ruleset + stringstream ss; + CrushWrapper *crush = NULL; + int r = shec->create_ruleset("myrule", *crush, &ss); + EXPECT_NE(0, r); //crush = NULL + + delete shec; + delete parameters; + } + */ + +TEST(ErasureCodeShec, create_ruleset_4) +{ + //create ruleset + CrushWrapper *crush = new CrushWrapper; + crush->create(); + crush->set_type_name(2, "root"); + crush->set_type_name(1, "host"); + crush->set_type_name(0, "osd"); + + int rootno; + crush->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1, 2, 0, NULL, + NULL, &rootno); + crush->set_item_name(rootno, "default"); + + map < string, string > loc; + loc["root"] = "default"; + + int num_host = 2; + int num_osd = 5; + int osd = 0; + for (int h = 0; h < num_host; ++h) { + loc["host"] = string("host-") + stringify(h); + for (int o = 0; o < num_osd; ++o, ++osd) { + crush->insert_item(g_ceph_context, osd, 1.0, + string("osd.") + stringify(osd), loc); + } + } + + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + shec->init(*parameters); + + //create_ruleset + int r = shec->create_ruleset("myrule", *crush, NULL); //ss = NULL + EXPECT_EQ(0, r); + + delete shec; + delete parameters; + delete crush; +} + +TEST(ErasureCodeShec, create_ruleset2_1) +{ + //create ruleset + CrushWrapper *crush = new CrushWrapper; + crush->create(); + crush->set_type_name(2, "root"); + crush->set_type_name(1, "host"); + crush->set_type_name(0, "osd"); + + int rootno; + crush->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1, 2, 0, NULL, + NULL, &rootno); + crush->set_item_name(rootno, "default"); + + map < string, string > loc; + loc["root"] = "default"; + + int num_host = 2; + int num_osd = 5; + int osd = 0; + for (int h = 0; h < num_host; ++h) { + loc["host"] = string("host-") + stringify(h); + for (int o = 0; o < num_osd; ++o, ++osd) { + crush->insert_item(g_ceph_context, osd, 1.0, + string("osd.") + stringify(osd), loc); + } + } + + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + shec->init(*parameters); + + //create_ruleset + stringstream ss; + + int r = shec->create_ruleset("myrule", *crush, &ss); + EXPECT_EQ(0, r); + EXPECT_STREQ("myrule", crush->rule_name_map[0].c_str()); + + delete shec; + delete parameters; + delete crush; +} + +TEST(ErasureCodeShec, create_ruleset2_2) +{ + //create ruleset + CrushWrapper *crush = new CrushWrapper; + crush->create(); + crush->set_type_name(2, "root"); + crush->set_type_name(1, "host"); + crush->set_type_name(0, "osd"); + + int rootno; + crush->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1, 2, 0, NULL, + NULL, &rootno); + crush->set_item_name(rootno, "default"); + + map < string, string > loc; + loc["root"] = "default"; + + int num_host = 2; + int num_osd = 5; + int osd = 0; + for (int h = 0; h < num_host; ++h) { + loc["host"] = string("host-") + stringify(h); + for (int o = 0; o < num_osd; ++o, ++osd) { + crush->insert_item(g_ceph_context, osd, 1.0, + string("osd.") + stringify(osd), loc); + } + } + + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + // init is not executed + + //create_ruleset + stringstream ss; + + int r = shec->create_ruleset("myrule", *crush, &ss); + EXPECT_EQ(0, r); + + delete shec; + delete parameters; + delete crush; +} + +struct CreateRuleset2_3_Param_d { + ErasureCodeShec *shec; + CrushWrapper *crush; +}; + +TEST(ErasureCodeShec, create_ruleset2_3) +{ + //create ruleset + CrushWrapper *crush = new CrushWrapper; + crush->create(); + crush->set_type_name(2, "root"); + crush->set_type_name(1, "host"); + crush->set_type_name(0, "osd"); + + int rootno; + crush->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1, 2, 0, NULL, + NULL, &rootno); + crush->set_item_name(rootno, "default"); + + map < string, string > loc; + loc["root"] = "default"; + + int num_host = 2; + int num_osd = 5; + int osd = 0; + for (int h = 0; h < num_host; ++h) { + loc["host"] = string("host-") + stringify(h); + for (int o = 0; o < num_osd; ++o, ++osd) { + crush->insert_item(g_ceph_context, osd, 1.0, + string("osd.") + stringify(osd), loc); + } + } + + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + shec->init(*parameters); + + //create_ruleset + stringstream ss; + + pthread_t tid; + g_flag = 0; + pthread_create(&tid, NULL, thread3, shec); + while (g_flag == 0) { + usleep(1); + } + sleep(1); + printf("*** test start ***\n"); + int r = (shec->create_ruleset("myrule", *crush, &ss)); + EXPECT_TRUE(r >= 0); + printf("*** test end ***\n"); + g_flag = 0; + pthread_join(tid, NULL); + + delete shec; + delete parameters; + delete crush; +} + +TEST(ErasureCodeShec, get_chunk_count_1) +{ + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + shec->init(*parameters); + + //get_chunk_count + EXPECT_EQ(10u, shec->get_chunk_count()); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, get_chunk_count_2) +{ + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + //init is not executed + + //get_chunk_count + EXPECT_EQ(10u, shec->get_chunk_count()); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, get_data_chunk_count_1) +{ + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + shec->init(*parameters); + + //get_data_chunk_count + EXPECT_EQ(6u, shec->get_data_chunk_count()); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, get_data_chunk_count_2) +{ + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + //init is not executed + + //get_data_chunk_count + EXPECT_EQ(6u, shec->get_data_chunk_count()); + + delete shec; + delete parameters; +} + +TEST(ErasureCodeShec, get_chunk_size_1_2) +{ + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + (*parameters)["w"] = "8"; + shec->init(*parameters); + + //when there is no padding(192=k*w*4) + EXPECT_EQ(32u, shec->get_chunk_size(192)); + //when there is padding(190=k*w*4-2) + EXPECT_EQ(32u, shec->get_chunk_size(190)); + + delete shec; + delete parameters; +} + +/* + * This test case cannot be executed because of environment issues, + * so this is intentionally commented out. + TEST(ErasureCodeShec, get_chunk_size2) + { + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = "6"; + (*parameters)["m"] = "4"; + (*parameters)["c"] = "3"; + (*parameters)["w"] = "8"; + //init is not executed + + //when there is no padding(192=k*w*4) + EXPECT_EQ(32u, shec->get_chunk_size(192)); + //when there is padding(190=k*w*4-2) + EXPECT_EQ(32u, shec->get_chunk_size(190)); + + delete shec; + delete parameters; + } + */ + +int main(int argc, char **argv) +{ + vector args; + argv_to_vec(argc, (const char **) argv, args); + + global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0); + common_init_finish(g_ceph_context); + + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +void* thread1(void* pParam) +{ + ErasureCodeShec* shec = (ErasureCodeShec*) pParam; + set want_to_decode; + set available_chunks; + set minimum_chunks; + + want_to_decode.insert(0); + want_to_decode.insert(1); + available_chunks.insert(0); + available_chunks.insert(1); + available_chunks.insert(2); + + printf("*** thread loop start ***\n"); + g_flag = 1; + while (g_flag == 1) { + shec->minimum_to_decode(want_to_decode, available_chunks, &minimum_chunks); + } + printf("*** thread loop end ***\n"); + + return NULL; +} + +void* thread2(void* pParam) +{ + ErasureCodeShec* shec = (ErasureCodeShec*) pParam; + set want_to_decode; + map available_chunks; + set minimum_chunks; + + want_to_decode.insert(0); + want_to_decode.insert(1); + available_chunks[0] = 0; + available_chunks[1] = 1; + available_chunks[2] = 2; + + printf("*** thread loop start ***\n"); + g_flag = 1; + while (g_flag == 1) { + shec->minimum_to_decode_with_cost(want_to_decode, available_chunks, + &minimum_chunks); + minimum_chunks.clear(); + } + printf("*** thread loop end ***\n"); + + return NULL; +} + +void* thread3(void* pParam) +{ + ErasureCodeShec* shec = (ErasureCodeShec*) pParam; + + CrushWrapper *crush = new CrushWrapper; + crush->create(); + crush->set_type_name(2, "root"); + crush->set_type_name(1, "host"); + crush->set_type_name(0, "osd"); + + int rootno; + crush->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1, 2, 0, NULL, + NULL, &rootno); + crush->set_item_name(rootno, "default"); + + map < string, string > loc; + loc["root"] = "default"; + + int num_host = 2; + int num_osd = 5; + int osd = 0; + for (int h = 0; h < num_host; ++h) { + loc["host"] = string("host-") + stringify(h); + for (int o = 0; o < num_osd; ++o, ++osd) { + crush->insert_item(g_ceph_context, osd, 1.0, + string("osd.") + stringify(osd), loc); + } + } + + stringstream ss; + int i = 0; + char name[30]; + + printf("*** thread loop start ***\n"); + g_flag = 1; + while (g_flag == 1) { + sprintf(name, "myrule%d", i); + shec->create_ruleset(name, *crush, &ss); + i++; + } + printf("*** thread loop end ***\n"); + + return NULL; +} + +void* thread4(void* pParam) +{ + ErasureCodeShec* shec = (ErasureCodeShec*) pParam; + + bufferlist in; + in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//length = 62 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//124 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//186 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//248 + ); + set want_to_encode; + for (unsigned int i = 0; i < shec->get_chunk_count(); i++) { + want_to_encode.insert(i); + } + + map encoded; + + printf("*** thread loop start ***\n"); + g_flag = 1; + while (g_flag == 1) { + shec->encode(want_to_encode, in, &encoded); + encoded.clear(); + } + printf("*** thread loop end ***\n"); + + return NULL; +} + +void* thread5(void* pParam) +{ + ErasureCodeShec* shec = (ErasureCodeShec*) pParam; + + bufferlist in; + in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//length = 62 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//124 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//186 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//248 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//310 + ); + set want_to_encode; + for (unsigned int i = 0; i < shec->get_chunk_count(); i++) { + want_to_encode.insert(i); + } + map encoded; + shec->encode(want_to_encode, in, &encoded); + + int want_to_decode[] = { 0, 1, 2, 3, 4, 5 }; + map decoded; + + printf("*** thread loop start ***\n"); + g_flag = 1; + while (g_flag == 1) { + shec->decode(set(want_to_decode, want_to_decode + 2), encoded, + &decoded); + decoded.clear(); + } + printf("*** thread loop end ***\n"); + + return NULL; +} diff --git a/src/test/erasure-code/TestErasureCodeShec_all.cc b/src/test/erasure-code/TestErasureCodeShec_all.cc new file mode 100644 index 00000000000..03c3ce6ac69 --- /dev/null +++ b/src/test/erasure-code/TestErasureCodeShec_all.cc @@ -0,0 +1,330 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2014,2015 FUJITSU LIMITED + * + * Author: Shotaro Kawaguchi + * Author: Takanori Nakao + * Author: Takeshi Miyamae + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + */ + +// SUMMARY: TestErasureCodeShec combination of k,m,c by 301 patterns + +#include + +#include "crush/CrushWrapper.h" +#include "osd/osd_types.h" +#include "include/stringify.h" +#include "global/global_init.h" +#include "erasure-code/shec/ErasureCodeShec.h" +#include "erasure-code/ErasureCodePlugin.h" +#include "common/ceph_argparse.h" +#include "global/global_context.h" +#include "gtest/gtest.h" + +struct Param_d { + char* k; + char* m; + char* c; + int ch_size; + char sk[16]; + char sm[16]; + char sc[16]; +}; +struct Param_d param[301]; + +unsigned int g_recover = 0; +unsigned int g_cannot_recover = 0; +struct Recover_d { + int k; + int m; + int c; + set want; + set avail; +}; +struct std::vector cannot_recover; + +class ParameterTest : public ::testing::TestWithParam { + +}; + +TEST_P(ParameterTest, parameter_all) +{ + int result; + //get parameters + char* k = GetParam().k; + char* m = GetParam().m; + char* c = GetParam().c; + int c_size = GetParam().ch_size; + int i_k = atoi(k); + int i_m = atoi(m); + int i_c = atoi(c); + + //init + ErasureCodeShecTableCache tcache; + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = ""; + (*parameters)["directory"] = "/usr/lib64/ceph/erasure-code"; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = k; + (*parameters)["m"] = m; + (*parameters)["c"] = c; + + result = shec->init(*parameters); + + //check parameters + EXPECT_EQ(i_k, shec->k); + EXPECT_EQ(i_m, shec->m); + EXPECT_EQ(i_c, shec->c); + EXPECT_EQ(8u, shec->w); + EXPECT_EQ(ErasureCodeShec::MULTIPLE, shec->technique); + EXPECT_STREQ("default", shec->ruleset_root.c_str()); + EXPECT_STREQ("osd", shec->ruleset_failure_domain.c_str()); + EXPECT_TRUE(shec->matrix != NULL); + EXPECT_EQ(0, result); + + //minimum_to_decode + //want_to_decode will be a combination that chooses 1~c from k+m + set want_to_decode, available_chunks, minimum_chunks; + int array_want_to_decode[shec->get_chunk_count()]; + struct Recover_d comb; + + for (int w = 1; w <= i_c; w++) { + const unsigned int r = w; // combination(k+m,r) + + for (unsigned int i = 0; i < r; ++i) { + array_want_to_decode[i] = 1; + } + for (unsigned int i = r; i < shec->get_chunk_count(); ++i) { + array_want_to_decode[i] = 0; + } + + do { + for (unsigned int i = 0; i < shec->get_chunk_count(); i++) { + available_chunks.insert(i); + } + for (unsigned int i = 0; i < shec->get_chunk_count(); i++) { + if (array_want_to_decode[i]) { + want_to_decode.insert(i); + available_chunks.erase(i); + } + } + + result = shec->minimum_to_decode(want_to_decode, available_chunks, + &minimum_chunks); + + if (result == 0){ + EXPECT_EQ(0, result); + EXPECT_TRUE(minimum_chunks.size()); + g_recover++; + } else { + EXPECT_EQ(-EIO, result); + EXPECT_EQ(0, minimum_chunks.size()); + g_cannot_recover++; + comb.k = shec->k; + comb.m = shec->m; + comb.c = shec->c; + comb.want = want_to_decode; + comb.avail = available_chunks; + cannot_recover.push_back(comb); + } + + want_to_decode.clear(); + available_chunks.clear(); + minimum_chunks.clear(); + } while (std::prev_permutation( + array_want_to_decode, + array_want_to_decode + shec->get_chunk_count())); + } + + //minimum_to_decode_with_cost + set want_to_decode_with_cost, minimum_chunks_with_cost; + map available_chunks_with_cost; + + for (unsigned int i = 0; i < 1; i++) { + want_to_decode_with_cost.insert(i); + } + for (unsigned int i = 0; i < shec->get_chunk_count(); i++) { + available_chunks_with_cost[i] = i; + } + + result = shec->minimum_to_decode_with_cost( + want_to_decode_with_cost, + available_chunks_with_cost, + &minimum_chunks_with_cost); + EXPECT_EQ(0, result); + EXPECT_TRUE(minimum_chunks_with_cost.size()); + + //encode + bufferlist in; + set want_to_encode; + map encoded; + + in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//length = 62 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//124 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//186 + "012345"//192 + ); + for (unsigned int i = 0; i < shec->get_chunk_count(); i++) { + want_to_encode.insert(i); + } + + result = shec->encode(want_to_encode, in, &encoded); + EXPECT_EQ(0, result); + EXPECT_EQ(i_k+i_m, encoded.size()); + EXPECT_EQ(c_size, encoded[0].length()); + + //decode + int want_to_decode2[i_k + i_m]; + map decoded; + + for (unsigned int i = 0; i < shec->get_chunk_count(); i++) { + want_to_decode2[i] = i; + } + + result = shec->decode(set(want_to_decode2, want_to_decode2 + 2), + encoded, &decoded); + EXPECT_EQ(0, result); + EXPECT_EQ(2u, decoded.size()); + EXPECT_EQ(c_size, decoded[0].length()); + + //check encoded,decoded + bufferlist out1, out2, usable; + + //out1 is "encoded" + for (unsigned int i = 0; i < encoded.size(); i++) { + out1.append(encoded[i]); + } + + //out2 is "decoded" + shec->decode_concat(encoded, &out2); + usable.substr_of(out2, 0, in.length()); + + EXPECT_FALSE(out1 == in); + EXPECT_TRUE(usable == in); + + //create_ruleset + stringstream ss; + CrushWrapper *crush = new CrushWrapper; + crush->create(); + crush->set_type_name(2, "root"); + crush->set_type_name(1, "host"); + crush->set_type_name(0, "osd"); + + int rootno; + crush->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1, 2, 0, NULL, + NULL, &rootno); + crush->set_item_name(rootno, "default"); + + map < string, string > loc; + loc["root"] = "default"; + + int num_host = 2; + int num_osd = 5; + int osd = 0; + for (int h = 0; h < num_host; ++h) { + loc["host"] = string("host-") + stringify(h); + for (int o = 0; o < num_osd; ++o, ++osd) { + crush->insert_item(g_ceph_context, osd, 1.0, + string("osd.") + stringify(osd), loc); + } + } + + result = shec->create_ruleset("myrule", *crush, &ss); + EXPECT_EQ(0, result); + EXPECT_STREQ("myrule", crush->rule_name_map[0].c_str()); + + //get_chunk_count + EXPECT_EQ(i_k+i_m, shec->get_chunk_count()); + + //get_data_chunk_count + EXPECT_EQ(i_k, shec->get_data_chunk_count()); + + //get_chunk_size + EXPECT_EQ(c_size, shec->get_chunk_size(192)); + + delete shec; + delete parameters; + delete crush; +} + +INSTANTIATE_TEST_CASE_P(Test, ParameterTest, ::testing::ValuesIn(param)); + +int main(int argc, char **argv) +{ + int i = 0; + int r; + const int kObjectSize = 192; + unsigned alignment, tail, padded_length; + float recovery_percentage; + + //make_kmc + for (unsigned int k = 1; k <= 12; k++) { + for (unsigned int m = 1; (m <= k) && (k + m <= 20); m++) { + for (unsigned int c = 1; c <= m; c++) { + sprintf(param[i].sk, "%d", k); + sprintf(param[i].sm, "%d", m); + sprintf(param[i].sc, "%d", c); + + param[i].k = param[i].sk; + param[i].m = param[i].sm; + param[i].c = param[i].sc; + + alignment = k * 8 * sizeof(int); + tail = kObjectSize % alignment; + padded_length = kObjectSize + (tail ? (alignment - tail) : 0); + param[i].ch_size = padded_length / k; + i++; + } + } + } + + vector args; + argv_to_vec(argc, (const char **) argv, args); + + global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0); + common_init_finish(g_ceph_context); + + ::testing::InitGoogleTest(&argc, argv); + + r = RUN_ALL_TESTS(); + + std::cout << "minimum_to_decode:recover_num = " << g_recover << std::endl; + std::cout << "minimum_to_decode:cannot_recover_num = " << g_cannot_recover + << std::endl; + recovery_percentage = 100.0 + - (float) (100.0 * g_cannot_recover / (g_recover + g_cannot_recover)); + printf("recovery_percentage:%f\n",recovery_percentage); + if (recovery_percentage > 99.0) { + std::cout << "[ OK ] Recovery percentage is more than 99.0%" + << std::endl; + } else { + std::cout << "[ NG ] Recovery percentage is less than 99.0%" + << std::endl; + } + std::cout << "cannot recovery patterns:" << std::endl; + for (std::vector::const_iterator i = cannot_recover.begin(); + i != cannot_recover.end(); i++) { + std::cout << "---" << std::endl; + std::cout << "k = " << i->k << ", m = " << i->m << ", c = " << i->c + << std::endl; + std::cout << "want_to_decode :" << i->want << std::endl; + std::cout << "available_chunks:" << i->avail << std::endl; + } + std::cout << "---" << std::endl; + + return r; +} diff --git a/src/test/erasure-code/TestErasureCodeShec_thread.cc b/src/test/erasure-code/TestErasureCodeShec_thread.cc new file mode 100644 index 00000000000..fdd6bfeee3b --- /dev/null +++ b/src/test/erasure-code/TestErasureCodeShec_thread.cc @@ -0,0 +1,231 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2014,2015 FUJITSU LIMITED + * + * Author: Shotaro Kawaguchi + * Author: Takanori Nakao + * Author: Takeshi Miyamae + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + */ + +// SUMMARY: TestErasureCodeShec executes some threads at the same time + +#include +#include + +#include "crush/CrushWrapper.h" +#include "osd/osd_types.h" +#include "include/stringify.h" +#include "global/global_init.h" +#include "erasure-code/shec/ErasureCodeShec.h" +#include "erasure-code/ErasureCodePlugin.h" +#include "common/ceph_argparse.h" +#include "global/global_context.h" +#include "gtest/gtest.h" + +void* thread1(void* pParam); + +class TestParam { +public: + string k, m, c, w; +}; + +TEST(ErasureCodeShec, thread) +{ + TestParam param1, param2, param3, param4, param5; + param1.k = "6"; + param1.m = "4"; + param1.c = "3"; + param1.w = "8"; + + param2.k = "4"; + param2.m = "3"; + param2.c = "2"; + param2.w = "16"; + + param3.k = "10"; + param3.m = "8"; + param3.c = "4"; + param3.w = "32"; + + param4.k = "5"; + param4.m = "5"; + param4.c = "5"; + param4.w = "8"; + + param5.k = "9"; + param5.m = "9"; + param5.c = "6"; + param5.w = "16"; + + pthread_t tid1, tid2, tid3, tid4, tid5; + pthread_create(&tid1, NULL, thread1, (void*) ¶m1); + std::cout << "thread1 start " << std::endl; + pthread_create(&tid2, NULL, thread1, (void*) ¶m2); + std::cout << "thread2 start " << std::endl; + pthread_create(&tid3, NULL, thread1, (void*) ¶m3); + std::cout << "thread3 start " << std::endl; + pthread_create(&tid4, NULL, thread1, (void*) ¶m4); + std::cout << "thread4 start " << std::endl; + pthread_create(&tid5, NULL, thread1, (void*) ¶m5); + std::cout << "thread5 start " << std::endl; + + pthread_join(tid1, NULL); + pthread_join(tid2, NULL); + pthread_join(tid3, NULL); + pthread_join(tid4, NULL); + pthread_join(tid5, NULL); +} + +int main(int argc, char **argv) +{ + vector args; + argv_to_vec(argc, (const char **) argv, args); + + global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0); + common_init_finish(g_ceph_context); + + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +void* thread1(void* pParam) +{ + TestParam* param = (TestParam*) pParam; + + time_t start, end; + int r; + + ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance(); + + instance.disable_dlclose = true; + { + Mutex::Locker l(instance.lock); + __erasure_code_init((char*) "shec", (char*) ""); + } + std::cout << "__erasure_code_init finish " << std::endl; + + //encode + bufferlist in; + set want_to_encode; + map encoded; + + in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" //length = 62 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//124 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//186 + "012345"//192 + ); + + //decode + int want_to_decode[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + map decoded; + bufferlist out1, out2, usable; + + time(&start); + time(&end); + const int kTestSec = 60; + ErasureCodeShecTableCache tcache; + + while (kTestSec >= (end - start)) { + //init + ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde( + tcache, + ErasureCodeShec::MULTIPLE); + map < std::string, std::string > *parameters = new map(); + (*parameters)["plugin"] = "shec"; + (*parameters)["technique"] = "multiple"; + (*parameters)["ruleset-failure-domain"] = "osd"; + (*parameters)["k"] = param->k; + (*parameters)["m"] = param->m; + (*parameters)["c"] = param->c; + (*parameters)["w"] = param->w; + r = shec->init(*parameters); + + int i_k = std::atoi(param->k.c_str()); + int i_m = std::atoi(param->m.c_str()); + int i_c = std::atoi(param->c.c_str()); + int i_w = std::atoi(param->w.c_str()); + + EXPECT_EQ(0, r); + EXPECT_EQ(i_k, shec->k); + EXPECT_EQ(i_m, shec->m); + EXPECT_EQ(i_c, shec->c); + EXPECT_EQ(i_w, shec->w); + EXPECT_EQ(ErasureCodeShec::MULTIPLE, shec->technique); + EXPECT_STREQ("default", shec->ruleset_root.c_str()); + EXPECT_STREQ("osd", shec->ruleset_failure_domain.c_str()); + EXPECT_TRUE(shec->matrix != NULL); + if ((shec->matrix == NULL)) { + std::cout << "matrix is null" << std::endl; + // error + break; + } + + //encode + for (unsigned int i = 0; i < shec->get_chunk_count(); i++) { + want_to_encode.insert(i); + } + r = shec->encode(want_to_encode, in, &encoded); + + EXPECT_EQ(0, r); + EXPECT_EQ(shec->get_chunk_count(), encoded.size()); + EXPECT_EQ(shec->get_chunk_size(in.length()), encoded[0].length()); + + if (r != 0) { + std::cout << "error in encode" << std::endl; + //error + break; + } + + //decode + r = shec->decode(set(want_to_decode, want_to_decode + 2), + encoded, + &decoded); + + EXPECT_EQ(0, r); + EXPECT_EQ(2u, decoded.size()); + EXPECT_EQ(shec->get_chunk_size(in.length()), decoded[0].length()); + + if (r != 0) { + std::cout << "error in decode" << std::endl; + //error + break; + } + + //out1 is "encoded" + for (unsigned int i = 0; i < encoded.size(); i++) { + out1.append(encoded[i]); + } + //out2 is "decoded" + shec->decode_concat(encoded, &out2); + usable.substr_of(out2, 0, in.length()); + EXPECT_FALSE(out1 == in); + EXPECT_TRUE(usable == in); + if (out1 == in || !(usable == in)) { + std::cout << "encode(decode) result is not correct" << std::endl; + break; + } + + delete shec; + delete parameters; + want_to_encode.clear(); + encoded.clear(); + decoded.clear(); + out1.clear(); + out2.clear(); + usable.clear(); + + time(&end); + } + + return NULL; +}