From 0114b3355bd968498aaa76dc4ff61d8ca860923d Mon Sep 17 00:00:00 2001 From: Loic Dachary Date: Tue, 3 Jun 2014 22:40:31 +0200 Subject: [PATCH] erasure-code: rework ErasureCode*::parse methods The ErasureCode::parse virtual function is derived in ErasureCode{Jerasure,Isa}. It is reworked to share parsing arguments the various techniques have in common. The logic is otherwise unmodified. http://tracker.ceph.com/issues/8496 Refs: #8496 Signed-off-by: Loic Dachary --- src/erasure-code/ErasureCode.cc | 6 + src/erasure-code/ErasureCode.h | 3 + src/erasure-code/isa/ErasureCodeIsa.cc | 37 ++++--- src/erasure-code/isa/ErasureCodeIsa.h | 11 +- .../jerasure/ErasureCodeJerasure.cc | 104 +++++++++++------- .../jerasure/ErasureCodeJerasure.h | 71 +++++++----- 6 files changed, 148 insertions(+), 84 deletions(-) diff --git a/src/erasure-code/ErasureCode.cc b/src/erasure-code/ErasureCode.cc index 25de8e137e3..498a06b0617 100644 --- a/src/erasure-code/ErasureCode.cc +++ b/src/erasure-code/ErasureCode.cc @@ -143,6 +143,12 @@ int ErasureCode::decode_chunks(const set &want_to_read, assert("ErasureCode::decode_chunks not implemented" == 0); } +int ErasureCode::parse(const map ¶meters, + ostream *ss) +{ + return 0; +} + int ErasureCode::to_int(const std::string &name, const map ¶meters, int *value, diff --git a/src/erasure-code/ErasureCode.h b/src/erasure-code/ErasureCode.h index 63306db2378..b70e7f87d7f 100644 --- a/src/erasure-code/ErasureCode.h +++ b/src/erasure-code/ErasureCode.h @@ -55,6 +55,9 @@ namespace ceph { const map &chunks, map *decoded); + virtual int parse(const map ¶meters, + ostream *ss); + static int to_int(const std::string &name, const map ¶meters, int *value, diff --git a/src/erasure-code/isa/ErasureCodeIsa.cc b/src/erasure-code/isa/ErasureCodeIsa.cc index 8d9e7c00ba5..b4cc4c37239 100644 --- a/src/erasure-code/isa/ErasureCodeIsa.cc +++ b/src/erasure-code/isa/ErasureCodeIsa.cc @@ -63,7 +63,9 @@ ErasureCodeIsa::init(const map ¶meters) parameter = parameters.find("ruleset-failure-domain"); if (parameter != parameters.end()) ruleset_failure_domain = parameter->second; - parse(parameters); + ostringstream ss; + if (parse(parameters, &ss)) + derr << ss.str() << dendl; prepare(); } @@ -385,38 +387,47 @@ ErasureCodeIsaDefault::get_alignment() const // ----------------------------------------------------------------------------- -void -ErasureCodeIsaDefault::parse(const map ¶meters) +int ErasureCodeIsaDefault::parse(const map ¶meters, + ostream *ss) { - k = to_int("k", parameters, DEFAULT_K); - m = to_int("m", parameters, DEFAULT_M); + int err = ErasureCode::parse(parameters, ss); + err |= to_int("k", parameters, &k, DEFAULT_K, ss); + err |= to_int("m", parameters, &m, DEFAULT_M, ss); if (matrixtype == kVandermonde) { - // these are verified safe values evaluted using the benchmarktool and 10*(combinatoric for maximum loss) random full erasures + // these are verified safe values evaluated using the + // benchmarktool and 10*(combinatoric for maximum loss) random + // full erasures if (k > 32) { - derr << "Vandermonde: m=" << m - << " should be less/equal than 32 : revert to k=32" << dendl; + *ss << "Vandermonde: m=" << m + << " should be less/equal than 32 : revert to k=32" << std::endl; k = 32; + err = -EINVAL; } if (m > 4) { - derr << "Vandermonde: m=" << m - << " should be less than 5 to guarantee an MDS codec: revert to m=4" << dendl; + *ss << "Vandermonde: m=" << m + << " should be less than 5 to guarantee an MDS codec:" + << " revert to m=4" << std::endl; m = 4; + err = -EINVAL; } switch (m) { case 4: if (k > 21) { - derr << "Vandermonde: K=" << k - << " should be less than 22 to guarantee an MDS codec with m=4: revert to k=21" << dendl; + *ss << "Vandermonde: k=" << k + << " should be less than 22 to guarantee an MDS" + << " codec with m=4: revert to k=21" << std::endl; k = 21; + err = -EINVAL; } break; default: ; } } + return err; } // ----------------------------------------------------------------------------- diff --git a/src/erasure-code/isa/ErasureCodeIsa.h b/src/erasure-code/isa/ErasureCodeIsa.h index f68fb56f962..8ca74b471fa 100644 --- a/src/erasure-code/isa/ErasureCodeIsa.h +++ b/src/erasure-code/isa/ErasureCodeIsa.h @@ -94,7 +94,9 @@ public: virtual unsigned get_alignment() const = 0; - virtual void parse(const map ¶meters) = 0; + virtual int parse(const map ¶meters, + ostream *ss) = 0; virtual void prepare() = 0; @@ -125,7 +127,8 @@ public: return g_decode_tbls_lru.size(); } - // we implement an LRU cache for coding matrix - the cache size is sufficient up to (12,4) decodings + // we implement an LRU cache for coding matrix - the cache size is + // sufficient up to (12,4) decodings typedef std::pair::iterator, bufferptr> lru_entry_t; std::map g_decode_tbls_map; @@ -161,7 +164,9 @@ public: virtual unsigned get_alignment() const; - virtual void parse(const map ¶meters); + virtual int parse(const map ¶meters, + ostream *ss); virtual void prepare(); diff --git a/src/erasure-code/jerasure/ErasureCodeJerasure.cc b/src/erasure-code/jerasure/ErasureCodeJerasure.cc index 536ce2918aa..e243f4bc442 100644 --- a/src/erasure-code/jerasure/ErasureCodeJerasure.cc +++ b/src/erasure-code/jerasure/ErasureCodeJerasure.cc @@ -55,10 +55,22 @@ void ErasureCodeJerasure::init(const map ¶meters) parameter = parameters.find("ruleset-failure-domain"); if (parameter != parameters.end()) ruleset_failure_domain = parameter->second; - parse(parameters); + ostringstream ss; + if (parse(parameters, &ss)) + derr << ss.str() << dendl; prepare(); } +int ErasureCodeJerasure::parse(const map ¶meters, + ostream *ss) +{ + int err = ErasureCode::parse(parameters, ss); + err |= to_int("k", parameters, &k, DEFAULT_K, ss); + err |= to_int("m", parameters, &m, DEFAULT_M, ss); + err |= to_int("w", parameters, &w, DEFAULT_W, ss); + return err; +} + unsigned int ErasureCodeJerasure::get_chunk_size(unsigned int object_size) const { unsigned alignment = get_alignment(); @@ -165,17 +177,19 @@ unsigned ErasureCodeJerasureReedSolomonVandermonde::get_alignment() const } } -void ErasureCodeJerasureReedSolomonVandermonde::parse(const map ¶meters) +int ErasureCodeJerasureReedSolomonVandermonde::parse(const map ¶meters, + ostream *ss) { - k = to_int("k", parameters, DEFAULT_K); - m = to_int("m", parameters, DEFAULT_M); - w = to_int("w", parameters, DEFAULT_W); + int err = ErasureCodeJerasure::parse(parameters, ss); if (w != 8 && w != 16 && w != 32) { - derr << "ReedSolomonVandermonde: w=" << w - << " must be one of {8, 16, 32} : revert to " << DEFAULT_W << dendl; + *ss << "ReedSolomonVandermonde: w=" << w + << " must be one of {8, 16, 32} : revert to DEFAULT_W " << std::endl; w = DEFAULT_W; + err = -EINVAL; } - per_chunk_alignment = to_bool("jerasure-per-chunk-alignment", parameters, false); + err |= to_bool("jerasure-per-chunk-alignement", parameters, + &per_chunk_alignment, false, ss); + return err; } void ErasureCodeJerasureReedSolomonVandermonde::prepare() @@ -213,16 +227,18 @@ unsigned ErasureCodeJerasureReedSolomonRAID6::get_alignment() const } } -void ErasureCodeJerasureReedSolomonRAID6::parse(const map ¶meters) +int ErasureCodeJerasureReedSolomonRAID6::parse(const map ¶meters, + ostream *ss) { - k = to_int("k", parameters, DEFAULT_K); + int err = ErasureCodeJerasure::parse(parameters, ss); m = 2; - w = to_int("w", parameters, DEFAULT_W); if (w != 8 && w != 16 && w != 32) { - derr << "ReedSolomonRAID6: w=" << w - << " must be one of {8, 16, 32} : revert to 8 " << dendl; + *ss << "ReedSolomonRAID6: w=" << w + << " must be one of {8, 16, 32} : revert to 8 " << std::endl; w = 8; + err = -EINVAL; } + return err; } void ErasureCodeJerasureReedSolomonRAID6::prepare() @@ -266,19 +282,21 @@ unsigned ErasureCodeJerasureCauchy::get_alignment() const } } -void ErasureCodeJerasureCauchy::parse(const map ¶meters) +int ErasureCodeJerasureCauchy::parse(const map ¶meters, + ostream *ss) { - k = to_int("k", parameters, DEFAULT_K); - m = to_int("m", parameters, DEFAULT_M); - w = to_int("w", parameters, DEFAULT_W); + int err = ErasureCodeJerasure::parse(parameters, ss); if (w != 8 && w != 16 && w != 32) { - derr << "Cauchy: w=" << w - << " must be one of {8, 16, 32} : revert to " - << DEFAULT_W << dendl; + *ss << "Cauchy: w=" << w + << " must be one of {8, 16, 32} : revert to " + << DEFAULT_W << std::endl; w = DEFAULT_W; + err = -EINVAL; } - packetsize = to_int("packetsize", parameters, DEFAULT_PACKETSIZE); - per_chunk_alignment = to_bool("jerasure-per-chunk-alignment", parameters, false); + err |= to_int("packetsize", parameters, &packetsize, DEFAULT_PACKETSIZE, ss); + err |= to_bool("jerasure-per-chunk-alignement", parameters, + &per_chunk_alignment, false, ss); + return err; } void ErasureCodeJerasureCauchy::prepare_schedule(int *matrix) @@ -343,38 +361,39 @@ unsigned ErasureCodeJerasureLiberation::get_alignment() const return alignment; } -void ErasureCodeJerasureLiberation::parse(const map ¶meters) +int ErasureCodeJerasureLiberation::parse(const map ¶meters, + ostream *ss) { - k = to_int("k", parameters, DEFAULT_K); - m = to_int("m", parameters, DEFAULT_M); - w = to_int("w", parameters, DEFAULT_W); - packetsize = to_int("packetsize", parameters, DEFAULT_PACKETSIZE); + int err = ErasureCodeJerasure::parse(parameters, ss); + err |= to_int("packetsize", parameters, &packetsize, DEFAULT_PACKETSIZE, ss); bool error = false; if (k > w) { - derr << "k=" << k << " must be less than or equal to w=" << w << dendl; + *ss << "k=" << k << " must be less than or equal to w=" << w << std::endl; error = true; } if (w <= 2 || !is_prime(w)) { - derr << "w=" << w << " must be greater than two and be prime" << dendl; + *ss << "w=" << w << " must be greater than two and be prime" << std::endl; error = true; } if (packetsize == 0) { - derr << "packetsize=" << packetsize << " must be set" << dendl; + *ss << "packetsize=" << packetsize << " must be set" << std::endl; error = true; } if ((packetsize%(sizeof(int))) != 0) { - derr << "packetsize=" << packetsize - << " must be a multiple of sizeof(int) = " << sizeof(int) << dendl; + *ss << "packetsize=" << packetsize + << " must be a multiple of sizeof(int) = " << sizeof(int) << std::endl; error = true; } if (error) { - derr << "reverting to k=" << DEFAULT_K << ", w=" - << DEFAULT_W << ", packetsize=" << DEFAULT_PACKETSIZE << dendl; + *ss << "reverting to k=" << DEFAULT_K << ", w=" + << DEFAULT_W << ", packetsize=" << DEFAULT_PACKETSIZE << std::endl; k = DEFAULT_K; w = DEFAULT_W; packetsize = DEFAULT_PACKETSIZE; + err = -EINVAL; } + return err; } void ErasureCodeJerasureLiberation::prepare() @@ -395,28 +414,31 @@ void ErasureCodeJerasureBlaumRoth::prepare() // // ErasureCodeJerasureLiber8tion // -void ErasureCodeJerasureLiber8tion::parse(const map ¶meters) +int ErasureCodeJerasureLiber8tion::parse(const map ¶meters, + ostream *ss) { - k = to_int("k", parameters, DEFAULT_K); + int err = ErasureCodeJerasure::parse(parameters, ss); m = DEFAULT_M; w = DEFAULT_W; - packetsize = to_int("packetsize", parameters, DEFAULT_PACKETSIZE); + err |= to_int("packetsize", parameters, &packetsize, DEFAULT_PACKETSIZE, ss); bool error = false; if (k > w) { - derr << "k=" << k << " must be less than or equal to w=" << w << dendl; + *ss << "k=" << k << " must be less than or equal to w=" << w << std::endl; error = true; } if (packetsize == 0) { - derr << "packetsize=" << packetsize << " must be set" << dendl; + *ss << "packetsize=" << packetsize << " must be set" << std::endl; error = true; } if (error) { - derr << "reverting to k=" << DEFAULT_K << ", packetsize=" - << DEFAULT_PACKETSIZE << dendl; + *ss << "reverting to k=" << DEFAULT_K << ", packetsize=" + << DEFAULT_PACKETSIZE << std::endl; k = DEFAULT_K; packetsize = DEFAULT_PACKETSIZE; + err = -EINVAL; } + return err; } void ErasureCodeJerasureLiber8tion::prepare() diff --git a/src/erasure-code/jerasure/ErasureCodeJerasure.h b/src/erasure-code/jerasure/ErasureCodeJerasure.h index 3f39f69d936..c7c16b8dcff 100644 --- a/src/erasure-code/jerasure/ErasureCodeJerasure.h +++ b/src/erasure-code/jerasure/ErasureCodeJerasure.h @@ -3,7 +3,7 @@ /* * Ceph distributed storage system * - * Copyright (C) 2013 Cloudwatt + * Copyright (C) 2013, 2014 Cloudwatt * * Author: Loic Dachary * @@ -22,14 +22,20 @@ class ErasureCodeJerasure : public ErasureCode { public: int k; + int DEFAULT_K; int m; + int DEFAULT_M; int w; + int DEFAULT_W; const char *technique; string ruleset_root; string ruleset_failure_domain; bool per_chunk_alignment; ErasureCodeJerasure(const char *_technique) : + DEFAULT_K(2), + DEFAULT_M(1), + DEFAULT_W(8), technique(_technique), ruleset_root("default"), ruleset_failure_domain("host"), @@ -38,6 +44,9 @@ public: virtual ~ErasureCodeJerasure() {} + virtual int parse(const map ¶meters, + ostream *ss); + virtual int create_ruleset(const string &name, CrushWrapper &crush, ostream *ss) const; @@ -68,22 +77,22 @@ public: char **coding, int blocksize) = 0; virtual unsigned get_alignment() const = 0; - virtual void parse(const map ¶meters) = 0; virtual void prepare() = 0; static bool is_prime(int value); }; class ErasureCodeJerasureReedSolomonVandermonde : public ErasureCodeJerasure { public: - static const int DEFAULT_K = 7; - static const int DEFAULT_M = 3; - static const int DEFAULT_W = 8; int *matrix; ErasureCodeJerasureReedSolomonVandermonde() : ErasureCodeJerasure("reed_sol_van"), matrix(0) - { } + { + DEFAULT_K = 7; + DEFAULT_M = 3; + DEFAULT_W = 8; + } virtual ~ErasureCodeJerasureReedSolomonVandermonde() { if (matrix) free(matrix); @@ -97,20 +106,22 @@ public: char **coding, int blocksize); virtual unsigned get_alignment() const; - virtual void parse(const map ¶meters); + virtual int parse(const map ¶meters, + ostream *ss); virtual void prepare(); }; class ErasureCodeJerasureReedSolomonRAID6 : public ErasureCodeJerasure { public: - static const int DEFAULT_K = 7; - static const int DEFAULT_W = 8; int *matrix; ErasureCodeJerasureReedSolomonRAID6() : ErasureCodeJerasure("reed_sol_r6_op"), matrix(0) - { } + { + DEFAULT_K = 7; + DEFAULT_W = 8; + } virtual ~ErasureCodeJerasureReedSolomonRAID6() { if (matrix) free(matrix); @@ -124,15 +135,13 @@ public: char **coding, int blocksize); virtual unsigned get_alignment() const; - virtual void parse(const map ¶meters); + virtual int parse(const map ¶meters, + ostream *ss); virtual void prepare(); }; class ErasureCodeJerasureCauchy : public ErasureCodeJerasure { public: - static const int DEFAULT_K = 7; - static const int DEFAULT_M = 3; - static const int DEFAULT_W = 8; static const int DEFAULT_PACKETSIZE = 2048; int *bitmatrix; int **schedule; @@ -142,7 +151,11 @@ public: ErasureCodeJerasure(technique), bitmatrix(0), schedule(0) - { } + { + DEFAULT_K = 7; + DEFAULT_M = 3; + DEFAULT_W = 8; + } virtual ~ErasureCodeJerasureCauchy() { if (bitmatrix) free(bitmatrix); @@ -158,7 +171,8 @@ public: char **coding, int blocksize); virtual unsigned get_alignment() const; - virtual void parse(const map ¶meters); + virtual int parse(const map ¶meters, + ostream *ss); void prepare_schedule(int *matrix); }; @@ -182,9 +196,6 @@ public: class ErasureCodeJerasureLiberation : public ErasureCodeJerasure { public: - static const int DEFAULT_K = 2; - static const int DEFAULT_M = 2; - static const int DEFAULT_W = 7; static const int DEFAULT_PACKETSIZE = 2048; int *bitmatrix; int **schedule; @@ -194,7 +205,11 @@ public: ErasureCodeJerasure(technique), bitmatrix(0), schedule(0) - { } + { + DEFAULT_K = 2; + DEFAULT_M = 2; + DEFAULT_W = 7; + } virtual ~ErasureCodeJerasureLiberation(); virtual void jerasure_encode(char **data, @@ -205,7 +220,8 @@ public: char **coding, int blocksize); virtual unsigned get_alignment() const; - virtual void parse(const map ¶meters); + virtual int parse(const map ¶meters, + ostream *ss); virtual void prepare(); }; @@ -220,15 +236,16 @@ public: class ErasureCodeJerasureLiber8tion : public ErasureCodeJerasureLiberation { public: - static const int DEFAULT_K = 2; - static const int DEFAULT_M = 2; - static const int DEFAULT_W = 8; - ErasureCodeJerasureLiber8tion() : ErasureCodeJerasureLiberation("liber8tion") - {} + { + DEFAULT_K = 2; + DEFAULT_M = 2; + DEFAULT_W = 8; + } - virtual void parse(const map ¶meters); + virtual int parse(const map ¶meters, + ostream *ss); virtual void prepare(); }; -- 2.47.3