From 1ff94ccf89b1bec45d602918474b6d89cb50519e Mon Sep 17 00:00:00 2001
From: Andreas-Joachim Peters <Andreas.Joachim.Peters@cern.ch>
Date: Tue, 9 Sep 2014 09:37:41 +0200
Subject: [PATCH] EC-ISA: avoid usage of table cache lock outside the class
 implementation by introducing the setEncodingTable/setEncodingCoefficient
 methods

---
 src/erasure-code/isa/ErasureCodeIsa.cc        | 18 ++++--
 .../isa/ErasureCodeIsaTableCache.cc           | 58 +++++++++++++++++--
 .../isa/ErasureCodeIsaTableCache.h            |  6 ++
 3 files changed, 72 insertions(+), 10 deletions(-)

diff --git a/src/erasure-code/isa/ErasureCodeIsa.cc b/src/erasure-code/isa/ErasureCodeIsa.cc
index f3a947e8180f..962ab81f0437 100644
--- a/src/erasure-code/isa/ErasureCodeIsa.cc
+++ b/src/erasure-code/isa/ErasureCodeIsa.cc
@@ -379,8 +379,6 @@ int ErasureCodeIsaDefault::parse(const map<std::string,
 void
 ErasureCodeIsaDefault::prepare()
 {
-  Mutex::Locker lock(tcache.codec_tables_guard);
-
   // setup shared encoding table and coefficients
   unsigned char** p_enc_table =
     tcache.getEncodingTable(matrixtype, k, m);
@@ -392,13 +390,17 @@ ErasureCodeIsaDefault::prepare()
     dout(10) << "[ cache tables ] creating coeff for k=" <<
       k << " m=" << m << dendl;
     // build encoding coefficients which need to be computed once for each (k,m)
-    *p_enc_coeff = (unsigned char*) malloc(k * (m + k));
-    encode_coeff = *p_enc_coeff;
+    encode_coeff = (unsigned char*) malloc(k * (m + k));
 
     if (matrixtype == kVandermonde)
       gf_gen_rs_matrix(encode_coeff, k + m, k);
     if (matrixtype == kCauchy)
       gf_gen_cauchy1_matrix(encode_coeff, k + m, k);
+
+      // either our new created coefficients are stored or if they have been
+      // created in the meanwhile the locally allocated coefficients will be
+      // freed by setEncodingCoefficient
+    encode_coeff = tcache.setEncodingCoefficient(matrixtype, k, m, encode_coeff);
   } else {
     encode_coeff = *p_enc_coeff;
   }
@@ -407,9 +409,13 @@ ErasureCodeIsaDefault::prepare()
     dout(10) << "[ cache tables ] creating tables for k=" <<
       k << " m=" << m << dendl;
     // build encoding table which needs to be computed once for each (k,m)
-    *p_enc_table = (unsigned char*) malloc(k * (m + k)*32);
-    encode_tbls = *p_enc_table;
+    encode_tbls = (unsigned char*) malloc(k * (m + k)*32);
     ec_init_tables(k, m, &encode_coeff[k * k], encode_tbls);
+
+    // either our new created table is stored or if it has been
+    // created in the meanwhile the locally allocated table will be
+    // freed by setEncodingTable
+    encode_tbls = tcache.setEncodingTable(matrixtype, k, m, encode_tbls);
   } else {
     encode_tbls = *p_enc_table;
   }
diff --git a/src/erasure-code/isa/ErasureCodeIsaTableCache.cc b/src/erasure-code/isa/ErasureCodeIsaTableCache.cc
index 88436bc290f1..003ff2224a4b 100644
--- a/src/erasure-code/isa/ErasureCodeIsaTableCache.cc
+++ b/src/erasure-code/isa/ErasureCodeIsaTableCache.cc
@@ -143,9 +143,15 @@ ErasureCodeIsaTableCache::getDecodingTablesLru(int matrix_type)
 unsigned char**
 ErasureCodeIsaTableCache::getEncodingTable(int matrix, int k, int m)
 {
-  // the caller must hold the guard mutex:
-  // => Mutex::Locker lock(codec_tables_guard);
+  Mutex::Locker lock(codec_tables_guard);
+  return getEncodingTableNoLock(matrix,k,m);
+}
 
+// -----------------------------------------------------------------------------
+
+unsigned char**
+ErasureCodeIsaTableCache::getEncodingTableNoLock(int matrix, int k, int m)
+{
   // create a pointer to store an encoding table address
   if (!encoding_table[matrix][k][m]) {
     encoding_table[matrix][k][m] = new (unsigned char*);
@@ -159,9 +165,15 @@ ErasureCodeIsaTableCache::getEncodingTable(int matrix, int k, int m)
 unsigned char**
 ErasureCodeIsaTableCache::getEncodingCoefficient(int matrix, int k, int m)
 {
-  // the caller must hold the guard mutex:
-  // => Mutex::Locker lock(codec_tables_guard);
+  Mutex::Locker lock(codec_tables_guard);
+  return getEncodingCoefficientNoLock(matrix,k,m);
+}
 
+// -----------------------------------------------------------------------------
+
+unsigned char**
+ErasureCodeIsaTableCache::getEncodingCoefficientNoLock(int matrix, int k, int m)
+{
   // create a pointer to store an encoding coefficients adddress
   if (!encoding_coefficient[matrix][k][m]) {
     encoding_coefficient[matrix][k][m] = new (unsigned char*);
@@ -172,6 +184,44 @@ ErasureCodeIsaTableCache::getEncodingCoefficient(int matrix, int k, int m)
 
 // -----------------------------------------------------------------------------
 
+unsigned char*
+ErasureCodeIsaTableCache::setEncodingTable(int matrix, int k, int m, unsigned char* ec_in_table)
+{
+  Mutex::Locker lock(codec_tables_guard);
+  unsigned char** ec_out_table = getEncodingTableNoLock(matrix, k, m);
+  if (*ec_out_table) {
+    // somebody might have deposited this table in the meanwhile, so clean
+    // the input table and return the stored one
+    free (ec_in_table);
+    return *ec_out_table;
+  } else {
+    // we store the provided input table and return this one
+    *encoding_table[matrix][k][m] = ec_in_table;
+    return ec_in_table;
+  }
+}
+
+// -----------------------------------------------------------------------------
+
+unsigned char*
+ErasureCodeIsaTableCache::setEncodingCoefficient(int matrix, int k, int m, unsigned char* ec_in_coeff)
+{
+  Mutex::Locker lock(codec_tables_guard);
+  unsigned char** ec_out_coeff = getEncodingCoefficientNoLock(matrix, k, m);
+  if (*ec_out_coeff) {
+    // somebody might have deposited these coefficients in the meanwhile, so clean
+    // the input coefficients and return the stored ones
+    free (ec_in_coeff);
+    return *ec_out_coeff;
+  } else {
+    // we store the provided input coefficients and return these
+    *encoding_coefficient[matrix][k][m] = ec_in_coeff;
+    return ec_in_coeff;
+  }
+}
+
+// -----------------------------------------------------------------------------
+
 Mutex*
 ErasureCodeIsaTableCache::getLock()
 {
diff --git a/src/erasure-code/isa/ErasureCodeIsaTableCache.h b/src/erasure-code/isa/ErasureCodeIsaTableCache.h
index 27ec603485d8..64aaae752af8 100644
--- a/src/erasure-code/isa/ErasureCodeIsaTableCache.h
+++ b/src/erasure-code/isa/ErasureCodeIsaTableCache.h
@@ -79,6 +79,12 @@ public:
   unsigned char** getEncodingTable(int matrix, int k, int m);
   unsigned char** getEncodingCoefficient(int matrix, int k, int m);
 
+  unsigned char** getEncodingTableNoLock(int matrix, int k, int m);
+  unsigned char** getEncodingCoefficientNoLock(int matrix, int k, int m);
+
+  unsigned char* setEncodingTable(int matrix, int k, int m, unsigned char*);
+  unsigned char* setEncodingCoefficient(int matrix, int k, int m, unsigned char*);
+
   int getDecodingTableCacheSize(int matrixtype = 0);
 
 private:
-- 
2.47.3