Compressor: Add Brotli Compressor

Signed-off-by: BI SHUN KE <aionshun@livemail.tw>

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 889b6f9fc7d20decc1530548108bb7c91268514e..9e8a21d0871daf1c59531c79d20105dcaa1d512c 100644 (file)
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -293,6 +293,11 @@ endif(WITH_LEVELDB)
 
 find_package(snappy REQUIRED)
 
+option(WITH_BROTLI "Brotli compression support" OFF)
+if(WITH_BROTLI)
+  set(HAVE_BROTLI TRUE)
+endif()
+
 option(WITH_LZ4 "LZ4 compression support" OFF)
 if(WITH_LZ4)
   find_package(LZ4 1.7 REQUIRED)

diff --git a/src/compressor/CMakeLists.txt b/src/compressor/CMakeLists.txt
index 798268449bccd44d4ed7364d0154b2b5fb24a46b..4caa3fca725ca013cf5b6b162c4e9517b88d903a 100644 (file)
--- a/src/compressor/CMakeLists.txt
+++ b/src/compressor/CMakeLists.txt
@@ -11,19 +11,27 @@ add_subdirectory(snappy)
 add_subdirectory(zlib)
 add_subdirectory(zstd)
 
-if (HAVE_LZ4)
+if(HAVE_LZ4)
   add_subdirectory(lz4)
 endif()
 
+if(HAVE_BROTLI)
+  add_subdirectory(brotli)
+endif()
+
 set(ceph_compressor_libs
     ceph_snappy
     ceph_zlib
     ceph_zstd)
 
-if (HAVE_LZ4)
+if(HAVE_LZ4)
   list(APPEND ceph_compressor_libs ceph_lz4)
 endif()
 
+if(HAVE_BROTLI)
+  list(APPEND ceph_compressor_libs ceph_brotli)
+endif()
+
 add_custom_target(compressor_plugins DEPENDS
     ${ceph_compressor_libs})
 
@@ -36,8 +44,11 @@ if(WITH_EMBEDDED)
       cephd_compressor_snappy
       cephd_compressor_zlib
       cephd_compressor_zstd)
-  if (HAVE_LZ4)
+  if(HAVE_LZ4)
     list(APPEND cephd_compressor_libs cephd_compressor_lz4)
   endif()
+  if(HAVE_BROTLI)
+    list(APPEND cephd_compressor_libs cephd_compressor_brotli)
+  endif()
   merge_static_libraries(cephd_compressor ${cephd_compressor_libs})
 endif()

diff --git a/src/compressor/Compressor.cc b/src/compressor/Compressor.cc
index ade3950fd7ee0f7c6cc79bb8d500ff82cd86cc1b..cca0c422d20ccd083bdf36f80b18df5374568e2d 100644 (file)
--- a/src/compressor/Compressor.cc
+++ b/src/compressor/Compressor.cc
@@ -30,6 +30,9 @@ const char * Compressor::get_comp_alg_name(int a) {
   case COMP_ALG_ZSTD: return "zstd";
 #ifdef HAVE_LZ4
   case COMP_ALG_LZ4: return "lz4";
+#endif
+#ifdef HAVE_BROTLI
+  case COMP_ALG_BROTLI: return "brotli";
 #endif
   default: return "???";
   }
@@ -45,6 +48,10 @@ boost::optional<Compressor::CompressionAlgorithm> Compressor::get_comp_alg_type(
 #ifdef HAVE_LZ4
   if (s == "lz4")
     return COMP_ALG_LZ4;
+#endif
+#ifdef HAVE_BROTLI
+  if (s == "brotli")
+    return COMP_ALG_BROTLI;
 #endif
   if (s == "" || s == "none")
     return COMP_ALG_NONE;

diff --git a/src/compressor/Compressor.h b/src/compressor/Compressor.h
index dba8f3460fea915c2ca06d851f7c719de310bb1f..cc343f8e8236e5a00325f3eea0dbefd39ab50e49 100644 (file)
--- a/src/compressor/Compressor.h
+++ b/src/compressor/Compressor.h
@@ -36,6 +36,9 @@ public:
     COMP_ALG_ZSTD = 3,
 #ifdef HAVE_LZ4
     COMP_ALG_LZ4 = 4,
+#endif
+#ifdef HAVE_BROTLI
+    COMP_ALG_BROTLI = 5,
 #endif
     COMP_ALG_LAST      //the last value for range checks
   };

diff --git a/src/compressor/brotli/BrotliCompressor.cc b/src/compressor/brotli/BrotliCompressor.cc
new file mode 100644 (file)
index 0000000..b9c1d1d
--- /dev/null
+++ b/src/compressor/brotli/BrotliCompressor.cc
@@ -0,0 +1,95 @@
+#include "brotli/encode.h"
+#include "brotli/decode.h"
+#include "BrotliCompressor.h"
+#include "include/scope_guard.h"
+
+#define MAX_LEN (CEPH_PAGE_SIZE)
+
+int BrotliCompressor::compress(const bufferlist &in, bufferlist &out) 
+{
+  BrotliEncoderState* s = BrotliEncoderCreateInstance(nullptr,
+                                                      nullptr,
+                                                      nullptr);
+  if (!s) {
+    return -1;
+  }
+  auto sg = make_scope_guard([&s] { BrotliEncoderDestroyInstance(s); });
+  BrotliEncoderSetParameter(s, BROTLI_PARAM_QUALITY, (uint32_t)9);
+  BrotliEncoderSetParameter(s, BROTLI_PARAM_LGWIN, 22);
+  for (auto i = in.buffers().begin(); i != in.buffers().end();) {
+    size_t available_in = i->length();
+    size_t max_comp_size = BrotliEncoderMaxCompressedSize(available_in);
+    size_t available_out =  max_comp_size;
+    bufferptr ptr = buffer::create_page_aligned(max_comp_size);
+    uint8_t* next_out = (uint8_t*)ptr.c_str();
+    const uint8_t* next_in = (uint8_t*)i->c_str();
+    ++i;
+    BrotliEncoderOperation finish = i != in.buffers().end() ?
+                                         BROTLI_OPERATION_PROCESS :
+                                         BROTLI_OPERATION_FINISH;
+    do {
+      if (!BrotliEncoderCompressStream(s,
+                                       finish,
+                                       &available_in,
+                                       &next_in,
+                                       &available_out,
+                                       &next_out,
+                                       nullptr)) {
+        return -1;
+      }
+      unsigned have = max_comp_size - available_out;
+      out.append(ptr, 0, have);
+    } while (available_out == 0);
+    if (BrotliEncoderIsFinished(s)) {
+      break;
+    }
+  }
+  return 0;
+}
+
+int BrotliCompressor::decompress(bufferlist::iterator &p,
+                                 size_t compressed_size,
+                                 bufferlist &out) 
+{
+  BrotliDecoderState* s = BrotliDecoderCreateInstance(nullptr,
+                                                      nullptr,
+                                                      nullptr);
+  if (!s) {
+    return -1;
+  }
+  auto sg = make_scope_guard([&s] { BrotliDecoderDestroyInstance(s); });
+  size_t remaining = std::min<size_t>(p.get_remaining(), compressed_size);
+  while (remaining) {
+    const uint8_t* next_in;
+    size_t len = p.get_ptr_and_advance(remaining, (const char**)&next_in);
+    remaining -= len;
+    size_t available_in = len;
+    BrotliDecoderResult result = BROTLI_DECODER_RESULT_ERROR;
+    do {
+      size_t available_out = MAX_LEN;
+      bufferptr ptr = buffer::create_page_aligned(MAX_LEN);
+      uint8_t* next_out = (uint8_t*)ptr.c_str();
+      result = BrotliDecoderDecompressStream(s,
+                                             &available_in,
+                                             &next_in,
+                                             &available_out,
+                                             &next_out,
+                                             0);
+      if (!result) {
+        return -1;
+      }
+      unsigned have = MAX_LEN - available_out;
+      out.append(ptr, 0, have);
+    } while (result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT);
+    if (BrotliDecoderIsFinished(s)) {
+      break;
+    }
+  }
+  return 0;
+}
+
+int BrotliCompressor::decompress(const bufferlist &in, bufferlist &out) 
+{  
+  bufferlist::iterator i = const_cast<bufferlist&>(in).begin();
+  return decompress(i, in.length(), out);
+}

diff --git a/src/compressor/brotli/BrotliCompressor.h b/src/compressor/brotli/BrotliCompressor.h
new file mode 100644 (file)
index 0000000..fcc4a80
--- /dev/null
+++ b/src/compressor/brotli/BrotliCompressor.h
@@ -0,0 +1,31 @@
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2017 BI SHUN KE <aionshun@livemail.tw>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef CEPH_BROTLICOMPRESSOR_H
+#define CEPH_BROTLICOMPRESSOR_H
+
+
+#include "include/buffer.h"
+#include "compressor/Compressor.h"
+
+class BrotliCompressor : public Compressor 
+{
+  public:
+  BrotliCompressor() : Compressor(COMP_ALG_BROTLI, "brotli") {}
+  
+  int compress(const bufferlist &in, bufferlist &out) override;
+  int decompress(const bufferlist &in, bufferlist &out) override;
+  int decompress(bufferlist::iterator &p, size_t compressed_len, bufferlist &out) override;
+};
+
+#endif //CEPH_BROTLICOMPRESSOR_H
+

diff --git a/src/compressor/brotli/CMakeLists.txt b/src/compressor/brotli/CMakeLists.txt
new file mode 100644 (file)
index 0000000..dc9a90a
--- /dev/null
+++ b/src/compressor/brotli/CMakeLists.txt
@@ -0,0 +1,44 @@
+# brotli
+
+set(brotli_sources 
+  CompressionPluginBrotli.cc
+  BrotliCompressor.cc
+)
+include(ExternalProject)
+ExternalProject_Add(brotli_ext
+  DOWNLOAD_DIR ${CMAKE_BINARY_DIR}/src/
+  GIT_REPOSITORY "https://github.com/google/brotli.git"
+  GIT_TAG "master"
+  SOURCE_DIR ${CMAKE_BINARY_DIR}/src/brotli
+  CONFIGURE_COMMAND ./configure-cmake --disable-debug 
+  INSTALL_COMMAND ""
+  BUILD_COMMAND $(MAKE)
+  BUILD_IN_SOURCE 1
+  INSTALL_COMMAND "")
+
+ExternalProject_Add_Step(brotli_ext forcebuild
+  DEPENDEES configure
+  DEPENDERS build
+  COMMAND "true"
+  ALWAYS 1)
+
+set(bortli_libs enc dec common)
+foreach(lib ${bortli_libs})
+  add_library(brotli::${lib} STATIC IMPORTED)
+  add_dependencies(brotli::${lib} brotli_ext)
+  set_property(TARGET brotli::${lib} PROPERTY IMPORTED_LOCATION "${CMAKE_BINARY_DIR}/src/brotli/libbrotli${lib}-static.a")
+  list(APPEND BROTLI_LIBRARIES brotli::${lib})
+endforeach()
+
+add_library(ceph_brotli SHARED ${brotli_sources})
+add_dependencies(ceph_brotli ${CMAKE_SOURCE_DIR}/src/ceph_ver.h)
+target_include_directories(ceph_brotli PRIVATE "${CMAKE_BINARY_DIR}/src/brotli/c/include")
+List(REVERSE bortli_libs)
+target_link_libraries(ceph_brotli ${BROTLI_LIBRARIES})
+install(TARGETS ceph_brotli DESTINATION ${compressor_plugin_dir})
+
+if(WITH_EMBEDDED)
+  add_library(cephd_compressor_brotli STATIC ${brotli_sources})
+  target_include_directories(cephd_compressor_brotli PRIVATE "${CMAKE_BINARY_DIR}/src/brotli/c/include")
+  set_target_properties(cephd_compressor_brotli PROPERTIES COMPILE_DEFINITIONS BUILDING_FOR_EMBEDDED)
+endif()

diff --git a/src/compressor/brotli/CompressionPluginBrotli.cc b/src/compressor/brotli/CompressionPluginBrotli.cc
new file mode 100644 (file)
index 0000000..9b11209
--- /dev/null
+++ b/src/compressor/brotli/CompressionPluginBrotli.cc
@@ -0,0 +1,20 @@
+#include "acconfig.h"
+#include "ceph_ver.h"
+#include "CompressionPluginBrotli.h"
+
+#ifndef BUILDING_FOR_EMBEDDED
+
+const char *__ceph_plugin_version()
+{
+  return CEPH_GIT_NICE_VER;
+}
+
+int __ceph_plugin_init(CephContext *cct,
+                       const std::string& type,
+                       const std::string& name)
+{
+  PluginRegistry *instance = cct->get_plugin_registry();
+  return instance->add(type, name, new CompressionPluginBrotli(cct));
+}
+
+#endif // !BUILDING_FOR_EMBEDDED

diff --git a/src/compressor/brotli/CompressionPluginBrotli.h b/src/compressor/brotli/CompressionPluginBrotli.h
new file mode 100644 (file)
index 0000000..641a6e1
--- /dev/null
+++ b/src/compressor/brotli/CompressionPluginBrotli.h
@@ -0,0 +1,36 @@
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2017 BI SHUN KE <aionshun@livemail.tw>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef CEPH_COMPRESSION_PLUGIN_BROTLI_H
+#define CEPH_COMPRESSION_PLUGIN_BROTLI_H
+
+#include "ceph_ver.h"
+#include "compressor/CompressionPlugin.h"
+#include "BrotliCompressor.h"
+
+class CompressionPluginBrotli : public CompressionPlugin {
+public:
+  explicit CompressionPluginBrotli(CephContext *cct) : CompressionPlugin(cct)
+  {}
+  
+  virtual int factory(CompressorRef *cs, std::ostream *ss)
+  {
+    if (compressor == nullptr) {
+      BrotliCompressor *interface = new BrotliCompressor();
+      compressor = CompressorRef(interface);
+    }
+    *cs = compressor;
+    return 0;
+  }
+};
+
+#endif

diff --git a/src/include/config-h.in.cmake b/src/include/config-h.in.cmake
index 46beb891ed6d8ff89d45e3959faa48f3215a077d..077a8a888fe4347c7eac6d71f7f476b26dc7ed5d 100644 (file)
--- a/src/include/config-h.in.cmake
+++ b/src/include/config-h.in.cmake
@@ -87,6 +87,9 @@
 /* Defined if you have LZ4 */
 #cmakedefine HAVE_LZ4
 
+/* Defined if you have BROTLI */
+#cmakedefine HAVE_BROTLI
+
 /* Defined if you have libaio */
 #cmakedefine HAVE_LIBAIO
 

diff --git a/src/test/compressor/test_compression.cc b/src/test/compressor/test_compression.cc
index 8e5cc3f485b3fc239ef41ed45583b270097f217b..bba0ca0c61238c44ed680e26239f9ff94d20649c 100644 (file)
--- a/src/test/compressor/test_compression.cc
+++ b/src/test/compressor/test_compression.cc
@@ -330,6 +330,9 @@ INSTANTIATE_TEST_CASE_P(
 #endif
     "zlib/noisal",
     "snappy",
+#ifdef HAVE_BROTLI
+    "brotli",
+#endif
     "zstd"));
 
 #ifdef __x86_64__