From: BI SHUN KE Date: Thu, 14 Dec 2017 20:13:23 +0000 (+0800) Subject: Compressor: Add Brotli Compressor X-Git-Tag: v13.0.2~637^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=f0aa22660c989867bcc2d23ab4254de8f953610f;p=ceph.git Compressor: Add Brotli Compressor Signed-off-by: BI SHUN KE --- diff --git a/CMakeLists.txt b/CMakeLists.txt index 889b6f9fc7d2..9e8a21d0871d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -293,6 +293,11 @@ endif(WITH_LEVELDB) find_package(snappy REQUIRED) +option(WITH_BROTLI "Brotli compression support" OFF) +if(WITH_BROTLI) + set(HAVE_BROTLI TRUE) +endif() + option(WITH_LZ4 "LZ4 compression support" OFF) if(WITH_LZ4) find_package(LZ4 1.7 REQUIRED) diff --git a/src/compressor/CMakeLists.txt b/src/compressor/CMakeLists.txt index 798268449bcc..4caa3fca725c 100644 --- a/src/compressor/CMakeLists.txt +++ b/src/compressor/CMakeLists.txt @@ -11,19 +11,27 @@ add_subdirectory(snappy) add_subdirectory(zlib) add_subdirectory(zstd) -if (HAVE_LZ4) +if(HAVE_LZ4) add_subdirectory(lz4) endif() +if(HAVE_BROTLI) + add_subdirectory(brotli) +endif() + set(ceph_compressor_libs ceph_snappy ceph_zlib ceph_zstd) -if (HAVE_LZ4) +if(HAVE_LZ4) list(APPEND ceph_compressor_libs ceph_lz4) endif() +if(HAVE_BROTLI) + list(APPEND ceph_compressor_libs ceph_brotli) +endif() + add_custom_target(compressor_plugins DEPENDS ${ceph_compressor_libs}) @@ -36,8 +44,11 @@ if(WITH_EMBEDDED) cephd_compressor_snappy cephd_compressor_zlib cephd_compressor_zstd) - if (HAVE_LZ4) + if(HAVE_LZ4) list(APPEND cephd_compressor_libs cephd_compressor_lz4) endif() + if(HAVE_BROTLI) + list(APPEND cephd_compressor_libs cephd_compressor_brotli) + endif() merge_static_libraries(cephd_compressor ${cephd_compressor_libs}) endif() diff --git a/src/compressor/Compressor.cc b/src/compressor/Compressor.cc index ade3950fd7ee..cca0c422d20c 100644 --- a/src/compressor/Compressor.cc +++ b/src/compressor/Compressor.cc @@ -30,6 +30,9 @@ const char * Compressor::get_comp_alg_name(int a) { case COMP_ALG_ZSTD: return "zstd"; #ifdef HAVE_LZ4 case COMP_ALG_LZ4: return "lz4"; +#endif +#ifdef HAVE_BROTLI + case COMP_ALG_BROTLI: return "brotli"; #endif default: return "???"; } @@ -45,6 +48,10 @@ boost::optional Compressor::get_comp_alg_type( #ifdef HAVE_LZ4 if (s == "lz4") return COMP_ALG_LZ4; +#endif +#ifdef HAVE_BROTLI + if (s == "brotli") + return COMP_ALG_BROTLI; #endif if (s == "" || s == "none") return COMP_ALG_NONE; diff --git a/src/compressor/Compressor.h b/src/compressor/Compressor.h index dba8f3460fea..cc343f8e8236 100644 --- a/src/compressor/Compressor.h +++ b/src/compressor/Compressor.h @@ -36,6 +36,9 @@ public: COMP_ALG_ZSTD = 3, #ifdef HAVE_LZ4 COMP_ALG_LZ4 = 4, +#endif +#ifdef HAVE_BROTLI + COMP_ALG_BROTLI = 5, #endif COMP_ALG_LAST //the last value for range checks }; diff --git a/src/compressor/brotli/BrotliCompressor.cc b/src/compressor/brotli/BrotliCompressor.cc new file mode 100644 index 000000000000..b9c1d1d52674 --- /dev/null +++ b/src/compressor/brotli/BrotliCompressor.cc @@ -0,0 +1,95 @@ +#include "brotli/encode.h" +#include "brotli/decode.h" +#include "BrotliCompressor.h" +#include "include/scope_guard.h" + +#define MAX_LEN (CEPH_PAGE_SIZE) + +int BrotliCompressor::compress(const bufferlist &in, bufferlist &out) +{ + BrotliEncoderState* s = BrotliEncoderCreateInstance(nullptr, + nullptr, + nullptr); + if (!s) { + return -1; + } + auto sg = make_scope_guard([&s] { BrotliEncoderDestroyInstance(s); }); + BrotliEncoderSetParameter(s, BROTLI_PARAM_QUALITY, (uint32_t)9); + BrotliEncoderSetParameter(s, BROTLI_PARAM_LGWIN, 22); + for (auto i = in.buffers().begin(); i != in.buffers().end();) { + size_t available_in = i->length(); + size_t max_comp_size = BrotliEncoderMaxCompressedSize(available_in); + size_t available_out = max_comp_size; + bufferptr ptr = buffer::create_page_aligned(max_comp_size); + uint8_t* next_out = (uint8_t*)ptr.c_str(); + const uint8_t* next_in = (uint8_t*)i->c_str(); + ++i; + BrotliEncoderOperation finish = i != in.buffers().end() ? + BROTLI_OPERATION_PROCESS : + BROTLI_OPERATION_FINISH; + do { + if (!BrotliEncoderCompressStream(s, + finish, + &available_in, + &next_in, + &available_out, + &next_out, + nullptr)) { + return -1; + } + unsigned have = max_comp_size - available_out; + out.append(ptr, 0, have); + } while (available_out == 0); + if (BrotliEncoderIsFinished(s)) { + break; + } + } + return 0; +} + +int BrotliCompressor::decompress(bufferlist::iterator &p, + size_t compressed_size, + bufferlist &out) +{ + BrotliDecoderState* s = BrotliDecoderCreateInstance(nullptr, + nullptr, + nullptr); + if (!s) { + return -1; + } + auto sg = make_scope_guard([&s] { BrotliDecoderDestroyInstance(s); }); + size_t remaining = std::min(p.get_remaining(), compressed_size); + while (remaining) { + const uint8_t* next_in; + size_t len = p.get_ptr_and_advance(remaining, (const char**)&next_in); + remaining -= len; + size_t available_in = len; + BrotliDecoderResult result = BROTLI_DECODER_RESULT_ERROR; + do { + size_t available_out = MAX_LEN; + bufferptr ptr = buffer::create_page_aligned(MAX_LEN); + uint8_t* next_out = (uint8_t*)ptr.c_str(); + result = BrotliDecoderDecompressStream(s, + &available_in, + &next_in, + &available_out, + &next_out, + 0); + if (!result) { + return -1; + } + unsigned have = MAX_LEN - available_out; + out.append(ptr, 0, have); + } while (result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT); + if (BrotliDecoderIsFinished(s)) { + break; + } + } + return 0; +} + +int BrotliCompressor::decompress(const bufferlist &in, bufferlist &out) +{ + bufferlist::iterator i = const_cast(in).begin(); + return decompress(i, in.length(), out); +} diff --git a/src/compressor/brotli/BrotliCompressor.h b/src/compressor/brotli/BrotliCompressor.h new file mode 100644 index 000000000000..fcc4a809c851 --- /dev/null +++ b/src/compressor/brotli/BrotliCompressor.h @@ -0,0 +1,31 @@ +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2017 BI SHUN KE + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#ifndef CEPH_BROTLICOMPRESSOR_H +#define CEPH_BROTLICOMPRESSOR_H + + +#include "include/buffer.h" +#include "compressor/Compressor.h" + +class BrotliCompressor : public Compressor +{ + public: + BrotliCompressor() : Compressor(COMP_ALG_BROTLI, "brotli") {} + + int compress(const bufferlist &in, bufferlist &out) override; + int decompress(const bufferlist &in, bufferlist &out) override; + int decompress(bufferlist::iterator &p, size_t compressed_len, bufferlist &out) override; +}; + +#endif //CEPH_BROTLICOMPRESSOR_H + diff --git a/src/compressor/brotli/CMakeLists.txt b/src/compressor/brotli/CMakeLists.txt new file mode 100644 index 000000000000..dc9a90acf549 --- /dev/null +++ b/src/compressor/brotli/CMakeLists.txt @@ -0,0 +1,44 @@ +# brotli + +set(brotli_sources + CompressionPluginBrotli.cc + BrotliCompressor.cc +) +include(ExternalProject) +ExternalProject_Add(brotli_ext + DOWNLOAD_DIR ${CMAKE_BINARY_DIR}/src/ + GIT_REPOSITORY "https://github.com/google/brotli.git" + GIT_TAG "master" + SOURCE_DIR ${CMAKE_BINARY_DIR}/src/brotli + CONFIGURE_COMMAND ./configure-cmake --disable-debug + INSTALL_COMMAND "" + BUILD_COMMAND $(MAKE) + BUILD_IN_SOURCE 1 + INSTALL_COMMAND "") + +ExternalProject_Add_Step(brotli_ext forcebuild + DEPENDEES configure + DEPENDERS build + COMMAND "true" + ALWAYS 1) + +set(bortli_libs enc dec common) +foreach(lib ${bortli_libs}) + add_library(brotli::${lib} STATIC IMPORTED) + add_dependencies(brotli::${lib} brotli_ext) + set_property(TARGET brotli::${lib} PROPERTY IMPORTED_LOCATION "${CMAKE_BINARY_DIR}/src/brotli/libbrotli${lib}-static.a") + list(APPEND BROTLI_LIBRARIES brotli::${lib}) +endforeach() + +add_library(ceph_brotli SHARED ${brotli_sources}) +add_dependencies(ceph_brotli ${CMAKE_SOURCE_DIR}/src/ceph_ver.h) +target_include_directories(ceph_brotli PRIVATE "${CMAKE_BINARY_DIR}/src/brotli/c/include") +List(REVERSE bortli_libs) +target_link_libraries(ceph_brotli ${BROTLI_LIBRARIES}) +install(TARGETS ceph_brotli DESTINATION ${compressor_plugin_dir}) + +if(WITH_EMBEDDED) + add_library(cephd_compressor_brotli STATIC ${brotli_sources}) + target_include_directories(cephd_compressor_brotli PRIVATE "${CMAKE_BINARY_DIR}/src/brotli/c/include") + set_target_properties(cephd_compressor_brotli PROPERTIES COMPILE_DEFINITIONS BUILDING_FOR_EMBEDDED) +endif() diff --git a/src/compressor/brotli/CompressionPluginBrotli.cc b/src/compressor/brotli/CompressionPluginBrotli.cc new file mode 100644 index 000000000000..9b11209f3e7c --- /dev/null +++ b/src/compressor/brotli/CompressionPluginBrotli.cc @@ -0,0 +1,20 @@ +#include "acconfig.h" +#include "ceph_ver.h" +#include "CompressionPluginBrotli.h" + +#ifndef BUILDING_FOR_EMBEDDED + +const char *__ceph_plugin_version() +{ + return CEPH_GIT_NICE_VER; +} + +int __ceph_plugin_init(CephContext *cct, + const std::string& type, + const std::string& name) +{ + PluginRegistry *instance = cct->get_plugin_registry(); + return instance->add(type, name, new CompressionPluginBrotli(cct)); +} + +#endif // !BUILDING_FOR_EMBEDDED diff --git a/src/compressor/brotli/CompressionPluginBrotli.h b/src/compressor/brotli/CompressionPluginBrotli.h new file mode 100644 index 000000000000..641a6e1c92ea --- /dev/null +++ b/src/compressor/brotli/CompressionPluginBrotli.h @@ -0,0 +1,36 @@ +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2017 BI SHUN KE + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#ifndef CEPH_COMPRESSION_PLUGIN_BROTLI_H +#define CEPH_COMPRESSION_PLUGIN_BROTLI_H + +#include "ceph_ver.h" +#include "compressor/CompressionPlugin.h" +#include "BrotliCompressor.h" + +class CompressionPluginBrotli : public CompressionPlugin { +public: + explicit CompressionPluginBrotli(CephContext *cct) : CompressionPlugin(cct) + {} + + virtual int factory(CompressorRef *cs, std::ostream *ss) + { + if (compressor == nullptr) { + BrotliCompressor *interface = new BrotliCompressor(); + compressor = CompressorRef(interface); + } + *cs = compressor; + return 0; + } +}; + +#endif diff --git a/src/include/config-h.in.cmake b/src/include/config-h.in.cmake index 46beb891ed6d..077a8a888fe4 100644 --- a/src/include/config-h.in.cmake +++ b/src/include/config-h.in.cmake @@ -87,6 +87,9 @@ /* Defined if you have LZ4 */ #cmakedefine HAVE_LZ4 +/* Defined if you have BROTLI */ +#cmakedefine HAVE_BROTLI + /* Defined if you have libaio */ #cmakedefine HAVE_LIBAIO diff --git a/src/test/compressor/test_compression.cc b/src/test/compressor/test_compression.cc index 8e5cc3f485b3..bba0ca0c6123 100644 --- a/src/test/compressor/test_compression.cc +++ b/src/test/compressor/test_compression.cc @@ -330,6 +330,9 @@ INSTANTIATE_TEST_CASE_P( #endif "zlib/noisal", "snappy", +#ifdef HAVE_BROTLI + "brotli", +#endif "zstd")); #ifdef __x86_64__