compressor/zlib: isa-l optimization for zlib algorithm on aarch64

Signed-off-by: Dai Zhiwei <daizhiwei3@huawei.com>

diff --git a/src/compressor/zlib/CMakeLists.txt b/src/compressor/zlib/CMakeLists.txt
index ade57cb08ee7e8c659cbfee8ee3d73b3d4be8cdb..050ff03fa28f1eed55ef83d87cc5e43b6fd88891 100644 (file)
--- a/src/compressor/zlib/CMakeLists.txt
+++ b/src/compressor/zlib/CMakeLists.txt
@@ -1,53 +1,93 @@
 # zlib
 
 if(HAVE_INTEL_SSE4_1 AND HAVE_NASM_X64_AVX2 AND (NOT APPLE))
-    set(CMAKE_ASM_FLAGS "-i ${PROJECT_SOURCE_DIR}/src/isa-l/igzip/ -i ${PROJECT_SOURCE_DIR}/src/isa-l/include/ ${CMAKE_ASM_FLAGS}")
-       set(zlib_sources
-         CompressionPluginZlib.cc
-         ZlibCompressor.cc
-         ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip.c
-         ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/hufftables_c.c
-         ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_base.c
-         ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_icf_base.c
-         ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/adler32_base.c
-         ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/flatten_ll.c
-         ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/encode_df.c
-         ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_icf_body.c
-         ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_inflate.c
-         ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/huff_codes.c
-         ${CMAKE_SOURCE_DIR}/src/isa-l/crc/crc_base_aliases.c
-         ${CMAKE_SOURCE_DIR}/src/isa-l/crc/crc_base.c
-         ${CMAKE_SOURCE_DIR}/src/isa-l/crc/crc64_base.c
-       )
-       list(APPEND zlib_sources
-         ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_body.asm
-         ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_finish.asm
-         ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_icf_body_h1_gr_bt.asm
-         ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_icf_finish.asm
-         ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/rfc1951_lookup.asm
-         ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/adler32_sse.asm
-         ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/adler32_avx2_4.asm
-         ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_multibinary.asm
-         ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_update_histogram_01.asm
-         ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_update_histogram_04.asm
-         ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_decode_block_stateless_01.asm
-         ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_decode_block_stateless_04.asm
-         ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_inflate_multibinary.asm
-         ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/encode_df_04.asm
-         ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/encode_df_06.asm
-         ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/proc_heap.asm
-         ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_deflate_hash.asm
-         ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_gen_icf_map_lh1_06.asm
-         ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_gen_icf_map_lh1_04.asm
-         ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_set_long_icf_fg_04.asm
-         ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_set_long_icf_fg_06.asm
-       )
-else(HAVE_INTEL_SSE4_1 AND HAVE_NASM_X64_AVX2 AND (NOT APPLE))
-       set(zlib_sources
-         CompressionPluginZlib.cc
-         ZlibCompressor.cc
-       )
-endif(HAVE_INTEL_SSE4_1 AND HAVE_NASM_X64_AVX2 AND (NOT APPLE))
+  set(CMAKE_ASM_FLAGS "-i ${PROJECT_SOURCE_DIR}/src/isa-l/igzip/ -i ${PROJECT_SOURCE_DIR}/src/isa-l/include/ ${CMAKE_ASM_FLAGS}")
+  set(zlib_sources
+    CompressionPluginZlib.cc
+    ZlibCompressor.cc
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/hufftables_c.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_base.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_icf_base.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/adler32_base.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/flatten_ll.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/encode_df.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_icf_body.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_inflate.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/huff_codes.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/crc/crc_base_aliases.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/crc/crc_base.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/crc/crc64_base.c
+  )
+  list(APPEND zlib_sources
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_body.asm
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_finish.asm
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_icf_body_h1_gr_bt.asm
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_icf_finish.asm
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/rfc1951_lookup.asm
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/adler32_sse.asm
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/adler32_avx2_4.asm
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_multibinary.asm
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_update_histogram_01.asm
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_update_histogram_04.asm
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_decode_block_stateless_01.asm
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_decode_block_stateless_04.asm
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_inflate_multibinary.asm
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/encode_df_04.asm
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/encode_df_06.asm
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/proc_heap.asm
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_deflate_hash.asm
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_gen_icf_map_lh1_06.asm
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_gen_icf_map_lh1_04.asm
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_set_long_icf_fg_04.asm
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_set_long_icf_fg_06.asm
+  )
+elseif(HAVE_ARMV8_SIMD)
+  set(zlib_asm_sources
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/aarch64/igzip_inflate_multibinary_arm64.S
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/aarch64/igzip_multibinary_arm64.S
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/aarch64/igzip_deflate_body_aarch64.S
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/aarch64/igzip_deflate_finish_aarch64.S
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/aarch64/isal_deflate_icf_body_hash_hist.S
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/aarch64/isal_deflate_icf_finish_hash_hist.S
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/aarch64/igzip_set_long_icf_fg.S
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/aarch64/isal_update_histogram.S
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/aarch64/igzip_deflate_hash_aarch64.S
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/aarch64/igzip_decode_huffman_code_block_aarch64.S
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/aarch64/igzip_isal_adler32_neon.S
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/aarch64/encode_df.S
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/aarch64/gen_icf_map.S
+  )
+  set(zlib_sources
+    CompressionPluginZlib.cc
+    ZlibCompressor.cc
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/hufftables_c.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_base.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_icf_base.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/adler32_base.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/flatten_ll.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/encode_df.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_icf_body.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_inflate.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/huff_codes.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/proc_heap_base.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/aarch64/igzip_multibinary_aarch64_dispatcher.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/crc/crc_base_aliases.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/crc/crc_base.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/crc/crc64_base.c
+    ${zlib_asm_sources}
+  )
+  set_source_files_properties(${zlib_asm_sources} PROPERTIES
+    COMPILE_DEFINITIONS "__ASSEMBLY__"
+    INCLUDE_DIRECTORIES "${PROJECT_SOURCE_DIR}/src/isa-l/igzip;${PROJECT_SOURCE_DIR}/src/isa-l/igzip/aarch64"
+  )
+else()
+  set(zlib_sources
+    CompressionPluginZlib.cc
+    ZlibCompressor.cc
+  )
+endif()
 
 add_library(ceph_zlib SHARED ${zlib_sources})
 target_link_libraries(ceph_zlib ZLIB::ZLIB compressor $<$<PLATFORM_ID:Windows>:ceph-common>)

diff --git a/src/compressor/zlib/CompressionPluginZlib.h b/src/compressor/zlib/CompressionPluginZlib.h
index fbdb635c6a407e2d4fcdd23839113f13dba2f3c4..597bc02a5d9a3cc12ada6ce1c17ee7c8280afee3 100644 (file)
--- a/src/compressor/zlib/CompressionPluginZlib.h
+++ b/src/compressor/zlib/CompressionPluginZlib.h
@@ -42,6 +42,11 @@ public:
       ceph_arch_probe();
       isal = (ceph_arch_intel_pclmul && ceph_arch_intel_sse41);
     }
+#elif defined(__aarch64__)
+    if (cct->_conf->compressor_zlib_isal) {
+      ceph_arch_probe();
+      isal = (ceph_arch_aarch64_pmull && ceph_arch_neon);
+    }
 #endif
     if (compressor == 0 || has_isal != isal) {
       compressor = std::make_shared<ZlibCompressor>(cct, isal);

diff --git a/src/compressor/zlib/ZlibCompressor.cc b/src/compressor/zlib/ZlibCompressor.cc
index 3bedbd8a1ebbdb6b21490897b51933c86da167ba..a38eb2e73031786d8785f04ec0224268769ee5a2 100644 (file)
--- a/src/compressor/zlib/ZlibCompressor.cc
+++ b/src/compressor/zlib/ZlibCompressor.cc
@@ -113,7 +113,7 @@ int ZlibCompressor::zlib_compress(const bufferlist &in, bufferlist &out, std::op
   return 0;
 }
 
-#if __x86_64__ && defined(HAVE_NASM_X64_AVX2)
+#if (__x86_64__ && defined(HAVE_NASM_X64_AVX2)) || defined(__aarch64__)
 int ZlibCompressor::isal_compress(const bufferlist &in, bufferlist &out, std::optional<int32_t> &compressor_message)
 {
   int ret;
@@ -174,7 +174,7 @@ int ZlibCompressor::compress(const bufferlist &in, bufferlist &out, std::optiona
   if (qat_enabled)
     return qat_accel.compress(in, out, compressor_message);
 #endif
-#if __x86_64__ && defined(HAVE_NASM_X64_AVX2)
+#if (__x86_64__ && defined(HAVE_NASM_X64_AVX2)) || defined(__aarch64__)
   if (isal_enabled)
     return isal_compress(in, out, compressor_message);
   else

diff --git a/src/test/compressor/test_compression.cc b/src/test/compressor/test_compression.cc
index cee9142742bb87369058fd01921c008308e40c61..6ae49daf544aa5e5f7d60432ca7b1c1f9d102d2a 100644 (file)
--- a/src/test/compressor/test_compression.cc
+++ b/src/test/compressor/test_compression.cc
@@ -378,7 +378,7 @@ INSTANTIATE_TEST_SUITE_P(
 #ifdef HAVE_LZ4
     "lz4",
 #endif
-#ifdef __x86_64__
+#if defined(__x86_64__) || defined(__aarch64__)
     "zlib/isal",
 #endif
     "zlib/noisal",
@@ -388,7 +388,7 @@ INSTANTIATE_TEST_SUITE_P(
 #endif
     "zstd"));
 
-#ifdef __x86_64__
+#if defined(__x86_64__) || defined(__aarch64__)
 
 TEST(ZlibCompressor, zlib_isal_compatibility)
 {
@@ -453,7 +453,7 @@ TEST(CompressionPlugin, all)
   }
 }
 
-#ifdef __x86_64__
+#if defined(__x86_64__) || defined(__aarch64__)
 
 TEST(ZlibCompressor, isal_compress_zlib_decompress_random)
 {