From: chenxuqiang <chenxuqiang3@hisilicon.com>
Date: Wed, 17 May 2023 09:34:50 +0000 (+0800)
Subject: inline_memory: optimized mem_is_zero for aarch64 by neon intrinsic
X-Git-Tag: v19.3.0~294^2~1
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=c7749f09bb7759d8696279882ce6852b60a2fae3;p=ceph.git

inline_memory: optimized mem_is_zero for aarch64 by neon intrinsic

The Neon instruction can be used to optimize mem_is_Zero. The Neon
    instruction can load 128-bit data. Most ARM CPU architectures have 2
    LD loading channels. Therefore, two 128-bit data can be loaded at the
    same time, which is good for block data.

    We performed a set of performance tests on TSV110 to compare the no-x64
    implementation, loop: 1,000,000, used time:

        size    |    no-x64      |     neon
     --------------------------------------------
        1024    |    0.091       |     0.044
     --------------------------------------------
        2048    |    0.177       |     0.092
     --------------------------------------------
        4096    |    0.348       |     0.175
     --------------------------------------------
        8192    |    0.689       |     0.345
     --------------------------------------------
        65536   |    5.471       |     2.746

Signed-off-by: chenxuqiang <chenxuqiang3@hisilicon.com>
---

diff --git a/src/include/inline_memory.h b/src/include/inline_memory.h
index 48d889763f7..6a1fe34800a 100644
--- a/src/include/inline_memory.h
+++ b/src/include/inline_memory.h
@@ -123,6 +123,49 @@ bool mem_is_zero(const char *data, size_t len)
   return true;
 }
 
+#elif defined(__GNUC__) && defined(__aarch64__) && defined(__ARM_NEON) // gcc and aarch64 neon
+
+#include <arm_neon.h>
+
+static inline bool mem_is_zero(const char *data, size_t len) {
+  const char *end = data + len;
+  const char *end256 = data + (len / sizeof(uint64x2x2_t)) * sizeof(uint64x2x2_t);
+  while (data < end256) {
+    uint64x2x2_t value = vld1q_u64_x2((uint64_t *)data);
+    if (value.val[0][0] != 0 || value.val[0][1] != 0 ||
+        value.val[1][0] != 0 || value.val[1][1] != 0) {
+      return false;
+    }
+    data += sizeof(uint64x2x2_t);
+  }
+
+  const char *end128 = data + sizeof(uint64x2_t);
+  if (end128 < end) {
+    uint64x2_t value = vld1q_u64((uint64_t *)data);
+    if (value[0] != 0 || value[1] != 0) {
+      return false;
+    }
+    data += sizeof(uint64x2_t);
+  }
+
+  const char *end64 = data + sizeof(uint64_t);
+  if (end64 < end) {
+    if(*(uint64_t *)data != 0) {
+      return false;
+    }
+    data += sizeof(uint64_t);
+  }
+
+  while (data < end) {
+    if (*data != 0) {
+      return false;
+    }
+    ++data;
+  }
+
+  return true;
+}
+
 #else  // gcc and x86_64
 
 static inline bool mem_is_zero(const char *data, size_t len) {