From: Sage Weil Date: Thu, 3 Jan 2019 17:59:36 +0000 (-0600) Subject: common/numa: cpu_set helpers and numa helpers X-Git-Tag: v14.1.0~436^2~9 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=c86f374081331b1074f3a6b0ff1b3abd8146f981;p=ceph.git common/numa: cpu_set helpers and numa helpers Note: the mask variants assume num_cpus is a factor of 4. Not sure what local_cpus looks like when that is not the case... we'll use the local_cpulist style instead. Signed-off-by: Sage Weil --- diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index 8a1d4e0f94c..1aa742d5ae5 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -73,6 +73,7 @@ set(common_srcs mempool.cc mime.c mutex_debug.cc + numa.cc options.cc page.cc perf_counters.cc diff --git a/src/common/numa.cc b/src/common/numa.cc new file mode 100644 index 00000000000..ff51d871c94 --- /dev/null +++ b/src/common/numa.cc @@ -0,0 +1,222 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "numa.h" + +#include +#include +#include + +#include "include/stringify.h" +#include "common/safe_io.h" + + +// list + +int parse_cpu_set_list(const char *s, + size_t *cpu_set_size, + cpu_set_t *cpu_set) +{ + CPU_ZERO(cpu_set); + while (*s) { + char *end; + int a = strtol(s, &end, 10); + if (end == s) { + return -EINVAL; + } + if (*end == '-') { + s = end + 1; + int b = strtol(s, &end, 10); + if (end == s) { + return -EINVAL; + } + for (; a <= b; ++a) { + CPU_SET(a, cpu_set); + } + *cpu_set_size = a; + } else { + CPU_SET(a, cpu_set); + *cpu_set_size = a + 1; + } + if (*end == 0) { + break; + } + if (*end != ',') { + return -EINVAL; + } + s = end + 1; + } + return 0; +} + +std::string cpu_set_to_str_list(size_t cpu_set_size, + const cpu_set_t *cpu_set) +{ + std::string r; + unsigned a = 0; + while (true) { + while (a < cpu_set_size && !CPU_ISSET(a, cpu_set)) { + ++a; + } + if (a >= cpu_set_size) { + break; + } + unsigned b = a + 1; + while (b < cpu_set_size && CPU_ISSET(b, cpu_set)) { + ++b; + } + if (r.size()) { + r += ","; + } + if (b > a + 1) { + r += stringify(a) + "-" + stringify(b - 1); + } else { + r += stringify(a); + } + a = b; + } + return r; +} + +std::set cpu_set_to_set(size_t cpu_set_size, + const cpu_set_t *cpu_set) +{ + set r; + unsigned a = 0; + while (true) { + while (a < cpu_set_size && !CPU_ISSET(a, cpu_set)) { + ++a; + } + if (a >= cpu_set_size) { + break; + } + unsigned b = a + 1; + while (b < cpu_set_size && CPU_ISSET(b, cpu_set)) { + ++b; + } + while (a < b) { + r.insert(a); + ++a; + } + } + return r; +} + +// mask +static int from_hex(char c) +{ + if (c >= '0' && c <= '9') { + return c - '0'; + } + if (c >= 'a' && c <= 'f') { + return c - 'a' + 10; + } + if (c >= 'A' && c <= 'F') { + return c - 'A' + 10; + } + return -1; +} + +static char to_hex(int v) +{ + if (v < 10) { + return '0' + v; + } + return 'a' + (v - 10); +} + +// FIXME: this is assuming the local_cpus value is little endian... is +// that right? + +// FIXME: this assumes the cpu count is a multiple of 4 (a nibble) + +int parse_cpu_set_mask( + const char *s, + size_t *cpu_set_size, + cpu_set_t *cpu_set) +{ + char *b = new char[CPU_SETSIZE]; + memset(b, 0, CPU_SETSIZE); + int r = -EINVAL; + unsigned cpu = 0; + int pos; + for (pos = 0; s[pos]; ++pos) { + if (pos/2 >= CPU_SETSIZE) { + goto out; + } + } + CPU_ZERO(cpu_set); + *cpu_set_size = pos * 4; + for (--pos; pos >= 0; --pos) { + int v = from_hex(s[pos]); + if (v < 0) { + goto out; + } + for (unsigned i = 0; i < 4; ++i, ++cpu) { + if (v & (1 << (cpu & 3))) { + CPU_SET(cpu, cpu_set); + } + } + } + r = 0; + + out: + delete b; + return r; +} + +std::string cpu_set_to_str_mask( + size_t cpu_set_size, + const cpu_set_t *cpu_set) +{ + std::string r; + unsigned v = 0; + for (int i = cpu_set_size - 1; i >= 0; --i) { + if (CPU_ISSET(i, cpu_set)) { + v |= 1 << (i & 7); + } + if ((i & 7) == 0) { + if (i + 4 < (int)cpu_set_size) { + r += to_hex((v & 0xf0) >> 4); + } + r += to_hex(v & 0xf); + v = 0; + } + } + return r; +} + +int get_numa_node_cpu_set( + int node, + size_t *cpu_set_size, + cpu_set_t *cpu_set) +{ + std::string fn = "/sys/devices/system/node/node"; + fn += stringify(node); + fn += "/cpulist"; + std::cout << __func__ << " fn " << fn << std::endl; + int fd = ::open(fn.c_str(), O_RDONLY); + if (fd < 0) { + std::cout << __func__ << " fail to open " << errno << std::endl; + return -errno; + } + char buf[1024]; + int r = safe_read(fd, &buf, sizeof(buf)); + if (r < 0) { + std::cout << __func__ << " fail to read " << errno << std::endl; + goto out; + } + buf[r] = 0; + while (r > 0 && ::isspace(buf[--r])) { + buf[r] = 0; + } + r = parse_cpu_set_list(buf, cpu_set_size, cpu_set); + if (r < 0) { + std::cout << __func__ << " fail to parse " << r << std::endl; + goto out; + } + r = 0; + out: + ::close(fd); + return r; +} diff --git a/src/common/numa.h b/src/common/numa.h new file mode 100644 index 00000000000..37f87872e6d --- /dev/null +++ b/src/common/numa.h @@ -0,0 +1,26 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#pragma once + +#include +#include +#include + +int parse_cpu_set_list(const char *s, + size_t *cpu_set_size, + cpu_set_t *cpu_set); +std::string cpu_set_to_str_list(size_t cpu_set_size, + const cpu_set_t *cpu_set); +std::set cpu_set_to_set(size_t cpu_set_size, + const cpu_set_t *cpu_set); + +int parse_cpu_set_mask(const char *s, + size_t *cpu_set_size, + cpu_set_t *cpu_set); +std::string cpu_set_to_str_mask(size_t cpu_set_size, + const cpu_set_t *cpu_set); + +int get_numa_node_cpu_set(int node, + size_t *cpu_set_size, + cpu_set_t *cpu_set); diff --git a/src/test/common/CMakeLists.txt b/src/test/common/CMakeLists.txt index 9398eac28d3..9578822c626 100644 --- a/src/test/common/CMakeLists.txt +++ b/src/test/common/CMakeLists.txt @@ -28,6 +28,13 @@ add_executable(unittest_lockdep add_ceph_unittest(unittest_lockdep) target_link_libraries(unittest_lockdep ceph-common global) +# unittest_numa +add_executable(unittest_numa + test_numa.cc + ) +add_ceph_unittest(unittest_numa) +target_link_libraries(unittest_numa ceph-common) + # unittest_bloom_filter add_executable(unittest_bloom_filter test_bloom_filter.cc diff --git a/src/test/common/test_numa.cc b/src/test/common/test_numa.cc new file mode 100644 index 00000000000..6070fc710f7 --- /dev/null +++ b/src/test/common/test_numa.cc @@ -0,0 +1,151 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "gtest/gtest.h" +#include "common/numa.h" + +TEST(cpu_set, parse_list) { + cpu_set_t cpu_set; + size_t size; + + ASSERT_EQ(0, parse_cpu_set_list("0-3", &size, &cpu_set)); + ASSERT_EQ(size, 4u); + for (unsigned i = 0; i < size; ++i) { + ASSERT_TRUE(CPU_ISSET(i, &cpu_set)); + } + + ASSERT_EQ(0, parse_cpu_set_list("0-3,6-7", &size, &cpu_set)); + ASSERT_EQ(size, 8u); + for (unsigned i = 0; i < 4; ++i) { + ASSERT_TRUE(CPU_ISSET(i, &cpu_set)); + } + for (unsigned i = 4; i < 6; ++i) { + ASSERT_FALSE(CPU_ISSET(i, &cpu_set)); + } + for (unsigned i = 6; i < 8; ++i) { + ASSERT_TRUE(CPU_ISSET(i, &cpu_set)); + } + + ASSERT_EQ(0, parse_cpu_set_list("0-31", &size, &cpu_set)); + ASSERT_EQ(size, 32u); + for (unsigned i = 0; i < size; ++i) { + ASSERT_TRUE(CPU_ISSET(i, &cpu_set)); + } +} + +TEST(cpu_set, to_str_list) { + cpu_set_t cpu_set; + CPU_ZERO(&cpu_set); + CPU_SET(0, &cpu_set); + ASSERT_EQ(std::string("0"), cpu_set_to_str_list(8, &cpu_set)); + CPU_SET(1, &cpu_set); + CPU_SET(2, &cpu_set); + CPU_SET(3, &cpu_set); + ASSERT_EQ(std::string("0-3"), cpu_set_to_str_list(8, &cpu_set)); + CPU_SET(5, &cpu_set); + ASSERT_EQ(std::string("0-3,5"), cpu_set_to_str_list(8, &cpu_set)); + CPU_SET(6, &cpu_set); + CPU_SET(7, &cpu_set); + ASSERT_EQ(std::string("0-3,5-7"), cpu_set_to_str_list(8, &cpu_set)); +} + +TEST(cpu_set, round_trip_list) +{ + for (unsigned i = 0; i < 100; ++i) { + cpu_set_t cpu_set; + size_t size = 32; + CPU_ZERO(&cpu_set); + for (unsigned i = 0; i < 32; ++i) { + if (rand() % 1) { + CPU_SET(i, &cpu_set); + } + } + std::string v = cpu_set_to_str_mask(size, &cpu_set); + cpu_set_t cpu_set_2; + size_t size2; + ASSERT_EQ(0, parse_cpu_set_mask(v.c_str(), &size2, &cpu_set_2)); + for (unsigned i = 0; i < 32; ++i) { + ASSERT_TRUE(CPU_ISSET(i, &cpu_set) == CPU_ISSET(i, &cpu_set_2)); + } + } +} + + + +TEST(cpu_set, parse_mask) { + cpu_set_t cpu_set; + size_t size; + + ASSERT_EQ(0, parse_cpu_set_mask("f", &size, &cpu_set)); + ASSERT_EQ(size, 4u); + for (unsigned i = 0; i < size; ++i) { + ASSERT_TRUE(CPU_ISSET(i, &cpu_set)); + } + + ASSERT_EQ(0, parse_cpu_set_mask("0f", &size, &cpu_set)); + ASSERT_EQ(size, 8u); + for (unsigned i = 0; i < 4; ++i) { + ASSERT_TRUE(CPU_ISSET(i, &cpu_set)); + } + + ASSERT_EQ(0, parse_cpu_set_mask("ffff", &size, &cpu_set)); + ASSERT_EQ(size, 16u); + for (unsigned i = 0; i < size; ++i) { + ASSERT_TRUE(CPU_ISSET(i, &cpu_set)); + } + + ASSERT_EQ(0, parse_cpu_set_mask("0000", &size, &cpu_set)); + ASSERT_EQ(size, 16u); + for (unsigned i = 0; i < size; ++i) { + ASSERT_FALSE(CPU_ISSET(i, &cpu_set)); + } + + ASSERT_EQ(0, parse_cpu_set_mask("00ff00ff", &size, &cpu_set)); + ASSERT_EQ(size, 32u); + for (unsigned i = 0; i < 8; ++i) { + ASSERT_TRUE(CPU_ISSET(i, &cpu_set)); + } + for (unsigned i = 8; i < 16; ++i) { + ASSERT_FALSE(CPU_ISSET(i, &cpu_set)); + } + for (unsigned i = 16; i < 24; ++i) { + ASSERT_TRUE(CPU_ISSET(i, &cpu_set)); + } + for (unsigned i = 24; i < 32; ++i) { + ASSERT_FALSE(CPU_ISSET(i, &cpu_set)); + } + + ASSERT_EQ(0, parse_cpu_set_mask("ffffffffffffffffffffffffffffffff", + &size, &cpu_set)); + ASSERT_EQ(size, 128u); + for (unsigned i = 0; i < size; ++i) { + ASSERT_TRUE(CPU_ISSET(i, &cpu_set)); + } +} + +TEST(cpu_set, to_str_mask) +{ + cpu_set_t cpu_set; + CPU_ZERO(&cpu_set); + CPU_SET(0, &cpu_set); + CPU_SET(1, &cpu_set); + CPU_SET(2, &cpu_set); + CPU_SET(3, &cpu_set); + ASSERT_EQ(std::string("f"), cpu_set_to_str_mask(4, &cpu_set)); + ASSERT_EQ(std::string("0f"), cpu_set_to_str_mask(8, &cpu_set)); +} + +TEST(cpu_set, round_trip_mask) +{ + for (unsigned i = 0; i < 100; ++i) { + char buf[50]; + snprintf(buf, sizeof(buf), "%08llx", (long long unsigned)rand()); + cpu_set_t cpu_set; + size_t size; + ASSERT_EQ(0, parse_cpu_set_mask(buf, &size, &cpu_set)); + ASSERT_EQ(32u, size); + std::string v = cpu_set_to_str_mask(size, &cpu_set); + ASSERT_EQ(v, buf); + } +} +