From b5d57c97c7f021304b8dea72feaa20a8e240e303 Mon Sep 17 00:00:00 2001 From: Bassam Tabbara Date: Fri, 4 Nov 2016 18:10:08 -0700 Subject: [PATCH] embedded: add compression and EC plugins to libcephd Compression and erasure coding plugins are now statically compiled into libcephd. A new method is added to load them into the respective registry. The static libraries are only built when WITH_EMBEDDED is enabled and existing plugins are unaffected. Signed-off-by: Bassam Tabbara --- src/ceph_osd.cc | 8 ++ src/compressor/CMakeLists.txt | 7 ++ src/compressor/snappy/CMakeLists.txt | 5 ++ .../snappy/CompressionPluginSnappy.cc | 26 ++---- .../snappy/CompressionPluginSnappy.h | 42 ++++++++++ src/compressor/zlib/CMakeLists.txt | 6 ++ src/compressor/zlib/CompressionPluginZlib.cc | 40 ++------- src/compressor/zlib/CompressionPluginZlib.h | 54 ++++++++++++ src/erasure-code/CMakeLists.txt | 8 ++ src/erasure-code/isa/CMakeLists.txt | 10 ++- src/erasure-code/isa/ErasureCodePluginIsa.cc | 4 + src/erasure-code/jerasure/CMakeLists.txt | 11 ++- .../jerasure/ErasureCodePluginJerasure.cc | 4 + src/erasure-code/lrc/CMakeLists.txt | 5 ++ src/erasure-code/lrc/ErasureCodePluginLrc.cc | 4 + src/erasure-code/shec/CMakeLists.txt | 10 ++- .../shec/ErasureCodePluginShec.cc | 4 + src/libcephd/CMakeLists.txt | 5 +- src/libcephd/libcephd.cc | 84 +++++++++++++++++++ 19 files changed, 276 insertions(+), 61 deletions(-) create mode 100644 src/compressor/snappy/CompressionPluginSnappy.h create mode 100644 src/compressor/zlib/CompressionPluginZlib.h diff --git a/src/ceph_osd.cc b/src/ceph_osd.cc index b7724e8eb4948..5f1f26e16d677 100644 --- a/src/ceph_osd.cc +++ b/src/ceph_osd.cc @@ -249,6 +249,10 @@ int main(int argc, const char **argv) return -ENODEV; } +#ifdef BUILDING_FOR_EMBEDDED + cephd_preload_embedded_plugins(); +#endif + if (mkfs) { common_init_finish(g_ceph_context); MonClient mc(g_ceph_context); @@ -603,6 +607,10 @@ int main(int argc, const char **argv) return 1; } +#ifdef BUILDING_FOR_EMBEDDED + cephd_preload_rados_classes(osd); +#endif + // install signal handlers init_async_signal_handler(); register_async_signal_handler(SIGHUP, sighup_handler); diff --git a/src/compressor/CMakeLists.txt b/src/compressor/CMakeLists.txt index 36b0a90ceb4b3..436ea3eb1df30 100644 --- a/src/compressor/CMakeLists.txt +++ b/src/compressor/CMakeLists.txt @@ -14,3 +14,10 @@ add_subdirectory(zlib) add_custom_target(compressor_plugins DEPENDS ceph_snappy ceph_zlib) + +if(WITH_EMBEDDED) + include(MergeStaticLibraries) + add_library(cephd_compressor_base STATIC ${compressor_srcs}) + set_target_properties(cephd_compressor_base PROPERTIES COMPILE_DEFINITIONS BUILDING_FOR_EMBEDDED) + merge_static_libraries(cephd_compressor cephd_compressor_base cephd_compressor_snappy cephd_compressor_zlib) +endif() diff --git a/src/compressor/snappy/CMakeLists.txt b/src/compressor/snappy/CMakeLists.txt index 1fa07c237b4ac..108aea204020a 100644 --- a/src/compressor/snappy/CMakeLists.txt +++ b/src/compressor/snappy/CMakeLists.txt @@ -9,3 +9,8 @@ add_dependencies(ceph_snappy ${CMAKE_SOURCE_DIR}/src/ceph_ver.h) target_link_libraries(ceph_snappy snappy common) set_target_properties(ceph_snappy PROPERTIES VERSION 2.0.0 SOVERSION 2) install(TARGETS ceph_snappy DESTINATION ${compressor_plugin_dir}) + +if(WITH_EMBEDDED) + add_library(cephd_compressor_snappy STATIC ${snappy_sources}) + set_target_properties(cephd_compressor_snappy PROPERTIES COMPILE_DEFINITIONS BUILDING_FOR_EMBEDDED) +endif() diff --git a/src/compressor/snappy/CompressionPluginSnappy.cc b/src/compressor/snappy/CompressionPluginSnappy.cc index e53444aa04af7..f4e8c3819d9ea 100644 --- a/src/compressor/snappy/CompressionPluginSnappy.cc +++ b/src/compressor/snappy/CompressionPluginSnappy.cc @@ -14,29 +14,11 @@ // ----------------------------------------------------------------------------- +#include "acconfig.h" #include "ceph_ver.h" -#include "compressor/CompressionPlugin.h" -#include "SnappyCompressor.h" -// ----------------------------------------------------------------------------- - -class CompressionPluginSnappy : public CompressionPlugin { - -public: +#include "CompressionPluginSnappy.h" - explicit CompressionPluginSnappy(CephContext* cct) : CompressionPlugin(cct) - {} - - virtual int factory(CompressorRef *cs, - std::ostream *ss) - { - if (compressor == 0) { - SnappyCompressor *interface = new SnappyCompressor(); - compressor = CompressorRef(interface); - } - *cs = compressor; - return 0; - } -}; +#ifndef BUILDING_FOR_EMBEDDED // ----------------------------------------------------------------------------- @@ -55,3 +37,5 @@ int __ceph_plugin_init(CephContext *cct, return instance->add(type, name, new CompressionPluginSnappy(cct)); } + +#endif // !BUILDING_FOR_EMBEDDED \ No newline at end of file diff --git a/src/compressor/snappy/CompressionPluginSnappy.h b/src/compressor/snappy/CompressionPluginSnappy.h new file mode 100644 index 0000000000000..a744a5df137fc --- /dev/null +++ b/src/compressor/snappy/CompressionPluginSnappy.h @@ -0,0 +1,42 @@ +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2015 Mirantis, Inc. + * + * Author: Alyona Kiseleva + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + */ + +#ifndef CEPH_COMPRESSION_PLUGIN_SNAPPY_H +#define CEPH_COMPRESSION_PLUGIN_SNAPPY_H + +// ----------------------------------------------------------------------------- +#include "compressor/CompressionPlugin.h" +#include "SnappyCompressor.h" +// ----------------------------------------------------------------------------- + +class CompressionPluginSnappy : public CompressionPlugin { + +public: + + explicit CompressionPluginSnappy(CephContext* cct) : CompressionPlugin(cct) + {} + + virtual int factory(CompressorRef *cs, + std::ostream *ss) + { + if (compressor == 0) { + SnappyCompressor *interface = new SnappyCompressor(); + compressor = CompressorRef(interface); + } + *cs = compressor; + return 0; + } +}; + +#endif diff --git a/src/compressor/zlib/CMakeLists.txt b/src/compressor/zlib/CMakeLists.txt index 72d6086d383d4..19a84da51a54f 100644 --- a/src/compressor/zlib/CMakeLists.txt +++ b/src/compressor/zlib/CMakeLists.txt @@ -33,3 +33,9 @@ target_link_libraries(ceph_zlib z common) target_include_directories(ceph_zlib PRIVATE "${CMAKE_SOURCE_DIR}/src/isa-l/include") set_target_properties(ceph_zlib PROPERTIES VERSION 2.0.0 SOVERSION 2) install(TARGETS ceph_zlib DESTINATION ${compressor_plugin_dir}) + +if(WITH_EMBEDDED) + add_library(cephd_compressor_zlib STATIC ${zlib_sources}) + target_include_directories(cephd_compressor_zlib PRIVATE "${CMAKE_SOURCE_DIR}/src/isa-l/include") + set_target_properties(cephd_compressor_zlib PROPERTIES COMPILE_DEFINITIONS BUILDING_FOR_EMBEDDED) +endif() diff --git a/src/compressor/zlib/CompressionPluginZlib.cc b/src/compressor/zlib/CompressionPluginZlib.cc index 7146bf9dca96d..26969f8377a29 100644 --- a/src/compressor/zlib/CompressionPluginZlib.cc +++ b/src/compressor/zlib/CompressionPluginZlib.cc @@ -14,43 +14,11 @@ // ----------------------------------------------------------------------------- +#include "acconfig.h" #include "ceph_ver.h" -#include "arch/probe.h" -#include "arch/intel.h" -#include "arch/arm.h" -#include "compressor/CompressionPlugin.h" -#include "ZlibCompressor.h" -#include "common/debug.h" - -#define dout_subsys ceph_subsys_mon -// ----------------------------------------------------------------------------- - -class CompressionPluginZlib : public CompressionPlugin { -public: - bool has_isal = false; - - explicit CompressionPluginZlib(CephContext *cct) : CompressionPlugin(cct) - {} - - virtual int factory(CompressorRef *cs, - std::ostream *ss) - { - bool isal; - if (cct->_conf->compressor_zlib_isal) { - ceph_arch_probe(); - isal = (ceph_arch_intel_pclmul && ceph_arch_intel_sse41); - } else { - isal = false; - } - if (compressor == 0 || has_isal != isal) { - compressor = CompressorRef(new ZlibCompressor(isal)); - has_isal = isal; - } - *cs = compressor; - return 0; - } -}; +#include "CompressionPluginZlib.h" +#ifndef BUILDING_FOR_EMBEDDED // ----------------------------------------------------------------------------- const char *__ceph_plugin_version() @@ -68,3 +36,5 @@ int __ceph_plugin_init(CephContext *cct, return instance->add(type, name, new CompressionPluginZlib(cct)); } + +#endif // !BUILDING_FOR_EMBEDDED diff --git a/src/compressor/zlib/CompressionPluginZlib.h b/src/compressor/zlib/CompressionPluginZlib.h new file mode 100644 index 0000000000000..e73afd802a3e5 --- /dev/null +++ b/src/compressor/zlib/CompressionPluginZlib.h @@ -0,0 +1,54 @@ +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2015 Mirantis, Inc. + * + * Author: Alyona Kiseleva + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + */ + +#ifndef CEPH_COMPRESSION_PLUGIN_ZLIB_H +#define CEPH_COMPRESSION_PLUGIN_ZLIB_H + +// ----------------------------------------------------------------------------- +#include "arch/probe.h" +#include "arch/intel.h" +#include "arch/arm.h" +#include "common/config.h" +#include "compressor/CompressionPlugin.h" +#include "ZlibCompressor.h" + +// ----------------------------------------------------------------------------- + +class CompressionPluginZlib : public CompressionPlugin { +public: + bool has_isal = false; + + explicit CompressionPluginZlib(CephContext *cct) : CompressionPlugin(cct) + {} + + virtual int factory(CompressorRef *cs, + std::ostream *ss) + { + bool isal; + if (cct->_conf->compressor_zlib_isal) { + ceph_arch_probe(); + isal = (ceph_arch_intel_pclmul && ceph_arch_intel_sse41); + } else { + isal = false; + } + if (compressor == 0 || has_isal != isal) { + compressor = CompressorRef(new ZlibCompressor(isal)); + has_isal = isal; + } + *cs = compressor; + return 0; + } +}; + +#endif diff --git a/src/erasure-code/CMakeLists.txt b/src/erasure-code/CMakeLists.txt index 1b4870403a366..450ec716973eb 100644 --- a/src/erasure-code/CMakeLists.txt +++ b/src/erasure-code/CMakeLists.txt @@ -24,6 +24,7 @@ add_subdirectory(shec) if (HAVE_BETTER_YASM_ELF64) add_subdirectory(isa) set(EC_ISA_LIB ec_isa) + set(EC_ISA_EMBEDDED_LIB cephd_ec_isa) endif (HAVE_BETTER_YASM_ELF64) add_library(erasure_code STATIC ErasureCodePlugin.cc) @@ -37,3 +38,10 @@ add_custom_target(erasure_code_plugins DEPENDS ec_lrc ec_jerasure ec_shec) + +if(WITH_EMBEDDED) + include(MergeStaticLibraries) + add_library(cephd_ec_base STATIC $) + set_target_properties(cephd_ec_base PROPERTIES COMPILE_DEFINITIONS BUILDING_FOR_EMBEDDED) + merge_static_libraries(cephd_ec cephd_ec_base ${EC_ISA_EMBEDDED_LIB} cephd_ec_jerasure cephd_ec_lrc cephd_ec_shec) +endif() diff --git a/src/erasure-code/isa/CMakeLists.txt b/src/erasure-code/isa/CMakeLists.txt index 1cb530424393e..63292fa72e34b 100644 --- a/src/erasure-code/isa/CMakeLists.txt +++ b/src/erasure-code/isa/CMakeLists.txt @@ -48,11 +48,17 @@ set(isa_srcs ErasureCodeIsaTableCache.cc ErasureCodePluginIsa.cc xor_op.cc - $ ) -add_library(ec_isa SHARED ${isa_srcs}) +add_library(ec_isa SHARED + ${isa_srcs} + $) add_dependencies(ec_isa ${CMAKE_SOURCE_DIR}/src/ceph_ver.h) target_link_libraries(ec_isa ${EXTRALIBS}) set_target_properties(ec_isa PROPERTIES VERSION 2.14.0 SOVERSION 2) install(TARGETS ec_isa DESTINATION ${erasure_plugin_dir}) + +if(WITH_EMBEDDED) + add_library(cephd_ec_isa STATIC ${isa_srcs}) + set_target_properties(cephd_ec_isa PROPERTIES COMPILE_DEFINITIONS BUILDING_FOR_EMBEDDED) +endif() diff --git a/src/erasure-code/isa/ErasureCodePluginIsa.cc b/src/erasure-code/isa/ErasureCodePluginIsa.cc index c4c37b1bf1d0f..1e95b0bc6a80e 100644 --- a/src/erasure-code/isa/ErasureCodePluginIsa.cc +++ b/src/erasure-code/isa/ErasureCodePluginIsa.cc @@ -65,6 +65,8 @@ int ErasureCodePluginIsa::factory(const std::string &directory, return 0; } +#ifndef BUILDING_FOR_EMBEDDED + // ----------------------------------------------------------------------------- const char *__erasure_code_version() @@ -80,3 +82,5 @@ int __erasure_code_init(char *plugin_name, char *directory) return instance.add(plugin_name, new ErasureCodePluginIsa()); } + +#endif \ No newline at end of file diff --git a/src/erasure-code/jerasure/CMakeLists.txt b/src/erasure-code/jerasure/CMakeLists.txt index 88c1ca40da8b4..968489e6592e3 100644 --- a/src/erasure-code/jerasure/CMakeLists.txt +++ b/src/erasure-code/jerasure/CMakeLists.txt @@ -1,8 +1,10 @@ # jerasure plugin -add_library(jerasure_utils OBJECT +set(jerasure_utils_src ErasureCodePluginJerasure.cc ErasureCodeJerasure.cc) + +add_library(jerasure_utils OBJECT ${jerasure_utils_src}) add_dependencies(jerasure_utils ${CMAKE_SOURCE_DIR}/src/ceph_ver.h) # Set the CFLAGS correctly for gf-complete based on SIMD compiler support @@ -97,3 +99,10 @@ foreach(flavor ${jerasure_legacy_flavors}) add_dependencies(ec_jerasure ${plugin_name}) endforeach() +if(WITH_EMBEDDED) + add_library(cephd_ec_jerasure STATIC + $ + $ + ${jerasure_utils_src}) + set_target_properties(cephd_ec_jerasure PROPERTIES COMPILE_DEFINITIONS BUILDING_FOR_EMBEDDED) +endif() diff --git a/src/erasure-code/jerasure/ErasureCodePluginJerasure.cc b/src/erasure-code/jerasure/ErasureCodePluginJerasure.cc index 6029eef4ea9d3..3ccedb9699db4 100644 --- a/src/erasure-code/jerasure/ErasureCodePluginJerasure.cc +++ b/src/erasure-code/jerasure/ErasureCodePluginJerasure.cc @@ -70,6 +70,8 @@ int ErasureCodePluginJerasure::factory(const std::string& directory, return 0; } +#ifndef BUILDING_FOR_EMBEDDED + const char *__erasure_code_version() { return CEPH_GIT_NICE_VER; } int __erasure_code_init(char *plugin_name, char *directory) @@ -82,3 +84,5 @@ int __erasure_code_init(char *plugin_name, char *directory) } return instance.add(plugin_name, new ErasureCodePluginJerasure()); } + +#endif \ No newline at end of file diff --git a/src/erasure-code/lrc/CMakeLists.txt b/src/erasure-code/lrc/CMakeLists.txt index 9c2ac7f81237e..80f9ffcf30420 100644 --- a/src/erasure-code/lrc/CMakeLists.txt +++ b/src/erasure-code/lrc/CMakeLists.txt @@ -11,3 +11,8 @@ add_library(ec_lrc SHARED ${lrc_srcs}) add_dependencies(ec_lrc ${CMAKE_SOURCE_DIR}/src/ceph_ver.h) target_link_libraries(ec_lrc crush json_spirit) install(TARGETS ec_lrc DESTINATION ${erasure_plugin_dir}) + +if(WITH_EMBEDDED) + add_library(cephd_ec_lrc STATIC ${lrc_srcs}) + set_target_properties(cephd_ec_lrc PROPERTIES COMPILE_DEFINITIONS BUILDING_FOR_EMBEDDED) +endif() diff --git a/src/erasure-code/lrc/ErasureCodePluginLrc.cc b/src/erasure-code/lrc/ErasureCodePluginLrc.cc index 003b3dc19ddef..c2be14b016a9b 100644 --- a/src/erasure-code/lrc/ErasureCodePluginLrc.cc +++ b/src/erasure-code/lrc/ErasureCodePluginLrc.cc @@ -42,6 +42,8 @@ int ErasureCodePluginLrc::factory(const std::string &directory, return 0; }; +#ifndef BUILDING_FOR_EMBEDDED + const char *__erasure_code_version() { return CEPH_GIT_NICE_VER; } int __erasure_code_init(char *plugin_name, char *directory) @@ -49,3 +51,5 @@ int __erasure_code_init(char *plugin_name, char *directory) ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance(); return instance.add(plugin_name, new ErasureCodePluginLrc()); } + +#endif diff --git a/src/erasure-code/shec/CMakeLists.txt b/src/erasure-code/shec/CMakeLists.txt index 64cac19465fe9..2ebc50e34c241 100644 --- a/src/erasure-code/shec/CMakeLists.txt +++ b/src/erasure-code/shec/CMakeLists.txt @@ -2,12 +2,14 @@ include_directories(.) -add_library(shec_utils OBJECT +set(shec_utils_srcs ${CMAKE_SOURCE_DIR}/src/erasure-code/ErasureCode.cc ErasureCodePluginShec.cc ErasureCodeShec.cc ErasureCodeShecTableCache.cc determinant.c) + +add_library(shec_utils OBJECT ${shec_utils_srcs}) add_dependencies(shec_utils ${CMAKE_SOURCE_DIR}/src/ceph_ver.h) set(ec_shec_objs @@ -27,3 +29,9 @@ foreach(flavor ${jerasure_legacy_flavors}) install(TARGETS ${plugin_name} DESTINATION ${erasure_plugin_dir}) add_dependencies(ec_shec ${plugin_name}) endforeach() + +if(WITH_EMBEDDED) + # note we rely on the fact this will always be statically linked with jerasure + add_library(cephd_ec_shec STATIC ${shec_utils_srcs}) + set_target_properties(cephd_ec_shec PROPERTIES COMPILE_DEFINITIONS BUILDING_FOR_EMBEDDED) +endif() diff --git a/src/erasure-code/shec/ErasureCodePluginShec.cc b/src/erasure-code/shec/ErasureCodePluginShec.cc index d31777e50d630..b55be77392969 100644 --- a/src/erasure-code/shec/ErasureCodePluginShec.cc +++ b/src/erasure-code/shec/ErasureCodePluginShec.cc @@ -66,6 +66,8 @@ int ErasureCodePluginShec::factory(const std::string &directory, return 0; } +#ifndef BUILDING_FOR_EMBEDDED + const char *__erasure_code_version() { return CEPH_GIT_NICE_VER; } int __erasure_code_init(char *plugin_name, char *directory = (char *)"") @@ -78,3 +80,5 @@ int __erasure_code_init(char *plugin_name, char *directory = (char *)"") } return instance.add(plugin_name, new ErasureCodePluginShec()); } + +#endif \ No newline at end of file diff --git a/src/libcephd/CMakeLists.txt b/src/libcephd/CMakeLists.txt index 11c739836f313..675e2d386a556 100644 --- a/src/libcephd/CMakeLists.txt +++ b/src/libcephd/CMakeLists.txt @@ -13,7 +13,10 @@ set(merge_libs kv global rocksdb - json_spirit) + json_spirit + erasure_code + cephd_compressor + cephd_ec) if(HAVE_ARMV8_CRC) list(APPEND merge_libs common_crc_aarch64) diff --git a/src/libcephd/libcephd.cc b/src/libcephd/libcephd.cc index 09a87c71dfc7d..f360017f5b69c 100644 --- a/src/libcephd/libcephd.cc +++ b/src/libcephd/libcephd.cc @@ -1,4 +1,16 @@ +#include "acconfig.h" #include "common/version.h" +#include "common/PluginRegistry.h" +#include "compressor/snappy/CompressionPluginSnappy.h" +#include "compressor/zlib/CompressionPluginZlib.h" +#include "erasure-code/ErasureCodePlugin.h" +#if __x86_64__ && defined(HAVE_BETTER_YASM_ELF64) +#include "erasure-code/isa/ErasureCodePluginIsa.h" +#endif +#include "erasure-code/jerasure/ErasureCodePluginJerasure.h" +#include "erasure-code/jerasure/jerasure_init.h" +#include "erasure-code/lrc/ErasureCodePluginLrc.h" +#include "erasure-code/shec/ErasureCodePluginShec.h" #include "include/cephd/libcephd.h" extern "C" void cephd_version(int *pmajor, int *pminor, int *ppatch) @@ -25,3 +37,75 @@ extern "C" const char *ceph_version(int *pmajor, int *pminor, int *ppatch) *ppatch = (n >= 3) ? patch : 0; return v; } + +// load the embedded plugins. This is safe to call multiple +// times in the same process +void cephd_preload_embedded_plugins() +{ + int r; + + // load erasure coding plugins + { + ErasureCodePlugin* plugin; + ErasureCodePluginRegistry& reg = ErasureCodePluginRegistry::instance(); + Mutex::Locker l(reg.lock); + reg.disable_dlclose = true; + + // initialize jerasure (and gf-complete) + int w[] = { 4, 8, 16, 32 }; + r = jerasure_init(4, w); + assert(r == 0); + + plugin = new ErasureCodePluginJerasure(); + r = reg.add("jerasure", plugin); + if (r == -EEXIST) { + delete plugin; + } + assert(r == 0); + + plugin = new ErasureCodePluginLrc(); + r = reg.add("lrc", plugin); + if (r == -EEXIST) { + delete plugin; + } + assert(r == 0); + + plugin = new ErasureCodePluginShec(); + r = reg.add("shec", plugin); + if (r == -EEXIST) { + delete plugin; + } + assert(r == 0); + +#if __x86_64__ && defined(HAVE_BETTER_YASM_ELF64) + plugin = new ErasureCodePluginIsa(); + r = reg.add("isa", plugin); + if (r == -EEXIST) { + delete plugin; + } + assert(r == 0); +#endif + } + + // now load the compression plugins + { + Plugin *plugin; + PluginRegistry *reg = g_ceph_context->get_plugin_registry(); + Mutex::Locker l(reg->lock); + reg->disable_dlclose = true; + + plugin = new CompressionPluginSnappy(g_ceph_context); + r = reg->add("compressor", "snappy", plugin); + if (r == -EEXIST) { + delete plugin; + } + assert(r == 0); + + plugin = new CompressionPluginZlib(g_ceph_context); + r = reg->add("compressor", "zlib", plugin); + if (r == -EEXIST) { + delete plugin; + } + assert(r == 0); + } +} -- 2.39.5