From: Sage Weil Date: Thu, 6 Oct 2016 17:43:13 +0000 (-0400) Subject: unittest_compression: subsume _zlib and _snappy test cases X-Git-Tag: v11.1.0~704^2~3 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=b49eda704634b80a9475948f8a88804a69ea8cef;p=ceph.git unittest_compression: subsume _zlib and _snappy test cases This includes zlib and isal interop. Signed-off-by: Sage Weil --- diff --git a/src/test/compressor/CMakeLists.txt b/src/test/compressor/CMakeLists.txt index 99edf0b7485c..7e7351596402 100644 --- a/src/test/compressor/CMakeLists.txt +++ b/src/test/compressor/CMakeLists.txt @@ -16,16 +16,3 @@ add_ceph_unittest(unittest_compression_plugin ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ target_link_libraries(unittest_compression_plugin global) add_dependencies(unittest_compression_plugin ceph_example) -# unittest_compression_snappy -add_executable(unittest_compression_snappy - test_compression_snappy.cc - ) -add_ceph_unittest(unittest_compression_snappy ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/unittest_compression_snappy) -target_link_libraries(unittest_compression_snappy global ceph_snappy) - -# unittest_compression_zlib -add_executable(unittest_compression_zlib - test_compression_zlib.cc - ) -add_ceph_unittest(unittest_compression_zlib ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/unittest_compression_zlib) -target_link_libraries(unittest_compression_zlib global ceph_zlib) diff --git a/src/test/compressor/test_compression.cc b/src/test/compressor/test_compression.cc index a3871fa81b78..2c81c3f6f61c 100644 --- a/src/test/compressor/test_compression.cc +++ b/src/test/compressor/test_compression.cc @@ -149,6 +149,168 @@ TEST_P(CompressionTest, big_round_trip_file) #endif +TEST_P(CompressionTest, compress_decompress) +{ + const char* test = "This is test text"; + int res; + int len = strlen(test); + bufferlist in, out; + bufferlist after; + bufferlist exp; + in.append(test, len); + res = compressor->compress(in, out); + EXPECT_EQ(res, 0); + res = compressor->decompress(out, after); + EXPECT_EQ(res, 0); + exp.append(test); + EXPECT_TRUE(exp.contents_equal(after)); + after.clear(); + size_t compressed_len = out.length(); + out.append_zero(12); + auto it = out.begin(); + res = compressor->decompress(it, compressed_len, after); + EXPECT_EQ(res, 0); + EXPECT_TRUE(exp.contents_equal(after)); + + //large block and non-begin iterator for continuous block + std::string data; + data.resize(0x10000 * 1); + for(size_t i = 0; i < data.size(); i++) + data[i] = i / 256; + in.clear(); + out.clear(); + in.append(data); + exp = in; + res = compressor->compress(in, out); + EXPECT_EQ(res, 0); + compressed_len = out.length(); + out.append_zero(0x10000 - out.length()); + after.clear(); + out.c_str(); + bufferlist prefix; + prefix.append(string("some prefix")); + size_t prefix_len = prefix.length(); + out.claim_prepend(prefix); + it = out.begin(); + it.advance(prefix_len); + res = compressor->decompress(it, compressed_len, after); + EXPECT_EQ(res, 0); + EXPECT_TRUE(exp.contents_equal(after)); +} + +TEST_P(CompressionTest, sharded_input_decompress) +{ + const size_t small_prefix_size=3; + + string test(128*1024,0); + int len = test.size(); + bufferlist in, out; + in.append(test.c_str(), len); + int res = compressor->compress(in, out); + EXPECT_EQ(res, 0); + EXPECT_GT(out.length(), small_prefix_size); + + bufferlist out2, tmp; + tmp.substr_of(out, 0, small_prefix_size ); + out2.append( tmp ); + size_t left = out.length()-small_prefix_size; + size_t offs = small_prefix_size; + while( left > 0 ){ + size_t shard_size = MIN( 2048, left ); + tmp.substr_of(out, offs, shard_size ); + out2.append( tmp ); + left -= shard_size; + offs += shard_size; + } + + bufferlist after; + res = compressor->decompress(out2, after); + EXPECT_EQ(res, 0); +} + +void test_compress(CompressorRef compressor, size_t size) +{ + char* data = (char*) malloc(size); + for (size_t t = 0; t < size; t++) { + data[t] = (t & 0xff) | (t >> 8); + } + bufferlist in; + in.append(data, size); + for (size_t t = 0; t < 100000; t++) { + bufferlist out; + int res = compressor->compress(in, out); + EXPECT_EQ(res, 0); + } +} + +void test_decompress(CompressorRef compressor, size_t size) +{ + char* data = (char*) malloc(size); + for (size_t t = 0; t < size; t++) { + data[t] = (t & 0xff) | (t >> 8); + } + bufferlist in, out; + in.append(data, size); + int res = compressor->compress(in, out); + EXPECT_EQ(res, 0); + for (size_t t = 0; t < 100000; t++) { + bufferlist out_dec; + int res = compressor->decompress(out, out_dec); + EXPECT_EQ(res, 0); + } +} + +TEST_P(CompressionTest, compress_1024) +{ + test_compress(compressor, 1024); +} + +TEST_P(CompressionTest, compress_2048) +{ + test_compress(compressor, 2048); +} + +TEST_P(CompressionTest, compress_4096) +{ + test_compress(compressor, 4096); +} + +TEST_P(CompressionTest, compress_8192) +{ + test_compress(compressor, 8192); +} + +TEST_P(CompressionTest, compress_16384) +{ + test_compress(compressor, 16384); +} + +TEST_P(CompressionTest, decompress_1024) +{ + test_decompress(compressor, 1024); +} + +TEST_P(CompressionTest, decompress_2048) +{ + test_decompress(compressor, 2048); +} + +TEST_P(CompressionTest, decompress_4096) +{ + test_decompress(compressor, 4096); +} + +TEST_P(CompressionTest, decompress_8192) +{ + test_decompress(compressor, 8192); +} + +TEST_P(CompressionTest, decompress_16384) +{ + test_decompress(compressor, 16384); +} + + INSTANTIATE_TEST_CASE_P( Compression, CompressionTest, @@ -157,6 +319,43 @@ INSTANTIATE_TEST_CASE_P( "zlib/noisal", "snappy")); +TEST(ZlibCompressor, zlib_isal_compatibility) +{ + g_conf->set_val("compressor_zlib_isal", "true"); + g_ceph_context->_conf->apply_changes(NULL); + CompressorRef isal = Compressor::create(g_ceph_context, "zlib"); + g_conf->set_val("compressor_zlib_isal", "false"); + g_ceph_context->_conf->apply_changes(NULL); + CompressorRef zlib = Compressor::create(g_ceph_context, "zlib"); + char test[101]; + srand(time(0)); + for (int i=0; i<100; ++i) + test[i] = 'a' + rand()%26; + test[100] = '\0'; + int len = strlen(test); + bufferlist in, out; + in.append(test, len); + // isal -> zlib + int res = isal->compress(in, out); + EXPECT_EQ(res, 0); + bufferlist after; + res = zlib->decompress(out, after); + EXPECT_EQ(res, 0); + bufferlist exp; + exp.append(test); + EXPECT_TRUE(exp.contents_equal(after)); + after.clear(); + out.clear(); + exp.clear(); + // zlib -> isal + res = zlib->compress(in, out); + EXPECT_EQ(res, 0); + res = isal->decompress(out, after); + EXPECT_EQ(res, 0); + exp.append(test); + EXPECT_TRUE(exp.contents_equal(after)); +} + int main(int argc, char **argv) { vector args; argv_to_vec(argc, (const char **)argv, args); diff --git a/src/test/compressor/test_compression_snappy.cc b/src/test/compressor/test_compression_snappy.cc deleted file mode 100644 index 296de4103b2c..000000000000 --- a/src/test/compressor/test_compression_snappy.cc +++ /dev/null @@ -1,99 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* - * Ceph distributed storage system - * - * Copyright (C) 2015 Mirantis, Inc. - * - * Author: Alyona Kiseleva - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - */ - -#include -#include -#include -#include "global/global_init.h" -#include "compressor/snappy/SnappyCompressor.h" -#include "common/ceph_argparse.h" -#include "global/global_context.h" -#include "common/config.h" - -TEST(SnappyCompressor, compress_decompress) -{ - SnappyCompressor sp; - EXPECT_EQ(sp.get_type(), "snappy"); - const char* test = "This is test text"; - int len = strlen(test); - bufferlist in, out; - in.append(test, len); - int res = sp.compress(in, out); - EXPECT_EQ(res, 0); - bufferlist after; - res = sp.decompress(out, after); - EXPECT_EQ(res, 0); - - after.clear(); - size_t compressed_len = out.length(); - out.append_zero(12); - auto it = out.begin(); - res = sp.decompress(it, compressed_len, after); - EXPECT_EQ(res, 0); -} - -TEST(SnappyCompressor, sharded_input_decompress) -{ - const size_t small_prefix_size=3; - - SnappyCompressor sp; - EXPECT_EQ(sp.get_type(), "snappy"); - string test(128*1024,0); - int len = test.size(); - bufferlist in, out; - in.append(test.c_str(), len); - int res = sp.compress(in, out); - EXPECT_EQ(res, 0); - EXPECT_GT(out.length(), small_prefix_size); - - bufferlist out2, tmp; - tmp.substr_of(out, 0, small_prefix_size ); - out2.append( tmp ); - size_t left = out.length()-small_prefix_size; - size_t offs = small_prefix_size; - while( left > 0 ){ - size_t shard_size = MIN( 2048, left ); - tmp.substr_of(out, offs, shard_size ); - out2.append( tmp ); - left -= shard_size; - offs += shard_size; - } - - bufferlist after; - res = sp.decompress(out2, after); - EXPECT_EQ(res, 0); -} - -int main(int argc, char **argv) { - vector args; - argv_to_vec(argc, (const char **)argv, args); - - global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0); - common_init_finish(g_ceph_context); - - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} - -/* - * Local Variables: - * compile-command: "cd ../.. ; make -j4 && - * make unittest_compression_snappy && - * valgrind --tool=memcheck \ - * ./unittest_compression_snappy \ - * --gtest_filter=*.* --log-to-stderr=true --debug-osd=20" - * End: - */ diff --git a/src/test/compressor/test_compression_zlib.cc b/src/test/compressor/test_compression_zlib.cc deleted file mode 100644 index daec78ebdd20..000000000000 --- a/src/test/compressor/test_compression_zlib.cc +++ /dev/null @@ -1,283 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* - * Ceph distributed storage system - * - * Copyright (C) 2015 Mirantis, Inc. - * - * Author: Alyona Kiseleva - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - */ - -#include -#include -#include -#include -#include "global/global_init.h" -#include "compressor/zlib/ZlibCompressor.h" -#include "common/ceph_argparse.h" -#include "global/global_context.h" -#include "common/config.h" - -TEST(ZlibCompressor, compress_decompress) -{ - ZlibCompressor sp(false); - EXPECT_STREQ(sp.get_type().c_str(), "zlib"); - const char* test = "This is test text"; - int res; - int len = strlen(test); - bufferlist in, out; - bufferlist after; - bufferlist exp; - in.append(test, len); - res = sp.compress(in, out); - EXPECT_EQ(res, 0); - res = sp.decompress(out, after); - EXPECT_EQ(res, 0); - exp.append(test); - EXPECT_TRUE(exp.contents_equal(after)); - after.clear(); - size_t compressed_len = out.length(); - out.append_zero(12); - auto it = out.begin(); - res = sp.decompress(it, compressed_len, after); - EXPECT_EQ(res, 0); - EXPECT_TRUE(exp.contents_equal(after)); - - //large block and non-begin iterator for continuous block - std::string data; - data.resize(0x10000 * 1); - for(size_t i = 0; i < data.size(); i++) - data[i] = i / 256; - in.clear(); - out.clear(); - in.append(data); - exp = in; - res = sp.compress(in, out); - EXPECT_EQ(res, 0); - compressed_len = out.length(); - out.append_zero(0x10000 - out.length()); - after.clear(); - out.c_str(); - bufferlist prefix; - prefix.append(string("some prefix")); - size_t prefix_len = prefix.length(); - out.claim_prepend(prefix); - it = out.begin(); - it.advance(prefix_len); - res = sp.decompress(it, compressed_len, after); - EXPECT_EQ(res, 0); - EXPECT_TRUE(exp.contents_equal(after)); -} - -TEST(ZlibCompressor, compress_decompress_chunk) -{ - ZlibCompressor sp(false); - EXPECT_STREQ(sp.get_type().c_str(), "zlib"); - const char* test = "This is test text"; - buffer::ptr test2 ("1234567890", 10); - int len = strlen(test); - bufferlist in, out; - in.append(test, len); - in.append(test2); - int res = sp.compress(in, out); - EXPECT_EQ(res, 0); - bufferlist after; - res = sp.decompress(out, after); - EXPECT_EQ(res, 0); - bufferlist exp; - exp.append("This is test text1234567890"); - EXPECT_TRUE(exp.contents_equal(after)); -} - -TEST(ZlibCompressor, compress_decompress_isal) -{ - ZlibCompressor sp(true); - EXPECT_STREQ(sp.get_type().c_str(), "zlib"); - const char* test = "This is test text"; - int len = strlen(test); - bufferlist in, out; - in.append(test, len); - int res = sp.compress(in, out); - EXPECT_EQ(res, 0); - bufferlist after; - res = sp.decompress(out, after); - EXPECT_EQ(res, 0); - bufferlist exp; - exp.append(test); - EXPECT_TRUE(exp.contents_equal(after)); - after.clear(); - size_t compressed_len = out.length(); - out.append_zero(12); - auto it = out.begin(); - res = sp.decompress(it, compressed_len, after); - EXPECT_EQ(res, 0); - EXPECT_TRUE(exp.contents_equal(after)); -} - -TEST(ZlibCompressor, compress_decompress_chunk_isal) -{ - ZlibCompressor sp(true); - EXPECT_STREQ(sp.get_type().c_str(), "zlib"); - const char* test = "This is test text"; - buffer::ptr test2 ("1234567890", 10); - int len = strlen(test); - bufferlist in, out; - in.append(test, len); - in.append(test2); - int res = sp.compress(in, out); - EXPECT_EQ(res, 0); - bufferlist after; - res = sp.decompress(out, after); - EXPECT_EQ(res, 0); - bufferlist exp; - exp.append("This is test text1234567890"); - EXPECT_TRUE(exp.contents_equal(after)); -} - -TEST(ZlibCompressor, zlib_isal_compatibility) -{ - ZlibCompressor isal(true); - EXPECT_STREQ(isal.get_type().c_str(), "zlib"); - ZlibCompressor zlib(false); - EXPECT_STREQ(zlib.get_type().c_str(), "zlib"); - char test[101]; - srand(time(0)); - for (int i=0; i<100; ++i) - test[i] = 'a' + rand()%26; - test[100] = '\0'; - int len = strlen(test); - bufferlist in, out; - in.append(test, len); - // isal -> zlib - int res = isal.compress(in, out); - EXPECT_EQ(res, 0); - bufferlist after; - res = zlib.decompress(out, after); - EXPECT_EQ(res, 0); - bufferlist exp; - exp.append(test); - EXPECT_TRUE(exp.contents_equal(after)); - after.clear(); - out.clear(); - exp.clear(); - // zlib -> isal - res = zlib.compress(in, out); - EXPECT_EQ(res, 0); - res = isal.decompress(out, after); - EXPECT_EQ(res, 0); - exp.append(test); - EXPECT_TRUE(exp.contents_equal(after)); -} - -void test_compress(size_t size) -{ - ZlibCompressor sp(false); - EXPECT_STREQ(sp.get_type().c_str(), "zlib"); - char* data = (char*) malloc(size); - for (size_t t = 0; t < size; t++) { - data[t] = (t & 0xff) | (t >> 8); - } - bufferlist in; - in.append(data, size); - for (size_t t = 0; t < 100000; t++) { - bufferlist out; - int res = sp.compress(in, out); - EXPECT_EQ(res, 0); - } -} - -void test_decompress(size_t size) -{ - ZlibCompressor sp(false); - EXPECT_STREQ(sp.get_type().c_str(), "zlib"); - char* data = (char*) malloc(size); - for (size_t t = 0; t < size; t++) { - data[t] = (t & 0xff) | (t >> 8); - } - bufferlist in, out; - in.append(data, size); - int res = sp.compress(in, out); - EXPECT_EQ(res, 0); - for (size_t t = 0; t < 100000; t++) { - bufferlist out_dec; - int res = sp.decompress(out, out_dec); - EXPECT_EQ(res, 0); - } -} - -TEST(ZlibCompressor, compress_1024) -{ - test_compress(1024); -} - -TEST(ZlibCompressor, compress_2048) -{ - test_compress(2048); -} - -TEST(ZlibCompressor, compress_4096) -{ - test_compress(4096); -} - -TEST(ZlibCompressor, compress_8192) -{ - test_compress(8192); -} - -TEST(ZlibCompressor, compress_16384) -{ - test_compress(16384); -} - -TEST(Zlibdecompressor, decompress_1024) -{ - test_decompress(1024); -} - -TEST(Zlibdecompressor, decompress_2048) -{ - test_decompress(2048); -} - -TEST(Zlibdecompressor, decompress_4096) -{ - test_decompress(4096); -} - -TEST(Zlibdecompressor, decompress_8192) -{ - test_decompress(8192); -} - -TEST(Zlibdecompressor, decompress_16384) -{ - test_decompress(16384); -} - -int main(int argc, char **argv) { - vector args; - argv_to_vec(argc, (const char **)argv, args); - - global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0); - common_init_finish(g_ceph_context); - - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} - -/* - * Local Variables: - * compile-command: "cd ../.. ; make -j4 && - * make unittest_compression_zlib && - * valgrind --tool=memcheck \ - * ./unittest_compression_zlib \ - * --gtest_filter=*.* --log-to-stderr=true --debug-osd=20" - * End: - */