From: Piotr Dałek Date: Thu, 27 Apr 2017 10:27:21 +0000 (+0200) Subject: os/bluestore: use denc for varint encoding X-Git-Tag: v12.0.3~42^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=ccba64ba211b07eef5a69c9628dc1f24db89d608;p=ceph.git os/bluestore: use denc for varint encoding This changes remaining bluestore types to use new denc code, getting rid of the old and slow small_encoding.h code in the process. Signed-off-by: Piotr Dałek --- diff --git a/src/include/small_encoding.h b/src/include/small_encoding.h deleted file mode 100644 index efd4953fc48a..000000000000 --- a/src/include/small_encoding.h +++ /dev/null @@ -1,245 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -#ifndef CEPH_SMALL_ENCODING_H -#define CEPH_SMALL_ENCODING_H - -#include "include/buffer.h" -#include "include/int_types.h" - -// varint encoding -// -// high bit of every byte indicates whether another byte follows. -template -inline void small_encode_varint(T v, bufferlist& bl) { - uint8_t byte = v & 0x7f; - v >>= 7; - while (v) { - byte |= 0x80; - ::encode(byte, bl); - byte = (v & 0x7f); - v >>= 7; - } - ::encode(byte, bl); -} - -template -inline void small_decode_varint(T& v, bufferlist::iterator& p) -{ - uint8_t byte; - ::decode(byte, p); - v = byte & 0x7f; - int shift = 7; - while (byte & 0x80) { - ::decode(byte, p); - v |= (T)(byte & 0x7f) << shift; - shift += 7; - } -} - -// signed varint encoding -// -// low bit = 1 = negative, 0 = positive -// high bit of every byte indicates whether another byte follows. -inline void small_encode_signed_varint(int64_t v, bufferlist& bl) { - if (v < 0) { - v = (-v << 1) | 1; - } else { - v <<= 1; - } - small_encode_varint(v, bl); -} - -template -inline void small_decode_signed_varint(T& v, bufferlist::iterator& p) -{ - int64_t i; - small_decode_varint(i, p); - if (i & 1) { - v = -(i >> 1); - } else { - v = i >> 1; - } -} - -// varint + lowz encoding -// -// first(low) 2 bits = how many low zero bits (nibbles) -// high bit of each byte = another byte follows -// (so, 5 bits data in first byte, 7 bits data thereafter) -inline void small_encode_varint_lowz(uint64_t v, bufferlist& bl) { - int lowznib = v ? (ctz(v) / 4) : 0; - if (lowznib > 3) - lowznib = 3; - v >>= lowznib * 4; - v <<= 2; - v |= lowznib; - small_encode_varint(v, bl); -} - -template -inline void small_decode_varint_lowz(T& v, bufferlist::iterator& p) -{ - uint64_t i; - small_decode_varint(i, p); - int lowznib = (i & 3); - i >>= 2; - i <<= lowznib * 4; - v = i; -} - -// signed varint + lowz encoding -// -// first low bit = 1 for negative, 0 for positive -// next 2 bits = how many low zero bits (nibbles) -// high bit of each byte = another byte follows -// (so, 4 bits data in first byte, 7 bits data thereafter) -inline void small_encode_signed_varint_lowz(int64_t v, bufferlist& bl) { - bool negative = false; - if (v < 0) { - v = -v; - negative = true; - } - int lowznib = v ? (ctz(v) / 4) : 0; - if (lowznib > 3) - lowznib = 3; - v >>= lowznib * 4; - v <<= 3; - v |= lowznib << 1; - v |= (int)negative; - small_encode_varint(v, bl); -} - -template -inline void small_decode_signed_varint_lowz(T& v, bufferlist::iterator& p) -{ - int64_t i; - small_decode_varint(i, p); - int lowznib = (i & 6) >> 1; - if (i & 1) { - i >>= 3; - i <<= lowznib * 4; - v = -i; - } else { - i >>= 3; - i <<= lowznib * 4; - v = i; - } -} - - -// LBA -// -// first 1-3 bits = how many low zero bits -// *0 = 12 (common 4 K alignment case) -// *01 = 16 -// *011 = 20 -// *111 = byte -// then 28-30 bits of data -// then last bit = another byte follows -// high bit of each subsequent byte = another byte follows -inline void small_encode_lba(uint64_t v, bufferlist& bl) { - int low_zero_nibbles = v ? (int)(ctz(v) / 4) : 0; - int pos; - uint32_t word; - int t = low_zero_nibbles - 3; - if (t < 0) { - pos = 3; - word = 0x7; - } else if (t < 3) { - v >>= (low_zero_nibbles * 4); - pos = t + 1; - word = (1 << t) - 1; - } else { - v >>= 20; - pos = 3; - word = 0x3; - } - word |= (v << pos) & 0x7fffffff; - v >>= 31 - pos; - if (!v) { - ::encode(word, bl); - return; - } - word |= 0x80000000; - ::encode(word, bl); - uint8_t byte = v & 0x7f; - v >>= 7; - while (v) { - byte |= 0x80; - ::encode(byte, bl); - byte = (v & 0x7f); - v >>= 7; - } - ::encode(byte, bl); -} - -inline void small_decode_lba(uint64_t& v, bufferlist::iterator& p) { - uint32_t word; - ::decode(word, p); - int shift; - switch (word & 7) { - case 0: - case 2: - case 4: - case 6: - v = (uint64_t)(word & 0x7ffffffe) << (12 - 1); - shift = 12 + 30; - break; - case 1: - case 5: - v = (uint64_t)(word & 0x7ffffffc) << (16 - 2); - shift = 16 + 29; - break; - case 3: - v = (uint64_t)(word & 0x7ffffff8) << (20 - 3); - shift = 20 + 28; - break; - case 7: - v = (uint64_t)(word & 0x7ffffff8) >> 3; - shift = 28; - } - uint8_t byte = word >> 24; - while (byte & 0x80) { - ::decode(byte, p); - v |= (uint64_t)(byte & 0x7f) << shift; - shift += 7; - } -} - - -// short bufferptrs, bufferlists, strings -template -inline void small_encode_buf_lowz(const T& bp, bufferlist& bl) { - size_t l = bp.length(); - small_encode_varint_lowz(l, bl); - bl.append(bp); -} -template -inline void small_decode_buf_lowz(T& bp, bufferlist::iterator& p) { - size_t l; - small_decode_varint_lowz(l, p); - p.copy_deep(l, bp); -} - -// STL containers - -template -inline void small_encode_obj(const std::vector& v, bufferlist& bl) { - size_t n = v.size(); - small_encode_varint(n, bl); - for (auto p = v.cbegin(); p != v.cend(); ++p) { - p->encode(bl); - } -} -template -inline void small_decode_obj(std::vector& v, bufferlist::iterator& p) { - size_t n; - small_decode_varint(n, p); - v.clear(); - while (n--) { - v.push_back(T()); - v.back().decode(p); - } -} - -#endif diff --git a/src/include/utime.h b/src/include/utime.h index da73f47235ec..732eaebcdff0 100644 --- a/src/include/utime.h +++ b/src/include/utime.h @@ -24,6 +24,7 @@ #include "include/timegm.h" #include "common/strtol.h" #include "common/ceph_time.h" +#include "include/denc.h" // -------- @@ -136,6 +137,12 @@ public: #endif } + DENC(utime_t, v, p) { + denc(v.tv.tv_sec, p); + denc(v.tv.tv_nsec, p); + } + + void encode_timeval(struct ceph_timespec *t) const { t->tv_sec = tv.tv_sec; t->tv_nsec = tv.tv_nsec; @@ -381,6 +388,7 @@ public: } }; WRITE_CLASS_ENCODER(utime_t) +WRITE_CLASS_DENC(utime_t) // arithmetic operators diff --git a/src/os/bluestore/bluefs_types.cc b/src/os/bluestore/bluefs_types.cc index a340de2f121c..e414c8b6d5be 100644 --- a/src/os/bluestore/bluefs_types.cc +++ b/src/os/bluestore/bluefs_types.cc @@ -5,28 +5,8 @@ #include "common/Formatter.h" #include "include/uuid.h" #include "include/stringify.h" -#include "include/small_encoding.h" // bluefs_extent_t - -void bluefs_extent_t::encode(bufferlist& bl) const -{ - ENCODE_START(1, 1, bl); - small_encode_lba(offset, bl); - small_encode_varint_lowz(length, bl); - ::encode(bdev, bl); - ENCODE_FINISH(bl); -} - -void bluefs_extent_t::decode(bufferlist::iterator& p) -{ - DECODE_START(1, p); - small_decode_lba(offset, p); - small_decode_varint_lowz(length, p); - ::decode(bdev, p); - DECODE_FINISH(p); -} - void bluefs_extent_t::dump(Formatter *f) const { f->dump_unsigned("offset", offset); @@ -118,29 +98,6 @@ mempool::bluefs::vector::iterator bluefs_fnode_t::seek( return p; } -void bluefs_fnode_t::encode(bufferlist& bl) const -{ - ENCODE_START(1, 1, bl); - small_encode_varint(ino, bl); - small_encode_varint(size, bl); - ::encode(mtime, bl); - ::encode(prefer_bdev, bl); - ::encode(extents, bl); - ENCODE_FINISH(bl); -} - -void bluefs_fnode_t::decode(bufferlist::iterator& p) -{ - DECODE_START(1, p); - small_decode_varint(ino, p); - small_decode_varint(size, p); - ::decode(mtime, p); - ::decode(prefer_bdev, p); - ::decode(extents, p); - DECODE_FINISH(p); - recalc_allocated(); -} - void bluefs_fnode_t::dump(Formatter *f) const { f->dump_unsigned("ino", ino); diff --git a/src/os/bluestore/bluefs_types.h b/src/os/bluestore/bluefs_types.h index 6586f32277f8..6a015892d70b 100644 --- a/src/os/bluestore/bluefs_types.h +++ b/src/os/bluestore/bluefs_types.h @@ -6,6 +6,7 @@ #include "bluestore_types.h" #include "include/utime.h" #include "include/encoding.h" +#include "include/denc.h" class bluefs_extent_t : public AllocExtent{ public: @@ -14,12 +15,18 @@ public: bluefs_extent_t(uint8_t b = 0, uint64_t o = 0, uint32_t l = 0) : AllocExtent(o, l), bdev(b) {} - void encode(bufferlist&) const; - void decode(bufferlist::iterator&); + DENC(bluefs_extent_t, v, p) { + DENC_START(1, 1, p); + denc_lba(v.offset, p); + denc_varint_lowz(v.length, p); + denc(v.bdev, p); + DENC_FINISH(p); + } + void dump(Formatter *f) const; static void generate_test_instances(list&); }; -WRITE_CLASS_ENCODER(bluefs_extent_t) +WRITE_CLASS_DENC(bluefs_extent_t) ostream& operator<<(ostream& out, bluefs_extent_t e); @@ -44,15 +51,23 @@ struct bluefs_fnode_t { allocated += p.length; } + DENC(bluefs_fnode_t, v, p) { + DENC_START(1, 1, p); + denc_varint(v.ino, p); + denc_varint(v.size, p); + denc(v.mtime, p); + denc(v.prefer_bdev, p); + denc(v.extents, p); + DENC_FINISH(p); + } + mempool::bluefs::vector::iterator seek( uint64_t off, uint64_t *x_off); - void encode(bufferlist& bl) const; - void decode(bufferlist::iterator& p); void dump(Formatter *f) const; static void generate_test_instances(list& ls); }; -WRITE_CLASS_ENCODER(bluefs_fnode_t) +WRITE_CLASS_DENC(bluefs_fnode_t) ostream& operator<<(ostream& out, const bluefs_fnode_t& file); diff --git a/src/os/bluestore/bluestore_types.h b/src/os/bluestore/bluestore_types.h index b62a32eea674..962d4909c71f 100644 --- a/src/os/bluestore/bluestore_types.h +++ b/src/os/bluestore/bluestore_types.h @@ -20,7 +20,6 @@ #include "include/types.h" #include "include/interval_set.h" #include "include/utime.h" -#include "include/small_encoding.h" #include "common/hobject.h" #include "compressor/Compressor.h" #include "common/Checksummer.h" diff --git a/src/test/encoding.cc b/src/test/encoding.cc index 29c3b99fddc3..2bcf7c01a38e 100644 --- a/src/test/encoding.cc +++ b/src/test/encoding.cc @@ -1,6 +1,5 @@ #include "include/buffer.h" #include "include/encoding.h" -#include "include/small_encoding.h" #include "gtest/gtest.h" @@ -350,39 +349,48 @@ TEST(small_encoding, varint) { for (unsigned i=0; v[i][1]; ++i) { { bufferlist bl; - small_encode_varint(v[i][0], bl); + { + auto app = bl.get_contiguous_appender(16, true); + denc_varint(v[i][0], app); + } cout << std::hex << v[i][0] << "\t" << v[i][1] << "\t"; bl.hexdump(cout, false); cout << std::endl; ASSERT_EQ(bl.length(), v[i][1]); uint32_t u; - auto p = bl.begin(); - small_decode_varint(u, p); + auto p = bl.begin().get_current_ptr().begin(); + denc_varint(u, p); ASSERT_EQ(v[i][0], u); } { bufferlist bl; - small_encode_signed_varint(v[i][0], bl); + { + auto app = bl.get_contiguous_appender(16, true); + denc_signed_varint(v[i][0], app); + } cout << std::hex << v[i][0] << "\t" << v[i][2] << "\t"; bl.hexdump(cout, false); cout << std::endl; ASSERT_EQ(bl.length(), v[i][2]); int32_t u; - auto p = bl.begin(); - small_decode_signed_varint(u, p); + auto p = bl.begin().get_current_ptr().begin(); + denc_signed_varint(u, p); ASSERT_EQ((int32_t)v[i][0], u); } { bufferlist bl; int64_t x = -(int64_t)v[i][0]; - small_encode_signed_varint(x, bl); + { + auto app = bl.get_contiguous_appender(16, true); + denc_signed_varint(x, app); + } cout << std::dec << x << std::hex << "\t" << v[i][3] << "\t"; bl.hexdump(cout, false); cout << std::endl; ASSERT_EQ(bl.length(), v[i][3]); int64_t u; - auto p = bl.begin(); - small_decode_signed_varint(u, p); + auto p = bl.begin().get_current_ptr().begin(); + denc_signed_varint(u, p); ASSERT_EQ(x, u); } } @@ -419,40 +427,49 @@ TEST(small_encoding, varint_lowz) { for (unsigned i=0; v[i][1]; ++i) { { bufferlist bl; - small_encode_varint_lowz(v[i][0], bl); + { + auto app = bl.get_contiguous_appender(16, true); + denc_varint_lowz(v[i][0], app); + } cout << std::hex << v[i][0] << "\t" << v[i][1] << "\t"; bl.hexdump(cout, false); cout << std::endl; ASSERT_EQ(bl.length(), v[i][1]); uint32_t u; - auto p = bl.begin(); - small_decode_varint_lowz(u, p); + auto p = bl.begin().get_current_ptr().begin(); + denc_varint_lowz(u, p); ASSERT_EQ(v[i][0], u); } { bufferlist bl; int64_t x = v[i][0]; - small_encode_signed_varint_lowz(x, bl); + { + auto app = bl.get_contiguous_appender(16, true); + denc_signed_varint_lowz(x, app); + } cout << std::hex << x << "\t" << v[i][1] << "\t"; bl.hexdump(cout, false); cout << std::endl; ASSERT_EQ(bl.length(), v[i][2]); int64_t u; - auto p = bl.begin(); - small_decode_signed_varint_lowz(u, p); + auto p = bl.begin().get_current_ptr().begin(); + denc_signed_varint_lowz(u, p); ASSERT_EQ(x, u); } { bufferlist bl; int64_t x = -(int64_t)v[i][0]; - small_encode_signed_varint_lowz(x, bl); + { + auto app = bl.get_contiguous_appender(16, true); + denc_signed_varint_lowz(x, app); + } cout << std::dec << x << "\t" << v[i][1] << "\t"; bl.hexdump(cout, false); cout << std::endl; ASSERT_EQ(bl.length(), v[i][3]); int64_t u; - auto p = bl.begin(); - small_decode_signed_varint_lowz(u, p); + auto p = bl.begin().get_current_ptr().begin(); + denc_signed_varint_lowz(u, p); ASSERT_EQ(x, u); } } @@ -481,14 +498,17 @@ TEST(small_encoding, lba) { }; for (unsigned i=0; v[i][1]; ++i) { bufferlist bl; - small_encode_lba(v[i][0], bl); + { + auto app = bl.get_contiguous_appender(16, true); + denc_lba(v[i][0], app); + } cout << std::hex << v[i][0] << "\t" << v[i][1] << "\t"; bl.hexdump(cout, false); cout << std::endl; ASSERT_EQ(bl.length(), v[i][1]); uint64_t u; - auto p = bl.begin(); - small_decode_lba(u, p); + auto p = bl.begin().get_current_ptr().begin(); + denc_lba(u, p); ASSERT_EQ(v[i][0], u); }