From 72bf7e777dc40eff6881ad13514bc483ccbeee04 Mon Sep 17 00:00:00 2001 From: sage Date: Mon, 5 Dec 2005 01:52:52 +0000 Subject: [PATCH] *** empty log message *** git-svn-id: https://ceph.svn.sf.net/svnroot/ceph@515 29311d96-e01e-0410-9327-a35deaab8ce9 --- ceph/ebofs/AlignedBufferPool.h | 1 + ceph/ebofs/Allocator.cc | 36 +++++- ceph/ebofs/Allocator.h | 2 + ceph/ebofs/BlockDevice.cc | 4 +- ceph/ebofs/BufferCache.cc | 14 +- ceph/ebofs/BufferCache.h | 83 +++++++++++- ceph/ebofs/Ebofs.cc | 229 +++++++++++++++++++++++++++------ ceph/ebofs/Ebofs.h | 2 +- ceph/ebofs/Table.h | 4 +- ceph/ebofs/mkfs.ebofs.cc | 64 ++++++++- ceph/include/bufferlist.h | 4 + 11 files changed, 382 insertions(+), 61 deletions(-) diff --git a/ceph/ebofs/AlignedBufferPool.h b/ceph/ebofs/AlignedBufferPool.h index d7b772e7aa31e..1ada70c3722d7 100644 --- a/ceph/ebofs/AlignedBufferPool.h +++ b/ceph/ebofs/AlignedBufferPool.h @@ -55,6 +55,7 @@ class AlignedBufferPool { } + // allocate a single buffer buffer* alloc() { // get more memory? if (freelist.empty()) { diff --git a/ceph/ebofs/Allocator.cc b/ceph/ebofs/Allocator.cc index 19625b8fd7173..b8c9c6bab6bc9 100644 --- a/ceph/ebofs/Allocator.cc +++ b/ceph/ebofs/Allocator.cc @@ -3,6 +3,28 @@ #include "Ebofs.h" +#undef dout +#define dout(x) if (x <= g_conf.debug) cout << "allocator." + + +void Allocator::dump_freelist() +{ + for (int b=0; bfree_tab[b]->get_num_keys() > 0) { + Table::Cursor cursor(fs->free_tab[b]); + fs->free_tab[b]->find(0, cursor); + while (1) { + cout << " ex " << cursor.current().key << " + " << cursor.current().value << endl; + if (cursor.move_right() < 0) break; + } + } else { + cout << " empty" << endl; + } + } +} + + int Allocator::find(Extent& ex, int bucket, block_t num, block_t near) { Table::Cursor cursor(fs->free_tab[bucket]); @@ -14,14 +36,14 @@ int Allocator::find(Extent& ex, int bucket, block_t num, block_t near) do { if (cursor.current().value >= num) found = true; - } while (!found && cursor.move_right() > 0); + } while (!found && cursor.move_right() >= 0); } if (!found) { // look to the left fs->free_tab[bucket]->find( near, cursor ); - while (!found && cursor.move_left() > 0) + while (!found && cursor.move_left() >= 0) if (cursor.current().value >= num) found = true; } @@ -88,6 +110,7 @@ int Allocator::allocate(Extent& ex, block_t num, block_t near) } dout(1) << "allocator.alloc " << ex << " near " << near << endl; + dump_freelist(); return num; } } @@ -101,11 +124,14 @@ int Allocator::allocate(Extent& ex, block_t num, block_t near) fs->free_tab[bucket]->remove(ex.start); fs->free_blocks -= ex.length; + dout(1) << "allocator.alloc partial " << ex << " near " << near << endl; + dump_freelist(); return ex.length; } } dout(1) << "allocate failed, fs full! " << fs->free_blocks << endl; + dump_freelist(); return -1; } @@ -113,6 +139,8 @@ int Allocator::release(Extent& ex) { Extent newex = ex; + dout(1) << "release " << ex << endl; + // one after us? for (int b=0; b::Cursor cursor(fs->free_tab[b]); @@ -132,7 +160,7 @@ int Allocator::release(Extent& ex) for (int b=0; b::Cursor cursor(fs->free_tab[b]); fs->free_tab[b]->find( newex.start+newex.length, cursor ); - if (cursor.move_left() > 0 && + if (cursor.move_left() >= 0 && (cursor.current().key + cursor.current().value == newex.start)) { // merge newex.start = cursor.current().key; @@ -149,6 +177,8 @@ int Allocator::release(Extent& ex) // ok, insert newex int b = pick_bucket(ex.length); fs->free_tab[b]->insert(ex.start, ex.length); + + dump_freelist(); return 0; } diff --git a/ceph/ebofs/Allocator.h b/ceph/ebofs/Allocator.h index a143e0f760ce9..1c8fce4a34026 100644 --- a/ceph/ebofs/Allocator.h +++ b/ceph/ebofs/Allocator.h @@ -22,6 +22,8 @@ class Allocator { int find(Extent& ex, int bucket, block_t num, block_t near); + void dump_freelist(); + public: Allocator(Ebofs *f) : fs(f) {} diff --git a/ceph/ebofs/BlockDevice.cc b/ceph/ebofs/BlockDevice.cc index 1743170c8834f..8497db14cd7af 100644 --- a/ceph/ebofs/BlockDevice.cc +++ b/ceph/ebofs/BlockDevice.cc @@ -95,7 +95,7 @@ void BlockDevice::do_io(biovec *bio) bio->rval = r; } else if (bio->context) { - bio->context->finish(r); + bio->context->finish((int)bio); delete bio->context; delete bio; } @@ -268,7 +268,7 @@ int BlockDevice::cancel_io(ioh_t ioh) // FIXME? if (r == 0 && pbio->context) { - pbio->context->finish(-1); + pbio->context->finish(0); delete pbio->context; delete pbio; } diff --git a/ceph/ebofs/BufferCache.cc b/ceph/ebofs/BufferCache.cc index b0d6150b08041..0cea676f0cdd3 100644 --- a/ceph/ebofs/BufferCache.cc +++ b/ceph/ebofs/BufferCache.cc @@ -22,7 +22,7 @@ -void ObjectCache::rx_finish(block_t start, block_t length) +void ObjectCache::rx_finish(ioh_t ioh, block_t start, block_t length) { list waiters; @@ -50,6 +50,8 @@ void ObjectCache::rx_finish(block_t start, block_t length) else { dout(10) << "rx_finish ignoring " << *p->second << endl; } + + if (p->second->ioh == ioh) p->second->ioh = 0; // trigger waiters waiters.splice(waiters.begin(), p->second->waitfor_read); @@ -61,7 +63,7 @@ void ObjectCache::rx_finish(block_t start, block_t length) } -void ObjectCache::tx_finish(block_t start, block_t length, version_t version) +void ObjectCache::tx_finish(ioh_t ioh, block_t start, block_t length, version_t version) { list waiters; @@ -90,6 +92,8 @@ void ObjectCache::tx_finish(block_t start, block_t length, version_t version) p->second->set_last_flushed(version); bc->mark_clean(p->second); + if (p->second->ioh == ioh) p->second->ioh = 0; + // trigger waiters waiters.splice(waiters.begin(), p->second->waitfor_flush); } @@ -112,7 +116,8 @@ int ObjectCache::map_read(block_t start, block_t len, block_t cur = start; block_t left = len; - if (p != data.begin() && p->first < cur) { + if (p != data.begin() && + (p->first > cur || p == data.end())) { p--; // might overlap! if (p->first + p->second->length() <= cur) p++; // doesn't overlap. @@ -192,7 +197,8 @@ int ObjectCache::map_write(block_t start, block_t len, block_t cur = start; block_t left = len; - if (p != data.begin() && p->first < cur) { + if (p != data.begin() && + (p->first > cur || p == data.end())) { p--; // might overlap! if (p->first + p->second->length() <= cur) p++; // doesn't overlap. diff --git a/ceph/ebofs/BufferCache.h b/ceph/ebofs/BufferCache.h index ccd376f0c0763..e647d87e2e9ae 100644 --- a/ceph/ebofs/BufferCache.h +++ b/ceph/ebofs/BufferCache.h @@ -108,7 +108,75 @@ class BufferHead : public LRUObject { } */ + void copy_partial_substr(off_t start, off_t end, bufferlist& bl) { + map::iterator i = partial.begin(); + + // skip first bits (fully to left) + while ((i->first + i->second.length() < start) && + i != partial.end()) + i++; + assert(i != partial.end()); + assert(i->first <= start); + + // first + unsigned bhoff = MAX(start, i->first) - i->first; + unsigned bhlen = MIN(end-start, i->second.length()); + bl.substr_of( i->second, bhoff, bhlen ); + + off_t pos = i->first + i->second.length(); + + // have continuous to end? + for (i++; i != partial.end(); i++) { + if (pos >= end) break; + assert(pos == i->first); + + pos = i->first + i->second.length(); + + if (pos <= end) { // this whole frag + bl.append( i->second ); + } else { // partial end + unsigned bhlen = end-start-bl.length(); + bufferlist frag; + frag.substr_of( i->second, 0, bhlen ); + bl.claim_append(frag); + break; // done. + } + } + + assert(pos >= end); + assert(bl.length() == (unsigned)(end-start)); + } + + bool have_partial_range(off_t start, off_t end) { + map::iterator i = partial.begin(); + + // skip first bits (fully to left) + while ((i->first + i->second.length() < start) && + i != partial.end()) + i++; + if (i == partial.end()) return false; + + // have start? + if (i->first > start) return false; + off_t pos = i->first + i->second.length(); + + // have continuous to end? + for (i++; i != partial.end(); i++) { + assert(pos <= i->first); + if (pos < i->first) return false; + assert(pos == i->first); + pos = i->first + i->second.length(); + if (pos >= end) break; // gone far enough + } + + if (pos >= end) return true; + return false; + } + bool partial_is_complete(off_t size) { + return have_partial_range( (off_t)(start()*EBOFS_BLOCK_SIZE), + MIN( size, (off_t)(end()*EBOFS_BLOCK_SIZE) ) ); + /* map::iterator i = partial.begin(); if (i == partial.end()) return false; if (i->first != (off_t)(object_loc.start * EBOFS_BLOCK_SIZE)) return false; @@ -122,6 +190,7 @@ class BufferHead : public LRUObject { off_t upto = MIN( size, (off_t)(end()*EBOFS_BLOCK_SIZE) ); if (pos == upto) return true; return false; + */ } void apply_partial() { const off_t bhstart = start() * EBOFS_BLOCK_SIZE; @@ -233,8 +302,8 @@ class ObjectCache { int scan_versions(block_t start, block_t len, version_t& low, version_t& high); - void rx_finish(block_t start, block_t length); - void tx_finish(block_t start, block_t length, version_t v); + void rx_finish(ioh_t ioh, block_t start, block_t length); + void tx_finish(ioh_t ioh, block_t start, block_t length, version_t v); }; @@ -245,8 +314,9 @@ public: C_OC_RxFinish(ObjectCache *o, block_t s, block_t l) : oc(o), start(s), length(l) {} void finish(int r) { - if (r == 0) - oc->rx_finish(start, length); + ioh_t ioh = (ioh_t)r; + if (ioh) + oc->rx_finish(ioh, start, length); } }; @@ -258,8 +328,9 @@ public: C_OC_TxFinish(ObjectCache *o, block_t s, block_t l, version_t v) : oc(o), start(s), length(l), version(v) {} void finish(int r) { - if (r == 0) - oc->tx_finish(start, length, version); + ioh_t ioh = (ioh_t)r; + if (ioh) + oc->tx_finish(ioh, start, length, version); } }; diff --git a/ceph/ebofs/Ebofs.cc b/ceph/ebofs/Ebofs.cc index f0d1a1808a7ca..30530638c89b5 100644 --- a/ceph/ebofs/Ebofs.cc +++ b/ceph/ebofs/Ebofs.cc @@ -375,42 +375,37 @@ void Ebofs::trim_buffer_cache() bc.lock.Unlock(); } -class C_E_Flush : public Context { - int *i; - Mutex *m; - Cond *c; -public: - C_E_Flush(int *_i, Mutex *_m, Cond *_c) : i(_i), m(_m), c(_c) {} - void finish(int r) { - (*i)--; - m->Lock(); - c->Signal(); - m->Unlock(); - } -}; void Ebofs::flush_all() { // FIXME what about partial heads? - // write all dirty bufferheads + dout(1) << "flush_all" << endl; + bc.lock.Lock(); - dout(1) << "flush_all writing dirty bufferheads" << endl; - while (!bc.dirty_bh.empty()) { - set::iterator i = bc.dirty_bh.begin(); - BufferHead *bh = *i; - if (bh->ioh) continue; - Onode *on = get_onode(bh->oc->get_object_id()); - bh_write(on, bh); - put_onode(on); - } - dout(1) << "flush_all submitted" << endl; + while (bc.get_stat_dirty() > 0 || // not strictly necessary + bc.get_stat_tx() > 0 || + bc.get_stat_partial() > 0 || + bc.get_stat_rx() > 0) { + + // write all dirty bufferheads + while (!bc.dirty_bh.empty()) { + set::iterator i = bc.dirty_bh.begin(); + BufferHead *bh = *i; + if (bh->ioh) continue; + Onode *on = get_onode(bh->oc->get_object_id()); + bh_write(on, bh); + put_onode(on); + } - - while (bc.get_stat_tx() > 0 || - bc.get_stat_partial() > 0) { - dout(1) << "flush_all waiting for " << bc.get_stat_tx() << " tx, " << bc.get_stat_partial() << " partial" << endl; + // wait for all tx and partial buffers to flush + dout(1) << "flush_all waiting for " + << bc.get_stat_dirty() << " dirty, " + << bc.get_stat_tx() << " tx, " + << bc.get_stat_rx() << " rx, " + << bc.get_stat_partial() << " partial" + << endl; bc.waitfor_stat(); } bc.lock.Unlock(); @@ -434,28 +429,36 @@ public: void Ebofs::bh_read(Onode *on, BufferHead *bh) { dout(5) << "bh_read " << *on << " on " << *bh << endl; - assert(bh->get_version() == 0); - assert(bh->is_rx() || bh->is_partial()); + + if (bh->is_missing()) { + bc.mark_rx(bh); + } else { + assert(bh->is_partial()); + } // get extents vector ex; on->map_extents(bh->start(), bh->length(), ex); + + // alloc new buffer + bc.bufferpool.alloc_list(bh->length(), bh->data); // new buffers! // lay out on disk - block_t ooff = 0; + block_t bhoff = 0; for (unsigned i=0; idata, ooff*EBOFS_BLOCK_SIZE, ex[i].length*EBOFS_BLOCK_SIZE); + sub.substr_of(bh->data, bhoff*EBOFS_BLOCK_SIZE, ex[i].length*EBOFS_BLOCK_SIZE); - if (bh->is_partial()) - bh->waitfor_read.push_back(new C_E_FlushPartial(this, on, bh)); + //if (bh->is_partial()) + //bh->waitfor_read.push_back(new C_E_FlushPartial(this, on, bh)); assert(bh->ioh == 0); bh->ioh = dev.read(ex[i].start, ex[i].length, sub, - new C_OC_RxFinish(on->oc, ooff, ex[i].length)); - - ooff += ex[i].length; + new C_OC_RxFinish(on->oc, + bhoff + bh->start(), ex[i].length)); + + bhoff += ex[i].length; } } @@ -548,7 +551,8 @@ void Ebofs::apply_write(Onode *on, size_t len, off_t off, bufferlist& bl) // partial at head or tail? if ((bh->start() == bstart && off % EBOFS_BLOCK_SIZE != 0) || - (bh->last() == blast && (len+off) % EBOFS_BLOCK_SIZE != 0)) { + (bh->last() == blast && (len+off) % EBOFS_BLOCK_SIZE != 0) || + (len % EBOFS_BLOCK_SIZE != 0)) { // locate ourselves in bh unsigned off_in_bh = opos - bh->start()*EBOFS_BLOCK_SIZE; assert(off_in_bh >= 0); @@ -598,6 +602,7 @@ void Ebofs::apply_write(Onode *on, size_t len, off_t off, bufferlist& bl) else if (bh->is_missing()) { dout(10) << "apply_write missing -> partial " << *bh << endl; bh_read(on, bh); + bc.mark_partial(bh); } else if (bh->is_partial()) { if (bh->ioh == 0) { @@ -635,11 +640,12 @@ void Ebofs::apply_write(Onode *on, size_t len, off_t off, bufferlist& bl) continue; } - // ok, we're talking full blocks now. - + // ok, we're talking full block(s) now. + assert(opos % EBOFS_BLOCK_SIZE == 0); + assert(zleft+left >= (off_t)(EBOFS_BLOCK_SIZE*bh->length())); // alloc new buffers. - bc.bufferpool.alloc_list(len, bh->data); + bc.bufferpool.alloc_list(bh->length(), bh->data); // copy! unsigned len_in_bh = bh->length()*EBOFS_BLOCK_SIZE; @@ -683,11 +689,150 @@ void Ebofs::apply_write(Onode *on, size_t len, off_t off, bufferlist& bl) // *** file i/o *** +class C_E_Cond : public Context { + Cond *cond; +public: + C_E_Cond(Cond *c) : cond(c) {} + void finish(int r) { + cond->Signal(); + } +}; + +bool Ebofs::attempt_read(Onode *on, size_t len, off_t off, bufferlist& bl, Cond *will_wait_on) +{ + dout(10) << "attempt_read " << *on << " len " << len << " off " << off << endl; + ObjectCache *oc = on->get_oc(&bc); + + // map + block_t bstart = off / EBOFS_BLOCK_SIZE; + block_t blast = (len+off-1) / EBOFS_BLOCK_SIZE; + block_t blen = blast-bstart+1; + + map hits; + map missing; // read these + map rx; // wait for these + map partials; // ?? + oc->map_read(bstart, blen, hits, missing, rx, partials); + + // missing buffers? + if (!missing.empty()) { + for (map::iterator i = missing.begin(); + i != missing.end(); + i++) { + dout(15) <<"attempt_read missing buffer " << *(i->second) << endl; + bh_read(on, i->second); + } + BufferHead *wait_on = missing.begin()->second; + wait_on->waitfor_read.push_back(new C_E_Cond(will_wait_on)); + return false; + } + + // are partials sufficient? + bool partials_ok = true; + for (map::iterator i = partials.begin(); + i != partials.end(); + i++) { + off_t start = MAX( off, (off_t)(i->second->start()*EBOFS_BLOCK_SIZE) ); + off_t end = MIN( off+len, (off_t)(i->second->end()*EBOFS_BLOCK_SIZE) ); + + if (!i->second->have_partial_range(start, end)) { + if (partials_ok) { + // wait on this one + dout(15) <<"attempt_read insufficient partial buffer " << *(i->second) << endl; + i->second->waitfor_read.push_back(new C_E_Cond(will_wait_on)); + } + partials_ok = false; + } + } + if (!partials_ok) return false; + + // wait on rx? + if (!rx.empty()) { + BufferHead *wait_on = rx.begin()->second; + dout(15) <<"attempt_read waiting for read to finish on " << *wait_on << endl; + wait_on->waitfor_read.push_back(new C_E_Cond(will_wait_on)); + return false; + } + + // yay, we have it all! + // concurrently walk thru hits, partials. + map::iterator h = hits.begin(); + map::iterator p = partials.begin(); + + off_t pos = off; + block_t curblock = bstart; + while (curblock <= blast) { + BufferHead *bh = 0; + if (h->first == curblock) { + bh = h->second; + h++; + } else if (p->first == curblock) { + bh = p->second; + p++; + } else assert(0); + + off_t bhstart = (off_t)(bh->start()*EBOFS_BLOCK_SIZE); + off_t bhend = (off_t)(bh->end()*EBOFS_BLOCK_SIZE); + off_t start = MAX( pos, bhstart ); + off_t end = MIN( off+len, bhend ); + + if (bh->is_partial()) { + // copy from a partial block. yuck! + bufferlist frag; + bh->copy_partial_substr( start, end, frag ); + bl.claim_append( frag ); + pos += frag.length(); + } else { + // copy from a full block. + if (bhstart == start && bhend == end) { + bl.append( bh->data ); + pos += bh->data.length(); + } else { + bufferlist frag; + frag.substr_of(bh->data, start-bhstart, end-start); + pos += frag.length(); + bl.claim_append( frag ); + } + } + + curblock = bh->end(); + assert((off_t)(curblock*EBOFS_BLOCK_SIZE) == pos || + end != bhend); + } + + assert(bl.length() == len); + return true; +} + int Ebofs::read(object_t oid, size_t len, off_t off, bufferlist& bl) { + Onode *on = get_onode(oid); + if (!on) + return -1; // object dne? + + // read data into bl. block as necessary. + Cond cond; + + bc.lock.Lock(); + while (1) { + // check size bound + if (off >= on->object_size) { + put_onode(on); + break; + } + size_t will_read = MIN( off+len, on->object_size ) - off; + + if (attempt_read(on, will_read, off, bl, &cond)) + break; // yay + + // wait + cond.Wait(bc.lock); + } + bc.lock.Unlock(); + return 0; } diff --git a/ceph/ebofs/Ebofs.h b/ceph/ebofs/Ebofs.h index 7231baaaf83f3..1d793b0cee6c0 100644 --- a/ceph/ebofs/Ebofs.h +++ b/ceph/ebofs/Ebofs.h @@ -76,7 +76,7 @@ class Ebofs { void zero(Onode *on, size_t len, off_t off, off_t write_thru); void apply_write(Onode *on, size_t len, off_t off, bufferlist& bl); - bool attempt_read(Onode *on, size_t len, off_t off, bufferlist& bl); + bool attempt_read(Onode *on, size_t len, off_t off, bufferlist& bl, Cond *will_wait_on); // io void bh_read(Onode *on, BufferHead *bh); diff --git a/ceph/ebofs/Table.h b/ceph/ebofs/Table.h index e661f49a95840..3785567701684 100644 --- a/ceph/ebofs/Table.h +++ b/ceph/ebofs/Table.h @@ -186,7 +186,7 @@ class Table : public _Table { open[l+1] = table->pool.get_node( open[l].index_item(pos[l]).node ); pos[l+1] = open[l+1].size() - 1; } - return 0; + return 1; } int move_right() { if (table->depth == 0) return OOB; @@ -213,7 +213,7 @@ class Table : public _Table { open[l+1] = table->pool.get_node( open[l].index_item(pos[l]).node ); pos[l+1] = 0; // furthest left } - return 0; + return 1; } // ** modifications ** diff --git a/ceph/ebofs/mkfs.ebofs.cc b/ceph/ebofs/mkfs.ebofs.cc index 46f880141955e..4bb5a0b39e29a 100644 --- a/ceph/ebofs/mkfs.ebofs.cc +++ b/ceph/ebofs/mkfs.ebofs.cc @@ -21,7 +21,7 @@ int main(int argc, char **argv) Ebofs fs(dev); fs.mkfs(); - if (1) { // test + if (0) { // test bufferlist bl; char crap[10000]; memset(crap, 0, 10000); @@ -33,6 +33,68 @@ int main(int argc, char **argv) fs.write(10, 5000, 3222, bl, 0); } + // test small writes + if (1) { + char crap[10000]; + memset(crap, 0, 10000); + bufferlist bl; + bl.append(crap, 10000); + + // write + srand(0); + for (int i=0; i<100; i++) { + off_t off = rand() % 1000000; + size_t len = 100; + cout << "writing bit at " << off << " len " << len << endl; + fs.write(10, len, off, bl, 0); + } + + if (0) { + // read + srand(0); + for (int i=0; i<100; i++) { + bufferlist bl; + off_t off = rand() % 1000000; + size_t len = 100; + cout << "read bit at " << off << " len " << len << endl; + int r = fs.read(10, len, off, bl); + assert(bl.length() == len); + assert(r == 0); + } + } + + // flush + fs.flush_all(); + fs.trim_buffer_cache(); + + if (0) { + // read again + srand(0); + for (int i=0; i<100; i++) { + bufferlist bl; + off_t off = rand() % 1000000; + size_t len = 100; + cout << "read bit at " << off << " len " << len << endl; + int r = fs.read(10, len, off, bl); + assert(bl.length() == len); + assert(r == 0); + } + + // flush + fs.trim_buffer_cache(); + } + + // write on empty cache + srand(0); + for (int i=0; i<100; i++) { + off_t off = rand() % 1000000; + size_t len = 100; + cout << "writing bit at " << off << " len " << len << endl; + fs.write(10, len, off, bl, 0); + } + + } + fs.flush_all(); fs.trim_buffer_cache(); fs.trim_onode_cache(); diff --git a/ceph/include/bufferlist.h b/ceph/include/bufferlist.h index 5b87a26728917..e5b4bd42c39c7 100644 --- a/ceph/include/bufferlist.h +++ b/ceph/include/bufferlist.h @@ -259,6 +259,10 @@ class bufferlist { bufferptr tempbp(bp, len, off); push_back(tempbp); } + void append(bufferlist& bl) { + bufferlist temp = bl; // copy list + claim_append(temp); // and append + } /* -- 2.39.5