From 96c4a26b46343f2850adae4f49e5b8be5104e0b5 Mon Sep 17 00:00:00 2001 From: Marcus Watts Date: Tue, 15 Aug 2017 21:57:06 -0400 Subject: [PATCH] radosgw: fix small sized chunked uploads with s3 The logic to compute the number of bytes to copy with a chunked file upload was failing to take into account the amount of data that had been consumed, but was still pending in the parse buffer. This would cause strange behavior ranging from "just works" to "fails". There was also a strange "works, one byte at a time" mode. Using the correct stream_pos offset eliminates the bad behavior. This fix also adds debug statements to make the correct behavior easier to see. Fixes: http://tracker.ceph.com/issues/21003 Signed-off-by: Marcus Watts --- src/rgw/rgw_auth_s3.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/rgw/rgw_auth_s3.cc b/src/rgw/rgw_auth_s3.cc index e183465b95c2f..ba137e3f59b50 100644 --- a/src/rgw/rgw_auth_s3.cc +++ b/src/rgw/rgw_auth_s3.cc @@ -993,8 +993,11 @@ size_t AWSv4ComplMulti::recv_body(char* const buf, const size_t buf_max) std::begin(parsing_buf) + consumed); } + size_t stream_pos_was = stream_pos - parsing_buf.size(); + size_t to_extract = \ - std::min(chunk_meta.get_data_size(stream_pos), buf_max); + std::min(chunk_meta.get_data_size(stream_pos_was), buf_max); + dout(30) << "AWSv4ComplMulti: stream_pos_was=" << stream_pos_was << ", to_extract=" << to_extract << dendl; /* It's quite probable we have a couple of real data bytes stored together * with meta-data in the parsing_buf. We need to extract them and move to @@ -1003,6 +1006,7 @@ size_t AWSv4ComplMulti::recv_body(char* const buf, const size_t buf_max) if (to_extract > 0 && parsing_buf.size() > 0) { const auto data_len = std::min(to_extract, parsing_buf.size()); const auto data_end_iter = std::begin(parsing_buf) + data_len; + dout(30) << "AWSv4ComplMulti: to_extract=" << to_extract << ", data_len=" << data_len << dendl; std::copy(std::begin(parsing_buf), data_end_iter, buf); parsing_buf.erase(std::begin(parsing_buf), data_end_iter); @@ -1017,6 +1021,7 @@ size_t AWSv4ComplMulti::recv_body(char* const buf, const size_t buf_max) * buffering. */ while (to_extract > 0) { const size_t received = io_base_t::recv_body(buf + buf_pos, to_extract); + dout(30) << "AWSv4ComplMulti: to_extract=" << to_extract << ", received=" << received << dendl; if (received == 0) { break; -- 2.39.5