From 3b864482d6aef2efe0b03be70ea83c38f7a6d99b Mon Sep 17 00:00:00 2001 From: "Robin H. Johnson" Date: Fri, 14 Sep 2018 14:23:49 -0700 Subject: [PATCH] rgw: fix chunked-encoding for chunks >1MiB For HTTP responses sent with chunked-encoding, and greater than 1MiB in size, the chunk-size field was being printed wrong. Specifically, the chunk-size field was being sent with a mangled or missing trailer of '\r\n'. This bug manifested as HTTP clients being unable to read the response: Chrome generates ERR_INCOMPLETE_CHUNKED_ENCODING Python/boto generates httplib.LineTooLong: got more than 65536 bytes when reading chunk size The wrong variable was being used to determine the size of the buffer used for the chunk-size field. Fix it by using the correct variable, and rename the variables to clearly reflect their purpose. Prior to PR#23940, this would only have been seen in some Swift operations. PR#23940 changed some S3 operations to also use chunked encoding to get responses sent faster, and made the bug easier to detect. It was initially reported for a ListBucket call with a high max-keys argument. Backport: luminous, mimic Reference: https://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.6.1 Reference: https://github.com/ceph/ceph/pull/23940 Fixes: http://tracker.ceph.com/issues/35990 Signed-off-by: Robin H. Johnson --- src/rgw/rgw_client_io_filters.h | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/rgw/rgw_client_io_filters.h b/src/rgw/rgw_client_io_filters.h index 82163cf8d746f..9ce83a936634c 100644 --- a/src/rgw/rgw_client_io_filters.h +++ b/src/rgw/rgw_client_io_filters.h @@ -286,11 +286,15 @@ public: return DecoratedRestfulClient::send_body(buf, len); } else { static constexpr char HEADER_END[] = "\r\n"; - char sizebuf[32]; - const auto slen = snprintf(sizebuf, sizeof(buf), "%" PRIx64 "\r\n", len); + /* https://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.6.1 */ + // TODO: we have no support for sending chunked-encoding + // extensions/trailing headers. + char chunk_size[32]; + const auto chunk_size_len = snprintf(chunk_size, sizeof(chunk_size), + "%" PRIx64 "\r\n", len); size_t sent = 0; - sent += DecoratedRestfulClient::send_body(sizebuf, slen); + sent += DecoratedRestfulClient::send_body(chunk_size, chunk_size_len); sent += DecoratedRestfulClient::send_body(buf, len); sent += DecoratedRestfulClient::send_body(HEADER_END, sizeof(HEADER_END) - 1); -- 2.39.5