#include "common/errno.h"
#include "librbd/AsioEngine.h"
#include "librbd/ImageCtx.h"
+#include "librbd/Utils.h"
#include "librbd/asio/Utils.h"
+#include "librbd/io/AioCompletion.h"
+#include "librbd/io/ReadResult.h"
#include "librbd/migration/Utils.h"
#include <boost/asio/buffer.hpp>
#include <boost/asio/post.hpp>
#include <boost/asio/read.hpp>
#include <boost/asio/ssl.hpp>
#include <boost/beast/core.hpp>
-#include <boost/beast/http/empty_body.hpp>
#include <boost/beast/http/read.hpp>
#include <boost/lexical_cast.hpp>
#include <deque>
std::deque<std::shared_ptr<Work>> m_receive_queue;
boost::beast::flat_buffer m_buffer;
- std::optional<boost::beast::http::parser<
- false, boost::beast::http::empty_body>> m_header_parser;
+ std::optional<boost::beast::http::parser<false, EmptyBody>> m_header_parser;
std::optional<boost::beast::http::parser<false, StringBody>> m_parser;
D& derived() {
return;
}
- boost::beast::http::response<StringBody> response;
+ Response response;
if (work->header_only()) {
m_parser.emplace(std::move(*m_header_parser));
}
ceph_assert(false);
}
- void complete_work(std::shared_ptr<Work> work, int r,
- boost::beast::http::response<StringBody>&& response) {
+ void complete_work(std::shared_ptr<Work> work, int r, Response&& response) {
auto cct = m_http_client->m_cct;
ldout(cct, 20) << "work=" << work.get() << ", r=" << r << dendl;
template <typename I>
HttpClient<I>::HttpClient(I* image_ctx, const std::string& url)
- : m_cct(image_ctx->cct), m_asio_engine(image_ctx->asio_engine), m_url(url),
- m_strand(*m_asio_engine),
+ : m_cct(image_ctx->cct), m_image_ctx(image_ctx),
+ m_asio_engine(image_ctx->asio_engine), m_url(url), m_strand(*m_asio_engine),
m_ssl_context(boost::asio::ssl::context::sslv23_client) {
m_ssl_context.set_default_verify_paths();
}
void HttpClient<I>::get_size(uint64_t* size, Context* on_finish) {
ldout(m_cct, 10) << dendl;
- boost::beast::http::request<boost::beast::http::empty_body> req;
+ boost::beast::http::request<EmptyBody> req;
req.method(boost::beast::http::verb::head);
issue(
- std::move(req), [this, size, on_finish]
- (int r, boost::beast::http::response<StringBody>&& response) {
+ std::move(req), [this, size, on_finish](int r, Response&& response) {
handle_get_size(r, std::move(response), size, on_finish);
});
}
template <typename I>
-void HttpClient<I>::handle_get_size(
- int r, boost::beast::http::response<StringBody>&& response,
- uint64_t* size, Context* on_finish) {
+void HttpClient<I>::handle_get_size(int r, Response&& response, uint64_t* size,
+ Context* on_finish) {
ldout(m_cct, 10) << "r=" << r << dendl;
if (r < 0) {
on_finish->complete(0);
}
+template <typename I>
+void HttpClient<I>::read(io::Extents&& byte_extents, bufferlist* data,
+ Context* on_finish) {
+ ldout(m_cct, 20) << dendl;
+
+ auto aio_comp = io::AioCompletion::create_and_start(
+ on_finish, librbd::util::get_image_ctx(m_image_ctx), io::AIO_TYPE_READ);
+ aio_comp->set_request_count(byte_extents.size());
+
+ // utilize ReadResult to assemble multiple byte extents into a single bl
+ // since boost::beast doesn't support multipart responses out-of-the-box
+ io::ReadResult read_result{data};
+ aio_comp->read_result = std::move(read_result);
+ aio_comp->read_result.set_image_extents(byte_extents);
+
+ // issue a range get request for each extent
+ uint64_t buffer_offset = 0;
+ for (auto [byte_offset, byte_length] : byte_extents) {
+ auto ctx = new io::ReadResult::C_ImageReadRequest(
+ aio_comp, buffer_offset, {{byte_offset, byte_length}});
+ buffer_offset += byte_length;
+
+ Request req;
+ req.method(boost::beast::http::verb::get);
+
+ std::stringstream range;
+ ceph_assert(byte_length > 0);
+ range << "bytes=" << byte_offset << "-" << (byte_offset + byte_length - 1);
+ req.set(boost::beast::http::field::range, range.str());
+
+ issue(
+ std::move(req),
+ [this, byte_offset=byte_offset, byte_length=byte_length, ctx]
+ (int r, Response&& response) {
+ handle_read(r, std::move(response), byte_offset, byte_length, &ctx->bl,
+ ctx);
+ });
+ }
+}
+
+template <typename I>
+void HttpClient<I>::handle_read(int r, Response&& response,
+ uint64_t byte_offset, uint64_t byte_length,
+ bufferlist* data, Context* on_finish) {
+ ldout(m_cct, 20) << "bytes=" << byte_offset << "~" << byte_length << ", "
+ << "r=" << r << dendl;
+
+ if (r < 0) {
+ lderr(m_cct) << "failed to read requested byte range: "
+ << cpp_strerror(r) << dendl;
+ on_finish->complete(r);
+ return;
+ } else if (response.result() != boost::beast::http::status::partial_content) {
+ lderr(m_cct) << "failed to retrieve requested byte range: HTTP "
+ << response.result() << dendl;
+ on_finish->complete(-EIO);
+ return;
+ } else if (byte_length != response.body().size()) {
+ lderr(m_cct) << "unexpected short range read: "
+ << "wanted=" << byte_length << ", "
+ << "received=" << response.body().size() << dendl;
+ on_finish->complete(-EINVAL);
+ return;
+ }
+
+ data->clear();
+ data->append(response.body());
+ on_finish->complete(data->length());
+}
+
template <typename I>
void HttpClient<I>::issue(std::shared_ptr<Work>&& work) {
boost::asio::post(m_strand, [this, work=std::move(work)]() mutable {
#include "include/common_fwd.h"
#include "include/int_types.h"
+#include "librbd/io/Types.h"
#include "librbd/migration/Types.h"
#include <boost/asio/io_context_strand.hpp>
#include <boost/asio/ip/tcp.hpp>
#include <boost/asio/ssl/context.hpp>
#include <boost/beast/version.hpp>
#include <boost/beast/core/tcp_stream.hpp>
+#include <boost/beast/http/empty_body.hpp>
#include <boost/beast/http/message.hpp>
#include <boost/beast/http/string_body.hpp>
#include <boost/beast/http/write.hpp>
template <typename ImageCtxT>
class HttpClient {
public:
+ using EmptyBody = boost::beast::http::empty_body;
+ using StringBody = boost::beast::http::string_body;
+ using Request = boost::beast::http::request<EmptyBody>;
+ using Response = boost::beast::http::response<StringBody>;
+
static HttpClient* create(ImageCtxT* image_ctx, const std::string& url) {
return new HttpClient(image_ctx, url);
}
void get_size(uint64_t* size, Context* on_finish);
+ void read(io::Extents&& byte_extents, bufferlist* data,
+ Context* on_finish);
+
void set_ignore_self_signed_cert(bool ignore) {
m_ignore_self_signed_cert = ignore;
}
- using StringBody = boost::beast::http::string_body;
-
template <class Body, typename Completion>
void issue(boost::beast::http::request<Body>&& request,
Completion&& completion) {
return (request.method() == boost::beast::http::verb::head);
}
- void complete(
- int r, boost::beast::http::response<StringBody>&& response) override {
+ void complete(int r, Response&& response) override {
completion(r, std::move(response));
}
virtual ~Work() {}
virtual bool need_eof() const = 0;
virtual bool header_only() const = 0;
- virtual void complete(
- int r, boost::beast::http::response<StringBody>&&) = 0;
+ virtual void complete(int r, Response&&) = 0;
virtual void operator()(
HttpSessionInterface* http_session,
boost::beast::tcp_stream& stream) = 0;
struct SslHttpSession;
CephContext* m_cct;
+ ImageCtxT* m_image_ctx;
std::shared_ptr<AsioEngine> m_asio_engine;
std::string m_url;
BOOST_BEAST_VERSION_STRING);
}
- void handle_get_size(
- int r, boost::beast::http::response<StringBody>&& response,
- uint64_t* size, Context* on_finish);
+ void handle_get_size(int r, Response&& response, uint64_t* size,
+ Context* on_finish);
+
+ void handle_read(int r, Response&& response, uint64_t byte_offset,
+ uint64_t byte_length, bufferlist* data, Context* on_finish);
void issue(std::shared_ptr<Work>&& work);
#include "common/errno.h"
#include "librbd/AsioEngine.h"
#include "librbd/ImageCtx.h"
-#include "librbd/Utils.h"
#include "librbd/asio/Utils.h"
-#include "librbd/io/AioCompletion.h"
-#include "librbd/io/ReadResult.h"
#include "librbd/migration/HttpClient.h"
#include <boost/beast/http.hpp>
template <typename I>
void HttpStream<I>::read(io::Extents&& byte_extents, bufferlist* data,
Context* on_finish) {
- using HttpRequest = boost::beast::http::request<
- boost::beast::http::empty_body>;
-
- ldout(m_cct, 20) << dendl;
-
- auto aio_comp = io::AioCompletion::create_and_start(
- on_finish, util::get_image_ctx(m_image_ctx), io::AIO_TYPE_READ);
- aio_comp->set_request_count(byte_extents.size());
-
- // utilize ReadResult to assemble multiple byte extents into a single bl
- // since boost::beast doesn't support multipart responses out-of-the-box
- io::ReadResult read_result{data};
- aio_comp->read_result = std::move(read_result);
- aio_comp->read_result.set_image_extents(byte_extents);
-
- // issue a range get request for each extent
- uint64_t buffer_offset = 0;
- for (auto [byte_offset, byte_length] : byte_extents) {
- auto ctx = new io::ReadResult::C_ImageReadRequest(
- aio_comp, buffer_offset, {{byte_offset, byte_length}});
- buffer_offset += byte_length;
-
- HttpRequest req;
- req.method(boost::beast::http::verb::get);
-
- std::stringstream range;
- ceph_assert(byte_length > 0);
- range << "bytes=" << byte_offset << "-" << (byte_offset + byte_length - 1);
- req.set(boost::beast::http::field::range, range.str());
-
- m_http_client->issue(std::move(req),
- [this, byte_offset=byte_offset, byte_length=byte_length, ctx](int r, HttpResponse&& response) {
- handle_read(r, std::move(response), byte_offset, byte_length, &ctx->bl,
- ctx);
- });
- }
-}
-
-template <typename I>
-void HttpStream<I>::handle_read(int r, HttpResponse&& response,
- uint64_t byte_offset, uint64_t byte_length,
- bufferlist* data, Context* on_finish) {
- ldout(m_cct, 20) << "bytes=" << byte_offset << "~" << byte_length << ", "
- << "r=" << r << dendl;
-
- if (r < 0) {
- lderr(m_cct) << "failed to read requested byte range: "
- << cpp_strerror(r) << dendl;
- on_finish->complete(r);
- return;
- } else if (response.result() != boost::beast::http::status::partial_content) {
- lderr(m_cct) << "failed to retrieve requested byte range: HTTP "
- << response.result() << dendl;
- on_finish->complete(-EIO);
- return;
- } else if (byte_length != response.body().size()) {
- lderr(m_cct) << "unexpected short range read: "
- << "wanted=" << byte_length << ", "
- << "received=" << response.body().size() << dendl;
- on_finish->complete(-EINVAL);
- return;
- }
+ ldout(m_cct, 20) << "byte_extents=" << byte_extents << dendl;
- data->clear();
- data->append(response.body());
- on_finish->complete(data->length());
+ m_http_client->read(std::move(byte_extents), data, on_finish);
}
} // namespace migration
std::unique_ptr<HttpClient<ImageCtxT>> m_http_client;
- void handle_read(int r, HttpResponse&& response, uint64_t byte_offset,
- uint64_t byte_length, bufferlist* data, Context* on_finish);
};
} // namespace migration
};
} // anonymous namespace
+
+namespace util {
+
+inline ImageCtx *get_image_ctx(MockTestImageCtx *image_ctx) {
+ return image_ctx->image_ctx;
+}
+
+} // namespace util
} // namespace librbd
#include "librbd/migration/HttpClient.cc"
ASSERT_EQ(0, ctx3.wait());
}
+TEST_F(TestMockMigrationHttpClient, Read) {
+ MockTestImageCtx mock_test_image_ctx(*m_image_ctx);
+ MockHttpClient http_client(&mock_test_image_ctx,
+ get_local_url(URL_SCHEME_HTTP));
+
+ boost::asio::ip::tcp::socket socket(*m_image_ctx->asio_engine);
+ C_SaferCond on_connect_ctx;
+ client_accept(&socket, false, &on_connect_ctx);
+
+ C_SaferCond ctx1;
+ http_client.open(&ctx1);
+ ASSERT_EQ(0, on_connect_ctx.wait());
+ ASSERT_EQ(0, ctx1.wait());
+
+ bufferlist bl;
+ C_SaferCond ctx2;
+ http_client.read({{0, 128}, {256, 64}}, &bl, &ctx2);
+
+ HttpRequest expected_req1;
+ expected_req1.method(boost::beast::http::verb::get);
+ expected_req1.set(boost::beast::http::field::range, "bytes=0-127");
+ client_read_request(socket, expected_req1);
+
+ HttpRequest expected_req2;
+ expected_req2.method(boost::beast::http::verb::get);
+ expected_req2.set(boost::beast::http::field::range, "bytes=256-319");
+ client_read_request(socket, expected_req2);
+
+ HttpResponse expected_res1;
+ expected_res1.result(boost::beast::http::status::partial_content);
+ expected_res1.body() = std::string(128, '1');
+ client_write_response(socket, expected_res1);
+
+ HttpResponse expected_res2;
+ expected_res2.result(boost::beast::http::status::partial_content);
+ expected_res2.body() = std::string(64, '2');
+ client_write_response(socket, expected_res2);
+
+ ASSERT_EQ(192, ctx2.wait());
+
+ bufferlist expect_bl;
+ expect_bl.append(std::string(128, '1'));
+ expect_bl.append(std::string(64, '2'));
+ ASSERT_EQ(expect_bl, bl);
+
+ C_SaferCond ctx3;
+ http_client.close(&ctx3);
+ ASSERT_EQ(0, ctx3.wait());
+}
+
} // namespace migration
} // namespace librbd
MOCK_METHOD1(open, void(Context*));
MOCK_METHOD1(close, void(Context*));
MOCK_METHOD2(get_size, void(uint64_t*, Context*));
-
- MOCK_METHOD3(issue, void(const boost::beast::http::request<
- boost::beast::http::empty_body>&,
- boost::beast::http::response<
- boost::beast::http::string_body>*,
- Context*));
-
- template <class Body, typename Completion>
- void issue(boost::beast::http::request<Body>&& request,
- Completion&& completion) {
- struct ContextImpl : public Context {
- boost::beast::http::response<boost::beast::http::string_body> res;
- Completion completion;
-
- ContextImpl(Completion&& completion) : completion(std::move(completion)) {
- }
-
- void finish(int r) override {
- completion(r, std::move(res));
- }
- };
-
- auto ctx = new ContextImpl(std::move(completion));
- issue(request, &ctx->res, ctx);
+ MOCK_METHOD3(do_read, void(const io::Extents&, bufferlist*, Context*));
+ void read(io::Extents&& extents, bufferlist* bl, Context* ctx) {
+ do_read(extents, bl, ctx);
}
HttpClient() {
HttpClient<MockTestImageCtx>* HttpClient<MockTestImageCtx>::s_instance = nullptr;
} // namespace migration
-
-namespace util {
-
-inline ImageCtx *get_image_ctx(MockTestImageCtx *image_ctx) {
- return image_ctx->image_ctx;
-}
-
-} // namespace util
} // namespace librbd
#include "librbd/migration/HttpStream.cc"
using ::testing::_;
using ::testing::Invoke;
using ::testing::InSequence;
+using ::testing::WithArgs;
class TestMockMigrationHttpStream : public TestMockFixture {
public:
}));
}
- void expect_read(MockHttpClient& mock_http_client, io::Extent byte_extent,
- const std::string& data, int r) {
- std::stringstream byte_range;
- byte_range << byte_extent.first << "-"
- << (byte_extent.first + byte_extent.second - 1);
-
- EXPECT_CALL(mock_http_client, issue(_, _, _))
- .WillOnce(Invoke(
- [data, r, byte_range=byte_range.str()]
- (const boost::beast::http::request<
- boost::beast::http::empty_body>& request,
- boost::beast::http::response<
- boost::beast::http::string_body>* response, Context* ctx) {
- ASSERT_EQ("bytes=" + byte_range,
- request[boost::beast::http::field::range]);
-
- response->result(boost::beast::http::status::partial_content);
- response->set(boost::beast::http::field::content_range,
- "bytes " + byte_range + "/*");
- response->body() = data;
- response->prepare_payload();
- ctx->complete(r);
- }));
+ void expect_read(MockHttpClient& mock_http_client, io::Extents byte_extents,
+ const bufferlist& bl, int r) {
+ uint64_t len = 0;
+ for (auto [_, byte_len] : byte_extents) {
+ len += byte_len;
+ }
+ EXPECT_CALL(mock_http_client, do_read(byte_extents, _, _))
+ .WillOnce(WithArgs<1, 2>(Invoke(
+ [len, bl, r](bufferlist* out_bl, Context* ctx) {
+ *out_bl = bl;
+ ctx->complete(r < 0 ? r : len);
+ })));
}
json_spirit::mObject json_object;
auto mock_http_client = new MockHttpClient();
expect_open(*mock_http_client, 0);
- expect_read(*mock_http_client, {0, 128}, std::string(128, '1'), 0);
- expect_read(*mock_http_client, {256, 64}, std::string(64, '2'), 0);
+ bufferlist expect_bl;
+ expect_bl.append(std::string(192, '1'));
+ expect_read(*mock_http_client, {{0, 128}, {256, 64}}, expect_bl, 0);
expect_close(*mock_http_client, 0);
bufferlist bl;
mock_http_stream.read({{0, 128}, {256, 64}}, &bl, &ctx2);
ASSERT_EQ(192, ctx2.wait());
-
- bufferlist expect_bl;
- expect_bl.append(std::string(128, '1'));
- expect_bl.append(std::string(64, '2'));
ASSERT_EQ(expect_bl, bl);
C_SaferCond ctx3;
projects / ceph.git / commitdiff