From c6f24c309cfb53344f6ecc48089ce2044156e19b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 30 Aug 2024 09:57:29 +0200 Subject: [PATCH 01/10] New changelog section to prepare for vNext (#8014) Co-authored-by: jedelbo <572755+jedelbo@users.noreply.github.com> --- CHANGELOG.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 69048190e9..d91f2c8a32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,26 @@ +# NEXT RELEASE + +### Enhancements +* (PR [#????](https://github.com/realm/realm-core/pull/????)) +* None. + +### Fixed +* ([#????](https://github.com/realm/realm-core/issues/????), since v?.?.?) +* None. + +### Breaking changes +* None. + +### Compatibility +* Fileformat: Generates files with format v24. Reads and automatically upgrade from fileformat v10. If you want to upgrade from an earlier file format version you will have to use RealmCore v13.x.y or earlier. + +----------- + +### Internals +* None. + +---------------------------------------------- + # 14.12.1 Release notes ### Enhancements From 07d57dee363b85fe304b12f7c44da2b194cabce1 Mon Sep 17 00:00:00 2001 From: Michael Wilkerson-Barker Date: Fri, 30 Aug 2024 10:48:39 -0400 Subject: [PATCH 02/10] RCORE-2222 Remove 308 redirect tests (#7994) * Removed redirect tests * Removed one more location redirect test case --- test/object-store/sync/app.cpp | 565 +++++++++++---------------------- 1 file changed, 182 insertions(+), 383 deletions(-) diff --git a/test/object-store/sync/app.cpp b/test/object-store/sync/app.cpp index 06d6fa56a0..a50edecf09 100644 --- a/test/object-store/sync/app.cpp +++ b/test/object-store/sync/app.cpp @@ -665,39 +665,6 @@ TEST_CASE("app: verify app utils helpers", "[sync][app][local]") { CHECK(!AppUtils::is_success_status_code(300)); CHECK(!AppUtils::is_success_status_code(99999)); } - - SECTION("is_redirect_status_code") { - // Only MovedPermanently(301) and PermanentRedirect(308) return true - CHECK(AppUtils::is_redirect_status_code(301)); - CHECK(AppUtils::is_redirect_status_code(308)); - CHECK(!AppUtils::is_redirect_status_code(0)); - CHECK(!AppUtils::is_redirect_status_code(200)); - CHECK(!AppUtils::is_redirect_status_code(300)); - CHECK(!AppUtils::is_redirect_status_code(403)); - CHECK(!AppUtils::is_redirect_status_code(99999)); - } - - SECTION("extract_redir_location") { - auto comp = AppUtils::extract_redir_location( - {{"Content-Type", "application/json"}, {"Location", "http://redirect.host"}}); - CHECK(comp == "http://redirect.host"); - comp = AppUtils::extract_redir_location({{"location", "http://redirect.host"}}); - CHECK(comp == "http://redirect.host"); - comp = AppUtils::extract_redir_location({{"LoCaTiOn", "http://redirect.host/"}}); - CHECK(comp == "http://redirect.host/"); - comp = AppUtils::extract_redir_location({{"LOCATION", "http://redirect.host/includes/path"}}); - CHECK(comp == "http://redirect.host/includes/path"); - comp = AppUtils::extract_redir_location({{"Content-Type", "application/json"}}); - CHECK(!comp); - comp = AppUtils::extract_redir_location({{"some-location", "http://redirect.host"}}); - CHECK(!comp); - comp = AppUtils::extract_redir_location({{"location", ""}}); - CHECK(!comp); - comp = AppUtils::extract_redir_location({}); - CHECK(!comp); - comp = AppUtils::extract_redir_location({{"location", "bad-server-url"}}); - CHECK(!comp); - } } // MARK: - Login with Credentials Tests @@ -2751,7 +2718,6 @@ TEST_CASE("app: sync integration", "[sync][pbs][app][baas]") { } else { INFO(request.url); - // any later requests (eg. redirect) won't have a current user REQUIRE(!user); } // simulate the server denying the refresh @@ -3386,104 +3352,6 @@ TEST_CASE("app: redirect handling", "[sync][pbs][app]") { auto app = oas.app(); const auto partition = random_string(100); - SECTION("invalid redirect response reports and error") { - int request_count = 0; - - // This will fail due to no Location header - transport->request_hook = [&](const Request& request) -> std::optional { - logger->trace("request.url (%1): %2", request_count, request.url); - REQUIRE(request_count++ == 0); - return Response{301, 0, {{"Content-Type", "application/json"}}, "Some body data"}; - }; - app->provider_client().register_email( - creds.email, creds.password, [&](util::Optional error) { - REQUIRE(error); - REQUIRE(error->is_client_error()); - REQUIRE(error->code() == ErrorCodes::ClientRedirectError); - REQUIRE(error->reason() == "Redirect response missing location header"); - }); - - // This will fail due to empty Location header - transport->request_hook = [&](const Request& request) -> std::optional { - logger->trace("request.url (%1): %2", request_count, request.url); - REQUIRE(request_count++ == 1); - return Response{301, 0, {{"Location", ""}, {"Content-Type", "application/json"}}, "Some body data"}; - }; - - app->provider_client().register_email( - creds.email, creds.password, [&](util::Optional error) { - REQUIRE(error); - REQUIRE(error->is_client_error()); - REQUIRE(error->code() == ErrorCodes::ClientRedirectError); - REQUIRE(error->reason() == "Redirect response missing location header"); - }); - } - - SECTION("valid redirect response") { - int request_count = 0; - const std::string second_host = "http://second.invalid:9091"; - const std::string third_host = "http://third.invalid:9092"; - - transport->request_hook = [&](const Request& request) -> std::optional { - logger->trace("Received request[%1]: %2", request_count, request.url); - switch (request_count++) { - case 0: - REQUIRE_THAT(request.url, ContainsSubstring("/location")); - REQUIRE_THAT(request.url, ContainsSubstring(*oas_config.base_url)); - return Response{301, 0, {{"Location", second_host}, {"Content-Type", "application/json"}}, ""}; - - case 1: - REQUIRE_THAT(request.url, ContainsSubstring("/location")); - REQUIRE_THAT(request.url, ContainsSubstring(second_host)); - return Response{301, 0, {{"Location", third_host}, {"Content-Type", "application/json"}}, ""}; - - case 2: - REQUIRE_THAT(request.url, ContainsSubstring("/location")); - REQUIRE_THAT(request.url, ContainsSubstring(third_host)); - return Response{301, 0, {{"Location", second_host}, {"Content-Type", "application/json"}}, ""}; - - case 3: - REQUIRE_THAT(request.url, ContainsSubstring("/location")); - REQUIRE_THAT(request.url, ContainsSubstring(second_host)); - return std::nullopt; - - default: - // some.fake.url is the location reported by UnitTestTransport - REQUIRE_THAT(request.url, ContainsSubstring("https://some.fake.url")); - return std::nullopt; - } - }; - - // This will be successful after a couple of retries due to the redirect response - app->provider_client().register_email( - creds.email, creds.password, [&](util::Optional error) { - REQUIRE(!error); - }); - } - - SECTION("too many redirects eventually reports an error") { - int request_count = 0; - transport->request_hook = [&](const Request& request) -> std::optional { - logger->trace("request.url (%1): %2", request_count, request.url); - REQUIRE(request_count < 21); - ++request_count; - return Response{request_count % 2 == 1 ? 308 : 301, - 0, - {{"Location", "http://somehost:9090"}, {"Content-Type", "application/json"}}, - "Some body data"}; - }; - - app->log_in_with_credentials(app::AppCredentials::username_password(creds.email, creds.password), - [&](std::shared_ptr user, util::Optional error) { - REQUIRE(!user); - REQUIRE(error); - REQUIRE(error->is_client_error()); - REQUIRE(error->code() == ErrorCodes::ClientTooManyRedirects); - REQUIRE(error->reason() == "number of redirections exceeded 20"); - }); - REQUIRE(request_count == 21); - } - SECTION("server in maintenance reports error") { transport->request_hook = [&](const Request&) -> std::optional { nlohmann::json maintenance_error = {{"error_code", "MaintenanceInProgress"}, @@ -3588,22 +3456,8 @@ TEST_CASE("app: redirect handling", "[sync][pbs][app]") { int request_count = 0; transport->request_hook = [&](const Request& request) -> std::optional { logger->trace("request.url (%1): %2", request_count, request.url); - ++request_count; - - // First request should be a location request against the original URL - if (request_count == 1) { + if (request.url.find("/location") != std::string::npos) { REQUIRE_THAT(request.url, ContainsSubstring("some.fake.url")); - REQUIRE_THAT(request.url, ContainsSubstring("/location")); - return Response{static_cast(sync::HTTPStatus::PermanentRedirect), - 0, - {{"Location", "http://asdf.invalid"}}, - ""}; - } - - // Second request should be a location request against the new URL - if (request_count == 2) { - REQUIRE_THAT(request.url, ContainsSubstring("/location")); - REQUIRE_THAT(request.url, ContainsSubstring("asdf.invalid")); return Response{200, 0, {}, @@ -3621,7 +3475,6 @@ TEST_CASE("app: redirect handling", "[sync][pbs][app]") { sync_session->resume(); REQUIRE(!wait_for_download(*r)); - REQUIRE(request_count > 1); REQUIRE(realm_config.sync_config->user->is_logged_in()); // Verify session is using the updated server url from the redirect @@ -3674,57 +3527,25 @@ TEST_CASE("app: redirect handling", "[sync][pbs][app]") { REQUIRE(result); REQUIRE_FALSE(realm_config.sync_config->user->is_logged_in()); } - - SECTION("too many websocket redirects logs out user") { - int request_count = 0; - const int max_http_redirects = 20; // from app.cpp in object-store - transport->request_hook = [&](const Request& request) -> std::optional { - logger->trace("request.url (%1): %2", request_count, request.url); - - // The test should never request anything other than /location - // even though the user is set to the logged-out state as trying - // to log out on the server needs to go through /location first too - REQUIRE_THAT(request.url, ContainsSubstring("/location")); - REQUIRE(request_count <= max_http_redirects); - - // First request should be a location request against the original URL - // and rest should use the redirect url - if (request_count++ == 0) { - REQUIRE_THAT(request.url, ContainsSubstring("some.fake.url")); - } - else { - REQUIRE_THAT(request.url, ContainsSubstring("asdf.invalid")); - } - // Keep returning the redirected response - return Response{static_cast(sync::HTTPStatus::MovedPermanently), - 0, - {{"Location", "http://asdf.invalid"}}, - ""}; - }; - - sync_session->resume(); - REQUIRE(wait_for_download(*r)); - std::unique_lock lk(logout_mutex); - auto result = logout_cv.wait_for(lk, std::chrono::seconds(15), [&]() { - return logged_out; - }); - REQUIRE(result); - REQUIRE_FALSE(realm_config.sync_config->user->is_logged_in()); - } } } TEST_CASE("app: base_url", "[sync][app][base_url]") { struct BaseUrlTransport : UnitTestTransport { std::string expected_url; - std::optional redirect_url; + std::string location_url; + std::string location_wsurl; bool location_requested = false; bool location_returns_error = false; - void reset(std::string_view expect_url, std::optional redir_url = std::nullopt) + void reset(std::string expect_url, std::optional url = std::nullopt, + std::optional wsurl = std::nullopt) { - expected_url = std::string(expect_url); - redirect_url = redir_url; + expected_url = expect_url; + REALM_ASSERT(!expected_url.empty()); + location_url = url.value_or(expect_url); + REALM_ASSERT(!location_url.empty()); + location_wsurl = wsurl.value_or(App::create_ws_host_url(location_url)); location_requested = false; location_returns_error = false; } @@ -3740,37 +3561,37 @@ TEST_CASE("app: base_url", "[sync][app][base_url]") { completion(app::Response{static_cast(sync::HTTPStatus::NotFound), 0, {}, "404 not found"}); return; } - if (redirect_url) { - // Update the expected url to be the redirect url - expected_url = std::string(*redirect_url); - redirect_url.reset(); - - completion(app::Response{static_cast(sync::HTTPStatus::PermanentRedirect), - 0, - {{"location", expected_url}}, - "308 permanent redirect"}); - return; - } - auto ws_url = App::create_ws_host_url(expected_url); completion( app::Response{static_cast(sync::HTTPStatus::Ok), 0, {}, util::format("{\"deployment_model\":\"GLOBAL\",\"location\":\"US-VA\",\"hostname\":" "\"%1\",\"ws_hostname\":\"%2\"}", - expected_url, ws_url)}); + location_url, location_wsurl)}); return; } - + if (location_requested) { + CHECK_THAT(request.url, ContainsSubstring(location_url)); + } + else { + CHECK_THAT(request.url, ContainsSubstring(expected_url)); + } UnitTestTransport::send_request_to_server(request, std::move(completion)); } }; auto logger = util::Logger::get_default_logger(); - - auto redir_transport = std::make_shared(); + std::string default_base_url = std::string(App::default_base_url()); + std::string default_base_wsurl = App::create_ws_host_url(App::default_base_url()); + std::string test_base_url = "https://base.someurl.fake"; + std::string test_base_wsurl = "wss://base.someurl.fake"; + std::string test_location_url = "https://loc.someurl.fake"; + std::string test_location_wsurl = "wss://loc.someurl.fake"; + std::string test_location_wsurl2 = "wss://ws.loc.someurl.fake"; + + auto location_transport = std::make_shared(); auto get_config_with_base_url = [&](std::optional base_url = std::nullopt) { - OfflineAppSession::Config config(redir_transport); + OfflineAppSession::Config config(location_transport); config.base_url = base_url; return config; }; @@ -3819,189 +3640,157 @@ TEST_CASE("app: base_url", "[sync][app][base_url]") { SECTION("Test app config baseurl") { { // First time through, base_url is empty; https://services.cloud.mongodb.com is expected - redir_transport->reset(App::default_base_url()); + location_transport->reset(std::string(App::default_base_url())); auto config = get_config_with_base_url(); OfflineAppSession oas(config); auto app = oas.app(); // Location is not requested until first app services request - CHECK(!redir_transport->location_requested); + CHECK(!location_transport->location_requested); // Initial hostname and ws hostname use base url, but aren't used until location is updated CHECK(app->get_host_url() == App::default_base_url()); CHECK(app->get_ws_host_url() == App::create_ws_host_url(App::default_base_url())); oas.make_user(); - CHECK(redir_transport->location_requested); + CHECK(location_transport->location_requested); CHECK(app->get_base_url() == App::default_base_url()); CHECK(app->get_host_url() == App::default_base_url()); CHECK(app->get_ws_host_url() == App::create_ws_host_url(App::default_base_url())); } { - // Second time through, base_url is set to https://alternate.someurl.fake is expected - redir_transport->reset("https://alternate.someurl.fake"); - auto config = get_config_with_base_url("https://alternate.someurl.fake"); + // Base_url is set to test_base_url and test_location_url is expected after + // location request + location_transport->reset(test_base_url, test_location_url); + auto config = get_config_with_base_url(test_base_url); OfflineAppSession oas(config); auto app = oas.app(); // Location is not requested until first app services request - CHECK(!redir_transport->location_requested); + CHECK(!location_transport->location_requested); // Initial hostname and ws hostname use base url, but aren't used until location is updated - CHECK(app->get_host_url() == "https://alternate.someurl.fake"); - CHECK(app->get_ws_host_url() == "wss://alternate.someurl.fake"); + CHECK(app->get_host_url() == test_base_url); + CHECK(app->get_ws_host_url() == test_base_wsurl); oas.make_user(); - CHECK(redir_transport->location_requested); - CHECK(app->get_base_url() == "https://alternate.someurl.fake"); - CHECK(app->get_host_url() == "https://alternate.someurl.fake"); - CHECK(app->get_ws_host_url() == "wss://alternate.someurl.fake"); + CHECK(location_transport->location_requested); + CHECK(app->get_base_url() == test_base_url); + CHECK(app->get_host_url() == test_location_url); + CHECK(app->get_ws_host_url() == test_location_wsurl); } { // Third time through, base_url is not set, expect https://services.cloud.mongodb.com, // since metadata is no longer used - std::string expected_url = std::string(App::default_base_url()); - std::string expected_wsurl = App::create_ws_host_url(App::default_base_url()); - redir_transport->reset(expected_url); + location_transport->reset(default_base_url); auto config = get_config_with_base_url(); OfflineAppSession oas(config); auto app = oas.app(); // Location is not requested until first app services request - CHECK(!redir_transport->location_requested); - // Initial hostname and ws hostname use base url, but aren't used until location is updated - CHECK(app->get_host_url() == expected_url); - CHECK(app->get_ws_host_url() == expected_wsurl); - - oas.make_user(); - CHECK(redir_transport->location_requested); - CHECK(app->get_base_url() == expected_url); - CHECK(app->get_host_url() == expected_url); - CHECK(app->get_ws_host_url() == expected_wsurl); - } - { - // Fourth time through, base_url is set to https://some-other.someurl.fake, with a redirect - redir_transport->reset("https://some-other.someurl.fake", "http://redirect.someurl.fake"); - auto config = get_config_with_base_url("https://some-other.someurl.fake"); - OfflineAppSession oas(config); - auto app = oas.app(); - - // Location is not requested until first app services request - CHECK(!redir_transport->location_requested); + CHECK(!location_transport->location_requested); // Initial hostname and ws hostname use base url, but aren't used until location is updated - CHECK(app->get_host_url() == "https://some-other.someurl.fake"); - CHECK(app->get_ws_host_url() == "wss://some-other.someurl.fake"); + CHECK(app->get_host_url() == default_base_url); + CHECK(app->get_ws_host_url() == default_base_wsurl); oas.make_user(); - CHECK(redir_transport->location_requested); - // Base URL is still set to the original value - CHECK(app->get_base_url() == "https://some-other.someurl.fake"); - // Hostname and ws hostname use the redirect URL values - CHECK(app->get_host_url() == "http://redirect.someurl.fake"); - CHECK(app->get_ws_host_url() == "ws://redirect.someurl.fake"); + CHECK(location_transport->location_requested); + CHECK(app->get_base_url() == default_base_url); + CHECK(app->get_host_url() == default_base_url); + CHECK(app->get_ws_host_url() == default_base_wsurl); } } - SECTION("Test update_baseurl") { - redir_transport->reset("https://alternate.someurl.fake"); - auto config = get_config_with_base_url("https://alternate.someurl.fake"); + SECTION("Test update_baseurl after first request") { + bool error_occurred = GENERATE(true, false); + + location_transport->reset(test_base_url, test_location_url); + auto config = get_config_with_base_url(test_base_url); OfflineAppSession oas(config); auto app = oas.app(); // Location is not requested until first app services request - CHECK(!redir_transport->location_requested); + CHECK(!location_transport->location_requested); + // Perform an operation prior to updating the base URL oas.make_user(); - CHECK(redir_transport->location_requested); - CHECK(app->get_base_url() == "https://alternate.someurl.fake"); - CHECK(app->get_host_url() == "https://alternate.someurl.fake"); - CHECK(app->get_ws_host_url() == "wss://alternate.someurl.fake"); + CHECK(location_transport->location_requested); + CHECK(app->get_base_url() == test_base_url); + CHECK(app->get_host_url() == test_location_url); + CHECK(app->get_ws_host_url() == test_location_wsurl); - redir_transport->reset(App::default_base_url()); + location_transport->reset(default_base_url); + location_transport->location_returns_error = error_occurred; // Revert the base URL to the default URL value using the empty string - app->update_base_url("", [](util::Optional error) { - CHECK(!error); - }); - CHECK(redir_transport->location_requested); - CHECK(app->get_base_url() == App::default_base_url()); - CHECK(app->get_host_url() == App::default_base_url()); - CHECK(app->get_ws_host_url() == App::create_ws_host_url(App::default_base_url())); - oas.make_user(); + app->update_base_url("", [error_occurred](util::Optional error) { + CHECK(error.has_value() == error_occurred); + }); + CHECK(location_transport->location_requested); + if (error_occurred) { + // Not updated due to the error + CHECK(app->get_base_url() == test_base_url); + CHECK(app->get_host_url() == test_location_url); + CHECK(app->get_ws_host_url() == test_location_wsurl); + } + else { + // updated successfully + CHECK(app->get_base_url() == default_base_url); + CHECK(app->get_host_url() == default_base_url); + CHECK(app->get_ws_host_url() == default_base_wsurl); + oas.make_user(); // try another operation + } } - SECTION("Test update_baseurl with redirect") { - redir_transport->reset("https://alternate.someurl.fake"); - auto config = get_config_with_base_url("https://alternate.someurl.fake"); - OfflineAppSession oas(config); - auto app = oas.app(); + SECTION("Test update_baseurl before first request") { + bool error_occurred = GENERATE(true, false); - // Location is not requested until first app services request - CHECK(!redir_transport->location_requested); - - oas.make_user(); - CHECK(redir_transport->location_requested); - CHECK(app->get_base_url() == "https://alternate.someurl.fake"); - CHECK(app->get_host_url() == "https://alternate.someurl.fake"); - CHECK(app->get_ws_host_url() == "wss://alternate.someurl.fake"); - - redir_transport->reset("http://some-other.someurl.fake", "https://redirect.otherurl.fake"); - - app->update_base_url("http://some-other.someurl.fake", [](util::Optional error) { - CHECK(!error); - }); - CHECK(redir_transport->location_requested); - CHECK(app->get_base_url() == "http://some-other.someurl.fake"); - CHECK(app->get_host_url() == "https://redirect.otherurl.fake"); - CHECK(app->get_ws_host_url() == "wss://redirect.otherurl.fake"); - // Expected URL is still "https://redirect.otherurl.fake" after redirect - oas.make_user(); - } - - SECTION("Test update_baseurl returns error") { - redir_transport->reset("http://alternate.someurl.fake"); - auto config = get_config_with_base_url("http://alternate.someurl.fake"); + location_transport->reset(default_base_url, test_location_url, test_location_wsurl2); + location_transport->location_returns_error = error_occurred; + auto config = get_config_with_base_url(test_base_url); OfflineAppSession oas(config); auto app = oas.app(); - // Location is not requested until first app services request - CHECK(!redir_transport->location_requested); + // Check updating the base URL before an initial app_services request. + CHECK(!location_transport->location_requested); - oas.make_user(); - CHECK(redir_transport->location_requested); - CHECK(app->get_base_url() == "http://alternate.someurl.fake"); - CHECK(app->get_host_url() == "http://alternate.someurl.fake"); - CHECK(app->get_ws_host_url() == "ws://alternate.someurl.fake"); - - redir_transport->reset("https://some-other.someurl.fake"); - redir_transport->location_returns_error = true; - - app->update_base_url("https://some-other.someurl.fake", [](util::Optional error) { - CHECK(error); - }); - CHECK(redir_transport->location_requested); - // Verify original url values are still being used - CHECK(app->get_base_url() == "http://alternate.someurl.fake"); - CHECK(app->get_host_url() == "http://alternate.someurl.fake"); - CHECK(app->get_ws_host_url() == "ws://alternate.someurl.fake"); + // Revert the base URL to the default URL value using the empty string + app->update_base_url("", [error_occurred](util::Optional error) { + CHECK(error.has_value() == error_occurred); + }); + CHECK(location_transport->location_requested); + if (error_occurred) { + // Not updated due to the error + CHECK(app->get_base_url() == test_base_url); + CHECK(app->get_host_url() == test_base_url); + CHECK(app->get_ws_host_url() == test_base_wsurl); + } + else { + // updated successfully + CHECK(app->get_base_url() == default_base_url); + CHECK(app->get_host_url() == test_location_url); + CHECK(app->get_ws_host_url() == test_location_wsurl2); + oas.make_user(); // try another operation + } } // Verify new sync session updates location when created with cached user SECTION("Verify new sync session updates location") { bool use_ssl = GENERATE(true, false); - std::string initial_host = "alternate.someurl.fake"; - unsigned initial_port = use_ssl ? 443 : 80; - std::string expected_host = "redirect.someurl.fake"; - unsigned expected_port = 8081; - std::string init_url = util::format("http%1://%2", use_ssl ? "s" : "", initial_host); - std::string init_wsurl = util::format("ws%1://%2", use_ssl ? "s" : "", initial_host); - std::string redir_url = util::format("http%1://%2:%3", use_ssl ? "s" : "", expected_host, expected_port); - std::string redir_wsurl = util::format("ws%1://%2:%3", use_ssl ? "s" : "", expected_host, expected_port); + std::string base_host = "base.url.fake"; + std::string location_host = "alternate.url.fake"; + std::string new_location_host = "new.url.fake"; + unsigned location_port = use_ssl ? 443 : 80; + std::string sync_base_url = util::format("http://%1", base_host); + std::string sync_location_url = util::format("http%1://%2", use_ssl ? "s" : "", location_host); + std::string sync_location_wsurl = util::format("ws%1://%2", use_ssl ? "s" : "", location_host); + std::string new_location_url = util::format("http%1://%2", use_ssl ? "s" : "", new_location_host); + std::string new_location_wsurl = util::format("ws%1://%2", use_ssl ? "s" : "", new_location_host); auto socket_provider = std::make_shared(logger, "some user agent"); socket_provider->websocket_connect_func = []() -> std::optional { return SocketProviderError(sync::websocket::WebSocketError::websocket_connection_failed, "404 not found"); }; - auto config = get_config_with_base_url(init_url); + auto config = get_config_with_base_url(sync_base_url); config.metadata_mode = AppConfig::MetadataMode::NoEncryption; config.socket_provider = socket_provider; config.storage_path = util::make_temp_dir(); @@ -4009,24 +3798,24 @@ TEST_CASE("app: base_url", "[sync][app][base_url]") { // Log in to get a cached user { - redir_transport->reset(init_url); + location_transport->reset(sync_base_url, sync_location_url, sync_location_wsurl); OfflineAppSession oas(config); auto app = oas.app(); { + CHECK_FALSE(location_transport->location_requested); auto [sync_route, verified] = app->sync_manager()->sync_route(); - CHECK_THAT(sync_route, ContainsSubstring(app::App::create_ws_host_url(init_url))); + CHECK_THAT(sync_route, ContainsSubstring(app::App::create_ws_host_url(sync_base_url))); CHECK_FALSE(verified); } oas.make_user(); - CHECK(redir_transport->location_requested); - CHECK(app->get_base_url() == init_url); - CHECK(app->get_host_url() == init_url); - CHECK(app->get_ws_host_url() == init_wsurl); + CHECK(location_transport->location_requested); + CHECK(app->get_base_url() == sync_base_url); + CHECK(app->get_host_url() == sync_location_url); + CHECK(app->get_ws_host_url() == sync_location_wsurl); auto [sync_route, verified] = app->sync_manager()->sync_route(); - CHECK_THAT(sync_route, ContainsSubstring(app::App::create_ws_host_url(init_url))); - CHECK_THAT(sync_route, ContainsSubstring(init_wsurl)); + CHECK_THAT(sync_route, ContainsSubstring(sync_location_wsurl)); CHECK(verified); } @@ -4034,37 +3823,42 @@ TEST_CASE("app: base_url", "[sync][app][base_url]") { config.delete_storage = true; // Recreate the app using the cached user and start a sync session, which will is set to fail on connect SECTION("Sync Session fails on connect after updating location") { - enum class TestState { start, session_started }; + enum class TestState { start, first_attempt, second_attempt, complete }; TestingStateMachine state(TestState::start); - redir_transport->reset(init_url, redir_url); + location_transport->reset(sync_base_url, new_location_url, new_location_wsurl); + // Reuse the config so the app uses the cached user OfflineAppSession oas(config); auto app = oas.app(); + REQUIRE(app->current_user()); - // Verify the default sync route, which has not been verified + // Verify the initial sync route, since the location hasn't been queried + // and the location is not "verified", the sync route host is based off + // the value provided in the AppConfig::base_url value { auto [sync_route, verified] = app->sync_manager()->sync_route(); - CHECK_THAT(sync_route, ContainsSubstring(app::App::create_ws_host_url(init_url))); + CHECK_THAT(sync_route, ContainsSubstring(app::App::create_ws_host_url(sync_base_url))); CHECK_FALSE(verified); } - REQUIRE(app->current_user()); - std::atomic connect_attempts = 0; socket_provider->endpoint_verify_func = [&](const sync::WebSocketEndpoint& ep) { - // First connection attempt is to the originally specified endpoint. Since - // it hasn't been verified, we swallow the error and do a location update, - // which will then try to connect to the redir target - auto attempt = connect_attempts++; - if (attempt == 0) { - CHECK(ep.address == initial_host); - CHECK(ep.port == initial_port); - CHECK(ep.is_ssl == use_ssl); - } - else { - CHECK(ep.address == expected_host); - CHECK(ep.port == expected_port); - CHECK(ep.is_ssl == use_ssl); - } + state.transition_with([&](TestState cur_state) -> std::optional { + if (cur_state == TestState::start) { + // First time through is using the original base URL + CHECK(ep.address == base_host); + CHECK(ep.port == 80); + CHECK(ep.is_ssl == false); + return TestState::first_attempt; + } + else if (cur_state == TestState::first_attempt) { + // Second time through is using the values from location endpoint + CHECK(ep.address == new_location_host); + CHECK(ep.port == location_port); + CHECK(ep.is_ssl == use_ssl); + return TestState::second_attempt; + } + return std::nullopt; + }); }; RealmConfig r_config; @@ -4076,42 +3870,47 @@ TEST_CASE("app: base_url", "[sync][app][base_url]") { CHECK(!error.status.is_ok()); CHECK(error.status.code() == ErrorCodes::SyncConnectFailed); CHECK(!error.is_fatal); - state.transition_to(TestState::session_started); + state.transition_with([&](TestState cur_state) -> std::optional { + CHECK(cur_state == TestState::second_attempt); + return TestState::complete; + }); }; auto realm = Realm::get_shared_realm(r_config); - state.wait_for(TestState::session_started); + state.wait_for(TestState::complete); - CHECK(redir_transport->location_requested); - CHECK(app->get_base_url() == init_url); - CHECK(app->get_host_url() == redir_url); - CHECK(app->get_ws_host_url() == redir_wsurl); + CHECK(location_transport->location_requested); + CHECK(app->get_base_url() == sync_base_url); + CHECK(app->get_host_url() == new_location_url); + CHECK(app->get_ws_host_url() == new_location_wsurl); auto [sync_route, verified] = app->sync_manager()->sync_route(); - CHECK_THAT(sync_route, ContainsSubstring(redir_wsurl)); + CHECK_THAT(sync_route, ContainsSubstring(new_location_wsurl)); CHECK(verified); } - SECTION("Sync Session retries after initial location failure") { enum class TestState { start, location_failed, session_started }; TestingStateMachine state(TestState::start); const int retry_count = GENERATE(1, 3); - redir_transport->reset(init_url); - redir_transport->location_returns_error = true; + location_transport->reset(sync_base_url, new_location_url, new_location_wsurl); + location_transport->location_returns_error = true; + // Reuse the config so the app uses the cached user OfflineAppSession oas(config); auto app = oas.app(); REQUIRE(app->current_user()); - // Verify the default sync route, which has not been verified + // Verify the initial sync route, since the location hasn't been queried + // and the location is not "verified", the sync route host is based off + // the value provided in the AppConfig::base_url value { auto [sync_route, verified] = app->sync_manager()->sync_route(); - CHECK_THAT(sync_route, ContainsSubstring(app::App::create_ws_host_url(init_url))); + CHECK_THAT(sync_route, ContainsSubstring(app::App::create_ws_host_url(sync_base_url))); CHECK_FALSE(verified); } socket_provider->endpoint_verify_func = [&](const sync::WebSocketEndpoint& ep) { - CHECK(ep.address == initial_host); - CHECK(ep.port == initial_port); - CHECK(ep.is_ssl == use_ssl); + CHECK(ep.address == base_host); + CHECK(ep.port == 80); + CHECK(ep.is_ssl == false); }; socket_provider->websocket_connect_func = [&, request_count = @@ -4120,33 +3919,33 @@ TEST_CASE("app: base_url", "[sync][app][base_url]") { // First connection attempt is to the unverified initial URL // since we have a valid access token but have never successfully // connected. This failing will trigger a location update. - CHECK_FALSE(redir_transport->location_requested); + CHECK_FALSE(location_transport->location_requested); } else { // All attempts after the first should have requested location - CHECK(redir_transport->location_requested); - redir_transport->location_requested = false; + CHECK(location_transport->location_requested); + location_transport->location_requested = false; } // Until we allow a location request to succeed we should keep // getting the original unverified route - if (redir_transport->location_returns_error) { - CHECK(app->get_base_url() == init_url); - CHECK(app->get_host_url() == init_url); - CHECK(app->get_ws_host_url() == init_wsurl); + if (location_transport->location_returns_error) { + CHECK(app->get_base_url() == sync_base_url); + CHECK(app->get_host_url() == sync_base_url); + CHECK(app->get_ws_host_url() == app::App::create_ws_host_url(sync_base_url)); { auto [sync_route, verified] = app->sync_manager()->sync_route(); - CHECK_THAT(sync_route, ContainsSubstring(app::App::create_ws_host_url(init_url))); + CHECK_THAT(sync_route, ContainsSubstring(app::App::create_ws_host_url(sync_base_url))); CHECK_FALSE(verified); } } // After the chosen number of attempts let the location request succeed if (request_count++ >= retry_count) { - redir_transport->reset(init_url, redir_url); + location_transport->reset(sync_base_url, new_location_url, new_location_wsurl); socket_provider->endpoint_verify_func = [&](const sync::WebSocketEndpoint& ep) { - CHECK(ep.address == expected_host); - CHECK(ep.port == expected_port); + CHECK(ep.address == new_location_host); + CHECK(ep.port == location_port); CHECK(ep.is_ssl == use_ssl); state.transition_to(TestState::location_failed); }; @@ -4177,11 +3976,11 @@ TEST_CASE("app: base_url", "[sync][app][base_url]") { auto realm = Realm::get_shared_realm(r_config); state.wait_for(TestState::session_started); - CHECK(app->get_base_url() == init_url); - CHECK(app->get_host_url() == redir_url); - CHECK(app->get_ws_host_url() == redir_wsurl); + CHECK(app->get_base_url() == sync_base_url); + CHECK(app->get_host_url() == new_location_url); + CHECK(app->get_ws_host_url() == new_location_wsurl); auto [sync_route, verified] = app->sync_manager()->sync_route(); - CHECK_THAT(sync_route, ContainsSubstring(redir_wsurl)); + CHECK_THAT(sync_route, ContainsSubstring(new_location_wsurl)); CHECK(verified); } } From 17600131a582f7bdc6bb5cbad335a1c6661198d9 Mon Sep 17 00:00:00 2001 From: Michael Wilkerson-Barker Date: Fri, 30 Aug 2024 12:09:03 -0400 Subject: [PATCH 03/10] RCORE-2222 Remove 308 redirect support from App/AppUser (#7996) * Removed redirect tests * Removed one more location redirect test case * Removed 301/308 redirection support from App * Updates from review * Updated changelog after release * Fixed misspelling and updated comment a bit --- CHANGELOG.md | 2 +- src/realm/object-store/sync/app.cpp | 164 ++++++++-------------- src/realm/object-store/sync/app.hpp | 20 +-- src/realm/object-store/sync/app_utils.cpp | 22 --- src/realm/object-store/sync/app_utils.hpp | 2 - 5 files changed, 62 insertions(+), 148 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d91f2c8a32..6cc80c4033 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ * None. ### Breaking changes -* None. +* Removed http 301/308 redirection support from app services operations provided by App. It is assumed that the SDK's http implementation will handle http redirects instead. ([PR #7996](https://github.com/realm/realm-core/pull/7996)) ### Compatibility * Fileformat: Generates files with format v24. Reads and automatically upgrade from fileformat v10. If you want to upgrade from an earlier file format version you will have to use RealmCore v13.x.y or earlier. diff --git a/src/realm/object-store/sync/app.cpp b/src/realm/object-store/sync/app.cpp index 9306807912..6f21614415 100644 --- a/src/realm/object-store/sync/app.cpp +++ b/src/realm/object-store/sync/app.cpp @@ -189,7 +189,6 @@ constexpr static std::string_view s_sync_path = "/realm-sync"; constexpr static uint64_t s_default_timeout_ms = 60000; constexpr static std::string_view s_username_password_provider_key = "local-userpass"; constexpr static std::string_view s_user_api_key_provider_key_path = "api_keys"; -constexpr static int s_max_http_redirects = 20; static util::FlatMap> s_apps_cache; // app_id -> base_url -> app std::mutex s_apps_mutex; } // anonymous namespace @@ -978,18 +977,17 @@ void App::delete_user(const std::shared_ptr& user, UniqueFunctionuser_id(); - user->detach_and_tear_down(); - m_metadata_store->delete_user(*m_file_manager, user_id); - emit_change_to_subscribers(); - } - completion(std::move(error)); - }); + do_authenticated_request(HttpMethod::del, url_for_path("/auth/delete"), "", user, RequestTokenType::AccessToken, + [completion = std::move(completion), user, this](const Response& response) { + auto error = AppUtils::check_for_errors(response); + if (!error) { + auto user_id = user->user_id(); + user->detach_and_tear_down(); + m_metadata_store->delete_user(*m_file_manager, user_id); + emit_change_to_subscribers(); + } + completion(std::move(error)); + }); } void App::link_user(const std::shared_ptr& user, const AppCredentials& credentials, @@ -1054,34 +1052,32 @@ std::string App::get_app_route(const Optional& hostname) const } void App::request_location(UniqueFunction)>&& completion, - std::optional&& new_hostname, std::optional&& redir_location, - int redirect_count) -{ - // Request the new location information at the new base url hostname; or redir response location if a redirect - // occurred during the initial location request. redirect_count is used to track the number of sequential - // redirect responses received during the location update and return an error if this count exceeds - // max_http_redirects. If neither new_hostname nor redir_location is provided, the current value of m_base_url - // will be used. - std::string app_route; - std::string base_url; + std::optional&& new_hostname) +{ + // Request the new location information the original configured base_url or the new_hostname + // if the base_url is being updated. If a new_hostname has not been provided and the location + // has already been requested, this function does nothing. + std::string app_route; // The app_route for the server to query the location + std::string base_url; // The configured base_url hostname used for querying the location { util::CheckedUniqueLock lock(m_route_mutex); // Skip if the location info has already been initialized and a new hostname is not provided - if (!new_hostname && !redir_location && m_location_updated) { + if (!new_hostname && m_location_updated) { // Release the lock before calling the completion function lock.unlock(); completion(util::none); return; } - base_url = new_hostname.value_or(m_base_url); - // If this is for a redirect after querying new_hostname, then use the redirect location - if (redir_location) - app_route = get_app_route(redir_location); - // If this is querying the new_hostname, then use that location - else if (new_hostname) + // If this is querying the new_hostname, then use that to query the location + if (new_hostname) { + base_url = *new_hostname; app_route = get_app_route(new_hostname); - else + } + // Otherwise, use the current hostname + else { app_route = get_app_route(); + base_url = m_base_url; + } REALM_ASSERT(!app_route.empty()); } @@ -1093,46 +1089,15 @@ void App::request_location(UniqueFunction)>&& compl log_debug("App: request location: %1", req.url); m_config.transport->send_request_to_server(req, [self = shared_from_this(), completion = std::move(completion), - base_url = std::move(base_url), - redirect_count](const Response& response) mutable { - // Check to see if a redirect occurred - if (AppUtils::is_redirect_status_code(response.http_status_code)) { - // Make sure we don't do too many redirects (max_http_redirects (20) is an arbitrary number) - if (redirect_count >= s_max_http_redirects) { - completion(AppError{ErrorCodes::ClientTooManyRedirects, - util::format("number of redirections exceeded %1", s_max_http_redirects), - {}, - response.http_status_code}); - return; - } - // Handle the redirect response when requesting the location - extract the - // new location header field and resend the request. - auto redir_location = AppUtils::extract_redir_location(response.headers); - if (!redir_location) { - // Location not found in the response, pass error response up the chain - completion(AppError{ErrorCodes::ClientRedirectError, - "Redirect response missing location header", - {}, - response.http_status_code}); - return; - } - // try to request the location info at the new location in the redirect response - // retry_count is passed in to track the number of subsequent redirection attempts - self->request_location(std::move(completion), std::move(base_url), std::move(redir_location), - redirect_count + 1); - return; - } - + base_url = std::move(base_url)](const Response& response) { // Location request was successful - update the location info - auto update_response = self->update_location(response, base_url); - if (update_response) { - self->log_error("App: request location failed (%1%2): %3", update_response->code_string(), - update_response->additional_status_code - ? util::format(" %1", *update_response->additional_status_code) - : "", - update_response->reason()); + auto error = self->update_location(response, base_url); + if (error) { + self->log_error("App: request location failed (%1%2): %3", error->code_string(), + error->additional_status_code ? util::format(" %1", *error->additional_status_code) : "", + error->reason()); } - completion(update_response); + completion(error); }); } @@ -1169,8 +1134,7 @@ std::optional App::update_location(const Response& response, const std return util::none; } -void App::update_location_and_resend(std::unique_ptr&& request, IntermediateCompletion&& completion, - Optional&& redir_location) +void App::update_location_and_resend(std::unique_ptr&& request, IntermediateCompletion&& completion) { // Update the location information if a redirect response was received or m_location_updated == false // and then send the request to the server with request.url updated to the new AppServices hostname. @@ -1192,13 +1156,13 @@ void App::update_location_and_resend(std::unique_ptr&& request, Interme // Retry the original request with the updated url auto& request_ref = *request; self->m_config.transport->send_request_to_server( - request_ref, [self = std::move(self), completion = std::move(completion), - request = std::move(request)](const Response& response) mutable { - self->check_for_redirect_response(std::move(request), response, std::move(completion)); + request_ref, + [completion = std::move(completion), request = std::move(request)](const Response& response) mutable { + completion(std::move(request), response); }); }, // The base_url is not changing for this request - util::none, std::move(redir_location)); + util::none); } void App::post(std::string&& route, UniqueFunction)>&& completion, const BsonDocument& body) @@ -1212,8 +1176,18 @@ void App::post(std::string&& route, UniqueFunction)>&& c void App::do_request(std::unique_ptr&& request, IntermediateCompletion&& completion, bool update_location) { + // NOTE: Since the calls to `send_request_to_server()` or `update_location_and_resend()` do not + // capture a shared_ptr to App as part of their callback, any function that calls `do_request()` + // or `do_authenticated_request()` needs to capture the App as `self = shared_from_this()` for + // the completion callback to ensure the lifetime of the App object is extended until the + // callback is called after the operation is complete. + // Verify the request URL to make sure it is valid - util::Uri::parse(request->url); + if (auto valid_url = util::Uri::try_parse(request->url); !valid_url.is_ok()) { + completion(std::move(request), AppUtils::make_apperror_response( + AppError{valid_url.get_status().code(), valid_url.get_status().reason()})); + return; + } // Refresh the location info when app is created or when requested (e.g. after a websocket redirect) // to ensure the http and websocket URL information is up to date. @@ -1235,36 +1209,12 @@ void App::do_request(std::unique_ptr&& request, IntermediateCompletion& // If location info has already been updated, then send the request directly auto& request_ref = *request; m_config.transport->send_request_to_server( - request_ref, [self = shared_from_this(), completion = std::move(completion), - request = std::move(request)](const Response& response) mutable { - self->check_for_redirect_response(std::move(request), response, std::move(completion)); + request_ref, + [completion = std::move(completion), request = std::move(request)](const Response& response) mutable { + completion(std::move(request), response); }); } -void App::check_for_redirect_response(std::unique_ptr&& request, const Response& response, - IntermediateCompletion&& completion) -{ - // If this isn't a redirect response, then we're done - if (!AppUtils::is_redirect_status_code(response.http_status_code)) { - return completion(std::move(request), response); - } - - // Handle a redirect response when sending the original request - extract the location - // header field and resend the request. - auto redir_location = AppUtils::extract_redir_location(response.headers); - if (!redir_location) { - // Location not found in the response, pass error response up the chain - return completion(std::move(request), - AppUtils::make_clienterror_response(ErrorCodes::ClientRedirectError, - "Redirect response missing location header", - response.http_status_code)); - } - - // Request the location info at the new location - once this is complete, the original - // request will be sent to the new server - update_location_and_resend(std::move(request), std::move(completion), std::move(redir_location)); -} - void App::do_authenticated_request(HttpMethod method, std::string&& route, std::string&& body, const std::shared_ptr& user, RequestTokenType token_type, util::UniqueFunction&& completion) @@ -1314,10 +1264,10 @@ void App::handle_auth_failure(const AppError& error, std::unique_ptr&& // Reissue the request with the new access token request->headers = get_request_headers(user, RequestTokenType::AccessToken); - self->do_request(std::move(request), - [completion = std::move(completion)](auto&&, auto& response) { - completion(response); - }); + self->do_request(std::move(request), [self = self, completion = std::move(completion)]( + auto&&, auto& response) { + completion(response); + }); }); } diff --git a/src/realm/object-store/sync/app.hpp b/src/realm/object-store/sync/app.hpp index 3fe735ca64..ddee73aecc 100644 --- a/src/realm/object-store/sync/app.hpp +++ b/src/realm/object-store/sync/app.hpp @@ -525,12 +525,8 @@ class App : public std::enable_shared_from_this, /// a new hostname is provided, the app metadata will be refreshed using the new hostname. /// @param completion The callback that will be called with the error on failure or empty on success /// @param new_hostname The (Original) new hostname to request the location from - /// @param redir_location The location provided by the last redirect response when querying location - /// @param redirect_count The current number of redirects that have occurred in a row - void request_location(util::UniqueFunction)>&& completion, - std::optional&& new_hostname = std::nullopt, - std::optional&& redir_location = std::nullopt, int redirect_count = 0) - REQUIRES(!m_route_mutex); + void request_location(util::UniqueFunction)>&& completion, + std::optional&& new_hostname) REQUIRES(!m_route_mutex); /// Update the location metadata from the location response /// @param response The response returned from the location request @@ -542,9 +538,8 @@ class App : public std::enable_shared_from_this, /// Update the app metadata and resend the request with the updated metadata /// @param request The original request object that needs to be sent after the update /// @param completion The original completion object that will be called with the response to the request - /// @param new_hostname If provided, the metadata will be requested from this hostname - void update_location_and_resend(std::unique_ptr&& request, IntermediateCompletion&& completion, - std::optional&& new_hostname = util::none) REQUIRES(!m_route_mutex); + void update_location_and_resend(std::unique_ptr&& request, IntermediateCompletion&& completion) + REQUIRES(!m_route_mutex); void post(std::string&& route, util::UniqueFunction)>&& completion, const bson::BsonDocument& body) REQUIRES(!m_route_mutex); @@ -559,13 +554,6 @@ class App : public std::enable_shared_from_this, std::unique_ptr make_request(HttpMethod method, std::string&& url, const std::shared_ptr& user, RequestTokenType, std::string&& body) const; - /// Process the redirect response received from the last request that was sent to the server - /// @param request The request to be performed (in case it needs to be sent again) - /// @param response The response from the send_request_to_server operation - /// @param completion Returns the response from the server if not a redirect - void check_for_redirect_response(std::unique_ptr&& request, const Response& response, - IntermediateCompletion&& completion) REQUIRES(!m_route_mutex); - void do_authenticated_request(HttpMethod, std::string&& route, std::string&& body, const std::shared_ptr& user, RequestTokenType, util::UniqueFunction&&) override REQUIRES(!m_route_mutex); diff --git a/src/realm/object-store/sync/app_utils.cpp b/src/realm/object-store/sync/app_utils.cpp index 2226496ea8..82e1074699 100644 --- a/src/realm/object-store/sync/app_utils.cpp +++ b/src/realm/object-store/sync/app_utils.cpp @@ -50,28 +50,6 @@ bool AppUtils::is_success_status_code(int status_code) return status_code == 0 || (status_code < 300 && status_code >= 200); } -bool AppUtils::is_redirect_status_code(int status_code) -{ - using namespace realm::sync; - // If the response contains a redirection, then return true - if (auto code = HTTPStatus(status_code); - code == HTTPStatus::MovedPermanently || code == HTTPStatus::PermanentRedirect) { - return true; - } - return false; -} - -std::optional AppUtils::extract_redir_location(const std::map& headers) -{ - // Look for case insensitive redirect "location" in headers - auto location = AppUtils::find_header("location", headers); - if (location && !location->second.empty() && util::Uri::try_parse(location->second).is_ok()) { - // If the location is valid, return it wholesale (e.g., it could include a path for API proxies) - return location->second; - } - return std::nullopt; -} - // Create a Response object with the given client error, message and optional http status code Response AppUtils::make_clienterror_response(ErrorCodes::Error code, const std::string_view message, std::optional http_status) diff --git a/src/realm/object-store/sync/app_utils.hpp b/src/realm/object-store/sync/app_utils.hpp index 2f6d1e4f66..371bbb3e45 100644 --- a/src/realm/object-store/sync/app_utils.hpp +++ b/src/realm/object-store/sync/app_utils.hpp @@ -38,8 +38,6 @@ class AppUtils { static const std::pair* find_header(const std::string& key_name, const std::map& search_map); static bool is_success_status_code(int status_code); - static bool is_redirect_status_code(int status_code); - static std::optional extract_redir_location(const std::map& headers); }; } // namespace realm::app From f010f7302735dcc8b042044c362c7637b490ec76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Edelbo?= Date: Tue, 3 Sep 2024 17:28:09 +0200 Subject: [PATCH 04/10] Add failing test --- test/object-store/object.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/object-store/object.cpp b/test/object-store/object.cpp index e8782166e3..f4bab829eb 100644 --- a/test/object-store/object.cpp +++ b/test/object-store/object.cpp @@ -575,6 +575,16 @@ TEST_CASE("object") { }); } + SECTION("modifying origin table 'table2', property 'value' " + "while observing related table 'table', property 'origin' " + "-> does NOT send a notification") { + auto token = require_no_change(object_target, kpa_target_backlink); + + write([&] { + object_origin.set_column_value("value", 105); + }); + } + SECTION("modifying related table 'table', property 'value 1' " "while observing related table 'table', property 'value 1' " "-> DOES send a notification") { From fd63c1c04c1aadc6dfae2cf4d15e89867e798ddc Mon Sep 17 00:00:00 2001 From: Jonathan Reams Date: Sun, 8 Sep 2024 23:02:00 -0400 Subject: [PATCH 05/10] Bump baasaas version --- dependencies.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dependencies.yml b/dependencies.yml index 3bf50ca20f..4788bba2c9 100644 --- a/dependencies.yml +++ b/dependencies.yml @@ -3,6 +3,6 @@ VERSION: 14.12.1 OPENSSL_VERSION: 3.3.1 ZLIB_VERSION: 1.2.13 # https://github.com/10gen/baas/commits -# 2f308db is 2024 July 10 -BAAS_VERSION: 2f308db6f65333728a101d1fecbb792f9659a5ce +# 04e3f27ad0e is 2024 Sep 8th +BAAS_VERSION: 04e3f27ad0eb9154bc4e3b631d179d702ac05215 BAAS_VERSION_TYPE: githash From 00eba1c788820607ecaaa150491a87e8ccf643f1 Mon Sep 17 00:00:00 2001 From: Jonathan Reams Date: Thu, 12 Sep 2024 09:00:58 -0400 Subject: [PATCH 06/10] RCORE-2257 Remove unneeded evergreen tasks/build variants (#8025) --- evergreen/bloaty_to_json.py | 175 ------- evergreen/config.yml | 574 +---------------------- evergreen/configure_baas_proxy.sh | 237 ---------- evergreen/proxy-network-faults.toxics | 4 - evergreen/proxy-nonideal-transfer.toxics | 5 - evergreen/setup_baas_host.sh | 279 ----------- evergreen/setup_baas_host_local.sh | 269 ----------- evergreen/setup_baas_proxy.sh | 263 ----------- evergreen/wait_for_baas.sh | 100 ---- evergreen/wait_for_remote_baas.sh | 126 ----- 10 files changed, 5 insertions(+), 2027 deletions(-) delete mode 100755 evergreen/bloaty_to_json.py delete mode 100755 evergreen/configure_baas_proxy.sh delete mode 100644 evergreen/proxy-network-faults.toxics delete mode 100644 evergreen/proxy-nonideal-transfer.toxics delete mode 100755 evergreen/setup_baas_host.sh delete mode 100755 evergreen/setup_baas_host_local.sh delete mode 100755 evergreen/setup_baas_proxy.sh delete mode 100755 evergreen/wait_for_baas.sh delete mode 100755 evergreen/wait_for_remote_baas.sh diff --git a/evergreen/bloaty_to_json.py b/evergreen/bloaty_to_json.py deleted file mode 100755 index 47b5967d08..0000000000 --- a/evergreen/bloaty_to_json.py +++ /dev/null @@ -1,175 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import argparse -import json -import re -from csv import DictReader -from pathlib import Path - -parser = argparse.ArgumentParser(description='Checks how bloated realm has become') -parser.add_argument( - '--short-symbols-input', - type=Path, - help='Path to CSV output of short symbols input file', -) -parser.add_argument( - '--sections-input', - type=Path, - help='Path to CSV output of sections input file', -) - -parser.add_argument( - '--compileunits-input', - type=Path, - help='Path to CSV output of compileunits input file', -) - -parser.add_argument( - '--analyzed-file', - type=str, - help='Name of file being analyzed by bloaty', -) - -evgOpts = parser.add_argument_group('Evergreen Metadata') -evgOpts.add_argument('--output', type=Path, help='The evergreen json output filename') -evgOpts.add_argument('--project', type=str, help='Evergreen project this script is running in') -evgOpts.add_argument('--execution', type=int, help='Execution # of this evergreen task') -evgOpts.add_argument( - '--is-patch', - type=bool, - dest='is_patch', - help='Specify if this is not a patch build', -) -evgOpts.add_argument( - '--build-variant', - type=str, - dest='build_variant', - help='Build variant of the evergreen task', -) -evgOpts.add_argument('--branch', type=str, help='Git branch that was being tested') -evgOpts.add_argument('--revision', type=str, help='Git sha being tested') -evgOpts.add_argument('--task-id', type=str, dest='task_id', help='Evergreen task ID of this task') -evgOpts.add_argument('--task-name', type=str, dest='task_name', help='Name of this evergreen task') -evgOpts.add_argument( - '--revision-order-id', - type=str, - dest='revision_order_id', - help='Evergreen revision order id', -) -evgOpts.add_argument('--version-id', type=str, dest='version_id', help='Name of this evergreen version') - -args = parser.parse_args() -patch_username : str = '' - -def parse_patch_order(): - global patch_username - patch_order_re = re.compile(r"(?P[\w\@\.]+)_(?P\d+)") - match_obj = patch_order_re.match(args.revision_order_id) - patch_username = match_obj.group('patch_username') - return int(match_obj.group('patch_order')) -evg_order = int(args.revision_order_id) if not args.is_patch else parse_patch_order() - -cxx_method_re = re.compile( - # namespaces/parent class name - r"(?P(?:(?:[_a-zA-Z][\w]*)(?:<.*>)?(?:::)|(?:\(anonymous namespace\)::))+)" + - r"(?P[\~a-zA-Z_][\w]*)(?:<.*>)?" + # function/class name - r"(?P\(\))?" + # if this is function, this will capture "()" - # will be a number if this is a lambda - r"(?:::\{lambda\(\)\#(?P\d+)\}::)?") - -elf_section_re = re.compile(r"\[section \.(?P[\w\.\-]+)\]") - -items : list[dict] = [] -sections_seen = set() -if args.short_symbols_input: - with open(args.short_symbols_input, 'r') as csv_file: - input_csv_reader = DictReader(csv_file) - for row in input_csv_reader: - raw_name = row['shortsymbols'] - if match := cxx_method_re.search(raw_name): - ns = match.group('ns').rstrip(':') - - node_name = match.group('name') - if match.group('lambda_number'): - node_name = "{} lambda #{}".format(node_name, match.group('lambda_number')) - - type_str: str = 'symbol' - if match.group('lambda_number'): - type_str = 'lambda' - elif match.group('is_function'): - type_str = 'function' - - items.append({ - 'type': type_str, - 'name': raw_name, - 'ns': ns, - 'file_size': int(row['filesize']), - 'vm_size': int(row['vmsize']), - }) - - elif match := elf_section_re.search(raw_name): - section_name = match.group('section_name') - type_str: str = 'section' if not section_name.startswith('.debug') else 'debug_section' - if section_name not in sections_seen: - items.append({ - 'type': type_str, - 'name': section_name, - 'file_size': int(row['filesize']), - 'vm_size': int(row['vmsize']) - }) - else: - items.append({ - 'type': 'symbol', - 'name': raw_name, - 'file_size': int(row['filesize']), - 'vm_size': int(row['vmsize']), - }) - -if args.sections_input: - with open(args.sections_input, 'r') as csv_file: - input_csv_reader = DictReader(csv_file) - - for row in input_csv_reader: - section_name = row['sections'] - type_str: str = 'section' if not section_name.startswith('.debug') else 'debug_section' - if section_name not in sections_seen: - items.append({ - 'name': section_name, - 'type': type_str, - 'file_size': int(row['filesize']), - 'vm_size': int(row['vmsize']) - }) - -if args.sections_input: - with open(args.compileunits_input, 'r') as csv_file: - input_csv_reader = DictReader(csv_file) - - for row in input_csv_reader: - compileunit_name = row['compileunits'] - if not elf_section_re.search(compileunit_name): - items.append({ - 'name': compileunit_name, - 'type': 'compileunit', - 'file_size': int(row['filesize']), - 'vm_size': int(row['vmsize']) - }) - -output_obj = { - 'items': items, - 'execution': args.execution, - 'is_mainline': (args.is_patch is not True), - 'analyzed_file': args.analyzed_file, - 'order': evg_order, - 'project': args.project, - 'branch': args.branch, - 'build_variant': args.build_variant, - 'revision': args.revision, - 'task_id': args.task_id, - 'task_name': args.task_name, - 'version_id': args.version_id, - 'patch_username': patch_username -} - -with open(args.output, 'w') as out_fp: - json.dump(output_obj, out_fp) diff --git a/evergreen/config.yml b/evergreen/config.yml index e14afde6c9..09d1c4e114 100644 --- a/evergreen/config.yml +++ b/evergreen/config.yml @@ -239,33 +239,6 @@ functions: make "-j$NPROC" 2>&1 popd > /dev/null # realm-core - "run benchmark": - - command: shell.exec - params: - working_dir: realm-core - shell: bash - script: |- - set -o errexit - - if [[ -z "${benchmark_name}" ]]; then - echo "No benchmark specified." - exit 1 - fi - - export UNITTEST_THREADS=1 - - BENCHMARK=$(./evergreen/abspath.sh ./build/test/benchmark-${benchmark_name}/${cmake_build_type|Debug}/realm-benchmark-${benchmark_name}) - echo "Going to run benchmark $BENCHMARK" - - [[ -d benchmark_results ]] && rm -rf benchmark_results - mkdir benchmark_results - cd benchmark_results - - $BENCHMARK "$(pwd)/" - - command: perf.send - params: - file: './realm-core/benchmark_results/results.latest.json' - "run tests": - command: expansions.update params: @@ -389,92 +362,6 @@ functions: file_location: realm-core/${task_name}_results.json "upload baas artifacts": - - command: shell.exec - params: - working_dir: realm-core - shell: bash - script: |- - set -o errexit - set -o pipefail - set -o verbose - - if [[ -n "${disable_tests_against_baas|}" ]]; then - exit 0 - fi - - # Copy the baas_server log from the remote baas host if it exists - if [[ ! -f baas_host.yml || ! -f .baas_ssh_key ]]; then - echo "Skipping - no remote baas host or remote baas host definitions not found" - exit 0 - fi - - BAAS_HOST_NAME=$(tr -d '"[]{}' < baas_host.yml | cut -d , -f 1 | awk -F : '{print $2}') - export BAAS_HOST_NAME - - ssh_user="$(printf "ubuntu@%s" "$BAAS_HOST_NAME")" - ssh_options="-o ForwardAgent=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=no -o ConnectTimeout=60 -i .baas_ssh_key" - - # Copy the baas_server.log, mongod.log and (optionally) baas_proxy.log files from the remote baas host - REMOTE_PATH=/data/baas-remote - REMOTE_BAAS_PATH="$REMOTE_PATH/baas-work-dir" - REMOTE_BAAS_DB_PATH="$REMOTE_BAAS_PATH/mongodb-dbpath" - REMOTE_PROXY_PATH="$REMOTE_PATH/baas-proxy-dir" - - LOCAL_BAAS_PATH=./baas-work-dir - LOCAL_BAAS_DB_PATH="$LOCAL_BAAS_PATH/mongodb-dbpath" - LOCAL_PROXY_PATH=./baas-proxy-dir - - mkdir -p "$LOCAL_BAAS_PATH/" - mkdir -p "$LOCAL_BAAS_DB_PATH/" - mkdir -p "$LOCAL_PROXY_PATH/" - scp $ssh_options $ssh_user:"$REMOTE_BAAS_PATH/baas_server.log" "$LOCAL_BAAS_PATH/baas_server.log" || true - scp $ssh_options $ssh_user:"$REMOTE_BAAS_DB_PATH/mongod.log" "$LOCAL_BAAS_DB_PATH/mongod.log" || true - scp $ssh_options $ssh_user:"$REMOTE_PROXY_PATH/baas_proxy.log" "$LOCAL_PROXY_PATH/baas_proxy.log" || true - - - command: s3.put - params: - aws_key: '${artifacts_aws_access_key}' - aws_secret: '${artifacts_aws_secret_key}' - local_file: 'realm-core/baas-work-dir/baas_server.log' - remote_file: 'realm-core-stable/${branch_name}/${task_id}/${execution}/baas_server.log' - bucket: mciuploads - permissions: public-read - content_type: text/plain - display_name: baas server logs - optional: true - - command: s3.put - params: - aws_key: '${artifacts_aws_access_key}' - aws_secret: '${artifacts_aws_secret_key}' - local_file: 'realm-core/install_baas_output.log' - remote_file: 'realm-core-stable/${branch_name}/${task_id}/${execution}/install_baas_output.log' - bucket: mciuploads - permissions: public-read - content_type: text/plain - display_name: install baas output - optional: true - - command: s3.put - params: - aws_key: '${artifacts_aws_access_key}' - aws_secret: '${artifacts_aws_secret_key}' - local_file: 'realm-core/baas-work-dir/mongodb-dbpath/mongod.log' - remote_file: 'realm-core-stable/${branch_name}/${task_id}/${execution}/mongod.log' - bucket: mciuploads - permissions: public-read - content_type: text/plain - display_name: mongod logs - optional: true - - command: s3.put - params: - aws_key: '${artifacts_aws_access_key}' - aws_secret: '${artifacts_aws_secret_key}' - local_file: 'realm-core/baas-proxy-dir/baas_proxy.log' - remote_file: 'realm-core-stable/${branch_name}/${task_id}/${execution}/baas_proxy.log' - bucket: mciuploads - permissions: public-read - content_type: text/plain - display_name: baas proxy logs - optional: true - command: s3.put params: aws_key: '${artifacts_aws_access_key}' @@ -487,53 +374,6 @@ functions: display_name: trace level test logs optional: true - "upload fuzzer results": - - command: shell.exec - params: - working_dir: realm-core/build/test/realm-fuzzer - script: |- - if ls crash-*> /dev/null 2>&1; then - echo "Found crash file" - #Rename the crash file and the realm file. - #If there is a crash, this will signal that something needs to be uploaded. - mv crash-* realm-fuzzer-crash.txt - mv fuzz-realm.realm fuzzer_realm.realm - fi - - - command: s3.put - params: - working_dir: realm-core/build/test/realm-fuzzer - aws_key: '${artifacts_aws_access_key}' - aws_secret: '${artifacts_aws_secret_key}' - local_file: 'realm-core/build/test/realm-fuzzer/realm-fuzzer-crash.txt' - remote_file: '${project}/${branch_name}/${task_id}/${execution}/realm-fuzzer-crash.txt' - bucket: mciuploads - permissions: public-read - content_type: text/plain - display_name: Crash input file - optional: true - - - command: s3.put - params: - working_dir: realm-core/build/test/realm-fuzzer - aws_key: '${artifacts_aws_access_key}' - aws_secret: '${artifacts_aws_secret_key}' - local_file: 'realm-core/build/test/realm-fuzzer/fuzzer_realm.realm' - remote_file: '${project}/${branch_name}/${task_id}/${execution}/fuzzer_realm.realm' - bucket: mciuploads - permissions: public-read - content_type: application/x-binary - display_name: Realm File - optional: true - - - command: shell.exec - params: - working_dir: realm-core/build/test/realm-fuzzer - script: |- - #remove the crash file and the realm produced by the fuzzer run - rm realm-fuzzer-crash.txt - rm fuzzer_realm.realm - "run hang analyzer": - command: shell.exec params: @@ -628,149 +468,6 @@ functions: python $HANG_ANALYZER_PATH - "launch remote baas": - - command: host.create - params: - provider: ec2 - distro: ubuntu2004-medium - - command: host.list - params: - num_hosts: 1 - path: realm-core/baas_host.yml - timeout_seconds: 600 - wait: true - - command: shell.exec - params: - working_dir: realm-core - shell: bash - # Do NOT use verbose for this script since it would reveal the values of secrets - script: |- - set -o errexit - set -o pipefail - - if [[ -n "${disable_tests_against_baas|}" ]]; then - echo "Should not be using remote baas if the tests are not being used" - exit 0 - fi - - # Save the remote host ssh key and details to files - echo "${__project_aws_ssh_key_value}" > .baas_ssh_key - chmod 600 .baas_ssh_key - - BAAS_HOST_NAME=$(tr -d '"[]{}' < baas_host.yml | cut -d , -f 1 | awk -F : '{print $2}') - - # Github SSH host key updated 06/26/2023 - # Use 'github_known_hosts' expansion for GITHUB_KNOWN_HOSTS below once EVG-20410 has been implemented - cat >baas_host_vars.sh <&1 | tee install_baas_output.log - - "wait for baas to start": - - command: shell.exec - params: - working_dir: realm-core - shell: bash - script: |- - set -o errexit - set -o verbose - - if [[ -n "${disable_tests_against_baas|}" ]]; then - exit 0 - fi - - # Don't print out the tail of the log file - ./evergreen/wait_for_baas.sh -s -w ./baas-work-dir -l "" - - echo "Baas is started!" - - "wait for remote baas to start": - - command: shell.exec - params: - working_dir: realm-core - shell: bash - script: |- - set -o errexit - set -o verbose - - if [[ -n "${disable_tests_against_baas|}" ]]; then - exit 0 - fi - - # Don't print out the tail of the log file - ./evergreen/wait_for_remote_baas.sh -i ./.baas_ssh_key ./baas_host_vars.sh ./.baas_ssh_key - - echo "Baas is started!" - - "setup proxy parameters": - - command: shell.exec - params: - working_dir: realm-core - shell: bash - script: |- - set -o errexit - set -o pipefail - set -o verbose - - if [[ -n "${disable_tests_against_baas|}" ]]; then - echo "Error: Baas is disabled for network tests" - exit 1 - fi - - if [[ -z "${proxy_toxics_file|}" ]]; then - echo "Error: Baas proxy toxics config file was not provided" - exit 1 - fi - - if [[ -n "${proxy_toxics_randoms|}" ]]; then - PROXY_RANDOMS="-r ${proxy_toxics_randoms}" - fi - - # Configure the toxics for the baas proxy - evergreen/configure_baas_proxy.sh $PROXY_RANDOMS "${proxy_toxics_file}" - # Display the list of configured toxics - curl --silent http://localhost:8474/proxies/baas_proxy/toxics - "check branch state": - command: shell.exec type: setup @@ -953,30 +650,6 @@ tasks: test_filter: CoreTests test_executable_name: "realm-tests" -- name: benchmark-common-tasks - exec_timeout_secs: 1800 - tags: [ "benchmark" ] - commands: - - func: "run benchmark" - vars: - benchmark_name: common-tasks - -- name: benchmark-crud - exec_timeout_secs: 1800 - tags: [ "benchmark" ] - commands: - - func: "run benchmark" - vars: - benchmark_name: crud - -- name: benchmark-sync - exec_timeout_secs: 1800 - tags: [ "benchmark" ] - commands: - - func: "run benchmark" - vars: - benchmark_name: sync - - name: sync-tests tags: [ "test_suite", "for_pull_requests" ] exec_timeout_secs: 1800 @@ -1020,31 +693,6 @@ tasks: verbose_test_output: true - func: "check branch state" -- name: baas-network-tests - # Uncomment once tests are passing - tags: [ "disabled" ] - # tags: [ "for_nightly_tests" ] - # These tests can be manually requested for patches and pull requests - allowed_requesters: [ "ad_hoc", "patch", "github_pr" ] - # The network tests can take a really long time, so we give the test up to 4 - # hours to complete - exec_timeout_secs: 14400 - commands: - - func: "launch remote baas" - vars: - baas_proxy: On - - func: "compile" - vars: - target_to_build: ObjectStoreTests - - func: "wait for baas to start" - - func: "setup proxy parameters" - - func: "run tests" - vars: - test_label: objstore-baas - test_executable_name: "realm-object-store-tests" - verbose_test_output: true - - func: "check branch state" - - name: process_coverage_data tags: [ "for_pull_requests" ] exec_timeout_secs: 1800 @@ -1190,16 +838,6 @@ tasks: echo "COMMAND: git clang-format -f --commit $format_ref" exit 1 -- name: fuzzer - tags: [ "for_nightly_tests" ] - allowed_requesters: [ "ad_hoc", "patch" ] - commands: - - command: shell.exec - params: - working_dir: realm-core/build/test/realm-fuzzer - shell: /bin/bash - script: |- - ${cmake_build_type|Debug}/realm-libfuzz -rss_limit_mb=0 -max_total_time=3600 - name: combined-tests-ios-simulator tags: [ "for_pull_requests" ] @@ -1237,56 +875,6 @@ tasks: 'io.realm.CombinedTests' \ $TEST_RESULTS_FILE -- name: generate-sync-corpus - tags: [ "for_nightly_tests" ] - exec_timeout_secs: 1800 - commands: - - func: "fetch source" - - func: "fetch binaries" - - func: "compile" - vars: - target_to_build: "SyncTests" - - command: shell.exec - params: - working_dir: realm-core - shell: bash - script: |- - set -o errexit - set -o verbose - - if [[ -n "${c_compiler}" && "$(basename ${c_compiler})" = "clang" && -f "$(dirname ${c_compiler})/llvm-symbolizer" ]]; then - LLVM_SYMBOLIZER="$(dirname ${c_compiler})/llvm-symbolizer" - export ASAN_SYMBOLIZER_PATH="$(./evergreen/abspath.sh $LLVM_SYMBOLIZER)" - export TSAN_OPTIONS="external_symbolizer_path=$(./evergreen/abspath.sh $LLVM_SYMBOLIZER)" - fi - - export UNITTEST_EVERGREEN_TEST_RESULTS="$(./evergreen/abspath.sh ${task_name}_results.json)" - if [[ -n "$UNITTEST_EVERGREEN_TEST_RESULTS" && -f "$UNITTEST_EVERGREEN_TEST_RESULTS" ]]; then - rm "$UNITTEST_EVERGREEN_TEST_RESULTS" - fi - export UNITTEST_PROGRESS=1 - export UNITTEST_FILTER="Array_Example Transform_* EmbeddedObjects_*" - export UNITTEST_DUMP_TRANSFORM="changeset_dump" - - export TSAN_OPTIONS="suppressions=$(pwd)/test/tsan.suppress history_size=4 $TSAN_OPTIONS" - export UBSAN_OPTIONS="print_stacktrace=1" - - ./build/test/${cmake_build_type}/realm-sync-tests - - CHANGESET_DUMP_DIR="$(find . -type d -name changeset_dump -print -quit)" - mv "$CHANGESET_DUMP_DIR" changeset_dump - tar -czvf changeset_dump.tgz changeset_dump - - command: s3.put - params: - aws_key: '${artifacts_aws_access_key}' - aws_secret: '${artifacts_aws_secret_key}' - local_file: 'realm-core/changeset_dump.tgz' - remote_file: 'realm-core-stable/${branch_name}/${task_id}/${execution}/changeset_dump.tgz' - bucket: mciuploads - permissions: public-read - content_type: application/gzip - display_name: changeset dump tarball - task_groups: - name: core_tests_group setup_group_can_fail_task: true @@ -1371,21 +959,6 @@ task_groups: tasks: - compile-emscripten -# Runs object-store-tests against baas running on remote host and runs -# the network simulation tests as a separate task for nightly builds -- name: network_tests - max_hosts: 1 - setup_group_can_fail_task: true - setup_group: - - func: "fetch source" - - func: "fetch binaries" - teardown_task: - - func: "upload test results" - - func: "upload baas artifacts" - timeout: - - func: "run hang analyzer" - tasks: - - baas-network-tests # Runs object-store-tests against baas running on remote host - name: compile_test_coverage @@ -1404,18 +977,6 @@ task_groups: - .test_suite - process_coverage_data -- name: benchmarks - setup_group_can_fail_task: true - setup_group: - - func: "fetch source" - - func: "fetch binaries" - - func: "compile" - vars: - target_to_build: "benchmarks" - timeout: - - func: "run hang analyzer" - tasks: - - .benchmark - name: long-running-tests setup_group_can_fail_task: true @@ -1433,18 +994,6 @@ task_groups: tasks: - long-running-core-tests -- name: fuzzer-tests - setup_group_can_fail_task: true - setup_group: - - func: "fetch source" - - func: "fetch binaries" - - func: "compile" - vars: - target_to_build: realm-libfuzz - teardown_task: - - func: "upload fuzzer results" - tasks: - - fuzzer - name: ios-simulator-tests max_hosts: 1 @@ -1592,11 +1141,8 @@ buildvariants: tasks: - name: core_tests_group -# TODO RCORE-2085 ubuntu2004-release/ubuntu2004-arm64 build variants are here until we've established -# a new baseline for the updated ubuntu 2204/clang 18 builders and to generate artifacts for the baas -# team. - name: ubuntu2004-release - display_name: "Ubuntu 20.04 x86_64 (Clang 11 release benchmarks/baas artifacts)" + display_name: "Ubuntu 20.04 x86_64 (Clang 11 release baas artifacts)" run_on: ubuntu2004-large expansions: clang_url: "https://s3.amazonaws.com/static.realm.io/evergreen-assets/clang%2Bllvm-11.0.0-x86_64-linux-gnu-ubuntu-20.04.tar.xz" @@ -1610,36 +1156,6 @@ buildvariants: python3: /opt/mongodbtoolchain/v3/bin/python3 tasks: - name: compile_test_and_package - - name: benchmarks - - name: generate-sync-corpus - -- name: ubuntu2004-arm64 - display_name: "Ubuntu 20.04 ARM64 (GCC 9 release benchmarks)" - run_on: ubuntu2004-arm64-large - expansions: - cmake_url: "https://s3.amazonaws.com/static.realm.io/evergreen-assets/cmake-3.26.3-linux-aarch64.tar.gz" - cmake_bindir: "./cmake_binaries/bin" - python3: "/opt/mongodbtoolchain/v3/bin/python3" - use_system_openssl: On - fetch_missing_dependencies: On - cmake_build_type: RelWithDebInfo - tasks: - - name: benchmarks - -- name: ubuntu-x86_64-benchmarks - display_name: "Ubuntu x86_64 benchmarks" - run_on: ubuntu2204-large - expansions: - fetch_missing_dependencies: On - cmake_url: "https://s3.amazonaws.com/static.realm.io/evergreen-assets/cmake-3.26.3-linux-x86_64.tar.gz" - cmake_bindir: "./cmake_binaries/bin" - clang_url: "https://s3.amazonaws.com/static.realm.io/evergreen-assets/clang%2Bllvm-18.1.2-x86_64-linux-gnu.tar.xz" - clang_sha256_sum: "feab4b0f5fba325bfe0f4245710fd19fd74f813f44b5e81eda794f4f75bca343" - c_compiler: "./clang_binaries/bin/clang" - cxx_compiler: "./clang_binaries/bin/clang++" - cmake_build_type: RelWithDebInfo - tasks: - - name: benchmarks - name: ubuntu-release display_name: "Ubuntu (Release build)" @@ -1652,7 +1168,6 @@ buildvariants: python3: /opt/mongodbtoolchain/v3/bin/python3 tasks: - name: compile_test_and_package - - name: benchmarks - name: long-running-tests - name: ubuntu-asan @@ -1708,20 +1223,6 @@ buildvariants: tasks: - name: compile_test -- name: ubuntu-fuzzer - display_name: "Ubuntu (Fuzzer)" - run_on: ubuntu2204-arm64-large - expansions: - fetch_missing_dependencies: On - enable_ubsan: On - c_compiler: "/opt/clang+llvm/bin/clang" - cxx_compiler: "/opt/clang+llvm/bin/clang++" - cmake_build_type: RelWithDebInfo - run_with_encryption: 1 - enable_fuzzer: On - tasks: - - name: fuzzer-tests - - name: ubuntu-emscripten display_name: "Ubuntu (Emscripten x86_64)" run_on: ubuntu2204-large @@ -1743,52 +1244,6 @@ buildvariants: tasks: - name: compile_emscripten -# disable these builders since there are constantly failing and not yet ready for nightly builds -# - name: ubuntu2004-network-nonideal -# display_name: "Ubuntu 20.04 x86_64 (Utunbu2004 - nonideal transfer)" -# run_on: ubuntu2004-large -# expansions: -# clang_url: "https://s3.amazonaws.com/static.realm.io/evergreen-assets/clang%2Bllvm-11.0.0-x86_64-linux-gnu-ubuntu-20.04.tar.xz" -# cmake_url: "https://s3.amazonaws.com/static.realm.io/evergreen-assets/cmake-3.26.3-linux-x86_64.tar.gz" -# cmake_bindir: "./cmake_binaries/bin" -# fetch_missing_dependencies: On -# c_compiler: "./clang_binaries/bin/clang" -# cxx_compiler: "./clang_binaries/bin/clang++" -# cmake_build_type: RelWithDebInfo -# run_with_encryption: On -# baas_admin_port: 9098 -# test_logging_level: debug -# test_timeout_extra: 60 -# proxy_toxics_file: evergreen/proxy-nonideal-transfer.toxics -# # RANDOM1: bandwidth-upstream limited to between 10-50 KB/s from the client to the server -# # RANDOM2: bandwidth-downstream limited to between 10-50 KB/s from the server to the client -# proxy_toxics_randoms: "10:50|10:50" -# tasks: -# - name: network_tests -# -# - name: ubuntu2004-network-faulty -# display_name: "Ubuntu 20.04 x86_64 (Utunbu2004 - network faults)" -# run_on: ubuntu2004-large -# expansions: -# clang_url: "https://s3.amazonaws.com/static.realm.io/evergreen-assets/clang%2Bllvm-11.0.0-x86_64-linux-gnu-ubuntu-20.04.tar.xz" -# cmake_url: "https://s3.amazonaws.com/static.realm.io/evergreen-assets/cmake-3.26.3-linux-x86_64.tar.gz" -# cmake_bindir: "./cmake_binaries/bin" -# fetch_missing_dependencies: On -# c_compiler: "./clang_binaries/bin/clang" -# cxx_compiler: "./clang_binaries/bin/clang++" -# cmake_build_type: RelWithDebInfo -# run_with_encryption: On -# baas_admin_port: 9098 -# test_logging_level: debug -# proxy_toxics_file: evergreen/proxy-network-faults.toxics -# # RANDOM1: limit-data-upstream to close connection after between 1000-3000 bytes have been sent -# # RANDOM2: limit-data-downstream to close connection after between 1000-3000 bytes have been received -# # RANDOM3: slow-close-upstream to keep connection to server open after 1000-1500 milliseconds after being closed -# # RANDOM4: reset-peer-upstream after 50-200 seconds to force close the connection to the server -# proxy_toxics_randoms: "1000:3000|1000:3000|1000:1500|50:200" -# tasks: -# - name: network_tests - - name: ubuntu-valgrind display_name: "Ubuntu 22.04 x86_64 (Valgrind)" run_on: ubuntu2204-large @@ -1848,35 +1303,19 @@ buildvariants: - name: core_tests_group - name: macos-release - display_name: "MacOS 11.0 x86_64 (Release build)" - run_on: macos-1100 + display_name: "MacOS 13.0 arm64 (Release build)" + run_on: macos-13-arm64 expansions: cmake_url: "https://s3.amazonaws.com/static.realm.io/evergreen-assets/cmake-3.26.3-macos-universal.tar.gz" cmake_bindir: "./cmake_binaries/CMake.app/Contents/bin" cmake_generator: Xcode max_jobs: $(sysctl -n hw.logicalcpu) cmake_build_type: Release - xcode_developer_dir: /Applications/Xcode13.1.app/Contents/Developer + xcode_developer_dir: /Applications/Xcode.app/Contents/Developer extra_flags: -DREALM_ENABLE_ASSERTIONS=ON tasks: - - name: benchmarks - - name: compile_test - name: test-on-exfat -- name: macos-1100-arm64-release - display_name: "MacOS 11 arm64 (Release benchmarks)" - run_on: macos-1100-arm64 - expansions: - cmake_url: "https://s3.amazonaws.com/static.realm.io/evergreen-assets/cmake-3.26.3-macos-universal.tar.gz" - cmake_bindir: "./cmake_binaries/CMake.app/Contents/bin" - cmake_generator: Xcode - max_jobs: $(sysctl -n hw.logicalcpu) - cmake_build_type: Release - xcode_developer_dir: /Applications/Xcode13.1.app/Contents/Developer - extra_flags: -DREALM_ENABLE_ASSERTIONS=ON - tasks: - - name: benchmarks - - name: macos display_name: "MacOS 14 arm64" run_on: macos-14-arm64 @@ -1907,8 +1346,6 @@ buildvariants: extra_flags: -DCMAKE_SYSTEM_NAME=Darwin -DCMAKE_OSX_ARCHITECTURES=arm64 tasks: - name: compile_test_and_package - # benchmarks are disabled for now because of perf problems on AWS macos instances. - # - name: benchmarks - name: long-running-tests - name: swift-build-and-test @@ -2107,9 +1544,8 @@ buildvariants: expansions: cmake_bindir: "/cygdrive/c/Program Files/CMake/bin/" cmake_generator: "Visual Studio 16 2019" - extra_flags: "-A x64" cmake_build_type: "Debug" - extra_flags: -DCMAKE_SYSTEM_NAME=WindowsStore -DCMAKE_SYSTEM_VERSION=10.0 + extra_flags: -A x64 -DCMAKE_SYSTEM_NAME=WindowsStore -DCMAKE_SYSTEM_VERSION=10.0 max_jobs: $(($(grep -c proc /proc/cpuinfo) / 2)) fetch_missing_dependencies: On python3: "/cygdrive/c/python/python37/python.exe" diff --git a/evergreen/configure_baas_proxy.sh b/evergreen/configure_baas_proxy.sh deleted file mode 100755 index fa2cc5c30a..0000000000 --- a/evergreen/configure_baas_proxy.sh +++ /dev/null @@ -1,237 +0,0 @@ -#!/usr/bin/env bash -# The script to send and execute the configuration to set up the baas proxy toxics. -# -# Usage: -# ./evergreen/configure_baas_proxy.sh [-c PORT] [-r NUM] [-p PATH] [-v] [-h] CONFIG_JSON -# - -set -o errexit -set -o errtrace -set -o pipefail - -CONFIG_PORT=8474 -PROXY_NAME="baas_proxy" -RND_STRING= -RND_MIN= -RND_MAX= -RND_DIFF=32767 -CURL=/usr/bin/curl -VERBOSE= -CONFIG_TMP_DIR= - -function usage() -{ - echo "Usage: configure_baas_proxy.sh [-c PORT] [-r MIN:MAX] [-p PATH] [-t NAME] [-v] [-h] CONFIG_JSON" - echo -e "\tCONFIG_JSON\tPath to baas proxy toxics config file (one toxic config JSON object per line)" - echo "Options:" - echo -e "\t-c PORT\t\tLocal configuration port for proxy HTTP API (default ${CONFIG_PORT})" - echo -e "\t-r MIN:MAX\tString containing one or more sets of min:max values to replace %RANDOM#% in toxics" - echo -e "\t-p PATH\t\tPath to the curl executable (default ${CURL})" - echo -e "\t-t NAME\t\tName of the proxy to be configured (default ${PROXY_NAME})" - echo -e "\t-v\t\tEnable verbose script debugging" - echo -e "\t-h\t\tShow this usage summary and exit" - # Default to 0 if exit code not provided - exit "${1:0}" -} - -while getopts "c:r:p:vh" opt; do - case "${opt}" in - c) CONFIG_PORT="${OPTARG}";; - r) RND_STRING="${OPTARG}";; - p) CURL="${OPTARG}";; - v) VERBOSE="yes";; - h) usage 0;; - *) usage 1;; - esac -done - -TOXIPROXY_URL="http://localhost:${CONFIG_PORT}" - -shift $((OPTIND - 1)) - -if [[ $# -lt 1 ]]; then - echo "Error: Baas proxy toxics config file not provided" - usage 1 -fi -PROXY_JSON_FILE="${1}"; shift; - -if [[ -z "${PROXY_JSON_FILE}" ]]; then - echo "Error: Baas proxy toxics config file value was empty" - usage 1 -elif [[ ! -f "${PROXY_JSON_FILE}" ]]; then - echo "Error: Baas proxy toxics config file not found: ${PROXY_JSON_FILE}" - usage 1 -fi - -if [[ -z "${CURL}" ]]; then - echo "Error: curl path is empty" - usage 1 -elif [[ ! -x "${CURL}" ]]; then - echo "Error: curl path is not valid: ${CURL}" - usage 1 -fi - -trap 'catch $? ${LINENO}' ERR -trap 'on_exit' INT TERM EXIT - -# Set up catch function that runs when an error occurs -function catch() -{ - # Usage: catch EXIT_CODE LINE_NUM - echo "${BASH_SOURCE[0]}: $2: Error $1 occurred while configuring baas proxy" -} - -function on_exit() -{ - # Usage: on_exit - if [[ -n "${CONFIG_TMP_DIR}" && -d "${CONFIG_TMP_DIR}" ]]; then - rm -rf "${CONFIG_TMP_DIR}" - fi -} - -function check_port() -{ - # Usage check_port PORT - port_num="${1}" - if [[ -n "${port_num}" && ${port_num} -gt 0 && ${port_num} -lt 65536 ]]; then - return 0 - fi - return 1 -} - -function check_port_ready() -{ - # Usage: check_port_active PORT PORT_NAME - port_num="${1}" - port_check=$(lsof -P "-i:${port_num}" | grep "LISTEN" || true) - if [[ -z "${port_check}" ]]; then - echo "Error: ${2} port (${port_num}) is not ready - is the Baas proxy running?" - exit 1 - fi - if ! curl "${TOXIPROXY_URL}/version" --silent --fail --connect-timeout 10 > /dev/null; then - echo "Error: No response from ${2} (${port_num}) - is the Baas proxy running?" - exit 1 - fi -} - -function parse_random() -{ - # Usage: parse_random RANDOM_STRING => RND_MIN, RND_MAX - random_string="${1}" - old_ifs="${IFS}" - - RND_MIN=() - RND_MAX=() - - if [[ "${random_string}" =~ .*|.* ]]; then - IFS='|' - read -ra random_list <<< "${random_string}" - else - random_list=("${random_string}") - fi - for random in "${random_list[@]}" - do - if [[ ! "${random}" =~ .*:.* ]]; then - IFS="${old_ifs}" - return 1 - fi - - # Setting IFS (input field separator) value as ":" and read the split string into array - IFS=':' - read -ra rnd_arr <<< "${random}" - - if [[ ${#rnd_arr[@]} -ne 2 ]]; then - IFS="${old_ifs}" - return 1 - elif [[ -z "${rnd_arr[0]}" || -z "${rnd_arr[0]}" ]]; then - IFS="${old_ifs}" - return 1 - fi - - if [[ ${rnd_arr[0]} -le ${rnd_arr[1]} ]]; then - RND_MIN+=("${rnd_arr[0]}") - RND_MAX+=("${rnd_arr[1]}") - else - RND_MIN+=("${rnd_arr[1]}") - RND_MAX+=("${rnd_arr[0]}") - fi - done - IFS="${old_ifs}" - return 0 -} - -function generate_random() -{ - # Usage: generate_random MINVAL MAXVAL => RAND_VAL - minval="${1}" - maxval="${2}" - diff=$(( "${maxval}" - "${minval}" )) - if [[ ${diff} -gt ${RND_DIFF} ]]; then - return 1 - fi - RAND_VAL=$(( "$minval" + $(("$RANDOM" % "$diff")) )) -} - -# Wait until after the functions are configured before enabling verbose tracing -if [[ -n "${VERBOSE}" ]]; then - set -o verbose - set -o xtrace -fi - -if ! check_port "${CONFIG_PORT}"; then - echo "Error: Baas proxy HTTP API config port was invalid: '${CONFIG_PORT}'" - usage 1 -fi - -# Parse and verify the random string, if provided -if [[ -n "${RND_STRING}" ]]; then - if ! parse_random "${RND_STRING}"; then - echo "Error: Malformed random string: ${random_string} - format 'MIN:MAX[|MIN:MAX[|...]]" - usage 1 - fi -fi - -# Verify the Baas proxy is ready to roll -check_port_ready "${CONFIG_PORT}" "Baas proxy HTTP API config" - -# Create a temp directory for constructing the updated config file -CONFIG_TMP_DIR=$(mktemp -d -t "proxy-config.XXXXXX") -cp "${PROXY_JSON_FILE}" "${CONFIG_TMP_DIR}" -json_file="$(basename "${PROXY_JSON_FILE}")" -TMP_CONFIG="${CONFIG_TMP_DIR}/${json_file}" - -if [[ ${#RND_MIN[@]} -gt 0 ]]; then - cnt=0 - while [[ cnt -lt ${#RND_MIN[@]} ]]; do - rndmin=${RND_MIN[cnt]} - rndmax=${RND_MAX[cnt]} - if ! generate_random "${rndmin}" "${rndmax}"; then - echo "Error: MAX - MIN cannot be more than ${RND_DIFF}" - exit 1 - fi - - cnt=$((cnt + 1)) - printf "Generated random value #%d from %d to %d: %d\n" "${cnt}" "${rndmin}" "${rndmax}" "${RAND_VAL}" - sed_pattern=$(printf "s/%%RANDOM%d%%/%d/g" "${cnt}" "${RAND_VAL}") - sed -i.bak "${sed_pattern}" "${TMP_CONFIG}" - done -fi - -# Get the current list of configured toxics for the baas_proxy proxy -TOXICS=$(${CURL} --silent "${TOXIPROXY_URL}/proxies/${PROXY_NAME}/toxics") -if [[ "${TOXICS}" != "[]" ]]; then - # Extract the toxic names from the toxics list JSON - # Steps: Remove brackets, split into lines, extract "name" value - mapfile -t TOXIC_LIST < <(echo "${TOXICS}" | sed 's/\[\(.*\)\]/\1/g' | sed 's/},{/}\n{/g' | sed 's/.*"name":"\([^"]*\).*/\1/g') - echo "Clearing existing set of toxics (${#TOXIC_LIST[@]}) for ${PROXY_NAME} proxy" - for toxic in "${TOXIC_LIST[@]}" - do - ${CURL} -X DELETE "${TOXIPROXY_URL}/proxies/${PROXY_NAME}/toxics/${toxic}" - done -fi - -# Configure the new set of toxics for the baas_proxy proxy -echo "Configuring toxics for ${PROXY_NAME} proxy with file: ${json_file}" -while IFS= read -r line; do - ${CURL} -X POST -H "Content-Type: application/json" --silent -d "${line}" "${TOXIPROXY_URL}/proxies/${PROXY_NAME}/toxics" > /dev/null -done < "${TMP_CONFIG}" diff --git a/evergreen/proxy-network-faults.toxics b/evergreen/proxy-network-faults.toxics deleted file mode 100644 index 8168089606..0000000000 --- a/evergreen/proxy-network-faults.toxics +++ /dev/null @@ -1,4 +0,0 @@ -{"name": "limit-data-upstream","type": "limit_data","stream": "upstream","toxicity": 0.6,"attributes": {"bytes": %RANDOM1%}} -{"name": "limit-data-downstream","type": "limit_data","stream": "downstream","toxicity": 0.6,"attributes": {"bytes": %RANDOM2%}} -{"name": "slow-close-upstream","type": "slow_close","stream": "upstream","toxicity": 0.6,"attributes": {"delay": %RANDOM3%}} -{"name": "reset-peer-upstream","type": "reset_peer","stream": "upstream","toxicity": 0.6,"attributes": {"timeout": %RANDOM4%}} diff --git a/evergreen/proxy-nonideal-transfer.toxics b/evergreen/proxy-nonideal-transfer.toxics deleted file mode 100644 index c0ff2d4b9b..0000000000 --- a/evergreen/proxy-nonideal-transfer.toxics +++ /dev/null @@ -1,5 +0,0 @@ -{"name": "latency-upstream","type": "latency","stream": "upstream","toxicity": 0.5,"attributes": {"latency": 250,"jitter": 250}} -{"name": "latency-downstream","type": "latency","stream": "downstream","toxicity": 0.5,"attributes": {"latency": 0,"jitter": 250}} -{"name": "bandwidth-upstream","type": "bandwidth","stream": "upstream","toxicity": 0.5,"attributes": {"rate": %RANDOM1%}} -{"name": "bandwidth-downstream","type": "bandwidth","stream": "downstream","toxicity": 0.5,"attributes": {"rate": %RANDOM2%}} -{"name": "slicer-downstream","type": "slicer","stream": "downstream","toxicity": 0.5,"attributes": {"average_size": 500,"size_variation": 250,"delay": 10000}} diff --git a/evergreen/setup_baas_host.sh b/evergreen/setup_baas_host.sh deleted file mode 100755 index e1d850b5e2..0000000000 --- a/evergreen/setup_baas_host.sh +++ /dev/null @@ -1,279 +0,0 @@ -#!/usr/bin/env bash -# The script to be run on the ubuntu host that will run baas for the evergreen windows tests -# -# Usage: -# ./evergreen/setup_baas_host.sh [-b BRANCH] [-d PATH] [-t PORT] [-v] [-h] HOST_VARS -# - -set -o errexit -set -o errtrace -set -o pipefail - -trap 'catch $? ${LINENO}' ERR -trap "exit" INT TERM - -# Set up catch function that runs when an error occurs -function catch() -{ - # Usage: catch EXIT_CODE LINE_NUM - echo "${BASH_SOURCE[0]}: $2: Error $1 occurred while starting remote baas" -} - -function usage() -{ - # Usage: usage [EXIT_CODE] - echo "Usage: setup_baas_host.sh [-b BRANCH] [-d PATH] [-t PORT] [-v] [-h] HOST_VARS" - echo -e "\tHOST_VARS\tPath to baas host vars script file" - echo "Options:" - echo -e "\t-b BRANCH\tOptional branch or git spec of baas to checkout/build" - echo -e "\t-d PATH\t\tSkip setting up the data device and use alternate data path" - echo -e "\t-v\t\tEnable verbose script debugging" - echo -e "\t-h\t\tShow this usage summary and exit" - echo "ToxiProxy Options:" - echo -e "\t-t PORT\t\tEnable Toxiproxy support (proxy between baas on :9090 and PORT)" - # Default to 0 if exit code not provided - exit "${1:0}" -} - -BAAS_BRANCH= -OPT_DATA_DIR= -PROXY_PORT= -VERBOSE= - -while getopts "b:d:t:vh" opt; do - case "${opt}" in - b) BAAS_BRANCH="${OPTARG}";; - d) if [[ -z "${OPTARG}" ]]; then - echo "Error: Alternate data directory was empty" - usage 1 - fi; OPT_DATA_DIR="${OPTARG}";; - t) if [[ -z "${OPTARG}" ]]; then - echo "Error: Baas proxy port was empty"; - usage 1 - fi; PROXY_PORT="${OPTARG}";; - v) VERBOSE="yes";; - h) usage 0;; - *) usage 1;; - esac -done - -shift $((OPTIND - 1)) - -if [[ $# -lt 1 ]]; then - echo "Error: Baas host vars script not provided" - usage 1 -fi - -BAAS_HOST_VARS="${1}"; shift; - -if [[ -z "${BAAS_HOST_VARS}" ]]; then - echo "Error: Baas host vars script value was empty" - usage 1 -elif [[ ! -f "${BAAS_HOST_VARS}" ]]; then - echo "Error: Baas host vars script not found: ${BAAS_HOST_VARS}" - usage 1 -fi - -# shellcheck disable=SC1090 -source "${BAAS_HOST_VARS}" - -if [[ -z "${AWS_ACCESS_KEY_ID}" ]]; then - echo "Error: AWS_ACCESS_KEY_ID was not provided by baas host vars script" - exit 1 -fi - -if [[ -z "${AWS_SECRET_ACCESS_KEY}" ]]; then - echo "Error: AWS_SECRET_ACCESS_KEY was not provided by baas host vars script" - exit 1 -fi - -if [[ -z "${GITHUB_KNOWN_HOSTS}" ]]; then - # Use a default if not defined, but this may become outdated one day... - GITHUB_KNOWN_HOSTS="github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=" - echo "Info: GITHUB_KNOWN_HOSTS not defined in baas host vars script - using default" -fi -KNOWN_HOSTS_FILE="${HOME}/.ssh/known_hosts" -if [[ -f "${KNOWN_HOSTS_FILE}" ]] && grep "${GITHUB_KNOWN_HOSTS}" < "${KNOWN_HOSTS_FILE}"; then - echo "Github known hosts entry found - skipping known_hosts update" -else - echo "${GITHUB_KNOWN_HOSTS}" | tee -a "${KNOWN_HOSTS_FILE}" -fi - -function init_data_device() -{ - #Usage: init_data_device - data_device= - - # Find /data ebs device to be mounted - devices=$(sudo lsblk | grep disk | awk '{print $1}') - for device in ${devices}; do - is_data=$(sudo file -s "/dev/${device}" | awk '{print $2}') - if [[ "${is_data}" == "data" ]]; then - data_device="/dev/${device}" - fi - done - - # If a data device was discovered, set up the device - if [[ -n "${data_device}" ]]; then - sudo umount /mnt || true - sudo umount "${data_device}" || true - sudo /sbin/mkfs.xfs -f "${data_device}" - sudo mkdir -p "${DATA_DIR}" - # get uuid of data device - data_uuid=$(sudo blkid | grep "${data_device}" | awk '{print $2}') - echo "Found data device: ${data_device}(${data_uuid})" - echo "${data_uuid} ${DATA_DIR} auto noatime 0 0" | sudo tee -a /etc/fstab - sudo mount "${DATA_DIR}" - echo "Successfully mounted ${data_device} to ${DATA_DIR}" - else - # Otherwise, create a local /data dir - sudo mkdir -p "${DATA_DIR}" - fi - - sudo chmod 777 "${DATA_DIR}" -} - -function setup_data_dir() -{ - # Usage: setup_data_dir - # Data directory is expected to be set in DATA_DIR variable - # Delete /data/baas-remote/ dir if is already exists - [[ -d "${BAAS_REMOTE_DIR}" ]] && sudo rm -rf "${BAAS_REMOTE_DIR}" - - # Create the baseline baas remote directories and set perms - DIR_PERMS="$(id -u):$(id -g)" - echo "Creating and setting ${BAAS_REMOTE_DIR} to '${DIR_PERMS}'" - mkdir -p "${BAAS_REMOTE_DIR}" - chown -R "${DIR_PERMS}" "${BAAS_REMOTE_DIR}" - chmod -R 755 "${BAAS_REMOTE_DIR}" - mkdir -p "${BAAS_WORK_DIR}" - chmod -R 755 "${BAAS_WORK_DIR}" - - # Set up the temp directory - it may already exist on evergreen spawn hosts - if [[ -d "${DATA_TEMP_DIR}" ]]; then - sudo chmod 1777 "${DATA_TEMP_DIR}" - else - mkdir -p "${DATA_TEMP_DIR}" - chmod 1777 "${DATA_TEMP_DIR}" - fi - export TMPDIR="${DATA_TEMP_DIR}" -} - -function on_exit() -{ - # Usage: on_exit - baas_pid= - proxy_pid= - if [[ -f "${PROXY_PID_FILE}" ]]; then - proxy_pid="$(< "${PROXY_PID_FILE}")" - fi - - if [[ -f "${SERVER_PID_FILE}" ]]; then - baas_pid="$(< "${SERVER_PID_FILE}")" - fi - - if [[ -n "${proxy_pid}" ]]; then - echo "Stopping baas proxy ${proxy_pid}" - kill "${proxy_pid}" || true - rm -f "${PROXY_PID_FILE}" || true - fi - - if [[ -n "${baas_pid}" ]]; then - echo "Stopping baas server ${baas_pid}" - kill "${baas_pid}" || true - rm -f "${SERVER_PID_FILE}" || true - fi - - echo "Waiting for processes to exit" - wait -} - -function start_baas_proxy() -{ - # Usage: start_baas_proxy PORT - listen_port="${1}" - # Delete the toxiproxy working directory if it currently exists - if [[ -n "${BAAS_PROXY_DIR}" && -d "${BAAS_PROXY_DIR}" ]]; then - rm -rf "${BAAS_PROXY_DIR}" - fi - - if [[ -f "${HOME}/setup_baas_proxy.sh" ]]; then - cp "${HOME}/setup_baas_proxy.sh" evergreen/ - fi - - proxy_options=("-w" "${BAAS_PROXY_DIR}" "-s" "${BAAS_WORK_DIR}" "-p" "${listen_port}") - if [[ -n "${VERBOSE}" ]]; then - proxy_options=("-v") - fi - - # Pass the baas work directory to the toxiproxy script for the go executable - echo "Staring baas proxy with listen port :${listen_port}" - ./evergreen/setup_baas_proxy.sh "${proxy_options[@]}" 2>&1 & - echo $! > "${PROXY_PID_FILE}" -} - - -# Wait until after the BAAS_HOST_VARS file is loaded to enable verbose tracing -if [[ -n "${VERBOSE}" ]]; then - set -o verbose - set -o xtrace -fi - -sudo chmod 600 "${HOME}/.ssh"/* - -# Should an alternate data directory location be used? If so, don't init the data device -if [[ -z "${OPT_DATA_DIR}" ]]; then - DATA_DIR=/data - init_data_device -else - DATA_DIR="${OPT_DATA_DIR}" -fi - -DATA_TEMP_DIR="${DATA_DIR}/tmp" -BAAS_REMOTE_DIR="${DATA_DIR}/baas-remote" -BAAS_WORK_DIR="${BAAS_REMOTE_DIR}/baas-work-dir" -SERVER_PID_FILE="${BAAS_REMOTE_DIR}/baas-server.pid" -BAAS_STOPPED_FILE="${BAAS_WORK_DIR}/baas_stopped" - -BAAS_PROXY_DIR="${BAAS_REMOTE_DIR}/baas-proxy-dir" -PROXY_PID_FILE="${BAAS_REMOTE_DIR}/baas-proxy.pid" -PROXY_STOPPED_FILE="${BAAS_PROXY_DIR}/baas_proxy_stopped" - -setup_data_dir - -pushd "${BAAS_REMOTE_DIR}" > /dev/null - -if [[ -d "${HOME}/remote-baas/evergreen/" ]]; then - cp -R "${HOME}/remote-baas/evergreen/" ./evergreen/ -else - echo "remote-baas/evergreen/ directory not found in ${HOME}" - exit 1 -fi - -# Set up the cleanup function that runs at exit and stops baas server and proxy (if run) -trap 'on_exit' EXIT - -BAAS_OPTIONS=() -if [[ -n "${BAAS_BRANCH}" ]]; then - BAAS_OPTIONS=("-b" "${BAAS_BRANCH}") -fi -if [[ -n "${VERBOSE}" ]]; then - BAAS_OPTIONS+=("-v") -fi - -echo "Staring baas server..." -./evergreen/install_baas.sh "${BAAS_OPTIONS[@]}" -w "${BAAS_WORK_DIR}" 2>&1 & -echo $! > "${SERVER_PID_FILE}" - -if [[ -n "${PROXY_PORT}" ]]; then - start_baas_proxy "${PROXY_PORT}" -fi - -# Turn off verbose logging since it's so noisy -set +o verbose -set +o xtrace -until [[ -f "${BAAS_STOPPED_FILE}" || -f "${PROXY_STOPPED_FILE}" ]]; do - sleep 1 -done - -popd > /dev/null # /data/baas-remote diff --git a/evergreen/setup_baas_host_local.sh b/evergreen/setup_baas_host_local.sh deleted file mode 100755 index d1cc4e907e..0000000000 --- a/evergreen/setup_baas_host_local.sh +++ /dev/null @@ -1,269 +0,0 @@ -#!/usr/bin/env bash -# The script to be run on the ubuntu host that will run baas for the evergreen windows tests -# -# Usage: -# ./evergreen/setup_baas_host_local.sh [-w PATH] [-u USER] [-b BRANCH] [-v] [-h] [-t] [-d PORT] [-l PORT] [-c PORT] HOST_VARS SSH_KEY -# - -set -o errexit -set -o errtrace -set -o pipefail - -EVERGREEN_PATH=./evergreen -BAAS_WORK_PATH=./baas-work-dir -BAAS_HOST_NAME= -BAAS_USER=ubuntu -BAAS_BRANCH= -VERBOSE= -BAAS_PROXY= -DIRECT_PORT=9098 -LISTEN_PORT=9092 -CONFIG_PORT=8474 -BAAS_PORT=9090 -BAAS_HOST_KEY= - -function usage() -{ - echo "Usage: setup_baas_host_local.sh [-w PATH] [-u USER] [-b BRANCH] [-v] [-h] [-t] [-d PORT] [-l PORT] [-c PORT] [-i SSH_KEY] HOST_VARS" - echo -e "\tHOST_VARS\tPath to baas host vars script file" - echo -e "\t -i SSH_KEY\t\tPath to baas host private key file" - echo "Options:" - echo -e "\t-w PATH\t\tPath to local baas server working directory (default ${BAAS_WORK_PATH})" - echo -e "\t-u USER\t\tUsername to connect to baas host (default ${BAAS_USER})" - echo -e "\t-b BRANCH\tOptional branch or git spec of baas to checkout/build" - echo -e "\t-v\t\tEnable verbose script debugging" - echo -e "\t-h\t\tShow this usage summary and exit" - echo "Baas Proxy Options:" - echo -e "\t-t\t\tEnable baas proxy support (proxy between baas on :9090 and listen port)" - echo -e "\t-d PORT\t\tPort for direct connection to baas - skips proxy (default ${DIRECT_PORT})" - echo -e "\t-l PORT\t\tBaas proxy listen port on remote host (default ${LISTEN_PORT})" - echo -e "\t-c PORT\t\tLocal configuration port for proxy HTTP API (default ${CONFIG_PORT})" - echo "Note: This script must be run from a cloned realm-core/ repository directory." - # Default to 0 if exit code not provided - exit "${1:0}" -} - -while getopts "w:u:b:ta:d:l:c:vhi:" opt; do - case "${opt}" in - w) BAAS_WORK_PATH="${OPTARG}";; - u) BAAS_USER="${OPTARG}";; - b) BAAS_BRANCH="${OPTARG}";; - t) BAAS_PROXY="yes";; - d) DIRECT_PORT="${OPTARG}";; - l) LISTEN_PORT="${OPTARG}";; - c) CONFIG_PORT="${OPTARG}";; - i) BAAS_HOST_KEY="${OPTARG}";; - v) VERBOSE="yes";; - h) usage 0;; - *) usage 1;; - esac -done - -shift $((OPTIND - 1)) - -if [[ $# -lt 1 ]]; then - echo "Error: Baas host vars script not provided" - usage 1 -fi -BAAS_HOST_VARS="${1}"; shift; - -if [[ -z "${BAAS_HOST_VARS}" ]]; then - echo "Error: Baas host vars script value was empty" - usage 1 -elif [[ ! -f "${BAAS_HOST_VARS}" ]]; then - echo "Error: Baas host vars script not found: ${BAAS_HOST_VARS}" - usage 1 -fi - -if [[ -n "${BAAS_HOST_KEY}" && ! -f "${BAAS_HOST_KEY}" ]]; then - echo "Error: Baas host private key not found: ${BAAS_HOST_KEY}" - usage 1 -fi - -if [[ "${BAAS_USER}" = "root" ]]; then - FILE_DEST_DIR="/root/remote-baas" -else - FILE_DEST_DIR="/home/${BAAS_USER}/remote-baas" -fi -EVERGREEN_DEST_DIR="${FILE_DEST_DIR}/evergreen" - -function check_port() -{ - # Usage check_port PORT - port_num="${1}" - if [[ -n "${port_num}" && ${port_num} -gt 0 && ${port_num} -lt 65536 ]]; then - return 0 - fi - return 1 -} - -function check_port_in_use() -{ - # Usage: check_port_in_use PORT PORT_NAME - port_num="${1}" - port_check=$(lsof -P "-i:${port_num}" | grep "LISTEN" || true) - if [[ -n "${port_check}" ]]; then - echo "Error: ${2} port (${port_num}) is already in use" - echo -e "${port_check}" - exit 1 - fi -} - -# Check the local baas port availability -check_port_in_use "${BAAS_PORT}" "Local baas server" - -# Check the port values and local ports in use for baas proxy -if [[ -n "${BAAS_PROXY}" ]]; then - if ! check_port "${CONFIG_PORT}"; then - echo "Error: Baas proxy local HTTP API config port was invalid: '${CONFIG_PORT}'" - usage 1 - elif ! check_port "${LISTEN_PORT}"; then - echo "Error: Baas proxy listen port was invalid: '${LISTEN_PORT}'" - usage 1 - fi - check_port_in_use "${CONFIG_PORT}" "Local baas proxy config" - - if [[ -n "${DIRECT_PORT}" ]]; then - if ! check_port "${DIRECT_PORT}"; then - echo "Error: Baas direct connect port was invalid: '${DIRECT_PORT}'" - usage 1 - fi - check_port_in_use "${DIRECT_PORT}" "Local baas server direct connect" - fi -fi - -trap 'catch $? ${LINENO}' ERR -trap 'on_exit' INT TERM EXIT - -# Set up catch function that runs when an error occurs -function catch() -{ - # Usage: catch EXIT_CODE LINE_NUM - echo "${BASH_SOURCE[0]}: $2: Error $1 occurred while starting baas (local)" -} - -function on_exit() -{ - # Usage: on_exit - if [[ -n "${BAAS_WORK_PATH}" ]]; then - # Create the baas_stopped file so wait_for_baas can exit early - [[ -d "${BAAS_WORK_PATH}" ]] || mkdir -p "${BAAS_WORK_PATH}" - touch "${BAAS_WORK_PATH}/baas_stopped" - fi -} - -# shellcheck disable=SC1090 -source "${BAAS_HOST_VARS}" - -# Wait until after the BAAS_HOST_VARS file is loaded to enable verbose tracing -if [[ -n "${VERBOSE}" ]]; then - set -o verbose - set -o xtrace -fi - -if [[ -z "${BAAS_HOST_NAME}" ]]; then - echo "Baas remote hostname (BAAS_HOST_NAME) not provided in baas host vars script" - usage 1 -fi - -if [[ -z "${BAAS_USER}" ]]; then - echo "Error: Baas host username was empty" - usage 1 -fi - -if [[ ! -d "${EVERGREEN_PATH}/" ]]; then - echo "This script must be run from the realm-core directory for accessing files in '${EVERGREEN_PATH}/'" - exit 1 -fi - -SSH_USER="$(printf "%s@%s" "${BAAS_USER}" "${BAAS_HOST_NAME}")" - -SSH_OPTIONS=(-o ForwardAgent=yes -o StrictHostKeyChecking=no ) -if [[ -n "${BAAS_HOST_KEY}" ]]; then - ssh-agent > ssh_agent_commands.sh - - # shellcheck disable=SC1091 - source ssh_agent_commands.sh - - if [[ -f ~/.ssh/id_rsa ]]; then - ssh-add ~/.ssh/id_rsa - fi - - ssh-add "${BAAS_HOST_KEY}" - SSH_OPTIONS+=(-o IdentitiesOnly=yes -i "${BAAS_HOST_KEY}") -fi - -echo "running ssh with ${SSH_OPTIONS[*]}" - -RETRY_COUNT=25 -WAIT_COUNTER=0 -WAIT_START=$(date -u +'%s') -CONNECT_COUNT=2 - -# Check for remote connectivity - try to connect twice to verify server is "really" ready -# The tests failed one time due to this ssh command passing, but the next scp command failed -while [[ ${CONNECT_COUNT} -gt 0 ]]; do - until ssh "${SSH_OPTIONS[@]}" -o ConnectTimeout=10 "${SSH_USER}" "mkdir -p ${EVERGREEN_DEST_DIR} && echo -n 'hello from '; hostname" ; do - if [[ ${WAIT_COUNTER} -ge ${RETRY_COUNT} ]] ; then - secs_spent_waiting=$(($(date -u +'%s') - WAIT_START)) - echo "Timed out after waiting ${secs_spent_waiting} seconds for host ${BAAS_HOST_NAME} to start" - exit 1 - fi - - ((++WAIT_COUNTER)) - printf "SSH connection attempt %d/%d failed. Retrying...\n" "${WAIT_COUNTER}" "${RETRY_COUNT}" - sleep 10 - done - - ((CONNECT_COUNT--)) -done - -echo "Transferring setup scripts to ${SSH_USER}:${FILE_DEST_DIR}" -# Copy the baas host vars script to the baas remote host -scp "${SSH_OPTIONS[@]}" -o ConnectTimeout=60 "${BAAS_HOST_VARS}" "${SSH_USER}:${FILE_DEST_DIR}/" -# Copy the entire evergreen/ directory from the working copy of realm-core to the remote host -# This ensures the remote host the latest copy, esp when running evergreen patches -# dependencies.yml contains the BAAS_VERSION to use -echo "Transferring evergreen scripts to ${SSH_USER}:${FILE_DEST_DIR}" -cp "${EVERGREEN_PATH}/../dependencies.yml" "${EVERGREEN_PATH}/" -scp -r "${SSH_OPTIONS[@]}" -o ConnectTimeout=60 "${EVERGREEN_PATH}/" "${SSH_USER}:${FILE_DEST_DIR}/" - -BAAS_TUNNELS=() -SETUP_OPTIONS=() - -if [[ -n "${VERBOSE}" ]]; then - SETUP_OPTIONS+=("-v") -fi - -if [[ -n "${BAAS_PROXY}" ]]; then - # Add extra tunnel for baas proxy HTTP API config interface and direct connection to baas - BAAS_TUNNELS+=("-L" "${CONFIG_PORT}:127.0.0.1:8474") - if [[ -n "${DIRECT_PORT}" ]]; then - BAAS_TUNNELS+=("-L" "${DIRECT_PORT}:127.0.0.1:9090") - fi - # Enable baas proxy and use LISTEN_PORT as the proxy listen port - SETUP_OPTIONS+=("-t" "${LISTEN_PORT}") -else - # Force remote port to 9090 if baas proxy is not used - connect directly to baas - LISTEN_PORT=9090 -fi - -BAAS_TUNNELS+=("-L" "9090:127.0.0.1:${LISTEN_PORT}") - -# Run the setup baas host script and provide the location of the baas host vars script -# Also sets up a forward tunnel for local port 9090 through the ssh connection to the baas remote host -# If baas proxy is enabled, a second forward tunnel is added for the HTTP API config interface -echo "Running setup script (with forward tunnel on :9090 to 127.0.0.1:${LISTEN_PORT})" -if [[ -n "${BAAS_BRANCH}" ]]; then - echo "- Starting remote baas with branch/commit: '${BAAS_BRANCH}'" - SETUP_OPTIONS+=("-b" "${BAAS_BRANCH}") -fi -if [[ -n "${BAAS_PROXY}" ]]; then - echo "- Baas proxy enabled - local HTTP API config port on :${CONFIG_PORT}" - if [[ -n "${DIRECT_PORT}" ]]; then - echo "- Baas direct connection on port :${DIRECT_PORT}" - fi -fi - -ssh -t "${SSH_OPTIONS[@]}" -o ConnectTimeout=60 "${BAAS_TUNNELS[@]}" "${SSH_USER}" \ - "${EVERGREEN_DEST_DIR}/setup_baas_host.sh" "${SETUP_OPTIONS[@]}" "${FILE_DEST_DIR}/baas_host_vars.sh" diff --git a/evergreen/setup_baas_proxy.sh b/evergreen/setup_baas_proxy.sh deleted file mode 100755 index 8a5d1ba444..0000000000 --- a/evergreen/setup_baas_proxy.sh +++ /dev/null @@ -1,263 +0,0 @@ -#!/usr/bin/env bash -# The script to download, build and run toxiproxy as a proxy to the baas server -# for simulating network error conditions for testing. -# -# Usage: -# ./evergreen/setup_baas_proxy.sh -w PATH [-p PORT] [-s PATH] [-b BRANCH] [-d] [-v] [-h] -# - -set -o errexit -set -o errtrace -set -o pipefail - -trap 'catch $? ${LINENO}' ERR -trap "exit" INT TERM - -function catch() -{ - echo "Error $1 occurred while starting baas proxy at line $2" -} - -WORK_PATH= -BAAS_PATH= -TOXIPROXY_VERSION="v2.5.0" -LISTEN_PORT=9092 -BAAS_PORT=9090 -SKIP_BAAS_WAIT= -CONFIG_PORT=8474 - -function usage() -{ - echo "Usage: setup_baas_proxy.sh -w PATH [-p PORT] [-s PATH] [-b BRANCH] [-d] [-v] [-h]" - echo -e "\t-w PATH\t\tPath to baas proxy working directory" - echo "Options:" - echo -e "\t-p PORT\t\tListen port for proxy connected to baas (default: ${LISTEN_PORT})" - echo -e "\t-s PATH\t\tOptional path to baas server working directory (for go binary)" - echo -e "\t-b BRANCH\tOptional branch or git spec to checkout/build (default: ${TOXIPROXY_VERSION})" - echo -e "\t-d\t\tDon't wait for baas to start before starting proxy" - echo -e "\t-v\t\tEnable verbose script debugging" - echo -e "\t-h\t\tShow this usage summary and exit" - # Default to 0 if exit code not provided - exit "${1:0}" -} - -BASE_PATH="$(cd "$(dirname "$0")"; pwd)" - -# Allow path to CURL to be overloaded by an environment variable -CURL="${CURL:=$LAUNCHER curl}" - -while getopts "w:p:s:b:dvh" opt; do - case "${opt}" in - w) WORK_PATH="${OPTARG}";; - p) LISTEN_PORT="${OPTARG}";; - s) BAAS_PATH="${OPTARG}";; - b) TOXIPROXY_VERSION="${OPTARG}";; - d) SKIP_BAAS_WAIT="yes";; - v) set -o verbose; set -o xtrace;; - h) usage 0;; - *) usage 1;; - esac -done - -if [[ -z "${WORK_PATH}" ]]; then - echo "Baas proxy work path was not provided" - usage 1 -fi -if [[ -z "${LISTEN_PORT}" ]]; then - echo "Baas proxy remote port was empty" - usage 1 -fi - -function check_port_in_use() -{ - # Usage: check_port_in_use PORT PORT_NAME - port_num="${1}" - port_check=$(lsof -P "-i:${port_num}" | grep "LISTEN" || true) - if [[ -n "${port_check}" ]]; then - echo "Error: ${2} port (${port_num}) is already in use" - echo -e "${port_check}" - exit 1 - fi -} - -# Check the baas proxy listen and Toxiproxy config port availability first -check_port_in_use "${LISTEN_PORT}" "baas proxy" -check_port_in_use "${CONFIG_PORT}" "Toxiproxy config" - -[[ -d "${WORK_PATH}" ]] || mkdir -p "${WORK_PATH}" -pushd "${WORK_PATH}" > /dev/null - -PROXY_CFG_FILE="${WORK_PATH}/baas_proxy.json" -PROXY_LOG="${WORK_PATH}/baas_proxy.log" -PROXY_PID_FILE="${WORK_PATH}/baas_proxy.pid" -PROXY_STOPPED_FILE="${WORK_PATH}/baas_proxy_stopped" -BAAS_STOPPED_FILE="${BAAS_PATH}/baas_stopped" - -# Remove some files from a previous run if they exist -if [[ -f "${CONFIG_FILE}" ]]; then - rm -f "${CONFIG_FILE}" -fi -if [[ -f "${PROXY_LOG}" ]]; then - rm -f "${PROXY_LOG}" -fi -if [[ -f "${PROXY_PID_FILE}" ]]; then - rm -f "${PROXY_PID_FILE}" -fi - -if [[ -f "${PROXY_STOPPED}" ]]; then - rm -f "${PROXY_STOPPED}" -fi - -# Set up the cleanup function that runs at exit and stops the toxiproxy server -trap 'on_exit' EXIT - -function on_exit() -{ - # Usage: on_exit - # Toxiproxy is being stopped (or never started), create a 'baas-proxy-stopped' file - touch "${PROXY_STOPPED_FILE}" || true - - proxy_pid= - if [[ -f "${PROXY_PID_FILE}" ]]; then - proxy_pid="$(< "${PROXY_PID_FILE}")" - fi - - if [[ -n "${proxy_pid}" ]]; then - echo "Stopping baas proxy ${proxy_pid}" - kill "${proxy_pid}" || true - echo "Waiting for baas proxy to stop" - wait - rm -f "${PROXY_PID_FILE}" || true - fi -} - -case $(uname -s) in - Darwin) - if [[ "$(uname -m)" == "arm64" ]]; then - export GOARCH=arm64 - GO_URL="https://s3.amazonaws.com/static.realm.io/evergreen-assets/go1.19.3.darwin-arm64.tar.gz" - # Go's scheduler is not BIG.little aware, and by default will spawn - # threads until they end up getting scheduled on efficiency cores, - # which is slower than just not using them. Limiting the threads to - # the number of performance cores results in them usually not - # running on efficiency cores. Checking the performance core count - # wasn't implemented until the first CPU with a performance core - # count other than 4 was released, so if it's unavailable it's 4. - GOMAXPROCS="$(sysctl -n hw.perflevel0.logicalcpu || echo 4)" - export GOMAXPROCS - else - export GOARCH=amd64 - GO_URL="https://s3.amazonaws.com/static.realm.io/evergreen-assets/go1.19.1.darwin-amd64.tar.gz" - fi - ;; - Linux) - GO_URL="https://s3.amazonaws.com/static.realm.io/evergreen-assets/go1.19.1.linux-amd64.tar.gz" - ;; -esac - -# Looking for go - first in the work path, then in the baas path (if provided), or -# download go into the work path -GOROOT= -# Was it found in the work path? -if [[ ! -x ${WORK_PATH}/go/bin/go ]]; then - # If the baas work path is set, check there first and wait - if [[ -n "${BAAS_PATH}" && -d "${BAAS_PATH}" ]]; then - WAIT_COUNTER=0 - RETRY_COUNT=10 - WAIT_START=$(date -u +'%s') - FOUND_GO="yes" - GO_ROOT_FILE="${BAAS_PATH}/go_root" - # Bass may be initializing at the same time, allow a bit of time for the two to sync - echo "Looking for go in baas work path for 20 secs in case both are starting concurrently" - until [[ -f "${GO_ROOT_FILE}" ]]; do - if [[ -n "${BAAS_STOPPED_FILE}" && -f "${BAAS_STOPPED_FILE}" ]]; then - echo "Error: Baas server failed to start (found baas_stopped file)" - exit 1 - fi - if [[ ${WAIT_COUNTER} -ge ${RETRY_COUNT} ]]; then - FOUND_GO= - secs_spent_waiting=$(($(date -u +'%s') - WAIT_START)) - echo "Error: Stopped after waiting ${secs_spent_waiting} seconds for baas go to become available" - break - fi - ((++WAIT_COUNTER)) - sleep 2 - done - if [[ -n "${FOUND_GO}" ]]; then - GOROOT="$(cat "${GO_ROOT_FILE}")" - echo "Found go in baas working directory: ${GOROOT}" - export GOROOT - fi - fi - - # If GOROOT is not set, then baas path was nor provided or go was not found - if [[ -z "${GOROOT}" ]]; then - # Download go since it wasn't found in the working directory - if [[ -z "${GO_URL}" ]]; then - echo "Error: go url not defined for current OS architecture" - uname -a - exit 1 - fi - echo "Downloading go to baas proxy working directory" - ${CURL} -sL "${GO_URL}" | tar -xz - # Set up the GOROOT for building/running baas - export GOROOT="${WORK_PATH}/go" - fi -else - echo "Found go in baas proxy working directory" - # Set up the GOROOT for building/running baas - export GOROOT="${WORK_PATH}/go" -fi -export PATH="${GOROOT}/bin":${PATH} -echo "Go version: $(go version)" - -if [[ ! -d "toxiproxy" ]]; then - git clone git@github.com:Shopify/toxiproxy.git toxiproxy -fi - -# Clone the baas repo and check out the specified version -if [[ ! -d "toxiproxy/.git" ]]; then - git clone git@github.com:Shopify/toxiproxy.git toxiproxy - pushd toxiproxy > /dev/null -else - pushd toxiproxy > /dev/null - git fetch -fi - -echo "Checking out Toxiproxy version '${TOXIPROXY_VERSION}'" -git checkout "${TOXIPROXY_VERSION}" -echo "Using Toxiproxy commit: $(git rev-parse HEAD)" - -# Build toxiproxy -make build - -if [[ -z "${SKIP_BAAS_WAIT}" ]]; then - # Wait for baas to start before starting Toxiproxy - OPT_WAIT_BAAS=() - if [[ -n "${BAAS_PATH}" ]]; then - OPT_WAIT_BAAS=("-w" "{$BAAS_PATH}") - fi - - "${BASE_PATH}/wait_for_baas.sh" "${OPT_WAIT_BAAS[@]}" -fi - -cat >"${PROXY_CFG_FILE}" < 127.0.0.1:${BAAS_PORT}" -./dist/toxiproxy-server -config "${PROXY_CFG_FILE}" > "${PROXY_LOG}" 2>&1 & -echo $! > "${PROXY_PID_FILE}" - -echo "---------------------------------------------" -echo "Baas proxy ready" -echo "---------------------------------------------" -wait - -popd > /dev/null # toxiproxy -popd > /dev/null # / diff --git a/evergreen/wait_for_baas.sh b/evergreen/wait_for_baas.sh deleted file mode 100755 index 508ec310ec..0000000000 --- a/evergreen/wait_for_baas.sh +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env bash -# The script to wait up to approx 600 seconds (default) for the baas server to -# start, which is indicated by being able to successfully "curl" the baas -# server endpoint. If a pid file is provided, it will be used to verify the -# baas server is still running. The retry count specifies the number of -# attempts to query the server endpoint, with a 5 sec wait between attempts. -# If a server log path is provided, the last entries will be printed to stdout -# using the 'tail' command after each attempt. -# -# Usage: -# ./evergreen/wait_for_baas.sh [-w PATH] [-p FILE] [-r COUNT] [-l FILE] [-s] [-v] [-h] -# - -set -o errexit -set -o pipefail - -CURL=${CURL:=curl} -BAAS_PID_FILE= -BAAS_STOPPED_FILE= -RETRY_COUNT=120 -BAAS_SERVER_LOG= -STATUS_OUT= - -function usage() -{ - echo "Usage: wait_for_baas.sh [-w PATH] [-p FILE] [-r COUNT] [-l FILE] [-s] [-v] [-h]" - echo "Options:" - echo -e "\t-w PATH\t\tPath to baas server working directory" - echo -e "\t-p FILE\t\tPath to baas server pid file (also set by -w option)" - echo -e "\t-r COUNT\tNumber of attempts to check for baas server (default 120)" - echo -e "\t-l FILE\t\tPath to baas server log file (also set by -w option)" - echo -e "\t-s\t\tDisplay a status for each attempt" - echo -e "\t-v\t\tEnable verbose script debugging" - echo -e "\t-h\t\tShow this usage summary and exit" - # Default to 0 if exit code not provided - exit "${1:0}" -} - -function update_paths() -{ - if [[ -n "${1}" ]]; then - BAAS_SERVER_LOG="${1}/baas_server.log" - BAAS_STOPPED_FILE="${1}/baas_stopped" - BAAS_PID_FILE="${1}/baas_server.pid" - fi -} - -while getopts "w:p:r:l:svh" opt; do - case "${opt}" in - w) update_paths "${OPTARG}";; - p) BAAS_PID_FILE="${OPTARG}";; - r) RETRY_COUNT="${OPTARG}";; - l) BAAS_SERVER_LOG="${OPTARG}";; - s) STATUS_OUT="yes";; - v) set -o verbose; set -o xtrace;; - h) usage 0;; - *) usage 1;; - esac -done - -WAIT_COUNTER=0 -WAIT_START=$(date -u +'%s') - -function output_log_tail() -{ - if [[ -n "${BAAS_SERVER_LOG}" && -f "${BAAS_SERVER_LOG}" ]]; then - tail -n 10 "${BAAS_SERVER_LOG}" - fi -} - -echo "Waiting for baas server to start..." -until $CURL --output /dev/null --head --fail http://localhost:9090 --silent ; do - if [[ -n "${BAAS_STOPPED_FILE}" && -f "${BAAS_STOPPED_FILE}" ]]; then - echo "Baas server failed to start (found baas_stopped file)" - output_log_tail - exit 1 - fi - - if [[ -n "${BAAS_PID_FILE}" && -f "${BAAS_PID_FILE}" ]]; then - if ! pgrep -F "${BAAS_PID_FILE}" > /dev/null; then - echo "Baas server $(< "${BAAS_PID_FILE}") is no longer running" - output_log_tail - exit 1 - fi - fi - - ((++WAIT_COUNTER)) - secs_spent_waiting=$(($(date -u +'%s') - WAIT_START)) - if [[ ${WAIT_COUNTER} -ge ${RETRY_COUNT} ]]; then - echo "Timed out after ${secs_spent_waiting} secs waiting for baas server to start" - output_log_tail - exit 1 - fi - - if [[ -n "${STATUS_OUT}" ]]; then - echo "Waiting for baas server to start... ${secs_spent_waiting} secs so far" - fi - - sleep 5 -done diff --git a/evergreen/wait_for_remote_baas.sh b/evergreen/wait_for_remote_baas.sh deleted file mode 100755 index 3eaf3da591..0000000000 --- a/evergreen/wait_for_remote_baas.sh +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/env bash -# Wait for baas to be setup on a remote host - -set -o errexit -set -o pipefail - -EVERGREEN_PATH=./evergreen -BAAS_WORK_PATH=./baas-work-dir -BAAS_HOST_NAME= -BAAS_USER=ubuntu -VERBOSE= -BAAS_HOST_KEY= - -function usage() -{ - echo "Usage: wait_for_remote_baas.sh [-v] [-h] [-i SSH_KEY] HOST_VARS" - echo -e "\tHOST_VARS\tPath to baas host vars script file" - echo -e "\t-i SSH_KEY\t\tPath to baas host private key file" - echo "Options:" - echo -e "\t-v\t\tEnable verbose script debugging" - echo -e "\t-h\t\tShow this usage summary and exit" - echo "If an SSH_KEY is not provided, the script will assume an ssh agent is already running with" - echo "an appropriate key" - exit "${1:0}" -} - -while getopts "vhi:" opt; do - case "${opt}" in - v) VERBOSE="yes";; - i) BAAS_HOST_KEY="${OPTARG}";; - h) usage 0;; - *) usage 1;; - esac -done - -shift $((OPTIND - 1)) - -if [[ $# -lt 1 ]]; then - echo "Error: Baas host vars script not provided" - usage 1 -fi -BAAS_HOST_VARS="${1}"; shift; - -if [[ -z "${BAAS_HOST_VARS}" ]]; then - echo "Error: Baas host vars script value was empty" - usage 1 -elif [[ ! -f "${BAAS_HOST_VARS}" ]]; then - echo "Error: Baas host vars script not found: ${BAAS_HOST_VARS}" - usage 1 -fi - -if [[ -n "${BAAS_HOST_KEY}" && ! -f "${BAAS_HOST_KEY}" ]]; then - echo "Error: Baas host private key not found: ${BAAS_HOST_KEY}" - usage 1 -fi - -if [[ "${BAAS_USER}" = "root" ]]; then - FILE_DEST_DIR="/root/remote-baas" -else - FILE_DEST_DIR="/home/${BAAS_USER}/remote-baas" -fi -EVERGREEN_DEST_DIR="${FILE_DEST_DIR}/evergreen" - -# shellcheck disable=SC1090 -source "${BAAS_HOST_VARS}" - -# Wait until after the BAAS_HOST_VARS file is loaded to enable verbose tracing -if [[ -n "${VERBOSE}" ]]; then - set -o verbose - set -o xtrace -fi - -if [[ -z "${BAAS_HOST_NAME}" ]]; then - echo "Baas remote hostname (BAAS_HOST_NAME) not provided in baas host vars script" - usage 1 -fi - -if [[ -z "${BAAS_USER}" ]]; then - echo "Error: Baas host username was empty" - usage 1 -fi - -if [[ ! -d "${EVERGREEN_PATH}/" ]]; then - echo "This script must be run from the realm-core directory for accessing files in '${EVERGREEN_PATH}/'" - exit 1 -fi - -SSH_USER="$(printf "%s@%s" "${BAAS_USER}" "${BAAS_HOST_NAME}")" -SSH_OPTIONS=(-o ForwardAgent=yes -o StrictHostKeyChecking=no) - -if [[ -n "${BAAS_HOST_KEY}" ]]; then - ssh-agent > ssh_agent_commands.sh - - # shellcheck disable=SC1091 - source ssh_agent_commands.sh - - ssh-add "${BAAS_HOST_KEY}" - SSH_OPTIONS+=(-o IdentitiesOnly=yes -i "${BAAS_HOST_KEY}") -fi - -echo "running ssh with ${SSH_OPTIONS[*]}" -RETRY_COUNT=25 -WAIT_COUNTER=0 -WAIT_START=$(date -u +'%s') -CONNECT_COUNT=2 -TEST_COMMAND="[[ -f /data/baas-remote/baas-work-dir/baas_ready ]]" - -# Check for remote connectivity - try to connect twice to verify server is "really" ready -# The tests failed one time due to this ssh command passing, but the next scp command failed -while [[ ${CONNECT_COUNT} -gt 0 ]]; do - until ssh "${SSH_OPTIONS[@]}" -o ConnectTimeout=10 "${SSH_USER}" "${TEST_COMMAND}" ; do - if [[ ${WAIT_COUNTER} -ge ${RETRY_COUNT} ]] ; then - secs_spent_waiting=$(($(date -u +'%s') - WAIT_START)) - echo "Timed out after waiting ${secs_spent_waiting} seconds for host ${BAAS_HOST_NAME} to start" - exit 1 - fi - - ((++WAIT_COUNTER)) - printf "SSH connection attempt %d/%d failed. Retrying...\n" "${WAIT_COUNTER}" "${RETRY_COUNT}" - sleep 10 - done - - ((CONNECT_COUNT--)) -done - -echo "Detected remote baas server ready" From a3cdfbbe179a73284589d26c051b4faaf1c08752 Mon Sep 17 00:00:00 2001 From: Jonathan Reams Date: Wed, 18 Sep 2024 11:54:37 -0400 Subject: [PATCH 07/10] Restore wait_for_baas.sh script (#8027) --- evergreen/wait_for_baas.sh | 100 +++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100755 evergreen/wait_for_baas.sh diff --git a/evergreen/wait_for_baas.sh b/evergreen/wait_for_baas.sh new file mode 100755 index 0000000000..508ec310ec --- /dev/null +++ b/evergreen/wait_for_baas.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +# The script to wait up to approx 600 seconds (default) for the baas server to +# start, which is indicated by being able to successfully "curl" the baas +# server endpoint. If a pid file is provided, it will be used to verify the +# baas server is still running. The retry count specifies the number of +# attempts to query the server endpoint, with a 5 sec wait between attempts. +# If a server log path is provided, the last entries will be printed to stdout +# using the 'tail' command after each attempt. +# +# Usage: +# ./evergreen/wait_for_baas.sh [-w PATH] [-p FILE] [-r COUNT] [-l FILE] [-s] [-v] [-h] +# + +set -o errexit +set -o pipefail + +CURL=${CURL:=curl} +BAAS_PID_FILE= +BAAS_STOPPED_FILE= +RETRY_COUNT=120 +BAAS_SERVER_LOG= +STATUS_OUT= + +function usage() +{ + echo "Usage: wait_for_baas.sh [-w PATH] [-p FILE] [-r COUNT] [-l FILE] [-s] [-v] [-h]" + echo "Options:" + echo -e "\t-w PATH\t\tPath to baas server working directory" + echo -e "\t-p FILE\t\tPath to baas server pid file (also set by -w option)" + echo -e "\t-r COUNT\tNumber of attempts to check for baas server (default 120)" + echo -e "\t-l FILE\t\tPath to baas server log file (also set by -w option)" + echo -e "\t-s\t\tDisplay a status for each attempt" + echo -e "\t-v\t\tEnable verbose script debugging" + echo -e "\t-h\t\tShow this usage summary and exit" + # Default to 0 if exit code not provided + exit "${1:0}" +} + +function update_paths() +{ + if [[ -n "${1}" ]]; then + BAAS_SERVER_LOG="${1}/baas_server.log" + BAAS_STOPPED_FILE="${1}/baas_stopped" + BAAS_PID_FILE="${1}/baas_server.pid" + fi +} + +while getopts "w:p:r:l:svh" opt; do + case "${opt}" in + w) update_paths "${OPTARG}";; + p) BAAS_PID_FILE="${OPTARG}";; + r) RETRY_COUNT="${OPTARG}";; + l) BAAS_SERVER_LOG="${OPTARG}";; + s) STATUS_OUT="yes";; + v) set -o verbose; set -o xtrace;; + h) usage 0;; + *) usage 1;; + esac +done + +WAIT_COUNTER=0 +WAIT_START=$(date -u +'%s') + +function output_log_tail() +{ + if [[ -n "${BAAS_SERVER_LOG}" && -f "${BAAS_SERVER_LOG}" ]]; then + tail -n 10 "${BAAS_SERVER_LOG}" + fi +} + +echo "Waiting for baas server to start..." +until $CURL --output /dev/null --head --fail http://localhost:9090 --silent ; do + if [[ -n "${BAAS_STOPPED_FILE}" && -f "${BAAS_STOPPED_FILE}" ]]; then + echo "Baas server failed to start (found baas_stopped file)" + output_log_tail + exit 1 + fi + + if [[ -n "${BAAS_PID_FILE}" && -f "${BAAS_PID_FILE}" ]]; then + if ! pgrep -F "${BAAS_PID_FILE}" > /dev/null; then + echo "Baas server $(< "${BAAS_PID_FILE}") is no longer running" + output_log_tail + exit 1 + fi + fi + + ((++WAIT_COUNTER)) + secs_spent_waiting=$(($(date -u +'%s') - WAIT_START)) + if [[ ${WAIT_COUNTER} -ge ${RETRY_COUNT} ]]; then + echo "Timed out after ${secs_spent_waiting} secs waiting for baas server to start" + output_log_tail + exit 1 + fi + + if [[ -n "${STATUS_OUT}" ]]; then + echo "Waiting for baas server to start... ${secs_spent_waiting} secs so far" + fi + + sleep 5 +done From 78636fe81a4319c27a10d6a643137bc51acf8bf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Edelbo?= Date: Thu, 19 Sep 2024 14:04:37 +0200 Subject: [PATCH 08/10] Fix comparison function for ConditionType The function should ensure strict weak ordering. According to the current one {1, 0} == {0, 1} and {0, 1} == {2, 1} but {1, 0} < {2, 1} It is not possible to construct a failing test as the outcome very much depends on how the runtime types are laid out. --- CHANGELOG.md | 2 +- src/realm/query_engine.hpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6cc80c4033..44c7a8218c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ * None. ### Fixed -* ([#????](https://github.com/realm/realm-core/issues/????), since v?.?.?) +* Having a query with a number of predicates ORed together may result in a crash on some platforms (strict weak ordering check failing on iphone) ([#8028](https://github.com/realm/realm-core/issues/8028), since v14.6.0) * None. ### Breaking changes diff --git a/src/realm/query_engine.hpp b/src/realm/query_engine.hpp index 39d39b43a5..573405b03e 100644 --- a/src/realm/query_engine.hpp +++ b/src/realm/query_engine.hpp @@ -2268,7 +2268,7 @@ class OrNode : public ParentNode { std::type_index m_type; bool operator<(const ConditionType& other) const { - return this->m_col < other.m_col && this->m_type < other.m_type; + return (this->m_col == other.m_col) ? this->m_type < other.m_type : this->m_col < other.m_col; } bool operator!=(const ConditionType& other) const { From e474a8d2270a8b12ac63ac9504e4757e39814b99 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 20 Sep 2024 10:33:48 +0200 Subject: [PATCH 09/10] Prepare for release 14.13.0 (#8030) Co-authored-by: jedelbo <572755+jedelbo@users.noreply.github.com> --- CHANGELOG.md | 3 +-- Package.swift | 2 +- dependencies.yml | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 44c7a8218c..6da5176e70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,6 @@ -# NEXT RELEASE +# 14.13.0 Release notes ### Enhancements -* (PR [#????](https://github.com/realm/realm-core/pull/????)) * None. ### Fixed diff --git a/Package.swift b/Package.swift index fa1e1f5095..eb3645e1de 100644 --- a/Package.swift +++ b/Package.swift @@ -3,7 +3,7 @@ import PackageDescription import Foundation -let versionStr = "14.12.1" +let versionStr = "14.13.0" let versionPieces = versionStr.split(separator: "-") let versionCompontents = versionPieces[0].split(separator: ".") let versionExtra = versionPieces.count > 1 ? versionPieces[1] : "" diff --git a/dependencies.yml b/dependencies.yml index 4788bba2c9..901dd5d82c 100644 --- a/dependencies.yml +++ b/dependencies.yml @@ -1,5 +1,5 @@ PACKAGE_NAME: realm-core -VERSION: 14.12.1 +VERSION: 14.13.0 OPENSSL_VERSION: 3.3.1 ZLIB_VERSION: 1.2.13 # https://github.com/10gen/baas/commits From c729fc803dc64753028f739bd2faf9d3f30bcad3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 20 Sep 2024 10:37:16 +0200 Subject: [PATCH 10/10] New changelog section to prepare for vNext (#8031) Co-authored-by: jedelbo <572755+jedelbo@users.noreply.github.com> --- CHANGELOG.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6da5176e70..933f9009d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,26 @@ +# NEXT RELEASE + +### Enhancements +* (PR [#????](https://github.com/realm/realm-core/pull/????)) +* None. + +### Fixed +* ([#????](https://github.com/realm/realm-core/issues/????), since v?.?.?) +* None. + +### Breaking changes +* None. + +### Compatibility +* Fileformat: Generates files with format v24. Reads and automatically upgrade from fileformat v10. If you want to upgrade from an earlier file format version you will have to use RealmCore v13.x.y or earlier. + +----------- + +### Internals +* None. + +---------------------------------------------- + # 14.13.0 Release notes ### Enhancements