From 50678db8a5fd6a282d029a69fce66ad9b7d671e8 Mon Sep 17 00:00:00 2001 From: Stachu Korick Date: Mon, 4 Nov 2024 22:52:46 -0500 Subject: [PATCH] fix some httpclient testing --- .../execution/stdlib/httpclient.dark | 210 +++++++----------- .../v0/basic-get-helper-function.test | 2 +- backend/tests/TestUtils/TestUtils.fs | 17 +- packages/darklang/github.dark | 2 +- packages/darklang/stdlib/httpclient.dark | 2 +- scripts/run-backend-server | 16 +- scripts/run-backend-tests | 4 +- 7 files changed, 102 insertions(+), 151 deletions(-) diff --git a/backend/testfiles/execution/stdlib/httpclient.dark b/backend/testfiles/execution/stdlib/httpclient.dark index 9dc6d060e6..87ebb3109e 100644 --- a/backend/testfiles/execution/stdlib/httpclient.dark +++ b/backend/testfiles/execution/stdlib/httpclient.dark @@ -1,8 +1,9 @@ -// Most of the httpclient tests are in testfiles/httpclient. +// Most of the httpclient tests are in testfiles/httpclient +// , but we test a few more explicit cases here. // Tests that don't use the internet -module NoInternal = +module NoInternet = Stdlib.HttpClient.ContentType.form_v0 = ("content-type", "application/x-www-form-urlencoded") Stdlib.HttpClient.ContentType.json_v0 = ("content-type", "application/json") Stdlib.HttpClient.ContentType.plainText_v0 = ("content-type", "text/plain; charset=utf-8") @@ -10,7 +11,6 @@ module NoInternal = Stdlib.HttpClient.bearerToken "YWxhZGRpbjpvcGVuc2VzYW1l" = (("authorization", "bearer YWxhZGRpbjpvcGVuc2VzYW1l")) - Stdlib.HttpClient.basicAuth "username" "password" = Stdlib.Result.Result.Ok(("authorization", "basic dXNlcm5hbWU6cGFzc3dvcmQ=")) Stdlib.HttpClient.basicAuth "" "" = Stdlib.Result.Result.Ok(("authorization", "basic Og==")) Stdlib.HttpClient.basicAuth "" "-" = Stdlib.Result.Result.Ok(("authorization", "basic Oi0=")) @@ -25,35 +25,42 @@ module NoInternal = // Tests that try to make requests to the internet -// basic requests work -( - (Stdlib.HttpClient.request "get" "https://example.com" [] []) - |> Stdlib.Result.map (fun response -> response.statusCode) - ) = Stdlib.Result.Result.Ok 200L +// ... but first, some helpers +let get (url: String) (headers: List): Stdlib.Result.Result = + Stdlib.HttpClient.get url headers + +let errInvalidHost = Stdlib.Result.Result.Error(Stdlib.HttpClient.RequestError.BadUrl(Stdlib.HttpClient.BadUrlDetails.InvalidHost)) +let errInvalidUri = Stdlib.Result.Result.Error(Stdlib.HttpClient.RequestError.BadUrl(Stdlib.HttpClient.BadUrlDetails.InvalidUri)) +let errInvalidRequest = Stdlib.Result.Result.Error(Stdlib.HttpClient.RequestError.BadUrl(Stdlib.HttpClient.BadUrlDetails.InvalidRequest)) + +let errNetworkError = Stdlib.Result.Result.Error(Stdlib.HttpClient.RequestError.NetworkError) -( - (Stdlib.HttpClient.request "get" "http://example.com" [] []) - |> Stdlib.Result.map (fun response -> response.statusCode) - ) = Stdlib.Result.Result.Ok 200L +let errBadHeaderEmptyKey = Stdlib.Result.Result.Error(Stdlib.HttpClient.RequestError.BadHeader(Stdlib.HttpClient.BadHeader.EmptyKey)) +let errBadMethod = Stdlib.Result.Result.Error(Stdlib.HttpClient.RequestError.BadMethod) + +let errUnsupportedProtocol = Stdlib.Result.Result.Error(Stdlib.HttpClient.RequestError.BadUrl(Stdlib.HttpClient.BadUrlDetails.UnsupportedProtocol)) + +// basic requests work +((get "https://example.com" []) |> Stdlib.Result.map (fun r -> r.statusCode)) = Stdlib.Result.Result.Ok 200L +((get "http://example.com" []) |> Stdlib.Result.map (fun r -> r.statusCode)) = Stdlib.Result.Result.Ok 200L // Stdlib.HttpClient.request "get" "https://darklang.com" [ 1L ] [] = // Builtin.testDerrorMessage "PACKAGE.Darklang.Stdlib.HttpClient.request's 3rd argument (`headers`) should be a List<(String * String)>. However, a List ([ 1]) was passed instead. // Expected: (headers: List<(String * String)>) // Actual: a List: [\n 1\n]" -Stdlib.HttpClient.request "get" "https://darklang.com" [ ("", "") ] [] = - Stdlib.Result.Result.Error(Stdlib.HttpClient.RequestError.BadHeader(Stdlib.HttpClient.BadHeader.EmptyKey)) +(get "https://darklang.com" [ ("", "") ]) = errBadHeaderEmptyKey // type errors for bad `method` are OK -Stdlib.HttpClient.request "" "https://darklang.com" [] [] = Stdlib.Result.Result.Error(Stdlib.HttpClient.RequestError.BadMethod) -Stdlib.HttpClient.request " get " "https://darklang.com" [] [] = Stdlib.Result.Result.Error(Stdlib.HttpClient.RequestError.BadMethod) -Stdlib.HttpClient.request "🇵🇷" "https://darklang.com" [] [] = Stdlib.Result.Result.Error(Stdlib.HttpClient.RequestError.BadMethod) +Stdlib.HttpClient.request "" "https://darklang.com" [] [] = errBadMethod +Stdlib.HttpClient.request " get " "https://darklang.com" [] [] = errBadMethod +Stdlib.HttpClient.request "🇵🇷" "https://darklang.com" [] [] = errBadMethod -// // unsupported protocols -// Stdlib.HttpClient.request "get" "ftp://darklang.com" [] [] = Stdlib.Result.Result.Error(Stdlib.HttpClient.RequestError.BadUrl(Stdlib.HttpClient.BadUrlDetails.UnsupportedProtocol)) -// Stdlib.HttpClient.request "put" "file:///etc/passwd" [] [] = Stdlib.Result.Result.Error(Stdlib.HttpClient.RequestError.BadUrl(Stdlib.HttpClient.BadUrlDetails.UnsupportedProtocol)) -// Stdlib.HttpClient.request "put" "/just-a-path" [] [] = Stdlib.Result.Result.Error(Stdlib.HttpClient.RequestError.BadUrl(Stdlib.HttpClient.BadUrlDetails.UnsupportedProtocol)) +// unsupported protocols +Stdlib.HttpClient.request "get" "ftp://darklang.com" [] [] = errUnsupportedProtocol +Stdlib.HttpClient.request "put" "file:///etc/passwd" [] [] = errUnsupportedProtocol +Stdlib.HttpClient.request "put" "/just-a-path" [] [] = errUnsupportedProtocol // totally bogus URLs Stdlib.HttpClient.request "get" "" [] [] = Stdlib.Result.Result.Error(Stdlib.HttpClient.RequestError.BadUrl(Stdlib.HttpClient.BadUrlDetails.InvalidUri)) @@ -65,111 +72,56 @@ Stdlib.HttpClient.request "get" "http://google.com:79" [] [] = Stdlib.Result.Res // Check for banned urls in the host name module Disallowed = - let errInvalidHost (url: String): Bool = - (Stdlib.HttpClient.request "get" url [] []) == Stdlib.Result.Result.Error(Stdlib.HttpClient.RequestError.BadUrl(Stdlib.HttpClient.BadUrlDetails.InvalidHost)) - - //(errInvalidHost "http://0.0.0.0") = true - // (errInvalidHost "http://0") = true - // (errInvalidHost "http://[0:0:0:0:0:0:0:0]" [] []) = true - // (errInvalidHost "localhost") = true - // (errInvalidHost "http://localhost") = true - // (errInvalidHost "http://127.0.0.1") = true - // (errInvalidHost "http://[::1]") = true - // (errInvalidHost "http://[0:0:0:0:0:0:0:1]") = true - // (errInvalidHost "http://[0000:0000:0000:0000:0000:0000:0000:0001]") = true - // (errInvalidHost "http://127.0.0.17") = true - // (errInvalidHost "http://[::ffff:7f00:11]") = true - // (errInvalidHost "http://[0:0:0:0:0:ffff:7f00:0011]") = true - // (errInvalidHost "http://[0000:0000:0000:0000:0000:ffff:7f00:0011]") = true - // (errInvalidHost "http://127.255.174.17") = true - // (errInvalidHost "http://metadata.google.internal") = true - // (errInvalidHost "http://metadata") = true - // (errInvalidHost "http://169.254.169.254") = true - // (errInvalidHost "http://[::ffff:a9fe:a9fe]") = true - // (errInvalidHost "http://[0:0:0:0:0:ffff:a9fe:a9fe]") = true - // (errInvalidHost "http://[0000:0000:0000:0000:0000:ffff:a9fe:a9fe]") = true - // (errInvalidHost "http://169.254.0.0") = true - // (errInvalidHost "http://172.16.0.1") = true - // (errInvalidHost "http://[::ffff:ac10:1]") = true - // (errInvalidHost "http://[0:0:0:0:0:ffff:ac10:0001]") = true - // (errInvalidHost "http://[0000:0000:0000:0000:0000:ffff:ac10:0001]") = true - // (errInvalidHost "http://192.168.1.1") = true - // (errInvalidHost "http://[::ffff:c0a8:101]") = true - // (errInvalidHost "http://[0:0:0:0:0:ffff:c0a8:0101]") = true - // (errInvalidHost "http://[0000:0000:0000:0000:0000:ffff:c0a8:0101]") = true - -// // Check for sneaky banned urls - blocked via connection callback -// // 127.0.0.1 -// Stdlib.HttpClient.request "get" "http://localtest.me" [] [] = Stdlib.Result.Result.Error(Stdlib.HttpClient.RequestError.NetworkError) -// // 0.0.0.0 -// Stdlib.HttpClient.request "get" "http://c.cx" [] [] = Stdlib.Result.Result.Error(Stdlib.HttpClient.RequestError.NetworkError) - -// // invalid headers -// Stdlib.HttpClient.request -// "get" -// "http://google.com" -// [ ("Metadata-Flavor", "Google") ] -// [] = Stdlib.Result.Result.Error( -// Stdlib.HttpClient.RequestError.BadUrl( -// Stdlib.HttpClient.BadUrlDetails.InvalidRequest -// ) -// ) - -// Stdlib.HttpClient.request -// "get" -// "http://google.com" -// [ ("metadata-flavor", "Google") ] -// [] = Stdlib.Result.Result.Error( -// Stdlib.HttpClient.RequestError.BadUrl( -// Stdlib.HttpClient.BadUrlDetails.InvalidRequest -// ) -// ) - -// Stdlib.HttpClient.request -// "get" -// "http://google.com" -// [ ("Metadata-Flavor", "google") ] -// [] = Stdlib.Result.Result.Error( -// Stdlib.HttpClient.RequestError.BadUrl( -// Stdlib.HttpClient.BadUrlDetails.InvalidRequest -// ) -// ) - -// Stdlib.HttpClient.request -// "get" -// "http://google.com" -// [ ("Metadata-Flavor", " Google ") ] -// [] = Stdlib.Result.Result.Error( -// Stdlib.HttpClient.RequestError.BadUrl( -// Stdlib.HttpClient.BadUrlDetails.InvalidRequest -// ) -// ) - -// Stdlib.HttpClient.request -// "get" -// "http://google.com" -// [ ("X-Google-Metadata-Request", " True ") ] -// [] = Stdlib.Result.Result.Error( -// Stdlib.HttpClient.RequestError.BadUrl( -// Stdlib.HttpClient.BadUrlDetails.InvalidRequest -// ) -// ) - -// Stdlib.HttpClient.request -// "get" -// "http://google.com" -// [ (" x-Google-metaData-Request", " True ") ] -// [] = Stdlib.Result.Result.Error( -// Stdlib.HttpClient.RequestError.BadUrl( -// Stdlib.HttpClient.BadUrlDetails.InvalidRequest -// ) -// ) - -// module BadSSL = -// Stdlib.HttpClient.request "get" "http://thenonexistingurlforsure.com" [] [] = Stdlib.Result.Result.Error(Stdlib.HttpClient.RequestError.NetworkError) - -// Stdlib.HttpClient.request "get" "https://self-signed.badssl.com" [] [] = Stdlib.Result.Result.Error(Stdlib.HttpClient.RequestError.NetworkError) - - - -// // TODO: http2, http3 \ No newline at end of file + // invalid host + (get "http://0.0.0.0" []) = errInvalidHost + (get "http://0" []) = errInvalidHost + (get "http://localhost" []) = errInvalidHost + (get "http://127.0.0.1" []) = errInvalidHost + (get "http://[::1]" []) = errInvalidHost + (get "http://[0:0:0:0:0:0:0:1]" []) = errInvalidHost + (get "http://[0000:0000:0000:0000:0000:0000:0000:0001]" []) = errInvalidHost + (get "http://127.0.0.17" []) = errInvalidHost + (get "http://[::ffff:7f00:11]" []) = errInvalidHost + (get "http://[0:0:0:0:0:ffff:7f00:0011]" []) = errInvalidHost + (get "http://[0000:0000:0000:0000:0000:ffff:7f00:0011]" []) = errInvalidHost + (get "http://127.255.174.17" []) = errInvalidHost + (get "http://metadata.google.internal" []) = errInvalidHost + (get "http://metadata" []) = errInvalidHost + (get "http://169.254.169.254" []) = errInvalidHost + (get "http://[::ffff:a9fe:a9fe]" []) = errInvalidHost + (get "http://[0:0:0:0:0:ffff:a9fe:a9fe]" []) = errInvalidHost + (get "http://[0000:0000:0000:0000:0000:ffff:a9fe:a9fe]" []) = errInvalidHost + (get "http://169.254.0.0" []) = errInvalidHost + (get "http://172.16.0.1" []) = errInvalidHost + (get "http://[::ffff:ac10:1]" []) = errInvalidHost + (get "http://[0:0:0:0:0:ffff:ac10:0001]" []) = errInvalidHost + (get "http://[0000:0000:0000:0000:0000:ffff:ac10:0001]" []) = errInvalidHost + (get "http://192.168.1.1" []) = errInvalidHost + (get "http://[::ffff:c0a8:101]" []) = errInvalidHost + (get "http://[0:0:0:0:0:ffff:c0a8:0101]" []) = errInvalidHost + (get "http://[0000:0000:0000:0000:0000:ffff:c0a8:0101]" []) = errInvalidHost + + // invalid uri + (get "localhost" []) = errInvalidUri + + // network error + (get "http://[0:0:0:0:0:0:0:0]" []) = errNetworkError + // Check for sneaky banned urls - blocked via connection callback + (get "http://localtest.me" []) = errNetworkError // 127.0.0.1 + (get "http://c.cx" []) = errNetworkError // 0.0.0.0 + + // invalid headers + // (we just happen to know that google will fail this way if we provide these headers..) + (get "http://google.com" [ ("Metadata-Flavor", "Google") ]) = errInvalidRequest + (get "http://google.com" [ ("metadata-flavor", "Google") ]) = errInvalidRequest + (get "http://google.com" [ ("Metadata-Flavor", " Google ") ]) = errInvalidRequest + (get "http://google.com" [ ("X-Google-Metadata-Request", " True ") ]) = errInvalidRequest + (get "http://google.com" [ (" x-Google-metaData-Request", " True ") ]) = errInvalidRequest + +module BadSSL = + (get "http://thenonexistingurlforsure.com" []) = errNetworkError + (get "https://self-signed.badssl.com" []) = errNetworkError + + + +// TODO: http2, http3 \ No newline at end of file diff --git a/backend/testfiles/httpclient/v0/basic-get-helper-function.test b/backend/testfiles/httpclient/v0/basic-get-helper-function.test index b58a183d82..c14cc447e8 100644 --- a/backend/testfiles/httpclient/v0/basic-get-helper-function.test +++ b/backend/testfiles/httpclient/v0/basic-get-helper-function.test @@ -12,7 +12,7 @@ Content-Length: LENGTH Hello back [test] -(let response = (PACKAGE.Darklang.Stdlib.HttpClient.get [] "http://URL") |> Builtin.unwrap +(let response = (PACKAGE.Darklang.Stdlib.HttpClient.get "http://URL" []) |> Builtin.unwrap let respHeaders = response.headers |> PACKAGE.Darklang.Stdlib.List.filter (fun h -> PACKAGE.Darklang.Stdlib.Tuple2.first h != "date") {response with headers = respHeaders}) == PACKAGE.Darklang.Stdlib.HttpClient.Response diff --git a/backend/tests/TestUtils/TestUtils.fs b/backend/tests/TestUtils/TestUtils.fs index a5067d3c28..269aee6479 100644 --- a/backend/tests/TestUtils/TestUtils.fs +++ b/backend/tests/TestUtils/TestUtils.fs @@ -126,12 +126,12 @@ let builtins BuiltinCli.Builtin.builtins ] [] -// let cloudBuiltIns (pm : PT.PackageManager) = -// let httpConfig = -// { LibCloudExecution.HttpClient.configuration with -// timeoutInMs = 5000 -// allowedIP = (fun _ -> true) } -// builtins httpConfig pm +let cloudBuiltIns (pm : PT.PackageManager) = + let httpConfig = + { LibCloudExecution.HttpClient.configuration with + timeoutInMs = 5000 + allowedIP = (fun _ -> true) } + builtins httpConfig pm let localBuiltIns (pm : PT.PackageManager) = let httpConfig = @@ -144,7 +144,7 @@ let executionStateFor (pmPT : PT.PackageManager) (canvasID : CanvasID) (internalFnsAllowed : bool) - (_allowLocalHttpAccess : bool) + (allowLocalHttpAccess : bool) //(dbs : Map) : Task = task { @@ -197,8 +197,7 @@ let executionStateFor let notifier : RT.Notifier = fun _state _msg _tags -> () let builtins = - //if allowLocalHttpAccess then localBuiltIns pmPT else cloudBuiltIns pmPT - localBuiltIns pmPT + if allowLocalHttpAccess then localBuiltIns pmPT else cloudBuiltIns pmPT let state = let pmRT = PT2RT.PackageManager.toRT pmPT Exe.createState builtins pmRT Exe.noTracing exceptionReporter notifier program diff --git a/packages/darklang/github.dark b/packages/darklang/github.dark index 732ea0804c..1a487e8463 100644 --- a/packages/darklang/github.dark +++ b/packages/darklang/github.dark @@ -18,7 +18,7 @@ module Darklang = ("accept", "application/vnd.github+json") ("user-Agent", "darklang") ] - match Stdlib.HttpClient.get headers url with + match Stdlib.HttpClient.get url headers with | Ok r -> r.body |> Stdlib.String.fromBytesWithReplacement |> Stdlib.Result.Result.Ok | Error _e -> Stdlib.Result.Result.Error "Failed to make GitHub API request" diff --git a/packages/darklang/stdlib/httpclient.dark b/packages/darklang/stdlib/httpclient.dark index 5c9ddb7f3c..0c96f7868a 100644 --- a/packages/darklang/stdlib/httpclient.dark +++ b/packages/darklang/stdlib/httpclient.dark @@ -65,8 +65,8 @@ module Darklang = let get - (headers: List) (uri: String) + (headers: List) : Stdlib.Result.Result = request "GET" uri headers [] diff --git a/scripts/run-backend-server b/scripts/run-backend-server index f5e5fc918e..36f0e8a9b3 100755 --- a/scripts/run-backend-server +++ b/scripts/run-backend-server @@ -63,8 +63,8 @@ LOCALEXEC_EXE="${LOCALEXEC_BINPATH}/LocalExec" # sudo pkill -f "CronChecker" || true # sudo pkill -f "QueueWorker" || true -# ./scripts/run-pubsub-emulator -# ./scripts/run-cloud-storage-emulator +./scripts/run-pubsub-emulator +./scripts/run-cloud-storage-emulator echo "Waiting for postgres" ./scripts/devcontainer/_wait-for-postgres @@ -84,12 +84,12 @@ do done echo "Done waiting for compiled servers" -# # Wait for cloud-storage-emulator (can be slow on CI) -# echo "Waiting for cloud-storage-emulator" -# until curl -s -o /dev/null "localhost:4444" ; do -# printf '.' -# sleep 0.1 -# done +# Wait for cloud-storage-emulator (can be slow on CI) +echo "Waiting for cloud-storage-emulator" +until curl -s -o /dev/null "localhost:4444" ; do + printf '.' + sleep 0.1 +done grey="\033[1;30m" reset="\033[0m" diff --git a/scripts/run-backend-tests b/scripts/run-backend-tests index 044d723bc1..48ec0c457d 100755 --- a/scripts/run-backend-tests +++ b/scripts/run-backend-tests @@ -68,8 +68,8 @@ esac LOGS="${DARK_CONFIG_RUNDIR}/logs" -# ./scripts/run-pubsub-emulator -# ./scripts/run-cloud-storage-emulator +./scripts/run-pubsub-emulator +./scripts/run-cloud-storage-emulator # Use random to avoid old items being in the pubsub queue. Use this instead of # $RANDOM as RANDOM is only 5 digits