Skip to content

Commit

Permalink
fix some httpclient testing
Browse files Browse the repository at this point in the history
  • Loading branch information
@StachuDotNet
StachuDotNet committed Nov 5, 2024
1 parent 53ce353 commit 50678db
Show file tree
Hide file tree
Showing 7 changed files with 102 additions and 151 deletions.
210 changes: 81 additions & 129 deletions backend/testfiles/execution/stdlib/httpclient.dark
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
// Most of the httpclient tests are in testfiles/httpclient.
// Most of the httpclient tests are in testfiles/httpclient
// , but we test a few more explicit cases here.


// Tests that don't use the internet
module NoInternal =
module NoInternet =
Stdlib.HttpClient.ContentType.form_v0 = ("content-type", "application/x-www-form-urlencoded")
Stdlib.HttpClient.ContentType.json_v0 = ("content-type", "application/json")
Stdlib.HttpClient.ContentType.plainText_v0 = ("content-type", "text/plain; charset=utf-8")
Stdlib.HttpClient.ContentType.html_v0 = ("content-type", "text/html; charset=utf-8")

Stdlib.HttpClient.bearerToken "YWxhZGRpbjpvcGVuc2VzYW1l" = (("authorization", "bearer YWxhZGRpbjpvcGVuc2VzYW1l"))


Stdlib.HttpClient.basicAuth "username" "password" = Stdlib.Result.Result.Ok(("authorization", "basic dXNlcm5hbWU6cGFzc3dvcmQ="))
Stdlib.HttpClient.basicAuth "" "" = Stdlib.Result.Result.Ok(("authorization", "basic Og=="))
Stdlib.HttpClient.basicAuth "" "-" = Stdlib.Result.Result.Ok(("authorization", "basic Oi0="))
Expand All @@ -25,35 +25,42 @@ module NoInternal =

// Tests that try to make requests to the internet

// basic requests work
(
(Stdlib.HttpClient.request "get" "https://example.com" [] [])
|> Stdlib.Result.map (fun response -> response.statusCode)
) = Stdlib.Result.Result.Ok 200L
// ... but first, some helpers
let get (url: String) (headers: List<String * String>): Stdlib.Result.Result<Stdlib.HttpClient.Response, Stdlib.HttpClient.RequestError> =
Stdlib.HttpClient.get url headers

let errInvalidHost = Stdlib.Result.Result.Error(Stdlib.HttpClient.RequestError.BadUrl(Stdlib.HttpClient.BadUrlDetails.InvalidHost))
let errInvalidUri = Stdlib.Result.Result.Error(Stdlib.HttpClient.RequestError.BadUrl(Stdlib.HttpClient.BadUrlDetails.InvalidUri))
let errInvalidRequest = Stdlib.Result.Result.Error(Stdlib.HttpClient.RequestError.BadUrl(Stdlib.HttpClient.BadUrlDetails.InvalidRequest))

let errNetworkError = Stdlib.Result.Result.Error(Stdlib.HttpClient.RequestError.NetworkError)

(
(Stdlib.HttpClient.request "get" "http://example.com" [] [])
|> Stdlib.Result.map (fun response -> response.statusCode)
) = Stdlib.Result.Result.Ok 200L
let errBadHeaderEmptyKey = Stdlib.Result.Result.Error(Stdlib.HttpClient.RequestError.BadHeader(Stdlib.HttpClient.BadHeader.EmptyKey))
let errBadMethod = Stdlib.Result.Result.Error(Stdlib.HttpClient.RequestError.BadMethod)

let errUnsupportedProtocol = Stdlib.Result.Result.Error(Stdlib.HttpClient.RequestError.BadUrl(Stdlib.HttpClient.BadUrlDetails.UnsupportedProtocol))

// basic requests work
((get "https://example.com" []) |> Stdlib.Result.map (fun r -> r.statusCode)) = Stdlib.Result.Result.Ok 200L
((get "http://example.com" []) |> Stdlib.Result.map (fun r -> r.statusCode)) = Stdlib.Result.Result.Ok 200L

// Stdlib.HttpClient.request "get" "https://darklang.com" [ 1L ] [] =
// Builtin.testDerrorMessage "PACKAGE.Darklang.Stdlib.HttpClient.request's 3rd argument (`headers`) should be a List<(String * String)>. However, a List<Int64> ([ 1]) was passed instead.
// Expected: (headers: List<(String * String)>)
// Actual: a List<Int64>: [\n 1\n]"

Stdlib.HttpClient.request "get" "https://darklang.com" [ ("", "") ] [] =
Stdlib.Result.Result.Error(Stdlib.HttpClient.RequestError.BadHeader(Stdlib.HttpClient.BadHeader.EmptyKey))
(get "https://darklang.com" [ ("", "") ]) = errBadHeaderEmptyKey


// type errors for bad `method` are OK
Stdlib.HttpClient.request "" "https://darklang.com" [] [] = Stdlib.Result.Result.Error(Stdlib.HttpClient.RequestError.BadMethod)
Stdlib.HttpClient.request " get " "https://darklang.com" [] [] = Stdlib.Result.Result.Error(Stdlib.HttpClient.RequestError.BadMethod)
Stdlib.HttpClient.request "🇵🇷" "https://darklang.com" [] [] = Stdlib.Result.Result.Error(Stdlib.HttpClient.RequestError.BadMethod)
Stdlib.HttpClient.request "" "https://darklang.com" [] [] = errBadMethod
Stdlib.HttpClient.request " get " "https://darklang.com" [] [] = errBadMethod
Stdlib.HttpClient.request "🇵🇷" "https://darklang.com" [] [] = errBadMethod

// // unsupported protocols
// Stdlib.HttpClient.request "get" "ftp://darklang.com" [] [] = Stdlib.Result.Result.Error(Stdlib.HttpClient.RequestError.BadUrl(Stdlib.HttpClient.BadUrlDetails.UnsupportedProtocol))
// Stdlib.HttpClient.request "put" "file:///etc/passwd" [] [] = Stdlib.Result.Result.Error(Stdlib.HttpClient.RequestError.BadUrl(Stdlib.HttpClient.BadUrlDetails.UnsupportedProtocol))
// Stdlib.HttpClient.request "put" "/just-a-path" [] [] = Stdlib.Result.Result.Error(Stdlib.HttpClient.RequestError.BadUrl(Stdlib.HttpClient.BadUrlDetails.UnsupportedProtocol))
// unsupported protocols
Stdlib.HttpClient.request "get" "ftp://darklang.com" [] [] = errUnsupportedProtocol
Stdlib.HttpClient.request "put" "file:///etc/passwd" [] [] = errUnsupportedProtocol
Stdlib.HttpClient.request "put" "/just-a-path" [] [] = errUnsupportedProtocol

// totally bogus URLs
Stdlib.HttpClient.request "get" "" [] [] = Stdlib.Result.Result.Error(Stdlib.HttpClient.RequestError.BadUrl(Stdlib.HttpClient.BadUrlDetails.InvalidUri))
Expand All @@ -65,111 +72,56 @@ Stdlib.HttpClient.request "get" "http://google.com:79" [] [] = Stdlib.Result.Res

// Check for banned urls in the host name
module Disallowed =
let errInvalidHost (url: String): Bool =
(Stdlib.HttpClient.request "get" url [] []) == Stdlib.Result.Result.Error(Stdlib.HttpClient.RequestError.BadUrl(Stdlib.HttpClient.BadUrlDetails.InvalidHost))

//(errInvalidHost "http://0.0.0.0") = true
// (errInvalidHost "http://0") = true
// (errInvalidHost "http://[0:0:0:0:0:0:0:0]" [] []) = true
// (errInvalidHost "localhost") = true
// (errInvalidHost "http://localhost") = true
// (errInvalidHost "http://127.0.0.1") = true
// (errInvalidHost "http://[::1]") = true
// (errInvalidHost "http://[0:0:0:0:0:0:0:1]") = true
// (errInvalidHost "http://[0000:0000:0000:0000:0000:0000:0000:0001]") = true
// (errInvalidHost "http://127.0.0.17") = true
// (errInvalidHost "http://[::ffff:7f00:11]") = true
// (errInvalidHost "http://[0:0:0:0:0:ffff:7f00:0011]") = true
// (errInvalidHost "http://[0000:0000:0000:0000:0000:ffff:7f00:0011]") = true
// (errInvalidHost "http://127.255.174.17") = true
// (errInvalidHost "http://metadata.google.internal") = true
// (errInvalidHost "http://metadata") = true
// (errInvalidHost "http://169.254.169.254") = true
// (errInvalidHost "http://[::ffff:a9fe:a9fe]") = true
// (errInvalidHost "http://[0:0:0:0:0:ffff:a9fe:a9fe]") = true
// (errInvalidHost "http://[0000:0000:0000:0000:0000:ffff:a9fe:a9fe]") = true
// (errInvalidHost "http://169.254.0.0") = true
// (errInvalidHost "http://172.16.0.1") = true
// (errInvalidHost "http://[::ffff:ac10:1]") = true
// (errInvalidHost "http://[0:0:0:0:0:ffff:ac10:0001]") = true
// (errInvalidHost "http://[0000:0000:0000:0000:0000:ffff:ac10:0001]") = true
// (errInvalidHost "http://192.168.1.1") = true
// (errInvalidHost "http://[::ffff:c0a8:101]") = true
// (errInvalidHost "http://[0:0:0:0:0:ffff:c0a8:0101]") = true
// (errInvalidHost "http://[0000:0000:0000:0000:0000:ffff:c0a8:0101]") = true

// // Check for sneaky banned urls - blocked via connection callback
// // 127.0.0.1
// Stdlib.HttpClient.request "get" "http://localtest.me" [] [] = Stdlib.Result.Result.Error(Stdlib.HttpClient.RequestError.NetworkError)
// // 0.0.0.0
// Stdlib.HttpClient.request "get" "http://c.cx" [] [] = Stdlib.Result.Result.Error(Stdlib.HttpClient.RequestError.NetworkError)

// // invalid headers
// Stdlib.HttpClient.request
// "get"
// "http://google.com"
// [ ("Metadata-Flavor", "Google") ]
// [] = Stdlib.Result.Result.Error(
// Stdlib.HttpClient.RequestError.BadUrl(
// Stdlib.HttpClient.BadUrlDetails.InvalidRequest
// )
// )

// Stdlib.HttpClient.request
// "get"
// "http://google.com"
// [ ("metadata-flavor", "Google") ]
// [] = Stdlib.Result.Result.Error(
// Stdlib.HttpClient.RequestError.BadUrl(
// Stdlib.HttpClient.BadUrlDetails.InvalidRequest
// )
// )

// Stdlib.HttpClient.request
// "get"
// "http://google.com"
// [ ("Metadata-Flavor", "google") ]
// [] = Stdlib.Result.Result.Error(
// Stdlib.HttpClient.RequestError.BadUrl(
// Stdlib.HttpClient.BadUrlDetails.InvalidRequest
// )
// )

// Stdlib.HttpClient.request
// "get"
// "http://google.com"
// [ ("Metadata-Flavor", " Google ") ]
// [] = Stdlib.Result.Result.Error(
// Stdlib.HttpClient.RequestError.BadUrl(
// Stdlib.HttpClient.BadUrlDetails.InvalidRequest
// )
// )

// Stdlib.HttpClient.request
// "get"
// "http://google.com"
// [ ("X-Google-Metadata-Request", " True ") ]
// [] = Stdlib.Result.Result.Error(
// Stdlib.HttpClient.RequestError.BadUrl(
// Stdlib.HttpClient.BadUrlDetails.InvalidRequest
// )
// )

// Stdlib.HttpClient.request
// "get"
// "http://google.com"
// [ (" x-Google-metaData-Request", " True ") ]
// [] = Stdlib.Result.Result.Error(
// Stdlib.HttpClient.RequestError.BadUrl(
// Stdlib.HttpClient.BadUrlDetails.InvalidRequest
// )
// )

// module BadSSL =
// Stdlib.HttpClient.request "get" "http://thenonexistingurlforsure.com" [] [] = Stdlib.Result.Result.Error(Stdlib.HttpClient.RequestError.NetworkError)

// Stdlib.HttpClient.request "get" "https://self-signed.badssl.com" [] [] = Stdlib.Result.Result.Error(Stdlib.HttpClient.RequestError.NetworkError)



// // TODO: http2, http3
// invalid host
(get "http://0.0.0.0" []) = errInvalidHost
(get "http://0" []) = errInvalidHost
(get "http://localhost" []) = errInvalidHost
(get "http://127.0.0.1" []) = errInvalidHost
(get "http://[::1]" []) = errInvalidHost
(get "http://[0:0:0:0:0:0:0:1]" []) = errInvalidHost
(get "http://[0000:0000:0000:0000:0000:0000:0000:0001]" []) = errInvalidHost
(get "http://127.0.0.17" []) = errInvalidHost
(get "http://[::ffff:7f00:11]" []) = errInvalidHost
(get "http://[0:0:0:0:0:ffff:7f00:0011]" []) = errInvalidHost
(get "http://[0000:0000:0000:0000:0000:ffff:7f00:0011]" []) = errInvalidHost
(get "http://127.255.174.17" []) = errInvalidHost
(get "http://metadata.google.internal" []) = errInvalidHost
(get "http://metadata" []) = errInvalidHost
(get "http://169.254.169.254" []) = errInvalidHost
(get "http://[::ffff:a9fe:a9fe]" []) = errInvalidHost
(get "http://[0:0:0:0:0:ffff:a9fe:a9fe]" []) = errInvalidHost
(get "http://[0000:0000:0000:0000:0000:ffff:a9fe:a9fe]" []) = errInvalidHost
(get "http://169.254.0.0" []) = errInvalidHost
(get "http://172.16.0.1" []) = errInvalidHost
(get "http://[::ffff:ac10:1]" []) = errInvalidHost
(get "http://[0:0:0:0:0:ffff:ac10:0001]" []) = errInvalidHost
(get "http://[0000:0000:0000:0000:0000:ffff:ac10:0001]" []) = errInvalidHost
(get "http://192.168.1.1" []) = errInvalidHost
(get "http://[::ffff:c0a8:101]" []) = errInvalidHost
(get "http://[0:0:0:0:0:ffff:c0a8:0101]" []) = errInvalidHost
(get "http://[0000:0000:0000:0000:0000:ffff:c0a8:0101]" []) = errInvalidHost

// invalid uri
(get "localhost" []) = errInvalidUri

// network error
(get "http://[0:0:0:0:0:0:0:0]" []) = errNetworkError
// Check for sneaky banned urls - blocked via connection callback
(get "http://localtest.me" []) = errNetworkError // 127.0.0.1
(get "http://c.cx" []) = errNetworkError // 0.0.0.0

// invalid headers
// (we just happen to know that google will fail this way if we provide these headers..)
(get "http://google.com" [ ("Metadata-Flavor", "Google") ]) = errInvalidRequest
(get "http://google.com" [ ("metadata-flavor", "Google") ]) = errInvalidRequest
(get "http://google.com" [ ("Metadata-Flavor", " Google ") ]) = errInvalidRequest
(get "http://google.com" [ ("X-Google-Metadata-Request", " True ") ]) = errInvalidRequest
(get "http://google.com" [ (" x-Google-metaData-Request", " True ") ]) = errInvalidRequest

module BadSSL =
(get "http://thenonexistingurlforsure.com" []) = errNetworkError
(get "https://self-signed.badssl.com" []) = errNetworkError



// TODO: http2, http3
2 changes: 1 addition & 1 deletion backend/testfiles/httpclient/v0/basic-get-helper-function.test
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ Content-Length: LENGTH
Hello back

[test]
(let response = (PACKAGE.Darklang.Stdlib.HttpClient.get [] "http://URL") |> Builtin.unwrap
(let response = (PACKAGE.Darklang.Stdlib.HttpClient.get "http://URL" []) |> Builtin.unwrap
let respHeaders = response.headers |> PACKAGE.Darklang.Stdlib.List.filter (fun h -> PACKAGE.Darklang.Stdlib.Tuple2.first h != "date")
{response with headers = respHeaders}) ==
PACKAGE.Darklang.Stdlib.HttpClient.Response
Expand Down
17 changes: 8 additions & 9 deletions backend/tests/TestUtils/TestUtils.fs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,12 +126,12 @@ let builtins
BuiltinCli.Builtin.builtins ]
[]

// let cloudBuiltIns (pm : PT.PackageManager) =
// let httpConfig =
// { LibCloudExecution.HttpClient.configuration with
// timeoutInMs = 5000
// allowedIP = (fun _ -> true) }
// builtins httpConfig pm
let cloudBuiltIns (pm : PT.PackageManager) =
let httpConfig =
{ LibCloudExecution.HttpClient.configuration with
timeoutInMs = 5000
allowedIP = (fun _ -> true) }
builtins httpConfig pm

let localBuiltIns (pm : PT.PackageManager) =
let httpConfig =
Expand All @@ -144,7 +144,7 @@ let executionStateFor
(pmPT : PT.PackageManager)
(canvasID : CanvasID)
(internalFnsAllowed : bool)
(_allowLocalHttpAccess : bool)
(allowLocalHttpAccess : bool)
//(dbs : Map<string, RT.DB.T>)
: Task<RT.ExecutionState> =
task {
Expand Down Expand Up @@ -197,8 +197,7 @@ let executionStateFor
let notifier : RT.Notifier = fun _state _msg _tags -> ()

let builtins =
//if allowLocalHttpAccess then localBuiltIns pmPT else cloudBuiltIns pmPT
localBuiltIns pmPT
if allowLocalHttpAccess then localBuiltIns pmPT else cloudBuiltIns pmPT
let state =
let pmRT = PT2RT.PackageManager.toRT pmPT
Exe.createState builtins pmRT Exe.noTracing exceptionReporter notifier program
Expand Down
2 changes: 1 addition & 1 deletion packages/darklang/github.dark
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ module Darklang =
("accept", "application/vnd.github+json")
("user-Agent", "darklang") ]

match Stdlib.HttpClient.get headers url with
match Stdlib.HttpClient.get url headers with
| Ok r ->
r.body |> Stdlib.String.fromBytesWithReplacement |> Stdlib.Result.Result.Ok
| Error _e -> Stdlib.Result.Result.Error "Failed to make GitHub API request"
Expand Down
2 changes: 1 addition & 1 deletion packages/darklang/stdlib/httpclient.dark
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,8 @@ module Darklang =


let get
(headers: List<String * String>)
(uri: String)
(headers: List<String * String>)
: Stdlib.Result.Result<Response, RequestError> =
request "GET" uri headers []

Expand Down
16 changes: 8 additions & 8 deletions scripts/run-backend-server
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,8 @@ LOCALEXEC_EXE="${LOCALEXEC_BINPATH}/LocalExec"
# sudo pkill -f "CronChecker" || true
# sudo pkill -f "QueueWorker" || true

# ./scripts/run-pubsub-emulator
# ./scripts/run-cloud-storage-emulator
./scripts/run-pubsub-emulator
./scripts/run-cloud-storage-emulator

echo "Waiting for postgres"
./scripts/devcontainer/_wait-for-postgres
Expand All @@ -84,12 +84,12 @@ do
done
echo "Done waiting for compiled servers"

# # Wait for cloud-storage-emulator (can be slow on CI)
# echo "Waiting for cloud-storage-emulator"
# until curl -s -o /dev/null "localhost:4444" ; do
# printf '.'
# sleep 0.1
# done
# Wait for cloud-storage-emulator (can be slow on CI)
echo "Waiting for cloud-storage-emulator"
until curl -s -o /dev/null "localhost:4444" ; do
printf '.'
sleep 0.1
done

grey="\033[1;30m"
reset="\033[0m"
Expand Down
4 changes: 2 additions & 2 deletions scripts/run-backend-tests
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,8 +68,8 @@ esac

LOGS="${DARK_CONFIG_RUNDIR}/logs"

# ./scripts/run-pubsub-emulator
# ./scripts/run-cloud-storage-emulator
./scripts/run-pubsub-emulator
./scripts/run-cloud-storage-emulator

# Use random to avoid old items being in the pubsub queue. Use this instead of
# $RANDOM as RANDOM is only 5 digits
Expand Down

0 comments on commit 50678db

Please sign in to comment.