From 0ea6a5ba8906f85802d6f1b7a1b472b705e5f7e2 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 20 Nov 2023 14:36:46 +0100 Subject: [PATCH] chore: sync with ooni/probe-cli@5c50be42 --- .github/workflows/build.yml | 2 +- UPSTREAM | 2 +- go.mod | 13 +- go.sum | 30 +- pkg/checkincache/checkincache.go | 17 + pkg/cmd/buildtool/android.go | 84 +- pkg/cmd/buildtool/android_test.go | 160 +- pkg/cmd/buildtool/builddeps.go | 5 + pkg/cmd/buildtool/cbuildenv.go | 45 +- pkg/cmd/buildtool/cdepslibevent.go | 16 +- pkg/cmd/buildtool/cdepsopenssl.go | 9 +- pkg/cmd/buildtool/cdepstor.go | 1 + .../internal/buildtoolmodel/buildtoolmodel.go | 8 +- .../internal/buildtooltest/buildtooltest.go | 21 +- pkg/cmd/buildtool/ios.go | 147 ++ pkg/cmd/buildtool/ios_test.go | 1286 +++++++++++++++++ pkg/cmd/buildtool/linuxcdeps.go | 36 +- pkg/cmd/buildtool/linuxcdeps_test.go | 40 +- pkg/cmd/ghgen/ios.go | 19 +- pkg/dslx/address.go | 10 + pkg/dslx/connpool.go | 42 - pkg/dslx/connpool_test.go | 100 -- pkg/dslx/dns.go | 317 ++-- pkg/dslx/dns_test.go | 174 +-- pkg/dslx/endpoint.go | 83 +- pkg/dslx/endpoint_test.go | 15 - pkg/dslx/fxasync.go | 72 +- pkg/dslx/fxasync_test.go | 23 +- pkg/dslx/fxcore.go | 184 +-- pkg/dslx/fxcore_test.go | 177 +-- pkg/dslx/fxgen.go | 228 +-- pkg/dslx/fxstream.go | 12 +- pkg/dslx/http_test.go | 488 ++++--- pkg/dslx/httpcore.go | 279 ++-- pkg/dslx/httpquic.go | 61 +- pkg/dslx/httptcp.go | 61 +- pkg/dslx/httptls.go | 61 +- pkg/dslx/integration_test.go | 38 +- pkg/dslx/observations.go | 11 +- pkg/dslx/qa_test.go | 311 ++++ pkg/dslx/quic.go | 223 +-- pkg/dslx/quic_test.go | 134 +- pkg/dslx/runtimecore.go | 49 + pkg/dslx/runtimemeasurex.go | 46 + pkg/dslx/runtimemeasurex_test.go | 24 + pkg/dslx/runtimeminimal.go | 206 +++ pkg/dslx/runtimeminimal_test.go | 252 ++++ pkg/dslx/tcp.go | 132 +- pkg/dslx/tcp_test.go | 48 +- pkg/dslx/tls.go | 228 ++- pkg/dslx/tls_test.go | 192 +-- pkg/dslx/trace.go | 74 + pkg/engine/experiment_integration_test.go | 7 + pkg/engine/experimentbuilder.go | 2 +- pkg/engine/inputloader.go | 6 + pkg/engine/session.go | 12 - pkg/experiment/echcheck/handshake.go | 23 +- pkg/experiment/echcheck/handshake_test.go | 4 +- pkg/experiment/echcheck/measure.go | 33 +- pkg/experiment/echcheck/measure_test.go | 133 +- pkg/experiment/riseupvpn/riseupvpn.go | 157 +- pkg/experiment/riseupvpn/riseupvpn_test.go | 368 +---- pkg/experiment/torsf/torsf.go | 2 +- pkg/experiment/torsf/torsf_test.go | 2 +- .../webconnectivitylte/cleartextflow.go | 10 +- .../webconnectivitylte/secureflow.go | 10 +- pkg/libtor/enabled.go | 27 +- pkg/libtor/iphoneos/arm64/.gitignore | 2 + pkg/libtor/iphonesimulator/amd64/.gitignore | 2 + pkg/libtor/iphonesimulator/arm64/.gitignore | 2 + pkg/measurexlite/conn.go | 24 +- pkg/measurexlite/dialer.go | 12 +- pkg/measurexlite/dns.go | 16 +- pkg/measurexlite/dns_test.go | 8 +- pkg/measurexlite/quic.go | 12 +- pkg/measurexlite/tls.go | 12 +- pkg/measurexlite/trace.go | 26 +- pkg/measurexlite/trace_test.go | 4 +- pkg/measurexlite/udp.go | 1 + pkg/netemx/scenario.go | 6 + pkg/progress/progress.go | 44 + pkg/progress/progress_test.go | 87 ++ pkg/registry/allexperiments.go | 3 + pkg/registry/dash.go | 7 +- pkg/registry/dnscheck.go | 5 +- pkg/registry/dnsping.go | 5 +- pkg/registry/example.go | 5 +- pkg/registry/factory.go | 88 +- pkg/registry/factory_test.go | 408 ++++++ pkg/registry/fbmessenger.go | 5 +- pkg/registry/hhfm.go | 5 +- pkg/registry/hirl.go | 5 +- pkg/registry/httphostheader.go | 5 +- pkg/registry/ndt.go | 7 +- pkg/registry/portfiltering.go | 7 +- pkg/registry/psiphon.go | 5 +- pkg/registry/quicping.go | 5 +- pkg/registry/run.go | 5 +- pkg/registry/signal.go | 5 +- pkg/registry/simplequicping.go | 5 +- pkg/registry/sniblocking.go | 5 +- pkg/registry/stunreachability.go | 5 +- pkg/registry/tcpping.go | 5 +- pkg/registry/telegram.go | 7 +- pkg/registry/tlsmiddlebox.go | 5 +- pkg/registry/tlsping.go | 5 +- pkg/registry/tlstool.go | 5 +- pkg/registry/tor.go | 5 +- pkg/registry/torsf.go | 5 +- pkg/registry/urlgetter.go | 5 +- pkg/registry/vanillator.go | 8 +- pkg/registry/webconnectivity.go | 7 +- pkg/registry/webconnectivityv05.go | 7 +- pkg/registry/whatsapp.go | 5 +- pkg/throttling/throttling.go | 33 +- pkg/tunnel/tordesktop.go | 4 +- pkg/tunnel/torembed.go | 9 +- pkg/tunnel/tormobile.go | 35 - pkg/tutorial/dslx/chapter02/README.md | 80 +- pkg/tutorial/dslx/chapter02/main.go | 80 +- 120 files changed, 5317 insertions(+), 2933 deletions(-) delete mode 100644 pkg/dslx/connpool.go delete mode 100644 pkg/dslx/connpool_test.go create mode 100644 pkg/dslx/qa_test.go create mode 100644 pkg/dslx/runtimecore.go create mode 100644 pkg/dslx/runtimemeasurex.go create mode 100644 pkg/dslx/runtimemeasurex_test.go create mode 100644 pkg/dslx/runtimeminimal.go create mode 100644 pkg/dslx/runtimeminimal_test.go create mode 100644 pkg/dslx/trace.go create mode 100644 pkg/libtor/iphoneos/arm64/.gitignore create mode 100644 pkg/libtor/iphonesimulator/amd64/.gitignore create mode 100644 pkg/libtor/iphonesimulator/arm64/.gitignore create mode 100644 pkg/progress/progress.go create mode 100644 pkg/progress/progress_test.go delete mode 100644 pkg/tunnel/tormobile.go diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 87255ace..0e748ea3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -11,5 +11,5 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-go@v3 with: - go-version: "1.20.8" + go-version: "1.20.10" - run: go build ./... diff --git a/UPSTREAM b/UPSTREAM index 7b2ea994..dbe2d418 100644 --- a/UPSTREAM +++ b/UPSTREAM @@ -1 +1 @@ -v3.19.0-alpha-1-gca2019be +v3.19.0-alpha-35-g5c50be42 diff --git a/go.mod b/go.mod index b6b3fc12..de3b1cf9 100644 --- a/go.mod +++ b/go.mod @@ -9,17 +9,16 @@ require ( github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 github.com/cloudflare/circl v1.3.3 github.com/cretz/bine v0.2.0 - github.com/google/go-cmp v0.5.9 + github.com/google/go-cmp v0.6.0 github.com/google/gopacket v1.1.19 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 github.com/gorilla/websocket v1.5.0 github.com/hexops/gotextdiff v1.0.3 github.com/miekg/dns v1.1.56 github.com/montanaflynn/stats v0.7.1 - github.com/ooni/go-libtor v1.1.8 github.com/ooni/netem v0.0.0-20230920215742-15f3ffec0107 - github.com/ooni/oocrypto v0.5.4 - github.com/ooni/oohttp v0.6.4 + github.com/ooni/oocrypto v0.5.5 + github.com/ooni/oohttp v0.6.5 github.com/ooni/probe-assets v0.19.0 github.com/pborman/getopt/v2 v2.1.0 github.com/pion/stun v0.6.1 @@ -34,7 +33,7 @@ require ( gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/goptlib v1.5.0 gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/v2 v2.6.1 golang.org/x/crypto v0.14.0 - golang.org/x/net v0.16.0 + golang.org/x/net v0.17.0 golang.org/x/sys v0.13.0 ) @@ -138,9 +137,9 @@ require ( github.com/xtaci/kcp-go/v5 v5.6.2 // indirect github.com/xtaci/smux v1.5.24 // indirect gitlab.com/yawning/bsaes.git v0.0.0-20190805113838-0a714cd429ec // indirect - golang.org/x/mod v0.12.0 // indirect + golang.org/x/mod v0.13.0 // indirect golang.org/x/term v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect - golang.org/x/tools v0.13.0 // indirect + golang.org/x/tools v0.14.0 // indirect google.golang.org/protobuf v1.31.0 // indirect ) diff --git a/go.sum b/go.sum index 57ef91a7..15ab4c57 100644 --- a/go.sum +++ b/go.sum @@ -64,7 +64,6 @@ github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= -github.com/cretz/bine v0.1.0/go.mod h1:6PF6fWAvYtwjRGkAuDEJeWNOv3a2hUouSP/yRYXmvHw= github.com/cretz/bine v0.2.0 h1:8GiDRGlTgz+o8H9DSnsl+5MeBK4HsExxgl6WgzOCuZo= github.com/cretz/bine v0.2.0/go.mod h1:WU4o9QR9wWp8AVKtTM1XD5vUHkEqnf2vVSo6dBqbetI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -149,8 +148,8 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 h1:pUa4ghanp6q4IJHwE9RwLgmVFfReJN+KbQ8ExNEUUoQ= @@ -304,14 +303,12 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= -github.com/ooni/go-libtor v1.1.8 h1:Wo3V3DVTxl5vZdxtQakqYP+DAHx7pPtAFSl1bnAa08w= -github.com/ooni/go-libtor v1.1.8/go.mod h1:q1YyLwRD9GeMyeerVvwc0vJ2YgwDLTp2bdVcrh/JXyI= github.com/ooni/netem v0.0.0-20230920215742-15f3ffec0107 h1:PktaCPQ1NYZOaK+J8pQGYiPCYFkGR5H3ZURg9zPkQsI= github.com/ooni/netem v0.0.0-20230920215742-15f3ffec0107/go.mod h1:5X3Lk4+cnrwrQiYgRlCWXgV33IMDgLaO5s1x0DD/fO0= -github.com/ooni/oocrypto v0.5.4 h1:/AkVZd+aq54+OXgOtWEmK8xgZsFQtlmtPf2VgY20YWw= -github.com/ooni/oocrypto v0.5.4/go.mod h1:HjEQ5pQBl6btcWgAsKKq1tFo8CfBrZu63C/vPAUGIDk= -github.com/ooni/oohttp v0.6.4 h1:QZyOO4e88AzLOHGTgapXmsjtn1EVR7Wl+BtHd8okIf4= -github.com/ooni/oohttp v0.6.4/go.mod h1:RipdYAUiw1UTnpm0ISd0r1Kiv/CGaRUgn08xbK1JgVo= +github.com/ooni/oocrypto v0.5.5 h1:x0wIgtBfghVu8Ok0tR/xVyfHlo646hN1LB/5bzuXcIg= +github.com/ooni/oocrypto v0.5.5/go.mod h1:HjEQ5pQBl6btcWgAsKKq1tFo8CfBrZu63C/vPAUGIDk= +github.com/ooni/oohttp v0.6.5 h1:hjMnX2fGNHYHqh1JmfxoTfnN9JmdgT0fa6yIEjoYhG8= +github.com/ooni/oohttp v0.6.5/go.mod h1:RipdYAUiw1UTnpm0ISd0r1Kiv/CGaRUgn08xbK1JgVo= github.com/ooni/probe-assets v0.19.0 h1:XloDJQt6uxn6EYVwfWCOnlgsJZbmzO7VPFsJ8RPW8Ns= github.com/ooni/probe-assets v0.19.0/go.mod h1:m0k2FFzcLfFm7dhgyYkLCUR3R0CoRPr0jcjctDS2+gU= github.com/oschwald/geoip2-golang v1.9.0 h1:uvD3O6fXAXs+usU+UGExshpdP13GAqp4GBrzN7IgKZc= @@ -530,7 +527,6 @@ go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -568,8 +564,8 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= +golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -596,8 +592,8 @@ golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= -golang.org/x/net v0.16.0 h1:7eBu7KsSvFDtSXUIDbh3aqlK4DPsZ1rByC8PFfBThos= -golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -606,7 +602,7 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -698,8 +694,8 @@ golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ= -golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= +golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/pkg/checkincache/checkincache.go b/pkg/checkincache/checkincache.go index db312b76..566f0d04 100644 --- a/pkg/checkincache/checkincache.go +++ b/pkg/checkincache/checkincache.go @@ -3,6 +3,7 @@ package checkincache import ( "encoding/json" + "fmt" "time" "github.com/ooni/probe-engine/pkg/model" @@ -25,6 +26,9 @@ type checkInFlagsWrapper struct { } // Store stores the result of the latest check-in in the given key-value store. +// +// We store check-in feature flags in a file called checkinflags.state. These flags +// are valid for 24 hours, after which we consider them stale. func Store(kvStore model.KeyValueStore, resp *model.OOAPICheckInResult) error { // store the check-in flags in the key-value store wrapper := &checkInFlagsWrapper{ @@ -52,3 +56,16 @@ func GetFeatureFlag(kvStore model.KeyValueStore, name string) bool { } return wrapper.Flags[name] // works even if map is nil } + +// ExperimentEnabledKey returns the [model.KeyValueStore] key to use to +// know whether a disabled experiment has been enabled via check-in. +func ExperimentEnabledKey(name string) string { + return fmt.Sprintf("%s_enabled", name) +} + +// ExperimentEnabled returns whether a given experiment has been enabled by a previous +// execution of check-in. Some experiments are disabled by default for different reasons +// and we use the check-in API to control whether and when they should be enabled. +func ExperimentEnabled(kvStore model.KeyValueStore, name string) bool { + return GetFeatureFlag(kvStore, ExperimentEnabledKey(name)) +} diff --git a/pkg/cmd/buildtool/android.go b/pkg/cmd/buildtool/android.go index f4a1e679..18386938 100644 --- a/pkg/cmd/buildtool/android.go +++ b/pkg/cmd/buildtool/android.go @@ -26,40 +26,57 @@ func androidSubcommand() *cobra.Command { Use: "android", Short: "Builds ooniprobe, miniooni, and oonimkall for android", } + cmd.AddCommand(&cobra.Command{ Use: "gomobile", Short: "Builds oonimkall for android using gomobile", Run: func(cmd *cobra.Command, args []string) { + // Implementation note: perform the check here such that we can + // run unit test for the building code from any system + runtimex.Assert( + runtime.GOOS == "darwin" || runtime.GOOS == "linux", + "this command requires darwin or linux", + ) androidBuildGomobile(&buildDeps{}) }, }) + cmd.AddCommand(&cobra.Command{ Use: "cli", Short: "Builds ooniprobe and miniooni for usage within termux", Run: func(cmd *cobra.Command, args []string) { + // Implementation note: perform the check here such that we can + // run unit test for the building code from any system + runtimex.Assert( + runtime.GOOS == "darwin" || runtime.GOOS == "linux", + "this command requires darwin or linux", + ) androidBuildCLIAll(&buildDeps{}) }, }) + cmd.AddCommand(&cobra.Command{ - Use: "cdeps {zlib|openssl|libevent|tor} [zlib|openssl|libevent|tor...]", + Use: "cdeps [zlib|openssl|libevent|tor...]", Short: "Cross compiles C dependencies for Android", Run: func(cmd *cobra.Command, args []string) { for _, arg := range args { + // Implementation note: perform the check here such that we can + // run unit test for the building code from any system + runtimex.Assert( + runtime.GOOS == "darwin" || runtime.GOOS == "linux", + "this command requires darwin or linux", + ) androidCdepsBuildMain(arg, &buildDeps{}) } }, Args: cobra.MinimumNArgs(1), }) + return cmd } // androidBuildGomobile invokes the gomobile build. func androidBuildGomobile(deps buildtoolmodel.Dependencies) { - runtimex.Assert( - runtime.GOOS == "darwin" || runtime.GOOS == "linux", - "this command requires darwin or linux", - ) - deps.PsiphonMaybeCopyConfigFiles() deps.GolangCheck() @@ -128,11 +145,6 @@ func androidNDKCheck(androidHome string) string { // androidBuildCLIAll builds all products in CLI mode for Android func androidBuildCLIAll(deps buildtoolmodel.Dependencies) { - runtimex.Assert( - runtime.GOOS == "darwin" || runtime.GOOS == "linux", - "this command requires darwin or linux", - ) - deps.PsiphonMaybeCopyConfigFiles() deps.GolangCheck() @@ -161,7 +173,7 @@ func androidBuildCLIProductArch( androidHome string, ndkDir string, ) { - cgo := newAndroidCBuildEnv(androidHome, ndkDir, ooniArch) + cgo := androidNewCBuildEnv(androidHome, ndkDir, ooniArch) log.Infof("building %s for android/%s", product.Pkg, ooniArch) @@ -203,33 +215,33 @@ func androidBuildCLIProductArch( runtimex.Try0(shellx.RunEx(defaultShellxConfig(), argv, envp)) } -// newAndroidCBuildEnv creates a new [cBuildEnv] for the +// androidNewCBuildEnv creates a new [cBuildEnv] for the // given ooniArch ("arm", "arm64", "386", "amd64"). -func newAndroidCBuildEnv(androidHome, ndkDir, ooniArch string) *cBuildEnv { +func androidNewCBuildEnv(androidHome, ndkDir, ooniArch string) *cBuildEnv { binpath := androidNDKBinPath(ndkDir) destdir := runtimex.Try1(filepath.Abs(filepath.Join( // must be absolute "internal", "libtor", "android", ooniArch, ))) out := &cBuildEnv{ - ANDROID_HOME: androidHome, - ANDROID_NDK_ROOT: ndkDir, - AS: "", // later - AR: filepath.Join(binpath, "llvm-ar"), - BINPATH: binpath, - CC: "", // later - CFLAGS: androidCflags(ooniArch), - CONFIGURE_HOST: "", // later - DESTDIR: destdir, - CXX: "", // later - CXXFLAGS: androidCflags(ooniArch), - GOARCH: ooniArch, - GOARM: "", // maybe later - LD: filepath.Join(binpath, "ld"), - LDFLAGS: []string{}, // empty - OPENSSL_API_DEFINE: "-D__ANDROID_API__=21", - OPENSSL_COMPILER: "", // later - RANLIB: filepath.Join(binpath, "llvm-ranlib"), - STRIP: filepath.Join(binpath, "llvm-strip"), + ANDROID_HOME: androidHome, + ANDROID_NDK_ROOT: ndkDir, + AS: "", // later + AR: filepath.Join(binpath, "llvm-ar"), + BINPATH: binpath, + CC: "", // later + CFLAGS: androidCflags(ooniArch), + CONFIGURE_HOST: "", // later + DESTDIR: destdir, + CXX: "", // later + CXXFLAGS: androidCflags(ooniArch), + GOARCH: ooniArch, + GOARM: "", // maybe later + LD: filepath.Join(binpath, "ld"), + LDFLAGS: []string{}, // empty + OPENSSL_COMPILER: "", // later + OPENSSL_POST_COMPILER_FLAGS: []string{"-D__ANDROID_API__=21"}, + RANLIB: filepath.Join(binpath, "llvm-ranlib"), + STRIP: filepath.Join(binpath, "llvm-strip"), } switch ooniArch { case "arm": @@ -375,10 +387,6 @@ func androidNDKBinPath(ndkDir string) string { // androidCdepsBuildMain builds C dependencies for android. func androidCdepsBuildMain(name string, deps buildtoolmodel.Dependencies) { - runtimex.Assert( - runtime.GOOS == "darwin" || runtime.GOOS == "linux", - "this command requires darwin or linux", - ) androidHome := deps.AndroidSDKCheck() ndkDir := deps.AndroidNDKCheck(androidHome) archs := []string{"arm", "arm64", "386", "amd64"} @@ -395,7 +403,7 @@ func androidCdepsBuildArch( ndkDir string, name string, ) { - cdenv := newAndroidCBuildEnv(androidHome, ndkDir, arch) + cdenv := androidNewCBuildEnv(androidHome, ndkDir, arch) switch name { case "libevent": cdepsLibeventBuildMain(cdenv, deps) diff --git a/pkg/cmd/buildtool/android_test.go b/pkg/cmd/buildtool/android_test.go index 231c4758..70645ee9 100644 --- a/pkg/cmd/buildtool/android_test.go +++ b/pkg/cmd/buildtool/android_test.go @@ -754,12 +754,6 @@ func TestAndroidBuildCdepsOpenSSL(t *testing.T) { "DESTDIR=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm", "install_dev", }, - }, { - Env: []string{}, - Argv: []string{ - "rm", "-rf", - faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm/lib/pkgconfig", - }, }, { Env: []string{}, Argv: []string{ @@ -815,12 +809,6 @@ func TestAndroidBuildCdepsOpenSSL(t *testing.T) { "DESTDIR=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm64", "install_dev", }, - }, { - Env: []string{}, - Argv: []string{ - "rm", "-rf", - faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm64/lib/pkgconfig", - }, }, { Env: []string{}, Argv: []string{ @@ -876,12 +864,6 @@ func TestAndroidBuildCdepsOpenSSL(t *testing.T) { "DESTDIR=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/386", "install_dev", }, - }, { - Env: []string{}, - Argv: []string{ - "rm", "-rf", - faketopdir + "/internal/cmd/buildtool/internal/libtor/android/386/lib/pkgconfig", - }, }, { Env: []string{}, Argv: []string{ @@ -937,12 +919,6 @@ func TestAndroidBuildCdepsOpenSSL(t *testing.T) { "DESTDIR=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/amd64", "install_dev", }, - }, { - Env: []string{}, - Argv: []string{ - "rm", "-rf", - faketopdir + "/internal/cmd/buildtool/internal/libtor/android/amd64/lib/pkgconfig", - }, }}, }} @@ -1045,6 +1021,7 @@ func TestAndroidBuildCdepsLibevent(t *testing.T) { "CXXFLAGS=-fdata-sections -ffunction-sections -fstack-protector-strong -funwind-tables -no-canonical-prefixes -D_FORTIFY_SOURCE=2 -fpic -mthumb -Oz -DANDROID", "-I"+faketopdir+"/internal/cmd/buildtool/internal/libtor/android/arm/include", ), + "PKG_CONFIG_PATH=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm/lib/pkgconfig", }, Argv: []string{ "./configure", @@ -1077,8 +1054,36 @@ func TestAndroidBuildCdepsLibevent(t *testing.T) { Env: []string{}, Argv: []string{ "rm", - "-rf", - faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm/lib/pkgconfig", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm/lib/pkgconfig/libevent.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm/lib/pkgconfig/libevent_core.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm/lib/pkgconfig/libevent_extra.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm/lib/pkgconfig/libevent_openssl.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm/lib/pkgconfig/libevent_pthreads.pc", }, }, { Env: []string{}, @@ -1195,6 +1200,7 @@ func TestAndroidBuildCdepsLibevent(t *testing.T) { "CXXFLAGS=-fdata-sections -ffunction-sections -fstack-protector-strong -funwind-tables -no-canonical-prefixes -D_FORTIFY_SOURCE=2 -fpic -O2 -DANDROID", "-I"+faketopdir+"/internal/cmd/buildtool/internal/libtor/android/arm64/include", ), + "PKG_CONFIG_PATH=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm64/lib/pkgconfig", }, Argv: []string{ "./configure", @@ -1227,8 +1233,36 @@ func TestAndroidBuildCdepsLibevent(t *testing.T) { Env: []string{}, Argv: []string{ "rm", - "-rf", - faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm64/lib/pkgconfig", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm64/lib/pkgconfig/libevent.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm64/lib/pkgconfig/libevent_core.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm64/lib/pkgconfig/libevent_extra.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm64/lib/pkgconfig/libevent_openssl.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm64/lib/pkgconfig/libevent_pthreads.pc", }, }, { Env: []string{}, @@ -1345,6 +1379,7 @@ func TestAndroidBuildCdepsLibevent(t *testing.T) { "CXXFLAGS=-fdata-sections -ffunction-sections -fstack-protector-strong -funwind-tables -no-canonical-prefixes -D_FORTIFY_SOURCE=2 -fPIC -O2 -DANDROID -mstackrealign", "-I"+faketopdir+"/internal/cmd/buildtool/internal/libtor/android/386/include", ), + "PKG_CONFIG_PATH=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/386/lib/pkgconfig", }, Argv: []string{ "./configure", @@ -1377,8 +1412,36 @@ func TestAndroidBuildCdepsLibevent(t *testing.T) { Env: []string{}, Argv: []string{ "rm", - "-rf", - faketopdir + "/internal/cmd/buildtool/internal/libtor/android/386/lib/pkgconfig", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/386/lib/pkgconfig/libevent.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/386/lib/pkgconfig/libevent_core.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/386/lib/pkgconfig/libevent_extra.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/386/lib/pkgconfig/libevent_openssl.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/386/lib/pkgconfig/libevent_pthreads.pc", }, }, { Env: []string{}, @@ -1495,6 +1558,7 @@ func TestAndroidBuildCdepsLibevent(t *testing.T) { "CXXFLAGS=-fdata-sections -ffunction-sections -fstack-protector-strong -funwind-tables -no-canonical-prefixes -D_FORTIFY_SOURCE=2 -fPIC -O2 -DANDROID", "-I"+faketopdir+"/internal/cmd/buildtool/internal/libtor/android/amd64/include", ), + "PKG_CONFIG_PATH=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/amd64/lib/pkgconfig", }, Argv: []string{ "./configure", @@ -1527,8 +1591,36 @@ func TestAndroidBuildCdepsLibevent(t *testing.T) { Env: []string{}, Argv: []string{ "rm", - "-rf", - faketopdir + "/internal/cmd/buildtool/internal/libtor/android/amd64/lib/pkgconfig", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/amd64/lib/pkgconfig/libevent.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/amd64/lib/pkgconfig/libevent_core.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/amd64/lib/pkgconfig/libevent_extra.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/amd64/lib/pkgconfig/libevent_openssl.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/amd64/lib/pkgconfig/libevent_pthreads.pc", }, }, { Env: []string{}, @@ -1701,6 +1793,7 @@ func TestAndroidBuildCdepsTor(t *testing.T) { "--disable-tool-name-check", "--disable-systemd", "--prefix=/", + "--disable-unittests", }, }, { Env: []string{}, @@ -1777,6 +1870,7 @@ func TestAndroidBuildCdepsTor(t *testing.T) { "--disable-tool-name-check", "--disable-systemd", "--prefix=/", + "--disable-unittests", }, }, { Env: []string{}, @@ -1853,6 +1947,7 @@ func TestAndroidBuildCdepsTor(t *testing.T) { "--disable-tool-name-check", "--disable-systemd", "--prefix=/", + "--disable-unittests", }, }, { Env: []string{}, @@ -1929,6 +2024,7 @@ func TestAndroidBuildCdepsTor(t *testing.T) { "--disable-tool-name-check", "--disable-systemd", "--prefix=/", + "--disable-unittests", }, }, { Env: []string{}, diff --git a/pkg/cmd/buildtool/builddeps.go b/pkg/cmd/buildtool/builddeps.go index 68121b3b..339736ca 100644 --- a/pkg/cmd/buildtool/builddeps.go +++ b/pkg/cmd/buildtool/builddeps.go @@ -82,3 +82,8 @@ func (*buildDeps) GOOS() string { func (*buildDeps) VerifySHA256(expectedSHA256 string, tarball string) { cdepsMustVerifySHA256(expectedSHA256, tarball) } + +// XCRun implements buildtoolmodel.Dependencies +func (*buildDeps) XCRun(args ...string) string { + return iosXCRun(args...) +} diff --git a/pkg/cmd/buildtool/cbuildenv.go b/pkg/cmd/buildtool/cbuildenv.go index c0e3a045..7d39f1ff 100644 --- a/pkg/cmd/buildtool/cbuildenv.go +++ b/pkg/cmd/buildtool/cbuildenv.go @@ -61,12 +61,12 @@ type cBuildEnv struct { // LDFLAGS contains the LDFLAGS to use when compiling. LDFLAGS []string - // OPENSSL_API_DEFINE is an extra define we need to add on Android. - OPENSSL_API_DEFINE string - // OPENSSL_COMPILER is the compiler name for OpenSSL. OPENSSL_COMPILER string + // OPENSSL_POST_COMPILER_FLAGS contains extra flags to pass after OPENSSL_COMPILER + OPENSSL_POST_COMPILER_FLAGS []string + // RANLIB is the path to the ranlib tool. RANLIB string @@ -86,29 +86,30 @@ type cBuildEnv struct { // environment variables to CFLAGS, CXXFLAGS, etc. func cBuildMerge(global, local *cBuildEnv) *cBuildEnv { out := &cBuildEnv{ - ANDROID_HOME: global.ANDROID_HOME, - ANDROID_NDK_ROOT: global.ANDROID_NDK_ROOT, - AR: global.AR, - AS: global.AS, - BINPATH: global.BINPATH, - CC: global.CC, - CFLAGS: append([]string{}, global.CFLAGS...), - CONFIGURE_HOST: global.CONFIGURE_HOST, - DESTDIR: global.DESTDIR, - CXX: global.CXX, - CXXFLAGS: append([]string{}, global.CXXFLAGS...), - GOARCH: global.GOARCH, - GOARM: global.GOARM, - LD: global.LD, - LDFLAGS: append([]string{}, global.LDFLAGS...), - OPENSSL_API_DEFINE: global.OPENSSL_API_DEFINE, - OPENSSL_COMPILER: global.OPENSSL_COMPILER, - RANLIB: global.RANLIB, - STRIP: global.STRIP, + ANDROID_HOME: global.ANDROID_HOME, + ANDROID_NDK_ROOT: global.ANDROID_NDK_ROOT, + AR: global.AR, + AS: global.AS, + BINPATH: global.BINPATH, + CC: global.CC, + CFLAGS: append([]string{}, global.CFLAGS...), + CONFIGURE_HOST: global.CONFIGURE_HOST, + DESTDIR: global.DESTDIR, + CXX: global.CXX, + CXXFLAGS: append([]string{}, global.CXXFLAGS...), + GOARCH: global.GOARCH, + GOARM: global.GOARM, + LD: global.LD, + LDFLAGS: append([]string{}, global.LDFLAGS...), + OPENSSL_COMPILER: global.OPENSSL_COMPILER, + OPENSSL_POST_COMPILER_FLAGS: append([]string{}, global.OPENSSL_POST_COMPILER_FLAGS...), + RANLIB: global.RANLIB, + STRIP: global.STRIP, } out.CFLAGS = append(out.CFLAGS, local.CFLAGS...) out.CXXFLAGS = append(out.CXXFLAGS, local.CXXFLAGS...) out.LDFLAGS = append(out.LDFLAGS, local.LDFLAGS...) + out.OPENSSL_POST_COMPILER_FLAGS = append(out.OPENSSL_POST_COMPILER_FLAGS, local.OPENSSL_POST_COMPILER_FLAGS...) return out } diff --git a/pkg/cmd/buildtool/cdepslibevent.go b/pkg/cmd/buildtool/cdepslibevent.go index 90eaf5bb..fe7e0599 100644 --- a/pkg/cmd/buildtool/cdepslibevent.go +++ b/pkg/cmd/buildtool/cdepslibevent.go @@ -49,6 +49,10 @@ func cdepsLibeventBuildMain(globalEnv *cBuildEnv, deps buildtoolmodel.Dependenci } envp := cBuildExportAutotools(cBuildMerge(globalEnv, localEnv)) + // On iOS, we need PKG_CONFIG_PATH to convince libevent to use the OpenSSL we built and + // always letting libevent's configure use pkgconfig is actually fine. + envp.Append("PKG_CONFIG_PATH", filepath.Join(globalEnv.DESTDIR, "lib", "pkgconfig")) + argv := runtimex.Try1(shellx.NewArgv("./configure")) if globalEnv.CONFIGURE_HOST != "" { argv.Append("--host=" + globalEnv.CONFIGURE_HOST) @@ -59,7 +63,17 @@ func cdepsLibeventBuildMain(globalEnv *cBuildEnv, deps buildtoolmodel.Dependenci must.Run(log.Log, "make", "V=1", "-j", strconv.Itoa(runtime.NumCPU())) must.Run(log.Log, "make", "DESTDIR="+globalEnv.DESTDIR, "install") must.Run(log.Log, "rm", "-rf", filepath.Join(globalEnv.DESTDIR, "bin")) - must.Run(log.Log, "rm", "-rf", filepath.Join(globalEnv.DESTDIR, "lib", "pkgconfig")) + + // We used to delete the whole pkgconfig directory but libevent's build needs OpenSSL's + // pkgconfig, so removing the whole directory means rebuilding libevent requires building + // OpenSSL because its pkgconfig is also removed. We discovered this need when working + // on the https://github.com/ooni/probe-cli/pull/1366 MVP. To keep the build idempotent, + // let's be more gentle and just remove libevent's pkgconfig files instead. + must.Run(log.Log, "rm", "-f", filepath.Join(globalEnv.DESTDIR, "lib", "pkgconfig", "libevent.pc")) + must.Run(log.Log, "rm", "-f", filepath.Join(globalEnv.DESTDIR, "lib", "pkgconfig", "libevent_core.pc")) + must.Run(log.Log, "rm", "-f", filepath.Join(globalEnv.DESTDIR, "lib", "pkgconfig", "libevent_extra.pc")) + must.Run(log.Log, "rm", "-f", filepath.Join(globalEnv.DESTDIR, "lib", "pkgconfig", "libevent_openssl.pc")) + must.Run(log.Log, "rm", "-f", filepath.Join(globalEnv.DESTDIR, "lib", "pkgconfig", "libevent_pthreads.pc")) // we just need libevent.a must.Run(log.Log, "rm", "-rf", filepath.Join(globalEnv.DESTDIR, "lib", "libevent.la")) diff --git a/pkg/cmd/buildtool/cdepsopenssl.go b/pkg/cmd/buildtool/cdepsopenssl.go index bb9fd193..75411bfa 100644 --- a/pkg/cmd/buildtool/cdepsopenssl.go +++ b/pkg/cmd/buildtool/cdepsopenssl.go @@ -67,9 +67,7 @@ func cdepsOpenSSLBuildMain(globalEnv *cBuildEnv, deps buildtoolmodel.Dependencie "no-rc2", "no-rc4", "no-rc5", "no-rmd160", "no-whirlpool", "no-dso", "no-ui-console", "no-shared", "no-unit-test", globalEnv.OPENSSL_COMPILER, )) - if globalEnv.OPENSSL_API_DEFINE != "" { - argv.Append(globalEnv.OPENSSL_API_DEFINE) - } + argv.Append(globalEnv.OPENSSL_POST_COMPILER_FLAGS...) argv.Append("--libdir=lib", "--prefix=/", "--openssldir=/") runtimex.Try0(shellx.RunEx(defaultShellxConfig(), argv, envp)) @@ -84,5 +82,8 @@ func cdepsOpenSSLBuildMain(globalEnv *cBuildEnv, deps buildtoolmodel.Dependencie )) must.Run(log.Log, "make", "DESTDIR="+globalEnv.DESTDIR, "install_dev") - must.Run(log.Log, "rm", "-rf", filepath.Join(globalEnv.DESTDIR, "lib", "pkgconfig")) + + // We used to delete the pkgconfig but it turns out this is important for libevent iOS builds, which + // means now we need to keep it. See https://github.com/ooni/probe-cli/pull/1369 for details. + //must.Run(log.Log, "rm", "-rf", filepath.Join(globalEnv.DESTDIR, "lib", "pkgconfig")) } diff --git a/pkg/cmd/buildtool/cdepstor.go b/pkg/cmd/buildtool/cdepstor.go index 52a7883b..37508d6e 100644 --- a/pkg/cmd/buildtool/cdepstor.go +++ b/pkg/cmd/buildtool/cdepstor.go @@ -56,6 +56,7 @@ func cdepsTorBuildMain(globalEnv *cBuildEnv, deps buildtoolmodel.Dependencies) { "--disable-tool-name-check", "--disable-systemd", "--prefix=/", + "--disable-unittests", ) runtimex.Try0(shellx.RunEx(defaultShellxConfig(), argv, envp)) diff --git a/pkg/cmd/buildtool/internal/buildtoolmodel/buildtoolmodel.go b/pkg/cmd/buildtool/internal/buildtoolmodel/buildtoolmodel.go index 67966e3f..bac8c685 100644 --- a/pkg/cmd/buildtool/internal/buildtoolmodel/buildtoolmodel.go +++ b/pkg/cmd/buildtool/internal/buildtoolmodel/buildtoolmodel.go @@ -18,6 +18,9 @@ type Dependencies interface { // function returns the Android home path. AndroidSDKCheck() string + // GOOS returns the current GOOS. + GOOS() string + // GOPATH returns the current GOPATH. GOPATH() string @@ -50,6 +53,7 @@ type Dependencies interface { // expected version of mingw-w64. WindowsMingwCheck() - // GOOS returns the current GOOS. - GOOS() string + // XCRun executes Xcode's xcrun tool with the given arguments and returns + // the first line of text emitted by xcrun or PANICS on failure. + XCRun(args ...string) string } diff --git a/pkg/cmd/buildtool/internal/buildtooltest/buildtooltest.go b/pkg/cmd/buildtool/internal/buildtooltest/buildtooltest.go index aa6969c1..62ae14b2 100644 --- a/pkg/cmd/buildtool/internal/buildtooltest/buildtooltest.go +++ b/pkg/cmd/buildtool/internal/buildtooltest/buildtooltest.go @@ -85,7 +85,7 @@ func CheckSingleCommand(cmd *execabs.Cmd, tee ExecExpectations) error { return err } if err := CompareEnv(tee.Env, shellxtesting.CmdEnvironMinusOsEnviron(cmd)); err != nil { - return err + return fmt.Errorf("in %v: %w", tee.Argv, err) } return nil } @@ -245,3 +245,22 @@ func (cc *DependenciesCallCounter) increment(name string) { } cc.Counter[name]++ } + +// XCRun implements buildtoolmodel.Dependencies. +func (*DependenciesCallCounter) XCRun(args ...string) string { + runtimex.Assert(len(args) >= 1, "expected at least one argument") + switch args[0] { + case "-sdk": + runtimex.Assert(len(args) == 3, "expected three arguments") + runtimex.Assert(args[2] == "--show-sdk-path", "the third argument must be --show-sdk-path") + return string(filepath.Separator) + filepath.Join("Developer", "SDKs", args[1]) + + case "-find": + runtimex.Assert(len(args) == 4, "expected four arguments") + runtimex.Assert(args[1] == "-sdk", "the second argument must be -sdk") + return string(filepath.Separator) + filepath.Join("Developer", "SDKs", args[2], "bin", args[3]) + + default: + panic(errors.New("the first argument must be -sdk or -find")) + } +} diff --git a/pkg/cmd/buildtool/ios.go b/pkg/cmd/buildtool/ios.go index 4d91d8f7..2daad99e 100644 --- a/pkg/cmd/buildtool/ios.go +++ b/pkg/cmd/buildtool/ios.go @@ -5,10 +5,15 @@ package main // import ( + "errors" + "fmt" "path/filepath" + "runtime" "github.com/apex/log" "github.com/ooni/probe-engine/pkg/cmd/buildtool/internal/buildtoolmodel" + "github.com/ooni/probe-engine/pkg/must" + "github.com/ooni/probe-engine/pkg/runtimex" "github.com/ooni/probe-engine/pkg/shellx" "github.com/spf13/cobra" ) @@ -19,6 +24,7 @@ func iosSubcommand() *cobra.Command { Use: "ios", Short: "Builds oonimkall and its dependencies for iOS", } + cmd.AddCommand(&cobra.Command{ Use: "gomobile", Short: "Builds oonimkall for iOS using gomobile", @@ -26,6 +32,21 @@ func iosSubcommand() *cobra.Command { iosBuildGomobile(&buildDeps{}) }, }) + + cmd.AddCommand(&cobra.Command{ + Use: "cdeps [zlib|openssl|libevent|tor...]", + Short: "Cross compiles C dependencies for iOS", + Run: func(cmd *cobra.Command, args []string) { + // Implementation note: perform the check here such that we can + // run unit test for the building code from any system + runtimex.Assert(runtime.GOOS == "darwin", "this command requires darwin") + for _, arg := range args { + iosCdepsBuildMain(arg, &buildDeps{}) + } + }, + Args: cobra.MinimumNArgs(1), + }) + return cmd } @@ -41,6 +62,132 @@ func iosBuildGomobile(deps buildtoolmodel.Dependencies) { output: filepath.Join("MOBILE", "ios", "oonimkall.xcframework"), target: "ios", } + log.Info("building the mobile library using gomobile") gomobileBuild(config) } + +// iosCdepsBuildMain builds C dependencies for ios. +func iosCdepsBuildMain(name string, deps buildtoolmodel.Dependencies) { + // The ooni/probe-ios app explicitly only targets amd64 and arm64. It also targets + // as the minimum version iOS 12, while one cannot target a version of iOS > 10 when + // building for 32-bit targets. Hence, using only 64 bit archs here is fine. + iosCdepsBuildArch(deps, name, "iphoneos", "arm64") + iosCdepsBuildArch(deps, name, "iphonesimulator", "arm64") + iosCdepsBuildArch(deps, name, "iphonesimulator", "amd64") +} + +// iosAppleArchForOONIArch maps the ooniArch to the corresponding apple arch +var iosAppleArchForOONIArch = map[string]string{ + "amd64": "x86_64", + "arm64": "arm64", +} + +// iosCdepsBuildArch builds the given dependency for the given arch +func iosCdepsBuildArch(deps buildtoolmodel.Dependencies, name, platform, ooniArch string) { + cdenv := iosNewCBuildEnv(deps, platform, ooniArch) + switch name { + case "libevent": + cdepsLibeventBuildMain(cdenv, deps) + case "openssl": + cdepsOpenSSLBuildMain(cdenv, deps) + case "tor": + cdepsTorBuildMain(cdenv, deps) + case "zlib": + cdepsZlibBuildMain(cdenv, deps) + default: + panic(fmt.Errorf("unknown dependency: %s", name)) + } +} + +// iosMinVersion is the minimum version that we support. We're using the +// same value used by the ooni/probe-ios app as of 2023-10.12. +const iosMinVersion = "12.0" + +// iosNewCBuildEnv creates a new [cBuildEnv] for the given ooniArch ("arm64" or "amd64"). +func iosNewCBuildEnv(deps buildtoolmodel.Dependencies, platform, ooniArch string) *cBuildEnv { + destdir := runtimex.Try1(filepath.Abs(filepath.Join( // must be absolute + "internal", "libtor", platform, ooniArch, + ))) + + var ( + appleArch = iosAppleArchForOONIArch[ooniArch] + + // minVersionFlag sets the correct flag for the compiler depending on whether + // we're using the iphoneos or iphonesimulator platform. + // + // Note: the documentation of clang fetched on 2023-10-12 explicitly mentions that + // ios-version-min is an alias for iphoneos-version-min. Likewise, ios-simulator-version-min + // aliaes iphonesimulator-version-min. + // + // See https://clang.llvm.org/docs/ClangCommandLineReference.html#cmdoption-clang-mios-simulator-version-min + minVersionFlag = fmt.Sprintf("-m%s-version-min=", platform) + ) + runtimex.Assert(appleArch != "", "empty appleArch") + runtimex.Assert(minVersionFlag != "", "empty minVersionFlag") + + isysroot := deps.XCRun("-sdk", platform, "--show-sdk-path") + + out := &cBuildEnv{ + ANDROID_HOME: "", // not needed + ANDROID_NDK_ROOT: "", // not needed + AS: deps.XCRun("-find", "-sdk", platform, "as"), + AR: deps.XCRun("-find", "-sdk", platform, "ar"), + BINPATH: "", // not needed + CC: deps.XCRun("-find", "-sdk", platform, "cc"), + CFLAGS: []string{ + "-isysroot", isysroot, + minVersionFlag + iosMinVersion, // tricky: they must be concatenated + "-arch", appleArch, + "-fembed-bitcode", + "-O2", + }, + CONFIGURE_HOST: "", // later + DESTDIR: destdir, + CXX: deps.XCRun("-find", "-sdk", platform, "c++"), + CXXFLAGS: []string{ + "-isysroot", isysroot, + minVersionFlag + iosMinVersion, // tricky: they must be concatenated + "-arch", appleArch, + "-fembed-bitcode", + "-O2", + }, + GOARCH: ooniArch, + GOARM: "", // not needed + LD: deps.XCRun("-find", "-sdk", platform, "ld"), + LDFLAGS: []string{ + "-isysroot", isysroot, + minVersionFlag + iosMinVersion, // tricky: they must be concatenated + "-arch", appleArch, + "-fembed-bitcode", + }, + OPENSSL_COMPILER: "", // later + OPENSSL_POST_COMPILER_FLAGS: []string{ + minVersionFlag + iosMinVersion, // tricky: they must be concatenated + "-fembed-bitcode", + }, + RANLIB: deps.XCRun("-find", "-sdk", platform, "ranlib"), + STRIP: deps.XCRun("-find", "-sdk", platform, "strip"), + } + + switch ooniArch { + case "arm64": + // TODO(https://github.com/ooni/probe/issues/2570): using ios64-xcrun here is wrong and + // we should instead use the simulator as discussed in the issue. + out.CONFIGURE_HOST = "arm-apple-darwin" + out.OPENSSL_COMPILER = "ios64-xcrun" + case "amd64": + out.CONFIGURE_HOST = "x86_64-apple-darwin" + out.OPENSSL_COMPILER = "iossimulator-xcrun" + default: + panic(errors.New("unsupported ooniArch")) + } + + return out +} + +// iosXCRun invokes `xcrun [args]` and returns its result of panics. This function +// is called indirectly by the iOS build through [buildtoolmodel.Dependencies]. +func iosXCRun(args ...string) string { + return string(must.FirstLineBytes(must.RunOutput(log.Log, "xcrun", args...))) +} diff --git a/pkg/cmd/buildtool/ios_test.go b/pkg/cmd/buildtool/ios_test.go index 4fe75329..ca140848 100644 --- a/pkg/cmd/buildtool/ios_test.go +++ b/pkg/cmd/buildtool/ios_test.go @@ -1,6 +1,9 @@ package main import ( + "fmt" + "runtime" + "strconv" "testing" "github.com/google/go-cmp/cmp" @@ -119,3 +122,1286 @@ func TestIOSBuildGomobile(t *testing.T) { }) } } + +func TestIOSBuildCdepsZlib(t *testing.T) { + faketopdir := (&buildtooltest.DependenciesCallCounter{}).AbsoluteCurDir() + + // testspec specifies a test case for this test + type testspec struct { + // name is the name of the test case + name string + + // expectations contains the commands we expect to see + expect []buildtooltest.ExecExpectations + } + + var testcases = []testspec{{ + name: "zlib", + expect: []buildtooltest.ExecExpectations{{ + Env: []string{}, + Argv: []string{ + "curl", "-fsSLO", "https://zlib.net/zlib-1.3.tar.gz", + }, + }, { + Env: []string{}, + Argv: []string{ + "tar", "-xf", "zlib-1.3.tar.gz", + }, + }, { + Env: []string{}, + Argv: []string{ + "git", "apply", faketopdir + "/CDEPS/zlib/000.patch", + }, + }, { + Env: []string{ + "AR=/Developer/SDKs/iphoneos/bin/ar", + "AS=/Developer/SDKs/iphoneos/bin/as", + "CC=/Developer/SDKs/iphoneos/bin/cc", + "CFLAGS=-isysroot /Developer/SDKs/iphoneos -miphoneos-version-min=12.0 -arch arm64 -fembed-bitcode -O2", + "CXX=/Developer/SDKs/iphoneos/bin/c++", + "CXXFLAGS=-isysroot /Developer/SDKs/iphoneos -miphoneos-version-min=12.0 -arch arm64 -fembed-bitcode -O2", + "LD=/Developer/SDKs/iphoneos/bin/ld", + "LDFLAGS=-isysroot /Developer/SDKs/iphoneos -miphoneos-version-min=12.0 -arch arm64 -fembed-bitcode", + "RANLIB=/Developer/SDKs/iphoneos/bin/ranlib", + "STRIP=/Developer/SDKs/iphoneos/bin/strip", + "CHOST=arm-apple-darwin", + }, + Argv: []string{ + "./configure", "--prefix=/", "--static", + }, + }, { + Env: []string{}, + Argv: []string{ + "make", "-j", strconv.Itoa(runtime.NumCPU()), + }, + }, { + Env: []string{}, + Argv: []string{ + "make", + "DESTDIR=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphoneos/arm64", + "install", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", "-rf", faketopdir + "/internal/cmd/buildtool/internal/libtor/iphoneos/arm64/lib/pkgconfig", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", "-rf", faketopdir + "/internal/cmd/buildtool/internal/libtor/iphoneos/arm64/share", + }, + }, { + Env: []string{}, + Argv: []string{ + "curl", "-fsSLO", "https://zlib.net/zlib-1.3.tar.gz", + }, + }, { + Env: []string{}, + Argv: []string{ + "tar", "-xf", "zlib-1.3.tar.gz", + }, + }, { + Env: []string{}, + Argv: []string{ + "git", "apply", faketopdir + "/CDEPS/zlib/000.patch", + }, + }, { + Env: []string{ + "AR=/Developer/SDKs/iphonesimulator/bin/ar", + "AS=/Developer/SDKs/iphonesimulator/bin/as", + "CC=/Developer/SDKs/iphonesimulator/bin/cc", + "CFLAGS=-isysroot /Developer/SDKs/iphonesimulator -miphonesimulator-version-min=12.0 -arch arm64 -fembed-bitcode -O2", + "CXX=/Developer/SDKs/iphonesimulator/bin/c++", + "CXXFLAGS=-isysroot /Developer/SDKs/iphonesimulator -miphonesimulator-version-min=12.0 -arch arm64 -fembed-bitcode -O2", + "LD=/Developer/SDKs/iphonesimulator/bin/ld", + "LDFLAGS=-isysroot /Developer/SDKs/iphonesimulator -miphonesimulator-version-min=12.0 -arch arm64 -fembed-bitcode", + "RANLIB=/Developer/SDKs/iphonesimulator/bin/ranlib", + "STRIP=/Developer/SDKs/iphonesimulator/bin/strip", + "CHOST=arm-apple-darwin", + }, + Argv: []string{ + "./configure", "--prefix=/", "--static", + }, + }, { + Env: []string{}, + Argv: []string{ + "make", "-j", strconv.Itoa(runtime.NumCPU()), + }, + }, { + Env: []string{}, + Argv: []string{ + "make", + "DESTDIR=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/arm64", + "install", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", "-rf", faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/arm64/lib/pkgconfig", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", "-rf", faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/arm64/share", + }, + }, { + Env: []string{}, + Argv: []string{ + "curl", "-fsSLO", "https://zlib.net/zlib-1.3.tar.gz", + }, + }, { + Env: []string{}, + Argv: []string{ + "tar", "-xf", "zlib-1.3.tar.gz", + }, + }, { + Env: []string{}, + Argv: []string{ + "git", "apply", faketopdir + "/CDEPS/zlib/000.patch", + }, + }, { + Env: []string{ + "AR=/Developer/SDKs/iphonesimulator/bin/ar", + "AS=/Developer/SDKs/iphonesimulator/bin/as", + "CC=/Developer/SDKs/iphonesimulator/bin/cc", + "CFLAGS=-isysroot /Developer/SDKs/iphonesimulator -miphonesimulator-version-min=12.0 -arch x86_64 -fembed-bitcode -O2", + "CXX=/Developer/SDKs/iphonesimulator/bin/c++", + "CXXFLAGS=-isysroot /Developer/SDKs/iphonesimulator -miphonesimulator-version-min=12.0 -arch x86_64 -fembed-bitcode -O2", + "LD=/Developer/SDKs/iphonesimulator/bin/ld", + "LDFLAGS=-isysroot /Developer/SDKs/iphonesimulator -miphonesimulator-version-min=12.0 -arch x86_64 -fembed-bitcode", + "RANLIB=/Developer/SDKs/iphonesimulator/bin/ranlib", + "STRIP=/Developer/SDKs/iphonesimulator/bin/strip", + "CHOST=x86_64-apple-darwin", + }, + Argv: []string{ + "./configure", "--prefix=/", "--static", + }, + }, { + Env: []string{}, + Argv: []string{ + "make", "-j", strconv.Itoa(runtime.NumCPU()), + }, + }, { + Env: []string{}, + Argv: []string{ + "make", + "DESTDIR=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/amd64", + "install", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", "-rf", faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/amd64/lib/pkgconfig", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", "-rf", faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/amd64/share", + }, + }}, + }} + + for _, testcase := range testcases { + t.Run(testcase.name, func(t *testing.T) { + + cc := &buildtooltest.SimpleCommandCollector{} + + deps := &buildtooltest.DependenciesCallCounter{ + HasPsiphon: false, + } + + shellxtesting.WithCustomLibrary(cc, func() { + iosCdepsBuildMain("zlib", deps) + }) + + expectCalls := map[string]int{ + buildtooltest.TagAbsoluteCurDir: 3, + buildtooltest.TagMustChdir: 3, + buildtooltest.TagVerifySHA256: 3, + } + + if diff := cmp.Diff(expectCalls, deps.Counter); diff != "" { + t.Fatal(diff) + } + + if err := buildtooltest.CheckManyCommands(cc.Commands, testcase.expect); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestIOSBuildCdepsOpenSSL(t *testing.T) { + faketopdir := (&buildtooltest.DependenciesCallCounter{}).AbsoluteCurDir() + + // testspec specifies a test case for this test + type testspec struct { + // name is the name of the test case + name string + + // expectations contains the commands we expect to see + expect []buildtooltest.ExecExpectations + } + + var testcases = []testspec{{ + name: "openssl", + expect: []buildtooltest.ExecExpectations{{ + Env: []string{}, + Argv: []string{ + "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.1.3.tar.gz", + }, + }, { + Env: []string{}, + Argv: []string{ + "tar", "-xf", "openssl-3.1.3.tar.gz", + }, + }, { + Env: []string{}, + Argv: []string{ + "git", "apply", faketopdir + "/CDEPS/openssl/000.patch", + }, + }, { + Env: []string{}, + Argv: []string{ + "git", "apply", faketopdir + "/CDEPS/openssl/001.patch", + }, + }, { + Env: []string{ + "CFLAGS=-isysroot /Developer/SDKs/iphoneos -miphoneos-version-min=12.0 -arch arm64 -fembed-bitcode -O2 -Wno-macro-redefined", + "LDFLAGS=-isysroot /Developer/SDKs/iphoneos -miphoneos-version-min=12.0 -arch arm64 -fembed-bitcode", + "CXXFLAGS=-isysroot /Developer/SDKs/iphoneos -miphoneos-version-min=12.0 -arch arm64 -fembed-bitcode -O2 -Wno-macro-redefined", + }, + Argv: []string{ + "./Configure", "no-comp", "no-dtls", "no-ec2m", "no-psk", "no-srp", + "no-ssl3", "no-camellia", "no-idea", "no-md2", "no-md4", "no-mdc2", + "no-rc2", "no-rc4", "no-rc5", "no-rmd160", "no-whirlpool", "no-dso", + "no-ui-console", "no-shared", "no-unit-test", "ios64-xcrun", + "-miphoneos-version-min=12.0", "-fembed-bitcode", + "--libdir=lib", "--prefix=/", "--openssldir=/", + }, + }, { + Env: []string{ + "CFLAGS=-isysroot /Developer/SDKs/iphoneos -miphoneos-version-min=12.0 -arch arm64 -fembed-bitcode -O2 -Wno-macro-redefined", + "LDFLAGS=-isysroot /Developer/SDKs/iphoneos -miphoneos-version-min=12.0 -arch arm64 -fembed-bitcode", + "CXXFLAGS=-isysroot /Developer/SDKs/iphoneos -miphoneos-version-min=12.0 -arch arm64 -fembed-bitcode -O2 -Wno-macro-redefined", + }, + Argv: []string{ + "make", "-j", strconv.Itoa(runtime.NumCPU()), + }, + }, { + Env: []string{}, + Argv: []string{ + "make", + "DESTDIR=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphoneos/arm64", + "install_dev", + }, + }, { + Env: []string{}, + Argv: []string{ + "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.1.3.tar.gz", + }, + }, { + Env: []string{}, + Argv: []string{ + "tar", "-xf", "openssl-3.1.3.tar.gz", + }, + }, { + Env: []string{}, + Argv: []string{ + "git", "apply", faketopdir + "/CDEPS/openssl/000.patch", + }, + }, { + Env: []string{}, + Argv: []string{ + "git", "apply", faketopdir + "/CDEPS/openssl/001.patch", + }, + }, { + Env: []string{ + "CFLAGS=-isysroot /Developer/SDKs/iphonesimulator -miphonesimulator-version-min=12.0 -arch arm64 -fembed-bitcode -O2 -Wno-macro-redefined", + "LDFLAGS=-isysroot /Developer/SDKs/iphonesimulator -miphonesimulator-version-min=12.0 -arch arm64 -fembed-bitcode", + "CXXFLAGS=-isysroot /Developer/SDKs/iphonesimulator -miphonesimulator-version-min=12.0 -arch arm64 -fembed-bitcode -O2 -Wno-macro-redefined", + }, + Argv: []string{ + "./Configure", "no-comp", "no-dtls", "no-ec2m", "no-psk", "no-srp", + "no-ssl3", "no-camellia", "no-idea", "no-md2", "no-md4", "no-mdc2", + "no-rc2", "no-rc4", "no-rc5", "no-rmd160", "no-whirlpool", "no-dso", + "no-ui-console", "no-shared", "no-unit-test", "ios64-xcrun", + "-miphonesimulator-version-min=12.0", "-fembed-bitcode", + "--libdir=lib", "--prefix=/", "--openssldir=/", + }, + }, { + Env: []string{ + "CFLAGS=-isysroot /Developer/SDKs/iphonesimulator -miphonesimulator-version-min=12.0 -arch arm64 -fembed-bitcode -O2 -Wno-macro-redefined", + "LDFLAGS=-isysroot /Developer/SDKs/iphonesimulator -miphonesimulator-version-min=12.0 -arch arm64 -fembed-bitcode", + "CXXFLAGS=-isysroot /Developer/SDKs/iphonesimulator -miphonesimulator-version-min=12.0 -arch arm64 -fembed-bitcode -O2 -Wno-macro-redefined", + }, + Argv: []string{ + "make", "-j", strconv.Itoa(runtime.NumCPU()), + }, + }, { + Env: []string{}, + Argv: []string{ + "make", + "DESTDIR=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/arm64", + "install_dev", + }, + }, { + Env: []string{}, + Argv: []string{ + "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.1.3.tar.gz", + }, + }, { + Env: []string{}, + Argv: []string{ + "tar", "-xf", "openssl-3.1.3.tar.gz", + }, + }, { + Env: []string{}, + Argv: []string{ + "git", "apply", faketopdir + "/CDEPS/openssl/000.patch", + }, + }, { + Env: []string{}, + Argv: []string{ + "git", "apply", faketopdir + "/CDEPS/openssl/001.patch", + }, + }, { + Env: []string{ + "CFLAGS=-isysroot /Developer/SDKs/iphonesimulator -miphonesimulator-version-min=12.0 -arch x86_64 -fembed-bitcode -O2 -Wno-macro-redefined", + "LDFLAGS=-isysroot /Developer/SDKs/iphonesimulator -miphonesimulator-version-min=12.0 -arch x86_64 -fembed-bitcode", + "CXXFLAGS=-isysroot /Developer/SDKs/iphonesimulator -miphonesimulator-version-min=12.0 -arch x86_64 -fembed-bitcode -O2 -Wno-macro-redefined", + }, + Argv: []string{ + "./Configure", "no-comp", "no-dtls", "no-ec2m", "no-psk", "no-srp", + "no-ssl3", "no-camellia", "no-idea", "no-md2", "no-md4", "no-mdc2", + "no-rc2", "no-rc4", "no-rc5", "no-rmd160", "no-whirlpool", "no-dso", + "no-ui-console", "no-shared", "no-unit-test", "iossimulator-xcrun", + "-miphonesimulator-version-min=12.0", "-fembed-bitcode", + "--libdir=lib", "--prefix=/", "--openssldir=/", + }, + }, { + Env: []string{ + "CFLAGS=-isysroot /Developer/SDKs/iphonesimulator -miphonesimulator-version-min=12.0 -arch x86_64 -fembed-bitcode -O2 -Wno-macro-redefined", + "LDFLAGS=-isysroot /Developer/SDKs/iphonesimulator -miphonesimulator-version-min=12.0 -arch x86_64 -fembed-bitcode", + "CXXFLAGS=-isysroot /Developer/SDKs/iphonesimulator -miphonesimulator-version-min=12.0 -arch x86_64 -fembed-bitcode -O2 -Wno-macro-redefined", + }, + Argv: []string{ + "make", "-j", strconv.Itoa(runtime.NumCPU()), + }, + }, { + Env: []string{}, + Argv: []string{ + "make", + "DESTDIR=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/amd64", + "install_dev", + }, + }}, + }} + + for _, testcase := range testcases { + t.Run(testcase.name, func(t *testing.T) { + + cc := &buildtooltest.SimpleCommandCollector{} + + deps := &buildtooltest.DependenciesCallCounter{ + HasPsiphon: false, + } + + shellxtesting.WithCustomLibrary(cc, func() { + iosCdepsBuildMain("openssl", deps) + }) + + expectCalls := map[string]int{ + buildtooltest.TagAbsoluteCurDir: 3, + buildtooltest.TagMustChdir: 3, + buildtooltest.TagVerifySHA256: 3, + } + + if diff := cmp.Diff(expectCalls, deps.Counter); diff != "" { + t.Fatal(diff) + } + + if err := buildtooltest.CheckManyCommands(cc.Commands, testcase.expect); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestIOSBuildCdepsLibevent(t *testing.T) { + faketopdir := (&buildtooltest.DependenciesCallCounter{}).AbsoluteCurDir() + + // testspec specifies a test case for this test + type testspec struct { + // name is the name of the test case + name string + + // expectations contains the commands we expect to see + expect []buildtooltest.ExecExpectations + } + + var testcases = []testspec{{ + name: "libevent", + expect: []buildtooltest.ExecExpectations{{ + Env: []string{}, + Argv: []string{ + "curl", + "-fsSLO", + "https://github.com/libevent/libevent/archive/release-2.1.12-stable.tar.gz", + }, + }, { + Env: []string{}, + Argv: []string{ + "tar", "-xf", "release-2.1.12-stable.tar.gz", + }, + }, { + Env: []string{}, + Argv: []string{ + "git", "apply", faketopdir + "/CDEPS/libevent/000.patch", + }, + }, { + Env: []string{}, + Argv: []string{ + "git", "apply", faketopdir + "/CDEPS/libevent/001.patch", + }, + }, { + Env: []string{}, + Argv: []string{ + "git", "apply", faketopdir + "/CDEPS/libevent/002.patch", + }, + }, { + Env: []string{}, + Argv: []string{ + "./autogen.sh", + }, + }, { + Env: []string{ + "AS=/Developer/SDKs/iphoneos/bin/as", + "LD=/Developer/SDKs/iphoneos/bin/ld", + "CXX=/Developer/SDKs/iphoneos/bin/c++", + "CC=/Developer/SDKs/iphoneos/bin/cc", + "AR=/Developer/SDKs/iphoneos/bin/ar", + "RANLIB=/Developer/SDKs/iphoneos/bin/ranlib", + "STRIP=/Developer/SDKs/iphoneos/bin/strip", + fmt.Sprintf( + "%s %s", + "LDFLAGS=-isysroot /Developer/SDKs/iphoneos -miphoneos-version-min=12.0 -arch arm64 -fembed-bitcode", + "-L"+faketopdir+"/internal/cmd/buildtool/internal/libtor/iphoneos/arm64/lib", + ), + fmt.Sprintf( + "%s %s", + "CFLAGS=-isysroot /Developer/SDKs/iphoneos -miphoneos-version-min=12.0 -arch arm64 -fembed-bitcode -O2", + "-I"+faketopdir+"/internal/cmd/buildtool/internal/libtor/iphoneos/arm64/include", + ), + fmt.Sprintf( + "%s %s", + "CXXFLAGS=-isysroot /Developer/SDKs/iphoneos -miphoneos-version-min=12.0 -arch arm64 -fembed-bitcode -O2", + "-I"+faketopdir+"/internal/cmd/buildtool/internal/libtor/iphoneos/arm64/include", + ), + "PKG_CONFIG_PATH=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphoneos/arm64/lib/pkgconfig", + }, + Argv: []string{ + "./configure", + "--host=arm-apple-darwin", + "--disable-libevent-regress", + "--disable-samples", + "--disable-shared", + "--prefix=/", + }, + }, { + Env: []string{}, + Argv: []string{ + "make", "V=1", "-j", strconv.Itoa(runtime.NumCPU()), + }, + }, { + Env: []string{}, + Argv: []string{ + "make", + "DESTDIR=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphoneos/arm64", + "install", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphoneos/arm64/bin", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphoneos/arm64/lib/pkgconfig/libevent.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphoneos/arm64/lib/pkgconfig/libevent_core.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphoneos/arm64/lib/pkgconfig/libevent_extra.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphoneos/arm64/lib/pkgconfig/libevent_openssl.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphoneos/arm64/lib/pkgconfig/libevent_pthreads.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphoneos/arm64/lib/libevent.la", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphoneos/arm64/lib/libevent_core.a", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphoneos/arm64/lib/libevent_core.la", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphoneos/arm64/lib/libevent_extra.a", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphoneos/arm64/lib/libevent_extra.la", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphoneos/arm64/lib/libevent_openssl.a", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphoneos/arm64/lib/libevent_openssl.la", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphoneos/arm64/lib/libevent_pthreads.a", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphoneos/arm64/lib/libevent_pthreads.la", + }, + }, { + Env: []string{}, + Argv: []string{ + "curl", + "-fsSLO", + "https://github.com/libevent/libevent/archive/release-2.1.12-stable.tar.gz", + }, + }, { + Env: []string{}, + Argv: []string{ + "tar", "-xf", "release-2.1.12-stable.tar.gz", + }, + }, { + Env: []string{}, + Argv: []string{ + "git", "apply", faketopdir + "/CDEPS/libevent/000.patch", + }, + }, { + Env: []string{}, + Argv: []string{ + "git", "apply", faketopdir + "/CDEPS/libevent/001.patch", + }, + }, { + Env: []string{}, + Argv: []string{ + "git", "apply", faketopdir + "/CDEPS/libevent/002.patch", + }, + }, { + Env: []string{}, + Argv: []string{ + "./autogen.sh", + }, + }, { + Env: []string{ + "AS=/Developer/SDKs/iphonesimulator/bin/as", + "LD=/Developer/SDKs/iphonesimulator/bin/ld", + "CXX=/Developer/SDKs/iphonesimulator/bin/c++", + "CC=/Developer/SDKs/iphonesimulator/bin/cc", + "AR=/Developer/SDKs/iphonesimulator/bin/ar", + "RANLIB=/Developer/SDKs/iphonesimulator/bin/ranlib", + "STRIP=/Developer/SDKs/iphonesimulator/bin/strip", + fmt.Sprintf( + "%s %s", + "LDFLAGS=-isysroot /Developer/SDKs/iphonesimulator -miphonesimulator-version-min=12.0 -arch arm64 -fembed-bitcode", + "-L"+faketopdir+"/internal/cmd/buildtool/internal/libtor/iphonesimulator/arm64/lib", + ), + fmt.Sprintf( + "%s %s", + "CFLAGS=-isysroot /Developer/SDKs/iphonesimulator -miphonesimulator-version-min=12.0 -arch arm64 -fembed-bitcode -O2", + "-I"+faketopdir+"/internal/cmd/buildtool/internal/libtor/iphonesimulator/arm64/include", + ), + fmt.Sprintf( + "%s %s", + "CXXFLAGS=-isysroot /Developer/SDKs/iphonesimulator -miphonesimulator-version-min=12.0 -arch arm64 -fembed-bitcode -O2", + "-I"+faketopdir+"/internal/cmd/buildtool/internal/libtor/iphonesimulator/arm64/include", + ), + "PKG_CONFIG_PATH=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/arm64/lib/pkgconfig", + }, + Argv: []string{ + "./configure", + "--host=arm-apple-darwin", + "--disable-libevent-regress", + "--disable-samples", + "--disable-shared", + "--prefix=/", + }, + }, { + Env: []string{}, + Argv: []string{ + "make", "V=1", "-j", strconv.Itoa(runtime.NumCPU()), + }, + }, { + Env: []string{}, + Argv: []string{ + "make", + "DESTDIR=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/arm64", + "install", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/arm64/bin", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/arm64/lib/pkgconfig/libevent.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/arm64/lib/pkgconfig/libevent_core.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/arm64/lib/pkgconfig/libevent_extra.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/arm64/lib/pkgconfig/libevent_openssl.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/arm64/lib/pkgconfig/libevent_pthreads.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/arm64/lib/libevent.la", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/arm64/lib/libevent_core.a", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/arm64/lib/libevent_core.la", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/arm64/lib/libevent_extra.a", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/arm64/lib/libevent_extra.la", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/arm64/lib/libevent_openssl.a", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/arm64/lib/libevent_openssl.la", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/arm64/lib/libevent_pthreads.a", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/arm64/lib/libevent_pthreads.la", + }, + }, { + Env: []string{}, + Argv: []string{ + "curl", + "-fsSLO", + "https://github.com/libevent/libevent/archive/release-2.1.12-stable.tar.gz", + }, + }, { + Env: []string{}, + Argv: []string{ + "tar", "-xf", "release-2.1.12-stable.tar.gz", + }, + }, { + Env: []string{}, + Argv: []string{ + "git", "apply", faketopdir + "/CDEPS/libevent/000.patch", + }, + }, { + Env: []string{}, + Argv: []string{ + "git", "apply", faketopdir + "/CDEPS/libevent/001.patch", + }, + }, { + Env: []string{}, + Argv: []string{ + "git", "apply", faketopdir + "/CDEPS/libevent/002.patch", + }, + }, { + Env: []string{}, + Argv: []string{ + "./autogen.sh", + }, + }, { + Env: []string{ + "AS=/Developer/SDKs/iphonesimulator/bin/as", + "LD=/Developer/SDKs/iphonesimulator/bin/ld", + "CXX=/Developer/SDKs/iphonesimulator/bin/c++", + "CC=/Developer/SDKs/iphonesimulator/bin/cc", + "AR=/Developer/SDKs/iphonesimulator/bin/ar", + "RANLIB=/Developer/SDKs/iphonesimulator/bin/ranlib", + "STRIP=/Developer/SDKs/iphonesimulator/bin/strip", + fmt.Sprintf( + "%s %s", + "LDFLAGS=-isysroot /Developer/SDKs/iphonesimulator -miphonesimulator-version-min=12.0 -arch x86_64 -fembed-bitcode", + "-L"+faketopdir+"/internal/cmd/buildtool/internal/libtor/iphonesimulator/amd64/lib", + ), + fmt.Sprintf( + "%s %s", + "CFLAGS=-isysroot /Developer/SDKs/iphonesimulator -miphonesimulator-version-min=12.0 -arch x86_64 -fembed-bitcode -O2", + "-I"+faketopdir+"/internal/cmd/buildtool/internal/libtor/iphonesimulator/amd64/include", + ), + fmt.Sprintf( + "%s %s", + "CXXFLAGS=-isysroot /Developer/SDKs/iphonesimulator -miphonesimulator-version-min=12.0 -arch x86_64 -fembed-bitcode -O2", + "-I"+faketopdir+"/internal/cmd/buildtool/internal/libtor/iphonesimulator/amd64/include", + ), + "PKG_CONFIG_PATH=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/amd64/lib/pkgconfig", + }, + Argv: []string{ + "./configure", + "--host=x86_64-apple-darwin", + "--disable-libevent-regress", + "--disable-samples", + "--disable-shared", + "--prefix=/", + }, + }, { + Env: []string{}, + Argv: []string{ + "make", "V=1", "-j", strconv.Itoa(runtime.NumCPU()), + }, + }, { + Env: []string{}, + Argv: []string{ + "make", + "DESTDIR=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/amd64", + "install", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/amd64/bin", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/amd64/lib/pkgconfig/libevent.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/amd64/lib/pkgconfig/libevent_core.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/amd64/lib/pkgconfig/libevent_extra.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/amd64/lib/pkgconfig/libevent_openssl.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/amd64/lib/pkgconfig/libevent_pthreads.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/amd64/lib/libevent.la", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/amd64/lib/libevent_core.a", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/amd64/lib/libevent_core.la", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/amd64/lib/libevent_extra.a", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/amd64/lib/libevent_extra.la", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/amd64/lib/libevent_openssl.a", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/amd64/lib/libevent_openssl.la", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/amd64/lib/libevent_pthreads.a", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-rf", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/amd64/lib/libevent_pthreads.la", + }, + }}, + }} + + for _, testcase := range testcases { + t.Run(testcase.name, func(t *testing.T) { + + cc := &buildtooltest.SimpleCommandCollector{} + + deps := &buildtooltest.DependenciesCallCounter{ + HasPsiphon: false, + } + + shellxtesting.WithCustomLibrary(cc, func() { + iosCdepsBuildMain("libevent", deps) + }) + + expectCalls := map[string]int{ + buildtooltest.TagAbsoluteCurDir: 3, + buildtooltest.TagMustChdir: 3, + buildtooltest.TagVerifySHA256: 3, + } + + if diff := cmp.Diff(expectCalls, deps.Counter); diff != "" { + t.Fatal(diff) + } + + if err := buildtooltest.CheckManyCommands(cc.Commands, testcase.expect); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestIOSBuildCdepsTor(t *testing.T) { + faketopdir := (&buildtooltest.DependenciesCallCounter{}).AbsoluteCurDir() + + // testspec specifies a test case for this test + type testspec struct { + // name is the name of the test case + name string + + // expectations contains the commands we expect to see + expect []buildtooltest.ExecExpectations + } + + var testcases = []testspec{{ + name: "tor", + expect: []buildtooltest.ExecExpectations{{ + Env: []string{}, + Argv: []string{ + "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.7.tar.gz", + }, + }, { + Env: []string{}, + Argv: []string{ + "tar", "-xf", "tor-0.4.8.7.tar.gz", + }, + }, { + Env: []string{}, + Argv: []string{ + "git", "apply", faketopdir + "/CDEPS/tor/000.patch", + }, + }, { + Env: []string{}, + Argv: []string{ + "git", "apply", faketopdir + "/CDEPS/tor/001.patch", + }, + }, { + Env: []string{}, + Argv: []string{ + "git", "apply", faketopdir + "/CDEPS/tor/002.patch", + }, + }, { + Env: []string{}, + Argv: []string{ + "git", "apply", faketopdir + "/CDEPS/tor/003.patch", + }, + }, { + Env: []string{ + "AS=/Developer/SDKs/iphoneos/bin/as", + "CC=/Developer/SDKs/iphoneos/bin/cc", + "RANLIB=/Developer/SDKs/iphoneos/bin/ranlib", + "STRIP=/Developer/SDKs/iphoneos/bin/strip", + "CXXFLAGS=-isysroot /Developer/SDKs/iphoneos -miphoneos-version-min=12.0 -arch arm64 -fembed-bitcode -O2", + "CFLAGS=-isysroot /Developer/SDKs/iphoneos -miphoneos-version-min=12.0 -arch arm64 -fembed-bitcode -O2", + "LDFLAGS=-isysroot /Developer/SDKs/iphoneos -miphoneos-version-min=12.0 -arch arm64 -fembed-bitcode", + "CXX=/Developer/SDKs/iphoneos/bin/c++", + "LD=/Developer/SDKs/iphoneos/bin/ld", + "AR=/Developer/SDKs/iphoneos/bin/ar", + }, + Argv: []string{ + "./configure", + "--host=arm-apple-darwin", + "--enable-pic", + "--enable-static-libevent", + "--with-libevent-dir=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphoneos/arm64", + "--enable-static-openssl", + "--with-openssl-dir=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphoneos/arm64", + "--enable-static-zlib", + "--with-zlib-dir=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphoneos/arm64", + "--disable-module-dirauth", + "--disable-zstd", + "--disable-lzma", + "--disable-tool-name-check", + "--disable-systemd", + "--prefix=/", + "--disable-unittests", + }, + }, { + Env: []string{}, + Argv: []string{ + "make", "V=1", "-j", strconv.Itoa(runtime.NumCPU()), + }, + }, { + Env: []string{}, + Argv: []string{ + "install", "-m644", "src/feature/api/tor_api.h", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphoneos/arm64/include", + }, + }, { + Env: []string{}, + Argv: []string{ + "install", "-m644", "libtor.a", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphoneos/arm64/lib", + }, + }, { + Env: []string{}, + Argv: []string{ + "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.7.tar.gz", + }, + }, { + Env: []string{}, + Argv: []string{ + "tar", "-xf", "tor-0.4.8.7.tar.gz", + }, + }, { + Env: []string{}, + Argv: []string{ + "git", "apply", faketopdir + "/CDEPS/tor/000.patch", + }, + }, { + Env: []string{}, + Argv: []string{ + "git", "apply", faketopdir + "/CDEPS/tor/001.patch", + }, + }, { + Env: []string{}, + Argv: []string{ + "git", "apply", faketopdir + "/CDEPS/tor/002.patch", + }, + }, { + Env: []string{}, + Argv: []string{ + "git", "apply", faketopdir + "/CDEPS/tor/003.patch", + }, + }, { + Env: []string{ + "AS=/Developer/SDKs/iphonesimulator/bin/as", + "CC=/Developer/SDKs/iphonesimulator/bin/cc", + "RANLIB=/Developer/SDKs/iphonesimulator/bin/ranlib", + "STRIP=/Developer/SDKs/iphonesimulator/bin/strip", + "CXXFLAGS=-isysroot /Developer/SDKs/iphonesimulator -miphonesimulator-version-min=12.0 -arch arm64 -fembed-bitcode -O2", + "CFLAGS=-isysroot /Developer/SDKs/iphonesimulator -miphonesimulator-version-min=12.0 -arch arm64 -fembed-bitcode -O2", + "LDFLAGS=-isysroot /Developer/SDKs/iphonesimulator -miphonesimulator-version-min=12.0 -arch arm64 -fembed-bitcode", + "CXX=/Developer/SDKs/iphonesimulator/bin/c++", + "LD=/Developer/SDKs/iphonesimulator/bin/ld", + "AR=/Developer/SDKs/iphonesimulator/bin/ar", + }, + Argv: []string{ + "./configure", + "--host=arm-apple-darwin", + "--enable-pic", + "--enable-static-libevent", + "--with-libevent-dir=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/arm64", + "--enable-static-openssl", + "--with-openssl-dir=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/arm64", + "--enable-static-zlib", + "--with-zlib-dir=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/arm64", + "--disable-module-dirauth", + "--disable-zstd", + "--disable-lzma", + "--disable-tool-name-check", + "--disable-systemd", + "--prefix=/", + "--disable-unittests", + }, + }, { + Env: []string{}, + Argv: []string{ + "make", "V=1", "-j", strconv.Itoa(runtime.NumCPU()), + }, + }, { + Env: []string{}, + Argv: []string{ + "install", "-m644", "src/feature/api/tor_api.h", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/arm64/include", + }, + }, { + Env: []string{}, + Argv: []string{ + "install", "-m644", "libtor.a", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/arm64/lib", + }, + }, { + Env: []string{}, + Argv: []string{ + "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.7.tar.gz", + }, + }, { + Env: []string{}, + Argv: []string{ + "tar", "-xf", "tor-0.4.8.7.tar.gz", + }, + }, { + Env: []string{}, + Argv: []string{ + "git", "apply", faketopdir + "/CDEPS/tor/000.patch", + }, + }, { + Env: []string{}, + Argv: []string{ + "git", "apply", faketopdir + "/CDEPS/tor/001.patch", + }, + }, { + Env: []string{}, + Argv: []string{ + "git", "apply", faketopdir + "/CDEPS/tor/002.patch", + }, + }, { + Env: []string{}, + Argv: []string{ + "git", "apply", faketopdir + "/CDEPS/tor/003.patch", + }, + }, { + Env: []string{ + "AS=/Developer/SDKs/iphonesimulator/bin/as", + "CC=/Developer/SDKs/iphonesimulator/bin/cc", + "RANLIB=/Developer/SDKs/iphonesimulator/bin/ranlib", + "STRIP=/Developer/SDKs/iphonesimulator/bin/strip", + "CXXFLAGS=-isysroot /Developer/SDKs/iphonesimulator -miphonesimulator-version-min=12.0 -arch x86_64 -fembed-bitcode -O2", + "CFLAGS=-isysroot /Developer/SDKs/iphonesimulator -miphonesimulator-version-min=12.0 -arch x86_64 -fembed-bitcode -O2", + "LDFLAGS=-isysroot /Developer/SDKs/iphonesimulator -miphonesimulator-version-min=12.0 -arch x86_64 -fembed-bitcode", + "CXX=/Developer/SDKs/iphonesimulator/bin/c++", + "LD=/Developer/SDKs/iphonesimulator/bin/ld", + "AR=/Developer/SDKs/iphonesimulator/bin/ar", + }, + Argv: []string{ + "./configure", + "--host=x86_64-apple-darwin", + "--enable-pic", + "--enable-static-libevent", + "--with-libevent-dir=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/amd64", + "--enable-static-openssl", + "--with-openssl-dir=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/amd64", + "--enable-static-zlib", + "--with-zlib-dir=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/amd64", + "--disable-module-dirauth", + "--disable-zstd", + "--disable-lzma", + "--disable-tool-name-check", + "--disable-systemd", + "--prefix=/", + "--disable-unittests", + }, + }, { + Env: []string{}, + Argv: []string{ + "make", "V=1", "-j", strconv.Itoa(runtime.NumCPU()), + }, + }, { + Env: []string{}, + Argv: []string{ + "install", "-m644", "src/feature/api/tor_api.h", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/amd64/include", + }, + }, { + Env: []string{}, + Argv: []string{ + "install", "-m644", "libtor.a", + faketopdir + "/internal/cmd/buildtool/internal/libtor/iphonesimulator/amd64/lib", + }, + }}, + }} + + for _, testcase := range testcases { + t.Run(testcase.name, func(t *testing.T) { + + cc := &buildtooltest.SimpleCommandCollector{} + + deps := &buildtooltest.DependenciesCallCounter{ + HasPsiphon: false, + } + + shellxtesting.WithCustomLibrary(cc, func() { + iosCdepsBuildMain("tor", deps) + }) + + expectCalls := map[string]int{ + buildtooltest.TagAbsoluteCurDir: 3, + buildtooltest.TagMustChdir: 3, + buildtooltest.TagVerifySHA256: 3, + } + + if diff := cmp.Diff(expectCalls, deps.Counter); diff != "" { + t.Fatal(diff) + } + + if err := buildtooltest.CheckManyCommands(cc.Commands, testcase.expect); err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/pkg/cmd/buildtool/linuxcdeps.go b/pkg/cmd/buildtool/linuxcdeps.go index 17be0501..45d98a2d 100644 --- a/pkg/cmd/buildtool/linuxcdeps.go +++ b/pkg/cmd/buildtool/linuxcdeps.go @@ -50,24 +50,24 @@ func linuxCdepsBuildMain(name string, deps buildtoolmodel.Dependencies) { "internal", "libtor", "linux", runtime.GOARCH, ))) globalEnv := &cBuildEnv{ - ANDROID_HOME: "", - ANDROID_NDK_ROOT: "", - AR: "", - BINPATH: "", - CC: "", - CFLAGS: cflags, - CONFIGURE_HOST: "", - DESTDIR: destdir, - CXX: "", - CXXFLAGS: cflags, - GOARCH: "", - GOARM: "", - LD: "", - LDFLAGS: []string{}, - OPENSSL_API_DEFINE: "", - OPENSSL_COMPILER: "linux-x86_64", - RANLIB: "", - STRIP: "", + ANDROID_HOME: "", + ANDROID_NDK_ROOT: "", + AR: "", + BINPATH: "", + CC: "", + CFLAGS: cflags, + CONFIGURE_HOST: "", + DESTDIR: destdir, + CXX: "", + CXXFLAGS: cflags, + GOARCH: "", + GOARM: "", + LD: "", + LDFLAGS: []string{}, + OPENSSL_COMPILER: "linux-x86_64", + OPENSSL_POST_COMPILER_FLAGS: []string{}, + RANLIB: "", + STRIP: "", } switch name { case "libevent": diff --git a/pkg/cmd/buildtool/linuxcdeps_test.go b/pkg/cmd/buildtool/linuxcdeps_test.go index 3b934d6e..cb0b3976 100644 --- a/pkg/cmd/buildtool/linuxcdeps_test.go +++ b/pkg/cmd/buildtool/linuxcdeps_test.go @@ -136,12 +136,6 @@ func TestLinuxCdepsBuildMain(t *testing.T) { "DESTDIR=" + faketopdir + "/" + sysDepDestDir, "install_dev", }, - }, { - Env: []string{}, - Argv: []string{ - "rm", "-rf", - faketopdir + "/" + sysDepDestDir + "/lib/pkgconfig", - }, }}, }, { name: "we can build libevent", @@ -195,6 +189,7 @@ func TestLinuxCdepsBuildMain(t *testing.T) { faketopdir, sysDepDestDir, ), + "PKG_CONFIG_PATH=" + faketopdir + "/" + sysDepDestDir + "/lib/pkgconfig", }, Argv: []string{ "./configure", @@ -226,8 +221,36 @@ func TestLinuxCdepsBuildMain(t *testing.T) { Env: []string{}, Argv: []string{ "rm", - "-rf", - faketopdir + "/" + sysDepDestDir + "/lib/pkgconfig", + "-f", + faketopdir + "/" + sysDepDestDir + "/lib/pkgconfig/libevent.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/" + sysDepDestDir + "/lib/pkgconfig/libevent_core.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/" + sysDepDestDir + "/lib/pkgconfig/libevent_extra.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/" + sysDepDestDir + "/lib/pkgconfig/libevent_openssl.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/" + sysDepDestDir + "/lib/pkgconfig/libevent_pthreads.pc", }, }, { Env: []string{}, @@ -346,6 +369,7 @@ func TestLinuxCdepsBuildMain(t *testing.T) { "--disable-tool-name-check", "--disable-systemd", "--prefix=/", + "--disable-unittests", }, }, { Env: []string{}, diff --git a/pkg/cmd/ghgen/ios.go b/pkg/cmd/ghgen/ios.go index 78fde23f..523a5aa5 100644 --- a/pkg/cmd/ghgen/ios.go +++ b/pkg/cmd/ghgen/ios.go @@ -15,6 +15,16 @@ func buildAndPublishMobileIOS(w io.Writer, job *Job) { buildJob := "build_ios_mobile" artifacts := []string{ + "./MOBILE/ios/libcrypto.xcframework.zip", + "./MOBILE/ios/libcrypto.podspec", + "./MOBILE/ios/libevent.xcframework.zip", + "./MOBILE/ios/libevent.podspec", + "./MOBILE/ios/libssl.xcframework.zip", + "./MOBILE/ios/libssl.podspec", + "./MOBILE/ios/libtor.xcframework.zip", + "./MOBILE/ios/libtor.podspec", + "./MOBILE/ios/libz.xcframework.zip", + "./MOBILE/ios/libz.podspec", "./MOBILE/ios/oonimkall.xcframework.zip", "./MOBILE/ios/oonimkall.podspec", } @@ -24,7 +34,8 @@ func buildAndPublishMobileIOS(w io.Writer, job *Job) { newStepCheckout(w) newStepSetupGo(w, "ios") newStepSetupPsiphon(w) - newStepMake(w, "EXPECTED_XCODE_VERSION=14.2 MOBILE/ios") + iosNewStepBrewInstall(w) + newStepMake(w, "EXPECTED_XCODE_VERSION=14.2 ios") newStepUploadArtifacts(w, artifacts) newJob(w, publishJob, runsOnUbuntu, buildJob, contentsWritePermissions) @@ -32,3 +43,9 @@ func buildAndPublishMobileIOS(w io.Writer, job *Job) { newStepDownloadArtifacts(w, artifacts) newStepGHPublish(w, artifacts) } + +func iosNewStepBrewInstall(w io.Writer) { + mustFprintf(w, " # ./internal/cmd/buildtool needs coreutils for sha256 plus GNU build tools\n") + mustFprintf(w, " - run: brew install autoconf automake coreutils libtool\n") + mustFprintf(w, "\n") +} diff --git a/pkg/dslx/address.go b/pkg/dslx/address.go index 2a2788bb..fc2f6f8c 100644 --- a/pkg/dslx/address.go +++ b/pkg/dslx/address.go @@ -57,12 +57,22 @@ func (as *AddressSet) RemoveBogons() *AddressSet { return as } +// Uniq returns the unique addresses. +func (as *AddressSet) Uniq() (uniq []string) { + for addr := range as.M { + uniq = append(uniq, addr) + } + return +} + // EndpointPort is the port for an endpoint. type EndpointPort uint16 // ToEndpoints transforms this set of IP addresses to a list of endpoints. We will // combine each IP address with the network and the port to construct an endpoint and // we will also apply any additional option to each endpoint. +// +// Deprecated: use MakeEndpoint instead. func (as *AddressSet) ToEndpoints( network EndpointNetwork, port EndpointPort, options ...EndpointOption) (v []*Endpoint) { for addr := range as.M { diff --git a/pkg/dslx/connpool.go b/pkg/dslx/connpool.go deleted file mode 100644 index 0636147b..00000000 --- a/pkg/dslx/connpool.go +++ /dev/null @@ -1,42 +0,0 @@ -package dslx - -// -// Connection pooling to streamline closing connections. -// - -import ( - "io" - "sync" -) - -// ConnPool tracks established connections. The zero value -// of this struct is ready to use. -type ConnPool struct { - mu sync.Mutex - v []io.Closer -} - -// MaybeTrack tracks the given connection if not nil. This -// method is safe for use by multiple goroutines. -func (p *ConnPool) MaybeTrack(c io.Closer) { - if c != nil { - defer p.mu.Unlock() - p.mu.Lock() - p.v = append(p.v, c) - } -} - -// Close closes all the tracked connections in reverse order. This -// method is safe for use by multiple goroutines. -func (p *ConnPool) Close() error { - // Implementation note: reverse order is such that we close TLS - // connections before we close the TCP connections they use. Hence - // we'll _gracefully_ close TLS connections. - defer p.mu.Unlock() - p.mu.Lock() - for idx := len(p.v) - 1; idx >= 0; idx-- { - _ = p.v[idx].Close() - } - p.v = nil // reset - return nil -} diff --git a/pkg/dslx/connpool_test.go b/pkg/dslx/connpool_test.go deleted file mode 100644 index 49d43dcf..00000000 --- a/pkg/dslx/connpool_test.go +++ /dev/null @@ -1,100 +0,0 @@ -package dslx - -import ( - "errors" - "io" - "testing" - - "github.com/ooni/probe-engine/pkg/mocks" - "github.com/quic-go/quic-go" -) - -/* -Test cases: -- Maybe track connections: - - with nil - - with connection - - with quic connection - -- Close ConnPool: - - all Close() calls succeed - - one Close() call fails -*/ - -func closeableConnWithErr(err error) io.Closer { - return &mocks.Conn{ - MockClose: func() error { - return err - }, - } -} - -func closeableQUICConnWithErr(err error) io.Closer { - return &quicCloserConn{ - &mocks.QUICEarlyConnection{ - MockCloseWithError: func(code quic.ApplicationErrorCode, reason string) error { - return err - }, - }, - } -} - -func TestConnPool(t *testing.T) { - type connpoolTest struct { - mockConn io.Closer - want int // len of connpool.v - } - - t.Run("Maybe track connections", func(t *testing.T) { - tests := map[string]connpoolTest{ - "with nil": {mockConn: nil, want: 0}, - "with connection": {mockConn: closeableConnWithErr(nil), want: 1}, - "with quic connection": {mockConn: closeableQUICConnWithErr(nil), want: 1}, - } - for name, tt := range tests { - t.Run(name, func(t *testing.T) { - connpool := &ConnPool{} - connpool.MaybeTrack(tt.mockConn) - if len(connpool.v) != tt.want { - t.Fatalf("expected %d tracked connections, got: %d", tt.want, len(connpool.v)) - } - }) - } - }) - - t.Run("Close ConnPool", func(t *testing.T) { - mockErr := errors.New("mocked") - tests := map[string]struct { - pool *ConnPool - }{ - "all Close() calls succeed": { - pool: &ConnPool{ - v: []io.Closer{ - closeableConnWithErr(nil), - closeableQUICConnWithErr(nil), - }, - }, - }, - "one Close() call fails": { - pool: &ConnPool{ - v: []io.Closer{ - closeableConnWithErr(nil), - closeableConnWithErr(mockErr), - }, - }, - }, - } - - for name, tt := range tests { - t.Run(name, func(t *testing.T) { - err := tt.pool.Close() - if err != nil { // Close() should always return nil - t.Fatalf("unexpected error %s", err) - } - if tt.pool.v != nil { - t.Fatalf("v should be reset but is not") - } - }) - } - }) -} diff --git a/pkg/dslx/dns.go b/pkg/dslx/dns.go index 95102838..7bc4ba4b 100644 --- a/pkg/dslx/dns.go +++ b/pkg/dslx/dns.go @@ -6,13 +6,10 @@ package dslx import ( "context" - "sync/atomic" + "errors" "time" "github.com/ooni/probe-engine/pkg/logx" - "github.com/ooni/probe-engine/pkg/measurexlite" - "github.com/ooni/probe-engine/pkg/model" - "github.com/ooni/probe-engine/pkg/netxlite" ) // DomainName is a domain name to resolve. @@ -21,22 +18,6 @@ type DomainName string // DNSLookupOption is an option you can pass to NewDomainToResolve. type DNSLookupOption func(*DomainToResolve) -// DNSLookupOptionIDGenerator configures a specific ID generator. -// See DomainToResolve docs for more information. -func DNSLookupOptionIDGenerator(value *atomic.Int64) DNSLookupOption { - return func(dis *DomainToResolve) { - dis.IDGenerator = value - } -} - -// DNSLookupOptionLogger configures a specific logger. -// See DomainToResolve docs for more information. -func DNSLookupOptionLogger(value model.Logger) DNSLookupOption { - return func(dis *DomainToResolve) { - dis.Logger = value - } -} - // DNSLookupOptionTags allows to set tags to tag observations. func DNSLookupOptionTags(value ...string) DNSLookupOption { return func(dis *DomainToResolve) { @@ -44,24 +25,13 @@ func DNSLookupOptionTags(value ...string) DNSLookupOption { } } -// DNSLookupOptionZeroTime configures the measurement's zero time. -// See DomainToResolve docs for more information. -func DNSLookupOptionZeroTime(value time.Time) DNSLookupOption { - return func(dis *DomainToResolve) { - dis.ZeroTime = value - } -} - // NewDomainToResolve creates input for performing DNS lookups. The only mandatory // argument is the domain name to resolve. You can also supply optional // values by passing options to this function. func NewDomainToResolve(domain DomainName, options ...DNSLookupOption) *DomainToResolve { state := &DomainToResolve{ - Domain: string(domain), - IDGenerator: &atomic.Int64{}, - Logger: model.DiscardLogger, - Tags: []string{}, - ZeroTime: time.Now(), + Domain: string(domain), + Tags: []string{}, } for _, option := range options { option(state) @@ -79,25 +49,8 @@ type DomainToResolve struct { // Domain is the MANDATORY domain name to lookup. Domain string - // IDGenerator is the MANDATORY ID generator. We will use this field - // to assign unique IDs to distinct sub-measurements. The default - // construction implemented by NewDomainToResolve creates a new generator - // that starts counting from zero, leading to the first trace having - // one as its index. - IDGenerator *atomic.Int64 - - // Logger is the MANDATORY logger to use. The default construction - // implemented by NewDomainToResolve uses model.DiscardLogger. - Logger model.Logger - // Tags contains OPTIONAL tags to tag observations. Tags []string - - // ZeroTime is the MANDATORY zero time of the measurement. We will - // use this field as the zero value to compute relative elapsed times - // when generating measurements. The default construction by - // NewDomainToResolve initializes this field with the current time. - ZeroTime time.Time } // ResolvedAddresses contains the results of DNS lookups. To initialize @@ -109,148 +62,158 @@ type ResolvedAddresses struct { // Domain is the domain we resolved. We inherit this field // from the value inside the DomainToResolve. Domain string +} - // IDGenerator is the ID generator. We inherit this field - // from the value inside the DomainToResolve. - IDGenerator *atomic.Int64 - - // Logger is the logger to use. We inherit this field - // from the value inside the DomainToResolve. - Logger model.Logger +// Flatten transforms a [ResolvedAddresses] into a slice of zero or more [ResolvedAddress]. +func (ra *ResolvedAddresses) Flatten() (out []*ResolvedAddress) { + for _, ipAddr := range ra.Addresses { + out = append(out, &ResolvedAddress{ + Address: ipAddr, + Domain: ra.Domain, + }) + } + return +} - // Trace is the trace we're currently using. This struct is - // created by the various Apply functions using values inside - // the DomainToResolve to initialize the Trace. - Trace *measurexlite.Trace +// ResolvedAddress is a single address resolved using a DNS lookup function. +type ResolvedAddress struct { + // Address is the address that was resolved. + Address string - // ZeroTime is the zero time of the measurement. We inherit this field - // from the value inside the DomainToResolve. - ZeroTime time.Time + // Domain is the domain from which we resolved the address. + Domain string } // DNSLookupGetaddrinfo returns a function that resolves a domain name to // IP addresses using libc's getaddrinfo function. -func DNSLookupGetaddrinfo() Func[*DomainToResolve, *Maybe[*ResolvedAddresses]] { - return &dnsLookupGetaddrinfoFunc{} -} - -// dnsLookupGetaddrinfoFunc is the function returned by DNSLookupGetaddrinfo. -type dnsLookupGetaddrinfoFunc struct { - resolver model.Resolver // for testing -} - -// Apply implements Func. -func (f *dnsLookupGetaddrinfoFunc) Apply( - ctx context.Context, input *DomainToResolve) *Maybe[*ResolvedAddresses] { - - // create trace - trace := measurexlite.NewTrace(input.IDGenerator.Add(1), input.ZeroTime, input.Tags...) - - // start the operation logger - ol := logx.NewOperationLogger( - input.Logger, - "[#%d] DNSLookup[getaddrinfo] %s", - trace.Index, - input.Domain, - ) - - // setup - const timeout = 4 * time.Second - ctx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - - resolver := f.resolver - if resolver == nil { - resolver = trace.NewStdlibResolver(input.Logger) - } - - // lookup - addrs, err := resolver.LookupHost(ctx, input.Domain) - - // stop the operation logger - ol.Stop(err) - - state := &ResolvedAddresses{ - Addresses: addrs, // maybe empty - Domain: input.Domain, - IDGenerator: input.IDGenerator, - Logger: input.Logger, - Trace: trace, - ZeroTime: input.ZeroTime, - } +func DNSLookupGetaddrinfo(rt Runtime) Func[*DomainToResolve, *ResolvedAddresses] { + return Operation[*DomainToResolve, *ResolvedAddresses](func(ctx context.Context, input *DomainToResolve) (*ResolvedAddresses, error) { + // create trace + trace := rt.NewTrace(rt.IDGenerator().Add(1), rt.ZeroTime(), input.Tags...) + + // start the operation logger + ol := logx.NewOperationLogger( + rt.Logger(), + "[#%d] DNSLookup[getaddrinfo] %s", + trace.Index(), + input.Domain, + ) - return &Maybe[*ResolvedAddresses]{ - Error: err, - Observations: maybeTraceToObservations(trace), - Operation: netxlite.ResolveOperation, - State: state, - } + // setup + const timeout = 4 * time.Second + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + // create the resolver + resolver := trace.NewStdlibResolver(rt.Logger()) + + // lookup + addrs, err := resolver.LookupHost(ctx, input.Domain) + + // save the observations + rt.SaveObservations(maybeTraceToObservations(trace)...) + + // handle error case + if err != nil { + ol.Stop(err) + return nil, err + } + + // handle success + ol.Stop(addrs) + state := &ResolvedAddresses{ + Addresses: addrs, + Domain: input.Domain, + } + return state, nil + }) } // DNSLookupUDP returns a function that resolves a domain name to // IP addresses using the given DNS-over-UDP resolver. -func DNSLookupUDP(resolver string) Func[*DomainToResolve, *Maybe[*ResolvedAddresses]] { - return &dnsLookupUDPFunc{ - Resolver: resolver, - } -} - -// dnsLookupUDPFunc is the function returned by DNSLookupUDP. -type dnsLookupUDPFunc struct { - // Resolver is the MANDATORY endpointed of the resolver to use. - Resolver string - mockResolver model.Resolver // for testing -} - -// Apply implements Func. -func (f *dnsLookupUDPFunc) Apply( - ctx context.Context, input *DomainToResolve) *Maybe[*ResolvedAddresses] { - - // create trace - trace := measurexlite.NewTrace(input.IDGenerator.Add(1), input.ZeroTime, input.Tags...) - - // start the operation logger - ol := logx.NewOperationLogger( - input.Logger, - "[#%d] DNSLookup[%s/udp] %s", - trace.Index, - f.Resolver, - input.Domain, - ) - - // setup - const timeout = 4 * time.Second - ctx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - - resolver := f.mockResolver - if resolver == nil { - resolver = trace.NewParallelUDPResolver( - input.Logger, - netxlite.NewDialerWithoutResolver(input.Logger), - f.Resolver, +func DNSLookupUDP(rt Runtime, endpoint string) Func[*DomainToResolve, *ResolvedAddresses] { + return Operation[*DomainToResolve, *ResolvedAddresses](func(ctx context.Context, input *DomainToResolve) (*ResolvedAddresses, error) { + // create trace + trace := rt.NewTrace(rt.IDGenerator().Add(1), rt.ZeroTime(), input.Tags...) + + // start the operation logger + ol := logx.NewOperationLogger( + rt.Logger(), + "[#%d] DNSLookup[%s/udp] %s", + trace.Index(), + endpoint, + input.Domain, ) - } - // lookup - addrs, err := resolver.LookupHost(ctx, input.Domain) + // setup + const timeout = 4 * time.Second + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() - // stop the operation logger - ol.Stop(err) + // create the resolver + resolver := trace.NewParallelUDPResolver( + rt.Logger(), + trace.NewDialerWithoutResolver(rt.Logger()), + endpoint, + ) - state := &ResolvedAddresses{ - Addresses: addrs, // maybe empty - Domain: input.Domain, - IDGenerator: input.IDGenerator, - Logger: input.Logger, - Trace: trace, - ZeroTime: input.ZeroTime, - } + // lookup + addrs, err := resolver.LookupHost(ctx, input.Domain) + + // save the observations + rt.SaveObservations(maybeTraceToObservations(trace)...) + + // handle error case + if err != nil { + ol.Stop(err) + return nil, err + } + + // handle success + ol.Stop(addrs) + state := &ResolvedAddresses{ + Addresses: addrs, + Domain: input.Domain, + } + return state, nil + }) +} - return &Maybe[*ResolvedAddresses]{ - Error: err, - Observations: maybeTraceToObservations(trace), - Operation: netxlite.ResolveOperation, - State: state, - } +// ErrDNSLookupParallel indicates that DNSLookupParallel failed. +var ErrDNSLookupParallel = errors.New("dslx: DNSLookupParallel failed") + +// DNSLookupParallel runs DNS lookups in parallel. On success, this function returns +// a unique list of IP addresses aggregated from all resolvers. On failure, this function +// returns [ErrDNSLookupParallel]. You can always obtain the individual errors by +// processing observations or by creating a per-DNS-resolver pipeline. +func DNSLookupParallel(fxs ...Func[*DomainToResolve, *ResolvedAddresses]) Func[*DomainToResolve, *ResolvedAddresses] { + return Operation[*DomainToResolve, *ResolvedAddresses](func(ctx context.Context, domain *DomainToResolve) (*ResolvedAddresses, error) { + // TODO(https://github.com/ooni/probe/issues/2619): we may want to configure this + const parallelism = Parallelism(3) + + // run all the DNS resolvers in parallel + results := Parallel(ctx, parallelism, domain, fxs...) + + // reduce addresses + addressSet := NewAddressSet() + for _, result := range results { + if err := result.Error; err != nil { + continue + } + addressSet.Add(result.State.Addresses...) + } + uniq := addressSet.Uniq() + + // handle the case where all the DNS resolvers failed + if len(uniq) < 1 { + return nil, ErrDNSLookupParallel + } + + // handle success + state := &ResolvedAddresses{ + Addresses: uniq, + Domain: domain.Domain, + } + return state, nil + }) } diff --git a/pkg/dslx/dns_test.go b/pkg/dslx/dns_test.go index 11270d16..58e1e595 100644 --- a/pkg/dslx/dns_test.go +++ b/pkg/dslx/dns_test.go @@ -3,6 +3,7 @@ package dslx import ( "context" "errors" + "net" "sync/atomic" "testing" "time" @@ -30,26 +31,13 @@ func TestNewDomainToResolve(t *testing.T) { t.Run("with options", func(t *testing.T) { idGen := &atomic.Int64{} idGen.Add(42) - zt := time.Now() domainToResolve := NewDomainToResolve( DomainName("www.example.com"), - DNSLookupOptionIDGenerator(idGen), - DNSLookupOptionLogger(model.DiscardLogger), - DNSLookupOptionZeroTime(zt), DNSLookupOptionTags("antani"), ) if domainToResolve.Domain != "www.example.com" { t.Fatalf("unexpected domain") } - if domainToResolve.IDGenerator != idGen { - t.Fatalf("unexpected id generator") - } - if domainToResolve.Logger != model.DiscardLogger { - t.Fatalf("unexpected logger") - } - if domainToResolve.ZeroTime != zt { - t.Fatalf("unexpected zerotime") - } if diff := cmp.Diff([]string{"antani"}, domainToResolve.Tags); diff != "" { t.Fatal(diff) } @@ -66,28 +54,19 @@ Test cases: - with success */ func TestGetaddrinfo(t *testing.T) { - t.Run("Get dnsLookupGetaddrinfoFunc", func(t *testing.T) { - f := DNSLookupGetaddrinfo() - if _, ok := f.(*dnsLookupGetaddrinfoFunc); !ok { - t.Fatal("unexpected type, want dnsLookupGetaddrinfoFunc") - } - }) - t.Run("Apply dnsLookupGetaddrinfoFunc", func(t *testing.T) { domain := &DomainToResolve{ - Domain: "example.com", - Logger: model.DiscardLogger, - IDGenerator: &atomic.Int64{}, - Tags: []string{"antani"}, - ZeroTime: time.Time{}, + Domain: "example.com", + Tags: []string{"antani"}, } t.Run("with nil resolver", func(t *testing.T) { - f := dnsLookupGetaddrinfoFunc{} + rt := NewRuntimeMeasurexLite(model.DiscardLogger, time.Now()) + f := DNSLookupGetaddrinfo(rt) ctx, cancel := context.WithCancel(context.Background()) cancel() // immediately cancel the lookup - res := f.Apply(ctx, domain) - if res.Observations == nil || len(res.Observations) <= 0 { + res := f.Apply(ctx, NewMaybeWithValue(domain)) + if obs := rt.Observations(); obs == nil || len(obs.Queries) <= 0 { t.Fatal("unexpected empty observations") } if res.Error == nil { @@ -97,36 +76,37 @@ func TestGetaddrinfo(t *testing.T) { t.Run("with lookup error", func(t *testing.T) { mockedErr := errors.New("mocked") - f := dnsLookupGetaddrinfoFunc{ - resolver: &mocks.Resolver{MockLookupHost: func(ctx context.Context, domain string) ([]string, error) { - return nil, mockedErr - }}, - } - res := f.Apply(context.Background(), domain) - if res.Observations == nil || len(res.Observations) <= 0 { - t.Fatal("unexpected empty observations") - } + rt := NewRuntimeMeasurexLite(model.DiscardLogger, time.Now(), RuntimeMeasurexLiteOptionMeasuringNetwork(&mocks.MeasuringNetwork{ + MockNewStdlibResolver: func(logger model.DebugLogger) model.Resolver { + return &mocks.Resolver{ + MockLookupHost: func(ctx context.Context, domain string) ([]string, error) { + return nil, mockedErr + }, + } + }, + })) + f := DNSLookupGetaddrinfo(rt) + res := f.Apply(context.Background(), NewMaybeWithValue(domain)) if res.Error != mockedErr { t.Fatalf("unexpected error type: %s", res.Error) } - if res.State == nil { - t.Fatal("unexpected nil state") - } - if res.State.Addresses != nil { - t.Fatal("expected empty addresses here") + if res.State != nil { + t.Fatal("expected nil state") } }) t.Run("with success", func(t *testing.T) { - f := dnsLookupGetaddrinfoFunc{ - resolver: &mocks.Resolver{MockLookupHost: func(ctx context.Context, domain string) ([]string, error) { - return []string{"93.184.216.34"}, nil - }}, - } - res := f.Apply(context.Background(), domain) - if res.Observations == nil || len(res.Observations) <= 0 { - t.Fatal("unexpected empty observations") - } + rt := NewRuntimeMeasurexLite(model.DiscardLogger, time.Now(), RuntimeMeasurexLiteOptionMeasuringNetwork(&mocks.MeasuringNetwork{ + MockNewStdlibResolver: func(logger model.DebugLogger) model.Resolver { + return &mocks.Resolver{ + MockLookupHost: func(ctx context.Context, domain string) ([]string, error) { + return []string{"93.184.216.34"}, nil + }, + } + }, + })) + f := DNSLookupGetaddrinfo(rt) + res := f.Apply(context.Background(), NewMaybeWithValue(domain)) if res.Error != nil { t.Fatalf("unexpected error: %s", res.Error) } @@ -136,9 +116,6 @@ func TestGetaddrinfo(t *testing.T) { if len(res.State.Addresses) != 1 || res.State.Addresses[0] != "93.184.216.34" { t.Fatal("unexpected addresses") } - if diff := cmp.Diff([]string{"antani"}, res.State.Trace.Tags()); diff != "" { - t.Fatal(diff) - } }) }) } @@ -152,28 +129,19 @@ Test cases: - with success */ func TestLookupUDP(t *testing.T) { - t.Run("Get dnsLookupUDPFunc", func(t *testing.T) { - f := DNSLookupUDP("1.1.1.1:53") - if _, ok := f.(*dnsLookupUDPFunc); !ok { - t.Fatal("unexpected type, want dnsLookupUDPFunc") - } - }) - - t.Run("Apply dnsLookupGetaddrinfoFunc", func(t *testing.T) { + t.Run("Apply dnsLookupUDPFunc", func(t *testing.T) { domain := &DomainToResolve{ - Domain: "example.com", - Logger: model.DiscardLogger, - IDGenerator: &atomic.Int64{}, - Tags: []string{"antani"}, - ZeroTime: time.Time{}, + Domain: "example.com", + Tags: []string{"antani"}, } t.Run("with nil resolver", func(t *testing.T) { - f := dnsLookupUDPFunc{Resolver: "1.1.1.1:53"} + rt := NewRuntimeMeasurexLite(model.DiscardLogger, time.Now()) + f := DNSLookupUDP(rt, "1.1.1.1:53") ctx, cancel := context.WithCancel(context.Background()) cancel() - res := f.Apply(ctx, domain) - if res.Observations == nil || len(res.Observations) <= 0 { + res := f.Apply(ctx, NewMaybeWithValue(domain)) + if obs := rt.Observations(); obs == nil || len(obs.Queries) <= 0 { t.Fatal("unexpected empty observations") } if res.Error == nil { @@ -183,38 +151,51 @@ func TestLookupUDP(t *testing.T) { t.Run("with lookup error", func(t *testing.T) { mockedErr := errors.New("mocked") - f := dnsLookupUDPFunc{ - Resolver: "1.1.1.1:53", - mockResolver: &mocks.Resolver{MockLookupHost: func(ctx context.Context, domain string) ([]string, error) { - return nil, mockedErr - }}, - } - res := f.Apply(context.Background(), domain) - if res.Observations == nil || len(res.Observations) <= 0 { - t.Fatal("unexpected empty observations") - } + rt := NewRuntimeMeasurexLite(model.DiscardLogger, time.Now(), RuntimeMeasurexLiteOptionMeasuringNetwork(&mocks.MeasuringNetwork{ + MockNewParallelUDPResolver: func(logger model.DebugLogger, dialer model.Dialer, endpoint string) model.Resolver { + return &mocks.Resolver{ + MockLookupHost: func(ctx context.Context, domain string) ([]string, error) { + return nil, mockedErr + }, + } + }, + MockNewDialerWithoutResolver: func(dl model.DebugLogger, w ...model.DialerWrapper) model.Dialer { + return &mocks.Dialer{ + MockDialContext: func(ctx context.Context, network, address string) (net.Conn, error) { + panic("should not be called") + }, + } + }, + })) + f := DNSLookupUDP(rt, "1.1.1.1:53") + res := f.Apply(context.Background(), NewMaybeWithValue(domain)) if res.Error != mockedErr { t.Fatalf("unexpected error type: %s", res.Error) } - if res.State == nil { - t.Fatal("unexpected nil state") - } - if res.State.Addresses != nil { - t.Fatal("expected empty addresses here") + if res.State != nil { + t.Fatal("expected nil state") } }) t.Run("with success", func(t *testing.T) { - f := dnsLookupUDPFunc{ - Resolver: "1.1.1.1:53", - mockResolver: &mocks.Resolver{MockLookupHost: func(ctx context.Context, domain string) ([]string, error) { - return []string{"93.184.216.34"}, nil - }}, - } - res := f.Apply(context.Background(), domain) - if res.Observations == nil || len(res.Observations) <= 0 { - t.Fatal("unexpected empty observations") - } + rt := NewRuntimeMeasurexLite(model.DiscardLogger, time.Now(), RuntimeMeasurexLiteOptionMeasuringNetwork(&mocks.MeasuringNetwork{ + MockNewParallelUDPResolver: func(logger model.DebugLogger, dialer model.Dialer, address string) model.Resolver { + return &mocks.Resolver{ + MockLookupHost: func(ctx context.Context, domain string) ([]string, error) { + return []string{"93.184.216.34"}, nil + }, + } + }, + MockNewDialerWithoutResolver: func(dl model.DebugLogger, w ...model.DialerWrapper) model.Dialer { + return &mocks.Dialer{ + MockDialContext: func(ctx context.Context, network, address string) (net.Conn, error) { + panic("should not be called") + }, + } + }, + })) + f := DNSLookupUDP(rt, "1.1.1.1:53") + res := f.Apply(context.Background(), NewMaybeWithValue(domain)) if res.Error != nil { t.Fatalf("unexpected error: %s", res.Error) } @@ -224,9 +205,6 @@ func TestLookupUDP(t *testing.T) { if len(res.State.Addresses) != 1 || res.State.Addresses[0] != "93.184.216.34" { t.Fatal("unexpected addresses") } - if diff := cmp.Diff([]string{"antani"}, res.State.Trace.Tags()); diff != "" { - t.Fatal(diff) - } }) }) } diff --git a/pkg/dslx/endpoint.go b/pkg/dslx/endpoint.go index 88b983d3..c9f5b0c4 100644 --- a/pkg/dslx/endpoint.go +++ b/pkg/dslx/endpoint.go @@ -5,10 +5,9 @@ package dslx // import ( - "sync/atomic" - "time" - - "github.com/ooni/probe-engine/pkg/model" + "context" + "net" + "strconv" ) type ( @@ -29,20 +28,11 @@ type Endpoint struct { // Domain is the OPTIONAL domain used to resolve the endpoints' IP address. Domain string - // IDGenerator is MANDATORY the ID generator to use. - IDGenerator *atomic.Int64 - - // Logger is the MANDATORY logger to use. - Logger model.Logger - // Network is the MANDATORY endpoint network. Network string // Tags contains OPTIONAL tags for tagging observations. Tags []string - - // ZeroTime is the MANDATORY zero time of the measurement. - ZeroTime time.Time } // EndpointOption is an option you can use to construct EndpointState. @@ -55,20 +45,6 @@ func EndpointOptionDomain(value string) EndpointOption { } } -// EndpointOptionIDGenerator allows to set the ID generator. -func EndpointOptionIDGenerator(value *atomic.Int64) EndpointOption { - return func(es *Endpoint) { - es.IDGenerator = value - } -} - -// EndpointOptionLogger allows to set the logger. -func EndpointOptionLogger(value model.Logger) EndpointOption { - return func(es *Endpoint) { - es.Logger = value - } -} - // EndpointOptionTags allows to set tags to tag observations. func EndpointOptionTags(value ...string) EndpointOption { return func(es *Endpoint) { @@ -76,13 +52,6 @@ func EndpointOptionTags(value ...string) EndpointOption { } } -// EndpointOptionZeroTime allows to set the zero time. -func EndpointOptionZeroTime(value time.Time) EndpointOption { - return func(es *Endpoint) { - es.ZeroTime = value - } -} - // NewEndpoint creates a new network endpoint (i.e., a three tuple composed // of a network protocol, an IP address, and a port). // @@ -97,16 +66,48 @@ func EndpointOptionZeroTime(value time.Time) EndpointOption { func NewEndpoint( network EndpointNetwork, address EndpointAddress, options ...EndpointOption) *Endpoint { epnt := &Endpoint{ - Address: string(address), - Domain: "", - IDGenerator: &atomic.Int64{}, - Logger: model.DiscardLogger, - Network: string(network), - Tags: []string{}, - ZeroTime: time.Now(), + Address: string(address), + Domain: "", + Network: string(network), + Tags: []string{}, } for _, option := range options { option(epnt) } return epnt } + +// MakeEndpoint returns a [Func] that creates an [*Endpoint] given [*ResolvedAddress]. +func MakeEndpoint(network EndpointNetwork, port EndpointPort, options ...EndpointOption) Func[*ResolvedAddress, *Endpoint] { + return Operation[*ResolvedAddress, *Endpoint](func(ctx context.Context, addr *ResolvedAddress) (*Endpoint, error) { + // create the destination endpoint address + addrport := EndpointAddress(net.JoinHostPort(addr.Address, strconv.Itoa(int(port)))) + + // make sure we include the proper domain name first but allow the caller + // to potentially override the domain name using options + allOptions := []EndpointOption{ + EndpointOptionDomain(addr.Domain), + } + allOptions = append(allOptions, options...) + + // build and return the endpoint + endpoint := NewEndpoint(network, addrport, allOptions...) + return endpoint, nil + }) +} + +// MeasureResolvedAddresses returns a [Func] that measures the resolved addresses provided +// as the input argument using each of the provided functions. +func MeasureResolvedAddresses(fxs ...Func[*ResolvedAddress, Void]) Func[*ResolvedAddresses, Void] { + return Operation[*ResolvedAddresses, Void](func(ctx context.Context, addrs *ResolvedAddresses) (Void, error) { + // TODO(https://github.com/ooni/probe/issues/2619): we may want to configure this + const parallelism = Parallelism(3) + + // run the matrix until the output is drained + for range Matrix(ctx, parallelism, addrs.Flatten(), fxs) { + // nothing + } + + return Void{}, nil + }) +} diff --git a/pkg/dslx/endpoint_test.go b/pkg/dslx/endpoint_test.go index b85e13e7..cc62eced 100644 --- a/pkg/dslx/endpoint_test.go +++ b/pkg/dslx/endpoint_test.go @@ -3,25 +3,19 @@ package dslx import ( "sync/atomic" "testing" - "time" "github.com/google/go-cmp/cmp" - "github.com/ooni/probe-engine/pkg/model" ) func TestEndpoint(t *testing.T) { idGen := &atomic.Int64{} idGen.Add(42) - zt := time.Now() t.Run("Create new endpoint", func(t *testing.T) { testEndpoint := NewEndpoint( "network", "10.9.8.76", EndpointOptionDomain("www.example.com"), - EndpointOptionIDGenerator(idGen), - EndpointOptionLogger(model.DiscardLogger), - EndpointOptionZeroTime(zt), EndpointOptionTags("antani"), ) if testEndpoint.Network != "network" { @@ -33,15 +27,6 @@ func TestEndpoint(t *testing.T) { if testEndpoint.Domain != "www.example.com" { t.Fatalf("unexpected domain") } - if testEndpoint.IDGenerator != idGen { - t.Fatalf("unexpected IDGenerator") - } - if testEndpoint.Logger != model.DiscardLogger { - t.Fatalf("unexpected logger") - } - if testEndpoint.ZeroTime != zt { - t.Fatalf("unexpected zero time") - } if diff := cmp.Diff([]string{"antani"}, testEndpoint.Tags); diff != "" { t.Fatal(diff) } diff --git a/pkg/dslx/fxasync.go b/pkg/dslx/fxasync.go index 7ecdc3fd..c11c553d 100644 --- a/pkg/dslx/fxasync.go +++ b/pkg/dslx/fxasync.go @@ -30,10 +30,12 @@ type Parallelism int // The return value is the channel generating fx(a) // for every a in inputs. This channel will also be closed // to signal EOF to the consumer. +// +// Deprecated: use Matrix instead. func Map[A, B any]( ctx context.Context, parallelism Parallelism, - fx Func[A, *Maybe[B]], + fx Func[A, B], inputs <-chan A, ) <-chan *Maybe[B] { // create channel for returning results @@ -49,7 +51,7 @@ func Map[A, B any]( go func() { defer wg.Done() for a := range inputs { - r <- fx.Apply(ctx, a) + r <- fx.Apply(ctx, NewMaybeWithValue(a)) } }() } @@ -77,11 +79,13 @@ func Map[A, B any]( // - fn is the list of functions. // // The return value is the list [fx(a)] for every fx in fn. +// +// Deprecated: use Matrix instead. func Parallel[A, B any]( ctx context.Context, parallelism Parallelism, input A, - fn ...Func[A, *Maybe[B]], + fn ...Func[A, B], ) []*Maybe[B] { c := ParallelAsync(ctx, parallelism, input, StreamList(fn...)) return Collect(c) @@ -90,11 +94,13 @@ func Parallel[A, B any]( // ParallelAsync is like Parallel but deals with channels. We assume the // input channel will be closed to signal EOF. We will close the output // channel to signal EOF to the consumer. +// +// Deprecated: use Matrix instead. func ParallelAsync[A, B any]( ctx context.Context, parallelism Parallelism, input A, - funcs <-chan Func[A, *Maybe[B]], + funcs <-chan Func[A, B], ) <-chan *Maybe[B] { // create channel for returning results r := make(chan *Maybe[B]) @@ -109,7 +115,7 @@ func ParallelAsync[A, B any]( go func() { defer wg.Done() for fx := range funcs { - r <- fx.Apply(ctx, input) + r <- fx.Apply(ctx, NewMaybeWithValue(input)) } }() } @@ -124,10 +130,64 @@ func ParallelAsync[A, B any]( } // ApplyAsync is equivalent to calling Apply but returns a channel. +// +// Deprecated: use Matrix instead. func ApplyAsync[A, B any]( ctx context.Context, - fx Func[A, *Maybe[B]], + fx Func[A, B], input A, ) <-chan *Maybe[B] { return Map(ctx, Parallelism(1), fx, StreamList(input)) } + +// matrixPoint is a point within the matrix used by [Matrix]. +type matrixPoint[A, B any] struct { + f Func[A, B] + in A +} + +// matrixMin can be replaced with the built-in min when we switch to go1.21. +func matrixMin(a, b Parallelism) Parallelism { + if a < b { + return a + } + return b +} + +// Matrix invokes each function on each input using N goroutines and streams the results to a channel. +func Matrix[A, B any](ctx context.Context, N Parallelism, inputs []A, functions []Func[A, B]) <-chan *Maybe[B] { + // make output + output := make(chan *Maybe[B]) + + // stream all the possible points + points := make(chan *matrixPoint[A, B]) + go func() { + defer close(points) + for _, input := range inputs { + for _, fx := range functions { + points <- &matrixPoint[A, B]{f: fx, in: input} + } + } + }() + + // spawn goroutines + wg := &sync.WaitGroup{} + N = matrixMin(1, N) + for i := Parallelism(0); i < N; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for p := range points { + output <- p.f.Apply(ctx, NewMaybeWithValue(p.in)) + } + }() + } + + // close output channel when done + go func() { + defer close(output) + wg.Wait() + }() + + return output +} diff --git a/pkg/dslx/fxasync_test.go b/pkg/dslx/fxasync_test.go index cf1af053..95499d1d 100644 --- a/pkg/dslx/fxasync_test.go +++ b/pkg/dslx/fxasync_test.go @@ -4,9 +4,11 @@ import ( "context" "sync" "testing" + + "github.com/ooni/probe-engine/pkg/runtimex" ) -func getFnWait(wg *sync.WaitGroup) Func[int, *Maybe[int]] { +func getFnWait(wg *sync.WaitGroup) Func[int, int] { return &fnWait{wg} } @@ -14,10 +16,11 @@ type fnWait struct { wg *sync.WaitGroup // set to n corresponding to the number of used goroutines } -func (f *fnWait) Apply(ctx context.Context, i int) *Maybe[int] { +func (f *fnWait) Apply(ctx context.Context, i *Maybe[int]) *Maybe[int] { + runtimex.Assert(i.Error == nil, "did not expect to see an error here") f.wg.Done() f.wg.Wait() // continue when n goroutines have reached this point - return &Maybe[int]{State: i + 1} + return &Maybe[int]{State: i.State + 1} } /* @@ -86,7 +89,7 @@ func TestParallel(t *testing.T) { t.Run(name, func(t *testing.T) { wg := sync.WaitGroup{} wg.Add(tt.funcs) - funcs := []Func[int, *Maybe[int]]{} + funcs := []Func[int, int]{} for i := 0; i < tt.funcs; i++ { funcs = append(funcs, getFnWait(&wg)) } @@ -98,3 +101,15 @@ func TestParallel(t *testing.T) { } }) } + +func TestMatrixMin(t *testing.T) { + if v := matrixMin(1, 7); v != 1 { + t.Fatal("expected to see 1, got", v) + } + if v := matrixMin(7, 4); v != 4 { + t.Fatal("expected to see 4, got", v) + } + if v := matrixMin(11, 11); v != 11 { + t.Fatal("expected to see 11, got", v) + } +} diff --git a/pkg/dslx/fxcore.go b/pkg/dslx/fxcore.go index 5e15cbfb..4dfcade4 100644 --- a/pkg/dslx/fxcore.go +++ b/pkg/dslx/fxcore.go @@ -6,15 +6,36 @@ package dslx import ( "context" - "sync/atomic" - - "github.com/ooni/probe-engine/pkg/netxlite" - "github.com/ooni/probe-engine/pkg/runtimex" + "errors" + "sync" ) // Func is a function f: (context.Context, A) -> B. type Func[A, B any] interface { - Apply(ctx context.Context, a A) B + Apply(ctx context.Context, a *Maybe[A]) *Maybe[B] +} + +// FuncAdapter adapts a func to be a [Func]. +type FuncAdapter[A, B any] func(ctx context.Context, a *Maybe[A]) *Maybe[B] + +// Apply implements Func. +func (fa FuncAdapter[A, B]) Apply(ctx context.Context, a *Maybe[A]) *Maybe[B] { + return fa(ctx, a) +} + +// Operation adapts a golang function to behave like a Func. +type Operation[A, B any] func(ctx context.Context, a A) (B, error) + +// Apply implements Func. +func (op Operation[A, B]) Apply(ctx context.Context, a *Maybe[A]) *Maybe[B] { + if err := a.Error; err != nil { + return NewMaybeWithError[B](err) + } + out, err := op(ctx, a.State) + if err != nil { + return NewMaybeWithError[B](err) + } + return NewMaybeWithValue(out) } // Maybe is the result of an operation implemented by this package @@ -23,19 +44,29 @@ type Maybe[State any] struct { // Error is either the error that occurred or nil. Error error - // Observations contains the collected observations. - Observations []*Observations - - // Operation contains the name of this operation. - Operation string - // State contains state passed between function calls. You should // only access State when Error is nil and Skipped is false. State State } +// NewMaybeWithValue constructs a Maybe containing the given value. +func NewMaybeWithValue[State any](value State) *Maybe[State] { + return &Maybe[State]{ + Error: nil, + State: value, + } +} + +// NewMaybeWithError constructs a Maybe containing the given error. +func NewMaybeWithError[State any](err error) *Maybe[State] { + return &Maybe[State]{ + Error: err, + State: *new(State), // zero value + } +} + // Compose2 composes two operations such as [TCPConnect] and [TLSHandshake]. -func Compose2[A, B, C any](f Func[A, *Maybe[B]], g Func[B, *Maybe[C]]) Func[A, *Maybe[C]] { +func Compose2[A, B, C any](f Func[A, B], g Func[B, C]) Func[A, C] { return &compose2Func[A, B, C]{ f: f, g: g, @@ -44,99 +75,76 @@ func Compose2[A, B, C any](f Func[A, *Maybe[B]], g Func[B, *Maybe[C]]) Func[A, * // compose2Func is the type returned by [Compose2]. type compose2Func[A, B, C any] struct { - f Func[A, *Maybe[B]] - g Func[B, *Maybe[C]] + f Func[A, B] + g Func[B, C] } // Apply implements Func -func (h *compose2Func[A, B, C]) Apply(ctx context.Context, a A) *Maybe[C] { - mb := h.f.Apply(ctx, a) - runtimex.Assert(mb != nil, "h.f.Apply returned a nil pointer") - if mb.Error != nil { - return &Maybe[C]{ - Error: mb.Error, - Observations: mb.Observations, - Operation: mb.Operation, - State: *new(C), // zero value - } - } - mc := h.g.Apply(ctx, mb.State) - runtimex.Assert(mc != nil, "h.g.Apply returned a nil pointer") - op := mc.Operation - if op == "" { // propagate the previous operation name, if this operation has none - op = mb.Operation - } - return &Maybe[C]{ - Error: mc.Error, - Observations: append(mb.Observations, mc.Observations...), // merge observations - Operation: op, - State: mc.State, - } +func (h *compose2Func[A, B, C]) Apply(ctx context.Context, a *Maybe[A]) *Maybe[C] { + return h.g.Apply(ctx, h.f.Apply(ctx, a)) } -// NewCounter generates an instance of *Counter -func NewCounter[T any]() *Counter[T] { - return &Counter[T]{} -} +// Void is the empty data structure. +type Void struct{} -// Counter allows to count how many times -// a Func[T, *Maybe[T]] is invoked. -type Counter[T any] struct { - n atomic.Int64 +// Discard transforms any type to [Void]. +func Discard[T any]() Func[T, Void] { + return Operation[T, Void](func(ctx context.Context, input T) (Void, error) { + return Void{}, nil + }) } -// Value returns the counter's value. -func (c *Counter[T]) Value() int64 { - return c.n.Load() -} - -// Func returns a Func[T, *Maybe[T]] that updates the counter. -func (c *Counter[T]) Func() Func[T, *Maybe[T]] { - return &counterFunc[T]{c} -} +// ErrSkip is an error that indicates that we already processed an error emitted +// by a previous stage, so we are using this error to avoid counting the original +// error more than once when computing statistics, e.g., in [*Stats]. +var ErrSkip = errors.New("dslx: error already processed by a previous stage") -// counterFunc is the Func returned by CounterFunc.Func. -type counterFunc[T any] struct { - c *Counter[T] +// Stats measures the number of successes and failures. +// +// The zero value is invalid; use [NewStats]. +type Stats[T any] struct { + m map[string]int64 + mu sync.Mutex } -// Apply implements Func. -func (c *counterFunc[T]) Apply(ctx context.Context, value T) *Maybe[T] { - c.c.n.Add(1) - return &Maybe[T]{ - Error: nil, - Observations: nil, - Operation: "", // we cannot fail, so no need to store operation name - State: value, +// NewStats creates a [*Stats] instance. +func NewStats[T any]() *Stats[T] { + return &Stats[T]{ + m: map[string]int64{}, + mu: sync.Mutex{}, } } -// FirstErrorExcludingBrokenIPv6Errors returns the first error and failed operation in a list of -// *Maybe[T] excluding errors known to be linked with IPv6 issues. -func FirstErrorExcludingBrokenIPv6Errors[T any](entries ...*Maybe[T]) (string, error) { - for _, entry := range entries { - if entry.Error == nil { - continue +// Observer returns a Func that observes the results of the previous pipeline stage. This function +// converts any error that it sees to [ErrSkip]. This function does not account for [ErrSkip], meaning +// that you will never see [ErrSkip] in the stats returned by [Stats.Export]. +func (s *Stats[T]) Observer() Func[T, T] { + return FuncAdapter[T, T](func(ctx context.Context, minput *Maybe[T]) *Maybe[T] { + defer s.mu.Unlock() + s.mu.Lock() + var r string + if err := minput.Error; err != nil { + if errors.Is(err, ErrSkip) { + return NewMaybeWithError[T](ErrSkip) // as documented + } + r = err.Error() } - err := entry.Error - switch err.Error() { - case netxlite.FailureNetworkUnreachable, netxlite.FailureHostUnreachable: - // This class of errors is often times linked with wrongly - // configured IPv6, therefore we skip them. - default: - return entry.Operation, err + s.m[r]++ + if r != "" { + return NewMaybeWithError[T](ErrSkip) // as documented } - } - return "", nil + return minput + }) } -// FirstError returns the first error and failed operation in a list of *Maybe[T]. -func FirstError[T any](entries ...*Maybe[T]) (string, error) { - for _, entry := range entries { - if entry.Error == nil { - continue - } - return entry.Operation, entry.Error +// Export exports the current stats without clearing the internally used map such that +// statistics accumulate over time and never reset for the [*Stats] lifecycle. +func (s *Stats[T]) Export() (out map[string]int64) { + out = make(map[string]int64) + defer s.mu.Unlock() + s.mu.Lock() + for r, cnt := range s.m { + out[r] = cnt } - return "", nil + return } diff --git a/pkg/dslx/fxcore_test.go b/pkg/dslx/fxcore_test.go index 6d2410eb..c3d62411 100644 --- a/pkg/dslx/fxcore_test.go +++ b/pkg/dslx/fxcore_test.go @@ -4,12 +4,14 @@ import ( "context" "errors" "testing" + "time" + "github.com/ooni/probe-engine/pkg/mocks" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/netxlite" ) -func getFn(err error, name string) Func[int, *Maybe[int]] { +func getFn(err error, name string) Func[int, int] { return &fn{err: err, name: name} } @@ -18,19 +20,45 @@ type fn struct { name string } -func (f *fn) Apply(ctx context.Context, i int) *Maybe[int] { +func (f *fn) Apply(ctx context.Context, i *Maybe[int]) *Maybe[int] { + if i.Error != nil { + return i + } return &Maybe[int]{ Error: f.err, - State: i + 1, - Observations: []*Observations{ - { - NetworkEvents: []*model.ArchivalNetworkEvent{{Tags: []string{"apply"}}}, - }, - }, - Operation: f.name, + State: i.State + 1, } } +func TestStageAdapter(t *testing.T) { + t.Run("make sure that we handle a previous stage failure", func(t *testing.T) { + unet := &mocks.UnderlyingNetwork{ + // explicitly empty so we crash if we try using underlying network functionality + } + netx := &netxlite.Netx{Underlying: unet} + + // create runtime + rt := NewMinimalRuntime(model.DiscardLogger, time.Now(), MinimalRuntimeOptionMeasuringNetwork(netx)) + + // create measurement pipeline where we run DNS lookups + pipeline := DNSLookupGetaddrinfo(rt) + + // create input that contains an error + input := &Maybe[*DomainToResolve]{ + Error: errors.New("mocked error"), + State: nil, + } + + // run the pipeline + output := pipeline.Apply(context.Background(), input) + + // make sure the output contains the same error as the input + if !errors.Is(output.Error, input.Error) { + t.Fatal("unexpected error") + } + }) +} + /* Test cases: - Compose 2 functions: @@ -53,16 +81,10 @@ func TestCompose2(t *testing.T) { f1 := getFn(tt.err, "maybe fail") f2 := getFn(nil, "succeed") composit := Compose2(f1, f2) - r := composit.Apply(context.Background(), tt.input) + r := composit.Apply(context.Background(), NewMaybeWithValue(tt.input)) if r.Error != tt.err { t.Fatalf("unexpected error") } - if tt.err != nil && r.Operation != "maybe fail" { - t.Fatalf("unexpected operation string") - } - if len(r.Observations) != tt.numObs { - t.Fatalf("unexpected number of (merged) observations") - } }) } }) @@ -73,133 +95,12 @@ func TestGen(t *testing.T) { incFunc := getFn(nil, "succeed") composit := Compose14(incFunc, incFunc, incFunc, incFunc, incFunc, incFunc, incFunc, incFunc, incFunc, incFunc, incFunc, incFunc, incFunc, incFunc) - r := composit.Apply(context.Background(), 0) + r := composit.Apply(context.Background(), NewMaybeWithValue(0)) if r.Error != nil { t.Fatalf("unexpected error: %s", r.Error) } if r.State != 14 { t.Fatalf("unexpected result state") } - if r.Operation != "succeed" { - t.Fatal("unexpected operation string") - } - }) -} - -func TestObservations(t *testing.T) { - t.Run("Extract observations", func(t *testing.T) { - fn1 := getFn(nil, "succeed") - fn2 := getFn(nil, "succeed") - composit := Compose2(fn1, fn2) - r1 := composit.Apply(context.Background(), 3) - r2 := composit.Apply(context.Background(), 42) - if len(r1.Observations) != 2 || len(r2.Observations) != 2 { - t.Fatalf("unexpected number of observations") - } - mergedObservations := ExtractObservations(r1, r2) - if len(mergedObservations) != 4 { - t.Fatalf("unexpected number of merged observations") - } - }) -} - -/* -Test cases: -- Success counter: - - pipeline succeeds - - pipeline fails -*/ -func TestCounter(t *testing.T) { - t.Run("Success counter", func(t *testing.T) { - tests := map[string]struct { - err error - expect int64 - }{ - "pipeline succeeds": {err: nil, expect: 1}, - "pipeline fails": {err: errors.New("mocked"), expect: 0}, - } - for name, tt := range tests { - t.Run(name, func(t *testing.T) { - fn := getFn(tt.err, "maybe fail") - cnt := NewCounter[int]() - composit := Compose2(fn, cnt.Func()) - r := composit.Apply(context.Background(), 42) - cntVal := cnt.Value() - if cntVal != tt.expect { - t.Fatalf("unexpected counter value") - } - if r.Operation != "maybe fail" { - t.Fatal("unexpected operation string") - } - }) - } - }) -} - -/* -Test cases: -- Extract first error from list of *Maybe: - - without errors - - with errors - -- Extract first error excluding broken IPv6 errors: - - without errors - - with errors -*/ -func TestFirstError(t *testing.T) { - networkUnreachable := errors.New(netxlite.FailureNetworkUnreachable) - mockErr := errors.New("mocked") - errRes := []*Maybe[string]{ - {Error: nil, Operation: "succeeds"}, - {Error: networkUnreachable, Operation: "broken IPv6"}, - {Error: mockErr, Operation: "mock error"}, - } - noErrRes := []*Maybe[int64]{ - {Error: nil, Operation: "succeeds"}, - {Error: nil, Operation: "succeeds"}, - } - - t.Run("Extract first error from list of *Maybe", func(t *testing.T) { - t.Run("without errors", func(t *testing.T) { - failedOp, firstErr := FirstError(noErrRes...) - if firstErr != nil { - t.Fatalf("unexpected error: %s", firstErr) - } - if failedOp != "" { - t.Fatalf("unexpected failed operation") - } - }) - - t.Run("with errors", func(t *testing.T) { - failedOp, firstErr := FirstError(errRes...) - if firstErr != networkUnreachable { - t.Fatalf("unexpected error: %s", firstErr) - } - if failedOp != "broken IPv6" { - t.Fatalf("unexpected failed operation") - } - }) - }) - - t.Run("Extract first error excluding broken IPv6 errors", func(t *testing.T) { - t.Run("without errors", func(t *testing.T) { - failedOp, firstErrExclIPv6 := FirstErrorExcludingBrokenIPv6Errors(noErrRes...) - if firstErrExclIPv6 != nil { - t.Fatalf("unexpected error: %s", firstErrExclIPv6) - } - if failedOp != "" { - t.Fatalf("unexpected failed operation") - } - }) - - t.Run("with errors", func(t *testing.T) { - failedOp, firstErrExclIPv6 := FirstErrorExcludingBrokenIPv6Errors(errRes...) - if firstErrExclIPv6 != mockErr { - t.Fatalf("unexpected error: %s", firstErrExclIPv6) - } - if failedOp != "mock error" { - t.Fatalf("unexpected failed operation") - } - }) }) } diff --git a/pkg/dslx/fxgen.go b/pkg/dslx/fxgen.go index 7d56cbf7..08ec72a6 100644 --- a/pkg/dslx/fxgen.go +++ b/pkg/dslx/fxgen.go @@ -11,10 +11,10 @@ func Compose3[ T2 any, T3 any, ]( - f0 Func[T0, *Maybe[T1]], - f1 Func[T1, *Maybe[T2]], - f2 Func[T2, *Maybe[T3]], -) Func[T0, *Maybe[T3]] { + f0 Func[T0, T1], + f1 Func[T1, T2], + f2 Func[T2, T3], +) Func[T0, T3] { return Compose2(f0, Compose2(f1, f2)) } @@ -26,11 +26,11 @@ func Compose4[ T3 any, T4 any, ]( - f0 Func[T0, *Maybe[T1]], - f1 Func[T1, *Maybe[T2]], - f2 Func[T2, *Maybe[T3]], - f3 Func[T3, *Maybe[T4]], -) Func[T0, *Maybe[T4]] { + f0 Func[T0, T1], + f1 Func[T1, T2], + f2 Func[T2, T3], + f3 Func[T3, T4], +) Func[T0, T4] { return Compose2(f0, Compose3(f1, f2, f3)) } @@ -43,12 +43,12 @@ func Compose5[ T4 any, T5 any, ]( - f0 Func[T0, *Maybe[T1]], - f1 Func[T1, *Maybe[T2]], - f2 Func[T2, *Maybe[T3]], - f3 Func[T3, *Maybe[T4]], - f4 Func[T4, *Maybe[T5]], -) Func[T0, *Maybe[T5]] { + f0 Func[T0, T1], + f1 Func[T1, T2], + f2 Func[T2, T3], + f3 Func[T3, T4], + f4 Func[T4, T5], +) Func[T0, T5] { return Compose2(f0, Compose4(f1, f2, f3, f4)) } @@ -62,13 +62,13 @@ func Compose6[ T5 any, T6 any, ]( - f0 Func[T0, *Maybe[T1]], - f1 Func[T1, *Maybe[T2]], - f2 Func[T2, *Maybe[T3]], - f3 Func[T3, *Maybe[T4]], - f4 Func[T4, *Maybe[T5]], - f5 Func[T5, *Maybe[T6]], -) Func[T0, *Maybe[T6]] { + f0 Func[T0, T1], + f1 Func[T1, T2], + f2 Func[T2, T3], + f3 Func[T3, T4], + f4 Func[T4, T5], + f5 Func[T5, T6], +) Func[T0, T6] { return Compose2(f0, Compose5(f1, f2, f3, f4, f5)) } @@ -83,14 +83,14 @@ func Compose7[ T6 any, T7 any, ]( - f0 Func[T0, *Maybe[T1]], - f1 Func[T1, *Maybe[T2]], - f2 Func[T2, *Maybe[T3]], - f3 Func[T3, *Maybe[T4]], - f4 Func[T4, *Maybe[T5]], - f5 Func[T5, *Maybe[T6]], - f6 Func[T6, *Maybe[T7]], -) Func[T0, *Maybe[T7]] { + f0 Func[T0, T1], + f1 Func[T1, T2], + f2 Func[T2, T3], + f3 Func[T3, T4], + f4 Func[T4, T5], + f5 Func[T5, T6], + f6 Func[T6, T7], +) Func[T0, T7] { return Compose2(f0, Compose6(f1, f2, f3, f4, f5, f6)) } @@ -106,15 +106,15 @@ func Compose8[ T7 any, T8 any, ]( - f0 Func[T0, *Maybe[T1]], - f1 Func[T1, *Maybe[T2]], - f2 Func[T2, *Maybe[T3]], - f3 Func[T3, *Maybe[T4]], - f4 Func[T4, *Maybe[T5]], - f5 Func[T5, *Maybe[T6]], - f6 Func[T6, *Maybe[T7]], - f7 Func[T7, *Maybe[T8]], -) Func[T0, *Maybe[T8]] { + f0 Func[T0, T1], + f1 Func[T1, T2], + f2 Func[T2, T3], + f3 Func[T3, T4], + f4 Func[T4, T5], + f5 Func[T5, T6], + f6 Func[T6, T7], + f7 Func[T7, T8], +) Func[T0, T8] { return Compose2(f0, Compose7(f1, f2, f3, f4, f5, f6, f7)) } @@ -131,16 +131,16 @@ func Compose9[ T8 any, T9 any, ]( - f0 Func[T0, *Maybe[T1]], - f1 Func[T1, *Maybe[T2]], - f2 Func[T2, *Maybe[T3]], - f3 Func[T3, *Maybe[T4]], - f4 Func[T4, *Maybe[T5]], - f5 Func[T5, *Maybe[T6]], - f6 Func[T6, *Maybe[T7]], - f7 Func[T7, *Maybe[T8]], - f8 Func[T8, *Maybe[T9]], -) Func[T0, *Maybe[T9]] { + f0 Func[T0, T1], + f1 Func[T1, T2], + f2 Func[T2, T3], + f3 Func[T3, T4], + f4 Func[T4, T5], + f5 Func[T5, T6], + f6 Func[T6, T7], + f7 Func[T7, T8], + f8 Func[T8, T9], +) Func[T0, T9] { return Compose2(f0, Compose8(f1, f2, f3, f4, f5, f6, f7, f8)) } @@ -158,17 +158,17 @@ func Compose10[ T9 any, T10 any, ]( - f0 Func[T0, *Maybe[T1]], - f1 Func[T1, *Maybe[T2]], - f2 Func[T2, *Maybe[T3]], - f3 Func[T3, *Maybe[T4]], - f4 Func[T4, *Maybe[T5]], - f5 Func[T5, *Maybe[T6]], - f6 Func[T6, *Maybe[T7]], - f7 Func[T7, *Maybe[T8]], - f8 Func[T8, *Maybe[T9]], - f9 Func[T9, *Maybe[T10]], -) Func[T0, *Maybe[T10]] { + f0 Func[T0, T1], + f1 Func[T1, T2], + f2 Func[T2, T3], + f3 Func[T3, T4], + f4 Func[T4, T5], + f5 Func[T5, T6], + f6 Func[T6, T7], + f7 Func[T7, T8], + f8 Func[T8, T9], + f9 Func[T9, T10], +) Func[T0, T10] { return Compose2(f0, Compose9(f1, f2, f3, f4, f5, f6, f7, f8, f9)) } @@ -187,18 +187,18 @@ func Compose11[ T10 any, T11 any, ]( - f0 Func[T0, *Maybe[T1]], - f1 Func[T1, *Maybe[T2]], - f2 Func[T2, *Maybe[T3]], - f3 Func[T3, *Maybe[T4]], - f4 Func[T4, *Maybe[T5]], - f5 Func[T5, *Maybe[T6]], - f6 Func[T6, *Maybe[T7]], - f7 Func[T7, *Maybe[T8]], - f8 Func[T8, *Maybe[T9]], - f9 Func[T9, *Maybe[T10]], - f10 Func[T10, *Maybe[T11]], -) Func[T0, *Maybe[T11]] { + f0 Func[T0, T1], + f1 Func[T1, T2], + f2 Func[T2, T3], + f3 Func[T3, T4], + f4 Func[T4, T5], + f5 Func[T5, T6], + f6 Func[T6, T7], + f7 Func[T7, T8], + f8 Func[T8, T9], + f9 Func[T9, T10], + f10 Func[T10, T11], +) Func[T0, T11] { return Compose2(f0, Compose10(f1, f2, f3, f4, f5, f6, f7, f8, f9, f10)) } @@ -218,19 +218,19 @@ func Compose12[ T11 any, T12 any, ]( - f0 Func[T0, *Maybe[T1]], - f1 Func[T1, *Maybe[T2]], - f2 Func[T2, *Maybe[T3]], - f3 Func[T3, *Maybe[T4]], - f4 Func[T4, *Maybe[T5]], - f5 Func[T5, *Maybe[T6]], - f6 Func[T6, *Maybe[T7]], - f7 Func[T7, *Maybe[T8]], - f8 Func[T8, *Maybe[T9]], - f9 Func[T9, *Maybe[T10]], - f10 Func[T10, *Maybe[T11]], - f11 Func[T11, *Maybe[T12]], -) Func[T0, *Maybe[T12]] { + f0 Func[T0, T1], + f1 Func[T1, T2], + f2 Func[T2, T3], + f3 Func[T3, T4], + f4 Func[T4, T5], + f5 Func[T5, T6], + f6 Func[T6, T7], + f7 Func[T7, T8], + f8 Func[T8, T9], + f9 Func[T9, T10], + f10 Func[T10, T11], + f11 Func[T11, T12], +) Func[T0, T12] { return Compose2(f0, Compose11(f1, f2, f3, f4, f5, f6, f7, f8, f9, f10, f11)) } @@ -251,20 +251,20 @@ func Compose13[ T12 any, T13 any, ]( - f0 Func[T0, *Maybe[T1]], - f1 Func[T1, *Maybe[T2]], - f2 Func[T2, *Maybe[T3]], - f3 Func[T3, *Maybe[T4]], - f4 Func[T4, *Maybe[T5]], - f5 Func[T5, *Maybe[T6]], - f6 Func[T6, *Maybe[T7]], - f7 Func[T7, *Maybe[T8]], - f8 Func[T8, *Maybe[T9]], - f9 Func[T9, *Maybe[T10]], - f10 Func[T10, *Maybe[T11]], - f11 Func[T11, *Maybe[T12]], - f12 Func[T12, *Maybe[T13]], -) Func[T0, *Maybe[T13]] { + f0 Func[T0, T1], + f1 Func[T1, T2], + f2 Func[T2, T3], + f3 Func[T3, T4], + f4 Func[T4, T5], + f5 Func[T5, T6], + f6 Func[T6, T7], + f7 Func[T7, T8], + f8 Func[T8, T9], + f9 Func[T9, T10], + f10 Func[T10, T11], + f11 Func[T11, T12], + f12 Func[T12, T13], +) Func[T0, T13] { return Compose2(f0, Compose12(f1, f2, f3, f4, f5, f6, f7, f8, f9, f10, f11, f12)) } @@ -286,20 +286,20 @@ func Compose14[ T13 any, T14 any, ]( - f0 Func[T0, *Maybe[T1]], - f1 Func[T1, *Maybe[T2]], - f2 Func[T2, *Maybe[T3]], - f3 Func[T3, *Maybe[T4]], - f4 Func[T4, *Maybe[T5]], - f5 Func[T5, *Maybe[T6]], - f6 Func[T6, *Maybe[T7]], - f7 Func[T7, *Maybe[T8]], - f8 Func[T8, *Maybe[T9]], - f9 Func[T9, *Maybe[T10]], - f10 Func[T10, *Maybe[T11]], - f11 Func[T11, *Maybe[T12]], - f12 Func[T12, *Maybe[T13]], - f13 Func[T13, *Maybe[T14]], -) Func[T0, *Maybe[T14]] { + f0 Func[T0, T1], + f1 Func[T1, T2], + f2 Func[T2, T3], + f3 Func[T3, T4], + f4 Func[T4, T5], + f5 Func[T5, T6], + f6 Func[T6, T7], + f7 Func[T7, T8], + f8 Func[T8, T9], + f9 Func[T9, T10], + f10 Func[T10, T11], + f11 Func[T11, T12], + f12 Func[T12, T13], + f13 Func[T13, T14], +) Func[T0, T14] { return Compose2(f0, Compose13(f1, f2, f3, f4, f5, f6, f7, f8, f9, f10, f11, f12, f13)) } diff --git a/pkg/dslx/fxstream.go b/pkg/dslx/fxstream.go index 7b15efe4..21d06722 100644 --- a/pkg/dslx/fxstream.go +++ b/pkg/dslx/fxstream.go @@ -18,13 +18,11 @@ func Collect[T any](c <-chan T) (v []T) { // StreamList creates a channel out of static values. This function will // close the channel when it has streamed all the available elements. func StreamList[T any](ts ...T) <-chan T { - c := make(chan T) - go func() { - defer close(c) // as documented - for _, t := range ts { - c <- t - } - }() + c := make(chan T, len(ts)) // buffer so writing does not block + defer close(c) // as documented + for _, t := range ts { + c <- t + } return c } diff --git a/pkg/dslx/http_test.go b/pkg/dslx/http_test.go index 0cb28fc9..44147aa2 100644 --- a/pkg/dslx/http_test.go +++ b/pkg/dslx/http_test.go @@ -3,7 +3,6 @@ package dslx import ( "context" "errors" - "fmt" "io" "net/http" "strings" @@ -17,56 +16,193 @@ import ( "github.com/ooni/probe-engine/pkg/model" ) -/* -Test cases: -- Get httpRequestFunc with options -- Apply httpRequestFunc: - - with EOF - - with invalid method - - with port-less address - - with success (https) - - with success (http) - - with header options -*/ -func TestHTTPRequest(t *testing.T) { - t.Run("Get httpRequestFunc with options", func(t *testing.T) { - f := HTTPRequest( +func TestHTTPNewRequest(t *testing.T) { + t.Run("without any option and with domain", func(t *testing.T) { + ctx := context.Background() + conn := &HTTPConnection{ + Address: "130.192.91.211:443", + Domain: "example.com", + Network: "tcp", + Scheme: "https", + TLSNegotiatedProtocol: "h2", + Trace: nil, + Transport: nil, + } + + req, err := httpNewRequest(ctx, conn, model.DiscardLogger) + if err != nil { + t.Fatal(err) + } + + if req.URL.Scheme != "https" { + t.Fatal("unexpected req.URL.Scheme", req.URL.Scheme) + } + if req.URL.Host != "example.com" { + t.Fatal("unexpected req.URL.Host", req.URL.Host) + } + if req.URL.Path != "/" { + t.Fatal("unexpected req.URL.Path", req.URL.Path) + } + if req.Method != "GET" { + t.Fatal("unexpected req.Method", req.Method) + } + if req.Host != "example.com" { + t.Fatal("unexpected req.Host", req.Host) + } + headers := http.Header{ + "Host": {"example.com"}, + } + if diff := cmp.Diff(headers, req.Header); diff != "" { + t.Fatal(diff) + } + }) + + t.Run("without any option, without domain but with standard port", func(t *testing.T) { + ctx := context.Background() + conn := &HTTPConnection{ + Address: "130.192.91.211:443", + Domain: "", + Network: "tcp", + Scheme: "https", + TLSNegotiatedProtocol: "h2", + Trace: nil, + Transport: nil, + } + + req, err := httpNewRequest(ctx, conn, model.DiscardLogger) + if err != nil { + t.Fatal(err) + } + + if req.URL.Scheme != "https" { + t.Fatal("unexpected req.URL.Scheme", req.URL.Scheme) + } + if req.URL.Host != "130.192.91.211" { + t.Fatal("unexpected req.URL.Host", req.URL.Host) + } + if req.URL.Path != "/" { + t.Fatal("unexpected req.URL.Path", req.URL.Path) + } + if req.Method != "GET" { + t.Fatal("unexpected req.Method", req.Method) + } + if req.Host != "130.192.91.211" { + t.Fatal("unexpected req.Host", req.Host) + } + headers := http.Header{ + "Host": {"130.192.91.211"}, + } + if diff := cmp.Diff(headers, req.Header); diff != "" { + t.Fatal(diff) + } + }) + + t.Run("without any option, without domain but with nonstandard port", func(t *testing.T) { + ctx := context.Background() + conn := &HTTPConnection{ + Address: "130.192.91.211:443", + Domain: "", + Network: "tcp", + Scheme: "http", + TLSNegotiatedProtocol: "h2", + Trace: nil, + Transport: nil, + } + + req, err := httpNewRequest(ctx, conn, model.DiscardLogger) + if err != nil { + t.Fatal(err) + } + + if req.URL.Scheme != "http" { + t.Fatal("unexpected req.URL.Scheme", req.URL.Scheme) + } + if req.URL.Host != "130.192.91.211:443" { + t.Fatal("unexpected req.URL.Host", req.URL.Host) + } + if req.URL.Path != "/" { + t.Fatal("unexpected req.URL.Path", req.URL.Path) + } + if req.Method != "GET" { + t.Fatal("unexpected req.Method", req.Method) + } + if req.Host != "130.192.91.211:443" { + t.Fatal("unexpected req.Host", req.Host) + } + headers := http.Header{ + "Host": {"130.192.91.211:443"}, + } + if diff := cmp.Diff(headers, req.Header); diff != "" { + t.Fatal(diff) + } + }) + + t.Run("with all options", func(t *testing.T) { + ctx := context.Background() + conn := &HTTPConnection{ + Address: "130.192.91.211:443", + Domain: "example.com", + Network: "tcp", + Scheme: "https", + TLSNegotiatedProtocol: "h2", + Trace: nil, + Transport: nil, + } + + options := []HTTPRequestOption{ HTTPRequestOptionAccept("text/html"), HTTPRequestOptionAcceptLanguage("de"), - HTTPRequestOptionHost("host"), + HTTPRequestOptionHost("www.x.org"), HTTPRequestOptionMethod("PUT"), HTTPRequestOptionReferer("https://example.com/"), HTTPRequestOptionURLPath("/path/to/example"), HTTPRequestOptionUserAgent("Mozilla/5.0 Gecko/geckotrail Firefox/firefoxversion"), - ) - var requestFunc *httpRequestFunc - var ok bool - if requestFunc, ok = f.(*httpRequestFunc); !ok { - t.Fatal("unexpected type. Expected: tlsHandshakeFunc") } - if requestFunc.Accept != "text/html" { - t.Fatalf("unexpected %s, expected %s, got %s", "Accept", "text/html", requestFunc.Accept) + + req, err := httpNewRequest(ctx, conn, model.DiscardLogger, options...) + if err != nil { + t.Fatal(err) + } + + if req.URL.Scheme != "https" { + t.Fatal("unexpected req.URL.Scheme", req.URL.Scheme) } - if requestFunc.AcceptLanguage != "de" { - t.Fatalf("unexpected %s, expected %s, got %s", "AcceptLanguage", "de", requestFunc.AcceptLanguage) + if req.URL.Host != "www.x.org" { + t.Fatal("unexpected req.URL.Host", req.URL.Host) } - if requestFunc.Host != "host" { - t.Fatalf("unexpected %s, expected %s, got %s", "Host", "host", requestFunc.Host) + if req.URL.Path != "/path/to/example" { + t.Fatal("unexpected req.URL.Path", req.URL.Path) } - if requestFunc.Method != "PUT" { - t.Fatalf("unexpected %s, expected %s, got %s", "Method", "PUT", requestFunc.Method) + if req.Method != "PUT" { + t.Fatal("unexpected req.Method", req.Method) } - if requestFunc.Referer != "https://example.com/" { - t.Fatalf("unexpected %s, expected %s, got %s", "Referer", "https://example.com/", requestFunc.Referer) + if req.Host != "www.x.org" { + t.Fatal("unexpected req.Host", req.Host) } - if requestFunc.URLPath != "/path/to/example" { - t.Fatalf("unexpected %s, expected %s, got %s", "URLPath", "example/to/path", requestFunc.URLPath) + headers := http.Header{ + "Accept": {"text/html"}, + "Accept-Language": {"de"}, + "Host": {"www.x.org"}, + "Referer": {"https://example.com/"}, + "User-Agent": {"Mozilla/5.0 Gecko/geckotrail Firefox/firefoxversion"}, } - if requestFunc.UserAgent != "Mozilla/5.0 Gecko/geckotrail Firefox/firefoxversion" { - t.Fatalf("unexpected %s, expected %s, got %s", "UserAgent", "Mozilla/5.0 Gecko/geckotrail Firefox/firefoxversion", requestFunc.UserAgent) + if diff := cmp.Diff(headers, req.Header); diff != "" { + t.Fatal(diff) } }) +} +/* +Test cases: +- Apply httpRequestFunc: + - with EOF + - with invalid method + - with port-less address + - with success (https) + - with success (http) + - with header options +*/ +func TestHTTPRequest(t *testing.T) { t.Run("Apply httpRequestFunc", func(t *testing.T) { mockResponse := &http.Response{ Status: "expected", @@ -95,62 +231,51 @@ func TestHTTPRequest(t *testing.T) { trace := measurexlite.NewTrace(idGen.Add(1), zeroTime, "antani") t.Run("with EOF", func(t *testing.T) { - httpTransport := HTTPTransport{ - Address: "1.2.3.4:567", - IDGenerator: idGen, - Logger: model.DiscardLogger, - Network: "tcp", - Scheme: "https", - Trace: trace, - Transport: eofTransport, - ZeroTime: zeroTime, + httpTransport := HTTPConnection{ + Address: "1.2.3.4:567", + Network: "tcp", + Scheme: "https", + Trace: trace, + Transport: eofTransport, } - httpRequest := &httpRequestFunc{} - res := httpRequest.Apply(context.Background(), &httpTransport) + httpRequest := HTTPRequest( + NewMinimalRuntime(model.DiscardLogger, time.Now()), + ) + res := httpRequest.Apply(context.Background(), NewMaybeWithValue(&httpTransport)) if res.Error != io.EOF { t.Fatal("not the error we expected") } - if res.State.HTTPResponse != nil { - t.Fatal("expected nil request here") - } }) - t.Run("with invalid method", func(t *testing.T) { - httpTransport := HTTPTransport{ - Address: "1.2.3.4:567", - IDGenerator: idGen, - Logger: model.DiscardLogger, - Network: "tcp", - Scheme: "https", - Trace: trace, - Transport: goodTransport, - ZeroTime: zeroTime, - } - httpRequest := &httpRequestFunc{ - Method: "€", - } - res := httpRequest.Apply(context.Background(), &httpTransport) - if res.Error == nil || !strings.HasPrefix(res.Error.Error(), "net/http: invalid method") { - t.Fatal("not the error we expected") + t.Run("with invalid domain", func(t *testing.T) { + httpTransport := HTTPConnection{ + Address: "1.2.3.4:567", + Domain: "\t", // invalid domain + Network: "tcp", + Scheme: "https", + Trace: trace, + Transport: goodTransport, } - if res.State.HTTPResponse != nil { - t.Fatal("expected nil request here") + rt := NewMinimalRuntime(model.DiscardLogger, time.Now()) + httpRequest := HTTPRequest(rt) + res := httpRequest.Apply(context.Background(), NewMaybeWithValue(&httpTransport)) + if res.Error == nil || !strings.HasPrefix(res.Error.Error(), `parse "https://%09/": invalid URL escape "%09"`) { + t.Fatal("not the error we expected", res.Error) } }) t.Run("with port-less address", func(t *testing.T) { - httpTransport := HTTPTransport{ - Address: "1.2.3.4", - IDGenerator: idGen, - Logger: model.DiscardLogger, - Network: "tcp", - Scheme: "https", - Trace: trace, - Transport: goodTransport, - ZeroTime: zeroTime, + httpTransport := HTTPConnection{ + Address: "1.2.3.4", + Network: "tcp", + Scheme: "https", + Trace: trace, + Transport: goodTransport, } - httpRequest := &httpRequestFunc{} - res := httpRequest.Apply(context.Background(), &httpTransport) + httpRequest := HTTPRequest( + NewMinimalRuntime(model.DiscardLogger, time.Now()), + ) + res := httpRequest.Apply(context.Background(), NewMaybeWithValue(&httpTransport)) if res.Error != nil { t.Fatal("expected error") } @@ -164,27 +289,20 @@ func TestHTTPRequest(t *testing.T) { // makeSureObservationsContainTags ensures the observations you can extract from // the given HTTPResponse contain the tags we configured when testing - makeSureObservationsContainTags := func(res *Maybe[*HTTPResponse]) error { - // exclude the case where there was an error - if res.Error != nil { - return fmt.Errorf("unexpected error: %w", res.Error) - } - - // obtain the observations - for _, obs := range ExtractObservations(res) { + makeSureObservationsContainTags := func(rt Runtime) error { + obs := rt.Observations() - // check the network events - for _, ev := range obs.NetworkEvents { - if diff := cmp.Diff([]string{"antani"}, ev.Tags); diff != "" { - return errors.New(diff) - } + // check the network events + for _, ev := range obs.NetworkEvents { + if diff := cmp.Diff([]string{"antani"}, ev.Tags); diff != "" { + return errors.New(diff) } + } - // check the HTTP events - for _, ev := range obs.Requests { - if diff := cmp.Diff([]string{"antani"}, ev.Tags); diff != "" { - return errors.New(diff) - } + // check the HTTP events + for _, ev := range obs.Requests { + if diff := cmp.Diff([]string{"antani"}, ev.Tags); diff != "" { + return errors.New(diff) } } @@ -192,70 +310,64 @@ func TestHTTPRequest(t *testing.T) { } t.Run("with success (https)", func(t *testing.T) { - httpTransport := HTTPTransport{ - Address: "1.2.3.4:443", - IDGenerator: idGen, - Logger: model.DiscardLogger, - Network: "tcp", - Scheme: "https", - Trace: trace, - Transport: goodTransport, - ZeroTime: zeroTime, + httpTransport := HTTPConnection{ + Address: "1.2.3.4:443", + Network: "tcp", + Scheme: "https", + Trace: trace, + Transport: goodTransport, } - httpRequest := &httpRequestFunc{} - res := httpRequest.Apply(context.Background(), &httpTransport) + rt := NewRuntimeMeasurexLite(model.DiscardLogger, time.Now()) + httpRequest := HTTPRequest(rt) + res := httpRequest.Apply(context.Background(), NewMaybeWithValue(&httpTransport)) if res.Error != nil { t.Fatal("unexpected error") } if res.State.HTTPResponse == nil || res.State.HTTPResponse.Status != "expected" { t.Fatal("unexpected request") } - makeSureObservationsContainTags(res) + makeSureObservationsContainTags(rt) }) t.Run("with success (http)", func(t *testing.T) { - httpTransport := HTTPTransport{ - Address: "1.2.3.4:80", - IDGenerator: idGen, - Logger: model.DiscardLogger, - Network: "tcp", - Scheme: "http", - Trace: trace, - Transport: goodTransport, - ZeroTime: zeroTime, + httpTransport := HTTPConnection{ + Address: "1.2.3.4:80", + Network: "tcp", + Scheme: "http", + Trace: trace, + Transport: goodTransport, } - httpRequest := &httpRequestFunc{} - res := httpRequest.Apply(context.Background(), &httpTransport) + rt := NewRuntimeMeasurexLite(model.DiscardLogger, time.Now()) + httpRequest := HTTPRequest(rt) + res := httpRequest.Apply(context.Background(), NewMaybeWithValue(&httpTransport)) if res.Error != nil { t.Fatal("unexpected error") } if res.State.HTTPResponse == nil || res.State.HTTPResponse.Status != "expected" { t.Fatal("unexpected request") } - makeSureObservationsContainTags(res) + makeSureObservationsContainTags(rt) }) t.Run("with header options", func(t *testing.T) { - httpTransport := HTTPTransport{ - Address: "1.2.3.4:567", - Domain: "domain.com", - IDGenerator: idGen, - Logger: model.DiscardLogger, - Network: "tcp", - Scheme: "https", - Trace: trace, - Transport: goodTransport, - ZeroTime: zeroTime, - } - httpRequest := &httpRequestFunc{ - Accept: "text/html", - AcceptLanguage: "de", - Host: "host", - Referer: "https://example.org", - URLPath: "/path/to/example", - UserAgent: "Mozilla/5.0 Gecko/geckotrail Firefox/firefoxversion", + httpTransport := HTTPConnection{ + Address: "1.2.3.4:567", + Domain: "domain.com", + Network: "tcp", + Scheme: "https", + Trace: trace, + Transport: goodTransport, } - res := httpRequest.Apply(context.Background(), &httpTransport) + rt := NewMinimalRuntime(model.DiscardLogger, time.Now()) + httpRequest := HTTPRequest(rt, + HTTPRequestOptionAccept("text/html"), + HTTPRequestOptionAcceptLanguage("de"), + HTTPRequestOptionHost("host"), + HTTPRequestOptionReferer("https://example.org"), + HTTPRequestOptionURLPath("/path/to/example"), + HTTPRequestOptionUserAgent("Mozilla/5.0 Gecko/geckotrail Firefox/firefoxversion"), + ) + res := httpRequest.Apply(context.Background(), NewMaybeWithValue(&httpTransport)) if res.Error != nil { t.Fatal("unexpected error") } @@ -278,21 +390,14 @@ func TestHTTPRequest(t *testing.T) { /* Test cases: -- Get httpTransportTCPFunc - Get composed function: TCP with HTTP - Apply httpTransportTCPFunc */ func TestHTTPTCP(t *testing.T) { - t.Run("Get httpTransportTCPFunc", func(t *testing.T) { - f := HTTPTransportTCP() - if _, ok := f.(*httpTransportTCPFunc); !ok { - t.Fatal("unexpected type") - } - }) - t.Run("Get composed function: TCP with HTTP", func(t *testing.T) { - f := HTTPRequestOverTCP() - if _, ok := f.(*compose2Func[*TCPConnection, *HTTPTransport, *HTTPResponse]); !ok { + rt := NewMinimalRuntime(model.DiscardLogger, time.Now()) + f := HTTPRequestOverTCP(rt) + if _, ok := f.(*compose2Func[*TCPConnection, *HTTPConnection, *HTTPResponse]); !ok { t.Fatal("unexpected type") } }) @@ -304,16 +409,15 @@ func TestHTTPTCP(t *testing.T) { trace := measurexlite.NewTrace(idGen.Add(1), zeroTime) address := "1.2.3.4:567" tcpConn := &TCPConnection{ - Address: address, - Conn: conn, - IDGenerator: idGen, - Logger: model.DiscardLogger, - Network: "tcp", - Trace: trace, - ZeroTime: zeroTime, - } - f := httpTransportTCPFunc{} - res := f.Apply(context.Background(), tcpConn) + Address: address, + Conn: conn, + Network: "tcp", + Trace: trace, + } + f := HTTPConnectionTCP( + NewMinimalRuntime(model.DiscardLogger, time.Now()), + ) + res := f.Apply(context.Background(), NewMaybeWithValue(tcpConn)) if res.Error != nil { t.Fatalf("unexpected error: %s", res.Error) } @@ -331,21 +435,14 @@ func TestHTTPTCP(t *testing.T) { /* Test cases: -- Get httpTransportQUICFunc - Get composed function: QUIC with HTTP - Apply httpTransportQUICFunc */ func TestHTTPQUIC(t *testing.T) { - t.Run("Get httpTransportQUICFunc", func(t *testing.T) { - f := HTTPTransportQUIC() - if _, ok := f.(*httpTransportQUICFunc); !ok { - t.Fatal("unexpected type") - } - }) - t.Run("Get composed function: QUIC with HTTP", func(t *testing.T) { - f := HTTPRequestOverQUIC() - if _, ok := f.(*compose2Func[*QUICConnection, *HTTPTransport, *HTTPResponse]); !ok { + rt := NewMinimalRuntime(model.DiscardLogger, time.Now()) + f := HTTPRequestOverQUIC(rt) + if _, ok := f.(*compose2Func[*QUICConnection, *HTTPConnection, *HTTPResponse]); !ok { t.Fatal("unexpected type") } }) @@ -357,16 +454,15 @@ func TestHTTPQUIC(t *testing.T) { trace := measurexlite.NewTrace(idGen.Add(1), zeroTime) address := "1.2.3.4:567" quicConn := &QUICConnection{ - Address: address, - QUICConn: conn, - IDGenerator: idGen, - Logger: model.DiscardLogger, - Network: "udp", - Trace: trace, - ZeroTime: zeroTime, - } - f := httpTransportQUICFunc{} - res := f.Apply(context.Background(), quicConn) + Address: address, + QUICConn: conn, + Network: "udp", + Trace: trace, + } + f := HTTPConnectionQUIC( + NewMinimalRuntime(model.DiscardLogger, time.Now()), + ) + res := f.Apply(context.Background(), NewMaybeWithValue(quicConn)) if res.Error != nil { t.Fatalf("unexpected error: %s", res.Error) } @@ -384,21 +480,14 @@ func TestHTTPQUIC(t *testing.T) { /* Test cases: -- Get httpTransportTLSFunc - Get composed function: TLS with HTTP - Apply httpTransportTLSFunc */ func TestHTTPTLS(t *testing.T) { - t.Run("Get httpTransportTLSFunc", func(t *testing.T) { - f := HTTPTransportTLS() - if _, ok := f.(*httpTransportTLSFunc); !ok { - t.Fatal("unexpected type") - } - }) - t.Run("Get composed function: TLS with HTTP", func(t *testing.T) { - f := HTTPRequestOverTLS() - if _, ok := f.(*compose2Func[*TLSConnection, *HTTPTransport, *HTTPResponse]); !ok { + rt := NewMinimalRuntime(model.DiscardLogger, time.Now()) + f := HTTPRequestOverTLS(rt) + if _, ok := f.(*compose2Func[*TLSConnection, *HTTPConnection, *HTTPResponse]); !ok { t.Fatal("unexpected type") } }) @@ -410,16 +499,15 @@ func TestHTTPTLS(t *testing.T) { trace := measurexlite.NewTrace(idGen.Add(1), zeroTime) address := "1.2.3.4:567" tlsConn := &TLSConnection{ - Address: address, - Conn: conn, - IDGenerator: idGen, - Logger: model.DiscardLogger, - Network: "tcp", - Trace: trace, - ZeroTime: zeroTime, - } - f := httpTransportTLSFunc{} - res := f.Apply(context.Background(), tlsConn) + Address: address, + Conn: conn, + Network: "tcp", + Trace: trace, + } + f := HTTPConnectionTLS( + NewMinimalRuntime(model.DiscardLogger, time.Now()), + ) + res := f.Apply(context.Background(), NewMaybeWithValue(tlsConn)) if res.Error != nil { t.Fatalf("unexpected error: %s", res.Error) } diff --git a/pkg/dslx/httpcore.go b/pkg/dslx/httpcore.go index 0bba3530..09810530 100644 --- a/pkg/dslx/httpcore.go +++ b/pkg/dslx/httpcore.go @@ -10,7 +10,6 @@ import ( "net" "net/http" "net/url" - "sync/atomic" "time" "github.com/ooni/probe-engine/pkg/logx" @@ -20,24 +19,18 @@ import ( "github.com/ooni/probe-engine/pkg/throttling" ) -// HTTPTransport is an HTTP transport bound to a TCP, TLS or QUIC connection +// HTTPConnection is an HTTP connection bound to a TCP, TLS or QUIC connection // that would use such a connection only and for any input URL. You generally // use [HTTPTransportTCP], [HTTPTransportTLS] or [HTTPTransportQUIC] to // create a new instance; if you want to initialize manually, make sure you // init the fields marked as MANDATORY. -type HTTPTransport struct { +type HTTPConnection struct { // Address is the MANDATORY address we're connected to. Address string // Domain is the OPTIONAL domain from which the address was resolved. Domain string - // IDGenerator is the MANDATORY ID generator. - IDGenerator *atomic.Int64 - - // Logger is the MANDATORY logger to use. - Logger model.Logger - // Network is the MANDATORY network used by the underlying conn. Network string @@ -48,221 +41,168 @@ type HTTPTransport struct { TLSNegotiatedProtocol string // Trace is the MANDATORY trace we're using. - Trace *measurexlite.Trace + Trace Trace // Transport is the MANDATORY HTTP transport we're using. Transport model.HTTPTransport - - // ZeroTime is the MANDATORY zero time of the measurement. - ZeroTime time.Time } // HTTPRequestOption is an option you can pass to HTTPRequest. -type HTTPRequestOption func(*httpRequestFunc) +type HTTPRequestOption func(req *http.Request) // HTTPRequestOptionAccept sets the Accept header. func HTTPRequestOptionAccept(value string) HTTPRequestOption { - return func(hrf *httpRequestFunc) { - hrf.Accept = value + return func(req *http.Request) { + req.Header.Set("Accept", value) } } // HTTPRequestOptionAcceptLanguage sets the Accept header. func HTTPRequestOptionAcceptLanguage(value string) HTTPRequestOption { - return func(hrf *httpRequestFunc) { - hrf.AcceptLanguage = value + return func(req *http.Request) { + req.Header.Set("Accept-Language", value) } } // HTTPRequestOptionHost sets the Host header. func HTTPRequestOptionHost(value string) HTTPRequestOption { - return func(hrf *httpRequestFunc) { - hrf.Host = value + return func(req *http.Request) { + req.URL.Host = value + req.Host = value } } // HTTPRequestOptionHost sets the request method. func HTTPRequestOptionMethod(value string) HTTPRequestOption { - return func(hrf *httpRequestFunc) { - hrf.Method = value + return func(req *http.Request) { + req.Method = value } } // HTTPRequestOptionReferer sets the Referer header. func HTTPRequestOptionReferer(value string) HTTPRequestOption { - return func(hrf *httpRequestFunc) { - hrf.Referer = value + return func(req *http.Request) { + req.Header.Set("Referer", value) } } // HTTPRequestOptionURLPath sets the URL path. func HTTPRequestOptionURLPath(value string) HTTPRequestOption { - return func(hrf *httpRequestFunc) { - hrf.URLPath = value + return func(req *http.Request) { + req.URL.Path = value } } // HTTPRequestOptionUserAgent sets the UserAgent header. func HTTPRequestOptionUserAgent(value string) HTTPRequestOption { - return func(hrf *httpRequestFunc) { - hrf.UserAgent = value + return func(req *http.Request) { + req.Header.Set("User-Agent", value) } } // HTTPRequest issues an HTTP request using a transport and returns a response. -func HTTPRequest(options ...HTTPRequestOption) Func[*HTTPTransport, *Maybe[*HTTPResponse]] { - f := &httpRequestFunc{} - for _, option := range options { - option(f) - } - return f -} - -// httpRequestFunc is the Func returned by HTTPRequest. -type httpRequestFunc struct { - // Accept is the OPTIONAL accept header. - Accept string - - // AcceptLanguage is the OPTIONAL accept-language header. - AcceptLanguage string - - // Host is the OPTIONAL host header. - Host string - - // Method is the OPTIONAL method. - Method string - - // Referer is the OPTIONAL referer header. - Referer string - - // URLPath is the OPTIONAL URL path. - URLPath string - - // UserAgent is the OPTIONAL user-agent header. - UserAgent string -} - -// Apply implements Func. -func (f *httpRequestFunc) Apply( - ctx context.Context, input *HTTPTransport) *Maybe[*HTTPResponse] { - // setup - const timeout = 10 * time.Second - ctx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - - var ( - body []byte - observations []*Observations - resp *http.Response - ) - - // create HTTP request - req, err := f.newHTTPRequest(ctx, input) - if err == nil { - - // start the operation logger - ol := logx.NewOperationLogger( - input.Logger, - "[#%d] HTTPRequest %s with %s/%s host=%s", - input.Trace.Index, - req.URL.String(), - input.Address, - input.Network, - req.Host, +func HTTPRequest(rt Runtime, options ...HTTPRequestOption) Func[*HTTPConnection, *HTTPResponse] { + return Operation[*HTTPConnection, *HTTPResponse](func(ctx context.Context, input *HTTPConnection) (*HTTPResponse, error) { + // setup + const timeout = 10 * time.Second + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + var ( + body []byte + observations []*Observations + resp *http.Response ) - // perform HTTP transaction and collect the related observations - resp, body, observations, err = f.do(ctx, input, req) - - // stop the operation logger - ol.Stop(err) - } - - observations = append(observations, maybeTraceToObservations(input.Trace)...) - - state := &HTTPResponse{ - Address: input.Address, - Domain: input.Domain, - HTTPRequest: req, // possibly nil - HTTPResponse: resp, // possibly nil - HTTPResponseBodySnapshot: body, // possibly nil - IDGenerator: input.IDGenerator, - Logger: input.Logger, - Network: input.Network, - Trace: input.Trace, - ZeroTime: input.ZeroTime, - } - - return &Maybe[*HTTPResponse]{ - Error: err, - Observations: observations, - Operation: netxlite.HTTPRoundTripOperation, - State: state, - } + // create HTTP request + req, err := httpNewRequest(ctx, input, rt.Logger(), options...) + if err == nil { + + // start the operation logger + ol := logx.NewOperationLogger( + rt.Logger(), + "[#%d] HTTPRequest %s with %s/%s host=%s", + input.Trace.Index(), + req.URL.String(), + input.Address, + input.Network, + req.Host, + ) + + // perform HTTP transaction and collect the related observations + resp, body, observations, err = httpRoundTrip(ctx, input, req) + + // stop the operation logger + ol.Stop(err) + } + + // merge and save observations + observations = append(observations, maybeTraceToObservations(input.Trace)...) + rt.SaveObservations(observations...) + + // handle error case + if err != nil { + return nil, err + } + + // handle success + state := &HTTPResponse{ + Address: input.Address, + Domain: input.Domain, + HTTPRequest: req, + HTTPResponse: resp, + HTTPResponseBodySnapshot: body, + Network: input.Network, + Trace: input.Trace, + } + return state, nil + }) } -func (f *httpRequestFunc) newHTTPRequest( - ctx context.Context, input *HTTPTransport) (*http.Request, error) { +// httpNewRequest is a convenience function for creating a new request. +func httpNewRequest( + ctx context.Context, input *HTTPConnection, logger model.Logger, options ...HTTPRequestOption) (*http.Request, error) { + // create the default HTTP request URL := &url.URL{ Scheme: input.Scheme, Opaque: "", User: nil, - Host: f.urlHost(input), - Path: f.urlPath(), + Host: httpNewURLHost(input, logger), + Path: "/", RawPath: "", ForceQuery: false, RawQuery: "", Fragment: "", RawFragment: "", } - - method := "GET" - if f.Method != "" { - method = f.Method - } - - req, err := http.NewRequestWithContext(ctx, method, URL.String(), nil) + req, err := http.NewRequestWithContext(ctx, "GET", URL.String(), nil) if err != nil { return nil, err } - if v := f.Host; v != "" { - req.Host = v - } else { - // Go would use URL.Host as "Host" header anyways in case we leave req.Host empty. - // We already set it here so that we can use req.Host for logging. - req.Host = URL.Host + // Go would use URL.Host as "Host" header anyways in case we leave req.Host empty. + // We already set it here so that we can use req.Host for logging. + req.Host = URL.Host + + // apply the user-specified options + for _, option := range options { + option(req) } + // req.Header["Host"] is ignored by Go but we want to have it in the measurement // to reflect what we think has been sent as HTTP headers. req.Header.Set("Host", req.Host) - - if v := f.Accept; v != "" { - req.Header.Set("Accept", v) - } - - if v := f.AcceptLanguage; v != "" { - req.Header.Set("Accept-Language", v) - } - - if v := f.Referer; v != "" { - req.Header.Set("Referer", v) - } - - if v := f.UserAgent; v != "" { // not setting means using Go's default - req.Header.Set("User-Agent", v) - } - return req, nil } -func (f *httpRequestFunc) urlHost(input *HTTPTransport) string { +// httpNewURLHost computes the URL host to use. +func httpNewURLHost(input *HTTPConnection, logger model.Logger) string { if input.Domain != "" { return input.Domain } addr, port, err := net.SplitHostPort(input.Address) if err != nil { - input.Logger.Warnf("httpRequestFunc: cannot SplitHostPort for input.Address") + logger.Warnf("httpRequestFunc: cannot SplitHostPort for input.Address") return input.Address } switch { @@ -275,20 +215,14 @@ func (f *httpRequestFunc) urlHost(input *HTTPTransport) string { } } -func (f *httpRequestFunc) urlPath() string { - if f.URLPath != "" { - return f.URLPath - } - return "/" -} - -func (f *httpRequestFunc) do( +// httpRoundTrip performs the actual HTTP round trip +func httpRoundTrip( ctx context.Context, - input *HTTPTransport, + input *HTTPConnection, req *http.Request, ) (*http.Response, []byte, []*Observations, error) { - const maxbody = 1 << 19 // TODO(bassosimone): allow to configure this value? - started := input.Trace.TimeSince(input.Trace.ZeroTime) + const maxbody = 1 << 19 // TODO(https://github.com/ooni/probe/issues/2621): allow to configure this value + started := input.Trace.TimeSince(input.Trace.ZeroTime()) // manually create a single 1-length observations structure because // the trace cannot automatically capture HTTP events @@ -298,7 +232,7 @@ func (f *httpRequestFunc) do( observations[0].NetworkEvents = append(observations[0].NetworkEvents, measurexlite.NewAnnotationArchivalNetworkEvent( - input.Trace.Index, + input.Trace.Index(), started, "http_transaction_start", input.Trace.Tags()..., @@ -315,17 +249,17 @@ func (f *httpRequestFunc) do( // read a snapshot of the response body reader := io.LimitReader(resp.Body, maxbody) - body, err = netxlite.ReadAllContext(ctx, reader) // TODO: enable streaming and measure speed + body, err = netxlite.ReadAllContext(ctx, reader) // TODO(https://github.com/ooni/probe/issues/2622) // collect and save download speed samples samples := sampler.ExtractSamples() observations[0].NetworkEvents = append(observations[0].NetworkEvents, samples...) } - finished := input.Trace.TimeSince(input.Trace.ZeroTime) + finished := input.Trace.TimeSince(input.Trace.ZeroTime()) observations[0].NetworkEvents = append(observations[0].NetworkEvents, measurexlite.NewAnnotationArchivalNetworkEvent( - input.Trace.Index, + input.Trace.Index(), finished, "http_transaction_done", input.Trace.Tags()..., @@ -333,7 +267,7 @@ func (f *httpRequestFunc) do( observations[0].Requests = append(observations[0].Requests, measurexlite.NewArchivalHTTPRequestResult( - input.Trace.Index, + input.Trace.Index(), started, input.Network, input.Address, @@ -369,19 +303,10 @@ type HTTPResponse struct { // HTTPResponseBodySnapshot is the response body or nil if Err != nil. HTTPResponseBodySnapshot []byte - // IDGenerator is the MANDATORY ID generator. - IDGenerator *atomic.Int64 - - // Logger is the MANDATORY logger to use. - Logger model.Logger - // Network is the MANDATORY network we're connected to. Network string // Trace is the MANDATORY trace we're using. The trace is drained // when you call the Observations method. - Trace *measurexlite.Trace - - // ZeroTime is the MANDATORY zero time of the measurement. - ZeroTime time.Time + Trace Trace } diff --git a/pkg/dslx/httpquic.go b/pkg/dslx/httpquic.go index c26e386b..c41d9684 100644 --- a/pkg/dslx/httpquic.go +++ b/pkg/dslx/httpquic.go @@ -11,44 +11,29 @@ import ( ) // HTTPRequestOverQUIC returns a Func that issues HTTP requests over QUIC. -func HTTPRequestOverQUIC(options ...HTTPRequestOption) Func[*QUICConnection, *Maybe[*HTTPResponse]] { - return Compose2(HTTPTransportQUIC(), HTTPRequest(options...)) +func HTTPRequestOverQUIC(rt Runtime, options ...HTTPRequestOption) Func[*QUICConnection, *HTTPResponse] { + return Compose2(HTTPConnectionQUIC(rt), HTTPRequest(rt, options...)) } -// HTTPTransportQUIC converts a QUIC connection into an HTTP transport. -func HTTPTransportQUIC() Func[*QUICConnection, *Maybe[*HTTPTransport]] { - return &httpTransportQUICFunc{} -} - -// httpTransportQUICFunc is the function returned by HTTPTransportQUIC. -type httpTransportQUICFunc struct{} - -// Apply implements Func. -func (f *httpTransportQUICFunc) Apply( - ctx context.Context, input *QUICConnection) *Maybe[*HTTPTransport] { - // create transport - httpTransport := netxlite.NewHTTP3Transport( - input.Logger, - netxlite.NewSingleUseQUICDialer(input.QUICConn), - input.TLSConfig, - ) - - state := &HTTPTransport{ - Address: input.Address, - Domain: input.Domain, - IDGenerator: input.IDGenerator, - Logger: input.Logger, - Network: input.Network, - Scheme: "https", - TLSNegotiatedProtocol: input.TLSState.NegotiatedProtocol, - Trace: input.Trace, - Transport: httpTransport, - ZeroTime: input.ZeroTime, - } - return &Maybe[*HTTPTransport]{ - Error: nil, - Observations: nil, - Operation: "", // we cannot fail, so no need to store operation name - State: state, - } +// HTTPConnectionQUIC converts a QUIC connection into an HTTP connection. +func HTTPConnectionQUIC(rt Runtime) Func[*QUICConnection, *HTTPConnection] { + return Operation[*QUICConnection, *HTTPConnection](func(ctx context.Context, input *QUICConnection) (*HTTPConnection, error) { + httpTransport := netxlite.NewHTTP3Transport( + rt.Logger(), + netxlite.NewSingleUseQUICDialer(input.QUICConn), + input.TLSConfig, + ) + + state := &HTTPConnection{ + Address: input.Address, + Domain: input.Domain, + Network: input.Network, + Scheme: "https", + TLSNegotiatedProtocol: input.TLSState.NegotiatedProtocol, + Trace: input.Trace, + Transport: httpTransport, + } + + return state, nil + }) } diff --git a/pkg/dslx/httptcp.go b/pkg/dslx/httptcp.go index 83b15367..198def6c 100644 --- a/pkg/dslx/httptcp.go +++ b/pkg/dslx/httptcp.go @@ -11,45 +11,32 @@ import ( ) // HTTPRequestOverTCP returns a Func that issues HTTP requests over TCP. -func HTTPRequestOverTCP(options ...HTTPRequestOption) Func[*TCPConnection, *Maybe[*HTTPResponse]] { - return Compose2(HTTPTransportTCP(), HTTPRequest(options...)) +func HTTPRequestOverTCP(rt Runtime, options ...HTTPRequestOption) Func[*TCPConnection, *HTTPResponse] { + return Compose2(HTTPConnectionTCP(rt), HTTPRequest(rt, options...)) } -// HTTPTransportTCP converts a TCP connection into an HTTP transport. -func HTTPTransportTCP() Func[*TCPConnection, *Maybe[*HTTPTransport]] { - return &httpTransportTCPFunc{} -} +// HTTPConnectionTCP converts a TCP connection into an HTTP connection. +func HTTPConnectionTCP(rt Runtime) Func[*TCPConnection, *HTTPConnection] { + return Operation[*TCPConnection, *HTTPConnection](func(ctx context.Context, input *TCPConnection) (*HTTPConnection, error) { + // TODO(https://github.com/ooni/probe/issues/2534): here we're using the QUIRKY netxlite.NewHTTPTransport + // function, but we can probably avoid using it, given that this code is + // not using tracing and does not care about those quirks. + httpTransport := netxlite.NewHTTPTransport( + rt.Logger(), + netxlite.NewSingleUseDialer(input.Conn), + netxlite.NewNullTLSDialer(), + ) -// httpTransportTCPFunc is the function returned by HTTPTransportTCP -type httpTransportTCPFunc struct{} + state := &HTTPConnection{ + Address: input.Address, + Domain: input.Domain, + Network: input.Network, + Scheme: "http", + TLSNegotiatedProtocol: "", + Trace: input.Trace, + Transport: httpTransport, + } -// Apply implements Func -func (f *httpTransportTCPFunc) Apply( - ctx context.Context, input *TCPConnection) *Maybe[*HTTPTransport] { - // TODO(https://github.com/ooni/probe/issues/2534): here we're using the QUIRKY netxlite.NewHTTPTransport - // function, but we can probably avoid using it, given that this code is - // not using tracing and does not care about those quirks. - httpTransport := netxlite.NewHTTPTransport( - input.Logger, - netxlite.NewSingleUseDialer(input.Conn), - netxlite.NewNullTLSDialer(), - ) - state := &HTTPTransport{ - Address: input.Address, - Domain: input.Domain, - IDGenerator: input.IDGenerator, - Logger: input.Logger, - Network: input.Network, - Scheme: "http", - TLSNegotiatedProtocol: "", - Trace: input.Trace, - Transport: httpTransport, - ZeroTime: input.ZeroTime, - } - return &Maybe[*HTTPTransport]{ - Error: nil, - Observations: nil, - Operation: "", // we cannot fail, so no need to store operation name - State: state, - } + return state, nil + }) } diff --git a/pkg/dslx/httptls.go b/pkg/dslx/httptls.go index 47df01dd..33ffa1f2 100644 --- a/pkg/dslx/httptls.go +++ b/pkg/dslx/httptls.go @@ -11,45 +11,32 @@ import ( ) // HTTPRequestOverTLS returns a Func that issues HTTP requests over TLS. -func HTTPRequestOverTLS(options ...HTTPRequestOption) Func[*TLSConnection, *Maybe[*HTTPResponse]] { - return Compose2(HTTPTransportTLS(), HTTPRequest(options...)) +func HTTPRequestOverTLS(rt Runtime, options ...HTTPRequestOption) Func[*TLSConnection, *HTTPResponse] { + return Compose2(HTTPConnectionTLS(rt), HTTPRequest(rt, options...)) } -// HTTPTransportTLS converts a TLS connection into an HTTP transport. -func HTTPTransportTLS() Func[*TLSConnection, *Maybe[*HTTPTransport]] { - return &httpTransportTLSFunc{} -} +// HTTPConnectionTLS converts a TLS connection into an HTTP connection. +func HTTPConnectionTLS(rt Runtime) Func[*TLSConnection, *HTTPConnection] { + return Operation[*TLSConnection, *HTTPConnection](func(ctx context.Context, input *TLSConnection) (*HTTPConnection, error) { + // TODO(https://github.com/ooni/probe/issues/2534): here we're using the QUIRKY netxlite.NewHTTPTransport + // function, but we can probably avoid using it, given that this code is + // not using tracing and does not care about those quirks. + httpTransport := netxlite.NewHTTPTransport( + rt.Logger(), + netxlite.NewNullDialer(), + netxlite.NewSingleUseTLSDialer(input.Conn), + ) -// httpTransportTLSFunc is the function returned by HTTPTransportTLS. -type httpTransportTLSFunc struct{} + state := &HTTPConnection{ + Address: input.Address, + Domain: input.Domain, + Network: input.Network, + Scheme: "https", + TLSNegotiatedProtocol: input.TLSState.NegotiatedProtocol, + Trace: input.Trace, + Transport: httpTransport, + } -// Apply implements Func. -func (f *httpTransportTLSFunc) Apply( - ctx context.Context, input *TLSConnection) *Maybe[*HTTPTransport] { - // TODO(https://github.com/ooni/probe/issues/2534): here we're using the QUIRKY netxlite.NewHTTPTransport - // function, but we can probably avoid using it, given that this code is - // not using tracing and does not care about those quirks. - httpTransport := netxlite.NewHTTPTransport( - input.Logger, - netxlite.NewNullDialer(), - netxlite.NewSingleUseTLSDialer(input.Conn), - ) - state := &HTTPTransport{ - Address: input.Address, - Domain: input.Domain, - IDGenerator: input.IDGenerator, - Logger: input.Logger, - Network: input.Network, - Scheme: "https", - TLSNegotiatedProtocol: input.TLSState.NegotiatedProtocol, - Trace: input.Trace, - Transport: httpTransport, - ZeroTime: input.ZeroTime, - } - return &Maybe[*HTTPTransport]{ - Error: nil, - Observations: nil, - Operation: "", // we cannot fail, so no need to store operation name - State: state, - } + return state, nil + }) } diff --git a/pkg/dslx/integration_test.go b/pkg/dslx/integration_test.go index 71ff3d02..5cbe1126 100644 --- a/pkg/dslx/integration_test.go +++ b/pkg/dslx/integration_test.go @@ -4,7 +4,6 @@ import ( "context" "net/http" "net/http/httptest" - "sync/atomic" "testing" "time" @@ -31,42 +30,37 @@ func TestMakeSureWeCollectSpeedSamples(t *testing.T) { })) defer server.Close() - // instantiate a connection pool - pool := &ConnPool{} - defer pool.Close() + // instantiate a runtime + rt := NewRuntimeMeasurexLite(model.DiscardLogger, time.Now()) + defer rt.Close() // create a measuring function f0 := Compose3( - TCPConnect(pool), - HTTPTransportTCP(), - HTTPRequest(), + TCPConnect(rt), + HTTPConnectionTCP(rt), + HTTPRequest(rt), ) // create the endpoint to measure epnt := &Endpoint{ - Address: server.Listener.Addr().String(), - Domain: "", - IDGenerator: &atomic.Int64{}, - Logger: model.DiscardLogger, - Network: "tcp", - Tags: []string{}, - ZeroTime: time.Now(), + Address: server.Listener.Addr().String(), + Domain: "", + Network: "tcp", + Tags: []string{}, } // measure the endpoint - result := f0.Apply(context.Background(), epnt) + _ = f0.Apply(context.Background(), NewMaybeWithValue(epnt)) // get observations - observations := ExtractObservations(result) + observations := rt.Observations() // process the network events and check for summary var foundSummary bool - for _, entry := range observations { - for _, ev := range entry.NetworkEvents { - if ev.Operation == throttling.BytesReceivedCumulativeOperation { - t.Log(ev) - foundSummary = true - } + for _, ev := range observations.NetworkEvents { + if ev.Operation == throttling.BytesReceivedCumulativeOperation { + t.Log(ev) + foundSummary = true } } if !foundSummary { diff --git a/pkg/dslx/observations.go b/pkg/dslx/observations.go index ca7fdde2..acfe6409 100644 --- a/pkg/dslx/observations.go +++ b/pkg/dslx/observations.go @@ -5,7 +5,6 @@ package dslx // import ( - "github.com/ooni/probe-engine/pkg/measurexlite" "github.com/ooni/probe-engine/pkg/model" ) @@ -44,17 +43,9 @@ func NewObservations() *Observations { } } -// ExtractObservations extracts observations from a list of [Maybe]. -func ExtractObservations[T any](rs ...*Maybe[T]) (out []*Observations) { - for _, r := range rs { - out = append(out, r.Observations...) - } - return -} - // maybeTraceToObservations returns the observations inside the // trace taking into account the case where trace is nil. -func maybeTraceToObservations(trace *measurexlite.Trace) (out []*Observations) { +func maybeTraceToObservations(trace Trace) (out []*Observations) { if trace != nil { out = append(out, &Observations{ NetworkEvents: trace.NetworkEvents(), diff --git a/pkg/dslx/qa_test.go b/pkg/dslx/qa_test.go new file mode 100644 index 00000000..a895e7e0 --- /dev/null +++ b/pkg/dslx/qa_test.go @@ -0,0 +1,311 @@ +package dslx_test + +import ( + "context" + "net" + "testing" + "time" + + "github.com/apex/log" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/google/gopacket/layers" + "github.com/ooni/netem" + "github.com/ooni/probe-engine/pkg/dslx" + "github.com/ooni/probe-engine/pkg/model" + "github.com/ooni/probe-engine/pkg/netemx" + "github.com/ooni/probe-engine/pkg/netxlite" +) + +// qaStringLessFunc is an utility function to force cmp.Diff to sort string +// slices before performing comparison so that the order doesn't matter +func qaStringLessFunc(a, b string) bool { + return a < b +} + +func TestDNSLookupQA(t *testing.T) { + // testcase is a test case implemented by this function + type testcase struct { + // name is the test case name + name string + + // newRuntime is the function creating a new runtime + newRuntime func(netx model.MeasuringNetwork) dslx.Runtime + + // configureDPI configures DPI + configureDPI func(dpi *netem.DPIEngine) + + // domain is the domain to resolve + domain dslx.DomainName + + // expectErr is the expected DNS error or nil + expectErr error + + // expectAddrs contains the expected DNS addresses + expectAddrs []string + } + + cases := []testcase{{ + name: "success with minimal runtime", + newRuntime: func(netx model.MeasuringNetwork) dslx.Runtime { + return dslx.NewMinimalRuntime(log.Log, time.Now(), dslx.MinimalRuntimeOptionMeasuringNetwork(netx)) + }, + configureDPI: func(dpi *netem.DPIEngine) { + // nothing + }, + domain: "dns.google", + expectErr: nil, + expectAddrs: []string{"8.8.8.8", "8.8.4.4"}, + }, { + name: "with injected nxdomain error and minimal runtime", + newRuntime: func(netx model.MeasuringNetwork) dslx.Runtime { + return dslx.NewMinimalRuntime(log.Log, time.Now(), dslx.MinimalRuntimeOptionMeasuringNetwork(netx)) + }, + configureDPI: func(dpi *netem.DPIEngine) { + dpi.AddRule(&netem.DPISpoofDNSResponse{ + Addresses: []string{}, // empty to cause NXDOMAIN + Logger: log.Log, + Domain: "dns.google", + }) + }, + domain: "dns.google", + expectErr: dslx.ErrDNSLookupParallel, + expectAddrs: []string{}, + }} + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // create an internet testing scenario + env := netemx.MustNewScenario(netemx.InternetScenario) + defer env.Close() + + // create a dslx.Runtime using the client stack + rt := tc.newRuntime(&netxlite.Netx{ + Underlying: &netxlite.NetemUnderlyingNetworkAdapter{UNet: env.ClientStack}, + }) + defer rt.Close() + + // configure the DPI engine + tc.configureDPI(env.DPIEngine()) + + // create DNS lookup function + function := dslx.DNSLookupParallel( + dslx.DNSLookupGetaddrinfo(rt), + dslx.DNSLookupUDP(rt, net.JoinHostPort(netemx.AddressDNSQuad9Net, "53")), + ) + + // create context + ctx := context.Background() + + // perform DNS lookup + results := function.Apply(ctx, dslx.NewMaybeWithValue(dslx.NewDomainToResolve(tc.domain))) + + // unpack the results + resolvedAddrs, err := results.State, results.Error + + // make sure the error matches expectations + switch { + case err == nil && tc.expectErr == nil: + // nothing + + case err != nil && tc.expectErr == nil: + t.Fatal("expected", tc.expectErr, "got", err) + + case err == nil && tc.expectErr != nil: + t.Fatal("expected", tc.expectErr, "got", err) + + case err != nil && tc.expectErr != nil: + if err.Error() != tc.expectErr.Error() { + t.Fatal("expected", tc.expectErr, "got", err) + } + return // no reason to continue + } + + // make sure that the domain has been correctly copied + if resolvedAddrs.Domain != string(tc.domain) { + t.Fatal("expected", tc.domain, "got", resolvedAddrs.Domain) + } + + // make sure we resolved the expected IP addresses + if diff := cmp.Diff(tc.expectAddrs, resolvedAddrs.Addresses, cmpopts.SortSlices(qaStringLessFunc)); diff != "" { + t.Fatal(diff) + } + + // TODO(https://github.com/ooni/probe/issues/2620): make sure the observations are OK + }) + } +} + +func TestMeasureResolvedAddressesQA(t *testing.T) { + // testcase is a test case implemented by this function + type testcase struct { + // name is the test case name + name string + + // newRuntime is the function creating a new runtime + newRuntime func(netx model.MeasuringNetwork) dslx.Runtime + + // configureDPI configures DPI + configureDPI func(dpi *netem.DPIEngine) + + // expectTCP contains the expected TCP connect stats + expectTCP map[string]int64 + + // expectTLS contains the expected TLS handshake stats + expectTLS map[string]int64 + + // expectQUIC contains the expected QUIC handshake stats + expectQUIC map[string]int64 + } + + cases := []testcase{{ + name: "success with minimal runtime", + newRuntime: func(netx model.MeasuringNetwork) dslx.Runtime { + return dslx.NewMinimalRuntime(log.Log, time.Now(), dslx.MinimalRuntimeOptionMeasuringNetwork(netx)) + }, + configureDPI: func(dpi *netem.DPIEngine) { + // nothing + }, + expectTCP: map[string]int64{"": 2}, + expectTLS: map[string]int64{"": 2}, + expectQUIC: map[string]int64{"": 2}, + }, { + name: "TCP connection refused with minimal runtime", + newRuntime: func(netx model.MeasuringNetwork) dslx.Runtime { + return dslx.NewMinimalRuntime(log.Log, time.Now(), dslx.MinimalRuntimeOptionMeasuringNetwork(netx)) + }, + configureDPI: func(dpi *netem.DPIEngine) { + dpi.AddRule(&netem.DPICloseConnectionForServerEndpoint{ + Logger: log.Log, + ServerIPAddress: "8.8.8.8", + ServerPort: 443, + }) + dpi.AddRule(&netem.DPICloseConnectionForServerEndpoint{ + Logger: log.Log, + ServerIPAddress: "8.8.4.4", + ServerPort: 443, + }) + }, + expectTCP: map[string]int64{ + "connection_refused": 2, + }, + expectTLS: map[string]int64{}, + expectQUIC: map[string]int64{"": 2}, + }, { + name: "TLS handshake reset with minimal runtime", + newRuntime: func(netx model.MeasuringNetwork) dslx.Runtime { + return dslx.NewMinimalRuntime(log.Log, time.Now(), dslx.MinimalRuntimeOptionMeasuringNetwork(netx)) + }, + configureDPI: func(dpi *netem.DPIEngine) { + dpi.AddRule(&netem.DPIResetTrafficForTLSSNI{ + Logger: log.Log, + SNI: "dns.google", + }) + }, + expectTCP: map[string]int64{"": 2}, + expectTLS: map[string]int64{ + "connection_reset": 2, + }, + expectQUIC: map[string]int64{"": 2}, + }, { + name: "QUIC handshake timeout with minimal runtime", + newRuntime: func(netx model.MeasuringNetwork) dslx.Runtime { + return dslx.NewMinimalRuntime(log.Log, time.Now(), dslx.MinimalRuntimeOptionMeasuringNetwork(netx)) + }, + configureDPI: func(dpi *netem.DPIEngine) { + dpi.AddRule(&netem.DPIDropTrafficForServerEndpoint{ + Logger: log.Log, + ServerIPAddress: "8.8.8.8", + ServerPort: 443, + ServerProtocol: layers.IPProtocolUDP, + }) + dpi.AddRule(&netem.DPIDropTrafficForServerEndpoint{ + Logger: log.Log, + ServerIPAddress: "8.8.4.4", + ServerPort: 443, + ServerProtocol: layers.IPProtocolUDP, + }) + }, + expectTCP: map[string]int64{"": 2}, + expectTLS: map[string]int64{"": 2}, + expectQUIC: map[string]int64{ + "generic_timeout_error": 2, + }, + }} + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // create an internet testing scenario + env := netemx.MustNewScenario(netemx.InternetScenario) + defer env.Close() + + // create a dslx.Runtime using the client stack + rt := tc.newRuntime(&netxlite.Netx{ + Underlying: &netxlite.NetemUnderlyingNetworkAdapter{UNet: env.ClientStack}, + }) + defer rt.Close() + + // configure the DPI engine + tc.configureDPI(env.DPIEngine()) + + // create stats + var ( + tcpConnectStats = dslx.NewStats[*dslx.TCPConnection]() + tlsHandshakeStats = dslx.NewStats[*dslx.TLSConnection]() + quicHandshakeStats = dslx.NewStats[*dslx.QUICConnection]() + ) + + // create endpoint measurement function + function := dslx.MeasureResolvedAddresses( + // measure 443/tcp + dslx.Compose7( + dslx.MakeEndpoint("tcp", 443), + dslx.TCPConnect(rt), + tcpConnectStats.Observer(), + dslx.TLSHandshake(rt), + tlsHandshakeStats.Observer(), + dslx.HTTPRequestOverTLS(rt), + dslx.Discard[*dslx.HTTPResponse](), + ), + + // measure 443/udp + dslx.Compose5( + dslx.MakeEndpoint("udp", 443), + dslx.QUICHandshake(rt), + quicHandshakeStats.Observer(), + dslx.HTTPRequestOverQUIC(rt), + dslx.Discard[*dslx.HTTPResponse](), + ), + ) + + // create context + ctx := context.Background() + + // fake out the resolved addresses + resolvedAddrs := &dslx.ResolvedAddresses{ + Addresses: []string{"8.8.8.8", "8.8.4.4"}, + Domain: "dns.google", + } + + // measure the endpoints + _ = function.Apply(ctx, dslx.NewMaybeWithValue(resolvedAddrs)) + + // make sure the TCP connect results are consistent + if diff := cmp.Diff(tc.expectTCP, tcpConnectStats.Export()); diff != "" { + t.Fatal(diff) + } + + // make sure the TLS handshake results are consistent + if diff := cmp.Diff(tc.expectTLS, tlsHandshakeStats.Export()); diff != "" { + t.Fatal(diff) + } + + // make sure the QUIC handshake results are consistent + if diff := cmp.Diff(tc.expectQUIC, quicHandshakeStats.Export()); diff != "" { + t.Fatal(diff) + } + + // TODO(https://github.com/ooni/probe/issues/2620): make sure the observations are OK + }) + } +} diff --git a/pkg/dslx/quic.go b/pkg/dslx/quic.go index e56fcca2..f935f2ea 100644 --- a/pkg/dslx/quic.go +++ b/pkg/dslx/quic.go @@ -7,165 +7,75 @@ package dslx import ( "context" "crypto/tls" - "crypto/x509" "io" - "net" - "sync/atomic" "time" "github.com/ooni/probe-engine/pkg/logx" - "github.com/ooni/probe-engine/pkg/measurexlite" - "github.com/ooni/probe-engine/pkg/model" - "github.com/ooni/probe-engine/pkg/netxlite" "github.com/quic-go/quic-go" ) -// QUICHandshakeOption is an option you can pass to QUICHandshake. -type QUICHandshakeOption func(*quicHandshakeFunc) - -// QUICHandshakeOptionInsecureSkipVerify controls whether QUIC verification is enabled. -func QUICHandshakeOptionInsecureSkipVerify(value bool) QUICHandshakeOption { - return func(thf *quicHandshakeFunc) { - thf.InsecureSkipVerify = value - } -} - -// QUICHandshakeOptionRootCAs allows to configure custom root CAs. -func QUICHandshakeOptionRootCAs(value *x509.CertPool) QUICHandshakeOption { - return func(thf *quicHandshakeFunc) { - thf.RootCAs = value - } -} - -// QUICHandshakeOptionServerName allows to configure the SNI to use. -func QUICHandshakeOptionServerName(value string) QUICHandshakeOption { - return func(thf *quicHandshakeFunc) { - thf.ServerName = value - } -} - // QUICHandshake returns a function performing QUIC handshakes. -func QUICHandshake(pool *ConnPool, options ...QUICHandshakeOption) Func[ - *Endpoint, *Maybe[*QUICConnection]] { - // See https://github.com/ooni/probe/issues/2413 to understand - // why we're using nil to force netxlite to use the cached - // default Mozilla cert pool. - f := &quicHandshakeFunc{ - InsecureSkipVerify: false, - Pool: pool, - RootCAs: nil, - ServerName: "", - } - for _, option := range options { - option(f) - } - return f -} - -// quicHandshakeFunc performs QUIC handshakes. -type quicHandshakeFunc struct { - // InsecureSkipVerify allows to skip TLS verification. - InsecureSkipVerify bool - - // Pool is the ConnPool that owns us. - Pool *ConnPool - - // RootCAs contains the Root CAs to use. - RootCAs *x509.CertPool - - // ServerName is the ServerName to handshake for. - ServerName string - - dialer model.QUICDialer // for testing -} - -// Apply implements Func. -func (f *quicHandshakeFunc) Apply( - ctx context.Context, input *Endpoint) *Maybe[*QUICConnection] { - // create trace - trace := measurexlite.NewTrace(input.IDGenerator.Add(1), input.ZeroTime, input.Tags...) - - // use defaults or user-configured overrides - serverName := f.serverName(input) - - // start the operation logger - ol := logx.NewOperationLogger( - input.Logger, - "[#%d] QUICHandshake with %s SNI=%s", - trace.Index, - input.Address, - serverName, - ) - - // setup - udpListener := netxlite.NewUDPListener() - quicDialer := f.dialer - if quicDialer == nil { - quicDialer = trace.NewQUICDialerWithoutResolver(udpListener, input.Logger) - } - config := &tls.Config{ - NextProtos: []string{"h3"}, - InsecureSkipVerify: f.InsecureSkipVerify, - RootCAs: f.RootCAs, - ServerName: serverName, - } - const timeout = 10 * time.Second - ctx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - - // handshake - quicConn, err := quicDialer.DialContext(ctx, input.Address, config, &quic.Config{}) - - var closerConn io.Closer - var tlsState tls.ConnectionState - if quicConn != nil { - closerConn = &quicCloserConn{quicConn} - tlsState = quicConn.ConnectionState().TLS // only quicConn can be nil - } - - // possibly track established conn for late close - f.Pool.MaybeTrack(closerConn) - - // stop the operation logger - ol.Stop(err) - - state := &QUICConnection{ - Address: input.Address, - QUICConn: quicConn, // possibly nil - Domain: input.Domain, - IDGenerator: input.IDGenerator, - Logger: input.Logger, - Network: input.Network, - TLSConfig: config, - TLSState: tlsState, - Trace: trace, - ZeroTime: input.ZeroTime, - } - - return &Maybe[*QUICConnection]{ - Error: err, - Observations: maybeTraceToObservations(trace), - Operation: netxlite.QUICHandshakeOperation, - State: state, - } -} - -func (f *quicHandshakeFunc) serverName(input *Endpoint) string { - if f.ServerName != "" { - return f.ServerName - } - if input.Domain != "" { - return input.Domain - } - addr, _, err := net.SplitHostPort(input.Address) - if err == nil { - return addr - } - // Note: golang requires a ServerName and fails if it's empty. If the provided - // ServerName is an IP address, however, golang WILL NOT emit any SNI extension - // in the ClientHello, consistently with RFC 6066 Section 3 requirements. - input.Logger.Warn("TLSHandshake: cannot determine which SNI to use") - return "" +func QUICHandshake(rt Runtime, options ...TLSHandshakeOption) Func[*Endpoint, *QUICConnection] { + return Operation[*Endpoint, *QUICConnection](func(ctx context.Context, input *Endpoint) (*QUICConnection, error) { + // create trace + trace := rt.NewTrace(rt.IDGenerator().Add(1), rt.ZeroTime(), input.Tags...) + + // create a suitable TLS configuration + config := tlsNewConfig(input.Address, []string{"h3"}, input.Domain, rt.Logger(), options...) + + // start the operation logger + ol := logx.NewOperationLogger( + rt.Logger(), + "[#%d] QUICHandshake with %s SNI=%s ALPN=%v", + trace.Index(), + input.Address, + config.ServerName, + config.NextProtos, + ) + + // setup + udpListener := trace.NewUDPListener() + quicDialer := trace.NewQUICDialerWithoutResolver(udpListener, rt.Logger()) + const timeout = 10 * time.Second + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + // handshake + quicConn, err := quicDialer.DialContext(ctx, input.Address, config, &quic.Config{}) + + var closerConn io.Closer + var tlsState tls.ConnectionState + if quicConn != nil { + closerConn = &quicCloserConn{quicConn} + tlsState = quicConn.ConnectionState().TLS // only quicConn can be nil + } + + // possibly track established conn for late close + rt.MaybeTrackConn(closerConn) + + // stop the operation logger + ol.Stop(err) + + // save the observations + rt.SaveObservations(maybeTraceToObservations(trace)...) + + // handle error case + if err != nil { + return nil, err + } + + // handle success + state := &QUICConnection{ + Address: input.Address, + QUICConn: quicConn, + Domain: input.Domain, + Network: input.Network, + TLSConfig: config, + TLSState: tlsState, + Trace: trace, + } + return state, nil + }) } // QUICConnection is an established QUIC connection. If you initialize @@ -180,12 +90,6 @@ type QUICConnection struct { // Domain is the OPTIONAL domain we resolved. Domain string - // IDGenerator is the MANDATORY ID generator to use. - IDGenerator *atomic.Int64 - - // Logger is the MANDATORY logger to use. - Logger model.Logger - // Network is the MANDATORY network we tried to use when connecting. Network string @@ -197,10 +101,7 @@ type QUICConnection struct { TLSState tls.ConnectionState // Trace is the MANDATORY trace we're using. - Trace *measurexlite.Trace - - // ZeroTime is the MANDATORY zero time of the measurement. - ZeroTime time.Time + Trace Trace } type quicCloserConn struct { diff --git a/pkg/dslx/quic_test.go b/pkg/dslx/quic_test.go index e32aeb09..496115e9 100644 --- a/pkg/dslx/quic_test.go +++ b/pkg/dslx/quic_test.go @@ -3,42 +3,25 @@ package dslx import ( "context" "crypto/tls" - "crypto/x509" "io" - "sync/atomic" "testing" "time" "github.com/google/go-cmp/cmp" "github.com/ooni/probe-engine/pkg/mocks" "github.com/ooni/probe-engine/pkg/model" + "github.com/ooni/probe-engine/pkg/netxlite" "github.com/quic-go/quic-go" ) /* Test cases: -- Get quicHandshakeFunc with options - Apply quicHandshakeFunc: - with EOF - success - with sni */ func TestQUICHandshake(t *testing.T) { - t.Run("Get quicHandshakeFunc with options", func(t *testing.T) { - certpool := x509.NewCertPool() - certpool.AddCert(&x509.Certificate{}) - - f := QUICHandshake( - &ConnPool{}, - QUICHandshakeOptionInsecureSkipVerify(true), - QUICHandshakeOptionServerName("sni"), - QUICHandshakeOptionRootCAs(certpool), - ) - if _, ok := f.(*quicHandshakeFunc); !ok { - t.Fatal("unexpected type. Expected: quicHandshakeFunc") - } - }) - t.Run("Apply quicHandshakeFunc", func(t *testing.T) { wasClosed := false plainConn := &mocks.QUICEarlyConnection{ @@ -99,28 +82,28 @@ func TestQUICHandshake(t *testing.T) { for name, tt := range tests { t.Run(name, func(t *testing.T) { - pool := &ConnPool{} - quicHandshake := &quicHandshakeFunc{ - Pool: pool, - dialer: tt.dialer, - ServerName: tt.sni, - } + rt := NewRuntimeMeasurexLite(model.DiscardLogger, time.Now(), RuntimeMeasurexLiteOptionMeasuringNetwork(&mocks.MeasuringNetwork{ + MockNewQUICDialerWithoutResolver: func(listener model.UDPListener, logger model.DebugLogger, w ...model.QUICDialerWrapper) model.QUICDialer { + return tt.dialer + }, + MockNewUDPListener: func() model.UDPListener { + return netxlite.NewUDPListener() + }, + })) + quicHandshake := QUICHandshake(rt, TLSHandshakeOptionServerName(tt.sni)) endpoint := &Endpoint{ - Address: "1.2.3.4:567", - Network: "udp", - IDGenerator: &atomic.Int64{}, - Logger: model.DiscardLogger, - Tags: tt.tags, - ZeroTime: time.Time{}, + Address: "1.2.3.4:567", + Network: "udp", + Tags: tt.tags, } - res := quicHandshake.Apply(context.Background(), endpoint) + res := quicHandshake.Apply(context.Background(), NewMaybeWithValue(endpoint)) if res.Error != tt.expectErr { t.Fatalf("unexpected error: %s", res.Error) } - if res.State == nil || res.State.QUICConn != tt.expectConn { - t.Fatal("unexpected conn") + if res.Error == nil && res.State.QUICConn != tt.expectConn { + t.Fatalf("unexpected conn %v", res.State) } - pool.Close() + rt.Close() if wasClosed != tt.closed { t.Fatalf("unexpected connection closed state: %v", wasClosed) } @@ -135,88 +118,5 @@ func TestQUICHandshake(t *testing.T) { }) wasClosed = false } - - t.Run("with nil dialer", func(t *testing.T) { - quicHandshake := &quicHandshakeFunc{Pool: &ConnPool{}, dialer: nil} - endpoint := &Endpoint{ - Address: "1.2.3.4:567", - Network: "udp", - IDGenerator: &atomic.Int64{}, - Logger: model.DiscardLogger, - ZeroTime: time.Time{}, - } - ctx, cancel := context.WithCancel(context.Background()) - cancel() - res := quicHandshake.Apply(ctx, endpoint) - - if res.Error == nil { - t.Fatalf("expected an error here") - } - if res.State.QUICConn != nil { - t.Fatalf("unexpected conn: %s", res.State.QUICConn) - } - }) - }) -} - -/* -Test cases: -- With input SNI -- With input domain -- With input host address -- With input IP address -*/ -func TestServerNameQUIC(t *testing.T) { - t.Run("With input SNI", func(t *testing.T) { - sni := "sni" - endpoint := &Endpoint{ - Address: "example.com:123", - Logger: model.DiscardLogger, - } - f := &quicHandshakeFunc{Pool: &ConnPool{}, ServerName: sni} - serverName := f.serverName(endpoint) - if serverName != sni { - t.Fatalf("unexpected server name: %s", serverName) - } - }) - - t.Run("With input domain", func(t *testing.T) { - domain := "domain" - endpoint := &Endpoint{ - Address: "example.com:123", - Domain: domain, - Logger: model.DiscardLogger, - } - f := &quicHandshakeFunc{Pool: &ConnPool{}} - serverName := f.serverName(endpoint) - if serverName != domain { - t.Fatalf("unexpected server name: %s", serverName) - } - }) - - t.Run("With input host address", func(t *testing.T) { - hostaddr := "example.com" - endpoint := &Endpoint{ - Address: hostaddr + ":123", - Logger: model.DiscardLogger, - } - f := &quicHandshakeFunc{Pool: &ConnPool{}} - serverName := f.serverName(endpoint) - if serverName != hostaddr { - t.Fatalf("unexpected server name: %s", serverName) - } - }) - - t.Run("With input IP address", func(t *testing.T) { - ip := "1.1.1.1" - endpoint := &Endpoint{ - Address: ip, - Logger: model.DiscardLogger, - } - f := &quicHandshakeFunc{Pool: &ConnPool{}} - serverName := f.serverName(endpoint) - if serverName != "" { - t.Fatalf("unexpected server name: %s", serverName) - } }) } diff --git a/pkg/dslx/runtimecore.go b/pkg/dslx/runtimecore.go new file mode 100644 index 00000000..fd52f35b --- /dev/null +++ b/pkg/dslx/runtimecore.go @@ -0,0 +1,49 @@ +package dslx + +import ( + "io" + "sync/atomic" + "time" + + "github.com/ooni/probe-engine/pkg/model" +) + +// Runtime is the runtime in which we execute the DSL. +type Runtime interface { + // Close closes all the connection tracked using MaybeTrackConn. + Close() error + + // IDGenerator returns an atomic counter used to generate + // separate unique IDs for each trace. + IDGenerator() *atomic.Int64 + + // Logger returns the base logger to use. + Logger() model.Logger + + // MaybeTrackConn tracks a connection such that it is closed + // when you call the Runtime's Close method. + MaybeTrackConn(conn io.Closer) + + // NewTrace creates a [Trace] instance. Note that each [Runtime] + // creates its own [Trace] type. A [Trace] is not guaranteed to collect + // [*Observations]. For example, [NewMinimalRuntime] creates a [Runtime] + // that does not collect any [*Observations]. + NewTrace(index int64, zeroTime time.Time, tags ...string) Trace + + // Observations returns the [*Observations] saved so far and clears our + // internal copy such that the next call to this method only returns + // the [*Observations] saved since the previous call. + // + // You can safely call this method from multiple goroutine contexts. + Observations() *Observations + + // SaveObservations saves [*Observations] inside the [Runtime]. You can + // get the saved [*Observations] by calling Observations. + // + // You can safely call this method from multiple goroutine contexts. + SaveObservations(obs ...*Observations) + + // ZeroTime returns the runtime's "zero" time, which is used as the + // starting point to generate observation's delta times. + ZeroTime() time.Time +} diff --git a/pkg/dslx/runtimemeasurex.go b/pkg/dslx/runtimemeasurex.go new file mode 100644 index 00000000..4a8590b4 --- /dev/null +++ b/pkg/dslx/runtimemeasurex.go @@ -0,0 +1,46 @@ +package dslx + +import ( + "time" + + "github.com/ooni/probe-engine/pkg/measurexlite" + "github.com/ooni/probe-engine/pkg/model" + "github.com/ooni/probe-engine/pkg/netxlite" +) + +// RuntimeMeasurexLiteOption is an option for initializing a [*RuntimeMeasurexLite]. +type RuntimeMeasurexLiteOption func(rt *RuntimeMeasurexLite) + +// RuntimeMeasurexLiteOptionMeasuringNetwork allows to configure which [model.MeasuringNetwork] to use. +func RuntimeMeasurexLiteOptionMeasuringNetwork(netx model.MeasuringNetwork) RuntimeMeasurexLiteOption { + return func(rt *RuntimeMeasurexLite) { + rt.netx = netx + } +} + +// NewRuntimeMeasurexLite creates a [Runtime] using [measurexlite] to collect [*Observations]. +func NewRuntimeMeasurexLite(logger model.Logger, zeroTime time.Time, options ...RuntimeMeasurexLiteOption) *RuntimeMeasurexLite { + rt := &RuntimeMeasurexLite{ + MinimalRuntime: NewMinimalRuntime(logger, zeroTime), + netx: &netxlite.Netx{Underlying: nil}, // implies using the host's network + } + for _, option := range options { + option(rt) + } + return rt +} + +// RuntimeMeasurexLite uses [measurexlite] to collect [*Observations.] +type RuntimeMeasurexLite struct { + *MinimalRuntime + netx model.MeasuringNetwork +} + +// NewTrace implements Runtime. +func (p *RuntimeMeasurexLite) NewTrace(index int64, zeroTime time.Time, tags ...string) Trace { + trace := measurexlite.NewTrace(index, zeroTime, tags...) + trace.Netx = p.netx + return trace +} + +var _ Runtime = &RuntimeMeasurexLite{} diff --git a/pkg/dslx/runtimemeasurex_test.go b/pkg/dslx/runtimemeasurex_test.go new file mode 100644 index 00000000..a36b93be --- /dev/null +++ b/pkg/dslx/runtimemeasurex_test.go @@ -0,0 +1,24 @@ +package dslx + +import ( + "testing" + "time" + + "github.com/ooni/probe-engine/pkg/measurexlite" + "github.com/ooni/probe-engine/pkg/mocks" + "github.com/ooni/probe-engine/pkg/model" +) + +func TestMeasurexLiteRuntime(t *testing.T) { + t.Run("we can configure a custom model.MeasuringNetwork", func(t *testing.T) { + netx := &mocks.MeasuringNetwork{} + rt := NewRuntimeMeasurexLite(model.DiscardLogger, time.Now(), RuntimeMeasurexLiteOptionMeasuringNetwork(netx)) + if rt.netx != netx { + t.Fatal("did not set the measuring network") + } + trace := rt.NewTrace(rt.IDGenerator().Add(1), rt.ZeroTime()).(*measurexlite.Trace) + if trace.Netx != netx { + t.Fatal("did not set the measuring network") + } + }) +} diff --git a/pkg/dslx/runtimeminimal.go b/pkg/dslx/runtimeminimal.go new file mode 100644 index 00000000..336971ca --- /dev/null +++ b/pkg/dslx/runtimeminimal.go @@ -0,0 +1,206 @@ +package dslx + +import ( + "io" + "sync" + "sync/atomic" + "time" + + "github.com/ooni/probe-engine/pkg/model" + "github.com/ooni/probe-engine/pkg/netxlite" +) + +// MinimalRuntimeOption is an option for configuring the [*MinimalRuntime]. +type MinimalRuntimeOption func(rt *MinimalRuntime) + +// MinimalRuntimeOptionMeasuringNetwork configures the [model.MeasuringNetwork] to use. +func MinimalRuntimeOptionMeasuringNetwork(netx model.MeasuringNetwork) MinimalRuntimeOption { + return func(rt *MinimalRuntime) { + rt.netx = netx + } +} + +// NewMinimalRuntime creates a minimal [Runtime] implementation. +// +// This [Runtime] implementation does not collect any [*Observations]. +func NewMinimalRuntime(logger model.Logger, zeroTime time.Time, options ...MinimalRuntimeOption) *MinimalRuntime { + rt := &MinimalRuntime{ + idg: &atomic.Int64{}, + logger: logger, + mu: sync.Mutex{}, + netx: &netxlite.Netx{Underlying: nil}, // implies using the host's network + ob: NewObservations(), + v: []io.Closer{}, + zeroT: zeroTime, + } + for _, option := range options { + option(rt) + } + return rt +} + +var _ Runtime = &MinimalRuntime{} + +// MinimalRuntime is a minimal [Runtime] implementation. +type MinimalRuntime struct { + idg *atomic.Int64 + logger model.Logger + mu sync.Mutex + netx model.MeasuringNetwork + ob *Observations + v []io.Closer + zeroT time.Time +} + +// Observations implements Runtime. +func (p *MinimalRuntime) Observations() *Observations { + defer p.mu.Unlock() + p.mu.Lock() + o := p.ob + p.ob = NewObservations() + return o +} + +// SaveObservations implements Runtime. +func (p *MinimalRuntime) SaveObservations(obs ...*Observations) { + defer p.mu.Unlock() + p.mu.Lock() + for _, o := range obs { + p.ob.NetworkEvents = append(p.ob.NetworkEvents, o.NetworkEvents...) + p.ob.QUICHandshakes = append(p.ob.QUICHandshakes, o.QUICHandshakes...) + p.ob.Queries = append(p.ob.Queries, o.Queries...) + p.ob.Requests = append(p.ob.Requests, o.Requests...) + p.ob.TCPConnect = append(p.ob.TCPConnect, o.TCPConnect...) + p.ob.TLSHandshakes = append(p.ob.TLSHandshakes, o.TLSHandshakes...) + } +} + +// IDGenerator implements Runtime. +func (p *MinimalRuntime) IDGenerator() *atomic.Int64 { + return p.idg +} + +// Logger implements Runtime. +func (p *MinimalRuntime) Logger() model.Logger { + return p.logger +} + +// ZeroTime implements Runtime. +func (p *MinimalRuntime) ZeroTime() time.Time { + return p.zeroT +} + +// MaybeTrackConn implements Runtime. +func (p *MinimalRuntime) MaybeTrackConn(conn io.Closer) { + if conn != nil { + defer p.mu.Unlock() + p.mu.Lock() + p.v = append(p.v, conn) + } +} + +// Close implements Runtime. +func (p *MinimalRuntime) Close() error { + // Implementation note: reverse order is such that we close TLS + // connections before we close the TCP connections they use. Hence + // we'll _gracefully_ close TLS connections. + defer p.mu.Unlock() + p.mu.Lock() + for idx := len(p.v) - 1; idx >= 0; idx-- { + _ = p.v[idx].Close() + } + p.v = nil // reset + return nil +} + +// NewTrace implements Runtime. +func (p *MinimalRuntime) NewTrace(index int64, zeroTime time.Time, tags ...string) Trace { + return &minimalTrace{idx: index, netx: p.netx, tags: tags, zt: zeroTime} +} + +type minimalTrace struct { + idx int64 + netx model.MeasuringNetwork + tags []string + zt time.Time +} + +// CloneBytesReceivedMap implements Trace. +func (tx *minimalTrace) CloneBytesReceivedMap() (out map[string]int64) { + return make(map[string]int64) +} + +// DNSLookupsFromRoundTrip implements Trace. +func (tx *minimalTrace) DNSLookupsFromRoundTrip() (out []*model.ArchivalDNSLookupResult) { + return []*model.ArchivalDNSLookupResult{} +} + +// Index implements Trace. +func (tx *minimalTrace) Index() int64 { + return tx.idx +} + +// NetworkEvents implements Trace. +func (tx *minimalTrace) NetworkEvents() (out []*model.ArchivalNetworkEvent) { + return []*model.ArchivalNetworkEvent{} +} + +// NewDialerWithoutResolver implements Trace. +func (tx *minimalTrace) NewDialerWithoutResolver(dl model.DebugLogger, wrappers ...model.DialerWrapper) model.Dialer { + return tx.netx.NewDialerWithoutResolver(dl, wrappers...) +} + +// NewParallelUDPResolver implements Trace. +func (tx *minimalTrace) NewParallelUDPResolver(logger model.DebugLogger, dialer model.Dialer, address string) model.Resolver { + return tx.netx.NewParallelUDPResolver(logger, dialer, address) +} + +// NewQUICDialerWithoutResolver implements Trace. +func (tx *minimalTrace) NewQUICDialerWithoutResolver(listener model.UDPListener, dl model.DebugLogger, wrappers ...model.QUICDialerWrapper) model.QUICDialer { + return tx.netx.NewQUICDialerWithoutResolver(listener, dl, wrappers...) +} + +// NewStdlibResolver implements Trace. +func (tx *minimalTrace) NewStdlibResolver(logger model.DebugLogger) model.Resolver { + return tx.netx.NewStdlibResolver(logger) +} + +// NewTLSHandshakerStdlib implements Trace. +func (tx *minimalTrace) NewTLSHandshakerStdlib(dl model.DebugLogger) model.TLSHandshaker { + return tx.netx.NewTLSHandshakerStdlib(dl) +} + +// NewUDPListener implements Trace +func (tx *minimalTrace) NewUDPListener() model.UDPListener { + return tx.netx.NewUDPListener() +} + +// QUICHandshakes implements Trace. +func (tx *minimalTrace) QUICHandshakes() (out []*model.ArchivalTLSOrQUICHandshakeResult) { + return []*model.ArchivalTLSOrQUICHandshakeResult{} +} + +// TCPConnects implements Trace. +func (tx *minimalTrace) TCPConnects() (out []*model.ArchivalTCPConnectResult) { + return []*model.ArchivalTCPConnectResult{} +} + +// TLSHandshakes implements Trace. +func (tx *minimalTrace) TLSHandshakes() (out []*model.ArchivalTLSOrQUICHandshakeResult) { + return []*model.ArchivalTLSOrQUICHandshakeResult{} +} + +// Tags implements Trace. +func (tx *minimalTrace) Tags() []string { + return tx.tags +} + +// TimeSince implements Trace. +func (tx *minimalTrace) TimeSince(t0 time.Time) time.Duration { + return time.Since(t0) +} + +// ZeroTime implements Trace. +func (tx *minimalTrace) ZeroTime() time.Time { + return tx.zt +} diff --git a/pkg/dslx/runtimeminimal_test.go b/pkg/dslx/runtimeminimal_test.go new file mode 100644 index 00000000..118475c3 --- /dev/null +++ b/pkg/dslx/runtimeminimal_test.go @@ -0,0 +1,252 @@ +package dslx + +import ( + "errors" + "io" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/ooni/probe-engine/pkg/mocks" + "github.com/ooni/probe-engine/pkg/model" + "github.com/quic-go/quic-go" +) + +/* +Test cases: +- Maybe track connections: + - with nil + - with connection + - with quic connection + +- Close MinimalRuntime: + - all Close() calls succeed + - one Close() call fails +*/ + +func closeableConnWithErr(err error) io.Closer { + return &mocks.Conn{ + MockClose: func() error { + return err + }, + } +} + +func closeableQUICConnWithErr(err error) io.Closer { + return &quicCloserConn{ + &mocks.QUICEarlyConnection{ + MockCloseWithError: func(code quic.ApplicationErrorCode, reason string) error { + return err + }, + }, + } +} + +func TestMinimalRuntime(t *testing.T) { + // testcase is a test case implemented by this function + type testcase struct { + mockConn io.Closer + want int // len of (*minimalRuntime).v + } + + t.Run("Maybe track connections", func(t *testing.T) { + tests := map[string]testcase{ + "with nil": {mockConn: nil, want: 0}, + "with connection": {mockConn: closeableConnWithErr(nil), want: 1}, + "with quic connection": {mockConn: closeableQUICConnWithErr(nil), want: 1}, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + rt := NewMinimalRuntime(model.DiscardLogger, time.Now()) + rt.MaybeTrackConn(tt.mockConn) + if len(rt.v) != tt.want { + t.Fatalf("expected %d tracked connections, got: %d", tt.want, len(rt.v)) + } + }) + } + }) + + t.Run("Close MinimalRuntime", func(t *testing.T) { + mockErr := errors.New("mocked") + tests := map[string]struct { + rt *MinimalRuntime + }{ + "all Close() calls succeed": { + rt: &MinimalRuntime{ + v: []io.Closer{ + closeableConnWithErr(nil), + closeableQUICConnWithErr(nil), + }, + }, + }, + "one Close() call fails": { + rt: &MinimalRuntime{ + v: []io.Closer{ + closeableConnWithErr(nil), + closeableConnWithErr(mockErr), + }, + }, + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + err := tt.rt.Close() + if err != nil { // Close() should always return nil + t.Fatalf("unexpected error %s", err) + } + if tt.rt.v != nil { + t.Fatalf("v should be reset but is not") + } + }) + } + }) + + t.Run("IDGenerator", func(t *testing.T) { + rt := NewMinimalRuntime(model.DiscardLogger, time.Now()) + out := rt.IDGenerator() + if out == nil { + t.Fatal("expected non-nil pointer") + } + }) + + t.Run("Logger", func(t *testing.T) { + rt := NewMinimalRuntime(model.DiscardLogger, time.Now()) + out := rt.Logger() + if out == nil { + t.Fatal("expected non-nil pointer") + } + }) + + t.Run("ZeroTime", func(t *testing.T) { + rt := NewMinimalRuntime(model.DiscardLogger, time.Now()) + out := rt.ZeroTime() + if out.IsZero() { + t.Fatal("expected non-zero time") + } + }) + + t.Run("Trace", func(t *testing.T) { + tags := []string{"antani", "mascetti", "melandri"} + rt := NewMinimalRuntime(model.DiscardLogger, time.Now()) + now := time.Now() + trace := rt.NewTrace(10, now, tags...) + + t.Run("CloneBytesReceivedMap", func(t *testing.T) { + out := trace.CloneBytesReceivedMap() + if out == nil || len(out) != 0 { + t.Fatal("expected zero-length map") + } + }) + + t.Run("DNSLookupsFromRoundTrip", func(t *testing.T) { + out := trace.DNSLookupsFromRoundTrip() + if out == nil || len(out) != 0 { + t.Fatal("expected zero-length slice") + } + }) + + t.Run("Index", func(t *testing.T) { + out := trace.Index() + if out != 10 { + t.Fatal("expected 10, got", out) + } + }) + + t.Run("NetworkEvents", func(t *testing.T) { + out := trace.NetworkEvents() + if out == nil || len(out) != 0 { + t.Fatal("expected zero-length slice") + } + }) + + t.Run("NewDialerWithoutResolver", func(t *testing.T) { + out := trace.NewDialerWithoutResolver(model.DiscardLogger) + if out == nil { + t.Fatal("expected non-nil pointer") + } + }) + + t.Run("NewParallelUDPResolver", func(t *testing.T) { + out := trace.NewParallelUDPResolver(model.DiscardLogger, &mocks.Dialer{}, "8.8.8.8:53") + if out == nil { + t.Fatal("expected non-nil pointer") + } + }) + + t.Run("NewQUICDialerWithoutResolver", func(t *testing.T) { + out := trace.NewQUICDialerWithoutResolver(&mocks.UDPListener{}, model.DiscardLogger) + if out == nil { + t.Fatal("expected non-nil pointer") + } + }) + + t.Run("NewStdlibResolver", func(t *testing.T) { + out := trace.NewStdlibResolver(model.DiscardLogger) + if out == nil { + t.Fatal("expected non-nil pointer") + } + }) + + t.Run("NewTLSHandshakerStdlib", func(t *testing.T) { + out := trace.NewTLSHandshakerStdlib(model.DiscardLogger) + if out == nil { + t.Fatal("expected non-nil pointer") + } + }) + + t.Run("QUICHandshakes", func(t *testing.T) { + out := trace.QUICHandshakes() + if out == nil || len(out) != 0 { + t.Fatal("expected zero-length slice") + } + }) + + t.Run("TCPConnects", func(t *testing.T) { + out := trace.TCPConnects() + if out == nil || len(out) != 0 { + t.Fatal("expected zero-length slice") + } + }) + + t.Run("TLSHandshakes", func(t *testing.T) { + out := trace.TLSHandshakes() + if out == nil || len(out) != 0 { + t.Fatal("expected zero-length slice") + } + }) + + t.Run("Tags", func(t *testing.T) { + out := trace.Tags() + if diff := cmp.Diff(tags, out); diff != "" { + t.Fatal(diff) + } + }) + + t.Run("TimeSince", func(t *testing.T) { + out := trace.TimeSince(now.Add(-10 * time.Second)) + if out == 0 { + t.Fatal("expected non-zero time") + } + }) + + t.Run("ZeroTime", func(t *testing.T) { + out := trace.ZeroTime() + if out.IsZero() { + t.Fatal("expected non-zero time") + } + }) + }) + + t.Run("we can use a custom model.MeasuringNetwork", func(t *testing.T) { + netx := &mocks.MeasuringNetwork{} + rt := NewMinimalRuntime(model.DiscardLogger, time.Now(), MinimalRuntimeOptionMeasuringNetwork(netx)) + if rt.netx != netx { + t.Fatal("did not set the measuring network") + } + trace := rt.NewTrace(rt.IDGenerator().Add(1), rt.ZeroTime()).(*minimalTrace) + if trace.netx != netx { + t.Fatal("did not set the measuring network") + } + }) +} diff --git a/pkg/dslx/tcp.go b/pkg/dslx/tcp.go index 8fb5971d..a524ee29 100644 --- a/pkg/dslx/tcp.go +++ b/pkg/dslx/tcp.go @@ -7,85 +7,60 @@ package dslx import ( "context" "net" - "sync/atomic" "time" "github.com/ooni/probe-engine/pkg/logx" - "github.com/ooni/probe-engine/pkg/measurexlite" - "github.com/ooni/probe-engine/pkg/model" - "github.com/ooni/probe-engine/pkg/netxlite" ) // TCPConnect returns a function that establishes TCP connections. -func TCPConnect(pool *ConnPool) Func[*Endpoint, *Maybe[*TCPConnection]] { - f := &tcpConnectFunc{pool, nil} - return f -} - -// tcpConnectFunc is a function that establishes TCP connections. -type tcpConnectFunc struct { - p *ConnPool - dialer model.Dialer // for testing -} - -// Apply applies the function to its arguments. -func (f *tcpConnectFunc) Apply( - ctx context.Context, input *Endpoint) *Maybe[*TCPConnection] { - - // create trace - trace := measurexlite.NewTrace(input.IDGenerator.Add(1), input.ZeroTime, input.Tags...) - - // start the operation logger - ol := logx.NewOperationLogger( - input.Logger, - "[#%d] TCPConnect %s", - trace.Index, - input.Address, - ) - - // setup - const timeout = 15 * time.Second - ctx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - - // obtain the dialer to use - dialer := f.dialerOrDefault(trace, input.Logger) - - // connect - conn, err := dialer.DialContext(ctx, "tcp", input.Address) - - // possibly register established conn for late close - f.p.MaybeTrack(conn) - - // stop the operation logger - ol.Stop(err) - - state := &TCPConnection{ - Address: input.Address, - Conn: conn, // possibly nil - Domain: input.Domain, - IDGenerator: input.IDGenerator, - Logger: input.Logger, - Network: input.Network, - Trace: trace, - ZeroTime: input.ZeroTime, - } - - return &Maybe[*TCPConnection]{ - Error: err, - Observations: maybeTraceToObservations(trace), - Operation: netxlite.ConnectOperation, - State: state, - } -} - -// dialerOrDefault is the function used to obtain a dialer -func (f *tcpConnectFunc) dialerOrDefault(trace *measurexlite.Trace, logger model.Logger) model.Dialer { - dialer := f.dialer - if dialer == nil { - dialer = trace.NewDialerWithoutResolver(logger) - } - return dialer +func TCPConnect(rt Runtime) Func[*Endpoint, *TCPConnection] { + return Operation[*Endpoint, *TCPConnection](func(ctx context.Context, input *Endpoint) (*TCPConnection, error) { + // create trace + trace := rt.NewTrace(rt.IDGenerator().Add(1), rt.ZeroTime(), input.Tags...) + + // start the operation logger + ol := logx.NewOperationLogger( + rt.Logger(), + "[#%d] TCPConnect %s", + trace.Index(), + input.Address, + ) + + // setup + const timeout = 15 * time.Second + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + // obtain the dialer to use + dialer := trace.NewDialerWithoutResolver(rt.Logger()) + + // connect + conn, err := dialer.DialContext(ctx, "tcp", input.Address) + + // possibly register established conn for late close + rt.MaybeTrackConn(conn) + + // stop the operation logger + ol.Stop(err) + + // save the observations + rt.SaveObservations(maybeTraceToObservations(trace)...) + + // handle error case + if err != nil { + return nil, err + } + + // handle success + state := &TCPConnection{ + Address: input.Address, + Conn: conn, + Domain: input.Domain, + Network: input.Network, + Trace: trace, + } + return state, nil + }) } // TCPConnection is an established TCP connection. If you initialize @@ -100,18 +75,9 @@ type TCPConnection struct { // Domain is the OPTIONAL domain from which we resolved the Address. Domain string - // IDGenerator is the MANDATORY ID generator. - IDGenerator *atomic.Int64 - - // Logger is the MANDATORY logger to use. - Logger model.Logger - // Network is the MANDATORY network we tried to use when connecting. Network string // Trace is the MANDATORY trace we're using. - Trace *measurexlite.Trace - - // ZeroTime is the MANDATORY zero time of the measurement. - ZeroTime time.Time + Trace Trace } diff --git a/pkg/dslx/tcp_test.go b/pkg/dslx/tcp_test.go index d5b51f37..5c86efdc 100644 --- a/pkg/dslx/tcp_test.go +++ b/pkg/dslx/tcp_test.go @@ -4,26 +4,15 @@ import ( "context" "io" "net" - "sync/atomic" "testing" "time" "github.com/google/go-cmp/cmp" - "github.com/ooni/probe-engine/pkg/measurexlite" "github.com/ooni/probe-engine/pkg/mocks" "github.com/ooni/probe-engine/pkg/model" ) func TestTCPConnect(t *testing.T) { - t.Run("Get tcpConnectFunc", func(t *testing.T) { - f := TCPConnect( - &ConnPool{}, - ) - if _, ok := f.(*tcpConnectFunc); !ok { - t.Fatal("unexpected type. Expected: tcpConnectFunc") - } - }) - t.Run("Apply tcpConnectFunc", func(t *testing.T) { wasClosed := false plainConn := &mocks.Conn{ @@ -69,24 +58,25 @@ func TestTCPConnect(t *testing.T) { for name, tt := range tests { t.Run(name, func(t *testing.T) { - pool := &ConnPool{} - tcpConnect := &tcpConnectFunc{pool, tt.dialer} + rt := NewRuntimeMeasurexLite(model.DiscardLogger, time.Now(), RuntimeMeasurexLiteOptionMeasuringNetwork(&mocks.MeasuringNetwork{ + MockNewDialerWithoutResolver: func(dl model.DebugLogger, w ...model.DialerWrapper) model.Dialer { + return tt.dialer + }, + })) + tcpConnect := TCPConnect(rt) endpoint := &Endpoint{ - Address: "1.2.3.4:567", - Network: "tcp", - IDGenerator: &atomic.Int64{}, - Logger: model.DiscardLogger, - Tags: tt.tags, - ZeroTime: time.Time{}, + Address: "1.2.3.4:567", + Network: "tcp", + Tags: tt.tags, } - res := tcpConnect.Apply(context.Background(), endpoint) + res := tcpConnect.Apply(context.Background(), NewMaybeWithValue(endpoint)) if res.Error != tt.expectErr { t.Fatalf("unexpected error: %s", res.Error) } - if res.State == nil || res.State.Conn != tt.expectConn { - t.Fatal("unexpected conn") + if res.Error == nil && res.State.Conn != tt.expectConn { + t.Fatalf("unexpected conn %v", res.State) } - pool.Close() + rt.Close() if wasClosed != tt.closed { t.Fatalf("unexpected connection closed state: %v", wasClosed) } @@ -103,15 +93,3 @@ func TestTCPConnect(t *testing.T) { } }) } - -// Make sure we get a valid dialer if no mocked dialer is configured -func TestDialerOrDefault(t *testing.T) { - f := &tcpConnectFunc{ - p: &ConnPool{}, - dialer: nil, - } - dialer := f.dialerOrDefault(measurexlite.NewTrace(0, time.Now()), model.DiscardLogger) - if dialer == nil { - t.Fatal("expected non-nil dialer here") - } -} diff --git a/pkg/dslx/tls.go b/pkg/dslx/tls.go index af504383..4d3307ec 100644 --- a/pkg/dslx/tls.go +++ b/pkg/dslx/tls.go @@ -9,183 +9,146 @@ import ( "crypto/tls" "crypto/x509" "net" - "sync/atomic" "time" "github.com/ooni/probe-engine/pkg/logx" - "github.com/ooni/probe-engine/pkg/measurexlite" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/netxlite" ) // TLSHandshakeOption is an option you can pass to TLSHandshake. -type TLSHandshakeOption func(*tlsHandshakeFunc) +type TLSHandshakeOption func(config *tls.Config) // TLSHandshakeOptionInsecureSkipVerify controls whether TLS verification is enabled. func TLSHandshakeOptionInsecureSkipVerify(value bool) TLSHandshakeOption { - return func(thf *tlsHandshakeFunc) { - thf.InsecureSkipVerify = value + return func(config *tls.Config) { + config.InsecureSkipVerify = value } } // TLSHandshakeOptionNextProto allows to configure the ALPN protocols. func TLSHandshakeOptionNextProto(value []string) TLSHandshakeOption { - return func(thf *tlsHandshakeFunc) { - thf.NextProto = value + return func(config *tls.Config) { + config.NextProtos = value } } // TLSHandshakeOptionRootCAs allows to configure custom root CAs. func TLSHandshakeOptionRootCAs(value *x509.CertPool) TLSHandshakeOption { - return func(thf *tlsHandshakeFunc) { - thf.RootCAs = value + return func(config *tls.Config) { + config.RootCAs = value } } // TLSHandshakeOptionServerName allows to configure the SNI to use. func TLSHandshakeOptionServerName(value string) TLSHandshakeOption { - return func(thf *tlsHandshakeFunc) { - thf.ServerName = value + return func(config *tls.Config) { + config.ServerName = value } } // TLSHandshake returns a function performing TSL handshakes. -func TLSHandshake(pool *ConnPool, options ...TLSHandshakeOption) Func[ - *TCPConnection, *Maybe[*TLSConnection]] { +func TLSHandshake(rt Runtime, options ...TLSHandshakeOption) Func[*TCPConnection, *TLSConnection] { + return Operation[*TCPConnection, *TLSConnection](func(ctx context.Context, input *TCPConnection) (*TLSConnection, error) { + // keep using the same trace + trace := input.Trace + + // create a suitable TLS configuration + config := tlsNewConfig(input.Address, []string{"h2", "http/1.1"}, input.Domain, rt.Logger(), options...) + + // start the operation logger + ol := logx.NewOperationLogger( + rt.Logger(), + "[#%d] TLSHandshake with %s SNI=%s ALPN=%v", + trace.Index(), + input.Address, + config.ServerName, + config.NextProtos, + ) + + // obtain the handshaker for use + handshaker := trace.NewTLSHandshakerStdlib(rt.Logger()) + + // setup + const timeout = 10 * time.Second + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + // handshake + conn, err := handshaker.Handshake(ctx, input.Conn, config) + + // possibly register established conn for late close + rt.MaybeTrackConn(conn) + + // stop the operation logger + ol.Stop(err) + + // save the observations + rt.SaveObservations(maybeTraceToObservations(trace)...) + + // handle error case + if err != nil { + return nil, err + } + + // handle success + state := &TLSConnection{ + Address: input.Address, + Conn: conn, + Domain: input.Domain, + Network: input.Network, + TLSState: netxlite.MaybeTLSConnectionState(conn), + Trace: trace, + } + return state, nil + }) +} + +// tlsNewConfig is an utility function to create a new TLS config. +// +// Arguments: +// +// - address is the endpoint address (e.g., 1.1.1.1:443); +// +// - defaultALPN contains the default to be used for configuring ALPN; +// +// - domain is the possibly empty domain to use; +// +// - logger is the logger to use; +// +// - options contains options to modify the TLS handshake defaults. +func tlsNewConfig(address string, defaultALPN []string, domain string, logger model.Logger, options ...TLSHandshakeOption) *tls.Config { // See https://github.com/ooni/probe/issues/2413 to understand // why we're using nil to force netxlite to use the cached // default Mozilla cert pool. - f := &tlsHandshakeFunc{ + config := &tls.Config{ + NextProtos: append([]string{}, defaultALPN...), InsecureSkipVerify: false, - NextProto: []string{}, - Pool: pool, RootCAs: nil, - ServerName: "", + ServerName: tlsServerName(address, domain, logger), } for _, option := range options { - option(f) + option(config) } - return f + return config } -// tlsHandshakeFunc performs TLS handshakes. -type tlsHandshakeFunc struct { - // InsecureSkipVerify allows to skip TLS verification. - InsecureSkipVerify bool - - // NextProto contains the ALPNs to negotiate. - NextProto []string - - // Pool is the Pool that owns us. - Pool *ConnPool - - // RootCAs contains the Root CAs to use. - RootCAs *x509.CertPool - - // ServerName is the ServerName to handshake for. - ServerName string - - // for testing - handshaker model.TLSHandshaker -} - -// Apply implements Func. -func (f *tlsHandshakeFunc) Apply( - ctx context.Context, input *TCPConnection) *Maybe[*TLSConnection] { - // keep using the same trace - trace := input.Trace - - // use defaults or user-configured overrides - serverName := f.serverName(input) - nextProto := f.nextProto() - - // start the operation logger - ol := logx.NewOperationLogger( - input.Logger, - "[#%d] TLSHandshake with %s SNI=%s ALPN=%v", - trace.Index, - input.Address, - serverName, - nextProto, - ) - - // obtain the handshaker for use - handshaker := f.handshakerOrDefault(trace, input.Logger) - - // setup - config := &tls.Config{ - NextProtos: nextProto, - InsecureSkipVerify: f.InsecureSkipVerify, - RootCAs: f.RootCAs, - ServerName: serverName, +// tlsServerName is an utility function to obtina the server name from a TCPConnection. +func tlsServerName(address, domain string, logger model.Logger) string { + if domain != "" { + return domain } - const timeout = 10 * time.Second - ctx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - - // handshake - conn, err := handshaker.Handshake(ctx, input.Conn, config) - - // possibly register established conn for late close - f.Pool.MaybeTrack(conn) - - // stop the operation logger - ol.Stop(err) - - state := &TLSConnection{ - Address: input.Address, - Conn: conn, // possibly nil - Domain: input.Domain, - IDGenerator: input.IDGenerator, - Logger: input.Logger, - Network: input.Network, - TLSState: netxlite.MaybeTLSConnectionState(conn), - Trace: trace, - ZeroTime: input.ZeroTime, - } - - return &Maybe[*TLSConnection]{ - Error: err, - Observations: maybeTraceToObservations(trace), - Operation: netxlite.TLSHandshakeOperation, - State: state, - } -} - -// handshakerOrDefault is the function used to obtain an handshaker -func (f *tlsHandshakeFunc) handshakerOrDefault(trace *measurexlite.Trace, logger model.Logger) model.TLSHandshaker { - handshaker := f.handshaker - if handshaker == nil { - handshaker = trace.NewTLSHandshakerStdlib(logger) - } - return handshaker -} - -func (f *tlsHandshakeFunc) serverName(input *TCPConnection) string { - if f.ServerName != "" { - return f.ServerName - } - if input.Domain != "" { - return input.Domain - } - addr, _, err := net.SplitHostPort(input.Address) + addr, _, err := net.SplitHostPort(address) if err == nil { return addr } // Note: golang requires a ServerName and fails if it's empty. If the provided // ServerName is an IP address, however, golang WILL NOT emit any SNI extension // in the ClientHello, consistently with RFC 6066 Section 3 requirements. - input.Logger.Warn("TLSHandshake: cannot determine which SNI to use") + logger.Warn("TLSHandshake: cannot determine which SNI to use") return "" } -func (f *tlsHandshakeFunc) nextProto() []string { - if len(f.NextProto) > 0 { - return f.NextProto - } - return []string{"h2", "http/1.1"} -} - // TLSConnection is an established TLS connection. If you initialize // manually, init at least the ones marked as MANDATORY. type TLSConnection struct { @@ -198,12 +161,6 @@ type TLSConnection struct { // Domain is the OPTIONAL domain we resolved. Domain string - // IDGenerator is the MANDATORY ID generator to use. - IDGenerator *atomic.Int64 - - // Logger is the MANDATORY logger to use. - Logger model.Logger - // Network is the MANDATORY network we tried to use when connecting. Network string @@ -211,8 +168,5 @@ type TLSConnection struct { TLSState tls.ConnectionState // Trace is the MANDATORY trace we're using. - Trace *measurexlite.Trace - - // ZeroTime is the MANDATORY zero time of the measurement. - ZeroTime time.Time + Trace Trace } diff --git a/pkg/dslx/tls_test.go b/pkg/dslx/tls_test.go index 29fb740b..6f450abf 100644 --- a/pkg/dslx/tls_test.go +++ b/pkg/dslx/tls_test.go @@ -10,52 +10,66 @@ import ( "testing" "time" - "github.com/ooni/probe-engine/pkg/measurexlite" + "github.com/google/go-cmp/cmp" "github.com/ooni/probe-engine/pkg/mocks" "github.com/ooni/probe-engine/pkg/model" ) -/* -Test cases: -- Get tlsHandshakeFunc with options -- Apply tlsHandshakeFunc: - - with EOF - - with invalid address - - with success - - with sni - - with options -*/ -func TestTLSHandshake(t *testing.T) { - t.Run("Get tlsHandshakeFunc with options", func(t *testing.T) { +func TestTLSNewConfig(t *testing.T) { + t.Run("without options", func(t *testing.T) { + config := tlsNewConfig("1.1.1.1:443", []string{"h2", "http/1.1"}, "sni", model.DiscardLogger) + + if config.InsecureSkipVerify { + t.Fatalf("unexpected %s, expected %v, got %v", "InsecureSkipVerify", false, config.InsecureSkipVerify) + } + if diff := cmp.Diff([]string{"h2", "http/1.1"}, config.NextProtos); diff != "" { + t.Fatal(diff) + } + if config.ServerName != "sni" { + t.Fatalf("unexpected %s, expected %s, got %s", "ServerName", "sni", config.ServerName) + } + if !config.RootCAs.Equal(nil) { + t.Fatalf("unexpected %s, expected %v, got %v", "RootCAs", nil, config.RootCAs) + } + }) + + t.Run("with options", func(t *testing.T) { certpool := x509.NewCertPool() certpool.AddCert(&x509.Certificate{}) - f := TLSHandshake( - &ConnPool{}, + config := tlsNewConfig( + "1.1.1.1:443", []string{"h2", "http/1.1"}, "sni", model.DiscardLogger, TLSHandshakeOptionInsecureSkipVerify(true), TLSHandshakeOptionNextProto([]string{"h2"}), - TLSHandshakeOptionServerName("sni"), + TLSHandshakeOptionServerName("example.domain"), TLSHandshakeOptionRootCAs(certpool), ) - var handshakeFunc *tlsHandshakeFunc - var ok bool - if handshakeFunc, ok = f.(*tlsHandshakeFunc); !ok { - t.Fatal("unexpected type. Expected: tlsHandshakeFunc") - } - if !handshakeFunc.InsecureSkipVerify { - t.Fatalf("unexpected %s, expected %v, got %v", "InsecureSkipVerify", true, false) + + if !config.InsecureSkipVerify { + t.Fatalf("unexpected %s, expected %v, got %v", "InsecureSkipVerify", true, config.InsecureSkipVerify) } - if len(handshakeFunc.NextProto) != 1 || handshakeFunc.NextProto[0] != "h2" { - t.Fatalf("unexpected %s, expected %v, got %v", "NextProto", []string{"h2"}, handshakeFunc.NextProto) + if diff := cmp.Diff([]string{"h2"}, config.NextProtos); diff != "" { + t.Fatal(diff) } - if handshakeFunc.ServerName != "sni" { - t.Fatalf("unexpected %s, expected %s, got %s", "ServerName", "sni", handshakeFunc.ServerName) + if config.ServerName != "example.domain" { + t.Fatalf("unexpected %s, expected %s, got %s", "ServerName", "example.domain", config.ServerName) } - if !handshakeFunc.RootCAs.Equal(certpool) { - t.Fatalf("unexpected %s, expected %v, got %v", "RootCAs", certpool, handshakeFunc.RootCAs) + if !config.RootCAs.Equal(certpool) { + t.Fatalf("unexpected %s, expected %v, got %v", "RootCAs", nil, config.RootCAs) } }) +} +/* +Test cases: +- Apply tlsHandshakeFunc: + - with EOF + - with invalid address + - with success + - with sni + - with options +*/ +func TestTLSHandshake(t *testing.T) { t.Run("Apply tlsHandshakeFunc", func(t *testing.T) { wasClosed := false @@ -133,37 +147,36 @@ func TestTLSHandshake(t *testing.T) { for name, tt := range tests { t.Run(name, func(t *testing.T) { - pool := &ConnPool{} - tlsHandshake := &tlsHandshakeFunc{ - NextProto: tt.config.nextProtos, - Pool: pool, - ServerName: tt.config.sni, - handshaker: tt.handshaker, - } + rt := NewMinimalRuntime(model.DiscardLogger, time.Now(), MinimalRuntimeOptionMeasuringNetwork(&mocks.MeasuringNetwork{ + MockNewTLSHandshakerStdlib: func(logger model.DebugLogger) model.TLSHandshaker { + return tt.handshaker + }, + })) + tlsHandshake := TLSHandshake(rt, + TLSHandshakeOptionNextProto(tt.config.nextProtos), + TLSHandshakeOptionServerName(tt.config.sni), + ) idGen := &atomic.Int64{} zeroTime := time.Time{} - trace := measurexlite.NewTrace(idGen.Add(1), zeroTime) + trace := rt.NewTrace(idGen.Add(1), zeroTime) address := tt.config.address if address == "" { address = "1.2.3.4:567" } tcpConn := TCPConnection{ - Address: address, - Conn: &tcpConn, - IDGenerator: idGen, - Logger: model.DiscardLogger, - Network: "tcp", - Trace: trace, - ZeroTime: zeroTime, + Address: address, + Conn: &tcpConn, + Network: "tcp", + Trace: trace, } - res := tlsHandshake.Apply(context.Background(), &tcpConn) + res := tlsHandshake.Apply(context.Background(), NewMaybeWithValue(&tcpConn)) if res.Error != tt.expectErr { t.Fatalf("unexpected error: %s", res.Error) } - if res.State.Conn != tt.expectConn { - t.Fatalf("unexpected conn %v", res.State.Conn) + if res.State != nil && res.State.Conn != tt.expectConn { + t.Fatalf("unexpected conn %v", res.State) } - pool.Close() + rt.Close() if wasClosed != tt.closed { t.Fatalf("unexpected connection closed state %v", wasClosed) } @@ -175,84 +188,29 @@ func TestTLSHandshake(t *testing.T) { /* Test cases: -- With input SNI -- With input domain -- With input host address -- With input IP address +- With domain +- With host address +- With IP address */ -func TestServerNameTLS(t *testing.T) { - t.Run("With input SNI", func(t *testing.T) { - sni := "sni" - tcpConn := TCPConnection{ - Address: "example.com:123", - Logger: model.DiscardLogger, - } - f := &tlsHandshakeFunc{ - Pool: &ConnPool{}, - ServerName: sni, - } - serverName := f.serverName(&tcpConn) - if serverName != sni { +func TestTLSServerName(t *testing.T) { + t.Run("With domain", func(t *testing.T) { + serverName := tlsServerName("example.com:123", "domain", model.DiscardLogger) + if serverName != "domain" { t.Fatalf("unexpected server name: %s", serverName) } }) - t.Run("With input domain", func(t *testing.T) { - domain := "domain" - tcpConn := TCPConnection{ - Address: "example.com:123", - Domain: domain, - Logger: model.DiscardLogger, - } - f := &tlsHandshakeFunc{ - Pool: &ConnPool{}, - } - serverName := f.serverName(&tcpConn) - if serverName != domain { - t.Fatalf("unexpected server name: %s", serverName) - } - }) - t.Run("With input host address", func(t *testing.T) { - hostaddr := "example.com" - tcpConn := TCPConnection{ - Address: hostaddr + ":123", - Logger: model.DiscardLogger, - } - f := &tlsHandshakeFunc{ - Pool: &ConnPool{}, - } - serverName := f.serverName(&tcpConn) - if serverName != hostaddr { + + t.Run("With host address", func(t *testing.T) { + serverName := tlsServerName("1.1.1.1:443", "", model.DiscardLogger) + if serverName != "1.1.1.1" { t.Fatalf("unexpected server name: %s", serverName) } }) - t.Run("With input IP address", func(t *testing.T) { - ip := "1.1.1.1" - tcpConn := TCPConnection{ - Address: ip, - Logger: model.DiscardLogger, - } - f := &tlsHandshakeFunc{ - Pool: &ConnPool{}, - } - serverName := f.serverName(&tcpConn) + + t.Run("With IP address", func(t *testing.T) { + serverName := tlsServerName("1.1.1.1", "", model.DiscardLogger) if serverName != "" { t.Fatalf("unexpected server name: %s", serverName) } }) } - -// Make sure we get a valid handshaker if no mocked handshaker is configured -func TestHandshakerOrDefault(t *testing.T) { - f := &tlsHandshakeFunc{ - InsecureSkipVerify: false, - NextProto: []string{}, - Pool: &ConnPool{}, - RootCAs: &x509.CertPool{}, - ServerName: "", - handshaker: nil, - } - handshaker := f.handshakerOrDefault(measurexlite.NewTrace(0, time.Now()), model.DiscardLogger) - if handshaker == nil { - t.Fatal("expected non-nil handshaker here") - } -} diff --git a/pkg/dslx/trace.go b/pkg/dslx/trace.go new file mode 100644 index 00000000..28d20d21 --- /dev/null +++ b/pkg/dslx/trace.go @@ -0,0 +1,74 @@ +package dslx + +import ( + "time" + + "github.com/ooni/probe-engine/pkg/model" +) + +// Trace collects [*Observations] using tracing. Specific implementations +// of this interface may be engineered to collect no [*Observations] for +// efficiency (i.e., when you don't care about collecting [*Observations] +// but you still want to use this package). +type Trace interface { + // CloneBytesReceivedMap returns a clone of the internal bytes received map. The key of the + // map is a string following the "EPNT_ADDRESS PROTO" pattern where the "EPNT_ADDRESS" contains + // the endpoint address and "PROTO" is "tcp" or "udp". + CloneBytesReceivedMap() (out map[string]int64) + + // DNSLookupsFromRoundTrip returns all the DNS lookup results collected so far. + DNSLookupsFromRoundTrip() (out []*model.ArchivalDNSLookupResult) + + // Index returns the unique index used by this trace. + Index() int64 + + // NewDialerWithoutResolver is equivalent to netxlite.NewDialerWithoutResolver + // except that it returns a model.Dialer that uses this trace. + // + // Caveat: the dialer wrappers are there to implement the + // model.MeasuringNetwork interface, but they're not used by this function. + NewDialerWithoutResolver(dl model.DebugLogger, wrappers ...model.DialerWrapper) model.Dialer + + // NewParallelUDPResolver returns a possibly-trace-ware parallel UDP resolver + NewParallelUDPResolver(logger model.DebugLogger, dialer model.Dialer, address string) model.Resolver + + // NewQUICDialerWithoutResolver is equivalent to + // netxlite.NewQUICDialerWithoutResolver except that it returns a + // model.QUICDialer that uses this trace. + // + // Caveat: the dialer wrappers are there to implement the + // model.MeasuringNetwork interface, but they're not used by this function. + NewQUICDialerWithoutResolver(listener model.UDPListener, + dl model.DebugLogger, wrappers ...model.QUICDialerWrapper) model.QUICDialer + + // NewTLSHandshakerStdlib is equivalent to netxlite.NewTLSHandshakerStdlib + // except that it returns a model.TLSHandshaker that uses this trace. + NewTLSHandshakerStdlib(dl model.DebugLogger) model.TLSHandshaker + + // NetworkEvents returns all the network events collected so far. + NetworkEvents() (out []*model.ArchivalNetworkEvent) + + // NewStdlibResolver returns a possibly-trace-ware system resolver. + NewStdlibResolver(logger model.DebugLogger) model.Resolver + + // NewUDPListener implements model.MeasuringNetwork. + NewUDPListener() model.UDPListener + + // QUICHandshakes collects all the QUIC handshake results collected so far. + QUICHandshakes() (out []*model.ArchivalTLSOrQUICHandshakeResult) + + // TCPConnects collects all the TCP connect results collected so far. + TCPConnects() (out []*model.ArchivalTCPConnectResult) + + // TLSHandshakes collects all the TLS handshake results collected so far. + TLSHandshakes() (out []*model.ArchivalTLSOrQUICHandshakeResult) + + // Tags returns the trace tags. + Tags() []string + + // TimeSince is equivalent to Trace.TimeNow().Sub(t0). + TimeSince(t0 time.Time) time.Duration + + // ZeroTime returns the "zero" time of this trace. + ZeroTime() time.Time +} diff --git a/pkg/engine/experiment_integration_test.go b/pkg/engine/experiment_integration_test.go index b2a732f5..dbc309e9 100644 --- a/pkg/engine/experiment_integration_test.go +++ b/pkg/engine/experiment_integration_test.go @@ -13,6 +13,7 @@ import ( "testing" "github.com/ooni/probe-engine/pkg/model" + "github.com/ooni/probe-engine/pkg/registry" ) func TestCreateAll(t *testing.T) { @@ -21,6 +22,12 @@ func TestCreateAll(t *testing.T) { } sess := newSessionForTesting(t) defer sess.Close() + + // Since https://github.com/ooni/probe-cli/pull/1355, some experiments are disabled + // by default and we need an environment variable to instantiate them + os.Setenv(registry.OONI_FORCE_ENABLE_EXPERIMENT, "1") + defer os.Unsetenv(registry.OONI_FORCE_ENABLE_EXPERIMENT) + for _, name := range AllExperiments() { builder, err := sess.NewExperimentBuilder(name) if err != nil { diff --git a/pkg/engine/experimentbuilder.go b/pkg/engine/experimentbuilder.go index daa8d1fa..7a8f4aff 100644 --- a/pkg/engine/experimentbuilder.go +++ b/pkg/engine/experimentbuilder.go @@ -62,7 +62,7 @@ func (b *experimentBuilder) NewExperiment() model.Experiment { // newExperimentBuilder creates a new experimentBuilder instance. func newExperimentBuilder(session *Session, name string) (*experimentBuilder, error) { - factory, err := registry.NewFactory(name) + factory, err := registry.NewFactory(name, session.kvStore, session.logger) if err != nil { return nil, err } diff --git a/pkg/engine/inputloader.go b/pkg/engine/inputloader.go index 7fa6ae91..23ee2212 100644 --- a/pkg/engine/inputloader.go +++ b/pkg/engine/inputloader.go @@ -219,6 +219,12 @@ func StaticBareInputForExperiment(name string) ([]string, error) { // Implementation note: we may be called from pkg/oonimkall // with a non-canonical experiment name, so we need to convert // the experiment name to be canonical before proceeding. + // + // TODO(https://github.com/ooni/probe/issues/1390): serve DNSCheck + // inputs using richer input (aka check-in v2). + // + // TODO(https://github.com/ooni/probe/issues/2557): server STUNReachability + // inputs using richer input (aka check-in v2). switch registry.CanonicalizeExperimentName(name) { case "dnscheck": return dnsCheckDefaultInput, nil diff --git a/pkg/engine/session.go b/pkg/engine/session.go index cd95689c..c034d54e 100644 --- a/pkg/engine/session.go +++ b/pkg/engine/session.go @@ -11,7 +11,6 @@ import ( "sync/atomic" "github.com/ooni/probe-engine/pkg/bytecounter" - "github.com/ooni/probe-engine/pkg/checkincache" "github.com/ooni/probe-engine/pkg/enginelocate" "github.com/ooni/probe-engine/pkg/enginenetx" "github.com/ooni/probe-engine/pkg/engineresolver" @@ -19,7 +18,6 @@ import ( "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/platform" "github.com/ooni/probe-engine/pkg/probeservices" - "github.com/ooni/probe-engine/pkg/registry" "github.com/ooni/probe-engine/pkg/runtimex" "github.com/ooni/probe-engine/pkg/tunnel" "github.com/ooni/probe-engine/pkg/version" @@ -406,16 +404,6 @@ var ErrAlreadyUsingProxy = errors.New( // for the experiment with the given name, or an error if // there's no such experiment with the given name func (s *Session) NewExperimentBuilder(name string) (model.ExperimentBuilder, error) { - name = registry.CanonicalizeExperimentName(name) - switch { - case name == "web_connectivity" && checkincache.GetFeatureFlag(s.kvStore, "webconnectivity_0.5"): - // use LTE rather than the normal webconnectivity when the - // feature flag has been set through the check-in API - s.Logger().Infof("using webconnectivity LTE") - name = "web_connectivity@v0.5" - default: - // nothing - } eb, err := newExperimentBuilder(s, name) if err != nil { return nil, err diff --git a/pkg/experiment/echcheck/handshake.go b/pkg/experiment/echcheck/handshake.go index c50a0e0e..7b47a403 100644 --- a/pkg/experiment/echcheck/handshake.go +++ b/pkg/experiment/echcheck/handshake.go @@ -8,6 +8,7 @@ import ( "time" "github.com/apex/log" + "github.com/ooni/probe-engine/pkg/logx" "github.com/ooni/probe-engine/pkg/measurexlite" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/netxlite" @@ -16,11 +17,13 @@ import ( const echExtensionType uint16 = 0xfe0d -func handshake(ctx context.Context, conn net.Conn, zeroTime time.Time, address string, sni string) *model.ArchivalTLSOrQUICHandshakeResult { - return handshakeWithExtension(ctx, conn, zeroTime, address, sni, []utls.TLSExtension{}) +func handshake(ctx context.Context, conn net.Conn, zeroTime time.Time, + address string, sni string, logger model.Logger) *model.ArchivalTLSOrQUICHandshakeResult { + return handshakeWithExtension(ctx, conn, zeroTime, address, sni, []utls.TLSExtension{}, logger) } -func handshakeWithEch(ctx context.Context, conn net.Conn, zeroTime time.Time, address string, sni string) *model.ArchivalTLSOrQUICHandshakeResult { +func handshakeWithEch(ctx context.Context, conn net.Conn, zeroTime time.Time, + address string, sni string, logger model.Logger) *model.ArchivalTLSOrQUICHandshakeResult { payload, err := generateGreaseExtension(rand.Reader) if err != nil { panic("failed to generate grease ECH: " + err.Error()) @@ -31,18 +34,28 @@ func handshakeWithEch(ctx context.Context, conn net.Conn, zeroTime time.Time, ad utlsEchExtension.Id = echExtensionType utlsEchExtension.Data = payload - return handshakeWithExtension(ctx, conn, zeroTime, address, sni, []utls.TLSExtension{&utlsEchExtension}) + return handshakeWithExtension(ctx, conn, zeroTime, address, sni, []utls.TLSExtension{&utlsEchExtension}, logger) } -func handshakeWithExtension(ctx context.Context, conn net.Conn, zeroTime time.Time, address string, sni string, extensions []utls.TLSExtension) *model.ArchivalTLSOrQUICHandshakeResult { +func handshakeMaybePrintWithECH(doprint bool) string { + if doprint { + return "WithECH" + } + return "" +} + +func handshakeWithExtension(ctx context.Context, conn net.Conn, zeroTime time.Time, address string, sni string, + extensions []utls.TLSExtension, logger model.Logger) *model.ArchivalTLSOrQUICHandshakeResult { tlsConfig := genTLSConfig(sni) handshakerConstructor := newHandshakerWithExtensions(extensions) tracedHandshaker := handshakerConstructor(log.Log, &utls.HelloFirefox_Auto) + ol := logx.NewOperationLogger(logger, "echcheck: TLSHandshake%s", handshakeMaybePrintWithECH(len(extensions) > 0)) start := time.Now() maybeTLSConn, err := tracedHandshaker.Handshake(ctx, conn, tlsConfig) finish := time.Now() + ol.Stop(err) connState := netxlite.MaybeTLSConnectionState(maybeTLSConn) return measurexlite.NewArchivalTLSOrQUICHandshakeResult(0, start.Sub(zeroTime), "tcp", address, tlsConfig, diff --git a/pkg/experiment/echcheck/handshake_test.go b/pkg/experiment/echcheck/handshake_test.go index 951916d9..ddfcbc72 100644 --- a/pkg/experiment/echcheck/handshake_test.go +++ b/pkg/experiment/echcheck/handshake_test.go @@ -9,6 +9,8 @@ import ( "net/url" "testing" "time" + + "github.com/ooni/probe-engine/pkg/model" ) func TestHandshake(t *testing.T) { @@ -31,7 +33,7 @@ func TestHandshake(t *testing.T) { t.Fatal(err) } - result := handshakeWithEch(ctx, conn, time.Now(), parsed.Host, "example.org") + result := handshakeWithEch(ctx, conn, time.Now(), parsed.Host, "crypto.cloudflare.com", model.DiscardLogger) if result == nil { t.Fatal("expected result") } diff --git a/pkg/experiment/echcheck/measure.go b/pkg/experiment/echcheck/measure.go index fb5adc91..cf30bba4 100644 --- a/pkg/experiment/echcheck/measure.go +++ b/pkg/experiment/echcheck/measure.go @@ -7,6 +7,7 @@ import ( "net/url" "time" + "github.com/ooni/probe-engine/pkg/logx" "github.com/ooni/probe-engine/pkg/measurexlite" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/netxlite" @@ -14,9 +15,9 @@ import ( ) const ( - testName = "echcheck" - testVersion = "0.1.1" - defaultDomain = "https://example.org" + testName = "echcheck" + testVersion = "0.1.2" + defaultURL = "https://crypto.cloudflare.com/cdn-cgi/trace" ) var ( @@ -54,7 +55,7 @@ func (m *Measurer) Run( args *model.ExperimentArgs, ) error { if args.Measurement.Input == "" { - args.Measurement.Input = defaultDomain + args.Measurement.Input = defaultURL } parsed, err := url.Parse(string(args.Measurement.Input)) if err != nil { @@ -65,9 +66,11 @@ func (m *Measurer) Run( } // 1. perform a DNSLookup + ol := logx.NewOperationLogger(args.Session.Logger(), "echcheck: DNSLookup[%s] %s", m.config.resolverURL(), parsed.Host) trace := measurexlite.NewTrace(0, args.Measurement.MeasurementStartTimeSaved) resolver := trace.NewParallelDNSOverHTTPSResolver(args.Session.Logger(), m.config.resolverURL()) addrs, err := resolver.LookupHost(ctx, parsed.Host) + ol.Stop(err) if err != nil { return err } @@ -75,13 +78,17 @@ func (m *Measurer) Run( address := net.JoinHostPort(addrs[0], "443") // 2. Set up TCP connections + ol = logx.NewOperationLogger(args.Session.Logger(), "echcheck: TCPConnect#1 %s", address) var dialer net.Dialer conn, err := dialer.DialContext(ctx, "tcp", address) + ol.Stop(err) if err != nil { return netxlite.NewErrWrapper(netxlite.ClassifyGenericError, netxlite.ConnectOperation, err) } + ol = logx.NewOperationLogger(args.Session.Logger(), "echcheck: TCPConnect#2 %s", address) conn2, err := dialer.DialContext(ctx, "tcp", address) + ol.Stop(err) if err != nil { return netxlite.NewErrWrapper(netxlite.ClassifyGenericError, netxlite.ConnectOperation, err) } @@ -93,11 +100,25 @@ func (m *Measurer) Run( defer cancel() go func() { - controlChannel <- *handshake(ctx, conn, args.Measurement.MeasurementStartTimeSaved, address, parsed.Host) + controlChannel <- *handshake( + ctx, + conn, + args.Measurement.MeasurementStartTimeSaved, + address, + parsed.Host, + args.Session.Logger(), + ) }() go func() { - targetChannel <- *handshakeWithEch(ctx, conn2, args.Measurement.MeasurementStartTimeSaved, address, parsed.Host) + targetChannel <- *handshakeWithEch( + ctx, + conn2, + args.Measurement.MeasurementStartTimeSaved, + address, + parsed.Host, + args.Session.Logger(), + ) }() control := <-controlChannel diff --git a/pkg/experiment/echcheck/measure_test.go b/pkg/experiment/echcheck/measure_test.go index 52962441..b57248a7 100644 --- a/pkg/experiment/echcheck/measure_test.go +++ b/pkg/experiment/echcheck/measure_test.go @@ -4,9 +4,9 @@ import ( "context" "testing" - "github.com/apex/log" - "github.com/ooni/probe-engine/pkg/legacy/mockable" + "github.com/ooni/probe-engine/pkg/mocks" "github.com/ooni/probe-engine/pkg/model" + "github.com/ooni/probe-engine/pkg/netemx" ) func TestNewExperimentMeasurer(t *testing.T) { @@ -14,84 +14,117 @@ func TestNewExperimentMeasurer(t *testing.T) { if measurer.ExperimentName() != "echcheck" { t.Fatal("unexpected name") } - if measurer.ExperimentVersion() != "0.1.1" { + if measurer.ExperimentVersion() != "0.1.2" { t.Fatal("unexpected version") } } -func TestMeasurerMeasureWithInvalidInput(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - cancel() // immediately cancel the context - sess := &mockable.Session{MockableLogger: log.Log} - callbacks := model.NewPrinterCallbacks(sess.Logger()) - measurer := NewExperimentMeasurer(Config{}) - measurement := &model.Measurement{ - Input: "http://example.org", - } - args := &model.ExperimentArgs{ - Callbacks: callbacks, - Measurement: measurement, - Session: sess, - } - err := measurer.Run( - ctx, - args, - ) - if err == nil { - t.Fatal("expected an error here") +// qaenv creates a [netemx.QAEnv] with a single crypto.cloudflare.com test server and a DoH server. +func qaenv() *netemx.QAEnv { + cfg := []*netemx.ScenarioDomainAddresses{ + { + Domains: []string{"crypto.cloudflare.com"}, + Addresses: []string{"130.192.91.7"}, + Role: netemx.ScenarioRoleWebServer, + WebServerFactory: netemx.ExampleWebPageHandlerFactory(), + }, + { + Domains: []string{"mozilla.cloudflare-dns.com"}, + Addresses: []string{"130.192.91.13"}, + Role: netemx.ScenarioRolePublicDNS, + }, } + return netemx.MustNewScenario(cfg) } -func TestMeasurerMeasureWithInvalidInput2(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - cancel() // immediately cancel the context - sess := &mockable.Session{MockableLogger: log.Log} - callbacks := model.NewPrinterCallbacks(sess.Logger()) +func TestMeasurerMeasureWithCancelledContext(t *testing.T) { + // create QAEnv + env := qaenv() + defer env.Close() + + env.Do(func() { + ctx, cancel := context.WithCancel(context.Background()) + cancel() // immediately cancel the context + + // create measurer + measurer := NewExperimentMeasurer(Config{}) + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(model.DiscardLogger), + Measurement: &model.Measurement{}, + Session: &mocks.Session{MockLogger: func() model.Logger { return model.DiscardLogger }}, + } + + // run measurement + err := measurer.Run(ctx, args) + if err == nil { + t.Fatal("expected an error here") + } + if err.Error() != "interrupted" { + t.Fatal("unexpected error type") + } + }) + +} + +func TestMeasurerMeasureWithInvalidInput(t *testing.T) { + // create QAEnv + env := qaenv() + defer env.Close() + + // create measurer measurer := NewExperimentMeasurer(Config{}) - measurement := &model.Measurement{ - // leading space to test url.Parse failure - Input: " https://example.org", - } args := &model.ExperimentArgs{ - Callbacks: callbacks, - Measurement: measurement, - Session: sess, + Callbacks: model.NewPrinterCallbacks(model.DiscardLogger), + Measurement: &model.Measurement{ + // leading space to test url.Parse failure + Input: " https://crypto.cloudflare.com/cdn-cgi/trace", + }, + Session: &mocks.Session{MockLogger: func() model.Logger { return model.DiscardLogger }}, } - err := measurer.Run( - ctx, - args, - ) + // run measurement + err := measurer.Run(context.Background(), args) if err == nil { t.Fatal("expected an error here") } + if err.Error() != "input is not an URL" { + t.Fatal("unexpected error type") + } } -func TestMeasurementSuccess(t *testing.T) { +func TestMeasurementSuccessRealWorld(t *testing.T) { if testing.Short() { + // this test uses the real internet so we want to skip this in short mode t.Skip("skip test in short mode") } - sess := &mockable.Session{MockableLogger: log.Log} - callbacks := model.NewPrinterCallbacks(sess.Logger()) + // create measurer measurer := NewExperimentMeasurer(Config{}) + msrmnt := &model.Measurement{} args := &model.ExperimentArgs{ - Callbacks: callbacks, - Measurement: &model.Measurement{}, - Session: sess, + Callbacks: model.NewPrinterCallbacks(model.DiscardLogger), + Measurement: msrmnt, + Session: &mocks.Session{MockLogger: func() model.Logger { return model.DiscardLogger }}, } - err := measurer.Run( - context.Background(), - args, - ) + + // run measurement + err := measurer.Run(context.Background(), args) if err != nil { t.Fatal("unexpected error: ", err) } + // check results summary, err := measurer.GetSummaryKeys(&model.Measurement{}) if err != nil { - t.Fatal(err) + t.Fatal("unexpected error: ", err) } if summary.(SummaryKeys).IsAnomaly != false { t.Fatal("expected false") } + tk := msrmnt.TestKeys.(TestKeys) + if tk.Control.Failure != nil { + t.Fatal("unexpected control failure:", *tk.Control.Failure) + } + if tk.Target.Failure != nil { + t.Fatal("unexpected target failure:", *tk.Target.Failure) + } } diff --git a/pkg/experiment/riseupvpn/riseupvpn.go b/pkg/experiment/riseupvpn/riseupvpn.go index 48d57739..4e6120e6 100644 --- a/pkg/experiment/riseupvpn/riseupvpn.go +++ b/pkg/experiment/riseupvpn/riseupvpn.go @@ -6,36 +6,38 @@ package riseupvpn import ( "context" "encoding/json" - "errors" "time" "github.com/ooni/probe-engine/pkg/experiment/urlgetter" - "github.com/ooni/probe-engine/pkg/legacy/tracex" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/netxlite" + "github.com/ooni/probe-engine/pkg/progress" ) const ( testName = "riseupvpn" - testVersion = "0.2.0" + testVersion = "0.3.0" eipServiceURL = "https://api.black.riseup.net:443/3/config/eip-service.json" providerURL = "https://riseup.net/provider.json" geoServiceURL = "https://api.black.riseup.net:9001/json" tcpConnect = "tcpconnect://" ) -// EipService is the main JSON object of eip-service.json. -type EipService struct { +// EIPServiceV3 is the main JSON object returned by eip-service.json. +type EIPServiceV3 struct { Gateways []GatewayV3 } +// CapabilitiesV3 is a list of transports a gateway supports +type CapabilitiesV3 struct { + Transport []TransportV3 +} + // GatewayV3 describes a gateway. type GatewayV3 struct { - Capabilities struct { - Transport []TransportV3 - } - Host string - IPAddress string `json:"ip_address"` + Capabilities CapabilitiesV3 + Host string + IPAddress string `json:"ip_address"` } // TransportV3 describes a transport. @@ -61,21 +63,15 @@ type Config struct { // TestKeys contains riseupvpn test keys. type TestKeys struct { urlgetter.TestKeys - APIFailure *string `json:"api_failure"` - APIStatus string `json:"api_status"` - CACertStatus bool `json:"ca_cert_status"` - FailingGateways []GatewayConnection `json:"failing_gateways"` - TransportStatus map[string]string `json:"transport_status"` + APIFailures []string `json:"api_failures"` + CACertStatus bool `json:"ca_cert_status"` } // NewTestKeys creates new riseupvpn TestKeys. func NewTestKeys() *TestKeys { return &TestKeys{ - APIFailure: nil, - APIStatus: "ok", - CACertStatus: true, - FailingGateways: nil, - TransportStatus: nil, + APIFailures: []string{}, + CACertStatus: true, } } @@ -86,12 +82,8 @@ func (tk *TestKeys) UpdateProviderAPITestKeys(v urlgetter.MultiOutput) { tk.Requests = append(tk.Requests, v.TestKeys.Requests...) tk.TCPConnect = append(tk.TCPConnect, v.TestKeys.TCPConnect...) tk.TLSHandshakes = append(tk.TLSHandshakes, v.TestKeys.TLSHandshakes...) - if tk.APIStatus != "ok" { - return // we already flipped the state - } if v.TestKeys.Failure != nil { - tk.APIStatus = "blocked" - tk.APIFailure = v.TestKeys.Failure + tk.APIFailures = append(tk.APIFailures, *v.TestKeys.Failure) return } } @@ -102,42 +94,6 @@ func (tk *TestKeys) UpdateProviderAPITestKeys(v urlgetter.MultiOutput) { func (tk *TestKeys) AddGatewayConnectTestKeys(v urlgetter.MultiOutput, transportType string) { tk.NetworkEvents = append(tk.NetworkEvents, v.TestKeys.NetworkEvents...) tk.TCPConnect = append(tk.TCPConnect, v.TestKeys.TCPConnect...) - for _, tcpConnect := range v.TestKeys.TCPConnect { - if !tcpConnect.Status.Success { - gatewayConnection := newGatewayConnection(tcpConnect, transportType) - tk.FailingGateways = append(tk.FailingGateways, *gatewayConnection) - } - } -} - -func (tk *TestKeys) updateTransportStatus(openvpnGatewayCount, obfs4GatewayCount int) { - failingOpenvpnGateways, failingObfs4Gateways := 0, 0 - for _, gw := range tk.FailingGateways { - if gw.TransportType == "openvpn" { - failingOpenvpnGateways++ - } else if gw.TransportType == "obfs4" { - failingObfs4Gateways++ - } - } - if failingOpenvpnGateways < openvpnGatewayCount { - tk.TransportStatus["openvpn"] = "ok" - } else { - tk.TransportStatus["openvpn"] = "blocked" - } - if failingObfs4Gateways < obfs4GatewayCount { - tk.TransportStatus["obfs4"] = "ok" - } else { - tk.TransportStatus["obfs4"] = "blocked" - } -} - -func newGatewayConnection( - tcpConnect tracex.TCPConnectEntry, transportType string) *GatewayConnection { - return &GatewayConnection{ - IP: tcpConnect.IP, - Port: tcpConnect.Port, - TransportType: transportType, - } } // AddCACertFetchTestKeys adds generic urlgetter.Get() testKeys to riseupvpn specific test keys @@ -147,11 +103,6 @@ func (tk *TestKeys) AddCACertFetchTestKeys(testKeys urlgetter.TestKeys) { tk.Requests = append(tk.Requests, testKeys.Requests...) tk.TCPConnect = append(tk.TCPConnect, testKeys.TCPConnect...) tk.TLSHandshakes = append(tk.TLSHandshakes, testKeys.TLSHandshakes...) - if testKeys.Failure != nil { - tk.APIStatus = "blocked" - tk.APIFailure = tk.Failure - tk.CACertStatus = false - } } // Measurer performs the measurement. @@ -204,20 +155,31 @@ func (m Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { FailOnHTTPError: true, }}, } - for entry := range multi.CollectOverall(ctx, inputs, 0, 50, "riseupvpn", callbacks) { + + // Q: why returning early if we cannot fetch the CA or the config? Cannot we just + // disable certificate verification and fetch the config? + // + // A: I do not feel comfortable with fetching without verying the certificates since + // this means the experiment could be person-in-the-middled and forced to perform TCP + // connect to arbitrary hosts, which maybe is harmless but still a bummer. + // + // TODO(https://github.com/ooni/probe/issues/2559): solve this problem by serving the + // correct CA and the endpoints to probes using check-in v2 (aka richer input). + + callbacksStage1 := progress.NewScaler(callbacks, 0, 0.25) + for entry := range multi.Collect(ctx, inputs, "riseupvpn", callbacksStage1) { tk := entry.TestKeys testkeys.AddCACertFetchTestKeys(tk) if tk.Failure != nil { - // TODO(bassosimone,cyberta): should we update the testkeys - // in this case (e.g., APIFailure?) - // See https://github.com/ooni/probe/issues/1432. + testkeys.CACertStatus = false + testkeys.APIFailures = append(testkeys.APIFailures, *tk.Failure) + // Note well: returning nil here causes the measurement to be submitted. return nil } if ok := certPool.AppendCertsFromPEM([]byte(tk.HTTPResponseBody)); !ok { testkeys.CACertStatus = false - testkeys.APIStatus = "blocked" - errorValue := "invalid_ca" - testkeys.APIFailure = &errorValue + testkeys.APIFailures = append(testkeys.APIFailures, "invalid_ca") + // Note well: returning nil here causes the measurement to be submitted. return nil } } @@ -242,20 +204,25 @@ func (m Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { FailOnHTTPError: true, }}, } - for entry := range multi.CollectOverall(ctx, inputs, 1, 50, "riseupvpn", callbacks) { + + callbacksStage2 := progress.NewScaler(callbacks, 0.25, 0.5) + for entry := range multi.Collect(ctx, inputs, "riseupvpn", callbacksStage2) { testkeys.UpdateProviderAPITestKeys(entry) + tk := entry.TestKeys + if tk.Failure != nil { + // Note well: returning nil here causes the measurement to be submitted. + return nil + } } // test gateways now - testkeys.TransportStatus = map[string]string{} gateways := parseGateways(testkeys) openvpnEndpoints := generateMultiInputs(gateways, "openvpn") obfs4Endpoints := generateMultiInputs(gateways, "obfs4") - overallCount := 1 + len(inputs) + len(openvpnEndpoints) + len(obfs4Endpoints) // measure openvpn in parallel - for entry := range multi.CollectOverall( - ctx, openvpnEndpoints, 1+len(inputs), overallCount, "riseupvpn", callbacks) { + callbacksStage3 := progress.NewScaler(callbacks, 0.5, 0.75) + for entry := range multi.Collect(ctx, openvpnEndpoints, "riseupvpn", callbacksStage3) { testkeys.AddGatewayConnectTestKeys(entry, "openvpn") } @@ -263,13 +230,12 @@ func (m Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { // TODO(bassosimone): when urlgetter is able to do obfs4 handshakes, here // can possibly also test for the obfs4 handshake. // See https://github.com/ooni/probe/issues/1463. - for entry := range multi.CollectOverall( - ctx, obfs4Endpoints, 1+len(inputs)+len(openvpnEndpoints), overallCount, "riseupvpn", callbacks) { + callbacksStage4 := progress.NewScaler(callbacks, 0.75, 1) + for entry := range multi.Collect(ctx, obfs4Endpoints, "riseupvpn", callbacksStage4) { testkeys.AddGatewayConnectTestKeys(entry, "obfs4") } - // set transport status based on gateway test results - testkeys.updateTransportStatus(len(openvpnEndpoints), len(obfs4Endpoints)) + // Note well: returning nil here causes the measurement to be submitted. return nil } @@ -304,7 +270,7 @@ func parseGateways(testKeys *TestKeys) []GatewayV3 { // TODO(bassosimone,cyberta): is it reasonable that we discard // the error when the JSON we fetched cannot be parsed? // See https://github.com/ooni/probe/issues/1432 - eipService, err := DecodeEIP3(string(requestEntry.Response.Body)) + eipService, err := DecodeEIPServiceV3(string(requestEntry.Response.Body)) if err == nil { return eipService.Gateways } @@ -313,9 +279,9 @@ func parseGateways(testKeys *TestKeys) []GatewayV3 { return nil } -// DecodeEIP3 decodes eip-service.json version 3 -func DecodeEIP3(body string) (*EipService, error) { - var eip EipService +// DecodeEIPServiceV3 decodes eip-service.json version 3 +func DecodeEIPServiceV3(body string) (*EIPServiceV3, error) { + var eip EIPServiceV3 err := json.Unmarshal([]byte(body), &eip) if err != nil { return nil, err @@ -333,28 +299,11 @@ func NewExperimentMeasurer(config Config) model.ExperimentMeasurer { // Note that this structure is part of the ABI contract with ooniprobe // therefore we should be careful when changing it. type SummaryKeys struct { - APIBlocked bool `json:"api_blocked"` - ValidCACert bool `json:"valid_ca_cert"` - FailingGateways int `json:"failing_gateways"` - TransportStatus map[string]string `json:"transport_status"` - IsAnomaly bool `json:"-"` + IsAnomaly bool `json:"-"` } // GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys. func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) { sk := SummaryKeys{IsAnomaly: false} - tk, ok := measurement.TestKeys.(*TestKeys) - if !ok { - return sk, errors.New("invalid test keys type") - } - sk.APIBlocked = tk.APIStatus != "ok" - sk.ValidCACert = tk.CACertStatus - sk.FailingGateways = len(tk.FailingGateways) - sk.TransportStatus = tk.TransportStatus - // Note: the order in the following OR chains matter: TransportStatus - // is nil if APIBlocked or !CACertStatus - sk.IsAnomaly = (sk.APIBlocked || !tk.CACertStatus || - tk.TransportStatus["openvpn"] == "blocked" || - tk.TransportStatus["obfs4"] == "blocked") return sk, nil } diff --git a/pkg/experiment/riseupvpn/riseupvpn_test.go b/pkg/experiment/riseupvpn/riseupvpn_test.go index df6ae8c3..d78b8579 100644 --- a/pkg/experiment/riseupvpn/riseupvpn_test.go +++ b/pkg/experiment/riseupvpn/riseupvpn_test.go @@ -2,7 +2,6 @@ package riseupvpn_test import ( "context" - "encoding/json" "errors" "fmt" "io" @@ -11,11 +10,10 @@ import ( "testing" "github.com/apex/log" - "github.com/google/go-cmp/cmp" "github.com/ooni/probe-engine/pkg/experiment/riseupvpn" "github.com/ooni/probe-engine/pkg/experiment/urlgetter" - "github.com/ooni/probe-engine/pkg/legacy/mockable" "github.com/ooni/probe-engine/pkg/legacy/tracex" + "github.com/ooni/probe-engine/pkg/mocks" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/netxlite" ) @@ -142,8 +140,9 @@ const ( "serial": 3, "version": 3 }` - geoservice = `{"ip":"51.15.0.88","cc":"NL","city":"Haarlem","lat":52.381,"lon":4.6275,"gateways":["test1.riseup.net","test2.riseup.net"]}` - cacert = `-----BEGIN CERTIFICATE----- + geoservice = `{"ip":"51.15.0.88","cc":"NL","city":"Haarlem","lat":52.381,"lon":4.6275,"gateways":["test1.riseup.net","test2.riseup.net"]}` + geoService_update = `{"ip":"51.15.0.88","cc":"NL","city":"Haarlem","lat":52.381,"lon":4.6275,"gateways":["test1.riseup.net","test2.riseup.net"], "sortedGateways": [{ "host": "test1.riseup.net", "fullness": 0.2, "overload": false }, { "host": "test2.riseup.net", "fullness": 0.9, "overload": true }]}` + cacert = `-----BEGIN CERTIFICATE----- MIIFjTCCA3WgAwIBAgIBATANBgkqhkiG9w0BAQ0FADBZMRgwFgYDVQQKDA9SaXNl dXAgTmV0d29ya3MxGzAZBgNVBAsMEmh0dHBzOi8vcmlzZXVwLm5ldDEgMB4GA1UE AwwXUmlzZXVwIE5ldHdvcmtzIFJvb3QgQ0EwHhcNMTQwNDI4MDAwMDAwWhcNMjQw @@ -184,9 +183,10 @@ UN9SaWRlWKSdP4haujnzCoJbM7dU9bjvlGZNyXEekgeT0W2qFeGGp+yyUWw8tNsp providerurl = "https://riseup.net/provider.json" geoserviceurl = "https://api.black.riseup.net:9001/json" cacerturl = "https://black.riseup.net/ca.crt" - openvpnurl1 = "tcpconnect://234.345.234.345:443" - openvpnurl2 = "tcpconnect://123.456.123.456:443" + openvpnurl1 = "tcpconnect://234.345.234.345:443" // "Seattle" + openvpnurl2 = "tcpconnect://123.456.123.456:443" // "Paris" obfs4url1 = "tcpconnect://234.345.234.345:23042" + obfs4url2 = "tcpconnect://123.456.123.456:444" ) var RequestResponse = map[string]string{ @@ -197,6 +197,7 @@ var RequestResponse = map[string]string{ openvpnurl1: "", openvpnurl2: "", obfs4url1: "", + obfs4url2: "", } func TestNewExperimentMeasurer(t *testing.T) { @@ -204,20 +205,22 @@ func TestNewExperimentMeasurer(t *testing.T) { if measurer.ExperimentName() != "riseupvpn" { t.Fatal("unexpected name") } - if measurer.ExperimentVersion() != "0.2.0" { + if measurer.ExperimentVersion() != "0.3.0" { t.Fatal("unexpected version") } } func TestGood(t *testing.T) { + // the gateaway openvpnurl2 is filtered out, since it doesn't support additionally obfs4 measurement := runDefaultMockTest(t, generateDefaultMockGetter(map[string]bool{ cacerturl: true, eipserviceurl: true, providerurl: true, geoserviceurl: true, openvpnurl1: true, - openvpnurl2: true, + openvpnurl2: false, obfs4url1: true, + obfs4url2: false, })) tk := measurement.TestKeys.(*riseupvpn.TestKeys) @@ -230,23 +233,21 @@ func TestGood(t *testing.T) { if tk.Failure != nil { t.Fatal("unexpected Failure") } - if tk.APIFailure != nil { + if len(tk.APIFailures) != 0 { t.Fatal("unexpected ApiFailure") } - if tk.APIStatus != "ok" { - t.Fatal("unexpected ApiStatus") - } if tk.CACertStatus != true { t.Fatal("unexpected CaCertStatus") } - if tk.FailingGateways != nil { - t.Fatal("unexpected FailingGateways value") - } - if tk.TransportStatus == nil { - t.Fatal("unexpected nil TransportStatus struct ") + + hasOpenvpn1 := false + for _, tcpConnect := range tk.TCPConnect { + if tcpConnect.IP == "234.345.234.345" { + hasOpenvpn1 = true + } } - if tk.TransportStatus["openvpn"] != "ok" { - t.Fatal("unexpected openvpn transport status") + if !hasOpenvpn1 { + t.Fatalf("Gateway tests should run %t", hasOpenvpn1) } } @@ -257,7 +258,7 @@ func TestUpdateWithMixedResults(t *testing.T) { tk.UpdateProviderAPITestKeys(urlgetter.MultiOutput{ Input: urlgetter.MultiInput{ Config: urlgetter.Config{Method: "GET"}, - Target: "https://api.black.riseup.net:443/3/config/eip-service.json", + Target: "https://riseup.net/provider.json", }, TestKeys: urlgetter.TestKeys{ HTTPResponseStatus: 200, @@ -266,9 +267,17 @@ func TestUpdateWithMixedResults(t *testing.T) { tk.UpdateProviderAPITestKeys(urlgetter.MultiOutput{ Input: urlgetter.MultiInput{ Config: urlgetter.Config{Method: "GET"}, - Target: "https://riseup.net/provider.json", + Target: "https://api.black.riseup.net:443/3/config/eip-service.json", }, TestKeys: urlgetter.TestKeys{ + Requests: []model.ArchivalHTTPRequestResult{ + { + Request: model.ArchivalHTTPRequest{URL: "https://api.black.riseup.net:443/3/config/eip-service.json"}, + Failure: (func() *string { + s := "eof" + return &s + })(), + }}, FailedOperation: (func() *string { s := netxlite.HTTPRoundTripOperation return &s @@ -288,18 +297,10 @@ func TestUpdateWithMixedResults(t *testing.T) { HTTPResponseStatus: 200, }, }) - if tk.APIStatus != "blocked" { - t.Fatal("ApiStatus should be blocked") - } - if *tk.APIFailure != netxlite.FailureEOFError { + + if len(tk.APIFailures) != 1 || tk.APIFailures[0] != netxlite.FailureEOFError { t.Fatal("invalid ApiFailure") } - if tk.FailingGateways != nil { - t.Fatal("invalid FailingGateways") - } - if tk.TransportStatus != nil { - t.Fatal("invalid TransportStatus") - } } func TestInvalidCaCert(t *testing.T) { @@ -311,6 +312,7 @@ func TestInvalidCaCert(t *testing.T) { openvpnurl1: "", openvpnurl2: "", obfs4url1: "", + obfs4url2: "", } measurer := riseupvpn.Measurer{ Config: riseupvpn.Config{}, @@ -319,13 +321,17 @@ func TestInvalidCaCert(t *testing.T) { eipserviceurl: true, providerurl: true, geoserviceurl: true, - openvpnurl1: false, - openvpnurl2: true, + openvpnurl1: true, + openvpnurl2: false, // filtered out, no obfs4 support obfs4url1: true, + obfs4url2: false, // filtered out }), } ctx := context.Background() - sess := &mockable.Session{MockableLogger: log.Log} + sess := &mocks.Session{MockLogger: func() model.Logger { + return model.DiscardLogger + }} + measurement := new(model.Measurement) callbacks := model.NewPrinterCallbacks(log.Log) args := &model.ExperimentArgs{ @@ -341,15 +347,6 @@ func TestInvalidCaCert(t *testing.T) { if tk.CACertStatus == true { t.Fatal("unexpected CaCertStatus") } - if tk.APIStatus != "blocked" { - t.Fatal("ApiStatus should be blocked") - } - if tk.FailingGateways != nil { - t.Fatal("invalid FailingGateways") - } - if tk.TransportStatus != nil { - t.Fatal("invalid TransportStatus") - } } func TestFailureCaCertFetch(t *testing.T) { @@ -367,21 +364,17 @@ func TestFailureCaCertFetch(t *testing.T) { if tk.CACertStatus != false { t.Fatal("invalid CACertStatus ") } - if tk.APIStatus != "blocked" { - t.Fatal("invalid ApiStatus") - } - if tk.APIFailure != nil { - t.Fatal("ApiFailure should be null") - } - if len(tk.Requests) > 1 { - t.Fatal("Unexpected requests") + if len(tk.APIFailures) != 1 || tk.APIFailures[0] != io.EOF.Error() { + t.Fatal("APIFailures should not be empty", tk.APIFailures) } - if tk.FailingGateways != nil { - t.Fatal("invalid FailingGateways") + if len(tk.Requests) != 1 { + t.Fatal("Expected a single request in this case") } - if tk.TransportStatus != nil { - t.Fatal("invalid TransportStatus") + for _, tcpConnect := range tk.TCPConnect { + if tcpConnect.IP == openvpnurl1 || tcpConnect.IP == openvpnurl2 || tcpConnect.IP == obfs4url1 || tcpConnect.IP == obfs4url2 { + t.Fatal("No gateaway tests should be run if API fails") + } } } @@ -408,12 +401,8 @@ func TestFailureEipServiceBlocked(t *testing.T) { } } - if tk.APIStatus != "blocked" { - t.Fatal("invalid ApiStatus") - } - - if tk.APIFailure == nil { - t.Fatal("ApiFailure should not be null") + if len(tk.APIFailures) <= 0 { + t.Fatal("APIFailures should not be empty") } } @@ -440,12 +429,9 @@ func TestFailureProviderUrlBlocked(t *testing.T) { if tk.CACertStatus != true { t.Fatal("invalid CACertStatus ") } - if tk.APIStatus != "blocked" { - t.Fatal("invalid ApiStatus") - } - if tk.APIFailure == nil { - t.Fatal("ApiFailure should not be null") + if len(tk.APIFailures) <= 0 { + t.Fatal("APIFailures should not be empty") } } @@ -472,253 +458,52 @@ func TestFailureGeoIpServiceBlocked(t *testing.T) { } } - if tk.APIStatus != "blocked" { - t.Fatal("invalid ApiStatus") - } - - if tk.APIFailure == nil { - t.Fatal("ApiFailure should not be null") + if len(tk.APIFailures) <= 0 { + t.Fatal("APIFailures should not be empty") } } -func TestFailureGateway1(t *testing.T) { +func TestFailureGateway1TransportNOK(t *testing.T) { measurement := runDefaultMockTest(t, generateDefaultMockGetter(map[string]bool{ cacerturl: true, eipserviceurl: true, providerurl: true, geoserviceurl: true, - openvpnurl1: false, + openvpnurl1: false, // failed gateway openvpnurl2: true, obfs4url1: true, + obfs4url2: false, })) tk := measurement.TestKeys.(*riseupvpn.TestKeys) if tk.CACertStatus != true { t.Fatal("invalid CACertStatus ") } - if tk.FailingGateways == nil || len(tk.FailingGateways) != 1 { - t.Fatal("unexpected amount of failing gateways") - } - - gw := tk.FailingGateways[0] - if gw.IP != "234.345.234.345" { - t.Fatal("invalid failed gateway ip: " + fmt.Sprint(gw.IP)) - } - if gw.Port != 443 { - t.Fatal("invalid failed gateway port: " + fmt.Sprint(gw.Port)) - } - if gw.TransportType != "openvpn" { - t.Fatal("invalid failed transport type: " + fmt.Sprint(gw.TransportType)) - } - - if tk.APIStatus == "blocked" { - t.Fatal("invalid ApiStatus") - } - - if tk.APIFailure != nil { - t.Fatal("ApiFailure should be null") - } - - if tk.TransportStatus == nil || tk.TransportStatus["openvpn"] == "blocked" { - t.Fatal("invalid TransportStatus: " + fmt.Sprint(tk.TransportStatus)) - } - - if tk.TransportStatus == nil || tk.TransportStatus["obfs4"] == "blocked" { - t.Fatal("invalid TransportStatus: " + fmt.Sprint(tk.TransportStatus)) - } -} - -func TestFailureTransport(t *testing.T) { - measurement := runDefaultMockTest(t, generateDefaultMockGetter(map[string]bool{ - cacerturl: true, - eipserviceurl: true, - providerurl: true, - geoserviceurl: true, - openvpnurl1: false, - openvpnurl2: false, - obfs4url1: false, - })) - tk := measurement.TestKeys.(*riseupvpn.TestKeys) - - if tk.TransportStatus == nil || tk.TransportStatus["openvpn"] != "blocked" { - t.Fatal("invalid TransportStatus: " + fmt.Sprint(tk.TransportStatus)) - } - - if tk.TransportStatus == nil || tk.TransportStatus["obfs4"] != "blocked" { - t.Fatal("invalid TransportStatus: " + fmt.Sprint(tk.TransportStatus)) - } -} - -func TestMissingTransport(t *testing.T) { - eipService, err := riseupvpn.DecodeEIP3(eipservice) - if err != nil { - t.Fatal("Preconditions for the test are not met.") - } - - //remove obfs4 capability from 2. gateway so that our - //mock provider supports only openvpn - index := -1 - transports := eipService.Gateways[1].Capabilities.Transport - for i, transport := range transports { - if transport.Type == "obfs4" { - index = i - break + for _, tcpConnect := range tk.TCPConnect { + if !tcpConnect.Status.Success { + if tcpConnect.IP != "234.345.234.345" { + t.Fatal("invalid failed gateway ip: " + fmt.Sprint(tcpConnect.IP)) + } + if tcpConnect.Port != 443 { + t.Fatal("invalid failed gateway port: " + fmt.Sprint(tcpConnect.Port)) + } } } - if index == -1 { - t.Fatal("Preconditions for the test are not met. Default eipservice string should contain obfs4 transport.") - } - - transports[index] = transports[len(transports)-1] - transports = transports[:len(transports)-1] - eipService.Gateways[1].Capabilities.Transport = transports - eipservicejson, err := json.Marshal(eipservice) - if err != nil { - t.Fatal(err) - } - - requestResponseMap := map[string]string{ - eipserviceurl: string(eipservicejson), - providerurl: provider, - geoserviceurl: geoservice, - cacerturl: cacert, - openvpnurl1: "", - openvpnurl2: "", - obfs4url1: "", - } - measurer := riseupvpn.Measurer{ - Config: riseupvpn.Config{}, - Getter: generateMockGetter(requestResponseMap, map[string]bool{ - cacerturl: true, - eipserviceurl: true, - providerurl: true, - geoserviceurl: true, - openvpnurl1: true, - openvpnurl2: true, - obfs4url1: false, - }), - } - - ctx := context.Background() - sess := &mockable.Session{MockableLogger: log.Log} - measurement := new(model.Measurement) - callbacks := model.NewPrinterCallbacks(log.Log) - args := &model.ExperimentArgs{ - Callbacks: callbacks, - Measurement: measurement, - Session: sess, - } - err = measurer.Run(ctx, args) - if err != nil { - t.Fatal(err) - } - tk := measurement.TestKeys.(*riseupvpn.TestKeys) - if tk.TransportStatus == nil || tk.TransportStatus["openvpn"] != "blocked" { - t.Fatal("invalid TransportStatus: " + fmt.Sprint(tk.TransportStatus)) - } - - if _, found := tk.TransportStatus["obfs"]; found { - t.Fatal("invalid TransportStatus: " + fmt.Sprint(tk.TransportStatus)) + if len(tk.APIFailures) != 0 { + t.Fatal("APIFailures should be empty") } } -func TestSummaryKeysInvalidType(t *testing.T) { +func TestSummaryKeysAlwaysReturnIsAnomalyFalse(t *testing.T) { measurement := new(model.Measurement) m := &riseupvpn.Measurer{} - _, err := m.GetSummaryKeys(measurement) - if err.Error() != "invalid test keys type" { - t.Fatal("not the error we expected") + result, err := m.GetSummaryKeys(measurement) + if err != nil { + t.Fatal("GetSummaryKeys should never return an error") } -} - -func TestSummaryKeysWorksAsIntended(t *testing.T) { - tests := []struct { - tk riseupvpn.TestKeys - sk riseupvpn.SummaryKeys - }{{ - tk: riseupvpn.TestKeys{ - APIStatus: "blocked", - CACertStatus: true, - FailingGateways: nil, - TransportStatus: nil, - }, - sk: riseupvpn.SummaryKeys{ - APIBlocked: true, - ValidCACert: true, - IsAnomaly: true, - TransportStatus: nil, - FailingGateways: 0, - }, - }, { - tk: riseupvpn.TestKeys{ - APIStatus: "ok", - CACertStatus: false, - FailingGateways: nil, - TransportStatus: nil, - }, - sk: riseupvpn.SummaryKeys{ - ValidCACert: false, - IsAnomaly: true, - FailingGateways: 0, - TransportStatus: nil, - }, - }, { - tk: riseupvpn.TestKeys{ - APIStatus: "ok", - CACertStatus: true, - FailingGateways: []riseupvpn.GatewayConnection{{ - IP: "1.1.1.1", - Port: 443, - TransportType: "obfs4", - }}, - TransportStatus: map[string]string{ - "obfs4": "blocked", - "openvpn": "ok", - }, - }, - sk: riseupvpn.SummaryKeys{ - FailingGateways: 1, - IsAnomaly: true, - ValidCACert: true, - TransportStatus: map[string]string{ - "obfs4": "blocked", - "openvpn": "ok", - }, - }, - }, { - tk: riseupvpn.TestKeys{ - APIStatus: "ok", - CACertStatus: true, - FailingGateways: nil, - TransportStatus: map[string]string{ - "openvpn": "ok", - }, - }, - sk: riseupvpn.SummaryKeys{ - ValidCACert: true, - IsAnomaly: false, - FailingGateways: 0, - TransportStatus: map[string]string{ - "openvpn": "ok", - }, - }, - }, - } - for idx, tt := range tests { - t.Run(fmt.Sprintf("%d", idx), func(t *testing.T) { - m := &riseupvpn.Measurer{} - measurement := &model.Measurement{TestKeys: &tt.tk} - got, err := m.GetSummaryKeys(measurement) - if err != nil { - t.Fatal(err) - return - } - sk := got.(riseupvpn.SummaryKeys) - if diff := cmp.Diff(tt.sk, sk); diff != "" { - t.Fatal(diff) - } - }) + if result.(riseupvpn.SummaryKeys).IsAnomaly { + t.Fatal("GetSummaryKeys should never return IsAnomaly true") } } @@ -781,6 +566,7 @@ func generateMockGetter(requestResponse map[string]string, responseStatus map[st responseBody, ), BodyIsTruncated: false, + Code: responseStatus, }}, }, TCPConnect: []tracex.TCPConnectEntry{tcpConnect}, @@ -803,9 +589,7 @@ func runDefaultMockTest(t *testing.T, multiGetter urlgetter.MultiGetter) *model. args := &model.ExperimentArgs{ Callbacks: model.NewPrinterCallbacks(log.Log), Measurement: measurement, - Session: &mockable.Session{ - MockableLogger: log.Log, - }, + Session: &mocks.Session{MockLogger: func() model.Logger { return log.Log }}, } err := measurer.Run(context.Background(), args) diff --git a/pkg/experiment/torsf/torsf.go b/pkg/experiment/torsf/torsf.go index 620d980d..adfa0e67 100644 --- a/pkg/experiment/torsf/torsf.go +++ b/pkg/experiment/torsf/torsf.go @@ -25,7 +25,7 @@ import ( // We may want to have a single implementation for both nettests in the future. // testVersion is the experiment version. -const testVersion = "0.4.0" +const testVersion = "0.5.0" // Config contains the experiment config. type Config struct { diff --git a/pkg/experiment/torsf/torsf_test.go b/pkg/experiment/torsf/torsf_test.go index 301bfa61..333bdc4c 100644 --- a/pkg/experiment/torsf/torsf_test.go +++ b/pkg/experiment/torsf/torsf_test.go @@ -25,7 +25,7 @@ func TestExperimentNameAndVersion(t *testing.T) { if m.ExperimentName() != "torsf" { t.Fatal("invalid experiment name") } - if m.ExperimentVersion() != "0.4.0" { + if m.ExperimentVersion() != "0.5.0" { t.Fatal("invalid experiment version") } } diff --git a/pkg/experiment/webconnectivitylte/cleartextflow.go b/pkg/experiment/webconnectivitylte/cleartextflow.go index bdfa78a5..e755f65d 100644 --- a/pkg/experiment/webconnectivitylte/cleartextflow.go +++ b/pkg/experiment/webconnectivitylte/cleartextflow.go @@ -242,9 +242,9 @@ func (t *CleartextFlow) newHTTPRequest(ctx context.Context) (*http.Request, erro func (t *CleartextFlow) httpTransaction(ctx context.Context, network, address, alpn string, txp model.HTTPTransport, req *http.Request, trace *measurexlite.Trace) (*http.Response, []byte, error) { const maxbody = 1 << 19 - started := trace.TimeSince(trace.ZeroTime) + started := trace.TimeSince(trace.ZeroTime()) t.TestKeys.AppendNetworkEvents(measurexlite.NewAnnotationArchivalNetworkEvent( - trace.Index, started, "http_transaction_start", + trace.Index(), started, "http_transaction_start", )) resp, err := txp.RoundTrip(req) var body []byte @@ -256,12 +256,12 @@ func (t *CleartextFlow) httpTransaction(ctx context.Context, network, address, a reader := io.LimitReader(resp.Body, maxbody) body, err = StreamAllContext(ctx, reader) } - finished := trace.TimeSince(trace.ZeroTime) + finished := trace.TimeSince(trace.ZeroTime()) t.TestKeys.AppendNetworkEvents(measurexlite.NewAnnotationArchivalNetworkEvent( - trace.Index, finished, "http_transaction_done", + trace.Index(), finished, "http_transaction_done", )) ev := measurexlite.NewArchivalHTTPRequestResult( - trace.Index, + trace.Index(), started, network, address, diff --git a/pkg/experiment/webconnectivitylte/secureflow.go b/pkg/experiment/webconnectivitylte/secureflow.go index 311abea1..ce4645c9 100644 --- a/pkg/experiment/webconnectivitylte/secureflow.go +++ b/pkg/experiment/webconnectivitylte/secureflow.go @@ -297,9 +297,9 @@ func (t *SecureFlow) newHTTPRequest(ctx context.Context) (*http.Request, error) func (t *SecureFlow) httpTransaction(ctx context.Context, network, address, alpn string, txp model.HTTPTransport, req *http.Request, trace *measurexlite.Trace) (*http.Response, []byte, error) { const maxbody = 1 << 19 - started := trace.TimeSince(trace.ZeroTime) + started := trace.TimeSince(trace.ZeroTime()) t.TestKeys.AppendNetworkEvents(measurexlite.NewAnnotationArchivalNetworkEvent( - trace.Index, started, "http_transaction_start", + trace.Index(), started, "http_transaction_start", )) resp, err := txp.RoundTrip(req) var body []byte @@ -311,12 +311,12 @@ func (t *SecureFlow) httpTransaction(ctx context.Context, network, address, alpn reader := io.LimitReader(resp.Body, maxbody) body, err = StreamAllContext(ctx, reader) } - finished := trace.TimeSince(trace.ZeroTime) + finished := trace.TimeSince(trace.ZeroTime()) t.TestKeys.AppendNetworkEvents(measurexlite.NewAnnotationArchivalNetworkEvent( - trace.Index, finished, "http_transaction_done", + trace.Index(), finished, "http_transaction_done", )) ev := measurexlite.NewArchivalHTTPRequestResult( - trace.Index, + trace.Index(), started, network, address, diff --git a/pkg/libtor/enabled.go b/pkg/libtor/enabled.go index 113030d3..060f4d25 100644 --- a/pkg/libtor/enabled.go +++ b/pkg/libtor/enabled.go @@ -18,11 +18,36 @@ package libtor // #cgo android,amd64 CFLAGS: -I${SRCDIR}/android/amd64/include // #cgo android,amd64 LDFLAGS: -L${SRCDIR}/android/amd64/lib -ltor -levent -lssl -lcrypto -lz -lm // +// #cgo ios CFLAGS: -I${SRCDIR} +// // #include // #include // #include // -// #include +// /* Select the correct header depending on the Apple's platform and architecture, otherwise, for +// other operating systems just use the header in the include path defined above. +// +// See https://stackoverflow.com/a/18729350 for details. */ +// #if defined(__APPLE__) && defined(__MACH__) +// #include +// #if TARGET_OS_IPHONE && TARGET_OS_SIMULATOR +// #if TARGET_CPU_X86_64 +// #include +// #elif TARGET_CPU_ARM64 +// #include +// #else +// #error "internal/libtor/enabled.go: unhandled Apple architecture" +// #endif +// #elif TARGET_OS_IPHONE && TARGET_OS_MACCATALYST +// #error "internal/libtor/enabled.go: unhandled Apple platform" +// #elif TARGET_OS_IPHONE +// #include +// #else +// #error "internal/libtor/enabled.go: unhandled Apple platform" +// #endif +// #else +// #include +// #endif // // /* Note: we need to define inline helpers because we cannot index C arrays in Go. */ // diff --git a/pkg/libtor/iphoneos/arm64/.gitignore b/pkg/libtor/iphoneos/arm64/.gitignore new file mode 100644 index 00000000..f32e7dc7 --- /dev/null +++ b/pkg/libtor/iphoneos/arm64/.gitignore @@ -0,0 +1,2 @@ +/include +/lib diff --git a/pkg/libtor/iphonesimulator/amd64/.gitignore b/pkg/libtor/iphonesimulator/amd64/.gitignore new file mode 100644 index 00000000..f32e7dc7 --- /dev/null +++ b/pkg/libtor/iphonesimulator/amd64/.gitignore @@ -0,0 +1,2 @@ +/include +/lib diff --git a/pkg/libtor/iphonesimulator/arm64/.gitignore b/pkg/libtor/iphonesimulator/arm64/.gitignore new file mode 100644 index 00000000..f32e7dc7 --- /dev/null +++ b/pkg/libtor/iphonesimulator/arm64/.gitignore @@ -0,0 +1,2 @@ +/include +/lib diff --git a/pkg/measurexlite/conn.go b/pkg/measurexlite/conn.go index 93787b06..5adfdb12 100644 --- a/pkg/measurexlite/conn.go +++ b/pkg/measurexlite/conn.go @@ -44,16 +44,16 @@ func (c *connTrace) Read(b []byte) (int, error) { // collect preliminary stats when the connection is surely active network := c.RemoteAddr().Network() addr := c.RemoteAddr().String() - started := c.tx.TimeSince(c.tx.ZeroTime) + started := c.tx.TimeSince(c.tx.ZeroTime()) // perform the underlying network operation count, err := c.Conn.Read(b) // emit the network event - finished := c.tx.TimeSince(c.tx.ZeroTime) + finished := c.tx.TimeSince(c.tx.ZeroTime()) select { case c.tx.networkEvent <- NewArchivalNetworkEvent( - c.tx.Index, started, netxlite.ReadOperation, network, addr, count, + c.tx.Index(), started, netxlite.ReadOperation, network, addr, count, err, finished, c.tx.tags...): default: // buffer is full } @@ -101,14 +101,14 @@ func (tx *Trace) CloneBytesReceivedMap() (out map[string]int64) { func (c *connTrace) Write(b []byte) (int, error) { network := c.RemoteAddr().Network() addr := c.RemoteAddr().String() - started := c.tx.TimeSince(c.tx.ZeroTime) + started := c.tx.TimeSince(c.tx.ZeroTime()) count, err := c.Conn.Write(b) - finished := c.tx.TimeSince(c.tx.ZeroTime) + finished := c.tx.TimeSince(c.tx.ZeroTime()) select { case c.tx.networkEvent <- NewArchivalNetworkEvent( - c.tx.Index, started, netxlite.WriteOperation, network, addr, count, + c.tx.Index(), started, netxlite.WriteOperation, network, addr, count, err, finished, c.tx.tags...): default: // buffer is full } @@ -143,17 +143,17 @@ type udpLikeConnTrace struct { // Read implements model.UDPLikeConn.ReadFrom and saves network events. func (c *udpLikeConnTrace) ReadFrom(b []byte) (int, net.Addr, error) { // record when we started measuring - started := c.tx.TimeSince(c.tx.ZeroTime) + started := c.tx.TimeSince(c.tx.ZeroTime()) // perform the network operation count, addr, err := c.UDPLikeConn.ReadFrom(b) // emit the network event - finished := c.tx.TimeSince(c.tx.ZeroTime) + finished := c.tx.TimeSince(c.tx.ZeroTime()) address := addrStringIfNotNil(addr) select { case c.tx.networkEvent <- NewArchivalNetworkEvent( - c.tx.Index, started, netxlite.ReadFromOperation, "udp", address, count, + c.tx.Index(), started, netxlite.ReadFromOperation, "udp", address, count, err, finished, c.tx.tags...): default: // buffer is full } @@ -176,15 +176,15 @@ func (tx *Trace) maybeUpdateBytesReceivedMapUDPLikeConn(addr net.Addr, count int // Write implements model.UDPLikeConn.WriteTo and saves network events. func (c *udpLikeConnTrace) WriteTo(b []byte, addr net.Addr) (int, error) { - started := c.tx.TimeSince(c.tx.ZeroTime) + started := c.tx.TimeSince(c.tx.ZeroTime()) address := addr.String() count, err := c.UDPLikeConn.WriteTo(b, addr) - finished := c.tx.TimeSince(c.tx.ZeroTime) + finished := c.tx.TimeSince(c.tx.ZeroTime()) select { case c.tx.networkEvent <- NewArchivalNetworkEvent( - c.tx.Index, started, netxlite.WriteToOperation, "udp", address, count, + c.tx.Index(), started, netxlite.WriteToOperation, "udp", address, count, err, finished, c.tx.tags...): default: // buffer is full } diff --git a/pkg/measurexlite/dialer.go b/pkg/measurexlite/dialer.go index d97c163e..f35eef4d 100644 --- a/pkg/measurexlite/dialer.go +++ b/pkg/measurexlite/dialer.go @@ -55,11 +55,11 @@ func (tx *Trace) OnConnectDone( // insert into the tcpConnect buffer select { case tx.tcpConnect <- NewArchivalTCPConnectResult( - tx.Index, - started.Sub(tx.ZeroTime), + tx.Index(), + started.Sub(tx.ZeroTime()), remoteAddr, err, - finished.Sub(tx.ZeroTime), + finished.Sub(tx.ZeroTime()), tx.tags..., ): default: // buffer is full @@ -69,14 +69,14 @@ func (tx *Trace) OnConnectDone( // see https://github.com/ooni/probe/issues/2254 select { case tx.networkEvent <- NewArchivalNetworkEvent( - tx.Index, - started.Sub(tx.ZeroTime), + tx.Index(), + started.Sub(tx.ZeroTime()), netxlite.ConnectOperation, "tcp", remoteAddr, 0, err, - finished.Sub(tx.ZeroTime), + finished.Sub(tx.ZeroTime()), tx.tags..., ): default: // buffer is full diff --git a/pkg/measurexlite/dns.go b/pkg/measurexlite/dns.go index 3694591b..7951aff4 100644 --- a/pkg/measurexlite/dns.go +++ b/pkg/measurexlite/dns.go @@ -52,7 +52,7 @@ func (r *resolverTrace) CloseIdleConnections() { func (r *resolverTrace) emitResolveStart() { select { case r.tx.networkEvent <- NewAnnotationArchivalNetworkEvent( - r.tx.Index, r.tx.TimeSince(r.tx.ZeroTime), "resolve_start", + r.tx.Index(), r.tx.TimeSince(r.tx.ZeroTime()), "resolve_start", r.tx.tags..., ): default: // buffer is full @@ -63,7 +63,7 @@ func (r *resolverTrace) emitResolveStart() { func (r *resolverTrace) emiteResolveDone() { select { case r.tx.networkEvent <- NewAnnotationArchivalNetworkEvent( - r.tx.Index, r.tx.TimeSince(r.tx.ZeroTime), "resolve_done", + r.tx.Index(), r.tx.TimeSince(r.tx.ZeroTime()), "resolve_done", r.tx.tags..., ): default: // buffer is full @@ -109,12 +109,12 @@ func (tx *Trace) NewParallelDNSOverHTTPSResolver(logger model.DebugLogger, URL s // OnDNSRoundTripForLookupHost implements model.Trace.OnDNSRoundTripForLookupHost func (tx *Trace) OnDNSRoundTripForLookupHost(started time.Time, reso model.Resolver, query model.DNSQuery, response model.DNSResponse, addrs []string, err error, finished time.Time) { - t := finished.Sub(tx.ZeroTime) + t := finished.Sub(tx.ZeroTime()) select { case tx.dnsLookup <- NewArchivalDNSLookupResultFromRoundTrip( - tx.Index, - started.Sub(tx.ZeroTime), + tx.Index(), + started.Sub(tx.ZeroTime()), reso, query, response, @@ -274,12 +274,12 @@ var ErrDelayedDNSResponseBufferFull = errors.New( // OnDelayedDNSResponse implements model.Trace.OnDelayedDNSResponse func (tx *Trace) OnDelayedDNSResponse(started time.Time, txp model.DNSTransport, query model.DNSQuery, response model.DNSResponse, addrs []string, err error, finished time.Time) error { - t := finished.Sub(tx.ZeroTime) + t := finished.Sub(tx.ZeroTime()) select { case tx.delayedDNSResponse <- NewArchivalDNSLookupResultFromRoundTrip( - tx.Index, - started.Sub(tx.ZeroTime), + tx.Index(), + started.Sub(tx.ZeroTime()), txp, query, response, diff --git a/pkg/measurexlite/dns_test.go b/pkg/measurexlite/dns_test.go index ef13122c..ffd52680 100644 --- a/pkg/measurexlite/dns_test.go +++ b/pkg/measurexlite/dns_test.go @@ -519,8 +519,8 @@ func TestDelayedDNSResponseWithTimeout(t *testing.T) { } for i := 0; i < events; i++ { // fill the trace - trace.delayedDNSResponse <- NewArchivalDNSLookupResultFromRoundTrip(trace.Index, started.Sub(trace.ZeroTime), - txp, query, dnsResponse, addrs, nil, finished.Sub(trace.ZeroTime)) + trace.delayedDNSResponse <- NewArchivalDNSLookupResultFromRoundTrip(trace.Index(), started.Sub(trace.ZeroTime()), + txp, query, dnsResponse, addrs, nil, finished.Sub(trace.ZeroTime())) } ctx, cancel := context.WithCancel(context.Background()) cancel() // we ensure that the context cancels before draining all the events @@ -566,8 +566,8 @@ func TestDelayedDNSResponseWithTimeout(t *testing.T) { return []byte{} }, } - trace.delayedDNSResponse <- NewArchivalDNSLookupResultFromRoundTrip(trace.Index, started.Sub(trace.ZeroTime), - txp, query, dnsResponse, addrs, nil, finished.Sub(trace.ZeroTime)) + trace.delayedDNSResponse <- NewArchivalDNSLookupResultFromRoundTrip(trace.Index(), started.Sub(trace.ZeroTime()), + txp, query, dnsResponse, addrs, nil, finished.Sub(trace.ZeroTime())) got := trace.DelayedDNSResponseWithTimeout(context.Background(), time.Second) if len(got) != 1 { t.Fatal("unexpected output from trace") diff --git a/pkg/measurexlite/quic.go b/pkg/measurexlite/quic.go index a7155cb4..5283e7bf 100644 --- a/pkg/measurexlite/quic.go +++ b/pkg/measurexlite/quic.go @@ -49,10 +49,10 @@ func (qdx *quicDialerTrace) CloseIdleConnections() { // OnQUICHandshakeStart implements model.Trace.OnQUICHandshakeStart func (tx *Trace) OnQUICHandshakeStart(now time.Time, remoteAddr string, config *quic.Config) { - t := now.Sub(tx.ZeroTime) + t := now.Sub(tx.ZeroTime()) select { case tx.networkEvent <- NewAnnotationArchivalNetworkEvent( - tx.Index, t, "quic_handshake_start", tx.tags...): + tx.Index(), t, "quic_handshake_start", tx.tags...): default: } } @@ -60,7 +60,7 @@ func (tx *Trace) OnQUICHandshakeStart(now time.Time, remoteAddr string, config * // OnQUICHandshakeDone implements model.Trace.OnQUICHandshakeDone func (tx *Trace) OnQUICHandshakeDone(started time.Time, remoteAddr string, qconn quic.EarlyConnection, config *tls.Config, err error, finished time.Time) { - t := finished.Sub(tx.ZeroTime) + t := finished.Sub(tx.ZeroTime()) state := tls.ConnectionState{} if qconn != nil { @@ -69,8 +69,8 @@ func (tx *Trace) OnQUICHandshakeDone(started time.Time, remoteAddr string, qconn select { case tx.quicHandshake <- NewArchivalTLSOrQUICHandshakeResult( - tx.Index, - started.Sub(tx.ZeroTime), + tx.Index(), + started.Sub(tx.ZeroTime()), "udp", remoteAddr, config, @@ -84,7 +84,7 @@ func (tx *Trace) OnQUICHandshakeDone(started time.Time, remoteAddr string, qconn select { case tx.networkEvent <- NewAnnotationArchivalNetworkEvent( - tx.Index, t, "quic_handshake_done", tx.tags...): + tx.Index(), t, "quic_handshake_done", tx.tags...): default: // buffer is full } } diff --git a/pkg/measurexlite/tls.go b/pkg/measurexlite/tls.go index e75aa426..0b6f5c81 100644 --- a/pkg/measurexlite/tls.go +++ b/pkg/measurexlite/tls.go @@ -41,10 +41,10 @@ func (thx *tlsHandshakerTrace) Handshake( // OnTLSHandshakeStart implements model.Trace.OnTLSHandshakeStart. func (tx *Trace) OnTLSHandshakeStart(now time.Time, remoteAddr string, config *tls.Config) { - t := now.Sub(tx.ZeroTime) + t := now.Sub(tx.ZeroTime()) select { case tx.networkEvent <- NewAnnotationArchivalNetworkEvent( - tx.Index, t, "tls_handshake_start", tx.tags...): + tx.Index(), t, "tls_handshake_start", tx.tags...): default: // buffer is full } } @@ -52,12 +52,12 @@ func (tx *Trace) OnTLSHandshakeStart(now time.Time, remoteAddr string, config *t // OnTLSHandshakeDone implements model.Trace.OnTLSHandshakeDone. func (tx *Trace) OnTLSHandshakeDone(started time.Time, remoteAddr string, config *tls.Config, state tls.ConnectionState, err error, finished time.Time) { - t := finished.Sub(tx.ZeroTime) + t := finished.Sub(tx.ZeroTime()) select { case tx.tlsHandshake <- NewArchivalTLSOrQUICHandshakeResult( - tx.Index, - started.Sub(tx.ZeroTime), + tx.Index(), + started.Sub(tx.ZeroTime()), "tcp", remoteAddr, config, @@ -71,7 +71,7 @@ func (tx *Trace) OnTLSHandshakeDone(started time.Time, remoteAddr string, config select { case tx.networkEvent <- NewAnnotationArchivalNetworkEvent( - tx.Index, t, "tls_handshake_done", tx.tags...): + tx.Index(), t, "tls_handshake_done", tx.tags...): default: // buffer is full } } diff --git a/pkg/measurexlite/trace.go b/pkg/measurexlite/trace.go index bc85d145..38fef011 100644 --- a/pkg/measurexlite/trace.go +++ b/pkg/measurexlite/trace.go @@ -25,10 +25,8 @@ import ( // // [step-by-step measurements]: https://github.com/ooni/probe-cli/blob/master/docs/design/dd-003-step-by-step.md type Trace struct { - // Index is the unique index of this trace within the - // current measurement. Note that this field MUST be read-only. Writing it - // once you have constructed a trace MAY lead to data races. - Index int64 + // index is the unique index of this trace within the current measurement. + index int64 // Netx is the network to use for measuring. The constructor inits this // field using a [*netxlite.Netx]. You MAY override this field for testing. Make @@ -69,10 +67,8 @@ type Trace struct { // to produce deterministic timing when testing. timeNowFn func() time.Time - // ZeroTime is the time when we started the current measurement. This field - // MUST be read-only. Writing it once you have constructed the trace will - // likely read to data races. - ZeroTime time.Time + // zeroTime is the time when we started the current measurement. + zeroTime time.Time } var _ model.MeasuringNetwork = &Trace{} @@ -111,7 +107,7 @@ const QUICHandshakeBufferSize = 8 // to identify that some traces belong to some submeasurements). func NewTrace(index int64, zeroTime time.Time, tags ...string) *Trace { return &Trace{ - Index: index, + index: index, Netx: &netxlite.Netx{Underlying: nil}, // use the host network bytesReceivedMap: make(map[string]int64), bytesReceivedMu: &sync.Mutex{}, @@ -141,10 +137,20 @@ func NewTrace(index int64, zeroTime time.Time, tags ...string) *Trace { ), tags: tags, timeNowFn: nil, // use default - ZeroTime: zeroTime, + zeroTime: zeroTime, } } +// Index returns the trace index. +func (tx *Trace) Index() int64 { + return tx.index +} + +// ZeroTime returns trace's zero time. +func (tx *Trace) ZeroTime() time.Time { + return tx.zeroTime +} + // TimeNow implements model.Trace.TimeNow. func (tx *Trace) TimeNow() time.Time { if tx.timeNowFn != nil { diff --git a/pkg/measurexlite/trace_test.go b/pkg/measurexlite/trace_test.go index 74268240..54ce14eb 100644 --- a/pkg/measurexlite/trace_test.go +++ b/pkg/measurexlite/trace_test.go @@ -25,7 +25,7 @@ func TestNewTrace(t *testing.T) { trace := NewTrace(index, zeroTime) t.Run("Index", func(t *testing.T) { - if trace.Index != index { + if trace.Index() != index { t.Fatal("invalid index") } }) @@ -164,7 +164,7 @@ func TestNewTrace(t *testing.T) { }) t.Run("ZeroTime", func(t *testing.T) { - if !trace.ZeroTime.Equal(zeroTime) { + if !trace.ZeroTime().Equal(zeroTime) { t.Fatal("invalid zero time") } }) diff --git a/pkg/measurexlite/udp.go b/pkg/measurexlite/udp.go index 46147b83..75b684ea 100644 --- a/pkg/measurexlite/udp.go +++ b/pkg/measurexlite/udp.go @@ -2,6 +2,7 @@ package measurexlite import "github.com/ooni/probe-engine/pkg/model" +// NewUDPListener implements model.Measuring Network. func (tx *Trace) NewUDPListener() model.UDPListener { return tx.Netx.NewUDPListener() } diff --git a/pkg/netemx/scenario.go b/pkg/netemx/scenario.go index d3776265..31508915 100644 --- a/pkg/netemx/scenario.go +++ b/pkg/netemx/scenario.go @@ -209,6 +209,12 @@ func MustNewScenario(config []*ScenarioDomainAddresses) *QAEnv { ServerNameMain: sad.ServerNameMain, ServerNameExtras: sad.ServerNameExtras, }, + &HTTP3ServerFactory{ + Factory: &DNSOverHTTPSHandlerFactory{}, + Ports: []int{443}, + ServerNameMain: sad.ServerNameMain, + ServerNameExtras: sad.ServerNameExtras, + }, )) } diff --git a/pkg/progress/progress.go b/pkg/progress/progress.go new file mode 100644 index 00000000..f8ef289b --- /dev/null +++ b/pkg/progress/progress.go @@ -0,0 +1,44 @@ +// Package progress contains utilities to emit progress. +package progress + +import ( + "github.com/ooni/probe-engine/pkg/model" + "github.com/ooni/probe-engine/pkg/runtimex" +) + +// Scaler implements [model.ExperimentCallbacks] and scales progress +// as instructed through the [NewScaler] constructor. +// +// The [*Scaler] is safe to use from multiple goroutine contexts. +type Scaler struct { + cbs model.ExperimentCallbacks + offset float64 + total float64 +} + +// NewScaler constructs a new [*Scaler] using the given offset and total +// and emitting progress using the given [model.ExperimentCallbacks]. +// +// The offset is added to each progress value we emit. The total is +// used to scale the 100% to a suitable subset. +// +// For example, with offset equal to 0.1 and total equal to 0.5, the value +// 0.5 corresponds to 0.3 and the value 1 (i.e., 100%) is 0.5. +// +// This func PANICS if offset<0, offset >= total, total<=0, total>1. +func NewScaler(callbacks model.ExperimentCallbacks, offset, total float64) *Scaler { + runtimex.Assert(offset >= 0.0 && offset < total, "NewScaler: offset must be >= 0 and < total") + runtimex.Assert(total > 0.0 && total <= 1, "NewScaler: total must be > 0 and <= 1") + return &Scaler{ + cbs: callbacks, + offset: offset, + total: total, + } +} + +var _ model.ExperimentCallbacks = &Scaler{} + +// OnProgress implements model.ExperimentCallbacks. +func (s *Scaler) OnProgress(percentage float64, message string) { + s.cbs.OnProgress(s.offset+percentage*(s.total-s.offset), message) +} diff --git a/pkg/progress/progress_test.go b/pkg/progress/progress_test.go new file mode 100644 index 00000000..0d6b231f --- /dev/null +++ b/pkg/progress/progress_test.go @@ -0,0 +1,87 @@ +package progress + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/ooni/probe-engine/pkg/model" +) + +type capturerCallbacks struct { + value float64 +} + +var _ model.ExperimentCallbacks = &capturerCallbacks{} + +// OnProgress implements model.ExperimentCallbacks. +func (v *capturerCallbacks) OnProgress(percentage float64, message string) { + v.value = percentage +} + +func TestScaler(t *testing.T) { + // testcase is a test case run by this function. + type testcase struct { + // name is the test case name. + name string + + // offset is the offset (>=0, 0, <=1) + total float64 + + // emit is the list of progress values to emit. + emit []float64 + + // expect is the list of progress values we expect in output. + expect []float64 + } + + cases := []testcase{{ + name: "with offset==0 and total=1", + offset: 0, + total: 1, + emit: []float64{0, 0.2, 0.4, 0.6, 0.8, 1}, + expect: []float64{0, 0.2, 0.4, 0.6, 0.8, 1}, + }, { + name: "with offset==0 and total=0.5", + offset: 0, + total: 0.5, + emit: []float64{0, 0.2, 0.4, 0.6, 0.8, 1}, + expect: []float64{0, 0.1, 0.2, 0.3, 0.4, 0.5}, + }, { + name: "with offset==0.5 and total=1", + offset: 0.5, + total: 1, + emit: []float64{0, 0.2, 0.4, 0.6, 0.8, 1}, + expect: []float64{0.5, 0.6, 0.7, 0.8, 0.9, 1}, + }, { + name: "with offset==0.2 and total=0.7", + offset: 0.2, + total: 0.7, + emit: []float64{0, 0.2, 0.4, 0.6, 0.8, 1}, + expect: []float64{0.2, 0.3, 0.4, 0.5, 0.6, 0.7}, + }, { + name: "with offset=0.4 and total=0.5", + offset: 0.4, + total: 0.5, + emit: []float64{0, 0.2, 0.4, 0.6, 0.8, 1}, + expect: []float64{0.4, 0.42, 0.44, 0.46, 0.48, 0.5}, + }} + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var got []float64 + for _, v := range tc.emit { + cc := &capturerCallbacks{} + wrapper := NewScaler(cc, tc.offset, tc.total) + wrapper.OnProgress(v, "") + got = append(got, cc.value) + } + if diff := cmp.Diff(tc.expect, got, cmpopts.EquateApprox(0, 0.01)); diff != "" { + t.Fatal(diff) + } + }) + } +} diff --git a/pkg/registry/allexperiments.go b/pkg/registry/allexperiments.go index d191b6c5..39ce6122 100644 --- a/pkg/registry/allexperiments.go +++ b/pkg/registry/allexperiments.go @@ -1,5 +1,7 @@ package registry +import "sort" + // Where we register all the available experiments. var AllExperiments = map[string]*Factory{} @@ -8,5 +10,6 @@ func ExperimentNames() (names []string) { for key := range AllExperiments { names = append(names, key) } + sort.Strings(names) // sort by name to always provide predictable output return } diff --git a/pkg/registry/dash.go b/pkg/registry/dash.go index c86560bf..d55656b2 100644 --- a/pkg/registry/dash.go +++ b/pkg/registry/dash.go @@ -16,8 +16,9 @@ func init() { *config.(*dash.Config), ) }, - config: &dash.Config{}, - interruptible: true, - inputPolicy: model.InputNone, + config: &dash.Config{}, + enabledByDefault: true, + interruptible: true, + inputPolicy: model.InputNone, } } diff --git a/pkg/registry/dnscheck.go b/pkg/registry/dnscheck.go index f16eea42..5f95fc4e 100644 --- a/pkg/registry/dnscheck.go +++ b/pkg/registry/dnscheck.go @@ -16,7 +16,8 @@ func init() { *config.(*dnscheck.Config), ) }, - config: &dnscheck.Config{}, - inputPolicy: model.InputOrStaticDefault, + config: &dnscheck.Config{}, + enabledByDefault: true, + inputPolicy: model.InputOrStaticDefault, } } diff --git a/pkg/registry/dnsping.go b/pkg/registry/dnsping.go index a360b13b..1c98ff17 100644 --- a/pkg/registry/dnsping.go +++ b/pkg/registry/dnsping.go @@ -16,7 +16,8 @@ func init() { *config.(*dnsping.Config), ) }, - config: &dnsping.Config{}, - inputPolicy: model.InputOrStaticDefault, + config: &dnsping.Config{}, + enabledByDefault: true, + inputPolicy: model.InputOrStaticDefault, } } diff --git a/pkg/registry/example.go b/pkg/registry/example.go index c3038edb..c3ee38ca 100644 --- a/pkg/registry/example.go +++ b/pkg/registry/example.go @@ -22,7 +22,8 @@ func init() { Message: "Good day from the example experiment!", SleepTime: int64(time.Second), }, - interruptible: true, - inputPolicy: model.InputNone, + enabledByDefault: true, + interruptible: true, + inputPolicy: model.InputNone, } } diff --git a/pkg/registry/factory.go b/pkg/registry/factory.go index 186d507f..c150adf3 100644 --- a/pkg/registry/factory.go +++ b/pkg/registry/factory.go @@ -7,9 +7,11 @@ package registry import ( "errors" "fmt" + "os" "reflect" "strconv" + "github.com/ooni/probe-engine/pkg/checkincache" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/strcasex" ) @@ -22,6 +24,9 @@ type Factory struct { // config contains the experiment's config. config any + // enabledByDefault indicates whether this experiment is enabled by default. + enabledByDefault bool + // inputPolicy contains the experiment's InputPolicy. inputPolicy model.InputPolicy @@ -218,12 +223,91 @@ func CanonicalizeExperimentName(name string) string { // ErrNoSuchExperiment indicates a given experiment does not exist. var ErrNoSuchExperiment = errors.New("no such experiment") +// ErrRequiresForceEnable is returned for experiments that are not enabled by default and are also +// not enabled by the most recent check-in API call. +var ErrRequiresForceEnable = errors.New("experiment not enabled by check-in API") + +const experimentDisabledByCheckInWarning = `We disabled the '%s' nettest. This usually happens in these cases: + +1. we just added the nettest to ooniprobe and we have not enabled it yet; + +2. the nettest is flaky and we are working on a fix; + +3. you ran Web Connectivity more than 24h ago, hence your check-in cache is stale. + +The last case is a known limitation in ooniprobe 3.19 that we will fix in a subsequent +release of ooniprobe by changing the nettests startup logic. + +If you really want to run this nettest, there is a way forward. You need to set the +OONI_FORCE_ENABLE_EXPERIMENT=1 environment variable. On a Unix like system, use: + + export OONI_FORCE_ENABLE_EXPERIMENT=1 + +on Windows use: + + set OONI_FORCE_ENABLE_EXPERIMENT=1 + +Re-running ooniprobe once you have set the environment variable would cause the +disabled nettest to run. Please, note that we usually have good reasons for disabling +nettests, including the following reasons: + +* making sure that we gradually introduce new nettests to all users by first introducing +them to a few users and monitoring whether they're working as intended; + +* avoid polluting our measurements database with measurements produced by experiments +that currently produce false positives or other data quality issues. +` + +// OONI_FORCE_ENABLE_EXPERIMENT is the name of the environment variable you should set to "1" +// to bypass the algorithm preventing disabled by default experiments to be instantiated. +const OONI_FORCE_ENABLE_EXPERIMENT = "OONI_FORCE_ENABLE_EXPERIMENT" + // NewFactory creates a new Factory instance. -func NewFactory(name string) (*Factory, error) { +func NewFactory(name string, kvStore model.KeyValueStore, logger model.Logger) (*Factory, error) { + // Make sure we are deadling with the canonical experiment name. Historically MK used + // names such as WebConnectivity and we want to continue supporting this use case. name = CanonicalizeExperimentName(name) + + // Handle A/B testing where we dynamically choose LTE for some users. The current policy + // only relates to a few users to collect data. + // + // TODO(https://github.com/ooni/probe/issues/2555): perform the actual comparison + // and improve the LTE implementation so that we can always use it. See the actual + // issue test for additional details on this planned A/B test. + switch { + case name == "web_connectivity" && checkincache.GetFeatureFlag(kvStore, "webconnectivity_0.5"): + // use LTE rather than the normal webconnectivity when the + // feature flag has been set through the check-in API + logger.Infof("using webconnectivity LTE") + name = "web_connectivity@v0.5" + + default: + // nothing + } + + // Obtain the factory for the canonical name. factory := AllExperiments[name] if factory == nil { return nil, fmt.Errorf("%w: %s", ErrNoSuchExperiment, name) } - return factory, nil + + // Some experiments are not enabled by default. To enable them we use + // the cached check-in response or an environment variable. + // + // Note: check-in flags expire after 24h. + // + // TODO(https://github.com/ooni/probe/issues/2554): we need to restructure + // how we run experiments to make sure check-in flags are always fresh. + if factory.enabledByDefault { + return factory, nil // enabled by default + } + if os.Getenv(OONI_FORCE_ENABLE_EXPERIMENT) == "1" { + return factory, nil // enabled by environment variable + } + if checkincache.ExperimentEnabled(kvStore, name) { + return factory, nil // enabled by check-in + } + + logger.Warnf(experimentDisabledByCheckInWarning, name) + return nil, fmt.Errorf("%s: %w", name, ErrRequiresForceEnable) } diff --git a/pkg/registry/factory_test.go b/pkg/registry/factory_test.go index f881ffdd..db7c4b6c 100644 --- a/pkg/registry/factory_test.go +++ b/pkg/registry/factory_test.go @@ -2,9 +2,15 @@ package registry import ( "errors" + "fmt" + "os" "testing" "github.com/google/go-cmp/cmp" + "github.com/ooni/probe-engine/pkg/checkincache" + "github.com/ooni/probe-engine/pkg/experiment/webconnectivitylte" + "github.com/ooni/probe-engine/pkg/kvstore" + "github.com/ooni/probe-engine/pkg/model" ) type fakeExperimentConfig struct { @@ -345,3 +351,405 @@ func TestExperimentBuilderSetOptionsAny(t *testing.T) { } }) } + +func TestNewFactory(t *testing.T) { + // experimentSpecificExpectations contains expectations for an experiment + type experimentSpecificExpectations struct { + // enabledByDefault contains the expected value for the enabledByDefault factory field. + enabledByDefault bool + + // inputPolicy contains the expected value for the input policy. + inputPolicy model.InputPolicy + + // interruptible contains the expected value for interrupted. + interruptible bool + } + + // expectationsMap contains expectations for each experiment that exists + expectationsMap := map[string]*experimentSpecificExpectations{ + "dash": { + enabledByDefault: true, + inputPolicy: model.InputNone, + interruptible: true, + }, + "dnscheck": { + enabledByDefault: true, + inputPolicy: model.InputOrStaticDefault, + }, + "dnsping": { + enabledByDefault: true, + inputPolicy: model.InputOrStaticDefault, + }, + "echcheck": { + // Note: echcheck is not enabled by default because we just introduced it + // into 3.19.0-alpha, which makes it a relatively new experiment. + //enabledByDefault: false, + inputPolicy: model.InputOptional, + }, + "example": { + enabledByDefault: true, + inputPolicy: model.InputNone, + interruptible: true, + }, + "facebook_messenger": { + enabledByDefault: true, + inputPolicy: model.InputNone, + }, + "http_header_field_manipulation": { + enabledByDefault: true, + inputPolicy: model.InputNone, + }, + "http_host_header": { + enabledByDefault: true, + inputPolicy: model.InputOrQueryBackend, + }, + "http_invalid_request_line": { + enabledByDefault: true, + inputPolicy: model.InputNone, + }, + "ndt": { + enabledByDefault: true, + inputPolicy: model.InputNone, + interruptible: true, + }, + "portfiltering": { + enabledByDefault: true, + inputPolicy: model.InputNone, + }, + "psiphon": { + enabledByDefault: true, + inputPolicy: model.InputOptional, + }, + "quicping": { + enabledByDefault: true, + inputPolicy: model.InputStrictlyRequired, + }, + "riseupvpn": { + // Note: riseupvpn is not enabled by default because it has been flaky + // in the past and we want to be defensive here. + //enabledByDefault: false, + inputPolicy: model.InputNone, + }, + "run": { + enabledByDefault: true, + inputPolicy: model.InputStrictlyRequired, + }, + "signal": { + enabledByDefault: true, + inputPolicy: model.InputNone, + }, + "simple_sni": { + // Note: simple_sni is not enabled by default because it has only been + // introduced for writing tutorials and should not be used. + //enabledByDefault: false, + inputPolicy: model.InputOrQueryBackend, + }, + "simplequicping": { + enabledByDefault: true, + inputPolicy: model.InputStrictlyRequired, + }, + "sni_blocking": { + enabledByDefault: true, + inputPolicy: model.InputOrQueryBackend, + }, + "stunreachability": { + enabledByDefault: true, + inputPolicy: model.InputOrStaticDefault, + }, + "tcpping": { + enabledByDefault: true, + inputPolicy: model.InputStrictlyRequired, + }, + "tlsmiddlebox": { + enabledByDefault: true, + inputPolicy: model.InputStrictlyRequired, + }, + "telegram": { + enabledByDefault: true, + inputPolicy: model.InputNone, + }, + "tlsping": { + enabledByDefault: true, + inputPolicy: model.InputStrictlyRequired, + }, + "tlstool": { + enabledByDefault: true, + inputPolicy: model.InputOrQueryBackend, + }, + "tor": { + enabledByDefault: true, + inputPolicy: model.InputNone, + }, + "torsf": { + // We suspect there will be changes in torsf SNI soon. We are not prepared to + // serve these changes using the check-in API. Hence, disable torsf by default + // and require enabling it using the check-in API feature flags. + //enabledByDefault: false, + inputPolicy: model.InputNone, + }, + "urlgetter": { + enabledByDefault: true, + inputPolicy: model.InputStrictlyRequired, + }, + "vanilla_tor": { + // The experiment crashes on Android and possibly also iOS. We want to + // control whether and when to run it using check-in. + //enabledByDefault: false, + inputPolicy: model.InputNone, + }, + "web_connectivity": { + enabledByDefault: true, + inputPolicy: model.InputOrQueryBackend, + }, + "web_connectivity@v0.5": { + enabledByDefault: true, + inputPolicy: model.InputOrQueryBackend, + }, + "whatsapp": { + enabledByDefault: true, + inputPolicy: model.InputNone, + }, + } + + // testCase is a test case checked by this func + type testCase struct { + // description describes the test case + description string + + // experimentName is the experiment experimentName + experimentName string + + // kvStore is the key-value store to use + kvStore model.KeyValueStore + + // setForceEnableExperiment sets the OONI_FORCE_ENABLE_EXPERIMENT=1 env variable + setForceEnableExperiment bool + + // expectErr is the error we expect when calling NewFactory + expectErr error + } + + // allCases contains all test cases + allCases := []*testCase{} + + // create test cases for canonical experiment names + for _, name := range ExperimentNames() { + allCases = append(allCases, &testCase{ + description: name, + experimentName: name, + kvStore: &kvstore.Memory{}, + expectErr: (func() error { + expectations := expectationsMap[name] + if expectations == nil { + t.Fatal("no expectations for", name) + } + if !expectations.enabledByDefault { + return ErrRequiresForceEnable + } + return nil + }()), + }) + } + + // add additional test for the ndt7 experiment name + allCases = append(allCases, &testCase{ + description: "the ndt7 name still works", + experimentName: "ndt7", + kvStore: &kvstore.Memory{}, + expectErr: nil, + }) + + // add additional test for the dns_check experiment name + allCases = append(allCases, &testCase{ + description: "the dns_check name still works", + experimentName: "dns_check", + kvStore: &kvstore.Memory{}, + expectErr: nil, + }) + + // add additional test for the stun_reachability experiment name + allCases = append(allCases, &testCase{ + description: "the stun_reachability name still works", + experimentName: "stun_reachability", + kvStore: &kvstore.Memory{}, + expectErr: nil, + }) + + // add additional test for the web_connectivity@v_0_5 experiment name + allCases = append(allCases, &testCase{ + description: "the web_connectivity@v_0_5 name still works", + experimentName: "web_connectivity@v_0_5", + kvStore: &kvstore.Memory{}, + expectErr: nil, + }) + + // make sure we can create default-not-enabled experiments if we + // configure the proper environment variable + for name, expectations := range expectationsMap { + if expectations.enabledByDefault { + continue + } + + allCases = append(allCases, &testCase{ + description: fmt.Sprintf("we can create %s with OONI_FORCE_ENABLE_EXPERIMENT=1", name), + experimentName: name, + kvStore: &kvstore.Memory{}, + setForceEnableExperiment: true, + expectErr: nil, + }) + } + + // make sure we can create default-not-enabled experiments if we + // configure the proper check-in flags + for name, expectations := range expectationsMap { + if expectations.enabledByDefault { + continue + } + + // create a check-in configuration with the experiment being enabled + store := &kvstore.Memory{} + checkincache.Store(store, &model.OOAPICheckInResult{ + Conf: model.OOAPICheckInResultConfig{ + Features: map[string]bool{ + checkincache.ExperimentEnabledKey(name): true, + }, + }, + }) + + allCases = append(allCases, &testCase{ + description: fmt.Sprintf("we can create %s with the proper check-in config", name), + experimentName: name, + kvStore: store, + setForceEnableExperiment: false, + expectErr: nil, + }) + } + + // perform checks for each name + for _, tc := range allCases { + t.Run(tc.description, func(t *testing.T) { + // make sure the bypass environment variable is not set + if os.Getenv(OONI_FORCE_ENABLE_EXPERIMENT) != "" { + t.Fatal("the OONI_FORCE_ENABLE_EXPERIMENT env variable shouldn't be set") + } + + // if needed, set the environment variable for the scope of the func + if tc.setForceEnableExperiment { + os.Setenv(OONI_FORCE_ENABLE_EXPERIMENT, "1") + defer os.Unsetenv(OONI_FORCE_ENABLE_EXPERIMENT) + } + + t.Log("experimentName:", tc.experimentName) + + // get experiment expectations -- note that here we must canonicalize the + // experiment name otherwise we won't find it into the map when testing non-canonical names + expectations := expectationsMap[CanonicalizeExperimentName(tc.experimentName)] + if expectations == nil { + t.Fatal("no expectations for", tc.experimentName) + } + + t.Logf("expectations: %+v", expectations) + + // get the experiment factory + factory, err := NewFactory(tc.experimentName, tc.kvStore, model.DiscardLogger) + + t.Logf("NewFactory returned: %+v %+v", factory, err) + + // make sure the returned error makes sense + switch { + case tc.expectErr == nil && err != nil: + t.Fatal(tc.experimentName, ": expected", tc.expectErr, "got", err) + + case tc.expectErr != nil && err == nil: + t.Fatal(tc.experimentName, ": expected", tc.expectErr, "got", err) + + case tc.expectErr != nil && err != nil: + if !errors.Is(err, tc.expectErr) { + t.Fatal(tc.experimentName, ": expected", tc.expectErr, "got", err) + } + return + + case tc.expectErr == nil && err == nil: + // fallthrough + } + + // make sure the enabled by default field is consistent with expectations + if factory.enabledByDefault != expectations.enabledByDefault { + t.Fatal(tc.experimentName, ": expected", expectations.enabledByDefault, "got", factory.enabledByDefault) + } + + // make sure the input policy is the expected one + if v := factory.InputPolicy(); v != expectations.inputPolicy { + t.Fatal(tc.experimentName, ": expected", expectations.inputPolicy, "got", v) + } + + // make sure the interruptible value is the expected one + if v := factory.Interruptible(); v != expectations.interruptible { + t.Fatal(tc.experimentName, ": expected", expectations.interruptible, "got", v) + } + + // make sure we can create the measurer + measurer := factory.NewExperimentMeasurer() + if measurer == nil { + t.Fatal("expected non-nil measurer, got nil") + } + }) + } + + // make sure we create web_connectivity@v0.5 when the check-in says so + t.Run("we honor check-in flags for web_connectivity@v0.5", func(t *testing.T) { + // create a keyvalue store with the proper flags + store := &kvstore.Memory{} + checkincache.Store(store, &model.OOAPICheckInResult{ + Conf: model.OOAPICheckInResultConfig{ + Features: map[string]bool{ + "webconnectivity_0.5": true, + }, + }, + }) + + // get the experiment factory + factory, err := NewFactory("web_connectivity", store, model.DiscardLogger) + if err != nil { + t.Fatal(err) + } + + // make sure the enabled by default field is consistent with expectations + if !factory.enabledByDefault { + t.Fatal("expected enabledByDefault to be true") + } + + // make sure the input policy is the expected one + if factory.InputPolicy() != model.InputOrQueryBackend { + t.Fatal("expected inputPolicy to be InputOrQueryBackend") + } + + // make sure the interrupted value is the expected one + if factory.Interruptible() { + t.Fatal("expected interruptible to be false") + } + + // make sure we can create the measurer + measurer := factory.NewExperimentMeasurer() + if measurer == nil { + t.Fatal("expected non-nil measurer, got nil") + } + + // make sure the type we're creating is the correct one + if _, good := measurer.(*webconnectivitylte.Measurer); !good { + t.Fatalf("expected to see an instance of *webconnectivitylte.Measurer, got %T", measurer) + } + }) + + // add a test case for a nonexistent experiment + t.Run("we correctly return an error for a nonexistent experiment", func(t *testing.T) { + // the empty string is a nonexistent experiment + factory, err := NewFactory("", &kvstore.Memory{}, model.DiscardLogger) + if !errors.Is(err, ErrNoSuchExperiment) { + t.Fatal("unexpected err", err) + } + if factory != nil { + t.Fatal("expected nil factory here") + } + }) +} diff --git a/pkg/registry/fbmessenger.go b/pkg/registry/fbmessenger.go index d7d3a43b..a8945607 100644 --- a/pkg/registry/fbmessenger.go +++ b/pkg/registry/fbmessenger.go @@ -16,7 +16,8 @@ func init() { *config.(*fbmessenger.Config), ) }, - config: &fbmessenger.Config{}, - inputPolicy: model.InputNone, + config: &fbmessenger.Config{}, + enabledByDefault: true, + inputPolicy: model.InputNone, } } diff --git a/pkg/registry/hhfm.go b/pkg/registry/hhfm.go index e5b0115b..1adfaecd 100644 --- a/pkg/registry/hhfm.go +++ b/pkg/registry/hhfm.go @@ -16,7 +16,8 @@ func init() { *config.(*hhfm.Config), ) }, - config: &hhfm.Config{}, - inputPolicy: model.InputNone, + config: &hhfm.Config{}, + enabledByDefault: true, + inputPolicy: model.InputNone, } } diff --git a/pkg/registry/hirl.go b/pkg/registry/hirl.go index 1a23c86a..6ffa57f2 100644 --- a/pkg/registry/hirl.go +++ b/pkg/registry/hirl.go @@ -16,7 +16,8 @@ func init() { *config.(*hirl.Config), ) }, - config: &hirl.Config{}, - inputPolicy: model.InputNone, + config: &hirl.Config{}, + enabledByDefault: true, + inputPolicy: model.InputNone, } } diff --git a/pkg/registry/httphostheader.go b/pkg/registry/httphostheader.go index 114cd832..b9503e0d 100644 --- a/pkg/registry/httphostheader.go +++ b/pkg/registry/httphostheader.go @@ -16,7 +16,8 @@ func init() { *config.(*httphostheader.Config), ) }, - config: &httphostheader.Config{}, - inputPolicy: model.InputOrQueryBackend, + config: &httphostheader.Config{}, + enabledByDefault: true, + inputPolicy: model.InputOrQueryBackend, } } diff --git a/pkg/registry/ndt.go b/pkg/registry/ndt.go index ac84aca1..49407b28 100644 --- a/pkg/registry/ndt.go +++ b/pkg/registry/ndt.go @@ -16,8 +16,9 @@ func init() { *config.(*ndt7.Config), ) }, - config: &ndt7.Config{}, - interruptible: true, - inputPolicy: model.InputNone, + config: &ndt7.Config{}, + enabledByDefault: true, + interruptible: true, + inputPolicy: model.InputNone, } } diff --git a/pkg/registry/portfiltering.go b/pkg/registry/portfiltering.go index 9ab3beab..5921e56e 100644 --- a/pkg/registry/portfiltering.go +++ b/pkg/registry/portfiltering.go @@ -16,8 +16,9 @@ func init() { config.(portfiltering.Config), ) }, - config: portfiltering.Config{}, - interruptible: false, - inputPolicy: model.InputNone, + config: portfiltering.Config{}, + enabledByDefault: true, + interruptible: false, + inputPolicy: model.InputNone, } } diff --git a/pkg/registry/psiphon.go b/pkg/registry/psiphon.go index 26cd368c..202f7e04 100644 --- a/pkg/registry/psiphon.go +++ b/pkg/registry/psiphon.go @@ -16,7 +16,8 @@ func init() { *config.(*psiphon.Config), ) }, - config: &psiphon.Config{}, - inputPolicy: model.InputOptional, + config: &psiphon.Config{}, + enabledByDefault: true, + inputPolicy: model.InputOptional, } } diff --git a/pkg/registry/quicping.go b/pkg/registry/quicping.go index 85582642..2229523c 100644 --- a/pkg/registry/quicping.go +++ b/pkg/registry/quicping.go @@ -16,7 +16,8 @@ func init() { *config.(*quicping.Config), ) }, - config: &quicping.Config{}, - inputPolicy: model.InputStrictlyRequired, + config: &quicping.Config{}, + enabledByDefault: true, + inputPolicy: model.InputStrictlyRequired, } } diff --git a/pkg/registry/run.go b/pkg/registry/run.go index 1c9a2026..85c44d87 100644 --- a/pkg/registry/run.go +++ b/pkg/registry/run.go @@ -16,7 +16,8 @@ func init() { *config.(*run.Config), ) }, - config: &run.Config{}, - inputPolicy: model.InputStrictlyRequired, + config: &run.Config{}, + enabledByDefault: true, + inputPolicy: model.InputStrictlyRequired, } } diff --git a/pkg/registry/signal.go b/pkg/registry/signal.go index d7efce22..4d700018 100644 --- a/pkg/registry/signal.go +++ b/pkg/registry/signal.go @@ -16,7 +16,8 @@ func init() { *config.(*signal.Config), ) }, - config: &signal.Config{}, - inputPolicy: model.InputNone, + config: &signal.Config{}, + enabledByDefault: true, + inputPolicy: model.InputNone, } } diff --git a/pkg/registry/simplequicping.go b/pkg/registry/simplequicping.go index 7f5c969f..1f1ded8f 100644 --- a/pkg/registry/simplequicping.go +++ b/pkg/registry/simplequicping.go @@ -16,7 +16,8 @@ func init() { *config.(*simplequicping.Config), ) }, - config: &simplequicping.Config{}, - inputPolicy: model.InputStrictlyRequired, + config: &simplequicping.Config{}, + enabledByDefault: true, + inputPolicy: model.InputStrictlyRequired, } } diff --git a/pkg/registry/sniblocking.go b/pkg/registry/sniblocking.go index cbdd44a1..db852135 100644 --- a/pkg/registry/sniblocking.go +++ b/pkg/registry/sniblocking.go @@ -16,7 +16,8 @@ func init() { *config.(*sniblocking.Config), ) }, - config: &sniblocking.Config{}, - inputPolicy: model.InputOrQueryBackend, + config: &sniblocking.Config{}, + enabledByDefault: true, + inputPolicy: model.InputOrQueryBackend, } } diff --git a/pkg/registry/stunreachability.go b/pkg/registry/stunreachability.go index 575e6139..865789f4 100644 --- a/pkg/registry/stunreachability.go +++ b/pkg/registry/stunreachability.go @@ -16,7 +16,8 @@ func init() { *config.(*stunreachability.Config), ) }, - config: &stunreachability.Config{}, - inputPolicy: model.InputOrStaticDefault, + config: &stunreachability.Config{}, + enabledByDefault: true, + inputPolicy: model.InputOrStaticDefault, } } diff --git a/pkg/registry/tcpping.go b/pkg/registry/tcpping.go index 042e011e..5786af75 100644 --- a/pkg/registry/tcpping.go +++ b/pkg/registry/tcpping.go @@ -16,7 +16,8 @@ func init() { *config.(*tcpping.Config), ) }, - config: &tcpping.Config{}, - inputPolicy: model.InputStrictlyRequired, + config: &tcpping.Config{}, + enabledByDefault: true, + inputPolicy: model.InputStrictlyRequired, } } diff --git a/pkg/registry/telegram.go b/pkg/registry/telegram.go index dd997a53..cef5786f 100644 --- a/pkg/registry/telegram.go +++ b/pkg/registry/telegram.go @@ -16,8 +16,9 @@ func init() { config.(telegram.Config), ) }, - config: telegram.Config{}, - interruptible: false, - inputPolicy: model.InputNone, + config: telegram.Config{}, + enabledByDefault: true, + interruptible: false, + inputPolicy: model.InputNone, } } diff --git a/pkg/registry/tlsmiddlebox.go b/pkg/registry/tlsmiddlebox.go index 40590613..1f956787 100644 --- a/pkg/registry/tlsmiddlebox.go +++ b/pkg/registry/tlsmiddlebox.go @@ -16,7 +16,8 @@ func init() { *config.(*tlsmiddlebox.Config), ) }, - config: &tlsmiddlebox.Config{}, - inputPolicy: model.InputStrictlyRequired, + config: &tlsmiddlebox.Config{}, + enabledByDefault: true, + inputPolicy: model.InputStrictlyRequired, } } diff --git a/pkg/registry/tlsping.go b/pkg/registry/tlsping.go index 6f5bfd0a..529bbb2a 100644 --- a/pkg/registry/tlsping.go +++ b/pkg/registry/tlsping.go @@ -16,7 +16,8 @@ func init() { *config.(*tlsping.Config), ) }, - config: &tlsping.Config{}, - inputPolicy: model.InputStrictlyRequired, + config: &tlsping.Config{}, + enabledByDefault: true, + inputPolicy: model.InputStrictlyRequired, } } diff --git a/pkg/registry/tlstool.go b/pkg/registry/tlstool.go index ab01eab6..1cc9d6a4 100644 --- a/pkg/registry/tlstool.go +++ b/pkg/registry/tlstool.go @@ -16,7 +16,8 @@ func init() { *config.(*tlstool.Config), ) }, - config: &tlstool.Config{}, - inputPolicy: model.InputOrQueryBackend, + config: &tlstool.Config{}, + enabledByDefault: true, + inputPolicy: model.InputOrQueryBackend, } } diff --git a/pkg/registry/tor.go b/pkg/registry/tor.go index 2dfcda4b..2248eb77 100644 --- a/pkg/registry/tor.go +++ b/pkg/registry/tor.go @@ -16,7 +16,8 @@ func init() { *config.(*tor.Config), ) }, - config: &tor.Config{}, - inputPolicy: model.InputNone, + config: &tor.Config{}, + enabledByDefault: true, + inputPolicy: model.InputNone, } } diff --git a/pkg/registry/torsf.go b/pkg/registry/torsf.go index dc4511f9..f0726734 100644 --- a/pkg/registry/torsf.go +++ b/pkg/registry/torsf.go @@ -16,7 +16,8 @@ func init() { *config.(*torsf.Config), ) }, - config: &torsf.Config{}, - inputPolicy: model.InputNone, + config: &torsf.Config{}, + enabledByDefault: false, + inputPolicy: model.InputNone, } } diff --git a/pkg/registry/urlgetter.go b/pkg/registry/urlgetter.go index 5ae2d600..6f29a8d8 100644 --- a/pkg/registry/urlgetter.go +++ b/pkg/registry/urlgetter.go @@ -16,7 +16,8 @@ func init() { *config.(*urlgetter.Config), ) }, - config: &urlgetter.Config{}, - inputPolicy: model.InputStrictlyRequired, + config: &urlgetter.Config{}, + enabledByDefault: true, + inputPolicy: model.InputStrictlyRequired, } } diff --git a/pkg/registry/vanillator.go b/pkg/registry/vanillator.go index 76128e75..e9865e4b 100644 --- a/pkg/registry/vanillator.go +++ b/pkg/registry/vanillator.go @@ -16,7 +16,11 @@ func init() { *config.(*vanillator.Config), ) }, - config: &vanillator.Config{}, - inputPolicy: model.InputNone, + config: &vanillator.Config{}, + // We discussed this topic with @aanorbel. On Android this experiment crashes + // frequently because of https://github.com/ooni/probe/issues/2406. So, it seems + // more cautious to disable it by default and let the check-in API decide. + enabledByDefault: false, + inputPolicy: model.InputNone, } } diff --git a/pkg/registry/webconnectivity.go b/pkg/registry/webconnectivity.go index 3975f176..e94ea6e8 100644 --- a/pkg/registry/webconnectivity.go +++ b/pkg/registry/webconnectivity.go @@ -16,8 +16,9 @@ func init() { config.(webconnectivity.Config), ) }, - config: webconnectivity.Config{}, - interruptible: false, - inputPolicy: model.InputOrQueryBackend, + config: webconnectivity.Config{}, + enabledByDefault: true, + interruptible: false, + inputPolicy: model.InputOrQueryBackend, } } diff --git a/pkg/registry/webconnectivityv05.go b/pkg/registry/webconnectivityv05.go index ad413a1c..77bd1519 100644 --- a/pkg/registry/webconnectivityv05.go +++ b/pkg/registry/webconnectivityv05.go @@ -18,8 +18,9 @@ func init() { config.(*webconnectivitylte.Config), ) }, - config: &webconnectivitylte.Config{}, - interruptible: false, - inputPolicy: model.InputOrQueryBackend, + config: &webconnectivitylte.Config{}, + enabledByDefault: true, + interruptible: false, + inputPolicy: model.InputOrQueryBackend, } } diff --git a/pkg/registry/whatsapp.go b/pkg/registry/whatsapp.go index 12f4bd35..bbd25285 100644 --- a/pkg/registry/whatsapp.go +++ b/pkg/registry/whatsapp.go @@ -16,7 +16,8 @@ func init() { *config.(*whatsapp.Config), ) }, - config: &whatsapp.Config{}, - inputPolicy: model.InputNone, + config: &whatsapp.Config{}, + enabledByDefault: true, + inputPolicy: model.InputNone, } } diff --git a/pkg/throttling/throttling.go b/pkg/throttling/throttling.go index fe48eca0..cf9be050 100644 --- a/pkg/throttling/throttling.go +++ b/pkg/throttling/throttling.go @@ -7,13 +7,32 @@ import ( "sync" "time" - "github.com/ooni/probe-engine/pkg/measurexlite" "github.com/ooni/probe-engine/pkg/memoryless" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/runtimex" ) -// Sampler periodically samples the bytes sent and received by a [*measurexlite.Trace]. The zero +// Trace is the [*measurexlite.Trace] abstraction used by this package. +type Trace interface { + // CloneBytesReceivedMap returns a clone of the internal bytes received map. The key of the + // map is a string following the "EPNT_ADDRESS PROTO" pattern where the "EPNT_ADDRESS" contains + // the endpoint address and "PROTO" is "tcp" or "udp". + CloneBytesReceivedMap() (out map[string]int64) + + // Index returns the unique index used by this trace. + Index() int64 + + // Tags returns the trace tags. + Tags() []string + + // TimeSince is equivalent to Trace.TimeNow().Sub(t0). + TimeSince(t0 time.Time) time.Duration + + // ZeroTime returns the "zero" time of this trace. + ZeroTime() time.Time +} + +// Sampler periodically samples the bytes sent and received by a [Trace]. The zero // value of this structure is invalid; please, construct using [NewSampler]. type Sampler struct { // cancel tells the background goroutine to stop @@ -29,16 +48,16 @@ type Sampler struct { q []*model.ArchivalNetworkEvent // tx is the trace we are sampling from - tx *measurexlite.Trace + tx Trace // wg is the waitgroup to wait for the sampler to join wg *sync.WaitGroup } -// NewSampler attaches a [*Sampler] to a [*measurexlite.Trace], starts sampling in the +// NewSampler attaches a [*Sampler] to a [Trace], starts sampling in the // background and returns the [*Sampler]. Remember to call [*Sampler.Close] to stop // the background goroutine that performs the sampling. -func NewSampler(tx *measurexlite.Trace) *Sampler { +func NewSampler(tx Trace) *Sampler { ctx, cancel := context.WithCancel(context.Background()) smpl := &Sampler{ cancel: cancel, @@ -95,7 +114,7 @@ const BytesReceivedCumulativeOperation = "bytes_received_cumulative" func (smpl *Sampler) collectSnapshot(stats map[string]int64) { // compute just once the events sampling time - now := smpl.tx.TimeSince(smpl.tx.ZeroTime).Seconds() + now := smpl.tx.TimeSince(smpl.tx.ZeroTime()).Seconds() // process each entry for key, count := range stats { @@ -116,7 +135,7 @@ func (smpl *Sampler) collectSnapshot(stats map[string]int64) { Proto: network, T0: now, T: now, - TransactionID: smpl.tx.Index, + TransactionID: smpl.tx.Index(), Tags: smpl.tx.Tags(), } diff --git a/pkg/tunnel/tordesktop.go b/pkg/tunnel/tordesktop.go index bda2bb74..a517ffc3 100644 --- a/pkg/tunnel/tordesktop.go +++ b/pkg/tunnel/tordesktop.go @@ -1,4 +1,4 @@ -//go:build !android && !ios && !ooni_libtor +//go:build !ooni_libtor package tunnel @@ -6,7 +6,7 @@ package tunnel // This file implements our strategy for running tor on desktop in most // configurations except for the ooni_libtor case, where we build tor and // its dependencies for Linux. The purpuse of this special case it that -// of testing the otherwise untested code that would run on Android. +// of testing the otherwise untested code that would run on mobile. // import ( diff --git a/pkg/tunnel/torembed.go b/pkg/tunnel/torembed.go index 346f7edc..715a980c 100644 --- a/pkg/tunnel/torembed.go +++ b/pkg/tunnel/torembed.go @@ -1,14 +1,13 @@ -//go:build ooni_libtor && android +//go:build ooni_libtor package tunnel // // This file implements the ooni_libtor strategy of embedding tor. We manually -// compile tor and its dependencies and link against it. We currently only adopt -// this technique for Android. We may possibly migrate also iOS in the future, -// provided that this functionality proves to be stable in the 3.17 cycle. +// compile tor and its dependencies and link against it. // -// See https://github.com/ooni/probe/issues/2365. +// See https://github.com/ooni/probe/issues/2365 and +// https://github.com/ooni/probe/issues/2564. // import ( diff --git a/pkg/tunnel/tormobile.go b/pkg/tunnel/tormobile.go deleted file mode 100644 index 651ed325..00000000 --- a/pkg/tunnel/tormobile.go +++ /dev/null @@ -1,35 +0,0 @@ -//go:build ios || (android && !ooni_libtor) - -package tunnel - -// -// This file implements our old strategy for running tor on mobile, which -// is based on integrating github.com/ooni/go-libtor. We currently only use -// this stategy on iOS. See https://github.com/ooni/probe/issues/2365. -// - -import ( - "strings" - - "github.com/cretz/bine/tor" - "github.com/ooni/go-libtor" -) - -// getTorStartConf in this configuration uses github.com/ooni/go-libtor. -func getTorStartConf(config *Config, dataDir string, extraArgs []string) (*tor.StartConf, error) { - config.logger().Infof("tunnel: tor: exec: %s %s", - dataDir, strings.Join(extraArgs, " ")) - return &tor.StartConf{ - // Implementation note: go-libtor leaks a file descriptor when you set - // UseEmbeddedControlConn, as documented by - // - // https://github.com/ooni/probe/issues/2405 - // - // This is why we're not using this field for now. The above mentioned - // issue also refers to what a possible fix would look like. - ProcessCreator: libtor.Creator, - DataDir: dataDir, - ExtraArgs: extraArgs, - NoHush: true, - }, nil -} diff --git a/pkg/tutorial/dslx/chapter02/README.md b/pkg/tutorial/dslx/chapter02/README.md index 57222181..adac8b74 100644 --- a/pkg/tutorial/dslx/chapter02/README.md +++ b/pkg/tutorial/dslx/chapter02/README.md @@ -43,7 +43,6 @@ import ( "context" "errors" "net" - "sync/atomic" "github.com/ooni/probe-cli/v3/internal/dslx" "github.com/ooni/probe-cli/v3/internal/model" @@ -108,21 +107,6 @@ type Subresult struct { ``` -Subresult.mergeObservations merges the observations collected during -a measurement with the Subresult output data format. - -```Go - -func (tk *Subresult) mergeObservations(obs []*dslx.Observations) { - for _, o := range obs { - tk.NetworkEvents = append(tk.NetworkEvents, o.NetworkEvents...) - tk.TCPConnect = append(tk.TCPConnect, o.TCPConnect...) - tk.TLSHandshakes = append(tk.TLSHandshakes, o.TLSHandshakes...) - } -} - -``` - ## The Measurer The `Measurer` performs the measurement and implements `ExperimentMeasurer`; i.e., the @@ -133,7 +117,6 @@ of dslx pipelines a unique identifier). ```Go type Measurer struct { config Config - idGen atomic.Int64 } var _ model.ExperimentMeasurer = &Measurer{} @@ -178,15 +161,6 @@ So, this is where we will use `dslx` to implement the SNI blocking experiment. func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { ``` -### Define measurement parameters - -`sess` is the session of this measurement run. - -```Go - sess := args.Session - -``` - `measurement` contains metadata, the (required) input in form of the target SNI, and the nettest results (`TestKeys`). @@ -249,25 +223,31 @@ experiment's start time. ```Go dnsInput := dslx.NewDomainToResolve( dslx.DomainName(thaddrHost), - dslx.DNSLookupOptionIDGenerator(&m.idGen), - dslx.DNSLookupOptionLogger(sess.Logger()), - dslx.DNSLookupOptionZeroTime(measurement.MeasurementStartTimeSaved), ) ``` +Next, we create a minimal runtime. This data structure helps us to manage +open connections and close them when `rt.Close` is invoked. + +```Go + rt := dslx.NewMinimalRuntime(args.Session.Logger(), args.Measurement.MeasurementStartTimeSaved) + defer rt.Close() + +``` + We construct the resolver dslx function which can be - like in this case - the system resolver, or a custom UDP resolver. ```Go - lookupFn := dslx.DNSLookupGetaddrinfo() + lookupFn := dslx.DNSLookupGetaddrinfo(rt) ``` Then we apply the `dnsInput` argument to `lookupFn` to get a `dnsResult`. ```Go - dnsResult := lookupFn.Apply(ctx, dnsInput) + dnsResult := lookupFn.Apply(ctx, dslx.NewMaybeWithValue(dnsInput)) ``` @@ -322,24 +302,12 @@ the protocol, address, and port three-tuple.) dslx.EndpointNetwork("tcp"), dslx.EndpointPort(443), dslx.EndpointOptionDomain(m.config.TestHelperAddress), - dslx.EndpointOptionIDGenerator(&m.idGen), - dslx.EndpointOptionLogger(sess.Logger()), - dslx.EndpointOptionZeroTime(measurement.MeasurementStartTimeSaved), ) runtimex.Assert(len(endpoints) >= 1, "expected at least one endpoint here") endpoint := endpoints[0] ``` -Next, we create a connection pool. This data structure helps us to manage -open connections and close them when `connpool.Close` is invoked. - -```Go - connpool := &dslx.ConnPool{} - defer connpool.Close() - -``` - In the following we compose step-by-step measurement "pipelines", represented by `dslx` functions. @@ -350,9 +318,9 @@ target SNI to be used within the TLS Client Hello. ```Go pipelineTarget := dslx.Compose2( - dslx.TCPConnect(connpool), + dslx.TCPConnect(rt), dslx.TLSHandshake( - connpool, + rt, dslx.TLSHandshakeOptionServerName(targetSNI), ), ) @@ -364,9 +332,9 @@ specify the *control* SNI to be used within the TLS Client Hello. ```Go pipelineControl := dslx.Compose2( - dslx.TCPConnect(connpool), + dslx.TCPConnect(rt), dslx.TLSHandshake( - connpool, + rt, dslx.TLSHandshakeOptionServerName(m.config.ControlSNI), ), ) @@ -379,8 +347,8 @@ data structure called `Maybe`, which contains either the endpoint measurement re (on success) or an error (in case of failure). ```Go - var targetResult *dslx.Maybe[*dslx.TLSConnection] = pipelineTarget.Apply(ctx, endpoint) - var controlResult *dslx.Maybe[*dslx.TLSConnection] = pipelineControl.Apply(ctx, endpoint) + var targetResult *dslx.Maybe[*dslx.TLSConnection] = pipelineTarget.Apply(ctx, dslx.NewMaybeWithValue(endpoint)) + var controlResult *dslx.Maybe[*dslx.TLSConnection] = pipelineControl.Apply(ctx, dslx.NewMaybeWithValue(endpoint)) ``` @@ -446,20 +414,6 @@ Store the control failure if any. ``` -The measurement result not only contains the potential error, but also -observations that have been collected during each step of the measurement pipeline. -Observations are for example network events like read and write operations, -TLS handshakes, or DNS queries. We as experiment programmers are responsible for -extracting these observations from the dslx measurement result and storing -them in the `TestKeys`, which is precisely what `Subresult.mergeObservations` -(implemented above) does. - -```Go - tk.Target.mergeObservations(targetResult.Observations) - tk.Control.mergeObservations(controlResult.Observations) - -``` - ### Return Finally, we can return, as the measurement ran successfully. diff --git a/pkg/tutorial/dslx/chapter02/main.go b/pkg/tutorial/dslx/chapter02/main.go index 6109adfb..6c6c3b0d 100644 --- a/pkg/tutorial/dslx/chapter02/main.go +++ b/pkg/tutorial/dslx/chapter02/main.go @@ -44,7 +44,6 @@ import ( "context" "errors" "net" - "sync/atomic" "github.com/ooni/probe-engine/pkg/dslx" "github.com/ooni/probe-engine/pkg/model" @@ -107,21 +106,6 @@ type Subresult struct { Cached bool `json:"-"` } -// ``` -// -// Subresult.mergeObservations merges the observations collected during -// a measurement with the Subresult output data format. -// -// ```Go - -func (tk *Subresult) mergeObservations(obs []*dslx.Observations) { - for _, o := range obs { - tk.NetworkEvents = append(tk.NetworkEvents, o.NetworkEvents...) - tk.TCPConnect = append(tk.TCPConnect, o.TCPConnect...) - tk.TLSHandshakes = append(tk.TLSHandshakes, o.TLSHandshakes...) - } -} - // ``` // // ## The Measurer @@ -134,7 +118,6 @@ func (tk *Subresult) mergeObservations(obs []*dslx.Observations) { // ```Go type Measurer struct { config Config - idGen atomic.Int64 } var _ model.ExperimentMeasurer = &Measurer{} @@ -177,15 +160,6 @@ func (m *Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, // // ```Go func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { - // ``` - // - // ### Define measurement parameters - // - // `sess` is the session of this measurement run. - // - // ```Go - sess := args.Session - // ``` // // `measurement` contains metadata, the (required) input in form of @@ -250,25 +224,31 @@ func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { // ```Go dnsInput := dslx.NewDomainToResolve( dslx.DomainName(thaddrHost), - dslx.DNSLookupOptionIDGenerator(&m.idGen), - dslx.DNSLookupOptionLogger(sess.Logger()), - dslx.DNSLookupOptionZeroTime(measurement.MeasurementStartTimeSaved), ) + // ``` + // + // Next, we create a minimal runtime. This data structure helps us to manage + // open connections and close them when `rt.Close` is invoked. + // + // ```Go + rt := dslx.NewMinimalRuntime(args.Session.Logger(), args.Measurement.MeasurementStartTimeSaved) + defer rt.Close() + // ``` // // We construct the resolver dslx function which can be - like in this case - the // system resolver, or a custom UDP resolver. // // ```Go - lookupFn := dslx.DNSLookupGetaddrinfo() + lookupFn := dslx.DNSLookupGetaddrinfo(rt) // ``` // // Then we apply the `dnsInput` argument to `lookupFn` to get a `dnsResult`. // // ```Go - dnsResult := lookupFn.Apply(ctx, dnsInput) + dnsResult := lookupFn.Apply(ctx, dslx.NewMaybeWithValue(dnsInput)) // ``` // @@ -323,22 +303,10 @@ func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { dslx.EndpointNetwork("tcp"), dslx.EndpointPort(443), dslx.EndpointOptionDomain(m.config.TestHelperAddress), - dslx.EndpointOptionIDGenerator(&m.idGen), - dslx.EndpointOptionLogger(sess.Logger()), - dslx.EndpointOptionZeroTime(measurement.MeasurementStartTimeSaved), ) runtimex.Assert(len(endpoints) >= 1, "expected at least one endpoint here") endpoint := endpoints[0] - // ``` - // - // Next, we create a connection pool. This data structure helps us to manage - // open connections and close them when `connpool.Close` is invoked. - // - // ```Go - connpool := &dslx.ConnPool{} - defer connpool.Close() - // ``` // // In the following we compose step-by-step measurement "pipelines", @@ -351,9 +319,9 @@ func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { // // ```Go pipelineTarget := dslx.Compose2( - dslx.TCPConnect(connpool), + dslx.TCPConnect(rt), dslx.TLSHandshake( - connpool, + rt, dslx.TLSHandshakeOptionServerName(targetSNI), ), ) @@ -365,9 +333,9 @@ func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { // // ```Go pipelineControl := dslx.Compose2( - dslx.TCPConnect(connpool), + dslx.TCPConnect(rt), dslx.TLSHandshake( - connpool, + rt, dslx.TLSHandshakeOptionServerName(m.config.ControlSNI), ), ) @@ -380,8 +348,8 @@ func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { // (on success) or an error (in case of failure). // // ```Go - var targetResult *dslx.Maybe[*dslx.TLSConnection] = pipelineTarget.Apply(ctx, endpoint) - var controlResult *dslx.Maybe[*dslx.TLSConnection] = pipelineControl.Apply(ctx, endpoint) + var targetResult *dslx.Maybe[*dslx.TLSConnection] = pipelineTarget.Apply(ctx, dslx.NewMaybeWithValue(endpoint)) + var controlResult *dslx.Maybe[*dslx.TLSConnection] = pipelineControl.Apply(ctx, dslx.NewMaybeWithValue(endpoint)) // ``` // @@ -445,20 +413,6 @@ func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { tk.Control.Failure = &failure } - // ``` - // - // The measurement result not only contains the potential error, but also - // observations that have been collected during each step of the measurement pipeline. - // Observations are for example network events like read and write operations, - // TLS handshakes, or DNS queries. We as experiment programmers are responsible for - // extracting these observations from the dslx measurement result and storing - // them in the `TestKeys`, which is precisely what `Subresult.mergeObservations` - // (implemented above) does. - // - // ```Go - tk.Target.mergeObservations(targetResult.Observations) - tk.Control.mergeObservations(controlResult.Observations) - // ``` // // ### Return