Merge pull request #2671 from fermyon/revert-2663-run-conformance

Revert "Run conformance tests as part of runtime tests"

Cargo.toml

@@ -100,8 +100,6 @@ runtime-tests = { path = "tests/runtime-tests" }
 test-components = { path = "tests/test-components" }
 test-codegen-macro = { path = "crates/test-codegen-macro" }
 test-environment = { git = "https://github.com/fermyon/conformance-tests", rev = "d2129a3fd73140a76c77f15a030a5273b37cbd11" }
-conformance-tests = { git = "https://github.com/fermyon/conformance-tests", rev = "d2129a3fd73140a76c77f15a030a5273b37cbd11" }
-conformance = { path = "tests/conformance-tests" }
 
 [build-dependencies]
 cargo-target-dep = { git = "https://github.com/fermyon/cargo-target-dep", rev = "482f269eceb7b1a7e8fc618bf8c082dd24979cf1" }

tests/conformance-tests/src/main.rs

@@ -1,7 +1,63 @@
+use anyhow::Context as _;
+use testing_framework::runtimes::spin_cli::{SpinCli, SpinConfig};
+
 fn main() {
     let spin_binary: std::path::PathBuf = std::env::args()
         .nth(1)
         .expect("expected first argument to be path to spin binary")
         .into();
-    conformance_tests::run_tests(move |test| conformance::run_test(test, &spin_binary)).unwrap();
+    conformance_tests::run_tests(move |test| run_test(test, &spin_binary)).unwrap();
+}
+
+fn run_test(test: conformance_tests::Test, spin_binary: &std::path::Path) -> anyhow::Result<()> {
+    let mut services = Vec::new();
+    for precondition in test.config.preconditions {
+        match precondition {
+            conformance_tests::config::Precondition::HttpEcho => {
+                services.push("http-echo");
+            }
+            conformance_tests::config::Precondition::TcpEcho => {
+                services.push("tcp-echo");
+            }
+            conformance_tests::config::Precondition::Redis => services.push("redis"),
+            conformance_tests::config::Precondition::Mqtt => services.push("mqtt"),
+            conformance_tests::config::Precondition::KeyValueStore(_) => {}
+            conformance_tests::config::Precondition::Sqlite => {}
+        }
+    }
+    let env_config = SpinCli::config(
+        SpinConfig {
+            binary_path: spin_binary.to_owned(),
+            spin_up_args: Vec::new(),
+            app_type: testing_framework::runtimes::SpinAppType::Http,
+        },
+        test_environment::services::ServicesConfig::new(services)?,
+        move |e| {
+            let mut manifest =
+                test_environment::manifest_template::EnvTemplate::from_file(&test.manifest)?;
+            manifest.substitute(e, |_| None)?;
+            e.write_file("spin.toml", manifest.contents())?;
+            e.copy_into(&test.component, test.component.file_name().unwrap())?;
+            Ok(())
+        },
+    );
+    let mut env = test_environment::TestEnvironment::up(env_config, |_| Ok(()))?;
+    for invocation in test.config.invocations {
+        let conformance_tests::config::Invocation::Http(mut invocation) = invocation;
+        invocation.request.substitute_from_env(&mut env)?;
+        let spin = env.runtime_mut();
+        let actual = invocation
+            .request
+            .send(|request| spin.make_http_request(request))?;
+
+        conformance_tests::assertions::assert_response(&invocation.response, &actual)
+            .with_context(|| {
+                format!(
+                    "Failed assertion.\nstdout: {}\nstderr: {}",
+                    spin.stdout().to_owned(),
+                    spin.stderr()
+                )
+            })?;
+    }
+    Ok(())
 }

tests/runtime-tests/tests/key-value-no-permission/error.txt

@@ -0,0 +1 @@
+Error::AccessDenied

tests/runtime-tests/tests/key-value-no-permission/spin.toml

@@ -0,0 +1,13 @@
+spin_manifest_version = 2
+
+[application]
+name = "key-value"
+authors = ["Fermyon Engineering <engineering@fermyon.com>"]
+version = "0.1.0"
+
+[[trigger.http]]
+route = "/"
+component = "test"
+
+[component.test]
+source = "%{source=key-value}"

tests/runtime-tests/tests/key-value/spin.toml

@@ -0,0 +1,14 @@
+spin_manifest_version = 2
+
+[application]
+name = "key-value"
+authors = ["Fermyon Engineering <engineering@fermyon.com>"]
+version = "0.1.0"
+
+[[trigger.http]]
+route = "/"
+component = "test"
+
+[component.test]
+source = "%{source=key-value}"
+key_value_stores = ["default"]

tests/runtime-tests/tests/outbound-mqtt-variable-permission/services

@@ -0,0 +1 @@
+mqtt

tests/runtime-tests/tests/outbound-mqtt-variable-permission/spin.toml

@@ -0,0 +1,19 @@
+spin_manifest_version = 2
+
+[application]
+name = "outbound-mqtt"
+authors = ["Suneet Nangia <suneetnangia@gmail.com>"]
+version = "0.1.0"
+
+[variables]
+mqtt_server = { default = "localhost" }
+
+[[trigger.http]]
+route = "/"
+component = "test"
+
+[component.test]
+source = "%{source=outbound-mqtt}"
+allowed_outbound_hosts = ["mqtt://{{ mqtt_server }}:%{port=1883}"]
+# To test anonymous MQTT authentication, remove the values from MQTT_USERNAME and MQTT_PASSWORD env variables.
+environment = { MQTT_ADDRESS = "mqtt://localhost:%{port=1883}?client_id=spintest", MQTT_USERNAME = "user", MQTT_PASSWORD = "password", MQTT_KEEP_ALIVE_INTERVAL = "30" }

tests/runtime-tests/tests/outbound-mqtt/services

@@ -0,0 +1 @@
+mqtt

tests/runtime-tests/tests/outbound-mqtt/spin.toml

@@ -0,0 +1,16 @@
+spin_manifest_version = 2
+
+[application]
+name = "outbound-mqtt"
+authors = ["Suneet Nangia <suneetnangia@gmail.com>"]
+version = "0.1.0"
+
+[[trigger.http]]
+route = "/"
+component = "test"
+
+[component.test]
+source = "%{source=outbound-mqtt}"
+allowed_outbound_hosts = ["mqtt://localhost:%{port=1883}"]
+# To test anonymous MQTT authentication, remove the values from MQTT_USERNAME and MQTT_PASSWORD env variables.
+environment = { MQTT_ADDRESS = "mqtt://localhost:%{port=1883}?client_id=spintest", MQTT_USERNAME = "user", MQTT_PASSWORD = "password", MQTT_KEEP_ALIVE_INTERVAL = "30" }

tests/runtime-tests/tests/outbound-redis-no-permission/error.txt

@@ -0,0 +1 @@
+Error::InvalidAddress

tests/runtime-tests/tests/outbound-redis-no-permission/spin.toml

@@ -0,0 +1,14 @@
+spin_manifest_version = 2
+
+[application]
+name = "outbound-redis"
+authors = ["Fermyon Engineering <engineering@fermyon.com>"]
+version = "0.1.0"
+
+[[trigger.http]]
+route = "/"
+component = "test"
+
+[component.test]
+source = "%{source=outbound-redis}"
+environment = { REDIS_ADDRESS = "redis://localhost:6379" }

tests/runtime-tests/tests/outbound-redis-variable-permission/services

@@ -0,0 +1 @@
+redis

tests/runtime-tests/tests/outbound-redis-variable-permission/spin.toml

@@ -0,0 +1,18 @@
+spin_manifest_version = 2
+
+[application]
+name = "outbound-redis"
+authors = ["Fermyon Engineering <engineering@fermyon.com>"]
+version = "0.1.0"
+
+[variables]
+redis_host = { default = "localhost" }
+
+[[trigger.http]]
+route = "/"
+component = "test"
+
+[component.test]
+source = "%{source=outbound-redis}"
+environment = { REDIS_ADDRESS = "redis://localhost:%{port=6379}" }
+allowed_outbound_hosts = ["redis://{{ redis_host }}:%{port=6379}"]

tests/runtime-tests/tests/outbound-redis/services

@@ -0,0 +1 @@
+redis

tests/runtime-tests/tests/outbound-redis/spin.toml

@@ -0,0 +1,15 @@
+spin_manifest_version = 2
+
+[application]
+name = "outbound-redis"
+authors = ["Fermyon Engineering <engineering@fermyon.com>"]
+version = "0.1.0"
+
+[[trigger.http]]
+route = "/"
+component = "test"
+
+[component.test]
+source = "%{source=outbound-redis}"
+environment = { REDIS_ADDRESS = "redis://localhost:%{port=6379}" }
+allowed_outbound_hosts = ["redis://localhost:%{port=6379}"]

tests/runtime-tests/tests/sqlite-no-permission/error.txt

@@ -0,0 +1 @@
+Error::AccessDenied

tests/runtime-tests/tests/sqlite-no-permission/spin.toml

@@ -0,0 +1,13 @@
+spin_manifest_version = 2
+
+[application]
+name = "sqlite"
+authors = ["Ryan Levick <ryan.levick@fermyon.com>"]
+version = "0.1.0"
+
+[[trigger.http]]
+route = "/"
+component = "test"
+
+[component.test]
+source = "%{source=sqlite}"

tests/runtime-tests/tests/sqlite/spin.toml

@@ -0,0 +1,14 @@
+spin_manifest_version = 2
+
+[application]
+name = "sqlite"
+authors = ["Ryan Levick <ryan.levick@fermyon.com>"]
+version = "0.1.0"
+
+[[trigger.http]]
+route = "/"
+component = "test"
+
+[component.test]
+source = "%{source=sqlite}"
+sqlite_databases = ["default"]

tests/runtime-tests/tests/tcp-sockets-ip-range-variable-permission/services

@@ -0,0 +1 @@
+tcp-echo

tests/runtime-tests/tests/tcp-sockets-ip-range-variable-permission/spin.toml

@@ -0,0 +1,19 @@
+spin_manifest_version = 2
+
+[application]
+name = "tcp-sockets"
+authors = ["Fermyon Engineering <engineering@fermyon.com>"]
+version = "0.1.0"
+
+[variables]
+addr_prefix = { default = "127.0.0.0" }
+prefix_len = { default = "24" }
+
+[[trigger.http]]
+route = "/"
+component = "test"
+
+[component.test]
+source = "%{source=tcp-sockets}"
+environment = { ADDRESS = "127.0.0.1:%{port=7}" }
+allowed_outbound_hosts = ["*://{{ addr_prefix }}/{{ prefix_len }}:%{port=7}"]

tests/runtime-tests/tests/tcp-sockets-ip-range/services

@@ -0,0 +1 @@
+tcp-echo

tests/runtime-tests/tests/tcp-sockets-ip-range/spin.toml

@@ -0,0 +1,15 @@
+spin_manifest_version = 2
+
+[application]
+name = "tcp-sockets"
+authors = ["Fermyon Engineering <engineering@fermyon.com>"]
+version = "0.1.0"
+
+[[trigger.http]]
+route = "/"
+component = "test"
+
+[component.test]
+source = "%{source=tcp-sockets}"
+environment = { ADDRESS = "127.0.0.1:%{port=7}" }
+allowed_outbound_hosts = ["*://127.0.0.0/24:%{port=7}"]

tests/runtime-tests/tests/tcp-sockets-no-ip-permission/error.txt

@@ -0,0 +1 @@
+access-denied

tests/runtime-tests/tests/tcp-sockets-no-ip-permission/spin.toml

@@ -0,0 +1,16 @@
+spin_manifest_version = 2
+
+[application]
+name = "tcp-sockets"
+authors = ["Fermyon Engineering <engineering@fermyon.com>"]
+version = "0.1.0"
+
+[[trigger.http]]
+route = "/"
+component = "test"
+
+[component.test]
+source = "%{source=tcp-sockets}"
+environment = { ADDRESS = "127.0.0.1:6001" }
+# Component expects 127.0.0.1 but we only allow 127.0.0.2
+allowed_outbound_hosts = ["*://127.0.0.2:6001"]

tests/runtime-tests/tests/tcp-sockets-no-port-permission/error.txt

@@ -0,0 +1 @@
+access-denied

tests/runtime-tests/tests/tcp-sockets-no-port-permission/spin.toml

@@ -0,0 +1,16 @@
+spin_manifest_version = 2
+
+[application]
+name = "tcp-sockets"
+authors = ["Fermyon Engineering <engineering@fermyon.com>"]
+version = "0.1.0"
+
+[[trigger.http]]
+route = "/"
+component = "test"
+
+[component.test]
+source = "%{source=tcp-sockets}"
+environment = { ADDRESS = "127.0.0.1:6001" }
+# Component expects port 6001 but we allow 6002
+allowed_outbound_hosts = ["*://127.0.0.1:6002"]

tests/runtime-tests/tests/tcp-sockets/services

@@ -0,0 +1 @@
+tcp-echo