From 5bbea5017103f3fc117e7f13d21d49a07ee9d532 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= <guillermo@otherplane.com>
Date: Tue, 10 Oct 2023 12:48:47 +0200
Subject: [PATCH 01/19] feat: implement HttpHead as new RADType
---
data_structures/src/chain/mod.rs | 10 ++++--
data_structures/src/proto/mod.rs | 2 ++
data_structures/src/serialization_helpers.rs | 4 +--
rad/src/lib.rs | 36 +++++++++++++++++---
schemas/witnet/witnet.proto | 3 +-
5 files changed, 45 insertions(+), 10 deletions(-)
diff --git a/data_structures/src/chain/mod.rs b/data_structures/src/chain/mod.rs
index 34910b8a5..757f6c46e 100644
--- a/data_structures/src/chain/mod.rs
+++ b/data_structures/src/chain/mod.rs
@@ -1645,11 +1645,14 @@ pub enum RADType {
/// HTTP POST request
#[serde(rename = "HTTP-POST")]
HttpPost,
+ /// HTTP HEAD request
+ #[serde(rename = "HTTP-HEAD")]
+ HttpHead,
}
impl RADType {
pub fn is_http(&self) -> bool {
- matches!(self, RADType::HttpGet | RADType::HttpPost)
+ matches!(self, RADType::HttpGet | RADType::HttpPost | RADType::HttpHead)
}
}
@@ -1701,7 +1704,7 @@ pub struct RADRetrieve {
pub script: Vec<u8>,
/// Body of a HTTP-POST request
pub body: Vec<u8>,
- /// Extra headers of a HTTP-GET or HTTP-POST request
+ /// Extra headers of a HTTP-GET, HTTP-POST or HTTP-HEAD request
pub headers: Vec<(String, String)>,
}
@@ -1809,7 +1812,8 @@ impl RADRetrieve {
&[Field::Kind, Field::Url, Field::Script],
&[Field::Body, Field::Headers],
)
- }
+ },
+ RADType::HttpHead => check(&[Field::Kind, Field::Url, Field::Script], &[Field::Headers])
}
}
diff --git a/data_structures/src/proto/mod.rs b/data_structures/src/proto/mod.rs
index 4d681d8b9..c1bb007ca 100644
--- a/data_structures/src/proto/mod.rs
+++ b/data_structures/src/proto/mod.rs
@@ -51,6 +51,7 @@ impl ProtobufConvert for chain::RADType {
chain::RADType::HttpGet => witnet::DataRequestOutput_RADRequest_RADType::HttpGet,
chain::RADType::Rng => witnet::DataRequestOutput_RADRequest_RADType::Rng,
chain::RADType::HttpPost => witnet::DataRequestOutput_RADRequest_RADType::HttpPost,
+ chain::RADType::HttpHead => witnet::DataRequestOutput_RADRequest_RADType::HttpHead,
}
}
@@ -60,6 +61,7 @@ impl ProtobufConvert for chain::RADType {
witnet::DataRequestOutput_RADRequest_RADType::HttpGet => chain::RADType::HttpGet,
witnet::DataRequestOutput_RADRequest_RADType::Rng => chain::RADType::Rng,
witnet::DataRequestOutput_RADRequest_RADType::HttpPost => chain::RADType::HttpPost,
+ witnet::DataRequestOutput_RADRequest_RADType::HttpHead => chain::RADType::HttpHead,
})
}
}
diff --git a/data_structures/src/serialization_helpers.rs b/data_structures/src/serialization_helpers.rs
index 7f7d7d4e6..a8d59ee69 100644
--- a/data_structures/src/serialization_helpers.rs
+++ b/data_structures/src/serialization_helpers.rs
@@ -360,7 +360,7 @@ struct RADRetrieveSerializationHelperJson {
/// Body of a HTTP-POST request
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub body: Vec<u8>,
- /// Extra headers of a HTTP-GET or HTTP-POST request
+ /// Extra headers of a HTTP-GET, HTTP-HEAD or HTTP-POST request
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub headers: Vec<(String, String)>,
}
@@ -377,7 +377,7 @@ struct RADRetrieveSerializationHelperBincode {
pub script: Vec<u8>,
/// Body of a HTTP-POST request
pub body: Vec<u8>,
- /// Extra headers of a HTTP-GET or HTTP-POST request
+ /// Extra headers of a HTTP-GET, HTTP-HEAD or HTTP-POST request
pub headers: Vec<(String, String)>,
}
diff --git a/rad/src/lib.rs b/rad/src/lib.rs
index 64c826780..a3781df42 100644
--- a/rad/src/lib.rs
+++ b/rad/src/lib.rs
@@ -25,10 +25,11 @@ use crate::{
create_radon_script_from_filters_and_reducer, execute_radon_script, unpack_radon_script,
RadonScriptExecutionSettings,
},
- types::{array::RadonArray, bytes::RadonBytes, string::RadonString, RadonTypes},
+ types::{array::RadonArray, bytes::RadonBytes, map::RadonMap, string::RadonString, RadonTypes},
user_agents::UserAgent,
};
use core::convert::From;
+use std::collections::BTreeMap;
use witnet_net::client::http::{WitnetHttpBody, WitnetHttpRequest};
pub mod conditions;
@@ -173,6 +174,25 @@ fn string_response_with_data_report(
execute_radon_script(input, &radon_script, context, settings)
}
+/// Handle HTTP-HEAD response with data, and return a `RadonReport`.
+fn headers_response_with_data_report(
+ retrieve: &RADRetrieve,
+ response: &str,
+ context: &mut ReportContext<RadonTypes>,
+ settings: RadonScriptExecutionSettings,
+) -> Result<RadonReport<RadonTypes>> {
+ let headers: BTreeMap<String, RadonTypes> = response.split("\r\n").map(|line| {
+ let parts: Vec<&str> = line.split(":").map(|part| part.trim()).collect();
+ // todo: check there are two parts, and two parts only
+ // todo: make sure that values from repeated keys get appended within a RadonArray
+ (String::from(parts[0]), RadonTypes::from(RadonString::from(parts[1])))
+ }).collect();
+ let input = RadonTypes::from(RadonMap::from(headers));
+ let radon_script = unpack_radon_script(&retrieve.script)?;
+
+ execute_radon_script(input, &radon_script, context, settings)
+}
+
/// Handle Rng response with data report
fn rng_response_with_data_report(
response: &str,
@@ -196,7 +216,10 @@ pub fn run_retrieval_with_data_report(
RADType::Rng => rng_response_with_data_report(response, context),
RADType::HttpPost => {
string_response_with_data_report(retrieve, response, context, settings)
- }
+ },
+ RADType::HttpHead => {
+ headers_response_with_data_report(retrieve, response, context, settings)
+ },
_ => Err(RadError::UnknownRetrieval),
}
}
@@ -214,7 +237,7 @@ pub fn run_retrieval_with_data(
.map(RadonReport::into_inner)
}
-/// Handle generic HTTP (GET/POST) response
+/// Handle generic HTTP (GET/POST/HEAD) response
async fn http_response(
retrieve: &RADRetrieve,
context: &mut ReportContext<RadonTypes>,
@@ -258,7 +281,11 @@ async fn http_response(
builder.method("POST").uri(&retrieve.url),
WitnetHttpBody::from(retrieve.body.clone()),
)
- }
+ },
+ RADType::HttpHead => (
+ builder.method("HEAD").uri(&retrieve.url),
+ WitnetHttpBody::empty(),
+ ),
_ => panic!(
"Called http_response with invalid retrieval kind {:?}",
retrieve.kind
@@ -357,6 +384,7 @@ pub async fn run_retrieval_report(
RADType::HttpGet => http_response(retrieve, context, settings, client).await,
RADType::Rng => rng_response(context, settings).await,
RADType::HttpPost => http_response(retrieve, context, settings, client).await,
+ RADType::HttpHead => http_response(retrieve, context, settings, client).await,
_ => Err(RadError::UnknownRetrieval),
}
}
diff --git a/schemas/witnet/witnet.proto b/schemas/witnet/witnet.proto
index 64b1b04e0..842009744 100644
--- a/schemas/witnet/witnet.proto
+++ b/schemas/witnet/witnet.proto
@@ -121,6 +121,7 @@ message DataRequestOutput {
HttpGet = 1;
Rng = 2;
HttpPost = 3;
+ HttpHead = 4;
}
message RADFilter {
uint32 op = 1;
@@ -133,7 +134,7 @@ message DataRequestOutput {
bytes script = 3;
// Body of HTTP-POST request
bytes body = 4;
- // Extra headers for HTTP-GET and HTTP-POST requests
+ // Extra headers for HTTP-GET, HTTP-HEAD and HTTP-POST requests
repeated StringPair headers = 5;
}
message RADAggregate {
From 3fdf8f973dcbb168149faafad727cbc084b6ed67 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= <guillermo@otherplane.com>
Date: Tue, 10 Oct 2023 13:26:31 +0200
Subject: [PATCH 02/19] chore: cargo fmt --all
---
data_structures/src/chain/mod.rs | 11 ++++++++---
rad/src/lib.rs | 22 ++++++++++++++--------
2 files changed, 22 insertions(+), 11 deletions(-)
diff --git a/data_structures/src/chain/mod.rs b/data_structures/src/chain/mod.rs
index 757f6c46e..7ed2e38e0 100644
--- a/data_structures/src/chain/mod.rs
+++ b/data_structures/src/chain/mod.rs
@@ -1652,7 +1652,10 @@ pub enum RADType {
impl RADType {
pub fn is_http(&self) -> bool {
- matches!(self, RADType::HttpGet | RADType::HttpPost | RADType::HttpHead)
+ matches!(
+ self,
+ RADType::HttpGet | RADType::HttpPost | RADType::HttpHead
+ )
}
}
@@ -1812,8 +1815,10 @@ impl RADRetrieve {
&[Field::Kind, Field::Url, Field::Script],
&[Field::Body, Field::Headers],
)
- },
- RADType::HttpHead => check(&[Field::Kind, Field::Url, Field::Script], &[Field::Headers])
+ }
+ RADType::HttpHead => {
+ check(&[Field::Kind, Field::Url, Field::Script], &[Field::Headers])
+ }
}
}
diff --git a/rad/src/lib.rs b/rad/src/lib.rs
index a3781df42..dfa623910 100644
--- a/rad/src/lib.rs
+++ b/rad/src/lib.rs
@@ -181,12 +181,18 @@ fn headers_response_with_data_report(
context: &mut ReportContext<RadonTypes>,
settings: RadonScriptExecutionSettings,
) -> Result<RadonReport<RadonTypes>> {
- let headers: BTreeMap<String, RadonTypes> = response.split("\r\n").map(|line| {
- let parts: Vec<&str> = line.split(":").map(|part| part.trim()).collect();
- // todo: check there are two parts, and two parts only
- // todo: make sure that values from repeated keys get appended within a RadonArray
- (String::from(parts[0]), RadonTypes::from(RadonString::from(parts[1])))
- }).collect();
+ let headers: BTreeMap<String, RadonTypes> = response
+ .split("\r\n")
+ .map(|line| {
+ let parts: Vec<&str> = line.split(":").map(|part| part.trim()).collect();
+ // todo: check there are two parts, and two parts only
+ // todo: make sure that values from repeated keys get appended within a RadonArray
+ (
+ String::from(parts[0]),
+ RadonTypes::from(RadonString::from(parts[1])),
+ )
+ })
+ .collect();
let input = RadonTypes::from(RadonMap::from(headers));
let radon_script = unpack_radon_script(&retrieve.script)?;
@@ -216,10 +222,10 @@ pub fn run_retrieval_with_data_report(
RADType::Rng => rng_response_with_data_report(response, context),
RADType::HttpPost => {
string_response_with_data_report(retrieve, response, context, settings)
- },
+ }
RADType::HttpHead => {
headers_response_with_data_report(retrieve, response, context, settings)
- },
+ }
_ => Err(RadError::UnknownRetrieval),
}
}
From 721347a1b316e385bdc0225fdc3862d5ff69b6b0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= <guillermo@otherplane.com>
Date: Wed, 11 Oct 2023 10:45:08 +0200
Subject: [PATCH 03/19] feat: handle repeated headers within http/head response
---
rad/src/lib.rs | 37 ++++++++++++++++++++++++++-----------
1 file changed, 26 insertions(+), 11 deletions(-)
diff --git a/rad/src/lib.rs b/rad/src/lib.rs
index dfa623910..532a67668 100644
--- a/rad/src/lib.rs
+++ b/rad/src/lib.rs
@@ -181,18 +181,33 @@ fn headers_response_with_data_report(
context: &mut ReportContext<RadonTypes>,
settings: RadonScriptExecutionSettings,
) -> Result<RadonReport<RadonTypes>> {
- let headers: BTreeMap<String, RadonTypes> = response
- .split("\r\n")
- .map(|line| {
- let parts: Vec<&str> = line.split(":").map(|part| part.trim()).collect();
- // todo: check there are two parts, and two parts only
- // todo: make sure that values from repeated keys get appended within a RadonArray
- (
- String::from(parts[0]),
- RadonTypes::from(RadonString::from(parts[1])),
- )
+ let mut headers: BTreeMap<String, Vec<RadonTypes>> = BTreeMap::new();
+
+ for line in response.split("\r\n") {
+ if let Some(first_colon_index) = line.find(":") {
+ // key: trim spaces and lower case all ascii chars left to the first colon character
+ let key = String::from(line[0..first_colon_index].trim().to_ascii_lowercase());
+ // value: trim spaces on the substring after the first colon character
+ let value = RadonTypes::from(RadonString::from(line[first_colon_index + 1..].trim()));
+ headers.entry(key).or_default().push(value);
+ }
+ }
+
+ let headers: BTreeMap<String, RadonTypes> = BTreeMap::from_iter(
+ headers.iter().map(|(key, value)| {
+ match value.len() {
+ len if len > 1 => (
+ key.clone(),
+ RadonTypes::from(RadonArray::from(value.to_vec()))
+ ),
+ _ => (
+ key.clone(),
+ value[0].clone()
+ )
+ }
})
- .collect();
+ );
+
let input = RadonTypes::from(RadonMap::from(headers));
let radon_script = unpack_radon_script(&retrieve.script)?;
From ef307337cc7b11b16b4c92a41839cbe2493229b8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= <guillermo@otherplane.com>
Date: Wed, 11 Oct 2023 10:47:30 +0200
Subject: [PATCH 04/19] test: run valid http head retrieval
---
rad/src/lib.rs | 32 ++++++++++++++++++++++++++++++++
1 file changed, 32 insertions(+)
diff --git a/rad/src/lib.rs b/rad/src/lib.rs
index 532a67668..d28a579ae 100644
--- a/rad/src/lib.rs
+++ b/rad/src/lib.rs
@@ -878,6 +878,38 @@ mod tests {
use super::*;
+ #[test]
+ fn test_run_http_head_retrieval() {
+ let script_r = Value::Array(vec![
+ Value::Array(vec![
+ Value::Integer(RadonOpCodes::MapGetString as i128),
+ Value::Text("etag".to_string()),
+ ]),
+ ]);
+ let packed_script_r = serde_cbor::to_vec(&script_r).unwrap();
+ println!("{:?}", packed_script_r);
+
+ let retrieve = RADRetrieve {
+ kind: RADType::HttpHead,
+ url: "https://witnet.io/_nuxt/img/dragon_reading.a37f8cb.png".to_string(),
+ script: packed_script_r,
+ body: vec![],
+ headers: vec![],
+ };
+ let response = "HTTP/1.1 200 OK\r\nContent-Type: image/png\r\nContent-Length: 498219\r\nContent-Length: 123456\r\neTag: \"64eca181-79a2b\"\r\n";
+ let result = run_retrieval_with_data(
+ &retrieve,
+ response,
+ RadonScriptExecutionSettings::disable_all(),
+ current_active_wips(),
+ ).unwrap();
+
+ match result {
+ RadonTypes::String(_) => {}
+ err => panic!("Error in run_retrieval: {:?}", err),
+ }
+ }
+
#[test]
fn test_run_retrieval() {
let script_r = Value::Array(vec![
From a6653977a7d3aa97da4bf1dcc19d07d239050b39 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= <guillermo@otherplane.com>
Date: Wed, 11 Oct 2023 10:48:31 +0200
Subject: [PATCH 05/19] test: try http-head data request w/ invalid request
header
---
rad/src/lib.rs | 54 +++++++++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 53 insertions(+), 1 deletion(-)
diff --git a/rad/src/lib.rs b/rad/src/lib.rs
index d28a579ae..e07bc4b32 100644
--- a/rad/src/lib.rs
+++ b/rad/src/lib.rs
@@ -302,7 +302,7 @@ async fn http_response(
builder.method("POST").uri(&retrieve.url),
WitnetHttpBody::from(retrieve.body.clone()),
)
- },
+ }
RADType::HttpHead => (
builder.method("HEAD").uri(&retrieve.url),
WitnetHttpBody::empty(),
@@ -1607,6 +1607,58 @@ mod tests {
}
}
+ #[test]
+ fn test_try_data_request_http_get_non_ascii_header_key() {
+ let script_r = Value::Array(vec![]);
+ let packed_script_r = serde_cbor::to_vec(&script_r).unwrap();
+ let body = Vec::from(String::from(""));
+ let headers = vec![("ñ", "value")];
+ let headers = headers
+ .into_iter()
+ .map(|(a, b)| (a.to_string(), b.to_string()))
+ .collect();
+ let request = RADRequest {
+ time_lock: 0,
+ retrieve: vec![RADRetrieve {
+ kind: RADType::HttpGet,
+ url: String::from("http://127.0.0.1"),
+ script: packed_script_r,
+ body,
+ headers,
+ }],
+ aggregate: RADAggregate {
+ filters: vec![],
+ reducer: RadonReducers::Mode as u32,
+ },
+ tally: RADTally {
+ filters: vec![],
+ reducer: RadonReducers::Mode as u32,
+ },
+ };
+ let report = try_data_request(
+ &request,
+ RadonScriptExecutionSettings::enable_all(),
+ None,
+ None,
+ );
+ let tally_result = report.tally.into_inner();
+
+ assert_eq!(
+ tally_result,
+ RadonTypes::RadonError(
+ RadonError::try_from(RadError::UnhandledIntercept {
+ inner: Some(Box::new(RadError::InvalidHttpHeader {
+ name: "ñ".to_string(),
+ value: "value".to_string(),
+ error: "invalid HTTP header name".to_string()
+ })),
+ message: None
+ })
+ .unwrap()
+ )
+ );
+ }
+
#[test]
fn test_try_data_request_http_post_non_ascii_header_key() {
let script_r = Value::Array(vec![]);
From 198e5b8f99f208f2a2662dcb81babc2eacf1d27c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= <guillermo@otherplane.com>
Date: Wed, 11 Oct 2023 10:49:09 +0200
Subject: [PATCH 06/19] chore: bump package versions
---
Cargo.lock | 14 +++++++-------
Cargo.toml | 2 +-
bridges/centralized-ethereum/Cargo.toml | 2 +-
data_structures/Cargo.toml | 2 +-
node/Cargo.toml | 2 +-
rad/Cargo.toml | 2 +-
toolkit/Cargo.toml | 2 +-
wallet/Cargo.toml | 2 +-
8 files changed, 14 insertions(+), 14 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index 2f0d09be9..4571cdfa6 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4958,7 +4958,7 @@ dependencies = [
[[package]]
name = "witnet"
-version = "1.6.7"
+version = "1.7.0"
dependencies = [
"ansi_term",
"bytecount",
@@ -5005,7 +5005,7 @@ dependencies = [
[[package]]
name = "witnet-centralized-ethereum-bridge"
-version = "1.6.7"
+version = "1.7.0"
dependencies = [
"actix",
"async-jsonrpc-client",
@@ -5069,7 +5069,7 @@ dependencies = [
[[package]]
name = "witnet_data_structures"
-version = "1.6.7"
+version = "1.7.0"
dependencies = [
"bech32",
"bencher",
@@ -5134,7 +5134,7 @@ dependencies = [
[[package]]
name = "witnet_node"
-version = "1.6.7"
+version = "1.7.0"
dependencies = [
"actix",
"ansi_term",
@@ -5198,7 +5198,7 @@ dependencies = [
[[package]]
name = "witnet_rad"
-version = "0.3.2"
+version = "0.3.3"
dependencies = [
"cbor-codec",
"failure",
@@ -5241,7 +5241,7 @@ dependencies = [
[[package]]
name = "witnet_toolkit"
-version = "1.6.7"
+version = "1.7.0"
dependencies = [
"failure",
"hex",
@@ -5284,7 +5284,7 @@ dependencies = [
[[package]]
name = "witnet_wallet"
-version = "1.6.7"
+version = "1.7.0"
dependencies = [
"actix",
"async-jsonrpc-client",
diff --git a/Cargo.toml b/Cargo.toml
index 0a428c2fb..4a1b64b65 100755
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,6 @@
[package]
name = "witnet"
-version = "1.6.7"
+version = "1.7.0"
authors = ["Witnet Foundation <info@witnet.foundation>"]
publish = false
repository = "witnet/witnet-rust"
diff --git a/bridges/centralized-ethereum/Cargo.toml b/bridges/centralized-ethereum/Cargo.toml
index 5492feb9e..b87084520 100644
--- a/bridges/centralized-ethereum/Cargo.toml
+++ b/bridges/centralized-ethereum/Cargo.toml
@@ -1,6 +1,6 @@
[package]
name = "witnet-centralized-ethereum-bridge"
-version = "1.6.7"
+version = "1.7.0"
authors = ["Witnet Foundation <info@witnet.foundation>"]
edition = "2018"
diff --git a/data_structures/Cargo.toml b/data_structures/Cargo.toml
index fa801502b..f09cf9699 100644
--- a/data_structures/Cargo.toml
+++ b/data_structures/Cargo.toml
@@ -3,7 +3,7 @@ authors = ["Witnet Foundation <info@witnet.foundation>"]
description = "data structures component"
edition = "2021"
name = "witnet_data_structures"
-version = "1.6.7"
+version = "1.7.0"
workspace = ".."
[features]
diff --git a/node/Cargo.toml b/node/Cargo.toml
index 796ca2737..88a31f51a 100644
--- a/node/Cargo.toml
+++ b/node/Cargo.toml
@@ -1,6 +1,6 @@
[package]
name = "witnet_node"
-version = "1.6.7"
+version = "1.7.0"
authors = ["Witnet Foundation <info@witnet.foundation>"]
workspace = ".."
description = "node component"
diff --git a/rad/Cargo.toml b/rad/Cargo.toml
index 806b62527..d9bef65f3 100644
--- a/rad/Cargo.toml
+++ b/rad/Cargo.toml
@@ -1,6 +1,6 @@
[package]
name = "witnet_rad"
-version = "0.3.2"
+version = "0.3.3"
authors = ["Witnet Foundation <info@witnet.foundation>"]
edition = "2021"
workspace = ".."
diff --git a/toolkit/Cargo.toml b/toolkit/Cargo.toml
index eb4a5096d..5ee64aecd 100644
--- a/toolkit/Cargo.toml
+++ b/toolkit/Cargo.toml
@@ -1,6 +1,6 @@
[package]
name = "witnet_toolkit"
-version = "1.6.7"
+version = "1.7.0"
authors = ["Adán SDPC <adan.sdpc@gmail.com>"]
edition = "2021"
diff --git a/wallet/Cargo.toml b/wallet/Cargo.toml
index 4bf3bb766..ec59fb12b 100644
--- a/wallet/Cargo.toml
+++ b/wallet/Cargo.toml
@@ -2,7 +2,7 @@
authors = ["Witnet Foundation <info@witnet.foundation>"]
edition = "2021"
name = "witnet_wallet"
-version = "1.6.7"
+version = "1.7.0"
workspace = ".."
[dependencies]
From 9bae448eb61a266b2fe36ebe6386fcd8a940e08e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= <guillermo@otherplane.com>
Date: Wed, 11 Oct 2023 10:52:59 +0200
Subject: [PATCH 07/19] chore: cargo fmt --all
---
rad/src/lib.rs | 37 +++++++++++++++----------------------
1 file changed, 15 insertions(+), 22 deletions(-)
diff --git a/rad/src/lib.rs b/rad/src/lib.rs
index e07bc4b32..d21e663ca 100644
--- a/rad/src/lib.rs
+++ b/rad/src/lib.rs
@@ -192,21 +192,15 @@ fn headers_response_with_data_report(
headers.entry(key).or_default().push(value);
}
}
-
- let headers: BTreeMap<String, RadonTypes> = BTreeMap::from_iter(
- headers.iter().map(|(key, value)| {
- match value.len() {
- len if len > 1 => (
- key.clone(),
- RadonTypes::from(RadonArray::from(value.to_vec()))
- ),
- _ => (
- key.clone(),
- value[0].clone()
- )
- }
- })
- );
+
+ let headers: BTreeMap<String, RadonTypes> =
+ BTreeMap::from_iter(headers.iter().map(|(key, value)| match value.len() {
+ len if len > 1 => (
+ key.clone(),
+ RadonTypes::from(RadonArray::from(value.to_vec())),
+ ),
+ _ => (key.clone(), value[0].clone()),
+ }));
let input = RadonTypes::from(RadonMap::from(headers));
let radon_script = unpack_radon_script(&retrieve.script)?;
@@ -880,12 +874,10 @@ mod tests {
#[test]
fn test_run_http_head_retrieval() {
- let script_r = Value::Array(vec![
- Value::Array(vec![
- Value::Integer(RadonOpCodes::MapGetString as i128),
- Value::Text("etag".to_string()),
- ]),
- ]);
+ let script_r = Value::Array(vec![Value::Array(vec![
+ Value::Integer(RadonOpCodes::MapGetString as i128),
+ Value::Text("etag".to_string()),
+ ])]);
let packed_script_r = serde_cbor::to_vec(&script_r).unwrap();
println!("{:?}", packed_script_r);
@@ -902,7 +894,8 @@ mod tests {
response,
RadonScriptExecutionSettings::disable_all(),
current_active_wips(),
- ).unwrap();
+ )
+ .unwrap();
match result {
RadonTypes::String(_) => {}
From af5ff8643a735a534fd322dc9423e5b4c9668cab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= <guillermo@otherplane.com>
Date: Wed, 11 Oct 2023 11:59:33 +0200
Subject: [PATCH 08/19] chore: solve clippy warnings
---
rad/src/lib.rs | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/rad/src/lib.rs b/rad/src/lib.rs
index d21e663ca..5fa1e8979 100644
--- a/rad/src/lib.rs
+++ b/rad/src/lib.rs
@@ -184,9 +184,9 @@ fn headers_response_with_data_report(
let mut headers: BTreeMap<String, Vec<RadonTypes>> = BTreeMap::new();
for line in response.split("\r\n") {
- if let Some(first_colon_index) = line.find(":") {
+ if let Some(first_colon_index) = line.find(':') {
// key: trim spaces and lower case all ascii chars left to the first colon character
- let key = String::from(line[0..first_colon_index].trim().to_ascii_lowercase());
+ let key = line[0..first_colon_index].trim().to_ascii_lowercase();
// value: trim spaces on the substring after the first colon character
let value = RadonTypes::from(RadonString::from(line[first_colon_index + 1..].trim()));
headers.entry(key).or_default().push(value);
@@ -692,7 +692,7 @@ fn validate_header(name: &str, value: &str) -> Result<()> {
Err(RadError::InvalidHttpHeader {
name: name.to_string(),
value: value.to_string(),
- error: error_message.to_string(),
+ error: error_message,
})
} else {
Ok(())
From ac57deddef050bba326bce951bc6f49ca592afb1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= <guillermo@otherplane.com>
Date: Wed, 11 Oct 2023 17:51:28 +0200
Subject: [PATCH 09/19] fix: use http::Response::headers(&self) to transform
response to HTTP/HEAD into a JSON string
Instead or parsing http::Response::body into a RadonMap
---
net/src/client/http/mod.rs | 2 +-
rad/src/lib.rs | 84 +++++++++++++-------------------------
2 files changed, 29 insertions(+), 57 deletions(-)
diff --git a/net/src/client/http/mod.rs b/net/src/client/http/mod.rs
index 55ae33e9e..3fca5533b 100644
--- a/net/src/client/http/mod.rs
+++ b/net/src/client/http/mod.rs
@@ -151,7 +151,7 @@ pub struct WitnetHttpResponse {
impl WitnetHttpResponse {
#[inline]
- /// Simple wrapper around `isahc::Response::status`.
+ /// Simple wrapper around `isahc::Response`.
pub fn inner(self) -> isahc::Response<isahc::AsyncBody> {
self.res
}
diff --git a/rad/src/lib.rs b/rad/src/lib.rs
index 5fa1e8979..3c3990dea 100644
--- a/rad/src/lib.rs
+++ b/rad/src/lib.rs
@@ -25,11 +25,10 @@ use crate::{
create_radon_script_from_filters_and_reducer, execute_radon_script, unpack_radon_script,
RadonScriptExecutionSettings,
},
- types::{array::RadonArray, bytes::RadonBytes, map::RadonMap, string::RadonString, RadonTypes},
+ types::{array::RadonArray, bytes::RadonBytes, string::RadonString, RadonTypes},
user_agents::UserAgent,
};
use core::convert::From;
-use std::collections::BTreeMap;
use witnet_net::client::http::{WitnetHttpBody, WitnetHttpRequest};
pub mod conditions;
@@ -174,40 +173,6 @@ fn string_response_with_data_report(
execute_radon_script(input, &radon_script, context, settings)
}
-/// Handle HTTP-HEAD response with data, and return a `RadonReport`.
-fn headers_response_with_data_report(
- retrieve: &RADRetrieve,
- response: &str,
- context: &mut ReportContext<RadonTypes>,
- settings: RadonScriptExecutionSettings,
-) -> Result<RadonReport<RadonTypes>> {
- let mut headers: BTreeMap<String, Vec<RadonTypes>> = BTreeMap::new();
-
- for line in response.split("\r\n") {
- if let Some(first_colon_index) = line.find(':') {
- // key: trim spaces and lower case all ascii chars left to the first colon character
- let key = line[0..first_colon_index].trim().to_ascii_lowercase();
- // value: trim spaces on the substring after the first colon character
- let value = RadonTypes::from(RadonString::from(line[first_colon_index + 1..].trim()));
- headers.entry(key).or_default().push(value);
- }
- }
-
- let headers: BTreeMap<String, RadonTypes> =
- BTreeMap::from_iter(headers.iter().map(|(key, value)| match value.len() {
- len if len > 1 => (
- key.clone(),
- RadonTypes::from(RadonArray::from(value.to_vec())),
- ),
- _ => (key.clone(), value[0].clone()),
- }));
-
- let input = RadonTypes::from(RadonMap::from(headers));
- let radon_script = unpack_radon_script(&retrieve.script)?;
-
- execute_radon_script(input, &radon_script, context, settings)
-}
-
/// Handle Rng response with data report
fn rng_response_with_data_report(
response: &str,
@@ -227,14 +192,10 @@ pub fn run_retrieval_with_data_report(
settings: RadonScriptExecutionSettings,
) -> Result<RadonReport<RadonTypes>> {
match retrieve.kind {
- RADType::HttpGet => string_response_with_data_report(retrieve, response, context, settings),
- RADType::Rng => rng_response_with_data_report(response, context),
- RADType::HttpPost => {
+ RADType::HttpGet | RADType::HttpPost | RADType::HttpHead => {
string_response_with_data_report(retrieve, response, context, settings)
}
- RADType::HttpHead => {
- headers_response_with_data_report(retrieve, response, context, settings)
- }
+ RADType::Rng => rng_response_with_data_report(response, context),
_ => Err(RadError::UnknownRetrieval),
}
}
@@ -340,13 +301,21 @@ async fn http_response(
// If at some point we want to support the retrieval of non-UTF8 data (e.g. raw bytes), this is
// where we need to decide how to read the response body
- let (_parts, mut body) = response.into_parts();
let mut response_string = String::default();
- body.read_to_string(&mut response_string)
- .await
- .map_err(|x| RadError::HttpOther {
- message: x.to_string(),
- })?;
+
+ let (parts, mut body) = response.into_parts();
+ match retrieve.kind {
+ RADType::HttpHead => {
+ response_string = format!("{:?}", parts.headers);
+ }
+ _ => {
+ body.read_to_string(&mut response_string)
+ .await
+ .map_err(|x| RadError::HttpOther {
+ message: x.to_string(),
+ })?;
+ }
+ }
let result = run_retrieval_with_data_report(retrieve, &response_string, context, settings);
@@ -396,10 +365,10 @@ pub async fn run_retrieval_report(
context.set_active_wips(active_wips);
match retrieve.kind {
- RADType::HttpGet => http_response(retrieve, context, settings, client).await,
+ RADType::HttpGet | RADType::HttpHead | RADType::HttpPost => {
+ http_response(retrieve, context, settings, client).await
+ }
RADType::Rng => rng_response(context, settings).await,
- RADType::HttpPost => http_response(retrieve, context, settings, client).await,
- RADType::HttpHead => http_response(retrieve, context, settings, client).await,
_ => Err(RadError::UnknownRetrieval),
}
}
@@ -874,10 +843,13 @@ mod tests {
#[test]
fn test_run_http_head_retrieval() {
- let script_r = Value::Array(vec![Value::Array(vec![
- Value::Integer(RadonOpCodes::MapGetString as i128),
- Value::Text("etag".to_string()),
- ])]);
+ let script_r = Value::Array(vec![
+ Value::Integer(RadonOpCodes::StringParseJSONMap as i128),
+ Value::Array(vec![
+ Value::Integer(RadonOpCodes::MapGetString as i128),
+ Value::Text("etag".to_string()),
+ ]),
+ ]);
let packed_script_r = serde_cbor::to_vec(&script_r).unwrap();
println!("{:?}", packed_script_r);
@@ -888,7 +860,7 @@ mod tests {
body: vec![],
headers: vec![],
};
- let response = "HTTP/1.1 200 OK\r\nContent-Type: image/png\r\nContent-Length: 498219\r\nContent-Length: 123456\r\neTag: \"64eca181-79a2b\"\r\n";
+ let response = r#"{"date": "Wed, 11 Oct 2023 15:18:42 GMT", "content-type": "image/png", "content-length": "498219", "x-origin-cache": "HIT", "last-modified": "Mon, 28 Aug 2023 13:30:41 GMT", "access-control-allow-origin": "*", "etag": "\"64eca181-79a2b\"", "expires": "Wed, 11 Oct 2023 15:28:41 GMT", "cache-control": "max-age=1800", "x-proxy-cache": "MISS", "x-github-request-id": "6750:35DB:BF8211:FEFD2B:652602FA", "via": "1.1 varnish", "x-served-by": "cache-hnd18736-HND", "x-cache": "MISS", "x-cache-hits": "0", "x-timer": "S1696989946.496383,VS0,VE487", "vary": "Accept-Encoding", "x-fastly-request-id": "118bdfd8a926cbdc781bc23079c3dc07a22d2223", "cf-cache-status": "REVALIDATED", "accept-ranges": "bytes", "report-to": "{\"endpoints\":[{\"url\":\"https:\/\/a.nel.cloudflare.com\/report\/v3?s=FlzxKRCYYN4SL0x%2FraG7ugKCqdC%2BeQqVrucvsfeDWf%2F7A0Nv9fv7TYRgU0WL4k1kbZyxt%2B04VjOyv0XK55sF37GEPwXHE%2FdXnoFlWutID762k2ktcX6hUml6oNk%3D\"}],\"group\":\"cf-nel\",\"max_age\":604800}", "nel": "{\"success_fraction\":0,\"report_to\":\"cf-nel\",\"max_age\":604800}", "strict-transport-security": "max-age=0", "x-content-type-options": "nosniff", "server": "cloudflare", "cf-ray": "814813bf3a73f689-NRT", "alt-svc": "h3=\":443\"; ma=86400"}"#;
let result = run_retrieval_with_data(
&retrieve,
response,
From 5e073f9ac26d577bc5cdbe4f619039f1f280ef6a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= <guillermo@otherplane.com>
Date: Fri, 20 Oct 2023 09:52:56 +0200
Subject: [PATCH 10/19] chore: attend pr review comments
---
Cargo.lock | 12 ++++++------
Cargo.toml | 2 +-
bridges/centralized-ethereum/Cargo.toml | 2 +-
data_structures/Cargo.toml | 2 +-
node/Cargo.toml | 2 +-
rad/src/lib.rs | 3 +--
toolkit/Cargo.toml | 2 +-
wallet/Cargo.toml | 2 +-
8 files changed, 13 insertions(+), 14 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index 4571cdfa6..8580a36aa 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4958,7 +4958,7 @@ dependencies = [
[[package]]
name = "witnet"
-version = "1.7.0"
+version = "2.0.0"
dependencies = [
"ansi_term",
"bytecount",
@@ -5005,7 +5005,7 @@ dependencies = [
[[package]]
name = "witnet-centralized-ethereum-bridge"
-version = "1.7.0"
+version = "2.0.0"
dependencies = [
"actix",
"async-jsonrpc-client",
@@ -5069,7 +5069,7 @@ dependencies = [
[[package]]
name = "witnet_data_structures"
-version = "1.7.0"
+version = "2.0.0"
dependencies = [
"bech32",
"bencher",
@@ -5134,7 +5134,7 @@ dependencies = [
[[package]]
name = "witnet_node"
-version = "1.7.0"
+version = "2.0.0"
dependencies = [
"actix",
"ansi_term",
@@ -5241,7 +5241,7 @@ dependencies = [
[[package]]
name = "witnet_toolkit"
-version = "1.7.0"
+version = "2.0.0"
dependencies = [
"failure",
"hex",
@@ -5284,7 +5284,7 @@ dependencies = [
[[package]]
name = "witnet_wallet"
-version = "1.7.0"
+version = "2.0.0"
dependencies = [
"actix",
"async-jsonrpc-client",
diff --git a/Cargo.toml b/Cargo.toml
index 4a1b64b65..21a934c11 100755
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,6 @@
[package]
name = "witnet"
-version = "1.7.0"
+version = "2.0.0"
authors = ["Witnet Foundation <info@witnet.foundation>"]
publish = false
repository = "witnet/witnet-rust"
diff --git a/bridges/centralized-ethereum/Cargo.toml b/bridges/centralized-ethereum/Cargo.toml
index b87084520..6e3a4f7a0 100644
--- a/bridges/centralized-ethereum/Cargo.toml
+++ b/bridges/centralized-ethereum/Cargo.toml
@@ -1,6 +1,6 @@
[package]
name = "witnet-centralized-ethereum-bridge"
-version = "1.7.0"
+version = "2.0.0"
authors = ["Witnet Foundation <info@witnet.foundation>"]
edition = "2018"
diff --git a/data_structures/Cargo.toml b/data_structures/Cargo.toml
index f09cf9699..9fd896c71 100644
--- a/data_structures/Cargo.toml
+++ b/data_structures/Cargo.toml
@@ -3,7 +3,7 @@ authors = ["Witnet Foundation <info@witnet.foundation>"]
description = "data structures component"
edition = "2021"
name = "witnet_data_structures"
-version = "1.7.0"
+version = "2.0.0"
workspace = ".."
[features]
diff --git a/node/Cargo.toml b/node/Cargo.toml
index 88a31f51a..57fe55a7a 100644
--- a/node/Cargo.toml
+++ b/node/Cargo.toml
@@ -1,6 +1,6 @@
[package]
name = "witnet_node"
-version = "1.7.0"
+version = "2.0.0"
authors = ["Witnet Foundation <info@witnet.foundation>"]
workspace = ".."
description = "node component"
diff --git a/rad/src/lib.rs b/rad/src/lib.rs
index 3c3990dea..e7eb65541 100644
--- a/rad/src/lib.rs
+++ b/rad/src/lib.rs
@@ -851,11 +851,10 @@ mod tests {
]),
]);
let packed_script_r = serde_cbor::to_vec(&script_r).unwrap();
- println!("{:?}", packed_script_r);
let retrieve = RADRetrieve {
kind: RADType::HttpHead,
- url: "https://witnet.io/_nuxt/img/dragon_reading.a37f8cb.png".to_string(),
+ url: "https://en.wikipedia.org/static/images/icons/wikipedia.png".to_string(),
script: packed_script_r,
body: vec![],
headers: vec![],
diff --git a/toolkit/Cargo.toml b/toolkit/Cargo.toml
index 5ee64aecd..7ac7b72d4 100644
--- a/toolkit/Cargo.toml
+++ b/toolkit/Cargo.toml
@@ -1,6 +1,6 @@
[package]
name = "witnet_toolkit"
-version = "1.7.0"
+version = "2.0.0"
authors = ["Adán SDPC <adan.sdpc@gmail.com>"]
edition = "2021"
diff --git a/wallet/Cargo.toml b/wallet/Cargo.toml
index ec59fb12b..5a7221a5f 100644
--- a/wallet/Cargo.toml
+++ b/wallet/Cargo.toml
@@ -2,7 +2,7 @@
authors = ["Witnet Foundation <info@witnet.foundation>"]
edition = "2021"
name = "witnet_wallet"
-version = "1.7.0"
+version = "2.0.0"
workspace = ".."
[dependencies]
From 6c4d1ac2006da442b55fa12cd2b8f7991be44547 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= <guillermo@otherplane.com>
Date: Fri, 20 Oct 2023 12:44:25 +0200
Subject: [PATCH 11/19] feat(rad): add support to binary sources
---
rad/src/lib.rs | 88 ++++++++++++++++++++++++++++----------------------
1 file changed, 50 insertions(+), 38 deletions(-)
diff --git a/rad/src/lib.rs b/rad/src/lib.rs
index e7eb65541..6f9b8f693 100644
--- a/rad/src/lib.rs
+++ b/rad/src/lib.rs
@@ -78,7 +78,12 @@ pub fn try_data_request(
.iter()
.zip(inputs.iter())
.map(|(retrieve, input)| {
- run_retrieval_with_data_report(retrieve, input, &mut retrieval_context, settings)
+ run_retrieval_with_data_report(
+ retrieve,
+ RadonTypes::from(RadonString::from(*input)),
+ &mut retrieval_context,
+ settings,
+ )
})
.collect()
} else {
@@ -160,42 +165,28 @@ pub fn try_data_request(
}
}
-/// Handle HTTP-GET and HTTP-POST response with data, and return a `RadonReport`.
-fn string_response_with_data_report(
+/// Execute Radon Script using as input the RadonTypes value deserialized from a retrieval response
+fn handle_response_with_data_report(
retrieve: &RADRetrieve,
- response: &str,
+ response: RadonTypes,
context: &mut ReportContext<RadonTypes>,
settings: RadonScriptExecutionSettings,
) -> Result<RadonReport<RadonTypes>> {
- let input = RadonTypes::from(RadonString::from(response));
let radon_script = unpack_radon_script(&retrieve.script)?;
-
- execute_radon_script(input, &radon_script, context, settings)
-}
-
-/// Handle Rng response with data report
-fn rng_response_with_data_report(
- response: &str,
- context: &mut ReportContext<RadonTypes>,
-) -> Result<RadonReport<RadonTypes>> {
- let response_bytes = response.as_bytes();
- let result = RadonTypes::from(RadonBytes::from(response_bytes.to_vec()));
-
- Ok(RadonReport::from_result(Ok(result), context))
+ execute_radon_script(response, &radon_script, context, settings)
}
/// Run retrieval without performing any external network requests, return `Result<RadonReport>`.
pub fn run_retrieval_with_data_report(
retrieve: &RADRetrieve,
- response: &str,
+ response: RadonTypes,
context: &mut ReportContext<RadonTypes>,
settings: RadonScriptExecutionSettings,
) -> Result<RadonReport<RadonTypes>> {
match retrieve.kind {
- RADType::HttpGet | RADType::HttpPost | RADType::HttpHead => {
- string_response_with_data_report(retrieve, response, context, settings)
+ RADType::HttpGet | RADType::HttpPost | RADType::HttpHead | RADType::Rng => {
+ handle_response_with_data_report(retrieve, response, context, settings)
}
- RADType::Rng => rng_response_with_data_report(response, context),
_ => Err(RadError::UnknownRetrieval),
}
}
@@ -203,7 +194,7 @@ pub fn run_retrieval_with_data_report(
/// Run retrieval without performing any external network requests, return `Result<RadonTypes>`.
pub fn run_retrieval_with_data(
retrieve: &RADRetrieve,
- response: &str,
+ response: RadonTypes,
settings: RadonScriptExecutionSettings,
active_wips: ActiveWips,
) -> Result<RadonTypes> {
@@ -299,26 +290,48 @@ async fn http_response(
});
}
- // If at some point we want to support the retrieval of non-UTF8 data (e.g. raw bytes), this is
- // where we need to decide how to read the response body
- let mut response_string = String::default();
-
let (parts, mut body) = response.into_parts();
- match retrieve.kind {
- RADType::HttpHead => {
- response_string = format!("{:?}", parts.headers);
+
+ let response: RadonTypes;
+ match parts.headers.get("accept-ranges") {
+ Some(_) => {
+ // http response is a binary stream
+ let mut response_bytes = Vec::<u8>::default();
+ match retrieve.kind {
+ RADType::HttpHead => {
+ // todo: assert http-head responses should never return binary streams
+ }
+ _ => {
+ // todo: before reading the response buffer, an error should thrown it was too big
+ body.read_to_end(&mut response_bytes).await.map_err(|x| {
+ RadError::HttpOther {
+ message: x.to_string(),
+ }
+ })?;
+ }
+ }
+ response = RadonTypes::from(RadonBytes::from(response_bytes));
}
_ => {
- body.read_to_string(&mut response_string)
- .await
- .map_err(|x| RadError::HttpOther {
- message: x.to_string(),
- })?;
+ // response is a string
+ let mut response_string = String::default();
+ match retrieve.kind {
+ RADType::HttpHead => {
+ response_string = format!("{:?}", parts.headers);
+ }
+ _ => {
+ body.read_to_string(&mut response_string)
+ .await
+ .map_err(|x| RadError::HttpOther {
+ message: x.to_string(),
+ })?;
+ }
+ }
+ response = RadonTypes::from(RadonString::from(response_string));
}
}
- let result = run_retrieval_with_data_report(retrieve, &response_string, context, settings);
-
+ let result = handle_response_with_data_report(retrieve, response, context, settings);
match &result {
Ok(report) => {
log::debug!(
@@ -329,7 +342,6 @@ async fn http_response(
}
Err(e) => log::debug!("Failed result for source {}: {:?}", retrieve.url, e),
}
-
result
}
From 0cfd81d4aad31b2212ee0b354fc45b6175d8940a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= <guillermo@otherplane.com>
Date: Fri, 20 Oct 2023 12:45:00 +0200
Subject: [PATCH 12/19] chore(rad): refactor existing tests
---
rad/src/lib.rs | 29 ++++++++++++++---------------
1 file changed, 14 insertions(+), 15 deletions(-)
diff --git a/rad/src/lib.rs b/rad/src/lib.rs
index 6f9b8f693..2af893083 100644
--- a/rad/src/lib.rs
+++ b/rad/src/lib.rs
@@ -871,10 +871,10 @@ mod tests {
body: vec![],
headers: vec![],
};
- let response = r#"{"date": "Wed, 11 Oct 2023 15:18:42 GMT", "content-type": "image/png", "content-length": "498219", "x-origin-cache": "HIT", "last-modified": "Mon, 28 Aug 2023 13:30:41 GMT", "access-control-allow-origin": "*", "etag": "\"64eca181-79a2b\"", "expires": "Wed, 11 Oct 2023 15:28:41 GMT", "cache-control": "max-age=1800", "x-proxy-cache": "MISS", "x-github-request-id": "6750:35DB:BF8211:FEFD2B:652602FA", "via": "1.1 varnish", "x-served-by": "cache-hnd18736-HND", "x-cache": "MISS", "x-cache-hits": "0", "x-timer": "S1696989946.496383,VS0,VE487", "vary": "Accept-Encoding", "x-fastly-request-id": "118bdfd8a926cbdc781bc23079c3dc07a22d2223", "cf-cache-status": "REVALIDATED", "accept-ranges": "bytes", "report-to": "{\"endpoints\":[{\"url\":\"https:\/\/a.nel.cloudflare.com\/report\/v3?s=FlzxKRCYYN4SL0x%2FraG7ugKCqdC%2BeQqVrucvsfeDWf%2F7A0Nv9fv7TYRgU0WL4k1kbZyxt%2B04VjOyv0XK55sF37GEPwXHE%2FdXnoFlWutID762k2ktcX6hUml6oNk%3D\"}],\"group\":\"cf-nel\",\"max_age\":604800}", "nel": "{\"success_fraction\":0,\"report_to\":\"cf-nel\",\"max_age\":604800}", "strict-transport-security": "max-age=0", "x-content-type-options": "nosniff", "server": "cloudflare", "cf-ray": "814813bf3a73f689-NRT", "alt-svc": "h3=\":443\"; ma=86400"}"#;
+ let response_string = r#"{"date": "Wed, 11 Oct 2023 15:18:42 GMT", "content-type": "image/png", "content-length": "498219", "x-origin-cache": "HIT", "last-modified": "Mon, 28 Aug 2023 13:30:41 GMT", "access-control-allow-origin": "*", "etag": "\"64eca181-79a2b\"", "expires": "Wed, 11 Oct 2023 15:28:41 GMT", "cache-control": "max-age=1800", "x-proxy-cache": "MISS", "x-github-request-id": "6750:35DB:BF8211:FEFD2B:652602FA", "via": "1.1 varnish", "x-served-by": "cache-hnd18736-HND", "x-cache": "MISS", "x-cache-hits": "0", "x-timer": "S1696989946.496383,VS0,VE487", "vary": "Accept-Encoding", "x-fastly-request-id": "118bdfd8a926cbdc781bc23079c3dc07a22d2223", "cf-cache-status": "REVALIDATED", "accept-ranges": "bytes", "report-to": "{\"endpoints\":[{\"url\":\"https:\/\/a.nel.cloudflare.com\/report\/v3?s=FlzxKRCYYN4SL0x%2FraG7ugKCqdC%2BeQqVrucvsfeDWf%2F7A0Nv9fv7TYRgU0WL4k1kbZyxt%2B04VjOyv0XK55sF37GEPwXHE%2FdXnoFlWutID762k2ktcX6hUml6oNk%3D\"}],\"group\":\"cf-nel\",\"max_age\":604800}", "nel": "{\"success_fraction\":0,\"report_to\":\"cf-nel\",\"max_age\":604800}", "strict-transport-security": "max-age=0", "x-content-type-options": "nosniff", "server": "cloudflare", "cf-ray": "814813bf3a73f689-NRT", "alt-svc": "h3=\":443\"; ma=86400"}"#;
let result = run_retrieval_with_data(
&retrieve,
- response,
+ RadonTypes::from(RadonString::from(response_string)),
RadonScriptExecutionSettings::disable_all(),
current_active_wips(),
)
@@ -908,11 +908,10 @@ mod tests {
body: vec![],
headers: vec![],
};
- let response = r#"{"coord":{"lon":13.41,"lat":52.52},"weather":[{"id":500,"main":"Rain","description":"light rain","icon":"10d"}],"base":"stations","main":{"temp":17.59,"pressure":1022,"humidity":67,"temp_min":15,"temp_max":20},"visibility":10000,"wind":{"speed":3.6,"deg":260},"rain":{"1h":0.51},"clouds":{"all":20},"dt":1567501321,"sys":{"type":1,"id":1275,"message":0.0089,"country":"DE","sunrise":1567484402,"sunset":1567533129},"timezone":7200,"id":2950159,"name":"Berlin","cod":200}"#;
-
+ let response_string = r#"{"coord":{"lon":13.41,"lat":52.52},"weather":[{"id":500,"main":"Rain","description":"light rain","icon":"10d"}],"base":"stations","main":{"temp":17.59,"pressure":1022,"humidity":67,"temp_min":15,"temp_max":20},"visibility":10000,"wind":{"speed":3.6,"deg":260},"rain":{"1h":0.51},"clouds":{"all":20},"dt":1567501321,"sys":{"type":1,"id":1275,"message":0.0089,"country":"DE","sunrise":1567484402,"sunset":1567533129},"timezone":7200,"id":2950159,"name":"Berlin","cod":200}"#;
let result = run_retrieval_with_data(
&retrieve,
- response,
+ RadonTypes::from(RadonString::from(response_string)),
RadonScriptExecutionSettings::disable_all(),
current_active_wips(),
)
@@ -967,7 +966,7 @@ mod tests {
body: vec![],
headers: vec![],
};
- let response = "84";
+ let response_string = "84";
let expected = RadonTypes::Float(RadonFloat::from(84));
let aggregate = RADAggregate {
@@ -982,7 +981,7 @@ mod tests {
let retrieved = run_retrieval_with_data(
&retrieve,
- response,
+ RadonTypes::from(RadonString::from(response_string)),
RadonScriptExecutionSettings::disable_all(),
current_active_wips(),
)
@@ -1005,7 +1004,7 @@ mod tests {
body: vec![],
headers: vec![],
};
- let response = "307";
+ let response_string = "307";
let expected = RadonTypes::Float(RadonFloat::from(307));
let aggregate = RADAggregate {
@@ -1019,7 +1018,7 @@ mod tests {
let retrieved = run_retrieval_with_data(
&retrieve,
- response,
+ RadonTypes::from(RadonString::from(response_string)),
RadonScriptExecutionSettings::disable_all(),
current_active_wips(),
)
@@ -1058,7 +1057,7 @@ mod tests {
headers: vec![],
};
// This response was modified because the original was about 100KB.
- let response = r#"[{"estacion_nombre":"Pza. de España","estacion_numero":4,"fecha":"03092019","hora0":{"estado":"Pasado","valor":"00008"}}]"#;
+ let response_string = r#"[{"estacion_nombre":"Pza. de España","estacion_numero":4,"fecha":"03092019","hora0":{"estado":"Pasado","valor":"00008"}}]"#;
let expected = RadonTypes::Float(RadonFloat::from(8));
let aggregate = RADAggregate {
@@ -1072,7 +1071,7 @@ mod tests {
let retrieved = run_retrieval_with_data(
&retrieve,
- response,
+ RadonTypes::from(RadonString::from(response_string)),
RadonScriptExecutionSettings::disable_all(),
current_active_wips(),
)
@@ -1102,7 +1101,7 @@ mod tests {
body: vec![],
headers: vec![],
};
- let response = r#"{"PSOE":123,"PP":66,"Cs":57,"UP":42,"VOX":24,"ERC-SOBIRANISTES":15,"JxCAT-JUNTS":7,"PNV":6,"EH Bildu":4,"CCa-PNC":2,"NA+":2,"COMPROMÍS 2019":1,"PRC":1,"PACMA":0,"FRONT REPUBLICÀ":0,"BNG":0,"RECORTES CERO-GV":0,"NCa":0,"PACT":0,"ARA-MES-ESQUERRA":0,"GBAI":0,"PUM+J":0,"EN MAREA":0,"PCTE":0,"EL PI":0,"AxSI":0,"PCOE":0,"PCPE":0,"AVANT ADELANTE LOS VERDES":0,"EB":0,"CpM":0,"SOMOS REGIÓN":0,"PCPA":0,"PH":0,"UIG-SOM-CUIDES":0,"ERPV":0,"IZQP":0,"PCPC":0,"AHORA CANARIAS":0,"CxG":0,"PPSO":0,"CNV":0,"PREPAL":0,"C.Ex-C.R.Ex-P.R.Ex":0,"PR+":0,"P-LIB":0,"CILU-LINARES":0,"ANDECHA ASTUR":0,"JF":0,"PYLN":0,"FIA":0,"FE de las JONS":0,"SOLIDARIA":0,"F8":0,"DPL":0,"UNIÓN REGIONALISTA":0,"centrados":0,"DP":0,"VOU":0,"PDSJE-UDEC":0,"IZAR":0,"RISA":0,"C 21":0,"+MAS+":0,"UDT":0}"#;
+ let response_string = r#"{"PSOE":123,"PP":66,"Cs":57,"UP":42,"VOX":24,"ERC-SOBIRANISTES":15,"JxCAT-JUNTS":7,"PNV":6,"EH Bildu":4,"CCa-PNC":2,"NA+":2,"COMPROMÍS 2019":1,"PRC":1,"PACMA":0,"FRONT REPUBLICÀ":0,"BNG":0,"RECORTES CERO-GV":0,"NCa":0,"PACT":0,"ARA-MES-ESQUERRA":0,"GBAI":0,"PUM+J":0,"EN MAREA":0,"PCTE":0,"EL PI":0,"AxSI":0,"PCOE":0,"PCPE":0,"AVANT ADELANTE LOS VERDES":0,"EB":0,"CpM":0,"SOMOS REGIÓN":0,"PCPA":0,"PH":0,"UIG-SOM-CUIDES":0,"ERPV":0,"IZQP":0,"PCPC":0,"AHORA CANARIAS":0,"CxG":0,"PPSO":0,"CNV":0,"PREPAL":0,"C.Ex-C.R.Ex-P.R.Ex":0,"PR+":0,"P-LIB":0,"CILU-LINARES":0,"ANDECHA ASTUR":0,"JF":0,"PYLN":0,"FIA":0,"FE de las JONS":0,"SOLIDARIA":0,"F8":0,"DPL":0,"UNIÓN REGIONALISTA":0,"centrados":0,"DP":0,"VOU":0,"PDSJE-UDEC":0,"IZAR":0,"RISA":0,"C 21":0,"+MAS+":0,"UDT":0}"#;
let expected = RadonTypes::Float(RadonFloat::from(123));
let aggregate = RADAggregate {
@@ -1116,7 +1115,7 @@ mod tests {
let retrieved = run_retrieval_with_data(
&retrieve,
- response,
+ RadonTypes::from(RadonString::from(response_string)),
RadonScriptExecutionSettings::disable_all(),
current_active_wips(),
)
@@ -1157,10 +1156,10 @@ mod tests {
body: vec![],
headers: vec![],
};
- let response = r#"{"event":{"homeTeam":{"name":"Ryazan-VDV","slug":"ryazan-vdv","gender":"F","national":false,"id":171120,"shortName":"Ryazan-VDV","subTeams":[]},"awayTeam":{"name":"Olympique Lyonnais","slug":"olympique-lyonnais","gender":"F","national":false,"id":26245,"shortName":"Lyon","subTeams":[]},"homeScore":{"current":0,"display":0,"period1":0,"normaltime":0},"awayScore":{"current":9,"display":9,"period1":5,"normaltime":9}}}"#;
+ let response_string = r#"{"event":{"homeTeam":{"name":"Ryazan-VDV","slug":"ryazan-vdv","gender":"F","national":false,"id":171120,"shortName":"Ryazan-VDV","subTeams":[]},"awayTeam":{"name":"Olympique Lyonnais","slug":"olympique-lyonnais","gender":"F","national":false,"id":26245,"shortName":"Lyon","subTeams":[]},"homeScore":{"current":0,"display":0,"period1":0,"normaltime":0},"awayScore":{"current":9,"display":9,"period1":5,"normaltime":9}}}"#;
let retrieved = run_retrieval_with_data(
&retrieve,
- response,
+ RadonTypes::from(RadonString::from(response_string)),
RadonScriptExecutionSettings::disable_all(),
current_active_wips(),
)
From 68c7648180863c17c4271cdbeaca9a8a9ed344d2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= <guillermo@otherplane.com>
Date: Thu, 26 Oct 2023 13:24:08 +0200
Subject: [PATCH 13/19] chore: attend pr review comments
---
rad/src/lib.rs | 68 ++++++++++++++++++++++++++------------------------
1 file changed, 35 insertions(+), 33 deletions(-)
diff --git a/rad/src/lib.rs b/rad/src/lib.rs
index 2af893083..a30cd0309 100644
--- a/rad/src/lib.rs
+++ b/rad/src/lib.rs
@@ -13,7 +13,7 @@ use witnet_data_structures::{
RADAggregate, RADRequest, RADRetrieve, RADTally, RADType,
},
radon_report::{RadonReport, ReportContext, RetrievalMetadata, Stage, TallyMetaData},
- witnessing::WitnessingConfig,
+ witnessing::WitnessingConfig, radon_error::RadonError,
};
use witnet_net::client::http::WitnetHttpClient;
pub use witnet_net::Uri;
@@ -293,42 +293,44 @@ async fn http_response(
let (parts, mut body) = response.into_parts();
let response: RadonTypes;
- match parts.headers.get("accept-ranges") {
- Some(_) => {
- // http response is a binary stream
- let mut response_bytes = Vec::<u8>::default();
- match retrieve.kind {
- RADType::HttpHead => {
- // todo: assert http-head responses should never return binary streams
- }
- _ => {
- // todo: before reading the response buffer, an error should thrown it was too big
- body.read_to_end(&mut response_bytes).await.map_err(|x| {
- RadError::HttpOther {
- message: x.to_string(),
- }
- })?;
- }
+ if parts.headers.contains_key("accept-ranges") {
+ // http response is a binary stream
+ let mut response_bytes = Vec::<u8>::default();
+ match retrieve.kind {
+ RADType::HttpHead => {
+ response = RadonTypes::RadonError(
+ RadonError::try_from(RadError::BufferIsNotValue {
+ description: String::from("Unsupported binary streams from HTTP/HEAD sources")
+ })
+ .unwrap()
+ );
+ }
+ _ => {
+ // todo: before reading the response buffer, an error should be thrown if it was too big
+ body.read_to_end(&mut response_bytes).await.map_err(|x| {
+ RadError::HttpOther {
+ message: x.to_string(),
+ }
+ })?;
+ response = RadonTypes::from(RadonBytes::from(response_bytes));
}
- response = RadonTypes::from(RadonBytes::from(response_bytes));
}
- _ => {
- // response is a string
- let mut response_string = String::default();
- match retrieve.kind {
- RADType::HttpHead => {
- response_string = format!("{:?}", parts.headers);
- }
- _ => {
- body.read_to_string(&mut response_string)
- .await
- .map_err(|x| RadError::HttpOther {
- message: x.to_string(),
- })?;
- }
+ } else {
+ // response is a string
+ let mut response_string = String::default();
+ match retrieve.kind {
+ RADType::HttpHead => {
+ response_string = format!("{:?}", parts.headers);
+ }
+ _ => {
+ body.read_to_string(&mut response_string)
+ .await
+ .map_err(|x| RadError::HttpOther {
+ message: x.to_string(),
+ })?;
}
- response = RadonTypes::from(RadonString::from(response_string));
}
+ response = RadonTypes::from(RadonString::from(response_string));
}
let result = handle_response_with_data_report(retrieve, response, context, settings);
From 184f768850a16a201679fc25f67b051f7d415f0b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= <guillermo@otherplane.com>
Date: Thu, 26 Oct 2023 13:27:44 +0200
Subject: [PATCH 14/19] chore: cargo clippy --fix
---
data_structures/src/chain/mod.rs | 4 ++--
data_structures/src/data_request.rs | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/data_structures/src/chain/mod.rs b/data_structures/src/chain/mod.rs
index 7ed2e38e0..8b9b3d729 100644
--- a/data_structures/src/chain/mod.rs
+++ b/data_structures/src/chain/mod.rs
@@ -2690,7 +2690,7 @@ impl TransactionsPool {
for input in &vt_tx.body.inputs {
self.output_pointer_map
.entry(input.output_pointer)
- .or_insert_with(Vec::new)
+ .or_default()
.push(vt_tx.hash());
}
@@ -2715,7 +2715,7 @@ impl TransactionsPool {
for input in &dr_tx.body.inputs {
self.output_pointer_map
.entry(input.output_pointer)
- .or_insert_with(Vec::new)
+ .or_default()
.push(dr_tx.hash());
}
diff --git a/data_structures/src/data_request.rs b/data_structures/src/data_request.rs
index 6dd3493a1..fc3823037 100644
--- a/data_structures/src/data_request.rs
+++ b/data_structures/src/data_request.rs
@@ -143,7 +143,7 @@ impl DataRequestPool {
self.data_requests_by_epoch
.entry(epoch)
- .or_insert_with(HashSet::new)
+ .or_default()
.insert(dr_hash);
self.data_request_pool.insert(dr_hash, dr_state);
From cd4641c84aae3593d2395dd3f63147c7875a2f49 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= <guillermo@otherplane.com>
Date: Thu, 26 Oct 2023 19:14:34 +0200
Subject: [PATCH 15/19] feat(rad): add new RadonErrors::BufferIsNotValue
---
data_structures/src/radon_error.rs | 2 ++
rad/src/error.rs | 5 +++++
rad/src/lib.rs | 13 +++++--------
3 files changed, 12 insertions(+), 8 deletions(-)
diff --git a/data_structures/src/radon_error.rs b/data_structures/src/radon_error.rs
index 7d3176629..6fdf30d26 100644
--- a/data_structures/src/radon_error.rs
+++ b/data_structures/src/radon_error.rs
@@ -34,6 +34,8 @@ pub enum RadonErrors {
HTTPError = 0x30,
/// Al least one of the sources could not be retrieved, timeout reached.
RetrieveTimeout = 0x31,
+ /// Value cannot be extracted from binary buffer
+ BufferIsNotValue = 0x32,
// Math errors
/// Math operator caused an underflow.
Underflow = 0x40,
diff --git a/rad/src/error.rs b/rad/src/error.rs
index a413f47c4..b9f2086b6 100644
--- a/rad/src/error.rs
+++ b/rad/src/error.rs
@@ -431,6 +431,10 @@ impl RadError {
}
Ok(RadonError::new(match kind {
+ RadonErrors::BufferIsNotValue => {
+ let (description,) = deserialize_args(error_args)?;
+ RadError::BufferIsNotValue { description }
+ }
RadonErrors::RequestTooManySources => RadError::RequestTooManySources,
RadonErrors::ScriptTooManyCalls => RadError::ScriptTooManyCalls,
RadonErrors::Overflow => RadError::Overflow,
@@ -574,6 +578,7 @@ impl RadError {
pub fn try_into_error_code(&self) -> Result<RadonErrors, RadError> {
Ok(match self {
RadError::Unknown => RadonErrors::Unknown,
+ RadError::BufferIsNotValue { .. } => RadonErrors::BufferIsNotValue,
RadError::SourceScriptNotCBOR => RadonErrors::SourceScriptNotCBOR,
RadError::SourceScriptNotArray => RadonErrors::SourceScriptNotArray,
RadError::SourceScriptNotRADON => RadonErrors::SourceScriptNotRADON,
diff --git a/rad/src/lib.rs b/rad/src/lib.rs
index a30cd0309..5de2f1379 100644
--- a/rad/src/lib.rs
+++ b/rad/src/lib.rs
@@ -13,7 +13,7 @@ use witnet_data_structures::{
RADAggregate, RADRequest, RADRetrieve, RADTally, RADType,
},
radon_report::{RadonReport, ReportContext, RetrievalMetadata, Stage, TallyMetaData},
- witnessing::WitnessingConfig, radon_error::RadonError,
+ witnessing::WitnessingConfig,
};
use witnet_net::client::http::WitnetHttpClient;
pub use witnet_net::Uri;
@@ -298,12 +298,9 @@ async fn http_response(
let mut response_bytes = Vec::<u8>::default();
match retrieve.kind {
RADType::HttpHead => {
- response = RadonTypes::RadonError(
- RadonError::try_from(RadError::BufferIsNotValue {
- description: String::from("Unsupported binary streams from HTTP/HEAD sources")
- })
- .unwrap()
- );
+ return Err(RadError::BufferIsNotValue {
+ description: String::from("Unsupported binary streams from HTTP/HEAD sources")
+ });
}
_ => {
// todo: before reading the response buffer, an error should be thrown if it was too big
@@ -312,9 +309,9 @@ async fn http_response(
message: x.to_string(),
}
})?;
- response = RadonTypes::from(RadonBytes::from(response_bytes));
}
}
+ response = RadonTypes::from(RadonBytes::from(response_bytes));
} else {
// response is a string
let mut response_string = String::default();
From f1314a4626a1f47bad478b0d4d6964897ba9a689 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= <guillermo@otherplane.com>
Date: Tue, 14 Nov 2023 10:36:02 +0100
Subject: [PATCH 16/19] fix(rad): http-head response headers can contain
'accept-ranges'
---
rad/src/lib.rs | 22 +++++++---------------
1 file changed, 7 insertions(+), 15 deletions(-)
diff --git a/rad/src/lib.rs b/rad/src/lib.rs
index 5de2f1379..289178f98 100644
--- a/rad/src/lib.rs
+++ b/rad/src/lib.rs
@@ -293,24 +293,16 @@ async fn http_response(
let (parts, mut body) = response.into_parts();
let response: RadonTypes;
- if parts.headers.contains_key("accept-ranges") {
+ if retrieve.kind != RADType::HttpHead && parts.headers.contains_key("accept-ranges") {
// http response is a binary stream
let mut response_bytes = Vec::<u8>::default();
- match retrieve.kind {
- RADType::HttpHead => {
- return Err(RadError::BufferIsNotValue {
- description: String::from("Unsupported binary streams from HTTP/HEAD sources")
- });
- }
- _ => {
- // todo: before reading the response buffer, an error should be thrown if it was too big
- body.read_to_end(&mut response_bytes).await.map_err(|x| {
- RadError::HttpOther {
- message: x.to_string(),
- }
- })?;
+
+ // todo: before reading the response buffer, an error should be thrown if it was too big
+ body.read_to_end(&mut response_bytes).await.map_err(|x| {
+ RadError::HttpOther {
+ message: x.to_string(),
}
- }
+ })?;
response = RadonTypes::from(RadonBytes::from(response_bytes));
} else {
// response is a string
From 569c04db508c54705dc5f68f401c70d66f4fe038 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= <guillermo@otherplane.com>
Date: Fri, 23 Feb 2024 11:36:52 +0100
Subject: [PATCH 17/19] fix(rad): determine binary sources by peeking first
operation in radon script
---
rad/src/lib.rs | 77 +++++++++++++++++++++++++---------------
rad/src/operators/mod.rs | 4 +--
2 files changed, 50 insertions(+), 31 deletions(-)
diff --git a/rad/src/lib.rs b/rad/src/lib.rs
index 289178f98..6852ecc55 100644
--- a/rad/src/lib.rs
+++ b/rad/src/lib.rs
@@ -3,6 +3,7 @@
extern crate witnet_data_structures;
use futures::{executor::block_on, future::join_all, AsyncReadExt};
+use script::RadonScript;
use serde::Serialize;
pub use serde_cbor::{to_vec as cbor_to_vec, Value as CborValue};
#[cfg(test)]
@@ -176,6 +177,16 @@ fn handle_response_with_data_report(
execute_radon_script(response, &radon_script, context, settings)
}
+/// Execute Radon Script using as input the RadonScript and the RadonTypes value deserialized from a retrieval response
+fn process_response_with_data_report(
+ response: RadonTypes,
+ radon_script: &RadonScript,
+ context: &mut ReportContext<RadonTypes>,
+ settings: RadonScriptExecutionSettings,
+) -> Result<RadonReport<RadonTypes>> {
+ execute_radon_script(response, radon_script, context, settings)
+}
+
/// Run retrieval without performing any external network requests, return `Result<RadonReport>`.
pub fn run_retrieval_with_data_report(
retrieve: &RADRetrieve,
@@ -219,6 +230,9 @@ async fn http_response(
})?
};
+ // Validate the retrieval's radon script before performing the http request
+ let radon_script: RadonScript = unpack_radon_script(&retrieve.script)?;
+
// Use the provided HTTP client, or instantiate a new one if none
let client = match client {
Some(client) => client,
@@ -292,37 +306,42 @@ async fn http_response(
let (parts, mut body) = response.into_parts();
- let response: RadonTypes;
- if retrieve.kind != RADType::HttpHead && parts.headers.contains_key("accept-ranges") {
- // http response is a binary stream
- let mut response_bytes = Vec::<u8>::default();
-
- // todo: before reading the response buffer, an error should be thrown if it was too big
- body.read_to_end(&mut response_bytes).await.map_err(|x| {
- RadError::HttpOther {
- message: x.to_string(),
- }
- })?;
- response = RadonTypes::from(RadonBytes::from(response_bytes));
- } else {
- // response is a string
- let mut response_string = String::default();
- match retrieve.kind {
- RADType::HttpHead => {
- response_string = format!("{:?}", parts.headers);
- }
- _ => {
- body.read_to_string(&mut response_string)
- .await
- .map_err(|x| RadError::HttpOther {
- message: x.to_string(),
- })?;
+ let response: RadonTypes = match retrieve.kind {
+ RADType::HttpHead => {
+ RadonTypes::from(RadonString::from(
+ format!("{:?}", parts.headers)
+ ))
+ }
+ RADType::HttpGet | RADType::HttpPost => {
+ let expect_binary_response = if let Some((first_opcode, _)) = radon_script.first() {
+ (0x30 .. 0x3f).contains(&u8::from(*first_opcode))
+ } else {
+ false
+ };
+ if expect_binary_response {
+ let mut response_bytes = Vec::<u8>::default();
+ match body.read_to_end(&mut response_bytes).await {
+ Ok(_) => RadonTypes::from(RadonBytes::from(response_bytes)),
+ Err(err) => {
+ return Err(RadError::InvalidHttpResponse {
+ error: err.to_string()
+ });
+ }
+ }
+ } else {
+ let mut response_string = String::default();
+ match body.read_to_string(&mut response_string).await {
+ Ok(_) => RadonTypes::from(RadonString::from(response_string)),
+ Err(err) => {
+ return Err(RadError::InvalidHttpResponse {
+ error: err.to_string()
+ });
+ }
+ }
}
}
- response = RadonTypes::from(RadonString::from(response_string));
- }
-
- let result = handle_response_with_data_report(retrieve, response, context, settings);
+ _ => unreachable!()
+ };
match &result {
Ok(report) => {
log::debug!(
diff --git a/rad/src/operators/mod.rs b/rad/src/operators/mod.rs
index 6964f2623..23e9955ba 100644
--- a/rad/src/operators/mod.rs
+++ b/rad/src/operators/mod.rs
@@ -1,6 +1,6 @@
use std::fmt;
-use num_enum::TryFromPrimitive;
+use num_enum::{IntoPrimitive, TryFromPrimitive};
use serde::Serialize;
use witnet_data_structures::radon_report::ReportContext;
@@ -17,7 +17,7 @@ pub mod string;
/// List of RADON operators.
/// **WARNING: these codes are consensus-critical.** They can be renamed but they cannot be
/// re-assigned without causing a non-backwards-compatible protocol upgrade.
-#[derive(Copy, Clone, Debug, PartialEq, Eq, Serialize, TryFromPrimitive)]
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Serialize, IntoPrimitive, TryFromPrimitive)]
#[repr(u8)]
pub enum RadonOpCodes {
/// Only for the sake of allowing catch-alls when matching
From 9a5d5ca6a7d756beb310abe318fd5985dd863c68 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= <guillermo@otherplane.com>
Date: Fri, 23 Feb 2024 11:37:48 +0100
Subject: [PATCH 18/19] chore(rad): cargo fmt --fix
---
rad/src/lib.rs | 16 ++++++----------
1 file changed, 6 insertions(+), 10 deletions(-)
diff --git a/rad/src/lib.rs b/rad/src/lib.rs
index 6852ecc55..f2e46de69 100644
--- a/rad/src/lib.rs
+++ b/rad/src/lib.rs
@@ -307,14 +307,10 @@ async fn http_response(
let (parts, mut body) = response.into_parts();
let response: RadonTypes = match retrieve.kind {
- RADType::HttpHead => {
- RadonTypes::from(RadonString::from(
- format!("{:?}", parts.headers)
- ))
- }
+ RADType::HttpHead => RadonTypes::from(RadonString::from(format!("{:?}", parts.headers))),
RADType::HttpGet | RADType::HttpPost => {
let expect_binary_response = if let Some((first_opcode, _)) = radon_script.first() {
- (0x30 .. 0x3f).contains(&u8::from(*first_opcode))
+ (0x30..0x3f).contains(&u8::from(*first_opcode))
} else {
false
};
@@ -323,8 +319,8 @@ async fn http_response(
match body.read_to_end(&mut response_bytes).await {
Ok(_) => RadonTypes::from(RadonBytes::from(response_bytes)),
Err(err) => {
- return Err(RadError::InvalidHttpResponse {
- error: err.to_string()
+ return Err(RadError::InvalidHttpResponse {
+ error: err.to_string(),
});
}
}
@@ -334,13 +330,13 @@ async fn http_response(
Ok(_) => RadonTypes::from(RadonString::from(response_string)),
Err(err) => {
return Err(RadError::InvalidHttpResponse {
- error: err.to_string()
+ error: err.to_string(),
});
}
}
}
}
- _ => unreachable!()
+ _ => unreachable!(),
};
match &result {
Ok(report) => {
From d88590bec3941fa2103fc012721df618ba4a2baa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= <guillermo@otherplane.com>
Date: Fri, 23 Feb 2024 11:46:22 +0100
Subject: [PATCH 19/19] chore(rad): cargo clippy --fix
---
rad/src/lib.rs | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/rad/src/lib.rs b/rad/src/lib.rs
index f2e46de69..e0d2a5459 100644
--- a/rad/src/lib.rs
+++ b/rad/src/lib.rs
@@ -319,8 +319,8 @@ async fn http_response(
match body.read_to_end(&mut response_bytes).await {
Ok(_) => RadonTypes::from(RadonBytes::from(response_bytes)),
Err(err) => {
- return Err(RadError::InvalidHttpResponse {
- error: err.to_string(),
+ return Err(RadError::HttpOther {
+ message: err.to_string(),
});
}
}
@@ -329,8 +329,8 @@ async fn http_response(
match body.read_to_string(&mut response_string).await {
Ok(_) => RadonTypes::from(RadonString::from(response_string)),
Err(err) => {
- return Err(RadError::InvalidHttpResponse {
- error: err.to_string(),
+ return Err(RadError::HttpOther {
+ message: err.to_string(),
});
}
}
@@ -338,6 +338,7 @@ async fn http_response(
}
_ => unreachable!(),
};
+ let result = process_response_with_data_report(response, &radon_script, context, settings);
match &result {
Ok(report) => {
log::debug!(