Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Kernel filesystem change #663

Open
wants to merge 36 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
36 commits
Select commit Hold shift + click to select a range
ede4113
Update the rust toolchain to 1.70.
ShaleXIONG Jul 25, 2023
b7eba4c
Update the lolrpop version.
ShaleXIONG Jul 25, 2023
e7de1ee
Update the wasmtime and wasmi version.
ShaleXIONG Jul 25, 2023
e4709b4
Use the kernal file system in wasmtime, and wire into freestanding ex…
ShaleXIONG Aug 22, 2023
49ecc60
Remove the vfs but use the kernel filesystem.
ShaleXIONG Sep 5, 2023
688e02d
update the example to use relative path.
ShaleXIONG Sep 28, 2023
e5b23f8
Rework on the engine and related, use the kernel file system.
ShaleXIONG Oct 2, 2023
e133c37
Update the makefiles for the new engine.
ShaleXIONG Oct 2, 2023
ffdb9a6
Remove appending the root `/` in veracruz client when calling write f…
ShaleXIONG Oct 3, 2023
f1fcfe6
Update the test suite on the engine rework.
ShaleXIONG Oct 3, 2023
fa0e80d
Fix a big in wrong import in freestanding.
ShaleXIONG Oct 3, 2023
02aa1d6
Temporarily comment out the test case for native module.
ShaleXIONG Oct 3, 2023
ec25967
Update all the cargo.toml file.
ShaleXIONG Oct 4, 2023
ade56f7
Rework on the permission check for (remote) clients.
ShaleXIONG Oct 6, 2023
b304a68
Remove dead code and unifies Cargo.toml.
ShaleXIONG Oct 6, 2023
a7aea77
Rework on the native module interface using the linux named pipeline.
ShaleXIONG Nov 1, 2023
b943904
Check the execution permission in the execution engine before running.
ShaleXIONG Nov 1, 2023
04fd9c2
Rework and simplify on the Sandbox for native binary.
ShaleXIONG Nov 24, 2023
67bd9ce
Fix a bug caused by type check of policy.
ShaleXIONG Nov 24, 2023
5a57695
Rework on the generate policy, use derive from clap.
ShaleXIONG Nov 24, 2023
f817133
Add the missing program hash when generating policy.
ShaleXIONG Nov 24, 2023
0e3054e
Update the generate policy script
ShaleXIONG Nov 24, 2023
445ce36
Fix a bug due to whitespace in policy generation.
ShaleXIONG Nov 24, 2023
8e3b2eb
Remove the application code for fd_create, which is no longer used.
ShaleXIONG Nov 27, 2023
786af80
Update the machnism to load internal native module by matching name.
ShaleXIONG Nov 28, 2023
0e12d5a
Generate the spec of the native service in the policy.
ShaleXIONG Nov 28, 2023
93913c3
Add the missing `Execution` Trait definition.
ShaleXIONG Nov 28, 2023
3f14c48
Remove an unused mod in execution-engine.
ShaleXIONG Nov 30, 2023
01caffe
Fix the quickstart test in the CI.
ShaleXIONG Dec 4, 2023
31f96a5
Update the shamir example.
ShaleXIONG Dec 4, 2023
8e1ee26
Update Cargo.lock.
ShaleXIONG Dec 8, 2023
7f66daf
TEST minor
ShaleXIONG Dec 7, 2023
258dc1f
Fix the directory mapping in Sandbox.
ShaleXIONG Jan 29, 2024
854c975
fix a merge mistake
ShaleXIONG Apr 2, 2024
dbea580
update cargo.lock
ShaleXIONG Apr 3, 2024
f825222
update the CI script.
ShaleXIONG Apr 5, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ jobs:
uses: actions/checkout@v3
with:
repository: 'veracruz-project/video-object-detection'
ref: '20230704'
#ref: '20230704'
submodules: recursive
set-safe-directory: true
- name: Build
Expand Down
18 changes: 11 additions & 7 deletions crates/examples/rust-examples/aesctr-native/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -57,13 +57,15 @@ fn main() -> anyhow::Result<()> {
let aes_ctr_enc_input = AesCtrInput {
key,
iv,
input_path: PathBuf::from("/output/data.dat"),
output_path: PathBuf::from("/output/enc.dat"),
input_path: PathBuf::from("./output/data.dat"),
output_path: PathBuf::from("./output/enc.dat"),
is_encryption: true,
};
write(&aes_ctr_enc_input.input_path, input)?;
let aes_ctr_enc_input_bytes = postcard::to_allocvec(&aes_ctr_enc_input)?;
write("/services/aesctr.dat", aes_ctr_enc_input_bytes)?;
write("/tmp/aes/input", aes_ctr_enc_input_bytes)?;
// wait the service finish
let _ = read("/tmp/aes/output");
let output = read(aes_ctr_enc_input.output_path)?;
if output != expected_output {
failed = true;
Expand All @@ -82,20 +84,22 @@ fn main() -> anyhow::Result<()> {
let aes_ctr_enc_input = AesCtrInput {
key,
iv,
input_path: PathBuf::from("/output/data.dat"),
output_path: PathBuf::from("/output/dec.dat"),
input_path: PathBuf::from("./output/data.dat"),
output_path: PathBuf::from("./output/dec.dat"),
is_encryption: false,
};
write(&aes_ctr_enc_input.input_path, input)?;
let aes_ctr_enc_input_bytes = postcard::to_allocvec(&aes_ctr_enc_input)?;
write("/services/aesctr.dat", aes_ctr_enc_input_bytes)?;
write("/tmp/aes/input", aes_ctr_enc_input_bytes)?;
// wait the service finish
let _ = read("/tmp/aes/output");
let output = read(aes_ctr_enc_input.output_path)?;
if output != expected_output {
failed = true;
}

if !failed {
write("/output/aesctr_native_pass.txt", [])?;
write("./output/aesctr_native_pass.txt", [])?;
}
Ok(())
}
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ fn main() -> anyhow::Result<()> {
// an input errors
let mut raw_events = Vec::new();
for i in 0.. {
let filename = format!("/input-{}", i);
let filename = format!("./input-{}", i);
let event = match fs::read(filename) {
Ok(event) => event,
Err(err) => match err.kind() {
Expand All @@ -172,6 +172,6 @@ fn main() -> anyhow::Result<()> {
let raw_location = encode_location(location);

// write our output through libveracruz
fs::write("/output", &raw_location)?;
fs::write("./output", &raw_location)?;
Ok(())
}
4 changes: 2 additions & 2 deletions crates/examples/rust-examples/file-seek/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@
use anyhow;
use std::{fs::File, io::prelude::*, io::SeekFrom};

const INPUT_FILENAME: &'static str = "/input/README.md";
const OUTPUT_FILENAME: &'static str = "/output/NEW_README.md";
const INPUT_FILENAME: &'static str = "./input/README.markdown";
const OUTPUT_FILENAME: &'static str = "./output/NEW_README.markdown";

fn main() -> anyhow::Result<()> {
let mut f = File::open(INPUT_FILENAME)?;
Expand Down
4 changes: 2 additions & 2 deletions crates/examples/rust-examples/huffman-encoding/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@
use anyhow::{self, Ok};
use std::{collections::HashMap, fs};

const INPUT_FILENAME: &'static str = "/input/hello-world-1.dat";
const OUTPUT_FILENAME: &'static str = "/output/encoded_output.dat";
const INPUT_FILENAME: &'static str = "./input/hello-world-1.dat";
const OUTPUT_FILENAME: &'static str = "./output/encoded_output.dat";

type Link = Option<Box<Node>>;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -285,7 +285,7 @@ fn true_ip(lhs: &[f64], rhs: &[f64]) -> anyhow::Result<f64> {
/// them together into a single compound dataset, then trains a logistic regressor on this new
/// dataset. Input and output are assumed to be encoded in Postcard.
fn main() -> anyhow::Result<()> {
for path in fs::read_dir("/input/idash2017")? {
for path in fs::read_dir("./input/idash2017")? {
let path = path?.path();
println!("path in: {:?}", path);
let file_name = path.file_name().ok_or(anyhow!("cannot get file name"))?;
Expand All @@ -302,7 +302,8 @@ fn main() -> anyhow::Result<()> {
)?;
println!("result: {:?}, {:?}, {:?}", w_data, correct, auc);
let result_encode = postcard::to_allocvec::<(Vec<f64>, f64, f64)>(&(w_data, correct, auc))?;
let mut output = PathBuf::from("/output/idash2017/");
fs::create_dir_all("./output/idash2017/")?;
let mut output = PathBuf::from("./output/idash2017/");
output.push(file_name);
println!("output {:?}", output);
std::fs::OpenOptions::new()
Expand Down
4 changes: 2 additions & 2 deletions crates/examples/rust-examples/image-processing/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ use image::{imageops, GenericImageView, ImageFormat};
fn main() -> anyhow::Result<()> {
// Use the open function to load an image from a Path.
// `open` returns a `DynamicImage` on success.
let mut img = image::open("/input/image-processing.png")?;
let mut img = image::open("./input/image-processing.png")?;

// Transform the image
let subimg = imageops::crop(&mut img, 0, 0, 100, 100);
Expand All @@ -39,7 +39,7 @@ fn main() -> anyhow::Result<()> {
// Write the contents of this image to the Writer in PNG format.
subimg
.to_image()
.save_with_format("/output/image-processing.png", ImageFormat::Png)?;
.save_with_format("./output/image-processing.png", ImageFormat::Png)?;

Ok(())
}
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,8 @@ struct Customer {
/// structs. Fails with [`return_code::ErrorCode::BadInput`] if the number of inputs provided is
/// not equal to 2, or if the inputs cannot be deserialized from Bincode.
fn read_inputs() -> anyhow::Result<(Vec<AdvertisementViewer>, Vec<Customer>)> {
let adverts = fs::read("/input/intersection-advertisement-viewer.dat")?;
let customs = fs::read("/input/intersection-customer.dat")?;
let adverts = fs::read("./input/intersection-advertisement-viewer.dat")?;
let customs = fs::read("./input/intersection-customer.dat")?;

let adverts = postcard::from_bytes(&adverts)?;
let customs = postcard::from_bytes(&customs)?;
Expand Down Expand Up @@ -97,6 +97,6 @@ fn main() -> anyhow::Result<()> {
let (adverts, customs) = read_inputs()?;
let total = intersection_set_sum(&adverts, &customs);
let result_encode = postcard::to_allocvec::<f64>(&total)?;
fs::write("/output/intersection-set-sum.dat", result_encode)?;
fs::write("./output/intersection-set-sum.dat", result_encode)?;
Ok(())
}
4 changes: 2 additions & 2 deletions crates/examples/rust-examples/linear-regression/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ use std::fs;
/// and fails with `return_code::ErrorCode::BadInput` if the input cannot be
/// decoded from `postcard` into a Rust vector of floating-point pairs.
fn read_input() -> anyhow::Result<Vec<(f64, f64)>> {
let input = fs::read("/input/linear-regression.dat")?;
let input = fs::read("./input/linear-regression.dat")?;
Ok(postcard::from_bytes(&input)?)
}

Expand Down Expand Up @@ -93,6 +93,6 @@ fn main() -> anyhow::Result<()> {
let data = read_input()?;
let result = linear_regression(&data);
let result_encode = postcard::to_allocvec(&result)?;
fs::write("/output/linear-regression.dat", result_encode)?;
fs::write("./output/linear-regression.dat", result_encode)?;
Ok(())
}
6 changes: 3 additions & 3 deletions crates/examples/rust-examples/logistic-regression/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -228,8 +228,8 @@ fn read_all_datasets(input: &[Vec<u8>]) -> anyhow::Result<Vec<Dataset>> {
/// if the deserialization of any dataset fails for any reason, or if the
/// datasets have differing dimensionalities.
fn read_input() -> anyhow::Result<Dataset> {
let i0 = fs::read("/input/logistic-regression-0.dat")?;
let i1 = fs::read("/input/logistic-regression-1.dat")?;
let i0 = fs::read("./input/logistic-regression-0.dat")?;
let i1 = fs::read("./input/logistic-regression-1.dat")?;
let datas = read_all_datasets(&vec![i0, i1])?;
Ok(flatten(&datas))
}
Expand Down Expand Up @@ -266,6 +266,6 @@ fn main() -> anyhow::Result<()> {
let dataset = read_input()?;
let model = train(&dataset)?;
let result_encode = postcard::to_allocvec::<Vec<f64>>(&model)?;
fs::write("/output/logistic-regression.dat", result_encode)?;
fs::write("./output/logistic-regression.dat", result_encode)?;
Ok(())
}
Original file line number Diff line number Diff line change
Expand Up @@ -162,13 +162,14 @@ fn dec_approx(data: &[f64], norm: f64) -> Vec<f64> {
/// Entry point: reads the vector of floats, processes them, and writes back a new vector of
/// floats as output.
fn main() -> anyhow::Result<()> {
for path in fs::read_dir("/input/macd/")? {
for path in fs::read_dir("./input/macd/")? {
let path = path?.path();
let dataset = read_inputs(&path)?;
let (_wma12, _wma26, _wma_diff, _wma9, _macd_wma, _decision_wma, decisions_wma_approx) =
computation(dataset.as_slice());
let result_encode = postcard::to_allocvec::<Vec<f64>>(&decisions_wma_approx)?;
let mut output = PathBuf::from("/output/macd/");
fs::create_dir_all("./output/macd/")?;
let mut output = PathBuf::from("./output/macd/");
output.push(path.file_name().ok_or(anyhow!("cannot get file name"))?);
fs::write(output, result_encode)?;
}
Expand Down
4 changes: 3 additions & 1 deletion crates/examples/rust-examples/nop/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,6 @@
//! and copyright information.

/// Entry point: immediately returns success.
fn main() {}
fn main() {
println!("hello");
}
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ fn main() -> anyhow::Result<()> {
count + 1,
(last_result_or_init + stream1 + stream2),
))?;
fs::write("/output/accumulation.dat", result_encode)?;
fs::write("./output/accumulation.dat", result_encode)?;
Ok(())
}

Expand All @@ -69,13 +69,13 @@ fn read_last_result_or_init() -> anyhow::Result<(u64, f64)> {

/// Read from 'stream-0' and 'stream-1' at `offset`
fn read_stream(offset: u64) -> anyhow::Result<(f64, f64)> {
let mut stream0 = File::open("/input/number-stream-1.dat")?;
let mut stream0 = File::open("./input/number-stream-1.dat")?;
stream0.seek(SeekFrom::Start(offset))?;
let mut data0 = Vec::new();
stream0.read_to_end(&mut data0)?;
let n1: f64 = postcard::from_bytes(&data0)?;

let mut stream1 = File::open("/input/number-stream-2.dat")?;
let mut stream1 = File::open("./input/number-stream-2.dat")?;
stream1.seek(SeekFrom::Start(offset))?;
let mut data1 = Vec::new();
stream1.read_to_end(&mut data1)?;
Expand Down
11 changes: 6 additions & 5 deletions crates/examples/rust-examples/postcard-native/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,12 @@
//! See the file `LICENSE.md` in the Veracruz root directory for licensing
//! and copyright information.

use std::fs;
use std::fs::{read, write};

fn main() -> anyhow::Result<()> {
let input = fs::read("/input/postcard.dat")?;
fs::write("/services/postcard_string.dat", input)?;
let rst = fs::read("/services/postcard_result.dat")?;
fs::write("/output/postcard_native.txt", &rst)?;
let input = read("./input/postcard.dat")?;
write("/tmp/postcard/input", input)?;
let rst = read("/tmp/postcard/output")?;
write("./output/postcard_native.txt", &rst)?;
Ok(())
}
4 changes: 2 additions & 2 deletions crates/examples/rust-examples/postcard-wasm/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -67,9 +67,9 @@ pub struct Struct3 {
}

fn main() -> anyhow::Result<()> {
let input = fs::read("/input/postcard.dat")?;
let input = fs::read("./input/postcard.dat")?;
let rst: Vec<Struct3> = from_bytes(&input)?;
let rst = serde_json::to_string(&rst)?;
fs::write("/output/postcard_wasm.txt", rst)?;
fs::write("./output/postcard_wasm.txt", rst)?;
Ok(())
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
use anyhow;
use std::fs;

const OUTPUT_FILENAME: &'static str = "/output/number-set.txt";
const OUTPUT_FILENAME: &'static str = "./output/number-set.txt";

fn main() -> anyhow::Result<()> {
let mut set: Vec<u32> = Vec::new();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
use anyhow;
use std::fs;

const FILENAME: &'static str = "/output/number-set.txt";
const FILENAME: &'static str = "./output/number-set.txt";

fn main() -> anyhow::Result<()> {
let content = String::from_utf8(fs::read(FILENAME)?)?;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,12 +68,13 @@ fn set_intersection_sum(data: Vec<((u64, u64), u32)>, sample: Vec<(u64, u64)>) -
/// The program entry point: reads exactly one input, decodes it and computes the set
/// intersection-sum before re-encoding it into Postcard and returning.
fn main() -> anyhow::Result<()> {
for path in fs::read_dir("/input/private-set-inter-sum/")? {
for path in fs::read_dir("./input/private-set-inter-sum/")? {
let path = path?.path();
let (data, sample) = read_inputs(&path)?;
let result = set_intersection_sum(data, sample);
let result_encode = postcard::to_allocvec::<(usize, u64)>(&result)?;
let mut output = PathBuf::from("/output/private-set-inter-sum/");
fs::create_dir_all("./output/private-set-inter-sum/")?;
let mut output = PathBuf::from("./output/private-set-inter-sum/");
output.push(path.file_name().ok_or(anyhow!("cannot get file name"))?);
fs::write(output, result_encode)?;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,9 @@ struct Person {
/// returns a `Vec` of all hash-sets, one from each input provider. Fails with
/// `return_code::ErrorCode::BadInput` if any input cannot be deserialized from Bincode.
fn read_inputs() -> anyhow::Result<Vec<HashSet<Person>>> {
let input0 = fs::read("/input/private-set-1.dat")?;
let input0 = fs::read("./input/private-set-1.dat")?;
let data0 = postcard::from_bytes(&input0)?;
let input1 = fs::read("/input/private-set-2.dat")?;
let input1 = fs::read("./input/private-set-2.dat")?;
let data1 = postcard::from_bytes(&input1)?;
Ok(vec![data0, data1])
}
Expand Down Expand Up @@ -70,6 +70,6 @@ fn main() -> anyhow::Result<()> {
let inputs = read_inputs()?;
let result = set_intersection(&inputs);
let result_encode = postcard::to_allocvec::<HashSet<Person>>(&result)?;
fs::write("/output/private-set.dat", result_encode)?;
fs::write("./output/private-set.dat", result_encode)?;
Ok(())
}
3 changes: 2 additions & 1 deletion crates/examples/rust-examples/random-source/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,10 @@ use std::fs;

/// Write 32 random bytes to 'output'. The result is a Postcard-encoded vector of u8.
fn main() -> anyhow::Result<()> {
let output = "/output/random.dat";
let output = "./output/random.dat";
let bytes = rand::thread_rng().gen::<[u8; 32]>();
let rst = postcard::to_allocvec(&bytes.to_vec())?;
fs::create_dir_all("output/")?;
fs::write(output, rst)?;
Ok(())
}
9 changes: 2 additions & 7 deletions crates/examples/rust-examples/random-u32-list/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,8 @@ use rand::Rng;
use std::fs;

fn main() -> anyhow::Result<()> {
let output = "/output/unsorted_numbers.txt";
let bytes = rand::thread_rng()
.gen::<[u32; 32]>()
.iter()
.map(|n| n.to_string())
.collect::<Vec<String>>()
.join(",");
let output = "./output/unsorted_numbers.txt";
let bytes = rand::thread_rng().gen::<[u32; 32]>().iter().map(|n| n.to_string()).collect::<Vec<String>>().join(",");
println!("{}", bytes);
fs::write(output, bytes)?;
Ok(())
Expand Down
10 changes: 5 additions & 5 deletions crates/examples/rust-examples/read-file/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,23 +18,23 @@ use std::{fs, io::Write};

/// Read data from a file, encode using postcard, then write to another file.
fn main() -> anyhow::Result<()> {
let input = "/input/hello-world-1.dat";
let output = "/output/hello-world-1.dat";
let input = "./input/hello-world-1.dat";
let output = "./output/hello-world-1.dat";

let mut input_string = fs::read(input)?;

println!("hello");
fs::create_dir_all("/output/test")?;
fs::create_dir_all("./output/test")?;
std::fs::OpenOptions::new()
.write(true)
.create(true)
.truncate(true)
.open("/output/test/test.txt")?
.open("./output/test/test.txt")?
.write(&postcard::to_allocvec("hello")?)?;
println!("rust");

input_string.append(&mut "\"read_dir on '/output':".as_bytes().to_vec());
for file in fs::read_dir("/output/")? {
for file in fs::read_dir("./output/")? {
input_string.append(&mut file?.path().to_str().unwrap().as_bytes().to_vec());
}
input_string.append(&mut "\"".as_bytes().to_vec());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -194,7 +194,7 @@ fn shares_read_all() -> io::Result<Vec<Vec<u8>>> {
// open files until one fails
let mut shares = vec![];
for i in 0.. {
let filename = format!("/input/shamir-{}.dat", i);
let filename = format!("./input/shamir-{}.dat", i);
let share = match fs::read(filename) {
Ok(share) => share,
Err(err) => match err.kind() {
Expand All @@ -218,6 +218,7 @@ fn main() -> anyhow::Result<()> {
let secret = shares_reconstruct(&shares);

// write our output
fs::write("/output/shamir.dat", &secret)?;
let _ = fs::create_dir_all("./output/");
fs::write("./output/shamir.dat", &secret)?;
Ok(())
}
Loading
Loading