diff --git a/compiler/serverless/ir_helper.py b/compiler/serverless/ir_helper.py index b048599bc..4eba3a0ec 100644 --- a/compiler/serverless/ir_helper.py +++ b/compiler/serverless/ir_helper.py @@ -160,7 +160,10 @@ def prepare_scripts_for_serverless_exec(ir: IR, shell_vars: dict, args: argparse mk_dirs = "mkdir -p "+config.PASH_TMP_PREFIX+" \n" for dir in dir_set: mk_dirs += "mkdir -p "+dir+" \n" - script = mk_dirs+to_shell(subgraph, args) + # rust_debug = "export RUST_BACKTRACE=full\n" + export_path = "export PATH=$PATH:runtime\n" + export_lib_path = "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:runtime/lib\n" + script = export_path+export_lib_path+mk_dirs+to_shell(subgraph, args) # generate scripts script_name = os.path.join(config.PASH_TMP_PREFIX, str(id_)) script_id_to_script[str(id_)] = script diff --git a/runtime/serverless/runtime/auto-split.sh b/runtime/serverless/runtime/auto-split.sh new file mode 100644 index 000000000..5478c701f --- /dev/null +++ b/runtime/serverless/runtime/auto-split.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +cd "$(dirname $0)" + +input="$1" +shift +outputs=("$@") +n_outputs="$#" + +# generate a temp file +temp="$(mktemp -u /tmp/pash_XXXXXXXXXX)" + +cat "$input" > "$temp" +total_lines=$(wc -l "$temp" | cut -f 1 -d ' ') +batch_size=$((total_lines / n_outputs)) +# echo "Input: $input" +# echo "Ouputs: $outputs" +# echo "Number of outputs: $n_outputs" +# echo "Total Lines: $total_lines" +# echo "Batch Size: $batch_size" + +cleanup() +{ + kill -SIGPIPE "$split_pid" > /dev/null 2>&1 +} +trap cleanup EXIT + + +# echo "$PASH_TOP/evaluation/tools/split $input $batch_size $outputs" +./split "$temp" "$batch_size" "${outputs[@]}" & +split_pid=$! +wait "$split_pid" +rm -f "$temp" diff --git a/runtime/serverless/runtime/col b/runtime/serverless/runtime/col new file mode 100755 index 000000000..8d617f032 Binary files /dev/null and b/runtime/serverless/runtime/col differ diff --git a/runtime/serverless/runtime/file b/runtime/serverless/runtime/file new file mode 100755 index 000000000..646f075ce Binary files /dev/null and b/runtime/serverless/runtime/file differ diff --git a/runtime/serverless/runtime/lib/libbsd.so.0 b/runtime/serverless/runtime/lib/libbsd.so.0 new file mode 100644 index 000000000..3a5509320 Binary files /dev/null and b/runtime/serverless/runtime/lib/libbsd.so.0 differ diff --git a/runtime/serverless/runtime/lib/libbz2.so.1.0 b/runtime/serverless/runtime/lib/libbz2.so.1.0 new file mode 100644 index 000000000..b4f6762c5 Binary files /dev/null and b/runtime/serverless/runtime/lib/libbz2.so.1.0 differ diff --git a/runtime/serverless/runtime/lib/libmagic.so.1 b/runtime/serverless/runtime/lib/libmagic.so.1 new file mode 100644 index 000000000..d194e3245 Binary files /dev/null and b/runtime/serverless/runtime/lib/libmagic.so.1 differ diff --git a/runtime/serverless/runtime/lib/magic.mgc b/runtime/serverless/runtime/lib/magic.mgc new file mode 100644 index 000000000..e701e04a2 Binary files /dev/null and b/runtime/serverless/runtime/lib/magic.mgc differ diff --git a/runtime/serverless/runtime/xargs b/runtime/serverless/runtime/xargs new file mode 100755 index 000000000..6ffe08b79 Binary files /dev/null and b/runtime/serverless/runtime/xargs differ