From 1401e9d0d3e0c746ec94c2e609391bf90330fd3a Mon Sep 17 00:00:00 2001 From: Nicolas Viennot Date: Fri, 9 Apr 2021 20:27:56 +0000 Subject: [PATCH] Add gcsthin for access to google storage --- .gitmodules | 3 +++ Makefile | 1 + README.md | 10 +++++----- deps/Makefile | 7 ++++++- deps/gcsthin | 1 + scripts/Dockerfile.build | 3 ++- scripts/fastfreeze | 2 +- src/cli/checkpoint.rs | 2 +- src/cli/extract.rs | 2 +- src/cli/run.rs | 2 +- src/store/gs.rs | 2 +- 11 files changed, 23 insertions(+), 12 deletions(-) create mode 160000 deps/gcsthin diff --git a/.gitmodules b/.gitmodules index a975ad5..aa9f504 100644 --- a/.gitmodules +++ b/.gitmodules @@ -13,3 +13,6 @@ [submodule "deps/set_ns_last_pid"] path = deps/set_ns_last_pid url = https://github.com/twosigma/set_ns_last_pid.git +[submodule "deps/gcsthin"] + path = deps/gcsthin + url = https://github.com/twosigma/gcsthin.git diff --git a/Makefile b/Makefile index 9d69170..350eed0 100644 --- a/Makefile +++ b/Makefile @@ -48,6 +48,7 @@ $(DIST_LIB_DIR): DIST_BINS := \ deps/criu/criu/criu \ deps/criu-image-streamer/criu-image-streamer \ + deps/gcsthin/gcsthin \ deps/set_ns_last_pid/set_ns_last_pid \ target/$(BUILD)/fastfreeze \ $(shell which pv) \ diff --git a/README.md b/README.md index 7502e7d..e969faf 100644 --- a/README.md +++ b/README.md @@ -71,11 +71,11 @@ FastFreeze includes the following high-level features: * **Low overhead**: FastFreeze needs less than 100MB of memory to perform a checkpoint or a restore. This memory headroom must be reserved in the - container in addition to what the application uses. Note that the standard S3 - and GCS uploaders (`aws s3` and `gsutil`) tend to use a lot of memory (500MB) - due to the fact that they are written in Python and use large buffers. In the - future, we plan to open-source our custom uploaders that can be used with - FastFreeze. + container in addition to what the application uses. Note that the standard AWS + S3 uploader (`aws s3`) tends to use a lot of memory (500MB) because it is + written in Python and uses generous buffers. However, when using Google + Cloud Storage, FastFreeze uses a bundled fast and memory efficient uploader, + [gcsthin](https://github.com/twosigma/gcsthin), which uses 10MB of memory. * **Compression**: Checkpoint images can be compressed on the fly with lz4 or zstd. Setting the `--cpu-budget` option when checkpointing provides ways to diff --git a/deps/Makefile b/deps/Makefile index 992f400..0cd4006 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -15,7 +15,7 @@ $(DEPS_FILE): build-essential pkg-config \ libprotobuf-dev libprotobuf-c-dev protobuf-c-compiler protobuf-compiler \ libbsd-dev iproute2 libcap-dev libnl-3-dev libnet-dev libaio-dev \ - patchelf pv liblz4-tool zstd jq wget openssl + patchelf pv liblz4-tool zstd jq wget openssl libssl-dev touch $@ BUILDS := \ @@ -23,6 +23,7 @@ BUILDS := \ build_libvirtcpuid \ build_libvirttime \ build_criu_image_streamer \ + build_gcsthin \ build_set_ns_last_pid \ .PHONY: $(BUILDS) clean @@ -43,6 +44,9 @@ build_libvirttime: | $(DEPS_FILE) build_criu_image_streamer: | $(DEPS_FILE) $(RUSTUP_DEP_FILE) $(MAKE) -C criu-image-streamer +build_gcsthin: | $(DEPS_FILE) $(RUSTUP_DEP_FILE) + $(MAKE) -C gcsthin + build_set_ns_last_pid: | $(DEPS_FILE) $(MAKE) -C set_ns_last_pid @@ -51,4 +55,5 @@ clean: $(MAKE) -C libvirtcpuid clean $(MAKE) -C libvirttime clean $(MAKE) -C criu-image-streamer clean + $(MAKE) -C gcsthin clean $(MAKE) -C set_ns_last_pid clean diff --git a/deps/gcsthin b/deps/gcsthin new file mode 160000 index 0000000..2f4cd33 --- /dev/null +++ b/deps/gcsthin @@ -0,0 +1 @@ +Subproject commit 2f4cd3331a964e06d69ce433dbfa57844e9df486 diff --git a/scripts/Dockerfile.build b/scripts/Dockerfile.build index 4bda76d..e3d7e59 100644 --- a/scripts/Dockerfile.build +++ b/scripts/Dockerfile.build @@ -21,7 +21,8 @@ RUN set -ex; \ # Build dependencies (CRIU, rust toolchain, libvirtcpuid, etc.) COPY deps deps # We clean first because we might have a copy of the host compiled binaries -RUN make -C deps clean && make -C deps -j4 +# No -j2 because compiling two rust programs will make the memory usage blow up +RUN make -C deps clean && make -C deps ENV CARGO=/root/.cargo/bin/cargo # Build FastFreeze Rust dependencies diff --git a/scripts/fastfreeze b/scripts/fastfreeze index b0765d9..2663d2a 100755 --- a/scripts/fastfreeze +++ b/scripts/fastfreeze @@ -26,6 +26,6 @@ unset TMUX # The metrics are formatted in JSON and passed as first argument # CRIU_OPTS Additional arguments to pass to CRIU, whitespace separated # S3_CMD Command to access AWS S3. Defaults to 'aws s3' -# GS_CMD Command to access Google Storage. Defaults to 'gsutil' +# GS_CMD Command to access Google Storage. Defaults to 'gcsthin' exec $FF_DIR/bin/fastfreeze "$@" diff --git a/src/cli/checkpoint.rs b/src/cli/checkpoint.rs index 4e9bdab..3a923e3 100644 --- a/src/cli/checkpoint.rs +++ b/src/cli/checkpoint.rs @@ -51,7 +51,7 @@ ENVS: The metrics are formatted in JSON and passed as first argument CRIU_OPTS Additional arguments to pass to CRIU, whitespace separated S3_CMD Command to access AWS S3. Defaults to 'aws s3' - GS_CMD Command to access Google Storage. Defaults to 'gsutil' + GS_CMD Command to access Google Storage. Defaults to 'gcsthin' TAR_CMD Command to tar the file system. Defaults to 'tar'" ))] pub struct Checkpoint { diff --git a/src/cli/extract.rs b/src/cli/extract.rs index e2366b6..5298522 100644 --- a/src/cli/extract.rs +++ b/src/cli/extract.rs @@ -29,7 +29,7 @@ use crate::{ #[structopt(after_help("\ ENVS: S3_CMD Command to access AWS S3. Defaults to 'aws s3' - GS_CMD Command to access Google Storage. Defaults to 'gsutil'" + GS_CMD Command to access Google Storage. Defaults to 'gcsthin'" ))] pub struct Extract { /// Image URL, which can also be a regular local path diff --git a/src/cli/run.rs b/src/cli/run.rs index c3f14ca..bf0282b 100644 --- a/src/cli/run.rs +++ b/src/cli/run.rs @@ -63,7 +63,7 @@ ENVS: FF_FAKE_ROOT Setting to 1 instructs FastFreeze to use uid=0 when creating user namespaces CRIU_OPTS Additional arguments to pass to CRIU, whitespace separated S3_CMD Command to access AWS S3. Defaults to 'aws s3' - GS_CMD Command to access Google Storage. Defaults to 'gsutil' + GS_CMD Command to access Google Storage. Defaults to 'gcsthin' TAR_CMD Command to untar the file system. Defaults to 'tar' EXIT CODES: diff --git a/src/store/gs.rs b/src/store/gs.rs index e6b1201..98d004d 100644 --- a/src/store/gs.rs +++ b/src/store/gs.rs @@ -20,7 +20,7 @@ use crate::util::UrlExt; lazy_static! { static ref GS_CMD: String = std::env::var("GS_CMD") - .unwrap_or_else(|_| "gsutil".to_string()); + .unwrap_or_else(|_| "gcsthin".to_string()); } pub struct Store {