Add gcsthin for access to google storage

.gitmodules

@@ -13,3 +13,6 @@
 [submodule "deps/set_ns_last_pid"]
 	path = deps/set_ns_last_pid
 	url = https://github.com/twosigma/set_ns_last_pid.git
+[submodule "deps/gcsthin"]
+	path = deps/gcsthin
+	url = https://github.com/twosigma/gcsthin.git

Makefile

@@ -48,6 +48,7 @@ $(DIST_LIB_DIR):
 DIST_BINS := \
 	deps/criu/criu/criu \
 	deps/criu-image-streamer/criu-image-streamer \
+	deps/gcsthin/gcsthin \
 	deps/set_ns_last_pid/set_ns_last_pid \
 	target/$(BUILD)/fastfreeze \
 	$(shell which pv) \

README.md

@@ -71,11 +71,11 @@ FastFreeze includes the following high-level features:
 
 * **Low overhead**: FastFreeze needs less than 100MB of memory to perform a
   checkpoint or a restore. This memory headroom must be reserved in the
-  container in addition to what the application uses. Note that the standard S3
-  and GCS uploaders (`aws s3` and `gsutil`) tend to use a lot of memory (500MB)
-  due to the fact that they are written in Python and use large buffers. In the
-  future, we plan to open-source our custom uploaders that can be used with
-  FastFreeze.
+  container in addition to what the application uses. Note that the standard AWS
+  S3 uploader (`aws s3`) tends to use a lot of memory (500MB) because it is
+  written in Python and uses generous buffers. However, when using Google
+  Cloud Storage, FastFreeze uses a bundled fast and memory efficient uploader,
+  [gcsthin](https://github.com/twosigma/gcsthin), which uses 10MB of memory.
 
 * **Compression**: Checkpoint images can be compressed on the fly with lz4 or
   zstd. Setting the `--cpu-budget` option when checkpointing provides ways to

deps/Makefile

@@ -15,14 +15,15 @@ $(DEPS_FILE):
 		build-essential pkg-config \
 		libprotobuf-dev libprotobuf-c-dev protobuf-c-compiler protobuf-compiler \
 		libbsd-dev iproute2 libcap-dev libnl-3-dev libnet-dev libaio-dev \
-		patchelf pv liblz4-tool zstd jq wget openssl
+		patchelf pv liblz4-tool zstd jq wget openssl libssl-dev
 	touch $@
 
 BUILDS := \
 	build_criu \
 	build_libvirtcpuid \
 	build_libvirttime \
 	build_criu_image_streamer \
+	build_gcsthin \
 	build_set_ns_last_pid \
 
 .PHONY: $(BUILDS) clean
@@ -43,6 +44,9 @@ build_libvirttime: | $(DEPS_FILE)
 build_criu_image_streamer: | $(DEPS_FILE) $(RUSTUP_DEP_FILE)
 	$(MAKE) -C criu-image-streamer
 
+build_gcsthin: | $(DEPS_FILE) $(RUSTUP_DEP_FILE)
+	$(MAKE) -C gcsthin
+
 build_set_ns_last_pid: | $(DEPS_FILE)
 	$(MAKE) -C set_ns_last_pid
 
@@ -51,4 +55,5 @@ clean:
 	$(MAKE) -C libvirtcpuid clean
 	$(MAKE) -C libvirttime clean
 	$(MAKE) -C criu-image-streamer clean
+	$(MAKE) -C gcsthin clean
 	$(MAKE) -C set_ns_last_pid clean

scripts/Dockerfile.build

@@ -21,7 +21,8 @@ RUN set -ex; \
 # Build dependencies (CRIU, rust toolchain, libvirtcpuid, etc.)
 COPY deps deps
 # We clean first because we might have a copy of the host compiled binaries
-RUN make -C deps clean && make -C deps -j4
+# No -j2 because compiling two rust programs will make the memory usage blow up
+RUN make -C deps clean && make -C deps
 ENV CARGO=/root/.cargo/bin/cargo
 
 # Build FastFreeze Rust dependencies

scripts/fastfreeze

@@ -26,6 +26,6 @@ unset TMUX
 #                            The metrics are formatted in JSON and passed as first argument
 # CRIU_OPTS                  Additional arguments to pass to CRIU, whitespace separated
 # S3_CMD                     Command to access AWS S3. Defaults to 'aws s3'
-# GS_CMD                     Command to access Google Storage. Defaults to 'gsutil'
+# GS_CMD                     Command to access Google Storage. Defaults to 'gcsthin'
 
 exec $FF_DIR/bin/fastfreeze "$@"

src/cli/checkpoint.rs

@@ -51,7 +51,7 @@ ENVS:
                                 The metrics are formatted in JSON and passed as first argument
     CRIU_OPTS                   Additional arguments to pass to CRIU, whitespace separated
     S3_CMD                      Command to access AWS S3. Defaults to 'aws s3'
-    GS_CMD                      Command to access Google Storage. Defaults to 'gsutil'
+    GS_CMD                      Command to access Google Storage. Defaults to 'gcsthin'
     TAR_CMD                     Command to tar the file system. Defaults to 'tar'"
 ))]
 pub struct Checkpoint {

src/cli/extract.rs

@@ -29,7 +29,7 @@ use crate::{
 #[structopt(after_help("\
 ENVS:
     S3_CMD   Command to access AWS S3. Defaults to 'aws s3'
-    GS_CMD   Command to access Google Storage. Defaults to 'gsutil'"
+    GS_CMD   Command to access Google Storage. Defaults to 'gcsthin'"
 ))]
 pub struct Extract {
     /// Image URL, which can also be a regular local path

src/cli/run.rs

@@ -63,7 +63,7 @@ ENVS:
     FF_FAKE_ROOT                Setting to 1 instructs FastFreeze to use uid=0 when creating user namespaces
     CRIU_OPTS                   Additional arguments to pass to CRIU, whitespace separated
     S3_CMD                      Command to access AWS S3. Defaults to 'aws s3'
-    GS_CMD                      Command to access Google Storage. Defaults to 'gsutil'
+    GS_CMD                      Command to access Google Storage. Defaults to 'gcsthin'
     TAR_CMD                     Command to untar the file system. Defaults to 'tar'
 
 EXIT CODES:

src/store/gs.rs

@@ -20,7 +20,7 @@ use crate::util::UrlExt;
 
 lazy_static! {
     static ref GS_CMD: String = std::env::var("GS_CMD")
-        .unwrap_or_else(|_| "gsutil".to_string());
+        .unwrap_or_else(|_| "gcsthin".to_string());
 }
 
 pub struct Store {