feat: Migrate to bun.lock (#5112)

* bun: Migrate to bun.lock

(In backwards compatible way)

Read more: https://bun.sh/blog/bun-lock-text-lockfile

* Clean up

* More clean up

* Mount bun.lock to jailed process

Author: pyranota

backend/windmill-worker/nsjail/run.bun.config.proto

@@ -77,6 +77,12 @@ mount {
     mandatory: false
 }
 
+mount {
+    src: "{JOB_DIR}/bun.lock"
+    dst: "/tmp/{LANG}/bun.lock"
+    is_bind: true
+    mandatory: false
+}
 
 mount {
     src: "{JOB_DIR}/wrapper.mjs"

backend/windmill-worker/src/bun_executor.rs

@@ -62,24 +62,37 @@ const RELATIVE_BUN_BUILDER: &str = include_str!("../loader_builder.bun.js");
 
 const NSJAIL_CONFIG_RUN_BUN_CONTENT: &str = include_str!("../nsjail/run.bun.config.proto");
 
+pub const BUN_LOCK_SPLIT: &str = "\n//bun.lock\n";
 pub const BUN_LOCKB_SPLIT: &str = "\n//bun.lockb\n";
+pub const BUN_LOCK_SPLIT_WINDOWS: &str = "\r\n//bun.lock\r\n";
 pub const BUN_LOCKB_SPLIT_WINDOWS: &str = "\r\n//bun.lockb\r\n";
 
 pub const EMPTY_FILE: &str = "<empty>";
 
-fn split_lockfile(lockfile: &str) -> (&str, Option<&str>, bool) {
-    if let Some(index) = lockfile.find(BUN_LOCKB_SPLIT) {
+/// Returns (package.json, bun.lock(b), is_empty, is_binary)
+fn split_lockfile(lockfile: &str) -> (&str, Option<&str>, bool, bool) {
+    if let Some(index) = lockfile.find(BUN_LOCK_SPLIT) {
+        // Split using "\n//bun.lock\n"
+        let (before, after_with_sep) = lockfile.split_at(index);
+        let after = &after_with_sep[BUN_LOCK_SPLIT.len()..];
+        (before, Some(after), after == EMPTY_FILE, false)
+    } else if let Some(index) = lockfile.find(BUN_LOCKB_SPLIT) {
         // Split using "\n//bun.lockb\n"
         let (before, after_with_sep) = lockfile.split_at(index);
         let after = &after_with_sep[BUN_LOCKB_SPLIT.len()..];
-        (before, Some(after), after == EMPTY_FILE)
+        (before, Some(after), after == EMPTY_FILE, true)
+    } else if let Some(index) = lockfile.find(BUN_LOCK_SPLIT_WINDOWS) {
+        // Split using "\r\n//bun.lock\r\n"
+        let (before, after_with_sep) = lockfile.split_at(index);
+        let after = &after_with_sep[BUN_LOCK_SPLIT_WINDOWS.len()..];
+        (before, Some(after), after == EMPTY_FILE, false)
     } else if let Some(index) = lockfile.find(BUN_LOCKB_SPLIT_WINDOWS) {
         // Split using "\r\n//bun.lockb\r\n"
         let (before, after_with_sep) = lockfile.split_at(index);
         let after = &after_with_sep[BUN_LOCKB_SPLIT_WINDOWS.len()..];
-        (before, Some(after), after == EMPTY_FILE)
+        (before, Some(after), after == EMPTY_FILE, true)
     } else {
-        (lockfile, None, false)
+        (lockfile, None, false, false)
     }
 }
 
@@ -199,18 +212,18 @@ pub async fn gen_bun_lockfile(
         }
         if !npm_mode {
             #[cfg(any(target_os = "linux", target_os = "macos"))]
-            content.push_str(BUN_LOCKB_SPLIT);
+            content.push_str(BUN_LOCK_SPLIT);
 
             #[cfg(target_os = "windows")]
-            content.push_str(BUN_LOCKB_SPLIT_WINDOWS);
+            content.push_str(BUN_LOCK_SPLIT_WINDOWS);
 
             {
-                let file = format!("{job_dir}/bun.lockb");
+                let file = format!("{job_dir}/bun.lock");
                 if !empty_deps && tokio::fs::metadata(&file).await.is_ok() {
                     let mut file = File::open(&file).await?;
-                    let mut buf = vec![];
-                    file.read_to_end(&mut buf).await?;
-                    content.push_str(&base64::engine::general_purpose::STANDARD.encode(&buf));
+                    let mut buf = String::default();
+                    file.read_to_string(&mut buf).await?;
+                    content.push_str(&buf);
                 } else {
                     content.push_str(&EMPTY_FILE);
                 }
@@ -277,7 +290,7 @@ pub async fn install_bun_lockfile(
         .env_clear()
         .envs(PROXY_ENVS.clone())
         .envs(common_bun_proc_envs)
-        .args(vec!["install"])
+        .args(vec!["install", "--save-text-lockfile"])
         .stdout(Stdio::piped())
         .stderr(Stdio::piped());
 
@@ -785,26 +798,30 @@ async fn compute_bundle_local_and_remote_path(
 }
 
 pub async fn prepare_job_dir(reqs: &str, job_dir: &str) -> Result<()> {
-    let (pkg, lock, empty) = split_lockfile(reqs);
+    let (pkg, lock, empty, is_binary) = split_lockfile(reqs);
     let _ = write_file(job_dir, "package.json", pkg)?;
 
     if !empty {
         if let Some(lock) = lock {
-            let _ = write_lockb(lock, job_dir).await?;
+            let _ = write_lock(lock, job_dir, is_binary).await?;
         }
     }
 
     Ok(())
 }
-async fn write_lockb(splitted_lockb_2: &str, job_dir: &str) -> Result<()> {
-    write_file_binary(
-        job_dir,
-        "bun.lockb",
-        &base64::engine::general_purpose::STANDARD
-            .decode(splitted_lockb_2)
-            .map_err(|_| error::Error::InternalErr("Could not decode bun.lockb".to_string()))?,
-    )
-    .await?;
+async fn write_lock(splitted_lockb_2: &str, job_dir: &str, is_binary: bool) -> Result<()> {
+    if is_binary {
+        write_file_binary(
+            job_dir,
+            "bun.lockb",
+            &base64::engine::general_purpose::STANDARD
+                .decode(splitted_lockb_2)
+                .map_err(|_| error::Error::InternalErr(format!("Could not decode bun.lockb")))?,
+        )
+        .await?;
+    } else {
+        write_file(job_dir, "bun.lock", splitted_lockb_2)?;
+    };
     Ok(())
 }
 
@@ -900,26 +917,26 @@ pub async fn handle_bun_job(
     } else if let Some(codebase) = codebase.as_ref() {
         pull_codebase(&job.workspace_id, codebase, job_dir).await?;
     } else if let Some(reqs) = requirements_o.as_ref() {
-        let (pkg, lock, empty) = split_lockfile(reqs);
+        let (pkg, lock, empty, is_binary) = split_lockfile(reqs);
 
         if lock.is_none() && !annotation.npm {
             return Err(error::Error::ExecutionErr(
-                format!("Invalid requirements, expected to find //bun.lockb split pattern in reqs. Found: |{reqs}|")
+                format!("Invalid requirements, expected to find //bun.lock{} split pattern in reqs. Found: |{reqs}|", if is_binary {"b"} else {""})
             ));
         }
 
         let _ = write_file(job_dir, "package.json", pkg)?;
-        let lockb = if annotation.npm { "" } else { lock.unwrap() };
+        let lock = if annotation.npm { "" } else { lock.unwrap() };
         if !empty {
             let mut skip_install = false;
             let mut create_buntar = false;
             let mut buntar_path = "".to_string();
 
             if !annotation.npm {
-                let _ = write_lockb(lockb, job_dir).await?;
+                let _ = write_lock(lock, job_dir, is_binary).await?;
 
                 let mut sha_path = sha2::Sha256::new();
-                sha_path.update(lockb.as_bytes());
+                sha_path.update(lock.as_bytes());
 
                 let buntar_name =
                     base64::engine::general_purpose::URL_SAFE.encode(sha_path.finalize());
@@ -962,7 +979,7 @@ pub async fn handle_bun_job(
                         Some(&vec![
                             "main.ts".to_string(),
                             "package.json".to_string(),
-                            "bun.lockb".to_string(),
+                            if is_binary { "bun.lockb" } else { "bun.lock" }.to_string(),
                             "shared".to_string(),
                             "bunfig.toml".to_string(),
                         ]),
@@ -1592,25 +1609,29 @@ pub async fn start_worker(
     if let Some(codebase) = codebase.as_ref() {
         pull_codebase(w_id, codebase, job_dir).await?;
     } else if let Some(reqs) = requirements_o {
-        let (pkg, lock, empty) = split_lockfile(&reqs);
+        let (pkg, lock, empty, is_binary) = split_lockfile(&reqs);
         if lock.is_none() {
             return Err(error::Error::ExecutionErr(
                 format!("Invalid requirements, expected to find //bun.lockb split pattern in reqs. Found: |{reqs}|")
             ));
         }
         let _ = write_file(job_dir, "package.json", pkg)?;
-        let lockb = lock.unwrap();
+        let lock = lock.unwrap();
         if !empty {
-            let _ = write_file_binary(
-                job_dir,
-                "bun.lockb",
-                &base64::engine::general_purpose::STANDARD
-                    .decode(lockb)
-                    .map_err(|_| {
-                        error::Error::InternalErr("Could not decode bun.lockb".to_string())
-                    })?,
-            )
-            .await?;
+            if is_binary {
+                let _ = write_file_binary(
+                    job_dir,
+                    "bun.lockb",
+                    &base64::engine::general_purpose::STANDARD
+                        .decode(lock)
+                        .map_err(|_| {
+                            error::Error::InternalErr("Could not decode bun.lockb".to_string())
+                        })?,
+                )
+                .await?;
+            } else {
+                write_file(job_dir, "bun.lock", lock)?;
+            }
 
             install_bun_lockfile(
                 &mut mem_peak,