From 76014a9226e3d7b9c127ab1fc99ca27441ac3920 Mon Sep 17 00:00:00 2001 From: Rob Schiemann Date: Fri, 14 Jun 2024 16:43:22 +0000 Subject: [PATCH 1/5] WIP --- scripts/razel.R | 27 +++++++++++++++++---------- scripts/repo_management.R | 8 ++++++++ 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/scripts/razel.R b/scripts/razel.R index 1bf3f90..cd8fe53 100644 --- a/scripts/razel.R +++ b/scripts/razel.R @@ -337,14 +337,13 @@ generateWorkspaceMacro <- function(local_repo_dir = NULL, stopifnot(!(rule_type == "http_archive" && is.null(build_file_format))) pkg_type <- match.arg(pkg_type) - if (getOption("pkgType") == "source") { - pkg_type <- "source" - } if (is.null(build_file_format)) { build_file_format <- NA_character_ } + sys_info <- Sys.info() + if (!is.null(package_list_csv)) { repo_pkgs <- read.csv(package_list_csv, header = TRUE, @@ -355,13 +354,16 @@ generateWorkspaceMacro <- function(local_repo_dir = NULL, if (pkg_type == "both") { r_version <- R.Version() minor_version <- gsub("\\..*", "", r_version$minor) - sys_info <- Sys.info() - stopifnot(sys_info["sysname"] == "Darwin") - stopifnot(as.integer(sub("\\..*", "", sys_info["release"])) >= 20) - if (sys_info["machine"] == "arm64") { - prefix <- "mac_arm_" - } else { - prefix <- "mac_intel_" + stopifnot(sys_info["sysname"] == "Darwin" || sys_info["sysname"] == "Linux") + if (sys_info["sysname"] == "Darwin") { + stopifnot(as.integer(sub("\\..*", "", sys_info["release"])) >= 20) + if (sys_info["machine"] == "arm64") { + prefix <- "mac_arm_" + } else { + prefix <- "mac_intel_" + } + } else if (sys_info["sysname"] == "Linux"){ + prefix <- "linux_" } sha256_col <- paste0(prefix, r_version$major, "_", minor_version, "_sha256") if (sha256_col %in% colnames(repo_pkgs)) { @@ -395,6 +397,11 @@ generateWorkspaceMacro <- function(local_repo_dir = NULL, ".tgz", ".tar.gz")), stringsAsFactors = FALSE) + linuxBinaryDir <- function(repos) { + r_version <- R.Version() + repo_r_version <- sprintf("%s.%s",r_version$major, gsub("\\..*", "", r_version$minor)) + } + findPackages <- function(repos, suffix) { mergeWithRemote <- function(type) { remote_pkgs <- as.data.frame( diff --git a/scripts/repo_management.R b/scripts/repo_management.R index a7f7179..bb051cd 100644 --- a/scripts/repo_management.R +++ b/scripts/repo_management.R @@ -53,6 +53,10 @@ macContribDirs <- function(r_version) { return(sprintf("/bin/macosx/%scontrib/%s", contrib_prefix, r_version)) } +linuxContribDir <- function(r_version){ + return(sprintf("/bin/linux/ubuntu/%s/src/contrib", r_version)) +} + # Returns windows binary archive locations of repos based on R version. winContribDir <- function(r_version) { stopifnot(r_version %in% getOption("RVersions")) @@ -201,12 +205,16 @@ updateRepoIndex <- function(repo_dir) { tools::write_PACKAGES(paste0(repo_dir, srcContribDir()), type = "source", latestOnly = FALSE) for (r_version in getOption("RVersions")) { macDirs <- paste0(repo_dir, macContribDirs(r_version)) + linuxDir <- paste0(repo_dir, linuxContribDir(r_version)) winDir <- paste0(repo_dir, winContribDir(r_version)) for (macDir in macDirs) { if (file.exists(macDir)) { tools::write_PACKAGES(macDir, type = "mac.binary", latestOnly = FALSE) } } + if (file.exists(linuxDir)) { + tools::write_PACKAGES(linuxDir, type = "source", latestOnly = FALSE) + } if (file.exists(winDir)) { tools::write_PACKAGES(winDir, type = "win.binary", latestOnly = FALSE) } From bf806694f37d1f53d79324cf01083101dd7fa982 Mon Sep 17 00:00:00 2001 From: Rob Schiemann Date: Mon, 17 Jun 2024 19:08:25 +0000 Subject: [PATCH 2/5] More work. generateWorkspaceMacro now returns list of linux binaries. repo_management will generate packages file for linux binary subrepo. --- scripts/razel.R | 56 ++++++++++++++++++++++++++++++--------- scripts/repo_management.R | 2 +- 2 files changed, 45 insertions(+), 13 deletions(-) diff --git a/scripts/razel.R b/scripts/razel.R index cd8fe53..bcb144a 100644 --- a/scripts/razel.R +++ b/scripts/razel.R @@ -329,6 +329,7 @@ generateWorkspaceMacro <- function(local_repo_dir = NULL, remote_repos = getOption("repos"), mirror_repo_url = NULL, use_only_mirror_repo = FALSE, + use_linux_binaries = TRUE, fail_fast = FALSE, ...) { stopifnot(!is.null(local_repo_dir) || !is.null(package_list_csv)) @@ -336,14 +337,21 @@ generateWorkspaceMacro <- function(local_repo_dir = NULL, rule_type <- match.arg(rule_type) stopifnot(!(rule_type == "http_archive" && is.null(build_file_format))) + sys_info <- Sys.info() + pkg_type <- match.arg(pkg_type) + if (getOption("pkgType") == "source") { + pkg_type <- "source" + } + + if (sys_info["sysname"] == "Linux" && use_linux_binaries) { + pkg_type <- "both" + } if (is.null(build_file_format)) { build_file_format <- NA_character_ } - sys_info <- Sys.info() - if (!is.null(package_list_csv)) { repo_pkgs <- read.csv(package_list_csv, header = TRUE, @@ -391,22 +399,30 @@ generateWorkspaceMacro <- function(local_repo_dir = NULL, vecCompareVersion(repo_merged_pkgs[, "Version.binary"], repo_merged_pkgs[, "Version"]) >= 0 } } + + binary_extension <- ".tgz" + if (sys_info["sysname"] == "Linux" && use_linux_binaries) binary_extension <- ".tar.gz" + repo_pkgs <- cbind(repo_pkgs, - Archive = paste0(repo_pkgs$Package, "_", repo_pkgs$Version, + Archive = paste0(repo_pkgs$Package, "_", repo_pkgs$Version, ifelse(repo_pkgs[, "binary_package_available"], - ".tgz", ".tar.gz")), - stringsAsFactors = FALSE) - - linuxBinaryDir <- function(repos) { - r_version <- R.Version() - repo_r_version <- sprintf("%s.%s",r_version$major, gsub("\\..*", "", r_version$minor)) + binary_extension, ".tar.gz")), + stringsAsFactors = FALSE) + + linuxBinarySubrepoPath <- function(repo) { + r_version <- R.Version() + repo_r_version <- sprintf("%s.%s",r_version$major, gsub("\\..*", "", r_version$minor)) + subrepo_path <- sprintf("/bin/linux/%s", repo_r_version) + return(paste0(repo, subrepo_path)) } findPackages <- function(repos, suffix) { + repos <- mirror_repo_url + suffix <- ".mirrored" mergeWithRemote <- function(type) { remote_pkgs <- as.data.frame( - available.packages(repos = repos, type = type)[, c("Package", "Repository")], - stringsAsFactors = FALSE) + available.packages(repos = repos, type = type)[, c("Package", "Repository")], + stringsAsFactors = FALSE) colnames(remote_pkgs) <- c("Package", paste0("Repository", suffix)) if (type == "binary") { pkg_idx <- which(repo_pkgs[, "binary_package_available"]) @@ -416,8 +432,24 @@ generateWorkspaceMacro <- function(local_repo_dir = NULL, merge(repo_pkgs[pkg_idx, ], remote_pkgs, by = "Package", all.x = TRUE, all.y = FALSE) } + mergeWithRemoteLinuxBinaries <- function() { + linuxBinaryRepos <- sapply(repos, linuxBinarySubrepoPath) + remote_pkgs <- as.data.frame( + available.packages(repos = linuxBinaryRepos, type = "source")[, c("Package", "Repository")], + stringsAsFactors = FALSE) + colnames(remote_pkgs) <- c("Package", paste0("Repository", suffix)) + pkg_idx <- which(repo_pkgs[, "binary_package_available"]) + merge(repo_pkgs[pkg_idx, ], remote_pkgs, by = "Package", all.x = TRUE, all.y = FALSE) + } + if (pkg_type == "both") { - unordered_df <- rbind(mergeWithRemote("binary"), mergeWithRemote("source"), + if(suffix == ".mirrored" && sys_info["sysname"] == "Linux" && use_linux_binaries){ + remoteBinaries <- mergeWithRemoteLinuxBinaries() + } else { + remoteBinaries <- mergeWithRemote("binary") + } + sourceBinaries <-mergeWithRemote("source") + unordered_df <- rbind(remoteBinaries, mergeWithRemote("source"), make.row.names = FALSE, stringsAsFactors = FALSE) ordered_df <- unordered_df[match(repo_pkgs[, "Package"], unordered_df[, "Package"]), ] stopifnot(identical(ordered_df[, "Package"], repo_pkgs[, "Package"])) diff --git a/scripts/repo_management.R b/scripts/repo_management.R index bb051cd..6dc7ac4 100644 --- a/scripts/repo_management.R +++ b/scripts/repo_management.R @@ -54,7 +54,7 @@ macContribDirs <- function(r_version) { } linuxContribDir <- function(r_version){ - return(sprintf("/bin/linux/ubuntu/%s/src/contrib", r_version)) + return(sprintf("/bin/linux/%s/src/contrib", r_version)) } # Returns windows binary archive locations of repos based on R version. From acfcb08306161f4131adb1128d97016cfd738d61 Mon Sep 17 00:00:00 2001 From: Rob Schiemann Date: Mon, 17 Jun 2024 22:51:39 +0000 Subject: [PATCH 3/5] Remove some testing code. --- scripts/razel.R | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/razel.R b/scripts/razel.R index bcb144a..af666c8 100644 --- a/scripts/razel.R +++ b/scripts/razel.R @@ -417,8 +417,6 @@ generateWorkspaceMacro <- function(local_repo_dir = NULL, } findPackages <- function(repos, suffix) { - repos <- mirror_repo_url - suffix <- ".mirrored" mergeWithRemote <- function(type) { remote_pkgs <- as.data.frame( available.packages(repos = repos, type = type)[, c("Package", "Repository")], From 86b66784ed8a9c0deb7dfe0dea9b05d0048b9354 Mon Sep 17 00:00:00 2001 From: Rob Schiemann Date: Mon, 17 Jun 2024 22:53:39 +0000 Subject: [PATCH 4/5] Remove testing code. --- scripts/razel.R | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/razel.R b/scripts/razel.R index af666c8..0881da5 100644 --- a/scripts/razel.R +++ b/scripts/razel.R @@ -446,7 +446,6 @@ generateWorkspaceMacro <- function(local_repo_dir = NULL, } else { remoteBinaries <- mergeWithRemote("binary") } - sourceBinaries <-mergeWithRemote("source") unordered_df <- rbind(remoteBinaries, mergeWithRemote("source"), make.row.names = FALSE, stringsAsFactors = FALSE) ordered_df <- unordered_df[match(repo_pkgs[, "Package"], unordered_df[, "Package"]), ] From ad51fd0934c83b171d9f5ba3f7d1faa88bfd3b0e Mon Sep 17 00:00:00 2001 From: Rob Schiemann Date: Thu, 3 Oct 2024 20:42:28 +0000 Subject: [PATCH 5/5] Add missing bioc_version for 4.3 in isValidBinRepo. --- scripts/repo_management.R | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/repo_management.R b/scripts/repo_management.R index 6dc7ac4..9a6ed78 100644 --- a/scripts/repo_management.R +++ b/scripts/repo_management.R @@ -74,7 +74,8 @@ isValidBinRepo <- function(repo, r_version) { } bioc_version <- gsub("(.*packages/|/bioc)", "", repo) - return((bioc_version == "3.14" && r_version == "4.1") || + return((bioc_version == "3.18" && r_version == "4.3") || + (bioc_version == "3.14" && r_version == "4.1") || (bioc_version == "3.13" && r_version == "4.1") || (bioc_version == "3.12" && r_version == "4.0") || (bioc_version == "3.11" && r_version == "4.0") ||