From 6809d4d09fada2ecf5e1d9daa8b75f11a8dfe803 Mon Sep 17 00:00:00 2001 From: Anthony Green Date: Sun, 21 Jul 2024 19:19:33 -0400 Subject: [PATCH] Add support for OCI registries. This patch introduces the ability to read repo contents from OCI registries, like ghcr.io, using the new 'oci' protocol. User can specify this protocol in their DNF .repo files as shown below: [oci-test] name=OCI Test baseurl=oci://ghcr.io/atgreen/librepo/gh-cli enabled=1 gpgcheck=1 To set up the server-side repository, create a public package repository in github, and populate it by pushing the repo file contents using the ORAS cli tool. createrepo . FILES=$(find . -type f | sed 's|^\./||') for FILE in $FILES; do oras push ghcr.io/atgreen/librepo/gh-cli/$FILE:latest $FILE done Currently, only public repositories are supported. To support private package repositories, a bearer token is required. Implementing this would necessitate changes to libdnf to allow for a bearer_token configuration option in .repo files. = changelog = msg: Add support for OCI registries type: enhancement --- CMakeLists.txt | 2 + README.md | 1 + librepo.spec | 1 + librepo/CMakeLists.txt | 1 + librepo/downloader.c | 197 ++++++++++++++++++++++++++++++++++------- librepo/lrmirrorlist.c | 3 + librepo/lrmirrorlist.h | 1 + 7 files changed, 173 insertions(+), 33 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 166ac03f3..8eae75f18 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -35,6 +35,7 @@ FIND_PACKAGE(PkgConfig) PKG_CHECK_MODULES(GLIB2 glib-2.0>=2.66 gio-2.0 REQUIRED) PKG_SEARCH_MODULE(LIBCRYPTO REQUIRED libcrypto openssl) PKG_CHECK_MODULES(LIBXML2 libxml-2.0 REQUIRED) +PKG_CHECK_MODULES(JSONGLIB json-glib-1.0 REQUIRED) FIND_PACKAGE(CURL 7.52.0 REQUIRED) IF (USE_GPGME) @@ -106,6 +107,7 @@ INCLUDE_DIRECTORIES("${CMAKE_CURRENT_BINARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/l INCLUDE_DIRECTORIES(${LIBXML2_INCLUDE_DIRS}) INCLUDE_DIRECTORIES(${CURL_INCLUDE_DIR}) +INCLUDE_DIRECTORIES(${JSONGLIB_INCLUDE_DIRS}) #INCLUDE_DIRECTORIES(${CHECK_INCLUDE_DIR}) IF (USE_GPGME AND ENABLE_SELINUX) INCLUDE_DIRECTORIES(${SELINUX_INCLUDE_DIRS}) diff --git a/README.md b/README.md index 1c058bc98..a2435fb3f 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ Fedora/Ubuntu name * glib2 (http://developer.gnome.org/glib/) - glib2-devel/libglib2.0-dev * libattr (https://savannah.nongnu.org/projects/attr) - libattr-devel/libattr1-dev * libcurl (http://curl.haxx.se/libcurl/) - libcurl-devel/libcurl4-openssl-dev +* json-glib (https://wiki.gnome.org/Projects/JsonGlib) - json-glib/json-glib-devel * openssl (http://www.openssl.org/) - openssl-devel/libssl-dev * python (http://python.org/) - python3-devel/libpython3-dev * One of the libraries: diff --git a/librepo.spec b/librepo.spec index 4608dea9e..16df29fe1 100644 --- a/librepo.spec +++ b/librepo.spec @@ -48,6 +48,7 @@ BuildRequires: pkgconfig(rpm) >= 4.18.0 %endif BuildRequires: libattr-devel BuildRequires: libcurl-devel >= %{libcurl_version} +BuildRequires: pkgconfig(json-glib-1.0) BuildRequires: pkgconfig(libxml-2.0) BuildRequires: pkgconfig(libcrypto) %if %{need_selinux} diff --git a/librepo/CMakeLists.txt b/librepo/CMakeLists.txt index 68d78677b..e9c2b9028 100644 --- a/librepo/CMakeLists.txt +++ b/librepo/CMakeLists.txt @@ -67,6 +67,7 @@ TARGET_LINK_LIBRARIES(librepo ${LIBXML2_LIBRARIES} ${CURL_LIBRARY} ${LIBCRYPTO_LIBRARIES} + ${JSONGLIB_LIBRARIES} ${GLIB2_LIBRARIES} ) IF (USE_GPGME) diff --git a/librepo/downloader.c b/librepo/downloader.c index 40dbeb264..e943ec601 100644 --- a/librepo/downloader.c +++ b/librepo/downloader.c @@ -35,6 +35,7 @@ #include #include #include +#include #ifdef WITH_ZCHUNK #include @@ -85,6 +86,16 @@ typedef enum { All headers which we were looking for are already found*/ } LrHeaderCbState; +/** Enum with OCI file status */ +typedef enum { + LR_OCI_DL_WAITING, /*!< + The OCI file is waiting to be processed. */ + LR_OCI_DL_MANIFEST, /*!< + The OCI manifest file is being downloaded. */ + LR_OCI_DL_LAYER, /*!< + The OCI layer is being downloaded. */ +} LrOciState; + /** Enum with zchunk file status */ typedef enum { LR_ZCK_DL_HEADER_CK, /*!< @@ -185,6 +196,8 @@ typedef struct { Last cb return code. */ struct curl_slist *curl_rqheaders; /*!< Extra headers for request. */ + LrOciState oci_state; /*!< + OCI download state. */ #ifdef WITH_ZCHUNK LrZckState zck_state; /*!< @@ -1399,7 +1412,19 @@ open_target_file(LrTarget *target, GError **err) int fd; FILE *f; - if (target->target->fd != -1) { + if (target->oci_state == LR_OCI_DL_MANIFEST) { + if (target->target->fn == NULL) { + // Create a temporary file for the OCI manifest + const char *tmpdir = getenv("TMPDIR"); + if (tmpdir == NULL) + tmpdir = "/tmp"; + char *tmpname = g_strdup_printf("%s/librepo-oci-XXXXXX", tmpdir); + close (mkstemp(tmpname)); + target->target->fn = tmpname; + } + } + + if (target->oci_state != LR_OCI_DL_MANIFEST && target->target->fd != -1) { // Use supplied filedescriptor fd = dup(target->target->fd); if (fd == -1) { @@ -1435,6 +1460,80 @@ open_target_file(LrTarget *target, GError **err) return f; } +/** Get the OCI manifest URL + */ +static char * +get_oci_manifest_url(char *oci_url, GError **err) +{ + assert(!err || *err == NULL); + + // Remove the 'oci://' prefix + char *hostname = strdup(oci_url + 6); + + char *first_slash = strchr(hostname, '/'); + if (first_slash == NULL || first_slash[1] == 0) { + g_set_error(err, LR_DOWNLOADER_ERROR, LRE_IO, + "invalid OCI URL format: %s", + oci_url); + return NULL; + } + *first_slash = 0; + + char *result = g_strdup_printf ("https://%s/v2/%s/manifests/latest", + hostname, + first_slash + 1); + free (hostname); + return result; +} + +static char * +get_oci_layer_url(char *oci_url, char *fn, GError **err) { + assert(!err || *err == NULL); + + // Load the JSON manifest file + JsonParser *parser = json_parser_new(); + + // Load the JSON manifest file + if (!json_parser_load_from_file(parser, fn, err)) { + g_object_unref(parser); + return NULL; + } + + // Delete the manifest file + unlink(fn); + + // Get the root object + JsonNode *root = json_parser_get_root(parser); + JsonObject *root_obj = json_node_get_object(root); + + // Navigate to the layers array + JsonArray *layers = json_object_get_array_member(root_obj, "layers"); + JsonObject *first_layer = json_array_get_object_element(layers, 0); + + // Extract the digest for the first layer + const char *digest = json_object_get_string_member(first_layer, "digest"); + + // Remove the 'oci://' prefix + char *hostname = strdup(oci_url + 6); + + char *first_slash = strchr(hostname, '/'); + if (first_slash == NULL || first_slash[1] == 0) { + g_set_error(err, LR_DOWNLOADER_ERROR, LRE_IO, + "invalid OCI URL format: %s", + oci_url); + return NULL; + } + *first_slash = 0; + + char *result = g_strdup_printf ("https://%s/v2/%s/blobs/%s", + hostname, + first_slash + 1, + digest); + free (hostname); + g_object_unref(parser); + return result; +} + /** Prepare next transfer */ static gboolean @@ -1478,6 +1577,17 @@ prepare_next_transfer(LrDownload *dd, gboolean *candidatefound, GError **err) protocol = lr_detect_protocol(full_url); + if (protocol == LR_PROTOCOL_OCI) { + if (target->oci_state == LR_OCI_DL_WAITING) { + target->oci_state = LR_OCI_DL_MANIFEST; + full_url = get_oci_manifest_url(full_url, err); + } else if (target->oci_state == LR_OCI_DL_LAYER) { + full_url = get_oci_layer_url(full_url, target->target->fn, err); + } + if (!full_url) + goto fail; + } + // Prepare CURL easy handle CURLcode c_rc; CURL *h; @@ -1656,6 +1766,18 @@ prepare_next_transfer(LrDownload *dd, gboolean *candidatefound, GError **err) if (!headers) lr_out_of_memory(); } + if (target->oci_state == LR_OCI_DL_MANIFEST) { + headers = curl_slist_append(headers, "Authorization: Bearer QQ=="); + if (!headers) + lr_out_of_memory(); + headers = curl_slist_append(headers, "Accept: application/vnd.oci.image.manifest.v1+json"); + if (!headers) + lr_out_of_memory(); + } else if (target->oci_state == LR_OCI_DL_LAYER) { + headers = curl_slist_append(headers, "Authorization: Bearer QQ=="); + if (!headers) + lr_out_of_memory(); + } target->curl_rqheaders = headers; c_rc = curl_easy_setopt(h, CURLOPT_HTTPHEADER, headers); assert(c_rc == CURLE_OK); @@ -2324,15 +2446,17 @@ check_transfer_statuses(LrDownload *dd, GError **err) // New file was downloaded - clear checksums cached in extended attributes lr_checksum_clear_cache(fd); - ret = check_finished_transfer_checksum(fd, - target->target->checksums, - &matches, - &transfer_err, - &tmp_err); + ret = target->oci_state != LR_OCI_DL_MANIFEST + ? check_finished_transfer_checksum(fd, + target->target->checksums, + &matches, + &transfer_err, + &tmp_err) + : 1; if (!ret) { // Error g_propagate_prefixed_error(err, tmp_err, "Downloading from %s" - "was successful but error encountered while " - "checksumming: ", effective_url); + "was successful but error encountered while " + "checksumming: ", effective_url); return FALSE; } #ifdef WITH_ZCHUNK @@ -2506,31 +2630,38 @@ check_transfer_statuses(LrDownload *dd, GError **err) target->tried_mirrors = g_slist_remove(target->tried_mirrors, target->mirror); } else { #endif /* WITH_ZCHUNK */ - target->state = LR_DS_FINISHED; - - // Remove xattr that states that the file is being downloaded - // by librepo, because the file is now completely downloaded - // and the xattr is not needed (is is useful only for resuming) - remove_librepo_xattr(target->target); - - // Call end callback - LrEndCb end_cb = target->target->endcb; - if (end_cb) { - int rc = end_cb(target->target->cbdata, - LR_TRANSFER_SUCCESSFUL, - NULL); - if (rc == LR_CB_ERROR) { - target->cb_return_code = LR_CB_ERROR; - g_debug("%s: Downloading was aborted by LR_CB_ERROR " - "from end callback", __func__); - g_set_error(&fail_fast_error, LR_DOWNLOADER_ERROR, - LRE_CBINTERRUPTED, - "Interrupted by LR_CB_ERROR from end callback"); - } - } - if (target->mirror) - lr_downloadtarget_set_usedmirror(target->target, - target->mirror->mirror->url); + if (target->oci_state == LR_OCI_DL_MANIFEST) { + // Now that we have the manifest, let's download the + // layer blob. + target->oci_state = LR_OCI_DL_LAYER; + target->state = LR_DS_WAITING; + } else { + target->state = LR_DS_FINISHED; + + // Remove xattr that states that the file is being downloaded + // by librepo, because the file is now completely downloaded + // and the xattr is not needed (is is useful only for resuming) + remove_librepo_xattr(target->target); + + // Call end callback + LrEndCb end_cb = target->target->endcb; + if (end_cb) { + int rc = end_cb(target->target->cbdata, + LR_TRANSFER_SUCCESSFUL, + NULL); + if (rc == LR_CB_ERROR) { + target->cb_return_code = LR_CB_ERROR; + g_debug("%s: Downloading was aborted by LR_CB_ERROR " + "from end callback", __func__); + g_set_error(&fail_fast_error, LR_DOWNLOADER_ERROR, + LRE_CBINTERRUPTED, + "Interrupted by LR_CB_ERROR from end callback"); + } + } + if (target->mirror) + lr_downloadtarget_set_usedmirror(target->target, + target->mirror->mirror->url); + } #ifdef WITH_ZCHUNK } #endif /* WITH_ZCHUNK */ diff --git a/librepo/lrmirrorlist.c b/librepo/lrmirrorlist.c index 91cdc4be3..d5e5e9be5 100644 --- a/librepo/lrmirrorlist.c +++ b/librepo/lrmirrorlist.c @@ -42,6 +42,9 @@ lr_detect_protocol(const char *url) if (g_str_has_prefix(url, "rsync://")) return LR_PROTOCOL_RSYNC; + if (g_str_has_prefix(url, "oci://")) + return LR_PROTOCOL_OCI; + return LR_PROTOCOL_OTHER; } diff --git a/librepo/lrmirrorlist.h b/librepo/lrmirrorlist.h index ca9c3484a..91af1310c 100644 --- a/librepo/lrmirrorlist.h +++ b/librepo/lrmirrorlist.h @@ -35,6 +35,7 @@ typedef enum { LR_PROTOCOL_HTTP, LR_PROTOCOL_FTP, LR_PROTOCOL_RSYNC, + LR_PROTOCOL_OCI, } LrProtocol; /** A internal representation of a mirror */