Skip to content

Commit

Permalink
Add support for OCI registries.
Browse files Browse the repository at this point in the history 
This patch introduces the ability to read repo contents from OCI
registries, like ghcr.io, using the new 'oci' protocol. User can
specify this protocol in their DNF .repo files as shown below:

[oci-test]
name=OCI Test
baseurl=oci://ghcr.io/atgreen/librepo/gh-cli
enabled=1
gpgcheck=1

To set up the server-side repository, create a public package
repository in github, and populate it by pushing the repo file
contents using the ORAS cli tool.

createrepo .
FILES=$(find . -type f | sed 's|^\./||')
for FILE in $FILES; do
    oras push ghcr.io/atgreen/librepo/gh-cli/$FILE:latest $FILE
done

Currently, only public repositories are supported. To support private
package repositories, a bearer token is required. Implementing this
would necessitate changes to libdnf to allow for a bearer_token
configuration option in .repo files.

= changelog =
msg:           Add support for OCI registries
type:          enhancement
  • Loading branch information
@atgreen
atgreen committed Jul 21, 2024
1 parent 3c85711 commit 3a5cbda
Show file tree
Hide file tree
Showing 6 changed files with 172 additions and 33 deletions.
2 changes: 2 additions & 0 deletions CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ FIND_PACKAGE(PkgConfig)
PKG_CHECK_MODULES(GLIB2 glib-2.0>=2.66 gio-2.0 REQUIRED)
PKG_SEARCH_MODULE(LIBCRYPTO REQUIRED libcrypto openssl)
PKG_CHECK_MODULES(LIBXML2 libxml-2.0 REQUIRED)
PKG_CHECK_MODULES(JSONGLIB json-glib-1.0 REQUIRED)
FIND_PACKAGE(CURL 7.52.0 REQUIRED)

IF (USE_GPGME)
Expand Down Expand Up @@ -106,6 +107,7 @@ INCLUDE_DIRECTORIES("${CMAKE_CURRENT_BINARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/l

INCLUDE_DIRECTORIES(${LIBXML2_INCLUDE_DIRS})
INCLUDE_DIRECTORIES(${CURL_INCLUDE_DIR})
INCLUDE_DIRECTORIES(${JSONGLIB_INCLUDE_DIRS})
#INCLUDE_DIRECTORIES(${CHECK_INCLUDE_DIR})
IF (USE_GPGME AND ENABLE_SELINUX)
INCLUDE_DIRECTORIES(${SELINUX_INCLUDE_DIRS})
Expand Down
1 change: 1 addition & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ Fedora/Ubuntu name
* glib2 (http://developer.gnome.org/glib/) - glib2-devel/libglib2.0-dev
* libattr (https://savannah.nongnu.org/projects/attr) - libattr-devel/libattr1-dev
* libcurl (http://curl.haxx.se/libcurl/) - libcurl-devel/libcurl4-openssl-dev
* json-glib (https://wiki.gnome.org/Projects/JsonGlib) - json-glib/json-glib-devel
* openssl (http://www.openssl.org/) - openssl-devel/libssl-dev
* python (http://python.org/) - python3-devel/libpython3-dev
* One of the libraries:
Expand Down
1 change: 1 addition & 0 deletions librepo/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ TARGET_LINK_LIBRARIES(librepo
${LIBXML2_LIBRARIES}
${CURL_LIBRARY}
${LIBCRYPTO_LIBRARIES}
${JSONGLIB_LIBRARIES}
${GLIB2_LIBRARIES}
)
IF (USE_GPGME)
Expand Down
197 changes: 164 additions & 33 deletions librepo/downloader.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
#include <sys/xattr.h>
#include <fcntl.h>
#include <curl/curl.h>
#include <json-glib/json-glib.h>

#ifdef WITH_ZCHUNK
#include <zck.h>
Expand Down Expand Up @@ -85,6 +86,16 @@ typedef enum {
All headers which we were looking for are already found*/
} LrHeaderCbState;

/** Enum with OCI file status */
typedef enum {
LR_OCI_DL_WAITING, /*!<
The OCI file is waiting to be processed. */
LR_OCI_DL_MANIFEST, /*!<
The OCI manifest file is being downloaded. */
LR_OCI_DL_LAYER, /*!<
The OCI layer is being downloaded. */
} LrOciState;

/** Enum with zchunk file status */
typedef enum {
LR_ZCK_DL_HEADER_CK, /*!<
Expand Down Expand Up @@ -185,6 +196,8 @@ typedef struct {
Last cb return code. */
struct curl_slist *curl_rqheaders; /*!<
Extra headers for request. */
LrOciState oci_state; /*!<
OCI download state. */

#ifdef WITH_ZCHUNK
LrZckState zck_state; /*!<
Expand Down Expand Up @@ -1399,7 +1412,19 @@ open_target_file(LrTarget *target, GError **err)
int fd;
FILE *f;

if (target->target->fd != -1) {
if (target->oci_state == LR_OCI_DL_MANIFEST) {
if (target->target->fn == NULL) {
// Create a temporary file for the OCI manifest
const char *tmpdir = getenv("TMPDIR");
if (tmpdir == NULL)
tmpdir = "/tmp";
char *tmpname = g_strdup_printf("%s/librepo-oci-XXXXXX", tmpdir);
close (mkstemp(tmpname));
target->target->fn = tmpname;
}
}

if (target->oci_state != LR_OCI_DL_MANIFEST && target->target->fd != -1) {
// Use supplied filedescriptor
fd = dup(target->target->fd);
if (fd == -1) {
Expand Down Expand Up @@ -1435,6 +1460,80 @@ open_target_file(LrTarget *target, GError **err)
return f;
}

/** Get the OCI manifest URL
*/
static char *
get_oci_manifest_url(char *oci_url, GError **err)
{
assert(!err || *err == NULL);

// Remove the 'oci://' prefix
char *hostname = strdup(oci_url + 6);

char *first_slash = strchr(hostname, '/');
if (first_slash == NULL || first_slash[1] == 0) {
g_set_error(err, LR_DOWNLOADER_ERROR, LRE_IO,
"invalid OCI URL format: %s",
oci_url);
return NULL;
}
*first_slash = 0;

char *result = g_strdup_printf ("https://%s/v2/%s/manifests/latest",
hostname,
first_slash + 1);
free (hostname);
return result;
}

static char *
get_oci_layer_url(char *oci_url, char *fn, GError **err) {
assert(!err || *err == NULL);

// Load the JSON manifest file
JsonParser *parser = json_parser_new();

// Load the JSON manifest file
if (!json_parser_load_from_file(parser, fn, err)) {
g_object_unref(parser);
return NULL;
}

// Delete the manifest file
unlink(fn);

// Get the root object
JsonNode *root = json_parser_get_root(parser);
JsonObject *root_obj = json_node_get_object(root);

// Navigate to the layers array
JsonArray *layers = json_object_get_array_member(root_obj, "layers");
JsonObject *first_layer = json_array_get_object_element(layers, 0);

// Extract the digest for the first layer
const char *digest = json_object_get_string_member(first_layer, "digest");

// Remove the 'oci://' prefix
char *hostname = strdup(oci_url + 6);

char *first_slash = strchr(hostname, '/');
if (first_slash == NULL || first_slash[1] == 0) {
g_set_error(err, LR_DOWNLOADER_ERROR, LRE_IO,
"invalid OCI URL format: %s",
oci_url);
return NULL;
}
*first_slash = 0;

char *result = g_strdup_printf ("https://%s/v2/%s/blobs/%s",
hostname,
first_slash + 1,
digest);
free (hostname);
g_object_unref(parser);
return result;
}

/** Prepare next transfer
*/
static gboolean
Expand Down Expand Up @@ -1478,6 +1577,17 @@ prepare_next_transfer(LrDownload *dd, gboolean *candidatefound, GError **err)

protocol = lr_detect_protocol(full_url);

if (protocol == LR_PROTOCOL_OCI) {
if (target->oci_state == LR_OCI_DL_WAITING) {
target->oci_state = LR_OCI_DL_MANIFEST;
full_url = get_oci_manifest_url(full_url, err);
} else if (target->oci_state == LR_OCI_DL_LAYER) {
full_url = get_oci_layer_url(full_url, target->target->fn, err);
}
if (!full_url)
goto fail;
}

// Prepare CURL easy handle
CURLcode c_rc;
CURL *h;
Expand Down Expand Up @@ -1656,6 +1766,18 @@ prepare_next_transfer(LrDownload *dd, gboolean *candidatefound, GError **err)
if (!headers)
lr_out_of_memory();
}
if (target->oci_state == LR_OCI_DL_MANIFEST) {
headers = curl_slist_append(headers, "Authorization: Bearer QQ==");
if (!headers)
lr_out_of_memory();
headers = curl_slist_append(headers, "Accept: application/vnd.oci.image.manifest.v1+json");
if (!headers)
lr_out_of_memory();
} else if (target->oci_state == LR_OCI_DL_LAYER) {
headers = curl_slist_append(headers, "Authorization: Bearer QQ==");
if (!headers)
lr_out_of_memory();
}
target->curl_rqheaders = headers;
c_rc = curl_easy_setopt(h, CURLOPT_HTTPHEADER, headers);
assert(c_rc == CURLE_OK);
Expand Down Expand Up @@ -2324,15 +2446,17 @@ check_transfer_statuses(LrDownload *dd, GError **err)
// New file was downloaded - clear checksums cached in extended attributes
lr_checksum_clear_cache(fd);

ret = check_finished_transfer_checksum(fd,
target->target->checksums,
&matches,
&transfer_err,
&tmp_err);
ret = target->oci_state == LR_OCI_DL_WAITING
? check_finished_transfer_checksum(fd,
target->target->checksums,
&matches,
&transfer_err,
&tmp_err)
: 1;
if (!ret) { // Error
g_propagate_prefixed_error(err, tmp_err, "Downloading from %s"
"was successful but error encountered while "
"checksumming: ", effective_url);
"was successful but error encountered while "
"checksumming: ", effective_url);
return FALSE;
}
#ifdef WITH_ZCHUNK
Expand Down Expand Up @@ -2506,31 +2630,38 @@ check_transfer_statuses(LrDownload *dd, GError **err)
target->tried_mirrors = g_slist_remove(target->tried_mirrors, target->mirror);
} else {
#endif /* WITH_ZCHUNK */
target->state = LR_DS_FINISHED;

// Remove xattr that states that the file is being downloaded
// by librepo, because the file is now completely downloaded
// and the xattr is not needed (is is useful only for resuming)
remove_librepo_xattr(target->target);

// Call end callback
LrEndCb end_cb = target->target->endcb;
if (end_cb) {
int rc = end_cb(target->target->cbdata,
LR_TRANSFER_SUCCESSFUL,
NULL);
if (rc == LR_CB_ERROR) {
target->cb_return_code = LR_CB_ERROR;
g_debug("%s: Downloading was aborted by LR_CB_ERROR "
"from end callback", __func__);
g_set_error(&fail_fast_error, LR_DOWNLOADER_ERROR,
LRE_CBINTERRUPTED,
"Interrupted by LR_CB_ERROR from end callback");
}
}
if (target->mirror)
lr_downloadtarget_set_usedmirror(target->target,
target->mirror->mirror->url);
if (target->oci_state == LR_OCI_DL_MANIFEST) {
// Now that we have the manifest, let's download the
// layer blob.
target->oci_state = LR_OCI_DL_LAYER;
target->state = LR_DS_WAITING;
} else {
target->state = LR_DS_FINISHED;

// Remove xattr that states that the file is being downloaded
// by librepo, because the file is now completely downloaded
// and the xattr is not needed (is is useful only for resuming)
remove_librepo_xattr(target->target);

// Call end callback
LrEndCb end_cb = target->target->endcb;
if (end_cb) {
int rc = end_cb(target->target->cbdata,
LR_TRANSFER_SUCCESSFUL,
NULL);
if (rc == LR_CB_ERROR) {
target->cb_return_code = LR_CB_ERROR;
g_debug("%s: Downloading was aborted by LR_CB_ERROR "
"from end callback", __func__);
g_set_error(&fail_fast_error, LR_DOWNLOADER_ERROR,
LRE_CBINTERRUPTED,
"Interrupted by LR_CB_ERROR from end callback");
}
}
if (target->mirror)
lr_downloadtarget_set_usedmirror(target->target,
target->mirror->mirror->url);
}
#ifdef WITH_ZCHUNK
}
#endif /* WITH_ZCHUNK */
Expand Down
3 changes: 3 additions & 0 deletions librepo/lrmirrorlist.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,9 @@ lr_detect_protocol(const char *url)
if (g_str_has_prefix(url, "rsync://"))
return LR_PROTOCOL_RSYNC;

if (g_str_has_prefix(url, "oci://"))
return LR_PROTOCOL_OCI;

return LR_PROTOCOL_OTHER;
}

Expand Down
1 change: 1 addition & 0 deletions librepo/lrmirrorlist.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ typedef enum {
LR_PROTOCOL_HTTP,
LR_PROTOCOL_FTP,
LR_PROTOCOL_RSYNC,
LR_PROTOCOL_OCI,
} LrProtocol;

/** A internal representation of a mirror */
Expand Down

0 comments on commit 3a5cbda

Please sign in to comment.