From cf26f31b0231549cae31a7291e5029621c31d93c Mon Sep 17 00:00:00 2001 From: "gan.wu" Date: Tue, 3 Dec 2024 23:00:27 +0800 Subject: [PATCH] =?UTF-8?q?[Feature][#11]=20=E6=94=AF=E6=8C=81=E6=8B=89?= =?UTF-8?q?=E5=88=97=E8=A1=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- examples/CMakeLists.txt | 5 +- examples/list.c | 62 ++++++++++++ lib/api.c | 26 +++++ lib/api.h | 30 ++++++ lib/auth.c | 29 ++++++ lib/auth.h | 7 ++ lib/http.c | 56 +++++++++-- lib/http.h | 11 +++ lib/json.c | 37 +++++++ lib/json.h | 9 ++ lib/ufile_list.c | 213 ++++++++++++++++++++++++++++++++++++++++ 11 files changed, 478 insertions(+), 7 deletions(-) create mode 100644 examples/list.c create mode 100644 lib/json.c create mode 100644 lib/json.h create mode 100644 lib/ufile_list.c diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 644d91b..97eb831 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -27,4 +27,7 @@ add_executable(delete delete.c) target_link_libraries(delete ${DEPEND_LIB}) add_executable(async_mput async_mput.c thpool.c) -target_link_libraries(async_mput ${DEPEND_LIB} pthread) #link pthread 库 \ No newline at end of file +target_link_libraries(async_mput ${DEPEND_LIB} pthread) #link pthread 库 + +add_executable(list list.c) +target_link_libraries(list ${DEPEND_LIB}) \ No newline at end of file diff --git a/examples/list.c b/examples/list.c new file mode 100644 index 0000000..8757600 --- /dev/null +++ b/examples/list.c @@ -0,0 +1,62 @@ +#include "../lib/api.h" +#include +#include +#include +#include + +void help(char *program_name) { + printf("Usage: %s [prefix=""]\n", program_name); +} + +int main(int argc, char *argv[]){ + if(argc < 2 || argc > 3){ + help(argv[0]); + return 1; + } + + const char *bucket_name = argv[1]; + const char *prefix = ""; + if (argc > 2) { + prefix = argv[2]; + } + + struct ufile_config cfg; + cfg.public_key = getenv("UFILE_PUBLIC_KEY"); + cfg.private_key = getenv("UFILE_PRIVATE_KEY"); + cfg.bucket_host = "api.ucloud.cn"; + cfg.file_host = "cn-bj.ufileos.com"; + + struct ufile_error error; + error = ufile_sdk_initialize(cfg, 1); + if(UFILE_HAS_ERROR(error.code)){ + ufile_sdk_cleanup(); + printf("Initialize SDK failed: %s\n", error.message); + return 1; + } + + int once_list_count = 5; + char *marker = ""; + char is_truncated = 1; + struct ufile_list_result result; + memset(&result, 0, sizeof(result)); + while (is_truncated) { + error = ufile_list(bucket_name, prefix, "", once_list_count, marker, &result); + if(UFILE_HAS_ERROR(error.code)){ + printf("List failed, error message: %s\n", error.message); + return 1; + } else{ + int i = 0; + for (i = 0; i < result.entry_num; i++) { + printf("key: %s, size: %ld, mime_type: %s, last_modified: %ld, create_time: %ld, etag: %s, storage_class: %s\n", + result.entries[i].filename, result.entries[i].size, result.entries[i].mime_type, result.entries[i].last_modified, + result.entries[i].create_time, result.entries[i].etag, result.entries[i].storage_class); + } + marker = result.next_marker; + is_truncated = result.is_truncated; + } + } + + ufile_free_list_result(result); + + ufile_sdk_cleanup(); +} \ No newline at end of file diff --git a/lib/api.c b/lib/api.c index fe95af6..b4abfd2 100644 --- a/lib/api.c +++ b/lib/api.c @@ -86,6 +86,32 @@ ufile_free_file_info(struct ufile_file_info info){ } } +void +ufile_free_list_result(struct ufile_list_result list_result) { + int i; + for (i = 0; i < list_result.entry_num; i++) { + struct ufile_list_result_entry *entry = &(list_result.entries[i]); + if (entry->filename != NULL) { + free((void*)(entry->filename)); + } + if (entry->mime_type != NULL) { + free((void*)(entry->mime_type)); + } + if (entry->filename != NULL) { + free((void*)(entry->etag)); + } + if (entry->storage_class != NULL) { + free((void*)(entry->storage_class)); + } + } + if(list_result.next_marker != NULL){ + free((void*)list_result.next_marker); + } + if (list_result.entries != NULL){ + free((void*)list_result.entries); + } +} + struct ufile_error ufile_load_config_from_json(const char* json_buf, struct ufile_config *cfg){ struct ufile_error error = NO_ERROR; diff --git a/lib/api.h b/lib/api.h index e41f20a..8e973a7 100644 --- a/lib/api.h +++ b/lib/api.h @@ -23,11 +23,15 @@ struct ufile_error{ #define UFILE_MULTIPLE_NOT_FINISH_ERROR_CODE -5 #define UFILE_BUCKET_REQ_ERROR_CODE -10 #define CURL_ERROR_CODE -20 +#define NO_MEMORY_ERROR_CODE -30 +#define UFILE_PARSE_JSON_ERROR_CODE -40 #define NO_ERROR {0, ""} #define CURL_INIT_ERROR_MSG "init curl failed." #define UFILE_HAS_ERROR(CODE) ((CODE) != 0 && (CODE)/100 != 2 ) + +#define UFILE_LIST_MAX_COUNT (1000) //****************************************************************** end //*********************************config @@ -154,6 +158,32 @@ ufile_bucket_create(const char *bucket_name, const char* region, const char* buc //bucket_name 为 bucket 名字 extern struct ufile_error ufile_bucket_delete(const char *bucket_name); + +struct ufile_list_result_entry { + char *filename; // 文件名称 + char *mime_type; // 文件mimetype + char *etag; // 标识文件内容 + long long size; // 文件大小 + char *storage_class; // 文件存储类型 + long long last_modified; //文件最后修改时间 + long long create_time; // 文件创建时间 +}; + +struct ufile_list_result { + struct ufile_list_result_entry *entries; + int entry_num; + char is_truncated; + char *next_marker; +}; + +//拉取指定前缀的对象列表 +extern struct ufile_error +ufile_list(const char *bucket_name, const char *prefix, const char *delimiter, int count, const char *marker, struct ufile_list_result *result); + +//清理list结果 +extern void +ufile_free_list_result(struct ufile_list_result list_result); + #ifdef __cplusplus } #endif diff --git a/lib/auth.c b/lib/auth.c index 7d91dc3..f6b04a8 100644 --- a/lib/auth.c +++ b/lib/auth.c @@ -28,6 +28,35 @@ char * ufile_file_authorization(const char *public_key, const char *private_key, return ufile_strconcat("UCloud ", public_key, ":", signature, NULL); } +char * ufile_file_authorization_for_list(const char *public_key, const char *private_key, + const char *method, const char *bucket, + const char *key, const char *mime_type, + const char *date, const char *md5, const char *prefix, + const char *marker, + const char *count, + const char *delimiter) { + char *sig_data = ufile_strconcat(method,"\n", + md5, "\n", + mime_type, "\n", + date, "\n", + "/",bucket, "/", key, + "\nlistobjects", + "\ndelimiter:", delimiter, + "\nmarker:", marker, + "\nmax-keys:", count, + "\nprefix:", prefix, + NULL); + unsigned char HMAC_str[HMAC_LEN]; + HMAC_SHA1(HMAC_str, (unsigned char*)private_key, strlen(private_key), + (unsigned char*)sig_data, strlen(sig_data)); + + char signature[20]; + base64encode(signature, (const char*)HMAC_str, 20); + + free(sig_data); + return ufile_strconcat("UCloud ", public_key, ":", signature, NULL); +} + char * ufile_download_authorization( const char* private_key, const char* bucket, diff --git a/lib/auth.h b/lib/auth.h index dfd4b9a..b0e71d4 100644 --- a/lib/auth.h +++ b/lib/auth.h @@ -14,6 +14,13 @@ char * ufile_file_authorization(const char *public_key, const char *private_key, const char *key, const char *mime_type, const char *date, const char *md5); +char * ufile_file_authorization_for_list(const char *public_key, const char *private_key, + const char *method, const char *bucket, + const char *key, const char *mime_type, + const char *date, const char *md5, const char *prefix, + const char *marker, + const char *count, + const char *delimiter); //文件下载签名，mime_type, date, md5 可以为空。 //如果你加入了 date 请在 http 的 header 里面加入 Date: md5 头 diff --git a/lib/http.c b/lib/http.c index d21ab63..2ddfb8c 100644 --- a/lib/http.c +++ b/lib/http.c @@ -36,12 +36,6 @@ set_http_options(struct http_options *opt, return error; } - if(strlen(key) == 0){ - error.code = UFILE_CONFIG_ERROR_CODE; - error.message = "key connot be empty"; - return error; - } - opt->header = NULL; char *buf; if (strlen(mime_type) != 0){ @@ -67,6 +61,56 @@ set_http_options(struct http_options *opt, return error; } +struct ufile_error +set_http_options_for_list(struct http_options *opt, + const char *method, + const char *mime_type, + const char *bucket, + const char *key, + const char *prefix, + const char *marker, + int count, + const char *delimiter) { + struct ufile_error error = NO_ERROR; + struct ufile_config *cfg = _global_config; + + if (cfg == NULL){ + error.code = UFILE_CONFIG_ERROR_CODE; + error.message = "global configuration has not been initialization yet."; + return error; + } + + if(strlen(bucket) == 0){ + error.code = UFILE_CONFIG_ERROR_CODE; + error.message = "bucket connot be empty"; + return error; + } + + char count_str[20]; + snprintf(count_str, sizeof(count_str), "%d", count); + + opt->header = NULL; + char *buf; + if (strlen(mime_type) != 0){ + buf = ufile_strconcat("Content-Type: ", mime_type, NULL); + opt->header = curl_slist_append(opt->header, buf); + free(buf); + } + opt->header = curl_slist_append(opt->header, "User-Agent: UFile CSDK/2.0.0"); + + char *auth = ufile_file_authorization_for_list(cfg->public_key,cfg->private_key,method, bucket, + key, mime_type, "", "", prefix, marker, count_str, delimiter); + buf = ufile_strconcat("Authorization: ", auth, NULL); + opt->header = curl_slist_append(opt->header, buf); + free(auth); + free(buf); + + opt->method = ufile_strconcat(method, NULL); + opt->url = ufile_strconcat(bucket, ".", cfg->file_host, "/", key, "?listobjects&delimiter=", delimiter, + "&marker=", marker, "&max-keys=", count_str, "&prefix=", prefix, NULL); + return error; +} + void set_curl_options(CURL *curl, struct http_options *opt) { diff --git a/lib/http.h b/lib/http.h index 3549213..3fd2668 100644 --- a/lib/http.h +++ b/lib/http.h @@ -26,6 +26,17 @@ set_http_options(struct http_options *opt, const char *key, const char *query); +struct ufile_error +set_http_options_for_list(struct http_options *opt, + const char *method, + const char *mime_type, + const char *bucket, + const char *key, + const char *prefix, + const char *marker, + int count, + const char *delimiter); + struct ufile_error set_download_options( CURL *curl, diff --git a/lib/json.c b/lib/json.c new file mode 100644 index 0000000..1c20952 --- /dev/null +++ b/lib/json.c @@ -0,0 +1,37 @@ +#include +#include "json.h" + +int JsonGetBool(cJSON *root, const char *key, char *value) { + cJSON *item = cJSON_GetObjectItemCaseSensitive(root, key); + if (!cJSON_IsBool(item)) { + return -1; // Error + } + *value = cJSON_IsTrue(item) ? 1 : 0; + return 0; +} + +int JsonGetString(cJSON *root, const char *key, char **value) { + cJSON *item = cJSON_GetObjectItemCaseSensitive(root, key); + if (!cJSON_IsString(item)) { + return -1; // Error + } + *value = strdup(item->valuestring); + return 0; +} + +int JsonGetArray(cJSON *root, const char *key, cJSON **array) { + *array = cJSON_GetObjectItemCaseSensitive(root, key); + if (!cJSON_IsArray(*array)) { + return -1; // Error + } + return 0; +} + +int JsonGetInt64(cJSON *root, const char *key, long long *value) { + cJSON *item = cJSON_GetObjectItemCaseSensitive(root, key); + if (!cJSON_IsNumber(item)) { + return -1; // Error + } + *value = (long long)item->valuedouble; + return 0; +} \ No newline at end of file diff --git a/lib/json.h b/lib/json.h new file mode 100644 index 0000000..1c54311 --- /dev/null +++ b/lib/json.h @@ -0,0 +1,9 @@ +#include "cJSON.h" + +int JsonGetBool(cJSON *root, const char *key, char *value); + +int JsonGetString(cJSON *root, const char *key, char **value); + +int JsonGetArray(cJSON *root, const char *key, cJSON **array); + +int JsonGetInt64(cJSON *root, const char *key, long long *value); \ No newline at end of file diff --git a/lib/ufile_list.c b/lib/ufile_list.c new file mode 100644 index 0000000..26c682a --- /dev/null +++ b/lib/ufile_list.c @@ -0,0 +1,213 @@ +#include "api.h" + +#include +#include +#include +#include "http.h" +#include "json.h" + + +#define UFILE_LIST_RESULT_BUFFER_SIZE (1024 * 1024) + +struct ufile_error +ufile_parse_list_response(char *body, struct ufile_list_result *result) { + struct ufile_error error = NO_ERROR; + + struct ufile_list_result out_result; + memset(&out_result, 0, sizeof(struct ufile_list_result)); + + cJSON *root = cJSON_Parse(body); + if (!root) { + error.code = UFILE_MULTIPLE_INIT_ERROR_CODE; + error.message = cJSON_GetErrorPtr(); + return error; + } + + int ret = 0; + + // Parse Contents + cJSON *contents = NULL; + cJSON *common_prefixes = NULL; + ret = JsonGetArray(root, "Contents", &contents); + if (ret) { + goto err; + } + + int num_contents = cJSON_GetArraySize(contents); + ret = JsonGetArray(root, "CommonPrefixes", &common_prefixes); + if (ret) { + goto err; + } + + int num_common_prefixes = cJSON_GetArraySize(common_prefixes); + + int entry_num = num_contents + num_common_prefixes; + if (entry_num > 0) { + out_result.entries = (struct ufile_list_result_entry *)malloc(entry_num * sizeof(struct ufile_list_result_entry)); + } + out_result.entry_num = entry_num; + + int i; + for (i = 0; i < num_contents; i++) { + cJSON *content = cJSON_GetArrayItem(contents, i); + struct ufile_list_result_entry *entry = &out_result.entries[i]; + + ret = JsonGetString(content, "Key", &entry->filename); + if (ret) { + goto err; + } + + ret = JsonGetString(content, "MimeType", &entry->mime_type); + if (ret) { + goto err; + } + + ret = JsonGetString(content, "ETag", &entry->etag); + if (ret) { + goto err; + } + + char *size_str = NULL; + ret = JsonGetString(content, "Size", &size_str); + if (ret) { + goto err; + } + entry->size = atoll(size_str); + free(size_str); + + ret = JsonGetString(content, "StorageClass", &entry->storage_class); + if (ret) { + goto err; + } + + ret = JsonGetInt64(content, "LastModified", &entry->last_modified); + if (ret) { + goto err; + } + + ret = JsonGetInt64(content, "CreateTime", &entry->create_time); + if (ret) { + goto err; + } + } + + int j; + for (j = 0; j < num_common_prefixes; j++) { + cJSON *common_prefix = cJSON_GetArrayItem(common_prefixes, j); + struct ufile_list_result_entry *entry = &out_result.entries[i + j]; + + ret = JsonGetString(common_prefix, "Prefix", &entry->filename); + if (ret) { + goto err; + } + } + + // Parse IsTruncated + ret = JsonGetBool(root, "IsTruncated", &out_result.is_truncated); + if (ret) { + goto err; + } + + // Parse NextMarker + ret = JsonGetString(root, "NextMarker", &out_result.next_marker); + if (ret) { + goto err; + } + + *result = out_result; + + cJSON_Delete(root); + return error; + +err: + cJSON_Delete(root); + error.code = UFILE_MULTIPLE_INIT_ERROR_CODE; + error.message = cJSON_GetErrorPtr(); + return error; +} + +extern struct ufile_error +ufile_list(const char *bucket_name, const char *prefix, const char *delimiter, + int count, const char *marker, struct ufile_list_result *result) { + struct ufile_error error = NO_ERROR; + if(!bucket_name || *bucket_name == '\0') { + error.code = UFILE_PARAM_ERROR_CODE; + error.message = "bucket_name cannot be nil."; + return error; + } + + if(!prefix) { + error.code = UFILE_PARAM_ERROR_CODE; + error.message = "prefix cannot be nil."; + return error; + } + + if (!delimiter) { + error.code = UFILE_PARAM_ERROR_CODE; + error.message = "delimiter cannot be nil."; + return error; + } + + if (!marker) { + error.code = UFILE_PARAM_ERROR_CODE; + error.message = "marker cannot be nil."; + return error; + } + + if (count > UFILE_LIST_MAX_COUNT) { + error.code = UFILE_PARAM_ERROR_CODE; + error.message = "list count too large."; + return error; + } + + if (!result) { + error.code = UFILE_PARAM_ERROR_CODE; + error.message = "result pointer cannot be nil."; + return error; + } + + CURL *curl = curl_easy_init(); + if(curl == NULL){ + error.code = CURL_ERROR_CODE; + error.message = CURL_INIT_ERROR_MSG; + return error; + } + + struct http_options opt; + error = set_http_options_for_list(&opt, "GET", "", bucket_name, "", prefix, marker, count, delimiter); + if(UFILE_HAS_ERROR(error.code)){ + http_cleanup(curl, &opt); + return error; + } + set_curl_options(curl, &opt); + struct http_body body; + size_t buffer_size = UFILE_LIST_RESULT_BUFFER_SIZE; + char *result_buf = (char *)malloc(buffer_size); + if (result_buf == NULL) { + error.code = NO_MEMORY_ERROR_CODE; + error.message = "no memory."; + return error; + } + memset(&body, 0, sizeof(struct http_body)); + body.buffer = result_buf; + if(body.buffer != NULL){ + body.buffer_size = buffer_size; + body.pos_n = 0; + } + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, http_write_cb); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &body); + + error = curl_do(curl); + + if (UFILE_HAS_ERROR(error.code)) { + http_cleanup(curl, &opt); + free(result_buf); + return error; + } + + error = ufile_parse_list_response(result_buf, result); + http_cleanup(curl, &opt); + free(result_buf); + + return error; +} \ No newline at end of file