From b2003bd93bfe9e4219bd34dbd26067e32c57c13f Mon Sep 17 00:00:00 2001
From: cupen <xcupen@gmail.com>
Date: Tue, 8 Oct 2024 22:44:19 +0800
Subject: [PATCH] feat: http api for upload/download
---
.github/workflows/release.yml | 65 +++++++++++++
.gitignore | 3 +
Cargo.toml | 14 ++-
README.md | 3 +-
src/cli.rs | 37 +++++--
src/files.rs | 15 +++
src/lib.rs | 0
src/main.rs | 175 +++++++++++++++++++++-------------
tests/upload_test.py | 152 ++++++++++++++++++++++-------
9 files changed, 351 insertions(+), 113 deletions(-)
create mode 100644 .github/workflows/release.yml
create mode 100644 src/files.rs
delete mode 100644 src/lib.rs
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
new file mode 100644
index 0000000..7b00e8c
--- /dev/null
+++ b/.github/workflows/release.yml
@@ -0,0 +1,65 @@
+name: Release
+
+permissions:
+ contents: write
+
+on:
+ push:
+ tags:
+ - "v*"
+
+
+env:
+ CARGO_TERM_COLOR: always
+
+jobs:
+ release:
+ name: Create Release
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v3
+ - name: Create Release
+ uses: actions/create-release@latest
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ with:
+ tag_name: ${{ github.ref }}
+ release_name: ${{ github.ref }}
+ draft: false
+ prerelease: false
+
+ publish:
+ name: publish ${{ matrix.name }}
+ needs:
+ - release
+ strategy:
+ fail-fast: true
+ matrix:
+ include:
+ - target: x86_64-pc-windows-gnu
+ suffix: windows-x86_64
+ archive: zip
+ name: x86_64-pc-windows-gnu
+ - target: x86_64-unknown-linux-gnu
+ suffix: linux-x86_64
+ archive: tar.xz
+ name: x86_64-unknown-linux-gnu
+ - target: x86_64-apple-darwin
+ suffix: darwin-x86_64
+ archive: tar.gz
+ name: x86_64-apple-darwin
+ runs-on: ubuntu-latest
+ steps:
+ - name: Clone test repository
+ uses: actions/checkout@v2
+ - uses: xhaiker/rust-release.action@v1.0.0
+ name: build ${{ matrix.name }}
+ with:
+ release: ${{ github.ref_name }}
+ rust_target: ${{ matrix.target }}
+ archive_suffix: ${{ matrix.suffix }}
+ archive_types: ${{ matrix.archive }}
+ extra_files: "README.md"
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index 210d03e..b2b5442 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,6 +6,9 @@ target/
.venv/
__pycache__/
*.html
+*.css
+*.js
+*.tar.gz
# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
diff --git a/Cargo.toml b/Cargo.toml
index 6fa9e53..707e4c8 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -3,18 +3,22 @@ name = "fake-cdn"
version = "0.1.0"
edition = "2021"
-[lib]
-proc-macro = true
+# [lib]
+# proc-macro = true
[dependencies]
actix-files = "0.6.6"
actix-multipart = "0.7.2"
actix-web = "4.9.0"
-clap = "4.5.18"
+clap = "4.5.20"
+colog = "1.3.0"
+flate2 = "1.0.34"
lazy_static = "1.5.0"
+log = "0.4.22"
# rocket = "0.5.1"
serde = { version = "1.0", features = ["derive"] }
-serde_json = "1.0.122"
+serde_json = "1.0.132"
structopt = "0.3.26"
-tokio = { version = "1.40.0", features= ["fs"] }
+tar = "0.4.42"
+tokio = { version = "1.41.1", features= ["fs"] }
toml = "0.8.19"
diff --git a/README.md b/README.md
index 195f01a..4fc003d 100644
--- a/README.md
+++ b/README.md
@@ -2,8 +2,7 @@
Fake CDN
# Features
-* Upload files
-* Download files
+* Upload / Download files
* Serve static site
# Dependencies
diff --git a/src/cli.rs b/src/cli.rs
index ef98c85..8a69ffd 100644
--- a/src/cli.rs
+++ b/src/cli.rs
@@ -1,6 +1,7 @@
-use serde::{Serialize};
use structopt::StructOpt;
+use std::sync::OnceLock;
+
#[derive(Debug, StructOpt)]
#[structopt(name = "args")]
@@ -8,19 +9,43 @@ pub struct Args {
#[structopt(subcommand)]
pub command: Command,
- #[structopt(short, long)]
- pub verbose: bool,
+ // #[structopt(short, long)]
+ // pub verbose: bool,
}
#[derive(StructOpt, Debug)]
pub enum Command {
Web {
- #[structopt(long)]
+ #[structopt(long, env="FAKECDN_LISTEN", default_value="127.0.0.1:9527")]
listen: String,
+
+ #[structopt(long, env="FAKECDN_DIR", default_value=".uploads")]
+ dir: String,
+
+ #[structopt(long, env="FAKECDN_TOKEN", default_value="")]
+ token: String,
},
}
+pub fn get_args() -> &'static Args {
+ static ARGS: OnceLock<Args> = OnceLock::new();
+ return ARGS.get_or_init(|| parse_args());
+}
+
+
+pub fn get_args_token() -> &'static String {
+ let args = get_args();
+ match &args.command {
+ Command::Web { token, .. } => return token,
+ }
+}
pub fn parse_args() -> Args {
- return Args::from_args()
-}
\ No newline at end of file
+ return Args::from_args();
+}
+
+// pub fn parse_args() -> &'static Mutex<Args> {
+// // return Args::from_args()
+// static ARGS: OnceLock<Mutex<Args>> = OnceLock::new();
+// return ARGS.get_or_init(|| Mutex::new(Args::from_args()))
+// }
\ No newline at end of file
diff --git a/src/files.rs b/src/files.rs
new file mode 100644
index 0000000..1a4d6c1
--- /dev/null
+++ b/src/files.rs
@@ -0,0 +1,15 @@
+use std::fs::File;
+use std::path::Path;
+use std::path::PathBuf;
+use flate2::read::GzDecoder;
+use tar::Archive;
+
+pub(crate) fn uncompress_tgz(path: &PathBuf, dest: &Path) -> Result<(), std::io::Error> {
+ // let path = "archive.tar.gz";
+ let tar_gz = File::open(path)?;
+ let tar = GzDecoder::new(tar_gz);
+ let mut archive = Archive::new(tar);
+ archive.unpack(dest)?;
+
+ Ok(())
+}
diff --git a/src/lib.rs b/src/lib.rs
deleted file mode 100644
index e69de29..0000000
diff --git a/src/main.rs b/src/main.rs
index 20b7c5e..d1d9aa1 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -1,12 +1,19 @@
+use colog;
+use log::{debug, error, info};
use std::io::Read;
use std::net::SocketAddr;
+use std::path::Path;
use std::path::PathBuf;
mod cli;
use cli::Command;
+mod files;
+
// use lazy_static::lazy_static;
-use actix_web::{get, post, web, App, HttpRequest, HttpResponse, HttpServer, Responder};
+use actix_web::{
+ get, post, web, App, HttpRequest, HttpResponse, HttpServer, Responder,
+};
// use actix_web::http::header::{HeaderMap, HeaderValue};
use actix_files::Files;
use tokio::fs;
@@ -16,6 +23,7 @@ use actix_multipart::form::{tempfile::TempFile, MultipartForm};
// use actix_multipart::Multipart;
// use serde::Deserialize;
use std::fmt::Debug;
+use serde_json::json;
const UPLOAD_DIR: &str = ".uploads";
// const listen: &str = "127.0.0.1:9527";
@@ -26,96 +34,131 @@ struct UploadForm {
file: TempFile,
}
-// #[post("/")]
-// async fn upload(req: HttpRequest) -> impl Responder {
-// println!("http upload: {:?}", req);
-// let path = req.path();
-// if path.contains("..") {
-// return HttpResponse::BadRequest().body("Bad path")
-// }
-// let full_path = format!("{}/{}", upload_dir, path);
-// // File::create(full_path).await.expect("Unable to create file");
-// fs::write(full_path, req.body()).await.expect("Unable to write file");
-// return HttpResponse::Ok().body("File created");
-// }
-
-
#[post("/{path:.*}")]
-async fn upload(args: web::Path<(String,)>, MultipartForm(mut form): MultipartForm<UploadForm>) -> impl Responder {
+async fn upload(
+ args: web::Path<(String,)>,
+ req: HttpRequest,
+ MultipartForm(mut form): MultipartForm<UploadForm>,
+) -> impl Responder {
let fpath = args.into_inner().0;
if fpath.contains("..") {
return HttpResponse::BadRequest().body("Bad path");
}
- let fname = form.file.file_name.unwrap();
- println!("[upload] path:{} fname:{} size:{}", fpath, fname, form.file.size);
- let full_path = PathBuf::from(UPLOAD_DIR).join(fpath);
+ debug!("[upload] {:?}", form);
+ let token = req.headers().get("Authorization");
+ match token {
+ Some(t) => {
+ info!("[upload] token: ...");
+ let token = cli::get_args_token();
+ if token.eq(t.to_str().unwrap()) {
+ info!("[upload] token: ok");
+ } else {
+ info!("[upload] token: invalid");
+ return HttpResponse::Unauthorized().body("Unauthorized");
+ }
+ }
+ None => {
+ info!("[upload] no token");
+ return HttpResponse::Unauthorized().body("Unauthorized");
+ }
+ }
+ info!("[upload] {} size: {}", fpath, form.file.size);
+ let full_path = PathBuf::from(UPLOAD_DIR).join(fpath.clone());
+
let dpath = full_path.parent().unwrap();
if !dpath.exists() {
- fs::create_dir_all(dpath).await.expect("Unable to create dir");
+ fs::create_dir_all(dpath)
+ .await
+ .expect("Unable to create dir");
}
- if cfg!(windows) {
- let buf :&mut Vec<u8> = &mut Vec::new();
- // let data = form.file.file.as_file();
- let fp = form.file.file.as_file_mut();
- let a = fp.read_to_end(buf).unwrap();
- // let a = fp.read_to_end(buf).unwrap();
- if a == 0 {
+ // if cfg!(windows) {
+ let buf: &mut Vec<u8> = &mut Vec::new();
+ // let data = form.file.file.as_file();
+ let fp = form.file.file.as_file_mut();
+ match fp.read_to_end(buf) {
+ Ok(s) => {
+ info!("[upload] => saved: {} {} bytes", full_path.display(), s);
+ fs::write(full_path.clone(), buf)
+ .await
+ .expect("Unable to write file");
+
+
+ let fname = form.file.file_name.unwrap();
+ if fname.contains(".") {
+ fname.split(".").last().unwrap();
+ let mut ext = Path::new(&fname)
+ .extension()
+ .and_then(|s| s.to_str())
+ .unwrap();
+ if fname.ends_with(".tar.gz") || fname.ends_with(".tgz") {
+ ext = "tar.gz";
+ }
+ match ext {
+ "tar.gz" => {
+ info!("[upload] {} is tar.gz", full_path.display());
+ let full_dir = full_path.parent().unwrap();
+ files::uncompress_tgz(&full_path, full_dir).expect("Unable to uncompress");
+ }
+ "zip" => {
+ info!("[upload] {} is zip", fpath);
+ }
+ "html" => {
+ info!("[upload] {} is html", fpath);
+ }
+ _ => {
+ info!("[upload] {} is {}", fpath, ext);
+ }
+ }
+ }
+ }
+
+ _ => {
+ error!("[upload] => save failed: {}", full_path.display());
return HttpResponse::BadRequest().body("Bad file");
}
- fs::write(full_path, buf).await.expect("Unable to write file");
- } else if cfg!(unix) {
- form.file.file.persist(full_path).unwrap();
}
return HttpResponse::Ok().body("");
}
-
-#[post("/echo")]
-async fn echo(req_body: String) -> impl Responder {
- println!("[echo] {:?}", req_body);
- HttpResponse::Ok().body(req_body)
-}
-
#[get("/status")]
async fn status() -> impl Responder {
- println!("[status] ok");
- HttpResponse::Ok()
+ info!("[status] ok");
+ HttpResponse::Ok().json(json!({
+ "status": "ok",
+ "version": "0.1.0"
+ }))
}
-
#[actix_web::main]
async fn main() -> std::io::Result<()> {
- let args = cli::parse_args();
- match args.command {
- Command::Web { listen } => {
- let lis = listen.parse::<SocketAddr>().expect("Invalid listen address");
- println!("Listening on: {}", lis);
- HttpServer::new(|| {
- App::new()
- .service(echo)
- .service(status)
- .service(upload)
- .service(
- Files::new("/" , UPLOAD_DIR)
+ colog::init();
+
+ let args = cli::get_args();
+ match &args.command {
+ Command::Web { listen, dir, token:_ } => {
+ let addr = listen
+ .parse::<SocketAddr>()
+ .expect("Invalid listen address");
+ println!("Listening on: {}", addr);
+ // let upload_dir = dir.clone();
+ let mut upload_dir = PathBuf::from(dir.clone());
+ if dir.eq("") {
+ upload_dir = PathBuf::from(UPLOAD_DIR);
+ }
+ if !upload_dir.exists() {
+ error!("--dir {} doesn't exists", upload_dir.display());
+ }
+ HttpServer::new(move || {
+ App::new().service(status).service(upload).service(
+ Files::new("/", &upload_dir)
.prefer_utf8(true)
- .show_files_listing())
+ .show_files_listing(),
+ )
})
- .bind(lis)?
+ .bind(addr)?
.run()
.await
- },
- _ => {
- panic!("Invalid command");
- // HttpServer::new(|| {
- // App::new()
- // .service(echo)
- // .service(status)
- // .service(upload)
- // .service(Files::new("/" , UPLOAD_DIR).prefer_utf8(true))
- // })
- // .run()
- // .await
}
}
}
diff --git a/tests/upload_test.py b/tests/upload_test.py
index 28c72cf..eff016e 100644
--- a/tests/upload_test.py
+++ b/tests/upload_test.py
@@ -1,34 +1,118 @@
-import requests
-from pathlib import Path
-
-test_dir = Path(__file__).parent
-
-def test_status():
- url = 'http://localhost:9527/status'
- resp = requests.get(url)
- assert resp.status_code == 200
-
-
-def test_echo():
- url = 'http://localhost:9527/echo'
- headers = {'Content-Type': 'text/plant; charset=utf-8'}
- resp = requests.post(url, data="abc", headers=headers)
- assert resp.status_code == 200
- assert resp.text == 'abc'
-
-
-def test_upload_file():
- url = 'http://localhost:9527/upload/file-abc.txt'
- files = {'file': open(__file__, 'rb')}
- resp = requests.post(url, files=files)
- assert resp.status_code == 200, resp.text
-
-
-def test_upload_html():
- url = 'http://localhost:9527/upload/index.html'
- fpath = test_dir.joinpath('index.html')
- with open(fpath, 'w') as f:
- f.write('<h1>Hello, World!</h1>')
- files = {'file': open(fpath, 'rb')}
- resp = requests.post(url, files=files)
- assert resp.status_code == 200, resp.text
\ No newline at end of file
+import requests
+from pathlib import Path
+import uuid
+import os
+
+ENV = lambda x, _default: os.environ.get(x, _default)
+
+base_url = ENV('BASE_URL', 'http://localhost:9527')
+
+TOKEN = "123456"
+test_dir = Path(__file__).parent
+
+class File:
+ @staticmethod
+ def upload(fpath, url_path, token=TOKEN):
+ files = {'file': open(fpath, 'rb')}
+ url = f'{base_url}/{url_path}'
+ if not token:
+ return requests.post(url, files=files)
+ headers = {'Authorization': token}
+ return requests.post(url, files=files, headers=headers)
+
+ @staticmethod
+ def download(url_path):
+ url = f'{base_url}/{url_path}'
+ return requests.get(url)
+
+ @staticmethod
+ def create(name, content):
+ fpath = test_dir / ".data" / name
+ os.makedirs(fpath.parent, exist_ok=True)
+ with open(fpath, 'w') as f:
+ f.write(content)
+ return fpath
+
+
+def test_upload_token():
+ resp = File.upload(__file__, 'file-abc.txt', token='invalid token')
+ assert resp.status_code == 401, resp.text
+
+ resp = File.upload(__file__, 'file-abc.txt', token='')
+ assert resp.status_code == 401, resp.text
+
+
+def test_status():
+ url = f'{base_url}/status'
+ resp = requests.get(url)
+ assert resp.status_code == 200
+ assert resp.json() == {'status': 'ok', 'version': '0.1.0'}
+
+
+def test_upload_file():
+ id = str(uuid.uuid4())
+ resp = File.upload(__file__, f"{id}/file-abc.txt")
+ assert resp.status_code == 200, resp.text
+
+ resp = File.download(f"{id}/file-abc.txt")
+ assert resp.status_code == 200
+ assert resp.headers['Content-Type'] == 'text/plain; charset=utf-8'
+
+
+def test_upload_html():
+ id = str(uuid.uuid4())
+ fpath = test_dir.joinpath('index.html')
+ with open(fpath, 'w') as f:
+ f.write('<h1>Hello, World!</h1>')
+ resp = File.upload(fpath, f"{id}/index.html")
+ assert resp.status_code == 200, resp.text
+
+ resp = File.download(f"{id}/index.html")
+ assert resp.status_code == 200
+ assert resp.text == '<h1>Hello, World!</h1>'
+ assert resp.headers['Content-Type'] == 'text/html; charset=utf-8'
+
+
+def test_upload_html_override():
+ id = str(uuid.uuid4())
+ def do(content):
+ fpath = File.create('override/index.html', content)
+ resp = File.upload(fpath, f"{id}/index.html")
+ assert resp.status_code == 200, resp.text
+
+ resp = File.download(f"{id}/index.html")
+ assert resp.status_code == 200
+ assert resp.text == content
+ assert resp.headers['Content-Type'] == 'text/html; charset=utf-8'
+ pass
+
+ do(f'<h1>Hello, World!</h1>')
+ do(f'<h1>Hello, World!</h1> {id}')
+ do(f'<h1>Hello, World!</h1> {id} again')
+ pass
+
+
+def test_upload_tar():
+ F = File.create
+ id = str(uuid.uuid4())
+ url_paths = [
+ 'tar/index.html',
+ 'tar/css/abc.css',
+ 'tar/js/abc.js',
+ ]
+ # create a tar file
+ fpath = F('index.html', '<h1>Hello, World!</h1>')
+ import tarfile
+ with tarfile.open(fpath.with_suffix('.tar.gz'), 'w:gz') as tar:
+ tar.add(F("index.html", '<h1>Hello, World!</h1>'), arcname='index.html')
+ tar.add(F("abc.css", 'abc'), arcname='css/abc.css')
+ tar.add(F("abc.js", '{}'), arcname='js/abc.js')
+ pass
+
+ resp = File.upload(fpath.with_suffix('.tar.gz'), f'tar/index.tar.gz')
+ assert resp.status_code == 200
+
+ for path in url_paths:
+ resp = File.download(path)
+ assert resp.status_code == 200
+ pass