From c466a786df27f89de44d1cd09dca58b96933f433 Mon Sep 17 00:00:00 2001 From: Alex Williams Date: Fri, 5 Jun 2020 05:36:32 +0000 Subject: [PATCH] Add 'native' support for faster AWS requests --- .github/workflows/main.yml | 7 +++- CHANGELOG.md | 5 +-- Makefile | 13 ++++++-- README.md | 2 ++ awscurl.l | 4 ++- libawscurl.l | 65 +++++++++++++++++++++++++++++++------- test.l | 6 ++++ 7 files changed, 83 insertions(+), 19 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9718d69..7a062e8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -21,5 +21,10 @@ jobs: version: ${{matrix.version}} architecture: ${{matrix.arch}} - - name: Run the tests on PicoLisp ${{matrix.arch}} v${{matrix.version}} + - name: Run the tests on PicoLisp (32-bit) v${{matrix.version}} + run: make test-multi + if: ${{ matrix.arch == 'src' }} + + - name: Run the tests on PicoLisp (64-bit) v${{matrix.version}} run: make check + if: ${{ matrix.arch == 'src64' }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 77f0788..75cee0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,9 @@ # Changelog -## 0.9.0 (2020-06-03) +## 0.10.0 (2020-06-05) - * Optionally output data to a file with '--output' parameter + * Optionally generate SHA256/HMAC hashes using 'libcrypto.so' with '--native' parameter + * Request creation is much faster with native hashing ## 0.8.0 (2020-06-03) diff --git a/Makefile b/Makefile index fcd8eb6..eb74b7c 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ TEST_REF = v3.0.0 .PHONY: all -all: check +all: check run-multi run-native $(TEST_DIR): mkdir -p $(TEST_DIR) && \ @@ -18,7 +18,14 @@ $(TEST_DIR): cd $(TEST_DIR) && \ git checkout $(TEST_REF) -check: $(TEST_DIR) run-tests +check: $(TEST_DIR) test-multi test-native -run-tests: +test-multi: $(TEST_DIR) run-multi + +run-multi: ./test.l + +test-native: $(TEST_DIR) run-native + +run-native: + ./test.l native diff --git a/README.md b/README.md index 727ec4e..c34b563 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ This command line tool can be used to sign [AWS Signature Version 4](https://doc # Requirements * `picolisp`: 32-bit or 64-bit `v3.1.11+`, tested up to PicoLisp `v20.5.26`, [see test runs](https://github.com/aw/picolisp-awscurl/actions/) + * `libcrypto.so`: for using `--native` functions with PicoLisp `v17.12+` * `picolisp-unit`: `v3.0.0+` for testing the library * `openssl`: `v1.0.0+` for signing and hashing strings * `curl`: for sending requests to the AWS APIs @@ -78,6 +79,7 @@ Options: --endpoint The API endpoint of the AWS service (default: /) --header HTTP header data (default: None) --host The Host of the AWS service (default: ec2.amazonaws.com) + --native" Use faster 'native' calls for hashing data (64-bit version only, default: False) --output Filename where data should be output (default: STDOUT) --protocol http|https Protocol for talking to AWS (default: https) --query The Query parameters of the AWS service (default: None) diff --git a/awscurl.l b/awscurl.l index ba0fc0c..7c2af40 100755 --- a/awscurl.l +++ b/awscurl.l @@ -2,7 +2,7 @@ [de APP_INFO ("name" "awscurl") - ("version" "0.9.0") + ("version" "0.10.0") ("summary" "PicoLisp AWS CLI tool using OpenSSL and Curl") ("source" "https://github.com/aw/picolisp-awscurl") ("author" "Alexander Williams") @@ -18,6 +18,7 @@ ("--endpoint " "The API endpoint of the AWS service (default: /)") ("--header " "HTTP header data (default: None)") ("--host " "The Host of the AWS service (default: ec2.amazonaws.com)") + ("--native" "Use faster 'native' calls for hashing data (64-bit version only, default: False)") ("--output " "Filename where data should be output (default: STDOUT)") ("--protocol http|https" "Protocol for talking to AWS (default: https)") ("--query " "The Query parameters of the AWS service (default: None)") @@ -52,6 +53,7 @@ (while (opt) (case @ (--verbose (on *Aws_verbose)) + (--native (on *Aws_native)) (--request (setq *Aws_method (opt))) (--data (awscurl-data (opt))) (--header (awscurl-headers (opt) (opt))) diff --git a/libawscurl.l b/libawscurl.l index 84f685a..d8bea14 100644 --- a/libawscurl.l +++ b/libawscurl.l @@ -17,38 +17,75 @@ *Aws_request_date (dat$ (date T)) *Aws_request_time (pack (mapcar '((S) (pad 2 S)) (time (time T)))) ) +### native +[de awscurl-native-sha256 (Payload) + (native "libcrypto.so" "SHA256" '(B . 32) Payload (length Payload) '(NIL (32) ] + +[de awscurl-native-hmac (Key Length Payload) + (let Evp (native "libcrypto.so" "EVP_sha256" 'N) + (native "libcrypto.so" "HMAC" '(B . 32) Evp Key Length Payload (length Payload) 0 '(NIL (32) ] + +# Key = hex string +[de awscurl-native-make-hmac (Key Payload) + (let (*Key (chop Key) + *Iterations (/ (length *Key) 2) + *P (native "@" "malloc" 'N *Iterations) ) # allocate some memory to store the key + + (for N *Iterations + [byte (+ *P (- N 1)) (hex (pack (cut 2 '*Key) ] # store the hex value in a byte + ) + (byte (+ *P *Iterations 1) 0) # store a null byte + (let Result (awscurl-list-to-hex (awscurl-native-hmac *P *Iterations Payload)) + (native "@" "free" NIL *P) # free the memory + Result ] + +[de awscurl-parse-native (Payload Key) + (if Key + (awscurl-native-make-hmac Key Payload) + (awscurl-list-to-hex (awscurl-native-sha256 Payload) ] + ### main [de awscurl-make-curl (Cmd) (prin (in Cmd (till NIL T) ] [de awscurl-make-request (Auth Headers) (make - (link 'curl "--silent" "--request" *Aws_method (pack *Aws_protocol "://" *Aws_host *Aws_endpoint (when *Aws_query (pack "?" @))) "--header" Auth) + (link 'curl "--silent" "--request" *Aws_method (pack *Aws_protocol "://" *Aws_host *Aws_endpoint (when *Aws_query (pack "?" *Aws_query))) "--header" Auth) (mapc '((S) (link "--header" (pack (car S) ": " (cdr S)))) Headers) (when *Aws_data (link "--data" (cdr *Aws_data))) (when *Aws_verbose (link "--verbose")) - (when *Aws_output (link "--output" @) ] + (when *Aws_output (link "--output" *Aws_output) ] [de awscurl-make-auth-header (Headers Signature) (pack "Authorization: AWS4-HMAC-SHA256 Credential=" *Aws_access_key "/" *Aws_request_date "/" *Aws_region "/" *Aws_service "/aws4_request, SignedHeaders=" (glue ";" (mapcar car Headers)) ", Signature=" Signature) ] +[de awscurl-list-to-hex (List) + (lowc (pack (mapcar '((N) (pad 2 (hex N))) List) ] + +[de awscurl-string-to-hex (String) + (awscurl-list-to-hex (mapcar char (chop String) ] + [de awscurl-make-hmac (Key Payload) - (awscurl-parse-openssl Payload "-mac" "HMAC" "-macopt" Key) ] + (if *Aws_native + (awscurl-parse-native Payload Key) + (awscurl-parse-openssl Payload "-mac" "HMAC" "-macopt" (pack "hexkey:" Key) ] [de awscurl-make-signature (Str2sign) - (let (A (awscurl-make-hmac (pack "key:AWS4" *Aws_secret_key) *Aws_request_date) - B (awscurl-make-hmac (pack "hexkey:" A) *Aws_region) - C (awscurl-make-hmac (pack "hexkey:" B) *Aws_service) - D (awscurl-make-hmac (pack "hexkey:" C) "aws4_request") ) + (let (A (awscurl-make-hmac (awscurl-string-to-hex (pack "AWS4" *Aws_secret_key)) *Aws_request_date) + B (awscurl-make-hmac A *Aws_region) + C (awscurl-make-hmac B *Aws_service) + D (awscurl-make-hmac C "aws4_request") ) - (awscurl-make-hmac (pack "hexkey:" D) Str2sign) ] + (awscurl-make-hmac D Str2sign) ] [de awscurl-make-canonical-hash (Canonical) (pack "AWS4-HMAC-SHA256" "^J" *Aws_request_date "T" *Aws_request_time "Z" "^J" *Aws_request_date "/" *Aws_region "/" *Aws_service "/aws4_request" "^J" - (awscurl-parse-openssl Canonical) ] + (if *Aws_native + (awscurl-parse-native Canonical) + (awscurl-parse-openssl Canonical) ] [de awscurl-make-canonical (Hash Headers) (pack @@ -67,7 +104,7 @@ (cons "x-amz-content-sha256" Sha256) (cons "x-amz-date" (pack *Aws_request_date "T" *Aws_request_time "Z")) ) (when *Aws_data (link (cons "content-length" (if (= "file" (car *Aws_data)) (car (info (cdr *Aws_data))) (length (cdr *Aws_data)))))) - (when *Aws_session_token (link (cons "x-amz-security-token" @))) + (when *Aws_session_token (link (cons "x-amz-security-token" *Aws_session_token))) (mapc link *Aws_headers) ] [de awscurl-file-openssl (Filename) @@ -91,7 +128,11 @@ [de awscurl-start () (awscurl-get-credentials) - (let (Sha256 (if (= "file" (car *Aws_data)) (awscurl-file-openssl (cdr *Aws_data)) (awscurl-parse-openssl (cdr *Aws_data))) + (let (Sha256 (if (= "file" (car *Aws_data)) + (awscurl-file-openssl (cdr *Aws_data)) + (if *Aws_native + (awscurl-parse-native (cdr *Aws_data)) + (awscurl-parse-openssl (cdr *Aws_data)) ) ) Headers (awscurl-sort-headers Sha256) Canonical (awscurl-make-canonical Sha256 Headers) Str2sign (awscurl-make-canonical-hash Canonical) @@ -103,7 +144,7 @@ [de awscurl-data (Value) (let Data (chop Value) (setq *Aws_data - (if (and (= "@" (car Data)) (info (pack (cdr Data)))) + (if (and (= (char 64) (car Data)) (info (pack (cdr Data)))) (cons "file" (pack (cdr Data))) (cons "data" Value) ] diff --git a/test.l b/test.l index d5ecee3..34f51ae 100755 --- a/test.l +++ b/test.l @@ -9,6 +9,8 @@ *Aws_request_date "20180713" *Aws_request_time "052018" ) +(when (= "native" (opt)) (on *Aws_native)) + # Default HOME to 'test' dir (sys "HOME" "test") @@ -86,6 +88,10 @@ '(test-headers-normalized) '(test-data-read) '(test-get-credentials) + (when *Aws_native + '(assert-equal "cf80cd8aed482d5d1527d7dc72fceff84e6326592848447d2dc0b0e87dfc9a90" (awscurl-parse-native "testing") "Ensure OpenSSL hashes a string (native)") + '(assert-equal "8f19bc12c7f2f3f31c2435645ff0e457d4e87f676b08d9db4ba3271bb2c3e87a" (awscurl-parse-native "testing" "cf80cd8aed482d5d1527d7dc72fceff84e6326592848447d2dc0b0e87dfc9a90") "Ensure OpenSSL hashes a string with HMAC (native)") + ) '(assert-equal "cf80cd8aed482d5d1527d7dc72fceff84e6326592848447d2dc0b0e87dfc9a90" (awscurl-parse-openssl "testing") "Ensure OpenSSL hashes a string") '(assert-equal "8f19bc12c7f2f3f31c2435645ff0e457d4e87f676b08d9db4ba3271bb2c3e87a" (awscurl-parse-openssl "testing" "-mac" "HMAC" "-macopt" "hexkey:cf80cd8aed482d5d1527d7dc72fceff84e6326592848447d2dc0b0e87dfc9a90") "Ensure OpenSSL hashes a string with HMAC") '(assert-equal "9a78a98c394ec90a0bc17d994c257bfbe9aa6f30f6837efb37c1f46ad86f75c6" (awscurl-file-openssl "test/data.json") "Ensure OpenSSL hashes a file")