From 4410310bb9325de607d0f3d379ab208b26fd13dc Mon Sep 17 00:00:00 2001 From: Alex Oloo Date: Tue, 6 Jul 2021 11:45:33 +0200 Subject: [PATCH] testing cluster mode --- README.md | 7 ++++++- install.sh | 4 ++++ project.clj | 3 ++- resources/PCP_VERSION | 2 +- resources/pcp.service | 3 ++- src/pcp/core.clj | 21 ++++++++++++++------ src/pcp/resp.clj | 1 + src/pcp/scgi.clj | 42 +++++++++++++++++++-------------------- src/pcp/utility.clj | 18 ++++++++--------- test/pcp/core_test.clj | 6 ++---- test/pcp/utility_test.clj | 2 +- 11 files changed, 64 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index a8b18d9..021cb39 100644 --- a/README.md +++ b/README.md @@ -252,7 +252,7 @@ You can see the spike in CPU and bandwidth on the droplet, but the CPU never sat These are by no means comprehensive benchmarks but give you a sense of what a PCP server can withstand for simple use case. Going above 400 req/s generally results in bad things happening. You can test your own site using [k6](https://k6.io/) with the instructions in [loadtest.js](./loadtest.js) -# Roadmap +# Roadmap & releases ## Release 0.0.1 - [x] Run arbitrary scripts @@ -268,6 +268,10 @@ Going above 400 req/s generally results in bad things happening. You can test yo - [x] Document PCP - [x] Load test a production deployment +## Release 0.0.2 +- [ ] Perfomance improvements +- [ ] Cluster mode + ## Release 0.1.0 - [ ] Store passphrases with [konserve](https://github.com/replikativ/konserve) - [ ] Add sponsorship button @@ -285,6 +289,7 @@ For the guidance and examples, special thanks to - [@BrunoBonacci](https://github.com/BrunoBonacci) - [@borkdude](https://github.com/borkdude) +- [@Baeldung](https://twitter.com/Baeldung) ## License diff --git a/install.sh b/install.sh index f5bb670..7ebd6f9 100644 --- a/install.sh +++ b/install.sh @@ -53,6 +53,10 @@ mv -f "$download_dir/pcp-server.jar" "$PWD/pcp-server.jar" rm -rf "$PWD/pcp-templates" mv -f "$download_dir/pcp-templates" "$PWD" +if [ ! -e "$PWD/pcp.conf" ] ; then + echo "PCP_CLUSTER=-c" > "$PWD/pcp.conf" +fi + case "$(uname -s)" in Linux*) mv -f "$download_dir/pcp.service" "/etc/systemd/system/pcp.service" diff --git a/project.clj b/project.clj index 5ad0c51..a59fe3a 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject pcp "0.0.2-beta.1" +(defproject pcp "0.0.2-beta.2" :description "PCP: Clojure Processor - A Clojure replacement for PHP" :url "https://github.com/alekcz/pcp" :license {:name "The MIT License" @@ -7,6 +7,7 @@ :dependencies [ ;core [org.clojure/clojure "1.10.3"] [org.clojure/tools.cli "1.0.194"] + [org.clojure/core.async "1.3.618"] [borkdude/sci "0.2.5"] [byte-streams "0.2.4"] [http-kit "2.5.0-RC1"] diff --git a/resources/PCP_VERSION b/resources/PCP_VERSION index 336fd11..ddf83ab 100644 --- a/resources/PCP_VERSION +++ b/resources/PCP_VERSION @@ -1 +1 @@ -v0.0.2-beta.1 \ No newline at end of file +v0.0.2-beta.2 \ No newline at end of file diff --git a/resources/pcp.service b/resources/pcp.service index d7ca6e9..90fa136 100644 --- a/resources/pcp.service +++ b/resources/pcp.service @@ -2,7 +2,8 @@ Description= pcp scgi server [Service] -ExecStart=/usr/bin/java -jar /usr/local/bin/pcp-server.jar +EnvironmentFile=/usr/local/bin/pcp.conf +ExecStart=/usr/bin/java -jar /usr/local/bin/pcp-server.jar $PCP_CLUSTER Restart=always [Install] diff --git a/src/pcp/core.clj b/src/pcp/core.clj index 746e038..a75d86f 100644 --- a/src/pcp/core.clj +++ b/src/pcp/core.clj @@ -89,8 +89,8 @@ (catch Exception _ nil))) (defn valid-path? [parent target] - (let [parent-path (-> parent io/file (.getParentFile) (.getCanonicalPath)) - target-path (-> target io/file (.getCanonicalPath))] + (let [parent-path (-> parent ^File io/file (.getParentFile) (.getCanonicalPath)) + target-path (-> target ^File io/file (.getCanonicalPath))] (str/starts-with? target-path parent-path))) (def persist ^:sci/macro @@ -105,9 +105,9 @@ (defn run-script [url-path &{:keys [root request]}] (let [path (URLDecoder/decode url-path "UTF-8") source (read-source path) - file (io/file path) - root (or root (-> file (.getParentFile) (.getCanonicalPath))) - parent (longer root (-> file (.getParentFile) (.getCanonicalPath))) + ^File file (io/file path) + root (or root (-> ^File file (.getParentFile) (.getCanonicalPath))) + parent (longer root (-> ^File file (.getParentFile) (.getCanonicalPath))) response (atom nil) keygen (fn [path k] (keyword (str (h/uuid [path k]))))] (if (string? source) @@ -200,7 +200,16 @@ ([path] (let [scgi-port (Integer/parseInt (or (System/getenv "SCGI_PORT") "9000"))] (case path - "" (scgi/serve scgi-handler scgi-port) + "" + (scgi/serve scgi-handler scgi-port) + + "-c" + (do + (scgi/serve scgi-handler scgi-port) + (scgi/serve scgi-handler 9007) + (scgi/serve scgi-handler 9014) + (scgi/serve scgi-handler 9021)) + (run-script path))))) \ No newline at end of file diff --git a/src/pcp/resp.clj b/src/pcp/resp.clj index b3ab005..9204749 100644 --- a/src/pcp/resp.clj +++ b/src/pcp/resp.clj @@ -51,6 +51,7 @@ :.bmp "image/bmp" :.bz "application/x-bzip" :.bz2 "application/x-bzip2" + :.cljs "application/x-scittle" :.csh "application/x-csh" :.css "text/css" :.csv "text/csv" diff --git a/src/pcp/scgi.clj b/src/pcp/scgi.clj index 4826970..50eaa9a 100644 --- a/src/pcp/scgi.clj +++ b/src/pcp/scgi.clj @@ -17,29 +17,29 @@ (let [data (str/split (:header req) #"\u0000") keys (map #(-> % (str/replace "_" "-") str/lower-case keyword) (take-nth 2 data)) values (take-nth 2 (rest data)) - h (zipmap keys values)] + h (transient (zipmap keys values))] ;make the ring linter happy. (-> h - (update :server-port #(Integer/parseInt (if (str/blank? %) "0" %))) - (update :content-length #(Integer/parseInt (if (str/blank? %) "0" %))) - (update :request-method #(-> % str str/lower-case keyword)) - (assoc :headers { "sec-fetch-site" (-> h :http-sec-fetch-site) - "host" (-> h :http-host) - "user-agent" (-> h :http-user-agent) - "cookie" (-> h :http-cookie) - "sec-fetch-user" (-> h :http-sec-fetch-user) - "connection" (-> h :hhttp-connection) - "upgrade-insecure-requests" (-> h :http-sec-fetch-site) - "accept" (-> h :http-accept) - "accept-language" (-> h :http-accept-language) - "sec-fetch-dest" (-> h :http-sec-fetch-dest) - "accept-encoding" (-> h :http-accept-encoding) - "sec-fetch-mode" (-> h :http-sec-fetch-mode) - "cache-control" (-> h :http-cache-control)}) - (assoc :headers {}) - (assoc :uri (:request-uri h)) - (assoc :scheme (-> h :request-scheme keyword)) - (assoc :body (:body req))))) + (assoc! :server-port (Integer/parseInt (if (str/blank? (:server-port h)) "0" (:server-port h)))) + (assoc! :content-length (Integer/parseInt (if (str/blank? (:content-length h)) "0" (:content-length h)))) + (assoc! :request-method (-> (:request-method h) str str/lower-case keyword)) + (assoc! :headers { "sec-fetch-site" (-> h :http-sec-fetch-site) + "host" (-> h :http-host) + "user-agent" (-> h :http-user-agent) + "cookie" (-> h :http-cookie) + "sec-fetch-user" (-> h :http-sec-fetch-user) + "connection" (-> h :hhttp-connection) + "upgrade-insecure-requests" (-> h :http-sec-fetch-site) + "accept" (-> h :http-accept) + "accept-language" (-> h :http-accept-language) + "sec-fetch-dest" (-> h :http-sec-fetch-dest) + "accept-encoding" (-> h :http-accept-encoding) + "sec-fetch-mode" (-> h :http-sec-fetch-mode) + "cache-control" (-> h :http-cache-control)}) + (assoc! :uri (:request-uri h)) + (assoc! :scheme (-> h :request-scheme keyword)) + (assoc! :body (:body req)) + (persistent!)))) (defn on-accept [^SelectionKey key] (let [^ServerSocketChannel channel (.channel key) diff --git a/src/pcp/utility.clj b/src/pcp/utility.clj index d9d7687..22df0d1 100644 --- a/src/pcp/utility.clj +++ b/src/pcp/utility.clj @@ -10,7 +10,7 @@ [taoensso.nippy :as nippy] [environ.core :refer [env]]) (:import [java.net Socket] - [java.io File ByteArrayOutputStream InputStream] + [java.io File ByteArrayOutputStream InputStream BufferedWriter] [org.apache.commons.io IOUtils] [org.apache.commons.codec.digest DigestUtils]) (:gen-class)) @@ -19,7 +19,7 @@ (def root (atom nil)) (def scgi (atom "9000")) -(def version "v0.0.2-beta.1") +(def version "v0.0.2-beta.2") (defn keydb [] (or (env :pcp-keydb) "/usr/local/etc/pcp-db")) @@ -97,8 +97,8 @@ Options: (resp/status status) (resp/content-type mime-type))) -(defn file-response [path ^File file] - (let [code (if (.exists file) 200 404) +(defn file-response [path file] + (let [code (if (.exists ^File file) 200 404) mime (resp/get-mime-type (re-find #"\.[0-9A-Za-z]{1,7}$" path))] (-> (resp/response file) (resp/status code) @@ -115,14 +115,14 @@ Options: {:status status :body body :headers headers})) (defn file-exists? [path] - (-> path io/file .exists)) + (-> path ^File io/file .exists)) (defn serve-file [path] (file-response path (io/file path))) (defn local-handler [opts] (fn [request] - (let [root (.getCanonicalPath (io/file (:root opts))) + (let [root (.getCanonicalPath ^File (io/file (:root opts))) path (str root (:uri request)) slashpath (str path "index.clj") exists (or (file-exists? path) (file-exists? slashpath)) @@ -141,7 +141,7 @@ Options: (defn run-file [path port] (let [path (str/replace (str "/" path) "//" "/") - root (.getCanonicalPath (io/file "./")) + root (.getCanonicalPath ^File (io/file "./")) scgi-port (Integer/parseInt (or (System/getenv "SCGI_PORT") (str port) "9000")) request {:document-root root :document-uri path :request-method :get}] (-> request http-to-scgi (forward scgi-port) create-resp :body))) @@ -235,7 +235,7 @@ Options: path (str (keydb) "/" project ".db")] (io/make-parents (keydb)) (println "adding passphrase for project:" project) - (with-open [w (io/writer path)] + (with-open [^BufferedWriter w (io/writer path)] (.write w ^String passphrase)) (println "done."))) @@ -256,7 +256,7 @@ Options: (spit (str path "/public/api/info.clj") (slurp (str (template-path) "/api/info.clj"))) - (println (str "Created pcp project `" project-name "` in directory") (.getAbsolutePath (io/file path))))) + (println (str "Created pcp project `" project-name "` in directory") (.getAbsolutePath ^File (io/file path))))) (defn -main ([] diff --git a/test/pcp/core_test.clj b/test/pcp/core_test.clj index 21d6d44..a2e93e3 100644 --- a/test/pcp/core_test.clj +++ b/test/pcp/core_test.clj @@ -4,8 +4,7 @@ [pcp.resp :as resp] [pcp.core :as core] [clojure.string :as str]) - (:import [java.io File] - [java.net Socket InetAddress ConnectException])) + (:import [java.io File])) (deftest read-source-test (testing "Test reading source" @@ -100,5 +99,4 @@ scgi-request {:document-root root :document-uri uri } _ (core/scgi-handler scgi-request) ans (try (core/run-script (str root uri)) (catch Exception _ "error"))] - (is (str/includes? ans "error"))))) - + (is (str/includes? ans "error"))))) \ No newline at end of file diff --git a/test/pcp/utility_test.clj b/test/pcp/utility_test.clj index ec141d3..1d9cf36 100644 --- a/test/pcp/utility_test.clj +++ b/test/pcp/utility_test.clj @@ -152,7 +152,7 @@ _ (io/make-parents (str "./test-resources/pcp-db/" project ".db")) _ (new-folder root) _ (utility/stop-scgi) - scgi (core/-main) + scgi (core/-main "-c") local (utility/-main "-s" root) _ (Thread/sleep 2000) file-eval (json/decode (with-out-str (utility/-main "-e" (str root "/index.clj"))) true)