From 0b1c6fb63871b389165c90c76a1de3baca3d3211 Mon Sep 17 00:00:00 2001 From: liquidz Date: Fri, 3 Jun 2022 22:22:24 +0900 Subject: [PATCH 01/41] deps: Add core.async --- deps.edn | 1 + 1 file changed, 1 insertion(+) diff --git a/deps.edn b/deps.edn index b34b9b51..09055189 100644 --- a/deps.edn +++ b/deps.edn @@ -4,6 +4,7 @@ org.clojure/data.xml {:mvn/version "0.2.0-alpha6"} org.clojure/data.zip {:mvn/version "1.0.0"} org.clojure/tools.cli {:mvn/version "1.0.206"} + org.clojure/core.async {:mvn/version "1.5.648"} ;; NOTE: tools.deps.alpha v0.12.1098 or later does not work with CLojure 1.9.0 org.clojure/tools.deps.alpha {:mvn/version "0.12.1090"} org.clojure/data.json {:mvn/version "2.4.0"} From 82359cb9e11fb9722d1c94e782c5dd9f793789a2 Mon Sep 17 00:00:00 2001 From: liquidz Date: Sat, 4 Jun 2022 07:58:16 +0900 Subject: [PATCH 02/41] feat: Add antq.log/async-print --- src/antq/log.clj | 41 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/src/antq/log.clj b/src/antq/log.clj index 4bb8f2b3..3ba113ff 100644 --- a/src/antq/log.clj +++ b/src/antq/log.clj @@ -1,18 +1,49 @@ -(ns antq.log) +(ns antq.log + (:require + [clojure.core.async :as async])) + +(defonce logger-ch nil) (def ^:dynamic *verbose* false) +(defn stop-async-logger! + [logger-end-ch] + (when logger-ch + (async/>!! logger-ch ::eol) + (async/! end-ch ::end) + (recur)))) + end-ch)) + (defn info [s] (println s)) +(defn warning + [s] + (when *verbose* + (binding [*out* *err*] + (println s)))) + (defn error [s] (binding [*out* *err*] (println s))) -(defn warning +(defn async-print [s] - (when *verbose* - (binding [*out* *err*] - (println s)))) + (async/>!! logger-ch s)) From 5ab25dedf22a531692eb5890bca102d5410e6e1c Mon Sep 17 00:00:00 2001 From: liquidz Date: Sat, 4 Jun 2022 07:59:11 +0900 Subject: [PATCH 03/41] feat: Update core to initialize async-logger --- src/antq/core.clj | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/antq/core.clj b/src/antq/core.clj index 12eb742f..1161f312 100644 --- a/src/antq/core.clj +++ b/src/antq/core.clj @@ -236,13 +236,15 @@ (defn antq [options deps] - (let [deps (->> deps + (let [alog (log/start-async-logger!) + deps (->> deps (mark-only-newest-version-flag) (unify-deps-having-only-newest-version-flag)) outdated (->> (outdated-deps deps options) (map assoc-diff-url) (concat (unverified-deps deps)))] (report/reporter outdated options) + (log/stop-async-logger! alog) outdated)) (defn main* From 27642d2427108f46e6d427aa937a3f145e6577b7 Mon Sep 17 00:00:00 2001 From: liquidz Date: Sat, 4 Jun 2022 08:01:55 +0900 Subject: [PATCH 04/41] feat: Add run-progress / init-progress to antq.report cf. #157 --- src/antq/report.clj | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/antq/report.clj b/src/antq/report.clj index d70d0fb1..f081a25f 100644 --- a/src/antq/report.clj +++ b/src/antq/report.clj @@ -9,3 +9,15 @@ (defmethod reporter :default [_ options] (log/error (str "Unknown reporter: " (:reporter options)))) + +(defmulti init-progress + (fn [_deps options] + (:reporter options))) + +(defmethod init-progress :default [_ _] nil) + +(defmulti run-progress + (fn [_dep options] + (:reporter options))) + +(defmethod run-progress :default [_ _] nil) From 21669cffa007e3af52a5bc43f8bfda7b8b163836 Mon Sep 17 00:00:00 2001 From: liquidz Date: Sat, 4 Jun 2022 08:03:30 +0900 Subject: [PATCH 05/41] feat: Update to show progress bar cf. #157 --- src/antq/core.clj | 10 ++++++---- src/antq/report/table.clj | 26 ++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/src/antq/core.clj b/src/antq/core.clj index 1161f312..f5abb96c 100644 --- a/src/antq/core.clj +++ b/src/antq/core.clj @@ -117,7 +117,9 @@ (defn- assoc-versions [dep options] - (assoc dep :_versions (ver/get-sorted-versions dep options))) + (let [res (assoc dep :_versions (ver/get-sorted-versions dep options))] + (report/run-progress dep options) + res)) (defn latest [arg-map] @@ -169,9 +171,9 @@ (let [org-deps (remove #(or (skip-artifacts? % options) (using-release-version? %)) deps) - uniq-deps-with-vers (->> org-deps - distinct-deps - (pmap #(assoc-versions % options))) + uniq-deps (distinct-deps org-deps) + _ (report/init-progress uniq-deps options) + uniq-deps-with-vers (pmap #(assoc-versions % options) uniq-deps) assoc-latest-version* #(assoc-latest-version % options)] (->> org-deps (pmap #(complete-versions-by % uniq-deps-with-vers)) diff --git a/src/antq/report/table.clj b/src/antq/report/table.clj index 8c3f9709..a0077687 100644 --- a/src/antq/report/table.clj +++ b/src/antq/report/table.clj @@ -1,5 +1,6 @@ (ns antq.report.table (:require + [antq.log :as log] [antq.report :as report] [antq.util.dep :as u.dep] [antq.util.file :as u.file] @@ -7,6 +8,8 @@ [clojure.pprint :as pprint] [clojure.set :as set])) +(def ^:private progress (atom nil)) + (defn skip-duplicated-file-name [sorted-deps] (loop [[dep & rest-deps] sorted-deps @@ -45,3 +48,26 @@ (println "\nAvailable diffs:") (doseq [u urls] (println "-" u))))) + +(defn- progress-text + [{:keys [width total-count current-count]}] + (let [width (or width 50) + ratio (int (* width (/ current-count total-count)))] + (format "[ %s%s ] %d/%d\r" + (apply str (repeat ratio "#")) + (apply str (repeat (- width ratio) "-")) + current-count + total-count))) + +(defmethod report/init-progress "table" + [deps _options] + (reset! progress {:total-count (count deps) + :count-atom (atom 0)})) + +(defmethod report/run-progress "table" + [_dep _options] + (when-let [{:keys [total-count count-atom]} @progress] + (when (< @count-atom total-count) + (swap! count-atom inc) + (log/async-print + (progress-text {:total-count total-count :current-count @count-atom}))))) From 4104cd864483225a527cd1c37e9d1a2e729cc208 Mon Sep 17 00:00:00 2001 From: liquidz Date: Sun, 5 Jun 2022 13:53:20 +0900 Subject: [PATCH 06/41] feat: Add antq.util.env/getlong --- src/antq/util/env.clj | 9 +++++++++ test/antq/util/env_test.clj | 14 ++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 test/antq/util/env_test.clj diff --git a/src/antq/util/env.clj b/src/antq/util/env.clj index f993cf8a..fc771175 100644 --- a/src/antq/util/env.clj +++ b/src/antq/util/env.clj @@ -3,3 +3,12 @@ (defn getenv [x] (System/getenv x)) + +(defn getlong + [env-name default-value] + (try + (Long/parseLong + (or (getenv env-name) + (str default-value))) + (catch Exception _ + default-value))) diff --git a/test/antq/util/env_test.clj b/test/antq/util/env_test.clj new file mode 100644 index 00000000..39502bcf --- /dev/null +++ b/test/antq/util/env_test.clj @@ -0,0 +1,14 @@ +(ns antq.util.env-test + (:require + [antq.util.env :as sut] + [clojure.test :as t])) + +(t/deftest getlong-test + (with-redefs [sut/getenv (constantly "100")] + (t/is (= 100 (sut/getlong "FOO" 200)))) + + (with-redefs [sut/getenv (constantly nil)] + (t/is (= 200 (sut/getlong "FOO" 200)))) + + (with-redefs [sut/getenv (constantly "INVALID")] + (t/is (= 200 (sut/getlong "FOO" 200))))) From 0844604212dd7f004200012db731c9f0df8cb412 Mon Sep 17 00:00:00 2001 From: liquidz Date: Sun, 5 Jun 2022 13:53:54 +0900 Subject: [PATCH 07/41] feat: Add antq.util.async/fn-with-timeout cf. #158 --- src/antq/util/async.clj | 22 ++++++++++++++++++++++ test/antq/util/async_test.clj | 28 ++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 src/antq/util/async.clj create mode 100644 test/antq/util/async_test.clj diff --git a/src/antq/util/async.clj b/src/antq/util/async.clj new file mode 100644 index 00000000..afd52773 --- /dev/null +++ b/src/antq/util/async.clj @@ -0,0 +1,22 @@ +(ns antq.util.async + (:require + [clojure.core.async :as async])) + +(defn fn-with-timeout + [f timeout-ms] + (fn [& args] + (let [ch (async/chan)] + (async/go + (let [ret (try + (apply f args) + (catch Throwable ex + [::exception ex]))] + (async/>! ch ret))) + (let [ret (-> [ch + (async/timeout timeout-ms)] + (async/alts!!) + (first))] + + (if (and (vector? ret) (= ::exception (first ret))) + (throw (second ret)) + ret))))) diff --git a/test/antq/util/async_test.clj b/test/antq/util/async_test.clj new file mode 100644 index 00000000..68140409 --- /dev/null +++ b/test/antq/util/async_test.clj @@ -0,0 +1,28 @@ +(ns antq.util.async-test + (:require + [antq.util.async :as sut] + [clojure.core.async :as async] + [clojure.test :as t])) + +(def ^:private test-async-fn + (sut/fn-with-timeout + (fn [x] + (cond + (= ::error x) + (throw (Exception. "test error")) + + (= ::timeout x) + (do (async/ Date: Sun, 5 Jun 2022 13:54:42 +0900 Subject: [PATCH 08/41] feat: Add timeout constant values cf. #158 --- src/antq/constant.clj | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/src/antq/constant.clj b/src/antq/constant.clj index 6a8eeba0..6360e080 100644 --- a/src/antq/constant.clj +++ b/src/antq/constant.clj @@ -1,5 +1,36 @@ -(ns antq.constant) +(ns antq.constant + (:require + [antq.util.env :as u.env])) -(def retry-limit 5) +(def retry-limit + "Retry count for + - antq.util.maven/read-pom + - antq.util.git/ls-remote*" + 5) -(def clojure-deps-keys #{:deps :default-deps :extra-deps :override-deps :replace-deps}) +(def clojure-deps-keys + "Keys for detecting dependencies in deps.edn" + #{:deps :default-deps :extra-deps :override-deps :replace-deps}) + +(def default-timeout-msec + (u.env/getlong "ANTQ_TIMEOUT" 5000)) + +(def ls-remote-timeout-msec + "Timeout msec for + - antq.util.git/ls-remote*" + (u.env/getlong "ANTQ_LS_REMOTE_TIMEOUT" default-timeout-msec)) + +(def github-api-timeout-msec + "Timeout msec for + - antq.ver.github-tag/get-sorted-versions-by-ls-remote*" + (u.env/getlong "ANTQ_GITHUB_API_TIMEOUT" default-timeout-msec)) + +(def maven-timeout-msec + "Timeout msec for + - antq.ver.java/get-versions" + (u.env/getlong "ANTQ_MAVEN_TIMEOUT" default-timeout-msec)) + +(def pom-timeout-msec + "Timeout msec for + - antq.util.maven/read-pom*" + (u.env/getlong "ANTQ_POM_TIMEOUT" default-timeout-msec)) From 669db97e08fc1e07f9d55db912e752cb406074e1 Mon Sep 17 00:00:00 2001 From: liquidz Date: Sun, 5 Jun 2022 13:55:15 +0900 Subject: [PATCH 09/41] feat: Apply timeout to antq.util.git/ls-remote* cf. #158 --- src/antq/util/git.clj | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/antq/util/git.clj b/src/antq/util/git.clj index 339deb2a..16822849 100644 --- a/src/antq/util/git.clj +++ b/src/antq/util/git.clj @@ -2,6 +2,7 @@ (:require [antq.constant :as const] [antq.log :as log] + [antq.util.async :as u.async] [clojure.java.shell :as sh] [clojure.string :as str])) @@ -51,8 +52,13 @@ (do (log/warning "git ls-remote timed out, retrying") (recur (inc i)))))))) +(def ^:private ls-remote*-with-timeout + (u.async/fn-with-timeout + ls-remote* + const/ls-remote-timeout-msec)) + (def ^:private ls-remote - (memoize ls-remote*)) + (memoize ls-remote*-with-timeout)) (defn tags-by-ls-remote* [url] From 111ef9610e389fd8ba8e19a629cd5a33946748b1 Mon Sep 17 00:00:00 2001 From: liquidz Date: Sun, 5 Jun 2022 13:56:28 +0900 Subject: [PATCH 10/41] feat: Apply timeout to antq.util.maven/read-pom* cf. #158 --- src/antq/util/maven.clj | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/antq/util/maven.clj b/src/antq/util/maven.clj index a2e128ba..d5c82c75 100644 --- a/src/antq/util/maven.clj +++ b/src/antq/util/maven.clj @@ -2,6 +2,7 @@ (:require [antq.constant :as const] [antq.log :as log] + [antq.util.async :as u.async] [antq.util.env :as u.env] [antq.util.leiningen :as u.lein] [antq.util.xml :as u.xml] @@ -127,6 +128,11 @@ (with-open [reader (io/reader url)] (.read (MavenXpp3Reader.) reader))) +(def ^:private read-pom*-with-timeout + (u.async/fn-with-timeout + read-pom* + const/pom-timeout-msec)) + (defn read-pom ^Model [^String url] @@ -134,7 +140,7 @@ (loop [i 0] (when (< i const/retry-limit) (or (try - (read-pom* url) + (read-pom*-with-timeout url) (catch java.net.ConnectException e (if (= "Operation timed out" (.getMessage e)) (log/warning (str "Fetching pom from " url " failed because it timed out, retrying")) From 205873fcf06d470e60d1f80664fd7c0f26a69d9c Mon Sep 17 00:00:00 2001 From: liquidz Date: Wed, 8 Jun 2022 06:23:25 +0900 Subject: [PATCH 11/41] feat: Apply timeout to antq.ver.java/get-versions cf. #158 --- src/antq/ver/java.clj | 12 +++++++++--- test/antq/ver/java_test.clj | 6 +++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/antq/ver/java.clj b/src/antq/ver/java.clj index 74f695f4..235bf91a 100644 --- a/src/antq/ver/java.clj +++ b/src/antq/ver/java.clj @@ -1,5 +1,7 @@ (ns antq.ver.java (:require + [antq.constant :as const] + [antq.util.async :as u.async] [antq.util.maven :as u.mvn] [antq.ver :as ver] [clojure.set :as set] @@ -13,7 +15,7 @@ (org.eclipse.aether.resolution VersionRangeRequest))) -(defn get-versions +(defn- get-versions [name opts] (let [{:keys [^RepositorySystem system ^DefaultRepositorySystemSession session @@ -25,11 +27,16 @@ (->> (.resolveVersionRange system session req) (.getVersions)))) +(def ^:private get-versions-with-timeout + (u.async/fn-with-timeout + get-versions + const/maven-timeout-msec)) + (defn get-sorted-versions-by-name* [name {:as dep-opts :keys [snapshots?]} options] - (let [maven-vers (->> (get-versions name dep-opts) + (let [maven-vers (->> (get-versions-with-timeout name dep-opts) (map str)) versions (if (:ignore-locals options) (seq (set/difference (set maven-vers) @@ -41,7 +48,6 @@ (cond->> sorted-versions (not snapshots?) (remove ver/snapshot?)))) - (def get-sorted-versions-by-name (memoize get-sorted-versions-by-name*)) diff --git a/test/antq/ver/java_test.clj b/test/antq/ver/java_test.clj index b06d9133..44df5880 100644 --- a/test/antq/ver/java_test.clj +++ b/test/antq/ver/java_test.clj @@ -13,8 +13,8 @@ [:deps 'org.clojure/clojure :mvn/version])) (t/deftest get-versions-test - (let [vers (sut/get-versions 'org.clojure/clojure - {:repositories u.mvn/default-repos})] + (let [vers (#'sut/get-versions 'org.clojure/clojure + {:repositories u.mvn/default-repos})] (t/is (seq vers)) (t/is (contains? (set (map str vers)) current-clojure-version)))) @@ -32,7 +32,7 @@ {})) (t/deftest get-sorted-versions-test - (with-redefs [sut/get-versions dummy-versions] + (with-redefs [sut/get-versions-with-timeout dummy-versions] (t/is (= ["2" "1"] (get-sorted-versions {:version "1.0.0"}))) (t/is (= ["2" "1.6-SNAPSHOT" "1"] From 41cfcac7fe43d364e9ee8b98f6909c727bd90da6 Mon Sep 17 00:00:00 2001 From: liquidz Date: Wed, 8 Jun 2022 06:24:10 +0900 Subject: [PATCH 12/41] feat: Apply timeout to antq.ver.github-tag/get-sorted-versions-by-url cf. #158 --- src/antq/ver/github_tag.clj | 18 ++++++++++++------ test/antq/ver/github_tag_test.clj | 6 ++++-- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/antq/ver/github_tag.clj b/src/antq/ver/github_tag.clj index e97c87a6..f74e74d2 100644 --- a/src/antq/ver/github_tag.clj +++ b/src/antq/ver/github_tag.clj @@ -1,6 +1,8 @@ (ns antq.ver.github-tag (:require + [antq.constant :as const] [antq.log :as log] + [antq.util.async :as u.async] [antq.util.git :as u.git] [antq.util.ver :as u.ver] [antq.ver :as ver] @@ -16,7 +18,7 @@ (format "https://api.github.com/repos/%s/tags" (str/join "/" (take 2 (str/split (:name dep) #"/"))))) -(defn get-sorted-versions-by-ls-remote* +(defn- get-sorted-versions-by-ls-remote* [dep] (let [url (format "https://github.com/%s" (str/join "/" (take 2 (str/split (:name dep) #"/"))))] @@ -32,10 +34,9 @@ (def get-sorted-versions-by-ls-remote (memoize get-sorted-versions-by-ls-remote*)) -(defn get-sorted-versions-by-url* +(defn- get-sorted-versions-by-url* [url] - (-> url - (slurp) + (-> (slurp url) (json/read-str :key-fn keyword) (->> (map :name) (filter (comp u.ver/sem-ver? @@ -46,9 +47,14 @@ (map u.ver/normalize-version args)))) (reverse)))) -(def get-sorted-versions-by-url +(def ^:private get-sorted-versions-by-url (memoize get-sorted-versions-by-url*)) +(def ^:private get-sorted-versions-by-url-with-timeout + (u.async/fn-with-timeout + get-sorted-versions-by-url + const/github-api-timeout-msec)) + (defn- fallback-to-ls-remote [dep] (try @@ -64,7 +70,7 @@ (try (-> dep (tag-api-url) - (get-sorted-versions-by-url)) + (get-sorted-versions-by-url-with-timeout)) (catch Exception ex (reset! failed-to-fetch-from-api true) (log/warning (str "Failed to fetch versions from GitHub, so fallback to `git ls-remote`: " diff --git a/test/antq/ver/github_tag_test.clj b/test/antq/ver/github_tag_test.clj index 39bb6bbd..99897946 100644 --- a/test/antq/ver/github_tag_test.clj +++ b/test/antq/ver/github_tag_test.clj @@ -39,7 +39,8 @@ (t/deftest get-sorted-versions-test (reset! @#'sut/failed-to-fetch-from-api false) - (with-redefs [slurp (constantly dummy-json)] + (with-redefs [sut/get-sorted-versions-by-url-with-timeout #'sut/get-sorted-versions-by-url + slurp (constantly dummy-json)] (t/is (= ["v3.0.0" "v2.0.0" "v2.0.0-alpha2" "v2.0.0-alpha1" "1.0.0"] (get-sorted-versions {:name "foo/bar"})))) @@ -57,7 +58,8 @@ ["bar-sha" "BAR"]] (map #(str/join "\t" %)) (str/join "\n"))] - (with-redefs [slurp (fn [& _] + (with-redefs [sut/get-sorted-versions-by-url-with-timeout #'sut/get-sorted-versions-by-url + slurp (fn [& _] (reset! api-errored true) (throw (Exception. "test exception"))) sh/sh (fn [& args] From 80d96c7d8f4bb45721fe17d61f1ea5565e4739e4 Mon Sep 17 00:00:00 2001 From: liquidz Date: Tue, 21 Jun 2022 06:06:27 +0900 Subject: [PATCH 13/41] chore: Set clojure.core.async.go-checking system prop while developing --- deps.edn | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deps.edn b/deps.edn index 09055189..08185197 100644 --- a/deps.edn +++ b/deps.edn @@ -27,7 +27,8 @@ {:extra-paths ["test" "test/resources"] :extra-deps {lambdaisland/kaocha {:mvn/version "1.66.1034"} - lambdaisland/deep-diff2 {:mvn/version "2.2.124"}}} + lambdaisland/deep-diff2 {:mvn/version "2.2.124"}} + :jvm-opts ["-Dclojure.core.async.go-checking=true"]} :nop {:extra-deps {org.slf4j/slf4j-nop {:mvn/version "RELEASE"}}} From 6a9cd0ddf10a4062ea5c95a9f24d52bc3a7ef161 Mon Sep 17 00:00:00 2001 From: liquidz Date: Tue, 21 Jun 2022 06:07:01 +0900 Subject: [PATCH 14/41] chore: Rename default timeout env --- src/antq/constant.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/antq/constant.clj b/src/antq/constant.clj index 6360e080..179c0e1c 100644 --- a/src/antq/constant.clj +++ b/src/antq/constant.clj @@ -13,7 +13,7 @@ #{:deps :default-deps :extra-deps :override-deps :replace-deps}) (def default-timeout-msec - (u.env/getlong "ANTQ_TIMEOUT" 5000)) + (u.env/getlong "ANTQ_DEFAULT_TIMEOUT" 5000)) (def ls-remote-timeout-msec "Timeout msec for From ebf760c8a5f0f7f697554ebc04a50b8a252fed5e Mon Sep 17 00:00:00 2001 From: liquidz Date: Tue, 21 Jun 2022 06:07:28 +0900 Subject: [PATCH 15/41] fix: Fix gradle dep extractor to use `--project-dir` option `--build-file` option is deprecated --- src/antq/dep/gradle.clj | 9 ++++++--- test/antq/dep/gradle_test.clj | 3 ++- .../build.gradle} | 0 3 files changed, 8 insertions(+), 4 deletions(-) rename test/resources/dep/{build_no_repo.gradle => no_repo_gradle/build.gradle} (100%) diff --git a/src/antq/dep/gradle.clj b/src/antq/dep/gradle.clj index 4e4e0ee8..7c00f59f 100644 --- a/src/antq/dep/gradle.clj +++ b/src/antq/dep/gradle.clj @@ -13,7 +13,9 @@ (defn- get-repositories [file-path] - (let [{:keys [exit out]} (sh/sh gradle-command "--build-file" file-path + (let [parent-path (.getParent (io/file file-path)) + {:keys [exit out]} (sh/sh gradle-command + "--project-dir" parent-path "antq_list_repositories")] (when (= 0 exit) (->> (str/split-lines out) @@ -24,8 +26,9 @@ (defn- filter-deps-from-gradle-dependencies [file-path] - (let [{:keys [exit out]} (sh/sh gradle-command - "--build-file" file-path + (let [parent-path (.getParent (io/file file-path)) + {:keys [exit out]} (sh/sh gradle-command + "--project-dir" parent-path "--quiet" "dependencies")] (if (= 0 exit) diff --git a/test/antq/dep/gradle_test.clj b/test/antq/dep/gradle_test.clj index 1dc8f63e..62c9219e 100644 --- a/test/antq/dep/gradle_test.clj +++ b/test/antq/dep/gradle_test.clj @@ -39,11 +39,12 @@ (t/deftest extract-deps-without-repositories-test (let [deps (sut/extract-deps file-path - (.getPath (io/resource "dep/build_no_repo.gradle"))) + (.getPath (io/resource "dep/no_repo_gradle/build.gradle"))) defined-deps (->> defined-deps (map #(assoc % :repositories nil)) (set)) actual-deps (set deps)] + (t/is (seq actual-deps)) (t/is (every? #(contains? actual-deps %) defined-deps)))) (t/deftest extract-deps-command-error-test diff --git a/test/resources/dep/build_no_repo.gradle b/test/resources/dep/no_repo_gradle/build.gradle similarity index 100% rename from test/resources/dep/build_no_repo.gradle rename to test/resources/dep/no_repo_gradle/build.gradle From 33266b04bbfbaf1df0ceba130d8bbb3bde5550ce Mon Sep 17 00:00:00 2001 From: liquidz Date: Tue, 21 Jun 2022 06:09:01 +0900 Subject: [PATCH 16/41] fix: Fix util.async not to do blocking IO in go block --- src/antq/util/async.clj | 19 +++++++++---------- test/antq/util/async_test.clj | 2 -- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/src/antq/util/async.clj b/src/antq/util/async.clj index afd52773..9b130cc1 100644 --- a/src/antq/util/async.clj +++ b/src/antq/util/async.clj @@ -6,17 +6,16 @@ [f timeout-ms] (fn [& args] (let [ch (async/chan)] - (async/go - (let [ret (try - (apply f args) - (catch Throwable ex - [::exception ex]))] - (async/>! ch ret))) + (async/pipe (async/thread + (try + (apply f args) + (catch Throwable ex + ex))) + ch) (let [ret (-> [ch (async/timeout timeout-ms)] (async/alts!!) (first))] - - (if (and (vector? ret) (= ::exception (first ret))) - (throw (second ret)) - ret))))) + (when (instance? Throwable ret) + (throw ret)) + ret)))) diff --git a/test/antq/util/async_test.clj b/test/antq/util/async_test.clj index 68140409..1d0762e0 100644 --- a/test/antq/util/async_test.clj +++ b/test/antq/util/async_test.clj @@ -24,5 +24,3 @@ (t/is (nil? (test-async-fn ::timeout))) (t/is (thrown-with-msg? Exception #"^test error$" (test-async-fn ::error)))) - - From cddd6afa9801778283bef979145bc68d1fbdd8e3 Mon Sep 17 00:00:00 2001 From: liquidz Date: Tue, 21 Jun 2022 06:13:41 +0900 Subject: [PATCH 17/41] test: Fix test not to be affected by memoize --- test/antq/ver/github_tag_test.clj | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/test/antq/ver/github_tag_test.clj b/test/antq/ver/github_tag_test.clj index 99897946..5a47cdef 100644 --- a/test/antq/ver/github_tag_test.clj +++ b/test/antq/ver/github_tag_test.clj @@ -1,6 +1,7 @@ (ns antq.ver.github-tag-test (:require [antq.record :as r] + [antq.util.git :as u.git] [antq.ver :as ver] [antq.ver.github-tag :as sut] [clojure.data.json :as json] @@ -39,8 +40,7 @@ (t/deftest get-sorted-versions-test (reset! @#'sut/failed-to-fetch-from-api false) - (with-redefs [sut/get-sorted-versions-by-url-with-timeout #'sut/get-sorted-versions-by-url - slurp (constantly dummy-json)] + (with-redefs [slurp (constantly dummy-json)] (t/is (= ["v3.0.0" "v2.0.0" "v2.0.0-alpha2" "v2.0.0-alpha1" "1.0.0"] (get-sorted-versions {:name "foo/bar"})))) @@ -49,6 +49,7 @@ (get-sorted-versions {:name "foo/bar"}))))) (t/deftest get-sorted-versions-fallback-test + (reset! @#'sut/failed-to-fetch-from-api false) (let [api-errored (atom false) dummy-out (->> [["foo-sha" "FOO"] ["one-sha" "refs/tags/1.0"] @@ -58,7 +59,12 @@ ["bar-sha" "BAR"]] (map #(str/join "\t" %)) (str/join "\n"))] - (with-redefs [sut/get-sorted-versions-by-url-with-timeout #'sut/get-sorted-versions-by-url + (with-redefs [;; Disable memoize + u.git/ls-remote #'u.git/ls-remote*-with-timeout + u.git/tags-by-ls-remote #'u.git/tags-by-ls-remote* + sut/get-sorted-versions-by-ls-remote #'sut/get-sorted-versions-by-ls-remote* + sut/get-sorted-versions-by-url #'sut/get-sorted-versions-by-url* + slurp (fn [& _] (reset! api-errored true) (throw (Exception. "test exception"))) From 8d869421e3dae539e84ddc92e15e8c226d696b30 Mon Sep 17 00:00:00 2001 From: liquidz Date: Tue, 21 Jun 2022 06:14:44 +0900 Subject: [PATCH 18/41] test: Rename test resources --- test/antq/dep/babashka_test.clj | 15 ++-- test/antq/dep/boot_test.clj | 8 +- test/antq/dep/clojure_test.clj | 77 ++++++++++--------- test/antq/dep/github_action_test.clj | 14 ++-- test/antq/dep/leiningen_test.clj | 8 +- test/antq/dep/pom_test.clj | 8 +- test/antq/dep/shadow_test.clj | 10 ++- test/antq/upgrade/boot_test.clj | 2 +- test/antq/upgrade/clojure_test.clj | 12 +-- test/antq/upgrade/github_action_test.clj | 16 ++-- test/antq/upgrade/leiningen_test.clj | 6 +- test/antq/upgrade/pom_test.clj | 2 +- test/antq/upgrade/shadow_test.clj | 2 +- .../cross-project/{deps.edn => test_deps.edn} | 0 .../local/nested/{deps.edn => test_deps.edn} | 0 .../dep/local/{deps.edn => test_deps.edn} | 0 test/resources/dep/{bb.edn => test_bb.edn} | 0 .../dep/{build.boot => test_build.boot} | 0 .../resources/dep/{deps.edn => test_deps.edn} | 0 ...thub_action.yml => test_github_action.yml} | 0 ...trix.yml => test_github_action_matrix.yml} | 0 ...yml => test_github_action_third_party.yml} | 0 test/resources/dep/{pom.xml => test_pom.xml} | 0 .../dep/{project.clj => test_project.clj} | 0 ...-cljs-env.edn => test_shadow-cljs-env.edn} | 0 .../{shadow-cljs.edn => test_shadow-cljs.edn} | 0 26 files changed, 96 insertions(+), 84 deletions(-) rename test/resources/dep/cross-project/{deps.edn => test_deps.edn} (100%) rename test/resources/dep/local/nested/{deps.edn => test_deps.edn} (100%) rename test/resources/dep/local/{deps.edn => test_deps.edn} (100%) rename test/resources/dep/{bb.edn => test_bb.edn} (100%) rename test/resources/dep/{build.boot => test_build.boot} (100%) rename test/resources/dep/{deps.edn => test_deps.edn} (100%) rename test/resources/dep/{github_action.yml => test_github_action.yml} (100%) rename test/resources/dep/{github_action_matrix.yml => test_github_action_matrix.yml} (100%) rename test/resources/dep/{github_action_third_party.yml => test_github_action_third_party.yml} (100%) rename test/resources/dep/{pom.xml => test_pom.xml} (100%) rename test/resources/dep/{project.clj => test_project.clj} (100%) rename test/resources/dep/{shadow-cljs-env.edn => test_shadow-cljs-env.edn} (100%) rename test/resources/dep/{shadow-cljs.edn => test_shadow-cljs.edn} (100%) diff --git a/test/antq/dep/babashka_test.clj b/test/antq/dep/babashka_test.clj index 351ae57b..b7b20078 100644 --- a/test/antq/dep/babashka_test.clj +++ b/test/antq/dep/babashka_test.clj @@ -5,10 +5,11 @@ [clojure.test :as t])) (t/deftest load-deps-test - (t/is (= [(r/map->Dependency {:type :java - :file "test/resources/dep/bb.edn" - :name "bb/core" - :version "1.0.0" - :project :clojure - :repositories nil})] - (sut/load-deps "test/resources/dep")))) + (with-redefs [sut/project-file "test_bb.edn"] + (t/is (= [(r/map->Dependency {:type :java + :file "test/resources/dep/test_bb.edn" + :name "bb/core" + :version "1.0.0" + :project :clojure + :repositories nil})] + (sut/load-deps "test/resources/dep"))))) diff --git a/test/antq/dep/boot_test.clj b/test/antq/dep/boot_test.clj index 74bf3afa..37aad675 100644 --- a/test/antq/dep/boot_test.clj +++ b/test/antq/dep/boot_test.clj @@ -19,7 +19,7 @@ (t/deftest extract-deps-test (let [deps (sut/extract-deps file-path - (slurp (io/resource "dep/build.boot")))] + (slurp (io/resource "dep/test_build.boot")))] (t/is (sequential? deps)) (t/is (every? #(instance? antq.record.Dependency %) deps)) (t/is (= #{(dependency {:name "foo/core" :version "1.0.0"}) @@ -28,5 +28,7 @@ (set deps))))) (t/deftest load-deps-test - (let [deps (sut/load-deps "test/resources/dep")] - (t/is (every? #(= :java (:type %)) deps)))) + (with-redefs [sut/project-file "test_build.boot"] + (let [deps (sut/load-deps "test/resources/dep")] + (t/is (seq deps)) + (t/is (every? #(= :java (:type %)) deps))))) diff --git a/test/antq/dep/clojure_test.clj b/test/antq/dep/clojure_test.clj index b863e514..261fe0de 100644 --- a/test/antq/dep/clojure_test.clj +++ b/test/antq/dep/clojure_test.clj @@ -8,7 +8,7 @@ (def ^:private file-path ;; "path/to/deps.edn" - (.getAbsolutePath (io/file (io/resource "dep/deps.edn")))) + (.getAbsolutePath (io/file (io/resource "dep/test_deps.edn")))) (defn- java-dependency [m] @@ -34,45 +34,46 @@ m))) (t/deftest extract-deps-test - (let [deps (sut/extract-deps - file-path - (slurp (io/resource "dep/deps.edn")))] - (t/is (sequential? deps)) - (t/is (every? #(instance? antq.record.Dependency %) deps)) - (t/is (= #{(java-dependency {:name "foo/core" :version "1.0.0"}) - (java-dependency {:name "foo/core" :version "1.1.0"}) - (java-dependency {:name "bar/bar" :version "2.0.0"}) - (java-dependency {:name "baz/baz" :version "3.0.0"}) - (java-dependency {:name "rep/rep" :version "4.0.0"}) - (java-dependency {:name "ovr/ovr" :version "5.0.0"}) - (java-dependency {:name "dft/dft" :version "6.0.0"}) - (git-sha-dependency {:name "sha/sha" :version "dummy-sha" - :extra {:url "https://github.com/example/sha.git"}}) - (git-sha-dependency {:name "git-sha/git-sha" :version "dummy-git-sha" - :extra {:url "https://github.com/example/git-sha.git"}}) - (git-tag-dependency {:name "tag-short-sha/tag-short-sha" :version "v1.2.3" - :extra {:url "https://github.com/example/tag-short.git" - :sha "123abcd"}}) - (git-tag-dependency {:name "git-tag-long-sha/git-tag-long-sha" :version "v2.3.4" - :extra {:url "https://github.com/example/git-tag-long.git" - :sha "1234567890abcdefghijklmnopqrstuvwxyz1234"}}) - (git-sha-dependency {:name "com.github.liquidz/dummy" - :version "dummy-inferring-url" - :extra {:url "https://github.com/liquidz/dummy.git"}}) - (java-dependency {:name "local/core" :version "9.9.9" - :file (.getAbsolutePath (io/file (io/resource "dep/local/deps.edn"))) - :repositories nil}) - (java-dependency {:name "local/nested-core" :version "8.8.8" - :file (.getAbsolutePath (io/file (io/resource "dep/local/nested/deps.edn"))) - :repositories nil})} - (set deps))))) + (with-redefs [sut/project-file "test_deps.edn"] + (let [deps (sut/extract-deps + file-path + (slurp file-path))] + (t/is (sequential? deps)) + (t/is (every? #(instance? antq.record.Dependency %) deps)) + (t/is (= #{(java-dependency {:name "foo/core" :version "1.0.0"}) + (java-dependency {:name "foo/core" :version "1.1.0"}) + (java-dependency {:name "bar/bar" :version "2.0.0"}) + (java-dependency {:name "baz/baz" :version "3.0.0"}) + (java-dependency {:name "rep/rep" :version "4.0.0"}) + (java-dependency {:name "ovr/ovr" :version "5.0.0"}) + (java-dependency {:name "dft/dft" :version "6.0.0"}) + (git-sha-dependency {:name "sha/sha" :version "dummy-sha" + :extra {:url "https://github.com/example/sha.git"}}) + (git-sha-dependency {:name "git-sha/git-sha" :version "dummy-git-sha" + :extra {:url "https://github.com/example/git-sha.git"}}) + (git-tag-dependency {:name "tag-short-sha/tag-short-sha" :version "v1.2.3" + :extra {:url "https://github.com/example/tag-short.git" + :sha "123abcd"}}) + (git-tag-dependency {:name "git-tag-long-sha/git-tag-long-sha" :version "v2.3.4" + :extra {:url "https://github.com/example/git-tag-long.git" + :sha "1234567890abcdefghijklmnopqrstuvwxyz1234"}}) + (git-sha-dependency {:name "com.github.liquidz/dummy" + :version "dummy-inferring-url" + :extra {:url "https://github.com/liquidz/dummy.git"}}) + (java-dependency {:name "local/core" :version "9.9.9" + :file (.getAbsolutePath (io/file (io/resource "dep/local/test_deps.edn"))) + :repositories nil}) + (java-dependency {:name "local/nested-core" :version "8.8.8" + :file (.getAbsolutePath (io/file (io/resource "dep/local/nested/test_deps.edn"))) + :repositories nil})} + (set deps)))))) (t/deftest extract-deps-cross-project-configuration-test (let [cross-project-path (.getAbsolutePath (io/file - (.getParentFile (io/file (io/resource "dep/deps.edn"))) + (.getParentFile (io/file (io/resource "dep/test_deps.edn"))) "cross-project" - "deps.edn")) + "test_deps.edn")) content (pr-str '{:deps {foo/bar {:mvn/version "0.0.1"}}})] (with-redefs [alpha/user-deps-path (constantly cross-project-path)] (t/is (= [(java-dependency @@ -88,5 +89,7 @@ (t/is (empty? (sut/extract-deps file-path "{:deps {foo/core \"bar\"}}")))) (t/deftest load-deps-test - (let [deps (sut/load-deps "test/resources/dep")] - (t/is (every? #(contains? #{:java :git-sha :git-tag-and-sha} (:type %)) deps)))) + (with-redefs [sut/project-file "test_deps.edn"] + (let [deps (sut/load-deps "test/resources/dep")] + (t/is (seq deps)) + (t/is (every? #(contains? #{:java :git-sha :git-tag-and-sha} (:type %)) deps))))) diff --git a/test/antq/dep/github_action_test.clj b/test/antq/dep/github_action_test.clj index 178c8509..4a0bbf7d 100644 --- a/test/antq/dep/github_action_test.clj +++ b/test/antq/dep/github_action_test.clj @@ -10,18 +10,18 @@ [m] (r/map->Dependency (merge {:project :github-action :type :github-tag - :file "dep/github_action.yml"} m))) + :file "dep/test_github_action.yml"} m))) (defn- git-sha-dependency [m] (r/map->Dependency (merge {:project :github-action :type :git-sha - :file "dep/github_action.yml"} m))) + :file "dep/test_github_action.yml"} m))) (t/deftest extract-deps-test (let [deps (sut/extract-deps - "dep/github_action.yml" - (slurp (io/resource "dep/github_action.yml")))] + "dep/test_github_action.yml" + (slurp (io/resource "dep/test_github_action.yml")))] (t/is (sequential? deps)) (t/is (every? #(instance? antq.record.Dependency %) deps)) (t/is (= #{(git-tag-dependency {:name "foo/bar" :version "v1.0.0" @@ -38,9 +38,9 @@ (t/deftest extract-deps-matrix-test (let [deps (sut/extract-deps - "dep/github_action_matrix.yml" - (slurp (io/resource "dep/github_action_matrix.yml"))) - git-tag-dependency #(git-tag-dependency (merge {:file "dep/github_action_matrix.yml"} %))] + "dep/test_github_action_matrix.yml" + (slurp (io/resource "dep/test_github_action_matrix.yml"))) + git-tag-dependency #(git-tag-dependency (merge {:file "dep/test_github_action_matrix.yml"} %))] (t/is (sequential? deps)) (t/is (every? #(instance? antq.record.Dependency %) deps)) (t/is (= #{(git-tag-dependency {:name "DeLaGuardo/setup-graalvm" :version "master" :only-newest-version? nil diff --git a/test/antq/dep/leiningen_test.clj b/test/antq/dep/leiningen_test.clj index 770fe202..161cf86f 100644 --- a/test/antq/dep/leiningen_test.clj +++ b/test/antq/dep/leiningen_test.clj @@ -20,7 +20,7 @@ (t/deftest extract-deps-test (let [deps (sut/extract-deps file-path - (slurp (io/resource "dep/project.clj")))] + (slurp (io/resource "dep/test_project.clj")))] (t/is (sequential? deps)) (t/is (every? #(instance? antq.record.Dependency %) deps)) (t/is (= #{(dependency {:name "foo/core" :version "1.0.0"}) @@ -32,5 +32,7 @@ (set deps))))) (t/deftest load-deps-test - (let [deps (sut/load-deps "test/resources/dep")] - (t/is (every? #(= :java (:type %)) deps)))) + (with-redefs [sut/project-file "test_project.clj"] + (let [deps (sut/load-deps "test/resources/dep")] + (t/is (seq deps)) + (t/is (every? #(= :java (:type %)) deps))))) diff --git a/test/antq/dep/pom_test.clj b/test/antq/dep/pom_test.clj index b400d038..dbd22d6a 100644 --- a/test/antq/dep/pom_test.clj +++ b/test/antq/dep/pom_test.clj @@ -19,7 +19,7 @@ (t/deftest extract-deps-test (let [deps (sut/extract-deps file-path - (slurp (io/resource "dep/pom.xml")))] + (slurp (io/resource "dep/test_pom.xml")))] (t/is (sequential? deps)) (t/is (every? #(instance? antq.record.Dependency %) deps)) (t/is (= #{(dependency {:name "foo/core" :version "1.0.0"}) @@ -29,5 +29,7 @@ (set deps))))) (t/deftest load-deps-test - (let [deps (sut/load-deps "test/resources/dep")] - (t/is (every? #(= :java (:type %)) deps)))) + (with-redefs [sut/project-file "test_pom.xml"] + (let [deps (sut/load-deps "test/resources/dep")] + (t/is (seq deps)) + (t/is (every? #(= :java (:type %)) deps))))) diff --git a/test/antq/dep/shadow_test.clj b/test/antq/dep/shadow_test.clj index 083189af..83e4f927 100644 --- a/test/antq/dep/shadow_test.clj +++ b/test/antq/dep/shadow_test.clj @@ -19,7 +19,7 @@ (t/deftest extract-deps-test (let [deps (sut/extract-deps file-path - (slurp (io/resource "dep/shadow-cljs.edn")))] + (slurp (io/resource "dep/test_shadow-cljs.edn")))] (t/is (sequential? deps)) (t/is (every? #(instance? antq.record.Dependency %) deps)) (t/is (= #{(dependency {:name "foo/core" :version "1.0.0"}) @@ -34,7 +34,7 @@ (let [deps (sut/extract-deps file-path - (slurp (io/resource "dep/shadow-cljs-env.edn")))] + (slurp (io/resource "dep/test_shadow-cljs-env.edn")))] (t/is (sequential? deps)) (t/is (every? #(instance? antq.record.Dependency %) deps)) (t/is (= #{(dependency {:name "foo1" :version "1.0.0"}) @@ -45,5 +45,7 @@ (set deps)))))) (t/deftest load-deps-test - (let [deps (sut/load-deps "test/resources/dep")] - (t/is (every? #(= :java (:type %)) deps)))) + (with-redefs [sut/project-file "test_shadow-cljs.edn"] + (let [deps (sut/load-deps "test/resources/dep")] + (t/is (seq deps)) + (t/is (every? #(= :java (:type %)) deps))))) diff --git a/test/antq/upgrade/boot_test.clj b/test/antq/upgrade/boot_test.clj index 7a6b5e30..114129b4 100644 --- a/test/antq/upgrade/boot_test.clj +++ b/test/antq/upgrade/boot_test.clj @@ -13,7 +13,7 @@ :type :java :name "bar/bar" :latest-version "9.0.0" - :file (io/resource "dep/build.boot")})) + :file (io/resource "dep/test_build.boot")})) (t/deftest upgrade-dep-test (let [from-deps (->> dummy-java-dep diff --git a/test/antq/upgrade/clojure_test.clj b/test/antq/upgrade/clojure_test.clj index 1e19a5fd..3d5b0127 100644 --- a/test/antq/upgrade/clojure_test.clj +++ b/test/antq/upgrade/clojure_test.clj @@ -14,28 +14,28 @@ :type :java :name "foo/core" :latest-version "9.0.0" - :file (io/resource "dep/deps.edn")})) + :file (io/resource "dep/test_deps.edn")})) (def ^:private dummy-git-dep (r/map->Dependency {:project :clojure :type :git-sha :name "sha/sha" :latest-version "new-sha" - :file (io/resource "dep/deps.edn")})) + :file (io/resource "dep/test_deps.edn")})) (def ^:private dummy-git-git-sha-dep (r/map->Dependency {:project :clojure :type :git-sha :name "git-sha/git-sha" :latest-version "new-sha" - :file (io/resource "dep/deps.edn")})) + :file (io/resource "dep/test_deps.edn")})) (def ^:private dummy-git-tag-short-sha-dep (r/map->Dependency {:project :clojure :type :git-tag-and-sha :name "tag-short-sha/tag-short-sha" :latest-version "v9.9.9" - :file (io/resource "dep/deps.edn") + :file (io/resource "dep/test_deps.edn") :extra {:sha "1234567"}})) (def ^:private dummy-git-tag-long-sha-dep @@ -43,7 +43,7 @@ :type :git-tag-and-sha :name "git-tag-long-sha/git-tag-long-sha" :latest-version "v9.9.9" - :file (io/resource "dep/deps.edn") + :file (io/resource "dep/test_deps.edn") :extra {:sha "123456789x123456789x123456789x123456789x"}})) (def ^:private dummy-no-version-dep @@ -51,7 +51,7 @@ :type :java :name "no-version" :latest-version "9.9.9" - :file (io/resource "dep/deps.edn")})) + :file (io/resource "dep/test_deps.edn")})) (t/deftest upgrade-dep-test (t/testing "java" diff --git a/test/antq/upgrade/github_action_test.clj b/test/antq/upgrade/github_action_test.clj index bb6c8740..f040760d 100644 --- a/test/antq/upgrade/github_action_test.clj +++ b/test/antq/upgrade/github_action_test.clj @@ -14,7 +14,7 @@ :type :github-tag :name "foo/bar" :latest-version "v9.0.0" - :file (io/resource "dep/github_action.yml") + :file (io/resource "dep/test_github_action.yml") :extra {const.gh-action/type-key "uses"}})) (def ^:private dummy-not-supported-dep @@ -22,14 +22,14 @@ :type :github-tag :name "bar/baz" :latest-version "v9.0.0" - :file (io/resource "dep/github_action.yml")})) + :file (io/resource "dep/test_github_action.yml")})) (def ^:private dummy-clojure-cli-dep (r/map->Dependency {:project :github-action :type :github-tag :name "clojure/brew-install" :latest-version "9.0.0" - :file (io/resource "dep/github_action_third_party.yml") + :file (io/resource "dep/test_github_action_third_party.yml") :extra {const.gh-action/type-key "DeLaGuardo/setup-clojure"}})) (def ^:private dummy-leiningen-dep @@ -37,7 +37,7 @@ :type :github-tag :name "technomancy/leiningen" :latest-version "9.0.0" - :file (io/resource "dep/github_action_third_party.yml") + :file (io/resource "dep/test_github_action_third_party.yml") :extra {const.gh-action/type-key "DeLaGuardo/setup-clojure"}})) (def ^:private dummy-boot-dep @@ -45,7 +45,7 @@ :type :github-tag :name "boot-clj/boot" :latest-version "9.0.0" - :file (io/resource "dep/github_action_third_party.yml") + :file (io/resource "dep/test_github_action_third_party.yml") :extra {const.gh-action/type-key "DeLaGuardo/setup-clojure"}})) (def ^:private dummy-clj-kondo-dep @@ -53,7 +53,7 @@ :type :java :name "clj-kondo/clj-kondo" :latest-version "9.0.0" - :file (io/resource "dep/github_action_third_party.yml") + :file (io/resource "dep/test_github_action_third_party.yml") :extra {const.gh-action/type-key "DeLaGuardo/setup-clj-kondo"}})) (def ^:private dummy-graalvm-dep @@ -61,7 +61,7 @@ :type :github-tag :name "graalvm/graalvm-ce-builds" :latest-version "9.0.0" - :file (io/resource "dep/github_action_third_party.yml") + :file (io/resource "dep/test_github_action_third_party.yml") :extra {const.gh-action/type-key "DeLaGuardo/setup-graalvm"}})) (def ^:private dummy-cljstyle-dep @@ -69,7 +69,7 @@ :type :github-tag :name "greglook/cljstyle" :latest-version "9.0.0" - :file (io/resource "dep/github_action_third_party.yml") + :file (io/resource "dep/test_github_action_third_party.yml") :extra {const.gh-action/type-key "0918nobita/setup-cljstyle"}})) (t/deftest upgrade-dep-test diff --git a/test/antq/upgrade/leiningen_test.clj b/test/antq/upgrade/leiningen_test.clj index ec708b6c..1cc599ab 100644 --- a/test/antq/upgrade/leiningen_test.clj +++ b/test/antq/upgrade/leiningen_test.clj @@ -13,7 +13,7 @@ :type :java :name "foo/core" :latest-version "9.0.0" - :file (io/resource "dep/project.clj")}) + :file (io/resource "dep/test_project.clj")}) from-deps (->> java-dep :file (slurp) @@ -30,7 +30,7 @@ :type :java :name "bar/bar" :latest-version "8.0.0" - :file (io/resource "dep/project.clj")}) + :file (io/resource "dep/test_project.clj")}) from-deps (->> meta-dep :file (slurp) @@ -46,7 +46,7 @@ :type :java :name "plug/plug" :latest-version "7.0.0" - :file (io/resource "dep/project.clj")}) + :file (io/resource "dep/test_project.clj")}) from-deps (->> meta-dep :file (slurp) diff --git a/test/antq/upgrade/pom_test.clj b/test/antq/upgrade/pom_test.clj index dbb6e94f..cfb3c9c4 100644 --- a/test/antq/upgrade/pom_test.clj +++ b/test/antq/upgrade/pom_test.clj @@ -13,7 +13,7 @@ :type :java :name "foo/core" :latest-version "9.0.0" - :file (io/resource "dep/pom.xml")})) + :file (io/resource "dep/test_pom.xml")})) (t/deftest upgrade-dep-test (let [from-deps (->> dummy-java-dep diff --git a/test/antq/upgrade/shadow_test.clj b/test/antq/upgrade/shadow_test.clj index c5826f5a..a8048575 100644 --- a/test/antq/upgrade/shadow_test.clj +++ b/test/antq/upgrade/shadow_test.clj @@ -13,7 +13,7 @@ :type :java :name "foo/core" :latest-version "9.0.0" - :file (io/resource "dep/shadow-cljs.edn")})) + :file (io/resource "dep/test_shadow-cljs.edn")})) (t/deftest upgrade-dep-test (let [from-deps (->> dummy-java-dep diff --git a/test/resources/dep/cross-project/deps.edn b/test/resources/dep/cross-project/test_deps.edn similarity index 100% rename from test/resources/dep/cross-project/deps.edn rename to test/resources/dep/cross-project/test_deps.edn diff --git a/test/resources/dep/local/nested/deps.edn b/test/resources/dep/local/nested/test_deps.edn similarity index 100% rename from test/resources/dep/local/nested/deps.edn rename to test/resources/dep/local/nested/test_deps.edn diff --git a/test/resources/dep/local/deps.edn b/test/resources/dep/local/test_deps.edn similarity index 100% rename from test/resources/dep/local/deps.edn rename to test/resources/dep/local/test_deps.edn diff --git a/test/resources/dep/bb.edn b/test/resources/dep/test_bb.edn similarity index 100% rename from test/resources/dep/bb.edn rename to test/resources/dep/test_bb.edn diff --git a/test/resources/dep/build.boot b/test/resources/dep/test_build.boot similarity index 100% rename from test/resources/dep/build.boot rename to test/resources/dep/test_build.boot diff --git a/test/resources/dep/deps.edn b/test/resources/dep/test_deps.edn similarity index 100% rename from test/resources/dep/deps.edn rename to test/resources/dep/test_deps.edn diff --git a/test/resources/dep/github_action.yml b/test/resources/dep/test_github_action.yml similarity index 100% rename from test/resources/dep/github_action.yml rename to test/resources/dep/test_github_action.yml diff --git a/test/resources/dep/github_action_matrix.yml b/test/resources/dep/test_github_action_matrix.yml similarity index 100% rename from test/resources/dep/github_action_matrix.yml rename to test/resources/dep/test_github_action_matrix.yml diff --git a/test/resources/dep/github_action_third_party.yml b/test/resources/dep/test_github_action_third_party.yml similarity index 100% rename from test/resources/dep/github_action_third_party.yml rename to test/resources/dep/test_github_action_third_party.yml diff --git a/test/resources/dep/pom.xml b/test/resources/dep/test_pom.xml similarity index 100% rename from test/resources/dep/pom.xml rename to test/resources/dep/test_pom.xml diff --git a/test/resources/dep/project.clj b/test/resources/dep/test_project.clj similarity index 100% rename from test/resources/dep/project.clj rename to test/resources/dep/test_project.clj diff --git a/test/resources/dep/shadow-cljs-env.edn b/test/resources/dep/test_shadow-cljs-env.edn similarity index 100% rename from test/resources/dep/shadow-cljs-env.edn rename to test/resources/dep/test_shadow-cljs-env.edn diff --git a/test/resources/dep/shadow-cljs.edn b/test/resources/dep/test_shadow-cljs.edn similarity index 100% rename from test/resources/dep/shadow-cljs.edn rename to test/resources/dep/test_shadow-cljs.edn From 255efd292a357a02e992215297d8e7d2fa4c6e3d Mon Sep 17 00:00:00 2001 From: liquidz Date: Tue, 21 Jun 2022 06:27:18 +0900 Subject: [PATCH 19/41] deps: Bump kaocha to 1.67.1055 --- deps.edn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps.edn b/deps.edn index 08185197..ff6a7a4a 100644 --- a/deps.edn +++ b/deps.edn @@ -26,7 +26,7 @@ :dev {:extra-paths ["test" "test/resources"] - :extra-deps {lambdaisland/kaocha {:mvn/version "1.66.1034"} + :extra-deps {lambdaisland/kaocha {:mvn/version "1.67.1055"} lambdaisland/deep-diff2 {:mvn/version "2.2.124"}} :jvm-opts ["-Dclojure.core.async.go-checking=true"]} From fdc149ed37fb56df53d597336e2c229d8d3d49d3 Mon Sep 17 00:00:00 2001 From: liquidz Date: Sun, 26 Jun 2022 07:00:57 +0900 Subject: [PATCH 20/41] chore: Change default timeout msec --- src/antq/constant.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/antq/constant.clj b/src/antq/constant.clj index 179c0e1c..97f0aa41 100644 --- a/src/antq/constant.clj +++ b/src/antq/constant.clj @@ -13,7 +13,7 @@ #{:deps :default-deps :extra-deps :override-deps :replace-deps}) (def default-timeout-msec - (u.env/getlong "ANTQ_DEFAULT_TIMEOUT" 5000)) + (u.env/getlong "ANTQ_DEFAULT_TIMEOUT" 10000)) (def ls-remote-timeout-msec "Timeout msec for From 0b44559688208339129f788a4c0e2674b4a21479 Mon Sep 17 00:00:00 2001 From: liquidz Date: Sun, 26 Jun 2022 07:01:22 +0900 Subject: [PATCH 21/41] deps: Bump kaocha to 1.68.1059 --- deps.edn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps.edn b/deps.edn index ff6a7a4a..80fc719c 100644 --- a/deps.edn +++ b/deps.edn @@ -26,7 +26,7 @@ :dev {:extra-paths ["test" "test/resources"] - :extra-deps {lambdaisland/kaocha {:mvn/version "1.67.1055"} + :extra-deps {lambdaisland/kaocha {:mvn/version "1.68.1059"} lambdaisland/deep-diff2 {:mvn/version "2.2.124"}} :jvm-opts ["-Dclojure.core.async.go-checking=true"]} From 00825d5677ba1427af4005b40ae6cce7551b405d Mon Sep 17 00:00:00 2001 From: liquidz Date: Sun, 26 Jun 2022 07:01:45 +0900 Subject: [PATCH 22/41] deps: Bump build.edn to 0.4.107 --- deps.edn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps.edn b/deps.edn index 80fc719c..860b2971 100644 --- a/deps.edn +++ b/deps.edn @@ -44,7 +44,7 @@ :main-opts ["-m" "cloverage.coverage" "--ns-exclude-regex" "leiningen.antq"]} :build - {:deps {com.github.liquidz/build.edn {:git/tag "0.3.90" :git/sha "e3a3e31"}} + {:deps {com.github.liquidz/build.edn {:git/tag "0.4.107" :git/sha "cf69a2a"}} :ns-default build} ;; -X From 42ab8651ffa13f04a77ba3cc81b8c482df5fab95 Mon Sep 17 00:00:00 2001 From: liquidz Date: Sun, 26 Jun 2022 07:02:09 +0900 Subject: [PATCH 23/41] deps: Bump clj-kondo action --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 7a496984..91d2489e 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -7,7 +7,7 @@ jobs: - uses: actions/checkout@v3 - uses: DeLaGuardo/setup-clj-kondo@master with: - version: '2022.05.31' + version: '2022.06.22' - run: clj-kondo --lint src:test --config '{:output {:pattern "::{{level}} file={{filename}},line={{row}},col={{col}}::{{message}}"}}' cljstyle: From 6297ecbfaee04ed84ee87feb7fdcae1ecab1fb91 Mon Sep 17 00:00:00 2001 From: liquidz Date: Sun, 26 Jun 2022 07:03:12 +0900 Subject: [PATCH 24/41] docs: Update Installing/updating Clojure CLI tool instruction --- README.adoc | 21 ++++++++++++--------- build.clj | 6 +----- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/README.adoc b/README.adoc index 50337c48..dbb6719b 100644 --- a/README.adoc +++ b/README.adoc @@ -81,23 +81,26 @@ Or add the following alias to your `$HOME/.clojure/deps.edn`. Then, run `clojure -M:outdated`. (run `clojure -A:outdated` for Clojure CLI Tool 1.10.1.645 or earlier). -=== Clojure CLI Tools (`1.10.3.933` or later) +=== Clojure CLI Tools (`1.11.1.1139` or later) -From Clojure CLI ver `1.10.3.933`, https://clojure.org/reference/deps_and_cli#tool_install[tool] installation is supported. +From Clojure CLI ver `1.11.1.1139`, https://clojure.org/reference/deps_and_cli#tool_install[tool] installation is supported. [source,sh] ---- -# install -clojure -Ttools install com.github.liquidz/antq '{:git/tag "1.7.804"}' :as antq -# uninstall +# Install tool +clojure -Ttools install-latest :lib com.github.liquidz/antq :as antq +# Uninstall tool clojure -Ttools remove :tool antq -# execute +# Update tool +clojure -Ttools install-latest :tool antq + +# Execute clojure -Tantq outdated -# execute with parameter +# Execute with parameter clojure -Tantq outdated :upgrade true -# help +# Show help clojure -A:deps -Tantq help/doc -# upgrade +# Upgrade oudated dependencies clojure -Tantq outdated :check-clojure-tools true :upgrade true ---- diff --git a/build.clj b/build.clj index 34dee3ab..b3ea8433 100644 --- a/build.clj +++ b/build.clj @@ -9,11 +9,7 @@ :documents [{:file "CHANGELOG.adoc" :match "Unreleased" :action :append-after - :text "\n== {{version}} ({{now/yyyy}}-{{now/mm}}-{{now/dd}})"} - {:file "README.adoc" - :match "install com\\.github\\.liquidz/antq" - :action :replace - :text "clojure -Ttools install com.github.liquidz/antq '{:git/tag \"{{version}}\"}' :as antq"}] + :text "\n== {{version}} ({{now/yyyy}}-{{now/mm}}-{{now/dd}})"}] :github-actions? true}) (defn jar From bdaf7d05a451eaeec83f3ec2e28240176dea3d2b Mon Sep 17 00:00:00 2001 From: liquidz Date: Sun, 26 Jun 2022 07:04:17 +0900 Subject: [PATCH 25/41] chore: Add description --- build.clj | 1 + 1 file changed, 1 insertion(+) diff --git a/build.clj b/build.clj index b3ea8433..8babc322 100644 --- a/build.clj +++ b/build.clj @@ -5,6 +5,7 @@ (def ^:private config {:lib 'com.github.liquidz/antq :version "1.7.{{git/commit-count}}" + :description "Point out your outdated dependencies" :main 'antq.core :documents [{:file "CHANGELOG.adoc" :match "Unreleased" From 67e6f83e12595b857a0a1d205274945d62b720a6 Mon Sep 17 00:00:00 2001 From: liquidz Date: Sun, 26 Jun 2022 07:45:50 +0900 Subject: [PATCH 26/41] docs: Add document for timeouts --- README.adoc | 6 ++++++ doc/timeout.adoc | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 doc/timeout.adoc diff --git a/README.adoc b/README.adoc index dbb6719b..c2206ca3 100644 --- a/README.adoc +++ b/README.adoc @@ -150,6 +150,11 @@ In another way, you can use the following action. Antq experimentally supports https://gradle.org[Gradle]. See link:./doc/gradle.adoc[here] for details. +=== Timeouts + +Antq has timeouts for acquiring various information. +See link:./doc/timeout.adoc[here] for details. + == Options === --upgrade Upgrade outdated versions interactively. @@ -292,6 +297,7 @@ You can also upgrade them with `--upgrade` option. * link:./doc/clojure-1.8.adoc[Antq with Clojure 1.8.0 or earlier] * link:./doc/gradle.adoc[Work with Gradle] * link:./doc/proxy.adoc[Run behind proxy] +* link:./doc/timeout.adoc[Timeouts] == License diff --git a/doc/timeout.adoc b/doc/timeout.adoc new file mode 100644 index 00000000..4e020ed6 --- /dev/null +++ b/doc/timeout.adoc @@ -0,0 +1,35 @@ += Timeouts + +Antq has timeouts for acquiring various information. +These timeouts are customizable by following environmental variables. + +[cols="1,4a,5a"] +|=== +| Name | Default | Description + +| ANTQ_DEFAULT_TIMEOUT +| `10,000` +| Unit is milli sec. + + +| ANTQ_LS_REMOTE_TIMEOUT +| Same as `ANTQ_DEFAULT_TIMEOUT`. +| Timeout for `git ls-remote`. + + Unit is milli sec. + +| ANTQ_GITHUB_API_TIMEOUT +| Same as `ANTQ_DEFAULT_TIMEOUT`. +| Timeout for GitHub API. + + Unit is milli sec. + +| ANTQ_MAVEN_TIMEOUT +| Same as `ANTQ_DEFAULT_TIMEOUT`. +| Timeout for accessing Maven repositories. + + Unit is milli sec. + +| ANTQ_POM_TIMEOUT +| Same as `ANTQ_DEFAULT_TIMEOUT`. +| Timeout for reading POM. + + Unit is milli sec. + +|=== From bb347aed9fb4be865dce1e2d8e1f942b8728c7c8 Mon Sep 17 00:00:00 2001 From: liquidz Date: Sun, 26 Jun 2022 10:54:31 +0900 Subject: [PATCH 27/41] refactor: Make inner functions private --- src/antq/util/git.clj | 6 +++--- test/antq/util/git_test.clj | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/antq/util/git.clj b/src/antq/util/git.clj index 16822849..a437e41b 100644 --- a/src/antq/util/git.clj +++ b/src/antq/util/git.clj @@ -60,7 +60,7 @@ (def ^:private ls-remote (memoize ls-remote*-with-timeout)) -(defn tags-by-ls-remote* +(defn- tags-by-ls-remote* [url] (-> (ls-remote url) (extract-tags))) @@ -68,12 +68,12 @@ (def tags-by-ls-remote (memoize tags-by-ls-remote*)) -(defn head-sha-by-ls-remote* +(defn- head-sha-by-ls-remote* [url] (-> (ls-remote url) (extract-sha-by-ref-name "HEAD"))) -(defn tag-sha-by-ls-remote* +(defn- tag-sha-by-ls-remote* [url tag-name] (-> (ls-remote url) (extract-sha-by-ref-name (str "refs/tags/" tag-name)))) diff --git a/test/antq/util/git_test.clj b/test/antq/util/git_test.clj index 8a8803e7..005493c7 100644 --- a/test/antq/util/git_test.clj +++ b/test/antq/util/git_test.clj @@ -34,20 +34,20 @@ :out dummy-ls-remote-tag-out :err ""})] (t/is (= ["v1" "v2"] - (sut/tags-by-ls-remote* (dummy-url)))))) + (#'sut/tags-by-ls-remote* (dummy-url)))))) (t/deftest head-sha-by-ls-remote*-test (with-redefs [sh/sh (constantly {:exit 0 :out dummy-ls-remote-sha-out :err ""})] (t/is (= "head-sha" - (sut/head-sha-by-ls-remote* (dummy-url)))))) + (#'sut/head-sha-by-ls-remote* (dummy-url)))))) (t/deftest tag-sha-by-ls-remote*-test (with-redefs [sh/sh (constantly {:exit 0 :out dummy-ls-remote-tag-out :err ""})] (t/is (= "dummy-sha4" - (sut/tag-sha-by-ls-remote* (dummy-url) "v1"))) + (#'sut/tag-sha-by-ls-remote* (dummy-url) "v1"))) (t/is (= "dummy-sha6" - (sut/tag-sha-by-ls-remote* (dummy-url) "v2"))))) + (#'sut/tag-sha-by-ls-remote* (dummy-url) "v2"))))) From 9ad4980c6e91aed82cd2a202c5912325e614ce58 Mon Sep 17 00:00:00 2001 From: liquidz Date: Sun, 26 Jun 2022 10:57:03 +0900 Subject: [PATCH 28/41] feat: Add antq.util.exception --- src/antq/util/exception.clj | 12 ++++++++++++ test/antq/util/exception_test.clj | 18 ++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 src/antq/util/exception.clj create mode 100644 test/antq/util/exception_test.clj diff --git a/src/antq/util/exception.clj b/src/antq/util/exception.clj new file mode 100644 index 00000000..c8481a6b --- /dev/null +++ b/src/antq/util/exception.clj @@ -0,0 +1,12 @@ +(ns antq.util.exception + (:import + clojure.lang.ExceptionInfo)) + +(defn ex-timeout + [msg] + (ex-info msg {:type ::timeout})) + +(defn ex-timeout? + [x] + (and (instance? ExceptionInfo x) + (= ::timeout (:type (ex-data x))))) diff --git a/test/antq/util/exception_test.clj b/test/antq/util/exception_test.clj new file mode 100644 index 00000000..fa62fe9b --- /dev/null +++ b/test/antq/util/exception_test.clj @@ -0,0 +1,18 @@ +(ns antq.util.exception-test + (:require + [antq.util.exception :as sut] + [clojure.test :as t]) + (:import + clojure.lang.ExceptionInfo)) + +(t/deftest ex-timeout-test + (let [ex (sut/ex-timeout "foo")] + (t/is (instance? ExceptionInfo ex)) + (t/is (= "foo" (.getMessage ex))) + (t/is (contains? (ex-data ex) :type)) + (t/is (qualified-keyword? (:type (ex-data ex)))))) + +(t/deftest ex-timeout?-test + (t/is (true? (sut/ex-timeout? (sut/ex-timeout "")))) + (t/is (false? (sut/ex-timeout? (ex-info "" {})))) + (t/is (false? (sut/ex-timeout? nil)))) From 05ad48d27e164129606d1d407dc43e91d83ae095 Mon Sep 17 00:00:00 2001 From: liquidz Date: Sun, 26 Jun 2022 11:05:47 +0900 Subject: [PATCH 29/41] fix: Update fn-with-timeout to throw ex-timeout --- src/antq/util/async.clj | 37 ++++++++++++++++++++--------------- test/antq/util/async_test.clj | 19 +++++++++++++++--- 2 files changed, 37 insertions(+), 19 deletions(-) diff --git a/src/antq/util/async.clj b/src/antq/util/async.clj index 9b130cc1..e915b1b8 100644 --- a/src/antq/util/async.clj +++ b/src/antq/util/async.clj @@ -1,21 +1,26 @@ (ns antq.util.async (:require + [antq.util.exception :as u.ex] [clojure.core.async :as async])) (defn fn-with-timeout - [f timeout-ms] - (fn [& args] - (let [ch (async/chan)] - (async/pipe (async/thread - (try - (apply f args) - (catch Throwable ex - ex))) - ch) - (let [ret (-> [ch - (async/timeout timeout-ms)] - (async/alts!!) - (first))] - (when (instance? Throwable ret) - (throw ret)) - ret)))) + ([f timeout-ms] + (fn-with-timeout f timeout-ms "")) + ([f timeout-ms timeouted-msg] + (fn [& args] + (let [ch (async/chan)] + (async/pipe (async/thread + (try + (apply f args) + (catch Throwable ex + ex))) + ch) + (let [ret (-> [ch + (async/go + (async/ Date: Sun, 26 Jun 2022 11:14:34 +0900 Subject: [PATCH 30/41] feat: Update to show timed out message for java deps --- src/antq/util/ver.clj | 26 +++++++++++++++++++++++--- src/antq/ver.clj | 9 +++++++-- src/antq/ver/java.clj | 29 ++++++++++++++++++----------- test/antq/ver/java_test.clj | 8 ++++++++ 4 files changed, 56 insertions(+), 16 deletions(-) diff --git a/src/antq/util/ver.clj b/src/antq/util/ver.clj index b1df2d19..e02708fa 100644 --- a/src/antq/util/ver.clj +++ b/src/antq/util/ver.clj @@ -1,10 +1,14 @@ (ns antq.util.ver (:require + [antq.util.exception :as u.ex] [clojure.string :as str])) (def ^:private no-latest-version-error "Failed to fetch") +(def ^:private timed-out-error + "Timed out") + (defn remove-qualifiers "c.f. https://github.com/xsc/version-clj/blob/v2.0.2/src/version_clj/qualifiers.cljc" [s] @@ -23,11 +27,27 @@ (defmethod normalize-latest-version :default [{:keys [latest-version]}] - (or latest-version - no-latest-version-error)) + (cond + (string? latest-version) + latest-version + + (u.ex/ex-timeout? latest-version) + timed-out-error + + :else + no-latest-version-error)) (defmethod normalize-latest-version :git-sha [{:keys [version latest-version]}] - (if (and version latest-version) + (cond + (and version + latest-version + (string? version) + (string? latest-version)) (subs latest-version 0 (count version)) + + (u.ex/ex-timeout? latest-version) + timed-out-error + + :else no-latest-version-error)) diff --git a/src/antq/ver.clj b/src/antq/ver.clj index 8937854b..e248e95a 100644 --- a/src/antq/ver.clj +++ b/src/antq/ver.clj @@ -8,13 +8,16 @@ (defn under-devleopment? [s] - (if-let [l (and s (str/lower-case s))] + (if-let [l (and s + (string? s) + (str/lower-case s))] (some? (some #(str/includes? l %) under-development-keywords)) false)) (defn snapshot? [s] - (if s + (if (and s + (string? s)) (str/includes? (str/lower-case s) "snapshot") false)) @@ -30,6 +33,8 @@ [dep] (and (:version dep) (:latest-version dep) + (string? (:version dep)) + (string? (:latest-version dep)) (<= 0 (version/version-compare (:version dep) (:latest-version dep))))) diff --git a/src/antq/ver/java.clj b/src/antq/ver/java.clj index 235bf91a..0238093e 100644 --- a/src/antq/ver/java.clj +++ b/src/antq/ver/java.clj @@ -2,11 +2,13 @@ (:require [antq.constant :as const] [antq.util.async :as u.async] + [antq.util.exception :as u.ex] [antq.util.maven :as u.mvn] [antq.ver :as ver] [clojure.set :as set] [version-clj.core :as version]) (:import + clojure.lang.ExceptionInfo (org.eclipse.aether DefaultRepositorySystemSession RepositorySystem) @@ -36,17 +38,22 @@ [name {:as dep-opts :keys [snapshots?]} options] - (let [maven-vers (->> (get-versions-with-timeout name dep-opts) - (map str)) - versions (if (:ignore-locals options) - (seq (set/difference (set maven-vers) - (set (u.mvn/get-local-versions name)))) - maven-vers) - sorted-versions (->> versions - (sort version/version-compare) - (reverse))] - (cond->> sorted-versions - (not snapshots?) (remove ver/snapshot?)))) + (try + (let [maven-vers (->> (get-versions-with-timeout name dep-opts) + (map str)) + versions (if (:ignore-locals options) + (seq (set/difference (set maven-vers) + (set (u.mvn/get-local-versions name)))) + maven-vers) + sorted-versions (->> versions + (sort version/version-compare) + (reverse))] + (cond->> sorted-versions + (not snapshots?) (remove ver/snapshot?))) + (catch ExceptionInfo ex + (if (u.ex/ex-timeout? ex) + [ex] + (throw ex))))) (def get-sorted-versions-by-name (memoize get-sorted-versions-by-name*)) diff --git a/test/antq/ver/java_test.clj b/test/antq/ver/java_test.clj index 44df5880..71bf05ca 100644 --- a/test/antq/ver/java_test.clj +++ b/test/antq/ver/java_test.clj @@ -1,6 +1,7 @@ (ns antq.ver.java-test (:require [antq.record :as r] + [antq.util.exception :as u.ex] [antq.util.maven :as u.mvn] [antq.ver :as ver] [antq.ver.java :as sut] @@ -47,3 +48,10 @@ (set u.mvn/default-repos))] (t/is (= #{["foo" {:url "s3://bar"}]} diff)))))) + +(t/deftest get-sorted-versions-timeout-test + (with-redefs [sut/get-sorted-versions-by-name sut/get-sorted-versions-by-name* + sut/get-versions-with-timeout (fn [& _] (throw (u.ex/ex-timeout "test timeout")))] + (let [deps (get-sorted-versions {:version "1.0.0"})] + (t/is (= 1 (count deps))) + (t/is (u.ex/ex-timeout? (first deps)))))) From 0a4ec76e3f3432f6e0d741eaa48875d1cefa2063 Mon Sep 17 00:00:00 2001 From: liquidz Date: Sun, 26 Jun 2022 11:17:45 +0900 Subject: [PATCH 31/41] fix: Fix to skip ex-timeout for associating diff url --- src/antq/core.clj | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/antq/core.clj b/src/antq/core.clj index f5abb96c..7e8bcad0 100644 --- a/src/antq/core.clj +++ b/src/antq/core.clj @@ -36,6 +36,7 @@ [antq.upgrade.leiningen] [antq.upgrade.pom] [antq.upgrade.shadow] + [antq.util.exception :as u.ex] [antq.util.maven :as u.maven] [antq.ver :as ver] [antq.ver.git-sha] @@ -44,7 +45,9 @@ [antq.ver.java] [clojure.string :as str] [clojure.tools.cli :as cli] - [version-clj.core :as version])) + [version-clj.core :as version]) + (:import + clojure.lang.ExceptionInfo)) (defn- concat-assoc-fn [opt k v] @@ -183,7 +186,10 @@ (defn assoc-diff-url [version-checked-dep] - (if-let [url (diff/get-diff-url version-checked-dep)] + (if-let [url (try (diff/get-diff-url version-checked-dep) + (catch ExceptionInfo ex + (when-not (u.ex/ex-timeout? ex) + (throw ex))))] (assoc version-checked-dep :diff-url url) version-checked-dep)) From ca455fcf90e84aacf1b1483f83cdc627d52bee8a Mon Sep 17 00:00:00 2001 From: liquidz Date: Sun, 26 Jun 2022 11:42:12 +0900 Subject: [PATCH 32/41] fix: Fix compare-deps to check args --- src/antq/util/dep.clj | 11 +++++---- test/antq/util/dep_test.clj | 47 ++++++++++++++++++++----------------- 2 files changed, 33 insertions(+), 25 deletions(-) diff --git a/src/antq/util/dep.clj b/src/antq/util/dep.clj index b804e268..12b8bfc0 100644 --- a/src/antq/util/dep.clj +++ b/src/antq/util/dep.clj @@ -7,10 +7,13 @@ (defn compare-deps [x y] - (let [prj (.compareTo ^String (:file x) ^String (:file y))] - (if (zero? prj) - (.compareTo ^String (:name x) ^String (:name y)) - prj))) + (if (and (string? (:file x)) + (string? (:file y))) + (let [prj (.compareTo ^String (:file x) ^String (:file y))] + (if (zero? prj) + (.compareTo ^String (:name x) ^String (:name y)) + prj)) + 0)) (defn relative-path [^File target-file] diff --git a/test/antq/util/dep_test.clj b/test/antq/util/dep_test.clj index 29d4cc5b..df2d5d88 100644 --- a/test/antq/util/dep_test.clj +++ b/test/antq/util/dep_test.clj @@ -6,30 +6,35 @@ [clojure.test :as t])) (t/deftest compare-deps-test - (let [aaa (h/test-dep {:file "aa" :name "bb"}) - bbb (h/test-dep {:file "aa" :name "xx"}) - ccc (h/test-dep {:file "xx" :name "bb"}) - ddd (h/test-dep {:file "xx" :name "xx"})] - (t/are [expected-fn x y] (expected-fn (sut/compare-deps x y)) - zero? aaa aaa - neg? aaa bbb - neg? aaa ccc - neg? aaa ddd + (t/testing "positive" + (let [aaa (h/test-dep {:file "aa" :name "bb"}) + bbb (h/test-dep {:file "aa" :name "xx"}) + ccc (h/test-dep {:file "xx" :name "bb"}) + ddd (h/test-dep {:file "xx" :name "xx"})] + (t/are [expected-fn x y] (expected-fn (sut/compare-deps x y)) + zero? aaa aaa + neg? aaa bbb + neg? aaa ccc + neg? aaa ddd - pos? bbb aaa - zero? bbb bbb - neg? bbb ccc - neg? bbb ddd + pos? bbb aaa + zero? bbb bbb + neg? bbb ccc + neg? bbb ddd - pos? ccc aaa - pos? ccc bbb - zero? ccc ccc - neg? ccc ddd + pos? ccc aaa + pos? ccc bbb + zero? ccc ccc + neg? ccc ddd - pos? ddd aaa - pos? ddd bbb - pos? ddd ccc - zero? ddd ddd))) + pos? ddd aaa + pos? ddd bbb + pos? ddd ccc + zero? ddd ddd))) + + (t/testing "negative" + (t/is (zero? (sut/compare-deps {} {}))) + (t/is (zero? (sut/compare-deps nil nil))))) (t/deftest relative-path-test (t/is (= "foo" (sut/relative-path (io/file "foo")))) From c192ae270281661819c3cba2ef088163a5e87ec2 Mon Sep 17 00:00:00 2001 From: liquidz Date: Sun, 26 Jun 2022 11:50:20 +0900 Subject: [PATCH 33/41] feat: Update to show timed out message for git sha deps --- src/antq/ver/git_sha.clj | 23 ++++++++++++++++------- test/antq/ver/git_sha_test.clj | 11 +++++++++++ 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/src/antq/ver/git_sha.clj b/src/antq/ver/git_sha.clj index 6544b601..017a9fa8 100644 --- a/src/antq/ver/git_sha.clj +++ b/src/antq/ver/git_sha.clj @@ -1,18 +1,27 @@ (ns antq.ver.git-sha (:require + [antq.util.exception :as u.ex] [antq.util.git :as u.git] [antq.ver :as ver])) (defmethod ver/get-sorted-versions :git-sha [dep _options] - (or (some-> (get-in dep [:extra :url]) - (u.git/head-sha-by-ls-remote) - (vector)) - [])) + (try + (or (some-> (get-in dep [:extra :url]) + (u.git/head-sha-by-ls-remote) + (vector)) + []) + (catch Exception ex + (if (u.ex/ex-timeout? ex) + [ex] + (throw ex))))) (defmethod ver/latest? :git-sha [dep] (let [current (some-> dep :version) - latest (some-> dep :latest-version - (subs 0 (count current)))] - (= latest current))) + latest (some-> dep :latest-version)] + (if (and (string? current) + (string? latest)) + (= (subs latest 0 (count current)) + current) + false))) diff --git a/test/antq/ver/git_sha_test.clj b/test/antq/ver/git_sha_test.clj index 239fe88a..cb4612e5 100644 --- a/test/antq/ver/git_sha_test.clj +++ b/test/antq/ver/git_sha_test.clj @@ -1,5 +1,6 @@ (ns antq.ver.git-sha-test (:require + [antq.util.exception :as u.ex] [antq.util.git :as u.git] [antq.ver :as ver] [antq.ver.git-sha] @@ -17,6 +18,13 @@ (ver/get-sorted-versions {:type :git-sha :extra {:url "failed to fetch"}} {}))))) +(t/deftest get-sorted-versions-timeout-test + (with-redefs [u.git/head-sha-by-ls-remote (fn [& _] (throw (u.ex/ex-timeout "test timeout")))] + (let [deps (ver/get-sorted-versions {:type :git-sha :extra {:url "https://example.com"}} + {})] + (t/is (= 1 (count deps))) + (t/is (u.ex/ex-timeout? (first deps)))))) + (t/deftest latest?-test (t/is (true? (ver/latest? {:type :git-sha :version "foo" :latest-version "foo"}))) (t/is (false? (ver/latest? {:type :git-sha :version "foo" :latest-version "bar"}))) @@ -26,3 +34,6 @@ :version "8be0919" :latest-version "8be09192b01d78912b03852f5d6141e8c48f4179"}))))) +(t/deftest latest?-timeout-test + (let [ex (u.ex/ex-timeout "dummy")] + (t/is (false? (ver/latest? {:type :git-sha :version "foo" :latest-version ex}))))) From 152f832e228c90de1a3a4d4152d5d0a2da4432f6 Mon Sep 17 00:00:00 2001 From: liquidz Date: Sun, 26 Jun 2022 11:53:04 +0900 Subject: [PATCH 34/41] feat: Update to show timed out message for git tag and sha deps --- src/antq/ver/git_tag_and_sha.clj | 27 ++++++++++++++++---------- test/antq/ver/git_tag_and_sha_test.clj | 8 ++++++++ 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/src/antq/ver/git_tag_and_sha.clj b/src/antq/ver/git_tag_and_sha.clj index 2f789977..7b86994e 100644 --- a/src/antq/ver/git_tag_and_sha.clj +++ b/src/antq/ver/git_tag_and_sha.clj @@ -1,5 +1,6 @@ (ns antq.ver.git-tag-and-sha (:require + [antq.util.exception :as u.ex] [antq.util.git :as u.git] [antq.util.ver :as u.ver] [antq.ver :as ver] @@ -7,13 +8,19 @@ (defmethod ver/get-sorted-versions :git-tag-and-sha [dep _options] - (or (some->> (get-in dep [:extra :url]) - (u.git/tags-by-ls-remote) - (filter (comp u.ver/sem-ver? - u.ver/remove-qualifiers - u.ver/normalize-version)) - (sort (fn [& args] - (apply version/version-compare - (map u.ver/normalize-version args)))) - (reverse)) - [])) + (try + (or (some->> (get-in dep [:extra :url]) + (u.git/tags-by-ls-remote) + (filter (comp u.ver/sem-ver? + u.ver/remove-qualifiers + u.ver/normalize-version)) + (sort (fn [& args] + (apply version/version-compare + (map u.ver/normalize-version args)))) + (reverse)) + []) + (catch Exception ex + (if (u.ex/ex-timeout? ex) + [ex] + (throw ex))))) + diff --git a/test/antq/ver/git_tag_and_sha_test.clj b/test/antq/ver/git_tag_and_sha_test.clj index c2e2baf2..67fd10af 100644 --- a/test/antq/ver/git_tag_and_sha_test.clj +++ b/test/antq/ver/git_tag_and_sha_test.clj @@ -1,6 +1,7 @@ (ns antq.ver.git-tag-and-sha-test (:require [antq.record :as r] + [antq.util.exception :as u.ex] [antq.util.git :as u.git] [antq.ver :as ver] [antq.ver.git-tag-and-sha] @@ -26,3 +27,10 @@ (t/testing "url is nil" (t/is (empty? (ver/get-sorted-versions (dep {}) {}))))) + +(t/deftest get-sorted-versions-timeout-test + (with-redefs [u.git/tags-by-ls-remote (fn [& _] (throw (u.ex/ex-timeout "test timeout")))] + (let [deps (ver/get-sorted-versions (dep {:extra {:url "https://example.com"}}) + {})] + (t/is (= 1 (count deps))) + (t/is (u.ex/ex-timeout? (first deps)))))) From d920d579ab15c2c17f4f6650da7bf1b14ff2bb7d Mon Sep 17 00:00:00 2001 From: liquidz Date: Sun, 26 Jun 2022 12:01:52 +0900 Subject: [PATCH 35/41] feat: Update to show timed out message for github tag deps --- src/antq/ver/github_tag.clj | 44 ++++++++++++++++++------------- test/antq/ver/github_tag_test.clj | 13 +++++++++ 2 files changed, 39 insertions(+), 18 deletions(-) diff --git a/src/antq/ver/github_tag.clj b/src/antq/ver/github_tag.clj index f74e74d2..47779a39 100644 --- a/src/antq/ver/github_tag.clj +++ b/src/antq/ver/github_tag.clj @@ -3,6 +3,7 @@ [antq.constant :as const] [antq.log :as log] [antq.util.async :as u.async] + [antq.util.exception :as u.ex] [antq.util.git :as u.git] [antq.util.ver :as u.ver] [antq.ver :as ver] @@ -60,8 +61,10 @@ (try (get-sorted-versions-by-ls-remote dep) (catch Exception ex - (log/error (str "Failed to fetch versions from GitHub: " - (.getMessage ex)))))) + (if (u.ex/ex-timeout? ex) + [ex] + (log/error (str "Failed to fetch versions from GitHub: " + (.getMessage ex))))))) (defmethod ver/get-sorted-versions :github-tag [dep _options] @@ -86,19 +89,24 @@ (defmethod ver/latest? :github-tag [dep] - (let [current (some-> dep :version u.ver/normalize-version version/version->seq) - latest (some-> dep :latest-version u.ver/normalize-version version/version->seq)] - (try - (when (and current latest) - (case (count (first current)) - 1 (nth-newer? current latest 0) - 2 (and (nth-newer? current latest 0) - (nth-newer? current latest 1)) - (<= 0 (version/version-seq-compare current latest)))) - (catch Throwable e - (log/error (format "Error determining latest version for GitHub dep %s (current: %s, latest: %s): %s" - (pr-str (:name dep)) - (pr-str (:version dep)) - (pr-str (:latest-version dep)) - (.getMessage e))) - true)))) + (let [current (some-> dep :version) + latest (some-> dep :latest-version)] + (if (and (string? current) + (string? latest)) + (let [current (-> current (u.ver/normalize-version) (version/version->seq)) + latest (-> latest (u.ver/normalize-version) (version/version->seq))] + (try + (when (and current latest) + (case (count (first current)) + 1 (nth-newer? current latest 0) + 2 (and (nth-newer? current latest 0) + (nth-newer? current latest 1)) + (<= 0 (version/version-seq-compare current latest)))) + (catch Throwable e + (log/error (format "Error determining latest version for GitHub dep %s (current: %s, latest: %s): %s" + (pr-str (:name dep)) + (pr-str (:version dep)) + (pr-str (:latest-version dep)) + (.getMessage e))) + true))) + false))) diff --git a/test/antq/ver/github_tag_test.clj b/test/antq/ver/github_tag_test.clj index 5a47cdef..890daa1e 100644 --- a/test/antq/ver/github_tag_test.clj +++ b/test/antq/ver/github_tag_test.clj @@ -1,6 +1,7 @@ (ns antq.ver.github-tag-test (:require [antq.record :as r] + [antq.util.exception :as u.ex] [antq.util.git :as u.git] [antq.ver :as ver] [antq.ver.github-tag :as sut] @@ -83,6 +84,15 @@ (t/is (true? @api-errored)) (t/is (true? @(deref #'sut/failed-to-fetch-from-api))))))) +(t/deftest get-sorted-versions-timeout-test + (reset! @#'sut/failed-to-fetch-from-api true) + (with-redefs [sut/get-sorted-versions-by-ls-remote #'sut/get-sorted-versions-by-ls-remote* + u.git/tags-by-ls-remote (fn [& _] (throw (u.ex/ex-timeout "test timeout")))] + + (let [deps (get-sorted-versions {:name "foo/bar"})] + (t/is (= 1 (count deps))) + (t/is (u.ex/ex-timeout? (first deps)))))) + (defn- latest? [m] (ver/latest? (dep m))) @@ -115,3 +125,6 @@ ;; if version tag is unparseable, just log an error and return true. true "v2.1.0" "v.2.x" true "v.2.x" "v2.1.0")) + +(t/deftest latest?-timeout-test + (t/is (false? (latest? {:version "1" :latest-version (u.ex/ex-timeout "dummy")})))) From 2a4c54041413fb1fed2beca91b994ca75d928da7 Mon Sep 17 00:00:00 2001 From: liquidz Date: Fri, 1 Jul 2022 05:42:34 +0900 Subject: [PATCH 36/41] fix: Fix to handle XmlPullParserException for read-pom cf. #166 --- src/antq/util/maven.clj | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/antq/util/maven.clj b/src/antq/util/maven.clj index d5c82c75..dc519141 100644 --- a/src/antq/util/maven.clj +++ b/src/antq/util/maven.clj @@ -146,6 +146,10 @@ (log/warning (str "Fetching pom from " url " failed because it timed out, retrying")) (throw e))) (catch java.io.IOException e + (log/warning (str "Fetching pom from " url " failed because of the following error: " (.getMessage e)))) + (catch org.codehaus.plexus.util.xml.pull.XmlPullParserException e + ;; e.g. This exception is thrown by reading the following pom.xml + ;; https://repo1.maven.org/maven2/jakarta/mail/jakarta.mail-api/2.0.1/jakarta.mail-api-2.0.1.pom (log/warning (str "Fetching pom from " url " failed because of the following error: " (.getMessage e))))) (recur (inc i))))))) From 5cf2ccbff23b72514383dd6b7efbdd1eb0ecb982 Mon Sep 17 00:00:00 2001 From: liquidz Date: Fri, 1 Jul 2022 06:13:49 +0900 Subject: [PATCH 37/41] chore: Bump minor version --- build.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.clj b/build.clj index 8babc322..0e60f9b7 100644 --- a/build.clj +++ b/build.clj @@ -4,7 +4,7 @@ (def ^:private config {:lib 'com.github.liquidz/antq - :version "1.7.{{git/commit-count}}" + :version "1.8.{{git/commit-count}}" :description "Point out your outdated dependencies" :main 'antq.core :documents [{:file "CHANGELOG.adoc" From 7f17c2c8cc80c0c1f013ab08f80ecf21d17cd212 Mon Sep 17 00:00:00 2001 From: liquidz Date: Fri, 1 Jul 2022 06:14:04 +0900 Subject: [PATCH 38/41] docs: Update CHANGELOG --- CHANGELOG.adoc | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/CHANGELOG.adoc b/CHANGELOG.adoc index 239ec19e..213835b4 100644 --- a/CHANGELOG.adoc +++ b/CHANGELOG.adoc @@ -1,6 +1,23 @@ All notable changes to this project will be documented in this file. This change log follows the conventions of http://keepachangelog.com/[keepachangelog.com]. == Unreleased (dev) +// {{{ +=== Added +* https://github.com/liquidz/antq/pull/157[#157]: Added progress bar to `table` reporter. +* https://github.com/liquidz/antq/pull/158[#158]: Added environmental variables to customize timeout milli secs. +** See details link:./doc/timeout.adoc[here]. + +=== Changed +* https://github.com/liquidz/antq/pull/158[#158]: Applied timeouts to the following operations. +** Fetching data from Maven repositories +** Fetching data from GitHub API +** Fetching data from git repositories +* Bumped kaocha to 1.68.1059. +* Bumped build.edn to 0.4.107. + +=== Fixed +* https://github.com/liquidz/antq/pull/166[#166]: Fixed to catch `XmlPullParserExceptio`. +// }}} == 1.7.804 (2022-06-11) // {{{ From c7bd00260dfbd044be8baa67d39bb4a389c29957 Mon Sep 17 00:00:00 2001 From: liquidz Date: Sat, 2 Jul 2022 06:00:57 +0900 Subject: [PATCH 39/41] deps: Bump deep-diff2 to 2.3.127 --- deps.edn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps.edn b/deps.edn index 860b2971..518dca6a 100644 --- a/deps.edn +++ b/deps.edn @@ -27,7 +27,7 @@ {:extra-paths ["test" "test/resources"] :extra-deps {lambdaisland/kaocha {:mvn/version "1.68.1059"} - lambdaisland/deep-diff2 {:mvn/version "2.2.124"}} + lambdaisland/deep-diff2 {:mvn/version "2.3.127"}} :jvm-opts ["-Dclojure.core.async.go-checking=true"]} :nop From 0844fdc0054c64a7553ff8b8c41330dd14559c9d Mon Sep 17 00:00:00 2001 From: liquidz Date: Sat, 2 Jul 2022 06:01:18 +0900 Subject: [PATCH 40/41] deps: Bump build.edn to 0.5.116 --- deps.edn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps.edn b/deps.edn index 518dca6a..2bf9d8de 100644 --- a/deps.edn +++ b/deps.edn @@ -44,7 +44,7 @@ :main-opts ["-m" "cloverage.coverage" "--ns-exclude-regex" "leiningen.antq"]} :build - {:deps {com.github.liquidz/build.edn {:git/tag "0.4.107" :git/sha "cf69a2a"}} + {:deps {com.github.liquidz/build.edn {:git/tag "0.5.116" :git/sha "f633fa3"}} :ns-default build} ;; -X From 50acc9d8d6ac9aba6e811ab7173801616e1ddb04 Mon Sep 17 00:00:00 2001 From: liquidz Date: Sat, 2 Jul 2022 06:03:11 +0900 Subject: [PATCH 41/41] docs: Update CHANGELOG --- CHANGELOG.adoc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.adoc b/CHANGELOG.adoc index 213835b4..a027c354 100644 --- a/CHANGELOG.adoc +++ b/CHANGELOG.adoc @@ -13,7 +13,8 @@ All notable changes to this project will be documented in this file. This change ** Fetching data from GitHub API ** Fetching data from git repositories * Bumped kaocha to 1.68.1059. -* Bumped build.edn to 0.4.107. +* Bumped deep-diff2 to 2.3.127. +* Bumped build.edn to 0.5.116. === Fixed * https://github.com/liquidz/antq/pull/166[#166]: Fixed to catch `XmlPullParserExceptio`.