From dc3a76e9e6ad45747b869ac53ddb142af7773354 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sun, 24 Mar 2024 16:23:46 +0000 Subject: [PATCH 01/10] updated linters --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9171d6e..ffa2f17 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,12 +8,12 @@ repos: - id: trailing-whitespace - id: mixed-line-ending - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.2.2 + rev: v0.3.4 hooks: - id: ruff-format - id: ruff - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.8.0 + rev: v1.9.0 hooks: # note: mypy runs in an isolated environment and so has no access to third party packages - id: mypy From 149bf619e8066ed6453cd957330e5e4b06377ee2 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sun, 24 Mar 2024 16:26:26 +0000 Subject: [PATCH 02/10] updated maturin version --- tests/maturin | 2 +- tests/package_resolver/Cargo.lock | 238 +++++++++--------- tests/requirements.txt | 4 +- tests/resolved.json | 9 +- .../test_import_hook/test_project_importer.py | 2 - 5 files changed, 125 insertions(+), 130 deletions(-) diff --git a/tests/maturin b/tests/maturin index c6932cd..7d711f0 160000 --- a/tests/maturin +++ b/tests/maturin @@ -1 +1 @@ -Subproject commit c6932cd40c0869d85028841ef8e717cb19a6de38 +Subproject commit 7d711f0c4a7c052608dc2e16d5c6721b9666d076 diff --git a/tests/package_resolver/Cargo.lock b/tests/package_resolver/Cargo.lock index 44f48d6..6c82441 100644 --- a/tests/package_resolver/Cargo.lock +++ b/tests/package_resolver/Cargo.lock @@ -85,9 +85,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.79" +version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca" +checksum = "0952808a6c2afd1aa8947271f3a60f1a6763c7b912d210184c5149b5cf147247" [[package]] name = "autocfg" @@ -200,9 +200,9 @@ dependencies = [ [[package]] name = "cargo-config2" -version = "0.1.17" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90811f7e462b86622e20549c1c43bb3861d1bf011a08c2ceceaeb51fdfc27402" +checksum = "3108787cb889089c29b110a3a7165c7c942f081c91d46c477818c33202b2a9cb" dependencies = [ "home", "serde", @@ -231,9 +231,9 @@ dependencies = [ [[package]] name = "cargo-xwin" -version = "0.16.3" +version = "0.16.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6afd4dab5fb38f9ec394ff7cd2aab95e5ddf581448afb9c42349ff849daed858" +checksum = "5e6c3dd7f20fdd197397532ac882e918cfe1d56f262a97ded7460a50e031e06b" dependencies = [ "anyhow", "cargo-config2", @@ -308,12 +308,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.83" +version = "1.0.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" -dependencies = [ - "libc", -] +checksum = "8cd6604a82acf3039f1144f54b8eb34e91ffba622051189e71b781822d5ee1f5" [[package]] name = "cfb" @@ -392,20 +389,9 @@ checksum = "183495371ea78d4c9ff638bfc6497d46fed2396e4f9c50aebc1278a4a9919a3d" dependencies = [ "clap", "clap_complete", - "clap_complete_fig", "clap_complete_nushell", ] -[[package]] -name = "clap_complete_fig" -version = "4.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87e571d70e22ec91d34e1c5317c8308035a2280d925167646bf094fc5de1737c" -dependencies = [ - "clap", - "clap_complete", -] - [[package]] name = "clap_complete_nushell" version = "0.1.11" @@ -546,6 +532,17 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "derivative" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "dialoguer" version = "0.11.0" @@ -644,7 +641,7 @@ version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d4c93f393add03d72bc10dd3dea43a1610ecb29e0c0a6459c70b53b82931adf" dependencies = [ - "goblin 0.8.0", + "goblin", ] [[package]] @@ -733,17 +730,6 @@ dependencies = [ "regex-syntax 0.8.2", ] -[[package]] -name = "goblin" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f27c1b4369c2cd341b5de549380158b105a04c331be5db9110eef7b6d2742134" -dependencies = [ - "log", - "plain", - "scroll 0.11.0", -] - [[package]] name = "goblin" version = "0.8.0" @@ -752,7 +738,7 @@ checksum = "bb07a4ffed2093b118a525b1d8f5204ae274faed5604537caf7135d0f18d9887" dependencies = [ "log", "plain", - "scroll 0.12.0", + "scroll", ] [[package]] @@ -824,9 +810,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.1.0" +version = "2.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" +checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", "hashbrown 0.14.3", @@ -866,9 +852,9 @@ dependencies = [ [[package]] name = "itertools" -version = "0.12.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25db6b064527c5d482d0423354fcd07a89a2dfe07b67892e62411946db7f07b0" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" dependencies = [ "either", ] @@ -893,7 +879,7 @@ checksum = "f88a93876d2485ede9c97d698c164cf5c024491908483964a998faae9705dea6" dependencies = [ "fs-err", "glob", - "goblin 0.8.0", + "goblin", ] [[package]] @@ -963,7 +949,7 @@ dependencies = [ [[package]] name = "maturin" -version = "1.4.0" +version = "1.5.1" dependencies = [ "anyhow", "base64 0.21.7", @@ -986,10 +972,10 @@ dependencies = [ "flate2", "fs-err", "glob", - "goblin 0.7.1", + "goblin", "ignore", - "indexmap 2.1.0", - "itertools 0.12.0", + "indexmap 2.2.6", + "itertools 0.12.1", "lddtree", "minijinja", "multipart", @@ -1015,10 +1001,11 @@ dependencies = [ "textwrap", "thiserror", "time", - "toml 0.8.8", + "toml 0.8.12", "toml_edit", "tracing", "tracing-subscriber", + "unicode-xid", "ureq", "url", "wild", @@ -1196,22 +1183,24 @@ checksum = "1e91099d4268b0e11973f036e885d652fb0b21fedcf69738c627f94db6a44f42" [[package]] name = "pep440_rs" -version = "0.3.12" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "887f66cc62717ea72caac4f1eb4e6f392224da3ffff3f40ec13ab427802746d6" +checksum = "15efd4d885c29126cc93e12af3087896e2518bd5ca0fb328c19c4ef9cecfa8be" dependencies = [ - "lazy_static", - "regex", + "once_cell", "serde", + "tracing", "unicode-width", + "unscanny", ] [[package]] name = "pep508_rs" -version = "0.2.3" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4516b53d9ea6112ebb38b4af08d5707d30b994fb7f98ff133c5dcf7ed8fa854" +checksum = "1455babf8edd3eedcdfcb39700e455a4bb189e71b4f1fa0eacc9b244cc5a55e6" dependencies = [ + "derivative", "once_cell", "pep440_rs", "regex", @@ -1220,6 +1209,7 @@ dependencies = [ "tracing", "unicode-width", "url", + "urlencoding", ] [[package]] @@ -1294,15 +1284,15 @@ dependencies = [ [[package]] name = "pyproject-toml" -version = "0.8.1" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46d4a5e69187f23a29f8aa0ea57491d104ba541bc55f76552c2a74962aa20e04" +checksum = "3b80f889b6d413c3f8963a2c7db03f95dd6e1d85e1074137cb2013ea2faa8898" dependencies = [ - "indexmap 2.1.0", + "indexmap 2.2.6", "pep440_rs", "pep508_rs", "serde", - "toml 0.8.8", + "toml 0.8.12", ] [[package]] @@ -1507,21 +1497,23 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.10" +version = "0.22.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" +checksum = "e87c9956bd9807afa1f77e0f7594af32566e830e088a5576d27c5b6f30f49d41" dependencies = [ "log", "ring", + "rustls-pki-types", "rustls-webpki", - "sct", + "subtle", + "zeroize", ] [[package]] name = "rustls-pemfile" -version = "2.0.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35e4980fa29e4c4b212ffb3db068a564cbf560e51d3944b7c88bd8bf5bec64f4" +checksum = "f48172685e6ff52a556baa527774f61fcaa884f59daf3375c62a3f1cd2549dab" dependencies = [ "base64 0.21.7", "rustls-pki-types", @@ -1529,17 +1521,18 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.1.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e9d979b3ce68192e42760c7810125eb6cf2ea10efae545a156063e61f314e2a" +checksum = "868e20fada228fefaf6b652e00cc73623d54f8171e7352c18bb281571f2d92da" [[package]] name = "rustls-webpki" -version = "0.101.7" +version = "0.102.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +checksum = "faaa0a62740bedb9b2ef5afa303da42764c012f743917351dc9a237ea1663610" dependencies = [ "ring", + "rustls-pki-types", "untrusted", ] @@ -1564,33 +1557,13 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "scroll" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04c565b551bafbef4157586fa379538366e4385d42082f255bfd96e4fe8519da" -dependencies = [ - "scroll_derive 0.11.1", -] - [[package]] name = "scroll" version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ab8598aa408498679922eff7fa985c25d58a90771bd6be794434c5277eab1a6" dependencies = [ - "scroll_derive 0.12.0", -] - -[[package]] -name = "scroll_derive" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1db149f81d46d2deba7cd3c50772474707729550221e69588478ebf9ada425ae" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.48", + "scroll_derive", ] [[package]] @@ -1604,39 +1577,29 @@ dependencies = [ "syn 2.0.48", ] -[[package]] -name = "sct" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "semver" -version = "1.0.21" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97ed7a9823b74f99c7742f5336af7be5ecd3eeafcb1507d1fa93347b1d589b0" +checksum = "92d43fe69e652f3df9bdc2b85b2854a0825b86e4fb76bc44d945137d053639ca" dependencies = [ "serde", ] [[package]] name = "serde" -version = "1.0.195" +version = "1.0.197" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02" +checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.195" +version = "1.0.197" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c" +checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" dependencies = [ "proc-macro2", "quote", @@ -1645,9 +1608,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.111" +version = "1.0.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "176e46fa42316f18edd598015a5166857fc835ec732f5215eac6b7bdbf0a84f4" +checksum = "c5f09b1bd632ef549eaa9f60a1f8de742bdbc698e6cee2095fc84dde5f549ae0" dependencies = [ "itoa", "ryu", @@ -1749,6 +1712,12 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "subtle" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" + [[package]] name = "syn" version = "1.0.109" @@ -1784,9 +1753,9 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.12.13" +version = "0.12.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69758bda2e78f098e4ccb393021a0963bb3442eac05f135c30f61b7370bbafae" +checksum = "e1fc403891a21bcfb7c37834ba66a547a8f402146eba7265b5a6d88059c9ff2f" [[package]] name = "tempfile" @@ -1822,9 +1791,9 @@ dependencies = [ [[package]] name = "textwrap" -version = "0.16.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" +checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" dependencies = [ "smawk", "unicode-linebreak", @@ -1833,18 +1802,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.56" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad" +checksum = "03468839009160513471e86a034bb2c5c0e4baae3b43f79ffc55c4a5427b3297" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.56" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" +checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7" dependencies = [ "proc-macro2", "quote", @@ -1905,9 +1874,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.8" +version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1a195ec8c9da26928f773888e0742ca3ca1040c6cd859c919c9f59c1954ab35" +checksum = "e9dd1545e8208b4a5af1aa9bbd0b4cf7e9ea08fabc5d0a5c67fcaafa17433aa3" dependencies = [ "serde", "serde_spanned", @@ -1926,11 +1895,11 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.21.0" +version = "0.22.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d34d383cd00a163b4a5b85053df514d45bc330f6de7737edfe0a93311d1eaa03" +checksum = "8e40bb779c5187258fd7aad0eb68cb8706a0a81fa712fbea808ab43c4b8374c4" dependencies = [ - "indexmap 2.1.0", + "indexmap 2.2.6", "serde", "serde_spanned", "toml_datetime", @@ -1943,7 +1912,6 @@ version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ - "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -2071,6 +2039,18 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" +[[package]] +name = "unicode-xid" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" + +[[package]] +name = "unscanny" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9df2af067a7953e9c3831320f35c1cc0600c30d44d9f7a12b01db1cd88d6b47" + [[package]] name = "untrusted" version = "0.9.0" @@ -2079,15 +2059,16 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "ureq" -version = "2.9.1" +version = "2.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8cdd25c339e200129fe4de81451814e5228c9b771d57378817d6117cc2b3f97" +checksum = "11f214ce18d8b2cbe84ed3aa6486ed3f5b285cf8d8fbdbce9f3f767a724adc35" dependencies = [ "base64 0.21.7", "flate2", "log", "once_cell", "rustls", + "rustls-pki-types", "rustls-webpki", "serde", "serde_json", @@ -2108,6 +2089,12 @@ dependencies = [ "serde", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf8parse" version = "0.2.1" @@ -2160,9 +2147,12 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "webpki-roots" -version = "0.25.3" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1778a42e8b3b90bff8d0f5032bf22250792889a5cdc752aa0020c84abe3aaf10" +checksum = "b3de34ae270483955a94f4b21bdaaeb83d508bb84a01435f393818edb0012009" +dependencies = [ + "rustls-pki-types", +] [[package]] name = "which" @@ -2364,9 +2354,9 @@ checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" [[package]] name = "winnow" -version = "0.5.34" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7cf47b659b318dccbd69cc4797a39ae128f533dce7902a1096044d1967b9c16" +checksum = "dffa400e67ed5a4dd237983829e66475f0a4a26938c4b04c21baede6262215b8" dependencies = [ "memchr", ] @@ -2406,7 +2396,7 @@ dependencies = [ "serde_json", "sha2", "tempfile", - "toml 0.8.8", + "toml 0.8.12", "tracing", "tracing-subscriber", "twox-hash", diff --git a/tests/requirements.txt b/tests/requirements.txt index a3110bd..9d0879b 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,9 +1,9 @@ -e .. virtualenv -maturin==1.4.0 +maturin==1.5.0 pytest junit2html -uniffi-bindgen==0.25.0 +uniffi-bindgen==0.26.0 cffi # required for pyo3-mixed diff --git a/tests/resolved.json b/tests/resolved.json index 000ac89..e6d9137 100644 --- a/tests/resolved.json +++ b/tests/resolved.json @@ -1,5 +1,5 @@ { - "commit": "c6932cd40c0869d85028841ef8e717cb19a6de38", + "commit": "7d711f0c4a7c052608dc2e16d5c6721b9666d076", "crates": { "cffi-mixed": { "cargo_manifest_path": "Cargo.toml", @@ -92,6 +92,13 @@ "python_dir": ".", "python_module": "pyo3_mixed_submodule" }, + "pyo3-mixed-with-path-dep": { + "cargo_manifest_path": "Cargo.toml", + "extension_module_dir": "pyo3_mixed_with_path_dep", + "module_full_name": "pyo3_mixed_with_path_dep", + "python_dir": ".", + "python_module": "pyo3_mixed_with_path_dep" + }, "pyo3-mixed-workspace": { "cargo_manifest_path": "rust/python/pyo3-mixed-workspace-py/Cargo.toml", "extension_module_dir": "src/pyo3_mixed_workspace", diff --git a/tests/test_import_hook/test_project_importer.py b/tests/test_import_hook/test_project_importer.py index 2435095..df94ab3 100644 --- a/tests/test_import_hook/test_project_importer.py +++ b/tests/test_import_hook/test_project_importer.py @@ -436,8 +436,6 @@ def test_import_multiple_projects(workspace: Path) -> None: assert _is_editable_installed_correctly("pyo3-pure", pure_dir, False) -# TODO(matt): remove skip -@pytest.mark.skip(reason="this crate is added in https://github.com/PyO3/maturin/pull/1958 which is not yet merged") def test_rebuild_on_change_to_path_dependency(workspace: Path) -> None: """This test ensures that the imported project is rebuilt if any of its path dependencies are edited. From a190ab38d43e01501fe23f88369e36e67a4facfa Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sun, 24 Mar 2024 16:29:27 +0000 Subject: [PATCH 03/10] raised minimum maturin version to the first release that creates editable installs with maturin develop --- src/maturin_import_hook/_building.py | 29 ------------------- src/maturin_import_hook/project_importer.py | 4 +-- src/maturin_import_hook/rust_file_importer.py | 2 +- .../test_import_hook/test_project_importer.py | 3 -- 4 files changed, 2 insertions(+), 36 deletions(-) diff --git a/src/maturin_import_hook/_building.py b/src/maturin_import_hook/_building.py index 32ee015..3a67745 100644 --- a/src/maturin_import_hook/_building.py +++ b/src/maturin_import_hook/_building.py @@ -5,7 +5,6 @@ import platform import re import shutil -import site import subprocess import sys import zipfile @@ -186,34 +185,6 @@ def develop_build_project( return output -# TODO(matt): remove once a maturin release can create editable installs and raise minimum supported version -def fix_direct_url(project_dir: Path, package_name: str) -> None: - """Seemingly due to a bug, installing with `pip install -e` will write the correct entry into `direct_url.json` to - point at the project directory, but calling `maturin develop` does not currently write this value correctly. - """ - logger.debug("fixing direct_url for %s", package_name) - for path in site.getsitepackages(): - dist_info = next(Path(path).glob(f"{package_name}-*.dist-info"), None) - if dist_info is None: - continue - direct_url_path = dist_info / "direct_url.json" - try: - with direct_url_path.open() as f: - direct_url = json.load(f) - except OSError: - continue - url = project_dir.as_uri() - if direct_url.get("url") != url: - logger.debug("fixing direct_url.json for package %s", package_name) - logger.debug('"%s" -> "%s"', direct_url.get("url"), url) - direct_url = {"dir_info": {"editable": True}, "url": url} - try: - with direct_url_path.open("w") as f: - json.dump(direct_url, f) - except OSError: - return - - def find_maturin(lower_version: Tuple[int, int, int], upper_version: Tuple[int, int, int]) -> Path: logger.debug("searching for maturin") maturin_path_str = shutil.which("maturin") diff --git a/src/maturin_import_hook/project_importer.py b/src/maturin_import_hook/project_importer.py index a145397..d81de24 100644 --- a/src/maturin_import_hook/project_importer.py +++ b/src/maturin_import_hook/project_importer.py @@ -24,7 +24,6 @@ LockedBuildCache, develop_build_project, find_maturin, - fix_direct_url, get_installation_freshness, get_installation_mtime, maturin_output_has_warnings, @@ -99,7 +98,7 @@ def get_settings(self, module_path: str, source_path: Path) -> MaturinSettings: def find_maturin(self) -> Path: """this method can be overridden to specify an alternative maturin binary to use""" if self._maturin_path is None: - self._maturin_path = find_maturin((1, 4, 0), (2, 0, 0)) + self._maturin_path = find_maturin((1, 5, 0), (2, 0, 0)) return self._maturin_path def find_spec( @@ -250,7 +249,6 @@ def _rebuild_project( logger.info('building "%s"', package_name) start = time.perf_counter() maturin_output = develop_build_project(self.find_maturin(), resolved.cargo_manifest_path, settings) - fix_direct_url(project_dir, package_name) logger.debug( 'compiled project "%s" in %.3fs', package_name, diff --git a/src/maturin_import_hook/rust_file_importer.py b/src/maturin_import_hook/rust_file_importer.py index 9379c0f..83fe34b 100644 --- a/src/maturin_import_hook/rust_file_importer.py +++ b/src/maturin_import_hook/rust_file_importer.py @@ -62,7 +62,7 @@ def get_settings(self, module_path: str, source_path: Path) -> MaturinSettings: def find_maturin(self) -> Path: """this method can be overridden to specify an alternative maturin binary to use""" if self._maturin_path is None: - self._maturin_path = find_maturin((1, 4, 0), (2, 0, 0)) + self._maturin_path = find_maturin((1, 5, 0), (2, 0, 0)) return self._maturin_path def get_source_files(self, source_path: Path) -> Iterator[Path]: diff --git a/tests/test_import_hook/test_project_importer.py b/tests/test_import_hook/test_project_importer.py index df94ab3..d2a2789 100644 --- a/tests/test_import_hook/test_project_importer.py +++ b/tests/test_import_hook/test_project_importer.py @@ -12,7 +12,6 @@ from typing import Iterator, Set, Tuple import pytest -from maturin_import_hook._building import fix_direct_url from maturin_import_hook.project_importer import DefaultProjectFileSearcher, _load_dist_info from .common import ( @@ -1378,8 +1377,6 @@ def _install_editable(project_dir: Path) -> None: env = os.environ.copy() env["VIRTUAL_ENV"] = sys.exec_prefix subprocess.check_call([maturin_path, "develop"], cwd=project_dir, env=env) - # TODO(matt): remove once maturin develop creates editable installs - fix_direct_url(project_dir, with_underscores(project_dir.name)) def _install_non_editable(project_dir: Path) -> None: From 002ac5083fc1807651d244839ae78e8876585d23 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sun, 24 Mar 2024 16:37:44 +0000 Subject: [PATCH 04/10] removed resolved TODOs --- .github/workflows/test.yml | 2 +- pyproject.toml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 790a831..5b30e59 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ on: options: - all - ubuntu-latest - - macos-13 # TODO(matt): switch to 14 once this is fixed: https://github.com/actions/setup-python/issues/825 + - macos-14 - windows-latest python_version: type: choice diff --git a/pyproject.toml b/pyproject.toml index 2e402bd..1d70fbb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,6 @@ classifiers = [ [project.urls] Homepage = "https://github.com/PyO3/maturin-import-hook" -# TODO(matt): should the import hook have its own documentation Documentation = "https://maturin.rs" Repository = "https://github.com/PyO3/maturin-import-hook.git" Issues = "https://github.com/PyO3/maturin-import-hook/issues" From b37223ece22c1da946fe015bec354145af19392f Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sun, 24 Mar 2024 16:41:52 +0000 Subject: [PATCH 05/10] raised minimum python version to 3.9 --- .github/workflows/test.yml | 4 +- pyproject.toml | 6 +-- src/maturin_import_hook/__main__.py | 3 +- src/maturin_import_hook/_building.py | 15 +++--- src/maturin_import_hook/_resolve_project.py | 22 ++++---- src/maturin_import_hook/project_importer.py | 33 ++++++------ src/maturin_import_hook/rust_file_importer.py | 7 +-- src/maturin_import_hook/settings.py | 24 ++++----- tests/runner.py | 53 +------------------ tests/test_import_hook/common.py | 37 ++++++------- tests/test_import_hook/conftest.py | 2 +- .../test_import_hook/test_project_importer.py | 6 +-- .../test_rust_file_importer.py | 3 +- tests/test_import_hook/test_utilities.py | 16 ++---- 14 files changed, 86 insertions(+), 145 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5b30e59..69f6c52 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,10 +15,9 @@ on: python_version: type: choice description: Python version to test with - default: '3.11' + default: '3.12' options: - 'all' - - '3.8' - '3.9' - '3.10' - '3.11' @@ -95,7 +94,6 @@ jobs: - macos-13 - windows-latest PYTHON_VERSION: | - - '3.8' - '3.9' - '3.10' - '3.11' diff --git a/pyproject.toml b/pyproject.toml index 1d70fbb..ce62ad7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ authors = [ ] readme = "README.md" version = "0.1.0" -requires-python = ">=3.8" +requires-python = ">=3.9" dependencies = [ "filelock", "tomli>=1.1.0 ; python_version<'3.11'" @@ -33,7 +33,7 @@ where = ["src"] [tool.ruff] line-length = 120 -target-version = "py38" +target-version = "py39" [tool.ruff.format] preview = true @@ -69,7 +69,7 @@ ignore = [ ] [tool.mypy] -python_version = "3.8" +python_version = "3.9" strict = true allow_redefinition = true exclude = [ diff --git a/src/maturin_import_hook/__main__.py b/src/maturin_import_hook/__main__.py index f48a311..3f8fb31 100644 --- a/src/maturin_import_hook/__main__.py +++ b/src/maturin_import_hook/__main__.py @@ -5,7 +5,6 @@ import shutil import subprocess from pathlib import Path -from typing import Dict from maturin_import_hook._building import get_default_build_dir from maturin_import_hook._site import ( @@ -119,7 +118,7 @@ def _dir_size_mib(dir_path: Path) -> str: return f"{cache_size / (1024 * 1024):.2f} MiB" -def _print_info(info: Dict[str, object], format_name: str) -> None: +def _print_info(info: dict[str, object], format_name: str) -> None: if format_name == "text": for k, v in info.items(): print(f"{k}: {v}") diff --git a/src/maturin_import_hook/_building.py b/src/maturin_import_hook/_building.py index 3a67745..14f360a 100644 --- a/src/maturin_import_hook/_building.py +++ b/src/maturin_import_hook/_building.py @@ -8,11 +8,12 @@ import subprocess import sys import zipfile +from collections.abc import Generator, Iterable from contextlib import contextmanager from dataclasses import dataclass from operator import itemgetter from pathlib import Path -from typing import Any, Dict, Generator, Iterable, List, Optional, Tuple +from typing import Any, Optional import filelock @@ -30,10 +31,10 @@ class BuildStatus: build_mtime: float source_path: Path - maturin_args: List[str] + maturin_args: list[str] maturin_output: str - def to_json(self) -> Dict[str, Any]: + def to_json(self) -> dict[str, Any]: return { "build_mtime": self.build_mtime, "source_path": str(self.source_path), @@ -42,7 +43,7 @@ def to_json(self) -> Dict[str, Any]: } @staticmethod - def from_json(json_data: Dict[Any, Any]) -> Optional["BuildStatus"]: + def from_json(json_data: dict[Any, Any]) -> Optional["BuildStatus"]: try: return BuildStatus( build_mtime=json_data["build_mtime"], @@ -185,7 +186,7 @@ def develop_build_project( return output -def find_maturin(lower_version: Tuple[int, int, int], upper_version: Tuple[int, int, int]) -> Path: +def find_maturin(lower_version: tuple[int, int, int], upper_version: tuple[int, int, int]) -> Path: logger.debug("searching for maturin") maturin_path_str = shutil.which("maturin") if maturin_path_str is None: @@ -202,7 +203,7 @@ def find_maturin(lower_version: Tuple[int, int, int], upper_version: Tuple[int, raise MaturinError(msg) -def get_maturin_version(maturin_path: Path) -> Tuple[int, int, int]: +def get_maturin_version(maturin_path: Path) -> tuple[int, int, int]: success, output = run_maturin(maturin_path, ["--version"]) if not success: msg = f'running "{maturin_path} --version" failed' @@ -214,7 +215,7 @@ def get_maturin_version(maturin_path: Path) -> Tuple[int, int, int]: return int(match.group(1)), int(match.group(2)), int(match.group(3)) -def run_maturin(maturin_path: Path, args: List[str]) -> Tuple[bool, str]: +def run_maturin(maturin_path: Path, args: list[str]) -> tuple[bool, str]: command = [str(maturin_path), *args] if logger.isEnabledFor(logging.DEBUG): logger.debug("running command: %s", subprocess.list2cmdline(command)) diff --git a/src/maturin_import_hook/_resolve_project.py b/src/maturin_import_hook/_resolve_project.py index b6c5fe9..254cd1f 100644 --- a/src/maturin_import_hook/_resolve_project.py +++ b/src/maturin_import_hook/_resolve_project.py @@ -1,7 +1,7 @@ import itertools from dataclasses import dataclass from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple, Type, TypeVar +from typing import Any, Optional, TypeVar from maturin_import_hook._logging import logger @@ -15,7 +15,7 @@ class _TomlFile: - def __init__(self, path: Path, data: Dict[Any, Any]) -> None: + def __init__(self, path: Path, data: dict[Any, Any]) -> None: self.path = path self.data = data @@ -25,11 +25,11 @@ def load(path: Path) -> "_TomlFile": data = tomllib.load(f) return _TomlFile(path, data) - def get_value_or_default(self, keys: List[str], required_type: Type[_T], default: _T) -> _T: + def get_value_or_default(self, keys: list[str], required_type: type[_T], default: _T) -> _T: value = self.get_value(keys, required_type) return default if value is None else value - def get_value(self, keys: List[str], required_type: Type[_T]) -> Optional[_T]: + def get_value(self, keys: list[str], required_type: type[_T]) -> Optional[_T]: assert keys current_data: Any = self.data num_keys = len(keys) @@ -78,7 +78,7 @@ def is_maybe_maturin_project(project_dir: Path) -> bool: class ProjectResolver: def __init__(self) -> None: - self._resolved_project_cache: Dict[Path, Optional[MaturinProject]] = {} + self._resolved_project_cache: dict[Path, Optional[MaturinProject]] = {} def resolve(self, project_dir: Path) -> Optional["MaturinProject"]: if project_dir not in self._resolved_project_cache: @@ -106,9 +106,9 @@ class MaturinProject: # the location that the compiled extension module is written to when installed in editable/unpacked mode extension_module_dir: Optional[Path] # path dependencies listed in the Cargo.toml of the main project - immediate_path_dependencies: List[Path] + immediate_path_dependencies: list[Path] # all path dependencies including transitive dependencies - _all_path_dependencies: Optional[List[Path]] = None + _all_path_dependencies: Optional[list[Path]] = None @property def package_name(self) -> str: @@ -124,13 +124,13 @@ def is_mixed(self) -> bool: return self.extension_module_dir is not None @property - def all_path_dependencies(self) -> List[Path]: + def all_path_dependencies(self) -> list[Path]: if self._all_path_dependencies is None: self._all_path_dependencies = _find_all_path_dependencies(self.immediate_path_dependencies) return self._all_path_dependencies -def _find_all_path_dependencies(immediate_path_dependencies: List[Path]) -> List[Path]: +def _find_all_path_dependencies(immediate_path_dependencies: list[Path]) -> list[Path]: if not immediate_path_dependencies: return [] all_path_dependencies = set() @@ -194,7 +194,7 @@ def _resolve_project(project_dir: Path) -> MaturinProject: ) -def _resolve_rust_module(python_dir: Path, module_name: str) -> Tuple[Path, Path, str]: +def _resolve_rust_module(python_dir: Path, module_name: str) -> tuple[Path, Path, str]: """This follows the same logic as project_layout.rs (ProjectLayout::determine). rust_module is the directory that the extension library gets written to when the package is @@ -234,7 +234,7 @@ def _resolve_module_name(pyproject: _TomlFile, cargo: _TomlFile) -> Optional[str return cargo.get_value(["package", "name"], str) -def _get_immediate_path_dependencies(manifest_dir_path: Path, cargo: _TomlFile) -> List[Path]: +def _get_immediate_path_dependencies(manifest_dir_path: Path, cargo: _TomlFile) -> list[Path]: path_dependencies = [] for dependency in cargo.get_value_or_default(["dependencies"], dict, {}).values(): if isinstance(dependency, dict): diff --git a/src/maturin_import_hook/project_importer.py b/src/maturin_import_hook/project_importer.py index d81de24..acdcdb1 100644 --- a/src/maturin_import_hook/project_importer.py +++ b/src/maturin_import_hook/project_importer.py @@ -13,10 +13,11 @@ import urllib.parse import urllib.request from abc import ABC, abstractmethod +from collections.abc import Iterator, Sequence from importlib.machinery import ExtensionFileLoader, ModuleSpec, PathFinder from pathlib import Path from types import ModuleType -from typing import ClassVar, Iterator, List, Optional, Sequence, Set, Tuple, Union +from typing import ClassVar, Optional, Union from maturin_import_hook._building import ( BuildCache, @@ -53,7 +54,7 @@ class ProjectFileSearcher(ABC): def get_source_paths( self, project_dir: Path, - all_path_dependencies: List[Path], + all_path_dependencies: list[Path], installed_package_root: Path, ) -> Iterator[Path]: """find the files corresponding to the source code of the given project""" @@ -211,7 +212,7 @@ def _rebuild_project( self, package_name: str, project_dir: Path, - ) -> Tuple[Optional[ModuleSpec], bool]: + ) -> tuple[Optional[ModuleSpec], bool]: resolved = self._resolver.resolve(project_dir) if resolved is None: return None, False @@ -283,7 +284,7 @@ def _get_spec_for_up_to_date_package( resolved: MaturinProject, settings: MaturinSettings, build_cache: LockedBuildCache, - ) -> Tuple[Optional[ModuleSpec], Optional[str]]: + ) -> tuple[Optional[ModuleSpec], Optional[str]]: """Return a spec for the package if it exists and is newer than the source code that it is derived from. """ @@ -381,7 +382,7 @@ def _find_maturin_project_above(path: Path) -> Optional[Path]: def _load_dist_info( path: Path, package_name: str, *, require_project_target: bool = True -) -> Tuple[Optional[Path], bool]: +) -> tuple[Optional[Path], bool]: dist_info_path = next(path.glob(f"{package_name}-*.dist-info"), None) if dist_info_path is None: return None, False @@ -457,7 +458,7 @@ class DefaultProjectFileSearcher(ProjectFileSearcher): # - https://github.com/github/gitignore/blob/main/Rust.gitignore # - https://github.com/github/gitignore/blob/main/Python.gitignore # - https://github.com/jupyter/notebook/blob/main/.gitignore - DEFAULT_SOURCE_EXCLUDED_DIR_NAMES: ClassVar[Set[str]] = { + DEFAULT_SOURCE_EXCLUDED_DIR_NAMES: ClassVar[set[str]] = { ".cache", ".env", ".git", @@ -481,10 +482,10 @@ class DefaultProjectFileSearcher(ProjectFileSearcher): "target", "venv", } - DEFAULT_SOURCE_EXCLUDED_DIR_MARKERS: ClassVar[Set[str]] = { + DEFAULT_SOURCE_EXCLUDED_DIR_MARKERS: ClassVar[set[str]] = { "CACHEDIR.TAG", # https://bford.info/cachedir/ } - DEFAULT_SOURCE_EXCLUDED_FILE_EXTENSIONS: ClassVar[Set[str]] = { + DEFAULT_SOURCE_EXCLUDED_FILE_EXTENSIONS: ClassVar[set[str]] = { ".so", ".pyc", } @@ -492,9 +493,9 @@ class DefaultProjectFileSearcher(ProjectFileSearcher): def __init__( self, *, - source_excluded_dir_names: Optional[Set[str]] = None, - source_excluded_dir_markers: Optional[Set[str]] = None, - source_excluded_file_extensions: Optional[Set[str]] = None, + source_excluded_dir_names: Optional[set[str]] = None, + source_excluded_dir_markers: Optional[set[str]] = None, + source_excluded_file_extensions: Optional[set[str]] = None, ) -> None: """ Args: @@ -525,7 +526,7 @@ def __init__( def get_source_paths( self, project_dir: Path, - all_path_dependencies: List[Path], + all_path_dependencies: list[Path], installed_package_root: Path, ) -> Iterator[Path]: excluded_dirs = set() @@ -557,10 +558,10 @@ def get_installation_paths(self, installed_package_root: Path) -> Iterator[Path] def get_files_in_dir( self, root_path: Path, - ignore_dirs: Set[Path], - excluded_dir_names: Set[str], - excluded_dir_markers: Set[str], - excluded_file_extensions: Set[str], + ignore_dirs: set[Path], + excluded_dir_names: set[str], + excluded_dir_markers: set[str], + excluded_file_extensions: set[str], ) -> Iterator[Path]: if root_path.name in excluded_dir_names: return diff --git a/src/maturin_import_hook/rust_file_importer.py b/src/maturin_import_hook/rust_file_importer.py index 83fe34b..92e4bb0 100644 --- a/src/maturin_import_hook/rust_file_importer.py +++ b/src/maturin_import_hook/rust_file_importer.py @@ -9,10 +9,11 @@ import sys import tempfile import time +from collections.abc import Iterator, Sequence from importlib.machinery import ExtensionFileLoader, ModuleSpec from pathlib import Path from types import ModuleType -from typing import TYPE_CHECKING, Iterator, Optional, Sequence, Tuple, Union +from typing import TYPE_CHECKING, Optional, Union from maturin_import_hook._building import ( BuildCache, @@ -189,7 +190,7 @@ def _handle_reload(self, module_path: str, spec: ModuleSpec) -> ModuleSpec: def _import_rust_file( self, module_path: str, module_name: str, file_path: Path - ) -> Tuple[Optional[ModuleSpec], bool]: + ) -> tuple[Optional[ModuleSpec], bool]: logger.debug('importing rust file "%s" as "%s"', file_path, module_path) with self._build_cache.lock() as build_cache: @@ -250,7 +251,7 @@ def _get_spec_for_up_to_date_extension_module( source_path: Path, settings: MaturinSettings, build_cache: LockedBuildCache, - ) -> Tuple[Optional[ModuleSpec], Optional[str]]: + ) -> tuple[Optional[ModuleSpec], Optional[str]]: """Return a spec for the given module at the given search_dir if it exists and is newer than the source code that it is derived from. """ diff --git a/src/maturin_import_hook/settings.py b/src/maturin_import_hook/settings.py index 1812bb5..6411374 100644 --- a/src/maturin_import_hook/settings.py +++ b/src/maturin_import_hook/settings.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from typing import Dict, List, Optional, Set +from typing import Optional __all__ = [ "MaturinSettings", @@ -17,7 +17,7 @@ class MaturinSettings: quiet: bool = False jobs: Optional[int] = None profile: Optional[str] = None - features: Optional[List[str]] = None + features: Optional[list[str]] = None all_features: bool = False no_default_features: bool = False target: Optional[str] = None @@ -26,13 +26,13 @@ class MaturinSettings: frozen: bool = False locked: bool = False offline: bool = False - config: Optional[Dict[str, str]] = None - unstable_flags: Optional[List[str]] = None + config: Optional[dict[str, str]] = None + unstable_flags: Optional[list[str]] = None verbose: int = 0 - rustc_flags: Optional[List[str]] = None + rustc_flags: Optional[list[str]] = None @staticmethod - def supported_commands() -> Set[str]: + def supported_commands() -> set[str]: return {"build", "develop"} @staticmethod @@ -42,7 +42,7 @@ def default() -> "MaturinSettings": color=True, ) - def to_args(self) -> List[str]: + def to_args(self) -> list[str]: args = [] if self.release: args.append("--release") @@ -103,10 +103,10 @@ class MaturinBuildSettings(MaturinSettings): zig: bool = False @staticmethod - def supported_commands() -> Set[str]: + def supported_commands() -> set[str]: return {"build"} - def to_args(self) -> List[str]: + def to_args(self) -> list[str]: args = [] if self.skip_auditwheel: args.append("--skip-auditwheel") @@ -120,14 +120,14 @@ def to_args(self) -> List[str]: class MaturinDevelopSettings(MaturinSettings): """settings for `maturin develop`.""" - extras: Optional[List[str]] = None + extras: Optional[list[str]] = None skip_install: bool = False @staticmethod - def supported_commands() -> Set[str]: + def supported_commands() -> set[str]: return {"develop"} - def to_args(self) -> List[str]: + def to_args(self) -> list[str]: args = [] if self.extras is not None: args.append("--extras") diff --git a/tests/runner.py b/tests/runner.py index d0b4e64..c5e7bcf 100644 --- a/tests/runner.py +++ b/tests/runner.py @@ -208,55 +208,6 @@ def _notify(message: str) -> None: pass -# TODO(matt): remove when 3.9 is the minimum supported version -if sys.version_info < (3, 9): - from typing import Any - # ruff: noqa: ANN401 - - class _BooleanOptionalAction(argparse.Action): - """a copy of argparse.BooleanOptionAction. This class is only available in python 3.9+""" - - def __init__( - self, - option_strings: Any, - dest: Any, - default: Any = None, - type: Any = None, # noqa: A002 - choices: Any = None, - required: Any = False, - help: Any = None, # noqa: A002 - metavar: Any = None, - ) -> None: - _option_strings = [] - for option_string in option_strings: - _option_strings.append(option_string) - - if option_string.startswith("--"): - option_string = "--no-" + option_string[2:] # noqa: PLW2901 - _option_strings.append(option_string) - - super().__init__( - option_strings=_option_strings, - dest=dest, - nargs=0, - default=default, - type=type, - choices=choices, - required=required, - help=help, - metavar=metavar, - ) - - def __call__(self, parser: Any, namespace: Any, values: Any, option_string: Any = None) -> None: - if option_string in self.option_strings: - setattr(namespace, self.dest, not option_string.startswith("--no-")) - - def format_usage(self) -> str: - return " | ".join(self.option_strings) - - argparse.BooleanOptionalAction = _BooleanOptionalAction # type: ignore[attr-defined] - - def main() -> None: parser = argparse.ArgumentParser(description="run the import hook tests in clean virtual environments") parser.add_argument( @@ -292,13 +243,13 @@ def main() -> None: parser.add_argument( "--html-report", - action=argparse.BooleanOptionalAction, # type: ignore[attr-defined] + action=argparse.BooleanOptionalAction, default=True, help="whether to create a html report from the junit test report", ) parser.add_argument( "--notify", - action=argparse.BooleanOptionalAction, # type: ignore[attr-defined] + action=argparse.BooleanOptionalAction, default=True, help="send a notification when finished", ) diff --git a/tests/test_import_hook/common.py b/tests/test_import_hook/common.py index c60cdfe..ac4f492 100644 --- a/tests/test_import_hook/common.py +++ b/tests/test_import_hook/common.py @@ -10,11 +10,12 @@ import sys import tempfile import time +from collections.abc import Iterator from contextlib import contextmanager from dataclasses import dataclass from io import StringIO from pathlib import Path -from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, TypeVar +from typing import Any, Callable, Optional, TypeVar script_dir = Path(__file__).resolve().parent log = logging.getLogger(__name__) @@ -59,7 +60,7 @@ class ResolvedPackage: python_module: Optional[Path] @staticmethod - def from_dict(data: Dict[str, Any]) -> "ResolvedPackage": + def from_dict(data: dict[str, Any]) -> "ResolvedPackage": return ResolvedPackage( cargo_manifest_path=Path(data["cargo_manifest_path"]), extension_module_dir=map_optional(data["extension_module_dir"], Path), @@ -72,10 +73,10 @@ def to_json(self) -> str: return json.dumps({k: str(v) for k, v in dataclasses.asdict(self).items()}, indent=2, sort_keys=True) -_RESOLVED_PACKAGES: Optional[Dict[str, Optional[ResolvedPackage]]] = None +_RESOLVED_PACKAGES: Optional[dict[str, Optional[ResolvedPackage]]] = None -def resolved_packages() -> Dict[str, Optional[ResolvedPackage]]: +def resolved_packages() -> dict[str, Optional[ResolvedPackage]]: global _RESOLVED_PACKAGES if _RESOLVED_PACKAGES is None: with (script_dir / "../resolved.json").open() as f: @@ -109,7 +110,7 @@ def with_underscores(project_name: str) -> str: return project_name.replace("-", "_") -def all_usable_test_crate_names() -> List[str]: +def all_usable_test_crate_names() -> list[str]: return sorted( p.name for p in TEST_CRATES_DIR.iterdir() @@ -118,7 +119,7 @@ def all_usable_test_crate_names() -> List[str]: ) -def mixed_test_crate_names() -> List[str]: +def mixed_test_crate_names() -> list[str]: return [name for name in all_usable_test_crate_names() if "mixed" in name] @@ -129,15 +130,15 @@ def __init__(self, output: str) -> None: def run_python( - args: List[str], + args: list[str], cwd: Path, *, quiet: bool = False, expect_error: bool = False, profile: Optional[Path] = None, - env: Optional[Dict[str, Any]] = None, + env: Optional[dict[str, Any]] = None, interpreter: Optional[Path] = None, -) -> Tuple[str, float]: +) -> tuple[str, float]: start = time.perf_counter() interpreter_path = sys.executable if interpreter is None else str(interpreter) @@ -189,13 +190,13 @@ def run_python( def run_python_code( python_script: str, *, - args: Optional[List[str]] = None, + args: Optional[list[str]] = None, cwd: Optional[Path] = None, quiet: bool = False, expect_error: bool = False, - env: Optional[Dict[str, Any]] = None, + env: Optional[dict[str, Any]] = None, interpreter: Optional[Path] = None, -) -> Tuple[str, float]: +) -> tuple[str, float]: with tempfile.TemporaryDirectory("run_python_code") as tmpdir_str: tmpdir = Path(tmpdir_str) tmp_script_path = tmpdir / "script.py" @@ -257,9 +258,9 @@ class PythonProcessOutput: def run_concurrent_python( - num: int, func: Callable[..., Tuple[str, float]], args: Dict[str, Any] -) -> List[PythonProcessOutput]: - outputs: List[PythonProcessOutput] = [] + num: int, func: Callable[..., tuple[str, float]], args: dict[str, Any] +) -> list[PythonProcessOutput]: + outputs: list[PythonProcessOutput] = [] with multiprocessing.Pool(processes=num) as pool: processes = [pool.apply_async(func, kwds=args) for _ in range(num)] @@ -283,7 +284,7 @@ def run_concurrent_python( return outputs -def get_file_times(path: Path) -> Tuple[float, float]: +def get_file_times(path: Path) -> tuple[float, float]: s = path.stat() times = (s.st_atime, s.st_mtime) if platform.system() == "Windows" and platform.python_implementation() == "PyPy": @@ -295,12 +296,12 @@ def get_file_times(path: Path) -> Tuple[float, float]: return times -def set_file_times_recursive(path: Path, times: Tuple[float, float]) -> None: +def set_file_times_recursive(path: Path, times: tuple[float, float]) -> None: for p in path.rglob("*"): os.utime(p, times) -def set_file_times(path: Path, times: Tuple[float, float]) -> None: +def set_file_times(path: Path, times: tuple[float, float]) -> None: os.utime(path, times) diff --git a/tests/test_import_hook/conftest.py b/tests/test_import_hook/conftest.py index 8f5516d..8747c0e 100644 --- a/tests/test_import_hook/conftest.py +++ b/tests/test_import_hook/conftest.py @@ -1,8 +1,8 @@ import logging import shutil import sys +from collections.abc import Iterator from pathlib import Path -from typing import Iterator import pytest from maturin_import_hook import reset_logger diff --git a/tests/test_import_hook/test_project_importer.py b/tests/test_import_hook/test_project_importer.py index d2a2789..b519517 100644 --- a/tests/test_import_hook/test_project_importer.py +++ b/tests/test_import_hook/test_project_importer.py @@ -7,9 +7,9 @@ import site import subprocess import sys +from collections.abc import Iterator from pathlib import Path from textwrap import dedent -from typing import Iterator, Set, Tuple import pytest from maturin_import_hook.project_importer import DefaultProjectFileSearcher, _load_dist_info @@ -591,7 +591,7 @@ class TestReload: """ @staticmethod - def _create_reload_project(output_dir: Path, mixed: bool) -> Tuple[Path, Path]: + def _create_reload_project(output_dir: Path, mixed: bool) -> tuple[Path, Path]: project_dir = _create_project_from_blank_template("my-project", output_dir / "my-project", mixed=mixed) if mixed: init = dedent("""\ @@ -1354,7 +1354,7 @@ def _uninstall(*project_names: str) -> None: ]) -def _get_installed_package_names() -> Set[str]: +def _get_installed_package_names() -> set[str]: packages = json.loads( subprocess.check_output([ sys.executable, diff --git a/tests/test_import_hook/test_rust_file_importer.py b/tests/test_import_hook/test_rust_file_importer.py index 22fdb32..82da662 100644 --- a/tests/test_import_hook/test_rust_file_importer.py +++ b/tests/test_import_hook/test_rust_file_importer.py @@ -5,7 +5,6 @@ import sys from pathlib import Path from textwrap import dedent -from typing import Tuple import pytest @@ -605,7 +604,7 @@ def test_submodule(self, workspace: Path) -> None: class TestLogging: """test the desired messages are visible to the user in the default logging configuration.""" - def _create_clean_package(self, package_path: Path, *, reload_helper: bool = False) -> Tuple[Path, Path]: + def _create_clean_package(self, package_path: Path, *, reload_helper: bool = False) -> tuple[Path, Path]: package_path.mkdir() rs_path = Path(shutil.copy(helpers_dir / "my_script_1.rs", package_path / "my_script.rs")) if reload_helper: diff --git a/tests/test_import_hook/test_utilities.py b/tests/test_import_hook/test_utilities.py index 5400a61..e74bed5 100644 --- a/tests/test_import_hook/test_utilities.py +++ b/tests/test_import_hook/test_utilities.py @@ -5,7 +5,7 @@ import subprocess import time from pathlib import Path -from typing import List, cast +from typing import cast import pytest from maturin_import_hook._building import BuildCache, BuildStatus, Freshness, get_installation_freshness @@ -277,7 +277,7 @@ def test_set_strictly_ordered_mtimes(tmp_path: Path) -> None: assert a.stat().st_mtime < c.stat().st_mtime < b.stat().st_mtime < d.stat().st_mtime -def _set_strictly_ordered_mtimes(paths: List[Path]) -> None: +def _set_strictly_ordered_mtimes(paths: list[Path]) -> None: atime, mtime = get_file_times(paths[0]) for i, p in enumerate(reversed(paths)): set_file_times(p, (atime, mtime - i)) @@ -426,7 +426,7 @@ def _mock_directory_as_unreadable(dir_path: Path, monkeypatch: pytest.MonkeyPatc original_stat = Path.stat def patched_stat(self: Path) -> object: - if _is_relative_to(self, dir_path): + if Path.is_relative_to(self, dir_path): e = PermissionError(13, "Permission denied") e.filename = str(self) raise e @@ -440,16 +440,6 @@ def patched_stat(self: Path) -> object: assert e_info.value.filename == str(dir_path / "abc") -def _is_relative_to(a: Path, b: Path) -> bool: - # TODO(matt): when 3.9 is the minimum supported python version, use Path.is_relative_to - try: - a.relative_to(b) - except ValueError: - return False - else: - return True - - def _file_not_found_message() -> str: return ( "The system cannot find the file specified" if platform.system() == "Windows" else "No such file or directory" From 909b2e05399bb00e03e7469efe0b22d2426de09b Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sun, 24 Mar 2024 17:28:47 +0000 Subject: [PATCH 06/10] improved matrix generation script in CI pipeline --- .github/workflows/test.yml | 120 ++++++++++++++++++++++++------------- 1 file changed, 77 insertions(+), 43 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 69f6c52..0a22b54 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -44,9 +44,20 @@ jobs: name: Generate Matrix runs-on: ubuntu-latest outputs: - os: ${{ steps.generate-matrix.outputs.os }} - python-version: ${{ steps.generate-matrix.outputs.python-version }} + platform: ${{ steps.generate-matrix.outputs.platform }} fail-fast: ${{ steps.generate-matrix.outputs.fail-fast }} + env: + OS_MATRIX: | + - ubuntu-latest + - macos-14 + - windows-latest + PYTHON_VERSION: | + - '3.9' + - '3.10' + - '3.11' + - '3.12' + - 'pypy3.9' + - 'pypy3.10' steps: - uses: actions/setup-node@v4 with: @@ -57,50 +68,74 @@ jobs: uses: actions/github-script@v7 with: script: | - const yaml = require('js-yaml') - const OS = yaml.load(process.env.OS_MATRIX) - const PYTHON_VERSIONS = yaml.load(process.env.PYTHON_VERSION) + const yaml = require("js-yaml"); + + const ALL_OS = yaml.load(process.env.OS_MATRIX); + const ALL_PYTHON_VERSIONS = yaml.load(process.env.PYTHON_VERSION); + + let platforms = []; + let fail_fast = false; + + let all_platforms = []; + ALL_OS.forEach(os => { + ALL_PYTHON_VERSIONS.forEach(python_version => { + if (os === "macos-14") { + if (python_version.startsWith("pypy")) { + return; // PyPy is only built for x64 + } else if (parseInt(python_version.slice(2)) < 11) { + return; // macOS ARM runners only have Python 3.11+ + } + } + all_platforms.push({ + "os": os, + "python-version": python_version + }) + }) + }); + core.info(`job triggered by: ${context.eventName}`); if (context.eventName == 'workflow_dispatch') { - const input_os = "${{ github.event.inputs.os }}"; - const input_python_version = "${{ github.event.inputs.python_version }}"; - core.setOutput('os', input_os == "all" ? OS : [input_os]); - core.setOutput('python-version', input_python_version == "all" ? PYTHON_VERSIONS : [input_python_version]); - core.setOutput('fail-fast', "${{ github.event.inputs.fail_fast }}"); + const INPUT_OS = "${{ github.event.inputs.os }}"; + const INPUT_PYTHON_VERSION = "${{ github.event.inputs.python_version }}"; + const INPUT_FAIL_FAST = "${{ github.event.inputs.fail_fast }}"; + + platforms = all_platforms.filter(platform => ( + (INPUT_OS == "all" || INPUT_OS.includes(platform.os)) + && (INPUT_PYTHON_VERSION == "all" || INPUT_PYTHON_VERSION.includes(platform['python-version'])) + )); + fail_fast = INPUT_FAIL_FAST; } else if (context.eventName == 'merge_group') { - core.setOutput('os', OS) - core.setOutput('python-version', PYTHON_VERSIONS) - core.setOutput('fail-fast', 'false') + platforms = all_platforms; + fail_fast = false; } else if (context.eventName == 'pull_request') { const { data: { labels: labels } } = await github.rest.pulls.get({ owner: context.repo.owner, repo: context.repo.repo, pull_number: context.payload.pull_request.number - }) - const labelNames = labels.map(label => label.name) - if (labelNames.includes('CI-no-fail-fast')) { - core.setOutput('fail-fast', 'false') + }); + const label_names = labels.map(label => label.name); + + if (label_names.includes("CI-test-all")) { + platforms = all_platforms; + } else { + // assumes versions are listed in ascending order + const latest_pypy_index = ALL_PYTHON_VERSIONS.findLastIndex(version => version.startsWith('pypy')); + const latest_cpython_index = ALL_PYTHON_VERSIONS.findLastIndex(version => /^\d/.test(version)); + const python_versions = [ALL_PYTHON_VERSIONS[latest_pypy_index], ALL_PYTHON_VERSIONS[latest_cpython_index]]; + platforms = all_platforms.filter(platform => ( + platform.os == "ubuntu-latest" && python_versions.includes(platform["python-version"]) + )); } - // Only run latest CPython and PyPy tests on pull requests - const firstPyPy = PYTHON_VERSIONS.findIndex(version => version.startsWith('pypy')) - const pythonVersions = [PYTHON_VERSIONS[firstPyPy - 1], PYTHON_VERSIONS[PYTHON_VERSIONS.length - 1]] - core.setOutput('python-version', pythonVersions) + fail_fast = !label_names.includes("CI-no-fail-fast"); } - env: - OS_MATRIX: | - - ubuntu-latest - - macos-13 - - windows-latest - PYTHON_VERSION: | - - '3.9' - - '3.10' - - '3.11' - - '3.12' - - 'pypy3.9' - - 'pypy3.10' + core.info(`platforms = ${JSON.stringify(platforms)}`); + core.setOutput("platform", platforms); + + core.info(`fail fast = ${fail_fast}`); + core.setOutput("fail-fast", fail_fast); test: name: Test @@ -108,25 +143,24 @@ jobs: strategy: fail-fast: ${{ needs.generate-matrix.outputs.fail-fast != 'false' }} matrix: - os: ${{ fromJson(needs.generate-matrix.outputs.os) }} - python-version: ${{ fromJson(needs.generate-matrix.outputs.python-version) }} - runs-on: ${{ matrix.os }} + platform: ${{ fromJson(needs.generate-matrix.outputs.platform) }} + runs-on: ${{ matrix.platform.os }} steps: - uses: actions/checkout@v4 with: submodules: recursive - uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} + python-version: ${{ matrix.platform.python-version }} architecture: "x64" cache: "pip" - uses: dtolnay/rust-toolchain@stable id: rustup - name: Install aarch64-apple-darwin Rust target - if: startsWith(matrix.os, 'macos') + if: startsWith(matrix.platform.os, 'macos') run: rustup target add aarch64-apple-darwin - name: Setup Xcode env - if: startsWith(matrix.os, 'macos') + if: startsWith(matrix.platform.os, 'macos') shell: bash run: | set -ex @@ -137,7 +171,7 @@ jobs: echo "SDKROOT=$(xcrun --sdk macosx --show-sdk-path)" >> "${GITHUB_ENV}" # To save disk space - name: Disable debuginfo on Windows - if: startsWith(matrix.os, 'windows') + if: startsWith(matrix.platform.os, 'windows') run: echo "RUSTFLAGS="-C debuginfo=0"" >> $GITHUB_ENV - name: Install test requirements run: cd tests && pip install --disable-pip-version-check -r requirements.txt @@ -151,21 +185,21 @@ jobs: python tests/runner.py \ --workspace ./test_workspace \ - --name "${{ matrix.os }}_${{ matrix.python-version }}" \ + --name "${{ matrix.platform.os }}_${{ matrix.platform.python-version }}" \ ${EXTRA_ARGS} \ "${{ github.event.inputs.test_specification || 'tests/test_import_hook' }}" - name: Upload HTML test report uses: actions/upload-artifact@v4 if: failure() with: - name: ${{ matrix.os }}-${{ matrix.python-version }}-test-report.html + name: ${{ matrix.platform.os }}-${{ matrix.platform.python-version }}-test-report.html path: './test_workspace/report.html' - name: Publish Test Report uses: mikepenz/action-junit-report@v4 if: success() || failure() with: report_paths: './test_workspace/reports/*.xml' - test_files_prefix: "${{ matrix.os }}_${{ matrix.python-version }}" + test_files_prefix: "${{ matrix.platform.os }}_${{ matrix.platform.python-version }}" conclusion: From 87d629f5dde86160231464757975fb3275f3489b Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sun, 24 Mar 2024 20:32:32 +0000 Subject: [PATCH 07/10] disable creating annotations from test failures since extra permissions are required --- .github/workflows/test.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0a22b54..ce1ba89 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -196,10 +196,12 @@ jobs: path: './test_workspace/report.html' - name: Publish Test Report uses: mikepenz/action-junit-report@v4 - if: success() || failure() + if: always() with: report_paths: './test_workspace/reports/*.xml' test_files_prefix: "${{ matrix.platform.os }}_${{ matrix.platform.python-version }}" + check_annotations: false + job_summary: true conclusion: From 87d21a2cbb99d39da66cc1d7317c89ffee7b4c31 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sun, 24 Mar 2024 20:37:49 +0000 Subject: [PATCH 08/10] fixed failing tests --- .../test_import_hook/test_project_importer.py | 19 ++++++++++--------- .../test_rust_file_importer.py | 2 +- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/tests/test_import_hook/test_project_importer.py b/tests/test_import_hook/test_project_importer.py index b519517..f40556d 100644 --- a/tests/test_import_hook/test_project_importer.py +++ b/tests/test_import_hook/test_project_importer.py @@ -449,16 +449,17 @@ def test_rebuild_on_change_to_path_dependency(workspace: Path) -> None: _install_editable(project_dir) assert _is_editable_installed_correctly(project_name, project_dir, True) - check_installed = dedent(f"""\ - {IMPORT_HOOK_HEADER} - - import pyo3_mixed_with_path_dep + check_installed = "{}\n{}".format( + IMPORT_HOOK_HEADER, + dedent("""\ + import pyo3_mixed_with_path_dep - assert pyo3_mixed_with_path_dep.get_42() == 42, 'get_42 did not return 42' + assert pyo3_mixed_with_path_dep.get_42() == 42, 'get_42 did not return 42' - print('21 is half 42:', pyo3_mixed_with_path_dep.is_half(21, 42)) - print('21 is half 63:', pyo3_mixed_with_path_dep.is_half(21, 63)) - """) + print('21 is half 42:', pyo3_mixed_with_path_dep.is_half(21, 42)) + print('21 is half 63:', pyo3_mixed_with_path_dep.is_half(21, 63)) + """), + ) output1, duration1 = run_python_code(check_installed) assert "21 is half 42: True" in output1 @@ -1066,7 +1067,7 @@ def test_maturin_detection(self, workspace: Path) -> None: assert output == ( 'building "test_project"\n' "caught MaturinError('unsupported maturin version: (0, 1, 2). " - "Import hook requires >=(1, 4, 0),<(2, 0, 0)')\n" + "Import hook requires >=(1, 5, 0),<(2, 0, 0)')\n" ) @pytest.mark.parametrize("is_mixed", [False, True]) diff --git a/tests/test_import_hook/test_rust_file_importer.py b/tests/test_import_hook/test_rust_file_importer.py index 82da662..b734ee5 100644 --- a/tests/test_import_hook/test_rust_file_importer.py +++ b/tests/test_import_hook/test_rust_file_importer.py @@ -632,7 +632,7 @@ def test_maturin_detection(self, workspace: Path) -> None: assert output == ( 'building "my_script"\n' "caught MaturinError('unsupported maturin version: (0, 1, 2). " - "Import hook requires >=(1, 4, 0),<(2, 0, 0)')\n" + "Import hook requires >=(1, 5, 0),<(2, 0, 0)')\n" ) def test_default_rebuild(self, workspace: Path) -> None: From 5b82c0d7b93784eea1e4816f5555e41649af5de7 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sun, 24 Mar 2024 21:02:03 +0000 Subject: [PATCH 09/10] do not specify x64 when not all runners are x64 --- .github/workflows/test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ce1ba89..095b025 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -152,7 +152,6 @@ jobs: - uses: actions/setup-python@v5 with: python-version: ${{ matrix.platform.python-version }} - architecture: "x64" cache: "pip" - uses: dtolnay/rust-toolchain@stable id: rustup From c025c89eda1e31ba7dbe52e77e6a4d6dd3faee55 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sun, 24 Mar 2024 21:21:16 +0000 Subject: [PATCH 10/10] added more permissions to test workflow --- .github/workflows/test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 095b025..1dac31f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -39,6 +39,10 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number }} cancel-in-progress: true +permissions: + contents: read + checks: write + jobs: generate-matrix: name: Generate Matrix