Merge pull request #13107 from adrianeboyd/chore/update-develop-from-master-v3.8-1

Update develop from master for v3.8

Author: adrianeboyd

.github/workflows/tests.yml

@@ -58,7 +58,7 @@ jobs:
       fail-fast: true
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python_version: ["3.11", "3.12.0-rc.2"]
+        python_version: ["3.12"]
         include:
           - os: windows-latest
             python_version: "3.7"
@@ -68,6 +68,8 @@ jobs:
             python_version: "3.9"
           - os: windows-latest
             python_version: "3.10"
+          - os: macos-latest
+            python_version: "3.11"
 
     runs-on: ${{ matrix.os }}
 
@@ -115,22 +117,22 @@ jobs:
       - name: Test import
         run: python -W error -c "import spacy"
 
-#      - name: "Test download CLI"
-#        run: |
-#          python -m spacy download ca_core_news_sm
-#          python -m spacy download ca_core_news_md
-#          python -c "import spacy; nlp=spacy.load('ca_core_news_sm'); doc=nlp('test')"
-#        if: matrix.python_version == '3.9'
-#
-#      - name: "Test download_url in info CLI"
-#        run: |
-#          python -W error -m spacy info ca_core_news_sm | grep -q download_url
-#        if: matrix.python_version == '3.9'
-#
-#      - name: "Test no warnings on load (#11713)"
-#        run: |
-#          python -W error -c "import ca_core_news_sm; nlp = ca_core_news_sm.load(); doc=nlp('test')"
-#        if: matrix.python_version == '3.9'
+      - name: "Test download CLI"
+        run: |
+          python -m spacy download ca_core_news_sm
+          python -m spacy download ca_core_news_md
+          python -c "import spacy; nlp=spacy.load('ca_core_news_sm'); doc=nlp('test')"
+        if: matrix.python_version == '3.9'
+
+      - name: "Test download_url in info CLI"
+        run: |
+          python -W error -m spacy info ca_core_news_sm | grep -q download_url
+        if: matrix.python_version == '3.9'
+
+      - name: "Test no warnings on load (#11713)"
+        run: |
+          python -W error -c "import ca_core_news_sm; nlp = ca_core_news_sm.load(); doc=nlp('test')"
+        if: matrix.python_version == '3.9'
 
       - name: "Test convert CLI"
         run: |
@@ -154,17 +156,17 @@ jobs:
           python -m spacy train ner.cfg --paths.train ner-token-per-line-conll2003.spacy --paths.dev ner-token-per-line-conll2003.spacy --training.max_steps 10 --gpu-id -1
         if: matrix.python_version == '3.9'
 
-#      - name: "Test assemble CLI"
-#        run: |
-#          python -c "import spacy; config = spacy.util.load_config('ner.cfg'); config['components']['ner'] = {'source': 'ca_core_news_sm'}; config.to_disk('ner_source_sm.cfg')"
-#          PYTHONWARNINGS="error,ignore::DeprecationWarning" python -m spacy assemble ner_source_sm.cfg output_dir
-#        if: matrix.python_version == '3.9'
-#
-#      - name: "Test assemble CLI vectors warning"
-#        run: |
-#          python -c "import spacy; config = spacy.util.load_config('ner.cfg'); config['components']['ner'] = {'source': 'ca_core_news_md'}; config.to_disk('ner_source_md.cfg')"
-#          python -m spacy assemble ner_source_md.cfg output_dir 2>&1 | grep -q W113
-#        if: matrix.python_version == '3.9'
+      - name: "Test assemble CLI"
+        run: |
+          python -c "import spacy; config = spacy.util.load_config('ner.cfg'); config['components']['ner'] = {'source': 'ca_core_news_sm'}; config.to_disk('ner_source_sm.cfg')"
+          PYTHONWARNINGS="error,ignore::DeprecationWarning" python -m spacy assemble ner_source_sm.cfg output_dir
+        if: matrix.python_version == '3.9'
+
+      - name: "Test assemble CLI vectors warning"
+        run: |
+          python -c "import spacy; config = spacy.util.load_config('ner.cfg'); config['components']['ner'] = {'source': 'ca_core_news_md'}; config.to_disk('ner_source_md.cfg')"
+          python -m spacy assemble ner_source_md.cfg output_dir 2>&1 | grep -q W113
+        if: matrix.python_version == '3.9'
 
       - name: "Install test requirements"
         run: |

LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (C) 2016-2022 ExplosionAI GmbH, 2016 spaCy GmbH, 2015 Matthew Honnibal
+Copyright (C) 2016-2023 ExplosionAI GmbH, 2016 spaCy GmbH, 2015 Matthew Honnibal
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -16,7 +16,7 @@ model packaging, deployment and workflow management. spaCy is commercial
 open-source software, released under the
 [MIT license](https://github.com/explosion/spaCy/blob/master/LICENSE).
 
-💫 **Version 3.6 out now!**
+💫 **Version 3.7 out now!**
 [Check out the release notes here.](https://github.com/explosion/spaCy/releases)
 
 [![tests](https://github.com/explosion/spaCy/actions/workflows/tests.yml/badge.svg)](https://github.com/explosion/spaCy/actions/workflows/tests.yml)

requirements.txt

@@ -10,7 +10,6 @@ wasabi>=0.9.1,<1.2.0
 srsly>=2.4.3,<3.0.0
 catalogue>=2.0.6,<2.1.0
 typer>=0.3.0,<0.10.0
-pathy>=0.10.0
 smart-open>=5.2.1,<7.0.0
 weasel>=0.1.0,<0.4.0
 # Third party dependencies

setup.cfg

@@ -56,7 +56,6 @@ install_requires =
     weasel>=0.1.0,<0.4.0
     # Third-party dependencies
     typer>=0.3.0,<0.10.0
-    pathy>=0.10.0
     smart-open>=5.2.1,<7.0.0
     tqdm>=4.38.0,<5.0.0
     numpy>=1.15.0; python_version < "3.9"

spacy/__init__.py

@@ -13,6 +13,7 @@
 from . import pipeline  # noqa: F401
 from . import util
 from .about import __version__  # noqa: F401
+from .cli.info import info  # noqa: F401
 from .errors import Errors
 from .glossary import explain  # noqa: F401
 from .language import Language
@@ -76,9 +77,3 @@ def blank(
     # We should accept both dot notation and nested dict here for consistency
     config = util.dot_to_dict(config)
     return LangClass.from_config(config, vocab=vocab, meta=meta)
-
-
-def info(*args, **kwargs):
-    from .cli.info import info as cli_info
-
-    return cli_info(*args, **kwargs)

spacy/about.py

@@ -1,5 +1,5 @@
 # fmt: off
 __title__ = "spacy"
-__version__ = "3.7.0"
+__version__ = "3.7.2"
 __download_url__ = "https://github.com/explosion/spacy-models/releases/download"
 __compatibility__ = "https://raw.githubusercontent.com/explosion/spacy-models/master/compatibility.json"

spacy/cli/__init__.py

@@ -22,8 +22,17 @@
 from .package import package  # noqa: F401
 from .pretrain import pretrain  # noqa: F401
 from .profile import profile  # noqa: F401
-from .train import train_cli  # noqa: F401
-from .validate import validate  # noqa: F401
+from .project.assets import project_assets  # type: ignore[attr-defined]  # noqa: F401
+from .project.clone import project_clone  # type: ignore[attr-defined]  # noqa: F401
+from .project.document import (  # type: ignore[attr-defined]  # noqa: F401
+    project_document,
+)
+from .project.dvc import project_update_dvc  # type: ignore[attr-defined]  # noqa: F401
+from .project.pull import project_pull  # type: ignore[attr-defined]  # noqa: F401
+from .project.push import project_push  # type: ignore[attr-defined]  # noqa: F401
+from .project.run import project_run  # type: ignore[attr-defined]  # noqa: F401
+from .train import train_cli  # type: ignore[attr-defined]  # noqa: F401
+from .validate import validate  # type: ignore[attr-defined]  # noqa: F401
 
 
 @app.command("link", no_args_is_help=True, deprecated=True, hidden=True)

spacy/cli/_util.py

@@ -41,10 +41,6 @@
     run_command,
 )
 
-if TYPE_CHECKING:
-    from pathy import FluidPath  # noqa: F401
-
-
 SDIST_SUFFIX = ".tar.gz"
 WHEEL_SUFFIX = "-py3-none-any.whl"
 

spacy/cli/project/__init__.py

@@ -0,0 +1 @@
+from weasel.cli.assets import *

spacy/cli/project/assets.py

@@ -0,0 +1 @@
+from weasel.cli.clone import *

spacy/cli/project/clone.py

@@ -0,0 +1 @@
+from weasel.cli.document import *

spacy/cli/project/document.py

@@ -0,0 +1 @@
+from weasel.cli.dvc import *

spacy/cli/project/dvc.py

@@ -0,0 +1 @@
+from weasel.cli.pull import *

spacy/cli/project/pull.py

@@ -0,0 +1 @@
+from weasel.cli.push import *

spacy/cli/project/push.py

@@ -0,0 +1 @@
+from weasel.cli.remote_storage import *

spacy/cli/project/remote_storage.py

@@ -0,0 +1 @@
+from weasel.cli.run import *

spacy/cli/project/run.py

@@ -142,7 +142,25 @@ def render_spans(
         spans (list): Individual entity spans and their start, end, label, kb_id and kb_url.
         title (str / None): Document title set in Doc.user_data['title'].
         """
-        per_token_info = []
+        per_token_info = self._assemble_per_token_info(tokens, spans)
+        markup = self._render_markup(per_token_info)
+        markup = TPL_SPANS.format(content=markup, dir=self.direction)
+        if title:
+            markup = TPL_TITLE.format(title=title) + markup
+        return markup
+
+    @staticmethod
+    def _assemble_per_token_info(
+        tokens: List[str], spans: List[Dict[str, Any]]
+    ) -> List[Dict[str, List[Dict[str, Any]]]]:
+        """Assembles token info used to generate markup in render_spans().
+        tokens (List[str]): Tokens in text.
+        spans (List[Dict[str, Any]]): Spans in text.
+        RETURNS (List[Dict[str, List[Dict, str, Any]]]): Per token info needed to render HTML markup for given tokens
+            and spans.
+        """
+        per_token_info: List[Dict[str, List[Dict[str, Any]]]] = []
+
         # we must sort so that we can correctly describe when spans need to "stack"
         # which is determined by their start token, then span length (longer spans on top),
         # then break any remaining ties with the span label
@@ -154,29 +172,35 @@ def render_spans(
                 s["label"],
             ),
         )
+
         for s in spans:
             # this is the vertical 'slot' that the span will be rendered in
             # vertical_position = span_label_offset + (offset_step * (slot - 1))
             s["render_slot"] = 0
+
         for idx, token in enumerate(tokens):
             # Identify if a token belongs to a Span (and which) and if it's a
             # start token of said Span. We'll use this for the final HTML render
             token_markup: Dict[str, Any] = {}
             token_markup["text"] = token
-            concurrent_spans = 0
+            intersecting_spans: List[Dict[str, Any]] = []
             entities = []
             for span in spans:
                 ent = {}
                 if span["start_token"] <= idx < span["end_token"]:
-                    concurrent_spans += 1
                     span_start = idx == span["start_token"]
                     ent["label"] = span["label"]
                     ent["is_start"] = span_start
                     if span_start:
                         # When the span starts, we need to know how many other
                         # spans are on the 'span stack' and will be rendered.
                         # This value becomes the vertical render slot for this entire span
-                        span["render_slot"] = concurrent_spans
+                        span["render_slot"] = (
+                            intersecting_spans[-1]["render_slot"]
+                            if len(intersecting_spans)
+                            else 0
+                        ) + 1
+                    intersecting_spans.append(span)
                     ent["render_slot"] = span["render_slot"]
                     kb_id = span.get("kb_id", "")
                     kb_url = span.get("kb_url", "#")
@@ -193,11 +217,8 @@ def render_spans(
                     span["render_slot"] = 0
             token_markup["entities"] = entities
             per_token_info.append(token_markup)
-        markup = self._render_markup(per_token_info)
-        markup = TPL_SPANS.format(content=markup, dir=self.direction)
-        if title:
-            markup = TPL_TITLE.format(title=title) + markup
-        return markup
+
+        return per_token_info
 
     def _render_markup(self, per_token_info: List[Dict[str, Any]]) -> str:
         """Render the markup from per-token information"""

spacy/displacy/render.py

@@ -1,3 +1,11 @@
 from .candidate import Candidate, get_candidates, get_candidates_batch
 from .kb import KnowledgeBase
 from .kb_in_memory import InMemoryLookupKB
+
+__all__ = [
+    "Candidate",
+    "KnowledgeBase",
+    "InMemoryLookupKB",
+    "get_candidates",
+    "get_candidates_batch",
+]

spacy/kb/__init__.py

@@ -3,4 +3,4 @@
 from .matcher import Matcher
 from .phrasematcher import PhraseMatcher
 
-__all__ = ["Matcher", "PhraseMatcher", "DependencyMatcher", "levenshtein"]
+__all__ = ["DependencyMatcher", "Matcher", "PhraseMatcher", "levenshtein"]

spacy/matcher/__init__.py

@@ -22,6 +22,7 @@
 __all__ = [
     "AttributeRuler",
     "DependencyParser",
+    "EditTreeLemmatizer",
     "EntityLinker",
     "EntityRecognizer",
     "EntityRuler",

spacy/pipeline/__init__.py

@@ -731,3 +731,12 @@ def test_for_no_ent_sents():
     sents = list(doc.ents[0].sents)
     assert len(sents) == 1
     assert str(sents[0]) == str(doc.ents[0].sent) == "ENTITY"
+
+
+def test_span_api_richcmp_other(en_tokenizer):
+    doc1 = en_tokenizer("a b")
+    doc2 = en_tokenizer("b c")
+    assert not doc1[1:2] == doc1[1]
+    assert not doc1[1:2] == doc2[0]
+    assert not doc1[1:2] == doc2[0:1]
+    assert not doc1[0:1] == doc2

spacy/tests/doc/test_span.py

@@ -294,3 +294,12 @@ def test_missing_head_dep(en_vocab):
     assert aligned_heads[0] == ref_heads[0]
     assert aligned_deps[5] == ref_deps[5]
     assert aligned_heads[5] == ref_heads[5]
+
+
+def test_token_api_richcmp_other(en_tokenizer):
+    doc1 = en_tokenizer("a b")
+    doc2 = en_tokenizer("b c")
+    assert not doc1[1] == doc1[0:1]
+    assert not doc1[1] == doc2[1:2]
+    assert not doc1[1] == doc2[0]
+    assert not doc1[0] == doc2

spacy/tests/doc/test_token_api.py

@@ -12,7 +12,6 @@
 
 import spacy
 from spacy import about
-from spacy import info as spacy_info
 from spacy.cli import info
 from spacy.cli._util import parse_config_overrides, string_to_list, walk_directory
 from spacy.cli.apply import apply
@@ -193,9 +192,6 @@ def test_cli_info():
         raw_data = info(tmp_dir, exclude=[""])
         assert raw_data["lang"] == "nl"
         assert raw_data["components"] == ["textcat"]
-        raw_data = spacy_info(tmp_dir, exclude=[""])
-        assert raw_data["lang"] == "nl"
-        assert raw_data["components"] == ["textcat"]
 
 
 def test_cli_converters_conllu_to_docs():
@@ -538,7 +534,6 @@ def test_string_to_list_intify(value):
     assert string_to_list(value, intify=True) == [1, 2, 3]
 
 
-@pytest.mark.skip(reason="Temporarily skip before 3.7 models are published")
 def test_download_compatibility():
     spec = SpecifierSet("==" + about.__version__)
     spec.prereleases = False
@@ -549,7 +544,6 @@ def test_download_compatibility():
         assert get_minor_version(about.__version__) == get_minor_version(version)
 
 
-@pytest.mark.skip(reason="Temporarily skip before 3.7 models are published")
 def test_validate_compatibility_table():
     spec = SpecifierSet("==" + about.__version__)
     spec.prereleases = False
@@ -1067,3 +1061,8 @@ def test_debug_data_trainable_lemmatizer_not_annotated():
 
     data = _compile_gold(train_examples, ["trainable_lemmatizer"], nlp, True)
     assert data["no_lemma_annotations"] == 2
+
+
+def test_project_api_imports():
+    from spacy.cli import project_run
+    from spacy.cli.project.run import project_run  # noqa: F401, F811

spacy/tests/test_cli.py

@@ -2,7 +2,7 @@
 import pytest
 
 from spacy import displacy
-from spacy.displacy.render import DependencyRenderer, EntityRenderer
+from spacy.displacy.render import DependencyRenderer, EntityRenderer, SpanRenderer
 from spacy.lang.en import English
 from spacy.lang.fa import Persian
 from spacy.tokens import Doc, Span
@@ -468,3 +468,23 @@ def test_issue12816(en_vocab) -> None:
     # Verify that the HTML tag is still escaped
     html = displacy.render(doc, style="span")
     assert "<TEST>" in html
+
+
+@pytest.mark.issue(13056)
+def test_displacy_span_stacking():
+    """Test whether span stacking works properly for multiple overlapping spans."""
+    spans = [
+        {"start_token": 2, "end_token": 5, "label": "SkillNC"},
+        {"start_token": 0, "end_token": 2, "label": "Skill"},
+        {"start_token": 1, "end_token": 3, "label": "Skill"},
+    ]
+    tokens = ["Welcome", "to", "the", "Bank", "of", "China", "."]
+    per_token_info = SpanRenderer._assemble_per_token_info(spans=spans, tokens=tokens)
+
+    assert len(per_token_info) == len(tokens)
+    assert all([len(per_token_info[i]["entities"]) == 1 for i in (0, 3, 4)])
+    assert all([len(per_token_info[i]["entities"]) == 2 for i in (1, 2)])
+    assert per_token_info[1]["entities"][0]["render_slot"] == 1
+    assert per_token_info[1]["entities"][1]["render_slot"] == 2
+    assert per_token_info[2]["entities"][0]["render_slot"] == 2
+    assert per_token_info[2]["entities"][1]["render_slot"] == 3