diff --git a/isort/core.py b/isort/core.py index 4c77a1d9..815aab13 100644 --- a/isort/core.py +++ b/isort/core.py @@ -5,6 +5,7 @@ import isort.literal from isort.settings import DEFAULT_CONFIG, Config +from isort.utils import is_module_dunder from . import output, parse from .exceptions import ExistingSyntaxErrors, FileSkipComment @@ -196,7 +197,11 @@ def process( first_comment_index_end = index - 1 was_in_quote = bool(in_quote) - if (not stripped_line.startswith("#") or in_quote) and '"' in line or "'" in line: + if ( + not is_module_dunder(stripped_line) + and (not stripped_line.startswith("#") or in_quote) + and ('"' in line or "'" in line) + ): char_index = 0 if first_comment_index_start == -1 and ( line.startswith('"') or line.startswith("'") @@ -222,6 +227,7 @@ def process( char_index += 1 not_imports = bool(in_quote) or was_in_quote or in_top_comment or isort_off + if not (in_quote or was_in_quote or in_top_comment): if isort_off: if not skip_file and stripped_line == "# isort: on": @@ -288,6 +294,22 @@ def process( and stripped_line not in config.treat_comments_as_code ): import_section += line + elif is_module_dunder(stripped_line): + # Handle module-level dunders. + dunder_statement = line + if stripped_line.endswith(("\\", "[", '= """', "= '''")): + # Handle multiline module dunder assignments. + while ( + stripped_line + and not stripped_line.endswith("]") + and stripped_line != '"""' + and stripped_line != "'''" + ): + line = input_stream.readline() + stripped_line = line.strip() + dunder_statement += line + import_section += dunder_statement + elif stripped_line.startswith(IMPORT_START_IDENTIFIERS): new_indent = line[: -len(line.lstrip())] import_statement = line @@ -295,6 +317,7 @@ def process( while stripped_line.endswith("\\") or ( "(" in stripped_line and ")" not in stripped_line ): + # Handle multiline import statements. if stripped_line.endswith("\\"): while stripped_line and stripped_line.endswith("\\"): line = input_stream.readline() @@ -429,6 +452,7 @@ def process( extension, import_type="cimport" if cimports else "import", ) + if not (import_section.strip() and not sorted_import_section): if indent: sorted_import_section = ( diff --git a/isort/output.py b/isort/output.py index 3cb3c08b..c79270ca 100644 --- a/isort/output.py +++ b/isort/output.py @@ -34,9 +34,12 @@ def sorted_imports( parsed.imports["no_sections"] = {"straight": {}, "from": {}} base_sections: Tuple[str, ...] = () for section in sections: + if section == "DUNDER": + continue if section == "FUTURE": base_sections = ("FUTURE",) continue + parsed.imports["no_sections"]["straight"].update( parsed.imports[section].get("straight", {}) ) @@ -46,7 +49,14 @@ def sorted_imports( output: List[str] = [] seen_headings: Set[str] = set() pending_lines_before = False + for section in sections: + if section == "DUNDER": + if parsed.module_dunders: + output += [""] * config.lines_between_sections + output.extend(parsed.module_dunders) + continue + straight_modules = parsed.imports[section]["straight"] if not config.only_sections: straight_modules = sorting.sort( diff --git a/isort/parse.py b/isort/parse.py index c60938d8..d6f06b47 100644 --- a/isort/parse.py +++ b/isort/parse.py @@ -5,6 +5,8 @@ from typing import TYPE_CHECKING, Any, Dict, List, NamedTuple, Optional, Set, Tuple from warnings import warn +from isort.utils import is_module_dunder + from . import place from .comments import parse as parse_comments from .exceptions import MissingSection @@ -132,6 +134,7 @@ class ParsedContent(NamedTuple): import_placements: Dict[str, str] as_map: Dict[str, Dict[str, List[str]]] imports: Dict[str, Dict[str, Any]] + module_dunders: List[str] categorized_comments: "CommentsDict" change_count: int original_line_count: int @@ -166,9 +169,12 @@ def file_contents(contents: str, config: Config = DEFAULT_CONFIG) -> ParsedConte "from": defaultdict(list), } imports: OrderedDict[str, Dict[str, Any]] = OrderedDict() + module_dunders: List[str] = [] verbose_output: List[str] = [] - for section in chain(config.sections, config.forced_separate): + section_names = [name for name in config.sections if name != "DUNDER"] + + for section in chain(section_names, config.forced_separate): imports[section] = {"straight": OrderedDict(), "from": OrderedDict()} categorized_comments: CommentsDict = { "from": {}, @@ -185,6 +191,23 @@ def file_contents(contents: str, config: Config = DEFAULT_CONFIG) -> ParsedConte while index < line_count: line = in_lines[index] index += 1 + + if is_module_dunder(line): + dunder_statement = line + if line.endswith(("\\", "[", '= """', "= '''")): + while ( + index < line_count + and line + and not line.endswith("]") + and line != '"""' + and line != "'''" + ): + line = in_lines[index] + index += 1 + dunder_statement += "\n" + line + module_dunders.append(dunder_statement) + continue + statement_index = index (skipping_line, in_quote) = skip_line( line, in_quote=in_quote, index=index, section_comments=config.section_comments @@ -265,8 +288,9 @@ def file_contents(contents: str, config: Config = DEFAULT_CONFIG) -> ParsedConte for statement in statements: line, raw_line = _normalize_line(statement) - type_of_import = import_type(line, config) or "" raw_lines = [raw_line] + type_of_import = import_type(line, config) or "" + if not type_of_import: out_lines.append(raw_line) continue @@ -587,6 +611,7 @@ def file_contents(contents: str, config: Config = DEFAULT_CONFIG) -> ParsedConte import_placements=import_placements, as_map=as_map, imports=imports, + module_dunders=module_dunders, categorized_comments=categorized_comments, change_count=change_count, original_line_count=original_line_count, diff --git a/isort/sections.py b/isort/sections.py index f59db692..930e28d5 100644 --- a/isort/sections.py +++ b/isort/sections.py @@ -2,8 +2,9 @@ from typing import Tuple FUTURE: str = "FUTURE" +DUNDER: str = "DUNDER" STDLIB: str = "STDLIB" THIRDPARTY: str = "THIRDPARTY" FIRSTPARTY: str = "FIRSTPARTY" LOCALFOLDER: str = "LOCALFOLDER" -DEFAULT: Tuple[str, ...] = (FUTURE, STDLIB, THIRDPARTY, FIRSTPARTY, LOCALFOLDER) +DEFAULT: Tuple[str, ...] = (FUTURE, DUNDER, STDLIB, THIRDPARTY, FIRSTPARTY, LOCALFOLDER) diff --git a/isort/utils.py b/isort/utils.py index 339c86f6..effee5f5 100644 --- a/isort/utils.py +++ b/isort/utils.py @@ -1,4 +1,5 @@ import os +import re import sys from pathlib import Path from typing import Any, Dict, Optional, Tuple @@ -70,3 +71,10 @@ def exists_case_sensitive(path: str) -> bool: directory, basename = os.path.split(path) result = basename in os.listdir(directory) return result + + +MODULE_DUNDER_PATTERN = re.compile(r"^__.*__\s*=") + + +def is_module_dunder(line: str) -> bool: + return bool(MODULE_DUNDER_PATTERN.match(line)) diff --git a/tests/unit/profiles/test_attrs.py b/tests/unit/profiles/test_attrs.py index c08f184e..6e033c38 100644 --- a/tests/unit/profiles/test_attrs.py +++ b/tests/unit/profiles/test_attrs.py @@ -9,6 +9,8 @@ def test_attrs_code_snippet_one(): attrs_isort_test( """from __future__ import absolute_import, division, print_function +__version__ = "20.2.0.dev0" + import sys from functools import partial @@ -28,9 +30,6 @@ def test_attrs_code_snippet_one(): validate, ) from ._version_info import VersionInfo - - -__version__ = "20.2.0.dev0" """ ) @@ -81,12 +80,6 @@ def test_attrs_code_snippet_three(): from __future__ import absolute_import, division, print_function -import re - -from ._make import _AndValidator, and_, attrib, attrs -from .exceptions import NotCallableError - - __all__ = [ "and_", "deep_iterable", @@ -98,5 +91,10 @@ def test_attrs_code_snippet_three(): "optional", "provides", ] + +import re + +from ._make import _AndValidator, and_, attrib, attrs +from .exceptions import NotCallableError ''' ) diff --git a/tests/unit/profiles/test_open_stack.py b/tests/unit/profiles/test_open_stack.py index 2def240f..685e429e 100644 --- a/tests/unit/profiles/test_open_stack.py +++ b/tests/unit/profiles/test_open_stack.py @@ -100,6 +100,19 @@ def test_open_stack_code_snippet_three(): # License for the specific language governing permissions and limitations # under the License. +__all__ = [ + 'init', + 'cleanup', + 'set_defaults', + 'add_extra_exmods', + 'clear_extra_exmods', + 'get_allowed_exmods', + 'RequestContextSerializer', + 'get_client', + 'get_server', + 'get_notifier', +] + import functools from oslo_log import log as logging @@ -115,19 +128,6 @@ def test_open_stack_code_snippet_three(): import nova.exception from nova.i18n import _ -__all__ = [ - 'init', - 'cleanup', - 'set_defaults', - 'add_extra_exmods', - 'clear_extra_exmods', - 'get_allowed_exmods', - 'RequestContextSerializer', - 'get_client', - 'get_server', - 'get_notifier', -] - profiler = importutils.try_import("osprofiler.profiler") """, known_first_party=["nova"], diff --git a/tests/unit/test_isort.py b/tests/unit/test_isort.py index 6f4f03a0..221fcae6 100644 --- a/tests/unit/test_isort.py +++ b/tests/unit/test_isort.py @@ -3730,7 +3730,6 @@ def test_new_lines_are_preserved() -> None: def test_forced_separate_is_deterministic_issue_774(tmpdir) -> None: - config_file = tmpdir.join("setup.cfg") config_file.write( "[isort]\n" @@ -5591,3 +5590,76 @@ def test_infinite_loop_in_unmatched_parenthesis() -> None: # ensure other cases are handled correctly assert isort.code(test_input) == "from os import path, walk\n" + + +def test_dunders() -> None: + """Test to ensure dunder imports are in the correct location.""" + test_input = """from __future__ import division + +import os +import sys + +__all__ = ["dla"] +__version__ = '0.1' +__author__ = 'someone' +""" + + expected_output = """from __future__ import division + +__all__ = ["dla"] +__version__ = '0.1' +__author__ = 'someone' + +import os +import sys +""" + assert isort.code(test_input) == expected_output + + +def test_multiline_dunders() -> None: + """Test to ensure isort correctly handles multiline dunders""" + test_input = """from __future__ import division + +import os +import sys + +__all__ = [ + "one", + "two", +] +__version__ = '0.1' +__author__ = ''' + first name + last name +''' +""" + + expected_output = """from __future__ import division + +__all__ = [ + "one", + "two", +] +__version__ = '0.1' +__author__ = ''' + first name + last name +''' + +import os +import sys +""" + assert isort.code(test_input) == expected_output + + +def test_dunders_needs_import() -> None: + """Test to ensure dunder definitions that need imports are not moved.""" + test_input = """from importlib import metadata + +__version__ = metadata.version("isort") +__all__ = ["dla"] +__author__ = 'someone' +""" + + expected_output = test_input + assert isort.code(test_input) == expected_output diff --git a/tests/unit/test_parse.py b/tests/unit/test_parse.py index 7d9e6606..94239b84 100644 --- a/tests/unit/test_parse.py +++ b/tests/unit/test_parse.py @@ -33,6 +33,7 @@ def test_file_contents(): _, _, _, + _, change_count, original_line_count, _,