Add functionality for .flyteignore file (#2479)

* Add functionality for .flyteignore file

Signed-off-by: Daniel Sola <daniel.sola@union.ai>

* copy docker ignore for flyteignore

---------

Signed-off-by: Daniel Sola <daniel.sola@union.ai>

Author: dansola

flytekit/tools/fast_registration.py

@@ -15,7 +15,7 @@
 
 from flytekit.core.context_manager import FlyteContextManager
 from flytekit.core.utils import timeit
-from flytekit.tools.ignore import DockerIgnore, GitIgnore, Ignore, IgnoreGroup, StandardIgnore
+from flytekit.tools.ignore import DockerIgnore, FlyteIgnore, GitIgnore, Ignore, IgnoreGroup, StandardIgnore
 from flytekit.tools.script_mode import tar_strip_file_attributes
 
 FAST_PREFIX = "fast"
@@ -46,7 +46,7 @@ def fast_package(
     :param bool deref_symlinks: Enables dereferencing symlinks when packaging directory
     :return os.PathLike:
     """
-    default_ignores = [GitIgnore, DockerIgnore, StandardIgnore]
+    default_ignores = [GitIgnore, DockerIgnore, StandardIgnore, FlyteIgnore]
     if options is not None:
         if options.keep_default_ignores:
             ignores = options.ignores + default_ignores

flytekit/tools/ignore.py

@@ -87,6 +87,27 @@ def _is_ignored(self, path: str) -> bool:
         return self.pm.matches(path)
 
 
+class FlyteIgnore(Ignore):
+    """Uses a .flyteignore file to determine ignored files."""
+
+    def __init__(self, root: Path):
+        super().__init__(root)
+        self.pm = self._parse()
+
+    def _parse(self) -> PatternMatcher:
+        patterns = []
+        flyteignore = os.path.join(self.root, ".flyteignore")
+        if os.path.isfile(flyteignore):
+            with open(flyteignore, "r") as f:
+                patterns = [l.strip() for l in f.readlines() if l and not l.startswith("#")]
+        else:
+            logger.info(f"No .flyteignore found in {self.root}, not applying any filters")
+        return PatternMatcher(patterns)
+
+    def _is_ignored(self, path: str) -> bool:
+        return self.pm.matches(path)
+
+
 class StandardIgnore(Ignore):
     """Retains the standard ignore functionality that previously existed. Could in theory
     by fed with custom ignore patterns from cli."""

tests/flytekit/unit/tools/test_ignore.py

@@ -8,7 +8,7 @@
 import pytest
 from docker.utils.build import PatternMatcher
 
-from flytekit.tools.ignore import DockerIgnore, GitIgnore, IgnoreGroup, StandardIgnore
+from flytekit.tools.ignore import DockerIgnore, GitIgnore, IgnoreGroup, StandardIgnore, FlyteIgnore
 
 
 def make_tree(root: Path, tree: Dict):
@@ -102,6 +102,19 @@ def all_ignore(tmp_path):
     return tmp_path
 
 
+@pytest.fixture
+def simple_flyteignore(tmp_path):
+    tree = {
+        "sub": {"some.bar": ""},
+        "test.foo": "",
+        "keep.foo": "",
+        ".flyteignore": "\n".join(["*.foo", "!keep.foo", "# A comment", "sub"]),
+    }
+
+    make_tree(tmp_path, tree)
+    return tmp_path
+
+
 def test_simple_gitignore(simple_gitignore):
     gitignore = GitIgnore(simple_gitignore)
     assert gitignore.is_ignored(str(simple_gitignore / "test.foo"))
@@ -219,7 +232,7 @@ def test_all_ignore(all_ignore):
 
 def test_all_ignore_tar_filter(all_ignore):
     """Test tar_filter method of all ignores grouped together"""
-    ignore = IgnoreGroup(all_ignore, [GitIgnore, DockerIgnore, StandardIgnore])
+    ignore = IgnoreGroup(all_ignore, [GitIgnore, DockerIgnore, StandardIgnore, FlyteIgnore])
     assert ignore.tar_filter(TarInfo(name="sub")).name == "sub"
     assert ignore.tar_filter(TarInfo(name="sub/some.bar")).name == "sub/some.bar"
     assert not ignore.tar_filter(TarInfo(name="sub/__pycache__/"))
@@ -232,4 +245,23 @@ def test_all_ignore_tar_filter(all_ignore):
     assert ignore.tar_filter(TarInfo(name="keep.foo")).name == "keep.foo"
     assert ignore.tar_filter(TarInfo(name=".gitignore")).name == ".gitignore"
     assert ignore.tar_filter(TarInfo(name=".dockerignore")).name == ".dockerignore"
+    assert ignore.tar_filter(TarInfo(name=".flyteignore")).name == ".flyteignore"
     assert not ignore.tar_filter(TarInfo(name=".git"))
+
+
+def test_flyteignore_parse(simple_flyteignore):
+    """Test .flyteignore file parsing"""
+    flyteignore = FlyteIgnore(simple_flyteignore)
+    assert flyteignore.pm.matches("whatever.foo")
+    assert not flyteignore.pm.matches("keep.foo")
+    assert flyteignore.pm.matches("sub")
+    assert flyteignore.pm.matches("sub/stuff.txt")
+
+
+def test_simple_flyteignore(simple_flyteignore):
+    flyteignore = FlyteIgnore(simple_flyteignore)
+    assert flyteignore.is_ignored(str(simple_flyteignore / "test.foo"))
+    assert flyteignore.is_ignored(str(simple_flyteignore / "sub"))
+    assert flyteignore.is_ignored(str(simple_flyteignore / "sub" / "some.bar"))
+    assert not flyteignore.is_ignored(str(simple_flyteignore / "keep.foo"))
+    assert not flyteignore.is_ignored(str(simple_flyteignore / ".flyteignore"))