Merge pull request #2074 from docker/3.4.1-release

3.4.1 release

docker/api/container.py

@@ -139,8 +139,9 @@ def commit(self, container, repository=None, tag=None, message=None,
             'changes': changes
         }
         u = self._url("/commit")
-        return self._result(self._post_json(u, data=conf, params=params),
-                            json=True)
+        return self._result(
+            self._post_json(u, data=conf, params=params), json=True
+        )
 
     def containers(self, quiet=False, all=False, trunc=False, latest=False,
                    since=None, before=None, limit=-1, size=False,

docker/auth.py

@@ -270,7 +270,7 @@ def load_config(config_path=None, config_dict=None):
         "Couldn't find auth-related section ; attempting to interpret"
         "as auth-only file"
     )
-    return parse_auth(config_dict)
+    return {'auths': parse_auth(config_dict)}
 
 
 def _load_legacy_config(config_file):
@@ -287,14 +287,14 @@ def _load_legacy_config(config_file):
                 )
 
         username, password = decode_auth(data[0])
-        return {
+        return {'auths': {
             INDEX_NAME: {
                 'username': username,
                 'password': password,
                 'email': data[1],
                 'serveraddress': INDEX_URL,
             }
-        }
+        }}
     except Exception as e:
         log.debug(e)
         pass

docker/utils/build.py

@@ -1,13 +1,13 @@
 import io
 import os
 import re
-import six
 import tarfile
 import tempfile
 
+import six
+
+from .fnmatch import fnmatch
 from ..constants import IS_WINDOWS_PLATFORM
-from fnmatch import fnmatch
-from itertools import chain
 
 
 _SEP = re.compile('/|\\\\') if IS_WINDOWS_PLATFORM else re.compile('/')
@@ -44,92 +44,9 @@ def exclude_paths(root, patterns, dockerfile=None):
     if dockerfile is None:
         dockerfile = 'Dockerfile'
 
-    def split_path(p):
-        return [pt for pt in re.split(_SEP, p) if pt and pt != '.']
-
-    def normalize(p):
-        # Leading and trailing slashes are not relevant. Yes,
-        # "foo.py/" must exclude the "foo.py" regular file. "."
-        # components are not relevant either, even if the whole
-        # pattern is only ".", as the Docker reference states: "For
-        # historical reasons, the pattern . is ignored."
-        # ".." component must be cleared with the potential previous
-        # component, regardless of whether it exists: "A preprocessing
-        # step [...]  eliminates . and .. elements using Go's
-        # filepath.".
-        i = 0
-        split = split_path(p)
-        while i < len(split):
-            if split[i] == '..':
-                del split[i]
-                if i > 0:
-                    del split[i - 1]
-                    i -= 1
-            else:
-                i += 1
-        return split
-
-    patterns = (
-        (True, normalize(p[1:]))
-        if p.startswith('!') else
-        (False, normalize(p))
-        for p in patterns)
-    patterns = list(reversed(list(chain(
-        # Exclude empty patterns such as "." or the empty string.
-        filter(lambda p: p[1], patterns),
-        # Always include the Dockerfile and .dockerignore
-        [(True, split_path(dockerfile)), (True, ['.dockerignore'])]))))
-    return set(walk(root, patterns))
-
-
-def walk(root, patterns, default=True):
-    """
-    A collection of file lying below root that should be included according to
-    patterns.
-    """
-
-    def match(p):
-        if p[1][0] == '**':
-            rec = (p[0], p[1][1:])
-            return [p] + (match(rec) if rec[1] else [rec])
-        elif fnmatch(f, p[1][0]):
-            return [(p[0], p[1][1:])]
-        else:
-            return []
-
-    for f in os.listdir(root):
-        cur = os.path.join(root, f)
-        # The patterns if recursing in that directory.
-        sub = list(chain(*(match(p) for p in patterns)))
-        # Whether this file is explicitely included / excluded.
-        hit = next((p[0] for p in sub if not p[1]), None)
-        # Whether this file is implicitely included / excluded.
-        matched = default if hit is None else hit
-        sub = list(filter(lambda p: p[1], sub))
-        if os.path.isdir(cur) and not os.path.islink(cur):
-            # Entirely skip directories if there are no chance any subfile will
-            # be included.
-            if all(not p[0] for p in sub) and not matched:
-                continue
-            # I think this would greatly speed up dockerignore handling by not
-            # recursing into directories we are sure would be entirely
-            # included, and only yielding the directory itself, which will be
-            # recursively archived anyway. However the current unit test expect
-            # the full list of subfiles and I'm not 100% sure it would make no
-            # difference yet.
-            # if all(p[0] for p in sub) and matched:
-            #     yield f
-            #     continue
-            children = False
-            for r in (os.path.join(f, p) for p in walk(cur, sub, matched)):
-                yield r
-                children = True
-            # The current unit tests expect directories only under those
-            # conditions. It might be simplifiable though.
-            if (not sub or not children) and hit or hit is None and default:
-                yield f
-        elif matched:
-            yield f
+    patterns.append('!' + dockerfile)
+    pm = PatternMatcher(patterns)
+    return set(pm.walk(root))
 
 
 def build_file_list(root):
@@ -217,3 +134,122 @@ def mkbuildcontext(dockerfile):
     t.close()
     f.seek(0)
     return f
+
+
+def split_path(p):
+    return [pt for pt in re.split(_SEP, p) if pt and pt != '.']
+
+
+def normalize_slashes(p):
+    if IS_WINDOWS_PLATFORM:
+        return '/'.join(split_path(p))
+    return p
+
+
+def walk(root, patterns, default=True):
+    pm = PatternMatcher(patterns)
+    return pm.walk(root)
+
+
+# Heavily based on
+# https://github.com/moby/moby/blob/master/pkg/fileutils/fileutils.go
+class PatternMatcher(object):
+    def __init__(self, patterns):
+        self.patterns = list(filter(
+            lambda p: p.dirs, [Pattern(p) for p in patterns]
+        ))
+        self.patterns.append(Pattern('!.dockerignore'))
+
+    def matches(self, filepath):
+        matched = False
+        parent_path = os.path.dirname(filepath)
+        parent_path_dirs = split_path(parent_path)
+
+        for pattern in self.patterns:
+            negative = pattern.exclusion
+            match = pattern.match(filepath)
+            if not match and parent_path != '':
+                if len(pattern.dirs) <= len(parent_path_dirs):
+                    match = pattern.match(
+                        os.path.sep.join(parent_path_dirs[:len(pattern.dirs)])
+                    )
+
+            if match:
+                matched = not negative
+
+        return matched
+
+    def walk(self, root):
+        def rec_walk(current_dir):
+            for f in os.listdir(current_dir):
+                fpath = os.path.join(
+                    os.path.relpath(current_dir, root), f
+                )
+                if fpath.startswith('.' + os.path.sep):
+                    fpath = fpath[2:]
+                match = self.matches(fpath)
+                if not match:
+                    yield fpath
+
+                cur = os.path.join(root, fpath)
+                if not os.path.isdir(cur) or os.path.islink(cur):
+                    continue
+
+                if match:
+                    # If we want to skip this file and it's a directory
+                    # then we should first check to see if there's an
+                    # excludes pattern (e.g. !dir/file) that starts with this
+                    # dir. If so then we can't skip this dir.
+                    skip = True
+
+                    for pat in self.patterns:
+                        if not pat.exclusion:
+                            continue
+                        if pat.cleaned_pattern.startswith(
+                                normalize_slashes(fpath)):
+                            skip = False
+                            break
+                    if skip:
+                        continue
+                for sub in rec_walk(cur):
+                    yield sub
+
+        return rec_walk(root)
+
+
+class Pattern(object):
+    def __init__(self, pattern_str):
+        self.exclusion = False
+        if pattern_str.startswith('!'):
+            self.exclusion = True
+            pattern_str = pattern_str[1:]
+
+        self.dirs = self.normalize(pattern_str)
+        self.cleaned_pattern = '/'.join(self.dirs)
+
+    @classmethod
+    def normalize(cls, p):
+
+        # Leading and trailing slashes are not relevant. Yes,
+        # "foo.py/" must exclude the "foo.py" regular file. "."
+        # components are not relevant either, even if the whole
+        # pattern is only ".", as the Docker reference states: "For
+        # historical reasons, the pattern . is ignored."
+        # ".." component must be cleared with the potential previous
+        # component, regardless of whether it exists: "A preprocessing
+        # step [...]  eliminates . and .. elements using Go's
+        # filepath.".
+        i = 0
+        split = split_path(p)
+        while i < len(split):
+            if split[i] == '..':
+                del split[i]
+                if i > 0:
+                    del split[i - 1]
+                    i -= 1
+            else:
+                i += 1
+        return split
+
+    def match(self, filepath):
+        return fnmatch(normalize_slashes(filepath), self.cleaned_pattern)

docker/utils/fnmatch.py

@@ -111,4 +111,5 @@ def translate(pat):
                 res = '%s[%s]' % (res, stuff)
         else:
             res = res + re.escape(c)
+
     return res + '$'

docker/version.py

@@ -1,2 +1,2 @@
-version = "3.4.0"
+version = "3.4.1"
 version_info = tuple([int(d) for d in version.split("-")[0].split(".")])

docs/change-log.md

@@ -1,6 +1,18 @@
 Change log
 ==========
 
+3.4.1
+-----
+
+[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/52?closed=1)
+
+### Bugfixes
+
+* Fixed a bug that caused auth values in config files written using one of the
+  legacy formats to be ignored
+* Fixed issues with handling of double-wildcard `**` patterns in
+  `.dockerignore` files
+
 3.4.0
 -----
 

tests/helpers.py

@@ -123,7 +123,12 @@ def assert_cat_socket_detached_with_keys(sock, inputs):
             sock.sendall(b'make sure the socket is closed\n')
     else:
         sock.sendall(b"make sure the socket is closed\n")
-        assert sock.recv(32) == b''
+        data = sock.recv(128)
+        # New in 18.06: error message is broadcast over the socket when reading
+        # after detach
+        assert data == b'' or data.startswith(
+            b'exec attach failed: error on attach stdin: read escape sequence'
+        )
 
 
 def ctrl_with(char):

tests/integration/api_client_test.py

@@ -1,6 +1,3 @@
-import base64
-import os
-import tempfile
 import time
 import unittest
 import warnings
@@ -24,43 +21,6 @@ def test_info(self):
         assert 'Debug' in res
 
 
-class LoadConfigTest(BaseAPIIntegrationTest):
-    def test_load_legacy_config(self):
-        folder = tempfile.mkdtemp()
-        self.tmp_folders.append(folder)
-        cfg_path = os.path.join(folder, '.dockercfg')
-        f = open(cfg_path, 'w')
-        auth_ = base64.b64encode(b'sakuya:izayoi').decode('ascii')
-        f.write('auth = {0}\n'.format(auth_))
-        f.write('email = sakuya@scarlet.net')
-        f.close()
-        cfg = docker.auth.load_config(cfg_path)
-        assert cfg[docker.auth.INDEX_NAME] is not None
-        cfg = cfg[docker.auth.INDEX_NAME]
-        assert cfg['username'] == 'sakuya'
-        assert cfg['password'] == 'izayoi'
-        assert cfg['email'] == 'sakuya@scarlet.net'
-        assert cfg.get('Auth') is None
-
-    def test_load_json_config(self):
-        folder = tempfile.mkdtemp()
-        self.tmp_folders.append(folder)
-        cfg_path = os.path.join(folder, '.dockercfg')
-        f = open(os.path.join(folder, '.dockercfg'), 'w')
-        auth_ = base64.b64encode(b'sakuya:izayoi').decode('ascii')
-        email_ = 'sakuya@scarlet.net'
-        f.write('{{"{0}": {{"auth": "{1}", "email": "{2}"}}}}\n'.format(
-            docker.auth.INDEX_URL, auth_, email_))
-        f.close()
-        cfg = docker.auth.load_config(cfg_path)
-        assert cfg[docker.auth.INDEX_URL] is not None
-        cfg = cfg[docker.auth.INDEX_URL]
-        assert cfg['username'] == 'sakuya'
-        assert cfg['password'] == 'izayoi'
-        assert cfg['email'] == 'sakuya@scarlet.net'
-        assert cfg.get('Auth') is None
-
-
 class AutoDetectVersionTest(unittest.TestCase):
     def test_client_init(self):
         client = docker.APIClient(version='auto', **kwargs_from_env())