Merge pull request #2062 from docker/3.4.0-release3.4.0

3.4.0 Release

21 files changed, 191 insertions, 53 deletions

diff --git a/docker/api/build.py b/docker/api/build.py
index f62a731..419255f 100644
--- a/docker/api/build.py
+++ b/docker/api/build.py
@@ -302,7 +302,8 @@ class BuildApiMixin(object):
                 # credentials/native_store.go#L68-L83
                 for registry in self._auth_configs.get('auths', {}).keys():
                     auth_data[registry] = auth.resolve_authconfig(
-                        self._auth_configs, registry
+                        self._auth_configs, registry,
+                        credstore_env=self.credstore_env,
                     )
             else:
                 auth_data = self._auth_configs.get('auths', {}).copy()
@@ -341,4 +342,9 @@ def process_dockerfile(dockerfile, path):
             )
 
     # Dockerfile is inside the context - return path relative to context root
-    return (os.path.relpath(abs_dockerfile, path), None)
+    if dockerfile == abs_dockerfile:
+        # Only calculate relpath if necessary to avoid errors
+        # on Windows client -> Linux Docker
+        # see https://github.com/docker/compose/issues/5969
+        dockerfile = os.path.relpath(abs_dockerfile, path)
+    return (dockerfile, None)
diff --git a/docker/api/client.py b/docker/api/client.py
index 13c292a..91da1c8 100644
--- a/docker/api/client.py
+++ b/docker/api/client.py
@@ -83,6 +83,8 @@ class APIClient(
             :py:class:`~docker.tls.TLSConfig` object to use custom
             configuration.
         user_agent (str): Set a custom user agent for requests to the server.
+        credstore_env (dict): Override environment variables when calling the
+            credential store process.
     """
 
     __attrs__ = requests.Session.__attrs__ + ['_auth_configs',
@@ -93,7 +95,8 @@ class APIClient(
 
     def __init__(self, base_url=None, version=None,
                  timeout=DEFAULT_TIMEOUT_SECONDS, tls=False,
-                 user_agent=DEFAULT_USER_AGENT, num_pools=DEFAULT_NUM_POOLS):
+                 user_agent=DEFAULT_USER_AGENT, num_pools=DEFAULT_NUM_POOLS,
+                 credstore_env=None):
         super(APIClient, self).__init__()
 
         if tls and not base_url:
@@ -109,6 +112,7 @@ class APIClient(
         self._auth_configs = auth.load_config(
             config_dict=self._general_configs
         )
+        self.credstore_env = credstore_env
 
         base_url = utils.parse_host(
             base_url, IS_WINDOWS_PLATFORM, tls=bool(tls)
diff --git a/docker/api/daemon.py b/docker/api/daemon.py
index fc3692c..76a94cf 100644
--- a/docker/api/daemon.py
+++ b/docker/api/daemon.py
@@ -128,7 +128,9 @@ class DaemonApiMixin(object):
         elif not self._auth_configs:
             self._auth_configs = auth.load_config()
 
-        authcfg = auth.resolve_authconfig(self._auth_configs, registry)
+        authcfg = auth.resolve_authconfig(
+            self._auth_configs, registry, credstore_env=self.credstore_env,
+        )
         # If we found an existing auth config for this registry and username
         # combination, we can return it immediately unless reauth is requested.
         if authcfg and authcfg.get('username', None) == username \
diff --git a/docker/api/plugin.py b/docker/api/plugin.py
index 73f1852..f6c0b13 100644
--- a/docker/api/plugin.py
+++ b/docker/api/plugin.py
@@ -44,7 +44,10 @@ class PluginApiMixin(object):
         """
         url = self._url('/plugins/create')
 
-        with utils.create_archive(root=plugin_data_dir, gzip=gzip) as archv:
+        with utils.create_archive(
+            root=plugin_data_dir, gzip=gzip,
+            files=set(utils.build.walk(plugin_data_dir, []))
+        ) as archv:
             res = self._post(url, params={'name': name}, data=archv)
         self._raise_for_status(res)
         return True
@@ -167,8 +170,16 @@ class PluginApiMixin(object):
             'remote': name,
         }
 
+        headers = {}
+        registry, repo_name = auth.resolve_repository_name(name)
+        header = auth.get_config_header(self, registry)
+        if header:
+            headers['X-Registry-Auth'] = header
+
         url = self._url('/plugins/privileges')
-        return self._result(self._get(url, params=params), True)
+        return self._result(
+            self._get(url, params=params, headers=headers), True
+        )
 
     @utils.minimum_version('1.25')
     @utils.check_resource('name')
diff --git a/docker/auth.py b/docker/auth.py
index 48fcd8b..0c0cb20 100644
--- a/docker/auth.py
+++ b/docker/auth.py
@@ -44,7 +44,9 @@ def get_config_header(client, registry):
             "No auth config in memory - loading from filesystem"
         )
         client._auth_configs = load_config()
-    authcfg = resolve_authconfig(client._auth_configs, registry)
+    authcfg = resolve_authconfig(
+        client._auth_configs, registry, credstore_env=client.credstore_env
+    )
     # Do not fail here if no authentication exists for this
     # specific registry as we can have a readonly pull. Just
     # put the header if we can.
@@ -76,7 +78,7 @@ def get_credential_store(authconfig, registry):
     )
 
 
-def resolve_authconfig(authconfig, registry=None):
+def resolve_authconfig(authconfig, registry=None, credstore_env=None):
     """
     Returns the authentication data from the given auth configuration for a
     specific registry. As with the Docker client, legacy entries in the config
@@ -91,7 +93,7 @@ def resolve_authconfig(authconfig, registry=None):
                 'Using credentials store "{0}"'.format(store_name)
             )
             cfg = _resolve_authconfig_credstore(
-                authconfig, registry, store_name
+                authconfig, registry, store_name, env=credstore_env
             )
             if cfg is not None:
                 return cfg
@@ -115,13 +117,14 @@ def resolve_authconfig(authconfig, registry=None):
     return None
 
 
-def _resolve_authconfig_credstore(authconfig, registry, credstore_name):
+def _resolve_authconfig_credstore(authconfig, registry, credstore_name,
+                                  env=None):
     if not registry or registry == INDEX_NAME:
         # The ecosystem is a little schizophrenic with index.docker.io VS
         # docker.io - in that case, it seems the full URL is necessary.
         registry = INDEX_URL
     log.debug("Looking for auth entry for {0}".format(repr(registry)))
-    store = dockerpycreds.Store(credstore_name)
+    store = dockerpycreds.Store(credstore_name, environment=env)
     try:
         data = store.get(registry)
         res = {
diff --git a/docker/client.py b/docker/client.py
index b4364c3..8d4a52b 100644
--- a/docker/client.py
+++ b/docker/client.py
@@ -33,6 +33,8 @@ class DockerClient(object):
             :py:class:`~docker.tls.TLSConfig` object to use custom
             configuration.
         user_agent (str): Set a custom user agent for requests to the server.
+        credstore_env (dict): Override environment variables when calling the
+            credential store process.
     """
     def __init__(self, *args, **kwargs):
         self.api = APIClient(*args, **kwargs)
@@ -66,6 +68,8 @@ class DockerClient(object):
             assert_hostname (bool): Verify the hostname of the server.
             environment (dict): The environment to read environment variables
                 from. Default: the value of ``os.environ``
+            credstore_env (dict): Override environment variables when calling
+                the credential store process.
 
         Example:
 
@@ -77,8 +81,9 @@ class DockerClient(object):
         """
         timeout = kwargs.pop('timeout', DEFAULT_TIMEOUT_SECONDS)
         version = kwargs.pop('version', None)
-        return cls(timeout=timeout, version=version,
-                   **kwargs_from_env(**kwargs))
+        return cls(
+            timeout=timeout, version=version, **kwargs_from_env(**kwargs)
+        )
 
     # Resources
     @property
diff --git a/docker/models/networks.py b/docker/models/networks.py
index 1c2fbf2..be3291a 100644
--- a/docker/models/networks.py
+++ b/docker/models/networks.py
@@ -211,5 +211,5 @@ class NetworkCollection(Collection):
         return networks
 
     def prune(self, filters=None):
-        self.client.api.prune_networks(filters=filters)
+        return self.client.api.prune_networks(filters=filters)
     prune.__doc__ = APIClient.prune_networks.__doc__
diff --git a/docker/models/services.py b/docker/models/services.py
index 125896b..458d2c8 100644
--- a/docker/models/services.py
+++ b/docker/models/services.py
@@ -126,7 +126,7 @@ class Service(Model):
 
         service_mode = ServiceMode('replicated', replicas)
         return self.client.api.update_service(self.id, self.version,
-                                              service_mode,
+                                              mode=service_mode,
                                               fetch_current_spec=True)
 
     def force_update(self):
@@ -276,7 +276,7 @@ CONTAINER_SPEC_KWARGS = [
     'labels',
     'mounts',
     'open_stdin',
-    'privileges'
+    'privileges',
     'read_only',
     'secrets',
     'stop_grace_period',
diff --git a/docker/transport/unixconn.py b/docker/transport/unixconn.py
index cc35d00..c59821a 100644
--- a/docker/transport/unixconn.py
+++ b/docker/transport/unixconn.py
@@ -1,14 +1,10 @@
 import six
 import requests.adapters
 import socket
+from six.moves import http_client as httplib
 
 from .. import constants
 
-if six.PY3:
-    import http.client as httplib
-else:
-    import httplib
-
 try:
     import requests.packages.urllib3 as urllib3
 except ImportError:
diff --git a/docker/types/daemon.py b/docker/types/daemon.py
index 852f3d8..ee8624e 100644
--- a/docker/types/daemon.py
+++ b/docker/types/daemon.py
@@ -57,6 +57,8 @@ class CancellableStream(object):
 
             else:
                 sock = sock_fp._sock
+            if isinstance(sock, urllib3.contrib.pyopenssl.WrappedSocket):
+                sock = sock.socket
 
             sock.shutdown(socket.SHUT_RDWR)
             sock.close()
diff --git a/docker/types/services.py b/docker/types/services.py
index 09eb05e..31f4750 100644
--- a/docker/types/services.py
+++ b/docker/types/services.py
@@ -82,7 +82,7 @@ class ContainerSpec(dict):
         args (:py:class:`list`): Arguments to the command.
         hostname (string): The hostname to set on the container.
         env (dict): Environment variables.
-        dir (string): The working directory for commands to run in.
+        workdir (string): The working directory for commands to run in.
         user (string): The user inside the container.
         labels (dict): A map of labels to associate with the service.
         mounts (:py:class:`list`): A list of specifications for mounts to be
diff --git a/docker/utils/socket.py b/docker/utils/socket.py
index 0945f0a..7b96d4f 100644
--- a/docker/utils/socket.py
+++ b/docker/utils/socket.py
@@ -1,6 +1,7 @@
 import errno
 import os
 import select
+import socket as pysocket
 import struct
 
 import six
@@ -28,6 +29,8 @@ def read(socket, n=4096):
     try:
         if hasattr(socket, 'recv'):
             return socket.recv(n)
+        if six.PY3 and isinstance(socket, getattr(pysocket, 'SocketIO')):
+            return socket.read(n)
         return os.read(socket.fileno(), n)
     except EnvironmentError as e:
         if e.errno not in recoverable_errors:
diff --git a/docker/version.py b/docker/version.py
index 8f6e651..c504327 100644
--- a/docker/version.py
+++ b/docker/version.py
@@ -1,2 +1,2 @@
-version = "3.3.0"
+version = "3.4.0"
 version_info = tuple([int(d) for d in version.split("-")[0].split(".")])
diff --git a/docs/change-log.md b/docs/change-log.md
index 0065c62..5a0d55a 100644
--- a/docs/change-log.md
+++ b/docs/change-log.md
@@ -1,6 +1,31 @@
 Change log
 ==========
 
+3.4.0
+-----
+
+[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/51?closed=1)
+
+### Features
+
+* The `APIClient` and `DockerClient` constructors now accept a `credstore_env`
+  parameter. When set, values in this dictionary are added to the environment
+  when executing the credential store process.
+
+### Bugfixes
+
+* `DockerClient.networks.prune` now properly returns the operation's result
+* Fixed a bug that caused custom Dockerfile paths in a subfolder of the build
+  context to be invalidated, preventing these builds from working
+* The `plugin_privileges` method can now be called for plugins requiring
+  authentication to access
+* Fixed a bug that caused attempts to read a data stream over an unsecured TCP
+  socket to crash on Windows clients
+* Fixed a bug where using the `read_only` parameter when creating a service using
+  the `DockerClient` was being ignored
+* Fixed an issue where `Service.scale` would not properly update the service's
+  mode, causing the operation to fail silently
+
 3.3.0
 -----
 
diff --git a/requirements.txt b/requirements.txt
index 9079315..6c5e7d0 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,7 +3,7 @@ asn1crypto==0.22.0
 backports.ssl-match-hostname==3.5.0.1
 cffi==1.10.0
 cryptography==1.9
-docker-pycreds==0.2.3
+docker-pycreds==0.3.0
 enum34==1.1.6
 idna==2.5
 ipaddress==1.0.18
diff --git a/setup.py b/setup.py
index c1eabcf..57b2b5a 100644
--- a/setup.py
+++ b/setup.py
@@ -13,7 +13,7 @@ requirements = [
     'requests >= 2.14.2, != 2.18.0',
     'six >= 1.4.0',
     'websocket-client >= 0.32.0',
-    'docker-pycreds >= 0.2.3'
+    'docker-pycreds >= 0.3.0'
 ]
 
 extras_require = {
diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py
index 92e0062..baaf33e 100644
--- a/tests/integration/api_build_test.py
+++ b/tests/integration/api_build_test.py
@@ -415,18 +415,20 @@ class BuildTest(BaseAPIIntegrationTest):
             f.write('hello world')
         with open(os.path.join(base_dir, '.dockerignore'), 'w') as f:
             f.write('.dockerignore\n')
-        df = tempfile.NamedTemporaryFile()
-        self.addCleanup(df.close)
-        df.write(('\n'.join([
-            'FROM busybox',
-            'COPY . /src',
-            'WORKDIR /src',
-        ])).encode('utf-8'))
-        df.flush()
+        df_dir = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, df_dir)
+        df_name = os.path.join(df_dir, 'Dockerfile')
+        with open(df_name, 'wb') as df:
+            df.write(('\n'.join([
+                'FROM busybox',
+                'COPY . /src',
+                'WORKDIR /src',
+            ])).encode('utf-8'))
+            df.flush()
         img_name = random_name()
         self.tmp_imgs.append(img_name)
         stream = self.client.build(
-            path=base_dir, dockerfile=df.name, tag=img_name,
+            path=base_dir, dockerfile=df_name, tag=img_name,
             decode=True
         )
         lines = []
@@ -472,6 +474,39 @@ class BuildTest(BaseAPIIntegrationTest):
             [b'.', b'..', b'file.txt', b'custom.dockerfile']
         ) == sorted(lsdata)
 
+    def test_build_in_context_nested_dockerfile(self):
+        base_dir = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, base_dir)
+        with open(os.path.join(base_dir, 'file.txt'), 'w') as f:
+            f.write('hello world')
+        subdir = os.path.join(base_dir, 'hello', 'world')
+        os.makedirs(subdir)
+        with open(os.path.join(subdir, 'custom.dockerfile'), 'w') as df:
+            df.write('\n'.join([
+                'FROM busybox',
+                'COPY . /src',
+                'WORKDIR /src',
+            ]))
+        img_name = random_name()
+        self.tmp_imgs.append(img_name)
+        stream = self.client.build(
+            path=base_dir, dockerfile='hello/world/custom.dockerfile',
+            tag=img_name, decode=True
+        )
+        lines = []
+        for chunk in stream:
+            lines.append(chunk)
+        assert 'Successfully tagged' in lines[-1]['stream']
+
+        ctnr = self.client.create_container(img_name, 'ls -a')
+        self.tmp_containers.append(ctnr)
+        self.client.start(ctnr)
+        lsdata = self.client.logs(ctnr).strip().split(b'\n')
+        assert len(lsdata) == 4
+        assert sorted(
+            [b'.', b'..', b'file.txt', b'hello']
+        ) == sorted(lsdata)
+
     def test_build_in_context_abs_dockerfile(self):
         base_dir = tempfile.mkdtemp()
         self.addCleanup(shutil.rmtree, base_dir)
diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py
index afd439f..ff70148 100644
--- a/tests/integration/api_container_test.py
+++ b/tests/integration/api_container_test.py
@@ -491,6 +491,9 @@ class CreateContainerTest(BaseAPIIntegrationTest):
         assert rule in self.client.logs(ctnr).decode('utf-8')
 
 
+@pytest.mark.xfail(
+    IS_WINDOWS_PLATFORM, reason='Test not designed for Windows platform'
+)
 class VolumeBindTest(BaseAPIIntegrationTest):
     def setUp(self):
         super(VolumeBindTest, self).setUp()
@@ -507,9 +510,6 @@ class VolumeBindTest(BaseAPIIntegrationTest):
             ['touch', os.path.join(self.mount_dest, self.filename)],
         )
 
-    @pytest.mark.xfail(
-        IS_WINDOWS_PLATFORM, reason='Test not designed for Windows platform'
-    )
     def test_create_with_binds_rw(self):
 
         container = self.run_with_volume(
@@ -525,9 +525,6 @@ class VolumeBindTest(BaseAPIIntegrationTest):
         inspect_data = self.client.inspect_container(container)
         self.check_container_data(inspect_data, True)
 
-    @pytest.mark.xfail(
-        IS_WINDOWS_PLATFORM, reason='Test not designed for Windows platform'
-    )
     def test_create_with_binds_ro(self):
         self.run_with_volume(
             False,
@@ -548,9 +545,6 @@ class VolumeBindTest(BaseAPIIntegrationTest):
         inspect_data = self.client.inspect_container(container)
         self.check_container_data(inspect_data, False)
 
-    @pytest.mark.xfail(
-        IS_WINDOWS_PLATFORM, reason='Test not designed for Windows platform'
-    )
     @requires_api_version('1.30')
     def test_create_with_mounts(self):
         mount = docker.types.Mount(
@@ -569,9 +563,6 @@ class VolumeBindTest(BaseAPIIntegrationTest):
         inspect_data = self.client.inspect_container(container)
         self.check_container_data(inspect_data, True)
 
-    @pytest.mark.xfail(
-        IS_WINDOWS_PLATFORM, reason='Test not designed for Windows platform'
-    )
     @requires_api_version('1.30')
     def test_create_with_mounts_ro(self):
         mount = docker.types.Mount(
@@ -1116,9 +1107,7 @@ class ContainerTopTest(BaseAPIIntegrationTest):
 
         self.client.start(container)
         res = self.client.top(container)
-        if IS_WINDOWS_PLATFORM:
-            assert res['Titles'] == ['PID', 'USER', 'TIME', 'COMMAND']
-        else:
+        if not IS_WINDOWS_PLATFORM:
             assert res['Titles'] == [
                 'UID', 'PID', 'PPID', 'C', 'STIME', 'TTY', 'TIME', 'CMD'
             ]
diff --git a/tests/integration/api_plugin_test.py b/tests/integration/api_plugin_test.py
index 433d44d..1150b09 100644
--- a/tests/integration/api_plugin_test.py
+++ b/tests/integration/api_plugin_test.py
@@ -135,7 +135,7 @@ class PluginTest(BaseAPIIntegrationTest):
 
     def test_create_plugin(self):
         plugin_data_dir = os.path.join(
-            os.path.dirname(__file__), 'testdata/dummy-plugin'
+            os.path.dirname(__file__), os.path.join('testdata', 'dummy-plugin')
         )
         assert self.client.create_plugin(
             'docker-sdk-py/dummy', plugin_data_dir
diff --git a/tests/integration/models_containers_test.py b/tests/integration/models_containers_test.py
index 6ddb034..ab41ea5 100644
--- a/tests/integration/models_containers_test.py
+++ b/tests/integration/models_containers_test.py
@@ -36,6 +36,9 @@ class ContainerCollectionTest(BaseIntegrationTest):
         with pytest.raises(docker.errors.ImageNotFound):
             client.containers.run("dockerpytest_does_not_exist")
 
+    @pytest.mark.skipif(
+        docker.constants.IS_WINDOWS_PLATFORM, reason="host mounts on Windows"
+    )
     def test_run_with_volume(self):
         client = docker.from_env(version=TEST_API_VERSION)
         path = tempfile.mkdtemp()
diff --git a/tests/unit/api_test.py b/tests/unit/api_test.py
index 46cbd68..af2bb1c 100644
--- a/tests/unit/api_test.py
+++ b/tests/unit/api_test.py
@@ -44,7 +44,7 @@ def response(status_code=200, content='', headers=None, reason=None, elapsed=0,
     return res
 
 
-def fake_resolve_authconfig(authconfig, registry=None):
+def fake_resolve_authconfig(authconfig, registry=None, *args, **kwargs):
     return None
 
 
@@ -365,7 +365,7 @@ class DockerApiTest(BaseAPIClientTest):
         assert result == content
 
 
-class StreamTest(unittest.TestCase):
+class UnixSocketStreamTest(unittest.TestCase):
     def setUp(self):
         socket_dir = tempfile.mkdtemp()
         self.build_context = tempfile.mkdtemp()
@@ -462,7 +462,61 @@ class StreamTest(unittest.TestCase):
                         raise e
 
             assert list(stream) == [
-                str(i).encode() for i in range(50)]
+                str(i).encode() for i in range(50)
+            ]
+
+
+class TCPSocketStreamTest(unittest.TestCase):
+    text_data = b'''
+    Now, those children out there, they're jumping through the
+    flames in the hope that the god of the fire will make them fruitful.
+    Really, you can't blame them. After all, what girl would not prefer the
+    child of a god to that of some acne-scarred artisan?
+    '''
+
+    def setUp(self):
+
+        self.server = six.moves.socketserver.ThreadingTCPServer(
+            ('', 0), self.get_handler_class()
+        )
+        self.thread = threading.Thread(target=self.server.serve_forever)
+        self.thread.setDaemon(True)
+        self.thread.start()
+        self.address = 'http://{}:{}'.format(
+            socket.gethostname(), self.server.server_address[1]
+        )
+
+    def tearDown(self):
+        self.server.shutdown()
+        self.server.server_close()
+        self.thread.join()
+
+    def get_handler_class(self):
+        text_data = self.text_data
+
+        class Handler(six.moves.BaseHTTPServer.BaseHTTPRequestHandler, object):
+            def do_POST(self):
+                self.send_response(101)
+                self.send_header(
+                    'Content-Type', 'application/vnd.docker.raw-stream'
+                )
+                self.send_header('Connection', 'Upgrade')
+                self.send_header('Upgrade', 'tcp')
+                self.end_headers()
+                self.wfile.flush()
+                time.sleep(0.2)
+                self.wfile.write(text_data)
+                self.wfile.flush()
+
+        return Handler
+
+    def test_read_from_socket(self):
+        with APIClient(base_url=self.address) as client:
+            resp = client._post(client._url('/dummy'), stream=True)
+            data = client._read_from_socket(resp, stream=True, tty=True)
+            results = b''.join(data)
+
+        assert results == self.text_data
 
 
 class UserAgentTest(unittest.TestCase):