# # Copyright (C) 2019 Bloomberg Finance LP # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2 of the License, or (at your option) any later version. # # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public # License along with this library. If not, see . # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name from contextlib import contextmanager import os import pytest import grpc from buildstream.testing import cli # pylint: disable=unused-import from tests.testutils import create_artifact_share, assert_shared # Project directory DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",) @contextmanager def limit_grpc_message_length(limit): orig_insecure_channel = grpc.insecure_channel def new_insecure_channel(target): return orig_insecure_channel(target, options=(("grpc.max_send_message_length", limit),)) grpc.insecure_channel = new_insecure_channel try: yield finally: grpc.insecure_channel = orig_insecure_channel @pytest.mark.datafiles(DATA_DIR) def test_large_directory(cli, tmpdir, datafiles): project = str(datafiles) # Number of files chosen to ensure the complete list of digests exceeds # our 1 MB gRPC message limit. I.e., test message splitting. MAX_MESSAGE_LENGTH = 1024 * 1024 NUM_FILES = MAX_MESSAGE_LENGTH // 64 + 1 large_directory_dir = os.path.join(project, "files", "large-directory") os.mkdir(large_directory_dir) for i in range(NUM_FILES): with open(os.path.join(large_directory_dir, str(i)), "w") as f: # The files need to have different content as we want different digests. f.write(str(i)) with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: # Configure bst to push to the artifact share cli.configure({"artifacts": [{"url": share.repo, "push": True},]}) # Enforce 1 MB gRPC message limit with limit_grpc_message_length(MAX_MESSAGE_LENGTH): # Build and push result = cli.run(project=project, args=["build", "import-large-directory.bst"]) result.assert_success() # Assert that we are now cached locally assert cli.get_element_state(project, "import-large-directory.bst") == "cached" # Assert that the push was successful assert_shared(cli, share, project, "import-large-directory.bst")