From ee8d66ca8ecbf40b011192939da890a9986f2e99 Mon Sep 17 00:00:00 2001 From: Aaron Virshup Date: Tue, 27 Feb 2018 11:45:14 -0800 Subject: [PATCH 1/4] Add test for bug --- pyccc/tests/test_job_types.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pyccc/tests/test_job_types.py b/pyccc/tests/test_job_types.py index 35faaeb..98b6661 100644 --- a/pyccc/tests/test_job_types.py +++ b/pyccc/tests/test_job_types.py @@ -456,3 +456,18 @@ def test_directory_input(fixture, request): assert job.exitcode == 0 assert job.stdout.strip() == 'a\nb' + +@pytest.mark.parametrize('fixture', fixture_types['engine']) +def test_passing_files_between_jobs(fixture, request): + engine = request.getfuncargvalue(fixture) + + # this is OK with docker but should fail with a subprocess + job1 = engine.launch(image='alpine', command='echo hello > world') + job1.wait() + assert job1.exitcode == 0 + + job2 = engine.launch(image='alpine', command='cat helloworld', + inputs={'helloworld': job1.get_output('world')}) + job2.wait() + assert job2.exitcode == 0 + assert job2.stdout.strip() == 'hello' From e51f01f4645c547bacc8a49b385d588452407da7 Mon Sep 17 00:00:00 2001 From: Aaron Virshup Date: Tue, 27 Feb 2018 11:50:02 -0800 Subject: [PATCH 2/4] Duck-typed approach to checking for local instances of file references --- pyccc/docker_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyccc/docker_utils.py b/pyccc/docker_utils.py index 6c9ee60..7c024cb 100644 --- a/pyccc/docker_utils.py +++ b/pyccc/docker_utils.py @@ -132,7 +132,7 @@ def make_tar_stream(build_context, buffer): """ tf = tarfile.TarFile(fileobj=buffer, mode='w') for context_path, fileobj in build_context.items(): - if isinstance(fileobj, (files.LocalDirectoryReference, files.LocalFile)): + if getattr(fileobj, 'localpath', None) is not None: tf.add(fileobj.localpath, arcname=context_path) else: tar_add_bytes(tf, context_path, fileobj.read('rb')) From 77153e3b4f720df06ece6d8ddef66bd5540892fc Mon Sep 17 00:00:00 2001 From: Aaron Virshup Date: Tue, 27 Feb 2018 11:52:55 -0800 Subject: [PATCH 3/4] Add test for LocalDirectoryReference objects --- pyccc/tests/test_file_refs.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pyccc/tests/test_file_refs.py b/pyccc/tests/test_file_refs.py index f9d6b0a..b5e4908 100644 --- a/pyccc/tests/test_file_refs.py +++ b/pyccc/tests/test_file_refs.py @@ -10,6 +10,8 @@ fixture_types = {} PYVERSION = sys.version_info.major +THISDIR = os.path.dirname(__file__) + if PYVERSION >= 3: unicode = str # we'll use "unicode" and "bytes" just to be completely clear @@ -115,3 +117,14 @@ def test_containers_are_pickleable(fixture, request): ctr = request.getfuncargvalue(fixture) newctr = pickle.loads(pickle.dumps(ctr)) assert newctr.read() == ctr.read() + + +def test_local_directory_reference(tmpdir): + import filecmp + src = os.path.join(THISDIR, 'data') + localdir = pyccc.files.LocalDirectoryReference(src) + target = os.path.join(tmpdir, 'data') + localdir.put(target) + match, mismatch, errors = filecmp.cmpfiles(src, target, ['a', 'b']) + assert not mismatch + assert not errors From b14f2f9466be2c91a2ee96d93b6f9177c39cc304 Mon Sep 17 00:00:00 2001 From: Aaron Virshup Date: Tue, 27 Feb 2018 12:18:38 -0800 Subject: [PATCH 4/4] Fix tmpdir type in directory ref test --- pyccc/tests/test_file_refs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyccc/tests/test_file_refs.py b/pyccc/tests/test_file_refs.py index b5e4908..077c5e5 100644 --- a/pyccc/tests/test_file_refs.py +++ b/pyccc/tests/test_file_refs.py @@ -121,6 +121,7 @@ def test_containers_are_pickleable(fixture, request): def test_local_directory_reference(tmpdir): import filecmp + tmpdir = str(tmpdir) src = os.path.join(THISDIR, 'data') localdir = pyccc.files.LocalDirectoryReference(src) target = os.path.join(tmpdir, 'data')