diff --git a/pyccc/engines/dockerengine.py b/pyccc/engines/dockerengine.py index 8b82945..e6df50c 100644 --- a/pyccc/engines/dockerengine.py +++ b/pyccc/engines/dockerengine.py @@ -41,7 +41,6 @@ def __init__(self, client=None, workingdir='/default_wdir'): self.default_wdir = workingdir self.hostname = self.client.base_url - def connect_to_docker(self, client=None): if isinstance(client, basestring): client = du.get_docker_apiclient(client) @@ -74,16 +73,48 @@ def submit(self, job): job.workingdir = self.default_wdir job.imageid = du.create_provisioned_image(self.client, job.image, job.workingdir, job.inputs) - cmdstring = "sh -c '%s'" % job.command - job.container = self.client.create_container(job.imageid, - command=cmdstring, - working_dir=job.workingdir, - environment={'PYTHONIOENCODING':'utf-8'}) + container_args = self._generate_container_args(job) + + job.container = self.client.create_container(job.imageid, **container_args) self.client.start(job.container) job.containerid = job.container['Id'] job.jobid = job.containerid + def _generate_container_args(self, job): + container_args = dict(command="sh -c '%s'" % job.command, + working_dir=job.workingdir, + environment={'PYTHONIOENCODING':'utf-8'}) + + if job.engine_options: + volumes = [] + binds = [] + + # mount the docker socket into the container + if job.engine_options.get('mount_docker_socket', False): + volumes.append('/var/run/docker.sock') + binds.append('/var/run/docker.sock:/var/run/docker.sock:rw') + + # handle other mounted volumes + for volume, mount in job.engine_options.get('volumes', {}).items(): + if isinstance(mount, (list, tuple)): + mountpoint, mode = mount + bind = '%s:%s:%s' % (volume, mountpoint, mode) + else: + mountpoint = mount + mode = None + bind = '%s:%s' % (volume, mountpoint) + + volumes.append(mountpoint) + binds.append(bind) + + if volumes or binds: + container_args['volumes'] = volumes + container_args['host_config'] = self.client.create_host_config(binds=binds) + + return container_args + + def wait(self, job): return self.client.wait(job.container) diff --git a/pyccc/job.py b/pyccc/job.py index 3afd99d..04a92b7 100644 --- a/pyccc/job.py +++ b/pyccc/job.py @@ -68,6 +68,7 @@ class Job(object): locally once, when this job completes numcpus (int): number of CPUs required (default:1) runtime (int): kill job if the runtime exceeds this value (in seconds) (default: 1 hour)` + engine_options (dict): additional engine-specific options """ def __init__(self, engine=None, image=None, @@ -79,12 +80,14 @@ def __init__(self, engine=None, numcpus=1, runtime=3600, on_status_update=None, - when_finished=None): + when_finished=None, + engine_options=None): self.name = name self.engine = engine self.image = image self.command = if_not_none(command, '') + self.engine_options = engine_options self.inputs = inputs if self.inputs is not None: # translate strings into file objects diff --git a/pyccc/tests/test_job_types.py b/pyccc/tests/test_job_types.py index e8f0534..ca7769c 100644 --- a/pyccc/tests/test_job_types.py +++ b/pyccc/tests/test_job_types.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +import os import sys import pytest import pyccc @@ -344,3 +345,55 @@ def _runcall(fixture, request, function, *args, **kwargs): job = engine.launch(image=PYIMAGE, command=fn, interpreter=PYVERSION) job.wait() return job.result + +@pytest.mark.skipif('CI_PROJECT_ID' in os.environ, + reason="Can't mount docker socket in codeship") +def test_docker_socket_mount(local_docker_engine): + engine = local_docker_engine + + job = engine.launch(image='docker', + command='docker ps -q --no-trunc', + engine_options={'mount_docker_socket': True}) + job.wait() + running = job.stdout.strip().splitlines() + assert job.jobid in running + + +def test_docker_volume_mount(local_docker_engine): + """ + Note: + The test context is not necessarily the same as the bind mount context! + These tests will run in containers themselves, so we can't assume + that any directories accessible to the tests are bind-mountable. + + Therefore we just test a named volume here. + """ + import subprocess, uuid + engine = local_docker_engine + key = uuid.uuid4() + volname = 'my-mounted-volume-%s' % key + + # Create a named volume with a file named "keyfile" containing a random uuid4 + subprocess.check_call(('docker volume rm {vn}; docker volume create {vn}; ' + 'docker run -v {vn}:/mounted alpine sh -c "echo {k} > /mounted/keyfile"') + .format(vn=volname, k=key), + shell=True) + + job = engine.launch(image='docker', + command='cat /mounted/keyfile', + engine_options={'volumes': {volname: '/mounted'}}) + job.wait() + result = job.stdout.strip() + assert result == str(key) + + +def test_readonly_docker_volume_mount(local_docker_engine): + engine = local_docker_engine + mountdir = '/tmp' + job = engine.launch(image='docker', + command='echo blah > /mounted/blah', + engine_options={'volumes': + {mountdir: ('/mounted', 'ro')}}) + job.wait() + assert isinstance(job.exitcode, int) + assert job.exitcode != 0 diff --git a/requirements.txt b/requirements.txt index df476d1..e3edd39 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ chardet -docker > 2.0 +docker >=2.0,<3.0 funcsigs ; python_version < '3.3' future requests