moved base code from pytest-lockable #1

.circleci/config.yml

@@ -0,0 +1,110 @@
+# Python CircleCI 2.0 configuration file
+#
+# Check https://circleci.com/docs/2.0/language-python/ for more details
+#
+version: 2.1
+
+workflows:
+  version: 2
+  test:
+    jobs:
+      - test-37:
+          filters:
+            tags:
+              only: /.*/
+      - test-38:
+          filters:
+            tags:
+              only: /.*/
+      - test-39:
+          filters:
+            tags:
+              only: /.*/
+      - deploy:
+          requires:
+            - test-39
+            - test-38
+            - test-37
+          filters:
+            tags:
+              only: /^v.*/
+            branches:
+              ignore: /.*/
+
+commands:
+  setup:
+    steps:
+      - checkout
+      # Download and cache dependencies
+      - restore_cache:
+          key: deps1-{{ .Branch }}-{{ checksum "setup.py" }}
+      - run:
+          name: install dependencies
+          command: |
+            python --version
+            virtualenv venv
+            . venv/bin/activate;
+            pip install -e .;
+            pip install -e .[dev]
+            pip install -e .[optional]
+      - save_cache:
+          paths:
+            - ./venv
+          key: deps1-{{ .Branch }}-{{ checksum "setup.py" }}
+
+jobs:
+  test-37: &test-template
+    docker:
+      - image: circleci/python:3.7
+    working_directory: ~/lockable
+    steps:
+      - setup
+      # run tests!
+      - run:
+          name: unit tests
+          command: |
+            mkdir junit || true
+            . venv/bin/activate;
+            nosetests --with-xunit --with-coverage --cover-package=lockable --cover-html --cover-html-dir=htmlcov --cover-xml-file=coverage.xml --xunit-file=junit/results.xml
+            coveralls || true
+      - run:
+          name: pylint
+          command: |
+            . venv/bin/activate;
+            pylint lockable
+      - store_artifacts:
+          path: htmlcov
+          destination: htmlcov
+      - store_test_results:
+          path: junit
+      - store_artifacts:
+          path: junit
+          destination: juni
+
+  test-38:
+    <<: *test-template
+    docker:
+    - image: circleci/python:3.8
+
+  test-39:
+    <<: *test-template
+    docker:
+    - image: circleci/python:3.9-rc-buster
+
+  deploy:
+    <<: *test-template
+    steps:
+      - setup
+      - run:
+          name: create packages
+          command: |
+            . venv/bin/activate;
+            python setup.py sdist
+            python setup.py bdist_wheel
+      - run:
+          name: upload to pypi
+          command: |
+            . venv/bin/activate;
+            pip install twine;
+            export PATH=$HOME/.local/bin:$PATH;
+            twine upload dist/*

.gitignore

@@ -3,15 +3,9 @@ __pycache__/
 *.py[cod]
 *$py.class
 
-# C extensions
-*.so
-
 # Distribution / packaging
 .Python
-build/
-develop-eggs/
 dist/
-downloads/
 eggs/
 .eggs/
 lib/
@@ -27,11 +21,6 @@ share/python-wheels/
 *.egg
 MANIFEST
 
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
 
 # Installer logs
 pip-log.txt
@@ -50,80 +39,3 @@ coverage.xml
 *.py,cover
 .hypothesis/
 .pytest_cache/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-db.sqlite3
-db.sqlite3-journal
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# IPython
-profile_default/
-ipython_config.py
-
-# pyenv
-.python-version
-
-# pipenv
-#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
-#   However, in case of collaboration, if having platform-specific dependencies or dependencies
-#   having no cross-platform support, pipenv may install dependencies that don't work, or not
-#   install all needed dependencies.
-#Pipfile.lock
-
-# PEP 582; used by e.g. github.com/David-OConnor/pyflow
-__pypackages__/
-
-# Celery stuff
-celerybeat-schedule
-celerybeat.pid
-
-# SageMath parsed files
-*.sage.py
-
-# Environments
-.env
-.venv
-env/
-venv/
-ENV/
-env.bak/
-venv.bak/
-
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
-
-# mypy
-.mypy_cache/
-.dmypy.json
-dmypy.json
-
-# Pyre type checker
-.pyre/

README.md

@@ -1,2 +1,7 @@
-# py-lockable
-py-lockable
+# lockable
+
+Resource locking module for python.
+
+Originally designed for following projects:
+* [pytest-lockable](https://github.com/jupe/pytest-lockable)
+* [robot-lockable](https://github.com/jupe/robot-lockable)

lockable/plugin.py

@@ -0,0 +1,116 @@
+""" Lockable library """
+import random
+import json
+import os
+import sys
+from time import sleep
+from contextlib import contextmanager
+from pydash import filter_, merge, count_by
+from func_timeout import func_timeout, FunctionTimedOut
+from filelock import Timeout, FileLock
+
+
+def read_resources_list(filename):
+    """ Read resources json file """
+    with open(filename) as json_file:
+        data = json.load(json_file)
+        assert isinstance(data, list), 'data is not an list'
+        validate_json(data)
+    return data
+
+
+def validate_json(data):
+    """ Validate json data """
+    counts = count_by(data, lambda obj: obj.get('id'))
+    no_ids = filter_(counts.keys(), lambda key: key is None)
+    if no_ids:
+        raise AssertionError('Invalid json, id property is missing')
+
+    duplicates = filter_(counts.keys(), lambda key: counts[key] > 1)
+    if duplicates:
+        print(duplicates)
+        raise AssertionError(f"Invalid json, duplicate ids in {duplicates}")
+
+
+def parse_requirements(requirements_str):
+    """ Parse requirements """
+    if not requirements_str:
+        return dict()
+    try:
+        return json.loads(requirements_str)
+    except json.decoder.JSONDecodeError as jsonerror:
+        parts = requirements_str.split('&')
+        if len(parts) == 0:
+            raise ValueError('no requirements given') from jsonerror
+        requirements = dict()
+        for part in parts:
+            try:
+                part.index("=")
+            except ValueError:
+                continue
+            key, value = part.split('=')
+            requirements[key] = value
+        return requirements
+
+
+def _try_lock(candidate, lock_folder):
+    """ Function that tries to lock given candidate resource """
+    resource_id = candidate.get("id")
+    try:
+        lock_file = os.path.join(lock_folder, f"{resource_id}.lock")
+        lockable = FileLock(lock_file)
+        lockable.acquire(timeout=0)
+        print(f'Allocated resource: {resource_id}')
+
+        def release():
+            print(f'Release resource: {resource_id}')
+            lockable.release()
+            try:
+                os.remove(lock_file)
+            except OSError as error:
+                print(error, file=sys.stderr)
+        return candidate, release
+    except Timeout as error:
+        raise AssertionError('not success') from error
+
+
+@contextmanager
+def _lock_some(candidates, timeout_s, lock_folder, retry_interval):
+    """ Contextmanager that lock some candidate that is free and release it finally """
+    print(f'Total match local resources: {len(candidates)}, timeout: {timeout_s}')
+    try:
+        def doit(candidates_inner):
+            while True:
+                for candidate in candidates_inner:
+                    try:
+                        return _try_lock(candidate, lock_folder)
+                    except AssertionError:
+                        pass
+                print('trying to lock after short period')
+                sleep(retry_interval)
+
+        resource, release = func_timeout(timeout_s, doit, args=(candidates,))
+        print(f'resource {resource["id"]} allocated ({json.dumps(resource)})')
+        yield resource
+        release()
+    except FunctionTimedOut as error:
+        raise TimeoutError(f'Allocation timeout ({timeout_s}s)') from error
+
+
+@contextmanager
+def lock(requirements: dict,
+         resource_list: list,
+         timeout_s: int,
+         lock_folder: str,
+         retry_interval=1):
+    """ Lock resource context """
+    local_resources = filter_(resource_list, requirements)
+    random.shuffle(local_resources)
+    with _lock_some(local_resources, timeout_s, lock_folder, retry_interval) as resource:
+        yield resource
+
+
+def _get_requirements(requirements, hostname):
+    """ Generate requirements"""
+    print(f'hostname: {hostname}')
+    return merge(dict(hostname=hostname, online=True), requirements)