From 90bf3c84d2ad381702602f34a803501a688148e7 Mon Sep 17 00:00:00 2001 From: Jussi Vatjus-Anttila Date: Fri, 2 Oct 2020 06:58:32 +0300 Subject: [PATCH 1/4] moved base code from pytest-lockable --- .circleci/config.yml | 110 ++++++++++++++++++++++++++++++++++ README.md | 9 ++- lockable/__init__.py | 0 lockable/plugin.py | 111 +++++++++++++++++++++++++++++++++++ setup.py | 69 ++++++++++++++++++++++ tests/test_plugin.py | 137 +++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 434 insertions(+), 2 deletions(-) create mode 100644 .circleci/config.yml create mode 100644 lockable/__init__.py create mode 100644 lockable/plugin.py create mode 100644 setup.py create mode 100644 tests/test_plugin.py diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..69b25e1 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,110 @@ +# Python CircleCI 2.0 configuration file +# +# Check https://circleci.com/docs/2.0/language-python/ for more details +# +version: 2.1 + +workflows: + version: 2 + test: + jobs: + - test-37: + filters: + tags: + only: /.*/ + - test-38: + filters: + tags: + only: /.*/ + - test-39: + filters: + tags: + only: /.*/ + - deploy: + requires: + - test-39 + - test-38 + - test-37 + filters: + tags: + only: /^v.*/ + branches: + ignore: /.*/ + +commands: + setup: + steps: + - checkout + # Download and cache dependencies + - restore_cache: + key: deps1-{{ .Branch }}-{{ checksum "setup.py" }} + - run: + name: install dependencies + command: | + python --version + virtualenv venv + . venv/bin/activate; + pip install -e .; + pip install -e .[dev] + pip install -e .[optional] + - save_cache: + paths: + - ./venv + key: deps1-{{ .Branch }}-{{ checksum "setup.py" }} + +jobs: + test-37: &test-template + docker: + - image: circleci/python:3.7 + working_directory: ~/lockable + steps: + - setup + # run tests! + - run: + name: unit tests + command: | + mkdir junit || true + . venv/bin/activate; + nosetests --with-xunit --with-coverage --cover-package=lockable --cover-html --cover-html-dir=htmlcov --cover-xml-file=coverage.xml --xunit-file=junit/results.xml + coveralls || true + - run: + name: pylint + command: | + . venv/bin/activate; + pylint lockable + - store_artifacts: + path: htmlcov + destination: htmlcov + - store_test_results: + path: junit + - store_artifacts: + path: junit + destination: juni + + test-38: + <<: *test-template + docker: + - image: circleci/python:3.8 + + test-39: + <<: *test-template + docker: + - image: circleci/python:3.9-rc-buster + + deploy: + <<: *test-template + steps: + - setup + - run: + name: create packages + command: | + . venv/bin/activate; + python setup.py sdist + python setup.py bdist_wheel + - run: + name: upload to pypi + command: | + . venv/bin/activate; + pip install twine; + export PATH=$HOME/.local/bin:$PATH; + twine upload dist/* diff --git a/README.md b/README.md index d1881ac..c3dba10 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,7 @@ -# py-lockable -py-lockable +# lockable + +Resource locking module for python. + +Originally designed for following projects: +* [pytest-lockable](https://github.com/jupe/pytest-lockable) +* [robot-lockable](https://github.com/jupe/robot-lockable) diff --git a/lockable/__init__.py b/lockable/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lockable/plugin.py b/lockable/plugin.py new file mode 100644 index 0000000..b384c49 --- /dev/null +++ b/lockable/plugin.py @@ -0,0 +1,111 @@ +""" Lockable library """ +import random +import json +import os +import sys +from time import sleep +from contextlib import contextmanager +from pydash import filter_, merge, count_by +from func_timeout import func_timeout, FunctionTimedOut +from filelock import Timeout, FileLock + + +def read_resources_list(filename): + """ Read resources json file """ + with open(filename) as json_file: + data = json.load(json_file) + assert isinstance(data, list), 'data is not an list' + validate_json(data) + return data + + +def validate_json(data): + counts = count_by(data, lambda obj: obj.get('id')) + no_ids = filter_(counts.keys(), lambda key: key is None) + if no_ids: + raise AssertionError('Invalid json, id property is missing') + + duplicates = filter_(counts.keys(), lambda key: counts[key] > 1) + if duplicates: + print(duplicates) + raise AssertionError(f"Invalid json, duplicate ids in {duplicates}") + + +def parse_requirements(requirements_str): + """ Parse requirements """ + if not requirements_str: + return dict() + try: + return json.loads(requirements_str) + except json.decoder.JSONDecodeError: + parts = requirements_str.split('&') + if len(parts) == 0: + raise ValueError('no requirements given') + requirements = dict() + for part in parts: + try: + part.index("=") + except ValueError: + continue + key, value = part.split('=') + requirements[key] = value + return requirements + + +def _try_lock(candidate, lock_folder): + """ Function that tries to lock given candidate resource """ + resource_id = candidate.get("id") + try: + lock_file = os.path.join(lock_folder, f"{resource_id}.lock") + lockable = FileLock(lock_file) + lockable.acquire(timeout=0) + print(f'Allocated resource: {resource_id}') + + def release(): + print(f'Release resource: {resource_id}') + lockable.release() + try: + os.remove(lock_file) + except OSError as error: + print(error, file=sys.stderr) + return candidate, release + except Timeout: + raise AssertionError('not success') + + +@contextmanager +def _lock_some(candidates, timeout_s, lock_folder, retry_interval): + """ Contextmanager that lock some candidate that is free and release it finally """ + print(f'Total match local resources: {len(candidates)}, timeout: {timeout_s}') + try: + def doit(candidates_inner): + while True: + for candidate in candidates_inner: + try: + return _try_lock(candidate, lock_folder) + except AssertionError: + pass + print('trying to lock after short period') + sleep(retry_interval) + + resource, release = func_timeout(timeout_s, doit, args=(candidates,)) + print(f'resource {resource["id"]} allocated ({json.dumps(resource)})') + yield resource + release() + except FunctionTimedOut: + raise TimeoutError(f'Allocation timeout ({timeout_s}s)') + + +@contextmanager +def lock(requirements: dict, resource_list: list, timeout_s: int, lock_folder: str, retry_interval=1): + """ Lock resource context """ + local_resources = filter_(resource_list, requirements) + random.shuffle(local_resources) + with _lock_some(local_resources, timeout_s, lock_folder, retry_interval) as resource: + yield resource + + +def _get_requirements(requirements, hostname): + """ Generate requirements""" + print(f'hostname: {hostname}') + return merge(dict(hostname=hostname, online=True), requirements) diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..72433fb --- /dev/null +++ b/setup.py @@ -0,0 +1,69 @@ +"""A setuptools based setup module. +See: +https://packaging.python.org/guides/distributing-packages-using-setuptools/ +https://github.com/pypa/sampleproject +""" + +# Always prefer setuptools over distutils +from setuptools import setup, find_packages +from os import path +# io.open is needed for projects that support Python 2.7 +# It ensures open() defaults to text mode with universal newlines, +# and accepts an argument to specify the text encoding +# Python 3 only projects can skip this import +from io import open + +here = path.abspath(path.dirname(__file__)) + +# Get the long description from the README file +with open(path.join(here, 'README.md'), encoding='utf-8') as f: + long_description = f.read() + +setup( + name='lockable', + use_scm_version=True, + setup_requires=["setuptools_scm"], + description='lockable resource module', + long_description=long_description, + long_description_content_type='text/markdown', + url='https://github.com/jupe/py-lockable', + author='Jussi Vatjus-Anttila', + author_email='jussiva@gmail.com', + + # Classifiers help users find your project by categorizing it. + # For a list of valid classifiers, see https://pypi.org/classifiers/ + classifiers=[ # Optional + 'Development Status :: 3 - Alpha', + "Intended Audience :: Developers", + # Indicate who your project is intended for + 'Intended Audience :: Developers', + "Operating System :: POSIX", + "Operating System :: Microsoft :: Windows", + "Operating System :: MacOS :: MacOS X", + "Topic :: Software Development :: Quality Assurance", + "Topic :: Software Development :: Testing", + "Topic :: Utilities", + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.7', + "Programming Language :: Python :: 3 :: Only", + ], + packages=find_packages(exclude=['tests']), # Required + keywords="py.test pytest lockable resource", + # Specify which Python versions you support. + python_requires='>=3.7, <4', + install_requires=[ + 'func_timeout', + 'filelock', + 'pydash' + ], + extras_require={ # Optional + 'dev': ['nose', 'coveralls', 'pylint', 'coverage'], + 'optional': ['pytest-metadata'] + }, + + project_urls={ # Optional + 'Bug Reports': 'https://github.com/jupe/pytest-lockable', + 'Source': 'https://github.com/jupe/pytest-lockable/', + } +) diff --git a/tests/test_plugin.py b/tests/test_plugin.py new file mode 100644 index 0000000..91c0914 --- /dev/null +++ b/tests/test_plugin.py @@ -0,0 +1,137 @@ +# pylint: disable=missing-function-docstring,missing-class-docstring +""" Unit test for lockable pytest plugin """ +import json +import unittest +import os +import socket +import time +from nose.tools import nottest +import multiprocessing as mp +from tempfile import mktemp +from contextlib import contextmanager +from lockable.plugin import read_resources_list, parse_requirements, lock, validate_json + + +HOSTNAME = socket.gethostname() + + +@contextmanager +def tmp_file(data): + filename = mktemp() + with open(filename, 'w') as file: + file.write(data) + yield filename + os.unlink(filename) + + +@nottest +def run_test(index, req, devices, duration, timeout): + print(f'waiting for test {index} resources') + with lock(requirements=req, resource_list=devices, lock_folder='.', timeout_s=timeout, retry_interval=0.1): + print(f'Run test {index}') + time.sleep(duration) + return index + + +class TestLockable(unittest.TestCase): + + def test_read_resources_list(self): + data = [] + with tmp_file(json.dumps(data)) as filename: + data = read_resources_list(filename) + self.assertIsInstance(data, list) + for obj in data: + self.assertIsInstance(obj, dict) + + def test_parse_requirements(self): + self.assertEqual(parse_requirements(""), dict()) + self.assertEqual(parse_requirements("a"), dict()) + self.assertEqual(parse_requirements("a=b"), dict(a='b')) + self.assertEqual(parse_requirements("a=b&b=2&"), dict(a='b', b="2")) + self.assertEqual(parse_requirements('{"a":"c"}'), dict(a='c')) + self.assertEqual(parse_requirements('{"a":"c", "d":2}'), dict(a='c', d=2)) + + def test_lock_success(self): + requirements = dict(id=1) + resource_list = [dict(hostame='12', id='1', online=True), dict(id=1, hostname=HOSTNAME, online=True)] + with lock(requirements=requirements, resource_list=resource_list, lock_folder='.', timeout_s=1) as resource: + self.assertTrue(os.path.exists('1.lock')) + self.assertEqual(resource, resource_list[1]) + self.assertFalse(os.path.exists('1.lock')) + + def test_not_available(self): + requirements = dict(id=1) + resource_list = [dict(hostame='12', id='1', online=True)] + try: + with lock(requirements=requirements, resource_list=resource_list, + lock_folder='.', timeout_s=0.1): + raise AssertionError('did not raise TimeoutError') + except TimeoutError: + pass + self.assertFalse(os.path.exists('1.lock')) + + def test_not_online(self): + requirements = dict(id=1) + resource_list = [dict(hostame=HOSTNAME, id='1', online=False)] + try: + with lock(requirements=requirements, resource_list=resource_list, + lock_folder='.', timeout_s=0.1): + raise AssertionError('did not raise TimeoutError') + except TimeoutError: + pass + self.assertFalse(os.path.exists('1.lock')) + + def test_raise_pending_timeout(self): + requirements = dict(id=1) + resource_list = [dict(id=1, hostname=HOSTNAME, online=True)] + with lock(requirements=requirements, resource_list=resource_list, lock_folder='.', timeout_s=1): + self.assertTrue(os.path.exists('1.lock')) + try: + with lock(requirements=requirements, resource_list=resource_list, lock_folder='.', + timeout_s=0.1): + pass + except TimeoutError: + self.assertTrue(os.path.exists('1.lock')) + else: + raise AssertionError('did not raise TimeoutError') + self.assertFalse(os.path.exists('1.lock')) + + def test_wait_pending_success(self): + requirements = dict(id=1) + resource_list = [dict(id=1, hostname=HOSTNAME, online=True)] + parallel_count = 5 + pool = mp.Pool(parallel_count) + + results = [] + + for index in range(parallel_count): + results.append(pool.apply_async(run_test, + args=(index, + requirements, + resource_list, + 0.1, # duration + 5 # allocation timeout + ))) + + pool.close() + pool.join() + + results = [result.get() for result in results] + results.sort() # we don't care for now how was first - just that all got resolved + + expected = list(range(parallel_count)) + self.assertEqual(results, expected) + + def test_valid_json(self): + data = [{"id": "12345"}] + validate_json(data) + + def test_duplicate_id_in_json(self): + data = [{"id": "1234"}, {"id": "12345"}, {"id": "12345"}] + with self.assertRaises(AssertionError): + validate_json(data) + + def test_missing_id_in_json(self): + data = [{"a": "1234"}, {"id": "12345"}, {"id": "123456"}] + with self.assertRaises(AssertionError): + validate_json(data) From 909cab58e88d82482a4b5d625be17b520f93917c Mon Sep 17 00:00:00 2001 From: Jussi Vatjus-Anttila Date: Fri, 2 Oct 2020 07:07:13 +0300 Subject: [PATCH 2/4] update gitignore --- .gitignore | 88 ------------------------------------------------------ 1 file changed, 88 deletions(-) diff --git a/.gitignore b/.gitignore index b6e4761..79247d8 100644 --- a/.gitignore +++ b/.gitignore @@ -3,15 +3,9 @@ __pycache__/ *.py[cod] *$py.class -# C extensions -*.so - # Distribution / packaging .Python -build/ -develop-eggs/ dist/ -downloads/ eggs/ .eggs/ lib/ @@ -27,11 +21,6 @@ share/python-wheels/ *.egg MANIFEST -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec # Installer logs pip-log.txt @@ -50,80 +39,3 @@ coverage.xml *.py,cover .hypothesis/ .pytest_cache/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -.python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ From 3ff704f38c2cab7bee3984ae37116df5ede9a638 Mon Sep 17 00:00:00 2001 From: Jussi Vatjus-Anttila Date: Fri, 2 Oct 2020 07:15:06 +0300 Subject: [PATCH 3/4] pylint cleanup --- lockable/plugin.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/lockable/plugin.py b/lockable/plugin.py index b384c49..29ff489 100644 --- a/lockable/plugin.py +++ b/lockable/plugin.py @@ -20,6 +20,7 @@ def read_resources_list(filename): def validate_json(data): + """ Validate json data """ counts = count_by(data, lambda obj: obj.get('id')) no_ids = filter_(counts.keys(), lambda key: key is None) if no_ids: @@ -37,10 +38,10 @@ def parse_requirements(requirements_str): return dict() try: return json.loads(requirements_str) - except json.decoder.JSONDecodeError: + except json.decoder.JSONDecodeError as jsonerror: parts = requirements_str.split('&') if len(parts) == 0: - raise ValueError('no requirements given') + raise ValueError('no requirements given') from jsonerror requirements = dict() for part in parts: try: @@ -97,7 +98,11 @@ def doit(candidates_inner): @contextmanager -def lock(requirements: dict, resource_list: list, timeout_s: int, lock_folder: str, retry_interval=1): +def lock(requirements: dict, + resource_list: list, + timeout_s: int, + lock_folder: str, + retry_interval=1): """ Lock resource context """ local_resources = filter_(resource_list, requirements) random.shuffle(local_resources) From 7c63fe4d9a9e56f58e467e230bf6567e329203fb Mon Sep 17 00:00:00 2001 From: Jussi Vatjus-Anttila Date: Fri, 2 Oct 2020 07:16:35 +0300 Subject: [PATCH 4/4] more pylint cleanup --- lockable/plugin.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lockable/plugin.py b/lockable/plugin.py index 29ff489..7b08e5d 100644 --- a/lockable/plugin.py +++ b/lockable/plugin.py @@ -70,8 +70,8 @@ def release(): except OSError as error: print(error, file=sys.stderr) return candidate, release - except Timeout: - raise AssertionError('not success') + except Timeout as error: + raise AssertionError('not success') from error @contextmanager @@ -93,8 +93,8 @@ def doit(candidates_inner): print(f'resource {resource["id"]} allocated ({json.dumps(resource)})') yield resource release() - except FunctionTimedOut: - raise TimeoutError(f'Allocation timeout ({timeout_s}s)') + except FunctionTimedOut as error: + raise TimeoutError(f'Allocation timeout ({timeout_s}s)') from error @contextmanager