From 7d421638a02fb68c1240edaafb22ffd817922206 Mon Sep 17 00:00:00 2001 From: Sergey Klyuykov Date: Tue, 15 Nov 2022 10:49:16 +0400 Subject: [PATCH 1/8] Fix: Remove useless `PIP_USE_FEATURE`. --- tox.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/tox.ini b/tox.ini index 745d885b..6655ff05 100644 --- a/tox.ini +++ b/tox.ini @@ -116,7 +116,6 @@ setenv = CCACHE_DIR = {envdir}/.ccache BUILD_OPTIMIZATION = true BUILD_COMPILE = true - PIP_USE_FEATURE = 2020-resolver passenv = * whitelist_externals = * commands = From 15e7042372f5e86d39b344d08ca34ad96859dd6b Mon Sep 17 00:00:00 2001 From: Sergey Klyuykov Date: Tue, 15 Nov 2022 13:37:53 +0400 Subject: [PATCH 2/8] Chore(deps): Update minimal vstutils to 5.1.7 --- requirements-doc.txt | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-doc.txt b/requirements-doc.txt index 3dffda61..2915c2ec 100644 --- a/requirements-doc.txt +++ b/requirements-doc.txt @@ -1,2 +1,2 @@ # Docs -vstutils[doc]~=5.1.6 +vstutils[doc]~=5.1.7 diff --git a/requirements.txt b/requirements.txt index b2c4605a..120c00d8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # Main -vstutils[rpc,ldap,doc,prod]~=5.1.6 +vstutils[rpc,ldap,doc,prod]~=5.1.7 docutils~=0.16.0 markdown2~=2.4.0 From 78739238435dd3fc56ac715b4dff4f205c424a21 Mon Sep 17 00:00:00 2001 From: Vladislav Korenkov Date: Thu, 1 Dec 2022 19:52:10 +1000 Subject: [PATCH 3/8] Feature: Execution plugins --- Dockerfile | 3 +- doc/api.rst | 2 +- doc/api_schema.yaml | 74 +-- doc/conf.py | 14 +- doc/config.rst | 29 +- doc/contribute.rst | 2 +- doc/gui.rst | 3 +- doc/index.rst | 1 + doc/plugins.rst | 110 ++++ frontend_src/inventory/InventoryFieldEdit.vue | 2 +- .../inventory/InventoryFieldReadonly.vue | 1 + frontend_src/inventory/index.js | 1 + frontend_src/project/detail/RunPlaybook.vue | 2 +- frontend_src/project/execute.js | 8 +- frontend_src/templates/index.js | 9 +- polemarch/__init__.py | 2 +- polemarch/api/v2/serializers.py | 20 +- polemarch/api/v2/views.py | 40 +- polemarch/api/v3/serializers.py | 97 ++- polemarch/api/v3/views.py | 68 ++- polemarch/main/acl/handlers.py | 6 +- polemarch/main/constants.py | 90 ++- polemarch/main/executions.py | 5 + polemarch/main/models/__init__.py | 24 +- polemarch/main/models/hosts.py | 5 +- polemarch/main/models/projects.py | 48 +- polemarch/main/models/tasks.py | 101 ++-- polemarch/main/models/utils.py | 454 +++++--------- polemarch/main/openapi.py | 11 +- polemarch/main/settings.py | 42 +- polemarch/main/tasks/tasks.py | 35 +- polemarch/main/utils.py | 82 ++- polemarch/plugins/__init__.py | 0 polemarch/plugins/ansible.py | 184 ++++++ polemarch/plugins/base.py | 209 +++++++ polemarch/settings.ini | 9 + polemarch/translations/ru.py | 1 + requirements-doc.txt | 4 +- requirements.txt | 2 +- setup.py | 16 +- tests.py | 559 ++++++++++++++---- tox.ini | 2 - yarn.lock | 30 +- 43 files changed, 1614 insertions(+), 793 deletions(-) create mode 100644 doc/plugins.rst create mode 100644 polemarch/main/executions.py create mode 100644 polemarch/plugins/__init__.py create mode 100644 polemarch/plugins/ansible.py create mode 100644 polemarch/plugins/base.py diff --git a/Dockerfile b/Dockerfile index 6a6007d5..8815d302 100644 --- a/Dockerfile +++ b/Dockerfile @@ -61,7 +61,8 @@ RUN --mount=type=cache,target=/var/cache/apt \ libldap-2.4-2 \ libsasl2-2 \ libffi7 \ - libssl1.1 && \ + libssl1.1 \ + openssh-client \ python3.8 -m pip install cryptography paramiko 'pip<22' && \ ln -s /usr/bin/python3.8 /usr/bin/python && \ mkdir -p /projects /hooks /run/openldap /etc/polemarch/hooks /var/lib/polemarch && \ diff --git a/doc/api.rst b/doc/api.rst index 5fc1b5b2..ce033c92 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -24,6 +24,6 @@ In Polemarch we have some entities that can be nested to another entities. Below **User** can be nested into **Team** -For add entities into another, you only need send ``[{"id": [instance_id]}, ...]`` to subpath. Also you can insert instead of data results of bulk request, inner mechanism add all entities in result to parent entity. +To add entities into another, you only need send ``[{"id": [instance_id]}, ...]`` to subpath. Also you can insert instead of data results of bulk request, inner mechanism add all entities in result to parent entity. .. vst_openapi:: ./api_schema.yaml diff --git a/doc/api_schema.yaml b/doc/api_schema.yaml index 4cce3d68..3f7b4454 100644 --- a/doc/api_schema.yaml +++ b/doc/api_schema.yaml @@ -69,7 +69,7 @@ info: x-versions: application: 2.1.2 library: 2.1.2 - vstutils: 5.1.1 + vstutils: 5.1.10 django: 3.2.16 djangorestframework: 3.14.0 drf_yasg: 1.21.4 @@ -5376,13 +5376,13 @@ paths: /project/{id}/execute_module/: post: operationId: project_execute_module - description: Execute `ansible -m [module]` with arguments. + description: Execute module plugin. parameters: - name: data in: body required: true schema: - $ref: '#/definitions/AnsibleModule' + $ref: '#/definitions/ExecuteModule' responses: '201': description: CREATED @@ -5401,13 +5401,13 @@ paths: /project/{id}/execute_playbook/: post: operationId: project_execute_playbook - description: Execute `ansible-playbook` with arguments. + description: Execute playbook plugin. parameters: - name: data in: body required: true schema: - $ref: '#/definitions/AnsiblePlaybook' + $ref: '#/definitions/ExecutePlaybook' responses: '201': description: CREATED @@ -11440,7 +11440,7 @@ definitions: - readme_content - execute_view_data x-view-field-name: name - AnsibleModule: + ExecuteModule: required: - module type: object @@ -11467,7 +11467,6 @@ definitions: title: Become description: run operations with become (does not imply password prompting) type: boolean - default: false become_method: title: Become method description: privilege escalation method to use (default=sudo), use `ansible-doc @@ -11478,7 +11477,6 @@ definitions: description: don't make any changes; instead, try to predict some of the changes that may occur type: boolean - default: false connection: title: Connection description: connection type to use (default=smart) @@ -11488,7 +11486,6 @@ definitions: description: when changing (small) files and templates, show the differences in those files; works great with --check type: boolean - default: false extra_vars: title: Extra vars description: set additional variables as key=value or YAML/JSON, if filename @@ -11512,12 +11509,10 @@ definitions: title: List hosts description: outputs a list of matching hosts; does not execute anything else type: boolean - default: false one_line: title: One line description: condense output type: boolean - default: false playbook_dir: title: Playbook dir description: Since this tool does not use playbooks, use this as a substitute @@ -11556,7 +11551,6 @@ definitions: title: Syntax check description: perform a syntax check on the playbook, but do not execute it type: boolean - default: false timeout: title: Timeout description: override the connection timeout in seconds (default=10) @@ -11581,7 +11575,6 @@ definitions: title: Verbose description: verbose mode (-vvv for more, -vvvv to enable connection debugging) type: integer - default: 0 maximum: 4 group: title: Group @@ -11646,7 +11639,7 @@ definitions: - history_id - executor x-view-field-name: history_id - AnsiblePlaybook: + ExecutePlaybook: required: - playbook type: object @@ -11669,7 +11662,6 @@ definitions: title: Become description: run operations with become (does not imply password prompting) type: boolean - default: false become_method: title: Become method description: privilege escalation method to use (default=sudo), use `ansible-doc @@ -11680,7 +11672,6 @@ definitions: description: don't make any changes; instead, try to predict some of the changes that may occur type: boolean - default: false connection: title: Connection description: connection type to use (default=smart) @@ -11690,7 +11681,6 @@ definitions: description: when changing (small) files and templates, show the differences in those files; works great with --check type: boolean - default: false extra_vars: title: Extra vars description: set additional variables as key=value or YAML/JSON, if filename @@ -11700,12 +11690,10 @@ definitions: title: Flush cache description: clear the fact cache for every host in inventory type: boolean - default: false force_handlers: title: Force handlers description: run handlers even if a task fails type: boolean - default: false forks: title: Forks description: specify number of parallel processes to use (default=5) @@ -11724,17 +11712,14 @@ definitions: title: List hosts description: outputs a list of matching hosts; does not execute anything else type: boolean - default: false list_tags: title: List tags description: list all available tags type: boolean - default: false list_tasks: title: List tasks description: list all tasks that would be executed type: boolean - default: false private_key: title: Private key description: use this file to authenticate the connection @@ -11771,12 +11756,10 @@ definitions: title: Step description: 'one-step-at-a-time: confirm each task before running' type: boolean - default: false syntax_check: title: Syntax check description: perform a syntax check on the playbook, but do not execute it type: boolean - default: false tags: title: Tags description: only run plays and tasks tagged with these values @@ -11801,7 +11784,6 @@ definitions: title: Verbose description: verbose mode (-vvv for more, -vvvv to enable connection debugging) type: integer - default: 0 maximum: 4 x-properties-groups: '': @@ -11892,7 +11874,17 @@ definitions: inventory: title: Inventory type: string - format: inventory + format: dynamic + x-options: + choices: {} + field: kind + types: + Module: + type: string + format: inventory + Task: + type: string + format: inventory data: title: Data type: string @@ -12237,7 +12229,17 @@ definitions: inventory: title: Inventory type: string - format: inventory + format: dynamic + x-options: + choices: {} + field: kind + types: + Module: + type: string + format: inventory + Task: + type: string + format: inventory data: title: Data type: string @@ -13829,7 +13831,6 @@ definitions: description: run operations with become (does not imply password prompting) type: boolean - default: false become_method: title: Become method description: privilege escalation method to use (default=sudo), @@ -13840,7 +13841,6 @@ definitions: description: don't make any changes; instead, try to predict some of the changes that may occur type: boolean - default: false connection: title: Connection description: connection type to use (default=smart) @@ -13850,7 +13850,6 @@ definitions: description: when changing (small) files and templates, show the differences in those files; works great with --check type: boolean - default: false extra_vars: title: Extra vars description: set additional variables as key=value or YAML/JSON, @@ -13873,12 +13872,10 @@ definitions: description: outputs a list of matching hosts; does not execute anything else type: boolean - default: false one_line: title: One line description: condense output type: boolean - default: false playbook_dir: title: Playbook dir description: Since this tool does not use playbooks, use this @@ -13922,7 +13919,6 @@ definitions: description: perform a syntax check on the playbook, but do not execute it type: boolean - default: false timeout: title: Timeout description: override the connection timeout in seconds (default=10) @@ -13948,7 +13944,6 @@ definitions: description: verbose mode (-vvv for more, -vvvv to enable connection debugging) type: integer - default: 0 maximum: 4 PLAYBOOK: format: dynamic @@ -13965,7 +13960,6 @@ definitions: description: run operations with become (does not imply password prompting) type: boolean - default: false become_method: title: Become method description: privilege escalation method to use (default=sudo), @@ -13976,7 +13970,6 @@ definitions: description: don't make any changes; instead, try to predict some of the changes that may occur type: boolean - default: false connection: title: Connection description: connection type to use (default=smart) @@ -13986,7 +13979,6 @@ definitions: description: when changing (small) files and templates, show the differences in those files; works great with --check type: boolean - default: false extra_vars: title: Extra vars description: set additional variables as key=value or YAML/JSON, @@ -13996,12 +13988,10 @@ definitions: title: Flush cache description: clear the fact cache for every host in inventory type: boolean - default: false force_handlers: title: Force handlers description: run handlers even if a task fails type: boolean - default: false forks: title: Forks description: specify number of parallel processes to use (default=5) @@ -14015,17 +14005,14 @@ definitions: description: outputs a list of matching hosts; does not execute anything else type: boolean - default: false list_tags: title: List tags description: list all available tags type: boolean - default: false list_tasks: title: List tasks description: list all tasks that would be executed type: boolean - default: false private_key: title: Private key description: use this file to authenticate the connection @@ -14067,13 +14054,11 @@ definitions: title: Step description: 'one-step-at-a-time: confirm each task before running' type: boolean - default: false syntax_check: title: Syntax check description: perform a syntax check on the playbook, but do not execute it type: boolean - default: false tags: title: Tags description: only run plays and tasks tagged with these values @@ -14099,7 +14084,6 @@ definitions: description: verbose mode (-vvv for more, -vvvv to enable connection debugging) type: integer - default: 0 maximum: 4 kind: name: kind diff --git a/doc/conf.py b/doc/conf.py index d5712ef9..26ab1cf9 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -21,11 +21,16 @@ import sys sys.path.insert(0, os.path.abspath('../')) import vstutils +from django import setup +from vstutils.environment import prepare_environment try: import polemarch except ImportError: import pmlib as polemarch +prepare_environment() +setup() + # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. @@ -36,6 +41,8 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ + 'sphinx.ext.autodoc', + 'sphinx_autodoc_typehints', 'sphinxcontrib.httpdomain', 'vstutils.api.doc_generator', 'sphinx.ext.autosectionlabel', @@ -188,4 +195,9 @@ extlinks = { 'wiki': ('https://en.wikipedia.org/wiki/%s', None), 'django_docs': ('https://docs.djangoproject.com/en/3.2/ref/%s', None), -} \ No newline at end of file +} + +set_type_checking_flag = True +typehints_fully_qualified = True +always_document_param_types = True +autodoc_inherit_docstrings = False diff --git a/doc/config.rst b/doc/config.rst index d66e6d87..c3a99405 100644 --- a/doc/config.rst +++ b/doc/config.rst @@ -415,4 +415,31 @@ This section contains additional information for configure additional elements. #. We strictly do not recommend running the web server from root. Use HTTP proxy to run on privileged ports. -.. note:: If you need more options you can find it in :doc:`vstutils:config` in the official vstutils documentation . +.. note:: If you need more options you can find it in :doc:`vstutils:config` in the official vstutils documentation. + + +.. _plugins_config: + +Execution plugins +----------------- + +To connect a plugin to Polemarch, there should be a section + +.. sourcecode:: ini + + [plugin.] + backend = import.path.to.plugin.Class + +Where + +* **** - name that will available in API to work with +* **backend** - is a python import path to plugin class + +Also you may add options which will be available in plugin: + +.. sourcecode:: ini + + [plugin..options] + some_option = some_option + +To read more about plugins, please see :doc:`plugins`. diff --git a/doc/contribute.rst b/doc/contribute.rst index 51eaf364..a6b16149 100644 --- a/doc/contribute.rst +++ b/doc/contribute.rst @@ -92,7 +92,7 @@ Here is how to proceed: `this topic `_. If you are encountered this problem, one of the solutions might be: - .. sourcecode:: bash:: + .. sourcecode:: bash git config --global protocol.file.allow always diff --git a/doc/gui.rst b/doc/gui.rst index c288ca6d..10d8e2f4 100644 --- a/doc/gui.rst +++ b/doc/gui.rst @@ -174,7 +174,7 @@ Once your project status changes to "OK", you can start working with Polemarch. `this topic `_. One of the solutions might be: - .. sourcecode:: bash:: + .. sourcecode:: bash git config --global protocol.file.allow always @@ -280,7 +280,6 @@ When status of your module execution changes to "OK" you will see the next page: .. image:: new_screenshots/execute_module_3.png .. image:: new_screenshots/execute_module_4.png - Execution templates ------------------- diff --git a/doc/index.rst b/doc/index.rst index 14d8466d..abc653f0 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -57,6 +57,7 @@ We also help you via: quickstart config gui + plugins api contribute cla diff --git a/doc/plugins.rst b/doc/plugins.rst new file mode 100644 index 00000000..31be639c --- /dev/null +++ b/doc/plugins.rst @@ -0,0 +1,110 @@ +Execution plugins +================= + +Plugins system allows you to execute any external command from Polemarch. To create a plugin you need to + +* create a python class describing the argument processing and API talking logic; +* configure this plugin in your ``settings.ini``. + +Quick start +----------- + +All built-in execution backends, such as ansible playbook or module execution, use plugins (starting from 2.2.0). + +To get started let's create a simple plugin allows you to run ``echo`` command. + +.. sourcecode:: python + + from polemarch.plugins.base import BasePlugin + from rest_framework.fields import BooleanField, empty + from vstutils.api.fields import VSTCharField + + + # Any plugin must be inherited from BasePlugin + class TestEcho(BasePlugin): + # Define fields which will be used to execute this plugin and create template with + serializer_fields = { + # You can use fields either from rest_framework or vstutils + 'string': VSTCharField(), + 'n': BooleanField(default=empty, required=False, label='No trailing newlines'), + 'e': BooleanField(default=empty, required=False, label='Interpret backslash escapes'), + } + # Value of this field will be shown in history detail page as Mode + arg_shown_on_history_as_mode = 'string' + + # This is our binary from which any command starts + @property + def base_command(self): + return ['echo'] + + # This method is called by get_args method of BasePlugin for each argument received from API. As we defined + # 'string', 'n', and 'e' arguments in serializer_fields, we can get their keys and values here. + def _process_arg(self, key, value): + # As 'string' argument in this case is a positional argument, let's return just it's value, so the final + # command will be something like ['echo', 'string to output', ...] + if key == 'string': + return value + # As this guys are just boolean flags, let's return them as '-n' or '-e' accordingly + if key in ('n', 'e') and value: + return f'-{key}' + # Note, that if we received for example `n` argument with False value, we are returning None, + # means that it won't be included to execution command. But of course, you may override this behavior + # in get_args method. + +Supposing that described plugin is located at ``polemarch.plugins.custom.Echo``, let's connect it to Polemarch. +In your ``/etc/polemarch/settings.ini`` add following section: + +.. sourcecode:: ini + + [plugins.echo] + backend = polemarch.plugins.custom.Echo + +Also you may want to provide additional options directly to plugin: + +.. sourcecode:: ini + + [plugins.echo.options] + some_option = 'some_option' + +In this example `some_option` will be available in any plugin's method as ``self.config['some_option']``, as all options are initialized in the constructor. + +So it's all done! After restarting your polemarch server and resetting schema, you can check +``/project//execute_echo/``. Here you should be able to execute echo plugin as any built-in one. +Also you should be able to create a :ref:`template` with it at ``/project//execution_templates/new/``. + +If you tried executing echo plugin with flags, you may see that this flags are being outputted too. This is because +they goes after ``string`` argument. To fix this issue, we may do something like this: + +.. sourcecode:: python + + ... + + class TestEcho(BasePlugin): + ... + + def get_args(self, raw_args): + # We know that 'string' is required so no default for .pop() is needed + string_value = raw_args.pop('string') + args = super().get_args(raw_args) + # Push our string to the end of command + args += [string_value] + return args + + @property + def base_command(self): + return ['echo'] + + def _process_arg(self, key, value): + if key in ('n', 'e') and value: + return f'-{key}' + +Now if you are passing flags to execution, they should work the same except not being outputted. + +To learn more about what plugins are provide, please check API reference. + +API reference +------------- + +.. autoclass:: polemarch.plugins.base::BasePlugin + :members: + :private-members: diff --git a/frontend_src/inventory/InventoryFieldEdit.vue b/frontend_src/inventory/InventoryFieldEdit.vue index df5fe123..cd91dc03 100644 --- a/frontend_src/inventory/InventoryFieldEdit.vue +++ b/frontend_src/inventory/InventoryFieldEdit.vue @@ -41,7 +41,7 @@ :data="{ [$parent.fkField.name]: realValue }" type="edit" style="flex: 2" - @set-value="({ value }) => setValue({ type: 'fk', value })" + @set-value="({ field, value, ...args }) => $emit('set-value', { type: 'fk', value }, args)" /> diff --git a/frontend_src/inventory/index.js b/frontend_src/inventory/index.js index 1ece4b29..132cf7ac 100644 --- a/frontend_src/inventory/index.js +++ b/frontend_src/inventory/index.js @@ -21,6 +21,7 @@ const InventoryFieldMixin = { value_field: 'id', view_field: 'name', makeLink: true, + usePrefetch: true, }, }); field.prepareFieldForView(this.view?.path); diff --git a/frontend_src/project/detail/RunPlaybook.vue b/frontend_src/project/detail/RunPlaybook.vue index 585d4319..4d4aedb0 100644 --- a/frontend_src/project/detail/RunPlaybook.vue +++ b/frontend_src/project/detail/RunPlaybook.vue @@ -37,7 +37,7 @@ mixins: [spa.fields.base.BaseFieldMixin], data() { return { - AnsibleModule: this.$app.modelsResolver.get('AnsibleModule'), + AnsibleModule: this.$app.modelsResolver.get('ExecuteModule'), mainData: {}, argsData: {}, }; diff --git a/frontend_src/project/execute.js b/frontend_src/project/execute.js index 3d502ea8..1a422cb2 100644 --- a/frontend_src/project/execute.js +++ b/frontend_src/project/execute.js @@ -1,12 +1,12 @@ export function setupExecuteViews() { - const executingViews = ['/project/{id}/execute_module/', '/project/{id}/execute_playbook/']; const HideNotRequiredMixin = { data: () => ({ hideNotRequired: true }), }; spa.signals.once('allViews.created', ({ views }) => { - for (const path of executingViews) { - const view = views.get(path); - view.mixins.push(HideNotRequiredMixin); + for (const [path, view] of views) { + if (path.startsWith('/project/{id}/execute_')) { + view.mixins.push(HideNotRequiredMixin); + } } }); } diff --git a/frontend_src/templates/index.js b/frontend_src/templates/index.js index 93fcd5d0..d1f4cf2a 100644 --- a/frontend_src/templates/index.js +++ b/frontend_src/templates/index.js @@ -31,5 +31,12 @@ spa.signals.once('app.afterInit', ({ app }) => { ['OneExecutionTemplate', 'CreateExecutionTemplate', 'CreateTemplateOption', 'OneTemplateOption'] .map((modelName) => app.modelsResolver.get(modelName)) .flatMap((model) => Object.values(model.fields.get('data').types)) - .forEach((field) => (field.nestedModel.fields.get('vars').hideNotRequired = true)); + .forEach((field) => { + const vars = field.nestedModel.fields.get('vars'); + if (vars === undefined) { + field.hideNotRequired = true; + } else { + vars.hideNotRequired = true; + } + }); }); diff --git a/polemarch/__init__.py b/polemarch/__init__.py index 11f83890..62ff2623 100644 --- a/polemarch/__init__.py +++ b/polemarch/__init__.py @@ -31,6 +31,6 @@ "VST_ROOT_URLCONF": os.getenv("VST_ROOT_URLCONF", 'vstutils.urls'), } -__version__ = "2.1.2" +__version__ = "2.2.0" prepare_environment(**default_settings) diff --git a/polemarch/api/v2/serializers.py b/polemarch/api/v2/serializers.py index 07d83eae..a8da8a20 100644 --- a/polemarch/api/v2/serializers.py +++ b/polemarch/api/v2/serializers.py @@ -10,7 +10,7 @@ from django.contrib.auth import get_user_model from django.db import transaction -from rest_framework import serializers, exceptions, fields +from rest_framework import serializers, exceptions from vstutils.api import auth as vst_auth from vstutils.api import serializers as vst_serializers, fields as vst_fields @@ -753,14 +753,11 @@ class Meta: 'execute_view_data',) -def generate_fields(ansible_reference: AnsibleArgumentsReference, ansible_type: str, no_default=False) -> OrderedDict: - if ansible_type is None: - return OrderedDict() # nocv - +def generate_fields(reference: dict, ansible_type: str, exclude=None) -> OrderedDict: fields_of_serializer = OrderedDict() - - for ref, settings in ansible_reference.raw_dict[ansible_type].items(): - if ref in {'help', 'version'}: + exclude = exclude or set() + for ref, settings in reference.items(): + if ref in {'help', 'version'} | exclude: continue ref_type = settings.get('type', None) kwargs = dict(help_text=settings.get('help', ''), required=False) @@ -790,8 +787,6 @@ def generate_fields(ansible_reference: AnsibleArgumentsReference, ansible_type: kwargs['default'] = 'all' field_name = ref.replace('-', '_') - if no_default: - kwargs['default'] = fields.empty fields_of_serializer[field_name] = field(**kwargs) return fields_of_serializer @@ -807,7 +802,10 @@ def __new__(cls, name, bases, attrs): ansible_type = 'playbook' elif isinstance(attrs.get('module', None), serializers.CharField): ansible_type = 'module' - attrs.update(generate_fields(attrs['ansible_reference'], ansible_type)) + attrs.update(generate_fields( + attrs['ansible_reference'].raw_dict[ansible_type], + ansible_type + )) return super(AnsibleSerializerMetaclass, cls).__new__(cls, name, bases, attrs) diff --git a/polemarch/api/v2/views.py b/polemarch/api/v2/views.py index d8b03b78..c8f8e450 100644 --- a/polemarch/api/v2/views.py +++ b/polemarch/api/v2/views.py @@ -10,7 +10,7 @@ from vstutils.api import auth as vst_auth from vstutils.api.permissions import StaffPermission from vstutils.api import base, serializers as vstsers, decorators as deco, responses -from vstutils.utils import KVExchanger +from vstutils.utils import KVExchanger, deprecated, translate as _ from vstutils.api.responses import HTTP_200_OK, HTTP_201_CREATED, HTTP_204_NO_CONTENT from . import filters @@ -575,8 +575,18 @@ def execute(self, request, *args, **kwargs): """ Execute template with option. """ - # returns HTTPResponse - return self.get_object().execute(request.user, request.data.get('option', None)) + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + + template = self.get_object() + history_id = template.execute(request.user, **serializer.validated_data) + + response_serializer = sers.ExecuteResponseSerializer(instance={ + 'history_id': history_id, + 'executor': request.user.id, + 'detail': _('{} plugin was executed.').format(template.get_plugin()) + }) + return HTTP_201_CREATED(response_serializer.data) class __ProjectHistoryViewSet(HistoryViewSet): @@ -658,34 +668,26 @@ def execute_module(self, request, *args, **kwargs): """ return self._execution("module", dict(request.data), request.user) + @deprecated def _execution(self, kind, data, user, **kwargs): - - template = data.pop("template", None) inventory = data.get("inventory", None) msg = "Started in the inventory {}.".format( inventory if inventory else 'specified in the project configuration.' ) instance = self.get_object() - if template is not None: - init_type = "template" - obj_id = template - msg = f'Start template [id={template}].' - else: - init_type = "project" - obj_id = instance.id - serializer = self._get_ansible_serializer(kind.lower()) - data = { - k: v for k, v in serializer.to_internal_value(data).items() - if k in data.keys() or v - } + serializer = self._get_ansible_serializer(kind.lower()) + data = { + k: v for k, v in serializer.to_internal_value(data).items() + if k in data.keys() or v + } target = data.pop(kind) try: target = str(target) except UnicodeEncodeError: # nocv target = target.encode('utf-8') + data[kind] = target history_id = instance.execute( - kind, str(target), - initiator=obj_id, initiator_type=init_type, executor=user, **data + kind.upper(), executor=user, execute_args=data ) rdata = sers.ExecuteResponseSerializer(data=dict( detail=msg, diff --git a/polemarch/api/v3/serializers.py b/polemarch/api/v3/serializers.py index 8c350caf..01a70533 100644 --- a/polemarch/api/v3/serializers.py +++ b/polemarch/api/v3/serializers.py @@ -5,56 +5,14 @@ from vstutils.utils import lazy_translate as __ from ...main import models -from ...main.constants import ExecutionTypesEnum, HiddenArgumentsEnum from ..v2.serializers import ( _WithVariablesSerializer, InventoryAutoCompletionField, ModuleSerializer, ExecuteResponseSerializer, OneProjectSerializer as V2OneProjectSerializer, - generate_fields ) -from ...main.constants import ANSIBLE_REFERENCE - - -class AnsibleArgumentsMetaSerializer(serializers.SerializerMetaclass): - @staticmethod - def __new__(mcs, name, bases, attrs): - args_type = attrs.get('type') - if args_type: - attrs.update( - generate_fields( - ansible_reference=ANSIBLE_REFERENCE, - ansible_type=args_type, - no_default=True, - ) - ) - for field in attrs.get('exclude_args', set()): - attrs[field] = None - return super().__new__(mcs, name, bases, attrs) - - -class BaseAnsibleArgumentsSerializer(BaseSerializer, metaclass=AnsibleArgumentsMetaSerializer): - def to_representation(self, instance): - representation = super().to_representation(instance) - HiddenArgumentsEnum.hide_values(representation) - return representation - - -class PlaybookAnsibleArgumentsSerializer(BaseAnsibleArgumentsSerializer): - type = 'playbook' - - -class ModuleAnsibleArgumentsSerializer(BaseAnsibleArgumentsSerializer): - type = 'module' - - -class TaskTemplateVarsSerializer(PlaybookAnsibleArgumentsSerializer): - exclude_args = {'inventory'} - - -class ModuleTemplateVarsSerializer(ModuleAnsibleArgumentsSerializer): - exclude_args = {'args', 'group', 'inventory'} +from ...main.executions import PLUGIN_HANDLERS class TaskTemplateParameters(BaseSerializer): @@ -63,7 +21,9 @@ class TaskTemplateParameters(BaseSerializer): autocomplete_property='playbook', autocomplete_represent='playbook', ) - vars = TaskTemplateVarsSerializer(required=False) + vars = PLUGIN_HANDLERS.backend('PLAYBOOK').get_serializer_class( + exclude_fields=('inventory', 'playbook') + )(required=False) class ModuleTemplateParameters(BaseSerializer): @@ -74,14 +34,35 @@ class ModuleTemplateParameters(BaseSerializer): autocomplete_represent='path' ) args = fields.CharField(label=__('Arguments'), required=False, default='', allow_blank=True) - vars = ModuleTemplateVarsSerializer(required=False) + vars = PLUGIN_HANDLERS.backend('MODULE').get_serializer_class( + exclude_fields=('args', 'group', 'inventory', 'module') + )(required=False) + + +template_kinds = ( + ('Task', 'Task'), + ('Module', 'Module'), +) + tuple( + (plugin, plugin) for plugin in PLUGIN_HANDLERS.keys() + if plugin not in {'PLAYBOOK', 'MODULE'} +) + +template_data_types = { + 'Task': TaskTemplateParameters(), + 'Module': ModuleTemplateParameters(), +} +template_data_types.update({ + plugin: backend.get_serializer_class(exclude_fields=('inventory',))(required=False) + for plugin, backend in PLUGIN_HANDLERS.items() + if plugin not in ('PLAYBOOK, MODULE') +}) class ExecutionTemplateSerializer(_WithVariablesSerializer): kind = fields.ChoiceField( - choices=ExecutionTypesEnum.to_choices(), + choices=template_kinds, required=False, - default=ExecutionTypesEnum.Task.value, + default=template_kinds[0][0], label=__('Type'), ) @@ -90,12 +71,20 @@ class Meta: fields = ['id', 'name', 'kind'] +template_inventory_types = { + 'Task': InventoryAutoCompletionField(allow_blank=True, required=False), + 'Module': InventoryAutoCompletionField(allow_blank=True, required=False), +} +template_inventory_types.update({ + plugin: InventoryAutoCompletionField(allow_blank=True, required=False) if backend.supports_inventory else 'hidden' + for plugin, backend in PLUGIN_HANDLERS.items() + if plugin not in ('PLAYBOOK', 'MODULE') +}) + + class CreateExecutionTemplateSerializer(ExecutionTemplateSerializer): - data = vst_fields.DependEnumField(field='kind', types={ - ExecutionTypesEnum.Task.value: TaskTemplateParameters(), - ExecutionTypesEnum.Module.value: ModuleTemplateParameters(), - }) - inventory = InventoryAutoCompletionField(allow_blank=True, required=False) + data = vst_fields.DependEnumField(field='kind', types=template_data_types) + inventory = vst_fields.DependEnumField(field='kind', types=template_inventory_types, required=False) class Meta(ExecutionTemplateSerializer.Meta): fields = ExecutionTemplateSerializer.Meta.fields + ['notes', 'inventory', 'data'] @@ -115,9 +104,9 @@ def create(self, validated_data): class OneExecutionTemplateSerializer(CreateExecutionTemplateSerializer): kind = fields.ChoiceField( - choices=ExecutionTypesEnum.to_choices(), + choices=template_kinds, label=__('Type'), - read_only=True + read_only=True, ) diff --git a/polemarch/api/v3/views.py b/polemarch/api/v3/views.py index 0f8a5f64..f88fd690 100644 --- a/polemarch/api/v3/views.py +++ b/polemarch/api/v3/views.py @@ -1,10 +1,13 @@ -from rest_framework import fields -from vstutils.api.decorators import nested_view +from rest_framework import fields, status +from vstutils.api.base import GenericViewSetMeta +from vstutils.api.decorators import nested_view, subaction from vstutils.api.fields import CharField, DependEnumField from vstutils.api.filters import CharFilter -from vstutils.utils import create_view +from vstutils.api.responses import HTTP_201_CREATED +from vstutils.utils import create_view, translate as _ -from ...main.models import TemplateOption, Group +from ...main.models import TemplateOption, Group, Project +from ...main.executions import PLUGIN_HANDLERS from ..v2.filters import variables_filter, vars_help from ..v2.views import ( HostViewSet, @@ -26,6 +29,7 @@ TaskTemplateParameters, ModuleTemplateParameters, OneProjectSerializer, + ExecuteResponseSerializer, ) @@ -67,13 +71,21 @@ class _ProjectInventoryViewSet(InventoryViewSet, __ProjectInventoryViewSet): __doc__ = InventoryViewSet.__doc__ +option_data = { + 'Task': TaskTemplateParameters(), + 'Module': ModuleTemplateParameters(), +} +option_data.update({ + plugin: backend.get_serializer_class()() + for plugin, backend in PLUGIN_HANDLERS.items() + if plugin not in ('PLAYBOOK, MODULE') +}) + + class CreateTemplateOptionSerializer(TemplateOption.generated_view.serializer_class_one): # pylint: disable=no-member id = fields.CharField(read_only=True) name = CharField(max_length=256) - data = DependEnumField(field='kind', types={ - 'Task': TaskTemplateParameters(), - 'Module': ModuleTemplateParameters(), - }) + data = DependEnumField(field='kind', types=option_data) class Meta: __inject_from__ = 'detail' @@ -93,10 +105,7 @@ class Meta: 'id': fields.CharField(read_only=True), 'name': fields.CharField(read_only=True), 'kind': fields.CharField(read_only=True), - 'data': DependEnumField(field='kind', types={ - 'Task': TaskTemplateParameters(), - 'Module': ModuleTemplateParameters(), - }), + 'data': DependEnumField(field='kind', types=option_data), } ) @@ -108,9 +117,42 @@ class ExecutionTemplateViewSet(__TemplateViewSet): serializer_class_create = CreateExecutionTemplateSerializer +class ProjectViewSetMeta(GenericViewSetMeta): + def __new__(mcs, name, bases, attrs): + for plugin, backend in PLUGIN_HANDLERS.items(): + action_name = f'execute_{plugin.lower()}' + + def action(self, request, plugin=plugin, *args, **kwargs): + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + project: Project = self.get_object() + + data = serializer.validated_data + history_id = project.execute(plugin, executor=request.user, execute_args=data) + + response_serializer = ExecuteResponseSerializer(instance={ + 'history_id': history_id, + 'executor': request.user.id, + 'detail': _('{} plugin was executed.').format(plugin), + }) + return HTTP_201_CREATED(response_serializer.data) + + action.__name__ = action_name + attrs[action_name] = subaction( + detail=True, + methods=['post'], + serializer_class=backend.get_serializer_class(), + response_serializer=ExecuteResponseSerializer, + response_code=status.HTTP_201_CREATED, + description=f'Execute {plugin.lower()} plugin.', + )(action) + + return super().__new__(mcs, name, bases, attrs) + + @nested_view('inventory', 'id', manager_name='inventories', allow_append=True, view=_ProjectInventoryViewSet) @nested_view('execution_templates', 'id', manager_name='template', view=ExecutionTemplateViewSet) @nested_view('template', 'id', manager_name='template', view=__TemplateViewSet, schema=None) -class ProjectViewSet(ProjectViewSetV2): +class ProjectViewSet(ProjectViewSetV2, metaclass=ProjectViewSetMeta): __doc__ = ProjectViewSetV2.__doc__ serializer_class_one = OneProjectSerializer diff --git a/polemarch/main/acl/handlers.py b/polemarch/main/acl/handlers.py index e13ac3ef..4300548a 100644 --- a/polemarch/main/acl/handlers.py +++ b/polemarch/main/acl/handlers.py @@ -1,4 +1,4 @@ -from typing import Type +from typing import Optional, Type from django.db.models import Model @@ -6,9 +6,9 @@ class Default: # pylint: disable=unused-argument qs_methods = [] - def __init__(self, model=None, instance=None): + def __init__(self, model: Optional[Type[Model]] = None, instance=None): self.instance = instance - self.model = model # type: Type[Model] + self.model = model def set_owner(self, user): ''' diff --git a/polemarch/main/constants.py b/polemarch/main/constants.py index 24c42806..d09a8195 100644 --- a/polemarch/main/constants.py +++ b/polemarch/main/constants.py @@ -8,12 +8,6 @@ ANSIBLE_REFERENCE = AnsibleArgumentsReference() -class ExecutionTypesEnum(BaseEnum): - # pylint: disable=invalid-name - Task = 'Task' - Module = 'Module' - - class BaseVariablesEnum(BaseEnum): @classmethod @lru_cache() @@ -33,45 +27,45 @@ def hide_values(cls, entries: Optional[Dict]): class InventoryVariablesEnum(BaseVariablesEnum): - ANSIBLE_HOST = 'ansible_host' - ANSIBLE_PORT = 'ansible_port' - ANSIBLE_USER = 'ansible_user' - ANSIBLE_CONNECTION = 'ansible_connection' - ANSIBLE_PASSWORD = 'ansible_password' - - ANSIBLE_SSH_PASS = 'ansible_ssh_pass' - ANSIBLE_SSH_PRIVATE_KEY_FILE = 'ansible_ssh_private_key_file' - ANSIBLE_SSH_COMMON_ARGS = 'ansible_ssh_common_args' - ANSIBLE_SFTP_EXTRA_ARGS = 'ansible_sftp_extra_args' - ANSIBLE_SCP_EXTRA_ARGS = 'ansible_scp_extra_args' - ANSIBLE_SSH_EXTRA_ARGS = 'ansible_ssh_extra_args' - ANSIBLE_SSH_EXECUTABLE = 'ansible_ssh_executable' - ANSIBLE_SSH_PIPELINING = 'ansible_ssh_pipelining' - - ANSIBLE_BECOME = 'ansible_become' - ANSIBLE_BECOME_METHOD = 'ansible_become_method' - ANSIBLE_BECOME_USER = 'ansible_become_user' - ANSIBLE_BECOME_PASS = 'ansible_become_pass' - ANSIBLE_BECOME_PASSWORD = 'ansible_become_password' - ANSIBLE_BECOME_EXE = 'ansible_become_exe' - ANSIBLE_BECOME_FLAGS = 'ansible_become_flags' - - ANSIBLE_SHELL_TYPE = 'ansible_shell_type' - ANSIBLE_PYTHON_INTERPRETER = 'ansible_python_interpreter' - ANSIBLE_RUBY_INTERPRETER = 'ansible_ruby_interpreter' - ANSIBLE_PERL_INTERPRETER = 'ansible_perl_interpreter' - ANSIBLE_SHELL_EXECUTABLE = 'ansible_shell_executable' + ANSIBLE_HOST = BaseEnum.LOWER + ANSIBLE_PORT = BaseEnum.LOWER + ANSIBLE_USER = BaseEnum.LOWER + ANSIBLE_CONNECTION = BaseEnum.LOWER + ANSIBLE_PASSWORD = BaseEnum.LOWER + + ANSIBLE_SSH_PASS = BaseEnum.LOWER + ANSIBLE_SSH_PRIVATE_KEY_FILE = BaseEnum.LOWER + ANSIBLE_SSH_COMMON_ARGS = BaseEnum.LOWER + ANSIBLE_SFTP_EXTRA_ARGS = BaseEnum.LOWER + ANSIBLE_SCP_EXTRA_ARGS = BaseEnum.LOWER + ANSIBLE_SSH_EXTRA_ARGS = BaseEnum.LOWER + ANSIBLE_SSH_EXECUTABLE = BaseEnum.LOWER + ANSIBLE_SSH_PIPELINING = BaseEnum.LOWER + + ANSIBLE_BECOME = BaseEnum.LOWER + ANSIBLE_BECOME_METHOD = BaseEnum.LOWER + ANSIBLE_BECOME_USER = BaseEnum.LOWER + ANSIBLE_BECOME_PASS = BaseEnum.LOWER + ANSIBLE_BECOME_PASSWORD = BaseEnum.LOWER + ANSIBLE_BECOME_EXE = BaseEnum.LOWER + ANSIBLE_BECOME_FLAGS = BaseEnum.LOWER + + ANSIBLE_SHELL_TYPE = BaseEnum.LOWER + ANSIBLE_PYTHON_INTERPRETER = BaseEnum.LOWER + ANSIBLE_RUBY_INTERPRETER = BaseEnum.LOWER + ANSIBLE_PERL_INTERPRETER = BaseEnum.LOWER + ANSIBLE_SHELL_EXECUTABLE = BaseEnum.LOWER class ProjectVariablesEnum(BaseVariablesEnum): - REPO_TYPE = 'repo_type' - REPO_SYNC_ON_RUN = 'repo_sync_on_run' - REPO_SYNC_ON_RUN_TIMEOUT = 'repo_sync_on_run_timeout' - REPO_BRANCH = 'repo_branch' - REPO_PASSWORD = 'repo_password' - REPO_KEY = 'repo_key' - PLAYBOOK_PATH = 'playbook_path' - CI_TEMPLATE = 'ci_template' + REPO_TYPE = BaseEnum.LOWER + REPO_SYNC_ON_RUN = BaseEnum.LOWER + REPO_SYNC_ON_RUN_TIMEOUT = BaseEnum.LOWER + REPO_BRANCH = BaseEnum.LOWER + REPO_PASSWORD = BaseEnum.LOWER + REPO_KEY = BaseEnum.LOWER + PLAYBOOK_PATH = BaseEnum.LOWER + CI_TEMPLATE = BaseEnum.LOWER class HiddenArgumentsEnum(BaseVariablesEnum): @@ -85,18 +79,6 @@ class HiddenArgumentsEnum(BaseVariablesEnum): def get_values(cls): return {x.value for x in cls} | {x.value.replace('-', '_') for x in cls} - @classmethod - @lru_cache() - def get_text_values(cls): - args = {cls.KEY_FILE.value, cls.PRIVATE_KEY.value} - return args | {x.replace('-', '_') for x in args} - - @classmethod - @lru_cache() - def get_file_values(cls): - args = {cls.VAULT_PASSWORD_FILE.value, cls.NEW_VAULT_PASSWORD_FILE.value} - return args | {x.replace('-', '_') for x in args} - class HiddenVariablesEnum(BaseVariablesEnum): ANSIBLE_SSH_PASS = InventoryVariablesEnum.ANSIBLE_SSH_PASS.value diff --git a/polemarch/main/executions.py b/polemarch/main/executions.py new file mode 100644 index 00000000..f27cca00 --- /dev/null +++ b/polemarch/main/executions.py @@ -0,0 +1,5 @@ +from django.conf import settings +from django.utils.module_loading import import_string + + +PLUGIN_HANDLERS = import_string(settings.PLUGIN_HANDLERS_CLASS)('PLUGINS', 'plugin not found') diff --git a/polemarch/main/models/__init__.py b/polemarch/main/models/__init__.py index b8fc0e70..11254efd 100644 --- a/polemarch/main/models/__init__.py +++ b/polemarch/main/models/__init__.py @@ -26,7 +26,7 @@ from ..validators import RegexValidator, validate_hostname, path_validator from ..exceptions import UnknownTypeException, Conflict from ..utils import CmdExecutor -from ...main.constants import ProjectVariablesEnum, ExecutionTypesEnum, ANSIBLE_REFERENCE +from ...main.constants import ProjectVariablesEnum, ANSIBLE_REFERENCE logger = logging.getLogger('polemarch') @@ -85,6 +85,8 @@ def check_variables_values(instance: Variable, *args, **kwargs) -> None: content_object = instance.content_object if isinstance(content_object, PeriodicTask): cmd = "module" if content_object.kind == "MODULE" else "playbook" + if instance.key == 'revision': + return # noce ANSIBLE_REFERENCE.validate_args(cmd, {instance.key: instance.value}) elif isinstance(content_object, Host): if instance.key == 'ansible_host': @@ -184,12 +186,12 @@ def validate_template_keys(instance: Template, **kwargs) -> None: if 'loaddata' in sys.argv or kwargs.get('raw', False): # nocv return errors = {} - for _, data in chain(((None, instance.data),), instance.options.items()): - for key in data.keys(): - if key not in instance.template_fields[instance.kind]: - errors[key] = ["Unknown key. Keys should be {}".format( - instance.template_fields[instance.kind] - )] + if instance.kind in instance.template_fields: + fields_to_validate = instance.template_fields[instance.kind] + for _, data in chain(((None, instance.data),), instance.options.items()): + for key in data.keys(): + if key not in fields_to_validate: + errors[key] = ["Unknown key. Keys should be {}".format(fields_to_validate)] if errors: raise drfValidationError(errors) @@ -200,10 +202,14 @@ def validate_template_args(instance: Template, **kwargs) -> None: return if instance.kind in ["Host", "Group"]: return # nocv - command = "playbook" ansible_args = dict(instance.data['vars']) - if instance.kind == "Module": + ansible_args.pop('revision', None) + if instance.kind == 'Task': + command = "playbook" + elif instance.kind == "Module": command = "module" + else: + return ANSIBLE_REFERENCE.validate_args(command, ansible_args) for _, data in dict(instance.options).items(): ANSIBLE_REFERENCE.validate_args( diff --git a/polemarch/main/models/hosts.py b/polemarch/main/models/hosts.py index dbf12066..8a156db2 100644 --- a/polemarch/main/models/hosts.py +++ b/polemarch/main/models/hosts.py @@ -19,7 +19,8 @@ from .base import models from .base import ManyToManyFieldACL, ManyToManyFieldACLReverse from .vars import AbstractModel, AbstractVarsQuerySet -from ...main import exceptions as ex, utils +from ...main import exceptions as ex +from ...main.utils import AnsibleInventoryParser from ..validators import RegexValidator logger = logging.getLogger("polemarch") @@ -228,7 +229,7 @@ class Inventory(InventoryItems): default_flow_style=False, allow_unicode=True ) - parser_class = utils.AnsibleInventoryParser + parser_class = AnsibleInventoryParser class Meta: default_related_name = "inventories" diff --git a/polemarch/main/models/projects.py b/polemarch/main/models/projects.py index 94e623ea..5fbe1682 100644 --- a/polemarch/main/models/projects.py +++ b/polemarch/main/models/projects.py @@ -30,6 +30,7 @@ from .hooks import Hook from ..utils import AnsibleModules, AnsibleConfigParser, SubCacheInterface + logger = logging.getLogger("polemarch") HISTORY_ID = TypeVar('HISTORY_ID', int, None) # pylint: disable=C0103 @@ -87,6 +88,7 @@ class Project(AbstractModel): objects = ProjectQuerySet.as_manager() repo_handlers = objects._queryset_class.repo_handlers task_handlers = objects._queryset_class.task_handlers + repository = models.CharField(max_length=2 * 1024) status = models.CharField(max_length=32, default="NEW") inventories = ManyToManyFieldACL(hosts_models.Inventory, blank=True) @@ -110,14 +112,6 @@ class ReadMe(ReadMe): 'repo_sync_on_run' ] - EXTRA_OPTIONS = { - 'initiator': 0, - 'initiator_type': 'project', - 'executor': None, - 'save_result': True, - 'template_option': None - } - PM_YAML_FORMATS = { 'unknown': str, 'string': str, @@ -266,36 +260,20 @@ def check_path(self, inventory) -> None: return path_validator(inventory) - def _prepare_kw(self, kind: str, mod_name: str, inventory=None, **extra) -> Dict: - if not mod_name: # nocv - raise PMException("Empty playbook/module name.") - history, extra = self.history.all().start( - self, kind, mod_name, inventory, **extra - ) - kwargs = dict( - target=mod_name, inventory=inventory, - history=history, project=self - ) - kwargs.update(extra) - return kwargs - def hook(self, when, msg) -> None: Hook.objects.all().execute(when, msg) - def execute(self, kind: str, *args, **extra) -> HISTORY_ID: - sync = extra.pop("sync", False) - if self.status != "OK" and not sync: - raise self.SyncError("ERROR project not synchronized") - kind = kind.upper() - task_class = self.task_handlers.backend(kind) - - kwargs = self._prepare_kw(kind, *args, **extra) - history = kwargs['history'] - if sync: - task_class(**kwargs) - else: - task_class.delay(**kwargs) - return history.id if history is not None else history + def execute(self, plugin: str, execute_args, **kwargs) -> HISTORY_ID: + from ..executions import PLUGIN_HANDLERS # pylint: disable=import-outside-toplevel + + return PLUGIN_HANDLERS.execute( + plugin, + self, + execute_args=execute_args, + initiator=self.id, + initiator_type='project', + **kwargs + ) def set_status(self, status) -> None: self.status = status diff --git a/polemarch/main/models/tasks.py b/polemarch/main/models/tasks.py index b565e019..2ae76f75 100644 --- a/polemarch/main/models/tasks.py +++ b/polemarch/main/models/tasks.py @@ -1,7 +1,7 @@ # pylint: disable=protected-access,no-member from __future__ import unicode_literals -from typing import Any, Dict, List, Tuple, Iterable, TypeVar, Text +from typing import Any, Dict, List, Iterable, Optional, TypeVar, Text import logging from collections import OrderedDict from datetime import timedelta, datetime @@ -18,9 +18,6 @@ from django.contrib.auth import get_user_model from django.db.models import functions as dbfunc, Count from django.utils.timezone import now -from django.test import Client -from django.conf import settings -from rest_framework.exceptions import UnsupportedMediaType from vstutils.custom_model import ListModel, CustomQuerySet from vstutils.utils import translate as _ @@ -71,7 +68,7 @@ def get_options_data(self) -> Dict: def get_data(self) -> Dict: data = json.loads(self.template_data) - if "inventory" in self.template_fields[self.kind] and self.inventory: + if self.inventory is not None: try: data['inventory'] = int(self.inventory) except ValueError: @@ -88,7 +85,7 @@ def inventory_object(self) -> InvOrString: self.project.check_path(self.data['inventory']) return self.data['inventory'] - def get_data_with_options(self, option: str, **extra) -> Dict[str, Any]: + def get_data_with_options(self, option: Optional[str], **extra) -> Dict[str, Any]: data = self.get_data() option_data = self.get_option_data(option) option_vars = option_data.pop("vars", {}) @@ -99,25 +96,24 @@ def get_data_with_options(self, option: str, **extra) -> Dict[str, Any]: data.update(extra) return data - def execute(self, user: User, option: str = None, **extra): - # pylint: disable=protected-access - tp = self._exec_types.get(self.kind, None) - if tp is None: - raise UnsupportedMediaType(media_type=self.kind) # nocv - client = Client(SERVER_NAME='TEMPLATE') - client.force_login(user) - url = "/{}/{}/project/{}/execute_{}/".format( - getattr(settings, 'API_URL'), getattr(settings, 'VST_API_VERSION'), - self.project.id, tp + def get_plugin(self): + plugin = self.kind.upper() + if self.kind in self._exec_types: + plugin = self._exec_types[self.kind].upper() + return plugin + + def execute(self, user: User, option: Optional[str] = None, **kwargs): + from ...main.executions import PLUGIN_HANDLERS # pylint: disable=import-outside-toplevel + + return PLUGIN_HANDLERS.execute( + self.get_plugin(), + project=self.project, + execute_args=self.get_data_with_options(option, **kwargs), + executor=user, + initiator=self.id, + initiator_type='template', + template_option=option, ) - data = dict( - template=self.id, template_option=option, - **self.get_data_with_options(option, **extra) - ) - response = client.post( - url, data=json.dumps(data), content_type="application/json" - ) - return response def ci_run(self): self.execute(self.project.owner) @@ -161,7 +157,7 @@ def set_options_data(self, value: Any) -> None: def set_data(self, value) -> None: data = self._convert_to_data(value) - data['vars'] = self.keep_encrypted_data(data.get('vars', None)) + data['vars'] = self.keep_encrypted_data(data.get('vars', {})) self.template_data = json.dumps(data) @property @@ -326,24 +322,30 @@ def get_schedule(self) -> Any: return float(self.schedule) def execute(self, sync: bool = True) -> HISTORY_ID: + from ...main.executions import PLUGIN_HANDLERS # pylint: disable=import-outside-toplevel + + execute_args = {} kwargs = dict( sync=sync, save_result=self.save_result, initiator=self.id, initiator_type="scheduler" ) + if self.kind == 'PLAYBOOK': + execute_args['playbook'] = self.mode + elif self.kind == 'MODULE': + execute_args['module'] = self.mode + if self.kind != 'TEMPLATE': - args = [self.kind, self.mode, self.inventory] - kwargs.update(self.vars) + plugin = self.kind + execute_args['inventory'] = self.inventory + execute_args.update(self.vars) else: data = self.template.get_data_with_options(self.template_opt) data.pop('inventory', None) - kind = self.template._exec_types[self.template.kind] - args = [ - kind.upper(), - data.pop(kind), - self.template.inventory_object - ] - kwargs.update(data) - return self.project.execute(*args, **kwargs) + data['inventory'] = self.template.inventory_object + plugin = self.template.get_plugin() + execute_args.update(data) + + return PLUGIN_HANDLERS.execute(plugin, self.project, execute_args, **kwargs) class HistoryQuerySet(BQuerySet): @@ -381,35 +383,6 @@ def stats(self, last: int) -> OrderedDict: result['year'] = self._get_history_stats_by(qs, 'year') return result - def __get_extra_options(self, extra, options) -> Dict[str, Any]: - return {opt: extra.pop(opt, options[opt]) for opt in options} - - def __get_additional_options(self, extra_options) -> Dict[str, Any]: - options = {} - if extra_options['template_option'] is not None: - options['template_option'] = extra_options['template_option'] - return options - - def start(self, project, kind, mod_name, inventory, **extra) -> Tuple[Any, Dict]: - extra_options = self.__get_extra_options(extra, project.EXTRA_OPTIONS) - if not extra_options['save_result']: - return None, extra - history_kwargs = dict( - mode=mod_name, start_time=timezone.now(), - inventory=inventory, project=project, - kind=kind, raw_stdout="", execute_args=extra, - initiator=extra_options['initiator'], - initiator_type=extra_options['initiator_type'], - executor=extra_options['executor'], hidden=project.hidden, - options=self.__get_additional_options(extra_options) - ) - if isinstance(inventory, str): - history_kwargs['inventory'] = None - extra['inventory'] = inventory - elif isinstance(inventory, int): - history_kwargs['inventory'] = project.inventories.get(pk=inventory) # nocv - return self.create(status="DELAY", **history_kwargs), extra - class History(BModel): ansi_escape = re.compile(r'\x1b[^m]*m') diff --git a/polemarch/main/models/utils.py b/polemarch/main/models/utils.py index 9d6c7584..5eb8ff04 100644 --- a/polemarch/main/models/utils.py +++ b/polemarch/main/models/utils.py @@ -1,9 +1,8 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals import threading -from typing import Text, Any, Iterable, Tuple, List, Dict, Union +from typing import Any, Iterable, Type, Union, Optional import os -import re import time import signal import shutil @@ -11,27 +10,19 @@ import tempfile import traceback from pathlib import Path -from collections import namedtuple, OrderedDict +from collections import OrderedDict from subprocess import Popen -from functools import reduce from django.apps import apps from django.utils import timezone -from vstutils.utils import tmp_file, KVExchanger, raise_context -from vstutils.tools import get_file_value +from vstutils.utils import KVExchanger from .hosts import Inventory from .tasks import History, Project -from ...main.utils import CmdExecutor, PMObject -from ..constants import HiddenArgumentsEnum, HiddenVariablesEnum, CYPHER, ANSIBLE_REFERENCE +from ...main.utils import CmdExecutor +from ...plugins.base import BasePlugin logger = logging.getLogger("polemarch") -InventoryDataType = Tuple[Text, List] -PolemarchInventory = namedtuple("PolemarchInventory", "raw keys") -AnsibleExtra = namedtuple('AnsibleExtraArgs', [ - 'args', - 'files', -]) # Classes and methods for support @@ -40,25 +31,25 @@ class DummyHistory: def __init__(self, *args, **kwargs): self.mode = kwargs.get('mode', None) - def __setattr__(self, key: Text, value: Any) -> None: + def __setattr__(self, key: str, value: Any) -> None: if key == 'raw_args': logger.info(value) - def __getattr__(self, item: Text) -> None: + def __getattr__(self, item: str) -> None: return None # nocv @property - def raw_stdout(self) -> Text: + def raw_stdout(self) -> str: return "" # nocv @raw_stdout.setter - def raw_stdout(self, value: Text) -> None: + def raw_stdout(self, value: str) -> None: logger.info(value) # nocv - def get_hook_data(self, when: Text) -> None: + def get_hook_data(self, when: str) -> None: return None - def write_line(self, value: Text, number: int, endl: Text = ''): # nocv + def write_line(self, value: str, number: int, endl: str = ''): # nocv # pylint: disable=unused-argument logger.info(value) @@ -67,17 +58,13 @@ def save(self) -> None: class Executor(CmdExecutor): - __slots__ = 'history', 'counter', 'exchanger', 'notificator', 'notificator_lock', 'notification_last_time' + __slots__ = ('history', 'counter', 'exchanger', 'notificator', 'notificator_lock', 'notification_last_time') - def __init__(self, history: History): + def __init__(self, history: Union[History, DummyHistory]): super().__init__() self.history = history self.counter = 0 self.exchanger = KVExchanger(self.CANCEL_PREFIX + str(self.history.id)) - env_vars = {} - if self.history.project is not None: - env_vars = self.history.project.env_vars - self.env = env_vars if isinstance(history, DummyHistory): self.notificator = None else: @@ -87,7 +74,7 @@ def __init__(self, history: History): self.notification_last_time = 0 @property - def output(self) -> Text: + def output(self) -> str: # Optimize for better performance. return '' @@ -116,23 +103,18 @@ def working_handler(self, proc: Popen): self.notification_last_time = time.time() self.notificator.send() - def write_output(self, line: Text): + def write_output(self, line: str): self.counter += 1 self.history.write_line(line, self.counter, '\n') if self.notificator: with self.notificator_lock: self.notificator.create_notification_from_instance(self.history) - def execute(self, cmd: Iterable[Text], cwd: Text): + def execute(self, cmd: Iterable[str], cwd: str, env_vars: dict): + self.env = env_vars pm_ansible_path = ' '.join(self.pm_ansible()) - new_cmd = [] - for one_cmd in cmd: - if isinstance(one_cmd, str): - with raise_context(): - one_cmd = one_cmd.decode('utf-8') - new_cmd.append(one_cmd) - self.history.raw_args = " ".join(new_cmd).replace(pm_ansible_path, '').lstrip() - ret = super().execute(new_cmd, cwd) + self.history.raw_args = " ".join(cmd).replace(pm_ansible_path, '').lstrip() + ret = super().execute(cmd, cwd) if self.notificator: self.notificator.disconnect_all() with self.notificator_lock: @@ -141,241 +123,98 @@ def execute(self, cmd: Iterable[Text], cwd: Text): return ret -class AnsibleCommand(PMObject): - ref_types = { - 'ansible-playbook': 'playbook', - 'ansible': 'module', - } - command_type = None - - status_codes = { - 4: "OFFLINE", - -9: "INTERRUPTED", - -15: "INTERRUPTED", - "other": "ERROR" - } - - class ExecutorClass(Executor): - ''' - Default executor class. - ''' - - class Inventory(object): - def __init__(self, inventory: Union[Inventory, int, Text], cwd: Text = "/tmp", tmpdir: Text = '/tmp'): - self.cwd = cwd - self.tmpdir = tmpdir - self._file = None - self.is_file = True - if isinstance(inventory, str): - self.raw, self.keys = self.get_from_file(inventory) - else: - self.raw, self.keys = self.get_from_int(inventory) - - def get_from_int(self, inventory: Union[Inventory, int]) -> InventoryDataType: - if isinstance(inventory, int): - inventory = Inventory.objects.get(pk=inventory) # nocv - return inventory.get_inventory() - - def get_from_file(self, inventory: Text) -> InventoryDataType: - _file = "{}/{}".format(self.cwd, inventory) - try: - new_filename = os.path.join(self.tmpdir, 'inventory') - shutil.copyfile(_file, new_filename) - if not os.path.exists(new_filename): - raise IOError # nocv - self._file = new_filename - return get_file_value(new_filename, ''), [] - except IOError: - self._file = inventory - self.is_file = False - return inventory.replace(',', '\n'), [] - - @property - def file(self) -> Union[tmp_file, Text]: - self._file = self._file or tmp_file(self.raw, dir=self.tmpdir) - return self._file - - @property - def file_name(self) -> Text: - # pylint: disable=no-member - if isinstance(self.file, str): - return self.file - return self.file.name - - def close(self) -> None: - # pylint: disable=no-member - map(lambda key_file: key_file.close(), self.keys) if self.keys else None - if not isinstance(self.file, str): - self._file.close() +class ProjectProxy: + __slots__ = ('__project__',) + + allowed_attrs = ( + 'config', + 'revision', + 'branch', + 'project_branch', + 'vars', + 'env_vars', + 'type', + 'repo_sync_on_run', + 'repo_sync_timeout', + ) + + def __init__(self, project: Project): + self.__project__ = project + + def __getattr__(self, name: str): + if name in self.allowed_attrs: + return getattr(self.__project__, name) + raise AttributeError(f'allowed attributes are {self.allowed_attrs}') + + +class PluginExecutor: + __slots__ = ( + 'project', + 'history', + 'raw_exec_args', + 'verbose_level', + '__execution_dir__', + 'plugin', + 'executor', + ) + + executor_class = Executor + + def __init__( + self, + plugin_class: Type[BasePlugin], + plugin_options: dict, + project: Project, + history: Optional[History], + exec_args, + ): + self.project = project + self.history = history or DummyHistory() + self.raw_exec_args = exec_args + self.__execution_dir__ = None + self.plugin = plugin_class(plugin_options, ProjectProxy(project), self.verbose_output) + self.verbose_level = self.plugin.get_verbose_level(exec_args) + + def execute(self) -> None: + try: + revision = self.get_execution_revision() + self.history.revision = revision or 'NO VCS' + self.project.repo_class.make_run_copy(str(self.execution_dir), revision) + + self.executor = self.executor_class(self.history) + cmd, env_vars, self.history.raw_inventory = self.plugin.get_execution_data( + self.execution_dir, + self.raw_exec_args + ) - def __init__(self, *args, **kwargs): - self.args = args - if 'verbose' in kwargs: - kwargs['verbose'] = int(float(kwargs.get('verbose', 0))) - self.kwargs = kwargs - self.__will_raise_exception = False - self.ref_type = self.ref_types[self.command_type] - self.ansible_ref = ANSIBLE_REFERENCE.raw_dict[self.ref_type] - self.verbose = kwargs.get('verbose', 0) - self.cwd = tempfile.mkdtemp() - self._verbose_output('Execution tmpdir created - [{}].'.format(self.cwd), 0) - self.env = {} - - def _verbose_output(self, value: Text, level: int = 3) -> None: - if self.verbose >= level: - if hasattr(self, 'executor'): - self.executor.write_output(value) - logger.debug(value) - - def _get_tmp_name(self) -> Text: - return os.path.join(self.cwd, 'project_sources') - - def _send_hook(self, when: Text, **kwargs) -> None: - msg = OrderedDict() - msg['execution_type'] = self.history.kind - msg['when'] = when - inventory = self.history.inventory - if isinstance(inventory, Inventory): - inventory = inventory.get_hook_data(when) - msg['target'] = OrderedDict() - msg['target']['name'] = self.history.mode - msg['target']['inventory'] = inventory - msg['target']['project'] = self.project.get_hook_data(when) - msg['history'] = self.history.get_hook_data(when) - msg['extra'] = kwargs - self.project.hook(when, msg) + self.history.status = 'RUN' + self.history.save() - def __generate_arg_file(self, value: Text) -> Tuple[Text, List[tmp_file]]: - file = tmp_file(value, dir=self.cwd) - return file.name, [file] - - def __parse_key(self, key: Text, value: Text) -> Tuple[Text, List]: - # pylint: disable=unused-argument, - if re.match(r"[-]+BEGIN .+ KEY[-]+", value): - # Add new line if not exists and generate tmpfile for private key value - value = value + '\n' if value[-1] != '\n' else value - return self.__generate_arg_file(value) - # Return path in project if it's path - path = (Path(self.workdir)/Path(value).expanduser()).resolve() - return str(path), [] - - def __convert_arg(self, ansible_extra: AnsibleExtra, item: Tuple[Text, Any]) -> Tuple[List, List]: - extra_args, files = ansible_extra - key, value = item - key = key.replace('_', '-') - if key == 'verbose': - extra_args += ['-' + ('v' * value)] if value else [] - return extra_args, files - result = [value, []] - if key in HiddenArgumentsEnum.get_text_values(): - result = self.__parse_key(key, value) - elif key in HiddenArgumentsEnum.get_file_values(): - result = self.__generate_arg_file(value) # nocv - value = result[0] - files += result[1] - - key_type = self.ansible_ref[key].get('type', None) - if (key_type is None and value) or key_type: - extra_args.append("--{}".format(key)) - extra_args += [str(value)] if key_type else [] - return extra_args, files - - def __parse_extra_args(self, **extra) -> AnsibleExtra: - handler_func = self.__convert_arg - return AnsibleExtra(*reduce( - handler_func, extra.items(), ([], []) - )) + self.history.status = 'OK' + self.send_hook('on_execution', execution_dir=self.execution_dir) + self.verbose_output(f'Executing command {cmd}') + self.executor.execute(cmd, str(self.execution_dir), env_vars) - @property - def workdir(self) -> Text: - return self.get_workdir() + except Exception as exception: + logger.error(traceback.format_exc()) + self.handle_error(exception) - @property - def path_to_ansible(self) -> List[Text]: - return self.pm_ansible(self.command_type) + finally: + self.history.stop_time = timezone.now() + self.history.save() + self.send_hook('after_execution') + self.__del__() - def get_inventory_arg(self, target: Text, extra_args: List[Text]) -> List[Text]: - # pylint: disable=unused-argument - args = [target] - if self.inventory_object is not None: - args += ['-i', self.inventory_object.file_name] - return args - - def get_args(self, target: Text, extra_args: List[Text]) -> List[Text]: - return ( - self.path_to_ansible + - self.get_inventory_arg(target, extra_args) + - extra_args - ) - - def get_workdir(self) -> Text: - return self._get_tmp_name() - - def get_kwargs(self, target, extra_args) -> Dict[Text, Any]: - # pylint: disable=unused-argument - return dict(cwd=self._get_tmp_name()) - - def hide_passwords(self, raw: Text) -> Text: - regex = r'|'.join(( - r"(?<=" + hide + r":\s).{1,}?(?=[\n\t\s])" - for hide in HiddenVariablesEnum.get_values() - )) - raw = re.sub(regex, CYPHER, raw, 0, re.MULTILINE) - return raw - - def get_execution_revision(self, project: Project): - if not project.repo_sync_on_run: - return project.branch - return project.vars.get('repo_branch', '') - - def prepare(self, target: Text, inventory: Any, history: History, project: Project) -> None: - self.target, self.project = target, project - self.history = history if history else DummyHistory() - self.history.status = "RUN" - if inventory: - self.inventory_object = self.Inventory(inventory, cwd=self.project.path, tmpdir=self.cwd) - self.history.raw_inventory = self.hide_passwords( - self.inventory_object.raw - ) - else: # nocv - self.inventory_object = None - - revision = self.get_execution_revision(project) - self.history.revision = revision or 'NO VCS' - self.history.save() - self.executor = self.ExecutorClass(self.history) - - work_dir = self._get_tmp_name() - project.repo_class.make_run_copy(work_dir, revision) - self._verbose_output(f'Copied project on execution to {work_dir}.', 2) - - project_cfg = self.executor.env.get('ANSIBLE_CONFIG') - if project_cfg is not None: - self.executor.env['ANSIBLE_CONFIG'] = str(Path(self.project.path) / project_cfg) - return - - project_cfg = Path(self.project.path) / 'ansible.cfg' - if project_cfg.is_file(): - self.executor.env['ANSIBLE_CONFIG'] = str(project_cfg) - return - - project_cfg = os.getenv('ANSIBLE_CONFIG') - if project_cfg is not None: - self.executor.env['ANSIBLE_CONFIG'] = project_cfg - - def error_handler(self, exception: BaseException) -> None: - # pylint: disable=no-else-return - default_code = self.status_codes["other"] + def handle_error(self, exception: BaseException) -> None: + default_status = 'ERROR' error_text = str(exception) - self.history.status = default_code + self.history.status = default_status - if isinstance(exception, self.ExecutorClass.CalledProcessError): # nocv - error_text = "{}".format(exception.output) - self.history.status = self.status_codes.get( - exception.returncode, default_code - ) + if isinstance(exception, self.executor_class.CalledProcessError): # nocv + error_text = f'{exception.output}' + self.history.status = self.plugin.error_codes.get(exception.returncode, default_status) elif isinstance(exception, self.project.SyncError): - self.__will_raise_exception = True + raise exception last_line_object = self.history.raw_history_line.last() last_line = 0 @@ -385,55 +224,44 @@ def error_handler(self, exception: BaseException) -> None: last_line += 1 self.history.write_line(line, last_line) - def execute(self, target: Text, inventory: Any, history: History, project: Project, **extra_args) -> None: - try: - self.prepare(target, inventory, history, project) - self.history.status = "OK" - extra = self.__parse_extra_args(**extra_args) - args = self.get_args(self.target, extra.args) - kwargs = self.get_kwargs(self.target, extra.args) - self._send_hook('on_execution', args=args, kwargs=kwargs) - self.executor.execute(args, **kwargs) - except Exception as exception: - logger.error(traceback.format_exc()) - self.error_handler(exception) - if self.__will_raise_exception: - raise - finally: - inventory_object = getattr(self, "inventory_object", None) - if inventory_object is not None: - inventory_object.close() - self.history.stop_time = timezone.now() - self.history.save() - self._send_hook('after_execution') - self.__del__() - - def run(self): - try: - return self.execute(*self.args, **self.kwargs) - except Exception: # nocv - logger.error(traceback.format_exc()) - raise - - def __del__(self): - if hasattr(self, 'cwd') and os.path.exists(self.cwd): - self._verbose_output('Tmpdir "{}" was cleared.'.format(self.cwd)) - shutil.rmtree(self.cwd, ignore_errors=True) - - -class AnsiblePlaybook(AnsibleCommand): - command_type = "ansible-playbook" - + @property + def execution_dir(self) -> Path: + if self.__execution_dir__ is not None: + return self.__execution_dir__ + exec_dir = Path(tempfile.mkdtemp()) / 'execution_dir' + self.verbose_output(f'Execution temp dir created: {exec_dir}') + self.__execution_dir__ = exec_dir + return exec_dir + + def send_hook(self, when: str, **kwargs) -> None: + msg = OrderedDict() + msg['execution_type'] = self.history.kind + msg['when'] = when + inventory = self.history.inventory + if isinstance(inventory, Inventory): + inventory = inventory.get_hook_data(when) + msg['target'] = OrderedDict() + msg['target']['name'] = self.history.mode + msg['target']['inventory'] = inventory + msg['target']['project'] = self.project.get_hook_data(when) + msg['history'] = self.history.get_hook_data(when) + msg['extra'] = kwargs + self.project.hook(when, msg) -class AnsibleModule(AnsibleCommand): - command_type = "ansible" + def get_execution_revision(self) -> str: + if not self.project.repo_sync_on_run: + return self.project.branch + return self.project.vars.get('repo_branch', '') - def __init__(self, target: Text, *pargs, **kwargs): - kwargs['module-name'] = target - if not kwargs.get('args', None): - kwargs.pop('args', None) - super().__init__(*pargs, **kwargs) - self.ansible_ref['module-name'] = {'type': 'string'} + def verbose_output(self, message: str, level: int = 3) -> None: + if self.verbose_level >= level: + executor = getattr(self, 'executor', None) + if executor is not None: + executor.write_output(message) + logger.debug(message) - def execute(self, group: Text = 'all', *args, **extra_args): - return super().execute(group, *args, **extra_args) + def __del__(self): + exec_dir = getattr(self, '__execution_dir__', None) + if exec_dir is not None and exec_dir.is_dir(): + self.verbose_output(f'Temp dir {exec_dir} was cleared.') + shutil.rmtree(exec_dir, ignore_errors=True) diff --git a/polemarch/main/openapi.py b/polemarch/main/openapi.py index 1c2b3fe4..7fab0664 100644 --- a/polemarch/main/openapi.py +++ b/polemarch/main/openapi.py @@ -67,7 +67,12 @@ def set_inventory_field(request, schema): def set_inventory(model): for name, field in model['properties'].items(): if name == 'inventory': - field['format'] = 'inventory' + if field.get('format') == 'dynamic' and field['x-options']['types']: + for type_field in field['x-options']['types'].values(): + if isinstance(type_field, dict): + type_field['format'] = 'inventory' + else: + field['format'] = 'inventory' elif field.get('format') == 'dynamic': for type_field in field['x-options']['types'].values(): @@ -86,13 +91,13 @@ def set_periodic_task_variable_value_field(request, schema): # pylint: disable= module_vars = { k: v for k, v - in definitions['AnsibleModule']['properties'].items() + in definitions['ExecuteModule']['properties'].items() if k not in {'module', 'inventory'} } playbook_vars = { k: v for k, v - in definitions['AnsiblePlaybook']['properties'].items() + in definitions['ExecutePlaybook']['properties'].items() if k not in {'playbook', 'inventory'} } definitions['PeriodicTaskVariable']['properties']['key'] = { diff --git a/polemarch/main/settings.py b/polemarch/main/settings.py index 9b2eec3a..a8c717fd 100644 --- a/polemarch/main/settings.py +++ b/polemarch/main/settings.py @@ -193,6 +193,34 @@ class ArchiveSection(BaseAppendSection): archive_section = ArchiveSection('archive', config, config['archive']).all() +class PluginSection(BaseAppendSection): + types_map = { + 'backend': cconfig.StrType(), + } + + +class PluginOptionsSection(PluginSection): + pass + + +PLUGINS = {} + +for plugin_name, plugin_config, in config['plugins'].items(): + if 'backend' in plugin_config: + plugin_section = PluginSection(f'plugins.{plugin_name}', config, config['plugins'][plugin_name]).all() + options_section = PluginOptionsSection( + f'plugins.{plugin_name}.options', + config, + plugin_section.get('options', {}) + ).all() + + PLUGINS[plugin_name.upper()] = { + "BACKEND": plugin_section['backend'], + "OPTIONS": options_section + } + +PLUGIN_HANDLERS_CLASS = f'{VST_PROJECT_LIB_NAME}.main.utils.ExecutionHandlers' + REPO_BACKENDS = { "MANUAL": { "BACKEND": "{}.main.repo.Manual".format(VST_PROJECT_LIB_NAME), @@ -227,11 +255,8 @@ class ArchiveSection(BaseAppendSection): "SCHEDULER": { "BACKEND": "{}.main.tasks.tasks.ScheduledTask".format(VST_PROJECT_LIB_NAME) }, - "MODULE": { - "BACKEND": "{}.main.tasks.tasks.ExecuteAnsibleModule".format(VST_PROJECT_LIB_NAME) - }, - "PLAYBOOK": { - "BACKEND": "{}.main.tasks.tasks.ExecuteAnsiblePlaybook".format(VST_PROJECT_LIB_NAME) + "EXECUTION": { + "BACKEND": "{}.main.tasks.tasks.PluginTask".format(VST_PROJECT_LIB_NAME) }, } @@ -300,3 +325,10 @@ class ArchiveSection(BaseAppendSection): HOOKS_DIR = '/tmp/polemarch_hooks' + str(KWARGS['PY_VER']) os.makedirs(PROJECTS_DIR) if not os.path.exists(PROJECTS_DIR) else None os.makedirs(HOOKS_DIR) if not os.path.exists(HOOKS_DIR) else None + + tests_module_name = 'tests' + if VST_PROJECT_LIB_NAME != 'polemarch': + tests_module_name = 'tests_ce' # noce + PLUGINS['TEST_ANSIBLE_DOC'] = {'BACKEND': f'{tests_module_name}.TestAnsibleDoc', 'OPTIONS': {}} + PLUGINS['TEST_ECHO'] = {'BACKEND': f'{tests_module_name}.TestEcho', 'OPTIONS': {}} + PLUGINS['TEST_MODULE'] = {'BACKEND': f'{tests_module_name}.TestModule', 'OPTIONS': {}} diff --git a/polemarch/main/tasks/tasks.py b/polemarch/main/tasks/tasks.py index 91693fa0..5609a55f 100644 --- a/polemarch/main/tasks/tasks.py +++ b/polemarch/main/tasks/tasks.py @@ -6,7 +6,7 @@ from ..utils import task, BaseTask from .exceptions import TaskError from ..models import PeriodicTask -from ..models.utils import AnsibleModule, AnsiblePlaybook +from ..executions import PLUGIN_HANDLERS logger = logging.getLogger("polemarch") clone_retry = getattr(settings, 'CLONE_RETRY', 5) @@ -56,20 +56,23 @@ def run(self): raise -class _ExecuteAnsible(BaseTask): - ansible_class = None - - def run(self): - # pylint: disable=not-callable - ansible_object = self.ansible_class(*self.args, **self.kwargs) - ansible_object.run() - - @task(app, ignore_result=True, bind=True) -class ExecuteAnsiblePlaybook(_ExecuteAnsible): - ansible_class = AnsiblePlaybook - +class PluginTask(BaseTask): + def __init__(self, *args, plugin: str, project, history, execute_args, **kwargs): + super().__init__(*args, **kwargs) + self.plugin = plugin + self.project = project + self.history = history + self.execute_args = execute_args -@task(app, ignore_result=True, bind=True) -class ExecuteAnsibleModule(_ExecuteAnsible): - ansible_class = AnsibleModule + def run(self): + try: + PLUGIN_HANDLERS.get_object( + self.plugin, + self.project, + self.history, + **self.execute_args, + ).execute() + except: + logger.error(traceback.format_exc()) + raise diff --git a/polemarch/main/utils.py b/polemarch/main/utils.py index 17932b18..34affa95 100644 --- a/polemarch/main/utils.py +++ b/polemarch/main/utils.py @@ -9,7 +9,10 @@ import re import json from os.path import dirname +from django.conf import settings +from django.utils import timezone from vstutils.models.cent_notify import Notificator +from vstutils.utils import ObjectHandlers try: from yaml import CLoader as Loader, CDumper as Dumper, load, dump @@ -25,7 +28,6 @@ Executor, UnhandledExecutor, ON_POSIX, ) -from ..main.settings import NOTIFY_WITHOUT_QUEUE_MODELS from . import __file__ as file @@ -349,5 +351,81 @@ class PolemarchNotificator(Notificator): def create_notification_from_instance(self, instance): super().create_notification_from_instance(instance) # pylint: disable=protected-access - if instance.__class__._meta.label in NOTIFY_WITHOUT_QUEUE_MODELS and self.label != 'history_lines': + if instance.__class__._meta.label in settings.NOTIFY_WITHOUT_QUEUE_MODELS and self.label != 'history_lines': self.send() + + +class ExecutionHandlers(ObjectHandlers): + def execute(self, plugin: str, project, execute_args, **kwargs): + sync = kwargs.pop('sync', False) + if project.status != 'OK' and not sync: + raise project.SyncError('ERROR project not synchronized') + + task_class = project.task_handlers.backend('EXECUTION') + plugin_class = self.backend(plugin) + + mode = f'[{plugin} plugin]' + if plugin_class.arg_shown_on_history_as_mode is not None: + mode = execute_args.get(plugin_class.arg_shown_on_history_as_mode, mode) + + history = self.create_history( + project, + plugin, + mode, + execute_args=execute_args, + initiator=kwargs.pop('initiator', 0), + initiator_type=kwargs.pop('initiator_type', 'project'), + executor=kwargs.pop('executor', None), + save_result=kwargs.pop('save_result', True), + template_option=kwargs.pop('template_option', None), + ) + task_kwargs = { + 'plugin': plugin, + 'project': project, + 'history': history, + 'execute_args': execute_args, + } + + if sync: + task_class(**task_kwargs) + else: + task_class.delay(**task_kwargs) + + return history.id if history else None + + def create_history(self, project, kind, mode, execute_args, **kwargs): + if not kwargs['save_result']: + return None + + history_execute_args = {**execute_args} + inventory = history_execute_args.pop('inventory', None) + if isinstance(inventory, str): + history_execute_args['inventory'] = inventory + inventory = None + elif isinstance(inventory, int): + inventory = project.inventories.get(id=inventory) + + options = {} + if kwargs['template_option'] is not None: + options['template_option'] = kwargs['template_option'] + + return project.history.create( + status='DELAY', + mode=mode, + start_time=timezone.now(), + inventory=inventory, + project=project, + kind=kind, + raw_stdout='', + execute_args=history_execute_args, + initiator=kwargs['initiator'], + initiator_type=kwargs['initiator_type'], + executor=kwargs['executor'], + hidden=project.hidden, + options=options, + ) + + def get_object(self, plugin: str, project, history, **exec_args): # noee + from .models.utils import PluginExecutor # pylint: disable=import-outside-toplevel + + return PluginExecutor(self.backend(plugin), self.opts(plugin), project, history, exec_args) diff --git a/polemarch/plugins/__init__.py b/polemarch/plugins/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/polemarch/plugins/ansible.py b/polemarch/plugins/ansible.py new file mode 100644 index 00000000..4b45b73c --- /dev/null +++ b/polemarch/plugins/ansible.py @@ -0,0 +1,184 @@ +import os +import re +from uuid import uuid1 +from functools import lru_cache +from typing import Type, Mapping, Optional, Union, Tuple +from django.conf import settings +from rest_framework import fields +from vstutils.api.serializers import BaseSerializer +from vstutils.api.fields import VSTCharField, SecretFileInString, AutoCompletionField +from ..main.constants import ANSIBLE_REFERENCE, CYPHER, HiddenArgumentsEnum, HiddenVariablesEnum +from ..main.models import Inventory +from ..api.v2.serializers import InventoryAutoCompletionField +from .base import BasePlugin + + +class BaseAnsiblePlugin(BasePlugin): + __slots__ = () + + reference = {} + base_command = settings.EXECUTOR + raw_inventory_hidden_vars = HiddenVariablesEnum.get_values() + supports_inventory = True + hide_passwords_regex = re.compile(r'|'.join(( + r"(?<=" + hidden_var + r":\s).{1,}?(?=[\n\t\s])" + for hidden_var in raw_inventory_hidden_vars + )), re.MULTILINE) + + error_codes = { + 4: 'OFFLINE', + -9: 'INTERRUPTED', + -15: 'INTERRUPTED', + } + + @classmethod + @lru_cache() + def get_serializer_class(cls, exclude_fields=()) -> Type[BaseSerializer]: + class AnsibleSerializer(super().get_serializer_class(exclude_fields=exclude_fields)): + def to_representation(self, instance): + representation = super().to_representation(instance) + HiddenArgumentsEnum.hide_values(representation) + return representation + + AnsibleSerializer.__name__ = f'{AnsibleSerializer.Meta.ref_name}Serializer' + return AnsibleSerializer + + def get_env_vars(self) -> Mapping[str, str]: + env_vars = super().get_env_vars() + ansible_config = env_vars.pop('ANSIBLE_CONFIG', None) + + if ansible_config is not None: + ansible_config = str(self.execution_dir / ansible_config) + elif (self.execution_dir / 'ansible.cfg').is_file(): + ansible_config = str(self.execution_dir / 'ansible.cfg') + elif os.getenv('ANSIBLE_CONFIG'): + ansible_config = os.getenv('ANSIBLE_CONFIG') + + if ansible_config is not None: + env_vars['ANSIBLE_CONFIG'] = ansible_config + + return env_vars + + def get_inventory(self, inventory: Optional[Union[Inventory, str, int]]) -> Tuple[Optional[str], str]: + if inventory is None: + return inventory, '' + + if isinstance(inventory, int) or (isinstance(inventory, str) and inventory.isdigit()): + inventory = Inventory.objects.get(id=int(inventory)) + + if isinstance(inventory, Inventory): + text = inventory.get_inventory()[0] + inventory_file = self.execution_dir / self.inventory_filename + inventory_file.write_text(text) + return str(inventory_file), self._get_raw_inventory(text) + + if (self.execution_dir / inventory).is_file(): + inventory_file = (self.execution_dir / inventory) + self._inventory_filename = inventory_file.name + text = inventory_file.read_text() + return str(inventory_file), self._get_raw_inventory(text) + + return inventory, self._get_raw_inventory(inventory) + + def get_verbose_level(self, raw_args: dict) -> int: + return int(raw_args.get('verbose', 0)) + + def _get_raw_inventory(self, raw_inventory: str) -> str: + return self.hide_passwords_regex.sub(CYPHER, raw_inventory, 0) + + def _process_arg(self, key: str, value) -> Optional[str]: + key = key.replace('_', '-') + if key in self.reference: + if not value: + return None + if key == 'verbose': + return '-' + 'v' * int(value) if value else '' + argtype = self.reference[key]['type'] + if argtype is None and value: + return f'--{key}' + if argtype == 'inner': + value = self._put_into_tmpfile(value) + return super()._process_arg(key, value) + + def _put_into_tmpfile(self, value) -> str: + tmpfile = self.execution_dir / f'inner_arg_{uuid1()}' + tmpfile.write_text(value) + return str(tmpfile) + + @classmethod + def _get_serializer_fields(cls, exclude_fields=()): + serializer_fields = super()._get_serializer_fields(exclude_fields=exclude_fields) + for field_name, field_def in cls.reference.items(): + if field_name in ('help', 'version', *exclude_fields): + continue + field_type = field_def.get('type') + kwargs = {'help_text': field_def.get('help', ''), 'required': False} + field = None + if field_type is None: + field = fields.BooleanField + elif field_type == 'int': + field = fields.IntegerField + elif field_type in ('string', 'choice'): + field = VSTCharField + kwargs['allow_blank'] = True + + if field_name == 'verbose': + field = fields.IntegerField + kwargs.update({'max_value': 4}) + if field_name in HiddenArgumentsEnum.get_values(): + field = SecretFileInString + if field_name == 'inventory': + field = InventoryAutoCompletionField + if field_name == 'group': + kwargs['default'] = 'all' + + if field is None: + continue + if 'default' not in kwargs: + kwargs['default'] = fields.empty + + serializer_fields[field_name.replace('-', '_')] = field(**kwargs) + + return serializer_fields + + +class Playbook(BaseAnsiblePlugin): + __slots__ = () + + reference = ANSIBLE_REFERENCE.raw_dict['playbook'] + arg_shown_on_history_as_mode = 'playbook' + serializer_fields = { + 'playbook': AutoCompletionField(autocomplete='Playbook', autocomplete_property='playbook') + } + + @property + def base_command(self): + return super().base_command + ['ansible-playbook'] + + def _process_arg(self, key, value): + if key == 'playbook': + return value + return super()._process_arg(key, value) + + +class Module(BaseAnsiblePlugin): + __slots__ = ('group', 'module') + + reference = ANSIBLE_REFERENCE.raw_dict['module'] + serializer_fields = { + 'module': AutoCompletionField( + autocomplete='Module', + autocomplete_property='name', + autocomplete_represent='path', + ) + } + arg_shown_on_history_as_mode = 'module' + + @property + def base_command(self): + return super().base_command + ['ansible', self.group, '-m', self.module] + + def get_args(self, raw_args): + self.group = raw_args.pop('group', 'all') + self.module = raw_args.pop('module') + return super().get_args(raw_args) diff --git a/polemarch/plugins/base.py b/polemarch/plugins/base.py new file mode 100644 index 00000000..5c61b12d --- /dev/null +++ b/polemarch/plugins/base.py @@ -0,0 +1,209 @@ +from typing import Callable, Union, Mapping, List, Tuple, Type, Optional, Any +from pathlib import Path +from rest_framework.fields import Field +from vstutils.api.serializers import BaseSerializer +from vstutils.utils import classproperty +from ..main.models import Inventory + + +class BasePlugin(metaclass=classproperty.meta): + """ + Plugin class from which any other plugin should inherit. The plugin itself is an entity which, on the one hand + provides appropriate fields for the API, and on the other hand, processes arguments received from it to build + execution command. + + For each configured plugin an endpoint will be generated allows you to choose arguments and execute it. + Also, this plugin will be available to create template with. + + :param config: ``settings.ini`` options mapping for this plugin. + :param project_data: proxy of the project instance, allows you to access it's readonly properties, such as + ``config`` ``vars``, ``env_vars`` etc. + :param output_handler: executor's function which handles logging and outputting to history. Used by + ``verbose_output`` method. + """ + + __slots__ = ('config', '_output_handler', 'execution_dir', '_inventory_filename', 'project_data') + + base_command: List[str] + """ + Base command (usually binary) from which execution command starts, e.g. ``['echo']``. You may also override this + attribute as a property if more complex logic needs to be performed. + """ + + serializer_fields: Mapping[str, Field] = {} + """Fields mapping used to generate serializer. By default returned by ``_get_serializer_fields`` method.""" + + arg_shown_on_history_as_mode: Optional[str] = None + """ + Name of argument presented in generated serializer which will be shown on detail history page as *Mode*. For + example, if you are executing some module with additional arguments and fields contains `module` field, than you + can set `'module'` here, and it's value will be shown after execution. If not set, *Mode* in the history will show + ``[ plugin]`` string. + """ + + supports_inventory: bool = False + """ + Flag shows if the plugin supports working with inventory. For now it's only used to show or hide inventory field + in execution template create page. + """ + + error_codes: Mapping[int, str] = {} + """ + This mapping will be looked up to choose an appropriate error message for history output if execution finished with + errors. If no code found, then just "ERROR" string outputs. + """ + + def __init__(self, config: dict, project_data, output_handler: Callable): + self.config = config + self._output_handler = output_handler + self._inventory_filename = 'inventory_file' + self.project_data = project_data + self.execution_dir = None + + @classproperty + def name(cls) -> str: + """ + Returns name of plugin, class name by default. Primarily used to generate an appropriate model name for + OpenAPI schema. + """ + + return cls.__name__ # pylint: disable=no-member + + @classmethod + def get_serializer_class(cls, exclude_fields: tuple = ()) -> Type[BaseSerializer]: + """ + Returns serializer class which will be used to generate fields for arguments. Uses metaclass returned by + ``_get_serializer_metaclass`` method. + + :param exclude_fields: field names that should not be presented in serializer. + """ + + class Serializer(BaseSerializer, metaclass=cls._get_serializer_metaclass(exclude_fields=exclude_fields)): + class Meta: + ref_name = f'Execute{cls.name}' + + Serializer.__name__ = f'{Serializer.Meta.ref_name}Serializer' + return Serializer + + def get_execution_data(self, execution_dir: Path, raw_args: dict) -> Tuple[List[str], dict, str]: + """ + Returns tuple of execution command, env variables and raw inventory string. This method will be called directly + by executor. + + :param execution_dir: path to execution directory in which project copy located. All additional files that + should be generated (e.g. inventory file) must be placed here. + :param raw_args: argument name-value mapping which should be processed. + """ + + self.prepare_execution_dir(execution_dir) + env_vars = self.get_env_vars() + inventory_arg, raw_inventory = self.get_inventory(raw_args.pop('inventory', None)) + raw_args['inventory'] = inventory_arg + args = self.get_args(raw_args) + return self.base_command + args, env_vars, raw_inventory + + def prepare_execution_dir(self, dir: Path) -> None: + """ + Gets execution directory with copied project. All files needed for execution (e.g. generated inventory file) + should be here. + + :param dir: path to execution directory in which project copy located. All additional files that + should be generated (e.g. inventory file) must be placed here. + """ + + self.execution_dir = dir + + def get_inventory(self, inventory: Optional[Union[Inventory, str, int]]) -> Tuple[Optional[str], str]: + """ + Returns tuple of inventory argument for execution command and raw inventory string used + for representation in history. If no inventory presented, should return ``(None, '')``. + + :param inventory: inventory, received from API, which can ``None`` if there is no inventory, + ``polemarch.main.models.Inventory`` instance, int or str. + """ + + return None, '' + + def get_env_vars(self) -> Mapping[str, str]: + """Returns env variables which will be used in execution, project's env variables by default.""" + + return self.project_data.env_vars + + @property + def inventory_filename(self) -> str: + """Returns name of file with inventory content.""" + + return self._inventory_filename + + def get_args(self, raw_args: dict) -> List[str]: + """ + Returns list of processed arguments which will be substituted into execution command. + + :param raw_args: argument name-value mapping which should be processed. + """ + + args = [] + for key, value in raw_args.items(): + arg = self._process_arg(key, value) + if arg: + args.append(arg) + return args + + def get_verbose_level(self, raw_args: dict) -> int: + """ + Returns verbose level used for history output and logging. Should be taken from execution arguments (usually + from ``verbose`` argument). This method will be called directly by executor. + + :param raw_args: argument name-value mapping which should be processed. + """ + + return 0 + + def verbose_output(self, message: str, level: int = 3) -> None: + """ + Logs value with logger and outputs it to history. + + :param message: message to output. + :param level: verbosity level from which message should be outputted. + """ + + self._output_handler(message, level) + + @classmethod + def _get_serializer_metaclass(cls, exclude_fields: tuple = ()) -> Type[Type[BaseSerializer]]: + """ + Returns serializer metaclass used to generate fields in serializer. + + :param exclude_fields: field names that should not be presented in serializer. + """ + + class SerializerMeta(type(BaseSerializer)): + def __new__(mcs, name, bases, attrs): + attrs.update(cls._get_serializer_fields(exclude_fields=exclude_fields)) + return super().__new__(mcs, name, bases, attrs) + + return SerializerMeta + + @classmethod + def _get_serializer_fields(cls, exclude_fields: tuple = ()) -> dict: + """ + Returns dict with field names and field instances used to generate fields for serializer. + + :param exclude_fields: field names that should not be presented in serializer. + """ + + return { + name: field for name, field in cls.serializer_fields.items() + if name not in exclude_fields + } + + def _process_arg(self, key: str, value: Any) -> Optional[str]: + """ + Returns single argument with value for ``get_args`` method. Should return ``None`` if argument must not be + included to the execution command. + + :param key: argument key (e.g. `verbose`). + :param value: argument value (e.g. `2`). + """ + + return f'--{key}={value}' diff --git a/polemarch/settings.ini b/polemarch/settings.ini index f8ed9282..cd311312 100644 --- a/polemarch/settings.ini +++ b/polemarch/settings.ini @@ -1,5 +1,14 @@ [web] notificator_client_class = {LIB_NAME}.main.utils.PolemarchNotificator +[uwsgi] +skip-atexit-teardown = true + [archive] max_content_length = 10mb + +[plugins.playbook] +backend = {LIB_NAME}.plugins.ansible.Playbook + +[plugins.module] +backend = {LIB_NAME}.plugins.ansible.Module diff --git a/polemarch/translations/ru.py b/polemarch/translations/ru.py index 415183df..4b9c5f22 100644 --- a/polemarch/translations/ru.py +++ b/polemarch/translations/ru.py @@ -3,6 +3,7 @@ 'User settings were successfully saved.': 'Пользовательские настройки были успешно сохранены.', 'Invalid path. Path must not contain "..", "~" or any other special characters and must be relative.': 'Неправильный путь. Путь не должен содержать "..", "~" и другие специальные символы, а также должен быть относительным.', + '{} plugin was executed.': 'Запущен плагин {}.', # field titles and names 'project id': 'id проекта', diff --git a/requirements-doc.txt b/requirements-doc.txt index 2915c2ec..c4acf281 100644 --- a/requirements-doc.txt +++ b/requirements-doc.txt @@ -1,2 +1,4 @@ # Docs -vstutils[doc]~=5.1.7 +vstutils[rpc,ldap,doc,prod]~=5.1.11 +markdown2~=2.4.0 +gitpython~=3.1.14 diff --git a/requirements.txt b/requirements.txt index 120c00d8..906635b4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # Main -vstutils[rpc,ldap,doc,prod]~=5.1.7 +vstutils[rpc,ldap,doc,prod]~=5.1.11 docutils~=0.16.0 markdown2~=2.4.0 diff --git a/setup.py b/setup.py index a13f5532..f0d29cc5 100644 --- a/setup.py +++ b/setup.py @@ -3,16 +3,16 @@ import re import os import sys +import subprocess import fnmatch import codecs import gzip -import glob import shutil # allow setup.py to be run from any path os.chdir(os.path.normpath(os.path.join(os.path.abspath(__file__), os.pardir))) -from setuptools import find_packages, setup, Command +from setuptools import find_packages, setup, errors, Command from setuptools.extension import Extension from setuptools.command.sdist import sdist as _sdist from setuptools.command.build_py import build_py as build_py_orig @@ -371,12 +371,11 @@ def make_setup(**opts): webpack_path = os.path.join(os.getcwd(), 'webpack.config.js') if os.path.exists(webpack_path) and is_build and os.environ.get('DONT_YARN', "") != 'true': - yarn_build_command = 'devBuild' if is_develop else 'build' try: - os.system('yarn install --pure-lockfile') - os.system('yarn ' + yarn_build_command) - except Extension as err: - print(err) + subprocess.check_call(['yarn', 'install', '--pure-lockfile'], stdout=sys.stdout, stderr=sys.stderr) + subprocess.check_call(['yarn', 'devBuild' if is_develop else 'build'], stdout=sys.stdout, stderr=sys.stderr) + except Exception as err: + raise errors.CompileError(str(err)) setup(**opts) @@ -396,8 +395,7 @@ def make_setup(**opts): 'polemarch/templates/gui/service-worker.js', ], install_requires=[ - ] + - load_requirements('requirements.txt', os.getcwd()), + ] + load_requirements('requirements.txt', os.getcwd()) + load_requirements('requirements-doc.txt', os.getcwd()), extras_require={ 'test': load_requirements('requirements-test.txt', os.getcwd()) + [ i.replace('prod', 'test,prod') diff --git a/tests.py b/tests.py index 64222c6b..318ad032 100644 --- a/tests.py +++ b/tests.py @@ -8,6 +8,7 @@ from http.server import BaseHTTPRequestHandler, HTTPServer from django.forms import ValidationError from django.test import override_settings +from rest_framework import fields as drffields from unittest import skipIf import git import yaml @@ -21,22 +22,27 @@ from django.contrib.contenttypes.models import ContentType from vstutils.tests import BaseTestCase as VSTBaseTestCase from vstutils.utils import raise_context +from vstutils.api import fields as vstfields from django.conf import settings try: - from polemarch.main import tasks + from polemarch.main.tasks import ScheduledTask from polemarch.main.openapi import PROJECT_MENU from polemarch.main.constants import CYPHER + from polemarch.plugins.ansible import BaseAnsiblePlugin, BasePlugin, Module except ImportError: - from pmlib.main import tasks + from pmlib.main.tasks import ScheduledTask from pmlib.main.constants import CYPHER + from pmlib.plugins.ansible import BaseAnsiblePlugin, BasePlugin, Module TEST_DATA_DIR = Path(__file__).parent.absolute() ORIGINAL_PROJECTS_DIR = settings.PROJECTS_DIR if settings.VST_PROJECT_LIB_NAME == 'polemarch': TEST_DATA_DIR /= 'test_data' + TESTS_MODULE = 'tests' else: TEST_DATA_DIR /= 'test_data_ce' + TESTS_MODULE = 'tests_ce' User = get_user_model() @@ -84,6 +90,8 @@ class MockServer: Stops server after leaving context. """ + __slots__ = ('handler', 'httpd') + def __init__(self, handler: BaseHTTPRequestHandler): self.handler = handler @@ -103,6 +111,53 @@ def __init__(self, msg='Test exception.', *args, **kwargs): super().__init__(msg, *args, **kwargs) +class TestAnsibleDoc(BaseAnsiblePlugin): + reference = { + 'module-path': {'type': 'string'}, + 'verbose': {'type': 'int', 'help': 'verbose level'}, + 'json': {'type': None}, + } + serializer_fields = {'target': vstfields.VSTCharField()} + arg_shown_on_history_as_mode = 'target' + supports_inventory = False + + @property + def base_command(self): + return super().base_command + ['ansible-doc'] + + def get_inventory(self, *args, **kwargs): + self.verbose_output('Test log from test plugin', level=2) + return super().get_inventory(*args, **kwargs) + + def _process_arg(self, key, value): + if key == 'target': + return value + return super()._process_arg(key, value) + + +class TestEcho(BasePlugin): + serializer_fields = { + 'string': vstfields.VSTCharField(), + 'n': drffields.BooleanField(default=drffields.empty, required=False, label='No trailing newlines'), + 'e': drffields.BooleanField(default=drffields.empty, required=False, label='Interpret backslash escapes'), + } + arg_shown_on_history_as_mode = 'string' + + @property + def base_command(self): + return ['echo'] + + def _process_arg(self, key, value): + if key == 'string': + return value + if key in ('n', 'e') and value: + return f'-{key}' + + +class TestModule(Module): + pass + + class BaseTestCase(VSTBaseTestCase): def setUp(self): self.host = self.get_model_class('main.Host').objects.create(name='localhost') @@ -201,13 +256,20 @@ def execute_module_bulk_data(self, project_id=None, inventory=None, module='ping 'data': {'module': module, 'inventory': inventory or self.inventory_path, **kwargs} } - def execute_playbook_bulk_data(self, project_id=None, inventory=None, playbook='bootstrap.yml', **kwargs): + def execute_playbook_bulk_data(self, project_id=None, inventory=None, playbook='playbook.yml', **kwargs): return { 'method': 'post', 'path': ['project', project_id or self.project.id, 'execute_playbook'], 'data': {'playbook': playbook, 'inventory': inventory or self.inventory_path, **kwargs} } + def execute_plugin_bulk_data(self, plugin, project_id=None, **kwargs): + return { + 'method': 'post', + 'path': ['project', project_id or self.project.id, f'execute_{plugin}'], + 'data': {'target': 'gitlab_runner', **kwargs} + } + def create_periodic_task_bulk_data(self, project_id=None, inventory=None, **kwargs): return { 'method': 'post', @@ -1586,31 +1648,39 @@ def test_community_templates(self, temp_dir): repo.index.commit('Initial commit') project_templates = (TEST_DATA_DIR / 'projects.yaml').read_text() % {'repo_url': repo_dir} - with self.patch('requests.models.Response.text', return_value=project_templates): - results = self.bulk([ - # [0] get possible templates - {'method': 'get', 'path': 'community_template'}, - # [1] use TestProject template (see test_data/projects.yaml) - { - 'method': 'post', - 'path': ['community_template', '<<0[data][results][0][id]>>', 'use_it'], - 'data': {'name': 'lol-name'} - }, - # [2] check created project - self.get_project_bulk_data(project_id='<<1[data][project_id]>>'), - # [3] sync project - self.sync_project_bulk_data(project_id='<<1[data][project_id]>>'), - # [4] check created project - self.get_project_bulk_data(project_id='<<1[data][project_id]>>'), - # [5] execute template playbook - { - 'method': 'post', - 'path': ['project', '<<1[data][project_id]>>', 'execute_playbook'], - 'data': {'playbook': 'main.yml'}, - }, - # [6] check execution - {'method': 'get', 'path': ['history', '<<5[data][history_id]>>']} - ]) + class MockHandler(BaseHTTPRequestHandler): + def do_GET(self, *args, **kwargs): + self.send_response(200) + self.send_header('Content-Type', 'application/yaml') + self.end_headers() + self.wfile.write(project_templates.encode()) + + with MockServer(MockHandler) as server: + with override_settings(COMMUNITY_REPOS_URL=f'http://localhost:{server.server_port}'): + results = self.bulk([ + # [0] get possible templates + {'method': 'get', 'path': 'community_template'}, + # [1] use TestProject template (see test_data/projects.yaml) + { + 'method': 'post', + 'path': ['community_template', '<<0[data][results][0][id]>>', 'use_it'], + 'data': {'name': 'lol-name'} + }, + # [2] check created project + self.get_project_bulk_data(project_id='<<1[data][project_id]>>'), + # [3] sync project + self.sync_project_bulk_data(project_id='<<1[data][project_id]>>'), + # [4] check created project + self.get_project_bulk_data(project_id='<<1[data][project_id]>>'), + # [5] execute template playbook + { + 'method': 'post', + 'path': ['project', '<<1[data][project_id]>>', 'execute_playbook'], + 'data': {'playbook': 'main.yml'}, + }, + # [6] check execution + {'method': 'get', 'path': ['history', '<<5[data][history_id]>>']} + ]) self.assertEqual(results[0]['status'], 200) self.assertEqual(results[0]['data']['count'], 1) self.assertEqual(results[0]['data']['results'][0]['name'], 'TestProject') @@ -1645,10 +1715,7 @@ def test_repo_sync_on_run_for_manual_project(self, temp_dir): (Path(project.path) / 'test.yml').touch() mock_dir = Path(temp_dir) / 'project' - with self.patch( - f'{settings.VST_PROJECT_LIB_NAME}.main.models.utils.AnsibleCommand._get_tmp_name', - return_value=mock_dir - ): + with self.patch(f'{settings.VST_PROJECT_LIB_NAME}.main.models.utils.PluginExecutor.execution_dir', mock_dir): results = self.bulk_transactional([ self.execute_module_bulk_data(project_id), self.get_history_bulk_data('<<0[data][history_id]>>'), @@ -1908,7 +1975,7 @@ def do_GET(self, *args, **kwargs): class PeriodicTaskTestCase(BaseProjectTestCase): def test_periodic_task(self): # check correct data - results = self.bulk([ + results = self.bulk_transactional([ # [0] self.create_periodic_task_bulk_data(type='INTERVAL', schedule='10'), # [1] @@ -1929,8 +1996,6 @@ def test_periodic_task(self): template='<<5[data][id]>>', ), ]) - for result in results: - self.assertEqual(result['status'], 201) playbook_task_id = results[0]['data']['id'] module_task_id = results[4]['data']['id'] template_id = results[5]['data']['id'] @@ -1961,7 +2026,7 @@ def test_periodic_task(self): self.assertIn("Invalid weekday literal", results[0]['data']['detail']['schedule'][0]) # execute template - results = self.bulk([ + results = self.bulk_transactional([ { # [0] execute playbook task 'method': 'post', 'path': ['project', self.project.id, 'periodic_task', playbook_task_id, 'execute'], @@ -2000,8 +2065,6 @@ def test_periodic_task(self): 'path': ['project', self.project.id, 'history', '<<6[data][history_id]>>'], }, ]) - for result in results: - self.assertIn(result['status'], {200, 201}) self.assertIn(f'Started at inventory {self.inventory.id}', results[0]['data']['detail']) self.assertEqual(results[1]['data']['status'], 'OK') self.assertEqual(results[1]['data']['initiator_type'], 'scheduler') @@ -2048,7 +2111,7 @@ def test_periodic_task(self): self.assertEqual(results[1]['data']['status'], 'OK') # emulate execute by scheduler - tasks.ScheduledTask.delay(module_task_id) + ScheduledTask.delay(module_task_id) results = self.bulk([ { 'method': 'get', @@ -2062,7 +2125,7 @@ def test_periodic_task(self): self.assertEqual(results[0]['data']['results'][0]['initiator_type'], 'scheduler') # try to execute not existing task - result = tasks.ScheduledTask.delay(999999) + result = ScheduledTask.delay(999999) self.assertEqual(result.status, 'SUCCESS') # check exception while execution @@ -2070,7 +2133,7 @@ def test_periodic_task(self): f'{settings.VST_PROJECT_LIB_NAME}.main.models.tasks.PeriodicTask.execute', side_effect=TestException ) as executor: - tasks.ScheduledTask.delay(playbook_task_id) + ScheduledTask.delay(playbook_task_id) self.assertEqual(executor.call_count, 1) # check patch schedule @@ -2103,9 +2166,67 @@ def test_periodic_task(self): ]) self.assertEqual(results[0]['status'], 204) + def test_echo_plugin(self): + results = self.bulk_transactional([ + # [0] create template with echo plugin + { + 'method': 'post', + 'path': ['project', self.project.id, 'execution_templates'], + 'data': { + 'name': 'echo', + 'kind': 'TEST_ECHO', + 'data': {'string': 'some text'}, + } + }, + # [1] + self.create_periodic_task_bulk_data( + type='INTERVAL', + schedule='10', + kind='TEMPLATE', + template='<<0[data][id]>>', + ), + # [2] execute periodic task + { + 'method': 'post', + 'path': ['project', self.project.id, 'periodic_task', '<<1[data][id]>>', 'execute'], + }, + # [3] + self.get_history_bulk_data('<<2[data][history_id]>>'), + # [4] + self.get_raw_history_bulk_data('<<2[data][history_id]>>'), + ]) + self.assertEqual(results[3]['data']['status'], 'OK') + self.assertEqual(results[3]['data']['kind'], 'TEST_ECHO') + self.assertEqual(results[3]['data']['initiator_type'], 'scheduler') + self.assertEqual(results[3]['data']['raw_args'], 'echo some text') + self.assertEqual(results[4]['data']['detail'], 'some text\n') + @own_projects_dir class PlaybookAndModuleTestCase(BaseProjectTestCase): + def test_v2_executions(self): + self.client.force_login(self.user) + result = self.client.post( + f'/api/v2/project/{self.project.id}/execute_module/', + {'module': 'ping', 'inventory': self.inventory.id}, + content_type='application/json' + ) + history1 = result.data['history_id'] + self.assertEqual(result.status_code, 201) + result = self.client.post( + f'/api/v2/project/{self.project.id}/execute_playbook/', + {'playbook': 'playbook.yml', 'inventory': self.inventory.id}, + content_type='application/json' + ) + history2 = result.data['history_id'] + self.assertEqual(result.status_code, 201) + results = self.bulk_transactional([ + self.get_history_bulk_data(history1), + self.get_history_bulk_data(history2), + ]) + self.assertEqual(results[0]['data']['status'], 'OK') + self.assertEqual(results[1]['data']['status'], 'OK') + def test_execute_playbook(self): # try to execute not synced project project = self.get_model_class('main.Project').objects.create(name='lol_project') @@ -2147,12 +2268,12 @@ def check_execution(): results = check_execution() self.assertEqual( - self.get_model_class('main.History').objects.get(pk=results[0]['data']['history_id']).initiator_object.id, + self.get_model_filter('main.History').get(pk=results[0]['data']['history_id']).initiator_object.id, self.project.id, ) # check again with repo_sync_on_run set - sync_on_run = self.get_model_class('main.Variable').objects.create( + sync_on_run = self.get_model_filter('main.Variable').create( key='repo_sync_on_run', value=False, object_id=self.project.id, @@ -2211,6 +2332,84 @@ def check_with_inventory(inventory, **kwargs): check_with_inventory(self.inventory_path) check_with_inventory('localhost,', extra_vars='ansible_connection=local') + def test_execute_echo_plugin(self): + results = self.bulk_transactional([ + # [0] + self.execute_plugin_bulk_data('test_echo', string='test string'), + # [1] + self.get_history_bulk_data('<<0[data][history_id]>>'), + # [2] + self.get_raw_history_bulk_data('<<0[data][history_id]>>'), + # [3] + self.execute_plugin_bulk_data('test_echo', string='test string 2'), + # [4] + self.get_history_bulk_data('<<3[data][history_id]>>'), + # [5] + self.get_raw_history_bulk_data('<<3[data][history_id]>>'), + ]) + self.assertEqual(results[1]['data']['status'], 'OK') + self.assertDictEqual(results[1]['data']['execute_args'], {'string': 'test string'}) + self.assertEqual(results[1]['data']['mode'], 'test string') + self.assertEqual(results[1]['data']['kind'], 'TEST_ECHO') + self.assertEqual(results[1]['data']['raw_args'], 'echo test string') + self.assertEqual(results[2]['data']['detail'], 'test string\n') + + self.assertEqual(results[4]['data']['status'], 'OK') + self.assertDictEqual(results[4]['data']['execute_args'], {'string': 'test string 2'}) + self.assertEqual(results[4]['data']['raw_args'], 'echo test string 2') + self.assertEqual(results[5]['data']['detail'], 'test string 2\n') + + # check that plugin cannot access other than allowed Project's attributes + def access_unsafe(inner_self, *args, **kwargs): + inner_self.project_data.objects.delete() + + with self.patch(f'{TESTS_MODULE}.TestEcho.get_args', side_effect=access_unsafe, autospec=True): + results = self.bulk_transactional([ + # [0] + self.execute_plugin_bulk_data('test_echo', string='test string'), + # [1] + self.get_history_bulk_data('<<0[data][history_id]>>'), + # [2] + self.get_raw_history_bulk_data('<<0[data][history_id]>>'), + ]) + self.assertEqual(results[1]['data']['status'], 'ERROR') + self.assertEqual( + results[2]['data']['detail'], + "allowed attributes are " + "('config', 'revision', 'branch', 'project_branch', 'vars', 'env_vars', 'type', " + "'repo_sync_on_run', 'repo_sync_timeout')" + ) + + def test_execute_ansible_doc_plugin(self): + results = self.bulk_transactional([ + # [0] + self.execute_plugin_bulk_data( + plugin='test_ansible_doc', + target='gitlab_runner', + verbose=2, + json=True, + module_path='some/path' + ), + # [1] + self.get_history_bulk_data('<<0[data][history_id]>>'), + # [2] + self.get_raw_history_bulk_data('<<0[data][history_id]>>'), + ]) + next_history_id = self.get_model_filter('main.History').order_by('-id').first().id + self.assertDictEqual(results[0]['data'], { + 'executor': self.user.id, + 'history_id': next_history_id, + 'detail': 'TEST_ANSIBLE_DOC plugin was executed.', + + }) + self.assertEqual(results[1]['data']['status'], 'OK') + self.assertIn('"module": "gitlab_runner"', results[2]['data']['detail']) + self.assertIn( + '"short_description": "Create, modify and delete GitLab Runners."', + results[2]['data']['detail'] + ) + self.assertIn('Test log from test plugin', results[2]['data']['detail']) + def test_gather_facts(self): results = self.bulk([ { # [0] run setup module @@ -2879,7 +3078,7 @@ def test_execution_templates_options(self): # NOTE: template api is deprecated. Remove after break support. def test_template(self): - results = self.bulk([ + results = self.bulk_transactional([ { # [0] create module template 'method': 'post', 'path': ['project', self.project.id, 'template'], @@ -2906,7 +3105,7 @@ def test_template(self): 'data': { 'kind': 'Task', 'name': 'Kek template', - 'data': {'playbook': 'playbook.yml', 'vars': {'forks': 8, 'connection': 'local', 'verbose': 8}}, + 'data': {'playbook': 'playbook.yml', 'vars': {'forks': 8, 'connection': 'local', 'verbose': 2}}, 'options': { 'one': {'vars': {'limit': 'localhost'}}, 'two': {'vars': {'forks': 1, 'private_key': 'id_rsa'}}, @@ -2942,8 +3141,6 @@ def test_template(self): } } ]) - for result in results: - self.assertIn(result['status'], {200, 201}) self.assertEqual(results[0]['data']['data']['vars']['vault_password_file'], CYPHER) self.assertEqual(results[6]['data']['options']['two']['vars']['private_key'], CYPHER) self.assertEqual(results[7]['data']['data']['vars']['private_key'], CYPHER) @@ -2963,7 +3160,7 @@ def test_template(self): 'data': { 'kind': 'Task', 'name': 'Kek template', - 'data': {'playbook': 'playbook.yml', 'vars': {'forks': 8, 'connection': 'local', 'verbose': 8}}, + 'data': {'playbook': 'playbook.yml', 'vars': {'forks': 8, 'connection': 'local', 'verbose': 2}}, 'options': {'vars': {'inventory': 'lol'}} } }, @@ -2973,7 +3170,7 @@ def test_template(self): self.assertEqual(results[1]['status'], 400) self.assertEqual(results[1]['data']['detail']['inventory'], ['Disallowed to override inventory.']) - results = self.bulk([ + results = self.bulk_transactional([ # [0] execute module template {'method': 'post', 'path': ['project', self.project.id, 'template', module_template['id'], 'execute']}, # [1] execute task template @@ -2983,14 +3180,10 @@ def test_template(self): # [3] check task execution history {'method': 'get', 'path': ['history', '<<1[data][history_id]>>']}, ]) - self.assertEqual(results[0]['status'], 201) - self.assertEqual(results[1]['status'], 201) - self.assertEqual(results[2]['status'], 200) self.assertEqual(results[2]['data']['status'], 'OK') self.assertEqual(results[2]['data']['initiator_type'], 'template') self.assertEqual(results[2]['data']['initiator'], module_template['id']) self.assertEqual(results[2]['data']['mode'], 'ping') - self.assertEqual(results[3]['status'], 200) self.assertEqual(results[3]['data']['status'], 'OK') self.assertEqual(results[3]['data']['initiator_type'], 'template') self.assertEqual(results[3]['data']['initiator'], task_template['id']) @@ -3016,6 +3209,151 @@ def send(*args): self.bulk_transactional([self.execute_module_bulk_data()]) client_getter.assert_any_call() + def test_execute_test_module(self): + results = self.bulk_transactional([ + # [0] + { + 'method': 'post', + 'path': ['project', self.project.id, 'execution_templates'], + 'data': { + 'name': 'test module', + 'kind': 'TEST_MODULE', + 'inventory': self.inventory.id, + 'data': {'module': 'ping'}, + } + }, + # [1] + { + 'method': 'post', + 'path': ['project', self.project.id, 'execution_templates', '<<0[data][id]>>', 'execute'] + }, + # [2] + self.get_history_bulk_data('<<1[data][history_id]>>'), + # [3] + self.get_raw_history_bulk_data('<<1[data][history_id]>>'), + ]) + self.assertEqual(results[2]['data']['status'], 'OK') + self.assertIn('"ping": "pong"', results[3]['data']['detail']) + + def test_execute_ansible_doc_plugin(self): + results = self.bulk_transactional([ + # [0] + { + 'method': 'post', + 'path': ['project', self.project.id, 'execution_templates'], + 'data': { + 'name': 'help a10_server', + 'notes': 'help a10_server', + 'kind': 'TEST_ANSIBLE_DOC', + 'data': { + 'target': 'a10_server', + 'verbose': 1, + 'json': False, + }, + } + }, + # [1] + { + 'method': 'post', + 'path': ['project', self.project.id, 'execution_templates', '<<0[data][id]>>', 'execute'] + }, + # [2] + self.get_history_bulk_data('<<1[data][history_id]>>'), + # [3] + self.get_raw_history_bulk_data('<<1[data][history_id]>>'), + ]) + template_id = results[0]['data']['id'] + next_history_id = self.get_model_filter('main.History').order_by('-id').first().id + self.assertDictEqual(results[1]['data'], { + 'executor': self.user.id, + 'history_id': next_history_id, + 'detail': 'TEST_ANSIBLE_DOC plugin was executed.', + + }) + self.assertEqual(results[2]['data']['status'], 'OK') + history = self.get_model_filter('main.History').get(id=next_history_id) + self.assertEqual(history.json_options, '{}') + self.assertIn( + 'Manage SLB (Server Load Balancer) server objects on A10', + results[3]['data']['detail'] + ) + self.assertIn( + 'AUTHOR: Eric Chou (@ericchou1), Mischa Peters (@mischapeters)', + results[3]['data']['detail'] + ) + self.assertNotIn('Test log from test plugin', results[3]['data']['detail']) + + # check with option + results = self.bulk_transactional([ + # [0] + { + 'method': 'post', + 'path': ['project', self.project.id, 'execution_templates', template_id, 'option'], + 'data': { + 'name': 'verbose_with_json', + 'data': { + 'target': 'a10_server', + 'verbose': 4, + 'json': True, + }, + } + }, + # [1] + { + 'method': 'post', + 'path': ['project', self.project.id, 'execution_templates', template_id, 'execute'], + 'data': {'option': '<<0[data][id]>>'}, + }, + # [2] + self.get_history_bulk_data('<<1[data][history_id]>>'), + # [3] + self.get_raw_history_bulk_data('<<1[data][history_id]>>'), + ]) + self.assertEqual(results[2]['data']['status'], 'OK') + history = self.get_model_filter('main.History').get(id=results[1]['data']['history_id']) + self.assertEqual(history.json_options, '{"template_option": "verbose_with_json"}') + # from verbose + self.assertIn( + 'Executing command', + results[3]['data']['detail'] + ) + self.assertIn( + "'pm_ansible', 'ansible-doc', 'a10_server', '-vvvv', '--json'", + results[3]['data']['detail'] + ) + self.assertIn("Test log from test plugin", results[3]['data']['detail']) + # json output + self.assertIn('"author": [', results[3]['data']['detail']) + self.assertIn('"Eric Chou (@ericchou1)",', results[3]['data']['detail']) + self.assertIn( + '"Manage SLB (Server Load Balancer) server objects on A10 Networks devices via aXAPIv2."', + results[3]['data']['detail'] + ) + + def test_edit_plugin_template(self): + results = self.bulk_transactional([ + # [0] + { + 'method': 'post', + 'path': ['project', self.project.id, 'execution_templates'], + 'data': { + 'name': 'echo', + 'kind': 'TEST_ECHO', + 'data': {'string': '1'}, + } + }, + # [1] + { + 'method': 'patch', + 'path': ['project', self.project.id, 'execution_templates', '<<0[data][id]>>'], + 'data': { + 'data': {'string': '2'}, + }, + }, + ]) + self.assertEqual(results[0]['data']['data']['string'], '1') + self.assertEqual(results[1]['data']['data']['string'], '2') + @own_projects_dir class VariableTestCase(BaseProjectTestCase): @@ -3040,57 +3378,41 @@ def test_override_ansible_cfg_in_project(self): ) # check if env_ANSIBLE_CONFIG is set than this config is used - def check(inner_self, *args, **kwargs): - self.assertTrue(inner_self.env['ANSIBLE_CONFIG'].endswith(f'{self.project.id}/dir0/dir1/ansible.cfg')) - - with self.patch( - f'{settings.VST_PROJECT_LIB_NAME}.main.models.utils.Executor.execute', - side_effect=check, - autospec=True - ): + with self.patch('subprocess.Popen.__init__', return_value=None) as popen: + popen.assert_not_called() results = self.bulk_transactional([ self.create_variable_bulk_data('env_ANSIBLE_CONFIG', 'dir0/dir1/ansible.cfg'), self.execute_playbook_bulk_data(playbook='playbook.yml'), - self.get_history_bulk_data('<<1[data][history_id]>>'), ]) - self.assertEqual(results[-1]['data']['status'], 'OK') - var_id = results[0]['data']['id'] + popen.assert_called_once() + self.assertTrue(popen.call_args[1]['env']['ANSIBLE_CONFIG'].endswith('/dir0/dir1/ansible.cfg')) + + var_id = self.get_model_filter('main.Variable').get(key='env_ANSIBLE_CONFIG').id # check if env_ANSIBLE_CONFIG is not set than root project ansible.cfg is used (Path(self.project.path) / 'ansible.cfg').write_text(test_ansible_cfg) - def check(inner_self, *args, **kwargs): - self.assertTrue(inner_self.env['ANSIBLE_CONFIG'].endswith(f'{self.project.id}/ansible.cfg')) - - with self.patch( - f'{settings.VST_PROJECT_LIB_NAME}.main.models.utils.Executor.execute', - side_effect=check, - autospec=True - ): + with self.patch('subprocess.Popen.__init__', return_value=None) as popen: + popen.assert_not_called() results = self.bulk_transactional([ {'method': 'delete', 'path': ['project', self.project.id, 'variables', var_id]}, self.execute_playbook_bulk_data(playbook='playbook.yml'), - self.get_history_bulk_data('<<1[data][history_id]>>'), ]) - self.assertEqual(results[-1]['data']['status'], 'OK') + popen.assert_called_once() + self.assertTrue(popen.call_args[1]['env']['ANSIBLE_CONFIG'].endswith('/ansible.cfg')) # check if env_ANSIBLE_CONFIG is not set and project's ansible.cfg does not exist than # os ANSIBLE_CONFIG env var is used os.remove(Path(self.project.path) / 'ansible.cfg') - def check(inner_self, *args, **kwargs): - self.assertEqual(inner_self.env['ANSIBLE_CONFIG'], '/some/global.cfg') - - with self.patch( - f'{settings.VST_PROJECT_LIB_NAME}.main.models.utils.Executor.execute', - side_effect=check, - autospec=True - ), self.patch('os.environ', {'ANSIBLE_CONFIG': '/some/global.cfg'}): + with self.patch('subprocess.Popen.__init__', return_value=None) as popen, \ + self.patch('os.environ', {'ANSIBLE_CONFIG': '/some/global.cfg'}): + popen.assert_not_called() results = self.bulk_transactional([ self.execute_playbook_bulk_data(playbook='playbook.yml'), - self.get_history_bulk_data('<<0[data][history_id]>>'), ]) - self.assertEqual(results[-1]['data']['status'], 'OK') + popen.assert_called_once() + self.assertEqual(popen.call_args[1]['env']['ANSIBLE_CONFIG'], '/some/global.cfg') def test_periodic_task_variables_validation(self): results = self.bulk([ @@ -3471,6 +3793,17 @@ def test_vars_property_caching(self): 'repo_sync_on_run': True }) + def test_env_vars_on_execution(self): + with self.patch('subprocess.Popen.__init__', return_value=None) as popen: + popen.assert_not_called() + self.bulk([ + self.create_variable_bulk_data('env_EXAMPLE', '1'), + self.execute_module_bulk_data() + ]) + popen.assert_called_once() + self.assertEqual(popen.call_args[1]['env']['EXAMPLE'], '1') + self.assertIn('VST_PROJECT', popen.call_args[1]['env']) + @own_projects_dir class HookTestCase(BaseProjectTestCase): @@ -3839,30 +4172,52 @@ class OpenAPITestCase(BaseOpenAPITestCase): """ def test_schema(self): - schema = self.schema() - openapi_schema_yml = yaml.load(Path(self.openapi_schema_yaml).read_text(), Loader=yaml.SafeLoader) - - openapi_schema_yml['host'] = self.server_name - openapi_schema_yml['schemes'][0] = 'https' - openapi_schema_yml['info']['contact'] = schema['info']['contact'] - openapi_schema_yml['info']['x-versions'] = schema['info']['x-versions'] - openapi_schema_yml['info']['x-links'] = schema['info']['x-links'] - openapi_schema_yml['info']['x-user-id'] = schema['info']['x-user-id'] + endpoint_schema = self.schema() + yml_schema = yaml.load(Path(self.openapi_schema_yaml).read_text(), Loader=yaml.SafeLoader) + + yml_schema['host'] = self.server_name + yml_schema['schemes'][0] = 'https' + yml_schema['info']['contact'] = endpoint_schema['info']['contact'] + yml_schema['info']['x-versions'] = endpoint_schema['info']['x-versions'] + yml_schema['info']['x-links'] = endpoint_schema['info']['x-links'] + yml_schema['info']['x-user-id'] = endpoint_schema['info']['x-user-id'] + + for key in list(filter(lambda x: 'Execute' in x, yml_schema['definitions'].keys())): + del yml_schema['definitions'][key] + del endpoint_schema['definitions'][key] + + template_kinds_depend_on_plugins = ('ExecutionTemplate', 'CreateExecutionTemplate', 'OneExecutionTemplate') + for key in template_kinds_depend_on_plugins: + endpoint_schema['definitions'][key]['properties']['kind']['enum'] = \ + yml_schema['definitions'][key]['properties']['kind']['enum'] + + template_types_depend_on_plugins = ( + 'CreateExecutionTemplate', + 'OneExecutionTemplate', + 'CreateTemplateOption', + 'OneTemplateOption', + ) + for key in template_types_depend_on_plugins: + endpoint_schema['definitions'][key]['properties']['data']['x-options']['types'] = \ + yml_schema['definitions'][key]['properties']['data']['x-options']['types'] + if 'inventory' in endpoint_schema['definitions'][key]['properties']: + endpoint_schema['definitions'][key]['properties']['inventory']['x-options']['types'] = \ + yml_schema['definitions'][key]['properties']['inventory']['x-options']['types'] - for key in list(filter(lambda x: 'Ansible' in x, openapi_schema_yml['definitions'].keys())): - del openapi_schema_yml['definitions'][key] - del schema['definitions'][key] + with raise_context(): + endpoint_schema['definitions']['PeriodicTaskVariable']['properties']['key']['x-options']['types'] = \ + yml_schema['definitions']['PeriodicTaskVariable']['properties']['key']['x-options']['types'] - del openapi_schema_yml['definitions']['_MainSettings'] - del schema['definitions']['_MainSettings'] + del yml_schema['definitions']['_MainSettings'] + del endpoint_schema['definitions']['_MainSettings'] with raise_context(): - openapi_schema_yml['definitions']['ProjectDir']['properties']['content']["x-options"]['types'] = \ - schema['definitions']['ProjectDir']['properties']['content']["x-options"]['types'] + yml_schema['definitions']['ProjectDir']['properties']['content']["x-options"]['types'] = \ + endpoint_schema['definitions']['ProjectDir']['properties']['content']["x-options"]['types'] for module in ('paths', 'definitions'): - for key, value in openapi_schema_yml[module].items(): - self.assertDictEqual(value, schema[module].get(key), key) + for key, value in yml_schema[module].items(): + self.assertDictEqual(value, endpoint_schema[module].get(key), f'Failed on {key}') @skipIf(settings.VST_PROJECT_LIB_NAME != 'polemarch', 'Menu may vary') def test_menu(self): diff --git a/tox.ini b/tox.ini index 6655ff05..8b2db20d 100644 --- a/tox.ini +++ b/tox.ini @@ -102,13 +102,11 @@ whitelist_externals = cp make commands = - pip install -U -r ../requirements-doc.txt make html # cp -rv _build/html ../public deps = -rrequirements-doc.txt - [testenv:build_for_docker] basepython = python3.8 skipsdist = True diff --git a/yarn.lock b/yarn.lock index f4d894dd..0957f7e7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2137,9 +2137,9 @@ loader-runner@^4.2.0: integrity sha512-3R/1M+yS3j5ou80Me59j7F9IMs4PXs3VqRrm0TU3AbKPxlmpoY1TNscJV/oGJXo8qCatFGTfDbY6W6ipGOYXfg== loader-utils@^1.0.2, loader-utils@^1.1.0: - version "1.4.1" - resolved "https://registry.yarnpkg.com/loader-utils/-/loader-utils-1.4.1.tgz#278ad7006660bccc4d2c0c1578e17c5c78d5c0e0" - integrity sha512-1Qo97Y2oKaU+Ro2xnDMR26g1BwMT29jNbem1EvcujW2jqt+j5COXyscjM7bLQkM9HaxI7pkWeW7gnI072yMI9Q== + version "1.4.2" + resolved "https://registry.yarnpkg.com/loader-utils/-/loader-utils-1.4.2.tgz#29a957f3a63973883eb684f10ffd3d151fec01a3" + integrity sha512-I5d00Pd/jwMD2QCduo657+YM/6L3KZu++pmX9VFncxaxvHcru9jx1lBaFft+r4Mt2jK0Yhp41XlRAihzPxHNCg== dependencies: big.js "^5.2.2" emojis-list "^3.0.0" @@ -2390,9 +2390,9 @@ postcss-modules-values@^4.0.0: icss-utils "^5.0.0" postcss-selector-parser@^6.0.2: - version "6.0.10" - resolved "https://registry.yarnpkg.com/postcss-selector-parser/-/postcss-selector-parser-6.0.10.tgz#79b61e2c0d1bfc2602d549e11d0876256f8df88d" - integrity sha512-IQ7TZdoaqbT+LCpShg46jnZVlhWD2w6iQYAcYXfHARZ7X1t/UGhhceQDs5X0cGqKvYlHNOuv7Oa1xmb0oQuA3w== + version "6.0.11" + resolved "https://registry.yarnpkg.com/postcss-selector-parser/-/postcss-selector-parser-6.0.11.tgz#2e41dc39b7ad74046e1615185185cd0b17d0c8dc" + integrity sha512-zbARubNdogI9j7WY4nQJBiNqQf3sLS3wCP4WfOidu+p28LofJqDH1tcXypGrcmMHhDk2t9wGhCsYe/+szLTy1g== dependencies: cssesc "^3.0.0" util-deprecate "^1.0.2" @@ -2445,9 +2445,9 @@ prettier-linter-helpers@^1.0.0: fast-diff "^1.1.2" "prettier@^1.18.2 || ^2.0.0": - version "2.7.1" - resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.7.1.tgz#e235806850d057f97bb08368a4f7d899f7760c64" - integrity sha512-ujppO+MkdPqoVINuDFDRLClm7D78qbDt0/NR+wp5FqEZOoTNAjPHWj17QRhu7geIHJfcNhRk1XVQmF8Bp3ye+g== + version "2.8.0" + resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.8.0.tgz#c7df58393c9ba77d6fba3921ae01faf994fb9dc9" + integrity sha512-9Lmg8hTFZKG0Asr/kW9Bp8tJjRVluO8EJQVfY2T7FMw9T5jy4I/Uvx0Rca/XWf50QQ1/SS48+6IJWnrb+2yemA== prettier@^2.5.1: version "2.6.0" @@ -2846,9 +2846,9 @@ vue-hot-reload-api@^2.3.0: integrity sha512-BXq3jwIagosjgNVae6tkHzzIk6a8MHFtzAdwhnV5VlvPTFxDCvIttgSiHWjdGoTJvXtmRu5HacExfdarRcFhog== vue-loader@^15.9.8: - version "15.10.0" - resolved "https://registry.yarnpkg.com/vue-loader/-/vue-loader-15.10.0.tgz#2a12695c421a2a2cc2138f05a949d04ed086e38b" - integrity sha512-VU6tuO8eKajrFeBzMssFUP9SvakEeeSi1BxdTH5o3+1yUyrldp8IERkSdXlMI2t4kxF2sqYUDsQY+WJBxzBmZg== + version "15.10.1" + resolved "https://registry.yarnpkg.com/vue-loader/-/vue-loader-15.10.1.tgz#c451c4cd05a911aae7b5dbbbc09fb913fb3cca18" + integrity sha512-SaPHK1A01VrNthlix6h1hq4uJu7S/z0kdLUb6klubo738NeQoLbS6V9/d8Pv19tU0XdQKju3D1HSKuI8wJ5wMA== dependencies: "@vue/component-compiler-utils" "^3.1.0" hash-sum "^1.0.2" @@ -2865,9 +2865,9 @@ vue-style-loader@^4.1.0: loader-utils "^1.0.2" vue-template-compiler@^2.7.12: - version "2.7.13" - resolved "https://registry.yarnpkg.com/vue-template-compiler/-/vue-template-compiler-2.7.13.tgz#1520a5aa6d1af51dd0622824e79814f6e8cb7058" - integrity sha512-jYM6TClwDS9YqP48gYrtAtaOhRKkbYmbzE+Q51gX5YDr777n7tNI/IZk4QV4l/PjQPNh/FVa/E92sh/RqKMrog== + version "2.7.14" + resolved "https://registry.yarnpkg.com/vue-template-compiler/-/vue-template-compiler-2.7.14.tgz#4545b7dfb88090744c1577ae5ac3f964e61634b1" + integrity sha512-zyA5Y3ArvVG0NacJDkkzJuPQDF8RFeRlzV2vLeSnhSpieO6LK2OVbdLPi5MPPs09Ii+gMO8nY4S3iKQxBxDmWQ== dependencies: de-indent "^1.0.2" he "^1.2.0" From 3bd446d07363841a6a94a536cd5da9d73f2b23f1 Mon Sep 17 00:00:00 2001 From: Vladislav Korenkov Date: Fri, 2 Dec 2022 18:14:22 +1000 Subject: [PATCH 4/8] Chore: Use requirements files in tox.ini --- tox_build.ini | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tox_build.ini b/tox_build.ini index 378e4046..b965ea2b 100644 --- a/tox_build.ini +++ b/tox_build.ini @@ -19,11 +19,8 @@ commands = deb: make deb PY=python {posargs} rpm: make rpm PY=python {posargs} deps = - cython>=0.29,<1.0 - wheel==0.31.1 - setuptools>=40.6.3 - jsmin==3.0.0 - csscompressor==0.9.5 + -rce/requirements.txt + -rrequirements.txt {rpm,deb}: virtualenv==16.0 {rpm,deb}: venvctrl From e3d3053dcf97276c0bd5e5274f79f5c4a0497f3f Mon Sep 17 00:00:00 2001 From: Vladislav Korenkov Date: Mon, 5 Dec 2022 16:12:48 +1000 Subject: [PATCH 5/8] Fix: tox_build --- .gitlab-ci.yml | 2 +- tox_build.ini | 15 +-------------- 2 files changed, 2 insertions(+), 15 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cbea6828..418969fc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -148,7 +148,7 @@ release_pypi: variables: TOX_ARGS: "" script: - - tox -c tox_build.ini $TOX_ARGS -e py37-build,py37-wheel + - tox -c tox_build.ini $TOX_ARGS - twine upload -u ${PYPI_UPLOAD_NAME} -p ${PYPI_UPLOAD_PASSWORD} $(find dist/*.{tar.gz,whl}) artifacts: name: "release-packages-${CI_BUILD_REF_NAME}.${CI_BUILD_ID}" diff --git a/tox_build.ini b/tox_build.ini index b965ea2b..9b4801b1 100644 --- a/tox_build.ini +++ b/tox_build.ini @@ -1,5 +1,5 @@ [tox] -envlist = py37-build,py{36,37,38,39,310}-wheel,auditwheel +envlist = py37-{build,wheel} skipsdist = True [testenv] @@ -19,19 +19,6 @@ commands = deb: make deb PY=python {posargs} rpm: make rpm PY=python {posargs} deps = - -rce/requirements.txt -rrequirements.txt {rpm,deb}: virtualenv==16.0 {rpm,deb}: venvctrl - -[testenv:auditwheel] -basepython = python3.6 -whitelist_externals = - bash - grep - rm - ls -commands = - bash -c "for whl in `ls dist/*.whl | grep -v manylinux | grep -v none-any`; do auditwheel repair $whl -w dist/ && rm $whl; done" -deps = - auditwheel~=5.1.2 From 96258e522eecb07068ebc4226779ad83b9d0be6b Mon Sep 17 00:00:00 2001 From: Sergey Klyuykov Date: Mon, 5 Dec 2022 13:50:21 +0400 Subject: [PATCH 6/8] Fix(packaging): Include polemarch/settings.ini to package. --- MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 6e71b3b1..c0176bcb 100755 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,6 @@ include polemarch/web.ini setup.cfg include README.rst LICENSE LICENSE_NAME -include polemarch/main/settings.ini +include polemarch/settings.ini include requirements.txt include requirements-git.txt include requirements-test.txt From 5bbef31e471244e8272e63c0a0bc5ceb1b194693 Mon Sep 17 00:00:00 2001 From: Sergey Klyuykov Date: Mon, 5 Dec 2022 14:41:59 +0400 Subject: [PATCH 7/8] Fix: Docs building and compiling. --- requirements-doc.txt | 4 +--- setup.py | 6 ++++-- tox_build.ini | 10 +++------- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/requirements-doc.txt b/requirements-doc.txt index c4acf281..9445a19e 100644 --- a/requirements-doc.txt +++ b/requirements-doc.txt @@ -1,4 +1,2 @@ # Docs -vstutils[rpc,ldap,doc,prod]~=5.1.11 -markdown2~=2.4.0 -gitpython~=3.1.14 +-rrequirements.txt diff --git a/setup.py b/setup.py index f0d29cc5..b1c5165f 100644 --- a/setup.py +++ b/setup.py @@ -387,6 +387,8 @@ def make_setup(**opts): if 'develop' in sys.argv: ext_list = [] +install_requirements = load_requirements('requirements.txt', os.getcwd()) + kwargs = dict( name='polemarch', packages=find_packages(), @@ -395,11 +397,11 @@ def make_setup(**opts): 'polemarch/templates/gui/service-worker.js', ], install_requires=[ - ] + load_requirements('requirements.txt', os.getcwd()) + load_requirements('requirements-doc.txt', os.getcwd()), + ] + install_requirements, extras_require={ 'test': load_requirements('requirements-test.txt', os.getcwd()) + [ i.replace('prod', 'test,prod') - for i in load_requirements('requirements.txt', os.getcwd()) + for i in install_requirements if isinstance(i, str) and 'vstutils' in i ], 'mysql': ['mysqlclient'], diff --git a/tox_build.ini b/tox_build.ini index 9b4801b1..342370e2 100644 --- a/tox_build.ini +++ b/tox_build.ini @@ -11,14 +11,10 @@ whitelist_externals = bash grep mkdir - make commands = rm -rf build - build: make compile PY=python {posargs} - wheel: make wheel PY=python {posargs} - deb: make deb PY=python {posargs} - rpm: make rpm PY=python {posargs} + build: python setup.py compile -v {posargs} + wheel: python setup.py compile_docs -v {posargs} + wheel: python setup.py bdist_wheel -v {posargs} deps = -rrequirements.txt - {rpm,deb}: virtualenv==16.0 - {rpm,deb}: venvctrl From 7e38b8057e3d490f1dd0d99a420927580fdf906b Mon Sep 17 00:00:00 2001 From: Sergey Klyuykov Date: Mon, 5 Dec 2022 21:15:52 +1000 Subject: [PATCH 8/8] Fix: Dockerfile missed `&&` --- Dockerfile | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8815d302..05612ccc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -53,16 +53,16 @@ RUN --mount=type=cache,target=/var/cache/apt \ --mount=type=bind,from=build,source=/usr/local/polemarch/,target=/polemarch_env \ apt update && \ apt -y install --no-install-recommends \ - git \ - sudo \ - sshpass \ - libmysqlclient21 \ - libpcre3 \ - libldap-2.4-2 \ - libsasl2-2 \ - libffi7 \ - libssl1.1 \ - openssh-client \ + git \ + sudo \ + sshpass \ + libmysqlclient21 \ + libpcre3 \ + libldap-2.4-2 \ + libsasl2-2 \ + libffi7 \ + libssl1.1 \ + openssh-client && \ python3.8 -m pip install cryptography paramiko 'pip<22' && \ ln -s /usr/bin/python3.8 /usr/bin/python && \ mkdir -p /projects /hooks /run/openldap /etc/polemarch/hooks /var/lib/polemarch && \