From c77fa38554c319aea6ead7aa358c58ec9d8f0b49 Mon Sep 17 00:00:00 2001 From: Wolfgang Medina-Erhardt Date: Tue, 26 Nov 2024 15:43:11 +0100 Subject: [PATCH] Add basic project structure --- .ansible-lint | 26 ++++++++++--- .editorconfig | 14 +++++++ .envrc | 24 ++++++++++++ .github/workflows/test.yml | 43 ++++++++++++++++++++++ .gitignore | 14 +++++++ .pre-commit-config.yaml | 3 +- .yamllint | 36 +++++------------- LICENSE | 21 +++++++++++ ansible.cfg | 6 +++ defaults/main.yaml | 8 ++-- handlers/main.yaml | 16 +++++--- inventory.ini | 30 +++++++++++++++ inventory.py | 54 +++++++++++++++++++++++++++ molecule/.molecule.yml.tmp | 27 ++++++++++++++ molecule/.verify.yml.tmp | 8 ++++ molecule/converge.yml | 9 +++++ molecule/converge.yml.template | 18 +++++++++ molecule/molecule.yml | 27 ++++++++++++++ molecule/verify.yml | 8 ++++ requirements.txt | 7 ++++ requirements.yml | 8 ++++ tasks/apache.yaml | 1 - tasks/config.yaml | 27 ++++++++++++++ tasks/main.yaml | 3 ++ tasks/nginx.yaml | 1 - tasks/solr.yaml | 61 ++++++++++++++++++++----------- tasks/tika.yaml | 9 ++++- templates/supervisord.d/solr.conf | 7 ++++ 28 files changed, 447 insertions(+), 69 deletions(-) create mode 100644 .editorconfig create mode 100644 .envrc create mode 100644 .github/workflows/test.yml create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 ansible.cfg create mode 100644 inventory.ini create mode 100755 inventory.py create mode 100644 molecule/.molecule.yml.tmp create mode 100644 molecule/.verify.yml.tmp create mode 100644 molecule/converge.yml create mode 100644 molecule/converge.yml.template create mode 100644 molecule/molecule.yml create mode 100644 molecule/verify.yml create mode 100644 requirements.txt create mode 100644 requirements.yml create mode 100644 tasks/config.yaml create mode 100644 templates/supervisord.d/solr.conf diff --git a/.ansible-lint b/.ansible-lint index 35bb270..2a150b2 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -1,11 +1,27 @@ --- skip_list: - - 'risky-shell-pipe' - - 'role-name' - - 'name[template]' + - meta-no-info + - var-naming[no-role-prefix] warn_list: - package-latest - unnamed-task - - command-instead-of-shell - - no-handler + +exclude_paths: + - group_vars/secrets.* + - roles/dehydrated + - roles/mailhog + - roles/postgresql + - roles/mail + - roles/mariadb + - roles/nginx + - roles/elasticsearch + - roles/oauth2_proxy + - roles/php + - roles/redis + - roles/supervisord + - roles/system + - roles/apache + - roles/solr + - roles/varnish + - roles/docker diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..f202fbd --- /dev/null +++ b/.editorconfig @@ -0,0 +1,14 @@ +# EditorConfig: http://EditorConfig.org + +# top-most EditorConfig file +root = true + +# Unix-style newlines with a newline ending every file +[*] +end_of_line = lf +insert_final_newline = true + +# 2 space indentation for YAML +[{*.yml,*.yaml}] +indent_style = space +indent_size = 2 diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..455c5ee --- /dev/null +++ b/.envrc @@ -0,0 +1,24 @@ +#!/bin/sh +if [ -n "$$DIRENV_DIFF" ]; then + # Do not `set -e` unless we're being run from direnv + set -e +fi + +if [ ! -f .venv/bin/activate ] +then + echo "Creating a virtual Python environment" + python3 -m venv .venv >/dev/null +fi + +if [ -f .venv/bin/activate ] +then + echo "Activating the Python virtual environment" + . .venv/bin/activate +fi + +if [ -f .venv/bin/pip3 ] +then + echo "Installing Python requirements" + .venv/bin/pip3 install --upgrade pip 1> /dev/null + .venv/bin/pip3 install -r requirements.txt 1> /dev/null +fi diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..74ab03a --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,43 @@ +--- +name: Test +run-name: Run molecule tests on the role +on: + push: + workflow_dispatch: + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + + - name: Install dependencies. + run: pip install -r requirements.txt + + - name: Run ansible-lint + run: "ansible-lint" + + molecule: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + + - name: Install dependencies. + run: | + pip install -r requirements.txt + ansible-galaxy install -r requirements.yml + + - name: Run molecule + run: molecule test diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b83c6a8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,14 @@ +.DS_Store +*.pyc +__pycache__ +.idea +.vscode +/*.retry +/.fact_cache +/venv/ +debug.log +id_ed25519* +/venv*/ +.collections +roles +.venv diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 53611d0..2830351 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,7 @@ --- repos: - repo: https://github.com/ansible/ansible-lint.git - rev: v6.17.2 + rev: v24.6.1 hooks: - id: ansible-lint + files: \.(yaml|yml)$ diff --git a/.yamllint b/.yamllint index 8827676..2ac072a 100644 --- a/.yamllint +++ b/.yamllint @@ -1,33 +1,15 @@ --- -# Based on ansible-lint config extends: default rules: + line-length: disable + truthy: + allowed-values: ['true', 'false', 'yes', 'no'] + comments: + min-spaces-from-content: 1 braces: + min-spaces-inside: 0 max-spaces-inside: 1 - level: error - brackets: - max-spaces-inside: 1 - level: error - colons: - max-spaces-after: -1 - level: error - commas: - max-spaces-after: -1 - level: error - comments: disable - comments-indentation: disable - document-start: disable - empty-lines: - max: 3 - level: error - hyphens: - level: error - indentation: disable - key-duplicates: enable - line-length: disable - new-line-at-end-of-file: disable - new-lines: - type: unix - trailing-spaces: disable - truthy: disable + +ignore: | + .cache diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..5609724 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2017 punkt.de GmbH + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/ansible.cfg b/ansible.cfg new file mode 100644 index 0000000..fac200a --- /dev/null +++ b/ansible.cfg @@ -0,0 +1,6 @@ +[defaults] +roles_path = ./roles +collections_path = ./.collections + +[ssh_connection] +pipelining = true diff --git a/defaults/main.yaml b/defaults/main.yaml index b1570da..ac909d2 100644 --- a/defaults/main.yaml +++ b/defaults/main.yaml @@ -2,12 +2,12 @@ solr: domain: prefix: - bin: /var/solr - var: /var/db/solr - version: 9.2.0 + bin: /var/opt/solr/solr + home: /var/opt/solr/home + version: 9.7.0 tika: prefix: bin: /var/opt/tika - version: 2.8.0 + version: 3.0.0 synced_config: oauth2_proxy: diff --git a/handlers/main.yaml b/handlers/main.yaml index 2f51df0..9937c7c 100644 --- a/handlers/main.yaml +++ b/handlers/main.yaml @@ -1,14 +1,18 @@ --- - name: Start Solr - ansible.builtin.service: - name: solr - state: started + ansible.builtin.shell: + cmd: | + set -e -u -o pipefail + supervisorctl update solr + supervisorctl start solr register: solr_service_result - name: Restart Solr - ansible.builtin.service: - name: solr - state: restarted + ansible.builtin.shell: + cmd: | + set -e -u -o pipefail + supervisorctl update solr + supervisorctl restart solr when: not (solr_service_result is defined and solr_service_result.changed) - name: Start Tika diff --git a/inventory.ini b/inventory.ini new file mode 100644 index 0000000..c6155f4 --- /dev/null +++ b/inventory.ini @@ -0,0 +1,30 @@ +[app_prd] +vpro0000.proserver.punkt.de ansible_user=vpro0000 + +[app_stg] +vpro0000.proserver.punkt.de ansible_user=vpro0000 + +[app] +[app:children] +app_stg +app_prd + +[production] +[production:children] +app_prd + +[staging] +[staging:children] +app_stg + +[mailhog] +[mailhog:children] +app_stg + +[secrets] +[secrets:children] +app + +[mariadb] +[mariadb:children] +app diff --git a/inventory.py b/inventory.py new file mode 100755 index 0000000..7a859c1 --- /dev/null +++ b/inventory.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +import json +import yaml +import requests +from typing import List + + +class Inventory: + inventory_url = None + vars = None + + def __init__(self, inventory_url='https://inventory.punkt.app'): + self.inventory_url = inventory_url + self._load_vars() + self._load_public_keys() + + def _load_vars(self) -> None: + with requests.get(f'{self.inventory_url}/defaults/main.yaml', stream=True, timeout=30) as response: + self.vars = yaml.safe_load(response.content) + + def _load_public_keys(self) -> None: + for user, user_info in self.vars['punktde']['people'].items(): + user_info['public_keys'] = self._get_public_keys(user, user_info['email']) + + def _get_public_keys(self, user: str, email: str) -> List: + response = requests.get(f'{self.inventory_url}/files/public_keys/{user}.pub', timeout=30) + if response.status_code != 200: + return [] + + public_keys = [] + for public_key in response.content.decode().splitlines(): + public_key = public_key.strip() + if not public_key or public_key.startswith('#'): + continue + public_key = public_key.split(' ', 2) + public_key = f'{public_key[0]} {public_key[1]} {email}' + public_keys.append(public_key) + break + + return public_keys + + def inventory(self): + return { + 'all': { + 'vars': self.vars, + } + } + + def __str__(self) -> str: + return json.dumps(self.inventory()) + + +if __name__ == '__main__': + print(str(Inventory())) diff --git a/molecule/.molecule.yml.tmp b/molecule/.molecule.yml.tmp new file mode 100644 index 0000000..c549054 --- /dev/null +++ b/molecule/.molecule.yml.tmp @@ -0,0 +1,27 @@ +--- +dependency: + name: galaxy +driver: + name: docker +platforms: + - name: instance + image: geerlingguy/docker-ubuntu2204-ansible + command: /lib/systemd/systemd + pre_build_image: true + privileged: true + cgroupns_mode: host + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw +provisioner: + name: ansible + playbooks: + converge: ${MOLECULE_PLAYBOOK:-converge.yml} +verifier: + name: ansible +scenario: + name: default + test_sequence: + - destroy + - create + - converge + - verify diff --git a/molecule/.verify.yml.tmp b/molecule/.verify.yml.tmp new file mode 100644 index 0000000..70761cd --- /dev/null +++ b/molecule/.verify.yml.tmp @@ -0,0 +1,8 @@ +--- +- name: Verify + hosts: all + gather_facts: false + tasks: + - name: Example assertion + ansible.builtin.assert: + that: true diff --git a/molecule/converge.yml b/molecule/converge.yml new file mode 100644 index 0000000..b67fd71 --- /dev/null +++ b/molecule/converge.yml @@ -0,0 +1,9 @@ +--- +- name: Converge + hosts: all + environment: + GITHUB_ACTIONS: "maybe" + tasks: + - name: "Include {{ role_name }}" + ansible.builtin.include_role: + name: "{{ role_name }}" diff --git a/molecule/converge.yml.template b/molecule/converge.yml.template new file mode 100644 index 0000000..b233329 --- /dev/null +++ b/molecule/converge.yml.template @@ -0,0 +1,18 @@ +--- +- name: Converge + hosts: all + environment: + GITHUB_ACTIONS: "maybe" + tasks: + - name: "Include {{ role_name }}" + ansible.builtin.include_role: + name: "{{ role_name }}" +--- +- name: Converge + hosts: all + environment: + GITHUB_ACTIONS: "maybe" + tasks: + - name: "Include {{ role_name }}" + ansible.builtin.include_role: + name: "{{ role_name }}" diff --git a/molecule/molecule.yml b/molecule/molecule.yml new file mode 100644 index 0000000..c549054 --- /dev/null +++ b/molecule/molecule.yml @@ -0,0 +1,27 @@ +--- +dependency: + name: galaxy +driver: + name: docker +platforms: + - name: instance + image: geerlingguy/docker-ubuntu2204-ansible + command: /lib/systemd/systemd + pre_build_image: true + privileged: true + cgroupns_mode: host + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw +provisioner: + name: ansible + playbooks: + converge: ${MOLECULE_PLAYBOOK:-converge.yml} +verifier: + name: ansible +scenario: + name: default + test_sequence: + - destroy + - create + - converge + - verify diff --git a/molecule/verify.yml b/molecule/verify.yml new file mode 100644 index 0000000..70761cd --- /dev/null +++ b/molecule/verify.yml @@ -0,0 +1,8 @@ +--- +- name: Verify + hosts: all + gather_facts: false + tasks: + - name: Example assertion + ansible.builtin.assert: + that: true diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..1042bff --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +ansible +ansible-lint +docker +molecule +molecule-plugins +yamllint +requests diff --git a/requirements.yml b/requirements.yml new file mode 100644 index 0000000..b24b298 --- /dev/null +++ b/requirements.yml @@ -0,0 +1,8 @@ +--- +collections: + - name: https://github.com/ansible-collections/community.general + type: git + - name: https://github.com/ansible-collections/community.crypto + type: git + - name: https://github.com/ansible-collections/ansible.posix + type: git diff --git a/tasks/apache.yaml b/tasks/apache.yaml index 80d4e8d..bd1bd5d 100644 --- a/tasks/apache.yaml +++ b/tasks/apache.yaml @@ -4,7 +4,6 @@ src: "{{ item }}" dest: "{{ render_path }}" owner: "root" - group: "{{ root_group }}" mode: "0644" loop_control: label: "{{ render_path }}" diff --git a/tasks/config.yaml b/tasks/config.yaml new file mode 100644 index 0000000..1038e4f --- /dev/null +++ b/tasks/config.yaml @@ -0,0 +1,27 @@ +--- +- name: Comment out unsupported options (Solr 8.x) + notify: Restart Solr + when: solr.version is ansible.builtin.version('9', '<') + loop: + - allowUrls + - modules + - zkCredentialsInjector + - distributedClusterStateUpdates + - distributedCollectionConfigSetExecution + - minStateByteLenForCompression + - stateCompressor + - hideStackTrace + ansible.builtin.lineinfile: + path: /var/db/solr/solr.xml + regexp: '^(?!<\!\-\-)(.*{{ item }}.*)$' + line: + state: present + backrefs: yes + +- name: Restore unsupported options (Solr 9.x) + notify: Restart Solr + when: solr.version is ansible.builtin.version('9', '>=') + ansible.builtin.replace: + path: /var/db/solr/solr.xml + regexp: "^.*$" + replace: \g<1> diff --git a/tasks/main.yaml b/tasks/main.yaml index a160c28..8facb34 100644 --- a/tasks/main.yaml +++ b/tasks/main.yaml @@ -2,6 +2,9 @@ - name: Install Apache Solr ansible.builtin.include_tasks: solr.yaml +- name: Configure Apache Solr + ansible.builtin.include_tasks: config.yaml + - name: Install Apache Tika ansible.builtin.include_tasks: tika.yaml diff --git a/tasks/nginx.yaml b/tasks/nginx.yaml index 9e62f44..5b6ea0f 100644 --- a/tasks/nginx.yaml +++ b/tasks/nginx.yaml @@ -4,7 +4,6 @@ src: "{{ item }}" dest: "{{ render_path }}" owner: "root" - group: "{{ root_group }}" mode: "0644" loop_control: label: "{{ render_path }}" diff --git a/tasks/solr.yaml b/tasks/solr.yaml index 1011b8d..0494619 100644 --- a/tasks/solr.yaml +++ b/tasks/solr.yaml @@ -6,8 +6,17 @@ owner: solr group: solr mode: "0755" - with_items: + recurse: yes + loop: - "{{ solr.prefix.bin }}" + - "{{ solr.prefix.home }}" + +- name: Create log directory for Solr + ansible.builtin.file: + path: "/var/log/solr" + state: directory + owner: nobody + mode: "0755" - name: Download Solr {{ solr.version }} ansible.builtin.get_url: @@ -31,9 +40,10 @@ when: solr_get_url_result.changed block: - name: Stop Solr service - ansible.builtin.service: - name: solr - state: stopped + ansible.builtin.shell: + cmd: | + set -e -u -o pipefail + sudo supervisorctl stop solr || true - name: Kill all solr processes changed_when: yes @@ -64,23 +74,30 @@ with_items: - "{{ solr.prefix.bin }}" - -- name: Sync Solr config - ansible.posix.synchronize: - src: "{{ solr.synced_config }}/" - dest: "{{ item }}" - rsync_opts: - - --chown=solr:solr - - --no-times - - --checksum - use_ssh_args: yes - with_items: - - "{{ solr.prefix.var }}" - when: solr.synced_config - notify: Restart Solr - -- name: Enable Solr service +- name: Stop and disable the native rc.d Solr service + failed_when: no ansible.builtin.service: name: solr - enabled: yes - notify: Start Solr + state: stopped + enabled: no + notify: + - Start Solr + - Restart Solr + +- name: Install Solr service + ansible.builtin.template: + src: supervisord.d/solr.conf + dest: "/usr/local/etc/supervisord.d/solr.conf" + owner: root + group: wheel + mode: "0755" + notify: + - Start Solr + - Restart Solr + +- name: Reload supervisord configuration + changed_when: no + ansible.builtin.shell: + cmd: | + set -e -u -o pipefail + sudo supervisorctl update diff --git a/tasks/tika.yaml b/tasks/tika.yaml index 6d1bb54..1a08ac9 100644 --- a/tasks/tika.yaml +++ b/tasks/tika.yaml @@ -4,7 +4,6 @@ path: "{{ item }}" state: directory owner: root - group: "{{ root_group }}" mode: "0755" with_items: - "{{ solr.tika.prefix.bin }}" @@ -23,7 +22,6 @@ dest: "{{ solr.tika.prefix.bin }}/tika-server.jar" checksum: "sha512:https://archive.apache.org/dist/tika/{{ solr.tika.version }}/tika-server-{{ tika_standard }}{{ solr.tika.version }}.jar.sha512" owner: root - group: "{{ root_group }}" mode: "0755" vars: tika_standard: "{{ 'standard-' if solr.tika.version is version('2', '>=') else '' }}" @@ -51,3 +49,10 @@ notify: - Start Tika - Restart Tika + +- name: Reload supervisord configuration + changed_when: no + ansible.builtin.shell: + cmd: | + set -e -u -o pipefail + sudo supervisorctl update diff --git a/templates/supervisord.d/solr.conf b/templates/supervisord.d/solr.conf new file mode 100644 index 0000000..f068a9a --- /dev/null +++ b/templates/supervisord.d/solr.conf @@ -0,0 +1,7 @@ +[program:tika] +user=nobody +command={{ solr.prefix.bin }}/bin/solr start -f -s {{ solr.prefix.home }} +startsecs=10 +environment=PATH=/usr/local/bin:/usr/bin,SOLR_INCLUDE="{{ solr.prefix.bin }}/bin/solr.in.sh" +stdout_logfile=/var/log/solr/solr.log +redirect_stderr=true