diff --git a/.github/workflows/check-scripts-updates.yml b/.github/workflows/check-scripts-updates.yml index afbe4f3..a2cd4be 100644 --- a/.github/workflows/check-scripts-updates.yml +++ b/.github/workflows/check-scripts-updates.yml @@ -20,8 +20,8 @@ jobs: changed_files=$(git diff --name-only "$BASE_COMMIT" "$HEAD_COMMIT") - # Check if any files in scripts/ were updated (excluding wrap_scripts_in_yaml.py) - scripts_updated=$(echo "$changed_files" | grep -v 'wrap_scripts_in_yaml.py' | grep 'scripts/' || true) + # Check if any files in scripts/ were updated + scripts_updated=$(echo "$changed_files" | grep 'scripts/' || true) echo "Scripts updated: $scripts_updated" # Check if any files in playbooks/ were updated diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1e0fdea..219e094 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,9 +29,10 @@ repos: "-sn", # Don't display the score "--rcfile=.pylintrc", # Link to your config file ] - - id: wrap-scripts-in-yaml - name: wrap-scripts-in-yaml - entry: python scripts/wrap_scripts_in_yaml.py + - id: sync-scripts-to-yaml + name: sync-scripts-to-yaml + entry: python misc/sync_scripts_to_yaml.py + args: ["--target", "repo"] language: python files: scripts/.*\.py$ diff --git a/misc/sync_scripts.py b/misc/sync_scripts.py index 193a239..1e84ae1 100644 --- a/misc/sync_scripts.py +++ b/misc/sync_scripts.py @@ -1,14 +1,17 @@ import os +import argparse import ruamel.yaml # Scripts located in this project SCRIPT_PATH = "scripts/leapp_script.py" -# Yaml playbooks in rhc-worker-script -PRE_UPGRADE_YAML_PATH = os.path.join( +REPO_PRE_UPGRADE_YAML_PATH = os.path.join(".", "playbooks/leapp_preupgrade_script.yaml") +REPO_UPGRADE_YAML_PATH = os.path.join(".", "playbooks/leapp_upgrade_script.yaml") + +WORKER_PRE_UPGRADE_YAML_PATH = os.path.join( "..", "rhc-worker-script/development/nginx/data/leapp_preupgrade.yml" ) -UPGRADE_YAML_PATH = os.path.join( +WORKER_UPGRADE_YAML_PATH = os.path.join( "..", "rhc-worker-script/development/nginx/data/leapp_upgrade.yml" ) @@ -44,7 +47,7 @@ def _get_updated_yaml_content(yaml_path, script_path): content = script.read() script_type = "PREUPGRADE" if "preupgrade" in yaml_path else "UPGRADE" - config[0]["name"] = "LEAPP %s" % script_type.title() + config[0]["name"] = "Leapp %s for rhc-worker-script" % script_type.title() config[0]["vars"]["content"] = content config[0]["vars"]["content_vars"]["LEAPP_SCRIPT_TYPE"] = script_type return config, mapping, offset @@ -59,14 +62,31 @@ def _write_content(config, path, mapping=None, offset=None): def main(): - config, mapping, offset = _get_updated_yaml_content( - PRE_UPGRADE_YAML_PATH, SCRIPT_PATH + parser = argparse.ArgumentParser() + parser.add_argument( + "--target", + choices=["repo", "worker"], + help="Target to sync scripts to", + default="worker", ) - print("Writing new content to %s" % PRE_UPGRADE_YAML_PATH) - _write_content(config, PRE_UPGRADE_YAML_PATH, mapping, offset) - config, mapping, offset = _get_updated_yaml_content(UPGRADE_YAML_PATH, SCRIPT_PATH) - print("Writing new content to %s" % UPGRADE_YAML_PATH) - _write_content(config, UPGRADE_YAML_PATH, mapping, offset) + args = parser.parse_args() + + if args.target == "repo": + print("Syncing scripts to repo") + pre_upgrade_path = REPO_PRE_UPGRADE_YAML_PATH + upgrade_path = REPO_UPGRADE_YAML_PATH + + elif args.target == "worker": + print("Syncing scripts to worker") + pre_upgrade_path = WORKER_PRE_UPGRADE_YAML_PATH + upgrade_path = WORKER_UPGRADE_YAML_PATH + + config, mapping, offset = _get_updated_yaml_content(pre_upgrade_path, SCRIPT_PATH) + print("Writing new content to %s" % pre_upgrade_path) + _write_content(config, pre_upgrade_path, mapping, offset) + config, mapping, offset = _get_updated_yaml_content(upgrade_path, SCRIPT_PATH) + print("Writing new content to %s" % upgrade_path) + _write_content(config, upgrade_path, mapping, offset) if __name__ == "__main__": diff --git a/playbooks/leapp_preupgrade_script.yaml b/playbooks/leapp_preupgrade_script.yaml index f5697b5..9666228 100644 --- a/playbooks/leapp_preupgrade_script.yaml +++ b/playbooks/leapp_preupgrade_script.yaml @@ -1,17 +1,22 @@ -- name: Leapp pre-upgrade for rhc-worker-script +- name: Leapp Preupgrade for rhc-worker-script vars: - insights_signature: !!binary | - needs signature - insights_signature_exclude: "/vars/insights_signature" + insights_signature: | + ascii_armored gpg signature + insights_signature_exclude: /vars/insights_signature,/vars/content_vars interpreter: /usr/bin/python content: | import json import os import subprocess - + # SCRIPT_TYPE is either 'PREUPGRADE' or 'UPGRADE' + # Value is set in signed yaml envelope in content_vars (RHC_WORKER_LEAPP_SCRIPT_TYPE) + SCRIPT_TYPE = os.environ.get("RHC_WORKER_LEAPP_SCRIPT_TYPE", "None") + IS_UPGRADE = SCRIPT_TYPE == "UPGRADE" + IS_PREUPGRADE = SCRIPT_TYPE == "PREUPGRADE" JSON_REPORT_PATH = "/var/log/leapp/leapp-report.json" TXT_REPORT_PATH = "/var/log/leapp/leapp-report.txt" + REBOOT_GUIDANCE_MESSAGE = "A reboot is required to continue. Please reboot your system." # Based on https://github.com/oamg/leapp/blob/master/report-schema-v110.json#L211 STATUS_CODE = { @@ -103,6 +108,7 @@ def is_non_eligible_releases(release): + """Check if the release is eligible for upgrade or preupgrade.""" print("Exit if not RHEL 7 or RHEL 8 ...") major_version, _ = release.split(".") if release is not None else (None, None) return release is None or major_version not in ["7", "8"] @@ -220,15 +226,18 @@ def setup_leapp(version): - print("Installing leapp ...") leapp_install_command, rhel_rhui_packages = _get_leapp_command_and_packages(version) - output, returncode = run_subprocess(leapp_install_command) - if returncode: - raise ProcessError( - message="Installation of leapp failed", - report="Installation of leapp failed with code '%s' and output: %s." - % (returncode, output.rstrip("\n")), - ) + if _check_if_package_installed('leapp-upgrade'): + print("'leapp-upgrade' already installed, skipping ...") + else: + print("Installing leapp ...") + output, returncode = run_subprocess(leapp_install_command) + if returncode: + raise ProcessError( + message="Installation of leapp failed", + report="Installation of leapp failed with code '%s' and output: %s." + % (returncode, output.rstrip("\n")), + ) print("Check installed rhui packages ...") for pkg in rhel_rhui_packages: @@ -252,7 +261,11 @@ ) if rhui_installed and not rhsm_repo_check_fail: - print("RHUI packages detected, adding --no-rhsm flag to preupgrade command") + print( + "RHUI packages detected, adding --no-rhsm flag to {} command".format( + SCRIPT_TYPE.title() + ) + ) command.append("--no-rhsm") return True return False @@ -274,7 +287,7 @@ def remove_previous_reports(): - print("Removing previous preupgrade reports at /var/log/leapp/leapp-report.* ...") + print("Removing previous leapp reports at /var/log/leapp/leapp-report.* ...") if os.path.exists(JSON_REPORT_PATH): os.remove(JSON_REPORT_PATH) @@ -283,17 +296,11 @@ os.remove(TXT_REPORT_PATH) - def execute_preupgrade(command): - print("Executing preupgrade ...") - _, _ = run_subprocess(command) + def execute_operation(command): + print("Executing {} ...".format(SCRIPT_TYPE.title())) + output, _ = run_subprocess(command) - # NOTE: we do not care about returncode because non-null always means actor error (or leapp error) - # if returncode: - # print( - # "The process leapp exited with code '%s' and output: %s\n" - # % (returncode, output) - # ) - # raise ProcessError(message="Leapp exited with code '%s'." % returncode) + return output def _find_highest_report_level(entries): @@ -312,8 +319,8 @@ return STATUS_CODE_NAME_MAP[valid_action_levels[0]] - def parse_results(output): - print("Processing preupgrade results ...") + def parse_results(output, reboot_required=False): + print("Processing {} results ...".format(SCRIPT_TYPE.title())) report_json = "Not found" message = "Can't open json report at " + JSON_REPORT_PATH @@ -325,16 +332,28 @@ with open(JSON_REPORT_PATH, mode="r") as handler: report_json = json.load(handler) - # NOTE: with newer schema we will need to parse groups instead of flags report_entries = report_json.get("entries", []) + + error_count = len( + [entry for entry in report_entries if "error" in entry.get("groups")] + ) inhibitor_count = len( - [entry for entry in report_entries if "inhibitor" in entry.get("flags")] + [entry for entry in report_entries if "inhibitor" in entry.get("groups")] ) - message = "Your system has %s inhibitors out of %s potential problems." % ( - inhibitor_count, - len(report_entries), + message = ( + "Your system has %s error%s and %s inhibitor%s out of %s potential problem%s." + % ( + error_count, + "" if error_count == 1 else "s", + inhibitor_count, + "" if inhibitor_count == 1 else "s", + len(report_entries), + "" if len(report_entries) == 1 else "s", + ) ) - alert = inhibitor_count > 0 + if reboot_required: + message += " System is ready to be upgraded. Rebooting system in 1 minute." + alert = inhibitor_count > 0 or error_count > 0 status = ( _find_highest_report_level(report_entries) if len(report_entries) > 0 @@ -355,24 +374,36 @@ output.report = report_txt - def update_insights_inventory(): + def update_insights_inventory(output): """Call insights-client to update insights inventory.""" print("Updating system status in Red Hat Insights.") - output, returncode = run_subprocess(cmd=["/usr/bin/insights-client"]) + _, returncode = run_subprocess(cmd=["/usr/bin/insights-client"]) if returncode: - raise ProcessError( - message="Failed to update Insights Inventory by registering the system again. See output the following output: %s" - % output, - report="insights-client execution exited with code '%s'." % returncode, - ) + print("System registration failed with exit code %s." % returncode) + output.message += " Failed to update Insights Inventory." + output.alert = True + return print("System registered with insights-client successfully.") + def reboot_system(): + print("Rebooting system in 1 minute.") + run_subprocess(["/usr/sbin/shutdown", "-r", "1"], wait=False) + + def main(): try: - # Exit if not RHEL 7 or 8 + # Exit if invalid value for SCRIPT_TYPE + if SCRIPT_TYPE not in ["PREUPGRADE", "UPGRADE"]: + raise ProcessError( + message="Allowed values for RHC_WORKER_LEAPP_SCRIPT_TYPE are 'PREUPGRADE' and 'UPGRADE'.", + report="Exiting because RHC_WORKER_LEAPP_SCRIPT_TYPE='%s'" + % SCRIPT_TYPE, + ) + + # Exit if not eligible release dist, version = get_rhel_version() if dist != "rhel" or is_non_eligible_releases(version): raise ProcessError( @@ -382,19 +413,24 @@ ) output = OutputCollector() - preupgrade_command = ["/usr/bin/leapp", "preupgrade", "--report-schema=1.1.0"] + preupgrade_command = ["/usr/bin/leapp", "preupgrade", "--report-schema=1.2.0"] + upgrade_command = ["/usr/bin/leapp", "upgrade", "--report-schema=1.2.0"] + operation_command = preupgrade_command if IS_PREUPGRADE else upgrade_command rhui_pkgs = setup_leapp(version) # Check for RHUI PKGs - use_no_rhsm = should_use_no_rhsm_check(len(rhui_pkgs) > 1, preupgrade_command) + use_no_rhsm = should_use_no_rhsm_check(len(rhui_pkgs) > 1, operation_command) if use_no_rhsm: install_leapp_pkg_corresponding_to_installed_rhui(rhui_pkgs) remove_previous_reports() - execute_preupgrade(preupgrade_command) - parse_results(output) - update_insights_inventory() - print("Pre-upgrade successfully executed.") + leapp_output = execute_operation(operation_command) + reboot_required = REBOOT_GUIDANCE_MESSAGE in leapp_output + parse_results(output, reboot_required) + update_insights_inventory(output) + print("Operation {} finished successfully.".format(SCRIPT_TYPE.title())) + if reboot_required: + reboot_system() except ProcessError as exception: print(exception.report) output = OutputCollector( @@ -422,3 +458,6 @@ if __name__ == "__main__": main() content_vars: + # variables that will be handed to the script as environment vars + # will be prefixed with RHC_WORKER_* + LEAPP_SCRIPT_TYPE: PREUPGRADE diff --git a/playbooks/leapp_upgrade_script.yaml b/playbooks/leapp_upgrade_script.yaml index e758677..c0dd9ff 100644 --- a/playbooks/leapp_upgrade_script.yaml +++ b/playbooks/leapp_upgrade_script.yaml @@ -1,15 +1,19 @@ -- name: Leapp upgrade for rhc-worker-script +- name: Leapp Upgrade for rhc-worker-script vars: - insights_signature: !!binary | - needs signature - insights_signature_exclude: "/vars/insights_signature" + insights_signature: | + ascii_armored gpg signature + insights_signature_exclude: /vars/insights_signature,/vars/content_vars interpreter: /usr/bin/python content: | import json import os import subprocess - + # SCRIPT_TYPE is either 'PREUPGRADE' or 'UPGRADE' + # Value is set in signed yaml envelope in content_vars (RHC_WORKER_LEAPP_SCRIPT_TYPE) + SCRIPT_TYPE = os.environ.get("RHC_WORKER_LEAPP_SCRIPT_TYPE", "None") + IS_UPGRADE = SCRIPT_TYPE == "UPGRADE" + IS_PREUPGRADE = SCRIPT_TYPE == "PREUPGRADE" JSON_REPORT_PATH = "/var/log/leapp/leapp-report.json" TXT_REPORT_PATH = "/var/log/leapp/leapp-report.txt" REBOOT_GUIDANCE_MESSAGE = "A reboot is required to continue. Please reboot your system." @@ -104,11 +108,10 @@ def is_non_eligible_releases(release): - print("Exit if not RHEL 7.9 or 8.4") - eligible_releases = ["7.9", "8.4"] - major_version, minor = release.split(".") if release is not None else (None, None) - version_str = major_version + "." + minor - return release is None or version_str not in eligible_releases + """Check if the release is eligible for upgrade or preupgrade.""" + print("Exit if not RHEL 7 or RHEL 8 ...") + major_version, _ = release.split(".") if release is not None else (None, None) + return release is None or major_version not in ["7", "8"] # Code taken from @@ -223,15 +226,18 @@ def setup_leapp(version): - print("Installing leapp ...") leapp_install_command, rhel_rhui_packages = _get_leapp_command_and_packages(version) - output, returncode = run_subprocess(leapp_install_command) - if returncode: - raise ProcessError( - message="Installation of leapp failed", - report="Installation of leapp failed with code '%s' and output: %s." - % (returncode, output.rstrip("\n")), - ) + if _check_if_package_installed('leapp-upgrade'): + print("'leapp-upgrade' already installed, skipping ...") + else: + print("Installing leapp ...") + output, returncode = run_subprocess(leapp_install_command) + if returncode: + raise ProcessError( + message="Installation of leapp failed", + report="Installation of leapp failed with code '%s' and output: %s." + % (returncode, output.rstrip("\n")), + ) print("Check installed rhui packages ...") for pkg in rhel_rhui_packages: @@ -255,7 +261,11 @@ ) if rhui_installed and not rhsm_repo_check_fail: - print("RHUI packages detected, adding --no-rhsm flag to preupgrade command") + print( + "RHUI packages detected, adding --no-rhsm flag to {} command".format( + SCRIPT_TYPE.title() + ) + ) command.append("--no-rhsm") return True return False @@ -277,7 +287,7 @@ def remove_previous_reports(): - print("Removing previous preupgrade reports at /var/log/leapp/leapp-report.* ...") + print("Removing previous leapp reports at /var/log/leapp/leapp-report.* ...") if os.path.exists(JSON_REPORT_PATH): os.remove(JSON_REPORT_PATH) @@ -286,20 +296,12 @@ os.remove(TXT_REPORT_PATH) - def execute_upgrade(command): - print("Executing upgrade ...") + def execute_operation(command): + print("Executing {} ...".format(SCRIPT_TYPE.title())) output, _ = run_subprocess(command) return output - # NOTE: we do not care about returncode because non-null always means actor error (or leapp error) - # if returncode: - # print( - # "The process leapp exited with code '%s' and output: %s\n" - # % (returncode, output) - # ) - # raise ProcessError(message="Leapp exited with code '%s'." % returncode) - def _find_highest_report_level(entries): """ @@ -318,7 +320,7 @@ def parse_results(output, reboot_required=False): - print("Processing upgrade results ...") + print("Processing {} results ...".format(SCRIPT_TYPE.title())) report_json = "Not found" message = "Can't open json report at " + JSON_REPORT_PATH @@ -330,18 +332,28 @@ with open(JSON_REPORT_PATH, mode="r") as handler: report_json = json.load(handler) - # NOTE: with newer schema we will need to parse groups instead of flags report_entries = report_json.get("entries", []) + + error_count = len( + [entry for entry in report_entries if "error" in entry.get("groups")] + ) inhibitor_count = len( - [entry for entry in report_entries if "inhibitor" in entry.get("flags")] + [entry for entry in report_entries if "inhibitor" in entry.get("groups")] ) - message = "Your system has %s inhibitors out of %s potential problems." % ( - inhibitor_count, - len(report_entries), + message = ( + "Your system has %s error%s and %s inhibitor%s out of %s potential problem%s." + % ( + error_count, + "" if error_count == 1 else "s", + inhibitor_count, + "" if inhibitor_count == 1 else "s", + len(report_entries), + "" if len(report_entries) == 1 else "s", + ) ) if reboot_required: message += " System is ready to be upgraded. Rebooting system in 1 minute." - alert = inhibitor_count > 0 + alert = inhibitor_count > 0 or error_count > 0 status = ( _find_highest_report_level(report_entries) if len(report_entries) > 0 @@ -362,17 +374,16 @@ output.report = report_txt - def update_insights_inventory(): + def update_insights_inventory(output): """Call insights-client to update insights inventory.""" print("Updating system status in Red Hat Insights.") - output, returncode = run_subprocess(["/usr/bin/insights-client"]) + _, returncode = run_subprocess(cmd=["/usr/bin/insights-client"]) if returncode: - raise ProcessError( - message="Failed to update Insights Inventory by registering the system again. See output the following output: %s" - % output, - report="insights-client execution exited with code '%s'." % returncode, - ) + print("System registration failed with exit code %s." % returncode) + output.message += " Failed to update Insights Inventory." + output.alert = True + return print("System registered with insights-client successfully.") @@ -384,7 +395,15 @@ def main(): try: - # Exit if not RHEL 7.9 or 8.4 + # Exit if invalid value for SCRIPT_TYPE + if SCRIPT_TYPE not in ["PREUPGRADE", "UPGRADE"]: + raise ProcessError( + message="Allowed values for RHC_WORKER_LEAPP_SCRIPT_TYPE are 'PREUPGRADE' and 'UPGRADE'.", + report="Exiting because RHC_WORKER_LEAPP_SCRIPT_TYPE='%s'" + % SCRIPT_TYPE, + ) + + # Exit if not eligible release dist, version = get_rhel_version() if dist != "rhel" or is_non_eligible_releases(version): raise ProcessError( @@ -394,20 +413,22 @@ ) output = OutputCollector() - upgrade_command = ["/usr/bin/leapp", "upgrade", "--report-schema=1.1.0"] + preupgrade_command = ["/usr/bin/leapp", "preupgrade", "--report-schema=1.2.0"] + upgrade_command = ["/usr/bin/leapp", "upgrade", "--report-schema=1.2.0"] + operation_command = preupgrade_command if IS_PREUPGRADE else upgrade_command rhui_pkgs = setup_leapp(version) # Check for RHUI PKGs - use_no_rhsm = should_use_no_rhsm_check(len(rhui_pkgs) > 1, upgrade_command) + use_no_rhsm = should_use_no_rhsm_check(len(rhui_pkgs) > 1, operation_command) if use_no_rhsm: install_leapp_pkg_corresponding_to_installed_rhui(rhui_pkgs) remove_previous_reports() - leapp_upgrade_output = execute_upgrade(upgrade_command) - reboot_required = REBOOT_GUIDANCE_MESSAGE in leapp_upgrade_output + leapp_output = execute_operation(operation_command) + reboot_required = REBOOT_GUIDANCE_MESSAGE in leapp_output parse_results(output, reboot_required) - update_insights_inventory() - print("Leapp upgrade command successfully executed.") + update_insights_inventory(output) + print("Operation {} finished successfully.".format(SCRIPT_TYPE.title())) if reboot_required: reboot_system() except ProcessError as exception: @@ -437,3 +458,6 @@ if __name__ == "__main__": main() content_vars: + # variables that will be handed to the script as environment vars + # will be prefixed with RHC_WORKER_* + LEAPP_SCRIPT_TYPE: UPGRADE diff --git a/scripts/wrap_scripts_in_yaml.py b/scripts/wrap_scripts_in_yaml.py deleted file mode 100644 index 3ce7703..0000000 --- a/scripts/wrap_scripts_in_yaml.py +++ /dev/null @@ -1,62 +0,0 @@ -""" -This file serves to be run during a pre-commit hook to wrap all scripts/ files -in yaml and convert them to ansible playbooks placed under playbooks/ folder. -""" - -import re -import sys -from pathlib import Path - - -def wrap_script_in_yaml(python_file): - yaml_file_path = f"playbooks/{Path(python_file).stem}_script.yaml" - yaml_content = generate_yaml_content(python_file) - - if not Path(yaml_file_path).exists() or open(yaml_file_path).read() != yaml_content: - with open(yaml_file_path, "w") as yaml_file: - yaml_file.write(yaml_content) - return True - return False - - -def generate_yaml_content(python_file): - with open(python_file, "r") as py_file: - content = "" - if python_file == "scripts/leapp_preupgrade.py": - content += "- name: Leapp pre-upgrade for rhc-worker-script\n" - elif python_file == "scripts/leapp_upgrade.py": - content += "- name: Leapp upgrade for rhc-worker-script\n" - content += " vars:\n" - content += " insights_signature: !!binary |\n" - content += " needs signature\n" - content += ' insights_signature_exclude: "/vars/insights_signature"\n' - content += " interpreter: /usr/bin/python\n" - content += " content: |\n" - for line in py_file: - if not line.strip(): - # Do not indent empty lines, causes errors with other pre-commit hooks - content += line - else: - content += f" {line}" - content += " content_vars:\n" - return content - - -def main(): - changes_detected = False - for filename in sys.argv[1:]: - if re.match(r"scripts/.*.py$", filename): - if filename == f"scripts/{Path(__file__).name}": - continue - if wrap_script_in_yaml(filename): - changes_detected = True - - if changes_detected: - print( - "Changes detected in ansible playbooks (coming from scripts). Please stage them and commit again." - ) - sys.exit(1) - - -if __name__ == "__main__": - main()