diff --git a/README.md b/README.md index 2cbb61f..622a4f2 100644 --- a/README.md +++ b/README.md @@ -121,11 +121,19 @@ docker build -t netbox-plugin . >Why FTP? Originally scp was used to transfer files, but based on experience, FTP is much faster. +>HTTP was added in 0.0.3, But FTP_USERNAME in configuration still required as it used as folder name for IOS upload + ```shell cd ftp docker build -t ftp_for_netbox . cd ../../ ``` +or +```shell +cd http +docker build -t http_for_netbox . +cd ../../ +``` ## 3. Change docker-compose.yml ```dockerfile @@ -175,6 +183,16 @@ cd ../../ - USERS=software-images|ftp_password # Your external (host) IP address, not contaner's IP - ADDRESS=192.168.0.1 + + # simple http server + http: + image: http_for_netbox + ports: + - "80:80" + volumes: + # Mount folder with IOS images. FTP has RO only. + - ./netbox-software-manager/software-images:/usr/share/nginx/html:z,ro + ``` ## 4. Change NetBox configuration.py @@ -193,7 +211,11 @@ PLUGINS_CONFIG = { 'FTP_USERNAME': 'software-images', 'FTP_PASSWORD': 'ftp_password', 'FTP_SERVER': '192.168.0.1', - + # HTTP server name with patch to images + "HTTP_SERVER": "http://192.168.0./", + # Default transport method, can be also changed while scheduling task, [tfp|http] + "DEFAULT_TRANSFER_METHOD": "ftp", + # Log file 'UPGRADE_LOG_FILE': '/var/log/upgrade.log', # Queue name. Check step 1. Should be the same diff --git a/http/default.conf b/http/default.conf new file mode 100644 index 0000000..0b901ff --- /dev/null +++ b/http/default.conf @@ -0,0 +1,9 @@ +server { + listen 80; + server_name localhost; + + location / { + root /usr/share/nginx/html; + autoindex on; + } +} \ No newline at end of file diff --git a/http/dockerfile b/http/dockerfile new file mode 100644 index 0000000..e63c9e6 --- /dev/null +++ b/http/dockerfile @@ -0,0 +1,5 @@ +FROM nginx + +ADD default.conf /etc/nginx/conf.d/ + +# VOLUME ["/usr/share/nginx/html"] diff --git a/setup.py b/setup.py index 319b0f0..3a08901 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name="netbox-plugin-software-manager", - version="0.0.2", + version="0.0.3", description="Software Manager for Cisco IOS/IOS-XE devices", long_description=long_description, long_description_content_type="text/markdown", diff --git a/software_manager/__init__.py b/software_manager/__init__.py index 91a4916..9fde8e9 100644 --- a/software_manager/__init__.py +++ b/software_manager/__init__.py @@ -5,7 +5,7 @@ class SoftwareManager(PluginConfig): name = "software_manager" verbose_name = "Software Manager" description = "Software Manager for Cisco IOS/IOS-XE devices" - version = "0.0.2" + version = "0.0.3" author = "Alexander Ignatov" author_email = "ignatov.alx@gmail.com" required_settings = [] diff --git a/software_manager/choices.py b/software_manager/choices.py index 4a21725..16b3123 100644 --- a/software_manager/choices.py +++ b/software_manager/choices.py @@ -2,52 +2,62 @@ class TaskTypeChoices(ChoiceSet): - TYPE_UPLOAD = 'upload' - TYPE_UPGRADE = 'upgrade' + TYPE_UPLOAD = "upload" + TYPE_UPGRADE = "upgrade" CHOICES = ( - (TYPE_UPLOAD, 'upload'), - (TYPE_UPGRADE, 'upgrade'), + (TYPE_UPLOAD, "upload"), + (TYPE_UPGRADE, "upgrade"), + ) + + +class TaskTransferMethod(ChoiceSet): + METHOD_FTP = "ftp" + METHOD_HTTP = "http" + + CHOICES = ( + (METHOD_FTP, "ftp"), + (METHOD_HTTP, "http"), ) class TaskStatusChoices(ChoiceSet): - STATUS_UNKNOWN = 'unknown' - STATUS_SCHEDULED = 'scheduled' - STATUS_FAILED = 'failed' - STATUS_RUNNING = 'running' - STATUS_SUCCEEDED = 'succeeded' - STATUS_SKIPPED = 'skipped' + STATUS_UNKNOWN = "unknown" + STATUS_SCHEDULED = "scheduled" + STATUS_FAILED = "failed" + STATUS_RUNNING = "running" + STATUS_SUCCEEDED = "succeeded" + STATUS_SKIPPED = "skipped" CHOICES = ( - (STATUS_UNKNOWN, 'unknown'), - (STATUS_SCHEDULED, 'scheduled'), - (STATUS_FAILED, 'failed'), - (STATUS_RUNNING, 'running'), - (STATUS_SUCCEEDED, 'succeeded'), - (STATUS_SKIPPED, 'skipped'), + (STATUS_UNKNOWN, "unknown"), + (STATUS_SCHEDULED, "scheduled"), + (STATUS_FAILED, "failed"), + (STATUS_RUNNING, "running"), + (STATUS_SUCCEEDED, "succeeded"), + (STATUS_SKIPPED, "skipped"), ) class TaskFailReasonChoices(ChoiceSet): - FAIL_UNKNOWN = 'fail-unknown' - FAIL_CHECK = 'fail-check' - FAIL_LOGIN = 'fail-login' - FAIL_CONFIG = 'fail-config' - FAIL_CONNECT = 'fail-connect' - FAIL_GENERAL = 'fail-general' - FAIL_ADD = 'fail-add' - FAIL_UPGRADE = 'fail-upgrade' - FAIL_UPLOAD = 'fail-upload' + FAIL_UNKNOWN = "fail-unknown" + FAIL_CHECK = "fail-check" + FAIL_LOGIN = "fail-login" + FAIL_CONFIG = "fail-config" + FAIL_CONNECT = "fail-connect" + FAIL_GENERAL = "fail-general" + FAIL_ADD = "fail-add" + FAIL_UPGRADE = "fail-upgrade" + FAIL_UPLOAD = "fail-upload" CHOICES = ( - (FAIL_UNKNOWN, 'fail-unknown'), - (FAIL_CHECK, 'fail-check'), - (FAIL_LOGIN, 'fail-login'), - (FAIL_CONFIG, 'fail-config'), - (FAIL_CONNECT, 'fail-connect'), - (FAIL_GENERAL, 'fail-general'), - (FAIL_ADD, 'fail-add'), - (FAIL_UPGRADE, 'fail-upgrade'), - (FAIL_UPLOAD, 'fail-upload'), + (FAIL_UNKNOWN, "fail-unknown"), + (FAIL_CHECK, "fail-check"), + (FAIL_LOGIN, "fail-login"), + (FAIL_CONFIG, "fail-config"), + (FAIL_CONNECT, "fail-connect"), + (FAIL_GENERAL, "fail-general"), + (FAIL_ADD, "fail-add"), + (FAIL_UPGRADE, "fail-upgrade"), + (FAIL_UPLOAD, "fail-upload"), ) diff --git a/software_manager/forms.py b/software_manager/forms.py index 589fcf7..c4fdbb9 100644 --- a/software_manager/forms.py +++ b/software_manager/forms.py @@ -5,161 +5,141 @@ from dcim.models import DeviceType, Device, DeviceRole from utilities.forms import ( - BootstrapMixin, DynamicModelMultipleChoiceField, TagFilterField, DateTimePicker, - StaticSelect2, StaticSelect2Multiple, BOOLEAN_WITH_BLANK_CHOICES, + BootstrapMixin, + DynamicModelMultipleChoiceField, + TagFilterField, + DateTimePicker, + StaticSelect2, + StaticSelect2Multiple, + BOOLEAN_WITH_BLANK_CHOICES, ) from extras.models import CustomField from tenancy.models import Tenant -from .choices import TaskTypeChoices, TaskStatusChoices +from .choices import TaskTransferMethod, TaskTypeChoices, TaskStatusChoices from .models import SoftwareImage, GoldenImage, ScheduledTask -PLUGIN_SETTINGS = settings.PLUGINS_CONFIG.get('software_manager', dict()) -CF_NAME_SW_VERSION = PLUGIN_SETTINGS.get('CF_NAME_SW_VERSION', '') +PLUGIN_SETTINGS = settings.PLUGINS_CONFIG.get("software_manager", dict()) +CF_NAME_SW_VERSION = PLUGIN_SETTINGS.get("CF_NAME_SW_VERSION", "") +DEFAULT_TRANSFER_METHOD = PLUGIN_SETTINGS.get("DEFAULT_TRANSFER_METHOD", TaskTransferMethod.METHOD_FTP) class SoftwareImageAddForm(BootstrapMixin, forms.ModelForm): image = forms.FileField( required=True, - label='IOS', - help_text='IOS Image File', + label="IOS", + help_text="IOS Image File", ) md5sum = forms.CharField( required=True, - label='MD5 Checksum', - help_text='Expected MD5 Checksum, ex: 0f58a02f3d3f1e1be8f509d2e5b58fb8', + label="MD5 Checksum", + help_text="Expected MD5 Checksum, ex: 0f58a02f3d3f1e1be8f509d2e5b58fb8", ) version = forms.CharField( required=True, - label='Version', - help_text='Verbose Software Version, ex: 15.5(3)M10', + label="Version", + help_text="Verbose Software Version, ex: 15.5(3)M10", ) class Meta: model = SoftwareImage - fields = ['image', 'md5sum', 'version'] + fields = ["image", "md5sum", "version"] class GoldenImageAddForm(BootstrapMixin, forms.ModelForm): device_pid = forms.CharField( required=True, - label='Device PID', + label="Device PID", ) sw = forms.ModelChoiceField( required=True, queryset=SoftwareImage.objects.all(), - label='Device Image File', + label="Device Image File", ) class Meta: model = GoldenImage - fields = [ - 'device_pid', 'sw' - ] + fields = ["device_pid", "sw"] def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.fields['device_pid'].widget.attrs['readonly'] = True - self.fields['device_pid'].initial = self.instance.pid + self.fields["device_pid"].widget.attrs["readonly"] = True + self.fields["device_pid"].initial = self.instance.pid class ScheduledTaskCreateForm(BootstrapMixin, forms.Form): model = ScheduledTask - pk = forms.ModelMultipleChoiceField( - queryset=Device.objects.all(), - widget=forms.MultipleHiddenInput() - ) + pk = forms.ModelMultipleChoiceField(queryset=Device.objects.all(), widget=forms.MultipleHiddenInput()) task_type = forms.ChoiceField( - choices=TaskTypeChoices, - required=True, - label='Job Type', - initial='', - widget=StaticSelect2() + choices=TaskTypeChoices, required=True, label="Job Type", initial="", widget=StaticSelect2() ) scheduled_time = forms.DateTimeField( - label='Scheduled Time', + label="Scheduled Time", required=False, widget=DateTimePicker(), ) mw_duration = forms.IntegerField( required=True, initial=6, - label='MW Duration, Hrs.', + label="MW Duration, Hrs.", ) - start_now = ['scheduled_time'] + start_now = ["scheduled_time"] + + transfer_method = forms.ChoiceField( + choices=TaskTransferMethod, + required=True, + label="Transfer Method", + initial=DEFAULT_TRANSFER_METHOD, + widget=StaticSelect2(), + ) class Meta: - start_now = ['scheduled_time'] + start_now = ["scheduled_time"] def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.fields['mw_duration'].widget.attrs['max'] = 8 - self.fields['mw_duration'].widget.attrs['min'] = 1 + self.fields["mw_duration"].widget.attrs["max"] = 8 + self.fields["mw_duration"].widget.attrs["min"] = 1 class ScheduledTaskFilterForm(BootstrapMixin, forms.ModelForm): model = ScheduledTask - q = forms.CharField( - required=False, - label='Search' - ) + q = forms.CharField(required=False, label="Search") task_type = forms.MultipleChoiceField( - label='Type', - choices=TaskTypeChoices, - required=False, - widget=StaticSelect2Multiple() + label="Type", choices=TaskTypeChoices, required=False, widget=StaticSelect2Multiple() ) status = forms.MultipleChoiceField( - label='Status', - choices=TaskStatusChoices, - required=False, - widget=StaticSelect2Multiple() + label="Status", choices=TaskStatusChoices, required=False, widget=StaticSelect2Multiple() ) confirmed = forms.NullBooleanField( - required=False, - label='Is Confirmed (ACK)', - widget=StaticSelect2(choices=BOOLEAN_WITH_BLANK_CHOICES) + required=False, label="Is Confirmed (ACK)", widget=StaticSelect2(choices=BOOLEAN_WITH_BLANK_CHOICES) ) scheduled_time_after = forms.DateTimeField( - label=mark_safe('
Scheduled After'), - required=False, - widget=DateTimePicker() - ) - scheduled_time_before = forms.DateTimeField( - label='Scheduled Before', - required=False, - widget=DateTimePicker() + label=mark_safe("
Scheduled After"), required=False, widget=DateTimePicker() ) + scheduled_time_before = forms.DateTimeField(label="Scheduled Before", required=False, widget=DateTimePicker()) start_time_after = forms.DateTimeField( - label=mark_safe('
Started After'), - required=False, - widget=DateTimePicker() - ) - start_time_before = forms.DateTimeField( - label='Started Before', - required=False, - widget=DateTimePicker() - ) - end_time_after = forms.DateTimeField( - label=mark_safe('
Ended After'), - required=False, - widget=DateTimePicker() - ) - end_time_before = forms.DateTimeField( - label='Ended Before', - required=False, - widget=DateTimePicker() + label=mark_safe("
Started After"), required=False, widget=DateTimePicker() ) + start_time_before = forms.DateTimeField(label="Started Before", required=False, widget=DateTimePicker()) + end_time_after = forms.DateTimeField(label=mark_safe("
Ended After"), required=False, widget=DateTimePicker()) + end_time_before = forms.DateTimeField(label="Ended Before", required=False, widget=DateTimePicker()) class Meta: model = ScheduledTask fields = [ - 'q', 'task_type', 'status', 'confirmed', - 'scheduled_time_after', 'scheduled_time_before', - 'start_time_after', 'start_time_before', - 'end_time_after', 'end_time_before', + "q", + "task_type", + "status", + "confirmed", + "scheduled_time_after", + "scheduled_time_before", + "start_time_after", + "start_time_before", + "end_time_after", + "end_time_before", ] @@ -168,36 +148,33 @@ def __init__(self, *args, **kwargs): self.obj_type = ContentType.objects.get_for_model(self.model) super().__init__(*args, **kwargs) custom_fields = CustomField.objects.get(content_types=self.obj_type, name=CF_NAME_SW_VERSION) - field_name = 'cf_{}'.format(custom_fields.name) + field_name = "cf_{}".format(custom_fields.name) self.fields[field_name] = custom_fields.to_form_field(set_initial=True, enforce_required=False) class UpgradeDeviceFilterForm(BootstrapMixin, CustomFieldVersionFilterForm): model = Device - field_order = ['q', 'role', 'tenant', 'device_type_id', 'tag', 'target_sw'] - q = forms.CharField( - required=False, - label='Search' - ) + field_order = ["q", "role", "tenant", "device_type_id", "tag", "target_sw"] + q = forms.CharField(required=False, label="Search") tenant = DynamicModelMultipleChoiceField( queryset=Tenant.objects.all(), - to_field_name='slug', + to_field_name="slug", required=False, ) role = DynamicModelMultipleChoiceField( queryset=DeviceRole.objects.all(), - to_field_name='slug', + to_field_name="slug", required=False, ) device_type_id = DynamicModelMultipleChoiceField( queryset=DeviceType.objects.all(), required=False, - label='Model', - display_field='model', + label="Model", + display_field="model", ) target_sw = forms.CharField( - label='Target SW', + label="Target SW", required=False, - help_text='Target SW Version', + help_text="Target SW Version", ) tag = TagFilterField(model) diff --git a/software_manager/migrations/0002_scheduledtask_transfer_method.py b/software_manager/migrations/0002_scheduledtask_transfer_method.py new file mode 100644 index 0000000..24af1ff --- /dev/null +++ b/software_manager/migrations/0002_scheduledtask_transfer_method.py @@ -0,0 +1,16 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("software_manager", "0001_initial"), + ] + + operations = [ + migrations.AddField( + model_name="scheduledtask", + name="transfer_method", + field=models.CharField(default="ftp", max_length=8), + ), + ] diff --git a/software_manager/models.py b/software_manager/models.py index b5d8a81..54b6538 100644 --- a/software_manager/models.py +++ b/software_manager/models.py @@ -12,17 +12,17 @@ from dcim.models import Device from utilities.querysets import RestrictedQuerySet -from .choices import TaskTypeChoices, TaskStatusChoices, TaskFailReasonChoices +from .choices import TaskTypeChoices, TaskStatusChoices, TaskFailReasonChoices, TaskTransferMethod -PLUGIN_SETTINGS = settings.PLUGINS_CONFIG.get('software_manager', dict()) -CF_NAME_SW_VERSION = PLUGIN_SETTINGS.get('CF_NAME_SW_VERSION', '') -FTP_USERNAME = PLUGIN_SETTINGS.get('FTP_USERNAME', '') +PLUGIN_SETTINGS = settings.PLUGINS_CONFIG.get("software_manager", dict()) +CF_NAME_SW_VERSION = PLUGIN_SETTINGS.get("CF_NAME_SW_VERSION", "") +FTP_USERNAME = PLUGIN_SETTINGS.get("FTP_USERNAME", "") class SoftwareImage(models.Model): timestamp = models.DateTimeField(auto_now_add=True) image = models.FileField( - upload_to=f'{FTP_USERNAME}/', unique=True, validators=[FileExtensionValidator(allowed_extensions=['bin'])] + upload_to=f"{FTP_USERNAME}/", unique=True, validators=[FileExtensionValidator(allowed_extensions=["bin"])] ) md5sum = models.CharField(max_length=36, blank=True) md5sum_calculated = models.CharField(max_length=36, blank=True) @@ -32,10 +32,10 @@ class SoftwareImage(models.Model): objects = RestrictedQuerySet.as_manager() class Meta: - ordering = ['-filename'] + ordering = ["-filename"] def save(self, *args, **kwargs): - self.filename = self.image.name.rsplit('/', 1)[-1] + self.filename = self.image.name.rsplit("/", 1)[-1] if self.pk: super(SoftwareImage, self).save(*args, **kwargs) else: @@ -54,28 +54,28 @@ def delete(self, *args, **kwargs): super(SoftwareImage, self).delete(*args, **kwargs) def __str__(self): - return self.image.name.rsplit('/', 1)[-1] + return self.image.name.rsplit("/", 1)[-1] class GoldenImage(models.Model): timestamp = models.DateTimeField(auto_now_add=True) - pid = models.OneToOneField(to='dcim.DeviceType', on_delete=models.CASCADE, related_name='golden_image') - sw = models.ForeignKey(to='SoftwareImage', on_delete=models.CASCADE, blank=True, null=True) + pid = models.OneToOneField(to="dcim.DeviceType", on_delete=models.CASCADE, related_name="golden_image") + sw = models.ForeignKey(to="SoftwareImage", on_delete=models.CASCADE, blank=True, null=True) objects = RestrictedQuerySet.as_manager() class Meta: - ordering = ['pid'] + ordering = ["pid"] def __str__(self): - return f'{self.pid.model}: {self.sw}' + return f"{self.pid.model}: {self.sw}" def get_progress(self): total = self.pid.instances.count() if total == 0: return 0 upgraded = Device.objects.filter( - **{f'custom_field_data__{CF_NAME_SW_VERSION}': self.sw.version}, + **{f"custom_field_data__{CF_NAME_SW_VERSION}": self.sw.version}, device_type=self.pid, ).count() return round(upgraded / total * 100, 2) @@ -84,7 +84,7 @@ def get_progress(self): class ScheduledTaskQuerySet(RestrictedQuerySet): def delete(self): exclude_list = [] - scheduler = get_scheduler('default') + scheduler = get_scheduler("default") for i in self: try: j = Job.fetch(i.job_id, scheduler.connection) @@ -105,7 +105,7 @@ def get_queryset(self): class ScheduledTask(models.Model): timestamp = models.DateTimeField(auto_now_add=True) - device = models.ForeignKey(to='dcim.Device', on_delete=models.SET_NULL, blank=True, null=True) + device = models.ForeignKey(to="dcim.Device", on_delete=models.SET_NULL, blank=True, null=True) task_type = models.CharField(max_length=255, choices=TaskTypeChoices, default=TaskTypeChoices.TYPE_UPLOAD) job_id = models.CharField(max_length=255, blank=True) status = models.CharField(max_length=255, choices=TaskStatusChoices, default=TaskStatusChoices.STATUS_UNKNOWN) @@ -120,17 +120,18 @@ class ScheduledTask(models.Model): mw_duration = models.PositiveIntegerField(blank=True) log = models.TextField(blank=True) user = models.CharField(max_length=255, blank=True) + transfer_method = models.CharField(max_length=8, choices=TaskTransferMethod, default=TaskTransferMethod.METHOD_FTP) objects = ScheduledTaskManager() def __str__(self): if not self.device: - return '' + return "" else: - return f'{self.device}: {self.job_id}' + return f"{self.device}: {self.job_id}" def delete(self): - scheduler = get_scheduler('default') + scheduler = get_scheduler("default") try: j = Job.fetch(self.job_id, scheduler.connection) if not j.is_started: @@ -141,4 +142,4 @@ def delete(self): return super().delete() class Meta: - ordering = ['-scheduled_time', '-start_time', '-end_time', 'job_id'] + ordering = ["-scheduled_time", "-start_time", "-end_time", "job_id"] diff --git a/software_manager/templates/software_manager/scheduledtask_info.html b/software_manager/templates/software_manager/scheduledtask_info.html index 4410cce..b4c5185 100644 --- a/software_manager/templates/software_manager/scheduledtask_info.html +++ b/software_manager/templates/software_manager/scheduledtask_info.html @@ -5,77 +5,81 @@ {% block title %}Task info{% endblock %} {% block content %} -
-
-
-
- Task -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Job ID{{ task.job_id }}
User{{ task.user }}
Device{{ task.device }}
Platform{{ task.device.device_type }}
IP Address{{ task.device.primary_ip.address.ip }}
Image{{ task.device.device_type.golden_image.sw.filename }}
Job Status{{ task.status }}
Message{{ task.message }}
Action{{ task.task_type }}
Created Time{{ task.timestamp|date:"M d, Y H:i:s" }}
Scheduled Time{{ task.scheduled_time|date:"M d, Y H:i:s" }}
Start Time{{ task.start_time|date:"M d, Y H:i:s" }}
End Time{{ task.end_time|date:"M d, Y H:i:s" }}
+
+
+
+
+ Task
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Job ID{{ task.job_id }}
User{{ task.user }}
Device{{ task.device }}
Platform{{ task.device.device_type }}
IP Address{{ task.device.primary_ip.address.ip }}
Image{{ task.device.device_type.golden_image.sw.filename }}
Transfer Method{{ task.transfer_method }}
Job Status{{ task.status }}
Message{{ task.message }}
Action{{ task.task_type }}
Created Time{{ task.timestamp|date:"M d, Y H:i:s" }}
Scheduled Time{{ task.scheduled_time|date:"M d, Y H:i:s" }}
Start Time{{ task.start_time|date:"M d, Y H:i:s" }}
End Time{{ task.end_time|date:"M d, Y H:i:s" }}
-
-
-
- Log -
-
- 
{{ task.log }}
-
+
+
+
+
+ Log +
+
+ 
{{ task.log }}
-{% endblock %} +
+{% endblock %} \ No newline at end of file diff --git a/software_manager/upgrade.py b/software_manager/upgrade.py index 7dbe950..af8f503 100644 --- a/software_manager/upgrade.py +++ b/software_manager/upgrade.py @@ -5,72 +5,75 @@ import os from django_rq import get_queue + # from django_rq import job, get_queue from django.conf import settings from datetime import timedelta + # from random import randint from scrapli.driver.core import IOSXEDriver from scrapli.exceptions import ScrapliAuthenticationFailed, ScrapliConnectionError, ScrapliTimeout from datetime import datetime from .logger import log -from .choices import TaskStatusChoices, TaskFailReasonChoices, TaskTypeChoices +from .choices import TaskStatusChoices, TaskFailReasonChoices, TaskTypeChoices, TaskTransferMethod from .models import ScheduledTask from .custom_exceptions import UpgradeException -PLUGIN_SETTINGS = settings.PLUGINS_CONFIG.get('software_manager', dict()) -CF_NAME_SW_VERSION = PLUGIN_SETTINGS.get('CF_NAME_SW_VERSION', '') -UPGRADE_QUEUE = PLUGIN_SETTINGS.get('UPGRADE_QUEUE', '') -UPGRADE_THRESHOLD = PLUGIN_SETTINGS.get('UPGRADE_THRESHOLD', '') -UPGRADE_MAX_ATTEMPTS_AFTER_RELOAD = PLUGIN_SETTINGS.get('UPGRADE_MAX_ATTEMPTS_AFTER_RELOAD', '') -UPGRADE_SECONDS_BETWEEN_ATTEMPTS = PLUGIN_SETTINGS.get('UPGRADE_SECONDS_BETWEEN_ATTEMPTS', '') -DEVICE_USERNAME = PLUGIN_SETTINGS.get('DEVICE_USERNAME', '') -DEVICE_PASSWORD = PLUGIN_SETTINGS.get('DEVICE_PASSWORD', '') -FTP_USERNAME = PLUGIN_SETTINGS.get('FTP_USERNAME', '') -FTP_PASSWORD = PLUGIN_SETTINGS.get('FTP_PASSWORD', '') -FTP_SERVER = PLUGIN_SETTINGS.get('FTP_SERVER', '') -TIME_ZONE = os.environ.get('TIME_ZONE', 'UTC') +PLUGIN_SETTINGS = settings.PLUGINS_CONFIG.get("software_manager", dict()) +CF_NAME_SW_VERSION = PLUGIN_SETTINGS.get("CF_NAME_SW_VERSION", "") +UPGRADE_QUEUE = PLUGIN_SETTINGS.get("UPGRADE_QUEUE", "") +UPGRADE_THRESHOLD = PLUGIN_SETTINGS.get("UPGRADE_THRESHOLD", "") +UPGRADE_MAX_ATTEMPTS_AFTER_RELOAD = PLUGIN_SETTINGS.get("UPGRADE_MAX_ATTEMPTS_AFTER_RELOAD", "") +UPGRADE_SECONDS_BETWEEN_ATTEMPTS = PLUGIN_SETTINGS.get("UPGRADE_SECONDS_BETWEEN_ATTEMPTS", "") +DEVICE_USERNAME = PLUGIN_SETTINGS.get("DEVICE_USERNAME", "") +DEVICE_PASSWORD = PLUGIN_SETTINGS.get("DEVICE_PASSWORD", "") +FTP_USERNAME = PLUGIN_SETTINGS.get("FTP_USERNAME", "") +FTP_PASSWORD = PLUGIN_SETTINGS.get("FTP_PASSWORD", "") +FTP_SERVER = PLUGIN_SETTINGS.get("FTP_SERVER", "") +HTTP_SERVER = PLUGIN_SETTINGS.get("HTTP_SERVER", "") +TIME_ZONE = os.environ.get("TIME_ZONE", "UTC") class UpgradeDevice: def __init__(self, task): self.task = task - self.log_id = f'{task.job_id} - {task.device.name}' + self.log_id = f"{task.job_id} - {task.device.name}" self.device = { - 'auth_username': DEVICE_USERNAME, - 'auth_password': DEVICE_PASSWORD, - 'auth_strict_key': False, + "auth_username": DEVICE_USERNAME, + "auth_password": DEVICE_PASSWORD, + "auth_strict_key": False, # 'ssh_config_file':'/var/lib/unit/.ssh/config', # 'ssh_config_file':'/root/.ssh/config', - 'port': 22, - 'timeout_socket': 5, - 'transport': 'paramiko', + "port": 22, + "timeout_socket": 5, + "transport": "paramiko", } if task.device.primary_ip: - self.device['host'] = str(task.device.primary_ip.address.ip) + self.device["host"] = str(task.device.primary_ip.address.ip) else: - msg = 'No primary (mgmt) address' + msg = "No primary (mgmt) address" self.warning(msg) self.skip_task(msg, TaskFailReasonChoices.FAIL_CHECK) def debug(self, msg): - log.debug(f'{self.log_id} - {msg}') + log.debug(f"{self.log_id} - {msg}") self.task.log += f'{datetime.now(pytz.timezone(TIME_ZONE)).strftime("%Y-%m-%d %H:%M:%S")} - DEBUG - {msg}\n' self.task.save() def info(self, msg): - log.info(f'{self.log_id} - {msg}') + log.info(f"{self.log_id} - {msg}") self.task.log += f'{datetime.now(pytz.timezone(TIME_ZONE)).strftime("%Y-%m-%d %H:%M:%S")} - INFO - {msg}\n' self.task.save() def warning(self, msg): - log.warning(f'{self.log_id} - {msg}') + log.warning(f"{self.log_id} - {msg}") self.task.log += f'{datetime.now(pytz.timezone(TIME_ZONE)).strftime("%Y-%m-%d %H:%M:%S")} - WARNING - {msg}\n' self.task.save() def error(self, msg): - log.error(f'{self.log_id} - {msg}') + log.error(f"{self.log_id} - {msg}") self.task.log += f'{datetime.now(pytz.timezone(TIME_ZONE)).strftime("%Y-%m-%d %H:%M:%S")} - ERROR - {msg}\n' self.task.save() @@ -84,39 +87,43 @@ def action_task(self, action, msg, reason): message=msg, ) - def skip_task(self, msg='', reason=''): + def skip_task(self, msg="", reason=""): self.action_task(TaskStatusChoices.STATUS_SKIPPED, msg, reason) - def drop_task(self, msg='', reason=''): + def drop_task(self, msg="", reason=""): self.action_task(TaskStatusChoices.STATUS_FAILED, msg, reason) def check(self): - if not hasattr(self.task.device.device_type, 'golden_image'): - msg = f'No Golden Image for {self.task.device.device_type.model}' + if not hasattr(self.task.device.device_type, "golden_image"): + msg = f"No Golden Image for {self.task.device.device_type.model}" self.warning(msg) self.skip_task(msg, TaskFailReasonChoices.FAIL_CHECK) else: - self.debug(f'Golden Image for {self.task.device.device_type.model} is {self.task.device.device_type.golden_image.sw}') + self.debug( + f"Golden Image for {self.task.device.device_type.model} is {self.task.device.device_type.golden_image.sw}" + ) if self.task.start_time > self.task.scheduled_time + timedelta(hours=int(self.task.mw_duration)): - msg = 'Maintenance Window is over' + msg = "Maintenance Window is over" self.warning(msg) self.skip_task(msg, TaskFailReasonChoices.FAIL_CHECK) else: - self.debug('MW is still active') + self.debug("MW is still active") if self.task.task_type == TaskTypeChoices.TYPE_UPGRADE: q = get_queue(UPGRADE_QUEUE) active_jobs = q.started_job_registry.count non_ack = ScheduledTask.objects.filter(start_time__isnull=False, confirmed=False).count() if non_ack >= active_jobs + UPGRADE_THRESHOLD: - msg = f'Reached failure threshold: Unconfirmed: {non_ack}, active: {active_jobs}, failed: {non_ack-active_jobs}, threshold: {UPGRADE_THRESHOLD}' + msg = f"Reached failure threshold: Unconfirmed: {non_ack}, active: {active_jobs}, failed: {non_ack-active_jobs}, threshold: {UPGRADE_THRESHOLD}" self.warning(msg) self.skip_task(msg, TaskFailReasonChoices.FAIL_CHECK) else: - self.debug(f'Unconfirmed: {non_ack}, active: {active_jobs}, failed: {non_ack - active_jobs}, threshold: {UPGRADE_THRESHOLD}') + self.debug( + f"Unconfirmed: {non_ack}, active: {active_jobs}, failed: {non_ack - active_jobs}, threshold: {UPGRADE_THRESHOLD}" + ) else: - self.debug(f'Task type is {self.task.task_type}, check against threshold was skipped') + self.debug(f"Task type is {self.task.task_type}, check against threshold was skipped") def connect_cli(self, **kwargs): def to_telnet(cli, **kwargs): @@ -125,10 +132,10 @@ def to_telnet(cli, **kwargs): except Exception: pass cli = False - if self.device['port'] != 23: - self.debug('Swiching to telnet') - self.device['port'] = 23 - self.device['transport'] = 'telnet' + if self.device["port"] != 23: + self.debug("Swiching to telnet") + self.device["port"] = 23 + self.device["transport"] = "telnet" cli = self.connect_cli(**kwargs) return cli @@ -153,36 +160,36 @@ def to_telnet(cli, **kwargs): def is_alive(self): try: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.settimeout(self.device.get('timeout_socket', 5)) - s.connect((self.device['host'], 22)) + s.settimeout(self.device.get("timeout_socket", 5)) + s.connect((self.device["host"], 22)) except Exception: - self.debug('no response on TCP/22') + self.debug("no response on TCP/22") try: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.settimeout(self.device.get('timeout_socket', 5)) - s.connect((self.device['host'], 23)) + s.settimeout(self.device.get("timeout_socket", 5)) + s.connect((self.device["host"], 23)) except Exception: - self.debug('no response on TCP/23') + self.debug("no response on TCP/23") time.sleep(2) return False else: - self.debug('got response on TCP/23') + self.debug("got response on TCP/23") else: - self.debug('got response on TCP/22') + self.debug("got response on TCP/22") time.sleep(2) return True def check_device(self): - pid = '' - sn = '' + pid = "" + sn = "" cmd = [ - 'show version', - 'dir /all', + "show version", + "dir /all", ] cli = self.connect_cli() if not cli: - msg = 'Can not connect to device CLI' + msg = "Can not connect to device CLI" self.error(msg) self.skip_task(msg, TaskFailReasonChoices.FAIL_CONNECT) @@ -190,269 +197,287 @@ def check_device(self): cli.close() if output.failed: - msg = 'Can not collect outputs from device' + msg = "Can not collect outputs from device" self.error(msg) self.skip_task(msg, TaskFailReasonChoices.FAIL_CONFIG) - self.debug('----------vv Outputs vv----------') + self.debug("----------vv Outputs vv----------") self.debug(output.result) - self.debug('----------^^ Outputs ^^----------') + self.debug("----------^^ Outputs ^^----------") - r = re.search(r'\n\w+\s+(\S+)\s+.*\(revision\s+', output[0].result) + r = re.search(r"\n\w+\s+(\S+)\s+.*\(revision\s+", output[0].result) if r: pid = r.group(1) # pid = re.sub('\+','plus',r.group(1)) - self.info(f'PID: {r.group(1)}') + self.info(f"PID: {r.group(1)}") else: - msg = 'Can not get device PID' + msg = "Can not get device PID" self.error(msg) self.skip_task(msg, reason=TaskFailReasonChoices.FAIL_CONFIG) - r = re.search(r'\n.*\s+board\s+ID\s+(\S+)', output[0].result) + r = re.search(r"\n.*\s+board\s+ID\s+(\S+)", output[0].result) if r: sn = r.group(1) - self.info(f'SN: {sn}') + self.info(f"SN: {sn}") else: - msg = 'Can not get device SN' + msg = "Can not get device SN" self.error(msg) self.skip_task(msg, reason=TaskFailReasonChoices.FAIL_CONFIG) if pid.upper() != self.task.device.device_type.model.upper() or sn.lower() != self.task.device.serial.lower(): - msg = 'Device PID/SN does not match with NetBox data' + msg = "Device PID/SN does not match with NetBox data" self.error(msg) self.skip_task(msg, reason=TaskFailReasonChoices.FAIL_CONFIG) - self.info(f'Device {pid}/{sn} matches with NetBox data') + self.info(f"Device {pid}/{sn} matches with NetBox data") self.files = output[1].textfsm_parse_output() - self.file_system = self.files[0]['file_system'].strip('/') + self.file_system = self.files[0]["file_system"].strip("/") self.target_image = self.task.device.device_type.golden_image.sw.filename self.target_path = self.task.device.device_type.golden_image.sw.image.path - self.image_on_device = list(filter(lambda x: x['name'] == self.target_image, self.files)) + self.image_on_device = list(filter(lambda x: x["name"] == self.target_image, self.files)) - self.debug(f'File system: {self.file_system}') - self.debug(f'Target Image: {self.target_image}') - self.debug(f'Target Path: {self.target_path}') - self.debug(f'Target Image on box: {self.image_on_device}') + self.debug(f"File system: {self.file_system}") + self.debug(f"Target Image: {self.target_image}") + self.debug(f"Target Path: {self.target_path}") + self.debug(f"Target Image on box: {self.image_on_device}") return True - def file_upload_ftp(self): - cmd_copy_ftp = f'copy ftp://{FTP_USERNAME}:{FTP_PASSWORD}@{FTP_SERVER}/{self.target_image} {self.file_system}/{self.target_image}' + def file_upload(self): + if self.task.transfer_method == TaskTransferMethod.METHOD_FTP: + cmd_copy = f"copy ftp://{FTP_USERNAME}:{FTP_PASSWORD}@{FTP_SERVER}/{self.target_image} {self.file_system}/{self.target_image}" + elif self.task.transfer_method == TaskTransferMethod.METHOD_HTTP: + cmd_copy = f"copy {HTTP_SERVER}{self.target_image} {self.file_system}/{self.target_image}" + else: + msg = "Unknown transfer method" + self.error(msg) + self.skip_task(msg, reason=TaskFailReasonChoices.FAIL_UPLOAD) + config = [ - 'file prompt quiet', - 'line vty 0 15', - 'exec-timeout 180 0', + "file prompt quiet", + "line vty 0 15", + "exec-timeout 180 0", ] config_undo = [ - 'no file prompt quiet', - 'line vty 0 15', - 'exec-timeout 30 0', + "no file prompt quiet", + "line vty 0 15", + "exec-timeout 30 0", ] cli = self.connect_cli(timeout_ops=7200, timeout_transport=7200) if not cli: - msg = 'Unable to connect to the device' + msg = "Unable to connect to the device" self.error(msg) self.skip_task(msg, TaskFailReasonChoices.FAIL_CONNECT) if not len(self.image_on_device): - self.info('No image on the device. Need to transfer') + self.info("No image on the device. Need to transfer") self.debug( f'Free on {self.file_system} {self.files[0]["total_free"]}, \ Image size (+10%) {int(int(self.task.device.device_type.golden_image.sw.image.size)*1.1)}' ) - if int(self.files[0]['total_free']) < int(int(self.task.device.device_type.golden_image.sw.image.size)*1.1): + if int(self.files[0]["total_free"]) < int( + int(self.task.device.device_type.golden_image.sw.image.size) * 1.1 + ): try: cli.close() except Exception: pass - msg = f'No enough space on {self.file_system}' + msg = f"No enough space on {self.file_system}" self.error(msg) self.skip_task(msg, TaskFailReasonChoices.FAIL_UPLOAD) - self.info('Download image from FTP...') + self.info("Download image from FTP...") output = cli.send_configs(config) - self.debug(f'Preparing for copy:\n{output.result}') + self.debug(f"Preparing for copy:\n{output.result}") if output.failed: try: cli.close() except Exception: pass - msg = 'Can not change configuration' + msg = "Can not change configuration" self.error(msg) self.skip_task(msg, TaskFailReasonChoices.FAIL_UPLOAD) - self.debug(f'Copy command: {cmd_copy_ftp}') - output = cli.send_command(cmd_copy_ftp) - self.debug(f'Copying process:\n{output.result}') - if output.failed or not re.search(r'OK', output.result): + self.debug(f"Copy command: {cmd_copy}") + output = cli.send_command(cmd_copy) + self.debug(f"Copying process:\n{output.result}") + if output.failed or not (re.search(r"OK", output.result) or re.search(r"bytes copied in", output.result)): try: cli.close() except Exception: pass - msg = 'Can not download image from FTP' + msg = "Can not download image from server" self.error(msg) self.skip_task(msg, TaskFailReasonChoices.FAIL_UPLOAD) output = cli.send_configs(config_undo) - self.debug(f'Rollback after copy:\n{output.result}') + self.debug(f"Rollback after copy:\n{output.result}") if output.failed: try: cli.close() except Exception: pass - msg = 'Can not do rollback configuration' + msg = "Can not do rollback configuration" self.error(msg) self.skip_task(msg, TaskFailReasonChoices.FAIL_UPLOAD) else: - self.info(f'Image {self.target_image} already exists') - self.info('MD5 verification ...') + self.info(f"Image {self.target_image} already exists") + self.info("MD5 verification ...") - md5 = cli.send_command(f'verify /md5 {self.file_system}/{self.target_image} {self.task.device.device_type.golden_image.sw.md5sum}') - self.debug(f'MD5 verication result:\n{md5.result[-200:]}') + md5 = cli.send_command( + f"verify /md5 {self.file_system}/{self.target_image} {self.task.device.device_type.golden_image.sw.md5sum}" + ) + self.debug(f"MD5 verication result:\n{md5.result[-200:]}") if md5.failed: try: cli.close() except Exception: pass - msg = 'Can not check MD5' + msg = "Can not check MD5" self.error(msg) self.skip_task(msg, TaskFailReasonChoices.FAIL_CHECK) - if re.search(r'Verified', md5.result): - self.info('MD5 was verified') + if re.search(r"Verified", md5.result): + self.info("MD5 was verified") else: try: cli.close() except Exception: pass - msg = 'Wrong M5' + msg = "Wrong M5" self.error(msg) self.skip_task(msg, TaskFailReasonChoices.FAIL_CHECK) try: cli.close() except Exception: pass - self.info('File was uploaded and verified') + self.info("File was uploaded and verified") return True def device_reload(self): cmd = [ - 'show run | i boot system', - 'show version', + "show run | i boot system", + "show version", ] cli = self.connect_cli() if not cli: - msg = 'Unable to connect to the device' + msg = "Unable to connect to the device" self.error(msg) self.skip_task(msg, TaskFailReasonChoices.FAIL_CONNECT) output = cli.send_commands(cmd) - self.debug(f'Collected outputs:------vvvvv\n{output.result}\n-----^^^^^') + self.debug(f"Collected outputs:------vvvvv\n{output.result}\n-----^^^^^") if output.failed: try: cli.close() except Exception: pass - msg = 'Can not collect outputs for upgrade' + msg = "Can not collect outputs for upgrade" self.error(msg) self.skip_task(msg, TaskFailReasonChoices.FAIL_UPGRADE) parsed = output[1].textfsm_parse_output() - sw_current = parsed[0].get('version', 'N/A') + sw_current = parsed[0].get("version", "N/A") sw_target = self.task.device.device_type.golden_image.sw.version - self.debug(f'Current version is {sw_current}') + self.debug(f"Current version is {sw_current}") if sw_current.upper() == sw_target.upper(): - msg = f'Current version {sw_current} matches with target {sw_target}' + msg = f"Current version {sw_current} matches with target {sw_target}" self.warning(msg) - self.info('Update custom field') + self.info("Update custom field") self.task.device.custom_field_data[CF_NAME_SW_VERSION] = sw_current self.task.device.save() self.skip_task(msg, TaskFailReasonChoices.FAIL_UPGRADE) if not len(self.image_on_device): - msg = 'No target image on the box' + msg = "No target image on the box" self.error(msg) self.skip_task(msg, TaskFailReasonChoices.FAIL_UPGRADE) - self.info('Image exists on the box') + self.info("Image exists on the box") cli.timeout_ops = 600 - self.info('MD5 verification ...') - md5 = cli.send_command(f'verify /md5 {self.file_system}/{self.target_image} {self.task.device.device_type.golden_image.sw.md5sum}') - self.debug(f'MD5 verication result:\n{md5.result[-200:]}') + self.info("MD5 verification ...") + md5 = cli.send_command( + f"verify /md5 {self.file_system}/{self.target_image} {self.task.device.device_type.golden_image.sw.md5sum}" + ) + self.debug(f"MD5 verication result:\n{md5.result[-200:]}") if md5.failed: try: cli.close() except Exception: pass - msg = 'Can not check MD5' + msg = "Can not check MD5" self.error(msg) self.skip_task(msg, TaskFailReasonChoices.FAIL_CHECK) - if re.search(r'Verified', md5.result): - self.info('MD5 was verified') + if re.search(r"Verified", md5.result): + self.info("MD5 was verified") else: try: cli.close() except Exception: pass - msg = 'Wrong M5' + msg = "Wrong M5" self.error(msg) self.skip_task(msg, TaskFailReasonChoices.FAIL_CHECK) cli.timeout_ops = 10 - self.info('Preparing boot system config') + self.info("Preparing boot system config") new_boot_lines = [] old_boot_lines = output[0].result.splitlines() - self.debug(f'Orginal boot lines:\n{old_boot_lines}') + self.debug(f"Orginal boot lines:\n{old_boot_lines}") for line in old_boot_lines: - new_boot_lines.append(f'no {line}') - new_boot_lines.append(f'boot system {self.file_system}/{self.target_image}') + new_boot_lines.append(f"no {line}") + new_boot_lines.append(f"boot system {self.file_system}/{self.target_image}") if len(old_boot_lines): new_boot_lines.append(old_boot_lines[0]) - self.debug(f'New boot lines:\n{new_boot_lines}') + self.debug(f"New boot lines:\n{new_boot_lines}") output = cli.send_configs(new_boot_lines) - self.debug(f'Changnig Boot vars:\n{output.result}') + self.debug(f"Changnig Boot vars:\n{output.result}") if output.failed: - msg = 'Unable to change bootvar' + msg = "Unable to change bootvar" self.error(msg) self.drop_task(msg, TaskFailReasonChoices.FAIL_UPGRADE) else: - self.info('Bootvar was changed') + self.info("Bootvar was changed") - self.info('Write memory before reload') + self.info("Write memory before reload") try: - output = cli.send_command('write memory') + output = cli.send_command("write memory") except (ScrapliTimeout, ScrapliConnectionError): - self.info('Interactive prompt was detected') + self.info("Interactive prompt was detected") time.sleep(2) cli.open() try: - output_tmp = cli.send_interactive([ - ('write', '[confirm]', False), - ('\n', '#', False), - ]) + output_tmp = cli.send_interactive( + [ + ("write", "[confirm]", False), + ("\n", "#", False), + ] + ) except (ScrapliTimeout, ScrapliConnectionError): - msg = 'Unable to write memory: ScrapliTimeout' + msg = "Unable to write memory: ScrapliTimeout" self.error(msg) self.drop_task(msg, TaskFailReasonChoices.FAIL_UPGRADE) else: output = output_tmp - if re.search(r'\[OK\]', output.result): - self.info('Config was saved') + if re.search(r"\[OK\]", output.result): + self.info("Config was saved") else: - msg = 'Can not save config' + msg = "Can not save config" self.error(msg) self.drop_task(msg, TaskFailReasonChoices.FAIL_UPGRADE) - self.info('Reloading the box') + self.info("Reloading the box") try: - output = cli.send_interactive([ - ('reload in 1', '[confirm]', False), - ('\n', '#', False), - ]) + output = cli.send_interactive( + [ + ("reload in 1", "[confirm]", False), + ("\n", "#", False), + ] + ) except ScrapliTimeout: - msg = 'Unable to reload: ScrapliTimeout' + msg = "Unable to reload: ScrapliTimeout" self.error(msg) self.drop_task(msg, TaskFailReasonChoices.FAIL_UPGRADE) else: - self.info('Reload was requested') + self.info("Reload was requested") try: cli.close() except Exception: @@ -461,99 +486,101 @@ def device_reload(self): def post_check(self): cmd = [ - 'show version', + "show version", ] cli = self.connect_cli() if not cli: - msg = 'Unable to connect to the device' + msg = "Unable to connect to the device" self.error(msg) self.drop_task(msg, TaskFailReasonChoices.FAIL_CONNECT) output = cli.send_commands(cmd) - self.debug(f'Commands output\n{output.result}') + self.debug(f"Commands output\n{output.result}") if output.failed: - msg = 'Can not collect outputs for post-chech' + msg = "Can not collect outputs for post-chech" self.error(msg) self.drop_task(msg, TaskFailReasonChoices.FAIL_UPGRADE) parsed = output[0].textfsm_parse_output() self.info(f'New version is {parsed[0].get("version", "N/A")}') - self.info('Write memory after reload') + self.info("Write memory after reload") try: - output = cli.send_command('write memory') + output = cli.send_command("write memory") except (ScrapliTimeout, ScrapliConnectionError): - self.info('Interactive prompt was detected') + self.info("Interactive prompt was detected") time.sleep(2) cli.open() try: - output_tmp = cli.send_interactive([ - ('write', '[confirm]', False), - ('\n', '#', False), - ]) + output_tmp = cli.send_interactive( + [ + ("write", "[confirm]", False), + ("\n", "#", False), + ] + ) except (ScrapliTimeout, ScrapliConnectionError): - msg = 'Unable to write memory: ScrapliTimeout' + msg = "Unable to write memory: ScrapliTimeout" self.error(msg) self.drop_task(msg, TaskFailReasonChoices.FAIL_UPGRADE) else: output = output_tmp - if re.search(r'\[OK\]', output.result): - self.info('Config was saved') + if re.search(r"\[OK\]", output.result): + self.info("Config was saved") else: - msg = 'Can not save config' + msg = "Can not save config" self.error(msg) self.drop_task(msg, TaskFailReasonChoices.FAIL_UPGRADE) cli.close() - self.info('Update custom field') - self.task.device.custom_field_data[CF_NAME_SW_VERSION] = parsed[0].get('version', 'N/A') + self.info("Update custom field") + self.task.device.custom_field_data[CF_NAME_SW_VERSION] = parsed[0].get("version", "N/A") self.task.device.save() - self.info('Post-checks have been done') + self.info("Post-checks have been done") return True def execute_task(self): - self.info(f'New Job {self.task.job_id} was started. Type {self.task.task_type}') - self.info('Initial task checking...') + self.info(f"New Job {self.task.job_id} was started. Type {self.task.task_type}") + self.info("Initial task checking...") self.check() - self.info('Initial task check has been completed') + self.info("Initial task check has been completed") - self.info('Checking if device alive...') + self.info("Checking if device alive...") if not self.is_alive(): - msg = f'Device {self.task.device.name}:{self.task.device.primary_ip.address.ip} is not reachable' + msg = f"Device {self.task.device.name}:{self.task.device.primary_ip.address.ip} is not reachable" self.warning(msg) self.skip_task(msg, TaskFailReasonChoices.FAIL_CONNECT) else: - msg = f'Device {self.task.device.name}:{self.task.device.primary_ip.address.ip} is reachable' + msg = f"Device {self.task.device.name}:{self.task.device.primary_ip.address.ip} is reachable" self.info(msg) - self.info('Device valiation...') + self.info("Device valiation...") self.check_device() - self.info('Device has been validated') + self.info("Device has been validated") if self.task.task_type == TaskTypeChoices.TYPE_UPLOAD: - self.info('Uploadng image on the box...') - self.file_upload_ftp() + self.info("Uploadng image on the box...") + self.file_upload() elif self.task.task_type == TaskTypeChoices.TYPE_UPGRADE: - self.info('Reloading the box...') + self.info("Reloading the box...") self.device_reload() hold_timer = 240 - self.info(f'Hold for {hold_timer} seconds') + self.info(f"Hold for {hold_timer} seconds") time.sleep(hold_timer) for try_number in range(1, UPGRADE_MAX_ATTEMPTS_AFTER_RELOAD + 1): - self.info(f'Connecting after reload {try_number}/{UPGRADE_MAX_ATTEMPTS_AFTER_RELOAD}...') + self.info(f"Connecting after reload {try_number}/{UPGRADE_MAX_ATTEMPTS_AFTER_RELOAD}...") if self.is_alive(): - self.info('Device became online') + self.info("Device became online") time.sleep(10) break else: - self.info(f'Device is not online, next try in {UPGRADE_SECONDS_BETWEEN_ATTEMPTS} seconds') + self.info(f"Device is not online, next try in {UPGRADE_SECONDS_BETWEEN_ATTEMPTS} seconds") time.sleep(UPGRADE_SECONDS_BETWEEN_ATTEMPTS) if not self.is_alive(): - msg = 'Device was lost after reload' + msg = "Device was lost after reload" self.error(msg) self.drop_task(msg, TaskFailReasonChoices.FAIL_UPGRADE) else: - self.info('Checks after reload') + self.info("Checks after reload") self.post_check() return True diff --git a/software_manager/views.py b/software_manager/views.py index 09b2c6e..4bcbc68 100644 --- a/software_manager/views.py +++ b/software_manager/views.py @@ -19,84 +19,96 @@ from .models import SoftwareImage, GoldenImage, ScheduledTask from .tables import SoftwareListTable, GoldenImageListTable, UpgradeDeviceListTable, ScheduledTaskTable from .filters import UpgradeDeviceFilter, ScheduledTaskFilter -from .forms import UpgradeDeviceFilterForm, ScheduledTaskCreateForm, ScheduledTaskFilterForm, SoftwareImageAddForm, GoldenImageAddForm +from .forms import ( + UpgradeDeviceFilterForm, + ScheduledTaskCreateForm, + ScheduledTaskFilterForm, + SoftwareImageAddForm, + GoldenImageAddForm, +) from .choices import TaskStatusChoices -TIME_ZONE = os.environ.get('TIME_ZONE', 'UTC') -PLUGIN_SETTINGS = settings.PLUGINS_CONFIG.get('software_manager', dict()) -CF_NAME_SW_VERSION = PLUGIN_SETTINGS.get('CF_NAME_SW_VERSION', '') -UPGRADE_QUEUE = PLUGIN_SETTINGS.get('UPGRADE_QUEUE', '') +TIME_ZONE = os.environ.get("TIME_ZONE", "UTC") +PLUGIN_SETTINGS = settings.PLUGINS_CONFIG.get("software_manager", dict()) +CF_NAME_SW_VERSION = PLUGIN_SETTINGS.get("CF_NAME_SW_VERSION", "") +UPGRADE_QUEUE = PLUGIN_SETTINGS.get("UPGRADE_QUEUE", "") class SoftwareList(ObjectListView): queryset = SoftwareImage.objects.all() table = SoftwareListTable - template_name = 'software_manager/software_list.html' + template_name = "software_manager/software_list.html" class SoftwareAdd(ObjectEditView): queryset = SoftwareImage.objects.all() model_form = SoftwareImageAddForm - default_return_url = 'plugins:software_manager:software_list' + default_return_url = "plugins:software_manager:software_list" class SoftwareDelele(ObjectDeleteView): queryset = SoftwareImage.objects.all() - default_return_url = 'plugins:software_manager:software_list' + default_return_url = "plugins:software_manager:software_list" class GoldenImageList(ObjectListView): queryset = DeviceType.objects.all() table = GoldenImageListTable - template_name = 'software_manager/golden_image_list.html' + template_name = "software_manager/golden_image_list.html" def export_to_excel(self): data = [] output = io.BytesIO() header = [ - {'header': 'Hostname'}, - {'header': 'PID'}, - {'header': 'IP Address'}, - {'header': 'Hub'}, - {'header': 'SW'}, - {'header': 'Golden Image'}, + {"header": "Hostname"}, + {"header": "PID"}, + {"header": "IP Address"}, + {"header": "Hub"}, + {"header": "SW"}, + {"header": "Golden Image"}, ] - width = [len(i['header']) + 2 for i in header] - devices = Device.objects.all().prefetch_related( - 'primary_ip4', - 'tenant', - 'device_type', - 'device_type__golden_image', - ).order_by('name') + width = [len(i["header"]) + 2 for i in header] + devices = ( + Device.objects.all() + .prefetch_related( + "primary_ip4", + "tenant", + "device_type", + "device_type__golden_image", + ) + .order_by("name") + ) for d in devices: if d.name: hostname = d.name else: - hostname = 'unnamed device' + hostname = "unnamed device" k = [ hostname, d.device_type.model, - str(d.primary_ip4).split('/')[0], + str(d.primary_ip4).split("/")[0], str(d.tenant), d.custom_field_data[CF_NAME_SW_VERSION], ] - if hasattr(d.device_type, 'golden_image'): - k.append(str(str(d.custom_field_data[CF_NAME_SW_VERSION]).upper() == str(d.device_type.golden_image.sw.version).upper())) + if hasattr(d.device_type, "golden_image"): + k.append( + str( + str(d.custom_field_data[CF_NAME_SW_VERSION]).upper() + == str(d.device_type.golden_image.sw.version).upper() + ) + ) else: - k.append('False') + k.append("False") data.append(k) w = [len(i) for i in k] width = [max(width[i], w[i]) for i in range(0, len(width))] workbook = xlsxwriter.Workbook( output, - {'remove_timezone': True, 'default_date_format': 'yyyy-mm-dd'}, - ) - worksheet = workbook.add_worksheet('SIAR') - worksheet.add_table( - 0, 0, Device.objects.all().count(), len(header) - 1, - {'columns': header, 'data': data} + {"remove_timezone": True, "default_date_format": "yyyy-mm-dd"}, ) + worksheet = workbook.add_worksheet("SIAR") + worksheet.add_table(0, 0, Device.objects.all().count(), len(header) - 1, {"columns": header, "data": data}) for i in range(0, len(width)): worksheet.set_column(i, i, width[i]) workbook.close() @@ -104,13 +116,13 @@ def export_to_excel(self): return output def get(self, request, *args, **kwargs): - if 'to_excel' in request.GET.keys(): + if "to_excel" in request.GET.keys(): filename = f'siar_{datetime.now().astimezone(pytz.timezone(TIME_ZONE)).strftime("%Y%m%d_%H%M%S")}.xlsx' response = HttpResponse( self.export_to_excel(), - content_type='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + content_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", ) - response['Content-Disposition'] = f'attachment; filename="{filename}"' + response["Content-Disposition"] = f'attachment; filename="{filename}"' return response return super().get(request, *args, **kwargs) @@ -118,7 +130,7 @@ def get(self, request, *args, **kwargs): class GoldenImageAdd(ObjectEditView): queryset = GoldenImage.objects.all() model_form = GoldenImageAddForm - default_return_url = 'plugins:software_manager:golden_image_list' + default_return_url = "plugins:software_manager:golden_image_list" def get(self, request, pk=None, pid_pk=None, *args, **kwargs): if pk is not None: @@ -127,95 +139,104 @@ def get(self, request, pk=None, pid_pk=None, *args, **kwargs): i = GoldenImage(pid=DeviceType.objects.get(pk=pid_pk)) i.pk = True form = GoldenImageAddForm(instance=i) - return render(request, 'generic/object_edit.html', { - 'obj': i, - 'obj_type': i._meta.verbose_name, - 'form': form, - 'return_url': reverse('plugins:software_manager:golden_image_list'), - }) + return render( + request, + "generic/object_edit.html", + { + "obj": i, + "obj_type": i._meta.verbose_name, + "form": form, + "return_url": reverse("plugins:software_manager:golden_image_list"), + }, + ) def post(self, request, pk=None, pid_pk=None, *args, **kwargs): - pid = request.POST.get('device_pid', None) + pid = request.POST.get("device_pid", None) if not pid: - messages.error(request, 'No PID') - return redirect(reverse('plugins:software_manager:golden_image_list')) + messages.error(request, "No PID") + return redirect(reverse("plugins:software_manager:golden_image_list")) - sw = request.POST.get('sw', None) + sw = request.POST.get("sw", None) if not sw: - messages.error(request, 'No SW') - return redirect(reverse('plugins:software_manager:golden_image_list')) + messages.error(request, "No SW") + return redirect(reverse("plugins:software_manager:golden_image_list")) if not DeviceType.objects.filter(model__iexact=pid).count(): - messages.error(request, 'Incorrect PID') - return redirect(reverse('plugins:software_manager:golden_image_list')) + messages.error(request, "Incorrect PID") + return redirect(reverse("plugins:software_manager:golden_image_list")) if not SoftwareImage.objects.filter(pk=sw).count(): - messages.error(request, 'Incorrect SW') - return redirect(reverse('plugins:software_manager:golden_image_list')) + messages.error(request, "Incorrect SW") + return redirect(reverse("plugins:software_manager:golden_image_list")) gi = GoldenImage.objects.create( - pid=DeviceType.objects.get(model__iexact=pid), - sw=SoftwareImage.objects.get(pk=sw) + pid=DeviceType.objects.get(model__iexact=pid), sw=SoftwareImage.objects.get(pk=sw) ) gi.save() - messages.success(request, f'Assigned golden image {pid}: {gi.sw}') - return redirect(reverse('plugins:software_manager:golden_image_list')) + messages.success(request, f"Assigned golden image {pid}: {gi.sw}") + return redirect(reverse("plugins:software_manager:golden_image_list")) class GoldenImageEdit(ObjectEditView): queryset = GoldenImage.objects.all() model_form = GoldenImageAddForm - default_return_url = 'plugins:software_manager:golden_image_list' + default_return_url = "plugins:software_manager:golden_image_list" class GoldenImageDelete(ObjectDeleteView): queryset = GoldenImage.objects.all() - default_return_url = 'plugins:software_manager:golden_image_list' + default_return_url = "plugins:software_manager:golden_image_list" class UpgradeDeviceList(ObjectListView): - queryset = Device.objects.all().prefetch_related( - 'primary_ip4', - 'tenant', - 'device_type', - 'device_type__golden_image', - ).order_by('name') + queryset = ( + Device.objects.all() + .prefetch_related( + "primary_ip4", + "tenant", + "device_type", + "device_type__golden_image", + ) + .order_by("name") + ) filterset = UpgradeDeviceFilter filterset_form = UpgradeDeviceFilterForm table = UpgradeDeviceListTable - template_name = 'software_manager/upgrade_device_list.html' + template_name = "software_manager/upgrade_device_list.html" class UpgradeDeviceScheduler(View): - def post(self, request): - if '_create' in request.POST: + if "_create" in request.POST: s = ScheduledTaskCreateForm(request.POST) if s.is_valid(): - checked_fields = request.POST.getlist('_nullify') + checked_fields = request.POST.getlist("_nullify") data = deepcopy(s.cleaned_data) - if 'scheduled_time' not in checked_fields and not data['scheduled_time']: - messages.error(request, 'Job start time was not set') - return redirect(reverse('plugins:software_manager:upgrade_device_list')) - for i in data['pk']: - if 'scheduled_time' in checked_fields: - data['scheduled_time'] = datetime.now().replace(microsecond=0).astimezone(pytz.timezone(TIME_ZONE)) + if "scheduled_time" not in checked_fields and not data["scheduled_time"]: + messages.error(request, "Job start time was not set") + return redirect(reverse("plugins:software_manager:upgrade_device_list")) + for i in data["pk"]: + if "scheduled_time" in checked_fields: + data["scheduled_time"] = ( + datetime.now().replace(microsecond=0).astimezone(pytz.timezone(TIME_ZONE)) + ) task = ScheduledTask( device=i, - task_type=data['task_type'], - scheduled_time=data['scheduled_time'], - mw_duration=int(data['mw_duration']), + task_type=data["task_type"], + scheduled_time=data["scheduled_time"], + mw_duration=int(data["mw_duration"]), status=TaskStatusChoices.STATUS_SCHEDULED, user=request.user.username, + transfer_method=data["transfer_method"], ) task.save() - if 'scheduled_time' in checked_fields: + if "scheduled_time" in checked_fields: queue = get_queue(UPGRADE_QUEUE) job = queue.enqueue_job( queue.create_job( - func='software_manager.worker.upgrade_device', + func="software_manager.worker.upgrade_device", args=[task.pk], timeout=9000, ) @@ -223,8 +244,8 @@ def post(self, request): else: scheduler = get_scheduler(UPGRADE_QUEUE) job = scheduler.schedule( - scheduled_time=data['scheduled_time'], - func='software_manager.worker.upgrade_device', + scheduled_time=data["scheduled_time"], + func="software_manager.worker.upgrade_device", args=[task.pk], timeout=9000, ) @@ -232,29 +253,33 @@ def post(self, request): task.save() messages.success(request, f'Task {data["task_type"]} was scheduled for {len(data["pk"])} devices') else: - messages.error(request, 'Error form is not valid') - return redirect(reverse('plugins:software_manager:upgrade_device_list')) - return redirect(reverse('plugins:software_manager:scheduled_task_list')) + messages.error(request, "Error form is not valid") + return redirect(reverse("plugins:software_manager:upgrade_device_list")) + return redirect(reverse("plugins:software_manager:scheduled_task_list")) else: - if '_device' in request.POST: - pk_list = [int(pk) for pk in request.POST.getlist('pk')] - elif '_task' in request.POST: - pk_list = [int(ScheduledTask.objects.get(pk=pk).device.pk) for pk in request.POST.getlist('pk')] + if "_device" in request.POST: + pk_list = [int(pk) for pk in request.POST.getlist("pk")] + elif "_task" in request.POST: + pk_list = [int(ScheduledTask.objects.get(pk=pk).device.pk) for pk in request.POST.getlist("pk")] selected_devices = Device.objects.filter(pk__in=pk_list) if not selected_devices: - messages.warning(request, 'No devices were selected.') - return redirect(reverse('plugins:software_manager:upgrade_device_list')) - - return render(request, 'software_manager/scheduledtask_add.html', { - 'form': ScheduledTaskCreateForm(initial={'pk': pk_list}), - 'parent_model_name': 'Devices', - 'model_name': 'Scheduled Tasks', - 'table': UpgradeDeviceListTable(selected_devices), - 'return_url': reverse('plugins:software_manager:upgrade_device_list'), - 'next_url': reverse('plugins:software_manager:scheduled_task_list'), - }) + messages.warning(request, "No devices were selected.") + return redirect(reverse("plugins:software_manager:upgrade_device_list")) + + return render( + request, + "software_manager/scheduledtask_add.html", + { + "form": ScheduledTaskCreateForm(initial={"pk": pk_list}), + "parent_model_name": "Devices", + "model_name": "Scheduled Tasks", + "table": UpgradeDeviceListTable(selected_devices), + "return_url": reverse("plugins:software_manager:upgrade_device_list"), + "next_url": reverse("plugins:software_manager:scheduled_task_list"), + }, + ) class ScheduledTaskList(ObjectListView): @@ -262,30 +287,30 @@ class ScheduledTaskList(ObjectListView): table = ScheduledTaskTable filterset = ScheduledTaskFilter filterset_form = ScheduledTaskFilterForm - template_name = 'software_manager/scheduledtask_list.html' + template_name = "software_manager/scheduledtask_list.html" def post(self, request, *args, **kwargs): - if '_confirm' in request.POST: - pk = request.POST.get('_confirm', '') + if "_confirm" in request.POST: + pk = request.POST.get("_confirm", "") if pk: task = ScheduledTask.objects.get(pk=int(pk)) task.confirmed = not task.confirmed task.save() messages.success(request, f'ACK changed to "{task.confirmed}" for job id "{task.job_id}"') else: - messages.warning(request, 'Missed pk, unknow Error') + messages.warning(request, "Missed pk, unknow Error") return redirect(request.get_full_path()) class ScheduledTaskBulkDelete(BulkDeleteView): queryset = ScheduledTask.objects.all() table = ScheduledTaskTable - default_return_url = 'plugins:software_manager:scheduled_task_list' + default_return_url = "plugins:software_manager:scheduled_task_list" class ScheduledTaskDelete(ObjectDeleteView): queryset = ScheduledTask.objects.all() - default_return_url = 'plugins:software_manager:scheduled_task_list' + default_return_url = "plugins:software_manager:scheduled_task_list" class ScheduledTaskInfo(ObjectDeleteView): @@ -293,6 +318,10 @@ class ScheduledTaskInfo(ObjectDeleteView): def get(self, request, pk): task = get_object_or_404(self.queryset, pk=pk) - return render(request, 'software_manager/scheduledtask_info.html', { - 'task': task, - }) + return render( + request, + "software_manager/scheduledtask_info.html", + { + "task": task, + }, + )