From fcc9aca708c9df899283116752912dc3ebd7b029 Mon Sep 17 00:00:00 2001 From: Victor BERTIN Date: Thu, 10 Jun 2021 16:00:26 +0200 Subject: [PATCH 1/2] Adapt update process for cases when both OS and apps are updated Fix notification system (except when the OS fails to be updated, the notification to Hawkbit server is always sent at the end of the update process) and change the update order (update OS -> reboot -> upadate containers -> restart containers -> notify hawkbit server). --- fullmetalupdate/fullmetalupdate_ddi_client.py | 116 ++++++++++-------- 1 file changed, 63 insertions(+), 53 deletions(-) diff --git a/fullmetalupdate/fullmetalupdate_ddi_client.py b/fullmetalupdate/fullmetalupdate_ddi_client.py index 0ac9f01..bdee5df 100755 --- a/fullmetalupdate/fullmetalupdate_ddi_client.py +++ b/fullmetalupdate/fullmetalupdate_ddi_client.py @@ -145,7 +145,6 @@ async def process_deployment(self, base): action_id, resource = match.groups() # fetch deployment information deploy_info = await self.ddi.deploymentBase[action_id](resource) - reboot_needed = False chunks_qty = len(deploy_info['deployment']['chunks']) @@ -157,6 +156,7 @@ async def process_deployment(self, base): status_execution, status_result, [msg]) raise APIError(msg) else: + # problem ? msg = "FullMetalUpdate:Proceeding" percentage = {"cnt": 0, "of": chunks_qty} status_execution = DeploymentStatusExecution.proceeding @@ -166,7 +166,7 @@ async def process_deployment(self, base): percentage=percentage) self.action_id = action_id - + final_result_os = True seq = ('name', 'version', 'rev', 'part', 'autostart', 'autoremove', 'status_execution', 'status_update', 'status_result', 'notify', 'timeout') updates = [] @@ -196,33 +196,40 @@ async def process_deployment(self, base): # checking if we just rebooted and we need to send the feedback in which # case we don't need to pull the update image again [feedback, reboot_data] = self.feedback_for_os_deployment(update['rev']) - if feedback: - await self.ddi.deploymentBase[reboot_data['action_id']].feedback( - DeploymentStatusExecution(reboot_data['status_execution']), - DeploymentStatusResult(reboot_data['status_result']), - [reboot_data['msg']]) - self.action_id = None - return - - self.logger.info("OS {} v.{} - updating...".format(update['name'], update['version'])) - update['status_update'] = self.update_system(update['rev']) - update['status_execution'] = DeploymentStatusExecution.closed - if not update['status_update']: - msg = "OS {} v.{} Deployment failed".format(update['name'], update['version']) - self.logger.error(msg) - update['status_result'] = DeploymentStatusResult.failure - await self.ddi.deploymentBase[self.action_id].feedback( - update['status_execution'], update['status_result'], [msg]) - return + if feedback: + final_result_os = (DeploymentStatusResult(reboot_data['status_result']) == DeploymentStatusResult.success) + if not final_result_os: + self.logger.info(reboot_data['msg']) + await self.ddi.deploymentBase[reboot_data['action_id']].feedback( + DeploymentStatusExecution(reboot_data['status_execution']), + DeploymentStatusResult(reboot_data['status_result']), + [reboot_data['msg']]) + action_id = None + return + else: - msg = "OS {} v.{} Deployment succeed".format(update['name'], update['version']) - self.logger.info(msg) - update['status_result'] = DeploymentStatusResult.success - reboot_needed = True - self.write_reboot_data(self.action_id, - update['status_execution'], - update['status_result'], - msg) + self.logger.info("OS {} v.{} - updating...".format(update['name'], update['version'])) + update['status_update'] = self.update_system(update['rev']) + update['status_execution'] = DeploymentStatusExecution.closed + if not update['status_update']: + msg = "OS {} v.{} Deployment failed".format(update['name'], update['version']) + self.logger.error(msg) + update['status_result'] = DeploymentStatusResult.failure + await self.ddi.deploymentBase[self.action_id].feedback( + update['status_execution'], update['status_result'], [msg]) + return + else: + msg = "OS {} v.{} Deployment succeed".format(update['name'], update['version']) + self.logger.info(msg) + update['status_result'] = DeploymentStatusResult.success + self.write_reboot_data(self.action_id, + update['status_execution'], + update['status_result'], + msg) + try: + subprocess.run("reboot") + except subprocess.CalledProcessError as e: + self.logger.error("Reboot failed: {}".format(e)) elif update['part'] == 'bApp': self.logger.info("App {} v.{} - updating...".format(update['name'], update['version'])) @@ -241,8 +248,9 @@ async def process_deployment(self, base): for update in updates: update['status_update'] &= self.handle_container(update['name'], update['autostart'], update['autoremove']) - final_result = True + final_result_apps = True fails = "" + feedbackMsg = "" feedbackThreadIt = iter(self.feedbackThreads) # Hawkbit server feedback process @@ -255,33 +263,35 @@ async def process_deployment(self, base): self.mutexResults.release() if not update['status_update']: - msg = "App {} v.{} Deployment failed\n {}".format(update['name'], update['version'], feedbackMsg) - self.logger.error(msg) - update['status_result'] = DeploymentStatusResult.failure - fails += update['name'] + " " + msg = "App {} v.{} Deployment failed : {}\n".format(update['name'], update['version'], feedbackMsg) + notifyMsg = msg + self.logger.error(msg) + update['status_result'] = DeploymentStatusResult.failure + fails += update['name'] + " " else: - msg = "App {} v.{} Deployment succeed".format(update['name'], update['version']) - self.logger.info(msg) - update['status_result'] = DeploymentStatusResult.success - - final_result &= (update['status_result'] == DeploymentStatusResult.success) - - if(final_result): - msg = "Hawkbit Update Success : All applications have been updated and correctly restarted." - self.logger.info(msg) + msg = "App {} v.{} Deployment succeed\n".format(update['name'], update['version']) + notifyMsg += msg + self.logger.info(msg) + update['status_result'] = DeploymentStatusResult.success + + final_result_apps &= (update['status_result'] == DeploymentStatusResult.success) + + if updates: + if(final_result_apps): + msg = "Hawkbit Update Success : All applications have been updated and correctly restarted." + self.logger.info(msg) + status_result = DeploymentStatusResult.success + else: + msg = "Hawkbit Update Failure : " + fails + "failed to update and / or to restart." + self.logger.error(msg) + status_result = DeploymentStatusResult.failure + notifyMsg = msg + "\n" + notifyMsg + if feedback: + notifyMsg += reboot_data['msg'] status_result = DeploymentStatusResult.success - else: - msg = "Hawkbit Update Failure : " + fails + "failed to update and / or to restart." - self.logger.error(msg) - status_result = DeploymentStatusResult.failure - await self.ddi.deploymentBase[self.action_id].feedback(DeploymentStatusExecution.closed, status_result, [msg]) - + + await self.ddi.deploymentBase[self.action_id].feedback(DeploymentStatusExecution.closed, status_result, [notifyMsg]) self.action_id = None - if reboot_needed: - try: - subprocess.run("reboot") - except subprocess.CalledProcessError as e: - self.logger.error("Reboot failed: {}".format(e)) async def sleep(self, base): """ From 1896a4ec5858e041c7e687f05347c024299ee9e3 Mon Sep 17 00:00:00 2001 From: Victor BERTIN Date: Thu, 10 Jun 2021 17:32:25 +0200 Subject: [PATCH 2/2] Add simple sort algorithm to process OS chunk first Previously, the chunks were processed in the order where the user dragged and dropped the "software updates" when creating the Hawkbit update on Hawkbit Web Client. I was a problem because for multi-updates OS + Apps. --- fullmetalupdate/fullmetalupdate_ddi_client.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/fullmetalupdate/fullmetalupdate_ddi_client.py b/fullmetalupdate/fullmetalupdate_ddi_client.py index bdee5df..38c97ba 100755 --- a/fullmetalupdate/fullmetalupdate_ddi_client.py +++ b/fullmetalupdate/fullmetalupdate_ddi_client.py @@ -156,7 +156,6 @@ async def process_deployment(self, base): status_execution, status_result, [msg]) raise APIError(msg) else: - # problem ? msg = "FullMetalUpdate:Proceeding" percentage = {"cnt": 0, "of": chunks_qty} status_execution = DeploymentStatusExecution.proceeding @@ -164,6 +163,14 @@ async def process_deployment(self, base): await self.ddi.deploymentBase[action_id].feedback( status_execution, status_result, [msg], percentage=percentage) + + # Algorithm to sort chunks : OS chunk first and then Apps chunk + count = 0 + for count in range(chunks_qty): + if deploy_info['deployment']['chunks'][count]['part'] == 'os': + deploy_info['deployment']['chunks'].insert(0, deploy_info['deployment']['chunks'][count]) + del deploy_info['deployment']['chunks'][count + 1] + break self.action_id = action_id final_result_os = True @@ -251,6 +258,7 @@ async def process_deployment(self, base): final_result_apps = True fails = "" feedbackMsg = "" + notifyMsg = "" feedbackThreadIt = iter(self.feedbackThreads) # Hawkbit server feedback process @@ -264,7 +272,7 @@ async def process_deployment(self, base): if not update['status_update']: msg = "App {} v.{} Deployment failed : {}\n".format(update['name'], update['version'], feedbackMsg) - notifyMsg = msg + notifyMsg += msg self.logger.error(msg) update['status_result'] = DeploymentStatusResult.failure fails += update['name'] + " " @@ -288,7 +296,6 @@ async def process_deployment(self, base): notifyMsg = msg + "\n" + notifyMsg if feedback: notifyMsg += reboot_data['msg'] - status_result = DeploymentStatusResult.success await self.ddi.deploymentBase[self.action_id].feedback(DeploymentStatusExecution.closed, status_result, [notifyMsg]) self.action_id = None