From 6217e1c5f714bde77abdc85e408c1c8a7fd43c61 Mon Sep 17 00:00:00 2001 From: Tanner Lewis Date: Fri, 22 Nov 2024 19:44:39 -0500 Subject: [PATCH] [Jenkins] Add Solutions CFN Create VPC Test (#1159) This change introduces a new Jenkins pipeline that synthesizes the Solutions Create VPC CFN template from this repository and deploys it to the test account. It will then perform the initBootstrap and ensure it exits properly as well as verifies that CLI tools such as cdk, docker, java, and python are now available on the box. --------- Signed-off-by: Tanner Lewis --- .../migration-assistant-solution/bin/app.ts | 11 +- .../lib/solutions-stack.ts | 8 +- .../solutionsCFNTestCover.groovy | 9 ++ test/awsRunInitBootstrap.sh | 107 ++++++++++++++++++ vars/solutionsCFNTest.groovy | 96 ++++++++++++++++ 5 files changed, 223 insertions(+), 8 deletions(-) create mode 100644 jenkins/migrationIntegPipelines/solutionsCFNTestCover.groovy create mode 100755 test/awsRunInitBootstrap.sh create mode 100644 vars/solutionsCFNTest.groovy diff --git a/deployment/migration-assistant-solution/bin/app.ts b/deployment/migration-assistant-solution/bin/app.ts index 4b0425365..de0cf883b 100644 --- a/deployment/migration-assistant-solution/bin/app.ts +++ b/deployment/migration-assistant-solution/bin/app.ts @@ -3,7 +3,7 @@ import { App, DefaultStackSynthesizer } from 'aws-cdk-lib'; import { SolutionsInfrastructureStack } from '../lib/solutions-stack'; const getProps = () => { - const { CODE_BUCKET, SOLUTION_NAME, CODE_VERSION } = process.env; + const { CODE_BUCKET, SOLUTION_NAME, CODE_VERSION, STACK_NAME_SUFFIX } = process.env; if (typeof CODE_BUCKET !== 'string' || CODE_BUCKET.trim() === '') { console.warn(`Missing environment variable CODE_BUCKET, using a default value`); } @@ -19,6 +19,7 @@ const getProps = () => { const codeBucket = CODE_BUCKET ?? "Unknown"; const solutionVersion = CODE_VERSION ?? "Unknown"; const solutionName = SOLUTION_NAME ?? "MigrationAssistant"; + const stackNameSuffix = STACK_NAME_SUFFIX ?? undefined; const solutionId = 'SO0290'; const description = `(${solutionId}) - The AWS CloudFormation template for deployment of the ${solutionName}. Version ${solutionVersion}`; return { @@ -26,19 +27,19 @@ const getProps = () => { solutionVersion, solutionId, solutionName, - description + description, + stackNameSuffix }; }; const app = new App(); const infraProps = getProps() - -new SolutionsInfrastructureStack(app, 'Migration-Assistant-Infra-Import-VPC', { +new SolutionsInfrastructureStack(app, "Migration-Assistant-Infra-Import-VPC", { synthesizer: new DefaultStackSynthesizer(), createVPC: false, ...infraProps }); -new SolutionsInfrastructureStack(app, 'Migration-Assistant-Infra-Create-VPC', { +new SolutionsInfrastructureStack(app, "Migration-Assistant-Infra-Create-VPC", { synthesizer: new DefaultStackSynthesizer(), createVPC: true, ...infraProps diff --git a/deployment/migration-assistant-solution/lib/solutions-stack.ts b/deployment/migration-assistant-solution/lib/solutions-stack.ts index 71b64a7e7..257e31693 100644 --- a/deployment/migration-assistant-solution/lib/solutions-stack.ts +++ b/deployment/migration-assistant-solution/lib/solutions-stack.ts @@ -38,6 +38,7 @@ export interface SolutionsInfrastructureStackProps extends StackProps { readonly solutionVersion: string; readonly codeBucket: string; readonly createVPC: boolean; + readonly stackNameSuffix?: string; } interface ParameterLabel { @@ -113,7 +114,8 @@ function getVpcEndpointForEFS(stack: Stack): InterfaceVpcEndpointAwsService { export class SolutionsInfrastructureStack extends Stack { constructor(scope: Construct, id: string, props: SolutionsInfrastructureStackProps) { - super(scope, id, props); + const finalId = props.stackNameSuffix ? `${id}-${props.stackNameSuffix}` : id + super(scope, finalId, props); this.templateOptions.templateFormatVersion = '2010-09-09'; new CfnMapping(this, 'Solution', { mapping: { @@ -189,7 +191,7 @@ export class SolutionsInfrastructureStack extends Stack { }); const serviceEndpoints = [ - // Logs and disk usage scales based on total data transfer + // Logs and disk usage scales based on total data transfer InterfaceVpcEndpointAwsService.CLOUDWATCH_LOGS, getVpcEndpointForEFS(this), @@ -197,7 +199,7 @@ export class SolutionsInfrastructureStack extends Stack { InterfaceVpcEndpointAwsService.ECR, InterfaceVpcEndpointAwsService.ECR_DOCKER, ]; - + serviceEndpoints.forEach(service => { new InterfaceVpcEndpoint(this, `${service.shortName}VpcEndpoint`, { service, diff --git a/jenkins/migrationIntegPipelines/solutionsCFNTestCover.groovy b/jenkins/migrationIntegPipelines/solutionsCFNTestCover.groovy new file mode 100644 index 000000000..9bbe43672 --- /dev/null +++ b/jenkins/migrationIntegPipelines/solutionsCFNTestCover.groovy @@ -0,0 +1,9 @@ +def gitBranch = params.GIT_BRANCH ?: 'main' +def gitUrl = params.GIT_REPO_URL ?: 'https://github.com/opensearch-project/opensearch-migrations.git' + +library identifier: "migrations-lib@${gitBranch}", retriever: modernSCM( + [$class: 'GitSCMSource', + remote: "${gitUrl}"]) + +// Shared library function (location from root: vars/solutionsCFNTest.groovy) +solutionsCFNTest() diff --git a/test/awsRunInitBootstrap.sh b/test/awsRunInitBootstrap.sh new file mode 100755 index 000000000..c0fefe718 --- /dev/null +++ b/test/awsRunInitBootstrap.sh @@ -0,0 +1,107 @@ +#!/bin/bash + +usage() { + echo "" + echo "Script to run initBootstrap.sh on Migration Assistant bootstrap box" + echo "" + echo "Usage: " + echo " ./awsRunInitBootstrap.sh [--stage] [--workflow]--" + echo "" + echo "Options:" + echo " --stage Deployment stage name, e.g. sol-integ" + echo " --workflow Workflow to execute, options include ALL(default)|INIT_BOOTSTRAP|VERIFY_INIT_BOOTSTRAP" + echo "" + exit 1 +} + +STAGE="aws-integ" +WORKFLOW="ALL" +REGION="us-east-1" +while [[ $# -gt 0 ]]; do + case $1 in + --stage) + STAGE="$2" + shift # past argument + shift # past value + ;; + --workflow) + WORKFLOW="$2" + shift # past argument + shift # past value + ;; + -h|--help) + usage + ;; + -*) + echo "Unknown option $1" + usage + ;; + *) + shift # past argument + ;; + esac +done + +execute_command_and_wait_for_result() { + local command="$1" + local instance_id="$2" + echo "Executing command: [$command] on node: $instance_id" + command_id=$(aws ssm send-command --instance-ids "$instance_id" --document-name "AWS-RunShellScript" --parameters commands="$command" --output text --query 'Command.CommandId') + if [[ -z "$command_id" ]]; then + echo "Error: Unable to retrieve command id from triggered SSM command" + exit 1 + fi + sleep 5 + command_status=$(aws ssm get-command-invocation --command-id "$command_id" --instance-id "$instance_id" --output text --query 'Status') + max_attempts=25 + attempt_count=0 + while [ "$command_status" != "Success" ] && [ "$command_status" != "Failed" ] && [ "$command_status" != "TimedOut" ] + do + ((attempt_count++)) + if [[ $attempt_count -ge $max_attempts ]]; then + echo "Error: Command did not complete within the maximum retry limit." + exit 1 + fi + echo "Waiting for command to complete, current status is $command_status" + sleep 60 + command_status=$(aws ssm get-command-invocation --command-id "$command_id" --instance-id "$instance_id" --output text --query 'Status') + done + echo "Command has completed with status: $command_status, appending output" + echo "Standard Output:" + aws ssm get-command-invocation --command-id "$command_id" --instance-id "$instance_id" --output text --query 'StandardOutputContent' + echo "Standard Error:" + aws ssm get-command-invocation --command-id "$command_id" --instance-id "$instance_id" --output text --query 'StandardErrorContent' + + if [[ "$command_status" != "Success" ]]; then + echo "Error: Command [$command] was not successful, see logs above" + exit 1 + fi +} + +get_instance_id() { + # Retrieve the instance ID + instance_id=$(aws ec2 describe-instances \ + --filters "Name=tag:Name,Values=bootstrap-instance-${STAGE}-${REGION}" "Name=instance-state-name,Values=running" \ + --query "Reservations[0].Instances[0].InstanceId" \ + --output text) + + if [[ -z "$instance_id" || "$instance_id" == "None" ]]; then + echo "Error: Running bootstrap EC2 instance not found" + exit 1 + fi + echo "$instance_id" +} + +instance_id=$(get_instance_id) +init_command="cd /opensearch-migrations && ./initBootstrap.sh" +verify_command="cdk --version && docker --version && java --version && python3 --version" +if [ "$WORKFLOW" = "ALL" ]; then + execute_command_and_wait_for_result "$init_command" "$instance_id" + execute_command_and_wait_for_result "$verify_command" "$instance_id" +elif [ "$WORKFLOW" = "INIT_BOOTSTRAP" ]; then + execute_command_and_wait_for_result "$init_command" "$instance_id" +elif [ "$WORKFLOW" = "VERIFY_INIT_BOOTSTRAP" ]; then + execute_command_and_wait_for_result "$verify_command" "$instance_id" +else + echo "Error: Unknown workflow: ${WORKFLOW} specified" +fi \ No newline at end of file diff --git a/vars/solutionsCFNTest.groovy b/vars/solutionsCFNTest.groovy new file mode 100644 index 000000000..194b7ba35 --- /dev/null +++ b/vars/solutionsCFNTest.groovy @@ -0,0 +1,96 @@ +def call(Map config = [:]) { + + pipeline { + agent { label config.workerAgent ?: 'Jenkins-Default-Agent-X64-C5xlarge-Single-Host' } + + parameters { + string(name: 'GIT_REPO_URL', defaultValue: 'https://github.com/opensearch-project/opensearch-migrations.git', description: 'Git repository url') + string(name: 'GIT_BRANCH', defaultValue: 'main', description: 'Git branch to use for repository') + string(name: 'STAGE', defaultValue: "sol-integ", description: 'Stage name for deployment environment') + } + + options { + // Acquire lock on a given deployment stage + lock(label: params.STAGE, quantity: 1, variable: 'stage') + timeout(time: 1, unit: 'HOURS') + buildDiscarder(logRotator(daysToKeepStr: '30')) + } + + stages { + stage('Checkout') { + steps { + script { + git branch: "${params.GIT_BRANCH}", url: "${params.GIT_REPO_URL}" + } + } + } + + stage('Deployment') { + steps { + timeout(time: 15, unit: 'MINUTES') { + dir('deployment/migration-assistant-solution') { + script { + env.STACK_NAME_SUFFIX = "${stage}-us-east-1" + sh "sudo npm install" + withCredentials([string(credentialsId: 'migrations-test-account-id', variable: 'MIGRATIONS_TEST_ACCOUNT_ID')]) { + withAWS(role: 'JenkinsDeploymentRole', roleAccount: "${MIGRATIONS_TEST_ACCOUNT_ID}", region: "us-east-1", duration: 3600, roleSessionName: 'jenkins-session') { + sh "sudo --preserve-env cdk deploy Migration-Assistant-Infra-Create-VPC-${env.STACK_NAME_SUFFIX} --parameters Stage=${stage} --require-approval never --concurrency 3" + } + } + // Wait for instance to be ready to accept SSM commands + sh "sleep 15" + } + } + } + } + } + + stage('Init Bootstrap') { + steps { + timeout(time: 30, unit: 'MINUTES') { + dir('test') { + script { + withCredentials([string(credentialsId: 'migrations-test-account-id', variable: 'MIGRATIONS_TEST_ACCOUNT_ID')]) { + withAWS(role: 'JenkinsDeploymentRole', roleAccount: "${MIGRATIONS_TEST_ACCOUNT_ID}", region: "us-east-1", duration: 3600, roleSessionName: 'jenkins-session') { + sh "sudo --preserve-env ./awsRunInitBootstrap.sh --stage ${stage} --workflow INIT_BOOTSTRAP" + } + } + } + } + } + } + } + + stage('Verify Bootstrap Instance') { + steps { + timeout(time: 5, unit: 'MINUTES') { + dir('test') { + script { + withCredentials([string(credentialsId: 'migrations-test-account-id', variable: 'MIGRATIONS_TEST_ACCOUNT_ID')]) { + withAWS(role: 'JenkinsDeploymentRole', roleAccount: "${MIGRATIONS_TEST_ACCOUNT_ID}", region: "us-east-1", duration: 3600, roleSessionName: 'jenkins-session') { + sh "sudo --preserve-env ./awsRunInitBootstrap.sh --stage ${stage} --workflow VERIFY_INIT_BOOTSTRAP" + } + } + } + } + } + } + } + } + post { + always { + timeout(time: 30, unit: 'MINUTES') { + dir('deployment/migration-assistant-solution') { + script { + withCredentials([string(credentialsId: 'migrations-test-account-id', variable: 'MIGRATIONS_TEST_ACCOUNT_ID')]) { + withAWS(role: 'JenkinsDeploymentRole', roleAccount: "${MIGRATIONS_TEST_ACCOUNT_ID}", region: "us-east-1", duration: 3600, roleSessionName: 'jenkins-session') { + sh "sudo --preserve-env cdk destroy Migration-Assistant-Infra-Create-VPC-${env.STACK_NAME_SUFFIX} --force" + } + } + } + } + } + } + } + } +}