-
Notifications
You must be signed in to change notification settings - Fork 31
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[Jenkins] Add Solutions CFN Create VPC Test (#1159)
This change introduces a new Jenkins pipeline that synthesizes the Solutions Create VPC CFN template from this repository and deploys it to the test account. It will then perform the initBootstrap and ensure it exits properly as well as verifies that CLI tools such as cdk, docker, java, and python are now available on the box. --------- Signed-off-by: Tanner Lewis <[email protected]>
- Loading branch information
Showing
5 changed files
with
223 additions
and
8 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,9 @@ | ||
def gitBranch = params.GIT_BRANCH ?: 'main' | ||
def gitUrl = params.GIT_REPO_URL ?: 'https://github.com/opensearch-project/opensearch-migrations.git' | ||
|
||
library identifier: "migrations-lib@${gitBranch}", retriever: modernSCM( | ||
[$class: 'GitSCMSource', | ||
remote: "${gitUrl}"]) | ||
|
||
// Shared library function (location from root: vars/solutionsCFNTest.groovy) | ||
solutionsCFNTest() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,107 @@ | ||
#!/bin/bash | ||
|
||
usage() { | ||
echo "" | ||
echo "Script to run initBootstrap.sh on Migration Assistant bootstrap box" | ||
echo "" | ||
echo "Usage: " | ||
echo " ./awsRunInitBootstrap.sh [--stage] [--workflow]--" | ||
echo "" | ||
echo "Options:" | ||
echo " --stage Deployment stage name, e.g. sol-integ" | ||
echo " --workflow Workflow to execute, options include ALL(default)|INIT_BOOTSTRAP|VERIFY_INIT_BOOTSTRAP" | ||
echo "" | ||
exit 1 | ||
} | ||
|
||
STAGE="aws-integ" | ||
WORKFLOW="ALL" | ||
REGION="us-east-1" | ||
while [[ $# -gt 0 ]]; do | ||
case $1 in | ||
--stage) | ||
STAGE="$2" | ||
shift # past argument | ||
shift # past value | ||
;; | ||
--workflow) | ||
WORKFLOW="$2" | ||
shift # past argument | ||
shift # past value | ||
;; | ||
-h|--help) | ||
usage | ||
;; | ||
-*) | ||
echo "Unknown option $1" | ||
usage | ||
;; | ||
*) | ||
shift # past argument | ||
;; | ||
esac | ||
done | ||
|
||
execute_command_and_wait_for_result() { | ||
local command="$1" | ||
local instance_id="$2" | ||
echo "Executing command: [$command] on node: $instance_id" | ||
command_id=$(aws ssm send-command --instance-ids "$instance_id" --document-name "AWS-RunShellScript" --parameters commands="$command" --output text --query 'Command.CommandId') | ||
if [[ -z "$command_id" ]]; then | ||
echo "Error: Unable to retrieve command id from triggered SSM command" | ||
exit 1 | ||
fi | ||
sleep 5 | ||
command_status=$(aws ssm get-command-invocation --command-id "$command_id" --instance-id "$instance_id" --output text --query 'Status') | ||
max_attempts=25 | ||
attempt_count=0 | ||
while [ "$command_status" != "Success" ] && [ "$command_status" != "Failed" ] && [ "$command_status" != "TimedOut" ] | ||
do | ||
((attempt_count++)) | ||
if [[ $attempt_count -ge $max_attempts ]]; then | ||
echo "Error: Command did not complete within the maximum retry limit." | ||
exit 1 | ||
fi | ||
echo "Waiting for command to complete, current status is $command_status" | ||
sleep 60 | ||
command_status=$(aws ssm get-command-invocation --command-id "$command_id" --instance-id "$instance_id" --output text --query 'Status') | ||
done | ||
echo "Command has completed with status: $command_status, appending output" | ||
echo "Standard Output:" | ||
aws ssm get-command-invocation --command-id "$command_id" --instance-id "$instance_id" --output text --query 'StandardOutputContent' | ||
echo "Standard Error:" | ||
aws ssm get-command-invocation --command-id "$command_id" --instance-id "$instance_id" --output text --query 'StandardErrorContent' | ||
|
||
if [[ "$command_status" != "Success" ]]; then | ||
echo "Error: Command [$command] was not successful, see logs above" | ||
exit 1 | ||
fi | ||
} | ||
|
||
get_instance_id() { | ||
# Retrieve the instance ID | ||
instance_id=$(aws ec2 describe-instances \ | ||
--filters "Name=tag:Name,Values=bootstrap-instance-${STAGE}-${REGION}" "Name=instance-state-name,Values=running" \ | ||
--query "Reservations[0].Instances[0].InstanceId" \ | ||
--output text) | ||
|
||
if [[ -z "$instance_id" || "$instance_id" == "None" ]]; then | ||
echo "Error: Running bootstrap EC2 instance not found" | ||
exit 1 | ||
fi | ||
echo "$instance_id" | ||
} | ||
|
||
instance_id=$(get_instance_id) | ||
init_command="cd /opensearch-migrations && ./initBootstrap.sh" | ||
verify_command="cdk --version && docker --version && java --version && python3 --version" | ||
if [ "$WORKFLOW" = "ALL" ]; then | ||
execute_command_and_wait_for_result "$init_command" "$instance_id" | ||
execute_command_and_wait_for_result "$verify_command" "$instance_id" | ||
elif [ "$WORKFLOW" = "INIT_BOOTSTRAP" ]; then | ||
execute_command_and_wait_for_result "$init_command" "$instance_id" | ||
elif [ "$WORKFLOW" = "VERIFY_INIT_BOOTSTRAP" ]; then | ||
execute_command_and_wait_for_result "$verify_command" "$instance_id" | ||
else | ||
echo "Error: Unknown workflow: ${WORKFLOW} specified" | ||
fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,96 @@ | ||
def call(Map config = [:]) { | ||
|
||
pipeline { | ||
agent { label config.workerAgent ?: 'Jenkins-Default-Agent-X64-C5xlarge-Single-Host' } | ||
|
||
parameters { | ||
string(name: 'GIT_REPO_URL', defaultValue: 'https://github.com/opensearch-project/opensearch-migrations.git', description: 'Git repository url') | ||
string(name: 'GIT_BRANCH', defaultValue: 'main', description: 'Git branch to use for repository') | ||
string(name: 'STAGE', defaultValue: "sol-integ", description: 'Stage name for deployment environment') | ||
} | ||
|
||
options { | ||
// Acquire lock on a given deployment stage | ||
lock(label: params.STAGE, quantity: 1, variable: 'stage') | ||
timeout(time: 1, unit: 'HOURS') | ||
buildDiscarder(logRotator(daysToKeepStr: '30')) | ||
} | ||
|
||
stages { | ||
stage('Checkout') { | ||
steps { | ||
script { | ||
git branch: "${params.GIT_BRANCH}", url: "${params.GIT_REPO_URL}" | ||
} | ||
} | ||
} | ||
|
||
stage('Deployment') { | ||
steps { | ||
timeout(time: 15, unit: 'MINUTES') { | ||
dir('deployment/migration-assistant-solution') { | ||
script { | ||
env.STACK_NAME_SUFFIX = "${stage}-us-east-1" | ||
sh "sudo npm install" | ||
withCredentials([string(credentialsId: 'migrations-test-account-id', variable: 'MIGRATIONS_TEST_ACCOUNT_ID')]) { | ||
withAWS(role: 'JenkinsDeploymentRole', roleAccount: "${MIGRATIONS_TEST_ACCOUNT_ID}", region: "us-east-1", duration: 3600, roleSessionName: 'jenkins-session') { | ||
sh "sudo --preserve-env cdk deploy Migration-Assistant-Infra-Create-VPC-${env.STACK_NAME_SUFFIX} --parameters Stage=${stage} --require-approval never --concurrency 3" | ||
} | ||
} | ||
// Wait for instance to be ready to accept SSM commands | ||
sh "sleep 15" | ||
} | ||
} | ||
} | ||
} | ||
} | ||
|
||
stage('Init Bootstrap') { | ||
steps { | ||
timeout(time: 30, unit: 'MINUTES') { | ||
dir('test') { | ||
script { | ||
withCredentials([string(credentialsId: 'migrations-test-account-id', variable: 'MIGRATIONS_TEST_ACCOUNT_ID')]) { | ||
withAWS(role: 'JenkinsDeploymentRole', roleAccount: "${MIGRATIONS_TEST_ACCOUNT_ID}", region: "us-east-1", duration: 3600, roleSessionName: 'jenkins-session') { | ||
sh "sudo --preserve-env ./awsRunInitBootstrap.sh --stage ${stage} --workflow INIT_BOOTSTRAP" | ||
} | ||
} | ||
} | ||
} | ||
} | ||
} | ||
} | ||
|
||
stage('Verify Bootstrap Instance') { | ||
steps { | ||
timeout(time: 5, unit: 'MINUTES') { | ||
dir('test') { | ||
script { | ||
withCredentials([string(credentialsId: 'migrations-test-account-id', variable: 'MIGRATIONS_TEST_ACCOUNT_ID')]) { | ||
withAWS(role: 'JenkinsDeploymentRole', roleAccount: "${MIGRATIONS_TEST_ACCOUNT_ID}", region: "us-east-1", duration: 3600, roleSessionName: 'jenkins-session') { | ||
sh "sudo --preserve-env ./awsRunInitBootstrap.sh --stage ${stage} --workflow VERIFY_INIT_BOOTSTRAP" | ||
} | ||
} | ||
} | ||
} | ||
} | ||
} | ||
} | ||
} | ||
post { | ||
always { | ||
timeout(time: 30, unit: 'MINUTES') { | ||
dir('deployment/migration-assistant-solution') { | ||
script { | ||
withCredentials([string(credentialsId: 'migrations-test-account-id', variable: 'MIGRATIONS_TEST_ACCOUNT_ID')]) { | ||
withAWS(role: 'JenkinsDeploymentRole', roleAccount: "${MIGRATIONS_TEST_ACCOUNT_ID}", region: "us-east-1", duration: 3600, roleSessionName: 'jenkins-session') { | ||
sh "sudo --preserve-env cdk destroy Migration-Assistant-Infra-Create-VPC-${env.STACK_NAME_SUFFIX} --force" | ||
} | ||
} | ||
} | ||
} | ||
} | ||
} | ||
} | ||
} | ||
} |