launch-cluster

#!/bin/bash +O sourcepath

# Copyright (C) 2015 Commerce Technologies, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


# Establishes an SSH session with local port forwarding per
# http://docs.aws.amazon.com/ElasticMapReduce/latest/DeveloperGuide/emr-ssh-tunnel-local.html
#
# For alternative methods of connecting, see
# http://docs.aws.amazon.com/ElasticMapReduce/latest/DeveloperGuide/emr-connect-master-node.html

LAUNCHER_HOME=${PWD}
CONFIGFILE=$1
if [ -z "${CONFIGFILE}" ]; then
    echo "Lauches a cluster and prints the cluster-id on stdout."
    echo ""
    echo "Usage: $0 <configFile>"
    exit 1
fi
. ${CONFIGFILE}

EMR_EC2_ATTRIBUTES="InstanceProfile=${EMR_EC2_ROLE},KeyName=${EMR_EC2_KEY_NAME}"
if [ -n "${EMR_EXTRA_EC2_ATTRIBUTES}" ]; then
    EMR_EC2_ATTRIBUTES="${EMR_EC2_ATTRIBUTES},${EMR_EXTRA_EC2_ATTRIBUTES}"
fi

EMR_BA_DRILL="Path=${RESOURCES_S3_URL}/emr/exec/setup_drill,Name=setup_drill"
EMR_BA_DRILL_LIB_AWS="Path=${RESOURCES_S3_URL}/emr/exec/setup_drill_lib_aws,Name=setup_drill_lib_aws"
EMR_BA_DRILL_STORAGE_STREAMS="Path=${RESOURCES_S3_URL}/emr/exec/configure_drill_storage_streams,Name=configure_drill_storage_streams"
EMR_BA_DRILL_STORAGE_STREAM_ARCHIVES="Path=${RESOURCES_S3_URL}/emr/exec/configure_drill_storage_stream_archives,Args=[${STREAM_ARCHIVES_DFS_URL},${STREAM_ARCHIVES_PATH}],Name=configure_drill_storage_stream_archives"
EMR_BOOTSTRAP_ACTIONS="${EMR_BA_DRILL} ${EMR_BA_DRILL_LIB_AWS} ${EMR_BA_HIVE_METASTORE} ${EMR_BA_DRILL_STORAGE_STREAMS} ${EMR_BA_DRILL_STORAGE_STREAM_ARCHIVES}"

EMR_CONFIG_STEPS="Name=hive_create_table_for_stream,Type=HIVE,Args=[-f,${RESOURCES_S3_URL}/emr/hive/create_table_for_stream,-hiveconf,tablename=${STREAM_HIVE_TABLE_NAME},-hiveconf,streamname=${STREAM_NAME}]"

aws --profile "${AWS_PROFILE}" emr create-cluster \
 --ami-version ${EMR_AMI_VERSION} \
 --instance-groups ${EMR_INSTANCE_GROUPS} \
 --name "${EMR_CLUSTER_NAME}" \
 --service-role ${EMR_SERVICE_ROLE} \
 --ec2-attributes ${EMR_EC2_ATTRIBUTES} \
 --applications ${EMR_APPLICATIONS} \
 --bootstrap-actions ${EMR_BOOTSTRAP_ACTIONS} \
 --steps ${EMR_CONFIG_STEPS} \
 ${EMR_EXTRA_CLUSTER_OPTIONS} \
 --query ClusterId --output text