Scripted launch of cluster with no apps, bootstrap actions, or steps

.gitignore

@@ -1,2 +1,2 @@
 conf/*.conf
-!conf/flux-capacitor-example.conf
+!conf/flux-capacitor-defaults.conf

README.md

@@ -27,8 +27,14 @@ Without using any costly database, this solution complements [Amazon Kinesis](ht
 * To the greatest extent possible without storing another copy of the data, provide a unified and de-duplicated view spanning current and archived Amazon Kinesis records.
 * (TBD) Provide a basic UI or API to initiate search and replay operations, and monitor progress.
 
+# Prerequisites
+* Bash shell installed at **/bin/bash**
+* [AWS CLI](http://aws.amazon.com/cli/) installed and configured with your credentials and default region (you can run **aws configure** to do so interactively)
+
 # Getting Started
-* Install the [AWS CLI](http://aws.amazon.com/cli/) and configure it with your credentials and default region
-* Create a config file by copying **conf/example-flux-capacitor.conf** and editing it
+* Create a config file. Either:
+ * Make a copy of **conf/flux-capacitor-defaults.conf** and edit the copy, or
+ * Create a new file that will contain only overrides, and import the defaults
+by following the directions at the top of **conf/flux-capacitor-defaults.conf**
 * Run **./upload-flux-capacitor-resources <config file>**
 * Run **./launch-flux-capacitor <config file>**

conf/flux-capacitor-defaults.conf

@@ -0,0 +1,53 @@
+## NOTE: flux-capacitor configuration files are sourced into a bash shell (executed).
+
+## NOTE: To complement the default configuration with a file containing only
+## a few custom overrides, create an override file whose content begins with:
+##
+## . ${FLUXCAP_HOME}/conf/flux-capacitor-defaults.conf
+##
+## and then sets some of the variables found below to custom values.
+##
+## Pass the location of the override file as the configFile argument to
+## flux-capacitor scripts.
+
+# AWS CLI profile for commands run locally
+AWS_PROFILE="default"
+
+# The S3 location at which flux-capacitor resources will be made available.
+# This will be done by the upload-flux-capacitor-resources script.
+FLUXCAP_S3_BUCKET="flux-capacitor"
+FLUXCAP_S3_PATH="/resources"
+
+#####################################
+##### BEGIN EMR CLUSTER OPTIONS #####
+#####################################
+EMR_CLUSTER_NAME="flux-capacitor"
+
+# The roles configured below must already exist.
+# To create the default Service Role "EMR_DefaultRole" and the default EC2
+# instance role "EMR_EC2_DefaultRole", use the command:
+#     aws emr create-default-roles
+EMR_SERVICE_ROLE="EMR_DefaultRole"
+EMR_EC2_ROLE="EMR_EC2_DefaultRole"
+
+EMR_AMI_VERSION="3.7.0"
+EMR_INSTANCE_GROUP_MASTER="InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge"
+EMR_INSTANCE_GROUP_CORE="InstanceGroupType=CORE,InstanceCount=1,InstanceType=m3.xlarge"
+EMR_INSTANCE_GROUP_TASK="InstanceGroupType=TASK,InstanceCount=1,InstanceType=m3.xlarge"
+EMR_INSTANCE_GROUPS="${EMR_INSTANCE_GROUP_MASTER} ${EMR_INSTANCE_GROUP_CORE} ${EMR_INSTANCE_GROUP_TASK}"
+
+# Additional options to 'aws emr create-cluster'
+# Refer to http://docs.aws.amazon.com/cli/latest/reference/emr/create-cluster.html
+# This value will not be quoted when expanded, so the aws command will recognize
+# multiple parameters.
+EMR_EXTRA_CLUSTER_OPTIONS=
+
+# Additional EC2 attributes to include in 'aws emr create-cluster --ec2-attributes'
+# Refer to http://docs.aws.amazon.com/cli/latest/reference/emr/create-cluster.html
+# This will be expanded into a quoted string with no further quotation of this
+# value specifically.
+EMR_EXTRA_EC2_ATTRIBUTES=
+
+#####################################
+#####  END EMR CLUSTER OPTIONS  #####
+#####################################

launch-flux-capacitor

@@ -1,10 +1,22 @@
-#!/bin/sh -x
+#!/bin/bash +O sourcepath
 
+FLUXCAP_HOME=${PWD}
 CONFIGFILE=$1
 if [ -z "${CONFIGFILE}" ]; then
     echo "Usage: $0 <configFile>"
     exit 1
 fi
 . ${CONFIGFILE}
 
-aws --profile ${AWS_PROFILE} emr help
+EMR_EC2_ATTRIBUTES="InstanceProfile=${EMR_EC2_ROLE}"
+if [ -n "${EMR_EXTRA_EC2_ATTRIBUTES}" ]; then
+    EMR_EC2_ATTRIBUTES="${EMR_EC2_ATTRIBUTES},${EMR_EXTRA_EC2_ATTRIBUTES}"
+fi
+
+aws --profile "${AWS_PROFILE}" emr create-cluster \
+ --ami-version "${EMR_AMI_VERSION}" \
+ --instance-groups ${EMR_INSTANCE_GROUPS} \
+ --name "${EMR_CLUSTER_NAME}" \
+ --service-role "${EMR_SERVICE_ROLE}" \
+ --ec2-attributes "${EMR_EC2_ATTRIBUTES}" \
+ ${EMR_EXTRA_CLUSTER_OPTIONS}

upload-flux-capacitor-resources

@@ -1,4 +1,4 @@
-#!/bin/sh -x
+#!/bin/bash +O sourcepath
 
 CONFIGFILE=$1
 if [ -z "${CONFIGFILE}" ]; then