diff --git a/.gitignore b/.gitignore index 8a4620e..3deb735 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ conf/*.conf -!conf/flux-capacitor-example.conf +!conf/flux-capacitor-defaults.conf diff --git a/README.md b/README.md index 7217807..e2cbc01 100644 --- a/README.md +++ b/README.md @@ -27,8 +27,14 @@ Without using any costly database, this solution complements [Amazon Kinesis](ht * To the greatest extent possible without storing another copy of the data, provide a unified and de-duplicated view spanning current and archived Amazon Kinesis records. * (TBD) Provide a basic UI or API to initiate search and replay operations, and monitor progress. +# Prerequisites +* Bash shell installed at **/bin/bash** +* [AWS CLI](http://aws.amazon.com/cli/) installed and configured with your credentials and default region (you can run **aws configure** to do so interactively) + # Getting Started -* Install the [AWS CLI](http://aws.amazon.com/cli/) and configure it with your credentials and default region -* Create a config file by copying **conf/example-flux-capacitor.conf** and editing it +* Create a config file. Either: + * Make a copy of **conf/flux-capacitor-defaults.conf** and edit the copy, or + * Create a new file that will contain only overrides, and import the defaults +by following the directions at the top of **conf/flux-capacitor-defaults.conf** * Run **./upload-flux-capacitor-resources ** * Run **./launch-flux-capacitor ** diff --git a/conf/flux-capacitor-defaults.conf b/conf/flux-capacitor-defaults.conf new file mode 100644 index 0000000..de007bf --- /dev/null +++ b/conf/flux-capacitor-defaults.conf @@ -0,0 +1,53 @@ +## NOTE: flux-capacitor configuration files are sourced into a bash shell (executed). + +## NOTE: To complement the default configuration with a file containing only +## a few custom overrides, create an override file whose content begins with: +## +## . ${FLUXCAP_HOME}/conf/flux-capacitor-defaults.conf +## +## and then sets some of the variables found below to custom values. +## +## Pass the location of the override file as the configFile argument to +## flux-capacitor scripts. + +# AWS CLI profile for commands run locally +AWS_PROFILE="default" + +# The S3 location at which flux-capacitor resources will be made available. +# This will be done by the upload-flux-capacitor-resources script. +FLUXCAP_S3_BUCKET="flux-capacitor" +FLUXCAP_S3_PATH="/resources" + +##################################### +##### BEGIN EMR CLUSTER OPTIONS ##### +##################################### +EMR_CLUSTER_NAME="flux-capacitor" + +# The roles configured below must already exist. +# To create the default Service Role "EMR_DefaultRole" and the default EC2 +# instance role "EMR_EC2_DefaultRole", use the command: +# aws emr create-default-roles +EMR_SERVICE_ROLE="EMR_DefaultRole" +EMR_EC2_ROLE="EMR_EC2_DefaultRole" + +EMR_AMI_VERSION="3.7.0" +EMR_INSTANCE_GROUP_MASTER="InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge" +EMR_INSTANCE_GROUP_CORE="InstanceGroupType=CORE,InstanceCount=1,InstanceType=m3.xlarge" +EMR_INSTANCE_GROUP_TASK="InstanceGroupType=TASK,InstanceCount=1,InstanceType=m3.xlarge" +EMR_INSTANCE_GROUPS="${EMR_INSTANCE_GROUP_MASTER} ${EMR_INSTANCE_GROUP_CORE} ${EMR_INSTANCE_GROUP_TASK}" + +# Additional options to 'aws emr create-cluster' +# Refer to http://docs.aws.amazon.com/cli/latest/reference/emr/create-cluster.html +# This value will not be quoted when expanded, so the aws command will recognize +# multiple parameters. +EMR_EXTRA_CLUSTER_OPTIONS= + +# Additional EC2 attributes to include in 'aws emr create-cluster --ec2-attributes' +# Refer to http://docs.aws.amazon.com/cli/latest/reference/emr/create-cluster.html +# This will be expanded into a quoted string with no further quotation of this +# value specifically. +EMR_EXTRA_EC2_ATTRIBUTES= + +##################################### +##### END EMR CLUSTER OPTIONS ##### +##################################### diff --git a/conf/flux-capacitor-example.conf b/conf/flux-capacitor-example.conf deleted file mode 100644 index 47360e3..0000000 --- a/conf/flux-capacitor-example.conf +++ /dev/null @@ -1,6 +0,0 @@ -# AWS CLI profile -AWS_PROFILE="default" - -# The S3 location at which flux-capacitor resources will be made available -FLUXCAP_S3_BUCKET="flux-capacitor" -FLUXCAP_S3_PATH="/resources" diff --git a/launch-flux-capacitor b/launch-flux-capacitor index 09b2664..518024e 100755 --- a/launch-flux-capacitor +++ b/launch-flux-capacitor @@ -1,5 +1,6 @@ -#!/bin/sh -x +#!/bin/bash +O sourcepath +FLUXCAP_HOME=${PWD} CONFIGFILE=$1 if [ -z "${CONFIGFILE}" ]; then echo "Usage: $0 " @@ -7,4 +8,15 @@ if [ -z "${CONFIGFILE}" ]; then fi . ${CONFIGFILE} -aws --profile ${AWS_PROFILE} emr help +EMR_EC2_ATTRIBUTES="InstanceProfile=${EMR_EC2_ROLE}" +if [ -n "${EMR_EXTRA_EC2_ATTRIBUTES}" ]; then + EMR_EC2_ATTRIBUTES="${EMR_EC2_ATTRIBUTES},${EMR_EXTRA_EC2_ATTRIBUTES}" +fi + +aws --profile "${AWS_PROFILE}" emr create-cluster \ + --ami-version "${EMR_AMI_VERSION}" \ + --instance-groups ${EMR_INSTANCE_GROUPS} \ + --name "${EMR_CLUSTER_NAME}" \ + --service-role "${EMR_SERVICE_ROLE}" \ + --ec2-attributes "${EMR_EC2_ATTRIBUTES}" \ + ${EMR_EXTRA_CLUSTER_OPTIONS} diff --git a/upload-flux-capacitor-resources b/upload-flux-capacitor-resources index 9accdf5..23fd78b 100755 --- a/upload-flux-capacitor-resources +++ b/upload-flux-capacitor-resources @@ -1,4 +1,4 @@ -#!/bin/sh -x +#!/bin/bash +O sourcepath CONFIGFILE=$1 if [ -z "${CONFIGFILE}" ]; then