diff --git a/client.go b/client.go index 9c6271d..a8469b7 100644 --- a/client.go +++ b/client.go @@ -7,7 +7,7 @@ type Client interface { Run(task *Task) error Wait() error Close() error - Prefix() string + Prefix() (string, int) Write(p []byte) (n int, err error) WriteClose() error Stdin() io.WriteCloser diff --git a/example/Supfile b/example/Supfile index 4120ef6..ca4e2dc 100644 --- a/example/Supfile +++ b/example/Supfile @@ -1,72 +1,65 @@ -# Supfile for "Example" server +# Supfile for "Example" Docker service --- +version: 0.3 -env: # Environment variables for the commands +env: + # Environment variables for all commands NAME: example - REPO: github.com/pressly/stackup + REPO: github.com/pressly/sup BRANCH: master IMAGE: pressly/example HOST_PORT: 8000 CONTAINER_PORT: 8000 -networks: # Groups of hosts +networks: + # Groups of hosts local: - env: - CONFIG: example.local.cfg hosts: - localhost dev: env: - CONFIG: example.dev.cfg + # Extra environment variable for dev hosts only + DOCKER_HOST: tcp://127.0.0.1:2375 hosts: - docker@192.168.59.103 stg: - env: - CONFIG: example.stg.cfg hosts: - ubuntu@stg.example.com prod: - inventory: - - for i in 4 5 6; do echo "$USER@prod$i.example.com\n"; done - env: - CONFIG: example.prod.cfg - hosts: - - ubuntu@prod1.example.com - - ubuntu@prod2.example.com - - ubuntu@prod3.example.com + inventory: for i in 1 2 3 4; do echo "ubuntu@prod$i.example.com"; done -commands: # Named set of commands to be run remotely +commands: + # Named set of commands to be run remotely ping: desc: Print uname and current date/time. run: uname -a; date - upload: - desc: Upload this repository + pre-build: + desc: Initialize directory + run: mkdir -p /tmp/$IMAGE + + build: + desc: Build Docker image from current directory, push to Docker Hub + # local: sup -f ./builder/Supfile $SUP_NETWORK build upload: - src: ./ dst: /tmp/$IMAGE - - build: - desc: Build Docker image script: ./scripts/docker-build.sh + once: true - image: - desc: List Docker image - run: sudo docker images | grep $IMAGE + pull: + desc: Pull latest Docker image + run: sudo docker pull $IMAGE config: desc: Upload/test config file. upload: - - src: ./$CONFIG + - src: ./example.$SUP_NETWORK.cfg dst: /tmp/ - run: test -f /tmp/$CONFIG - - # pull: - # desc: Pull git repository - # script: ./scripts/docker-pull.sh + run: test -f /tmp/example.$SUP_NETWORK.cfg stop: desc: Stop Docker container @@ -82,12 +75,29 @@ commands: # Named set of commands to be run remotely run: desc: Run Docker container - script: ./scripts/docker-run.sh + run: > + sudo docker run -d \ + -p $HOST_PORT:$CONTAINER_PORT \ + -v /tmp/example.$SUP_NETWORK.cfg:/etc/example.cfg \ + --restart=always \ + --name $NAME $IMAGE restart: desc: Restart Docker container run: sudo docker restart $NAME || exit 0 + stop-rm-run: + desc: Stop & remove old Docker container, run new one + run: > + sudo docker stop $NAME || :; \ + sudo docker rm $NAME || :; \ + sudo docker run -d \ + -p $HOST_PORT:$CONTAINER_PORT \ + -v /tmp/example.$SUP_NETWORK.cfg:/etc/example.cfg \ + --restart=always \ + --name $NAME $IMAGE + serial: 1 + ps: desc: List running Docker containers run: sudo docker ps | grep $NAME @@ -104,6 +114,12 @@ commands: # Named set of commands to be run remotely desc: Application health check run: curl localhost:$HOST_PORT + slack-notify: + desc: Notify Slack about new deployment + local: > + curl -X POST --data-urlencode 'payload={"channel": "#_team_", "text": "['$SUP_NETWORK'] '$(whoami)' deployed '$NAME'"}' \ + https://hooks.slack.com/services/X/Y/Z + shell: desc: Interactive shell on all hosts stdin: true @@ -112,18 +128,17 @@ commands: # Named set of commands to be run remotely exec: desc: Interactive docker exec on all hosts stdin: true - run: docker exec -i $NAME bash + run: sudo docker exec -i $NAME bash -targets: # Aliases to run multiple commands at once +targets: + # Aliases to run multiple commands at once deploy: - #- pull - - upload + - pre-build - build - - image + - pull - config - - stop - - rm - - run + - stop-rm-run - ps - logs - - health \ No newline at end of file + - health + - slack-notify diff --git a/example/scripts/docker-build.sh b/example/scripts/docker-build.sh index 39933a8..acb5e40 100644 --- a/example/scripts/docker-build.sh +++ b/example/scripts/docker-build.sh @@ -10,7 +10,9 @@ sudo rm -rf bin sudo docker run --rm \ -v $(pwd):/go/src/$REPO/$NAME \ -w /go/src/$REPO/$NAME \ - golang:1.4 go build + golang:1.5.2 go build # Bake bin/* into the resulting image. sudo docker build --no-cache -t $IMAGE . + +sudo docker push $IMAGE diff --git a/example/scripts/docker-pull.sh b/example/scripts/docker-pull.sh deleted file mode 100644 index ca6cbbe..0000000 --- a/example/scripts/docker-pull.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -set -e - -# Clone the repo for the first time. -if [[ ! -d /tmp/$IMAGE || ! -d /tmp/$IMAGE/.git ]]; then - mkdir -p /tmp/$IMAGE - git clone -b $BRANCH $REPO /tmp/$IMAGE -fi - -cd /tmp/$IMAGE - -# Fix permissions. -sudo chown -R $USER:$USER ./ - -# Clean up the repository. -git reset --hard && git clean -f -d - -# Switch to master and update. -git checkout master && git pull - -# Remove all branches except for master. -# (This prevents error when someone rebases his branch.) -for branch in $(git branch | grep -v master || :); do - git branch -D $branch -done - -# If we're deploying a custom branch, we need to pull it first. -if [ "$BRANCH" != "master" ]; then - git checkout -b $BRANCH origin/$BRANCH -fi diff --git a/example/scripts/docker-run.sh b/example/scripts/docker-run.sh deleted file mode 100644 index 10e7e0e..0000000 --- a/example/scripts/docker-run.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -set -e - -sudo docker run -d \ - -p $HOST_PORT:$CONTAINER_PORT \ - -v /tmp/$CONFIG:/etc/example.cfg \ - --restart=always \ - --name $NAME $IMAGE diff --git a/localhost.go b/localhost.go index 263704e..debb0e8 100644 --- a/localhost.go +++ b/localhost.go @@ -86,8 +86,9 @@ func (c *LocalhostClient) Stdout() io.Reader { return c.stdout } -func (c *LocalhostClient) Prefix() string { - return c.user + "@localhost" +func (c *LocalhostClient) Prefix() (string, int) { + host := c.user + "@localhost" + " | " + return ResetColor + host, len(host) } func (c *LocalhostClient) Write(p []byte) (n int, err error) { diff --git a/ssh.go b/ssh.go index 5aa97e1..106c415 100644 --- a/ssh.go +++ b/ssh.go @@ -27,6 +27,7 @@ type SSHClient struct { sessOpened bool running bool env string //export FOO="bar"; export BAR="baz"; + color string } type ErrConnect struct { @@ -247,8 +248,9 @@ func (c *SSHClient) Stdout() io.Reader { return c.remoteStdout } -func (c *SSHClient) Prefix() string { - return c.user + "@" + c.host +func (c *SSHClient) Prefix() (string, int) { + host := c.user + "@" + c.host + " | " + return c.color + host + ResetColor, len(host) } func (c *SSHClient) Write(p []byte) (n int, err error) { diff --git a/sup.go b/sup.go index 963aa5d..20d12fd 100644 --- a/sup.go +++ b/sup.go @@ -12,7 +12,7 @@ import ( "golang.org/x/crypto/ssh" ) -const VERSION = "0.2.2" +const VERSION = "0.3" type Stackup struct { conf *Supfile @@ -42,8 +42,6 @@ func (sup *Stackup) Run(network *Network, commands ...*Command) error { env += `export ` + name + `="` + value + `";` } - var paddingLen int - // Create clients for every host (either SSH or Localhost). var ( clients []Client @@ -57,46 +55,44 @@ func (sup *Stackup) Run(network *Network, commands ...*Command) error { } } - for _, host := range network.Hosts { - var c Client - - if host == "localhost" { // LocalhostClient - + for i, host := range network.Hosts { + // Localhost client. + if host == "localhost" { local := &LocalhostClient{ env: env + `export SUP_HOST="` + host + `";`, } if err := local.Connect(host); err != nil { return err } + clients = append(clients, local) + continue + } - c = local - - } else { // SSHClient - - remote := &SSHClient{ - env: env + `export SUP_HOST="` + host + `";`, - } + // SSH client. + remote := &SSHClient{ + env: env + `export SUP_HOST="` + host + `";`, + color: Colors[i%len(Colors)], + } - var err error - if bastion != nil { - err = remote.ConnectWith(host, bastion.DialThrough) - } else { - err = remote.Connect(host) + if bastion != nil { + if err := remote.ConnectWith(host, bastion.DialThrough); err != nil { + return err } - if err != nil { + } else { + if err := remote.Connect(host); err != nil { return err } - defer remote.Close() - - c = remote } + defer remote.Close() + clients = append(clients, remote) + } - len := len(c.Prefix()) - if len > paddingLen { - paddingLen = len + maxLen := 0 + for _, c := range clients { + _, prefixLen := c.Prefix() + if prefixLen > maxLen { + maxLen = prefixLen } - - clients = append(clients, c) } // Run command or run multiple commands defined by target sequentially. @@ -104,24 +100,24 @@ func (sup *Stackup) Run(network *Network, commands ...*Command) error { // Translate command into task(s). tasks, err := CreateTasks(cmd, clients, env) if err != nil { - return fmt.Errorf("TasksFromConfigCommand(): %s", err) + return fmt.Errorf("CreateTasks(): %s", err) } - // Run tasks sequentally. + // Run tasks sequentially. for _, task := range tasks { var writers []io.Writer var wg sync.WaitGroup // Run tasks on the provided clients. - for i, c := range task.Clients { - padding := strings.Repeat(" ", paddingLen-(len(c.Prefix()))) - color := Colors[i%len(Colors)] - i++ - prefix := color + padding + c.Prefix() + " | " + for _, c := range task.Clients { + prefix, prefixLen := c.Prefix() + if len(prefix) < maxLen { // Left padding. + prefix = strings.Repeat(" ", maxLen-prefixLen) + prefix + } err := c.Run(task) if err != nil { - return fmt.Errorf("%sexit %v", prefix, err) + return fmt.Errorf("%s%v", prefix, err) } // Copy over tasks's STDOUT. @@ -132,7 +128,7 @@ func (sup *Stackup) Run(network *Network, commands ...*Command) error { if err != nil && err != io.EOF { // TODO: io.Copy() should not return io.EOF at all. // Upstream bug? Or prefixer.WriteTo() bug? - fmt.Fprintf(os.Stderr, "%sSTDOUT: %v", c.Prefix(), err) + fmt.Fprintf(os.Stderr, "%sSTDOUT: %v", prefix, err) } }(c) @@ -142,7 +138,7 @@ func (sup *Stackup) Run(network *Network, commands ...*Command) error { defer wg.Done() _, err := io.Copy(os.Stderr, prefixer.New(c.Stderr(), prefix)) if err != nil && err != io.EOF { - fmt.Fprintf(os.Stderr, "%sSTDERR: %v", c.Prefix(), err) + fmt.Fprintf(os.Stderr, "%sSTDERR: %v", prefix, err) } }(c) @@ -173,14 +169,16 @@ func (sup *Stackup) Run(network *Network, commands ...*Command) error { go func(c Client) { defer wg.Done() if err := c.Wait(); err != nil { + prefix, prefixLen := c.Prefix() + if len(prefix) < maxLen { // Left padding. + prefix = strings.Repeat(" ", maxLen-prefixLen) + prefix + } if e, ok := err.(*ssh.ExitError); ok && e.ExitStatus() != 15 { - // TODO: Prefix should be with color. // TODO: Store all the errors, and print them after Wait(). - fmt.Fprintf(os.Stderr, "%s | exit %v\n", c.Prefix(), e.ExitStatus()) + fmt.Fprintf(os.Stderr, "%sexit %v\n", prefix, e.ExitStatus()) os.Exit(e.ExitStatus()) } - // TODO: Prefix should be with color. - fmt.Fprintf(os.Stderr, "%s | %v\n", c.Prefix(), err) + fmt.Fprintf(os.Stderr, "%s%v\n", prefix, err) os.Exit(1) } }(c) diff --git a/supfile.go b/supfile.go index 7b21d18..1cde8f5 100644 --- a/supfile.go +++ b/supfile.go @@ -3,6 +3,7 @@ package sup import ( "bytes" "errors" + "fmt" "io" "io/ioutil" "os" @@ -12,7 +13,7 @@ import ( "gopkg.in/yaml.v2" ) -// Supfile represents the Stackup configuration YAML file. +// Supfile represents the Stack Up configuration YAML file. type Supfile struct { Networks map[string]Network `yaml:"networks"` Commands map[string]Command `yaml:"commands"` @@ -31,15 +32,18 @@ type Network struct { // Command represents command(s) to be run remotely. type Command struct { - Name string `yaml:"-"` // Command name. - Desc string `yaml:"desc"` // Command description. - Run string `yaml:"run"` // Command(s) to be run remotelly. - Script string `yaml:"script"` // Load command(s) from script and run it remotelly. - Upload []Upload `yaml:"upload"` // See below. - Stdin bool `yaml:"stdin"` // Attach localhost STDOUT to remote commands' STDIN? - Max int `yaml:"max"` // Max number of clients processing a task in parallel. - RunOnce bool `yaml:"run_once"` // The command should be run once only. - // TODO: RunSerial int `yaml:"run_serial"` // Max number of clients processing the command in parallel. + Name string `yaml:"-"` // Command name. + Desc string `yaml:"desc"` // Command description. + Local string `yaml:"local"` // Command(s) to be run locally. + Run string `yaml:"run"` // Command(s) to be run remotelly. + Script string `yaml:"script"` // Load command(s) from script and run it remotelly. + Upload []Upload `yaml:"upload"` // See Upload struct. + Stdin bool `yaml:"stdin"` // Attach localhost STDOUT to remote commands' STDIN? + Once bool `yaml:"once"` // The command should be run "once" (on one host only). + Serial int `yaml:"serial"` // Max number of clients processing a task in parallel. + + // API backward compatibility. Will be deprecated in v1.0. + RunOnce bool `yaml:"run_once"` // The command should be run once only. } // Upload represents file copy operation from localhost Src path to Dst @@ -62,15 +66,43 @@ func NewSupfile(file string) (*Supfile, error) { return nil, err } + // API backward compatibility. Will be deprecated in v1.0. switch conf.Version { - case "", "0.1": + case "": + conf.Version = "0.1" + fallthrough + case "0.1": for _, cmd := range conf.Commands { if cmd.RunOnce { - return nil, errors.New("command.run_once is not supported in Supfile version 0.1") + return nil, errors.New("command.run_once is not supported in Supfile v" + conf.Version) } } + fallthrough case "0.2": - // latest; skip + for _, cmd := range conf.Commands { + if cmd.Once { + return nil, errors.New("command.once is not supported in Supfile v" + conf.Version) + } + if cmd.Local != "" { + return nil, errors.New("command.local is not supported in Supfile v" + conf.Version) + } + if cmd.Serial != 0 { + return nil, errors.New("command.serial is not supported in Supfile v" + conf.Version) + } + } + for _, network := range conf.Networks { + if network.Inventory != "" { + return nil, errors.New("network.inventory is not supported in Supfile v" + conf.Version) + } + } + case "0.3": + for _, cmd := range conf.Commands { + if cmd.RunOnce { + fmt.Fprintf(os.Stderr, "Warning: command.run_once was deprecated by command.once in Supfile v"+conf.Version+"\n") + cmd.Once = cmd.RunOnce + break + } + } default: return nil, errors.New("unsupported version, please update sup by `go get -u github.com/pressly/sup`") } diff --git a/task.go b/task.go index c28e4d4..ec055c0 100644 --- a/task.go +++ b/task.go @@ -19,21 +19,32 @@ func CreateTasks(cmd *Command, clients []Client, env string) ([]*Task, error) { // Anything to upload? for _, upload := range cmd.Upload { - task := &Task{ + task := Task{ Run: RemoteTarCommand(upload.Dst), Input: NewTarStreamReader(upload.Src, upload.Exc, env), } - if cmd.RunOnce { + if cmd.Once { task.Clients = []Client{clients[0]} - tasks = append(tasks, task) + tasks = append(tasks, &task) + } else if cmd.Serial > 0 { + // Each "serial" task client group is executed sequentially. + for i := 0; i < len(clients); i += cmd.Serial { + j := i + cmd.Serial + if j > len(clients) { + j = len(clients) + } + copy := task + copy.Clients = clients[i:j] + tasks = append(tasks, ©) + } } else { task.Clients = clients - tasks = append(tasks, task) + tasks = append(tasks, &task) } } - // Script? Read the file as a multiline input command. + // Script. Read the file as a multiline input command. if cmd.Script != "" { f, err := os.Open(cmd.Script) if err != nil { @@ -44,37 +55,73 @@ func CreateTasks(cmd *Command, clients []Client, env string) ([]*Task, error) { return nil, err } - task := &Task{ + task := Task{ Run: string(data), } if cmd.Stdin { task.Input = os.Stdin } - - if cmd.RunOnce { + if cmd.Once { task.Clients = []Client{clients[0]} - tasks = append(tasks, task) + tasks = append(tasks, &task) + } else if cmd.Serial > 0 { + // Each "serial" task client group is executed sequentially. + for i := 0; i < len(clients); i += cmd.Serial { + j := i + cmd.Serial + if j > len(clients) { + j = len(clients) + } + copy := task + copy.Clients = clients[i:j] + tasks = append(tasks, ©) + } } else { task.Clients = clients - tasks = append(tasks, task) + tasks = append(tasks, &task) } } - // Command? - if cmd.Run != "" { + // Local command. + if cmd.Local != "" { + local := &LocalhostClient{ + env: env + `export SUP_HOST="localhost";`, + } + local.Connect("localhost") task := &Task{ - Run: cmd.Run, + Run: cmd.Local, + Clients: []Client{local}, } if cmd.Stdin { task.Input = os.Stdin } + tasks = append(tasks, task) + } - if cmd.RunOnce { + // Remote command. + if cmd.Run != "" { + task := Task{ + Run: cmd.Run, + } + if cmd.Stdin { + task.Input = os.Stdin + } + if cmd.Once { task.Clients = []Client{clients[0]} - tasks = append(tasks, task) + tasks = append(tasks, &task) + } else if cmd.Serial > 0 { + // Each "serial" task client group is executed sequentially. + for i := 0; i < len(clients); i += cmd.Serial { + j := i + cmd.Serial + if j > len(clients) { + j = len(clients) + } + copy := task + copy.Clients = clients[i:j] + tasks = append(tasks, ©) + } } else { task.Clients = clients - tasks = append(tasks, task) + tasks = append(tasks, &task) } }