Merge pull request #1 from ConvertKit/spec-and-refactor-for-prime-time

CircleCI/Test + Refactor for official usage

.circleci/config.yml

@@ -15,6 +15,7 @@ jobs:
           keys:
             - v1-pkg-cache
 
+      - run: go get github.com/google/uuid
       - run:
           name: Run unit tests
           environment:

README.md

@@ -0,0 +1,26 @@
+# Stories
+
+Stories is a lightweight agent for StoryTeller logs.
+
+## Configuration
+
+Currently, the agent has a hardcoded dependency to Scalyr as a provider. This means that the agent will look for the `SCALYR_WRITE_TOKEN` Environment variable to be set. Other than that, `stories` can be configured with the following flags at startup.
+
+### `--buffer=int`
+_default: 1000_
+The buffer size is how many stories that the agent can store before it sends it in a batch to the provider.
+
+### `--interval=int`
+_default: 1_
+The interval, in seconds, before the agent batch the stories that it has buffered.
+
+
+### `--socket=string`
+_default: /tmp/stories.sock_
+The path where the unix socket will be created. This is used by libraries to send over events.
+
+#### Batching against an interval
+
+The way the agent works is that there is a runloop set to the `interval` set at runtime that will send in batch any stories that were collected during the interval.
+
+If, during the interval, the buffer reach its limit, the agent will trigger a batch send to the provider. If all default settings are set and you send twice the size of the buffer over the interval period, 2 batch will be sent to the provider.

integrations/integrations.go

@@ -2,8 +2,8 @@ package integrations
 
 import (
 	"errors"
-	"github.com/pothibo/stories/integrations/scalyr"
-	"github.com/pothibo/stories/stories"
+	"github.com/convertkit/stories/integrations/scalyr"
+	"github.com/convertkit/stories/stories"
 	"net/http"
 )
 

integrations/scalyr/event.go

@@ -2,7 +2,7 @@ package scalyr
 
 import (
 	"encoding/json"
-	"github.com/pothibo/stories/stories"
+	"github.com/convertkit/stories/stories"
 )
 
 type Event stories.Story
@@ -11,7 +11,18 @@ func (e *Event) MarshalJSON() ([]byte, error) {
 	data := make(map[string]interface{})
 	data["ts"] = e.Timestamp
 	data["sev"] = e.Severity
-	data["attrs"] = e.Data
+
+	attributes := e.Data
+
+	message, err := json.Marshal(e.Message)
+
+	if err != nil {
+		return nil, err
+	}
+
+	attributes["message"] = message
+
+	data["attrs"] = attributes
 
 	return json.Marshal(data)
 }

integrations/scalyr/event_test.go

@@ -1,9 +1,79 @@
 package scalyr
 
 import (
+	"encoding/json"
+	"github.com/convertkit/stories/stories"
 	"testing"
 )
 
-func TestEventJSONIsAcceptable(t *testing.T) {
+func story(t *testing.T) *stories.Story {
+	story, err := stories.NewStory([]byte(`{
+    "severity": 4,
+    "timestamp": "1541354132811",
+    "message": "Hello world!",
+    "data": {
+      "foo": {
+        "bar": "Something",
+        "yolo": true
+      },
+      "object_id": 1234,
+      "boolean": true,
+      "content": "Stuff"
+    }
+  }`))
 
+	if err != nil {
+		t.Fail()
+	}
+
+	return story
+}
+
+func payloadForEventTest(story *stories.Story, t *testing.T) map[string]interface{} {
+	event := Event(*story)
+	content, err := json.Marshal(&event)
+
+	if err != nil {
+		t.Fail()
+	}
+
+	var payload map[string]interface{}
+
+	err = json.Unmarshal(content, &payload)
+
+	if err != nil {
+		t.Error(err)
+		t.Fail()
+	}
+
+	return payload
+}
+
+func TestTimestampInJSON(t *testing.T) {
+	story := story(t)
+	payload := payloadForEventTest(story, t)
+
+	if payload["ts"] != string("1541354132811") {
+		t.Fail()
+	}
+}
+
+func TestSevInJSON(t *testing.T) {
+	story := story(t)
+	payload := payloadForEventTest(story, t)
+
+	if payload["sev"].(float64) != float64(4) {
+		t.Fail()
+	}
+}
+
+func TestMessageInAttributesJSON(t *testing.T) {
+	story := story(t)
+	payload := payloadForEventTest(story, t)
+
+	attributes := payload["attrs"].(map[string]interface{})
+
+	if attributes["message"] != string("Hello world!") {
+		t.Fail()
+	}
 }

integrations/scalyr/instance.go

@@ -4,8 +4,8 @@ import (
 	"bytes"
 	"encoding/json"
 	"errors"
+	"github.com/convertkit/stories/stories"
 	"github.com/google/uuid"
-	"github.com/pothibo/stories/stories"
 	"log"
 	"net/http"
 	"os"
@@ -34,7 +34,7 @@ func (i *Instance) Configure() error {
 
 	i.SessionInfo = make(map[string]string)
 	i.SessionInfo["logfile"] = "stories"
-	i.SessionInfo["serverHost"] = "test"
+	i.SessionInfo["serverHost"] = os.Getenv("RACK_ENV")
 
 	i.Url = "https://www.scalyr.com/addEvents"
 

integrations/scalyr/instance_test.go

@@ -1,11 +1,14 @@
 package scalyr
 
 import (
+	"os"
 	"strings"
 	"testing"
 )
 
 func TestConfigureInstanceGenerateASessionUUID(t *testing.T) {
+	os.Setenv("SCALYR_WRITE_TOKEN", "test")
+
 	instance := &Instance{}
 	err := instance.Configure()
 

integrations/scalyr/payload.go

@@ -2,7 +2,7 @@ package scalyr
 
 import (
 	"encoding/json"
-	"github.com/pothibo/stories/stories"
+	"github.com/convertkit/stories/stories"
 )
 
 type Payload struct {

integrations/scalyr/payload_test.go

@@ -0,0 +1,71 @@
+package scalyr
+
+import (
+	"encoding/json"
+	"github.com/convertkit/stories/stories"
+	"os"
+	"testing"
+)
+
+func instanceForInstanceTest(t *testing.T) *Instance {
+	os.Setenv("SCALYR_WRITE_TOKEN", "test")
+
+	instance := &Instance{}
+	err := instance.Configure()
+
+	if err != nil {
+		t.Error(err)
+		t.Fail()
+	}
+
+	return instance
+}
+
+func payloadForInstanceTest(t *testing.T) *Payload {
+	story, err := stories.NewStory([]byte(`{
+    "severity": 4,
+    "timestamp": "1541354132811",
+    "message": "Hello world!",
+    "data": {
+      "foo": {
+        "bar": "Something",
+        "yolo": true
+      },
+      "object_id": 1234,
+      "boolean": true,
+      "content": "Stuff"
+    }
+  }`))
+
+	if err != nil {
+		t.Fail()
+	}
+
+	stories := []*stories.Story{story}
+	return NewPayload(instanceForInstanceTest(t), stories)
+}
+
+func TestTokenPresentInJSON(t *testing.T) {
+	payload := payloadForInstanceTest(t)
+	body, err := json.Marshal(&payload)
+
+	if err != nil {
+		t.Error(err)
+	}
+
+	var data map[string]interface{}
+
+	err = json.Unmarshal(body, &data)
+
+	if err != nil {
+		t.Error(err)
+	}
+
+	if data["token"] == nil {
+		t.Fail()
+	}
+
+	if data["token"] != payload.Token {
+		t.Fail()
+	}
+}

main.go

@@ -2,8 +2,8 @@ package main
 
 import (
 	"flag"
-	"github.com/pothibo/stories/integrations"
-	"github.com/pothibo/stories/stories"
+	"github.com/convertkit/stories/integrations"
+	"github.com/convertkit/stories/stories"
 	"log"
 	"net"
 	"os"

stories/queue.go

@@ -59,11 +59,11 @@ func (q *Queue) Collect() []*Story {
 	return stories
 }
 
-func (q *Queue) Size() int {
+func (q *Queue) Capacity() int {
 	return cap(q.channel)
 }
 
-func (q *Queue) InQueue() int {
+func (q *Queue) Size() int {
 	return len(q.channel)
 }
 
@@ -72,7 +72,7 @@ func (q *Queue) IsEmpty() bool {
 }
 
 func (q *Queue) IsFull() bool {
-	return q.InQueue() == q.Size()
+	return q.Size() >= q.Capacity()
 }
 
 func read(c net.Conn) ([]byte, int, error) {

stories/queue_test.go

@@ -2,22 +2,77 @@ package stories
 
 import (
 	"testing"
+  "net"
 )
 
 func TestQueueInitializedWithSize(t *testing.T) {
 	size := 100
 	queue := NewQueueOfSize(size)
 
-	if queue.Size() != size {
+	if queue.Capacity() != size {
 		t.Fail()
 	}
 }
 
 func TestCollectingWillClearTheQueue(t *testing.T) {
+  size := 10
+	queue := NewQueueOfSize(size)
+
+  for i := 0; i < size; i++ {
+    server, client := net.Pipe()
+    go func() {
+      server.Write(storyJSON())
+      server.Close()
+    }()
+
+    err := queue.Add(client)
+
+    if err != nil {
+      t.Fatal(err)
+    }
+  }
+
+  if queue.IsEmpty() {
+    t.Fatal("Queue should not have been empty")
+  }
+
+  if len(queue.Collect()) != size {
+    t.Fatalf("Wanted a queue of %d. Got %d", queue.Capacity(), size)
+  }
+
+  if queue.IsEmpty() != true {
+    t.Fatalf("Queue should have been empty, has a size of %d", queue.Capacity())
+  }
 }
 
 func TestQueueIsFullAtMaxSize(t *testing.T) {
+  size := 10
+	queue := NewQueueOfSize(size)
+
+  for i := 0; i < size; i++ {
+    server, client := net.Pipe()
+    go func() {
+      server.Write(storyJSON())
+      server.Close()
+    }()
+
+    err := queue.Add(client)
+
+    if err != nil {
+      t.Fatal(err)
+    }
+  }
+
+  if queue.IsFull() != true {
+    t.Fatal("Queue should have been full")
+  }
 }
 
 func TestEmptyQueue(t *testing.T) {
+  size := 10
+	queue := NewQueueOfSize(size)
+
+  if queue.IsEmpty() != true {
+    t.Fatal("Queue should have been empty")
+  }
 }