Merge pull request #16 from mia-platform/feat/multi-pipeline-per-source

mia-platform · Nov 25, 2024 · 65240b3 · 65240b3
2 parents 3885a6e + b4d8582
commit 65240b3
Show file tree

Hide file tree

Showing 23 changed files with 454 additions and 1,047 deletions.
diff --git a/README.md b/README.md
@@ -3,12 +3,13 @@
 ## Development Local
 
 To develop the service locally you need:
-	- Go 1.23+
+
+- Go 1.23+
 
 To start the application locally
 
 ```go
-go run integration-connector-agent
+go run .
 ```
 
 By default the service will run on port 8080, to change the port please set `HTTP_PORT` env variable
@@ -20,6 +21,3 @@ To test the application use:
 ```go
 make test
 ```
-
-[logger]: https://github.com/mia-platform/glogger
-[library]: https://github.com/mia-platform/configlib
diff --git a/docs/10_overview.md b/docs/10_overview.md
@@ -17,7 +17,14 @@ real-time updates across all configured sinks.
 
 The following image shows the architecture of the Integration Connector Agent.
 
-![architecture](./img/architecture.svg)
+![architecture](./img/architecture.excalidraw.svg)
+
+### Data processing
+
+For each source, it is possible to configure different pipelines, each pipeline can have one or more
+processors and must have one or more sinks.
+
+![data-processing](./img/data-processing.excalidraw.svg)
 
 ### Use Cases
 
@@ -53,28 +60,33 @@ to save data into the collection `jira-issues`.
           }
         }
       },
-      "processors": [
+      "pipelines": [
         {
-          "type": "mapper",
-          "outputEvent": {
-            "key": "{{ issue.key }}",
-            "summary": "{{ issue.fields.summary }}",
-            "createdAt": "{{ issue.fields.created }}",
-            "description": "{{ issue.fields.description }}"
-          }
-        }
-      ],
-      "sinks": [
-        {
-          "type": "mongo",
-          "url": {
-            "fromEnv": "MONGO_URL"
-          },
-          "collection": "jira-issues"
+          "processors": [
+            {
+              "type": "mapper",
+              "outputEvent": {
+                "key": "{{ issue.key }}",
+                "summary": "{{ issue.fields.summary }}",
+                "createdAt": "{{ issue.fields.created }}",
+                "description": "{{ issue.fields.description }}"
+              }
+            }
+          ],
+          "sinks": [
+            {
+              "type": "mongo",
+              "url": {
+                "fromEnv": "MONGO_URL"
+              },
+              "collection": "jira-issues"
+            }
+          ]
         }
       ]
     }
   ]
 }
-</details>
 ```
+
+</details>
diff --git a/docs/20_install.md b/docs/20_install.md
@@ -42,24 +42,28 @@ The following is an example of a configuration for integrate source `Jira` with
           }
         },
       },
-      "processors": [
+      "pipelines": [
         {
-          "type": "mapper",
-          "outputEvent": {
-            "key": "{{ issue.key }}",
-            "summary": "{{ issue.fields.summary }}",
-            "createdAt": "{{ issue.fields.created }}",
-            "description": "{{ issue.fields.description }}"
-          }
-        }
-      ],
-      "sinks": [
-        {
-          "type": "mongo",
-          "url": {
-            "fromEnv": "TEST_LOAD_SERVICE_MONGO_URL"
-          },
-          "collection": "my-collection"
+          "processors": [
+            {
+              "type": "mapper",
+              "outputEvent": {
+                "key": "{{ issue.key }}",
+                "summary": "{{ issue.fields.summary }}",
+                "createdAt": "{{ issue.fields.created }}",
+                "description": "{{ issue.fields.description }}"
+              }
+            }
+          ],
+          "sinks": [
+            {
+              "type": "mongo",
+              "url": {
+                "fromEnv": "TEST_LOAD_SERVICE_MONGO_URL"
+              },
+              "collection": "my-collection"
+            }
+          ]
         }
       ]
     }