From da08df321a62edd39080a912c6ee30671a868b26 Mon Sep 17 00:00:00 2001 From: mithun <51120171+MithunKinarullathil@users.noreply.github.com> Date: Thu, 12 May 2022 13:52:13 +0100 Subject: [PATCH 01/28] Remov exposed 11311, as it is curretly stoping user from running roscore outside Better healthcheck for roscore container --- docker-compose.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index f38f99e..ce5a9bc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -172,10 +172,8 @@ services: options: mode: "non-blocking" max-buffer-size: "4m" - ports: - - "${STUDIO_ROS_MASTER_PORT:-11311}:11311" healthcheck: - test: ["CMD", "echo", ">", "/dev/tcp/localhost/11311"] + test: ["CMD", "rostopic", "list", "|", "grep", "-w", "'/rosout'", ">>", "/dev/null"] start_period: 3s interval: 10s timeout: 2s From 277046826b5a2af0a69416a20ac907f988be64c4 Mon Sep 17 00:00:00 2001 From: mithun <51120171+MithunKinarullathil@users.noreply.github.com> Date: Thu, 12 May 2022 14:33:17 +0100 Subject: [PATCH 02/28] Add sonarqube quality gate status badge --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 2a0a0b4..4af5de5 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ Official Release Pre Release PR Checks + Quality Gate Status Twitter

From 2619032c9df97e3f3722bd8775ec722fe11e801f Mon Sep 17 00:00:00 2001 From: Mithun Kinarullathil <51120171+MithunKinarullathil@users.noreply.github.com> Date: Fri, 13 May 2022 10:11:58 +0100 Subject: [PATCH 03/28] Fix duplicated items in issue template --- .github/ISSUE_TEMPLATE.md | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 73261e7..6ad506c 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -22,16 +22,6 @@ Have you checked our docs before proceeding? -### Versions - -- movai-flow: - -- movai-ide: - -### Possible workarounds - -- - ### Versions - movai-flow: From ee3ca1d795b32ab5c928fb9b725e234b82463e6c Mon Sep 17 00:00:00 2001 From: mithun <51120171+MithunKinarullathil@users.noreply.github.com> Date: Fri, 13 May 2022 11:18:31 +0100 Subject: [PATCH 04/28] Update links to flow components --- README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 4af5de5..492c85d 100644 --- a/README.md +++ b/README.md @@ -138,14 +138,14 @@ Installing the MOV.AI Flow™ package will provide a few command line tools to c MOV.AI Flow™ initiates a set of services running as a *docker-compose* cluster, namely: - - redis-master: master DB of the cluster - - redis-local: local DB of the cluster - - backend: web service application - - ros-master: ROS core service - - spawner: Flow orchestrator of ROS nodes - - haproxy: web proxy - - simulator: Ignition Fortress container with graphical capabilities enabled - - ros-tools: rviz container with graphical capabilities enabled + - [redis-master](https://github.com/MOV-AI/containers-redis2): Master DB of the cluster + - [redis-local](https://github.com/MOV-AI/containers-redis2): Local DB of the cluster + - [backend](https://github.com/MOV-AI/backend): Web service application + - [ros-master](https://github.com/MOV-AI/containers-ros-master): ROS core service + - [spawner](https://github.com/MOV-AI/containers-spawner-base): Flow orchestrator of ROS and MOV.AI nodes + - [haproxy](https://github.com/MOV-AI/containers-haproxy): Web proxy + - [simulator](https://github.com/MOV-AI/containers-ign-simulator): Ignition Fortress container with graphical capabilities enabled + - [ros-tools](https://github.com/MOV-AI/containers-ros-tools): RVIZ container with graphical capabilities enabled > Some directories in the containers are mounted, which means that their contents are synchronized between your computer (host) and the container. From 777011c66388b1c3947c5583c65d10a5d9f787e6 Mon Sep 17 00:00:00 2001 From: mithun <51120171+MithunKinarullathil@users.noreply.github.com> Date: Tue, 7 Jun 2022 09:42:19 +0100 Subject: [PATCH 05/28] Mention movai-developer-tools in README --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 4af5de5..463e7e4 100644 --- a/README.md +++ b/README.md @@ -266,3 +266,6 @@ The best way to do it is to: To stop and delete containers, delete volumes with database data and downloaded images, run: docker-compose down --volumes --rmi all + +## MOV.AI Developer Tools +A python package build to facilitate development when using MOV.AI. The tool can be used to, but not limited to, expose the ros topics to the host as well as export/import MOV.AI metadata. Please follow [this link](https://github.com/MOV-AI/movai-developer-tools) for more detailed documentation. From 719b1c009fb1d9cb88c4fdd764008caac0b02cf1 Mon Sep 17 00:00:00 2001 From: Lucia Carreira Date: Mon, 13 Jun 2022 16:08:47 +0100 Subject: [PATCH 06/28] Changing the saturation of red on the documentation template to make it more comfortable to the eyes. --- haproxy/503-movai.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/haproxy/503-movai.css b/haproxy/503-movai.css index 9771f71..ff17805 100644 --- a/haproxy/503-movai.css +++ b/haproxy/503-movai.css @@ -32,7 +32,7 @@ body { } .info-container h1 { - color: red; + color: rgb(220, 0, 0); text-align: left; } From 0fe2e58feec1016da3a0c192d85e76013d21d131 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 Jun 2022 14:02:16 +0100 Subject: [PATCH 07/28] Bump isbang/compose-action from 1.0.0 to 1.1.0 (#160) * Update changelog for 1.2.0-4 release * Update changelog for 1.2.0-5 release * Update changelog for 1.2.0-6 release * Bump isbang/compose-action from 1.0.0 to 1.1.0 Bumps [isbang/compose-action](https://github.com/isbang/compose-action) from 1.0.0 to 1.1.0. - [Release notes](https://github.com/isbang/compose-action/releases) - [Commits](https://github.com/isbang/compose-action/compare/v1.0.0...v1.1.0) --- updated-dependencies: - dependency-name: isbang/compose-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: Mithun Kinarullathil <51120171+MithunKinarullathil@users.noreply.github.com> Co-authored-by: OttoMation-Movai Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/TestOnPR.yml | 2 +- debian/changelog | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/TestOnPR.yml b/.github/workflows/TestOnPR.yml index 9290ba5..a7273e1 100644 --- a/.github/workflows/TestOnPR.yml +++ b/.github/workflows/TestOnPR.yml @@ -50,7 +50,7 @@ jobs: sudo chown 1000:1000 ./{logs,shared,userspace,models_database} -R - name: run docker-compose cluster - uses: isbang/compose-action@v1.0.0 + uses: isbang/compose-action@v1.1.0 with: compose-file: './docker-compose.yml' down-flags: '--volumes' diff --git a/debian/changelog b/debian/changelog index 913c4f5..cf5d36b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -movai-flow (1.2.0-5) UNRELEASED; urgency=medium +movai-flow (1.2.0-6) UNRELEASED; urgency=medium [ Alex Fernandes ] * @@ -66,5 +66,6 @@ movai-flow (1.2.0-5) UNRELEASED; urgency=medium * UNRELEASED * UNRELEASED * UNRELEASED + * UNRELEASED - -- root Mon, 09 May 2022 17:42:34 +0000 + -- root Wed, 11 May 2022 14:35:41 +0000 From c13339d4c7120ce10c86fdc03ec0e1853e9ded3e Mon Sep 17 00:00:00 2001 From: Mithun Kinarullathil <51120171+MithunKinarullathil@users.noreply.github.com> Date: Wed, 15 Jun 2022 08:21:43 +0100 Subject: [PATCH 08/28] Update haproxy link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0da70f0..7322b2c 100644 --- a/README.md +++ b/README.md @@ -143,7 +143,7 @@ MOV.AI Flow™ initiates a set of services running as a *docker-compose* cluster - [backend](https://github.com/MOV-AI/backend): Web service application - [ros-master](https://github.com/MOV-AI/containers-ros-master): ROS core service - [spawner](https://github.com/MOV-AI/containers-spawner-base): Flow orchestrator of ROS and MOV.AI nodes - - [haproxy](https://github.com/MOV-AI/containers-haproxy): Web proxy + - [haproxy](https://github.com/docker-library/haproxy): Web proxy - [simulator](https://github.com/MOV-AI/containers-ign-simulator): Ignition Fortress container with graphical capabilities enabled - [ros-tools](https://github.com/MOV-AI/containers-ros-tools): RVIZ container with graphical capabilities enabled From 50f7f585746ece66f317b62e41516b8682937d1b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 13 Jul 2022 10:09:01 +0100 Subject: [PATCH 09/28] Bump isbang/compose-action from 1.0.0 to 1.1.0 (#163) Bumps [isbang/compose-action](https://github.com/isbang/compose-action) from 1.0.0 to 1.1.0. - [Release notes](https://github.com/isbang/compose-action/releases) - [Commits](https://github.com/isbang/compose-action/compare/v1.0.0...v1.1.0) --- updated-dependencies: - dependency-name: isbang/compose-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> From b1207454cf20a723c4b77994661e9540ce08e31d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 21 Sep 2022 12:02:07 +0100 Subject: [PATCH 10/28] Bump isbang/compose-action from 1.1.0 to 1.2.0 (#170) Bumps [isbang/compose-action](https://github.com/isbang/compose-action) from 1.1.0 to 1.2.0. - [Release notes](https://github.com/isbang/compose-action/releases) - [Commits](https://github.com/isbang/compose-action/compare/v1.1.0...v1.2.0) --- updated-dependencies: - dependency-name: isbang/compose-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/TestOnPR.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/TestOnPR.yml b/.github/workflows/TestOnPR.yml index a7273e1..0511aca 100644 --- a/.github/workflows/TestOnPR.yml +++ b/.github/workflows/TestOnPR.yml @@ -50,7 +50,7 @@ jobs: sudo chown 1000:1000 ./{logs,shared,userspace,models_database} -R - name: run docker-compose cluster - uses: isbang/compose-action@v1.1.0 + uses: isbang/compose-action@v1.2.0 with: compose-file: './docker-compose.yml' down-flags: '--volumes' From 9f4b80c6ca4c0810538ac1696c84b418a8c3721d Mon Sep 17 00:00:00 2001 From: OttoMation-Movai Date: Wed, 21 Sep 2022 13:20:40 +0100 Subject: [PATCH 11/28] hotfix : ros-master health-check --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index ce5a9bc..661e4b3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -173,7 +173,7 @@ services: mode: "non-blocking" max-buffer-size: "4m" healthcheck: - test: ["CMD", "rostopic", "list", "|", "grep", "-w", "'/rosout'", ">>", "/dev/null"] + test: ["CMD-SHELL", ". /opt/ros/noetic/setup.sh && /opt/ros/noetic/bin/rostopic list | grep -w '/rosout' >> /dev/null"] start_period: 3s interval: 10s timeout: 2s From 7f494060e28ea740d08f7d5b1c801b55206abed1 Mon Sep 17 00:00:00 2001 From: OttoMation-Movai Date: Wed, 21 Sep 2022 13:29:51 +0100 Subject: [PATCH 12/28] fix cicd : update pack image --- .github/workflows/DeployOnMergeMain.yml | 2 +- .github/workflows/TestOnPR.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/DeployOnMergeMain.yml b/.github/workflows/DeployOnMergeMain.yml index 60d5c87..99131eb 100644 --- a/.github/workflows/DeployOnMergeMain.yml +++ b/.github/workflows/DeployOnMergeMain.yml @@ -12,7 +12,7 @@ jobs: Deploy-App-Internal-Release: runs-on: ubuntu-20.04 container: - image: registry.cloud.mov.ai/devops/ros-buildtools-noetic:main + image: registry.cloud.mov.ai/qa/ros-buildtools-noetic:v1.0.14 options: --user root credentials: username: ${{secrets.PORTUS_APP_USER}} diff --git a/.github/workflows/TestOnPR.yml b/.github/workflows/TestOnPR.yml index 0511aca..8b1cb37 100644 --- a/.github/workflows/TestOnPR.yml +++ b/.github/workflows/TestOnPR.yml @@ -82,7 +82,7 @@ jobs: name: "Pack Checks" runs-on: ubuntu-20.04 container: - image: registry.cloud.mov.ai/devops/ros-buildtools-noetic:main + image: registry.cloud.mov.ai/qa/ros-buildtools-noetic:v1.0.14 options: --user root credentials: username: ${{secrets.PORTUS_APP_USER}} From cda3185c3292aadfe371cc18aa642cee82961e86 Mon Sep 17 00:00:00 2001 From: OttoMation-Movai Date: Wed, 21 Sep 2022 13:37:02 +0100 Subject: [PATCH 13/28] cicd: fix git issue --- .github/workflows/DeployOnMergeMain.yml | 1 + .github/workflows/TestOnPR.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/DeployOnMergeMain.yml b/.github/workflows/DeployOnMergeMain.yml index 99131eb..448b32a 100644 --- a/.github/workflows/DeployOnMergeMain.yml +++ b/.github/workflows/DeployOnMergeMain.yml @@ -42,6 +42,7 @@ jobs: - name: Pack debian run: | + git config --global --add safe.directory $(pwd) git config --global user.name ${{ secrets.RAISE_BOT_COMMIT_USER }} git config --global user.email ${{ secrets.RAISE_BOT_COMMIT_MAIL }} diff --git a/.github/workflows/TestOnPR.yml b/.github/workflows/TestOnPR.yml index 8b1cb37..715a202 100644 --- a/.github/workflows/TestOnPR.yml +++ b/.github/workflows/TestOnPR.yml @@ -111,6 +111,7 @@ jobs: - name: Pack debian run: | + git config --global --add safe.directory $(pwd) git config --global user.name '${{ secrets.RAISE_BOT_COMMIT_USER }}' git config --global user.email '${{ secrets.RAISE_BOT_COMMIT_MAIL }}' From e1e3e94a54d5156b0306c48306b42fa2db25f8d2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Sep 2022 14:30:11 +0100 Subject: [PATCH 14/28] Bump isbang/compose-action from 1.1.0 to 1.2.0 (#172) Bumps [isbang/compose-action](https://github.com/isbang/compose-action) from 1.1.0 to 1.2.0. - [Release notes](https://github.com/isbang/compose-action/releases) - [Commits](https://github.com/isbang/compose-action/compare/v1.1.0...v1.2.0) --- updated-dependencies: - dependency-name: isbang/compose-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> From b14fa63c3c00999281cf14da340445b1ca9482c7 Mon Sep 17 00:00:00 2001 From: Alex Fernandes <84147256+AlexFernandes-MOVAI@users.noreply.github.com> Date: Wed, 28 Sep 2022 10:38:14 +0100 Subject: [PATCH 15/28] Feat : update all docker images (#173) * update all docker images * update backend * update spawner * revert backend not to bring new devs * fix haproxy version * Bump isbang/compose-action from 1.1.0 to 1.2.0 (#174) Bumps [isbang/compose-action](https://github.com/isbang/compose-action) from 1.1.0 to 1.2.0. - [Release notes](https://github.com/isbang/compose-action/releases) - [Commits](https://github.com/isbang/compose-action/compare/v1.1.0...v1.2.0) --- updated-dependencies: - dependency-name: isbang/compose-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- docker-compose.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 661e4b3..4144fb6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -84,7 +84,7 @@ x-movai-common: services: redis-master: - image: ${MOVAI_REGISTRY:-pubregistry.aws.cloud.mov.ai}/ce/redis2:v1.0.6 + image: ${MOVAI_REGISTRY:-pubregistry.aws.cloud.mov.ai}/ce/redis2:v1.0.10 container_name: redis-master-$MOVAI_DEVICE_NAME logging: driver: "journald" @@ -109,7 +109,7 @@ services: restart: unless-stopped redis-local: - image: ${MOVAI_REGISTRY:-pubregistry.aws.cloud.mov.ai}/ce/redis2:v1.0.6 + image: ${MOVAI_REGISTRY:-pubregistry.aws.cloud.mov.ai}/ce/redis2:v1.0.10 container_name: redis-local-$MOVAI_DEVICE_NAME logging: driver: "journald" @@ -159,7 +159,7 @@ services: restart: unless-stopped ros-master: - image: ${MOVAI_REGISTRY:-pubregistry.aws.cloud.mov.ai}/ce/ros-master-noetic:v1.0.0 + image: ${MOVAI_REGISTRY:-pubregistry.aws.cloud.mov.ai}/ce/ros-master-noetic:v1.0.2 privileged: true networks: - private @@ -182,7 +182,7 @@ services: spawner: <<: *movai-common - image: ${MOVAI_REGISTRY:-pubregistry.aws.cloud.mov.ai}/ce/spawner-ign-noetic:v3.0.5 + image: ${MOVAI_REGISTRY:-pubregistry.aws.cloud.mov.ai}/ce/spawner-ign-noetic:v3.0.9 privileged: true networks: - private @@ -218,7 +218,7 @@ services: restart: unless-stopped simulator: - image: ${MOVAI_REGISTRY:-pubregistry.aws.cloud.mov.ai}/ce/ignition-gazebo:v1.0.4 + image: ${MOVAI_REGISTRY:-pubregistry.aws.cloud.mov.ai}/ce/ignition-gazebo:v1.0.5 container_name: simulator-$MOVAI_DEVICE_NAME logging: driver: "journald" @@ -252,7 +252,7 @@ services: command: ign gui -s $STUDIO_SIMU_GUI ros-tools: - image: ${MOVAI_REGISTRY:-pubregistry.aws.cloud.mov.ai}/ce/ros-tools-ce:v1.0.0 + image: ${MOVAI_REGISTRY:-pubregistry.aws.cloud.mov.ai}/ce/ros-tools-ce:v1.0.4 container_name: ros-tools-$MOVAI_DEVICE_NAME logging: driver: "journald" @@ -280,7 +280,7 @@ services: exec /usr/bin/rviz haproxy: - image: ${MOVAI_REGISTRY:-pubregistry.aws.cloud.mov.ai}/ce/haproxy:2.4.15 + image: ${MOVAI_REGISTRY:-pubregistry.aws.cloud.mov.ai}/ce/haproxy:v1.0.7 container_name: haproxy-$MOVAI_DEVICE_NAME logging: driver: "journald" From f7a5822179be6b4077424df1d1682fc137a33718 Mon Sep 17 00:00:00 2001 From: Alex Fernandes <84147256+AlexFernandes-MOVAI@users.noreply.github.com> Date: Mon, 3 Oct 2022 15:10:23 +0100 Subject: [PATCH 16/28] DP-806 : add support for host network usage (#176) * update all docker images * update backend * update spawner * revert backend not to bring new devs * fix haproxy version * add config for cluster in host mode * fix lint * revert fe app bump * fixes fro NVIDIA and ports * fix test * review HTTP port to always match 8080 * fix switch between modes needing always a reset of containers and networks * add warning * review redis containers * use var instead of hardcoded value * Update .yamllint-config.yml --- .env | 8 +- .github/workflows/TestOnPR.yml | 98 +++++++++-- .gitignore | 2 + .yamllint-config.yml | 4 +- debian/install | 3 + debian/movai-flow.postinst | 31 ++-- debian/movai-flow.prerm | 3 +- docker-compose-host.yml | 299 +++++++++++++++++++++++++++++++++ docker-compose-nvidia-host.yml | 38 +++++ docker-compose-nvidia.yml | 2 +- docker-compose.yml | 29 ++-- haproxy/haproxy.cfg | 12 +- scripts/movai-flow-launch | 96 +++++++++-- scripts/movai-flow-reset | 58 +++++++ scripts/movai-flow-rviz | 58 +++++-- scripts/movai-flow-simulator | 55 ++++-- scripts/movai-flow-stop | 32 +++- 17 files changed, 732 insertions(+), 96 deletions(-) create mode 100644 docker-compose-host.yml create mode 100644 docker-compose-nvidia-host.yml create mode 100755 scripts/movai-flow-reset diff --git a/.env b/.env index e32dc26..23f232e 100644 --- a/.env +++ b/.env @@ -1,7 +1,8 @@ -STUDIO_WEB_PORT=5004 +STUDIO_HTTP_PORT=8080 +STUDIO_HTTPS_PORT=8083 STUDIO_DB_PACKAGES= STUDIO_APP_PACKAGES="movai-launcher=2.2.2-2,movai-ide-ce=3.0.0-7" -STUDIO_SPAWNER_PACKAGES="ros-noetic-movai-ce-demos=1.0.0-8,ros-noetic-movai-ports-and-messages-ce=1.0.0-1" +STUDIO_SPAWNER_PACKAGES="flow-initiator=1.0.1.9,ros-noetic-movai-ce-demos=1.0.0-8,ros-noetic-movai-ports-and-messages-ce=1.0.0-1" STUDIO_SIMU_PLUGIN_PACKAGES="movai-ign-plugin-world-launcher=1.0.1-4" STUDIO_SIMU_GUI="WorldLauncher" MOVAI_DEVICE_NAME=robot1 @@ -11,9 +12,10 @@ JWT_SECRET_KEY=m0va1f10wl0v3s0p3n50rc3@2022 FLEET_NAME=ce LOCAL_IP=0.0.0.0 PUBLIC_IP=127.0.0.1 -MANAGER_URI=http://localhost:8080 SPAWNER_GROUPS= EXTERNAL_HOST_IP= IGN_PARTITION="movai_ce_flow" XAUTH=/tmp/.docker.xauth DISPLAY=${DISPLAY:-:1} +REDIS_LOCAL_PORT=6380 +REDIS_MASTER_PORT=6379 \ No newline at end of file diff --git a/.github/workflows/TestOnPR.yml b/.github/workflows/TestOnPR.yml index 715a202..3df1c04 100644 --- a/.github/workflows/TestOnPR.yml +++ b/.github/workflows/TestOnPR.yml @@ -9,9 +9,9 @@ on: - 'debian/changelog' jobs: - Test-Project: + Lint-Project: timeout-minutes: 30 - name: "Sanity checks" + name: "Linting checks" runs-on: ubuntu-20.04 steps: - name: Checkout @@ -24,13 +24,20 @@ jobs: - name: Yamllint uses: karancode/yamllint-github-action@master with: - yamllint_file_or_dir: docker-compose.yml docker-compose-nvidia.yml + yamllint_file_or_dir: docker-compose.yml docker-compose-nvidia.yml docker-compose-host.yml docker-compose-nvidia-host.yml yamllint_strict: false yamllint_comment: true yamllint_config_filepath: .yamllint-config.yml env: GITHUB_ACCESS_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: run docker-compose lint + run: | + docker-compose -f docker-compose.yml config + docker-compose -f docker-compose-host.yml config + docker-compose -f docker-compose-nvidia.yml config + docker-compose -f docker-compose-nvidia-host.yml config + # - name: Validate json # uses: anyone-developer/anyone-validate-json@main # with: @@ -39,10 +46,18 @@ jobs: # ignore-directories: '.git' # read-path: 'robots' - - name: run docker-compose lint + Test-Project: + timeout-minutes: 30 + name: "Cluster sanity checks" + runs-on: ubuntu-20.04 + needs: [ Lint-Project ] + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: install requirements run: | - docker-compose -f docker-compose.yml config - docker-compose -f docker-compose-nvidia.yml config + python3 -m pip install docker-compose - name: prepare docker-compose cluster run: | @@ -57,7 +72,10 @@ jobs: - name: test cluster frontend run: | - export token_url=http://localhost:8080/token-auth/ + export base_url="http://localhost:8080" + export token_url="$base_url/token-auth/" + export authenticated_url="$base_url/api/v1/User/movai/?" + # retry for 1 minute count=10 set +e @@ -69,18 +87,72 @@ jobs: echo "Waiting for MOV.AI Flow™ availability (ret:$status_code)" sleep 10 done - [ $status_code == 200 ] && exit 0 + [ $status_code == 200 ] || exit -1 + + err_status_code=$(curl "$authenticated_url" -k -s -o /dev/null -w "%{http_code}") + [ $err_status_code == 401 ] || exit -1 + + json_token=$(curl -k -s --cookie-jar cookies --header 'Content-Type: application/json' -X POST --data '{"username":"movai","password":"movai","remember":true}' $token_url) + token=$( jq -r ".access_token" <<<"$json_token" ) + ok_status_code=$(curl "$authenticated_url" -k -s -o /dev/null -H "Authorization: Bearer $token" -w "%{http_code}") + [ $ok_status_code == 200 ] || exit -1 + + Test-Project-HostMode: + timeout-minutes: 30 + name: "Host Mode Cluster checks" + runs-on: ubuntu-20.04 + needs: [ Lint-Project ] + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: install requirements + run: | + python3 -m pip install docker-compose + + - name: prepare docker-compose cluster + run: | + mkdir -p ./{logs,shared,userspace,models_database} || true + sudo chown 1000:1000 ./{logs,shared,userspace,models_database} -R + + - name: run docker-compose cluster in host mode + uses: isbang/compose-action@v1.2.0 + with: + compose-file: './docker-compose-host.yml' + down-flags: '--volumes' + + - name: test cluster frontend in host mode + run: | + export base_url="http://localhost:8080" + export token_url="$base_url/token-auth/" + export authenticated_url="$base_url/api/v1/User/movai/?" + + # retry for 1 minute + count=10 + set +e + for i in $(seq $count); do + status_code=$(curl -k -s -o /dev/null --cookie-jar cookies --header 'Content-Type: application/json' -X POST --data '{"username":"movai","password":"movai","remember":true}' -w "%{http_code}" $token_url) + if [ $status_code == 200 ]; then + break + fi + echo "Waiting for MOV.AI Flow™ availability (ret:$status_code)" + sleep 10 + done + [ $status_code == 200 ] || exit -1 + + err_status_code=$(curl "$authenticated_url" -k -s -o /dev/null -w "%{http_code}") + [ $err_status_code == 401 ] || exit -1 - export url=http://localhost:8080/api/v1/apps/mov-fe-app-ide-ce/ - #export url=https://localhost:8443/api/v1/apps/mov-fe-app-ide-ce/ - json=$( status_code=$(curl -k -s --cookie-jar cookies --header 'Content-Type: application/json' -X POST --data '{"username":"movai","password":"movai","remember":true}' -w "%{http_code}" $token_url)' ) - token=$( jq -r ".access_token" <<<"$json" ) - curl "$url" -k -X POST -H "X-Cookie:token=$token" + json_token=$(curl -k -s --cookie-jar cookies --header 'Content-Type: application/json' -X POST --data '{"username":"movai","password":"movai","remember":true}' $token_url) + token=$( jq -r ".access_token" <<<"$json_token" ) + ok_status_code=$(curl "$authenticated_url" -k -s -o /dev/null -H "Authorization: Bearer $token" -w "%{http_code}") + [ $ok_status_code == 200 ] || exit -1 Pack-Project: timeout-minutes: 30 name: "Pack Checks" runs-on: ubuntu-20.04 + needs: [ Lint-Project ] container: image: registry.cloud.mov.ai/qa/ros-buildtools-noetic:v1.0.14 options: --user root diff --git a/.gitignore b/.gitignore index 903875d..e8dd529 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,5 @@ debian/files debian/movai-flow.debhelper.log debian/movai-flow.substvars debian/.debhelper/generated/movai-flow/installed-by-dh_install +*.code-workspace +config/config.env diff --git a/.yamllint-config.yml b/.yamllint-config.yml index 28a238f..35a6050 100644 --- a/.yamllint-config.yml +++ b/.yamllint-config.yml @@ -3,7 +3,7 @@ extends: default rules: # 120 chars should be enough, but don't fail if a line is longer line-length: - max: 120 + max: 132 level: warning # accept both key: @@ -12,4 +12,4 @@ rules: # and key: # - item indentation: - indent-sequences: whatever \ No newline at end of file + indent-sequences: whatever diff --git a/debian/install b/debian/install index c5234e4..e9a6c01 100644 --- a/debian/install +++ b/debian/install @@ -3,11 +3,14 @@ rviz usr/share/movai-flow models_database usr/share/movai-flow docker-compose.yml usr/share/movai-flow docker-compose-nvidia.yml usr/share/movai-flow +docker-compose-host.yml usr/share/movai-flow +docker-compose-nvidia-host.yml usr/share/movai-flow .env usr/share/movai-flow scripts/movai-flow-simulator usr/bin/ scripts/movai-flow-rviz usr/bin/ scripts/movai-flow-launch usr/bin/ scripts/movai-flow-stop usr/bin/ +scripts/movai-flow-reset usr/bin/ misc/movai-flow.desktop usr/share/applications misc/movai-flow-sim.desktop usr/share/applications misc/movai-flow-rviz.desktop usr/share/applications diff --git a/debian/movai-flow.postinst b/debian/movai-flow.postinst index 1848391..2aa5794 100644 --- a/debian/movai-flow.postinst +++ b/debian/movai-flow.postinst @@ -1,6 +1,14 @@ #!/bin/bash # post install script +SERVICE_NAME=movai-flow +APP_PATH=/usr/share/$SERVICE_NAME +CONFIG_PATH=/etc/$SERVICE_NAME +DOCKER_COMPOSE_CONF=$APP_PATH/docker-compose.yml +DOCKER_COMPOSE_CONF_MHOST=$APP_PATH/docker-compose-host.yml +DOCKER_COMPOSE_ENV=$APP_PATH/.env +DOCKER_COMPOSE_BIN=$(which docker-compose) + # Function to output to log and console. # # Usage: @@ -9,14 +17,6 @@ # # Note: Blank lines (Null text strings "") are not written to the system log. There is no need to waste that space. -SERVICE_NAME=movai-flow -SERVICE_PORTS="8080" #8083 -APP_PATH=/usr/share/$SERVICE_NAME -CONFIG_PATH=/etc/$SERVICE_NAME -DOCKER_COMPOSE_CONF=$APP_PATH/docker-compose.yml -DOCKER_COMPOSE_ENV=$APP_PATH/.env -DOCKER_COMPOSE_BIN=$(which docker-compose) - Output() { # Prepare base message Message="MOV.AI Flow™ install: $1" @@ -76,8 +76,7 @@ fi if [ $HaveFreePorts -gt 0 ]; then Output "Needed ports are available" else - Output "Error: Needed ports are not available" - exit 1 + Output "Warning: Needed ports are not available you might face networking issues when starting MOV.AI Flow™" fi if [ "$1" == "configure" ]; then @@ -92,17 +91,23 @@ if [ "$1" == "configure" ]; then Output "Created $SERVICE_NAME fleet token" fi - Output "Setting Nvidia config" + Output "Setting NVIDIA config" if ! grep -q HAVE_NVIDIA $CONFIG_PATH/config.env ; then echo "HAVE_NVIDIA=$HaveNvidia" >> $CONFIG_PATH/config.env else sed -i -e "/^HAVE_NVIDIA=/s/=.*/=\'$HaveNvidia\'/" $CONFIG_PATH/config.env fi - mkdir -p $APP_PATH/{logs,shared,userspace,models_database} - sudo chown 1000:1000 $APP_PATH/{logs,shared,userspace,models_database} -R + Output "Setting empty network config" + echo "NETWORK_MODE=" > $CONFIG_PATH/config.env + + mkdir -p $APP_PATH/{logs,shared,userspace,models_database,config} + chown 1000:1000 $APP_PATH/{logs,shared,userspace,models_database,config} -R + + cp $CONFIG_PATH/config.env $APP_PATH/config/ elif [ "$1" == "abort-remove" ]; then Output "Stopping cluster of $SERVICE_NAME" $DOCKER_COMPOSE_BIN -f $DOCKER_COMPOSE_CONF --env-file $DOCKER_COMPOSE_ENV down || true + $DOCKER_COMPOSE_BIN -f $DOCKER_COMPOSE_CONF_MHOST --env-file $DOCKER_COMPOSE_ENV down || true fi diff --git a/debian/movai-flow.prerm b/debian/movai-flow.prerm index 4582417..087f0cd 100644 --- a/debian/movai-flow.prerm +++ b/debian/movai-flow.prerm @@ -5,13 +5,14 @@ SERVICE_NAME=movai-flow APP_PATH=/usr/share/$SERVICE_NAME DOCKER_COMPOSE_CONF=$APP_PATH/docker-compose.yml -DOCKER_COMPOSE_ENV=$APP_PATH/.env +DOCKER_COMPOSE_CONF_MHOST=$APP_PATH/docker-compose-host.yml DOCKER_COMPOSE_BIN=$(which docker-compose || echo /usr/local/bin/docker-compose) # Remove the package (but not its configuration files) if [ "$1" = "remove" ]; then pushd $APP_PATH || echo "" $DOCKER_COMPOSE_BIN -f $DOCKER_COMPOSE_CONF down || true + $DOCKER_COMPOSE_BIN -f $DOCKER_COMPOSE_CONF_MHOST down || true popd || echo "" find /home -type l -wholename "*MovaiFlow/models_database" -delete diff --git a/docker-compose-host.yml b/docker-compose-host.yml new file mode 100644 index 0000000..f9395fd --- /dev/null +++ b/docker-compose-host.yml @@ -0,0 +1,299 @@ +# Basic MOV.AI Flow™ cluster configuration with Redis and Redis local databases. +# +# WARNING: +# This configuration is for local development. Do not use it in a production deployment. +# +# This configuration supports basic configuration using environment variables or an .env file +# The following variables are supported: +# +# MOVAI_REGISTRY - Docker images registry +# Default: pubregistry.aws.cloud.mov.ai +# MOVAI_UID - User ID in MOVAI containers +# Default: 1000 +# MOVAI_DEVICE_NAME - Docker containers identifier +# Default: Flow +# MOVAI_ENV - Docker containers environment (develop/qa/release) +# Default: qa +# +# Those configurations are useful mostly in case of standalone testing/running MOV.AI Flow™ in test/try-out mode +# +# STUDIO_USER_USERNAME - Username for the administrator account (if requested). +# Default: movai +# STUDIO_USER_PASSWORD - Password for the administrator account (if requested). +# Default: movai +# +# STUDIO_SIMU_GUI - Default Simulation GUI: WorldLauncher +# +# STUDIO_SIMU_PLUGIN_PACKAGES - List of movai ignition plugin packages to be installed. +# Default: movai-ign-plugin-world-launcher +# +# STUDIO_DB_PACKAGES - List of packages initializing the DB. +# Default: movai-baseline +# +# STUDIO_APP_PACKAGES - List of packages initializing the web apps. +# Default: movai-launcher movai-ide +# +# STUDIO_SPAWNER_PACKAGES - List of packages initializing the spawner. +# Default: ros-noetic-movai-ce-demos +# +# SPAWNER_PIP_ADDITIONAL_REQUIREMENTS - Additional PIP requirements to add when starting ROS spawner. +# Default: '' +# + +# Feel free to modify this file to suit your needs. +--- +version: "3.9" +x-movai-common: + &movai-common + environment: + &movai-common-env + MOVAI_ENV: ${MOVAI_ENV:-qa} + MOVAI_STDOUT_VERBOSITY_LEVEL: 10 # DEBUG + MOVAI_LOGFILE_VERBOSITY_LEVEL: 30 # WARNING + MOVAI_GENERAL_VERBOSITY_LEVEL: 10 # DEBUG + MOVAI_HEALTHNODE_VERBOSITY_LEVEL: 0 + MOVAI_USERSPACE: /opt/mov.ai/user + MOVAI_MANAGER_URI: http://localhost:$STUDIO_HTTP_PORT + DEVICE_NAME: ${MOVAI_DEVICE_NAME:-flow} + FLEET_NAME: ${FLEET_NAME:?missing FLEET_NAME ENV var} + FLEET_TOKEN: ${FLEET_TOKEN:?missing FLEET_TOKEN ENV var} + REDIS_LOCAL_PORT: $REDIS_LOCAL_PORT + REDIS_LOCAL_HOST: localhost + REDIS_MASTER_PORT: $REDIS_MASTER_PORT + REDIS_MASTER_HOST: localhost + ROS_MASTER_URI: http://localhost:11311/ + PUBLIC_IP: ${PUBLIC_IP:-127.0.0.1} + LOG_HTTP_HOST: http://localhost:$STUDIO_HTTP_PORT + IGNITION_MOVAI: "true" + IGN_PARTITION: ${IGN_PARTITION:?missing IGN_PARTITION ENV var} + IGNITION_VERSION: fortress + APT_AUTOINSTALL: "once" + APT_KEYS_URL_LIST: "https://artifacts.aws.cloud.mov.ai/repository/movai-applications/gpg" + APT_REPOS_LIST: "deb [arch=all] https://artifacts.aws.cloud.mov.ai/repository/ppa-public main main" + volumes: + - ./userspace:/opt/mov.ai/user + - ./shared:/opt/mov.ai/shared + - ./logs:/opt/mov.ai/logs + # user: "${MOVAI_UID:-1000}:0" + depends_on: + &movai-common-depend + redis-local: + condition: service_healthy + redis-master: + condition: service_healthy + extra_hosts: + &movai-common-extra-hosts + backend: 127.0.0.1 + spawner: 127.0.0.1 + redis-master: 127.0.0.1 + redis-local: 127.0.0.1 + redis-slave: 127.0.0.1 + ros-master: 127.0.0.1 + +services: + redis-master: + image: ${MOVAI_REGISTRY:-pubregistry.aws.cloud.mov.ai}/ce/redis2:v1.0.11 + container_name: redis-master-$MOVAI_DEVICE_NAME + logging: + driver: "journald" + options: + mode: "non-blocking" + max-buffer-size: "4m" + network_mode: host + healthcheck: + test: ["CMD", "echo", ">", "/dev/tcp/localhost/${REDIS_MASTER_PORT}"] + start_period: 3s + interval: 10s + timeout: 2s + retries: 3 + volumes: + - redis-master-flow-volume:/data + environment: + <<: *movai-common-env + APT_INSTALL_LIST: $STUDIO_DB_PACKAGES + REDIS_PORT: $REDIS_MASTER_PORT + restart: unless-stopped + extra_hosts: + <<: *movai-common-extra-hosts + + redis-local: + image: ${MOVAI_REGISTRY:-pubregistry.aws.cloud.mov.ai}/ce/redis2:v1.0.11 + container_name: redis-local-$MOVAI_DEVICE_NAME + logging: + driver: "journald" + options: + mode: "non-blocking" + max-buffer-size: "4m" + network_mode: host + healthcheck: + test: ["CMD", "echo", ">", "/dev/tcp/localhost/${REDIS_LOCAL_PORT}"] + start_period: 3s + interval: 10s + timeout: 2s + retries: 3 + volumes: + - redis-local-flow-volume:/data + restart: unless-stopped + environment: + - REDIS_PORT=$REDIS_LOCAL_PORT + extra_hosts: + <<: *movai-common-extra-hosts + + backend: + <<: *movai-common + image: ${MOVAI_REGISTRY:-pubregistry.aws.cloud.mov.ai}/ce/backend-noetic:1.0.0.23 + container_name: backend-$MOVAI_DEVICE_NAME + logging: + driver: "journald" + options: + mode: "non-blocking" + max-buffer-size: "4m" + network_mode: host + depends_on: + <<: *movai-common-depend + environment: + <<: *movai-common-env + APT_INSTALL_LIST: "$STUDIO_APP_PACKAGES" + HTTP_PORT: ${STUDIO_HTTP_PORT} + DEFAULT_USERNAME: movai + DEFAULT_PASSWORD: movai + JWT_SECRET_KEY: ${JWT_SECRET_KEY:?missing JWT_SECRET_KEY ENV var} + healthcheck: + test: ["CMD", "echo", ">", "/dev/tcp/localhost/${STUDIO_HTTP_PORT}"] + start_period: 5s + interval: 10s + timeout: 2s + retries: 3 + restart: unless-stopped + extra_hosts: + <<: *movai-common-extra-hosts + + ros-master: + image: ${MOVAI_REGISTRY:-pubregistry.aws.cloud.mov.ai}/ce/ros-master-noetic:v1.0.2 + privileged: true + network_mode: host + depends_on: + backend: + condition: service_healthy + container_name: ros-master-$MOVAI_DEVICE_NAME + logging: + driver: "journald" + options: + mode: "non-blocking" + max-buffer-size: "4m" + healthcheck: + test: ["CMD-SHELL", ". /opt/ros/noetic/setup.sh && /opt/ros/noetic/bin/rostopic list | grep -w '/rosout' >> /dev/null"] + start_period: 3s + interval: 10s + timeout: 2s + retries: 3 + restart: unless-stopped + extra_hosts: + <<: *movai-common-extra-hosts + + spawner: + <<: *movai-common + image: ${MOVAI_REGISTRY:-pubregistry.aws.cloud.mov.ai}/ce/spawner-ign-noetic:v3.0.9 + privileged: true + network_mode: host + depends_on: + <<: *movai-common-depend + backend: + condition: service_healthy + healthcheck: + test: pgrep -f "/usr/bin/python3 -m flow_initiator" + start_period: 10s + interval: 10s + timeout: 3s + retries: 1 + container_name: spawner-$MOVAI_DEVICE_NAME + logging: + driver: "journald" + options: + mode: "non-blocking" + max-buffer-size: "4m" + volumes: + - /dev:/dev + - /sys/devices:/sys/devices + - /var/run/dbus/system_bus_socket:/var/run/dbus/system_bus_socket + - /run/movai:/var/run/movai + - ./userspace:/opt/mov.ai/user + - ./shared:/opt/mov.ai/shared + - ./logs:/opt/mov.ai/logs + environment: + <<: *movai-common-env + PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + APP_NAME: spawner + APT_INSTALL_LIST: $STUDIO_SPAWNER_PACKAGES + restart: unless-stopped + extra_hosts: + <<: *movai-common-extra-hosts + + simulator: + image: ${MOVAI_REGISTRY:-pubregistry.aws.cloud.mov.ai}/ce/ignition-gazebo:v1.0.5 + container_name: simulator-$MOVAI_DEVICE_NAME + logging: + driver: "journald" + options: + mode: "non-blocking" + max-buffer-size: "4m" + profiles: ["simulator"] + environment: + <<: *movai-common-env + DISPLAY: "$DISPLAY" + QT_X11_NO_MITSHM: "1" + XAUTHORITY: "$XAUTH" + IGN_GAZEBO_SYSTEM_PLUGIN_PATH: /movai_ign_plugins/system:/models_database/plugins/system + IGN_GUI_PLUGIN_PATH: /movai_ign_plugins/gui:/models_database/plugins/gui + IGN_GAZEBO_RESOURCE_PATH: /models_database/my_models + APT_INSTALL_LIST: $STUDIO_SIMU_PLUGIN_PACKAGES + volumes: + - /tmp/.X11-unix:/tmp/.X11-unix:rw + - $XAUTH:$XAUTH + - ./models_database:/models_database + - ./models_database/fuel:/root/.ignition/fuel + network_mode: host + privileged: true + tty: true + healthcheck: + test: pgrep -f "ign gui -s $STUDIO_SIMU_GUI" + start_period: 3s + interval: 30s + timeout: 3s + retries: 10 + command: ign gui -s $STUDIO_SIMU_GUI + extra_hosts: + <<: *movai-common-extra-hosts + + ros-tools: + image: ${MOVAI_REGISTRY:-pubregistry.aws.cloud.mov.ai}/ce/ros-tools-ce:v1.0.4 + container_name: ros-tools-$MOVAI_DEVICE_NAME + logging: + driver: "journald" + options: + mode: "non-blocking" + max-buffer-size: "4m" + profiles: ["tools"] + environment: + <<: *movai-common-env + DISPLAY: "$DISPLAY" + QT_X11_NO_MITSHM: "1" + XAUTHORITY: "$XAUTH" + volumes: + - /tmp/.X11-unix:/tmp/.X11-unix:rw + - $XAUTH:$XAUTH + - ./rviz/:/headless/.rviz/:rw + network_mode: host + privileged: true + tty: true + entrypoint: /bin/bash + command: + - -ic + - | + exec /usr/bin/rviz + extra_hosts: + <<: *movai-common-extra-hosts + +volumes: + redis-master-flow-volume: + redis-local-flow-volume: diff --git a/docker-compose-nvidia-host.yml b/docker-compose-nvidia-host.yml new file mode 100644 index 0000000..17306b5 --- /dev/null +++ b/docker-compose-nvidia-host.yml @@ -0,0 +1,38 @@ +# MOV.AI Flow™ configuration of Rviz and IGN Gazebo with NVidia support +# +# WARNING: +# This configuration is for local development. Do not use it in a production deployment. +# +# This configuration supports basic configuration using environment variables or an .env file +# The following variables are supported: +# +# Feel free to modify this file to suit your needs. +--- +version: "3.9" +services: + simulator: + extends: + file: docker-compose-host.yml + service: simulator + environment: + NVIDIA_VISIBLE_DEVICES: ${NVIDIA_VISIBLE_DEVICES:-all} + NVIDIA_DRIVER_CAPABILITIES: ${NVIDIA_DRIVER_CAPABILITIES:-graphics} + deploy: + resources: + reservations: + devices: + - driver: nvidia + capabilities: ["gpu"] + ros-tools: + extends: + file: docker-compose-host.yml + service: ros-tools + environment: + NVIDIA_VISIBLE_DEVICES: ${NVIDIA_VISIBLE_DEVICES:-all} + NVIDIA_DRIVER_CAPABILITIES: ${NVIDIA_DRIVER_CAPABILITIES:-graphics} + deploy: + resources: + reservations: + devices: + - driver: nvidia + capabilities: ["gpu"] diff --git a/docker-compose-nvidia.yml b/docker-compose-nvidia.yml index a75615b..ed35f7c 100644 --- a/docker-compose-nvidia.yml +++ b/docker-compose-nvidia.yml @@ -39,4 +39,4 @@ services: networks: private: external: - name: flow-private + name: flow_private diff --git a/docker-compose.yml b/docker-compose.yml index 4144fb6..e376a0e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -53,17 +53,17 @@ x-movai-common: MOVAI_GENERAL_VERBOSITY_LEVEL: 10 # DEBUG MOVAI_HEALTHNODE_VERBOSITY_LEVEL: 0 MOVAI_USERSPACE: /opt/mov.ai/user - MOVAI_MANAGER_URI: ${MANAGER_URI:-http://localhost} + MOVAI_MANAGER_URI: http://localhost:8080 DEVICE_NAME: ${MOVAI_DEVICE_NAME:-flow} FLEET_NAME: ${FLEET_NAME:?missing FLEET_NAME ENV var} FLEET_TOKEN: ${FLEET_TOKEN:?missing FLEET_TOKEN ENV var} - REDIS_LOCAL_PORT: 6379 + REDIS_LOCAL_PORT: $REDIS_LOCAL_PORT REDIS_LOCAL_HOST: redis-local - REDIS_MASTER_PORT: 6379 + REDIS_MASTER_PORT: $REDIS_MASTER_PORT REDIS_MASTER_HOST: redis-master ROS_MASTER_URI: http://ros-master:11311/ PUBLIC_IP: ${PUBLIC_IP:-127.0.0.1} - LOG_HTTP_HOST: ${MANAGER_URI:-http://localhost} + LOG_HTTP_HOST: http://localhost:8080 IGNITION_MOVAI: "true" IGN_PARTITION: ${IGN_PARTITION:?missing IGN_PARTITION ENV var} IGNITION_VERSION: fortress @@ -84,7 +84,7 @@ x-movai-common: services: redis-master: - image: ${MOVAI_REGISTRY:-pubregistry.aws.cloud.mov.ai}/ce/redis2:v1.0.10 + image: ${MOVAI_REGISTRY:-pubregistry.aws.cloud.mov.ai}/ce/redis2:v1.0.11 container_name: redis-master-$MOVAI_DEVICE_NAME logging: driver: "journald" @@ -94,7 +94,7 @@ services: networks: - private ports: - - "6379:6379" + - "$REDIS_MASTER_PORT:$REDIS_MASTER_PORT" healthcheck: test: ["CMD", "echo", ">", "/dev/tcp/localhost/6379"] start_period: 3s @@ -106,10 +106,11 @@ services: environment: <<: *movai-common-env APT_INSTALL_LIST: $STUDIO_DB_PACKAGES + REDIS_PORT: $REDIS_MASTER_PORT restart: unless-stopped redis-local: - image: ${MOVAI_REGISTRY:-pubregistry.aws.cloud.mov.ai}/ce/redis2:v1.0.10 + image: ${MOVAI_REGISTRY:-pubregistry.aws.cloud.mov.ai}/ce/redis2:v1.0.11 container_name: redis-local-$MOVAI_DEVICE_NAME logging: driver: "journald" @@ -119,9 +120,9 @@ services: networks: - private ports: - - "6380:6379" + - "$REDIS_LOCAL_PORT:$REDIS_LOCAL_PORT" healthcheck: - test: ["CMD", "echo", ">", "/dev/tcp/localhost/6380"] + test: ["CMD", "echo", ">", "/dev/tcp/localhost/${REDIS_LOCAL_PORT}"] start_period: 3s interval: 10s timeout: 2s @@ -129,6 +130,8 @@ services: volumes: - redis-local-flow-volume:/data restart: unless-stopped + environment: + - REDIS_PORT=$REDIS_LOCAL_PORT backend: <<: *movai-common @@ -146,12 +149,12 @@ services: environment: <<: *movai-common-env APT_INSTALL_LIST: "$STUDIO_APP_PACKAGES" - HTTP_PORT: ${STUDIO_WEB_PORT} + HTTP_PORT: ${STUDIO_HTTP_PORT} DEFAULT_USERNAME: movai DEFAULT_PASSWORD: movai JWT_SECRET_KEY: ${JWT_SECRET_KEY:?missing JWT_SECRET_KEY ENV var} healthcheck: - test: ["CMD", "echo", ">", "/dev/tcp/localhost/${STUDIO_WEB_PORT}"] + test: ["CMD", "echo", ">", "/dev/tcp/localhost/${STUDIO_HTTP_PORT}"] start_period: 5s interval: 10s timeout: 2s @@ -307,6 +310,6 @@ volumes: networks: proxy: - name: flow-proxy + name: flow_proxy private: - name: flow-private + name: flow_private diff --git a/haproxy/haproxy.cfg b/haproxy/haproxy.cfg index 69bbfa0..3a2c8c8 100644 --- a/haproxy/haproxy.cfg +++ b/haproxy/haproxy.cfg @@ -149,23 +149,23 @@ frontend ha-frontend-spawner backend ha-backend-monitoring mode http #server container-monitoring 127.0.0.1 disabled - server container-monitoring monitoring.flow-private check resolvers dockerdns init-addr libc,none + server container-monitoring monitoring.flow_private check resolvers dockerdns init-addr libc,none backend ha-backend-http mode http #server container-backend 127.0.0.1 disabled - server container-backend backend.flow-private:5004 check resolvers dockerdns init-addr libc,none + server container-backend backend.flow_private:8080 check resolvers dockerdns init-addr libc,none backend ha-backend-npm mode http #server container-frontend 127.0.0.1 disabled - server container-frontend frontend.flow-private:3000 check resolvers dockerdns init-addr libc,none + server container-frontend frontend.flow_private:3000 check resolvers dockerdns init-addr libc,none backend ha-backend-redis-master mode tcp # master/ssl #server container-redis-master 127.0.0.1 disabled - server container-redis-master redis-master.flow-private:6379 check resolvers dockerdns init-addr libc,none + server container-redis-master redis-master.flow_private:6379 check resolvers dockerdns init-addr libc,none backend ha-backend-redis-slave mode tcp @@ -176,9 +176,9 @@ backend ha-backend-redis-slave backend ha-backend-ros-tools mode http #server container-ros-tools 127.0.0.1 disabled - server container-ros-tools ros-tools.flow-private check resolvers dockerdns init-addr libc,none + server container-ros-tools ros-tools.flow_private check resolvers dockerdns init-addr libc,none backend ha-backend-spawner mode tcp #server container-spawner 127.0.0.1 disabled - server container-spawner spawner.flow-private:11411 check resolvers dockerdns init-addr libc,none + server container-spawner spawner.flow_private:11411 check resolvers dockerdns init-addr libc,none diff --git a/scripts/movai-flow-launch b/scripts/movai-flow-launch index e01f090..ceafc30 100755 --- a/scripts/movai-flow-launch +++ b/scripts/movai-flow-launch @@ -2,10 +2,14 @@ SCRIPTNAME=Launcher SERVICE_NAME=movai-flow APP_PATH=${APP_PATH:-/usr/share/$SERVICE_NAME} -CONFIG_PATH=${CONFIG_PATH:-/etc/$SERVICE_NAME} +CONFIG_PATH=${CONFIG_PATH:-/$APP_PATH/config} + DOCKER_COMPOSE_CONF=$APP_PATH/docker-compose.yml +DOCKER_COMPOSE_CONF_MHOST=$APP_PATH/docker-compose-host.yml + DOCKER_COMPOSE_BIN=$(which docker-compose || echo /usr/local/bin/docker-compose) DOCKER_COMPOSE_UP_OPTIONS=${DOCKER_COMPOSE_UP_OPTIONS:---no-recreate} + APP_URL="http://localhost:8080/api/v1/apps/mov-fe-app-ide-ce/" Output() { @@ -23,44 +27,104 @@ Output() { # Output to log if which journalctl > /dev/null 2>&1 then - echo MESSAGE="$Message" | logger --journald + echo MESSAGE="$Message" | head -n 1 | logger --journald else echo "$Message" | logger fi } FilterRunningService() { - $DOCKER_COMPOSE_BIN -f "$DOCKER_COMPOSE_CONF" ps --services --filter "status=running" | grep "$1" + $DOCKER_COMPOSE_BIN -f "$compose_conf" ps --services --filter "status=running" | grep "$1" } -Output "This terminal window will verify all MOV.AI Flow™ services are running \ -and then launch the web application" -Output "(U) [^_^] (U)" +Output "This terminal window will verify all MOV.AI Flow™ services are running +and then launch the web application -if [ -f $CONFIG_PATH/config.env ]; then - Output "Sourcing fleet configuration" - . $CONFIG_PATH/config.env +(U) [^_^] (U) + +" + +if [ -f "$CONFIG_PATH/config.env" ]; then + . "$CONFIG_PATH/config.env" else - Output "No fleet configuration found" + Output "No configuration found" +fi + +if [ "$NETWORK_MODE" = "" ]; then + Output " + + MOV.AI Flow™ can either run in an isolated docker network or use the host's networking interfaces + - isolated network is recommended when working with a single host + - host network is recommended when working with multiple hosts (RPi, sensors, ...) which will need to access MOV.AI Flow™ containers + + If you don't know what this is all about just press ENTER + + See doc: https://docs.docker.com/network/host/. + " + while true; do + read -p "Do you want to run MOV.AI Flow™ in isolated network [Y/n]?" yn + case ${yn:-Y} in + [Yy]* ) NETWORK_MODE="bridge"; break;; + [Nn]* ) NETWORK_MODE="host"; break;; + esac + done + + # Update config file + if [ -f "$CONFIG_PATH/config.env" ]; then + sed -i -e "/^NETWORK_MODE=/s/=.*/=$NETWORK_MODE/" "$CONFIG_PATH/config.env" + else + mkdir -p "$CONFIG_PATH" + echo "NETWORK_MODE=$NETWORK_MODE" > "$CONFIG_PATH/config.env" + fi +fi + +# check if haproxy was ever created +HaveProxy=0 +[ -n "$(docker ps -a --filter "label=com.docker.compose.project=movai-flow" -f "label=com.docker.compose.service=haproxy" -q)" ] && HaveProxy=1 + +if [ "$NETWORK_MODE" = "host" ]; then + if [ $HaveProxy -gt 0 ];then + DOCKER_COMPOSE_UP_OPTIONS="--remove-orphans" + fi + compose_conf="$DOCKER_COMPOSE_CONF_MHOST" + +else + if [ $HaveProxy -eq 0 ];then + DOCKER_COMPOSE_UP_OPTIONS="--remove-orphans" + fi + compose_conf="$DOCKER_COMPOSE_CONF" +fi + +# warn in case of container recreation +if [ -n "$(docker volume ls --filter "label=com.docker.compose.project=movai-flow" -q)" ]; then + if ! [[ "$DOCKER_COMPOSE_UP_OPTIONS" =~ .*"no-recreate".* ]]; then + while true; do + read -p "Containers need to be recreated, MOV.AI Flow™ might suffer data loss. Continue [y/N]?" yn + case ${yn:-N} in + [Yy]* ) break;; + [Nn]* ) Output "Aborted"; exit 0;; + esac + done + fi fi pushd "$APP_PATH" || exit Output "Downloading images" -$DOCKER_COMPOSE_BIN -f "$DOCKER_COMPOSE_CONF" pull +$DOCKER_COMPOSE_BIN -f "$compose_conf" pull -Output "Starting services" -$DOCKER_COMPOSE_BIN -f "$DOCKER_COMPOSE_CONF" up -d "$DOCKER_COMPOSE_UP_OPTIONS" +Output "Starting services (network: $NETWORK_MODE, opts=$DOCKER_COMPOSE_UP_OPTIONS)" +$DOCKER_COMPOSE_BIN -f "$compose_conf" up -d $DOCKER_COMPOSE_UP_OPTIONS while [[ -z $(FilterRunningService spawner) ]]; do Output "waiting for spawner" sleep 5 done -while [[ -z $( $DOCKER_COMPOSE_BIN -f $DOCKER_COMPOSE_CONF exec -u movai spawner bash -c 'pgrep -f "/usr/bin/python3 -m flow_initiator"') ]] +while [[ -z $( $DOCKER_COMPOSE_BIN -f "$compose_conf" exec -u movai spawner bash -c 'pgrep -f "/usr/bin/python3 -m flow_initiator"') ]] do - sleep 10 - Output "loading ..." + sleep 10 + Output "loading ..." done # Symlink userspace and models_database to ~/Documents/MovaiFlow diff --git a/scripts/movai-flow-reset b/scripts/movai-flow-reset new file mode 100755 index 0000000..e5dcf62 --- /dev/null +++ b/scripts/movai-flow-reset @@ -0,0 +1,58 @@ +#!/bin/bash +SCRIPTNAME=Reset +SERVICE_NAME=movai-flow +APP_PATH=${APP_PATH:-/usr/share/$SERVICE_NAME} +CONFIG_PATH=${CONFIG_PATH:-/$APP_PATH/config} + +DOCKER_COMPOSE_CONF=$APP_PATH/docker-compose.yml +DOCKER_COMPOSE_CONF_MHOST=$APP_PATH/docker-compose-host.yml + +DOCKER_COMPOSE_BIN=$(which docker-compose || echo /usr/local/bin/docker-compose) + +Output() { + # Prepare base message + Message="MOV.AI Flow™ $SCRIPTNAME: $1" + + # Output the message even if a blank message (make output nice) + echo "$Message" + + # Return if blank. + if [ "$1" = "" ]; then + return + fi + + # Output to log + if which journalctl > /dev/null 2>&1 + then + echo MESSAGE="$Message" | head -n 1 | logger --journald + else + echo "$Message" | logger + fi +} + +StopServices() { + Output "Stopping containers and removing containers, networks, volumes, but not images. + + (U) [^_^] (U) + + Terminal will close on completion, wait or press CTRL+C on failure" + + if [ "$NETWORK_MODE" = "host" ]; then + compose_conf="$DOCKER_COMPOSE_CONF_MHOST" + else + compose_conf="$DOCKER_COMPOSE_CONF" + fi + + sed -i -e "/^NETWORK_MODE=/s/=.*/=/" "$CONFIG_PATH/config.env" + + $DOCKER_COMPOSE_BIN -f "$compose_conf" down -v --remove-orphans +} + +if [ -f "$CONFIG_PATH/config.env" ]; then + Output "Sourcing configuration" + . "$CONFIG_PATH/config.env" +else + Output "No configuration found" +fi + +StopServices \ No newline at end of file diff --git a/scripts/movai-flow-rviz b/scripts/movai-flow-rviz index 2086e34..6464466 100755 --- a/scripts/movai-flow-rviz +++ b/scripts/movai-flow-rviz @@ -2,9 +2,11 @@ SCRIPTNAME=RVIZ SERVICE_NAME=movai-flow APP_PATH=${APP_PATH:-/usr/share/$SERVICE_NAME} -CONFIG_PATH=${CONFIG_PATH:-/etc/$SERVICE_NAME} -SERVICE_CONF=$APP_PATH/docker-compose.yml -SERVICE_NVIDIA_CONF=$APP_PATH/docker-compose-nvidia.yml +CONFIG_PATH=${CONFIG_PATH:-/$APP_PATH/config} +DOCKER_COMPOSE_CONF=$APP_PATH/docker-compose.yml +DOCKER_COMPOSE_CONF_MHOST=$APP_PATH/docker-compose-host.yml +DOCKER_COMPOSE_NVIDIA_CONF=$APP_PATH/docker-compose-nvidia.yml +DOCKER_COMPOSE_NVIDIA_CONF_MHOST=$APP_PATH/docker-compose-nvidia-host.yml DOCKER_COMPOSE_BIN=$(which docker-compose || echo /usr/local/bin/docker-compose) Output() { @@ -22,18 +24,20 @@ Output() { # Output to log if which journalctl > /dev/null 2>&1 then - echo MESSAGE="$Message" | logger --journald + echo MESSAGE="$Message" | head -n 1 | logger --journald else echo "$Message" | logger fi } FilterRunningService() { - $DOCKER_COMPOSE_BIN -f $SERVICE_CONF ps --services --filter "status=running" | grep "$1" + $DOCKER_COMPOSE_BIN -f "$compose_conf" ps --services --filter "status=running" | grep "$1" } -Output "This terminal window will verify all MOV.AI Flow™ services are running and then launch the RVIZ application" -Output "(U) [^_^] (U)" +Output "This terminal window will verify all MOV.AI Flow™ services are running and then launch the RVIZ application + +(U) [^_^] (U) +" if [ -f "$CONFIG_PATH/config.env" ]; then Output "Sourcing configuration" @@ -42,19 +46,51 @@ else Output "No configuration found" fi +if [ "$NETWORK_MODE" = "host" ]; then + compose_conf="$DOCKER_COMPOSE_CONF_MHOST" + compose_conf_nvidia="$DOCKER_COMPOSE_NVIDIA_CONF_MHOST" +else + compose_conf="$DOCKER_COMPOSE_CONF" + compose_conf_nvidia="$DOCKER_COMPOSE_NVIDIA_CONF" +fi + +# Check NVIDIA +Architecture="$(uname -m)" +if [ "$Architecture" = "x86_64" ]; then + # Is there a Nvidia card present (possible HW enhanced graphical rendering) + [ "$(echo /sys/module/nvidia/drivers/pci:nvidia/*:*:*.*)" != "/sys/module/nvidia/drivers/pci:nvidia/*:*:*.*" ] && HaveNvidia=1 + # Is there NVIDIA Container Toolkit to build and run GPU accelerated Docker containers + [ "$(dpkg -l | grep ^ii | grep -iw nvidia-container-toolkit)" != "" ] && HaveDockerNvidia=1 +fi +if [ $HaveNvidia -gt 0 ]; then + Output " NVIDIA GPU card found" +else + Output " NVIDIA GPU card not found" +fi +if [ $HaveDockerNvidia -gt 0 ]; then + Output " NVIDIA Container Toolkit found" +else + Output " NVIDIA Container Toolkit not found" + if [ $HaveNvidia -gt 0 ]; then + echo -e "\033[1;33mWARNING: NVIDIA GPU found but no NVIDIA Container Toolkit\e[0m" + echo "Follow instructions on getting started with the NVIDIA Container Toolkit" + echo -e "\thttps://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html#docker" + fi +fi + xhost +local:docker Output "Launching ros-master" -$DOCKER_COMPOSE_BIN -f "$SERVICE_CONF" up -d ros-master +$DOCKER_COMPOSE_BIN -f "$compose_conf" up -d ros-master while [[ -z $(FilterRunningService ros-master) ]]; do Output "waiting for ros-master" sleep 5 done -if [ "$HAVE_NVIDIA" = "1" ]; then +if [ $HaveNvidia -gt 0 ]; then Output "Launching RViz for Nvidia GPU" - exec $DOCKER_COMPOSE_BIN -f "$SERVICE_NVIDIA_CONF" up ros-tools + exec $DOCKER_COMPOSE_BIN -f "$compose_conf_nvidia" up ros-tools else Output "Launching RViz for integrated GPU" - exec $DOCKER_COMPOSE_BIN -f "$SERVICE_CONF" up ros-tools + exec $DOCKER_COMPOSE_BIN -f "$compose_conf" up ros-tools fi diff --git a/scripts/movai-flow-simulator b/scripts/movai-flow-simulator index f1bad95..c28570d 100755 --- a/scripts/movai-flow-simulator +++ b/scripts/movai-flow-simulator @@ -2,12 +2,13 @@ SCRIPTNAME=SIMULATOR SERVICE_NAME=movai-flow APP_PATH=${APP_PATH:-/usr/share/$SERVICE_NAME} -CONFIG_PATH=${CONFIG_PATH:-/etc/$SERVICE_NAME} -SERVICE_CONF=$APP_PATH/docker-compose.yml -SERVICE_NVIDIA_CONF=$APP_PATH/docker-compose-nvidia.yml +CONFIG_PATH=${CONFIG_PATH:-/$APP_PATH/config} +DOCKER_COMPOSE_CONF=$APP_PATH/docker-compose.yml +DOCKER_COMPOSE_CONF_MHOST=$APP_PATH/docker-compose-host.yml +DOCKER_COMPOSE_NVIDIA_CONF=$APP_PATH/docker-compose-nvidia.yml +DOCKER_COMPOSE_NVIDIA_CONF_MHOST=$APP_PATH/docker-compose-nvidia-host.yml DOCKER_COMPOSE_BIN=$(which docker-compose || echo /usr/local/bin/docker-compose) - Output() { # Prepare base message Message="MOV.AI Flow™ $SCRIPTNAME: $1" @@ -23,7 +24,7 @@ Output() { # Output to log if which journalctl > /dev/null 2>&1 then - echo MESSAGE="$Message" | logger --journald + echo MESSAGE="$Message" | head -n 1 | logger --journald else echo "$Message" | logger fi @@ -39,8 +40,10 @@ if [ ! -e "${user_documents_path}/MovaiFlow" ]; then Output "User and models folder ${user_documents_path}/MovaiFlow" fi -Output "This terminal window will verify all MOV.AI Flow™ services are running and then launch the Ignition Simulator GUI application" -Output "(U) [^_^] (U)" +Output "This terminal window will verify all MOV.AI Flow™ services are running and then launch the Ignition Simulator GUI application + +(U) [^_^] (U) +" if [ -f "$CONFIG_PATH/config.env" ]; then Output "Sourcing configuration" @@ -49,11 +52,43 @@ else Output "No configuration found" fi +if [ "$NETWORK_MODE" = "host" ]; then + compose_conf="$DOCKER_COMPOSE_CONF_MHOST" + compose_conf_nvidia="$DOCKER_COMPOSE_NVIDIA_CONF_MHOST" +else + compose_conf="$DOCKER_COMPOSE_CONF" + compose_conf_nvidia="$DOCKER_COMPOSE_NVIDIA_CONF" +fi + +# Check NVIDIA +Architecture="$(uname -m)" +if [ "$Architecture" = "x86_64" ]; then + # Is there a Nvidia card present (possible HW enhanced graphical rendering) + [ "$(echo /sys/module/nvidia/drivers/pci:nvidia/*:*:*.*)" != "/sys/module/nvidia/drivers/pci:nvidia/*:*:*.*" ] && HaveNvidia=1 + # Is there NVIDIA Container Toolkit to build and run GPU accelerated Docker containers + [ "$(dpkg -l | grep ^ii | grep -iw nvidia-container-toolkit)" != "" ] && HaveDockerNvidia=1 +fi +if [ $HaveNvidia -gt 0 ]; then + Output " NVIDIA GPU card found" +else + Output " NVIDIA GPU card not found" +fi +if [ $HaveDockerNvidia -gt 0 ]; then + Output " NVIDIA Container Toolkit found" +else + Output " NVIDIA Container Toolkit not found" + if [ $HaveNvidia -gt 0 ]; then + echo -e "\033[1;33mWARNING: NVIDIA GPU found but no NVIDIA Container Toolkit\e[0m" + echo "Follow instructions on getting started with the NVIDIA Container Toolkit" + echo -e "\thttps://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html#docker" + fi +fi + xhost +local:docker -if [ "$HAVE_NVIDIA" = "1" ]; then +if [ $HaveNvidia -gt 0 ]; then Output "Launching simulator for Nvidia GPU" - exec $DOCKER_COMPOSE_BIN -f "$SERVICE_NVIDIA_CONF" up simulator + exec $DOCKER_COMPOSE_BIN -f "$compose_conf_nvidia" up simulator else Output "Launching simulator for integrated GPU" - exec $DOCKER_COMPOSE_BIN -f "$SERVICE_CONF" up simulator + exec $DOCKER_COMPOSE_BIN -f "$compose_conf" up simulator fi diff --git a/scripts/movai-flow-stop b/scripts/movai-flow-stop index 4cb177e..dd318d6 100755 --- a/scripts/movai-flow-stop +++ b/scripts/movai-flow-stop @@ -2,10 +2,12 @@ SCRIPTNAME=Stopper SERVICE_NAME=movai-flow APP_PATH=${APP_PATH:-/usr/share/$SERVICE_NAME} -CONFIG_PATH=${CONFIG_PATH:-/etc/$SERVICE_NAME} +CONFIG_PATH=${CONFIG_PATH:-/$APP_PATH/config} + DOCKER_COMPOSE_CONF=$APP_PATH/docker-compose.yml +DOCKER_COMPOSE_CONF_MHOST=$APP_PATH/docker-compose-host.yml + DOCKER_COMPOSE_BIN=$(which docker-compose || echo /usr/local/bin/docker-compose) -APP_URL="http://localhost:8080/api/v1/apps/mov-fe-app-ide-ce/" Output() { # Prepare base message @@ -22,17 +24,33 @@ Output() { # Output to log if which journalctl > /dev/null 2>&1 then - echo MESSAGE="$Message" | logger --journald + echo MESSAGE="$Message" | head -n 1 | logger --journald else echo "$Message" | logger fi } StopServices() { - Output "Stopping services..." - Output "(U) [^_^] (U)" - Output "Terminal will close on completion, wait or press CTRL+C on failure" - $DOCKER_COMPOSE_BIN -f $DOCKER_COMPOSE_CONF stop + Output "Stopping services... + + (U) [^_^] (U) + + Terminal will close on completion, wait or press CTRL+C on failure" + + if [ "$NETWORK_MODE" = "host" ]; then + compose_conf="$DOCKER_COMPOSE_CONF_MHOST" + else + compose_conf="$DOCKER_COMPOSE_CONF" + fi + + $DOCKER_COMPOSE_BIN -f "$compose_conf" stop } +if [ -f "$CONFIG_PATH/config.env" ]; then + Output "Sourcing configuration" + . "$CONFIG_PATH/config.env" +else + Output "No configuration found" +fi + StopServices \ No newline at end of file From 898ff001294bd95e018045938c6f5a6e1cd2418f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Oct 2022 15:11:42 +0100 Subject: [PATCH 17/28] Bump isbang/compose-action from 1.1.0 to 1.2.0 (#175) Bumps [isbang/compose-action](https://github.com/isbang/compose-action) from 1.1.0 to 1.2.0. - [Release notes](https://github.com/isbang/compose-action/releases) - [Commits](https://github.com/isbang/compose-action/compare/v1.1.0...v1.2.0) --- updated-dependencies: - dependency-name: isbang/compose-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> From 21a797a345c0e107d7966eaa694e5f38d26c0a5b Mon Sep 17 00:00:00 2001 From: OttoMation-Movai Date: Thu, 6 Oct 2022 11:33:41 +0100 Subject: [PATCH 18/28] bump version --- debian/changelog | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index 7e43d7b..bb43d65 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,7 +1,9 @@ -movai-flow (1.2.0-7) UNRELEASED; urgency=medium +movai-flow (1.3.0-0) UNRELEASED; urgency=medium + + [ Alex Fernandes ] + * add host_mode feature [ Alex Fernandes ] - * * review doc and compose * remove cli build * add ci From fccd11ba8f675770c8b8981256b217e1b9127e3d Mon Sep 17 00:00:00 2001 From: OttoMation-Movai Date: Wed, 12 Oct 2022 18:25:37 +0100 Subject: [PATCH 19/28] prompt for ROS_IP value to use --- .env | 3 ++- docker-compose-host.yml | 1 + scripts/movai-flow-launch | 20 +++++++++++++++++++- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/.env b/.env index 23f232e..c305f0e 100644 --- a/.env +++ b/.env @@ -18,4 +18,5 @@ IGN_PARTITION="movai_ce_flow" XAUTH=/tmp/.docker.xauth DISPLAY=${DISPLAY:-:1} REDIS_LOCAL_PORT=6380 -REDIS_MASTER_PORT=6379 \ No newline at end of file +REDIS_MASTER_PORT=6379 +ROS_IP=${MOVAI_FLOW_ROS_IP:-0.0.0.0} diff --git a/docker-compose-host.yml b/docker-compose-host.yml index f9395fd..1fb65af 100644 --- a/docker-compose-host.yml +++ b/docker-compose-host.yml @@ -225,6 +225,7 @@ services: PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin APP_NAME: spawner APT_INSTALL_LIST: $STUDIO_SPAWNER_PACKAGES + ROS_IP: $ROS_IP restart: unless-stopped extra_hosts: <<: *movai-common-extra-hosts diff --git a/scripts/movai-flow-launch b/scripts/movai-flow-launch index ceafc30..8e81654 100755 --- a/scripts/movai-flow-launch +++ b/scripts/movai-flow-launch @@ -56,10 +56,13 @@ if [ "$NETWORK_MODE" = "" ]; then MOV.AI Flow™ can either run in an isolated docker network or use the host's networking interfaces - isolated network is recommended when working with a single host - host network is recommended when working with multiple hosts (RPi, sensors, ...) which will need to access MOV.AI Flow™ containers + When selecting host network, the host's networking interface to be used as ROS_IP will be asked. If you don't know what this is all about just press ENTER - See doc: https://docs.docker.com/network/host/. + --- + See docker doc: https://docs.docker.com/network/host/. + See ROS doc: http://wiki.ros.org/ROS/EnvironmentVariables#ROS_IP.2FROS_HOSTNAME. " while true; do read -p "Do you want to run MOV.AI Flow™ in isolated network [Y/n]?" yn @@ -69,6 +72,21 @@ if [ "$NETWORK_MODE" = "" ]; then esac done + if [ $NETWORK_MODE = "host" ]; then + DETECTED_IP=$(hostname -I | cut -d " " -f 1) + while true; do + read -p "Please confirm the host IP to be used [$DETECTED_IP]?" ip_input + if [[ "${ip_input:-$DETECTED_IP}" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + export MOVAI_FLOW_ROS_IP=${ip_input:-$DETECTED_IP} + Output "Using $MOVAI_FLOW_ROS_IP" + break; + else + echo "Please enter a valid IP address" + fi + done + + fi + # Update config file if [ -f "$CONFIG_PATH/config.env" ]; then sed -i -e "/^NETWORK_MODE=/s/=.*/=$NETWORK_MODE/" "$CONFIG_PATH/config.env" From 609a255ff38e4856ec11dd0b711673c95c417456 Mon Sep 17 00:00:00 2001 From: mithun <51120171+MithunKinarullathil@users.noreply.github.com> Date: Wed, 12 Oct 2022 18:55:41 +0100 Subject: [PATCH 20/28] There are several issues associated with the new image of the simulator and no peronnel to support the fixes. Reverting back to a stable version. --- docker-compose-host.yml | 2 +- docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose-host.yml b/docker-compose-host.yml index f9395fd..f94cc6b 100644 --- a/docker-compose-host.yml +++ b/docker-compose-host.yml @@ -230,7 +230,7 @@ services: <<: *movai-common-extra-hosts simulator: - image: ${MOVAI_REGISTRY:-pubregistry.aws.cloud.mov.ai}/ce/ignition-gazebo:v1.0.5 + image: ${MOVAI_REGISTRY:-pubregistry.aws.cloud.mov.ai}/ce/ignition-gazebo:v1.0.4 container_name: simulator-$MOVAI_DEVICE_NAME logging: driver: "journald" diff --git a/docker-compose.yml b/docker-compose.yml index e376a0e..51c8256 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -221,7 +221,7 @@ services: restart: unless-stopped simulator: - image: ${MOVAI_REGISTRY:-pubregistry.aws.cloud.mov.ai}/ce/ignition-gazebo:v1.0.5 + image: ${MOVAI_REGISTRY:-pubregistry.aws.cloud.mov.ai}/ce/ignition-gazebo:v1.0.4 container_name: simulator-$MOVAI_DEVICE_NAME logging: driver: "journald" From 0248df6e570e1589f578f42f2f8a35fc8034b1db Mon Sep 17 00:00:00 2001 From: OttoMation-Movai Date: Fri, 14 Oct 2022 11:54:24 +0100 Subject: [PATCH 21/28] fix DP-911 --- debian/movai-flow.postrm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/movai-flow.postrm b/debian/movai-flow.postrm index 0b457a8..255f21e 100644 --- a/debian/movai-flow.postrm +++ b/debian/movai-flow.postrm @@ -41,7 +41,7 @@ if [ "$1" = "purge" ]; then $DOCKER_BIN volume rm $volumes 2>/dev/null fi - images=$($DOCKER_BIN images | awk '$1 ~ "mov.ai" { print $3 }') + images=$($DOCKER_BIN images | awk '$1 ~ "mov.ai/ce" { print $3 }') if [ -n "$images" ] then Output "removing docker images" From 95f7a9c1ba9e6caa7598592635e8542fe8aa9139 Mon Sep 17 00:00:00 2001 From: OttoMation-Movai Date: Fri, 14 Oct 2022 12:03:29 +0100 Subject: [PATCH 22/28] fix DP-915 --- scripts/movai-flow-launch | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/scripts/movai-flow-launch b/scripts/movai-flow-launch index 8e81654..e03b0f1 100755 --- a/scripts/movai-flow-launch +++ b/scripts/movai-flow-launch @@ -150,9 +150,13 @@ user_documents_path=$(xdg-user-dir DOCUMENTS) Output "Creating user and models folder" if [ ! -e "${user_documents_path}/MovaiFlow" ]; then mkdir -p "${user_documents_path}/MovaiFlow" - ln -sf "$APP_PATH/userspace" "${user_documents_path}/MovaiFlow" - ln -sf "$APP_PATH/models_database" "${user_documents_path}/MovaiFlow" - Output "User and models folder ${user_documents_path}/MovaiFlow" + Output "User and models folder created in ${user_documents_path}/MovaiFlow" +fi +if [ ! -e "${user_documents_path}/MovaiFlow/userspace" ]; then + ln -sf "$APP_PATH/userspace" "${user_documents_path}/MovaiFlow/userspace" +fi +if [ ! -e "${user_documents_path}/MovaiFlow/models_database" ]; then + ln -sf "$APP_PATH/models_database" "${user_documents_path}/MovaiFlow/models_database" fi Output "Starting web application" From 1f499f4b0e66751e581c18a3ea164732092e65e7 Mon Sep 17 00:00:00 2001 From: OttoMation-Movai Date: Fri, 14 Oct 2022 12:08:10 +0100 Subject: [PATCH 23/28] fix DP-916 --- scripts/movai-flow-rviz | 2 ++ scripts/movai-flow-simulator | 2 ++ 2 files changed, 4 insertions(+) diff --git a/scripts/movai-flow-rviz b/scripts/movai-flow-rviz index 6464466..fa590ac 100755 --- a/scripts/movai-flow-rviz +++ b/scripts/movai-flow-rviz @@ -56,6 +56,8 @@ fi # Check NVIDIA Architecture="$(uname -m)" +HaveNvidia=0 +HaveDockerNvidia=0 if [ "$Architecture" = "x86_64" ]; then # Is there a Nvidia card present (possible HW enhanced graphical rendering) [ "$(echo /sys/module/nvidia/drivers/pci:nvidia/*:*:*.*)" != "/sys/module/nvidia/drivers/pci:nvidia/*:*:*.*" ] && HaveNvidia=1 diff --git a/scripts/movai-flow-simulator b/scripts/movai-flow-simulator index c28570d..3fb6569 100755 --- a/scripts/movai-flow-simulator +++ b/scripts/movai-flow-simulator @@ -62,6 +62,8 @@ fi # Check NVIDIA Architecture="$(uname -m)" +HaveNvidia=0 +HaveDockerNvidia=0 if [ "$Architecture" = "x86_64" ]; then # Is there a Nvidia card present (possible HW enhanced graphical rendering) [ "$(echo /sys/module/nvidia/drivers/pci:nvidia/*:*:*.*)" != "/sys/module/nvidia/drivers/pci:nvidia/*:*:*.*" ] && HaveNvidia=1 From 53a261cd7cfd9ecdc33821abf078c03444ab95e2 Mon Sep 17 00:00:00 2001 From: OttoMation-Movai Date: Fri, 14 Oct 2022 12:13:47 +0100 Subject: [PATCH 24/28] fix DP-917 --- debian/movai-flow.postinst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/debian/movai-flow.postinst b/debian/movai-flow.postinst index 2aa5794..28358eb 100644 --- a/debian/movai-flow.postinst +++ b/debian/movai-flow.postinst @@ -102,7 +102,8 @@ if [ "$1" == "configure" ]; then echo "NETWORK_MODE=" > $CONFIG_PATH/config.env mkdir -p $APP_PATH/{logs,shared,userspace,models_database,config} - chown 1000:1000 $APP_PATH/{logs,shared,userspace,models_database,config} -R + chown 1000:1000 $APP_PATH/{logs,shared,userspace,models_database} -R + chmod 777 $APP_PATH/config -R cp $CONFIG_PATH/config.env $APP_PATH/config/ From ae650a8fff2e7f8bb60555702dbbb067b502663d Mon Sep 17 00:00:00 2001 From: duartecoelhomovai <84720623+duartecoelhomovai@users.noreply.github.com> Date: Fri, 14 Oct 2022 14:42:38 +0100 Subject: [PATCH 25/28] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 7322b2c..c1f6568 100644 --- a/README.md +++ b/README.md @@ -269,3 +269,4 @@ To stop and delete containers, delete volumes with database data and downloaded ## MOV.AI Developer Tools A python package build to facilitate development when using MOV.AI. The tool can be used to, but not limited to, expose the ros topics to the host as well as export/import MOV.AI metadata. Please follow [this link](https://github.com/MOV-AI/movai-developer-tools) for more detailed documentation. + From 47ef7e3b3cf974d25bf4c0b9361765a5de5f1704 Mon Sep 17 00:00:00 2001 From: Mithun Kinarullathil <51120171+MithunKinarullathil@users.noreply.github.com> Date: Fri, 14 Oct 2022 15:35:32 +0100 Subject: [PATCH 26/28] Update README.md --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index c1f6568..7322b2c 100644 --- a/README.md +++ b/README.md @@ -269,4 +269,3 @@ To stop and delete containers, delete volumes with database data and downloaded ## MOV.AI Developer Tools A python package build to facilitate development when using MOV.AI. The tool can be used to, but not limited to, expose the ros topics to the host as well as export/import MOV.AI metadata. Please follow [this link](https://github.com/MOV-AI/movai-developer-tools) for more detailed documentation. - From 7b951ea595e76745d898031df1d1a4b7e80dd18f Mon Sep 17 00:00:00 2001 From: mithun <51120171+MithunKinarullathil@users.noreply.github.com> Date: Tue, 18 Oct 2022 12:49:28 +0100 Subject: [PATCH 27/28] New spawner version: The new image contains new base packages for ROS noetic and a Fix for rospkg package. --- docker-compose-host.yml | 2 +- docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose-host.yml b/docker-compose-host.yml index 1b96c7b..b29dee7 100644 --- a/docker-compose-host.yml +++ b/docker-compose-host.yml @@ -193,7 +193,7 @@ services: spawner: <<: *movai-common - image: ${MOVAI_REGISTRY:-pubregistry.aws.cloud.mov.ai}/ce/spawner-ign-noetic:v3.0.9 + image: ${MOVAI_REGISTRY:-pubregistry.aws.cloud.mov.ai}/ce/spawner-ign-noetic:v3.0.10 privileged: true network_mode: host depends_on: diff --git a/docker-compose.yml b/docker-compose.yml index 51c8256..0e4b23f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -185,7 +185,7 @@ services: spawner: <<: *movai-common - image: ${MOVAI_REGISTRY:-pubregistry.aws.cloud.mov.ai}/ce/spawner-ign-noetic:v3.0.9 + image: ${MOVAI_REGISTRY:-pubregistry.aws.cloud.mov.ai}/ce/spawner-ign-noetic:v3.0.10 privileged: true networks: - private From da81a1273fa487136fb4e89897adda47512c3567 Mon Sep 17 00:00:00 2001 From: Mithun Kinarullathil <51120171+MithunKinarullathil@users.noreply.github.com> Date: Tue, 18 Oct 2022 14:56:37 +0100 Subject: [PATCH 28/28] To trigger pipeline. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 7322b2c..c1f6568 100644 --- a/README.md +++ b/README.md @@ -269,3 +269,4 @@ To stop and delete containers, delete volumes with database data and downloaded ## MOV.AI Developer Tools A python package build to facilitate development when using MOV.AI. The tool can be used to, but not limited to, expose the ros topics to the host as well as export/import MOV.AI metadata. Please follow [this link](https://github.com/MOV-AI/movai-developer-tools) for more detailed documentation. +