Visualize NAP logs in mock collector grafana

Makefile

@@ -69,12 +69,12 @@ OLD_BENCHMARK_RESULTS_FILE ?= $(TEST_BUILD_DIR)/benchmark.txt
 uname_m    := $(shell uname -m)
 
 ifeq ($(uname_m),aarch64)
-	OSARCH = arm64
+	OSARCH ?= arm64
 else
 	ifeq ($(uname_m),x86_64)
-		OSARCH = amd64
+		OSARCH ?= amd64
 	else
-		OSARCH = $(uname_m)
+		OSARCH ?= $(uname_m)
 	endif
 endif
 
@@ -150,9 +150,9 @@ build-mock-management-plane-grpc:
 	mkdir -p $(BUILD_DIR)/mock-management-plane-grpc
 	@CGO_ENABLED=0 GOARCH=$(OSARCH) GOOS=linux $(GOBUILD) -o $(BUILD_DIR)/mock-management-plane-grpc/server test/mock/grpc/cmd/main.go
 
-build-mock-management-plane-collector:
-	mkdir -p $(BUILD_DIR)/mock-management-plane-collector
-	@CGO_ENABLED=0 GOARCH=$(OSARCH) GOOS=linux $(GOBUILD) -o $(BUILD_DIR)/mock-management-plane-collector/collector test/mock/collector/mock-collector/main.go
+build-mock-management-otel-collector:
+	mkdir -p $(BUILD_DIR)/mock-management-otel-collector
+	@CGO_ENABLED=0 GOARCH=$(OSARCH) GOOS=linux $(GOBUILD) -o $(BUILD_DIR)/mock-management-otel-collector/collector test/mock/collector/mock-collector/main.go
 
 integration-test: $(SELECTED_PACKAGE) build-mock-management-plane-grpc
 	TEST_ENV="Container" CONTAINER_OS_TYPE=$(CONTAINER_OS_TYPE) BUILD_TARGET="install-agent-local" CONTAINER_NGINX_IMAGE_REGISTRY=${CONTAINER_NGINX_IMAGE_REGISTRY} \
@@ -211,8 +211,8 @@ build-test-oss-image:
 		--build-arg BASE_IMAGE=$(BASE_IMAGE) \
 		--build-arg ENTRY_POINT=./test/docker/entrypoint.sh
 
-.PHONY: build-mock-collector-image
-build-mock-collector-image: build-mock-management-plane-collector
+.PHONY: build-mock-management-otel-collector-image
+build-mock-management-otel-collector-image: build-mock-management-otel-collector
 	$(CONTAINER_BUILDENV) $(CONTAINER_CLITOOL) build -t mock-collector . \
 		--no-cache -f ./test/mock/collector/mock-collector/Dockerfile
 

test/docker/entrypoint.sh

@@ -15,14 +15,15 @@ trap 'handle_term' TERM
 
 # Launch nginx
 echo "starting nginx ..."
+/bin/su -s /bin/sh -c "/usr/share/ts/bin/bd-socket-plugin tmm_count 4 proc_cpuinfo_cpu_mhz 2000000 total_xml_memory 307200000 total_umu_max_size 3129344 sys_max_account_id 1024 no_static_config 2>&1 >> /var/log/app_protect/bd-socket-plugin.log &" nginx
 /usr/sbin/nginx -g "daemon off;" &
 
 nginx_pid=$!
 
 SECONDS=0
 
 while ! ps -ef | grep "nginx: master process" | grep -v grep; do 
-    if (( SECONDS > 5 )); then 
+    if (( SECONDS > 30 )); then
         echo "couldn't find nginx master process"
         exit 1
     fi

test/docker/nginx-plus/deb/Dockerfile

@@ -1,5 +1,5 @@
 ARG BASE_IMAGE
-FROM ${BASE_IMAGE} as install-nginx
+FROM --platform=linux/amd64 ${BASE_IMAGE} as install-nginx
 LABEL maintainer="NGINX Docker Maintainers <[email protected]>"
 
 ARG DEBIAN_FRONTEND=noninteractive
@@ -19,14 +19,19 @@ RUN --mount=type=secret,id=nginx-crt,dst=nginx-repo.crt \
 # Create nginx user/group first, to be consistent throughout Docker variants
     && groupadd --system --gid 101 nginx \
     && useradd --system --gid nginx --no-create-home --home-dir /nonexistent --uid 101 nginx \
-    && apt-get update \
+    && apt-get update --allow-releaseinfo-change \
     && apt-get install --no-install-recommends --no-install-suggests -y \
                         ca-certificates \
                         gnupg1 \
                         lsb-release \
+                        apt-transport-https \
                         git \
                         wget \
                         make \
+                        gnupg2 \
+                        ubuntu-keyring \
+    && wget -qO - https://cs.nginx.com/static/keys/app-protect-security-updates.key | gpg --dearmor | tee /usr/share/keyrings/app-protect-security-updates.gpg >/dev/null \
+    && wget -qO - https://cs.nginx.com/static/keys/nginx_signing.key | gpg --dearmor | tee /usr/share/keyrings/nginx-archive-keyring.gpg >/dev/null \
     && \
     NGINX_GPGKEY=573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62; \
     found=''; \
@@ -43,13 +48,15 @@ RUN --mount=type=secret,id=nginx-crt,dst=nginx-repo.crt \
 # Uncomment individual modules if necessary
 # Use versioned packages over defaults to specify a release
     && nginxPackages=" \
-        nginx-plus \
+        app-protect \
     " \
     && echo "Acquire::https::pkgs.nginx.com::Verify-Peer \"true\";" > /etc/apt/apt.conf.d/90nginx \
     && echo "Acquire::https::pkgs.nginx.com::Verify-Host \"true\";" >> /etc/apt/apt.conf.d/90nginx \
     && echo "Acquire::https::pkgs.nginx.com::SslCert     \"/etc/ssl/nginx/nginx-repo.crt\";" >> /etc/apt/apt.conf.d/90nginx \
     && echo "Acquire::https::pkgs.nginx.com::SslKey      \"/etc/ssl/nginx/nginx-repo.key\";" >> /etc/apt/apt.conf.d/90nginx \
     && printf "deb https://pkgs.nginx.com/plus/${PLUS_VERSION}/ubuntu/ `lsb_release -cs` nginx-plus\n" > /etc/apt/sources.list.d/nginx-plus.list \
+    && printf "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] https://pkgs.nginx.com/app-protect/${PLUS_VERSION}/ubuntu `lsb_release -cs` nginx-plus\n" | tee /etc/apt/sources.list.d/nginx-app-protect.list \
+    && printf "deb [signed-by=/usr/share/keyrings/app-protect-security-updates.gpg] https://pkgs.nginx.com/app-protect-security-updates/ubuntu `lsb_release -cs` nginx-plus\n" | tee -a /etc/apt/sources.list.d/nginx-app-protect.list \
     && mkdir -p /etc/ssl/nginx \
     && cat nginx-repo.crt > /etc/ssl/nginx/nginx-repo.crt \
     && cat nginx-repo.key > /etc/ssl/nginx/nginx-repo.key \
@@ -59,9 +66,8 @@ RUN --mount=type=secret,id=nginx-crt,dst=nginx-repo.crt \
                         curl \
                         gettext-base \
                         jq \
-                        gnupg2 \
     && apt-get remove --purge -y lsb-release \
-    && apt-get remove --purge --auto-remove -y && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/nginx-plus.list \
+    && apt-get remove --purge --auto-remove -y && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/nginx-plus.list /etc/apt/sources.list.d/nginx-app-protect.list \
     && rm -rf /etc/apt/apt.conf.d/90nginx /etc/ssl/nginx
 
 EXPOSE 80

test/mock/collector/README.md

@@ -1,30 +1,35 @@
 # Mock Management OTel Collector
 
 There are 3 images that need to be built in order to use the mock management OTel collector
-* Agent with NGINX Plus image
+* Agent with NGINX Plus & NGINX App Protect image
 * Agent with NGINX OSS image
 * Custom OTel collector image
 
 To build these images run the following
 ```
-make local-deb-package build-test-plus-image build-test-oss-image build-mock-collector-image
+OSARCH=amd64 make local-deb-package build-test-plus-image
+make local-deb-package build-test-oss-image build-mock-management-otel-collector-image
 ```
 
+[**Note:** We need to build the test NGINX Plus image with the environment variable `OSARCH=amd64` since NGINX App Protect doesn't support ARM yet.]
+
 To start run everything run the following
 ```
 make run-mock-management-otel-collector
 ```
 
-Once everything is started there should be 5 containers running
-```
-8e6df6d0bc73  localhost/nginx_plus_agent_ubuntu_22.04:latest                        4 minutes ago  Up 4 minutes              80/tcp, 443/tcp                                 mock-collector-agent-with-nginx-plus
-a65a7efaf2b3  localhost/nginx_oss_agent_ubuntu_22.04:latest                         4 minutes ago  Up 4 minutes              80/tcp, 443/tcp                                 mock-collector-agent-with-nginx-oss
-bf0f247991c0  localhost/mock-collector:latest                 go run main.go        4 minutes ago  Up 4 minutes              0.0.0.0:4320->4317/tcp, 0.0.0.0:9775->9090/tcp  mock-collector-otel-collector
-67bb7bde6392  docker.io/prom/prometheus:latest                --config.file=/et...  4 minutes ago  Up 4 minutes              0.0.0.0:9090->9090/tcp, 9090/tcp                mock-collector-prometheus
-a83a997eb652  docker.io/grafana/grafana:latest                                      4 minutes ago  Up 4 minutes              0.0.0.0:3002->3000/tcp, 3000/tcp                mock-collector-grafana
+Once everything is started there should be 6 containers running
+```
+98d601aeb602  localhost/nginx_plus_agent_ubuntu_22.04:latest                        14 seconds ago  Up 14 seconds  80/tcp, 443/tcp                                 mock-collector-agent-with-nginx-plus
+abaa9f70bfb6  localhost/nginx_oss_agent_ubuntu_22.04:latest                         12 seconds ago  Up 13 seconds  80/tcp, 443/tcp                                 mock-collector-agent-with-nginx-oss
+7158c3d0035b  localhost/mock-collector:latest                 /mock-management-...  11 seconds ago  Up 11 seconds  0.0.0.0:4320->4317/tcp, 0.0.0.0:9775->9090/tcp  mock-collector-otel-collector
+3f3a50c5fff4  docker.io/prom/prometheus:latest                --config.file=/et...  9 seconds ago   Up 9 seconds   0.0.0.0:9090->9090/tcp, 9090/tcp                mock-collector-prometheus
+b4541072ef8b  docker.io/grafana/grafana:latest                                      7 seconds ago   Up 8 seconds   0.0.0.0:3002->3000/tcp, 3000/tcp                mock-collector-grafana
+52b16ef06a10  docker.io/grafana/loki:latest                   -config.file=/etc...  6 seconds ago   Up 6 seconds   0.0.0.0:3100->3100/tcp, 3100/tcp                mock-collector-loki
+
 ```
 
-To view the metrics, the grafana UI can be used by accessing this URL http://localhost:3002/login (Note: username/password is admin/admin)
+To view the metrics & logs, the grafana UI can be used by accessing this URL http://localhost:3002/login (Note: username/password is admin/admin)
 
 To stop everything run the following
 ```

test/mock/collector/docker-compose.yaml

@@ -12,7 +12,8 @@ services:
     container_name: mock-collector-agent-with-nginx-plus
     volumes:
       - ./nginx-agent.conf:/etc/nginx-agent/nginx-agent.conf
-      - ./nginx-plus:/etc/nginx/
+      - ./nginx-plus/nginx.conf:/etc/nginx/nginx.conf
+      - ./nginx-plus/conf.d/default.conf:/etc/nginx/conf.d/default.conf
     networks:
       - metrics
 
@@ -53,12 +54,27 @@ services:
     image: grafana/grafana:latest
     container_name: mock-collector-grafana
     restart: unless-stopped
+    environment:
+      GF_INSTALL_PLUGINS: "grafana-lokiexplore-app"
     ports:
       - "3002:3000"
     volumes:
       - grafana-storage:/var/lib/grafana
       - ./grafana/provisioning/datasources:/etc/grafana/provisioning/datasources
       - ./grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards
+      - ./grafana/provisioning/plugins:/etc/grafana/provisioning/plugins
       - ./grafana/provisioning/dashboards:/var/lib/grafana/dashboards
     networks:
       - metrics
+
+  loki:
+    image: grafana/loki:latest
+    container_name: mock-collector-loki
+    restart: unless-stopped
+    ports:
+      - "3100:3100"
+    volumes:
+      - ./loki-config.yaml:/etc/loki/local-config.yaml
+    command: -config.file=/etc/loki/local-config.yaml
+    networks:
+      - metrics

test/mock/collector/grafana/provisioning/datasources/datasource.yml

@@ -7,3 +7,8 @@ datasources:
     access: proxy
     url: http://prometheus:9090
     isDefault: true
+  - name: Loki
+    type: loki
+    uid: otel-loki-scraper
+    access: proxy
+    url: http://loki:3100

test/mock/collector/grafana/provisioning/plugins/plugin.yaml

@@ -0,0 +1,5 @@
+apiVersion: 1
+
+apps:
+  - type: grafana-lokiexplore-app
+    org_id: 1

test/mock/collector/loki-config.yaml

@@ -0,0 +1,36 @@
+auth_enabled: false
+
+limits_config:
+  allow_structured_metadata: true
+  volume_enabled: true
+
+server:
+  http_listen_port: 3100
+
+common:
+  ring:
+    instance_addr: 0.0.0.0
+    kvstore:
+      store: inmemory
+  replication_factor: 1
+  path_prefix: /tmp/loki
+
+schema_config:
+  configs:
+    - from: 2020-05-15
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+storage_config:
+  tsdb_shipper:
+    active_index_directory: /tmp/loki/index
+    cache_location: /tmp/loki/index_cache
+  filesystem:
+    directory: /tmp/loki/chunks
+
+pattern_ingester:
+  enabled: true

test/mock/collector/mock-collector/Dockerfile

@@ -1,6 +1,6 @@
 FROM golang:bookworm
 
-WORKDIR /mock-management-plane-collector
-COPY ./build/mock-management-plane-collector ./
+WORKDIR /mock-management-otel-collector
+COPY ./build/mock-management-otel-collector ./
 
-CMD ["/mock-management-plane-collector/collector"]
+CMD ["/mock-management-otel-collector/collector"]

test/mock/collector/mock-collector/main.go

@@ -16,10 +16,12 @@ import (
 
 	"github.com/nginx/agent/v3/test/mock/collector/mock-collector/auth"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusexporter"
+	"github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourceprocessor"
 	"go.opentelemetry.io/collector/connector"
 	"go.opentelemetry.io/collector/exporter"
 	"go.opentelemetry.io/collector/exporter/debugexporter"
 	"go.opentelemetry.io/collector/exporter/otlpexporter"
+	"go.opentelemetry.io/collector/exporter/otlphttpexporter"
 	"go.opentelemetry.io/collector/extension"
 	"go.opentelemetry.io/collector/processor"
 	"go.opentelemetry.io/collector/processor/batchprocessor"
@@ -85,13 +87,15 @@ func components() (otelcol.Factories, error) {
 		debugexporter.NewFactory(),
 		otlpexporter.NewFactory(),
 		prometheusexporter.NewFactory(),
+		otlphttpexporter.NewFactory(),
 	)
 	if err != nil {
 		return otelcol.Factories{}, err
 	}
 
 	factories.Processors, err = processor.MakeFactoryMap(
 		batchprocessor.NewFactory(),
+		resourceprocessor.NewFactory(),
 	)
 	if err != nil {
 		return otelcol.Factories{}, err

test/mock/collector/nginx-plus/nginx.conf

@@ -1,6 +1,8 @@
 user  nginx;
 worker_processes  auto;
-
+
+load_module modules/ngx_http_app_protect_module.so;
+
 error_log  /var/log/nginx/error.log notice;
 pid        /var/run/nginx.pid;
 
@@ -80,6 +82,23 @@ http {
             status_zone my_location_zone2;
        }
     }
+
+     server {
+         listen       8099;
+         server_name  localhost;
+         proxy_http_version 1.1;
+
+         app_protect_enable on;
+         app_protect_policy_file "/etc/app_protect/conf/NginxDefaultPolicy.json";
+         app_protect_security_log_enable on;
+         app_protect_security_log "/etc/app_protect/conf/log_default.json" syslog:server=127.0.0.1:5141; 
+
+         location / {
+             client_max_body_size 0;
+             default_type text/html;
+             proxy_pass http://172.29.38.211/;
+         }
+     }
 
    include /etc/nginx/conf.d/*.conf;
 }

test/mock/collector/otel-collector.yaml

@@ -15,9 +15,16 @@ exporters:
     verbosity: detailed
     sampling_initial: 5
     sampling_thereafter: 200
+  otlphttp:
+    endpoint: http://loki:3100/otlp
 
 processors:
   batch:
+  resource:
+    attributes:
+      - key: service.name
+        value: "nginx.app.protect"
+        action: insert
 
 extensions:
   headers_check:
@@ -34,5 +41,5 @@ service:
       exporters: [prometheus]
     logs:
       receivers: [otlp]
-      processors: [batch]
-      exporters: [debug]
+      processors: [resource, batch]
+      exporters: [otlphttp]