From c5deb61da23e0d5bf8b0d5f49793737a34571a86 Mon Sep 17 00:00:00 2001 From: RRashmit Date: Wed, 13 Nov 2024 16:25:57 +0000 Subject: [PATCH 01/24] feat: add syslog receiver --- internal/collector/factories.go | 2 ++ internal/collector/factories_test.go | 2 +- .../collector/otel_collector_plugin_test.go | 10 +++++++++ internal/collector/otelcol.tmpl | 22 +++++++++++++++++++ internal/config/types.go | 7 ++++++ .../test-opentelemetry-collector-agent.yaml | 12 ++++++++++ test/docker/nginx-plus/deb/Dockerfile | 19 +++++++++------- test/types/config.go | 5 +++++ 8 files changed, 70 insertions(+), 9 deletions(-) diff --git a/internal/collector/factories.go b/internal/collector/factories.go index df3a431f64..75e6e50198 100644 --- a/internal/collector/factories.go +++ b/internal/collector/factories.go @@ -32,6 +32,7 @@ import ( "github.com/open-telemetry/opentelemetry-collector-contrib/processor/tailsamplingprocessor" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor" "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/hostmetricsreceiver" + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/syslogreceiver" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/connector" "go.opentelemetry.io/collector/exporter" @@ -110,6 +111,7 @@ func createReceiverFactories() (map[component.Type]receiver.Factory, error) { hostmetricsreceiver.NewFactory(), nginxreceiver.NewFactory(), nginxplusreceiver.NewFactory(), + syslogreceiver.NewFactory(), } return receiver.MakeFactoryMap(receiverList...) diff --git a/internal/collector/factories_test.go b/internal/collector/factories_test.go index 5cefda51eb..ca3d9423d5 100644 --- a/internal/collector/factories_test.go +++ b/internal/collector/factories_test.go @@ -17,7 +17,7 @@ func TestOTelComponentFactories(t *testing.T) { require.NoError(t, err, "OTelComponentFactories should not return an error") assert.NotNil(t, factories, "factories should not be nil") - assert.Len(t, factories.Receivers, 4) + assert.Len(t, factories.Receivers, 5) assert.Len(t, factories.Processors, 20) assert.Len(t, factories.Exporters, 4) assert.Len(t, factories.Extensions, 3) diff --git a/internal/collector/otel_collector_plugin_test.go b/internal/collector/otel_collector_plugin_test.go index 9e14c73cf4..99ae3c3055 100644 --- a/internal/collector/otel_collector_plugin_test.go +++ b/internal/collector/otel_collector_plugin_test.go @@ -120,6 +120,11 @@ func TestCollector_ProcessNginxConfigUpdateTopic(t *testing.T) { Network: &config.NetworkScraper{}, }, }, + SyslogReceiver: &config.SyslogReceiver{ + Host: "127.0.0.1", + Port: "515", + Protocol: "rfc5424", + }, OtlpReceivers: types.OtlpReceivers(), NginxPlusReceivers: []config.NginxPlusReceiver{ { @@ -157,6 +162,11 @@ func TestCollector_ProcessNginxConfigUpdateTopic(t *testing.T) { }, }, OtlpReceivers: types.OtlpReceivers(), + SyslogReceiver: &config.SyslogReceiver{ + Host: "127.0.0.1", + Port: "515", + Protocol: "rfc5424", + }, NginxReceivers: []config.NginxReceiver{ { InstanceID: "123", diff --git a/internal/collector/otelcol.tmpl b/internal/collector/otelcol.tmpl index e6b3baca83..b49e5f9f6f 100644 --- a/internal/collector/otelcol.tmpl +++ b/internal/collector/otelcol.tmpl @@ -1,4 +1,10 @@ receivers: +{{- if ne .Receivers.SyslogReceiver nil }} + syslog: + tcp: + listen_address: "{{- .Receivers.SyslogReceiver.Host -}}:{{- .Receivers.SyslogReceiver.Port -}}" + protocol: {{ .Receivers.SyslogReceiver.Protocol }} +{{- end }} {{- if ne .Receivers.HostMetrics nil }} hostmetrics: collection_interval: {{ .Receivers.HostMetrics.CollectionInterval }} @@ -233,3 +239,19 @@ service: {{- if ne .Exporters.Debug nil }} - debug {{- end }} + logs: + receivers: + {{- if ne .Receivers.SyslogReceiver nil }} + - syslog + {{- end }} + processors: + {{- if ne .Processors.Batch nil }} + - batch + {{- end }} + exporters: + {{- range $index, $otlpExporter := .Exporters.OtlpExporters }} + - otlp/{{$index}} + {{- end }} + {{- if ne .Exporters.Debug nil }} + - debug + {{- end }} diff --git a/internal/config/types.go b/internal/config/types.go index 0e36ae0819..2ec27c880b 100644 --- a/internal/config/types.go +++ b/internal/config/types.go @@ -163,6 +163,7 @@ type ( OtlpReceivers []OtlpReceiver `yaml:"-" mapstructure:"otlp_receivers"` NginxReceivers []NginxReceiver `yaml:"-" mapstructure:"nginx_receivers"` NginxPlusReceivers []NginxPlusReceiver `yaml:"-" mapstructure:"nginx_plus_receivers"` + SyslogReceiver *SyslogReceiver `yaml:"-" mapstructure:"syslog_receiver"` } OtlpReceiver struct { @@ -171,6 +172,12 @@ type ( OtlpTLSConfig *OtlpTLSConfig `yaml:"-" mapstructure:"tls"` } + SyslogReceiver struct { + Host string `yaml:"-" mapstructure:"host"` + Port string `yaml:"-" mapstructure:"port"` + Protocol string `yaml:"-" mapstructure:"protocol"` + } + NginxReceiver struct { InstanceID string `yaml:"-" mapstructure:"instance_id"` StubStatus string `yaml:"-" mapstructure:"stub_status"` diff --git a/test/config/collector/test-opentelemetry-collector-agent.yaml b/test/config/collector/test-opentelemetry-collector-agent.yaml index a4f5084a26..c88d3929ee 100644 --- a/test/config/collector/test-opentelemetry-collector-agent.yaml +++ b/test/config/collector/test-opentelemetry-collector-agent.yaml @@ -1,4 +1,8 @@ receivers: + syslog: + tcp: + listen_address: "127.0.0.1:515" + protocol: rfc5424 hostmetrics: collection_interval: 1m0s initial_delay: 1s @@ -84,3 +88,11 @@ service: - otlp/0 - prometheus - debug + logs: + receivers: + - syslog + processors: + - batch + exporters: + - otlp/0 + - debug diff --git a/test/docker/nginx-plus/deb/Dockerfile b/test/docker/nginx-plus/deb/Dockerfile index 0754e4bd02..6fa083c6fb 100644 --- a/test/docker/nginx-plus/deb/Dockerfile +++ b/test/docker/nginx-plus/deb/Dockerfile @@ -14,7 +14,7 @@ COPY $ENTRY_POINT /agent/entrypoint.sh RUN --mount=type=secret,id=nginx-crt,dst=nginx-repo.crt \ --mount=type=secret,id=nginx-key,dst=nginx-repo.key \ set -x \ -# Create nginx user/group first, to be consistent throughout Docker variants + # Create nginx user/group first, to be consistent throughout Docker variants && groupadd --system --gid 101 nginx \ && useradd --system --gid nginx --no-create-home --home-dir /nonexistent --uid 101 nginx \ && apt-get update \ @@ -37,20 +37,23 @@ RUN --mount=type=secret,id=nginx-crt,dst=nginx-repo.crt \ done; \ test -z "$found" && echo >&2 "error: failed to fetch GPG key $NGINX_GPGKEY" && exit 1; \ apt-get remove --purge --auto-remove -y gnupg1 && rm -rf /var/lib/apt/lists/* \ -# Install the latest release of NGINX Plus and/or NGINX Plus modules -# Uncomment individual modules if necessary -# Use versioned packages over defaults to specify a release - && nginxPackages=" \ - nginx-plus \ - " \ + # Install the latest release of NGINX Plus and/or NGINX Plus modules + # Uncomment individual modules if necessary + # Use versioned packages over defaults to specify a release + && nginxPackages="nginx-plus app-protect" \ + # Enable certificate-based authentication for the NGINX repository && echo "Acquire::https::pkgs.nginx.com::Verify-Peer \"true\";" > /etc/apt/apt.conf.d/90nginx \ && echo "Acquire::https::pkgs.nginx.com::Verify-Host \"true\";" >> /etc/apt/apt.conf.d/90nginx \ && echo "Acquire::https::pkgs.nginx.com::SslCert \"/etc/ssl/nginx/nginx-repo.crt\";" >> /etc/apt/apt.conf.d/90nginx \ && echo "Acquire::https::pkgs.nginx.com::SslKey \"/etc/ssl/nginx/nginx-repo.key\";" >> /etc/apt/apt.conf.d/90nginx \ + # Set up repository sources for NGINX Plus and App Protect && printf "deb https://pkgs.nginx.com/plus/ubuntu `lsb_release -cs` nginx-plus\n" > /etc/apt/sources.list.d/nginx-plus.list \ + && printf "deb https://pkgs.nginx.com/app-protect/ubuntu `lsb_release -cs` nginx-plus\n" > /etc/apt/sources.list.d/nginx-app-protect.list \ + # Copy certificates and keys for repository access && mkdir -p /etc/ssl/nginx \ && cat nginx-repo.crt > /etc/ssl/nginx/nginx-repo.crt \ && cat nginx-repo.key > /etc/ssl/nginx/nginx-repo.key \ + # Update and install NGINX packages && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ $nginxPackages \ @@ -59,7 +62,7 @@ RUN --mount=type=secret,id=nginx-crt,dst=nginx-repo.crt \ jq \ gnupg2 \ && apt-get remove --purge -y lsb-release \ - && apt-get remove --purge --auto-remove -y && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/nginx-plus.list \ + && apt-get remove --purge --auto-remove -y && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/nginx-plus.list /etc/apt/sources.list.d/nginx-app-protect.list \ && rm -rf /etc/apt/apt.conf.d/90nginx /etc/ssl/nginx EXPOSE 80 diff --git a/test/types/config.go b/test/types/config.go index 3d570c9152..ecc97c8608 100644 --- a/test/types/config.go +++ b/test/types/config.go @@ -67,6 +67,11 @@ func AgentConfig() *config.Config { }, Receivers: config.Receivers{ OtlpReceivers: OtlpReceivers(), + SyslogReceiver: &config.SyslogReceiver{ + Host: "127.0.0.1", + Port: "515", + Protocol: "rfc5424", + }, HostMetrics: &config.HostMetrics{ CollectionInterval: time.Minute, InitialDelay: time.Second, From 016275714d8d11f0dc54b70bbe11e4157c776248 Mon Sep 17 00:00:00 2001 From: RRashmit Date: Wed, 13 Nov 2024 16:55:41 +0000 Subject: [PATCH 02/24] chore: update port to non priviledged --- internal/collector/otel_collector_plugin_test.go | 4 ++-- test/config/collector/test-opentelemetry-collector-agent.yaml | 2 +- test/types/config.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/collector/otel_collector_plugin_test.go b/internal/collector/otel_collector_plugin_test.go index 99ae3c3055..608a7a73fa 100644 --- a/internal/collector/otel_collector_plugin_test.go +++ b/internal/collector/otel_collector_plugin_test.go @@ -122,7 +122,7 @@ func TestCollector_ProcessNginxConfigUpdateTopic(t *testing.T) { }, SyslogReceiver: &config.SyslogReceiver{ Host: "127.0.0.1", - Port: "515", + Port: "1515", Protocol: "rfc5424", }, OtlpReceivers: types.OtlpReceivers(), @@ -164,7 +164,7 @@ func TestCollector_ProcessNginxConfigUpdateTopic(t *testing.T) { OtlpReceivers: types.OtlpReceivers(), SyslogReceiver: &config.SyslogReceiver{ Host: "127.0.0.1", - Port: "515", + Port: "1515", Protocol: "rfc5424", }, NginxReceivers: []config.NginxReceiver{ diff --git a/test/config/collector/test-opentelemetry-collector-agent.yaml b/test/config/collector/test-opentelemetry-collector-agent.yaml index c88d3929ee..29bd572f54 100644 --- a/test/config/collector/test-opentelemetry-collector-agent.yaml +++ b/test/config/collector/test-opentelemetry-collector-agent.yaml @@ -1,7 +1,7 @@ receivers: syslog: tcp: - listen_address: "127.0.0.1:515" + listen_address: "127.0.0.1:1515" protocol: rfc5424 hostmetrics: collection_interval: 1m0s diff --git a/test/types/config.go b/test/types/config.go index ecc97c8608..bd88cfc19a 100644 --- a/test/types/config.go +++ b/test/types/config.go @@ -69,7 +69,7 @@ func AgentConfig() *config.Config { OtlpReceivers: OtlpReceivers(), SyslogReceiver: &config.SyslogReceiver{ Host: "127.0.0.1", - Port: "515", + Port: "1515", Protocol: "rfc5424", }, HostMetrics: &config.HostMetrics{ From 036aba93c7bfc2786dd754eab5703452bbf65ad8 Mon Sep 17 00:00:00 2001 From: RRashmit Date: Mon, 18 Nov 2024 10:00:04 +0000 Subject: [PATCH 03/24] chore: update the syslog type and template --- internal/collector/otel_collector_plugin.go | 11 +++ .../collector/otel_collector_plugin_test.go | 18 ++-- internal/collector/otelcol.tmpl | 12 +-- internal/config/defaults.go | 1 + internal/config/types.go | 8 +- internal/model/config.go | 7 ++ .../watcher/instance/nginx_config_parser.go | 17 ++++ .../test-opentelemetry-collector-agent.yaml | 6 +- test/docker/nginx-plus/deb/Dockerfile | 96 ++++++++++--------- test/types/config.go | 10 +- 10 files changed, 115 insertions(+), 71 deletions(-) diff --git a/internal/collector/otel_collector_plugin.go b/internal/collector/otel_collector_plugin.go index d2879dbfbc..822071d8cd 100644 --- a/internal/collector/otel_collector_plugin.go +++ b/internal/collector/otel_collector_plugin.go @@ -365,6 +365,16 @@ func (oc *Collector) checkForNewNginxReceivers(nginxConfigContext *model.NginxCo PlusAPI: nginxConfigContext.PlusAPI, }, ) + if nginxConfigContext.Syslog != nil && nginxConfigContext.Syslog.SyslogServer != "" { + oc.config.Collector.Receivers.SyslogReceivers = append( + oc.config.Collector.Receivers.SyslogReceivers, + config.SyslogReceiver{ + InstanceID: nginxConfigContext.InstanceID, + Server: nginxConfigContext.Syslog.SyslogServer, + Protocol: "rfc3164", // default value, need to get from the agent conf + }, + ) + } reloadCollector = true } else if nginxConfigContext.PlusAPI == "" { @@ -387,6 +397,7 @@ func (oc *Collector) checkForNewNginxReceivers(nginxConfigContext *model.NginxCo return reloadCollector } +// need to update new syslogreceivers func (oc *Collector) updateExistingNginxPlusReceiver( nginxConfigContext *model.NginxConfigContext, ) (nginxReceiverFound, reloadCollector bool) { diff --git a/internal/collector/otel_collector_plugin_test.go b/internal/collector/otel_collector_plugin_test.go index 608a7a73fa..5727e764dc 100644 --- a/internal/collector/otel_collector_plugin_test.go +++ b/internal/collector/otel_collector_plugin_test.go @@ -120,10 +120,11 @@ func TestCollector_ProcessNginxConfigUpdateTopic(t *testing.T) { Network: &config.NetworkScraper{}, }, }, - SyslogReceiver: &config.SyslogReceiver{ - Host: "127.0.0.1", - Port: "1515", - Protocol: "rfc5424", + SyslogReceivers: []config.SyslogReceiver{ + { + Server: "127.0.0.1:1515", + InstanceID: "1", + Protocol: "rfc3164"}, }, OtlpReceivers: types.OtlpReceivers(), NginxPlusReceivers: []config.NginxPlusReceiver{ @@ -162,10 +163,11 @@ func TestCollector_ProcessNginxConfigUpdateTopic(t *testing.T) { }, }, OtlpReceivers: types.OtlpReceivers(), - SyslogReceiver: &config.SyslogReceiver{ - Host: "127.0.0.1", - Port: "1515", - Protocol: "rfc5424", + SyslogReceivers: []config.SyslogReceiver{ + { + Server: "127.0.0.1:1515", + InstanceID: "1", + Protocol: "rfc3164"}, }, NginxReceivers: []config.NginxReceiver{ { diff --git a/internal/collector/otelcol.tmpl b/internal/collector/otelcol.tmpl index b49e5f9f6f..8d5de0fdfc 100644 --- a/internal/collector/otelcol.tmpl +++ b/internal/collector/otelcol.tmpl @@ -1,9 +1,9 @@ receivers: -{{- if ne .Receivers.SyslogReceiver nil }} - syslog: +{{- range .Receivers.SyslogReceivers }} + syslog/{{- .InstanceID -}}: tcp: - listen_address: "{{- .Receivers.SyslogReceiver.Host -}}:{{- .Receivers.SyslogReceiver.Port -}}" - protocol: {{ .Receivers.SyslogReceiver.Protocol }} + listen_address: "{{- .Server -}}" + protocol: rfc3164 {{- end }} {{- if ne .Receivers.HostMetrics nil }} hostmetrics: @@ -241,8 +241,8 @@ service: {{- end }} logs: receivers: - {{- if ne .Receivers.SyslogReceiver nil }} - - syslog + {{- range .Receivers.SyslogReceivers }} + - syslog/{{- .InstanceID -}} {{- end }} processors: {{- if ne .Processors.Batch nil }} diff --git a/internal/config/defaults.go b/internal/config/defaults.go index 526ea914c9..b5de7ffcb0 100644 --- a/internal/config/defaults.go +++ b/internal/config/defaults.go @@ -12,6 +12,7 @@ import ( ) const ( + // Add default nap parameter DefGracefulShutdownPeriod = 5 * time.Second DefNginxReloadMonitoringPeriod = 10 * time.Second DefTreatErrorsAsWarnings = false diff --git a/internal/config/types.go b/internal/config/types.go index 2ec27c880b..996c188168 100644 --- a/internal/config/types.go +++ b/internal/config/types.go @@ -163,7 +163,7 @@ type ( OtlpReceivers []OtlpReceiver `yaml:"-" mapstructure:"otlp_receivers"` NginxReceivers []NginxReceiver `yaml:"-" mapstructure:"nginx_receivers"` NginxPlusReceivers []NginxPlusReceiver `yaml:"-" mapstructure:"nginx_plus_receivers"` - SyslogReceiver *SyslogReceiver `yaml:"-" mapstructure:"syslog_receiver"` + SyslogReceivers []SyslogReceiver `yaml:"-" mapstructure:"syslog_receiver"` } OtlpReceiver struct { @@ -173,9 +173,9 @@ type ( } SyslogReceiver struct { - Host string `yaml:"-" mapstructure:"host"` - Port string `yaml:"-" mapstructure:"port"` - Protocol string `yaml:"-" mapstructure:"protocol"` + InstanceID string `yaml:"-" mapstructure:"instance_id"` + Server string `yaml:"-" mapstructure:"host"` + Protocol string `yaml:"-" mapstructure:"protocol"` } NginxReceiver struct { diff --git a/internal/model/config.go b/internal/model/config.go index 3f37428912..ef60d0e568 100644 --- a/internal/model/config.go +++ b/internal/model/config.go @@ -18,6 +18,7 @@ type NginxConfigContext struct { Files []*v1.File AccessLogs []*AccessLog ErrorLogs []*ErrorLog + Syslog *NAP } func (ncc *NginxConfigContext) Equal(otherNginxConfigContext *NginxConfigContext) bool { @@ -85,6 +86,12 @@ type ErrorLog struct { Readable bool } +type NAP struct { + Enable bool + Syslog bool + SyslogServer string +} + type ( WriteStatus int ) diff --git a/internal/watcher/instance/nginx_config_parser.go b/internal/watcher/instance/nginx_config_parser.go index ab147e2bd5..e0f7621e1d 100644 --- a/internal/watcher/instance/nginx_config_parser.go +++ b/internal/watcher/instance/nginx_config_parser.go @@ -16,6 +16,7 @@ import ( "net/http" "os" "path/filepath" + "regexp" "slices" "strconv" "strings" @@ -36,6 +37,7 @@ const ( stubStatusAPIDirective = "stub_status" apiFormat = "http://%s%s" locationDirective = "location" + napDirective = "app_protect_security_log" ) type ( @@ -104,6 +106,7 @@ func (ncp *NginxConfigParser) createNginxConfigContext( case "log_format": formatMap = ncp.formatMap(directive) case "access_log": + fmt.Println("am here") if !ncp.ignoreLog(directive.Args[0]) { accessLog := ncp.accessLog(directive.Args[0], ncp.accessLogDirectiveFormat(directive), formatMap) @@ -120,6 +123,20 @@ func (ncp *NginxConfigParser) createNginxConfigContext( case "ssl_certificate", "proxy_ssl_certificate", "ssl_client_certificate", "ssl_trusted_certificate": sslCertFile := ncp.sslCert(ctx, directive.Args[0], rootDir) nginxConfigContext.Files = append(nginxConfigContext.Files, sslCertFile) + case "app_protect_security_log": + // Extract the syslog:server part of the app_protect_security_log directive + if len(directive.Args) > 1 { + syslogArg := directive.Args[1] + re := regexp.MustCompile(`syslog:server=([\S]+)`) + matches := re.FindStringSubmatch(syslogArg) + if len(matches) > 1 { + syslogServer := matches[1] + nginxConfigContext.Syslog.SyslogServer = syslogServer + slog.InfoContext(ctx, "Captured syslog server", "syslog_server", syslogServer) + fmt.Println("syslogServer") + fmt.Println(syslogServer) + } + } } return nil diff --git a/test/config/collector/test-opentelemetry-collector-agent.yaml b/test/config/collector/test-opentelemetry-collector-agent.yaml index 29bd572f54..8d6be2900c 100644 --- a/test/config/collector/test-opentelemetry-collector-agent.yaml +++ b/test/config/collector/test-opentelemetry-collector-agent.yaml @@ -1,8 +1,8 @@ receivers: - syslog: + syslog/1: tcp: listen_address: "127.0.0.1:1515" - protocol: rfc5424 + protocol: rfc3164 hostmetrics: collection_interval: 1m0s initial_delay: 1s @@ -90,7 +90,7 @@ service: - debug logs: receivers: - - syslog + - syslog/1 processors: - batch exporters: diff --git a/test/docker/nginx-plus/deb/Dockerfile b/test/docker/nginx-plus/deb/Dockerfile index 6fa083c6fb..9951d5114c 100644 --- a/test/docker/nginx-plus/deb/Dockerfile +++ b/test/docker/nginx-plus/deb/Dockerfile @@ -1,79 +1,83 @@ +# Base Image Argument ARG BASE_IMAGE FROM ${BASE_IMAGE} as install-nginx LABEL maintainer="NGINX Docker Maintainers " +# Define arguments for flexibility ARG DEBIAN_FRONTEND=noninteractive ARG ENTRY_POINT ARG PACKAGE_NAME ARG PACKAGES_REPO +# Set working directory WORKDIR /agent COPY ./build /agent/build COPY $ENTRY_POINT /agent/entrypoint.sh +# Install NGINX Plus and NGINX App Protect RUN --mount=type=secret,id=nginx-crt,dst=nginx-repo.crt \ --mount=type=secret,id=nginx-key,dst=nginx-repo.key \ - set -x \ - # Create nginx user/group first, to be consistent throughout Docker variants + set -eux \ && groupadd --system --gid 101 nginx \ && useradd --system --gid nginx --no-create-home --home-dir /nonexistent --uid 101 nginx \ + \ && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ - ca-certificates \ - gnupg1 \ - lsb-release \ - git \ - wget \ - make \ - && \ - NGINX_GPGKEY=573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62; \ - found=''; \ - for server in \ - hkp://keyserver.ubuntu.com:80 \ - pgp.mit.edu \ - ; do \ - echo "Fetching GPG key $NGINX_GPGKEY from $server"; \ - apt-key adv --keyserver "$server" --keyserver-options timeout=10 --recv-keys "$NGINX_GPGKEY" && found=yes && break; \ - done; \ - test -z "$found" && echo >&2 "error: failed to fetch GPG key $NGINX_GPGKEY" && exit 1; \ - apt-get remove --purge --auto-remove -y gnupg1 && rm -rf /var/lib/apt/lists/* \ - # Install the latest release of NGINX Plus and/or NGINX Plus modules - # Uncomment individual modules if necessary - # Use versioned packages over defaults to specify a release - && nginxPackages="nginx-plus app-protect" \ - # Enable certificate-based authentication for the NGINX repository - && echo "Acquire::https::pkgs.nginx.com::Verify-Peer \"true\";" > /etc/apt/apt.conf.d/90nginx \ - && echo "Acquire::https::pkgs.nginx.com::Verify-Host \"true\";" >> /etc/apt/apt.conf.d/90nginx \ - && echo "Acquire::https::pkgs.nginx.com::SslCert \"/etc/ssl/nginx/nginx-repo.crt\";" >> /etc/apt/apt.conf.d/90nginx \ - && echo "Acquire::https::pkgs.nginx.com::SslKey \"/etc/ssl/nginx/nginx-repo.key\";" >> /etc/apt/apt.conf.d/90nginx \ - # Set up repository sources for NGINX Plus and App Protect - && printf "deb https://pkgs.nginx.com/plus/ubuntu `lsb_release -cs` nginx-plus\n" > /etc/apt/sources.list.d/nginx-plus.list \ - && printf "deb https://pkgs.nginx.com/app-protect/ubuntu `lsb_release -cs` nginx-plus\n" > /etc/apt/sources.list.d/nginx-app-protect.list \ - # Copy certificates and keys for repository access + ca-certificates \ + gnupg2 \ + lsb-release \ + git \ + wget \ + make \ + apt-transport-https \ + ubuntu-keyring \ + \ && mkdir -p /etc/ssl/nginx \ && cat nginx-repo.crt > /etc/ssl/nginx/nginx-repo.crt \ && cat nginx-repo.key > /etc/ssl/nginx/nginx-repo.key \ - # Update and install NGINX packages + \ + # Add NGINX repositories securely + && wget -qO - https://cs.nginx.com/static/keys/nginx_signing.key | gpg --dearmor > /usr/share/keyrings/nginx-archive-keyring.gpg \ + && wget -qO - https://cs.nginx.com/static/keys/app-protect-security-updates.key | gpg --dearmor > /usr/share/keyrings/app-protect-security-updates.gpg \ + \ + && printf "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] https://pkgs.nginx.com/plus/ubuntu $(lsb_release -cs) nginx-plus\n" > /etc/apt/sources.list.d/nginx-plus.list \ + && printf "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] https://pkgs.nginx.com/app-protect/ubuntu $(lsb_release -cs) nginx-plus\n" > /etc/apt/sources.list.d/nginx-app-protect.list \ + && printf "deb [signed-by=/usr/share/keyrings/app-protect-security-updates.gpg] https://pkgs.nginx.com/app-protect-security-updates/ubuntu $(lsb_release -cs) nginx-plus\n" >> /etc/apt/sources.list.d/nginx-app-protect.list \ + \ + && wget -P /etc/apt/apt.conf.d https://cs.nginx.com/static/files/90pkgs-nginx \ + \ && apt-get update \ + && apt-cache show nginx-plus || echo "nginx-plus package not found" \ + && apt-cache show app-protect || echo "app-protect package not found" \ && apt-get install --no-install-recommends --no-install-suggests -y \ - $nginxPackages \ - curl \ - gettext-base \ - jq \ - gnupg2 \ + nginx-plus \ + app-protect \ + app-protect-attack-signatures \ + curl \ + gettext-base \ + jq \ + \ && apt-get remove --purge -y lsb-release \ - && apt-get remove --purge --auto-remove -y && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/nginx-plus.list /etc/apt/sources.list.d/nginx-app-protect.list \ - && rm -rf /etc/apt/apt.conf.d/90nginx /etc/ssl/nginx + && apt-get remove --purge --auto-remove -y \ + \ + # Clean up + && rm -rf /var/lib/apt/lists/* \ + /etc/apt/sources.list.d/nginx-plus.list \ + /etc/apt/sources.list.d/nginx-app-protect.list \ + /etc/apt/apt.conf.d/90nginx \ + /etc/ssl/nginx -EXPOSE 80 +# Expose ports +EXPOSE 80 443 +# Set stop signal STOPSIGNAL SIGQUIT +# Set executable permissions for entrypoint RUN chmod +x /agent/entrypoint.sh -RUN apt install -y /agent/build/${PACKAGE_NAME}.deb - -STOPSIGNAL SIGTERM -EXPOSE 80 443 +# Install NGINX Agent package +RUN apt install -y /agent/build/${PACKAGE_NAME}.deb || { echo "Failed to install ${PACKAGE_NAME}"; exit 1; } +# Define entrypoint ENTRYPOINT ["/agent/entrypoint.sh"] diff --git a/test/types/config.go b/test/types/config.go index bd88cfc19a..d1e5cfccf3 100644 --- a/test/types/config.go +++ b/test/types/config.go @@ -67,10 +67,12 @@ func AgentConfig() *config.Config { }, Receivers: config.Receivers{ OtlpReceivers: OtlpReceivers(), - SyslogReceiver: &config.SyslogReceiver{ - Host: "127.0.0.1", - Port: "1515", - Protocol: "rfc5424", + SyslogReceivers: []config.SyslogReceiver{ + { + Server: "127.0.0.1:1515", + InstanceID: "1", + Protocol: "rfc3164", + }, }, HostMetrics: &config.HostMetrics{ CollectionInterval: time.Minute, From 0e138e7ccb330b248603591c947b5486c4bae498 Mon Sep 17 00:00:00 2001 From: RRashmit Date: Mon, 18 Nov 2024 10:14:28 +0000 Subject: [PATCH 04/24] chore: update comments --- internal/collector/otel_collector_plugin.go | 2 +- internal/config/defaults.go | 2 +- .../watcher/instance/nginx_config_parser.go | 1 - test/docker/nginx-plus/deb/Dockerfile | 67 ++++++------------- 4 files changed, 23 insertions(+), 49 deletions(-) diff --git a/internal/collector/otel_collector_plugin.go b/internal/collector/otel_collector_plugin.go index 822071d8cd..f0d8c813ca 100644 --- a/internal/collector/otel_collector_plugin.go +++ b/internal/collector/otel_collector_plugin.go @@ -397,7 +397,7 @@ func (oc *Collector) checkForNewNginxReceivers(nginxConfigContext *model.NginxCo return reloadCollector } -// need to update new syslogreceivers +// Todo: consider update scenario for new syslogreceivers func (oc *Collector) updateExistingNginxPlusReceiver( nginxConfigContext *model.NginxConfigContext, ) (nginxReceiverFound, reloadCollector bool) { diff --git a/internal/config/defaults.go b/internal/config/defaults.go index b5de7ffcb0..b386a236b6 100644 --- a/internal/config/defaults.go +++ b/internal/config/defaults.go @@ -12,7 +12,7 @@ import ( ) const ( - // Add default nap parameter + //Todo: Add default Protocol for nap DefGracefulShutdownPeriod = 5 * time.Second DefNginxReloadMonitoringPeriod = 10 * time.Second DefTreatErrorsAsWarnings = false diff --git a/internal/watcher/instance/nginx_config_parser.go b/internal/watcher/instance/nginx_config_parser.go index e0f7621e1d..52f0c09eec 100644 --- a/internal/watcher/instance/nginx_config_parser.go +++ b/internal/watcher/instance/nginx_config_parser.go @@ -124,7 +124,6 @@ func (ncp *NginxConfigParser) createNginxConfigContext( sslCertFile := ncp.sslCert(ctx, directive.Args[0], rootDir) nginxConfigContext.Files = append(nginxConfigContext.Files, sslCertFile) case "app_protect_security_log": - // Extract the syslog:server part of the app_protect_security_log directive if len(directive.Args) > 1 { syslogArg := directive.Args[1] re := regexp.MustCompile(`syslog:server=([\S]+)`) diff --git a/test/docker/nginx-plus/deb/Dockerfile b/test/docker/nginx-plus/deb/Dockerfile index 9951d5114c..1cd29f0b37 100644 --- a/test/docker/nginx-plus/deb/Dockerfile +++ b/test/docker/nginx-plus/deb/Dockerfile @@ -1,15 +1,12 @@ -# Base Image Argument ARG BASE_IMAGE FROM ${BASE_IMAGE} as install-nginx LABEL maintainer="NGINX Docker Maintainers " -# Define arguments for flexibility ARG DEBIAN_FRONTEND=noninteractive ARG ENTRY_POINT ARG PACKAGE_NAME ARG PACKAGES_REPO -# Set working directory WORKDIR /agent COPY ./build /agent/build COPY $ENTRY_POINT /agent/entrypoint.sh @@ -17,67 +14,45 @@ COPY $ENTRY_POINT /agent/entrypoint.sh # Install NGINX Plus and NGINX App Protect RUN --mount=type=secret,id=nginx-crt,dst=nginx-repo.crt \ --mount=type=secret,id=nginx-key,dst=nginx-repo.key \ - set -eux \ + set -x \ && groupadd --system --gid 101 nginx \ && useradd --system --gid nginx --no-create-home --home-dir /nonexistent --uid 101 nginx \ - \ && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ - ca-certificates \ - gnupg2 \ - lsb-release \ - git \ - wget \ - make \ - apt-transport-https \ - ubuntu-keyring \ - \ + ca-certificates \ + gnupg2 \ + lsb-release \ + git \ + wget \ + make \ + apt-transport-https \ + ubuntu-keyring \ && mkdir -p /etc/ssl/nginx \ && cat nginx-repo.crt > /etc/ssl/nginx/nginx-repo.crt \ && cat nginx-repo.key > /etc/ssl/nginx/nginx-repo.key \ - \ - # Add NGINX repositories securely && wget -qO - https://cs.nginx.com/static/keys/nginx_signing.key | gpg --dearmor > /usr/share/keyrings/nginx-archive-keyring.gpg \ && wget -qO - https://cs.nginx.com/static/keys/app-protect-security-updates.key | gpg --dearmor > /usr/share/keyrings/app-protect-security-updates.gpg \ - \ - && printf "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] https://pkgs.nginx.com/plus/ubuntu $(lsb_release -cs) nginx-plus\n" > /etc/apt/sources.list.d/nginx-plus.list \ - && printf "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] https://pkgs.nginx.com/app-protect/ubuntu $(lsb_release -cs) nginx-plus\n" > /etc/apt/sources.list.d/nginx-app-protect.list \ - && printf "deb [signed-by=/usr/share/keyrings/app-protect-security-updates.gpg] https://pkgs.nginx.com/app-protect-security-updates/ubuntu $(lsb_release -cs) nginx-plus\n" >> /etc/apt/sources.list.d/nginx-app-protect.list \ - \ + && printf "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] https://pkgs.nginx.com/plus/ubuntu `lsb_release -cs` nginx-plus\n" > /etc/apt/sources.list.d/nginx-plus.list \ + && printf "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] https://pkgs.nginx.com/app-protect/ubuntu `lsb_release -cs` nginx-plus\n" > /etc/apt/sources.list.d/nginx-app-protect.list \ + && printf "deb [signed-by=/usr/share/keyrings/app-protect-security-updates.gpg] https://pkgs.nginx.com/app-protect-security-updates/ubuntu `lsb_release -cs` nginx-plus\n" >> /etc/apt/sources.list.d/nginx-app-protect.list \ && wget -P /etc/apt/apt.conf.d https://cs.nginx.com/static/files/90pkgs-nginx \ - \ && apt-get update \ - && apt-cache show nginx-plus || echo "nginx-plus package not found" \ - && apt-cache show app-protect || echo "app-protect package not found" \ && apt-get install --no-install-recommends --no-install-suggests -y \ - nginx-plus \ - app-protect \ - app-protect-attack-signatures \ - curl \ - gettext-base \ - jq \ - \ + nginx-plus \ + app-protect \ + app-protect-attack-signatures \ + curl \ + gettext-base \ + jq \ && apt-get remove --purge -y lsb-release \ - && apt-get remove --purge --auto-remove -y \ - \ - # Clean up - && rm -rf /var/lib/apt/lists/* \ - /etc/apt/sources.list.d/nginx-plus.list \ - /etc/apt/sources.list.d/nginx-app-protect.list \ - /etc/apt/apt.conf.d/90nginx \ - /etc/ssl/nginx + && apt-get remove --purge --auto-remove -y && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/nginx-plus.list /etc/apt/sources.list.d/nginx-app-protect.list \ + && rm -rf /etc/apt/apt.conf.d/90nginx /etc/ssl/nginx -# Expose ports EXPOSE 80 443 -# Set stop signal STOPSIGNAL SIGQUIT -# Set executable permissions for entrypoint RUN chmod +x /agent/entrypoint.sh +RUN apt install -y /agent/build/${PACKAGE_NAME}.deb -# Install NGINX Agent package -RUN apt install -y /agent/build/${PACKAGE_NAME}.deb || { echo "Failed to install ${PACKAGE_NAME}"; exit 1; } - -# Define entrypoint ENTRYPOINT ["/agent/entrypoint.sh"] From 4f6db4ae8b55624d629377e89135ffafccfa3ce8 Mon Sep 17 00:00:00 2001 From: RRashmit Date: Mon, 18 Nov 2024 15:49:36 +0000 Subject: [PATCH 05/24] chore: update the docker --- test/docker/nginx-plus/deb/Dockerfile | 39 ++++++++++++++++++--------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/test/docker/nginx-plus/deb/Dockerfile b/test/docker/nginx-plus/deb/Dockerfile index 1cd29f0b37..9b1aa04047 100644 --- a/test/docker/nginx-plus/deb/Dockerfile +++ b/test/docker/nginx-plus/deb/Dockerfile @@ -11,10 +11,10 @@ WORKDIR /agent COPY ./build /agent/build COPY $ENTRY_POINT /agent/entrypoint.sh -# Install NGINX Plus and NGINX App Protect RUN --mount=type=secret,id=nginx-crt,dst=nginx-repo.crt \ --mount=type=secret,id=nginx-key,dst=nginx-repo.key \ set -x \ + # Create nginx user/group first, to be consistent throughout Docker variants && groupadd --system --gid 101 nginx \ && useradd --system --gid nginx --no-create-home --home-dir /nonexistent --uid 101 nginx \ && apt-get update \ @@ -25,34 +25,47 @@ RUN --mount=type=secret,id=nginx-crt,dst=nginx-repo.crt \ git \ wget \ make \ - apt-transport-https \ - ubuntu-keyring \ && mkdir -p /etc/ssl/nginx \ - && cat nginx-repo.crt > /etc/ssl/nginx/nginx-repo.crt \ - && cat nginx-repo.key > /etc/ssl/nginx/nginx-repo.key \ - && wget -qO - https://cs.nginx.com/static/keys/nginx_signing.key | gpg --dearmor > /usr/share/keyrings/nginx-archive-keyring.gpg \ - && wget -qO - https://cs.nginx.com/static/keys/app-protect-security-updates.key | gpg --dearmor > /usr/share/keyrings/app-protect-security-updates.gpg \ + && cat nginx-repo.crt > /etc/ssl/nginx/nginx-repo.crt \ + && cat nginx-repo.key > /etc/ssl/nginx/nginx-repo.key \ + && echo "Acquire::https::pkgs.nginx.com::Verify-Peer \"true\";" > /etc/apt/apt.conf.d/90nginx \ + && echo "Acquire::https::pkgs.nginx.com::Verify-Host \"true\";" >> /etc/apt/apt.conf.d/90nginx \ + && echo "Acquire::https::pkgs.nginx.com::SslCert \"/etc/ssl/nginx/nginx-repo.crt\";" >> /etc/apt/apt.conf.d/90nginx \ + && echo "Acquire::https::pkgs.nginx.com::SslKey \"/etc/ssl/nginx/nginx-repo.key\";" >> /etc/apt/apt.conf.d/90nginx \ + # Add GPG keys for nginx-plus and app-protect + && wget -qO - https://cs.nginx.com/static/keys/nginx_signing.key | gpg --dearmor | tee /usr/share/keyrings/nginx-archive-keyring.gpg >/dev/null \ + && wget -qO - https://cs.nginx.com/static/keys/app-protect-security-updates.key | gpg --dearmor | tee /usr/share/keyrings/app-protect-security-updates.gpg >/dev/null \ + #&& ls -l /usr/share/keyrings/nginx-archive-keyring.gpg /usr/share/keyrings/app-protect-security-updates.gpg \ + + + # Install the latest release of NGINX Plus and/or NGINX Plus modules + # Set up repository sources for NGINX Plus and App Protect && printf "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] https://pkgs.nginx.com/plus/ubuntu `lsb_release -cs` nginx-plus\n" > /etc/apt/sources.list.d/nginx-plus.list \ - && printf "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] https://pkgs.nginx.com/app-protect/ubuntu `lsb_release -cs` nginx-plus\n" > /etc/apt/sources.list.d/nginx-app-protect.list \ - && printf "deb [signed-by=/usr/share/keyrings/app-protect-security-updates.gpg] https://pkgs.nginx.com/app-protect-security-updates/ubuntu `lsb_release -cs` nginx-plus\n" >> /etc/apt/sources.list.d/nginx-app-protect.list \ - && wget -P /etc/apt/apt.conf.d https://cs.nginx.com/static/files/90pkgs-nginx \ + && printf "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] https://pkgs.nginx.com/app-protect/ubuntu `lsb_release -cs` nginx-plus\n" | tee /etc/apt/sources.list.d/nginx-app-protect.list \ + && printf "deb [signed-by=/usr/share/keyrings/app-protect-security-updates.gpg] https://pkgs.nginx.com/app-protect-security-updates/ubuntu `lsb_release -cs` nginx-plus\n" | tee -a /etc/apt/sources.list.d/nginx-app-protect.list \ + # && wget -P /etc/apt/apt.conf.d https://cs.nginx.com/static/files/90pkgs-nginx \ + # && printf "deb [signed-by=/usr/share/keyrings/app-protect-security-updates.gpg] https://pkgs.nginx.com/app-protect/ubuntu `lsb_release -cs` nginx-plus\n" > /etc/apt/sources.list.d/nginx-app-protect.list \ && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ nginx-plus \ app-protect \ - app-protect-attack-signatures \ curl \ gettext-base \ jq \ && apt-get remove --purge -y lsb-release \ - && apt-get remove --purge --auto-remove -y && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/nginx-plus.list /etc/apt/sources.list.d/nginx-app-protect.list \ + && apt-get remove --purge --auto-remove -y \ + && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/nginx-plus.list /etc/apt/sources.list.d/nginx-app-protect.list \ && rm -rf /etc/apt/apt.conf.d/90nginx /etc/ssl/nginx -EXPOSE 80 443 +EXPOSE 80 STOPSIGNAL SIGQUIT RUN chmod +x /agent/entrypoint.sh RUN apt install -y /agent/build/${PACKAGE_NAME}.deb +STOPSIGNAL SIGTERM + +EXPOSE 80 443 + ENTRYPOINT ["/agent/entrypoint.sh"] From 409242c1f959cabf71674c86f4320b21acc1de7d Mon Sep 17 00:00:00 2001 From: RRashmit Date: Mon, 25 Nov 2024 11:06:46 +0000 Subject: [PATCH 06/24] chore: updated the otel collector pipeline --- internal/collector/otelcol.tmpl | 2 ++ internal/config/types.go | 2 +- internal/watcher/instance/nginx_config_parser.go | 4 ---- test/mock/collector/otel-collector.yaml | 8 ++++++++ 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/internal/collector/otelcol.tmpl b/internal/collector/otelcol.tmpl index 8d5de0fdfc..94524705be 100644 --- a/internal/collector/otelcol.tmpl +++ b/internal/collector/otelcol.tmpl @@ -239,6 +239,7 @@ service: {{- if ne .Exporters.Debug nil }} - debug {{- end }} + {{- if and (ne .Receivers.SyslogReceivers nil) (ne .Exporters.OtlpExporters nil) }} logs: receivers: {{- range .Receivers.SyslogReceivers }} @@ -255,3 +256,4 @@ service: {{- if ne .Exporters.Debug nil }} - debug {{- end }} + {{- end }} diff --git a/internal/config/types.go b/internal/config/types.go index 996c188168..20e27d9b94 100644 --- a/internal/config/types.go +++ b/internal/config/types.go @@ -174,7 +174,7 @@ type ( SyslogReceiver struct { InstanceID string `yaml:"-" mapstructure:"instance_id"` - Server string `yaml:"-" mapstructure:"host"` + Server string `yaml:"-" mapstructure:"server"` Protocol string `yaml:"-" mapstructure:"protocol"` } diff --git a/internal/watcher/instance/nginx_config_parser.go b/internal/watcher/instance/nginx_config_parser.go index 52f0c09eec..541f5c4251 100644 --- a/internal/watcher/instance/nginx_config_parser.go +++ b/internal/watcher/instance/nginx_config_parser.go @@ -106,7 +106,6 @@ func (ncp *NginxConfigParser) createNginxConfigContext( case "log_format": formatMap = ncp.formatMap(directive) case "access_log": - fmt.Println("am here") if !ncp.ignoreLog(directive.Args[0]) { accessLog := ncp.accessLog(directive.Args[0], ncp.accessLogDirectiveFormat(directive), formatMap) @@ -132,12 +131,9 @@ func (ncp *NginxConfigParser) createNginxConfigContext( syslogServer := matches[1] nginxConfigContext.Syslog.SyslogServer = syslogServer slog.InfoContext(ctx, "Captured syslog server", "syslog_server", syslogServer) - fmt.Println("syslogServer") - fmt.Println(syslogServer) } } } - return nil }, ) diff --git a/test/mock/collector/otel-collector.yaml b/test/mock/collector/otel-collector.yaml index 1fb86a2560..b1e801a9cc 100644 --- a/test/mock/collector/otel-collector.yaml +++ b/test/mock/collector/otel-collector.yaml @@ -11,6 +11,10 @@ exporters: resource_to_telemetry_conversion: enabled: true add_metric_suffixes: false + debug: + verbosity: detailed + sampling_initial: 5 + sampling_thereafter: 200 processors: batch: @@ -28,3 +32,7 @@ service: receivers: [otlp] processors: [batch] exporters: [prometheus] + logs: + receivers: [otlp] + processors: [batch] + exporters: [debug] From 3fde54ce306e5156bc25abb64f45bc087363fa4d Mon Sep 17 00:00:00 2001 From: RRashmit Date: Mon, 25 Nov 2024 17:11:57 +0000 Subject: [PATCH 07/24] chore: update multiple syslog servers --- api/grpc/mpi/v1/command.pb.go | 2 + internal/collector/otel_collector_plugin.go | 7 ++- .../collector/otel_collector_plugin_test.go | 8 +-- internal/collector/otelcol.tmpl | 9 ++-- internal/config/types.go | 6 +-- internal/model/config.go | 2 +- .../watcher/instance/nginx_config_parser.go | 6 ++- .../instance/nginx_config_parser_test.go | 4 ++ .../test-opentelemetry-collector-agent.yaml | 4 +- test/docker/nginx-plus/deb/Dockerfile | 49 ++++++++++--------- test/model/config.go | 2 + test/protos/instances.go | 3 ++ test/types/config.go | 4 +- 13 files changed, 66 insertions(+), 40 deletions(-) diff --git a/api/grpc/mpi/v1/command.pb.go b/api/grpc/mpi/v1/command.pb.go index 7f6e52f857..9645d9c5e8 100644 --- a/api/grpc/mpi/v1/command.pb.go +++ b/api/grpc/mpi/v1/command.pb.go @@ -1737,6 +1737,8 @@ type NGINXPlusRuntimeInfo struct { DynamicModules []string `protobuf:"bytes,5,rep,name=dynamic_modules,json=dynamicModules,proto3" json:"dynamic_modules,omitempty"` // the plus API location PlusApi string `protobuf:"bytes,6,opt,name=plus_api,json=plusApi,proto3" json:"plus_api,omitempty"` + // a list of App Protect directives + AppProtectDirectives []string `protobuf:"bytes,7,rep,name=app_protect_directives,json=appProtectDirectives,proto3" json:"app_protect_directives,omitempty"` } func (x *NGINXPlusRuntimeInfo) Reset() { diff --git a/internal/collector/otel_collector_plugin.go b/internal/collector/otel_collector_plugin.go index f0d8c813ca..aa329dab81 100644 --- a/internal/collector/otel_collector_plugin.go +++ b/internal/collector/otel_collector_plugin.go @@ -365,17 +365,20 @@ func (oc *Collector) checkForNewNginxReceivers(nginxConfigContext *model.NginxCo PlusAPI: nginxConfigContext.PlusAPI, }, ) - if nginxConfigContext.Syslog != nil && nginxConfigContext.Syslog.SyslogServer != "" { + + if nginxConfigContext.Syslog != nil { oc.config.Collector.Receivers.SyslogReceivers = append( oc.config.Collector.Receivers.SyslogReceivers, config.SyslogReceiver{ InstanceID: nginxConfigContext.InstanceID, - Server: nginxConfigContext.Syslog.SyslogServer, + Server: nginxConfigContext.Syslog, Protocol: "rfc3164", // default value, need to get from the agent conf }, ) } + // slog.Error("hello", oc.config.Collector.Receivers) + reloadCollector = true } else if nginxConfigContext.PlusAPI == "" { nginxReceiverFound, reloadCollector = oc.updateExistingNginxOSSReceiver(nginxConfigContext) diff --git a/internal/collector/otel_collector_plugin_test.go b/internal/collector/otel_collector_plugin_test.go index 5727e764dc..f9138b3db9 100644 --- a/internal/collector/otel_collector_plugin_test.go +++ b/internal/collector/otel_collector_plugin_test.go @@ -122,8 +122,8 @@ func TestCollector_ProcessNginxConfigUpdateTopic(t *testing.T) { }, SyslogReceivers: []config.SyslogReceiver{ { - Server: "127.0.0.1:1515", - InstanceID: "1", + Server: []string{"127.0.0.1:1515"}, + InstanceID: "00", Protocol: "rfc3164"}, }, OtlpReceivers: types.OtlpReceivers(), @@ -165,8 +165,8 @@ func TestCollector_ProcessNginxConfigUpdateTopic(t *testing.T) { OtlpReceivers: types.OtlpReceivers(), SyslogReceivers: []config.SyslogReceiver{ { - Server: "127.0.0.1:1515", - InstanceID: "1", + Server: []string{"127.0.0.1:1515"}, + InstanceID: "00", Protocol: "rfc3164"}, }, NginxReceivers: []config.NginxReceiver{ diff --git a/internal/collector/otelcol.tmpl b/internal/collector/otelcol.tmpl index 94524705be..8850551407 100644 --- a/internal/collector/otelcol.tmpl +++ b/internal/collector/otelcol.tmpl @@ -1,9 +1,12 @@ receivers: -{{- range .Receivers.SyslogReceivers }} - syslog/{{- .InstanceID -}}: +{{- range $index, $otlpReceiver := .Receivers.SyslogReceivers }} + {{- $instanceID := .InstanceID }} + {{- range $serverIndex, $server := .Server }} + syslog/{{$index}}{{$serverIndex}}: tcp: - listen_address: "{{- .Server -}}" + listen_address: "{{ $server }}" protocol: rfc3164 + {{- end }} {{- end }} {{- if ne .Receivers.HostMetrics nil }} hostmetrics: diff --git a/internal/config/types.go b/internal/config/types.go index 20e27d9b94..36bb3fae9f 100644 --- a/internal/config/types.go +++ b/internal/config/types.go @@ -173,9 +173,9 @@ type ( } SyslogReceiver struct { - InstanceID string `yaml:"-" mapstructure:"instance_id"` - Server string `yaml:"-" mapstructure:"server"` - Protocol string `yaml:"-" mapstructure:"protocol"` + InstanceID string `yaml:"-" mapstructure:"instance_id"` + Server []string `yaml:"-" mapstructure:"server"` + Protocol string `yaml:"-" mapstructure:"protocol"` } NginxReceiver struct { diff --git a/internal/model/config.go b/internal/model/config.go index ef60d0e568..faeb3c121e 100644 --- a/internal/model/config.go +++ b/internal/model/config.go @@ -18,7 +18,7 @@ type NginxConfigContext struct { Files []*v1.File AccessLogs []*AccessLog ErrorLogs []*ErrorLog - Syslog *NAP + Syslog []string } func (ncc *NginxConfigContext) Equal(otherNginxConfigContext *NginxConfigContext) bool { diff --git a/internal/watcher/instance/nginx_config_parser.go b/internal/watcher/instance/nginx_config_parser.go index 541f5c4251..e93bd8bbde 100644 --- a/internal/watcher/instance/nginx_config_parser.go +++ b/internal/watcher/instance/nginx_config_parser.go @@ -100,6 +100,7 @@ func (ncp *NginxConfigParser) createNginxConfigContext( for _, conf := range payload.Config { formatMap := make(map[string]string) + syslogMap := make(map[string]bool) err := ncp.crossplaneConfigTraverse(ctx, &conf, func(ctx context.Context, parent, directive *crossplane.Directive) error { switch directive.Directive { @@ -129,7 +130,10 @@ func (ncp *NginxConfigParser) createNginxConfigContext( matches := re.FindStringSubmatch(syslogArg) if len(matches) > 1 { syslogServer := matches[1] - nginxConfigContext.Syslog.SyslogServer = syslogServer + if !syslogMap[syslogServer] { + nginxConfigContext.Syslog = append(nginxConfigContext.Syslog, syslogServer) + syslogMap[syslogServer] = true + } slog.InfoContext(ctx, "Captured syslog server", "syslog_server", syslogServer) } } diff --git a/internal/watcher/instance/nginx_config_parser_test.go b/internal/watcher/instance/nginx_config_parser_test.go index e4c493e530..f3ac1790af 100644 --- a/internal/watcher/instance/nginx_config_parser_test.go +++ b/internal/watcher/instance/nginx_config_parser_test.go @@ -279,6 +279,7 @@ func TestNginxConfigParser_Parse(t *testing.T) { tests := []struct { instance *mpi.Instance name string + syslog []string }{ { name: "Test 1: Valid response", @@ -287,6 +288,7 @@ func TestNginxConfigParser_Parse(t *testing.T) { { name: "Test 2: Error response", instance: protos.GetNginxPlusInstance([]string{}), + syslog: []string{"127.0.0.1:1515"}, }, } @@ -298,7 +300,9 @@ func TestNginxConfigParser_Parse(t *testing.T) { ltsvAccessLog.Name(), errorLog.Name(), test.instance.GetInstanceMeta().GetInstanceId(), + test.syslog, ) + expectedConfigContext.Files = append(expectedConfigContext.Files, &mpi.File{ FileMeta: fileMeta, }) diff --git a/test/config/collector/test-opentelemetry-collector-agent.yaml b/test/config/collector/test-opentelemetry-collector-agent.yaml index 8d6be2900c..f8970a4dec 100644 --- a/test/config/collector/test-opentelemetry-collector-agent.yaml +++ b/test/config/collector/test-opentelemetry-collector-agent.yaml @@ -1,5 +1,5 @@ receivers: - syslog/1: + syslog/00: tcp: listen_address: "127.0.0.1:1515" protocol: rfc3164 @@ -90,7 +90,7 @@ service: - debug logs: receivers: - - syslog/1 + - syslog/00 processors: - batch exporters: diff --git a/test/docker/nginx-plus/deb/Dockerfile b/test/docker/nginx-plus/deb/Dockerfile index 9b1aa04047..d160ff0cbc 100644 --- a/test/docker/nginx-plus/deb/Dockerfile +++ b/test/docker/nginx-plus/deb/Dockerfile @@ -14,47 +14,52 @@ COPY $ENTRY_POINT /agent/entrypoint.sh RUN --mount=type=secret,id=nginx-crt,dst=nginx-repo.crt \ --mount=type=secret,id=nginx-key,dst=nginx-repo.key \ set -x \ - # Create nginx user/group first, to be consistent throughout Docker variants +# Create nginx user/group first, to be consistent throughout Docker variants && groupadd --system --gid 101 nginx \ && useradd --system --gid nginx --no-create-home --home-dir /nonexistent --uid 101 nginx \ && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates \ - gnupg2 \ + gnupg1 \ lsb-release \ git \ wget \ make \ - && mkdir -p /etc/ssl/nginx \ - && cat nginx-repo.crt > /etc/ssl/nginx/nginx-repo.crt \ - && cat nginx-repo.key > /etc/ssl/nginx/nginx-repo.key \ + && \ + NGINX_GPGKEY=573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62; \ + found=''; \ + for server in \ + hkp://keyserver.ubuntu.com:80 \ + pgp.mit.edu \ + ; do \ + echo "Fetching GPG key $NGINX_GPGKEY from $server"; \ + apt-key adv --keyserver "$server" --keyserver-options timeout=10 --recv-keys "$NGINX_GPGKEY" && found=yes && break; \ + done; \ + test -z "$found" && echo >&2 "error: failed to fetch GPG key $NGINX_GPGKEY" && exit 1; \ + apt-get remove --purge --auto-remove -y gnupg1 && rm -rf /var/lib/apt/lists/* \ +# Install the latest release of NGINX Plus and/or NGINX Plus modules +# Uncomment individual modules if necessary +# Use versioned packages over defaults to specify a release + && nginxPackages=" \ + nginx-plus \ + " \ && echo "Acquire::https::pkgs.nginx.com::Verify-Peer \"true\";" > /etc/apt/apt.conf.d/90nginx \ && echo "Acquire::https::pkgs.nginx.com::Verify-Host \"true\";" >> /etc/apt/apt.conf.d/90nginx \ && echo "Acquire::https::pkgs.nginx.com::SslCert \"/etc/ssl/nginx/nginx-repo.crt\";" >> /etc/apt/apt.conf.d/90nginx \ && echo "Acquire::https::pkgs.nginx.com::SslKey \"/etc/ssl/nginx/nginx-repo.key\";" >> /etc/apt/apt.conf.d/90nginx \ - # Add GPG keys for nginx-plus and app-protect - && wget -qO - https://cs.nginx.com/static/keys/nginx_signing.key | gpg --dearmor | tee /usr/share/keyrings/nginx-archive-keyring.gpg >/dev/null \ - && wget -qO - https://cs.nginx.com/static/keys/app-protect-security-updates.key | gpg --dearmor | tee /usr/share/keyrings/app-protect-security-updates.gpg >/dev/null \ - #&& ls -l /usr/share/keyrings/nginx-archive-keyring.gpg /usr/share/keyrings/app-protect-security-updates.gpg \ - - - # Install the latest release of NGINX Plus and/or NGINX Plus modules - # Set up repository sources for NGINX Plus and App Protect - && printf "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] https://pkgs.nginx.com/plus/ubuntu `lsb_release -cs` nginx-plus\n" > /etc/apt/sources.list.d/nginx-plus.list \ - && printf "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] https://pkgs.nginx.com/app-protect/ubuntu `lsb_release -cs` nginx-plus\n" | tee /etc/apt/sources.list.d/nginx-app-protect.list \ - && printf "deb [signed-by=/usr/share/keyrings/app-protect-security-updates.gpg] https://pkgs.nginx.com/app-protect-security-updates/ubuntu `lsb_release -cs` nginx-plus\n" | tee -a /etc/apt/sources.list.d/nginx-app-protect.list \ - # && wget -P /etc/apt/apt.conf.d https://cs.nginx.com/static/files/90pkgs-nginx \ - # && printf "deb [signed-by=/usr/share/keyrings/app-protect-security-updates.gpg] https://pkgs.nginx.com/app-protect/ubuntu `lsb_release -cs` nginx-plus\n" > /etc/apt/sources.list.d/nginx-app-protect.list \ + && printf "deb https://pkgs.nginx.com/plus/R32/ubuntu `lsb_release -cs` nginx-plus\n" > /etc/apt/sources.list.d/nginx-plus.list \ + && mkdir -p /etc/ssl/nginx \ + && cat nginx-repo.crt > /etc/ssl/nginx/nginx-repo.crt \ + && cat nginx-repo.key > /etc/ssl/nginx/nginx-repo.key \ && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ - nginx-plus \ - app-protect \ + $nginxPackages \ curl \ gettext-base \ jq \ + gnupg2 \ && apt-get remove --purge -y lsb-release \ - && apt-get remove --purge --auto-remove -y \ - && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/nginx-plus.list /etc/apt/sources.list.d/nginx-app-protect.list \ + && apt-get remove --purge --auto-remove -y && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/nginx-plus.list \ && rm -rf /etc/apt/apt.conf.d/90nginx /etc/ssl/nginx EXPOSE 80 diff --git a/test/model/config.go b/test/model/config.go index 8d8cdc5594..c5a242a49d 100644 --- a/test/model/config.go +++ b/test/model/config.go @@ -20,6 +20,7 @@ func GetConfigContextWithNames( ltsvAccessLogName, errorLogName string, instanceID string, + syslog []string, ) *model.NginxConfigContext { return &model.NginxConfigContext{ AccessLogs: []*model.AccessLog{ @@ -52,5 +53,6 @@ func GetConfigContextWithNames( }, }, InstanceID: instanceID, + Syslog: syslog, } } diff --git a/test/protos/instances.go b/test/protos/instances.go index 6ea7c9b1eb..fa6a3eb6ee 100644 --- a/test/protos/instances.go +++ b/test/protos/instances.go @@ -102,6 +102,9 @@ func GetNginxPlusInstance(expectedModules []string) *v1.Instance { "stream_proxy_protocol_vendor_module", "stream_realip_module", "stream_ssl_module", "stream_ssl_preread_module", }, + AppProtectDirectives: []string{ + `app_protect_security_log "/etc/app_protect/conf/log_default.json" syslog:server=127.0.0.1:514`, + }, PlusApi: "", }, }, diff --git a/test/types/config.go b/test/types/config.go index d1e5cfccf3..109298ebfb 100644 --- a/test/types/config.go +++ b/test/types/config.go @@ -69,8 +69,8 @@ func AgentConfig() *config.Config { OtlpReceivers: OtlpReceivers(), SyslogReceivers: []config.SyslogReceiver{ { - Server: "127.0.0.1:1515", - InstanceID: "1", + Server: []string{"127.0.0.1:1515"}, + InstanceID: "00", Protocol: "rfc3164", }, }, From 7ddc9b836d8b414a0ad749b1326b60a47224b233 Mon Sep 17 00:00:00 2001 From: dhurley Date: Thu, 28 Nov 2024 14:22:26 +0000 Subject: [PATCH 08/24] Replace syslog receiver with tcplog receiver --- api/grpc/mpi/v1/command.pb.go | 2 - go.mod | 1 + go.sum | 2 + internal/collector/factories.go | 4 +- internal/collector/otel_collector_plugin.go | 75 ++++++++++++++----- .../collector/otel_collector_plugin_test.go | 14 +--- internal/collector/otelcol.tmpl | 26 ++++--- internal/collector/settings_test.go | 21 ++++++ internal/config/defaults.go | 9 --- internal/config/types.go | 17 +++-- internal/model/config.go | 24 +++--- .../watcher/instance/nginx_config_parser.go | 16 ++-- .../instance/nginx_config_parser_test.go | 19 ++--- .../test-opentelemetry-collector-agent.yaml | 14 ++-- .../nginx-with-multiple-access-logs.conf | 3 + test/model/config.go | 7 +- test/protos/instances.go | 3 - test/types/config.go | 7 -- 18 files changed, 158 insertions(+), 106 deletions(-) diff --git a/api/grpc/mpi/v1/command.pb.go b/api/grpc/mpi/v1/command.pb.go index 8a572c8e63..e390eb5ac1 100644 --- a/api/grpc/mpi/v1/command.pb.go +++ b/api/grpc/mpi/v1/command.pb.go @@ -1746,8 +1746,6 @@ type NGINXPlusRuntimeInfo struct { DynamicModules []string `protobuf:"bytes,5,rep,name=dynamic_modules,json=dynamicModules,proto3" json:"dynamic_modules,omitempty"` // the plus API location PlusApi string `protobuf:"bytes,6,opt,name=plus_api,json=plusApi,proto3" json:"plus_api,omitempty"` - // a list of App Protect directives - AppProtectDirectives []string `protobuf:"bytes,7,rep,name=app_protect_directives,json=appProtectDirectives,proto3" json:"app_protect_directives,omitempty"` } func (x *NGINXPlusRuntimeInfo) Reset() { diff --git a/go.mod b/go.mod index e19b6d469a..ff91a835e8 100644 --- a/go.mod +++ b/go.mod @@ -45,6 +45,7 @@ require ( github.com/open-telemetry/opentelemetry-collector-contrib/processor/tailsamplingprocessor v0.108.0 github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor v0.108.0 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/hostmetricsreceiver v0.108.0 + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/tcplogreceiver v0.108.0 github.com/open-telemetry/opentelemetry-collector-contrib/testbed v0.108.0 github.com/shirou/gopsutil/v4 v4.24.7 github.com/spf13/pflag v1.0.5 diff --git a/go.sum b/go.sum index dd098cbcba..a2af99d112 100644 --- a/go.sum +++ b/go.sum @@ -693,6 +693,8 @@ github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusrec github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.108.0/go.mod h1:VUCVWsYWUlxut3MQbH3ADOz7bQBrtC3NlzJl9PQpKAk= github.com/open-telemetry/opentelemetry-collector-contrib/receiver/syslogreceiver v0.108.0 h1:FyiVuycmSmI4pg5PKvRjpUT0qaBVdFW9AeX1giolQG8= github.com/open-telemetry/opentelemetry-collector-contrib/receiver/syslogreceiver v0.108.0/go.mod h1:bmgqS6N2W31oEVg82zAkCJxmMrVamkODmZWr9qpCdK8= +github.com/open-telemetry/opentelemetry-collector-contrib/receiver/tcplogreceiver v0.108.0 h1:zm8uZlzIgK/QajEte0ccj9LZhgSAakhy0P+WPDo5DVc= +github.com/open-telemetry/opentelemetry-collector-contrib/receiver/tcplogreceiver v0.108.0/go.mod h1:KQshgnBLGVFaRtmEiCIEzYa+DFHTzfDxRjzMx5KIH2Y= github.com/open-telemetry/opentelemetry-collector-contrib/receiver/zipkinreceiver v0.108.0 h1:fR/TiCF1jkTSLIpvxJ8ycBYx3wAIMo5sOjyna28qNPc= github.com/open-telemetry/opentelemetry-collector-contrib/receiver/zipkinreceiver v0.108.0/go.mod h1:VdIAkr+2oTSwkl+CQ9UW3ZiV0TXobqOznSV5I5CB7Rc= github.com/open-telemetry/opentelemetry-collector-contrib/testbed v0.108.0 h1:rZlnwqE9x/MX+13+tgqx5tizLoRpJ4m3qD6ww1hdf5k= diff --git a/internal/collector/factories.go b/internal/collector/factories.go index cac41fd248..74d9a5f7f1 100644 --- a/internal/collector/factories.go +++ b/internal/collector/factories.go @@ -32,7 +32,7 @@ import ( "github.com/open-telemetry/opentelemetry-collector-contrib/processor/tailsamplingprocessor" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor" "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/hostmetricsreceiver" - "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/syslogreceiver" + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/tcplogreceiver" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/connector" "go.opentelemetry.io/collector/exporter" @@ -111,7 +111,7 @@ func createReceiverFactories() (map[component.Type]receiver.Factory, error) { hostmetricsreceiver.NewFactory(), nginxreceiver.NewFactory(), nginxplusreceiver.NewFactory(), - syslogreceiver.NewFactory(), + tcplogreceiver.NewFactory(), } return receiver.MakeFactoryMap(receiverList...) diff --git a/internal/collector/otel_collector_plugin.go b/internal/collector/otel_collector_plugin.go index 6b513705c5..72d74ee3f6 100644 --- a/internal/collector/otel_collector_plugin.go +++ b/internal/collector/otel_collector_plugin.go @@ -24,8 +24,10 @@ import ( ) const ( - maxTimeToWaitForShutdown = 30 * time.Second - filePermission = 0o600 + maxTimeToWaitForShutdown = 30 * time.Second + filePermission = 0o600 + timestampConversionExpression = `EXPR(split(body, ">")[0] + ">" + ` + + `date(split(split(body, ">")[1], " ")[0]).Format("Jan 02 15:04:05") + " " + split(body, " ", 2)[1])` ) type ( @@ -237,7 +239,7 @@ func (oc *Collector) handleNginxConfigUpdate(ctx context.Context, msg *bus.Messa return } - reloadCollector := oc.checkForNewNginxReceivers(nginxConfigContext) + reloadCollector := oc.checkForNewReceivers(nginxConfigContext) if reloadCollector { slog.InfoContext(ctx, "Reloading OTel collector config") @@ -362,7 +364,7 @@ func (oc *Collector) restartCollector(ctx context.Context) { } } -func (oc *Collector) checkForNewNginxReceivers(nginxConfigContext *model.NginxConfigContext) bool { +func (oc *Collector) checkForNewReceivers(nginxConfigContext *model.NginxConfigContext) bool { nginxReceiverFound, reloadCollector := oc.updateExistingNginxPlusReceiver(nginxConfigContext) if !nginxReceiverFound && nginxConfigContext.PlusAPI != "" { @@ -374,19 +376,6 @@ func (oc *Collector) checkForNewNginxReceivers(nginxConfigContext *model.NginxCo }, ) - if nginxConfigContext.Syslog != nil { - oc.config.Collector.Receivers.SyslogReceivers = append( - oc.config.Collector.Receivers.SyslogReceivers, - config.SyslogReceiver{ - InstanceID: nginxConfigContext.InstanceID, - Server: nginxConfigContext.Syslog, - Protocol: "rfc3164", // default value, need to get from the agent conf - }, - ) - } - - // slog.Error("hello", oc.config.Collector.Receivers) - reloadCollector = true } else if nginxConfigContext.PlusAPI == "" { nginxReceiverFound, reloadCollector = oc.updateExistingNginxOSSReceiver(nginxConfigContext) @@ -405,10 +394,14 @@ func (oc *Collector) checkForNewNginxReceivers(nginxConfigContext *model.NginxCo } } + tcplogReceiversFound := oc.updateTcplogReceivers(nginxConfigContext) + if tcplogReceiversFound { + reloadCollector = true + } + return reloadCollector } -// Todo: consider update scenario for new syslogreceivers func (oc *Collector) updateExistingNginxPlusReceiver( nginxConfigContext *model.NginxConfigContext, ) (nginxReceiverFound, reloadCollector bool) { @@ -472,6 +465,52 @@ func (oc *Collector) updateExistingNginxOSSReceiver( return nginxReceiverFound, reloadCollector } +func (oc *Collector) updateTcplogReceivers(nginxConfigContext *model.NginxConfigContext) bool { + oc.config.Collector.Receivers.TcplogReceivers = make([]config.TcplogReceiver, 0) + + if nginxConfigContext.NAPSysLogServers != nil { + for _, napSysLogServer := range nginxConfigContext.NAPSysLogServers { + oc.config.Collector.Receivers.TcplogReceivers = append( + oc.config.Collector.Receivers.TcplogReceivers, + config.TcplogReceiver{ + ListenAddress: napSysLogServer, + Operators: []config.Operator{ + { + Type: "add", + Fields: map[string]string{ + "field": "body", + "value": timestampConversionExpression, + }, + }, + { + Type: "syslog_parser", + Fields: map[string]string{ + "protocol": "rfc3164", + }, + }, + { + Type: "key_value_parser", + Fields: map[string]string{ + "parse_from": "attributes.message", + "parse_to": "body", + "pair_delimiter": "\",\"", + }, + }, + { + Type: "remove", + Fields: map[string]string{ + "field": "attributes.message", + }, + }, + }, + }, + ) + } + } + + return len(oc.config.Collector.Receivers.TcplogReceivers) > 0 +} + // nolint: revive func (oc *Collector) updateResourceAttributes( attributesToAdd []config.ResourceAttribute, diff --git a/internal/collector/otel_collector_plugin_test.go b/internal/collector/otel_collector_plugin_test.go index 9953263605..ddc4bd91df 100644 --- a/internal/collector/otel_collector_plugin_test.go +++ b/internal/collector/otel_collector_plugin_test.go @@ -178,18 +178,13 @@ func TestCollector_ProcessNginxConfigUpdateTopic(t *testing.T) { receivers: config.Receivers{ HostMetrics: nil, OtlpReceivers: nil, - SyslogReceivers: []config.SyslogReceiver{ - { - Server: []string{"127.0.0.1:1515"}, - InstanceID: "00", - Protocol: "rfc3164"}, - }, NginxPlusReceivers: []config.NginxPlusReceiver{ { InstanceID: "123", PlusAPI: "", }, }, + TcplogReceivers: make([]config.TcplogReceiver, 0), }, }, { @@ -210,12 +205,6 @@ func TestCollector_ProcessNginxConfigUpdateTopic(t *testing.T) { receivers: config.Receivers{ HostMetrics: nil, OtlpReceivers: nil, - SyslogReceivers: []config.SyslogReceiver{ - { - Server: []string{"127.0.0.1:1515"}, - InstanceID: "00", - Protocol: "rfc3164"}, - }, NginxReceivers: []config.NginxReceiver{ { InstanceID: "123", @@ -228,6 +217,7 @@ func TestCollector_ProcessNginxConfigUpdateTopic(t *testing.T) { }, }, }, + TcplogReceivers: make([]config.TcplogReceiver, 0), }, }, } diff --git a/internal/collector/otelcol.tmpl b/internal/collector/otelcol.tmpl index 9f915eff58..bcae88d423 100644 --- a/internal/collector/otelcol.tmpl +++ b/internal/collector/otelcol.tmpl @@ -1,13 +1,4 @@ receivers: -{{- range $index, $otlpReceiver := .Receivers.SyslogReceivers }} - {{- $instanceID := .InstanceID }} - {{- range $serverIndex, $server := .Server }} - syslog/{{$index}}{{$serverIndex}}: - tcp: - listen_address: "{{ $server }}" - protocol: rfc3164 - {{- end }} -{{- end }} {{- if ne .Receivers.HostMetrics nil }} hostmetrics: collection_interval: {{ .Receivers.HostMetrics.CollectionInterval }} @@ -83,6 +74,17 @@ receivers: endpoint: "{{- .PlusAPI -}}" collection_interval: 10s {{- end }} +{{- range $index, $tcplogReceiver := .Receivers.TcplogReceivers }} + tcplog/{{$index}}: + listen_address: "{{- .ListenAddress -}}" + operators: +{{- range $index, $operator := .Operators }} + - type: {{.Type}} +{{- range $key, $value := .Fields }} + {{$key}}: {{$value}} +{{- end }} +{{- end }} +{{- end }} processors: {{- if ne .Processors.Resource nil }} @@ -244,11 +246,11 @@ service: {{- if ne .Exporters.Debug nil }} - debug {{- end }} - {{- if and (ne .Receivers.SyslogReceivers nil) (ne .Exporters.OtlpExporters nil) }} + {{- if ne .Receivers.TcplogReceivers nil }} logs: receivers: - {{- range .Receivers.SyslogReceivers }} - - syslog/{{- .InstanceID -}} + {{- range $index, $tcplogReceiver := .Receivers.TcplogReceivers }} + - tcplog/{{$index}} {{- end }} processors: {{- if ne .Processors.Batch nil }} diff --git a/internal/collector/settings_test.go b/internal/collector/settings_test.go index baf29807fe..47b764c16b 100644 --- a/internal/collector/settings_test.go +++ b/internal/collector/settings_test.go @@ -100,6 +100,27 @@ func TestTemplateWrite(t *testing.T) { }, }) + cfg.Collector.Receivers.TcplogReceivers = []config.TcplogReceiver{ + { + ListenAddress: "localhost:151", + Operators: []config.Operator{ + { + Type: "add", + Fields: map[string]string{ + "field": "body", + "value": `EXPR(split(body, ",")[0])`, + }, + }, + { + Type: "remove", + Fields: map[string]string{ + "field": "attributes.message", + }, + }, + }, + }, + } + cfg.Collector.Extensions.HeadersSetter = &config.HeadersSetter{ Headers: []config.Header{ { diff --git a/internal/config/defaults.go b/internal/config/defaults.go index b386a236b6..1716320b34 100644 --- a/internal/config/defaults.go +++ b/internal/config/defaults.go @@ -12,7 +12,6 @@ import ( ) const ( - //Todo: Add default Protocol for nap DefGracefulShutdownPeriod = 5 * time.Second DefNginxReloadMonitoringPeriod = 10 * time.Second DefTreatErrorsAsWarnings = false @@ -64,14 +63,6 @@ const ( DefCollectorExtensionsHealthTLSCAPath = "" DefCollectorExtensionsHealthTLSSkipVerify = false DefCollectorExtensionsHealthTLServerNameKey = "" - - DefCollectorPrometheusExporterServerHost = "" - DefCollectorPrometheusExporterServerPort = 0 - DefCollectorPrometheusExporterTLSCertPath = "" - DefCollectorPrometheusExporterTLSKeyPath = "" - DefCollectorPrometheusExporterTLSCAPath = "" - DefCollectorPrometheusExporterTLSSkipVerify = false - DefCollectorPrometheusExporterTLServerNameKey = "" ) func DefaultFeatures() []string { diff --git a/internal/config/types.go b/internal/config/types.go index a37797f239..3f17a06093 100644 --- a/internal/config/types.go +++ b/internal/config/types.go @@ -164,7 +164,7 @@ type ( OtlpReceivers []OtlpReceiver `yaml:"-" mapstructure:"otlp_receivers"` NginxReceivers []NginxReceiver `yaml:"-" mapstructure:"nginx_receivers"` NginxPlusReceivers []NginxPlusReceiver `yaml:"-" mapstructure:"nginx_plus_receivers"` - SyslogReceivers []SyslogReceiver `yaml:"-" mapstructure:"syslog_receiver"` + TcplogReceivers []TcplogReceiver `yaml:"-" mapstructure:"tcplog_receivers"` } OtlpReceiver struct { @@ -173,10 +173,17 @@ type ( OtlpTLSConfig *OtlpTLSConfig `yaml:"-" mapstructure:"tls"` } - SyslogReceiver struct { - InstanceID string `yaml:"-" mapstructure:"instance_id"` - Server []string `yaml:"-" mapstructure:"server"` - Protocol string `yaml:"-" mapstructure:"protocol"` + TcplogReceiver struct { + ListenAddress string `yaml:"-" mapstructure:"listen_address"` + Operators []Operator `yaml:"-" mapstructure:"operators"` + } + + // There are many types of operators with different field names so we use a generic map to store the fields. + // See here for more info: + // https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/pkg/stanza/docs/operators/README.md + Operator struct { + Fields map[string]string `yaml:"-" mapstructure:"fields"` + Type string `yaml:"-" mapstructure:"type"` } NginxReceiver struct { diff --git a/internal/model/config.go b/internal/model/config.go index faeb3c121e..2e44c4fef0 100644 --- a/internal/model/config.go +++ b/internal/model/config.go @@ -12,13 +12,13 @@ import ( ) type NginxConfigContext struct { - StubStatus string - PlusAPI string - InstanceID string - Files []*v1.File - AccessLogs []*AccessLog - ErrorLogs []*ErrorLog - Syslog []string + StubStatus string + PlusAPI string + InstanceID string + Files []*v1.File + AccessLogs []*AccessLog + ErrorLogs []*ErrorLog + NAPSysLogServers []string } func (ncc *NginxConfigContext) Equal(otherNginxConfigContext *NginxConfigContext) bool { @@ -46,6 +46,10 @@ func (ncc *NginxConfigContext) Equal(otherNginxConfigContext *NginxConfigContext return false } + if !reflect.DeepEqual(ncc.NAPSysLogServers, otherNginxConfigContext.NAPSysLogServers) { + return false + } + return true } @@ -86,12 +90,6 @@ type ErrorLog struct { Readable bool } -type NAP struct { - Enable bool - Syslog bool - SyslogServer string -} - type ( WriteStatus int ) diff --git a/internal/watcher/instance/nginx_config_parser.go b/internal/watcher/instance/nginx_config_parser.go index 5e555ff125..bf2d812b2e 100644 --- a/internal/watcher/instance/nginx_config_parser.go +++ b/internal/watcher/instance/nginx_config_parser.go @@ -37,7 +37,6 @@ const ( stubStatusAPIDirective = "stub_status" apiFormat = "http://%s%s" locationDirective = "location" - napDirective = "app_protect_security_log" ) type ( @@ -92,6 +91,8 @@ func (ncp *NginxConfigParser) createNginxConfigContext( instance *mpi.Instance, payload *crossplane.Payload, ) (*model.NginxConfigContext, error) { + napSyslogServersFound := make(map[string]bool) + nginxConfigContext := &model.NginxConfigContext{ InstanceID: instance.GetInstanceMeta().GetInstanceId(), } @@ -100,7 +101,6 @@ func (ncp *NginxConfigParser) createNginxConfigContext( for _, conf := range payload.Config { formatMap := make(map[string]string) - syslogMap := make(map[string]bool) err := ncp.crossplaneConfigTraverse(ctx, &conf, func(ctx context.Context, parent, directive *crossplane.Directive) error { switch directive.Directive { @@ -130,14 +130,18 @@ func (ncp *NginxConfigParser) createNginxConfigContext( matches := re.FindStringSubmatch(syslogArg) if len(matches) > 1 { syslogServer := matches[1] - if !syslogMap[syslogServer] { - nginxConfigContext.Syslog = append(nginxConfigContext.Syslog, syslogServer) - syslogMap[syslogServer] = true + if !napSyslogServersFound[syslogServer] { + nginxConfigContext.NAPSysLogServers = append( + nginxConfigContext.NAPSysLogServers, + syslogServer, + ) + napSyslogServersFound[syslogServer] = true + slog.DebugContext(ctx, "Found NAP syslog server", "address", syslogServer) } - slog.InfoContext(ctx, "Captured syslog server", "syslog_server", syslogServer) } } } + return nil }, ) diff --git a/internal/watcher/instance/nginx_config_parser_test.go b/internal/watcher/instance/nginx_config_parser_test.go index adfa084eb6..2d49840aae 100644 --- a/internal/watcher/instance/nginx_config_parser_test.go +++ b/internal/watcher/instance/nginx_config_parser_test.go @@ -277,18 +277,19 @@ func TestNginxConfigParser_Parse(t *testing.T) { require.NoError(t, err) tests := []struct { - instance *mpi.Instance - name string - syslog []string + instance *mpi.Instance + name string + syslogServers []string }{ { - name: "Test 1: Valid response", - instance: protos.GetNginxOssInstance([]string{}), + name: "Test 1: Valid response", + instance: protos.GetNginxOssInstance([]string{}), + syslogServers: []string{"127.0.0.1:1515"}, }, { - name: "Test 2: Error response", - instance: protos.GetNginxPlusInstance([]string{}), - syslog: []string{"127.0.0.1:1515"}, + name: "Test 2: Error response", + instance: protos.GetNginxPlusInstance([]string{}), + syslogServers: []string{"127.0.0.1:1515"}, }, } @@ -300,7 +301,7 @@ func TestNginxConfigParser_Parse(t *testing.T) { ltsvAccessLog.Name(), errorLog.Name(), test.instance.GetInstanceMeta().GetInstanceId(), - test.syslog, + test.syslogServers, ) expectedConfigContext.Files = append(expectedConfigContext.Files, &mpi.File{ diff --git a/test/config/collector/test-opentelemetry-collector-agent.yaml b/test/config/collector/test-opentelemetry-collector-agent.yaml index cf2a0bdd1b..1ef1532443 100644 --- a/test/config/collector/test-opentelemetry-collector-agent.yaml +++ b/test/config/collector/test-opentelemetry-collector-agent.yaml @@ -1,8 +1,4 @@ receivers: - syslog/00: - tcp: - listen_address: "127.0.0.1:1515" - protocol: rfc3164 hostmetrics: collection_interval: 1m0s initial_delay: 1s @@ -26,6 +22,14 @@ receivers: access_logs: - log_format: "$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\" \"$http_x_forwarded_for\"\"$upstream_cache_status\"" file_path: "/var/log/nginx/access-custom.conf" + tcplog/0: + listen_address: "localhost:151" + operators: + - type: add + field: body + value: EXPR(split(body, ",")[0]) + - type: remove + field: attributes.message processors: batch: @@ -90,7 +94,7 @@ service: - debug logs: receivers: - - syslog/00 + - tcplog/0 processors: - batch exporters: diff --git a/test/config/nginx/nginx-with-multiple-access-logs.conf b/test/config/nginx/nginx-with-multiple-access-logs.conf index 83582169a6..4d22a456e9 100644 --- a/test/config/nginx/nginx-with-multiple-access-logs.conf +++ b/test/config/nginx/nginx-with-multiple-access-logs.conf @@ -4,6 +4,7 @@ worker_processes auto; error_log %s notice; pid /var/run/nginx.pid; +load_module modules/ngx_http_app_protect_module.so; events { worker_connections 1024; @@ -33,5 +34,7 @@ http { server { access_log %s ltsv; + + app_protect_security_log "/etc/app_protect/conf/log_default.json" syslog:server=127.0.0.1:1515; } } diff --git a/test/model/config.go b/test/model/config.go index c5a242a49d..59c82395de 100644 --- a/test/model/config.go +++ b/test/model/config.go @@ -14,13 +14,14 @@ func GetConfigContext() *model.NginxConfigContext { } } +// nolint: revive func GetConfigContextWithNames( accessLogName, combinedAccessLogName, ltsvAccessLogName, errorLogName string, instanceID string, - syslog []string, + syslogServers []string, ) *model.NginxConfigContext { return &model.NginxConfigContext{ AccessLogs: []*model.AccessLog{ @@ -52,7 +53,7 @@ func GetConfigContextWithNames( Permissions: "0600", }, }, - InstanceID: instanceID, - Syslog: syslog, + InstanceID: instanceID, + NAPSysLogServers: syslogServers, } } diff --git a/test/protos/instances.go b/test/protos/instances.go index fa6a3eb6ee..6ea7c9b1eb 100644 --- a/test/protos/instances.go +++ b/test/protos/instances.go @@ -102,9 +102,6 @@ func GetNginxPlusInstance(expectedModules []string) *v1.Instance { "stream_proxy_protocol_vendor_module", "stream_realip_module", "stream_ssl_module", "stream_ssl_preread_module", }, - AppProtectDirectives: []string{ - `app_protect_security_log "/etc/app_protect/conf/log_default.json" syslog:server=127.0.0.1:514`, - }, PlusApi: "", }, }, diff --git a/test/types/config.go b/test/types/config.go index 9073b160f5..681aa0cc02 100644 --- a/test/types/config.go +++ b/test/types/config.go @@ -75,13 +75,6 @@ func AgentConfig() *config.Config { }, }, }, - SyslogReceivers: []config.SyslogReceiver{ - { - Server: []string{"127.0.0.1:1515"}, - InstanceID: "00", - Protocol: "rfc3164", - }, - }, HostMetrics: &config.HostMetrics{ CollectionInterval: time.Minute, InitialDelay: time.Second, From 1b7a3598600341dc2d40b293f6e73581e205dfed Mon Sep 17 00:00:00 2001 From: dhurley Date: Thu, 28 Nov 2024 15:22:01 +0000 Subject: [PATCH 09/24] Replace syslog receiver with tcplog receiver --- internal/collector/otel_collector_plugin_test.go | 9 +++++++-- internal/collector/settings_test.go | 4 +++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/internal/collector/otel_collector_plugin_test.go b/internal/collector/otel_collector_plugin_test.go index ddc4bd91df..d27c931e76 100644 --- a/internal/collector/otel_collector_plugin_test.go +++ b/internal/collector/otel_collector_plugin_test.go @@ -9,6 +9,7 @@ import ( "context" "errors" "fmt" + "path/filepath" "strings" "testing" @@ -27,6 +28,8 @@ import ( ) func TestCollector_New(t *testing.T) { + tmpDir := t.TempDir() + tests := []struct { config *config.Config expectedError error @@ -57,7 +60,7 @@ func TestCollector_New(t *testing.T) { name: "Successful initialization", config: &config.Config{ Collector: &config.Collector{ - Log: &config.Log{Path: "/tmp/test.log"}, + Log: &config.Log{Path: filepath.Join(tmpDir, "test.log")}, }, }, expectedError: nil, @@ -80,6 +83,8 @@ func TestCollector_New(t *testing.T) { } func TestCollector_Init(t *testing.T) { + tmpDir := t.TempDir() + tests := []struct { name string expectedLog string @@ -105,7 +110,7 @@ func TestCollector_Init(t *testing.T) { logBuf := &bytes.Buffer{} stub.StubLoggerWith(logBuf) - conf.Collector.Log = &config.Log{Path: "/tmp/test.log"} + conf.Collector.Log = &config.Log{Path: filepath.Join(tmpDir, "test.log")} if tt.expectedError { conf.Collector.Receivers = config.Receivers{} diff --git a/internal/collector/settings_test.go b/internal/collector/settings_test.go index 47b764c16b..f9857f31fc 100644 --- a/internal/collector/settings_test.go +++ b/internal/collector/settings_test.go @@ -49,8 +49,10 @@ func TestConfigProviderSettings(t *testing.T) { } func TestTemplateWrite(t *testing.T) { + tmpDir := t.TempDir() + cfg := types.AgentConfig() - actualConfPath := filepath.Join("/tmp/", "nginx-agent-otelcol-test.yaml") + actualConfPath := filepath.Join(tmpDir, "nginx-agent-otelcol-test.yaml") cfg.Collector.ConfigPath = actualConfPath cfg.Collector.Exporters.PrometheusExporter = &config.PrometheusExporter{ From 6f0eddeb8c845fc8c5bcc73d04c35886a821282c Mon Sep 17 00:00:00 2001 From: dhurley Date: Thu, 28 Nov 2024 15:46:40 +0000 Subject: [PATCH 10/24] Replace syslog receiver with tcplog receiver --- internal/collector/otel_collector_plugin.go | 7 +++++++ internal/collector/otelcol.tmpl | 5 +++++ internal/collector/settings_test.go | 9 +++++++++ .../collector/test-opentelemetry-collector-agent.yaml | 7 +++++++ 4 files changed, 28 insertions(+) diff --git a/internal/collector/otel_collector_plugin.go b/internal/collector/otel_collector_plugin.go index 72d74ee3f6..c6dbd2599d 100644 --- a/internal/collector/otel_collector_plugin.go +++ b/internal/collector/otel_collector_plugin.go @@ -502,6 +502,13 @@ func (oc *Collector) updateTcplogReceivers(nginxConfigContext *model.NginxConfig "field": "attributes.message", }, }, + { + Type: "add", + Fields: map[string]string{ + "field": "resource.instance.id", + "value": nginxConfigContext.InstanceID, + }, + }, }, }, ) diff --git a/internal/collector/otelcol.tmpl b/internal/collector/otelcol.tmpl index bcae88d423..52fa876cf0 100644 --- a/internal/collector/otelcol.tmpl +++ b/internal/collector/otelcol.tmpl @@ -253,6 +253,11 @@ service: - tcplog/{{$index}} {{- end }} processors: + {{- if ne .Processors.Resource nil }} + {{- if .Processors.Resource.Attributes }} + - resource + {{- end }} + {{- end }} {{- if ne .Processors.Batch nil }} - batch {{- end }} diff --git a/internal/collector/settings_test.go b/internal/collector/settings_test.go index f9857f31fc..c2bf75e31a 100644 --- a/internal/collector/settings_test.go +++ b/internal/collector/settings_test.go @@ -54,6 +54,15 @@ func TestTemplateWrite(t *testing.T) { cfg := types.AgentConfig() actualConfPath := filepath.Join(tmpDir, "nginx-agent-otelcol-test.yaml") cfg.Collector.ConfigPath = actualConfPath + cfg.Collector.Processors.Resource = &config.Resource{ + Attributes: []config.ResourceAttribute{ + { + Key: "resource.id", + Action: "add", + Value: "12345", + }, + }, + } cfg.Collector.Exporters.PrometheusExporter = &config.PrometheusExporter{ Server: &config.ServerConfig{ diff --git a/test/config/collector/test-opentelemetry-collector-agent.yaml b/test/config/collector/test-opentelemetry-collector-agent.yaml index 1ef1532443..35fec11fb4 100644 --- a/test/config/collector/test-opentelemetry-collector-agent.yaml +++ b/test/config/collector/test-opentelemetry-collector-agent.yaml @@ -32,6 +32,11 @@ receivers: field: attributes.message processors: + resource: + attributes: + - key: resource.id + action: add + value: 12345 batch: send_batch_size: 8192 timeout: 200ms @@ -87,6 +92,7 @@ service: - otlp/0 - nginx/123 processors: + - resource - batch exporters: - otlp/0 @@ -96,6 +102,7 @@ service: receivers: - tcplog/0 processors: + - resource - batch exporters: - otlp/0 From 5faae621d55fd521a17e96c3adf3ebffefc3fbae Mon Sep 17 00:00:00 2001 From: dhurley Date: Thu, 28 Nov 2024 16:52:39 +0000 Subject: [PATCH 11/24] Replace syslog receiver with tcplog receiver --- internal/collector/otel_collector_plugin.go | 22 ++++++++-- .../collector/otel_collector_plugin_test.go | 40 ++++++++++++++++++- 2 files changed, 57 insertions(+), 5 deletions(-) diff --git a/internal/collector/otel_collector_plugin.go b/internal/collector/otel_collector_plugin.go index c6dbd2599d..a614bbd9be 100644 --- a/internal/collector/otel_collector_plugin.go +++ b/internal/collector/otel_collector_plugin.go @@ -466,10 +466,14 @@ func (oc *Collector) updateExistingNginxOSSReceiver( } func (oc *Collector) updateTcplogReceivers(nginxConfigContext *model.NginxConfigContext) bool { - oc.config.Collector.Receivers.TcplogReceivers = make([]config.TcplogReceiver, 0) - + newTcplogReceiverAdded := false if nginxConfigContext.NAPSysLogServers != nil { + napLoop: for _, napSysLogServer := range nginxConfigContext.NAPSysLogServers { + if oc.doesTcplogReceiverAlreadyExist(napSysLogServer) { + continue napLoop + } + oc.config.Collector.Receivers.TcplogReceivers = append( oc.config.Collector.Receivers.TcplogReceivers, config.TcplogReceiver{ @@ -512,10 +516,22 @@ func (oc *Collector) updateTcplogReceivers(nginxConfigContext *model.NginxConfig }, }, ) + + newTcplogReceiverAdded = true + } + } + + return newTcplogReceiverAdded +} + +func (oc *Collector) doesTcplogReceiverAlreadyExist(listenAddress string) bool { + for _, tcplogReceiver := range oc.config.Collector.Receivers.TcplogReceivers { + if listenAddress == tcplogReceiver.ListenAddress { + return true } } - return len(oc.config.Collector.Receivers.TcplogReceivers) > 0 + return false } // nolint: revive diff --git a/internal/collector/otel_collector_plugin_test.go b/internal/collector/otel_collector_plugin_test.go index d27c931e76..28c76c3867 100644 --- a/internal/collector/otel_collector_plugin_test.go +++ b/internal/collector/otel_collector_plugin_test.go @@ -189,7 +189,6 @@ func TestCollector_ProcessNginxConfigUpdateTopic(t *testing.T) { PlusAPI: "", }, }, - TcplogReceivers: make([]config.TcplogReceiver, 0), }, }, { @@ -222,7 +221,6 @@ func TestCollector_ProcessNginxConfigUpdateTopic(t *testing.T) { }, }, }, - TcplogReceivers: make([]config.TcplogReceiver, 0), }, }, } @@ -652,6 +650,44 @@ func TestCollector_updateResourceAttributes(t *testing.T) { } } +func TestCollector_updateTcplogReceivers(t *testing.T) { + conf := types.OTelConfig(t) + conf.Collector.Log.Path = "" + conf.Collector.Processors.Batch = nil + conf.Collector.Processors.Attribute = nil + conf.Collector.Processors.Resource = nil + + collector, err := New(conf) + require.NoError(t, err) + + nginxConfigContext := &model.NginxConfigContext{ + NAPSysLogServers: []string{ + "localhost:151", + }, + } + + assert.Empty(t, conf.Collector.Receivers.TcplogReceivers) + + t.Run("Test 1: New TcplogReceiver added", func(tt *testing.T) { + tcplogReceiverAdded := collector.updateTcplogReceivers(nginxConfigContext) + + assert.True(tt, tcplogReceiverAdded) + assert.Len(tt, conf.Collector.Receivers.TcplogReceivers, 1) + assert.Equal(tt, "localhost:151", conf.Collector.Receivers.TcplogReceivers[0].ListenAddress) + assert.Len(tt, conf.Collector.Receivers.TcplogReceivers[0].Operators, 5) + }) + + // Calling updateTcplogReceivers shouldn't update the TcplogReceivers slice + // since there is already a receiver with the same ListenAddress + t.Run("Test 2: TcplogReceiver already exists", func(tt *testing.T) { + tcplogReceiverAdded := collector.updateTcplogReceivers(nginxConfigContext) + assert.False(t, tcplogReceiverAdded) + assert.Len(t, conf.Collector.Receivers.TcplogReceivers, 1) + assert.Equal(t, "localhost:151", conf.Collector.Receivers.TcplogReceivers[0].ListenAddress) + assert.Len(t, conf.Collector.Receivers.TcplogReceivers[0].Operators, 5) + }) +} + func createFakeCollector() *typesfakes.FakeCollectorInterface { fakeCollector := &typesfakes.FakeCollectorInterface{} fakeCollector.RunStub = func(ctx context.Context) error { return nil } From bbc3eb550e593a845f241958ea55683fc4bf677f Mon Sep 17 00:00:00 2001 From: dhurley Date: Thu, 28 Nov 2024 17:10:49 +0000 Subject: [PATCH 12/24] Replace syslog receiver with tcplog receiver --- internal/collector/otel_collector_plugin.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/collector/otel_collector_plugin.go b/internal/collector/otel_collector_plugin.go index a614bbd9be..baf8255c8a 100644 --- a/internal/collector/otel_collector_plugin.go +++ b/internal/collector/otel_collector_plugin.go @@ -509,7 +509,7 @@ func (oc *Collector) updateTcplogReceivers(nginxConfigContext *model.NginxConfig { Type: "add", Fields: map[string]string{ - "field": "resource.instance.id", + "field": "resource[\"instance.id\"]", "value": nginxConfigContext.InstanceID, }, }, From f3b063685f27ca6b906b1539f745306c0a8b161c Mon Sep 17 00:00:00 2001 From: dhurley Date: Mon, 2 Dec 2024 16:03:31 +0000 Subject: [PATCH 13/24] Visualize NAP logs in mock collector grafana --- test/docker/entrypoint.sh | 3 ++- test/docker/nginx-plus/deb/Dockerfile | 14 +++++++++---- test/mock/collector/docker-compose.yaml | 16 +++++++++++++- .../provisioning/datasources/datasource.yml | 5 +++++ test/mock/collector/mock-collector/main.go | 4 ++++ test/mock/collector/nginx-plus/nginx.conf | 21 ++++++++++++++++++- test/mock/collector/otel-collector.yaml | 11 ++++++++-- 7 files changed, 65 insertions(+), 9 deletions(-) diff --git a/test/docker/entrypoint.sh b/test/docker/entrypoint.sh index b7f137bc01..5379ccb334 100644 --- a/test/docker/entrypoint.sh +++ b/test/docker/entrypoint.sh @@ -15,6 +15,7 @@ trap 'handle_term' TERM # Launch nginx echo "starting nginx ..." +/bin/su -s /bin/sh -c "/usr/share/ts/bin/bd-socket-plugin tmm_count 4 proc_cpuinfo_cpu_mhz 2000000 total_xml_memory 307200000 total_umu_max_size 3129344 sys_max_account_id 1024 no_static_config 2>&1 >> /var/log/app_protect/bd-socket-plugin.log &" nginx /usr/sbin/nginx -g "daemon off;" & nginx_pid=$! @@ -22,7 +23,7 @@ nginx_pid=$! SECONDS=0 while ! ps -ef | grep "nginx: master process" | grep -v grep; do - if (( SECONDS > 5 )); then + if (( SECONDS > 15 )); then echo "couldn't find nginx master process" exit 1 fi diff --git a/test/docker/nginx-plus/deb/Dockerfile b/test/docker/nginx-plus/deb/Dockerfile index 392ed5fb70..bc4a15dbb0 100644 --- a/test/docker/nginx-plus/deb/Dockerfile +++ b/test/docker/nginx-plus/deb/Dockerfile @@ -19,14 +19,19 @@ RUN --mount=type=secret,id=nginx-crt,dst=nginx-repo.crt \ # Create nginx user/group first, to be consistent throughout Docker variants && groupadd --system --gid 101 nginx \ && useradd --system --gid nginx --no-create-home --home-dir /nonexistent --uid 101 nginx \ - && apt-get update \ + && apt-get update --allow-releaseinfo-change \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates \ gnupg1 \ lsb-release \ + apt-transport-https \ git \ wget \ make \ + gnupg2 \ + ubuntu-keyring \ + && wget -qO - https://cs.nginx.com/static/keys/app-protect-security-updates.key | gpg --dearmor | tee /usr/share/keyrings/app-protect-security-updates.gpg >/dev/null \ + && wget -qO - https://cs.nginx.com/static/keys/nginx_signing.key | gpg --dearmor | tee /usr/share/keyrings/nginx-archive-keyring.gpg >/dev/null \ && \ NGINX_GPGKEY=573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62; \ found=''; \ @@ -43,13 +48,15 @@ RUN --mount=type=secret,id=nginx-crt,dst=nginx-repo.crt \ # Uncomment individual modules if necessary # Use versioned packages over defaults to specify a release && nginxPackages=" \ - nginx-plus \ + app-protect \ " \ && echo "Acquire::https::pkgs.nginx.com::Verify-Peer \"true\";" > /etc/apt/apt.conf.d/90nginx \ && echo "Acquire::https::pkgs.nginx.com::Verify-Host \"true\";" >> /etc/apt/apt.conf.d/90nginx \ && echo "Acquire::https::pkgs.nginx.com::SslCert \"/etc/ssl/nginx/nginx-repo.crt\";" >> /etc/apt/apt.conf.d/90nginx \ && echo "Acquire::https::pkgs.nginx.com::SslKey \"/etc/ssl/nginx/nginx-repo.key\";" >> /etc/apt/apt.conf.d/90nginx \ && printf "deb https://pkgs.nginx.com/plus/${PLUS_VERSION}/ubuntu/ `lsb_release -cs` nginx-plus\n" > /etc/apt/sources.list.d/nginx-plus.list \ + && printf "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] https://pkgs.nginx.com/app-protect/${PLUS_VERSION}/ubuntu `lsb_release -cs` nginx-plus\n" | tee /etc/apt/sources.list.d/nginx-app-protect.list \ + && printf "deb [signed-by=/usr/share/keyrings/app-protect-security-updates.gpg] https://pkgs.nginx.com/app-protect-security-updates/ubuntu `lsb_release -cs` nginx-plus\n" | tee -a /etc/apt/sources.list.d/nginx-app-protect.list \ && mkdir -p /etc/ssl/nginx \ && cat nginx-repo.crt > /etc/ssl/nginx/nginx-repo.crt \ && cat nginx-repo.key > /etc/ssl/nginx/nginx-repo.key \ @@ -59,9 +66,8 @@ RUN --mount=type=secret,id=nginx-crt,dst=nginx-repo.crt \ curl \ gettext-base \ jq \ - gnupg2 \ && apt-get remove --purge -y lsb-release \ - && apt-get remove --purge --auto-remove -y && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/nginx-plus.list \ + && apt-get remove --purge --auto-remove -y && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/nginx-plus.list /etc/apt/sources.list.d/nginx-app-protect.list \ && rm -rf /etc/apt/apt.conf.d/90nginx /etc/ssl/nginx EXPOSE 80 diff --git a/test/mock/collector/docker-compose.yaml b/test/mock/collector/docker-compose.yaml index 9330c9b4e6..050837dc23 100644 --- a/test/mock/collector/docker-compose.yaml +++ b/test/mock/collector/docker-compose.yaml @@ -12,7 +12,8 @@ services: container_name: mock-collector-agent-with-nginx-plus volumes: - ./nginx-agent.conf:/etc/nginx-agent/nginx-agent.conf - - ./nginx-plus:/etc/nginx/ + - ./nginx-plus/nginx.conf:/etc/nginx/nginx.conf + - ./nginx-plus/conf.d/default.conf:/etc/nginx/conf.d/default.conf networks: - metrics @@ -59,6 +60,19 @@ services: - grafana-storage:/var/lib/grafana - ./grafana/provisioning/datasources:/etc/grafana/provisioning/datasources - ./grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards + - ./grafana/provisioning/plugins:/etc/grafana/provisioning/plugins - ./grafana/provisioning/dashboards:/var/lib/grafana/dashboards networks: - metrics + + loki: + image: grafana/loki:latest + container_name: mock-collector-loki + restart: unless-stopped + ports: + - "3100:3100" + volumes: + - ./loki-config.yaml:/etc/loki/local-config.yaml + command: -config.file=/etc/loki/local-config.yaml + networks: + - metrics diff --git a/test/mock/collector/grafana/provisioning/datasources/datasource.yml b/test/mock/collector/grafana/provisioning/datasources/datasource.yml index 8732222c7f..356ff441f2 100644 --- a/test/mock/collector/grafana/provisioning/datasources/datasource.yml +++ b/test/mock/collector/grafana/provisioning/datasources/datasource.yml @@ -7,3 +7,8 @@ datasources: access: proxy url: http://prometheus:9090 isDefault: true + - name: Loki + type: loki + uid: otel-loki-scraper + access: proxy + url: http://loki:3100 diff --git a/test/mock/collector/mock-collector/main.go b/test/mock/collector/mock-collector/main.go index e036ad12e6..3a7bdda754 100644 --- a/test/mock/collector/mock-collector/main.go +++ b/test/mock/collector/mock-collector/main.go @@ -16,10 +16,12 @@ import ( "github.com/nginx/agent/v3/test/mock/collector/mock-collector/auth" "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusexporter" + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourceprocessor" "go.opentelemetry.io/collector/connector" "go.opentelemetry.io/collector/exporter" "go.opentelemetry.io/collector/exporter/debugexporter" "go.opentelemetry.io/collector/exporter/otlpexporter" + "go.opentelemetry.io/collector/exporter/otlphttpexporter" "go.opentelemetry.io/collector/extension" "go.opentelemetry.io/collector/processor" "go.opentelemetry.io/collector/processor/batchprocessor" @@ -85,6 +87,7 @@ func components() (otelcol.Factories, error) { debugexporter.NewFactory(), otlpexporter.NewFactory(), prometheusexporter.NewFactory(), + otlphttpexporter.NewFactory(), ) if err != nil { return otelcol.Factories{}, err @@ -92,6 +95,7 @@ func components() (otelcol.Factories, error) { factories.Processors, err = processor.MakeFactoryMap( batchprocessor.NewFactory(), + resourceprocessor.NewFactory(), ) if err != nil { return otelcol.Factories{}, err diff --git a/test/mock/collector/nginx-plus/nginx.conf b/test/mock/collector/nginx-plus/nginx.conf index e26d0b5a18..6c95de7b44 100644 --- a/test/mock/collector/nginx-plus/nginx.conf +++ b/test/mock/collector/nginx-plus/nginx.conf @@ -1,6 +1,8 @@ user nginx; worker_processes auto; - + +load_module modules/ngx_http_app_protect_module.so; + error_log /var/log/nginx/error.log notice; pid /var/run/nginx.pid; @@ -80,6 +82,23 @@ http { status_zone my_location_zone2; } } + + server { + listen 8099; + server_name localhost; + proxy_http_version 1.1; + + app_protect_enable on; + app_protect_policy_file "/etc/app_protect/conf/NginxDefaultPolicy.json"; + app_protect_security_log_enable on; + app_protect_security_log "/etc/app_protect/conf/log_default.json" syslog:server=127.0.0.1:5141; + + location / { + client_max_body_size 0; + default_type text/html; + proxy_pass http://172.29.38.211/; + } + } include /etc/nginx/conf.d/*.conf; } diff --git a/test/mock/collector/otel-collector.yaml b/test/mock/collector/otel-collector.yaml index b1e801a9cc..26883be811 100644 --- a/test/mock/collector/otel-collector.yaml +++ b/test/mock/collector/otel-collector.yaml @@ -15,9 +15,16 @@ exporters: verbosity: detailed sampling_initial: 5 sampling_thereafter: 200 + otlphttp: + endpoint: http://loki:3100/otlp processors: batch: + resource: + attributes: + - key: service.name + value: "nginx.app.protect" + action: insert extensions: headers_check: @@ -34,5 +41,5 @@ service: exporters: [prometheus] logs: receivers: [otlp] - processors: [batch] - exporters: [debug] + processors: [resource, batch] + exporters: [otlphttp] From 2597fd6478afb37792a75dbd1f10a34e1ca2037c Mon Sep 17 00:00:00 2001 From: dhurley Date: Mon, 2 Dec 2024 17:05:07 +0000 Subject: [PATCH 14/24] Visualize NAP logs in mock collector grafana --- .../grafana/provisioning/plugins/plugin.yaml | 7 ++++ test/mock/collector/loki-config.yaml | 36 +++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 test/mock/collector/grafana/provisioning/plugins/plugin.yaml create mode 100644 test/mock/collector/loki-config.yaml diff --git a/test/mock/collector/grafana/provisioning/plugins/plugin.yaml b/test/mock/collector/grafana/provisioning/plugins/plugin.yaml new file mode 100644 index 0000000000..accccf6229 --- /dev/null +++ b/test/mock/collector/grafana/provisioning/plugins/plugin.yaml @@ -0,0 +1,7 @@ +apiVersion: 1 + +apps: + - type: "grafana-lokiexplore-app" + org_id: 1 + org_name: "Grafana" + disabled: false diff --git a/test/mock/collector/loki-config.yaml b/test/mock/collector/loki-config.yaml new file mode 100644 index 0000000000..23aadb4502 --- /dev/null +++ b/test/mock/collector/loki-config.yaml @@ -0,0 +1,36 @@ +auth_enabled: false + +limits_config: + allow_structured_metadata: true + volume_enabled: true + +server: + http_listen_port: 3100 + +common: + ring: + instance_addr: 0.0.0.0 + kvstore: + store: inmemory + replication_factor: 1 + path_prefix: /tmp/loki + +schema_config: + configs: + - from: 2020-05-15 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +storage_config: + tsdb_shipper: + active_index_directory: /tmp/loki/index + cache_location: /tmp/loki/index_cache + filesystem: + directory: /tmp/loki/chunks + +pattern_ingester: + enabled: true From 079d594d038e0091bcdf7d510da58dde14436a28 Mon Sep 17 00:00:00 2001 From: dhurley Date: Tue, 3 Dec 2024 16:39:47 +0000 Subject: [PATCH 15/24] Visualize NAP logs in mock collector grafana --- test/mock/collector/docker-compose.yaml | 2 ++ test/mock/collector/grafana/provisioning/plugins/plugin.yaml | 4 +--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/mock/collector/docker-compose.yaml b/test/mock/collector/docker-compose.yaml index 050837dc23..39324e1206 100644 --- a/test/mock/collector/docker-compose.yaml +++ b/test/mock/collector/docker-compose.yaml @@ -54,6 +54,8 @@ services: image: grafana/grafana:latest container_name: mock-collector-grafana restart: unless-stopped + environment: + GF_INSTALL_PLUGINS: "grafana-lokiexplore-app" ports: - "3002:3000" volumes: diff --git a/test/mock/collector/grafana/provisioning/plugins/plugin.yaml b/test/mock/collector/grafana/provisioning/plugins/plugin.yaml index accccf6229..14a369088a 100644 --- a/test/mock/collector/grafana/provisioning/plugins/plugin.yaml +++ b/test/mock/collector/grafana/provisioning/plugins/plugin.yaml @@ -1,7 +1,5 @@ apiVersion: 1 apps: - - type: "grafana-lokiexplore-app" + - type: grafana-lokiexplore-app org_id: 1 - org_name: "Grafana" - disabled: false From ddb02df1f4919328e219a2c67667b05fcff09eb1 Mon Sep 17 00:00:00 2001 From: dhurley Date: Tue, 3 Dec 2024 17:01:50 +0000 Subject: [PATCH 16/24] Remove key_value_parser operator --- internal/collector/otel_collector_plugin.go | 8 -------- internal/collector/otel_collector_plugin_test.go | 4 ++-- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/internal/collector/otel_collector_plugin.go b/internal/collector/otel_collector_plugin.go index baf8255c8a..c4899dc2db 100644 --- a/internal/collector/otel_collector_plugin.go +++ b/internal/collector/otel_collector_plugin.go @@ -492,14 +492,6 @@ func (oc *Collector) updateTcplogReceivers(nginxConfigContext *model.NginxConfig "protocol": "rfc3164", }, }, - { - Type: "key_value_parser", - Fields: map[string]string{ - "parse_from": "attributes.message", - "parse_to": "body", - "pair_delimiter": "\",\"", - }, - }, { Type: "remove", Fields: map[string]string{ diff --git a/internal/collector/otel_collector_plugin_test.go b/internal/collector/otel_collector_plugin_test.go index 28c76c3867..67e5ecdd0a 100644 --- a/internal/collector/otel_collector_plugin_test.go +++ b/internal/collector/otel_collector_plugin_test.go @@ -674,7 +674,7 @@ func TestCollector_updateTcplogReceivers(t *testing.T) { assert.True(tt, tcplogReceiverAdded) assert.Len(tt, conf.Collector.Receivers.TcplogReceivers, 1) assert.Equal(tt, "localhost:151", conf.Collector.Receivers.TcplogReceivers[0].ListenAddress) - assert.Len(tt, conf.Collector.Receivers.TcplogReceivers[0].Operators, 5) + assert.Len(tt, conf.Collector.Receivers.TcplogReceivers[0].Operators, 4) }) // Calling updateTcplogReceivers shouldn't update the TcplogReceivers slice @@ -684,7 +684,7 @@ func TestCollector_updateTcplogReceivers(t *testing.T) { assert.False(t, tcplogReceiverAdded) assert.Len(t, conf.Collector.Receivers.TcplogReceivers, 1) assert.Equal(t, "localhost:151", conf.Collector.Receivers.TcplogReceivers[0].ListenAddress) - assert.Len(t, conf.Collector.Receivers.TcplogReceivers[0].Operators, 5) + assert.Len(t, conf.Collector.Receivers.TcplogReceivers[0].Operators, 4) }) } From 5d0f9b009cfee0160e995eece291419bde6509e8 Mon Sep 17 00:00:00 2001 From: dhurley Date: Wed, 4 Dec 2024 15:57:04 +0000 Subject: [PATCH 17/24] Update tcplog timestamp operator --- internal/collector/otel_collector_plugin.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/internal/collector/otel_collector_plugin.go b/internal/collector/otel_collector_plugin.go index c4899dc2db..2d8e326938 100644 --- a/internal/collector/otel_collector_plugin.go +++ b/internal/collector/otel_collector_plugin.go @@ -24,10 +24,18 @@ import ( ) const ( - maxTimeToWaitForShutdown = 30 * time.Second - filePermission = 0o600 - timestampConversionExpression = `EXPR(split(body, ">")[0] + ">" + ` + - `date(split(split(body, ">")[1], " ")[0]).Format("Jan 02 15:04:05") + " " + split(body, " ", 2)[1])` + maxTimeToWaitForShutdown = 30 * time.Second + filePermission = 0o600 + // To conform to the rfc3164 spec the timestamp in the logs need to be formatted correctly. + // Here are some examples of what the timestamp conversions look like. + // Notice how if the day begins with a zero that the zero is replaced with an empty space. + + // 2024-11-06T17:19:24+00:00 ---> Nov 6 17:19:24 + // 2024-11-16T17:19:24+00:00 ---> Nov 16 17:19:24 + timestampConversionExpression = `'EXPR(let timestamp = split(split(body, ">")[1], " ")[0]; ` + + `let newTimestamp = timestamp matches "(\\d{4})-(\\d{2})-(0\\d{1})T(\\d{2}):(\\d{2}):(\\d{2}).*" ` + + `? date(timestamp).Format("Jan 2 15:04:05") : date(timestamp).Format("Jan 02 15:04:05"); ` + + `split(body, ">")[0] + ">" + newTimestamp + " " + split(body, " ", 2)[1])'` ) type ( From a65e3d32221d50619a72843ad3495dd04cd77876 Mon Sep 17 00:00:00 2001 From: dhurley Date: Tue, 17 Dec 2024 16:50:57 +0000 Subject: [PATCH 18/24] Update NGINX plus dockerfile to always build with platform set to amd64. Updated makefile target names to be more consistent. --- Makefile | 10 +- api/grpc/mpi/v1/command.pb.go | 321 +++++++++--------- api/grpc/mpi/v1/common.pb.go | 20 +- api/grpc/mpi/v1/files.pb.go | 167 +++++---- test/docker/nginx-plus/deb/Dockerfile | 2 +- test/mock/collector/README.md | 22 +- test/mock/collector/mock-collector/Dockerfile | 6 +- 7 files changed, 267 insertions(+), 281 deletions(-) diff --git a/Makefile b/Makefile index dd0c84e651..8d5d7a6dcb 100644 --- a/Makefile +++ b/Makefile @@ -150,9 +150,9 @@ build-mock-management-plane-grpc: mkdir -p $(BUILD_DIR)/mock-management-plane-grpc @CGO_ENABLED=0 GOARCH=$(OSARCH) GOOS=linux $(GOBUILD) -o $(BUILD_DIR)/mock-management-plane-grpc/server test/mock/grpc/cmd/main.go -build-mock-management-plane-collector: - mkdir -p $(BUILD_DIR)/mock-management-plane-collector - @CGO_ENABLED=0 GOARCH=$(OSARCH) GOOS=linux $(GOBUILD) -o $(BUILD_DIR)/mock-management-plane-collector/collector test/mock/collector/mock-collector/main.go +build-mock-management-otel-collector: + mkdir -p $(BUILD_DIR)/mock-management-otel-collector + @CGO_ENABLED=0 GOARCH=$(OSARCH) GOOS=linux $(GOBUILD) -o $(BUILD_DIR)/mock-management-otel-collector/collector test/mock/collector/mock-collector/main.go integration-test: $(SELECTED_PACKAGE) build-mock-management-plane-grpc TEST_ENV="Container" CONTAINER_OS_TYPE=$(CONTAINER_OS_TYPE) BUILD_TARGET="install-agent-local" CONTAINER_NGINX_IMAGE_REGISTRY=${CONTAINER_NGINX_IMAGE_REGISTRY} \ @@ -211,8 +211,8 @@ build-test-oss-image: --build-arg BASE_IMAGE=$(BASE_IMAGE) \ --build-arg ENTRY_POINT=./test/docker/entrypoint.sh -.PHONY: build-mock-collector-image -build-mock-collector-image: build-mock-management-plane-collector +.PHONY: build-mock-management-otel-collector-image +build-mock-collector-image: build-mock-management-otel-collector $(CONTAINER_BUILDENV) $(CONTAINER_CLITOOL) build -t mock-collector . \ --no-cache -f ./test/mock/collector/mock-collector/Dockerfile diff --git a/api/grpc/mpi/v1/command.pb.go b/api/grpc/mpi/v1/command.pb.go index e390eb5ac1..bfdaa782eb 100644 --- a/api/grpc/mpi/v1/command.pb.go +++ b/api/grpc/mpi/v1/command.pb.go @@ -8,7 +8,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.35.2 +// protoc-gen-go v1.36.0 // protoc (unknown) // source: mpi/v1/command.proto @@ -150,14 +150,13 @@ func (InstanceMeta_InstanceType) EnumDescriptor() ([]byte, []int) { // The connection request is an initial handshake to establish a connection, sending NGINX Agent instance information type CreateConnectionRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // Meta-information associated with a message MessageMeta *MessageMeta `protobuf:"bytes,1,opt,name=message_meta,json=messageMeta,proto3" json:"message_meta,omitempty"` // Instance and infrastructure information associated with the NGINX Agent - Resource *Resource `protobuf:"bytes,2,opt,name=resource,proto3" json:"resource,omitempty"` + Resource *Resource `protobuf:"bytes,2,opt,name=resource,proto3" json:"resource,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *CreateConnectionRequest) Reset() { @@ -206,21 +205,20 @@ func (x *CreateConnectionRequest) GetResource() *Resource { // A representation of instances and runtime resource information type Resource struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // A resource identifier ResourceId string `protobuf:"bytes,1,opt,name=resource_id,json=resourceId,proto3" json:"resource_id,omitempty"` // A list of instances associated with this resource Instances []*Instance `protobuf:"bytes,2,rep,name=instances,proto3" json:"instances,omitempty"` // Runtime information of where the instances reside. Host or container based // - // Types that are assignable to Info: + // Types that are valid to be assigned to Info: // // *Resource_HostInfo // *Resource_ContainerInfo - Info isResource_Info `protobuf_oneof:"info"` + Info isResource_Info `protobuf_oneof:"info"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *Resource) Reset() { @@ -267,23 +265,27 @@ func (x *Resource) GetInstances() []*Instance { return nil } -func (m *Resource) GetInfo() isResource_Info { - if m != nil { - return m.Info +func (x *Resource) GetInfo() isResource_Info { + if x != nil { + return x.Info } return nil } func (x *Resource) GetHostInfo() *HostInfo { - if x, ok := x.GetInfo().(*Resource_HostInfo); ok { - return x.HostInfo + if x != nil { + if x, ok := x.Info.(*Resource_HostInfo); ok { + return x.HostInfo + } } return nil } func (x *Resource) GetContainerInfo() *ContainerInfo { - if x, ok := x.GetInfo().(*Resource_ContainerInfo); ok { - return x.ContainerInfo + if x != nil { + if x, ok := x.Info.(*Resource_ContainerInfo); ok { + return x.ContainerInfo + } } return nil } @@ -308,16 +310,15 @@ func (*Resource_ContainerInfo) isResource_Info() {} // Represents the host system information type HostInfo struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // The host identifier HostId string `protobuf:"bytes,1,opt,name=host_id,json=hostId,proto3" json:"host_id,omitempty"` // The name of the host Hostname string `protobuf:"bytes,2,opt,name=hostname,proto3" json:"hostname,omitempty"` // Release information of the host - ReleaseInfo *ReleaseInfo `protobuf:"bytes,3,opt,name=release_info,json=releaseInfo,proto3" json:"release_info,omitempty"` + ReleaseInfo *ReleaseInfo `protobuf:"bytes,3,opt,name=release_info,json=releaseInfo,proto3" json:"release_info,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *HostInfo) Reset() { @@ -373,10 +374,7 @@ func (x *HostInfo) GetReleaseInfo() *ReleaseInfo { // Release information of the host type ReleaseInfo struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // OS type (e.g. freebsd, linux, etc) Codename string `protobuf:"bytes,1,opt,name=codename,proto3" json:"codename,omitempty"` // OS name (e.g. ubuntu, linuxmint, etc) @@ -386,7 +384,9 @@ type ReleaseInfo struct { // Version of the OS kernel VersionId string `protobuf:"bytes,4,opt,name=version_id,json=versionId,proto3" json:"version_id,omitempty"` // Version of the OS - Version string `protobuf:"bytes,5,opt,name=version,proto3" json:"version,omitempty"` + Version string `protobuf:"bytes,5,opt,name=version,proto3" json:"version,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ReleaseInfo) Reset() { @@ -456,14 +456,13 @@ func (x *ReleaseInfo) GetVersion() string { // Container information type ContainerInfo struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // The identifier of the container ContainerId string `protobuf:"bytes,1,opt,name=container_id,json=containerId,proto3" json:"container_id,omitempty"` // The name of the host - Hostname string `protobuf:"bytes,2,opt,name=hostname,proto3" json:"hostname,omitempty"` + Hostname string `protobuf:"bytes,2,opt,name=hostname,proto3" json:"hostname,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ContainerInfo) Reset() { @@ -512,14 +511,13 @@ func (x *ContainerInfo) GetHostname() string { // A response to a CreateConnectionRequest type CreateConnectionResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // The success or failure of the CreateConnectionRequest Response *CommandResponse `protobuf:"bytes,1,opt,name=response,proto3" json:"response,omitempty"` // The recommendation NGINX Agent configurations provided by the ManagementPlane - AgentConfig *AgentConfig `protobuf:"bytes,2,opt,name=agent_config,json=agentConfig,proto3" json:"agent_config,omitempty"` + AgentConfig *AgentConfig `protobuf:"bytes,2,opt,name=agent_config,json=agentConfig,proto3" json:"agent_config,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *CreateConnectionResponse) Reset() { @@ -568,14 +566,13 @@ func (x *CreateConnectionResponse) GetAgentConfig() *AgentConfig { // Report on the status of the Data Plane type UpdateDataPlaneStatusRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // Meta-information associated with a message MessageMeta *MessageMeta `protobuf:"bytes,1,opt,name=message_meta,json=messageMeta,proto3" json:"message_meta,omitempty"` // the representation of a data plane - Resource *Resource `protobuf:"bytes,2,opt,name=resource,proto3" json:"resource,omitempty"` + Resource *Resource `protobuf:"bytes,2,opt,name=resource,proto3" json:"resource,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *UpdateDataPlaneStatusRequest) Reset() { @@ -624,9 +621,9 @@ func (x *UpdateDataPlaneStatusRequest) GetResource() *Resource { // Respond to a UpdateDataPlaneStatusRequest - intentionally empty type UpdateDataPlaneStatusResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *UpdateDataPlaneStatusResponse) Reset() { @@ -660,15 +657,14 @@ func (*UpdateDataPlaneStatusResponse) Descriptor() ([]byte, []int) { } type InstanceHealth struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - InstanceId string `protobuf:"bytes,1,opt,name=instance_id,json=instanceId,proto3" json:"instance_id,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + InstanceId string `protobuf:"bytes,1,opt,name=instance_id,json=instanceId,proto3" json:"instance_id,omitempty"` // Health status InstanceHealthStatus InstanceHealth_InstanceHealthStatus `protobuf:"varint,2,opt,name=instance_health_status,json=instanceHealthStatus,proto3,enum=mpi.v1.InstanceHealth_InstanceHealthStatus" json:"instance_health_status,omitempty"` // Provides a human readable context around why a health status is a particular state - Description string `protobuf:"bytes,3,opt,name=description,proto3" json:"description,omitempty"` + Description string `protobuf:"bytes,3,opt,name=description,proto3" json:"description,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *InstanceHealth) Reset() { @@ -724,14 +720,13 @@ func (x *InstanceHealth) GetDescription() string { // Health report of a set of instances type UpdateDataPlaneHealthRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // Meta-information associated with a message MessageMeta *MessageMeta `protobuf:"bytes,1,opt,name=message_meta,json=messageMeta,proto3" json:"message_meta,omitempty"` // Health report of a set of instances InstanceHealths []*InstanceHealth `protobuf:"bytes,2,rep,name=instance_healths,json=instanceHealths,proto3" json:"instance_healths,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *UpdateDataPlaneHealthRequest) Reset() { @@ -780,9 +775,9 @@ func (x *UpdateDataPlaneHealthRequest) GetInstanceHealths() []*InstanceHealth { // Response to a UpdateDataPlaneHealthRequest - intentionally empty type UpdateDataPlaneHealthResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *UpdateDataPlaneHealthResponse) Reset() { @@ -817,16 +812,15 @@ func (*UpdateDataPlaneHealthResponse) Descriptor() ([]byte, []int) { // Reports the status of an associated command. This may be in response to a ManagementPlaneRequest type DataPlaneResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // Meta-information associated with a message MessageMeta *MessageMeta `protobuf:"bytes,1,opt,name=message_meta,json=messageMeta,proto3" json:"message_meta,omitempty"` // The command response with the associated request CommandResponse *CommandResponse `protobuf:"bytes,2,opt,name=command_response,json=commandResponse,proto3" json:"command_response,omitempty"` // The instance identifier, if applicable, for this response - InstanceId string `protobuf:"bytes,3,opt,name=instance_id,json=instanceId,proto3" json:"instance_id,omitempty"` + InstanceId string `protobuf:"bytes,3,opt,name=instance_id,json=instanceId,proto3" json:"instance_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *DataPlaneResponse) Reset() { @@ -882,13 +876,10 @@ func (x *DataPlaneResponse) GetInstanceId() string { // A Management Plane request for information, triggers an associated rpc on the Data Plane type ManagementPlaneRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // Meta-information associated with a message MessageMeta *MessageMeta `protobuf:"bytes,1,opt,name=message_meta,json=messageMeta,proto3" json:"message_meta,omitempty"` - // Types that are assignable to Request: + // Types that are valid to be assigned to Request: // // *ManagementPlaneRequest_StatusRequest // *ManagementPlaneRequest_HealthRequest @@ -896,7 +887,9 @@ type ManagementPlaneRequest struct { // *ManagementPlaneRequest_ConfigUploadRequest // *ManagementPlaneRequest_ActionRequest // *ManagementPlaneRequest_CommandStatusRequest - Request isManagementPlaneRequest_Request `protobuf_oneof:"request"` + Request isManagementPlaneRequest_Request `protobuf_oneof:"request"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ManagementPlaneRequest) Reset() { @@ -936,51 +929,63 @@ func (x *ManagementPlaneRequest) GetMessageMeta() *MessageMeta { return nil } -func (m *ManagementPlaneRequest) GetRequest() isManagementPlaneRequest_Request { - if m != nil { - return m.Request +func (x *ManagementPlaneRequest) GetRequest() isManagementPlaneRequest_Request { + if x != nil { + return x.Request } return nil } func (x *ManagementPlaneRequest) GetStatusRequest() *StatusRequest { - if x, ok := x.GetRequest().(*ManagementPlaneRequest_StatusRequest); ok { - return x.StatusRequest + if x != nil { + if x, ok := x.Request.(*ManagementPlaneRequest_StatusRequest); ok { + return x.StatusRequest + } } return nil } func (x *ManagementPlaneRequest) GetHealthRequest() *HealthRequest { - if x, ok := x.GetRequest().(*ManagementPlaneRequest_HealthRequest); ok { - return x.HealthRequest + if x != nil { + if x, ok := x.Request.(*ManagementPlaneRequest_HealthRequest); ok { + return x.HealthRequest + } } return nil } func (x *ManagementPlaneRequest) GetConfigApplyRequest() *ConfigApplyRequest { - if x, ok := x.GetRequest().(*ManagementPlaneRequest_ConfigApplyRequest); ok { - return x.ConfigApplyRequest + if x != nil { + if x, ok := x.Request.(*ManagementPlaneRequest_ConfigApplyRequest); ok { + return x.ConfigApplyRequest + } } return nil } func (x *ManagementPlaneRequest) GetConfigUploadRequest() *ConfigUploadRequest { - if x, ok := x.GetRequest().(*ManagementPlaneRequest_ConfigUploadRequest); ok { - return x.ConfigUploadRequest + if x != nil { + if x, ok := x.Request.(*ManagementPlaneRequest_ConfigUploadRequest); ok { + return x.ConfigUploadRequest + } } return nil } func (x *ManagementPlaneRequest) GetActionRequest() *APIActionRequest { - if x, ok := x.GetRequest().(*ManagementPlaneRequest_ActionRequest); ok { - return x.ActionRequest + if x != nil { + if x, ok := x.Request.(*ManagementPlaneRequest_ActionRequest); ok { + return x.ActionRequest + } } return nil } func (x *ManagementPlaneRequest) GetCommandStatusRequest() *CommandStatusRequest { - if x, ok := x.GetRequest().(*ManagementPlaneRequest_CommandStatusRequest); ok { - return x.CommandStatusRequest + if x != nil { + if x, ok := x.Request.(*ManagementPlaneRequest_CommandStatusRequest); ok { + return x.CommandStatusRequest + } } return nil } @@ -1034,9 +1039,9 @@ func (*ManagementPlaneRequest_CommandStatusRequest) isManagementPlaneRequest_Req // Additional information associated with a StatusRequest type StatusRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *StatusRequest) Reset() { @@ -1071,9 +1076,9 @@ func (*StatusRequest) Descriptor() ([]byte, []int) { // Additional information associated with a HealthRequest type HealthRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *HealthRequest) Reset() { @@ -1108,12 +1113,11 @@ func (*HealthRequest) Descriptor() ([]byte, []int) { // Additional information associated with a ConfigApplyRequest type ConfigApplyRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // set of files related to the request - Overview *FileOverview `protobuf:"bytes,1,opt,name=overview,proto3" json:"overview,omitempty"` + Overview *FileOverview `protobuf:"bytes,1,opt,name=overview,proto3" json:"overview,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ConfigApplyRequest) Reset() { @@ -1155,12 +1159,11 @@ func (x *ConfigApplyRequest) GetOverview() *FileOverview { // Additional information associated with a ConfigUploadRequest type ConfigUploadRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // set of files related to the request - Overview *FileOverview `protobuf:"bytes,1,opt,name=overview,proto3" json:"overview,omitempty"` + Overview *FileOverview `protobuf:"bytes,1,opt,name=overview,proto3" json:"overview,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ConfigUploadRequest) Reset() { @@ -1202,9 +1205,9 @@ func (x *ConfigUploadRequest) GetOverview() *FileOverview { // Perform an associated API action on an instance type APIActionRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *APIActionRequest) Reset() { @@ -1239,9 +1242,9 @@ func (*APIActionRequest) Descriptor() ([]byte, []int) { // Request an update on a particular command type CommandStatusRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *CommandStatusRequest) Reset() { @@ -1276,16 +1279,15 @@ func (*CommandStatusRequest) Descriptor() ([]byte, []int) { // This represents an instance being reported on type Instance struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // Meta-information associated with an instance InstanceMeta *InstanceMeta `protobuf:"bytes,1,opt,name=instance_meta,json=instanceMeta,proto3" json:"instance_meta,omitempty"` // Read and write configuration associated with an instance that can be modified via this definition InstanceConfig *InstanceConfig `protobuf:"bytes,2,opt,name=instance_config,json=instanceConfig,proto3" json:"instance_config,omitempty"` // Read-only meta data associated with the instance running in it's environment InstanceRuntime *InstanceRuntime `protobuf:"bytes,3,opt,name=instance_runtime,json=instanceRuntime,proto3" json:"instance_runtime,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *Instance) Reset() { @@ -1341,16 +1343,15 @@ func (x *Instance) GetInstanceRuntime() *InstanceRuntime { // Meta-information relating to the reported instance type InstanceMeta struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // the identifier associated with the instance InstanceId string `protobuf:"bytes,1,opt,name=instance_id,json=instanceId,proto3" json:"instance_id,omitempty"` // the types of instances possible InstanceType InstanceMeta_InstanceType `protobuf:"varint,2,opt,name=instance_type,json=instanceType,proto3,enum=mpi.v1.InstanceMeta_InstanceType" json:"instance_type,omitempty"` // the version of the instance - Version string `protobuf:"bytes,3,opt,name=version,proto3" json:"version,omitempty"` + Version string `protobuf:"bytes,3,opt,name=version,proto3" json:"version,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *InstanceMeta) Reset() { @@ -1406,16 +1407,15 @@ func (x *InstanceMeta) GetVersion() string { // Instance Configuration options type InstanceConfig struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // provided actions associated with a particular instance. These are runtime based and provided by a particular version of the NGINX Agent Actions []*InstanceAction `protobuf:"bytes,1,rep,name=actions,proto3" json:"actions,omitempty"` - // Types that are assignable to Config: + // Types that are valid to be assigned to Config: // // *InstanceConfig_AgentConfig - Config isInstanceConfig_Config `protobuf_oneof:"config"` + Config isInstanceConfig_Config `protobuf_oneof:"config"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *InstanceConfig) Reset() { @@ -1455,16 +1455,18 @@ func (x *InstanceConfig) GetActions() []*InstanceAction { return nil } -func (m *InstanceConfig) GetConfig() isInstanceConfig_Config { - if m != nil { - return m.Config +func (x *InstanceConfig) GetConfig() isInstanceConfig_Config { + if x != nil { + return x.Config } return nil } func (x *InstanceConfig) GetAgentConfig() *AgentConfig { - if x, ok := x.GetConfig().(*InstanceConfig_AgentConfig); ok { - return x.AgentConfig + if x != nil { + if x, ok := x.Config.(*InstanceConfig_AgentConfig); ok { + return x.AgentConfig + } } return nil } @@ -1481,10 +1483,7 @@ type InstanceConfig_AgentConfig struct { func (*InstanceConfig_AgentConfig) isInstanceConfig_Config() {} type InstanceRuntime struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // the process identifier ProcessId int32 `protobuf:"varint,1,opt,name=process_id,json=processId,proto3" json:"process_id,omitempty"` // the binary path location @@ -1493,13 +1492,15 @@ type InstanceRuntime struct { ConfigPath string `protobuf:"bytes,3,opt,name=config_path,json=configPath,proto3" json:"config_path,omitempty"` // more detailed runtime objects // - // Types that are assignable to Details: + // Types that are valid to be assigned to Details: // // *InstanceRuntime_NginxRuntimeInfo // *InstanceRuntime_NginxPlusRuntimeInfo Details isInstanceRuntime_Details `protobuf_oneof:"details"` // List of worker processes InstanceChildren []*InstanceChild `protobuf:"bytes,6,rep,name=instance_children,json=instanceChildren,proto3" json:"instance_children,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *InstanceRuntime) Reset() { @@ -1553,23 +1554,27 @@ func (x *InstanceRuntime) GetConfigPath() string { return "" } -func (m *InstanceRuntime) GetDetails() isInstanceRuntime_Details { - if m != nil { - return m.Details +func (x *InstanceRuntime) GetDetails() isInstanceRuntime_Details { + if x != nil { + return x.Details } return nil } func (x *InstanceRuntime) GetNginxRuntimeInfo() *NGINXRuntimeInfo { - if x, ok := x.GetDetails().(*InstanceRuntime_NginxRuntimeInfo); ok { - return x.NginxRuntimeInfo + if x != nil { + if x, ok := x.Details.(*InstanceRuntime_NginxRuntimeInfo); ok { + return x.NginxRuntimeInfo + } } return nil } func (x *InstanceRuntime) GetNginxPlusRuntimeInfo() *NGINXPlusRuntimeInfo { - if x, ok := x.GetDetails().(*InstanceRuntime_NginxPlusRuntimeInfo); ok { - return x.NginxPlusRuntimeInfo + if x != nil { + if x, ok := x.Details.(*InstanceRuntime_NginxPlusRuntimeInfo); ok { + return x.NginxPlusRuntimeInfo + } } return nil } @@ -1600,12 +1605,11 @@ func (*InstanceRuntime_NginxRuntimeInfo) isInstanceRuntime_Details() {} func (*InstanceRuntime_NginxPlusRuntimeInfo) isInstanceRuntime_Details() {} type InstanceChild struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // the process identifier - ProcessId int32 `protobuf:"varint,1,opt,name=process_id,json=processId,proto3" json:"process_id,omitempty"` + ProcessId int32 `protobuf:"varint,1,opt,name=process_id,json=processId,proto3" json:"process_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *InstanceChild) Reset() { @@ -1647,10 +1651,7 @@ func (x *InstanceChild) GetProcessId() int32 { // A set of runtime NGINX OSS settings type NGINXRuntimeInfo struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // the stub status API URL StubStatus string `protobuf:"bytes,1,opt,name=stub_status,json=stubStatus,proto3" json:"stub_status,omitempty"` // a list of access_logs @@ -1661,6 +1662,8 @@ type NGINXRuntimeInfo struct { LoadableModules []string `protobuf:"bytes,4,rep,name=loadable_modules,json=loadableModules,proto3" json:"loadable_modules,omitempty"` // List of NGINX dynamic modules. DynamicModules []string `protobuf:"bytes,5,rep,name=dynamic_modules,json=dynamicModules,proto3" json:"dynamic_modules,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NGINXRuntimeInfo) Reset() { @@ -1730,10 +1733,7 @@ func (x *NGINXRuntimeInfo) GetDynamicModules() []string { // A set of runtime NGINX Plus settings type NGINXPlusRuntimeInfo struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // the stub status API URL StubStatus string `protobuf:"bytes,1,opt,name=stub_status,json=stubStatus,proto3" json:"stub_status,omitempty"` // a list of access_logs @@ -1745,7 +1745,9 @@ type NGINXPlusRuntimeInfo struct { // List of NGINX dynamic modules. DynamicModules []string `protobuf:"bytes,5,rep,name=dynamic_modules,json=dynamicModules,proto3" json:"dynamic_modules,omitempty"` // the plus API location - PlusApi string `protobuf:"bytes,6,opt,name=plus_api,json=plusApi,proto3" json:"plus_api,omitempty"` + PlusApi string `protobuf:"bytes,6,opt,name=plus_api,json=plusApi,proto3" json:"plus_api,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NGINXPlusRuntimeInfo) Reset() { @@ -1822,9 +1824,9 @@ func (x *NGINXPlusRuntimeInfo) GetPlusApi() string { // A set of actions that can be performed on an instance type InstanceAction struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *InstanceAction) Reset() { @@ -1859,10 +1861,7 @@ func (*InstanceAction) Descriptor() ([]byte, []int) { // This contains a series of NGINX Agent configurations type AgentConfig struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // Command server settings Command *CommandServer `protobuf:"bytes,1,opt,name=command,proto3" json:"command,omitempty"` // Metrics server settings @@ -1875,6 +1874,8 @@ type AgentConfig struct { Features []string `protobuf:"bytes,5,rep,name=features,proto3" json:"features,omitempty"` // Message buffer size, maximum not acknowledged messages from the subscribe perspective MessageBufferSize string `protobuf:"bytes,6,opt,name=message_buffer_size,json=messageBufferSize,proto3" json:"message_buffer_size,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *AgentConfig) Reset() { @@ -1951,9 +1952,9 @@ func (x *AgentConfig) GetMessageBufferSize() string { // The command settings, associated with messaging from an external source type CommandServer struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *CommandServer) Reset() { @@ -1988,9 +1989,9 @@ func (*CommandServer) Descriptor() ([]byte, []int) { // The metrics settings associated with origins (sources) of the metrics and destinations (exporter) type MetricsServer struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *MetricsServer) Reset() { @@ -2025,9 +2026,9 @@ func (*MetricsServer) Descriptor() ([]byte, []int) { // The file settings associated with file server for configurations type FileServer struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *FileServer) Reset() { diff --git a/api/grpc/mpi/v1/common.pb.go b/api/grpc/mpi/v1/common.pb.go index 8f28d16ed7..94bda91fde 100644 --- a/api/grpc/mpi/v1/common.pb.go +++ b/api/grpc/mpi/v1/common.pb.go @@ -5,7 +5,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.35.2 +// protoc-gen-go v1.36.0 // protoc (unknown) // source: mpi/v1/common.proto @@ -90,16 +90,15 @@ func (CommandResponse_CommandStatus) EnumDescriptor() ([]byte, []int) { // Meta-information associated with a message type MessageMeta struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // uuid v7 monotonically increasing string MessageId string `protobuf:"bytes,1,opt,name=message_id,json=messageId,proto3" json:"message_id,omitempty"` // if 2 or more messages associated with the same workflow, use this field as an association CorrelationId string `protobuf:"bytes,2,opt,name=correlation_id,json=correlationId,proto3" json:"correlation_id,omitempty"` // timestamp for human readable timestamp in UTC format - Timestamp *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=timestamp,proto3" json:"timestamp,omitempty"` + Timestamp *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=timestamp,proto3" json:"timestamp,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *MessageMeta) Reset() { @@ -155,16 +154,15 @@ func (x *MessageMeta) GetTimestamp() *timestamppb.Timestamp { // Represents a the status response of an command type CommandResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // Command status Status CommandResponse_CommandStatus `protobuf:"varint,1,opt,name=status,proto3,enum=mpi.v1.CommandResponse_CommandStatus" json:"status,omitempty"` // Provides a user friendly message to describe the response Message string `protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"` // Provides an error message of why the command failed, only populated when CommandStatus is COMMAND_STATUS_ERROR - Error string `protobuf:"bytes,3,opt,name=error,proto3" json:"error,omitempty"` + Error string `protobuf:"bytes,3,opt,name=error,proto3" json:"error,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *CommandResponse) Reset() { diff --git a/api/grpc/mpi/v1/files.pb.go b/api/grpc/mpi/v1/files.pb.go index 7503e90e77..86c01dc006 100644 --- a/api/grpc/mpi/v1/files.pb.go +++ b/api/grpc/mpi/v1/files.pb.go @@ -5,7 +5,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.35.2 +// protoc-gen-go v1.36.0 // protoc (unknown) // source: mpi/v1/files.proto @@ -199,14 +199,13 @@ func (File_FileAction) EnumDescriptor() ([]byte, []int) { // Represents a request payload for a file overview type GetOverviewRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // Meta-information associated with a message MessageMeta *MessageMeta `protobuf:"bytes,1,opt,name=message_meta,json=messageMeta,proto3" json:"message_meta,omitempty"` // The config version of the overview you are requesting ConfigVersion *ConfigVersion `protobuf:"bytes,2,opt,name=config_version,json=configVersion,proto3" json:"config_version,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *GetOverviewRequest) Reset() { @@ -255,12 +254,11 @@ func (x *GetOverviewRequest) GetConfigVersion() *ConfigVersion { // Represents the response payload to a GetOverviewRequest, requesting a list of logically grouped files e.g. configuration payload type GetOverviewResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // The file overview of an instance - Overview *FileOverview `protobuf:"bytes,1,opt,name=overview,proto3" json:"overview,omitempty"` + Overview *FileOverview `protobuf:"bytes,1,opt,name=overview,proto3" json:"overview,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *GetOverviewResponse) Reset() { @@ -302,14 +300,13 @@ func (x *GetOverviewResponse) GetOverview() *FileOverview { // Represents a list of logically grouped files that have changed e.g. configuration payload type UpdateOverviewRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // Meta-information associated with a message MessageMeta *MessageMeta `protobuf:"bytes,1,opt,name=message_meta,json=messageMeta,proto3" json:"message_meta,omitempty"` // The file overview of an instance - Overview *FileOverview `protobuf:"bytes,2,opt,name=overview,proto3" json:"overview,omitempty"` + Overview *FileOverview `protobuf:"bytes,2,opt,name=overview,proto3" json:"overview,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *UpdateOverviewRequest) Reset() { @@ -358,9 +355,9 @@ func (x *UpdateOverviewRequest) GetOverview() *FileOverview { // Represents a the response from an UpdateOverviewRequest - intentionally left empty type UpdateOverviewResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *UpdateOverviewResponse) Reset() { @@ -395,14 +392,13 @@ func (*UpdateOverviewResponse) Descriptor() ([]byte, []int) { // Represents a specific configuration version associated with an instance type ConfigVersion struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // The instance identifier InstanceId string `protobuf:"bytes,1,opt,name=instance_id,json=instanceId,proto3" json:"instance_id,omitempty"` // The version of the configuration - Version string `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"` + Version string `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ConfigVersion) Reset() { @@ -451,14 +447,13 @@ func (x *ConfigVersion) GetVersion() string { // Represents a collection of files type FileOverview struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // A list of files Files []*File `protobuf:"bytes,1,rep,name=files,proto3" json:"files,omitempty"` // The configuration version of the current set of files ConfigVersion *ConfigVersion `protobuf:"bytes,2,opt,name=config_version,json=configVersion,proto3" json:"config_version,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *FileOverview) Reset() { @@ -507,14 +502,13 @@ func (x *FileOverview) GetConfigVersion() *ConfigVersion { // Represents meta data about a file type File struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // Meta information about the file, the name (including path) and hash FileMeta *FileMeta `protobuf:"bytes,1,opt,name=file_meta,json=fileMeta,proto3" json:"file_meta,omitempty"` // Optional action - Action *File_FileAction `protobuf:"varint,2,opt,name=action,proto3,enum=mpi.v1.File_FileAction,oneof" json:"action,omitempty"` + Action *File_FileAction `protobuf:"varint,2,opt,name=action,proto3,enum=mpi.v1.File_FileAction,oneof" json:"action,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *File) Reset() { @@ -563,14 +557,13 @@ func (x *File) GetAction() File_FileAction { // Represents the get file request type GetFileRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // Meta-information associated with a message MessageMeta *MessageMeta `protobuf:"bytes,1,opt,name=message_meta,json=messageMeta,proto3" json:"message_meta,omitempty"` // Meta-information associated with the file - FileMeta *FileMeta `protobuf:"bytes,2,opt,name=file_meta,json=fileMeta,proto3" json:"file_meta,omitempty"` + FileMeta *FileMeta `protobuf:"bytes,2,opt,name=file_meta,json=fileMeta,proto3" json:"file_meta,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *GetFileRequest) Reset() { @@ -619,12 +612,11 @@ func (x *GetFileRequest) GetFileMeta() *FileMeta { // Represents the response to a get file request type GetFileResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // The contents of a file - Contents *FileContents `protobuf:"bytes,1,opt,name=contents,proto3" json:"contents,omitempty"` + Contents *FileContents `protobuf:"bytes,1,opt,name=contents,proto3" json:"contents,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *GetFileResponse) Reset() { @@ -666,12 +658,11 @@ func (x *GetFileResponse) GetContents() *FileContents { // Represents the bytes contents of the file https://protobuf.dev/programming-guides/api/#dont-encode-data-in-a-string type FileContents struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // Byte representation of a file without encoding - Contents []byte `protobuf:"bytes,1,opt,name=contents,proto3" json:"contents,omitempty"` + Contents []byte `protobuf:"bytes,1,opt,name=contents,proto3" json:"contents,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *FileContents) Reset() { @@ -713,10 +704,7 @@ func (x *FileContents) GetContents() []byte { // Meta information about the file, the name (including path) and hash type FileMeta struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // The full path of the file Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` // The hash of the file contents sha256, hex encoded @@ -729,10 +717,12 @@ type FileMeta struct { Size int64 `protobuf:"varint,5,opt,name=size,proto3" json:"size,omitempty"` // additional file information // - // Types that are assignable to FileType: + // Types that are valid to be assigned to FileType: // // *FileMeta_CertificateMeta - FileType isFileMeta_FileType `protobuf_oneof:"file_type"` + FileType isFileMeta_FileType `protobuf_oneof:"file_type"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *FileMeta) Reset() { @@ -800,16 +790,18 @@ func (x *FileMeta) GetSize() int64 { return 0 } -func (m *FileMeta) GetFileType() isFileMeta_FileType { - if m != nil { - return m.FileType +func (x *FileMeta) GetFileType() isFileMeta_FileType { + if x != nil { + return x.FileType } return nil } func (x *FileMeta) GetCertificateMeta() *CertificateMeta { - if x, ok := x.GetFileType().(*FileMeta_CertificateMeta); ok { - return x.CertificateMeta + if x != nil { + if x, ok := x.FileType.(*FileMeta_CertificateMeta); ok { + return x.CertificateMeta + } } return nil } @@ -826,16 +818,15 @@ func (*FileMeta_CertificateMeta) isFileMeta_FileType() {} // Represents the update file request type UpdateFileRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // The file requested to be updated File *File `protobuf:"bytes,1,opt,name=file,proto3" json:"file,omitempty"` // The contents of a file Contents *FileContents `protobuf:"bytes,2,opt,name=contents,proto3" json:"contents,omitempty"` // Meta-information associated with a message - MessageMeta *MessageMeta `protobuf:"bytes,3,opt,name=message_meta,json=messageMeta,proto3" json:"message_meta,omitempty"` + MessageMeta *MessageMeta `protobuf:"bytes,3,opt,name=message_meta,json=messageMeta,proto3" json:"message_meta,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *UpdateFileRequest) Reset() { @@ -891,12 +882,11 @@ func (x *UpdateFileRequest) GetMessageMeta() *MessageMeta { // Represents the response to an update file request type UpdateFileResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // Meta-information associated with the updated file - FileMeta *FileMeta `protobuf:"bytes,1,opt,name=file_meta,json=fileMeta,proto3" json:"file_meta,omitempty"` + FileMeta *FileMeta `protobuf:"bytes,1,opt,name=file_meta,json=fileMeta,proto3" json:"file_meta,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *UpdateFileResponse) Reset() { @@ -939,10 +929,7 @@ func (x *UpdateFileResponse) GetFileMeta() *FileMeta { // Define the certificate message based on https://pkg.go.dev/crypto/x509#Certificate // and https://github.com/googleapis/googleapis/blob/005df4681b89bd204a90b76168a6dc9d9e7bf4fe/google/cloud/iot/v1/resources.proto#L341 type CertificateMeta struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // Serial number of the certificate, usually a unique identifier, RFC5280 states the upper limit for serial number is 20 octets SerialNumber []byte `protobuf:"bytes,1,opt,name=serial_number,json=serialNumber,proto3" json:"serial_number,omitempty"` // Issuer details (who issued the certificate) @@ -957,6 +944,8 @@ type CertificateMeta struct { SignatureAlgorithm SignatureAlgorithm `protobuf:"varint,6,opt,name=signature_algorithm,json=signatureAlgorithm,proto3,enum=mpi.v1.SignatureAlgorithm" json:"signature_algorithm,omitempty"` // The type of public key in the certificate. PublicKeyAlgorithm string `protobuf:"bytes,7,opt,name=public_key_algorithm,json=publicKeyAlgorithm,proto3" json:"public_key_algorithm,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *CertificateMeta) Reset() { @@ -1040,14 +1029,13 @@ func (x *CertificateMeta) GetPublicKeyAlgorithm() string { // Represents the dates for which a certificate is valid type CertificateDates struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // The start date that for when the certificate is valid NotBefore int64 `protobuf:"varint,1,opt,name=not_before,json=notBefore,proto3" json:"not_before,omitempty"` // The end date that for when the certificate is valid - NotAfter int64 `protobuf:"varint,2,opt,name=not_after,json=notAfter,proto3" json:"not_after,omitempty"` + NotAfter int64 `protobuf:"varint,2,opt,name=not_after,json=notAfter,proto3" json:"not_after,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *CertificateDates) Reset() { @@ -1096,14 +1084,13 @@ func (x *CertificateDates) GetNotAfter() int64 { // Represents the Subject Alternative Names for a certificate type SubjectAlternativeNames struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // List of DNS names in the Subject Alternative Name (SAN) extension DnsNames []string `protobuf:"bytes,1,rep,name=dns_names,json=dnsNames,proto3" json:"dns_names,omitempty"` // List of ip addresses in the SAN extension - IpAddresses []string `protobuf:"bytes,2,rep,name=ip_addresses,json=ipAddresses,proto3" json:"ip_addresses,omitempty"` + IpAddresses []string `protobuf:"bytes,2,rep,name=ip_addresses,json=ipAddresses,proto3" json:"ip_addresses,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *SubjectAlternativeNames) Reset() { @@ -1152,10 +1139,7 @@ func (x *SubjectAlternativeNames) GetIpAddresses() []string { // Represents the dates for which a certificate is valid as seen at https://pkg.go.dev/crypto/x509/pkix#Name type X509Name struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // Country name (C): Two-letter country code as per ISO 3166. // Must be exactly 2 characters. Country []string `protobuf:"bytes,1,rep,name=country,proto3" json:"country,omitempty"` @@ -1181,7 +1165,9 @@ type X509Name struct { Names []*AttributeTypeAndValue `protobuf:"bytes,10,rep,name=names,proto3" json:"names,omitempty"` // Additional attributes to be included in the marshaled distinguished names. // These override any attributes with the same OID in `names`. - ExtraNames []*AttributeTypeAndValue `protobuf:"bytes,11,rep,name=extra_names,json=extraNames,proto3" json:"extra_names,omitempty"` + ExtraNames []*AttributeTypeAndValue `protobuf:"bytes,11,rep,name=extra_names,json=extraNames,proto3" json:"extra_names,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *X509Name) Reset() { @@ -1292,14 +1278,13 @@ func (x *X509Name) GetExtraNames() []*AttributeTypeAndValue { } type AttributeTypeAndValue struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // The type (or identifier) of the attribute (e.g., OID). Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` // The value associated with the attribute. - Value string `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + Value string `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *AttributeTypeAndValue) Reset() { diff --git a/test/docker/nginx-plus/deb/Dockerfile b/test/docker/nginx-plus/deb/Dockerfile index bc4a15dbb0..163a6fbce7 100644 --- a/test/docker/nginx-plus/deb/Dockerfile +++ b/test/docker/nginx-plus/deb/Dockerfile @@ -1,5 +1,5 @@ ARG BASE_IMAGE -FROM ${BASE_IMAGE} as install-nginx +FROM --platform=linux/amd64 ${BASE_IMAGE} as install-nginx LABEL maintainer="NGINX Docker Maintainers " ARG DEBIAN_FRONTEND=noninteractive diff --git a/test/mock/collector/README.md b/test/mock/collector/README.md index 92a53afc17..3527a3c5e2 100644 --- a/test/mock/collector/README.md +++ b/test/mock/collector/README.md @@ -1,13 +1,13 @@ # Mock Management OTel Collector There are 3 images that need to be built in order to use the mock management OTel collector -* Agent with NGINX Plus image +* Agent with NGINX Plus & NGINX App Protect image * Agent with NGINX OSS image * Custom OTel collector image To build these images run the following ``` -make local-deb-package build-test-plus-image build-test-oss-image build-mock-collector-image +make local-deb-package build-test-plus-image build-test-oss-image build-mock-management-otel-collector-image ``` To start run everything run the following @@ -15,16 +15,18 @@ To start run everything run the following make run-mock-management-otel-collector ``` -Once everything is started there should be 5 containers running -``` -8e6df6d0bc73 localhost/nginx_plus_agent_ubuntu_22.04:latest 4 minutes ago Up 4 minutes 80/tcp, 443/tcp mock-collector-agent-with-nginx-plus -a65a7efaf2b3 localhost/nginx_oss_agent_ubuntu_22.04:latest 4 minutes ago Up 4 minutes 80/tcp, 443/tcp mock-collector-agent-with-nginx-oss -bf0f247991c0 localhost/mock-collector:latest go run main.go 4 minutes ago Up 4 minutes 0.0.0.0:4320->4317/tcp, 0.0.0.0:9775->9090/tcp mock-collector-otel-collector -67bb7bde6392 docker.io/prom/prometheus:latest --config.file=/et... 4 minutes ago Up 4 minutes 0.0.0.0:9090->9090/tcp, 9090/tcp mock-collector-prometheus -a83a997eb652 docker.io/grafana/grafana:latest 4 minutes ago Up 4 minutes 0.0.0.0:3002->3000/tcp, 3000/tcp mock-collector-grafana +Once everything is started there should be 6 containers running +``` +98d601aeb602 localhost/nginx_plus_agent_ubuntu_22.04:latest 14 seconds ago Up 14 seconds 80/tcp, 443/tcp mock-collector-agent-with-nginx-plus +abaa9f70bfb6 localhost/nginx_oss_agent_ubuntu_22.04:latest 12 seconds ago Up 13 seconds 80/tcp, 443/tcp mock-collector-agent-with-nginx-oss +7158c3d0035b localhost/mock-collector:latest /mock-management-... 11 seconds ago Up 11 seconds 0.0.0.0:4320->4317/tcp, 0.0.0.0:9775->9090/tcp mock-collector-otel-collector +3f3a50c5fff4 docker.io/prom/prometheus:latest --config.file=/et... 9 seconds ago Up 9 seconds 0.0.0.0:9090->9090/tcp, 9090/tcp mock-collector-prometheus +b4541072ef8b docker.io/grafana/grafana:latest 7 seconds ago Up 8 seconds 0.0.0.0:3002->3000/tcp, 3000/tcp mock-collector-grafana +52b16ef06a10 docker.io/grafana/loki:latest -config.file=/etc... 6 seconds ago Up 6 seconds 0.0.0.0:3100->3100/tcp, 3100/tcp mock-collector-loki + ``` -To view the metrics, the grafana UI can be used by accessing this URL http://localhost:3002/login (Note: username/password is admin/admin) +To view the metrics & logs, the grafana UI can be used by accessing this URL http://localhost:3002/login (Note: username/password is admin/admin) To stop everything run the following ``` diff --git a/test/mock/collector/mock-collector/Dockerfile b/test/mock/collector/mock-collector/Dockerfile index 57637c1ee2..dc916663de 100644 --- a/test/mock/collector/mock-collector/Dockerfile +++ b/test/mock/collector/mock-collector/Dockerfile @@ -1,6 +1,6 @@ FROM golang:bookworm -WORKDIR /mock-management-plane-collector -COPY ./build/mock-management-plane-collector ./ +WORKDIR /mock-management-otel-collector +COPY ./build/mock-management-otel-collector ./ -CMD ["/mock-management-plane-collector/collector"] +CMD ["/mock-management-otel-collector/collector"] From 771a1cf664b3cb02698b82dc06f9d5d326a7d72a Mon Sep 17 00:00:00 2001 From: dhurley Date: Fri, 20 Dec 2024 10:14:07 +0000 Subject: [PATCH 19/24] Update how long to wait for nginx master process to start in dockerfile --- test/docker/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/docker/entrypoint.sh b/test/docker/entrypoint.sh index 5379ccb334..93920d97dd 100644 --- a/test/docker/entrypoint.sh +++ b/test/docker/entrypoint.sh @@ -23,7 +23,7 @@ nginx_pid=$! SECONDS=0 while ! ps -ef | grep "nginx: master process" | grep -v grep; do - if (( SECONDS > 15 )); then + if (( SECONDS > 30 )); then echo "couldn't find nginx master process" exit 1 fi From b8bd1b32a9ffa9902cc96a8e6596eb5ecbd5d1e6 Mon Sep 17 00:00:00 2001 From: dhurley Date: Fri, 20 Dec 2024 14:48:35 +0000 Subject: [PATCH 20/24] Fix OSARCH variable in Makefile --- Makefile | 11 ++++++----- test/mock/collector/README.md | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 8d5d7a6dcb..cf58c30753 100644 --- a/Makefile +++ b/Makefile @@ -69,12 +69,12 @@ OLD_BENCHMARK_RESULTS_FILE ?= $(TEST_BUILD_DIR)/benchmark.txt uname_m := $(shell uname -m) ifeq ($(uname_m),aarch64) - OSARCH = arm64 + OSARCH ?= arm64 else ifeq ($(uname_m),x86_64) - OSARCH = amd64 + OSARCH ?= amd64 else - OSARCH = $(uname_m) + OSARCH ?= $(uname_m) endif endif @@ -237,8 +237,9 @@ local-apk-package: ## Create local apk package ARCH=$(OSARCH) VERSION=$(shell echo $(VERSION) | tr -d 'v') $(GORUN) $(NFPM) pkg --config ./scripts/packages/.local-nfpm.yaml --packager apk --target $(APK_PACKAGE); local-deb-package: ## Create local deb package - @CGO_ENABLED=0 GOARCH=$(OSARCH) GOOS=linux $(GOBUILD) -o $(BUILD_DIR)/$(BINARY_NAME) -pgo=default.pgo -ldflags=$(LDFLAGS) $(PROJECT_DIR)/$(PROJECT_FILE) - ARCH=$(OSARCH) VERSION=$(shell echo $(VERSION) | tr -d 'v') $(GORUN) $(NFPM) pkg --config ./scripts/packages/.local-nfpm.yaml --packager deb --target $(DEB_PACKAGE); + echo $(OSARCH) +# @CGO_ENABLED=0 GOARCH=$(OSARCH) GOOS=linux $(GOBUILD) -o $(BUILD_DIR)/$(BINARY_NAME) -pgo=default.pgo -ldflags=$(LDFLAGS) $(PROJECT_DIR)/$(PROJECT_FILE) +# ARCH=$(OSARCH) VERSION=$(shell echo $(VERSION) | tr -d 'v') $(GORUN) $(NFPM) pkg --config ./scripts/packages/.local-nfpm.yaml --packager deb --target $(DEB_PACKAGE); local-rpm-package: ## Create local rpm package @CGO_ENABLED=0 GOARCH=$(OSARCH) GOOS=linux $(GOBUILD) -o $(BUILD_DIR)/$(BINARY_NAME) -pgo=default.pgo -ldflags=$(LDFLAGS) $(PROJECT_DIR)/$(PROJECT_FILE) diff --git a/test/mock/collector/README.md b/test/mock/collector/README.md index 3527a3c5e2..d110d6fef2 100644 --- a/test/mock/collector/README.md +++ b/test/mock/collector/README.md @@ -7,7 +7,7 @@ There are 3 images that need to be built in order to use the mock management OTe To build these images run the following ``` -make local-deb-package build-test-plus-image build-test-oss-image build-mock-management-otel-collector-image +OSARCH=amd64 make local-deb-package build-test-plus-image build-test-oss-image build-mock-management-otel-collector-image ``` To start run everything run the following From 1593c264d3ed6045661bd49e93c3400663b9f2a5 Mon Sep 17 00:00:00 2001 From: dhurley Date: Fri, 20 Dec 2024 14:50:55 +0000 Subject: [PATCH 21/24] Fix OSARCH variable in Makefile --- Makefile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index cf58c30753..28480bca80 100644 --- a/Makefile +++ b/Makefile @@ -237,9 +237,8 @@ local-apk-package: ## Create local apk package ARCH=$(OSARCH) VERSION=$(shell echo $(VERSION) | tr -d 'v') $(GORUN) $(NFPM) pkg --config ./scripts/packages/.local-nfpm.yaml --packager apk --target $(APK_PACKAGE); local-deb-package: ## Create local deb package - echo $(OSARCH) -# @CGO_ENABLED=0 GOARCH=$(OSARCH) GOOS=linux $(GOBUILD) -o $(BUILD_DIR)/$(BINARY_NAME) -pgo=default.pgo -ldflags=$(LDFLAGS) $(PROJECT_DIR)/$(PROJECT_FILE) -# ARCH=$(OSARCH) VERSION=$(shell echo $(VERSION) | tr -d 'v') $(GORUN) $(NFPM) pkg --config ./scripts/packages/.local-nfpm.yaml --packager deb --target $(DEB_PACKAGE); + @CGO_ENABLED=0 GOARCH=$(OSARCH) GOOS=linux $(GOBUILD) -o $(BUILD_DIR)/$(BINARY_NAME) -pgo=default.pgo -ldflags=$(LDFLAGS) $(PROJECT_DIR)/$(PROJECT_FILE) + ARCH=$(OSARCH) VERSION=$(shell echo $(VERSION) | tr -d 'v') $(GORUN) $(NFPM) pkg --config ./scripts/packages/.local-nfpm.yaml --packager deb --target $(DEB_PACKAGE); local-rpm-package: ## Create local rpm package @CGO_ENABLED=0 GOARCH=$(OSARCH) GOOS=linux $(GOBUILD) -o $(BUILD_DIR)/$(BINARY_NAME) -pgo=default.pgo -ldflags=$(LDFLAGS) $(PROJECT_DIR)/$(PROJECT_FILE) From 29f338e0d838345d83fca877d5669b76452727c8 Mon Sep 17 00:00:00 2001 From: dhurley Date: Tue, 7 Jan 2025 11:29:49 +0000 Subject: [PATCH 22/24] Update README --- api/grpc/mpi/v1/command.pb.go | 2 +- api/grpc/mpi/v1/common.pb.go | 2 +- api/grpc/mpi/v1/files.pb.go | 2 +- test/mock/collector/README.md | 5 ++++- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/api/grpc/mpi/v1/command.pb.go b/api/grpc/mpi/v1/command.pb.go index 7a29c30dd7..6e46a0c118 100644 --- a/api/grpc/mpi/v1/command.pb.go +++ b/api/grpc/mpi/v1/command.pb.go @@ -8,7 +8,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.0 +// protoc-gen-go v1.36.1 // protoc (unknown) // source: mpi/v1/command.proto diff --git a/api/grpc/mpi/v1/common.pb.go b/api/grpc/mpi/v1/common.pb.go index 493030c7db..8d51fa6b93 100644 --- a/api/grpc/mpi/v1/common.pb.go +++ b/api/grpc/mpi/v1/common.pb.go @@ -5,7 +5,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.0 +// protoc-gen-go v1.36.1 // protoc (unknown) // source: mpi/v1/common.proto diff --git a/api/grpc/mpi/v1/files.pb.go b/api/grpc/mpi/v1/files.pb.go index 38592679c8..896e045753 100644 --- a/api/grpc/mpi/v1/files.pb.go +++ b/api/grpc/mpi/v1/files.pb.go @@ -5,7 +5,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.0 +// protoc-gen-go v1.36.1 // protoc (unknown) // source: mpi/v1/files.proto diff --git a/test/mock/collector/README.md b/test/mock/collector/README.md index d110d6fef2..8374d22adb 100644 --- a/test/mock/collector/README.md +++ b/test/mock/collector/README.md @@ -7,9 +7,12 @@ There are 3 images that need to be built in order to use the mock management OTe To build these images run the following ``` -OSARCH=amd64 make local-deb-package build-test-plus-image build-test-oss-image build-mock-management-otel-collector-image +OSARCH=amd64 make local-deb-package build-test-plus-image +make local-deb-package build-test-oss-image build-mock-management-otel-collector-image ``` +[**Note:** We need to build the test NGINX Plus image with the environment variable `OSARCH=amd64` since NGINX App Protect doesn't support ARM yet.] + To start run everything run the following ``` make run-mock-management-otel-collector From 4413e5e3c4fd98debcdce9e2c88af3b1bee8613c Mon Sep 17 00:00:00 2001 From: dhurley Date: Thu, 9 Jan 2025 16:29:22 +0000 Subject: [PATCH 23/24] Fix makefile target --- Makefile | 2 +- api/grpc/mpi/v1/command.pb.go | 2 +- api/grpc/mpi/v1/common.pb.go | 2 +- api/grpc/mpi/v1/files.pb.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 28480bca80..fbe9fcdf66 100644 --- a/Makefile +++ b/Makefile @@ -212,7 +212,7 @@ build-test-oss-image: --build-arg ENTRY_POINT=./test/docker/entrypoint.sh .PHONY: build-mock-management-otel-collector-image -build-mock-collector-image: build-mock-management-otel-collector +build-mock-management-otel-collector-image: build-mock-management-otel-collector $(CONTAINER_BUILDENV) $(CONTAINER_CLITOOL) build -t mock-collector . \ --no-cache -f ./test/mock/collector/mock-collector/Dockerfile diff --git a/api/grpc/mpi/v1/command.pb.go b/api/grpc/mpi/v1/command.pb.go index 6e46a0c118..94368f7d33 100644 --- a/api/grpc/mpi/v1/command.pb.go +++ b/api/grpc/mpi/v1/command.pb.go @@ -8,7 +8,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.1 +// protoc-gen-go v1.36.2 // protoc (unknown) // source: mpi/v1/command.proto diff --git a/api/grpc/mpi/v1/common.pb.go b/api/grpc/mpi/v1/common.pb.go index 8d51fa6b93..763aae6c0c 100644 --- a/api/grpc/mpi/v1/common.pb.go +++ b/api/grpc/mpi/v1/common.pb.go @@ -5,7 +5,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.1 +// protoc-gen-go v1.36.2 // protoc (unknown) // source: mpi/v1/common.proto diff --git a/api/grpc/mpi/v1/files.pb.go b/api/grpc/mpi/v1/files.pb.go index 896e045753..cd41aa7389 100644 --- a/api/grpc/mpi/v1/files.pb.go +++ b/api/grpc/mpi/v1/files.pb.go @@ -5,7 +5,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.1 +// protoc-gen-go v1.36.2 // protoc (unknown) // source: mpi/v1/files.proto From c5dd6467aee2a448e23286d6d65586dce7b8e8c1 Mon Sep 17 00:00:00 2001 From: dhurley Date: Fri, 17 Jan 2025 14:05:00 +0000 Subject: [PATCH 24/24] Clean up --- Makefile | 16 +++- api/grpc/mpi/v1/command.pb.go | 2 +- api/grpc/mpi/v1/common.pb.go | 2 +- api/grpc/mpi/v1/files.pb.go | 2 +- test/docker/nginx-plus-and-nap/deb/Dockerfile | 84 +++++++++++++++++++ test/docker/nginx-plus/deb/Dockerfile | 16 ++-- test/mock/collector/README.md | 25 +++--- test/mock/collector/docker-compose.yaml | 10 +++ .../nginx-plus-and-nap/conf.d/default.conf | 59 +++++++++++++ .../collector/nginx-plus-and-nap/nginx.conf | 46 ++++++++++ test/mock/collector/nginx-plus/nginx.conf | 19 ----- 11 files changed, 235 insertions(+), 46 deletions(-) create mode 100644 test/docker/nginx-plus-and-nap/deb/Dockerfile create mode 100644 test/mock/collector/nginx-plus-and-nap/conf.d/default.conf create mode 100644 test/mock/collector/nginx-plus-and-nap/nginx.conf diff --git a/Makefile b/Makefile index fbe9fcdf66..ca5a562a73 100644 --- a/Makefile +++ b/Makefile @@ -190,6 +190,18 @@ run-mock-management-grpc-server: ## Run mock management plane gRPC server @echo "🖲️ Running mock management plane gRPC server" $(GORUN) test/mock/grpc/cmd/main.go -configDirectory=$(MOCK_MANAGEMENT_PLANE_CONFIG_DIRECTORY) -logLevel=$(MOCK_MANAGEMENT_PLANE_LOG_LEVEL) -grpcAddress=$(MOCK_MANAGEMENT_PLANE_GRPC_ADDRESS) -apiAddress=$(MOCK_MANAGEMENT_PLANE_API_ADDRESS) + +.PHONY: build-test-nginx-plus-and-nap-image +build-test-nginx-plus-and-nap-image: + $(CONTAINER_BUILDENV) $(CONTAINER_CLITOOL) build -t nginx_plus_and_nap_$(IMAGE_TAG) . \ + --no-cache -f ./test/docker/nginx-plus-and-nap/deb/Dockerfile \ + --secret id=nginx-crt,src=$(CERTS_DIR)/nginx-repo.crt \ + --secret id=nginx-key,src=$(CERTS_DIR)/nginx-repo.key \ + --build-arg PACKAGE_NAME=$(PACKAGE_NAME) \ + --build-arg PACKAGES_REPO=$(OSS_PACKAGES_REPO) \ + --build-arg BASE_IMAGE=$(BASE_IMAGE) \ + --build-arg ENTRY_POINT=./test/docker/entrypoint.sh + .PHONY: build-test-plus-image build-test-plus-image: $(CONTAINER_BUILDENV) $(CONTAINER_CLITOOL) build -t nginx_plus_$(IMAGE_TAG) . \ @@ -219,12 +231,12 @@ build-mock-management-otel-collector-image: build-mock-management-otel-collector .PHONY: run-mock-management-otel-collector run-mock-management-otel-collector: ## Run mock management plane OTel collector @echo "🚀 Running mock management plane OTel collector" - AGENT_IMAGE_WITH_NGINX_PLUS=nginx_plus_$(IMAGE_TAG):latest AGENT_IMAGE_WITH_NGINX_OSS=nginx_oss_$(IMAGE_TAG):latest $(CONTAINER_COMPOSE) -f ./test/mock/collector/docker-compose.yaml up -d + AGENT_IMAGE_WITH_NGINX_PLUS=nginx_plus_$(IMAGE_TAG):latest AGENT_IMAGE_WITH_NGINX_OSS=nginx_oss_$(IMAGE_TAG):latest AGENT_IMAGE_WITH_NGINX_PLUS_AND_NAP=nginx_plus_and_nap_$(IMAGE_TAG):latest $(CONTAINER_COMPOSE) -f ./test/mock/collector/docker-compose.yaml up -d .PHONY: stop-mock-management-otel-collector stop-mock-management-otel-collector: ## Stop running mock management plane OTel collector @echo "Stopping mock management plane OTel collector" - AGENT_IMAGE_WITH_NGINX_PLUS=nginx_plus_$(IMAGE_TAG):latest AGENT_IMAGE_WITH_NGINX_OSS=nginx_oss_$(IMAGE_TAG):latest $(CONTAINER_COMPOSE) -f ./test/mock/collector/docker-compose.yaml down + AGENT_IMAGE_WITH_NGINX_PLUS=nginx_plus_$(IMAGE_TAG):latest AGENT_IMAGE_WITH_NGINX_OSS=nginx_oss_$(IMAGE_TAG):latest AGENT_IMAGE_WITH_NGINX_PLUS_AND_NAP=nginx_plus_and_nap_$(IMAGE_TAG):latest $(CONTAINER_COMPOSE) -f ./test/mock/collector/docker-compose.yaml down generate: ## Generate golang code @echo "🗄️ Generating proto files" diff --git a/api/grpc/mpi/v1/command.pb.go b/api/grpc/mpi/v1/command.pb.go index 94368f7d33..b3cdd5cf1f 100644 --- a/api/grpc/mpi/v1/command.pb.go +++ b/api/grpc/mpi/v1/command.pb.go @@ -8,7 +8,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.2 +// protoc-gen-go v1.36.3 // protoc (unknown) // source: mpi/v1/command.proto diff --git a/api/grpc/mpi/v1/common.pb.go b/api/grpc/mpi/v1/common.pb.go index 763aae6c0c..d2cb5d35b5 100644 --- a/api/grpc/mpi/v1/common.pb.go +++ b/api/grpc/mpi/v1/common.pb.go @@ -5,7 +5,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.2 +// protoc-gen-go v1.36.3 // protoc (unknown) // source: mpi/v1/common.proto diff --git a/api/grpc/mpi/v1/files.pb.go b/api/grpc/mpi/v1/files.pb.go index cd41aa7389..2cbfb57ec1 100644 --- a/api/grpc/mpi/v1/files.pb.go +++ b/api/grpc/mpi/v1/files.pb.go @@ -5,7 +5,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.2 +// protoc-gen-go v1.36.3 // protoc (unknown) // source: mpi/v1/files.proto diff --git a/test/docker/nginx-plus-and-nap/deb/Dockerfile b/test/docker/nginx-plus-and-nap/deb/Dockerfile new file mode 100644 index 0000000000..163a6fbce7 --- /dev/null +++ b/test/docker/nginx-plus-and-nap/deb/Dockerfile @@ -0,0 +1,84 @@ +ARG BASE_IMAGE +FROM --platform=linux/amd64 ${BASE_IMAGE} as install-nginx +LABEL maintainer="NGINX Docker Maintainers " + +ARG DEBIAN_FRONTEND=noninteractive +ARG ENTRY_POINT +ARG PACKAGE_NAME +ARG PACKAGES_REPO + +WORKDIR /agent +COPY ./build /agent/build +COPY $ENTRY_POINT /agent/entrypoint.sh + +ENV PLUS_VERSION=R32 + +RUN --mount=type=secret,id=nginx-crt,dst=nginx-repo.crt \ + --mount=type=secret,id=nginx-key,dst=nginx-repo.key \ + set -x \ +# Create nginx user/group first, to be consistent throughout Docker variants + && groupadd --system --gid 101 nginx \ + && useradd --system --gid nginx --no-create-home --home-dir /nonexistent --uid 101 nginx \ + && apt-get update --allow-releaseinfo-change \ + && apt-get install --no-install-recommends --no-install-suggests -y \ + ca-certificates \ + gnupg1 \ + lsb-release \ + apt-transport-https \ + git \ + wget \ + make \ + gnupg2 \ + ubuntu-keyring \ + && wget -qO - https://cs.nginx.com/static/keys/app-protect-security-updates.key | gpg --dearmor | tee /usr/share/keyrings/app-protect-security-updates.gpg >/dev/null \ + && wget -qO - https://cs.nginx.com/static/keys/nginx_signing.key | gpg --dearmor | tee /usr/share/keyrings/nginx-archive-keyring.gpg >/dev/null \ + && \ + NGINX_GPGKEY=573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62; \ + found=''; \ + for server in \ + hkp://keyserver.ubuntu.com:80 \ + pgp.mit.edu \ + ; do \ + echo "Fetching GPG key $NGINX_GPGKEY from $server"; \ + apt-key adv --keyserver "$server" --keyserver-options timeout=10 --recv-keys "$NGINX_GPGKEY" && found=yes && break; \ + done; \ + test -z "$found" && echo >&2 "error: failed to fetch GPG key $NGINX_GPGKEY" && exit 1; \ + apt-get remove --purge --auto-remove -y gnupg1 && rm -rf /var/lib/apt/lists/* \ +# Install the latest release of NGINX Plus and/or NGINX Plus modules +# Uncomment individual modules if necessary +# Use versioned packages over defaults to specify a release + && nginxPackages=" \ + app-protect \ + " \ + && echo "Acquire::https::pkgs.nginx.com::Verify-Peer \"true\";" > /etc/apt/apt.conf.d/90nginx \ + && echo "Acquire::https::pkgs.nginx.com::Verify-Host \"true\";" >> /etc/apt/apt.conf.d/90nginx \ + && echo "Acquire::https::pkgs.nginx.com::SslCert \"/etc/ssl/nginx/nginx-repo.crt\";" >> /etc/apt/apt.conf.d/90nginx \ + && echo "Acquire::https::pkgs.nginx.com::SslKey \"/etc/ssl/nginx/nginx-repo.key\";" >> /etc/apt/apt.conf.d/90nginx \ + && printf "deb https://pkgs.nginx.com/plus/${PLUS_VERSION}/ubuntu/ `lsb_release -cs` nginx-plus\n" > /etc/apt/sources.list.d/nginx-plus.list \ + && printf "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] https://pkgs.nginx.com/app-protect/${PLUS_VERSION}/ubuntu `lsb_release -cs` nginx-plus\n" | tee /etc/apt/sources.list.d/nginx-app-protect.list \ + && printf "deb [signed-by=/usr/share/keyrings/app-protect-security-updates.gpg] https://pkgs.nginx.com/app-protect-security-updates/ubuntu `lsb_release -cs` nginx-plus\n" | tee -a /etc/apt/sources.list.d/nginx-app-protect.list \ + && mkdir -p /etc/ssl/nginx \ + && cat nginx-repo.crt > /etc/ssl/nginx/nginx-repo.crt \ + && cat nginx-repo.key > /etc/ssl/nginx/nginx-repo.key \ + && apt-get update \ + && apt-get install --no-install-recommends --no-install-suggests -y \ + $nginxPackages \ + curl \ + gettext-base \ + jq \ + && apt-get remove --purge -y lsb-release \ + && apt-get remove --purge --auto-remove -y && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/nginx-plus.list /etc/apt/sources.list.d/nginx-app-protect.list \ + && rm -rf /etc/apt/apt.conf.d/90nginx /etc/ssl/nginx + +EXPOSE 80 + +STOPSIGNAL SIGQUIT + +RUN chmod +x /agent/entrypoint.sh +RUN apt install -y /agent/build/${PACKAGE_NAME}.deb + +STOPSIGNAL SIGTERM + +EXPOSE 80 443 + +ENTRYPOINT ["/agent/entrypoint.sh"] diff --git a/test/docker/nginx-plus/deb/Dockerfile b/test/docker/nginx-plus/deb/Dockerfile index 163a6fbce7..392ed5fb70 100644 --- a/test/docker/nginx-plus/deb/Dockerfile +++ b/test/docker/nginx-plus/deb/Dockerfile @@ -1,5 +1,5 @@ ARG BASE_IMAGE -FROM --platform=linux/amd64 ${BASE_IMAGE} as install-nginx +FROM ${BASE_IMAGE} as install-nginx LABEL maintainer="NGINX Docker Maintainers " ARG DEBIAN_FRONTEND=noninteractive @@ -19,19 +19,14 @@ RUN --mount=type=secret,id=nginx-crt,dst=nginx-repo.crt \ # Create nginx user/group first, to be consistent throughout Docker variants && groupadd --system --gid 101 nginx \ && useradd --system --gid nginx --no-create-home --home-dir /nonexistent --uid 101 nginx \ - && apt-get update --allow-releaseinfo-change \ + && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates \ gnupg1 \ lsb-release \ - apt-transport-https \ git \ wget \ make \ - gnupg2 \ - ubuntu-keyring \ - && wget -qO - https://cs.nginx.com/static/keys/app-protect-security-updates.key | gpg --dearmor | tee /usr/share/keyrings/app-protect-security-updates.gpg >/dev/null \ - && wget -qO - https://cs.nginx.com/static/keys/nginx_signing.key | gpg --dearmor | tee /usr/share/keyrings/nginx-archive-keyring.gpg >/dev/null \ && \ NGINX_GPGKEY=573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62; \ found=''; \ @@ -48,15 +43,13 @@ RUN --mount=type=secret,id=nginx-crt,dst=nginx-repo.crt \ # Uncomment individual modules if necessary # Use versioned packages over defaults to specify a release && nginxPackages=" \ - app-protect \ + nginx-plus \ " \ && echo "Acquire::https::pkgs.nginx.com::Verify-Peer \"true\";" > /etc/apt/apt.conf.d/90nginx \ && echo "Acquire::https::pkgs.nginx.com::Verify-Host \"true\";" >> /etc/apt/apt.conf.d/90nginx \ && echo "Acquire::https::pkgs.nginx.com::SslCert \"/etc/ssl/nginx/nginx-repo.crt\";" >> /etc/apt/apt.conf.d/90nginx \ && echo "Acquire::https::pkgs.nginx.com::SslKey \"/etc/ssl/nginx/nginx-repo.key\";" >> /etc/apt/apt.conf.d/90nginx \ && printf "deb https://pkgs.nginx.com/plus/${PLUS_VERSION}/ubuntu/ `lsb_release -cs` nginx-plus\n" > /etc/apt/sources.list.d/nginx-plus.list \ - && printf "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] https://pkgs.nginx.com/app-protect/${PLUS_VERSION}/ubuntu `lsb_release -cs` nginx-plus\n" | tee /etc/apt/sources.list.d/nginx-app-protect.list \ - && printf "deb [signed-by=/usr/share/keyrings/app-protect-security-updates.gpg] https://pkgs.nginx.com/app-protect-security-updates/ubuntu `lsb_release -cs` nginx-plus\n" | tee -a /etc/apt/sources.list.d/nginx-app-protect.list \ && mkdir -p /etc/ssl/nginx \ && cat nginx-repo.crt > /etc/ssl/nginx/nginx-repo.crt \ && cat nginx-repo.key > /etc/ssl/nginx/nginx-repo.key \ @@ -66,8 +59,9 @@ RUN --mount=type=secret,id=nginx-crt,dst=nginx-repo.crt \ curl \ gettext-base \ jq \ + gnupg2 \ && apt-get remove --purge -y lsb-release \ - && apt-get remove --purge --auto-remove -y && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/nginx-plus.list /etc/apt/sources.list.d/nginx-app-protect.list \ + && apt-get remove --purge --auto-remove -y && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/nginx-plus.list \ && rm -rf /etc/apt/apt.conf.d/90nginx /etc/ssl/nginx EXPOSE 80 diff --git a/test/mock/collector/README.md b/test/mock/collector/README.md index 8374d22adb..b191e858e0 100644 --- a/test/mock/collector/README.md +++ b/test/mock/collector/README.md @@ -1,31 +1,34 @@ # Mock Management OTel Collector -There are 3 images that need to be built in order to use the mock management OTel collector +There are 4 images that need to be built in order to use the mock management OTel collector +* Agent with NGINX Plus image * Agent with NGINX Plus & NGINX App Protect image * Agent with NGINX OSS image * Custom OTel collector image To build these images run the following ``` -OSARCH=amd64 make local-deb-package build-test-plus-image -make local-deb-package build-test-oss-image build-mock-management-otel-collector-image +OSARCH=amd64 make local-deb-package build-test-nginx-plus-and-nap-image +make local-deb-package build-test-oss-image build-test-plus-image build-mock-management-otel-collector-image ``` -[**Note:** We need to build the test NGINX Plus image with the environment variable `OSARCH=amd64` since NGINX App Protect doesn't support ARM yet.] +[**Note:** We need to build the test NGINX Plus with NAP image with the environment variable `OSARCH=amd64` since NGINX App Protect doesn't support ARM yet.] To start run everything run the following ``` make run-mock-management-otel-collector ``` -Once everything is started there should be 6 containers running +Once everything is started there should be 7 containers running ``` -98d601aeb602 localhost/nginx_plus_agent_ubuntu_22.04:latest 14 seconds ago Up 14 seconds 80/tcp, 443/tcp mock-collector-agent-with-nginx-plus -abaa9f70bfb6 localhost/nginx_oss_agent_ubuntu_22.04:latest 12 seconds ago Up 13 seconds 80/tcp, 443/tcp mock-collector-agent-with-nginx-oss -7158c3d0035b localhost/mock-collector:latest /mock-management-... 11 seconds ago Up 11 seconds 0.0.0.0:4320->4317/tcp, 0.0.0.0:9775->9090/tcp mock-collector-otel-collector -3f3a50c5fff4 docker.io/prom/prometheus:latest --config.file=/et... 9 seconds ago Up 9 seconds 0.0.0.0:9090->9090/tcp, 9090/tcp mock-collector-prometheus -b4541072ef8b docker.io/grafana/grafana:latest 7 seconds ago Up 8 seconds 0.0.0.0:3002->3000/tcp, 3000/tcp mock-collector-grafana -52b16ef06a10 docker.io/grafana/loki:latest -config.file=/etc... 6 seconds ago Up 6 seconds 0.0.0.0:3100->3100/tcp, 3100/tcp mock-collector-loki +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +e0e57897223d localhost/nginx_plus_and_nap_agent_ubuntu_22.04:latest 17 seconds ago Up 17 seconds 80/tcp, 443/tcp mock-collector-agent-with-nginx-plus-and-nap +f4e7264ceb0b localhost/nginx_plus_agent_ubuntu_22.04:latest 15 seconds ago Up 16 seconds 80/tcp, 443/tcp mock-collector-agent-with-nginx-plus +97e9e84679a2 localhost/nginx_oss_agent_ubuntu_22.04:latest 14 seconds ago Up 14 seconds 80/tcp, 443/tcp mock-collector-agent-with-nginx-oss +04a3bce14c7a localhost/mock-collector:latest /mock-management-... 12 seconds ago Up 13 seconds 0.0.0.0:4320->4317/tcp, 0.0.0.0:9775->9090/tcp mock-collector-otel-collector +8efe150cbd96 docker.io/prom/prometheus:latest --config.file=/et... 11 seconds ago Up 11 seconds 0.0.0.0:9090->9090/tcp, 9090/tcp mock-collector-prometheus +e55572131d2a docker.io/grafana/grafana:latest 9 seconds ago Up 10 seconds 0.0.0.0:3002->3000/tcp, 3000/tcp mock-collector-grafana +69c8efab591b docker.io/grafana/loki:latest -config.file=/etc... 8 seconds ago Up 8 seconds 0.0.0.0:3100->3100/tcp, 3100/tcp mock-collector-loki ``` diff --git a/test/mock/collector/docker-compose.yaml b/test/mock/collector/docker-compose.yaml index 39324e1206..63e37e1a9e 100644 --- a/test/mock/collector/docker-compose.yaml +++ b/test/mock/collector/docker-compose.yaml @@ -7,6 +7,16 @@ volumes: grafana-storage: services: + agent-with-nginx-plus-and-nap: + image: ${AGENT_IMAGE_WITH_NGINX_PLUS_AND_NAP} + container_name: mock-collector-agent-with-nginx-plus-and-nap + volumes: + - ./nginx-agent.conf:/etc/nginx-agent/nginx-agent.conf + - ./nginx-plus-and-nap/nginx.conf:/etc/nginx/nginx.conf + - ./nginx-plus-and-nap/conf.d/default.conf:/etc/nginx/conf.d/default.conf + networks: + - metrics + agent-with-nginx-plus: image: ${AGENT_IMAGE_WITH_NGINX_PLUS} container_name: mock-collector-agent-with-nginx-plus diff --git a/test/mock/collector/nginx-plus-and-nap/conf.d/default.conf b/test/mock/collector/nginx-plus-and-nap/conf.d/default.conf new file mode 100644 index 0000000000..5534818abb --- /dev/null +++ b/test/mock/collector/nginx-plus-and-nap/conf.d/default.conf @@ -0,0 +1,59 @@ +server { + listen 80 default_server; + server_name localhost; + + #access_log /var/log/nginx/host.access.log main; + + location / { + root /usr/share/nginx/html; + index index.html index.htm; + } + + #error_page 404 /404.html; + + # redirect server error pages to the static page /50x.html + # + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } + + # proxy the PHP scripts to Apache listening on 127.0.0.1:80 + # + #location ~ \.php$ { + # proxy_pass http://127.0.0.1; + #} + + # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000 + # + #location ~ \.php$ { + # root html; + # fastcgi_pass 127.0.0.1:9000; + # fastcgi_index index.php; + # fastcgi_param SCRIPT_FILENAME /scripts$fastcgi_script_name; + # include fastcgi_params; + #} + + # deny access to .htaccess files, if Apache's document root + # concurs with nginx's one + # + #location ~ /\.ht { + # deny all; + #} + + # enable /api/ location with appropriate access control in order + # to make use of NGINX Plus API + # + #location /api/ { + # api write=on; + # allow 127.0.0.1; + # deny all; + #} + + # enable NGINX Plus Dashboard; requires /api/ location to be + # enabled and appropriate access control for remote access + # + #location = /dashboard.html { + # root /usr/share/nginx/html; + #} +} diff --git a/test/mock/collector/nginx-plus-and-nap/nginx.conf b/test/mock/collector/nginx-plus-and-nap/nginx.conf new file mode 100644 index 0000000000..15bc8304f5 --- /dev/null +++ b/test/mock/collector/nginx-plus-and-nap/nginx.conf @@ -0,0 +1,46 @@ +user nginx; +worker_processes auto; + +load_module modules/ngx_http_app_protect_module.so; + +error_log /var/log/nginx/error.log notice; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"' + '"$upstream_cache_status"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + server { + listen 8099; + server_name localhost; + proxy_http_version 1.1; + + app_protect_enable on; + app_protect_policy_file "/etc/app_protect/conf/NginxDefaultPolicy.json"; + app_protect_security_log_enable on; + app_protect_security_log "/etc/app_protect/conf/log_default.json" syslog:server=127.0.0.1:5141; + + location / { + client_max_body_size 0; + default_type text/html; + proxy_pass http://172.29.38.211/; + } + } + + include /etc/nginx/conf.d/*.conf; +} diff --git a/test/mock/collector/nginx-plus/nginx.conf b/test/mock/collector/nginx-plus/nginx.conf index 6c95de7b44..e83b9bec9d 100644 --- a/test/mock/collector/nginx-plus/nginx.conf +++ b/test/mock/collector/nginx-plus/nginx.conf @@ -1,8 +1,6 @@ user nginx; worker_processes auto; -load_module modules/ngx_http_app_protect_module.so; - error_log /var/log/nginx/error.log notice; pid /var/run/nginx.pid; @@ -82,23 +80,6 @@ http { status_zone my_location_zone2; } } - - server { - listen 8099; - server_name localhost; - proxy_http_version 1.1; - - app_protect_enable on; - app_protect_policy_file "/etc/app_protect/conf/NginxDefaultPolicy.json"; - app_protect_security_log_enable on; - app_protect_security_log "/etc/app_protect/conf/log_default.json" syslog:server=127.0.0.1:5141; - - location / { - client_max_body_size 0; - default_type text/html; - proxy_pass http://172.29.38.211/; - } - } include /etc/nginx/conf.d/*.conf; }