Move extension plugins to the extensions folder (#183)

* Move advanced metrics plugin to the extensions folder

---------

Co-authored-by: o.omahony <[email protected]>

Makefile

@@ -102,6 +102,8 @@ deps: ## Update dependencies in vendor folders
 	done
 	go mod download
 	go work sync
+
+no-local-changes:
 	git diff --quiet || { echo "Depenency changes detected. Please commit these before pushing." >&2; exit 1; }
 
 lint: ## Run linter

README.md

@@ -24,6 +24,7 @@ NGINX Agent is a companion daemon for your NGINX Open Source or NGINX Plus insta
   - [Installing Go](#installing-go)
   - [Starting the gRPC Mock Control Plane](#starting-the-grpc-mock-control-plane)
   - [NGINX Agent Settings](#nginx-agent-settings)
+  - [Extensions](#extensions)
   - [Starting NGINX Agent](#starting-nginx-agent)
 - [Development Environment Setup](#development-environment-setup)
   - [Selecting an Operating System](#selecting-an-operating-system)
@@ -195,6 +196,17 @@ make launch-swagger-ui
 
 Open a web browser to view the Swagger UI at http://localhost:8082/docs.
 
+## Extensions
+An extension is a piece of code, not critical to the main functionality that the NGINX agent is responsible for. This generally falls outside the remit of managing NGINX Configuration and reporting NGINX metrics.
+
+To enable an extension, it must be added to the extensions list in the `/etc/nginx-agent/nginx-agent.conf`. 
+Here is an example of enabling the advanced metrics extension:
+
+```yaml
+extensions:
+  - advanced-metrics
+```
+
 ## Starting NGINX Agent
 If already running, restart NGINX Agent to apply the new configuration. Alternatively, if NGINX Agent is not running, you may run it from the source code root directory.
 

docs/proto/proto.md

@@ -1221,6 +1221,8 @@ Represents App Protect WAF details
 | attack_signatures_version | [string](#string) |  | Attack signatures version (This is being deprecated and will be removed in a future release) |
 | threat_campaigns_version | [string](#string) |  | Threat signatures version (This is being deprecated and will be removed in a future release) |
 | health | [AppProtectWAFHealth](#f5-nginx-agent-sdk-AppProtectWAFHealth) |  | App Protect Health details (This is being deprecated and will be removed in a future release) |
+| waf_location | [string](#string) |  | Location of WAF metadata file |
+| precompiled_publication | [bool](#bool) |  | Determines whether the publication of NGINX App Protect pre-compiled content from an external source is supported |
 
 
 

hugo/content/getting-started.md

@@ -37,16 +37,13 @@ INFO[0000] grpc listening at 54789 # grpc control plane port which NGINX Agent w
 ```
 
 ## NGINX Agent Settings
-
-If it doesn't already exist, create the `/etc/nginx-agent/` directory and copy the `nginx-agent.conf` file into it from the project root directory.
-
-```bash
+If it doesn't already exist, create the `/etc/nginx-agent/` directory and copy the `nginx-agent.conf` file into it from the project root directory. 
+```
 sudo mkdir /etc/nginx-agent
-sudo cp nginx-agent.conf /etc/nginx-agent/
+sudo cp <project_root_directory>/nginx-agent.conf /etc/nginx-agent/
+```
+Create the `agent-dynamic.conf` file in the `/etc/nginx-agent/` directory, which is required for NGINX Agent to run. 
 ```
-Create the `agent-dynamic.conf` file in the `/etc/nginx-agent/` directory, which is required for NGINX Agent to run.
-
-```bash
 sudo touch /etc/nginx-agent/agent-dynamic.conf
 ```
 
@@ -68,12 +65,11 @@ tls:
 For more information, see [Agent Protocol Definitions and Documentation](https://github.com/nginx/agent/tree/main/docs/proto/README.md)
 
 ### Enable the REST interface
-
 The NGINX Agent REST interface can be exposed by validating the following lines in the `/etc/nginx-agent/nginx-agent.conf` file are present:
 
-```nginx
+```yaml
 api:
-  port: 9090 # port to expose REST API
+  port: 8081 # port to expose REST API
   
   # REST TLS parameters
   cert: "<TLS-CERTIFICATE>.crt"
@@ -91,6 +87,17 @@ To launch the Swagger UI for the REST interface run the following command
 make launch-swagger-ui
 ```
 
+## Extensions
+An extension is a piece of code, not critical to the main functionality that the NGINX agent is responsible for. This generally falls outside the remit of managing NGINX Configuration and reporting NGINX metrics.
+
+To enable an extension, it must be added to the extensions list in the `/etc/nginx-agent/nginx-agent.conf`. 
+Here is an example of enabling the advanced metrics extension:
+
+```yaml
+extensions:
+  - advanced-metrics
+```
+
 ## Start NGINX Agent
 
 If already running, restart NGINX Agent to apply the new configuration. Alternatively, if NGINX Agent is not running, you may run it from the source code root directory.

lefthook.yml

@@ -9,3 +9,5 @@ pre-push:
       run: make format
     3_dependencies:
       run: make deps
+    4_check_local_changes:
+      run: make no-local-changes

main.go

@@ -25,7 +25,10 @@ import (
 	"github.com/nginx/agent/v2/src/core"
 	"github.com/nginx/agent/v2/src/core/config"
 	"github.com/nginx/agent/v2/src/core/logger"
+	"github.com/nginx/agent/v2/src/extensions"
 	"github.com/nginx/agent/v2/src/plugins"
+
+	agent_config "github.com/nginx/agent/sdk/v2/agent/config"
 )
 
 var (
@@ -34,10 +37,6 @@ var (
 	version = ""
 )
 
-const (
-	DEFAULT_PLUGIN_SIZE = 100
-)
-
 func init() {
 	config.SetVersion(version, commit)
 	config.SetDefaults()
@@ -87,9 +86,9 @@ func main() {
 
 		binary := core.NewNginxBinary(env, loadedConfig)
 
-		corePlugins := loadPlugins(commander, binary, env, reporter, loadedConfig)
+		corePlugins, extensionPlugins := loadPlugins(commander, binary, env, reporter, loadedConfig)
 
-		pipe := initializeMessagePipe(ctx, corePlugins)
+		pipe := initializeMessagePipe(ctx, corePlugins, extensionPlugins)
 
 		pipe.Process(core.NewMessage(core.AgentStarted,
 			plugins.NewAgentEventMeta(version, strconv.Itoa(os.Getpid()))),
@@ -187,8 +186,9 @@ func createGrpcClients(ctx context.Context, loadedConfig *config.Config) (client
 	return controller, commander, reporter
 }
 
-func loadPlugins(commander client.Commander, binary *core.NginxBinaryType, env *core.EnvironmentType, reporter client.MetricReporter, loadedConfig *config.Config) []core.Plugin {
+func loadPlugins(commander client.Commander, binary *core.NginxBinaryType, env *core.EnvironmentType, reporter client.MetricReporter, loadedConfig *config.Config) ([]core.Plugin, []core.ExtensionPlugin) {
 	var corePlugins []core.Plugin
+	var extensionPlugins []core.ExtensionPlugin
 
 	if commander != nil {
 		corePlugins = append(corePlugins,
@@ -226,34 +226,38 @@ func loadPlugins(commander client.Commander, binary *core.NginxBinaryType, env *
 		corePlugins = append(corePlugins, plugins.NewNginxCounter(loadedConfig, binary, env))
 	}
 
-	if (config.AdvancedMetrics{}) != loadedConfig.AdvancedMetrics {
-		corePlugins = append(corePlugins, plugins.NewAdvancedMetrics(env, loadedConfig))
-	}
-
-	if loadedConfig.IsNginxAppProtectConfigured() {
-		napPlugin, err := plugins.NewNginxAppProtect(loadedConfig, env)
-		if err == nil {
-			corePlugins = append(corePlugins, napPlugin)
-		} else {
-			log.Errorf("Unable to load the Nginx App Protect plugin due to the following error: %v", err)
-		}
-	}
-
-	if loadedConfig.NAPMonitoring != (config.NAPMonitoring{}) {
-		nm, err := plugins.NewNAPMonitoring(env, loadedConfig)
-		if err != nil {
-			log.Errorf("Unable to load the Nginx App Protect Monitoring plugin due to the following error: %v", err)
-		} else {
-			corePlugins = append(corePlugins, nm)
+	if loadedConfig.Extensions != nil && len(loadedConfig.Extensions) > 0 {
+		for _, extension := range loadedConfig.Extensions {
+			switch {
+			case extension == agent_config.AdvancedMetricsExtensionPlugin:
+				advancedMetricsExtensionPlugin := extensions.NewAdvancedMetrics(env, loadedConfig, config.Viper.Get(agent_config.AdvancedMetricsExtensionPluginConfigKey))
+				extensionPlugins = append(extensionPlugins, advancedMetricsExtensionPlugin)
+			case extension == agent_config.NginxAppProtectExtensionPlugin:
+				nginxAppProtectExtensionPlugin, err := extensions.NewNginxAppProtect(loadedConfig, env, config.Viper.Get(agent_config.NginxAppProtectExtensionPluginConfigKey))
+				if err != nil {
+					log.Errorf("Unable to load the Nginx App Protect plugin due to the following error: %v", err)
+				} else {
+					extensionPlugins = append(extensionPlugins, nginxAppProtectExtensionPlugin)
+				}
+			case extension == agent_config.NginxAppProtectMonitoringExtensionPlugin:
+				nginxAppProtectMonitoringExtensionPlugin, err := extensions.NewNAPMonitoring(env, loadedConfig, config.Viper.Get(agent_config.NginxAppProtectMonitoringExtensionPluginConfigKey))
+				if err != nil {
+					log.Errorf("Unable to load the Nginx App Protect Monitoring plugin due to the following error: %v", err)
+				} else {
+					extensionPlugins = append(extensionPlugins, nginxAppProtectMonitoringExtensionPlugin)
+				}
+			default:
+				log.Warnf("unknown extension configured: %s", extension)
+			}
 		}
 	}
 
-	return corePlugins
+	return corePlugins, extensionPlugins
 }
 
-func initializeMessagePipe(ctx context.Context, corePlugins []core.Plugin) core.MessagePipeInterface {
+func initializeMessagePipe(ctx context.Context, corePlugins []core.Plugin, extensionPlugins []core.ExtensionPlugin) core.MessagePipeInterface {
 	pipe := core.NewMessagePipe(ctx)
-	err := pipe.Register(DEFAULT_PLUGIN_SIZE, corePlugins...)
+	err := pipe.Register(agent_config.DefaultPluginSize, corePlugins, extensionPlugins)
 	if err != nil {
 		log.Warnf("Failed to start agent successfully, error loading plugins %v", err)
 	}

scripts/packages/postinstall.sh

@@ -249,6 +249,28 @@ EOF
     fi
 }
 
+upgrade_config_file() {
+    if [ -f "${BSD_HIER}"/etc/nginx-agent/nginx-agent.conf ]; then
+        extensions=""
+        if grep -q "advanced_metrics:" "${BSD_HIER}"/etc/nginx-agent/nginx-agent.conf; then
+            extensions="${extensions} advanced-metrics"
+        fi
+        if grep -q "nginx_app_protect:" "${BSD_HIER}"/etc/nginx-agent/nginx-agent.conf; then
+            extensions="${extensions} nginx-app-protect"
+        fi
+        if grep -q "nap_monitoring:" "${BSD_HIER}"/etc/nginx-agent/nginx-agent.conf; then
+            extensions="${extensions} nap-monitoring"
+        fi
+        if ! grep -q "extensions:" "${BSD_HIER}"/etc/nginx-agent/nginx-agent.conf && [ "${#extensions}" -ne "0" ]; then
+            printf "PostInstall: Updating nginx-agent.conf to include extensions array\n"
+            printf "\nextensions:\n" >> "${BSD_HIER}"/etc/nginx-agent/nginx-agent.conf
+            for extension in ${extensions}; do
+                echo "  - $extension" >> "${BSD_HIER}"/etc/nginx-agent/nginx-agent.conf
+            done
+        fi
+    fi
+}
+
 summary() {
     echo "----------------------------------------------------------------------"
     echo " NGINX Agent package has been successfully installed."
@@ -278,6 +300,7 @@ summary() {
     create_run_dir
     update_unit_file
     add_default_config_file
+    upgrade_config_file
     summary
 }
 

sdk/agent/config/config_helpers.go

@@ -7,10 +7,15 @@
 
 package config
 
+import (
+	"github.com/mitchellh/mapstructure"
+)
+
 const (
-	KeyDelimiter = "_"
+	DefaultPluginSize = 100
+	KeyDelimiter      = "_"
 
-	// viper keys used in config
+	// Features
 	FeaturesKey             = "features"
 	FeatureRegistration     = FeaturesKey + KeyDelimiter + "registration"
 	FeatureNginxConfig      = FeaturesKey + KeyDelimiter + "nginx-config"
@@ -24,8 +29,37 @@ const (
 	FeatureFileWatcher      = FeaturesKey + KeyDelimiter + "file-watcher"
 	FeatureActivityEvents   = FeaturesKey + KeyDelimiter + "activity-events"
 	FeatureAgentAPI         = FeaturesKey + KeyDelimiter + "agent-api"
+
+	// Extensions
+	ExtensionsKey                            = "extensions"
+	AdvancedMetricsExtensionPlugin           = "advanced-metrics"
+	NginxAppProtectExtensionPlugin           = "nginx-app-protect"
+	NginxAppProtectMonitoringExtensionPlugin = "nap-monitoring"
+
+	// Configuration Keys
+	AdvancedMetricsExtensionPluginConfigKey           = "advanced_metrics"
+	NginxAppProtectExtensionPluginConfigKey           = "nginx_app_protect"
+	NginxAppProtectMonitoringExtensionPluginConfigKey = "nap_monitoring"
 )
 
+func GetKnownExtensions() []string {
+	return []string{
+		AdvancedMetricsExtensionPlugin,
+		NginxAppProtectExtensionPlugin,
+		NginxAppProtectMonitoringExtensionPlugin,
+	}
+}
+
+func IsKnownExtension(extension string) bool {
+	for _, knownExtension := range GetKnownExtensions() {
+		if knownExtension == extension {
+			return true
+		}
+	}
+
+	return false
+}
+
 func GetDefaultFeatures() []string {
 	return []string{
 		FeatureRegistration,
@@ -42,3 +76,23 @@ func GetDefaultFeatures() []string {
 		FeatureAgentAPI,
 	}
 }
+
+func DecodeConfig[T interface{}](input interface{}) (output T, err error) {
+	decoder, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
+		WeaklyTypedInput: true,
+		DecodeHook:       mapstructure.ComposeDecodeHookFunc(mapstructure.StringToTimeDurationHookFunc()),
+		Result:           &output,
+	})
+
+	if err != nil {
+		return output, err
+	}
+
+	err = decoder.Decode(input)
+
+	if err != nil {
+		return output, err
+	}
+
+	return output, nil
+}

sdk/agent/config/config_helpers_test.go

@@ -0,0 +1,33 @@
+/**
+ * Copyright (c) F5, Inc.
+ *
+ * This source code is licensed under the Apache License, Version 2.0 license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+package config
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type TestConfig struct {
+	Name string `mapstructure:"name"`
+}
+
+func Test_DecodeConfig(t *testing.T) {
+	// Valid input
+	input := map[string]string{"name": "test-name"}
+	output, err := DecodeConfig[*TestConfig](input)
+	require.NoError(t, err)
+	assert.NotNil(t, output)
+	assert.Equal(t, "test-name", output.Name)
+
+	// Invalid input
+	output, err = DecodeConfig[*TestConfig]("invalid")
+	require.Error(t, err)
+	assert.Nil(t, output)
+}

sdk/go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/cenkalti/backoff/v4 v4.2.0
 	github.com/gogo/protobuf v1.3.2
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
+	github.com/mitchellh/mapstructure v1.5.0
 	github.com/nginxinc/nginx-go-crossplane v0.4.1
 	github.com/sirupsen/logrus v1.9.0
 	github.com/stretchr/testify v1.8.2

sdk/go.sum

@@ -62,6 +62,8 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/maxbrunsfeld/counterfeiter/v6 v6.5.0/go.mod h1:fJ0UAZc1fx3xZhU4eSHQDJ1ApFmTVhp5VTpV9tm2ogg=
+github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
+github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/nginxinc/nginx-go-crossplane v0.4.1 h1:swWcI437atMpMT/l6GuEu0oRhkMBOhh0DGHCOd2QgOc=
 github.com/nginxinc/nginx-go-crossplane v0.4.1/go.mod h1:NH9Gmsd1gxoLFJHZPxL9I4Z3qeA2n1BXCKiN2TtW2bc=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=