Add RequiredDaemons list to DirectiveBreakdown (#202)

Allow a user to specify a list of required daemons to their "#DW jobdw ..." directive. The new keyword is "requires=" and it takes a list of strings: requires=copy-offload,future-thing The dwdparse library is updated to support parsing a comma-separated list of strings, with a list of RE patterns for each element allowed in the list. The DirectiveBreakdownStatus has a new field RequiredDaemons to hold the list of daemons. Signed-off-by: Dean Roehrich <[email protected]>
DataWorkflowServices · Jun 26, 2024 · 0eac099 · 0eac099
1 parent 7c9f194
commit 0eac099
Show file tree

Hide file tree

Showing 9 changed files with 137 additions and 70 deletions.
diff --git a/api/v1alpha1/conversion.go b/api/v1alpha1/conversion.go
@@ -190,6 +190,10 @@ func (src *DirectiveBreakdown) ConvertTo(dstRaw conversion.Hub) error {
 		dst.Status.Error.Severity = dwsv1alpha2.SeverityFatal
 	}
 
+	if hasAnno {
+		dst.Status.RequiredDaemons = restored.Status.RequiredDaemons
+	}
+
 	return nil
 }
 
@@ -543,6 +547,10 @@ func Convert_v1alpha2_ClientMountStatus_To_v1alpha1_ClientMountStatus(in *dwsv1a
 	return autoConvert_v1alpha2_ClientMountStatus_To_v1alpha1_ClientMountStatus(in, out, s)
 }
 
+func Convert_v1alpha2_DirectiveBreakdownStatus_To_v1alpha1_DirectiveBreakdownStatus(in *dwsv1alpha2.DirectiveBreakdownStatus, out *DirectiveBreakdownStatus, s apiconversion.Scope) error {
+	return autoConvert_v1alpha2_DirectiveBreakdownStatus_To_v1alpha1_DirectiveBreakdownStatus(in, out, s)
+}
+
 func Convert_v1alpha1_ResourceErrorInfo_To_v1alpha2_ResourceErrorInfo(in *ResourceErrorInfo, out *dwsv1alpha2.ResourceErrorInfo, s apiconversion.Scope) error {
 	return autoConvert_v1alpha1_ResourceErrorInfo_To_v1alpha2_ResourceErrorInfo(in, out, s)
 }

diff --git a/api/v1alpha1/zz_generated.conversion.go b/api/v1alpha1/zz_generated.conversion.go
diff --git a/api/v1alpha2/directivebreakdown_types.go b/api/v1alpha2/directivebreakdown_types.go
@@ -1,5 +1,5 @@
 /*
- * Copyright 2021-2023 Hewlett Packard Enterprise Development LP
+ * Copyright 2021-2024 Hewlett Packard Enterprise Development LP
  * Other additional copyright holders may be indicated within.
  *
  * The entirety of this work is licensed under the Apache License,
@@ -182,6 +182,9 @@ type DirectiveBreakdownStatus struct {
 	// Ready indicates whether AllocationSets have been generated (true) or not (false)
 	Ready bool `json:"ready"`
 
+	// RequiredDeamons tells the WLM about any driver-specific daemons it must enable for the job; it is assumed that the WLM knows about the driver-specific daemons and that if the users are specifying these then the WLM knows how to start them
+	RequiredDaemons []string `json:"requiredDaemons,omitempty"`
+
 	// Error information
 	ResourceError `json:",inline"`
 }

diff --git a/api/v1alpha2/zz_generated.deepcopy.go b/api/v1alpha2/zz_generated.deepcopy.go
diff --git a/config/crd/bases/dataworkflowservices.github.io_directivebreakdowns.yaml b/config/crd/bases/dataworkflowservices.github.io_directivebreakdowns.yaml
@@ -501,6 +501,14 @@ spec:
                 description: Ready indicates whether AllocationSets have been generated
                   (true) or not (false)
                 type: boolean
+              requiredDaemons:
+                description: RequiredDeamons tells the WLM about any driver-specific
+                  daemons it must enable for the job; it is assumed that the WLM knows
+                  about the driver-specific daemons and that if the users are specifying
+                  these then the WLM knows how to start them
+                items:
+                  type: string
+                type: array
               storage:
                 description: Storage is the storage breakdown for the directive
                 properties:

diff --git a/config/crd/bases/dataworkflowservices.github.io_dwdirectiverules.yaml b/config/crd/bases/dataworkflowservices.github.io_dwdirectiverules.yaml
@@ -67,6 +67,10 @@ spec:
                         type: integer
                       pattern:
                         type: string
+                      patterns:
+                        items:
+                          type: string
+                        type: array
                       type:
                         type: string
                       uniqueWithin:
@@ -143,6 +147,10 @@ spec:
                         type: integer
                       pattern:
                         type: string
+                      patterns:
+                        items:
+                          type: string
+                        type: array
                       type:
                         type: string
                       uniqueWithin:

diff --git a/utils/dwdparse/dwdparse.go b/utils/dwdparse/dwdparse.go
@@ -1,5 +1,5 @@
 /*
- * Copyright 2021, 2022 Hewlett Packard Enterprise Development LP
+ * Copyright 2021-2024 Hewlett Packard Enterprise Development LP
  * Other additional copyright holders may be indicated within.
  *
  * The entirety of this work is licensed under the Apache License,
@@ -29,14 +29,15 @@ import (
 // DWDirectiveRuleDef defines the DWDirective parser rules
 // +kubebuilder:object:generate=true
 type DWDirectiveRuleDef struct {
-	Key             string `json:"key"`
-	Type            string `json:"type"`
-	Pattern         string `json:"pattern,omitempty"`
-	Min             int    `json:"min,omitempty"`
-	Max             int    `json:"max,omitempty"`
-	IsRequired      bool   `json:"isRequired,omitempty"`
-	IsValueRequired bool   `json:"isValueRequired,omitempty"`
-	UniqueWithin    string `json:"uniqueWithin,omitempty"`
+	Key             string   `json:"key"`
+	Type            string   `json:"type"`
+	Pattern         string   `json:"pattern,omitempty"`
+	Patterns        []string `json:"patterns,omitempty"`
+	Min             int      `json:"min,omitempty"`
+	Max             int      `json:"max,omitempty"`
+	IsRequired      bool     `json:"isRequired,omitempty"`
+	IsValueRequired bool     `json:"isValueRequired,omitempty"`
+	UniqueWithin    string   `json:"uniqueWithin,omitempty"`
 }
 
 // DWDirectiveRuleSpec defines the desired state of DWDirective
@@ -170,6 +171,35 @@ func ValidateArgs(spec DWDirectiveRuleSpec, args map[string]string, uniqueMap ma
 					return fmt.Errorf("argument '%s' invalid string '%s'", k, v)
 				}
 			}
+		case "list-of-string":
+			words := strings.Split(v, ",")
+			if len(rule.Patterns) > 0 {
+				wordsMatched := make(map[string]bool)
+				for _, word := range words {
+					for idx := range rule.Patterns {
+						re, err := regexp.Compile(rule.Patterns[idx])
+						if err != nil {
+							return fmt.Errorf("invalid regular expression '%s'", rule.Patterns[idx])
+						}
+
+						if re.MatchString(word) {
+							wordsMatched[word] = true
+							break
+						}
+					}
+					if !wordsMatched[word] {
+						return fmt.Errorf("argument '%s' invalid string '%s' in list '%s'", k, word, v)
+					}
+				}
+			}
+			// All of the words in the list are valid, but were any words repeated?
+			wordMap := make(map[string]bool)
+			for _, word := range words {
+				if _, present := wordMap[word]; present {
+					return fmt.Errorf("argument '%s' word '%s' repeated in list '%s'", k, word, v)
+				}
+				wordMap[word] = true
+			}
 		default:
 			return fmt.Errorf("unsupported rule type '%s'", rule.Type)
 		}

diff --git a/utils/dwdparse/dwdparse_test.go b/utils/dwdparse/dwdparse_test.go
@@ -1,5 +1,5 @@
 /*
- * Copyright 2021, 2022 Hewlett Packard Enterprise Development LP
+ * Copyright 2021-2024 Hewlett Packard Enterprise Development LP
  * Other additional copyright holders may be indicated within.
  *
  * The entirety of this work is licensed under the Apache License,
@@ -20,8 +20,9 @@
 package dwdparse
 
 import (
-	. "github.com/onsi/ginkgo/v2"
 	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
 )
 
 var dWDRules = []DWDirectiveRuleSpec{
@@ -57,19 +58,6 @@ var dWDRules = []DWDirectiveRuleSpec{
 				IsRequired:      false,
 				IsValueRequired: true,
 			},
-			{
-				Key:             "combined_mgtmdt",
-				Type:            "bool",
-				IsRequired:      false,
-				IsValueRequired: false,
-			},
-			{
-				Key:             "external_mgs",
-				Type:            "string",
-				Pattern:         "^[A-Za-z0-9\\-_\\.@,:]+$",
-				IsRequired:      false,
-				IsValueRequired: true,
-			},
 		},
 	},
 	{
@@ -104,19 +92,6 @@ var dWDRules = []DWDirectiveRuleSpec{
 				IsRequired:      false,
 				IsValueRequired: true,
 			},
-			{
-				Key:             "combined_mgtmdt",
-				Type:            "bool",
-				IsRequired:      false,
-				IsValueRequired: false,
-			},
-			{
-				Key:             "external_mgs",
-				Type:            "string",
-				Pattern:         "^[A-Za-z0-9\\-_\\.@,:]+$",
-				IsRequired:      false,
-				IsValueRequired: true,
-			},
 		},
 	},
 	{
@@ -261,15 +236,6 @@ var dwDirectiveTests = []struct {
 	{[]string{"#DW jobdw type=lustre capacity=100GB name=UncoolProfile3 profile=_this"}, fail},
 	{[]string{"#DW jobdw type=lustre capacity=100GB name=UncoolProfile4 profile=-this"}, fail},
 
-	{[]string{"#DW jobdw type=lustre capacity=100GB combined_mgtmdt name=prettierGoodName"}, pass},
-	{[]string{"#DW jobdw type=lustre capacity=100GB combined_mgtmdt=true name=prettierGoodName"}, pass},
-
-	{[]string{"#DW jobdw type=lustre external_mgs=rabbit-01@tcp capacity=100GB name=Extern1"}, pass},
-	{[]string{"#DW jobdw type=lustre external_mgs=rabbit-01@tcp,rabbit-02@tcp:rabbit-03@tcp capacity=100GB name=Extern1"}, pass},
-	{[]string{"#DW jobdw type=lustre external_mgs=rabbit-01@tcp combined_mgtmdt capacity=100GB name=Extern1"}, pass},
-	{[]string{"#DW jobdw type=lustre external_mgs=rabbit-01@tcp0 capacity=100GB name=Extern1"}, pass},
-	{[]string{"#DW jobdw type=lustre external_mgs=10.0.0.1@o2ib capacity=100GB name=Extern2"}, pass},
-
 	{[]string{"#DW create_persistent type=raw    capacity=100GB name=prettyGoodName  "}, pass},
 	{[]string{"#DW create_persistent type=xfs    capacity=100GB name=prettyGoodName  "}, pass},
 	{[]string{"#DW create_persistent type=gfs2   capacity=100GB name=prettyGoodName  "}, pass},
@@ -288,14 +254,6 @@ var dwDirectiveTests = []struct {
 	{[]string{"#DW create_persistent type=lustre capacity=100GB name=UncoolProfile3 profile=_this"}, fail},
 	{[]string{"#DW create_persistent type=lustre capacity=100GB name=UncoolProfile4 profile=-this"}, fail},
 
-	{[]string{"#DW create_persistent type=lustre capacity=100GB combined_mgtmdt name=prettierGoodName"}, pass},
-	{[]string{"#DW create_persistent type=lustre capacity=100GB combined_mgtmdt=true name=prettierGoodName"}, pass},
-
-	{[]string{"#DW create_persistent type=lustre external_mgs=rabbit-01@tcp capacity=100GB name=Extern1"}, pass},
-	{[]string{"#DW create_persistent type=lustre external_mgs=rabbit-01@tcp combined_mgtmdt capacity=100GB name=Extern1"}, pass},
-	{[]string{"#DW create_persistent type=lustre external_mgs=rabbit-01@tcp0 capacity=100GB name=Extern1"}, pass},
-	{[]string{"#DW create_persistent type=lustre external_mgs=10.0.0.1@o2ib capacity=100GB name=Extern2"}, pass},
-
 	{[]string{"#DW stage_in  type=file      destination=$DW_JOB_STRIPED source=/pfs/dld-input "}, pass},
 	{[]string{"#DW stage_in  type=directory destination=$DW_JOB_STRIPED source=/pfs/dld-input "}, pass},
 	{[]string{"#DW stage_in  type=list      destination=$DW_JOB_STRIPED source=/pfs/dld-input "}, pass},
@@ -362,7 +320,7 @@ var dwDirectiveTests = []struct {
 	//       contains individualized test cases.
 }
 
-func _TestDWParse(t *testing.T) {
+func TestDWParse(t *testing.T) {
 	for index, tt := range dwDirectiveTests {
 
 		err := Validate(dWDRules, tt.directiveList, func(int, DWDirectiveRuleSpec) {})
@@ -375,7 +333,6 @@ func _TestDWParse(t *testing.T) {
 
 // testCase defines an individual test case
 type testCase struct {
-	rules      []DWDirectiveRuleSpec
 	directives []string
 	result     bool
 }
@@ -391,7 +348,52 @@ func test(t *testing.T, rules []DWDirectiveRuleSpec, tests []testCase) {
 	}
 }
 
-func _TestUniqueWithin(t *testing.T) {
+func TestRequiresKeyword(t *testing.T) {
+
+	rules := []DWDirectiveRuleSpec{
+		{
+			Command: "check_requires",
+			RuleDefs: []DWDirectiveRuleDef{
+				{
+					Key:  "requires",
+					Type: "list-of-string",
+					Patterns: []string{
+						"^copy-offload$",
+						"^other-thing$",
+					},
+					IsRequired:      false,
+					IsValueRequired: true,
+				},
+			},
+		},
+	}
+
+	tests := []testCase{
+		{
+			directives: []string{
+				"#DW check_requires requires=copy-offload,other-thing",
+			},
+			result: pass,
+		},
+		{
+			directives: []string{
+				"#DW check_requires requires=none-such,copy-offload",
+			},
+			result: fail,
+		},
+		{
+			// Catch a repeated word in a list of valid words.
+			directives: []string{
+				"#DW check_requires requires=copy-offload,other-thing,copy-offload",
+			},
+			result: fail,
+		},
+	}
+
+	test(t, rules, tests)
+}
+
+func TestUniqueWithin(t *testing.T) {
 	rules := []DWDirectiveRuleSpec{{
 		Command: "unique",
 		RuleDefs: []DWDirectiveRuleDef{{
@@ -417,7 +419,7 @@ func _TestUniqueWithin(t *testing.T) {
 	test(t, rules, tests)
 }
 
-func _TestKeyRegexp(t *testing.T) {
+func TestKeyRegexp(t *testing.T) {
 	rules := []DWDirectiveRuleSpec{{
 		Command: "regexp",
 		RuleDefs: []DWDirectiveRuleDef{{

diff --git a/utils/dwdparse/zz_generated.deepcopy.go b/utils/dwdparse/zz_generated.deepcopy.go