diff --git a/api/v1alpha1/conversion.go b/api/v1alpha1/conversion.go index 8095f03c..f409f9b2 100644 --- a/api/v1alpha1/conversion.go +++ b/api/v1alpha1/conversion.go @@ -190,6 +190,10 @@ func (src *DirectiveBreakdown) ConvertTo(dstRaw conversion.Hub) error { dst.Status.Error.Severity = dwsv1alpha2.SeverityFatal } + if hasAnno { + dst.Status.RequiredDaemons = restored.Status.RequiredDaemons + } + return nil } @@ -543,6 +547,10 @@ func Convert_v1alpha2_ClientMountStatus_To_v1alpha1_ClientMountStatus(in *dwsv1a return autoConvert_v1alpha2_ClientMountStatus_To_v1alpha1_ClientMountStatus(in, out, s) } +func Convert_v1alpha2_DirectiveBreakdownStatus_To_v1alpha1_DirectiveBreakdownStatus(in *dwsv1alpha2.DirectiveBreakdownStatus, out *DirectiveBreakdownStatus, s apiconversion.Scope) error { + return autoConvert_v1alpha2_DirectiveBreakdownStatus_To_v1alpha1_DirectiveBreakdownStatus(in, out, s) +} + func Convert_v1alpha1_ResourceErrorInfo_To_v1alpha2_ResourceErrorInfo(in *ResourceErrorInfo, out *dwsv1alpha2.ResourceErrorInfo, s apiconversion.Scope) error { return autoConvert_v1alpha1_ResourceErrorInfo_To_v1alpha2_ResourceErrorInfo(in, out, s) } diff --git a/api/v1alpha1/zz_generated.conversion.go b/api/v1alpha1/zz_generated.conversion.go index 6aeca58d..adf28cd2 100644 --- a/api/v1alpha1/zz_generated.conversion.go +++ b/api/v1alpha1/zz_generated.conversion.go @@ -293,11 +293,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1alpha2.DirectiveBreakdownStatus)(nil), (*DirectiveBreakdownStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1alpha2_DirectiveBreakdownStatus_To_v1alpha1_DirectiveBreakdownStatus(a.(*v1alpha2.DirectiveBreakdownStatus), b.(*DirectiveBreakdownStatus), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*Node)(nil), (*v1alpha2.Node)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha1_Node_To_v1alpha2_Node(a.(*Node), b.(*v1alpha2.Node), scope) }); err != nil { @@ -618,6 +613,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*v1alpha2.DirectiveBreakdownStatus)(nil), (*DirectiveBreakdownStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha2_DirectiveBreakdownStatus_To_v1alpha1_DirectiveBreakdownStatus(a.(*v1alpha2.DirectiveBreakdownStatus), b.(*DirectiveBreakdownStatus), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*v1alpha2.ResourceErrorInfo)(nil), (*ResourceErrorInfo)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha2_ResourceErrorInfo_To_v1alpha1_ResourceErrorInfo(a.(*v1alpha2.ResourceErrorInfo), b.(*ResourceErrorInfo), scope) }); err != nil { @@ -1307,17 +1307,13 @@ func autoConvert_v1alpha2_DirectiveBreakdownStatus_To_v1alpha1_DirectiveBreakdow out.Storage = (*StorageBreakdown)(unsafe.Pointer(in.Storage)) out.Compute = (*ComputeBreakdown)(unsafe.Pointer(in.Compute)) out.Ready = in.Ready + // WARNING: in.RequiredDaemons requires manual conversion: does not exist in peer-type if err := Convert_v1alpha2_ResourceError_To_v1alpha1_ResourceError(&in.ResourceError, &out.ResourceError, s); err != nil { return err } return nil } -// Convert_v1alpha2_DirectiveBreakdownStatus_To_v1alpha1_DirectiveBreakdownStatus is an autogenerated conversion function. -func Convert_v1alpha2_DirectiveBreakdownStatus_To_v1alpha1_DirectiveBreakdownStatus(in *v1alpha2.DirectiveBreakdownStatus, out *DirectiveBreakdownStatus, s conversion.Scope) error { - return autoConvert_v1alpha2_DirectiveBreakdownStatus_To_v1alpha1_DirectiveBreakdownStatus(in, out, s) -} - func autoConvert_v1alpha1_Node_To_v1alpha2_Node(in *Node, out *v1alpha2.Node, s conversion.Scope) error { out.Name = in.Name out.Status = v1alpha2.ResourceStatus(in.Status) diff --git a/api/v1alpha2/directivebreakdown_types.go b/api/v1alpha2/directivebreakdown_types.go index 4a57756e..dfbc5d12 100644 --- a/api/v1alpha2/directivebreakdown_types.go +++ b/api/v1alpha2/directivebreakdown_types.go @@ -1,5 +1,5 @@ /* - * Copyright 2021-2023 Hewlett Packard Enterprise Development LP + * Copyright 2021-2024 Hewlett Packard Enterprise Development LP * Other additional copyright holders may be indicated within. * * The entirety of this work is licensed under the Apache License, @@ -182,6 +182,9 @@ type DirectiveBreakdownStatus struct { // Ready indicates whether AllocationSets have been generated (true) or not (false) Ready bool `json:"ready"` + // RequiredDeamons tells the WLM about any driver-specific daemons it must enable for the job; it is assumed that the WLM knows about the driver-specific daemons and that if the users are specifying these then the WLM knows how to start them + RequiredDaemons []string `json:"requiredDaemons,omitempty"` + // Error information ResourceError `json:",inline"` } diff --git a/api/v1alpha2/zz_generated.deepcopy.go b/api/v1alpha2/zz_generated.deepcopy.go index ef8f0a27..f177a88b 100644 --- a/api/v1alpha2/zz_generated.deepcopy.go +++ b/api/v1alpha2/zz_generated.deepcopy.go @@ -601,6 +601,11 @@ func (in *DirectiveBreakdownStatus) DeepCopyInto(out *DirectiveBreakdownStatus) *out = new(ComputeBreakdown) (*in).DeepCopyInto(*out) } + if in.RequiredDaemons != nil { + in, out := &in.RequiredDaemons, &out.RequiredDaemons + *out = make([]string, len(*in)) + copy(*out, *in) + } in.ResourceError.DeepCopyInto(&out.ResourceError) } diff --git a/config/crd/bases/dataworkflowservices.github.io_directivebreakdowns.yaml b/config/crd/bases/dataworkflowservices.github.io_directivebreakdowns.yaml index 296b6650..63f9cde1 100644 --- a/config/crd/bases/dataworkflowservices.github.io_directivebreakdowns.yaml +++ b/config/crd/bases/dataworkflowservices.github.io_directivebreakdowns.yaml @@ -501,6 +501,14 @@ spec: description: Ready indicates whether AllocationSets have been generated (true) or not (false) type: boolean + requiredDaemons: + description: RequiredDeamons tells the WLM about any driver-specific + daemons it must enable for the job; it is assumed that the WLM knows + about the driver-specific daemons and that if the users are specifying + these then the WLM knows how to start them + items: + type: string + type: array storage: description: Storage is the storage breakdown for the directive properties: diff --git a/config/crd/bases/dataworkflowservices.github.io_dwdirectiverules.yaml b/config/crd/bases/dataworkflowservices.github.io_dwdirectiverules.yaml index 1ebc1659..8d77c240 100644 --- a/config/crd/bases/dataworkflowservices.github.io_dwdirectiverules.yaml +++ b/config/crd/bases/dataworkflowservices.github.io_dwdirectiverules.yaml @@ -67,6 +67,10 @@ spec: type: integer pattern: type: string + patterns: + items: + type: string + type: array type: type: string uniqueWithin: @@ -143,6 +147,10 @@ spec: type: integer pattern: type: string + patterns: + items: + type: string + type: array type: type: string uniqueWithin: diff --git a/utils/dwdparse/dwdparse.go b/utils/dwdparse/dwdparse.go index 8fd03bb7..b3df9840 100644 --- a/utils/dwdparse/dwdparse.go +++ b/utils/dwdparse/dwdparse.go @@ -1,5 +1,5 @@ /* - * Copyright 2021, 2022 Hewlett Packard Enterprise Development LP + * Copyright 2021-2024 Hewlett Packard Enterprise Development LP * Other additional copyright holders may be indicated within. * * The entirety of this work is licensed under the Apache License, @@ -29,14 +29,15 @@ import ( // DWDirectiveRuleDef defines the DWDirective parser rules // +kubebuilder:object:generate=true type DWDirectiveRuleDef struct { - Key string `json:"key"` - Type string `json:"type"` - Pattern string `json:"pattern,omitempty"` - Min int `json:"min,omitempty"` - Max int `json:"max,omitempty"` - IsRequired bool `json:"isRequired,omitempty"` - IsValueRequired bool `json:"isValueRequired,omitempty"` - UniqueWithin string `json:"uniqueWithin,omitempty"` + Key string `json:"key"` + Type string `json:"type"` + Pattern string `json:"pattern,omitempty"` + Patterns []string `json:"patterns,omitempty"` + Min int `json:"min,omitempty"` + Max int `json:"max,omitempty"` + IsRequired bool `json:"isRequired,omitempty"` + IsValueRequired bool `json:"isValueRequired,omitempty"` + UniqueWithin string `json:"uniqueWithin,omitempty"` } // DWDirectiveRuleSpec defines the desired state of DWDirective @@ -170,6 +171,35 @@ func ValidateArgs(spec DWDirectiveRuleSpec, args map[string]string, uniqueMap ma return fmt.Errorf("argument '%s' invalid string '%s'", k, v) } } + case "list-of-string": + words := strings.Split(v, ",") + if len(rule.Patterns) > 0 { + wordsMatched := make(map[string]bool) + for _, word := range words { + for idx := range rule.Patterns { + re, err := regexp.Compile(rule.Patterns[idx]) + if err != nil { + return fmt.Errorf("invalid regular expression '%s'", rule.Patterns[idx]) + } + + if re.MatchString(word) { + wordsMatched[word] = true + break + } + } + if !wordsMatched[word] { + return fmt.Errorf("argument '%s' invalid string '%s' in list '%s'", k, word, v) + } + } + } + // All of the words in the list are valid, but were any words repeated? + wordMap := make(map[string]bool) + for _, word := range words { + if _, present := wordMap[word]; present { + return fmt.Errorf("argument '%s' word '%s' repeated in list '%s'", k, word, v) + } + wordMap[word] = true + } default: return fmt.Errorf("unsupported rule type '%s'", rule.Type) } diff --git a/utils/dwdparse/dwdparse_test.go b/utils/dwdparse/dwdparse_test.go index b9669bf7..d6b88d5f 100644 --- a/utils/dwdparse/dwdparse_test.go +++ b/utils/dwdparse/dwdparse_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2021, 2022 Hewlett Packard Enterprise Development LP + * Copyright 2021-2024 Hewlett Packard Enterprise Development LP * Other additional copyright holders may be indicated within. * * The entirety of this work is licensed under the Apache License, @@ -20,8 +20,9 @@ package dwdparse import ( - . "github.com/onsi/ginkgo/v2" "testing" + + . "github.com/onsi/ginkgo/v2" ) var dWDRules = []DWDirectiveRuleSpec{ @@ -57,19 +58,6 @@ var dWDRules = []DWDirectiveRuleSpec{ IsRequired: false, IsValueRequired: true, }, - { - Key: "combined_mgtmdt", - Type: "bool", - IsRequired: false, - IsValueRequired: false, - }, - { - Key: "external_mgs", - Type: "string", - Pattern: "^[A-Za-z0-9\\-_\\.@,:]+$", - IsRequired: false, - IsValueRequired: true, - }, }, }, { @@ -104,19 +92,6 @@ var dWDRules = []DWDirectiveRuleSpec{ IsRequired: false, IsValueRequired: true, }, - { - Key: "combined_mgtmdt", - Type: "bool", - IsRequired: false, - IsValueRequired: false, - }, - { - Key: "external_mgs", - Type: "string", - Pattern: "^[A-Za-z0-9\\-_\\.@,:]+$", - IsRequired: false, - IsValueRequired: true, - }, }, }, { @@ -261,15 +236,6 @@ var dwDirectiveTests = []struct { {[]string{"#DW jobdw type=lustre capacity=100GB name=UncoolProfile3 profile=_this"}, fail}, {[]string{"#DW jobdw type=lustre capacity=100GB name=UncoolProfile4 profile=-this"}, fail}, - {[]string{"#DW jobdw type=lustre capacity=100GB combined_mgtmdt name=prettierGoodName"}, pass}, - {[]string{"#DW jobdw type=lustre capacity=100GB combined_mgtmdt=true name=prettierGoodName"}, pass}, - - {[]string{"#DW jobdw type=lustre external_mgs=rabbit-01@tcp capacity=100GB name=Extern1"}, pass}, - {[]string{"#DW jobdw type=lustre external_mgs=rabbit-01@tcp,rabbit-02@tcp:rabbit-03@tcp capacity=100GB name=Extern1"}, pass}, - {[]string{"#DW jobdw type=lustre external_mgs=rabbit-01@tcp combined_mgtmdt capacity=100GB name=Extern1"}, pass}, - {[]string{"#DW jobdw type=lustre external_mgs=rabbit-01@tcp0 capacity=100GB name=Extern1"}, pass}, - {[]string{"#DW jobdw type=lustre external_mgs=10.0.0.1@o2ib capacity=100GB name=Extern2"}, pass}, - {[]string{"#DW create_persistent type=raw capacity=100GB name=prettyGoodName "}, pass}, {[]string{"#DW create_persistent type=xfs capacity=100GB name=prettyGoodName "}, pass}, {[]string{"#DW create_persistent type=gfs2 capacity=100GB name=prettyGoodName "}, pass}, @@ -288,14 +254,6 @@ var dwDirectiveTests = []struct { {[]string{"#DW create_persistent type=lustre capacity=100GB name=UncoolProfile3 profile=_this"}, fail}, {[]string{"#DW create_persistent type=lustre capacity=100GB name=UncoolProfile4 profile=-this"}, fail}, - {[]string{"#DW create_persistent type=lustre capacity=100GB combined_mgtmdt name=prettierGoodName"}, pass}, - {[]string{"#DW create_persistent type=lustre capacity=100GB combined_mgtmdt=true name=prettierGoodName"}, pass}, - - {[]string{"#DW create_persistent type=lustre external_mgs=rabbit-01@tcp capacity=100GB name=Extern1"}, pass}, - {[]string{"#DW create_persistent type=lustre external_mgs=rabbit-01@tcp combined_mgtmdt capacity=100GB name=Extern1"}, pass}, - {[]string{"#DW create_persistent type=lustre external_mgs=rabbit-01@tcp0 capacity=100GB name=Extern1"}, pass}, - {[]string{"#DW create_persistent type=lustre external_mgs=10.0.0.1@o2ib capacity=100GB name=Extern2"}, pass}, - {[]string{"#DW stage_in type=file destination=$DW_JOB_STRIPED source=/pfs/dld-input "}, pass}, {[]string{"#DW stage_in type=directory destination=$DW_JOB_STRIPED source=/pfs/dld-input "}, pass}, {[]string{"#DW stage_in type=list destination=$DW_JOB_STRIPED source=/pfs/dld-input "}, pass}, @@ -362,7 +320,7 @@ var dwDirectiveTests = []struct { // contains individualized test cases. } -func _TestDWParse(t *testing.T) { +func TestDWParse(t *testing.T) { for index, tt := range dwDirectiveTests { err := Validate(dWDRules, tt.directiveList, func(int, DWDirectiveRuleSpec) {}) @@ -375,7 +333,6 @@ func _TestDWParse(t *testing.T) { // testCase defines an individual test case type testCase struct { - rules []DWDirectiveRuleSpec directives []string result bool } @@ -391,7 +348,51 @@ func test(t *testing.T, rules []DWDirectiveRuleSpec, tests []testCase) { } } -func _TestUniqueWithin(t *testing.T) { +func TestRequiresKeyword(t *testing.T) { + + rules := []DWDirectiveRuleSpec{ + { + Command: "check_requires", + RuleDefs: []DWDirectiveRuleDef{ + { + Key: "requires", + Type: "list-of-string", + Patterns: []string{ + "^copy-offload$", + "^other-thing$", + }, + IsRequired: false, + IsValueRequired: true, + }, + }, + }, + } + + tests := []testCase{ + { + directives: []string{ + "#DW check_requires requires=copy-offload,other-thing", + }, + result: pass, + }, + { + directives: []string{ + "#DW check_requires requires=none-such,copy-offload", + }, + result: fail, + }, + { + directives: []string{ + "#DW check_requires requires=copy-offload,other-thing,copy-offload", + }, + result: fail, + }, + } + + test(t, rules, tests) +} + +func TestUniqueWithin(t *testing.T) { rules := []DWDirectiveRuleSpec{{ Command: "unique", RuleDefs: []DWDirectiveRuleDef{{ @@ -417,7 +418,7 @@ func _TestUniqueWithin(t *testing.T) { test(t, rules, tests) } -func _TestKeyRegexp(t *testing.T) { +func TestKeyRegexp(t *testing.T) { rules := []DWDirectiveRuleSpec{{ Command: "regexp", RuleDefs: []DWDirectiveRuleDef{{ diff --git a/utils/dwdparse/zz_generated.deepcopy.go b/utils/dwdparse/zz_generated.deepcopy.go index dbeaaa38..94223369 100644 --- a/utils/dwdparse/zz_generated.deepcopy.go +++ b/utils/dwdparse/zz_generated.deepcopy.go @@ -28,6 +28,11 @@ import () // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DWDirectiveRuleDef) DeepCopyInto(out *DWDirectiveRuleDef) { *out = *in + if in.Patterns != nil { + in, out := &in.Patterns, &out.Patterns + *out = make([]string, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DWDirectiveRuleDef. @@ -46,7 +51,9 @@ func (in *DWDirectiveRuleSpec) DeepCopyInto(out *DWDirectiveRuleSpec) { if in.RuleDefs != nil { in, out := &in.RuleDefs, &out.RuleDefs *out = make([]DWDirectiveRuleDef, len(*in)) - copy(*out, *in) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } } }