From d439d03c21b1687354820cc780ab3882d59f31c5 Mon Sep 17 00:00:00 2001 From: VaniHaripriya Date: Fri, 13 Dec 2024 11:07:06 -0600 Subject: [PATCH] Add ilab kfp pipeline to the DSPO repo Signed-off-by: VaniHaripriya Co-authored-by: Matt Prahl --- api/v1/dspipeline_types.go | 10 ++++--- api/v1alpha1/dspipeline_types.go | 10 ++++--- ...b.io_datasciencepipelinesapplications.yaml | 10 +++++++ .../apiserver/default/deployment.yaml.tmpl | 12 ++++----- .../sample-pipeline/sample-config.yaml.tmpl | 26 +++++++++++++------ controllers/apiserver.go | 2 +- controllers/dspipeline_params.go | 10 ++++--- tests/pipeline_test.go | 25 ++++++++++++++++++ tests/resources/dspa-lite.yaml | 1 + tests/resources/test-pipeline-run.yaml | 2 +- ...t-pipeline-with-custom-pip-server-run.yaml | 4 +-- tests/util/rest.go | 11 ++++++++ 12 files changed, 95 insertions(+), 28 deletions(-) diff --git a/api/v1/dspipeline_types.go b/api/v1/dspipeline_types.go index 8f6253318..b79ad4ac4 100644 --- a/api/v1/dspipeline_types.go +++ b/api/v1/dspipeline_types.go @@ -69,9 +69,13 @@ type APIServer struct { // Include sample pipelines with the deployment of this DSP API Server. Default: true // +kubebuilder:default:=false // +kubebuilder:validation:Optional - EnableSamplePipeline bool `json:"enableSamplePipeline"` - ArgoLauncherImage string `json:"argoLauncherImage,omitempty"` - ArgoDriverImage string `json:"argoDriverImage,omitempty"` + EnableSamplePipeline bool `json:"enableSamplePipeline"` + // Enable the Instructlab Multi-Phase Training pipeline with the deployment of this DSP API server. Default: false + // +kubebuilder:default:=false + // +kubebuilder:validation:Optional + EnableInstructLabPipeline bool `json:"enableInstructLabPipeline"` + ArgoLauncherImage string `json:"argoLauncherImage,omitempty"` + ArgoDriverImage string `json:"argoDriverImage,omitempty"` // Specify custom Pod resource requirements for this component. Resources *ResourceRequirements `json:"resources,omitempty"` diff --git a/api/v1alpha1/dspipeline_types.go b/api/v1alpha1/dspipeline_types.go index 6a01914ac..6d5deb4e3 100644 --- a/api/v1alpha1/dspipeline_types.go +++ b/api/v1alpha1/dspipeline_types.go @@ -69,9 +69,13 @@ type APIServer struct { // Include sample pipelines with the deployment of this DSP API Server. Default: true // +kubebuilder:default:=false // +kubebuilder:validation:Optional - EnableSamplePipeline bool `json:"enableSamplePipeline"` - ArgoLauncherImage string `json:"argoLauncherImage,omitempty"` - ArgoDriverImage string `json:"argoDriverImage,omitempty"` + EnableSamplePipeline bool `json:"enableSamplePipeline"` + // Enable the Instructlab Multi-Phase Training pipeline with the deployment of this DSP API server. Default: false + // +kubebuilder:default:=false + // +kubebuilder:validation:Optional + EnableInstructLabPipeline bool `json:"enableInstructLabPipeline"` + ArgoLauncherImage string `json:"argoLauncherImage,omitempty"` + ArgoDriverImage string `json:"argoDriverImage,omitempty"` // Specify custom Pod resource requirements for this component. Resources *ResourceRequirements `json:"resources,omitempty"` diff --git a/config/crd/bases/datasciencepipelinesapplications.opendatahub.io_datasciencepipelinesapplications.yaml b/config/crd/bases/datasciencepipelinesapplications.opendatahub.io_datasciencepipelinesapplications.yaml index 380aaeb5c..dab76db84 100644 --- a/config/crd/bases/datasciencepipelinesapplications.opendatahub.io_datasciencepipelinesapplications.yaml +++ b/config/crd/bases/datasciencepipelinesapplications.opendatahub.io_datasciencepipelinesapplications.yaml @@ -105,6 +105,11 @@ spec: Server. Setting Deploy to false disables operator reconciliation. Default: true' type: boolean + enableInstructLabPipeline: + default: false + description: 'Enable the Instructlab Multi-Phase Training pipeline + with the deployment of this DSP API server. Default: false' + type: boolean enableOauth: default: true description: 'Create an Openshift Route for this DSP API Server. @@ -990,6 +995,11 @@ spec: Server. Setting Deploy to false disables operator reconciliation. Default: true' type: boolean + enableInstructLabPipeline: + default: false + description: 'Enable the Instructlab Multi-Phase Training pipeline + with the deployment of this DSP API server. Default: false' + type: boolean enableOauth: default: true description: 'Create an Openshift Route for this DSP API Server. diff --git a/config/internal/apiserver/default/deployment.yaml.tmpl b/config/internal/apiserver/default/deployment.yaml.tmpl index ee555d763..c4c180be7 100644 --- a/config/internal/apiserver/default/deployment.yaml.tmpl +++ b/config/internal/apiserver/default/deployment.yaml.tmpl @@ -147,7 +147,7 @@ spec: args: - --config=/config - -logtostderr=true - {{ if .APIServer.EnableSamplePipeline }} + {{ if or .APIServer.EnableSamplePipeline .APIServer.EnableInstructLabPipeline }} - --sampleconfig=/config/sample_config.json {{ end }} {{ if .PodToPodTLS }} @@ -206,18 +206,16 @@ spec: - mountPath: /etc/tls/private name: proxy-tls {{ end }} - {{ if or .APIServer.EnableSamplePipeline .CustomCABundle }} - {{ if .APIServer.EnableSamplePipeline }} + {{ if or .APIServer.EnableSamplePipeline .APIServer.EnableInstructLabPipeline}} - name: sample-config mountPath: /config/sample_config.json subPath: sample_config.json - name: sample-pipeline mountPath: /samples/ - {{ end }} - {{ if .CustomCABundle }} + {{ end }} + {{ if .CustomCABundle }} - mountPath: {{ .CustomCABundleRootMountPath }} name: ca-bundle - {{ end }} {{ end }} {{ if .APIServer.EnableRoute }} - name: oauth-proxy @@ -287,7 +285,7 @@ spec: configMap: name: {{ .CustomCABundle.ConfigMapName }} {{ end }} - {{ if .APIServer.EnableSamplePipeline }} + {{ if or .APIServer.EnableSamplePipeline .APIServer.EnableInstructLabPipeline}} - name: sample-config configMap: name: sample-config-{{.Name}} diff --git a/config/internal/apiserver/sample-pipeline/sample-config.yaml.tmpl b/config/internal/apiserver/sample-pipeline/sample-config.yaml.tmpl index 76e64d65d..0f0baed0f 100644 --- a/config/internal/apiserver/sample-pipeline/sample-config.yaml.tmpl +++ b/config/internal/apiserver/sample-pipeline/sample-config.yaml.tmpl @@ -7,11 +7,21 @@ metadata: app: {{.APIServerDefaultResourceName}} component: data-science-pipelines data: - sample_config.json: |- - [ - { - "name": "[Demo] iris-training", - "description": "[source code](https://github.com/opendatahub-io/data-science-pipelines/tree/master/samples/iris-sklearn) A simple pipeline to demonstrate a basic ML Training workflow", - "file": "/samples/iris-pipeline-compiled.yaml" - } - ] + sample_config.json: |- + [ + {{- if .EnableSamplePipeline }} + { + "name": "[Demo] iris-training", + "description": "[source code](https://github.com/opendatahub-io/data-science-pipelines/tree/master/samples/iris-sklearn) A simple pipeline to demonstrate a basic ML Training workflow", + "file": "/samples/iris-pipeline-compiled.yaml" + } + {{- if and .EnableSamplePipeline .EnableInstructLabPipeline }},{{ end }} + {{- end }} + {{- if .EnableInstructLabPipeline }} + { + "name": "[InstructLab] Multi-Phase Training Pipeline", + "description": "[source code](https://github.com/opendatahub-io/ilab-on-ocp) Instructlab Multi-Phase Training Pipeline", + "file": "/pipelines/instructlab.yaml" + } + {{- end }} + ] diff --git a/controllers/apiserver.go b/controllers/apiserver.go index 929ede232..8b1b47938 100644 --- a/controllers/apiserver.go +++ b/controllers/apiserver.go @@ -68,7 +68,7 @@ func (r *DSPAReconciler) ReconcileAPIServer(ctx context.Context, dsp *dspav1.Dat } for cmName, template := range samplePipelineTemplates { - if dsp.Spec.APIServer.EnableSamplePipeline { + if dsp.Spec.APIServer.EnableSamplePipeline || dsp.Spec.APIServer.EnableInstructLabPipeline { err := r.Apply(dsp, params, template) if err != nil { return err diff --git a/controllers/dspipeline_params.go b/controllers/dspipeline_params.go index 4efdbe4af..9ee424c50 100644 --- a/controllers/dspipeline_params.go +++ b/controllers/dspipeline_params.go @@ -92,9 +92,10 @@ type DSPAParams struct { CustomCABundle *dspa.CABundle DSPONamespace string // Use to enable tls communication between component pods. - PodToPodTLS bool - - APIServerServiceDNSName string + PodToPodTLS bool + EnableSamplePipeline bool + EnableInstructLabPipeline bool + APIServerServiceDNSName string } type DBConnection struct { @@ -601,6 +602,9 @@ func (p *DSPAParams) ExtractParams(ctx context.Context, dsp *dspa.DataSciencePip setStringDefault(argoDriverImageFromConfig, &p.APIServer.ArgoDriverImage) setResourcesDefault(config.APIServerResourceRequirements, &p.APIServer.Resources) + p.EnableSamplePipeline = dsp.Spec.APIServer.EnableSamplePipeline + p.EnableInstructLabPipeline = dsp.Spec.APIServer.EnableInstructLabPipeline + if p.APIServer.CustomServerConfig == nil { p.APIServer.CustomServerConfig = &dspa.ScriptConfigMap{ Name: config.CustomServerConfigMapNamePrefix + dsp.Name, diff --git a/tests/pipeline_test.go b/tests/pipeline_test.go index 057ce9233..3ab09e858 100644 --- a/tests/pipeline_test.go +++ b/tests/pipeline_test.go @@ -75,4 +75,29 @@ func (suite *IntegrationTestSuite) TestAPIServerDeployment() { require.NoError(t, err) assert.Equal(t, 200, response.StatusCode) }) + + suite.T().Run("Should check for InstructLab pipeline existence based on EnableInstructLabPipeline flag", func(t *testing.T) { + expectedDisplayName := "[InstructLab] Multi-Phase Training Pipeline" + + // Retrieve pipelines + pipelines, err := TestUtil.RetrievePipelines(t, suite.Clientmgr.httpClient, APIServerURL) + require.NoError(t, err, "Failed to retrieve pipelines") + + found := false + for _, pipeline := range pipelines.Pipelines { + if pipeline.DisplayName == expectedDisplayName { + found = true + break + } + } + if suite.DSPA.Spec.APIServer.EnableInstructLabPipeline { + expectedCount := 4 + assert.Equal(t, expectedCount, len(pipelines.Pipelines), "Pipeline count should match when EnableInstructLabPipeline flag is enabled") + assert.True(t, found, "InstructLab pipeline should exist when the flag is enabled") + } else { + expectedCount := 3 + assert.Equal(t, expectedCount, len(pipelines.Pipelines), "Pipeline count should match when EnableInstructLabPipeline flag is disabled") + assert.False(t, found, "InstructLab pipeline should not exist when the flag is disabled") + } + }) } diff --git a/tests/resources/dspa-lite.yaml b/tests/resources/dspa-lite.yaml index ff36007b1..d9789f19c 100644 --- a/tests/resources/dspa-lite.yaml +++ b/tests/resources/dspa-lite.yaml @@ -9,6 +9,7 @@ spec: deploy: true enableOauth: false enableSamplePipeline: true + enableInstructLabPipeline: true cABundle: configMapName: nginx-tls-config configMapKey: rootCA.crt diff --git a/tests/resources/test-pipeline-run.yaml b/tests/resources/test-pipeline-run.yaml index f92d80046..e57e88869 100644 --- a/tests/resources/test-pipeline-run.yaml +++ b/tests/resources/test-pipeline-run.yaml @@ -29,7 +29,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.10.1'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.11.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh diff --git a/tests/resources/test-pipeline-with-custom-pip-server-run.yaml b/tests/resources/test-pipeline-with-custom-pip-server-run.yaml index da3e6352c..704c77a56 100644 --- a/tests/resources/test-pipeline-with-custom-pip-server-run.yaml +++ b/tests/resources/test-pipeline-with-custom-pip-server-run.yaml @@ -24,7 +24,7 @@ deploymentSpec: - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ \ python3 -m pip install --quiet --no-warn-script-location --index-url https://nginx-service.test-pypiserver.svc.cluster.local/simple/\ - \ 'kfp==2.10.1' '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"\ + \ 'kfp==2.11.0' '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"\ 3.9\"' && python3 -m pip install --quiet --no-warn-script-location --index-url\ \ https://nginx-service.test-pypiserver.svc.cluster.local/simple/ 'numpy'\ \ && \"$0\" \"$@\"\n" @@ -66,4 +66,4 @@ root: Output: parameterType: STRING schemaVersion: 2.1.0 -sdkVersion: kfp-2.10.1 +sdkVersion: kfp-2.11.0 diff --git a/tests/util/rest.go b/tests/util/rest.go index 3a2b9d52c..4baad3bb2 100644 --- a/tests/util/rest.go +++ b/tests/util/rest.go @@ -94,6 +94,17 @@ func RetrievePipelineId(t *testing.T, httpClient http.Client, APIServerURL strin } } +func RetrievePipelines(t *testing.T, httpClient http.Client, APIServerURL string) (Pipeline, error) { + response, err := httpClient.Get(fmt.Sprintf("%s/apis/v2beta1/pipelines", APIServerURL)) + require.NoError(t, err) + responseData, err := io.ReadAll(response.Body) + require.NoError(t, err) + var pipelineData Pipeline + err = json.Unmarshal(responseData, &pipelineData) + require.NoError(t, err) + return pipelineData, nil +} + func FormatRequestBody(t *testing.T, pipelineID string, PipelineDisplayName string) []byte { requestBody := PipelineRequest{ DisplayName: PipelineDisplayName,