Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add ilab kfp pipeline to the DSPO repo #764

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions api/v1/dspipeline_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,9 +69,13 @@ type APIServer struct {
// Include sample pipelines with the deployment of this DSP API Server. Default: true
// +kubebuilder:default:=false
// +kubebuilder:validation:Optional
EnableSamplePipeline bool `json:"enableSamplePipeline"`
ArgoLauncherImage string `json:"argoLauncherImage,omitempty"`
ArgoDriverImage string `json:"argoDriverImage,omitempty"`
EnableSamplePipeline bool `json:"enableSamplePipeline"`
// Enable the Instructlab Multi-Phase Training pipeline with the deployment of this DSP API server. Default: false
// +kubebuilder:default:=false
// +kubebuilder:validation:Optional
EnableInstructLabPipeline bool `json:"enableInstructLabPipeline"`
ArgoLauncherImage string `json:"argoLauncherImage,omitempty"`
ArgoDriverImage string `json:"argoDriverImage,omitempty"`
// Specify custom Pod resource requirements for this component.
Resources *ResourceRequirements `json:"resources,omitempty"`

Expand Down
10 changes: 7 additions & 3 deletions api/v1alpha1/dspipeline_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,9 +69,13 @@ type APIServer struct {
// Include sample pipelines with the deployment of this DSP API Server. Default: true
// +kubebuilder:default:=false
// +kubebuilder:validation:Optional
EnableSamplePipeline bool `json:"enableSamplePipeline"`
ArgoLauncherImage string `json:"argoLauncherImage,omitempty"`
ArgoDriverImage string `json:"argoDriverImage,omitempty"`
EnableSamplePipeline bool `json:"enableSamplePipeline"`
// Enable the Instructlab Multi-Phase Training pipeline with the deployment of this DSP API server. Default: false
// +kubebuilder:default:=false
// +kubebuilder:validation:Optional
EnableInstructLabPipeline bool `json:"enableInstructLabPipeline"`
ArgoLauncherImage string `json:"argoLauncherImage,omitempty"`
ArgoDriverImage string `json:"argoDriverImage,omitempty"`
// Specify custom Pod resource requirements for this component.
Resources *ResourceRequirements `json:"resources,omitempty"`

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,11 @@ spec:
Server. Setting Deploy to false disables operator reconciliation.
Default: true'
type: boolean
enableInstructLabPipeline:
default: false
description: 'Enable the Instructlab Multi-Phase Training pipeline
with the deployment of this DSP API server. Default: false'
type: boolean
enableOauth:
default: true
description: 'Create an Openshift Route for this DSP API Server.
Expand Down Expand Up @@ -990,6 +995,11 @@ spec:
Server. Setting Deploy to false disables operator reconciliation.
Default: true'
type: boolean
enableInstructLabPipeline:
default: false
description: 'Enable the Instructlab Multi-Phase Training pipeline
with the deployment of this DSP API server. Default: false'
type: boolean
enableOauth:
default: true
description: 'Create an Openshift Route for this DSP API Server.
Expand Down
12 changes: 5 additions & 7 deletions config/internal/apiserver/default/deployment.yaml.tmpl
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ spec:
args:
- --config=/config
- -logtostderr=true
{{ if .APIServer.EnableSamplePipeline }}
{{ if or .APIServer.EnableSamplePipeline .APIServer.EnableInstructLabPipeline }}
- --sampleconfig=/config/sample_config.json
{{ end }}
{{ if .PodToPodTLS }}
Expand Down Expand Up @@ -206,18 +206,16 @@ spec:
- mountPath: /etc/tls/private
name: proxy-tls
{{ end }}
{{ if or .APIServer.EnableSamplePipeline .CustomCABundle }}
{{ if .APIServer.EnableSamplePipeline }}
{{ if or .APIServer.EnableSamplePipeline .APIServer.EnableInstructLabPipeline}}
- name: sample-config
mountPath: /config/sample_config.json
subPath: sample_config.json
- name: sample-pipeline
mountPath: /samples/
{{ end }}
{{ if .CustomCABundle }}
{{ end }}
{{ if .CustomCABundle }}
- mountPath: {{ .CustomCABundleRootMountPath }}
name: ca-bundle
{{ end }}
{{ end }}
{{ if .APIServer.EnableRoute }}
- name: oauth-proxy
Expand Down Expand Up @@ -287,7 +285,7 @@ spec:
configMap:
name: {{ .CustomCABundle.ConfigMapName }}
{{ end }}
{{ if .APIServer.EnableSamplePipeline }}
{{ if or .APIServer.EnableSamplePipeline .APIServer.EnableInstructLabPipeline}}
- name: sample-config
configMap:
name: sample-config-{{.Name}}
Expand Down
26 changes: 18 additions & 8 deletions config/internal/apiserver/sample-pipeline/sample-config.yaml.tmpl
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,21 @@ metadata:
app: {{.APIServerDefaultResourceName}}
component: data-science-pipelines
data:
sample_config.json: |-
[
{
"name": "[Demo] iris-training",
"description": "[source code](https://github.com/opendatahub-io/data-science-pipelines/tree/master/samples/iris-sklearn) A simple pipeline to demonstrate a basic ML Training workflow",
"file": "/samples/iris-pipeline-compiled.yaml"
}
]
sample_config.json: |-
[
{{- if .EnableSamplePipeline }}
{
"name": "[Demo] iris-training",
"description": "[source code](https://github.com/opendatahub-io/data-science-pipelines/tree/master/samples/iris-sklearn) A simple pipeline to demonstrate a basic ML Training workflow",
"file": "/samples/iris-pipeline-compiled.yaml"
}
{{- if and .EnableSamplePipeline .EnableInstructLabPipeline }},{{ end }}
{{- end }}
{{- if .EnableInstructLabPipeline }}
{
"name": "[InstructLab] Multi-Phase Training Pipeline",
"description": "[source code](https://github.com/opendatahub-io/ilab-on-ocp) Instructlab Multi-Phase Training Pipeline",
"file": "/pipelines/instructlab.yaml"
}
{{- end }}
]
2 changes: 1 addition & 1 deletion controllers/apiserver.go
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ func (r *DSPAReconciler) ReconcileAPIServer(ctx context.Context, dsp *dspav1.Dat
}

for cmName, template := range samplePipelineTemplates {
if dsp.Spec.APIServer.EnableSamplePipeline {
if dsp.Spec.APIServer.EnableSamplePipeline || dsp.Spec.APIServer.EnableInstructLabPipeline {
err := r.Apply(dsp, params, template)
if err != nil {
return err
Expand Down
10 changes: 7 additions & 3 deletions controllers/dspipeline_params.go
Original file line number Diff line number Diff line change
Expand Up @@ -92,9 +92,10 @@ type DSPAParams struct {
CustomCABundle *dspa.CABundle
DSPONamespace string
// Use to enable tls communication between component pods.
PodToPodTLS bool

APIServerServiceDNSName string
PodToPodTLS bool
EnableSamplePipeline bool
EnableInstructLabPipeline bool
APIServerServiceDNSName string
}

type DBConnection struct {
Expand Down Expand Up @@ -601,6 +602,9 @@ func (p *DSPAParams) ExtractParams(ctx context.Context, dsp *dspa.DataSciencePip
setStringDefault(argoDriverImageFromConfig, &p.APIServer.ArgoDriverImage)
setResourcesDefault(config.APIServerResourceRequirements, &p.APIServer.Resources)

p.EnableSamplePipeline = dsp.Spec.APIServer.EnableSamplePipeline
p.EnableInstructLabPipeline = dsp.Spec.APIServer.EnableInstructLabPipeline

if p.APIServer.CustomServerConfig == nil {
p.APIServer.CustomServerConfig = &dspa.ScriptConfigMap{
Name: config.CustomServerConfigMapNamePrefix + dsp.Name,
Expand Down
25 changes: 25 additions & 0 deletions tests/pipeline_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -75,4 +75,29 @@ func (suite *IntegrationTestSuite) TestAPIServerDeployment() {
require.NoError(t, err)
assert.Equal(t, 200, response.StatusCode)
})

suite.T().Run("Should check for InstructLab pipeline existence based on EnableInstructLabPipeline flag", func(t *testing.T) {
expectedDisplayName := "[InstructLab] Multi-Phase Training Pipeline"

// Retrieve pipelines
pipelines, err := TestUtil.RetrievePipelines(t, suite.Clientmgr.httpClient, APIServerURL)
require.NoError(t, err, "Failed to retrieve pipelines")

found := false
for _, pipeline := range pipelines.Pipelines {
if pipeline.DisplayName == expectedDisplayName {
found = true
break
}
}
if suite.DSPA.Spec.APIServer.EnableInstructLabPipeline {
expectedCount := 4
assert.Equal(t, expectedCount, len(pipelines.Pipelines), "Pipeline count should match when EnableInstructLabPipeline flag is enabled")
assert.True(t, found, "InstructLab pipeline should exist when the flag is enabled")
} else {
expectedCount := 3
assert.Equal(t, expectedCount, len(pipelines.Pipelines), "Pipeline count should match when EnableInstructLabPipeline flag is disabled")
assert.False(t, found, "InstructLab pipeline should not exist when the flag is disabled")
}
})
}
1 change: 1 addition & 0 deletions tests/resources/dspa-lite.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ spec:
deploy: true
enableOauth: false
enableSamplePipeline: true
enableInstructLabPipeline: true
cABundle:
configMapName: nginx-tls-config
configMapKey: rootCA.crt
Expand Down
2 changes: 1 addition & 1 deletion tests/resources/test-pipeline-run.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ deploymentSpec:
- -c
- "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\
\ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
\ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.10.1'\
\ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.11.0'\
\ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\
$0\" \"$@\"\n"
- sh
Expand Down
4 changes: 2 additions & 2 deletions tests/resources/test-pipeline-with-custom-pip-server-run.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ deploymentSpec:
- "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\
\ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
\ python3 -m pip install --quiet --no-warn-script-location --index-url https://nginx-service.test-pypiserver.svc.cluster.local/simple/\
\ 'kfp==2.10.1' '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"\
\ 'kfp==2.11.0' '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"\
3.9\"' && python3 -m pip install --quiet --no-warn-script-location --index-url\
\ https://nginx-service.test-pypiserver.svc.cluster.local/simple/ 'numpy'\
\ && \"$0\" \"$@\"\n"
Expand Down Expand Up @@ -66,4 +66,4 @@ root:
Output:
parameterType: STRING
schemaVersion: 2.1.0
sdkVersion: kfp-2.10.1
sdkVersion: kfp-2.11.0
11 changes: 11 additions & 0 deletions tests/util/rest.go
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,17 @@ func RetrievePipelineId(t *testing.T, httpClient http.Client, APIServerURL strin
}
}

func RetrievePipelines(t *testing.T, httpClient http.Client, APIServerURL string) (Pipeline, error) {
response, err := httpClient.Get(fmt.Sprintf("%s/apis/v2beta1/pipelines", APIServerURL))
require.NoError(t, err)
responseData, err := io.ReadAll(response.Body)
require.NoError(t, err)
var pipelineData Pipeline
err = json.Unmarshal(responseData, &pipelineData)
require.NoError(t, err)
return pipelineData, nil
}

func FormatRequestBody(t *testing.T, pipelineID string, PipelineDisplayName string) []byte {
requestBody := PipelineRequest{
DisplayName: PipelineDisplayName,
Expand Down
Loading