Skip to content

Commit

Permalink
[Exporter] Improve exporting of DLT and test coverage (#3898)
Browse files Browse the repository at this point in the history
## Changes
<!-- Summary of your changes that are easy to understand -->

This PR includes following changes:

- emitting of email notifications recipients
- checking the generated code to detect problems earlier

## Tests
<!-- 
How is this tested? Please see the checklist below and also describe any
other relevant tests
-->

- [x] `make test` run locally
- [ ] relevant change in `docs/` folder
- [ ] covered with integration tests in `internal/acceptance`
- [ ] relevant acceptance tests are passing
- [ ] using Go SDK
  • Loading branch information
alexott authored Aug 21, 2024
1 parent 2d2b4d1 commit cfe23c3
Show file tree
Hide file tree
Showing 3 changed files with 49 additions and 1 deletion.
38 changes: 37 additions & 1 deletion exporter/exporter_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2117,7 +2117,7 @@ func TestImportingDLTPipelines(t *testing.T) {
Resources: []scim.User{
{
ID: "id",
UserName: "id",
UserName: "id@domain.com",
},
},
},
Expand Down Expand Up @@ -2184,6 +2184,42 @@ func TestImportingDLTPipelines(t *testing.T) {

err := ic.Run()
assert.NoError(t, err)
content, err := os.ReadFile(tmpDir + "/dlt.tf")
assert.NoError(t, err)
contentStr := string(content)
assert.True(t, strings.Contains(contentStr, `resource "databricks_pipeline" "test_dlt_123"`))
assert.True(t, strings.Contains(contentStr, `library {
notebook {
path = databricks_notebook.users_user_domain_com_test_dlt_123.id
}
}`))
assert.True(t, strings.Contains(contentStr, `cluster {
spark_conf = {
"fs.azure.account.auth.type" = "OAuth"
"fs.azure.account.oauth.provider.type" = "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider"
"fs.azure.account.oauth2.client.secret" = "{{secrets/some-kv-scope/test-secret}}"
}
num_workers = 1
label = "default"
instance_pool_id = "123"
init_scripts {
dbfs {
destination = "dbfs:/FileStore/jars/test.jar"
}
}
init_scripts {
workspace {
destination = databricks_workspace_file.init_sh_789.id
}
}
}`))
assert.True(t, strings.Contains(contentStr, `notification {
email_recipients = [databricks_user.user_123.user_name]
alerts = ["on-flow-failure", "on-update-failure"]
}`))
assert.True(t, strings.Contains(contentStr, `edition = "advanced"
development = true
continuous = true`))
})
}

Expand Down
6 changes: 6 additions & 0 deletions exporter/importables.go
Original file line number Diff line number Diff line change
Expand Up @@ -2084,6 +2084,11 @@ var resourcesMap map[string]importable = map[string]importable{
ic.emitSecretsFromSecretsPathMap(pipeline.Configuration)
ic.emitPermissionsIfNotIgnored(r, fmt.Sprintf("/pipelines/%s", r.ID),
"pipeline_"+ic.Importables["databricks_pipeline"].Name(ic, r.Data))
if pipeline.Notifications != nil {
for _, n := range pipeline.Notifications {
ic.emitListOfUsers(n.EmailRecipients)
}
}
return nil
},
ShouldOmitField: func(ic *importContext, pathString string, as *schema.Schema, d *schema.ResourceData) bool {
Expand Down Expand Up @@ -2136,6 +2141,7 @@ var resourcesMap map[string]importable = map[string]importable{
{Path: "library.file.path", Resource: "databricks_workspace_file", Match: "workspace_path"},
{Path: "library.jar", Resource: "databricks_dbfs_file", Match: "dbfs_path"},
{Path: "library.whl", Resource: "databricks_dbfs_file", Match: "dbfs_path"},
{Path: "notification.email_recipients", Resource: "databricks_user", Match: "user_name", MatchType: MatchCaseInsensitive},
{Path: "configuration", Resource: "databricks_repo", Match: "workspace_path",
MatchType: MatchPrefix, SearchValueTransformFunc: appendEndingSlashToDirName},
{Path: "library.notebook.path", Resource: "databricks_repo", Match: "path",
Expand Down
6 changes: 6 additions & 0 deletions exporter/test-data/get-dlt-pipeline.json
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,12 @@
}
}
],
"notifications": [
{
"email_recipients": ["[email protected]"],
"alerts": ["on-flow-failure", "on-update-failure"]
}
],
"name": "Test DLT",
"photon": false,
"storage": "dbfs:/pipelines/9ab12322-ccef-4539-8bf1-a43a95667dac",
Expand Down

0 comments on commit cfe23c3

Please sign in to comment.