Skip to content

Commit a3d1a10

Browse files
wjohnsonwjohnson
committed
Update test expectations to make room for additional Databricks Runtimes and add Kusto support (#182)
Fix test runner to support situations where we don't provide a process entity expectation. Update requirements.txt for test.
1 parent 917e271 commit a3d1a10

16 files changed

+932
-17
lines changed

‎.github/workflows/build-release.yml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,13 @@ jobs:
133133
AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
134134
AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
135135

136+
- name: Start Integration ADX Cluster
137+
run: source tests/integration/manage-adx-cluster.sh start ${{ secrets.INT_SUBSCRIPTION_ID }} ${{ secrets.ADX_RG_NAME }} ${{ secrets.ADX_CLUSTER_NAME }}
138+
env:
139+
AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
140+
AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
141+
AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
142+
136143
- name: Install Python Requirements and Databricks CLI
137144
run: pip install pyapacheatlas==0.12.0 azure-identity databricks-cli
138145

@@ -172,6 +179,13 @@ jobs:
172179
AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
173180
AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
174181
AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
182+
183+
- name: Stop Integration ADX Cluster
184+
run: source tests/integration/manage-adx-cluster.sh stop ${{ secrets.INT_SUBSCRIPTION_ID }} ${{ secrets.ADX_RG_NAME }} ${{ secrets.ADX_CLUSTER_NAME }}
185+
env:
186+
AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
187+
AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
188+
AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
175189

176190
createRelease:
177191
name: Create Release

‎tests/integration/jobdefs/hive3-tests-def.json‎ renamed to ‎tests/integration/jobdefs-inactive/hive3-tests-def.json‎

File renamed without changes.

‎tests/integration/jobdefs/hive3-tests-expectations.json‎ renamed to ‎tests/integration/jobdefs-inactive/hive3-tests-expectations.json‎

File renamed without changes.

‎tests/integration/jobdefs/spark3-tests-def.json‎ renamed to ‎tests/integration/jobdefs-inactive/spark3-tests-def.json‎

File renamed without changes.

‎tests/integration/jobdefs/spark3-tests-expectations.json‎ renamed to ‎tests/integration/jobdefs-inactive/spark3-tests-expectations.json‎

File renamed without changes.

‎tests/integration/jobdefs/dbx-delta-lts-104-def.json‎

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
{
2+
"name": "dbx-delta-lts-104",
3+
"email_notifications": {
4+
"no_alert_for_skipped_runs": false
5+
},
6+
"webhook_notifications": {},
7+
"timeout_seconds": 0,
8+
"max_concurrent_runs": 1,
9+
"tasks": [
10+
{
11+
"task_key": "delta-merge-task",
12+
"run_if": "ALL_SUCCESS",
13+
"notebook_task": {
14+
"notebook_path": "/Shared/examples/delta-in-delta-merge",
15+
"source": "WORKSPACE"
16+
},
17+
"existing_cluster_id": "0104-045638-iaecf5ne",
18+
"timeout_seconds": 0,
19+
"email_notifications": {},
20+
"notification_settings": {
21+
"no_alert_for_skipped_runs": false,
22+
"no_alert_for_canceled_runs": false,
23+
"alert_on_last_attempt": false
24+
},
25+
"webhook_notifications": {}
26+
},
27+
{
28+
"task_key": "delta-merge-package-task",
29+
"depends_on": [
30+
{
31+
"task_key": "delta-merge-task"
32+
}
33+
],
34+
"run_if": "ALL_DONE",
35+
"notebook_task": {
36+
"notebook_path": "/Shared/examples/delta-in-delta-merge-package",
37+
"source": "WORKSPACE"
38+
},
39+
"existing_cluster_id": "0104-045638-iaecf5ne",
40+
"timeout_seconds": 0,
41+
"email_notifications": {},
42+
"notification_settings": {
43+
"no_alert_for_skipped_runs": false,
44+
"no_alert_for_canceled_runs": false,
45+
"alert_on_last_attempt": false
46+
},
47+
"webhook_notifications": {}
48+
},
49+
{
50+
"task_key": "delta-with-abfss-task",
51+
"depends_on": [
52+
{
53+
"task_key": "delta-merge-package-task"
54+
}
55+
],
56+
"run_if": "ALL_DONE",
57+
"notebook_task": {
58+
"notebook_path": "/Shared/examples/delta-in-delta-out-abfss",
59+
"source": "WORKSPACE"
60+
},
61+
"existing_cluster_id": "0104-045638-iaecf5ne",
62+
"timeout_seconds": 0,
63+
"email_notifications": {},
64+
"notification_settings": {
65+
"no_alert_for_skipped_runs": false,
66+
"no_alert_for_canceled_runs": false,
67+
"alert_on_last_attempt": false
68+
},
69+
"webhook_notifications": {}
70+
},
71+
{
72+
"task_key": "delta-fs-task",
73+
"depends_on": [
74+
{
75+
"task_key": "delta-with-abfss-task"
76+
}
77+
],
78+
"run_if": "ALL_DONE",
79+
"notebook_task": {
80+
"notebook_path": "/Shared/examples/delta-in-delta-out-fs",
81+
"source": "WORKSPACE"
82+
},
83+
"existing_cluster_id": "0104-045638-iaecf5ne",
84+
"timeout_seconds": 0,
85+
"email_notifications": {},
86+
"notification_settings": {
87+
"no_alert_for_skipped_runs": false,
88+
"no_alert_for_canceled_runs": false,
89+
"alert_on_last_attempt": false
90+
},
91+
"webhook_notifications": {}
92+
},
93+
{
94+
"task_key": "delta-fs-col-changes-task",
95+
"depends_on": [
96+
{
97+
"task_key": "delta-fs-task"
98+
}
99+
],
100+
"run_if": "ALL_DONE",
101+
"notebook_task": {
102+
"notebook_path": "/Shared/examples/delta-in-delta-out-fs-with-col-changes",
103+
"source": "WORKSPACE"
104+
},
105+
"existing_cluster_id": "0104-045638-iaecf5ne",
106+
"timeout_seconds": 0,
107+
"email_notifications": {},
108+
"notification_settings": {
109+
"no_alert_for_skipped_runs": false,
110+
"no_alert_for_canceled_runs": false,
111+
"alert_on_last_attempt": false
112+
},
113+
"webhook_notifications": {}
114+
},
115+
{
116+
"task_key": "delta-mnt-task",
117+
"depends_on": [
118+
{
119+
"task_key": "delta-fs-col-changes-task"
120+
}
121+
],
122+
"run_if": "ALL_DONE",
123+
"notebook_task": {
124+
"notebook_path": "/Shared/examples/delta-in-delta-out-mnt",
125+
"source": "WORKSPACE"
126+
},
127+
"existing_cluster_id": "0104-045638-iaecf5ne",
128+
"timeout_seconds": 0,
129+
"email_notifications": {},
130+
"notification_settings": {
131+
"no_alert_for_skipped_runs": false,
132+
"no_alert_for_canceled_runs": false,
133+
"alert_on_last_attempt": false
134+
},
135+
"webhook_notifications": {}
136+
}
137+
],
138+
"run_as": {
139+
"user_name": "examplemicrosoft.com"
140+
}
141+
}

‎tests/integration/jobdefs/dbx-delta-lts-104-expectations.json‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
[
2+
"databricks://<WORKSPACE_ID>.azuredatabricks.net/jobs/<JOB_ID>",
3+
"databricks://<WORKSPACE_ID>.azuredatabricks.net/jobs/<JOB_ID>/tasks/delta-merge-task",
4+
"databricks://<WORKSPACE_ID>.azuredatabricks.net/notebooks/Shared/examples/delta-in-delta-merge",
5+
"databricks://<WORKSPACE_ID>.azuredatabricks.net/jobs/<JOB_ID>/tasks/delta-merge-package-task",
6+
"databricks://<WORKSPACE_ID>.azuredatabricks.net/notebooks/Shared/examples/delta-in-delta-merge-package",
7+
"databricks://<WORKSPACE_ID>.azuredatabricks.net/jobs/<JOB_ID>/tasks/delta-with-abfss-task",
8+
"databricks://<WORKSPACE_ID>.azuredatabricks.net/notebooks/Shared/examples/delta-in-delta-out-abfss",
9+
"databricks://<WORKSPACE_ID>.azuredatabricks.net/jobs/<JOB_ID>/tasks/delta-fs-task",
10+
"databricks://<WORKSPACE_ID>.azuredatabricks.net/notebooks/Shared/examples/delta-in-delta-out-fs",
11+
"databricks://<WORKSPACE_ID>.azuredatabricks.net/jobs/<JOB_ID>/tasks/delta-fs-col-changes-task",
12+
"databricks://<WORKSPACE_ID>.azuredatabricks.net/notebooks/Shared/examples/delta-in-delta-out-fs-with-col-changes",
13+
"databricks://<WORKSPACE_ID>.azuredatabricks.net/jobs/<JOB_ID>/tasks/delta-mnt-task",
14+
"databricks://<WORKSPACE_ID>.azuredatabricks.net/notebooks/Shared/examples/delta-in-delta-out-mnt"
15+
]

‎tests/integration/jobdefs/dbx-hdfs-core-lts-104-def.json‎

Lines changed: 201 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,201 @@
1+
{
2+
"name": "dbx-hdfs-core-lts-104",
3+
"email_notifications": {
4+
"no_alert_for_skipped_runs": false
5+
},
6+
"webhook_notifications": {},
7+
"timeout_seconds": 0,
8+
"max_concurrent_runs": 1,
9+
"tasks": [
10+
{
11+
"task_key": "abfss-in-abfss-out-task",
12+
"run_if": "ALL_SUCCESS",
13+
"notebook_task": {
14+
"notebook_path": "/Shared/examples/abfss-in-abfss-out",
15+
"source": "WORKSPACE"
16+
},
17+
"existing_cluster_id": "0104-045638-iaecf5ne",
18+
"timeout_seconds": 0,
19+
"email_notifications": {},
20+
"notification_settings": {
21+
"no_alert_for_skipped_runs": false,
22+
"no_alert_for_canceled_runs": false,
23+
"alert_on_last_attempt": false
24+
},
25+
"webhook_notifications": {}
26+
},
27+
{
28+
"task_key": "abfss-in-abfss-out-oauth-task",
29+
"depends_on": [
30+
{
31+
"task_key": "abfss-in-abfss-out-task"
32+
}
33+
],
34+
"run_if": "ALL_DONE",
35+
"notebook_task": {
36+
"notebook_path": "/Shared/examples/abfss-in-abfss-out-oauth",
37+
"source": "WORKSPACE"
38+
},
39+
"existing_cluster_id": "0104-045638-iaecf5ne",
40+
"timeout_seconds": 0,
41+
"email_notifications": {},
42+
"notification_settings": {
43+
"no_alert_for_skipped_runs": false,
44+
"no_alert_for_canceled_runs": false,
45+
"alert_on_last_attempt": false
46+
},
47+
"webhook_notifications": {}
48+
},
49+
{
50+
"task_key": "abfss-in-abfss-out-root-task",
51+
"depends_on": [
52+
{
53+
"task_key": "abfss-in-abfss-out-oauth-task"
54+
}
55+
],
56+
"run_if": "ALL_DONE",
57+
"notebook_task": {
58+
"notebook_path": "/Shared/examples/abfss-in-abfss-out-root",
59+
"source": "WORKSPACE"
60+
},
61+
"existing_cluster_id": "0104-045638-iaecf5ne",
62+
"timeout_seconds": 0,
63+
"email_notifications": {},
64+
"notification_settings": {
65+
"no_alert_for_skipped_runs": false,
66+
"no_alert_for_canceled_runs": false,
67+
"alert_on_last_attempt": false
68+
},
69+
"webhook_notifications": {}
70+
},
71+
{
72+
"task_key": "wasbs-in-wasbs-out-task",
73+
"depends_on": [
74+
{
75+
"task_key": "abfss-in-abfss-out-root-task"
76+
}
77+
],
78+
"run_if": "ALL_DONE",
79+
"notebook_task": {
80+
"notebook_path": "/Shared/examples/wasbs-in-wasbs-out",
81+
"source": "WORKSPACE"
82+
},
83+
"existing_cluster_id": "0104-045638-iaecf5ne",
84+
"timeout_seconds": 0,
85+
"email_notifications": {},
86+
"notification_settings": {
87+
"no_alert_for_skipped_runs": false,
88+
"no_alert_for_canceled_runs": false,
89+
"alert_on_last_attempt": false
90+
},
91+
"webhook_notifications": {}
92+
},
93+
{
94+
"task_key": "wasbs-in-wasbs-out-with-param-task",
95+
"depends_on": [
96+
{
97+
"task_key": "wasbs-in-wasbs-out-task"
98+
}
99+
],
100+
"run_if": "ALL_DONE",
101+
"notebook_task": {
102+
"notebook_path": "/Shared/examples/wasbs-in-wasbs-out-with-param",
103+
"source": "WORKSPACE"
104+
},
105+
"existing_cluster_id": "0104-045638-iaecf5ne",
106+
"timeout_seconds": 0,
107+
"email_notifications": {},
108+
"notification_settings": {
109+
"no_alert_for_skipped_runs": false,
110+
"no_alert_for_canceled_runs": false,
111+
"alert_on_last_attempt": false
112+
},
113+
"webhook_notifications": {}
114+
},
115+
{
116+
"task_key": "intermix-languages-task",
117+
"depends_on": [
118+
{
119+
"task_key": "wasbs-in-wasbs-out-with-param-task"
120+
}
121+
],
122+
"run_if": "ALL_DONE",
123+
"notebook_task": {
124+
"notebook_path": "/Shared/examples/intermix-languages",
125+
"source": "WORKSPACE"
126+
},
127+
"existing_cluster_id": "0104-045638-iaecf5ne",
128+
"timeout_seconds": 0,
129+
"email_notifications": {},
130+
"notification_settings": {
131+
"no_alert_for_skipped_runs": false,
132+
"no_alert_for_canceled_runs": false,
133+
"alert_on_last_attempt": false
134+
},
135+
"webhook_notifications": {}
136+
},
137+
{
138+
"task_key": "mount-task",
139+
"depends_on": [
140+
{
141+
"task_key": "intermix-languages-task"
142+
}
143+
],
144+
"run_if": "ALL_DONE",
145+
"notebook_task": {
146+
"notebook_path": "/Shared/examples/mnt-in-mnt-out",
147+
"source": "WORKSPACE"
148+
},
149+
"existing_cluster_id": "0104-045638-iaecf5ne",
150+
"timeout_seconds": 0,
151+
"email_notifications": {},
152+
"notification_settings": {
153+
"no_alert_for_skipped_runs": false,
154+
"no_alert_for_canceled_runs": false,
155+
"alert_on_last_attempt": false
156+
},
157+
"webhook_notifications": {}
158+
},
159+
{
160+
"task_key": "name-period-task",
161+
"depends_on": [
162+
{
163+
"task_key": "mount-task"
164+
}
165+
],
166+
"run_if": "ALL_DONE",
167+
"notebook_task": {
168+
"notebook_path": "/Shared/examples/name-with-periods",
169+
"source": "WORKSPACE"
170+
},
171+
"existing_cluster_id": "0104-045638-iaecf5ne",
172+
"timeout_seconds": 0,
173+
"email_notifications": {},
174+
"notification_settings": {
175+
"no_alert_for_skipped_runs": false,
176+
"no_alert_for_canceled_runs": false,
177+
"alert_on_last_attempt": false
178+
},
179+
"webhook_notifications": {}
180+
},
181+
{
182+
"task_key": "nested-parent-task",
183+
"depends_on": [
184+
{
185+
"task_key": "name-period-task"
186+
}
187+
],
188+
"run_if": "ALL_DONE",
189+
"notebook_task": {
190+
"notebook_path": "/Shared/examples/nested-parent",
191+
"source": "WORKSPACE"
192+
},
193+
"existing_cluster_id": "0104-045638-iaecf5ne",
194+
"timeout_seconds": 0,
195+
"email_notifications": {}
196+
}
197+
],
198+
"run_as": {
199+
"user_name": "examplemicrosoft.com"
200+
}
201+
}

0 commit comments

Comments
 (0)