Skip to content

Commit 8f193e6

Browse files
tushar-gupta-1995tushar.gupta
tushar-gupta-1995
and
tushar.gupta
authored
feat: Add validate dependency chain flag to validate broken dependencies without running the pipeline (#1657)
* feat: add --validate-dependency-chain flag to verify all local job needs exist * feat: add --validate-dependency-chain flag to verify all local job needs exist * feat: add --validate-dependency-chain flag to verify all local job needs exist * feat: add --validate-dependency-chain flag to verify all local job needs exist * feat: add --validate-dependency-chain flag to verify all local job needs exist * feat: add --validate-dependency-chain flag to verify all local job needs exist * feat: add --validate-dependency-chain flag to verify all local job needs exist * feat: add --validate-dependency-chain flag to verify all local job needs exist * feat: add --validate-dependency-chain flag to verify all local job needs exist * feat: add --validate-dependency-chain flag to verify all local job needs exist * feat: add --validate-dependency-chain flag to verify all local job needs exist * feat: add --validate-dependency-chain flag to verify all local job needs exist --------- Co-authored-by: tushar.gupta <tushar.gupta@indexexchange.com>
1 parent d435154 commit 8f193e6

File tree

6 files changed

+174
-0
lines changed

6 files changed

+174
-0
lines changed

‎src/argv.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -266,6 +266,10 @@ export class Argv {
266266
return this.map.get("preview") ?? false;
267267
}
268268

269+
get validateDependencyChain (): boolean {
270+
return this.map.get("validateDependencyChain") ?? false;
271+
}
272+
269273
get shellIsolation (): boolean {
270274
// TODO: default to true in 5.x.x
271275
return this.map.get("shellIsolation") ?? false;

‎src/commander.ts‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -272,4 +272,24 @@ export class Commander {
272272
});
273273
}
274274

275+
static validateDependencyChain (parser: Parser) {
276+
const allJobs = parser.jobs;
277+
// This is only the jobs that will actually run
278+
const activeJobs = allJobs.filter(j => j.when !== "never");
279+
const stages = parser.stages;
280+
// This will throw an assertion errror if the dependency chain is broken due to needs keyword on specific events without having to run the full pipeline
281+
Executor.getStartCandidates(allJobs, stages, activeJobs, []);
282+
283+
const activeJobNames = new Set(activeJobs.map(job => job.name));
284+
// This willl throw an assertion error if the dependency chain is broken due to dependencies keyword (a job depending on artifacts from a job that will never run) without having to run the full pipeline
285+
for (const job of activeJobs) {
286+
if (job.dependencies) {
287+
for (const dependency of job.dependencies) {
288+
if (!activeJobNames.has(dependency)) {
289+
throw new AssertionError({message: chalk`{blueBright ${dependency}} is when:never, but its depended on by {blueBright ${job.name}}`});
290+
}
291+
}
292+
}
293+
}
294+
}
275295
}

‎src/handler.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,11 @@ export async function handler (args: any, writeStreams: WriteStreams, jobs: Job[
5454
const pipelineIid = await state.getPipelineIid(cwd, stateDir);
5555
parser = await Parser.create(argv, writeStreams, pipelineIid, jobs);
5656
Commander.runList(parser, writeStreams, argv.listAll);
57+
} else if (argv.validateDependencyChain) {
58+
const pipelineIid = await state.getPipelineIid(cwd, stateDir);
59+
parser = await Parser.create(argv, writeStreams, pipelineIid, jobs);
60+
Commander.validateDependencyChain(parser);
61+
writeStreams.stdout(chalk`{green ✓ All job dependencies are valid}\n`);
5762
} else if (argv.listJson) {
5863
const pipelineIid = await state.getPipelineIid(cwd, stateDir);
5964
parser = await Parser.create(argv, writeStreams, pipelineIid, jobs);

‎src/index.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,11 @@ process.on("SIGUSR2", async () => await cleanupJobResources(jobs));
9595
description: "List job information in csv format, when:never included",
9696
requiresArg: false,
9797
})
98+
.option("validate-dependency-chain", {
99+
type: "boolean",
100+
description: "Validate that jobs needed or dependent by active jobs under specified conditions are also active without actually running the jobs. Uses fail-fast approach - stops at first validation error for both 'needs' and 'dependencies' keywords. If validation fails, use --list flag to see which jobs will run under specified conditions",
101+
requiresArg: false,
102+
})
98103
.option("preview", {
99104
type: "boolean",
100105
description: "Print YML with defaults, includes, extends and reference's expanded",

‎tests/test-cases/validate-dependency-chain/.gitlab-ci.yml‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
---
2+
stages:
3+
- build
4+
- test
5+
6+
alpine-guest:
7+
image: nginxinc/nginx-unprivileged:alpine3.18
8+
needs: ["alpine-root"]
9+
script:
10+
- stat -c "%a %n %u %g" one.txt
11+
- stat -c "%a %n %u %g" script.sh
12+
rules:
13+
- if: '$RUN_ALL == "true"'
14+
- if: '$RUN_SINGLE == "alpine-root"'
15+
- if: '$RUN_SINGLE == "alpine-guest"'
16+
17+
alpine-root:
18+
needs: ["kaniko-root"]
19+
image: nginx:alpine3.18
20+
rules:
21+
- if: '$RUN_ALL == "true"'
22+
- if: '$RUN_SINGLE == "alpine-root"'
23+
- if: '$RUN_SINGLE == "alpine-guest"'
24+
when: never
25+
script:
26+
- stat -c "%a %n %u %g" one.txt
27+
- stat -c "%a %n %u %g" script.sh
28+
29+
kaniko-root:
30+
image:
31+
name: gcr.io/kaniko-project/executor:v1.23.0-debug
32+
entrypoint: [""]
33+
rules:
34+
- if: '$RUN_ALL == "true"'
35+
script:
36+
- stat -c "%a %n %u %g" one.txt
37+
- stat -c "%a %n %u %g" script.sh
38+
39+
40+
kaniko-guest:
41+
image:
42+
name: gcr.io/kaniko-project/executor:v1.23.0-debug
43+
entrypoint: [""]
44+
rules:
45+
- if: '$RUN_ALL == "true"'
46+
script:
47+
- stat -c "%a %n %u %g" one.txt
48+
- stat -c "%a %n %u %g" script.sh
49+
50+
51+
build-job-1:
52+
stage: build
53+
script: echo "build 1"
54+
artifacts:
55+
paths: [build1.txt]
56+
rules:
57+
- if: '$RUN_ALL == "true"'
58+
- if: '$TEST_DEPENDENCIES == "true"'
59+
60+
61+
test-job:
62+
stage: test
63+
dependencies: [build-job-1]
64+
rules:
65+
- if: '$RUN_ALL == "true"'
66+
script: echo "testing"
67+
68+
build-job-2:
69+
stage: build
70+
script: echo "build 2"
71+
artifacts:
72+
paths: [build2.txt]
73+
rules:
74+
- if: '$RUN_ALL == "true"'
75+
76+
broken-dependencies-job:
77+
stage: test
78+
dependencies: [build-job-1, build-job-2]
79+
rules:
80+
- if: '$TEST_DEPENDENCIES == "true"'
81+
script: echo "This has broken dependencies"

‎tests/test-cases/validate-dependency-chain/integration.test.ts‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
import {WriteStreamsMock} from "../../../src/write-streams.js";
2+
import {handler} from "../../../src/handler.js";
3+
import chalk from "chalk";
4+
import {initSpawnSpy} from "../../mocks/utils.mock.js";
5+
import {WhenStatics} from "../../mocks/when-statics.js";
6+
7+
beforeAll(() => {
8+
initSpawnSpy(WhenStatics.all);
9+
});
10+
11+
describe("validate-dependency-chain", () => {
12+
test("should pass when all dependencies are valid", async () => {
13+
const writeStreams = new WriteStreamsMock();
14+
await handler({
15+
cwd: "tests/test-cases/validate-dependency-chain",
16+
validateDependencyChain: true,
17+
variable: ["RUN_ALL=true"],
18+
}, writeStreams);
19+
20+
const output = writeStreams.stdoutLines.join("\n");
21+
expect(output).toContain(chalk`{green ✓ All job dependencies are valid}`);
22+
23+
// Check that there are no validation errors in stderr (only info messages)
24+
const validationErrors = writeStreams.stderrLines.filter(line =>
25+
line.includes("Dependency chain validation will fail with event"),
26+
);
27+
expect(validationErrors.length).toBe(0);
28+
});
29+
30+
test("should fail when dependency chain is broken due to a non-existent job", async () => {
31+
const writeStreams = new WriteStreamsMock();
32+
33+
await expect(handler({
34+
cwd: "tests/test-cases/validate-dependency-chain",
35+
validateDependencyChain: true,
36+
variable: ["RUN_SINGLE=alpine-root"],
37+
}, writeStreams)).rejects.toThrow(chalk`{blueBright kaniko-root} is when:never, but its needed by {blueBright alpine-root}`);
38+
});
39+
40+
test("should fail when dependency chain is broken due to a job that never runs", async () => {
41+
const writeStreams = new WriteStreamsMock();
42+
43+
await expect(handler({
44+
cwd: "tests/test-cases/validate-dependency-chain",
45+
validateDependencyChain: true,
46+
variable: ["RUN_SINGLE=alpine-guest"],
47+
}, writeStreams)).rejects.toThrow(chalk`{blueBright alpine-root} is when:never, but its needed by {blueBright alpine-guest}`);
48+
});
49+
50+
test("should fail when dependencies keyword references missing artifact jobs", async () => {
51+
const writeStreams = new WriteStreamsMock();
52+
53+
await expect(handler({
54+
cwd: "tests/test-cases/validate-dependency-chain",
55+
validateDependencyChain: true,
56+
variable: ["TEST_DEPENDENCIES=true"],
57+
}, writeStreams)).rejects.toThrow(chalk`{blueBright build-job-2} is when:never, but its depended on by {blueBright broken-dependencies-job}`);
58+
});
59+
});

0 commit comments

Comments
 (0)