diff --git a/extensions/ql-vscode/src/codeql-cli/cli.ts b/extensions/ql-vscode/src/codeql-cli/cli.ts index 6ead0b338fc..fa5aa9fcd85 100644 --- a/extensions/ql-vscode/src/codeql-cli/cli.ts +++ b/extensions/ql-vscode/src/codeql-cli/cli.ts @@ -9,6 +9,7 @@ import { SemVer } from "semver"; import type { Readable } from "stream"; import tk from "tree-kill"; import type { CancellationToken, Disposable, Uri } from "vscode"; +import { dir } from "tmp-promise"; import type { BqrsInfo, @@ -202,9 +203,7 @@ interface BqrsDecodeOptions { entities?: string[]; } -type OnLineCallback = ( - line: string, -) => Promise | string | undefined; +type OnLineCallback = (line: string) => Promise; type VersionChangedListener = ( newVersionAndFeatures: VersionAndFeatures | undefined, @@ -368,12 +367,11 @@ export class CodeQLCliServer implements Disposable { */ private async launchProcess(): Promise { const codeQlPath = await this.getCodeQlPath(); - const args = []; - if (shouldDebugCliServer()) { - args.push( - "-J=-agentlib:jdwp=transport=dt_socket,address=localhost:9012,server=n,suspend=y,quiet=y", - ); - } + const args = shouldDebugCliServer() + ? [ + "-J=-agentlib:jdwp=transport=dt_socket,address=localhost:9012,server=n,suspend=y,quiet=y", + ] + : []; return spawnServer( codeQlPath, @@ -399,15 +397,11 @@ export class CodeQLCliServer implements Disposable { } this.commandInProcess = true; try { - //Launch the process if it doesn't exist - if (!this.process) { - this.process = await this.launchProcess(); - } - // Grab the process so that typescript know that it is always defined. - const process = this.process; + // Launch the process if it doesn't exist + this.process ??= await this.launchProcess(); // Compute the full args array - const args = command.concat(LOGGING_FLAGS).concat(commandArgs); + const args = command.concat(LOGGING_FLAGS, commandArgs); const argsString = args.join(" "); // If we are running silently, we don't want to print anything to the console. if (!silent) { @@ -416,7 +410,7 @@ export class CodeQLCliServer implements Disposable { ); } try { - return await this.handleProcessOutput(process, { + return await this.handleProcessOutput(this.process, { handleNullTerminator: true, onListenStart: (process) => { // Write the command followed by a null terminator. @@ -451,7 +445,7 @@ export class CodeQLCliServer implements Disposable { ): Promise { const codeqlPath = await this.getCodeQlPath(); - const args = command.concat(LOGGING_FLAGS).concat(commandArgs); + const args = command.concat(LOGGING_FLAGS, commandArgs); const argsString = args.join(" "); // If we are running silently, we don't want to print anything to the console. @@ -569,16 +563,15 @@ export class CodeQLCliServer implements Disposable { stdoutBuffers.push(newData); - if (handleNullTerminator) { + if ( + handleNullTerminator && // If the buffer ends in '0' then exit. // We don't have to check the middle as no output will be written after the null until // the next command starts - if ( - newData.length > 0 && - newData.readUInt8(newData.length - 1) === 0 - ) { - resolve(); - } + newData.length > 0 && + newData.readUInt8(newData.length - 1) === 0 + ) { + resolve(); } }; stderrListener = (newData: Buffer) => { @@ -693,9 +686,7 @@ export class CodeQLCliServer implements Disposable { */ private runNext(): void { const callback = this.commandQueue.shift(); - if (callback) { - callback(); - } + callback?.(); } /** @@ -813,7 +804,7 @@ export class CodeQLCliServer implements Disposable { * is false or not specified, this option is ignored. * @returns The contents of the command's stdout, if the command succeeded. */ - runCodeQlCliCommand( + private runCodeQlCliCommand( command: string[], commandArgs: string[], description: string, @@ -825,9 +816,7 @@ export class CodeQLCliServer implements Disposable { token, }: RunOptions = {}, ): Promise { - if (progressReporter) { - progressReporter.report({ message: description }); - } + progressReporter?.report({ message: description }); if (runInNewProcess) { return this.runCodeQlCliInNewProcess( @@ -874,18 +863,17 @@ export class CodeQLCliServer implements Disposable { * @param progressReporter Used to output progress messages, e.g. to the status bar. * @returns The contents of the command's stdout, if the command succeeded. */ - async runJsonCodeQlCliCommand( + private async runJsonCodeQlCliCommand( command: string[], commandArgs: string[], description: string, { addFormat = true, ...runOptions }: JsonRunOptions = {}, ): Promise { - let args: string[] = []; - if (addFormat) { + const args = [ // Add format argument first, in case commandArgs contains positional parameters. - args = args.concat(["--format", "json"]); - } - args = args.concat(commandArgs); + ...(addFormat ? ["--format", "json"] : []), + ...commandArgs, + ]; const result = await this.runCodeQlCliCommand( command, args, @@ -922,7 +910,7 @@ export class CodeQLCliServer implements Disposable { * @param runOptions Options for running the command. * @returns The contents of the command's stdout, if the command succeeded. */ - async runJsonCodeQlCliCommandWithAuthentication( + private async runJsonCodeQlCliCommandWithAuthentication( command: string[], commandArgs: string[], description: string, @@ -1226,8 +1214,8 @@ export class CodeQLCliServer implements Disposable { } /** - * Gets the results from a bqrs. - * @param bqrsPath The path to the bqrs. + * Gets the results from a bqrs file. + * @param bqrsPath The path to the bqrs file. * @param resultSet The result set to get. * @param options Optional BqrsDecodeOptions arguments */ @@ -1240,11 +1228,11 @@ export class CodeQLCliServer implements Disposable { `--entities=${entities.join(",")}`, "--result-set", resultSet, - ] - .concat(pageSize ? ["--rows", pageSize.toString()] : []) - .concat(offset ? ["--start-at", offset.toString()] : []) - .concat([bqrsPath]); - return await this.runJsonCodeQlCliCommand( + ...(pageSize ? ["--rows", pageSize.toString()] : []), + ...(offset ? ["--start-at", offset.toString()] : []), + bqrsPath, + ]; + return this.runJsonCodeQlCliCommand( ["bqrs", "decode"], subcommandArgs, "Reading bqrs data", @@ -1252,18 +1240,49 @@ export class CodeQLCliServer implements Disposable { } /** - * Gets all results from a bqrs. - * @param bqrsPath The path to the bqrs. + * Gets all results from a bqrs file. + * @param bqrsPath The path to the bqrs file. */ async bqrsDecodeAll(bqrsPath: string): Promise { - return await this.runJsonCodeQlCliCommand( + return this.runJsonCodeQlCliCommand( ["bqrs", "decode"], [bqrsPath], "Reading all bqrs data", ); } - async runInterpretCommand( + /** Gets the difference between two bqrs files. */ + async bqrsDiff( + bqrsPath1: string, + bqrsPath2: string, + options?: BqrsDiffOptions, + ): Promise<{ + uniquePath1: string; + uniquePath2: string; + path: string; + cleanup: () => Promise; + }> { + const { path, cleanup } = await dir({ unsafeCleanup: true }); + const uniquePath1 = join(path, "left.bqrs"); + const uniquePath2 = join(path, "right.bqrs"); + await this.runCodeQlCliCommand( + ["bqrs", "diff"], + [ + "--left", + uniquePath1, + "--right", + uniquePath2, + "--retain-result-sets", + options?.retainResultSets?.join(",") ?? "", + bqrsPath1, + bqrsPath2, + ], + "Diffing bqrs files", + ); + return { uniquePath1, uniquePath2, path, cleanup }; + } + + private async runInterpretCommand( format: string, additonalArgs: string[], metadata: QueryMetadata, @@ -1278,21 +1297,22 @@ export class CodeQLCliServer implements Disposable { format, // Forward all of the query metadata. ...Object.entries(metadata).map(([key, value]) => `-t=${key}=${value}`), - ].concat(additonalArgs); - if (sourceInfo !== undefined) { - args.push( - "--source-archive", - sourceInfo.sourceArchive, - "--source-location-prefix", - sourceInfo.sourceLocationPrefix, - ); - } - - args.push("--threads", this.cliConfig.numberThreads.toString()); - - args.push("--max-paths", this.cliConfig.maxPaths.toString()); + ...additonalArgs, + ...(sourceInfo !== undefined + ? [ + "--source-archive", + sourceInfo.sourceArchive, + "--source-location-prefix", + sourceInfo.sourceLocationPrefix, + ] + : []), + "--threads", + this.cliConfig.numberThreads.toString(), + "--max-paths", + this.cliConfig.maxPaths.toString(), + resultsPath, + ]; - args.push(resultsPath); await this.runCodeQlCliCommand( ["bqrs", "interpret"], args, @@ -1797,7 +1817,7 @@ export class CodeQLCliServer implements Disposable { * Spawns a child server process using the CodeQL CLI * and attaches listeners to it. * - * @param config The configuration containing the path to the CLI. + * @param codeqlPath The configuration containing the path to the CLI. * @param name Name of the server being started, to be shown in log and error messages. * @param command The `codeql` command to be run, provided as an array of command/subcommand names. * @param commandArgs The arguments to pass to the `codeql` command. @@ -1823,9 +1843,9 @@ export function spawnServer( // Start the server process. const base = codeqlPath; const argsString = args.join(" "); - if (progressReporter !== undefined) { - progressReporter.report({ message: `Starting ${name}` }); - } + + progressReporter?.report({ message: `Starting ${name}` }); + void logger.log(`Starting ${name} using CodeQL CLI: ${base} ${argsString}`); const child = spawnChildProcess(base, args); if (!child || !child.pid) { @@ -1859,9 +1879,8 @@ export function spawnServer( child.stdout.on("data", stdoutListener); } - if (progressReporter !== undefined) { - progressReporter.report({ message: `Started ${name}` }); - } + progressReporter?.report({ message: `Started ${name}` }); + void logger.log(`${name} started on PID: ${child.pid}`); return child; } diff --git a/extensions/ql-vscode/src/codeql-cli/query-language.ts b/extensions/ql-vscode/src/codeql-cli/query-language.ts index d5cef76e697..1fe27474092 100644 --- a/extensions/ql-vscode/src/codeql-cli/query-language.ts +++ b/extensions/ql-vscode/src/codeql-cli/query-language.ts @@ -19,7 +19,7 @@ export async function findLanguage( cliServer: CodeQLCliServer, queryUri: Uri | undefined, ): Promise { - const uri = queryUri || window.activeTextEditor?.document.uri; + const uri = queryUri ?? window.activeTextEditor?.document.uri; if (uri !== undefined) { try { const queryInfo = await cliServer.resolveQueryByLanguage( diff --git a/extensions/ql-vscode/src/common/bqrs-raw-results-mapper.ts b/extensions/ql-vscode/src/common/bqrs-raw-results-mapper.ts index 10bd5fe9e33..ec6a7bf5ee4 100644 --- a/extensions/ql-vscode/src/common/bqrs-raw-results-mapper.ts +++ b/extensions/ql-vscode/src/common/bqrs-raw-results-mapper.ts @@ -27,20 +27,13 @@ export function bqrsToResultSet( schema: BqrsResultSetSchema, chunk: DecodedBqrsChunk, ): RawResultSet { - const name = schema.name; - const totalRowCount = schema.rows; - - const columns = schema.columns.map(mapColumn); - - const rows = chunk.tuples.map( - (tuple): Row => tuple.map((cell): CellValue => mapCellValue(cell)), - ); - const resultSet: RawResultSet = { - name, - totalRowCount, - columns, - rows, + name: schema.name, + totalRowCount: schema.rows, + columns: schema.columns.map(mapColumn), + rows: chunk.tuples.map( + (tuple): Row => tuple.map((cell): CellValue => mapCellValue(cell)), + ), }; if (chunk.next) { diff --git a/extensions/ql-vscode/src/compare/compare-view.ts b/extensions/ql-vscode/src/compare/compare-view.ts index 93538f52e6b..ff3018a409c 100644 --- a/extensions/ql-vscode/src/compare/compare-view.ts +++ b/extensions/ql-vscode/src/compare/compare-view.ts @@ -45,6 +45,14 @@ interface ComparePair { commonResultSetNames: readonly string[]; } +function findSchema(info: BqrsInfo, name: string) { + const schema = info["result-sets"].find((schema) => schema.name === name); + if (schema === undefined) { + throw new Error(`Schema ${name} not found.`); + } + return schema; +} + export class CompareView extends AbstractWebview< ToCompareViewMessage, FromCompareViewMessage @@ -160,8 +168,10 @@ export class CompareView extends AbstractWebview< currentResultSetDisplayName, fromResultSetName, toResultSetName, - } = await this.findResultSetsToCompare( - this.comparePair, + } = await findResultSetNames( + this.comparePair.fromInfo, + this.comparePair.toInfo, + this.comparePair.commonResultSetNames, selectedResultSetName, ); if (currentResultSetDisplayName) { @@ -169,7 +179,12 @@ export class CompareView extends AbstractWebview< let message: string | undefined; try { if (currentResultSetName === ALERTS_TABLE_NAME) { - result = await this.compareInterpretedResults(this.comparePair); + result = await compareInterpretedResults( + this.databaseManager, + this.cliServer, + this.comparePair.from, + this.comparePair.to, + ); } else { result = await this.compareResults( this.comparePair, @@ -336,31 +351,6 @@ export class CompareView extends AbstractWebview< } } - private async findResultSetsToCompare( - { fromInfo, toInfo, commonResultSetNames }: ComparePair, - selectedResultSetName: string | undefined, - ) { - const { - currentResultSetName, - currentResultSetDisplayName, - fromResultSetName, - toResultSetName, - } = await findResultSetNames( - fromInfo, - toInfo, - commonResultSetNames, - selectedResultSetName, - ); - - return { - commonResultSetNames, - currentResultSetName, - currentResultSetDisplayName, - fromResultSetName, - toResultSetName, - }; - } - private async changeTable(newResultSetName: string) { await this.showResultsInternal(newResultSetName); } @@ -370,12 +360,7 @@ export class CompareView extends AbstractWebview< resultSetName: string, resultsPath: string, ): Promise { - const schema = bqrsInfo["result-sets"].find( - (schema) => schema.name === resultSetName, - ); - if (!schema) { - throw new Error(`Schema ${resultSetName} not found.`); - } + const schema = findSchema(bqrsInfo, resultSetName); const chunk = await this.cliServer.bqrsDecode(resultsPath, resultSetName); return bqrsToResultSet(schema, chunk); } @@ -385,32 +370,66 @@ export class CompareView extends AbstractWebview< fromResultSetName: string, toResultSetName: string, ): Promise { - const [fromResultSet, toResultSet] = await Promise.all([ - this.getResultSet( - fromInfo.schemas, - fromResultSetName, - from.completedQuery.query.resultsPath, - ), - this.getResultSet( - toInfo.schemas, - toResultSetName, - to.completedQuery.query.resultsPath, - ), - ]); + const fromPath = from.completedQuery.query.resultsPath; + const toPath = to.completedQuery.query.resultsPath; - return resultsDiff(fromResultSet, toResultSet); - } + const fromSchema = findSchema(fromInfo.schemas, fromResultSetName); + const toSchema = findSchema(toInfo.schemas, toResultSetName); - private async compareInterpretedResults({ - from, - to, - }: ComparePair): Promise { - return compareInterpretedResults( - this.databaseManager, - this.cliServer, - from, - to, - ); + if (fromSchema.columns.length !== toSchema.columns.length) { + throw new Error("CodeQL Compare: Columns do not match."); + } + if (fromSchema.rows === 0) { + throw new Error("CodeQL Compare: Source query has no results."); + } + if (toSchema.rows === 0) { + throw new Error("CodeQL Compare: Target query has no results."); + } + + // If the result set names are the same, we use `bqrs diff`. This is more + // efficient, but we can't use it in general as it does not support + // comparing different result sets. + if (fromResultSetName === toResultSetName) { + const { uniquePath1, uniquePath2, cleanup } = + await this.cliServer.bqrsDiff(fromPath, toPath); + try { + const info1 = await this.cliServer.bqrsInfo(uniquePath1); + const info2 = await this.cliServer.bqrsInfo(uniquePath2); + + // We avoid decoding the results sets if there is no overlap + if ( + fromSchema.rows === findSchema(info1, fromResultSetName).rows && + toSchema.rows === findSchema(info2, toResultSetName).rows + ) { + throw new Error( + "CodeQL Compare: No overlap between the selected queries.", + ); + } + + const fromUniqueResults = bqrsToResultSet( + fromSchema, + await this.cliServer.bqrsDecode(uniquePath1, fromResultSetName), + ); + const toUniqueResults = bqrsToResultSet( + toSchema, + await this.cliServer.bqrsDecode(uniquePath2, toResultSetName), + ); + return { + kind: "raw", + columns: fromUniqueResults.columns, + from: fromUniqueResults.rows, + to: toUniqueResults.rows, + }; + } finally { + await cleanup(); + } + } else { + const [fromResultSet, toResultSet] = await Promise.all([ + this.getResultSet(fromInfo.schemas, fromResultSetName, fromPath), + this.getResultSet(toInfo.schemas, toResultSetName, toPath), + ]); + return resultsDiff(fromResultSet, toResultSet); + } } private async openQuery(kind: "from" | "to") { diff --git a/extensions/ql-vscode/src/compare/interpreted-results.ts b/extensions/ql-vscode/src/compare/interpreted-results.ts index f98913a15e8..9cf76eff34c 100644 --- a/extensions/ql-vscode/src/compare/interpreted-results.ts +++ b/extensions/ql-vscode/src/compare/interpreted-results.ts @@ -1,7 +1,5 @@ import { Uri } from "vscode"; -import type { Log } from "sarif"; -import { pathExists } from "fs-extra"; -import { sarifParser } from "../common/sarif-parser"; +import { join } from "path"; import type { CompletedLocalQueryInfo } from "../query-results"; import type { DatabaseManager } from "../databases/local-databases"; import type { CodeQLCliServer } from "../codeql-cli/cli"; @@ -9,16 +7,6 @@ import type { InterpretedQueryCompareResult } from "../common/interface-types"; import { sarifDiff } from "./sarif-diff"; -async function getInterpretedResults( - interpretedResultsPath: string, -): Promise { - if (!(await pathExists(interpretedResultsPath))) { - return undefined; - } - - return await sarifParser(interpretedResultsPath); -} - export async function compareInterpretedResults( databaseManager: DatabaseManager, cliServer: CodeQLCliServer, @@ -34,37 +22,65 @@ export async function compareInterpretedResults( ); } - const [fromResultSet, toResultSet, sourceLocationPrefix] = await Promise.all([ - getInterpretedResults( - fromQuery.completedQuery.query.interpretedResultsPath, - ), - getInterpretedResults(toQuery.completedQuery.query.interpretedResultsPath), - database.getSourceLocationPrefix(cliServer), - ]); + const { uniquePath1, uniquePath2, path, cleanup } = await cliServer.bqrsDiff( + fromQuery.completedQuery.query.resultsPath, + toQuery.completedQuery.query.resultsPath, + { retainResultSets: ["nodes", "edges", "subpaths"] }, + ); + try { + const sarifOutput1 = join(path, "from.sarif"); + const sarifOutput2 = join(path, "to.sarif"); - if (!fromResultSet || !toResultSet) { - throw new Error( - "Could not find interpreted results for one or both queries.", + const sourceLocationPrefix = + await database.getSourceLocationPrefix(cliServer); + const sourceArchiveUri = database.sourceArchive; + const sourceInfo = + sourceArchiveUri === undefined + ? undefined + : { + sourceArchive: sourceArchiveUri.fsPath, + sourceLocationPrefix, + }; + + const fromResultSet = await cliServer.interpretBqrsSarif( + toQuery.completedQuery.query.metadata!, + uniquePath1, + sarifOutput1, + sourceInfo, + ); + const toResultSet = await cliServer.interpretBqrsSarif( + toQuery.completedQuery.query.metadata!, + uniquePath2, + sarifOutput2, + sourceInfo, ); - } - const fromResults = fromResultSet.runs[0].results; - const toResults = toResultSet.runs[0].results; + if (!fromResultSet || !toResultSet) { + throw new Error( + "Could not find interpreted results for one or both queries.", + ); + } - if (!fromResults) { - throw new Error("No results found in the 'from' query."); - } + const fromResults = fromResultSet.runs[0].results; + const toResults = toResultSet.runs[0].results; - if (!toResults) { - throw new Error("No results found in the 'to' query."); - } + if (!fromResults) { + throw new Error("CodeQL Compare: Source query has no results."); + } + + if (!toResults) { + throw new Error("CodeQL Compare: Target query has no results."); + } - const { from, to } = sarifDiff(fromResults, toResults); + const { from, to } = sarifDiff(fromResults, toResults); - return { - kind: "interpreted", - sourceLocationPrefix, - from, - to, - }; + return { + kind: "interpreted", + sourceLocationPrefix, + from, + to, + }; + } finally { + await cleanup(); + } } diff --git a/extensions/ql-vscode/src/compare/resultsDiff.ts b/extensions/ql-vscode/src/compare/resultsDiff.ts index 01efa5dbc04..a0b240a05ac 100644 --- a/extensions/ql-vscode/src/compare/resultsDiff.ts +++ b/extensions/ql-vscode/src/compare/resultsDiff.ts @@ -23,18 +23,6 @@ export default function resultsDiff( fromResults: RawResultSet, toResults: RawResultSet, ): RawQueryCompareResult { - if (fromResults.columns.length !== toResults.columns.length) { - throw new Error("CodeQL Compare: Columns do not match."); - } - - if (!fromResults.rows.length) { - throw new Error("CodeQL Compare: Source query has no results."); - } - - if (!toResults.rows.length) { - throw new Error("CodeQL Compare: Target query has no results."); - } - const results: RawQueryCompareResult = { kind: "raw", columns: fromResults.columns, diff --git a/extensions/ql-vscode/src/compare/sarif-diff.ts b/extensions/ql-vscode/src/compare/sarif-diff.ts index 158c63ae487..50a71286b12 100644 --- a/extensions/ql-vscode/src/compare/sarif-diff.ts +++ b/extensions/ql-vscode/src/compare/sarif-diff.ts @@ -97,14 +97,6 @@ function toCanonicalResult(result: Result): Result { * 2. If the queries are 100% disjoint */ export function sarifDiff(fromResults: Result[], toResults: Result[]) { - if (!fromResults.length) { - throw new Error("CodeQL Compare: Source query has no results."); - } - - if (!toResults.length) { - throw new Error("CodeQL Compare: Target query has no results."); - } - const canonicalFromResults = fromResults.map(toCanonicalResult); const canonicalToResults = toResults.map(toCanonicalResult); @@ -113,13 +105,6 @@ export function sarifDiff(fromResults: Result[], toResults: Result[]) { to: arrayDiff(canonicalToResults, canonicalFromResults), }; - if ( - fromResults.length === diffResults.from.length && - toResults.length === diffResults.to.length - ) { - throw new Error("CodeQL Compare: No overlap between the selected queries."); - } - // We don't want to return the canonical results, we want to return the original results. // We can retrieve this by finding the index of the canonical result in the canonical results // and then using that index to find the original result. This is possible because we know that diff --git a/extensions/ql-vscode/src/view/compare/Compare.tsx b/extensions/ql-vscode/src/view/compare/Compare.tsx index 18412bac5f2..d52ffc5a5e4 100644 --- a/extensions/ql-vscode/src/view/compare/Compare.tsx +++ b/extensions/ql-vscode/src/view/compare/Compare.tsx @@ -46,7 +46,7 @@ export function Compare(_: Record): React.JSX.Element { null, ); - const message = comparison?.message || "Empty comparison"; + const message = comparison?.message ?? "Empty comparison"; const hasRows = comparison?.result && (comparison.result.to.length || comparison.result.from.length); @@ -170,7 +170,7 @@ export function Compare(_: Record): React.JSX.Element { queryInfo={queryInfo} comparison={comparison} userSettings={userSettings} - > + /> ) : ( {message} )} diff --git a/extensions/ql-vscode/src/view/compare/CompareTable.tsx b/extensions/ql-vscode/src/view/compare/CompareTable.tsx index ccc805aeda7..18f390002a8 100644 --- a/extensions/ql-vscode/src/view/compare/CompareTable.tsx +++ b/extensions/ql-vscode/src/view/compare/CompareTable.tsx @@ -31,6 +31,10 @@ const Table = styled.table` } `; +async function openQuery(kind: "from" | "to") { + vscode.postMessage({ t: "openQuery", kind }); +} + export default function CompareTable({ queryInfo, comparison, @@ -38,13 +42,6 @@ export default function CompareTable({ }: Props) { const result = comparison.result!; - async function openQuery(kind: "from" | "to") { - vscode.postMessage({ - t: "openQuery", - kind, - }); - } - return (