Partially implement javascript source map upload command (#9447)

* Implement partial source map upload command

* lint

* move test fixture data

* improve coverage

* refactor

* format

* fix non-determinism in test

Author: andrewbrook

src/archiveFile.spec.ts

@@ -0,0 +1,17 @@
+import * as fs from "fs";
+import { expect } from "chai";
+import { archiveFile } from "./archiveFile";
+import { FIXTURE_DIR } from "./test/fixtures/config-imports";
+
+describe("archiveFile", () => {
+  it("should archive files", async () => {
+    const outputPath = await archiveFile(`${FIXTURE_DIR}/firebase.json`);
+
+    expect(outputPath).to.match(/\.zip$/);
+    expect(fs.existsSync(outputPath)).to.be.true;
+    const archiveStats = fs.statSync(outputPath);
+    const origStats = fs.statSync(`${FIXTURE_DIR}/firebase.json`);
+    expect(archiveStats.size).to.be.greaterThan(0);
+    expect(archiveStats.size).not.to.equal(origStats.size);
+  });
+});

src/archiveFile.ts

@@ -0,0 +1,32 @@
+import * as archiver from "archiver";
+import * as fs from "fs";
+import * as path from "path";
+import * as tmp from "tmp";
+
+export interface ArchiveOptions {
+  /** Optionally override the name of the file being archived */
+  archivedFileName?: string;
+}
+
+/** Archives (zips) a file and returns a path to the tmp output file. */
+export async function archiveFile(filePath: string, options?: ArchiveOptions): Promise<string> {
+  const tmpFile = tmp.fileSync({ postfix: ".zip" }).name;
+  const fileStream = fs.createWriteStream(tmpFile, {
+    flags: "w",
+    encoding: "binary",
+  });
+  const archive = archiver("zip");
+  const name = options?.archivedFileName ?? path.basename(filePath);
+  archive.file(filePath, { name });
+  await pipeAsync(archive, fileStream);
+  return tmpFile;
+}
+
+async function pipeAsync(from: archiver.Archiver, to: fs.WriteStream): Promise<void> {
+  from.pipe(to);
+  await from.finalize();
+  return new Promise((resolve, reject) => {
+    to.on("finish", resolve);
+    to.on("error", reject);
+  });
+}

src/commands/crashlytics-sourcemap-upload.spec.ts

@@ -0,0 +1,101 @@
+import * as chai from "chai";
+import * as sinon from "sinon";
+
+import { command } from "./crashlytics-sourcemap-upload";
+import * as gcs from "../gcp/storage";
+import * as projectUtils from "../projectUtils";
+import * as getProjectNumber from "../getProjectNumber";
+import { FirebaseError } from "../error";
+
+const expect = chai.expect;
+
+const PROJECT_ID = "test-project";
+const PROJECT_NUMBER = "12345";
+const BUCKET_NAME = "test-bucket";
+const DIR_PATH = "src/test/fixtures/mapping-files";
+const FILE_PATH = "src/test/fixtures/mapping-files/mock_mapping.js.map";
+
+describe("crashlytics:sourcemap:upload", () => {
+  let sandbox: sinon.SinonSandbox;
+  let gcsMock: sinon.SinonStubbedInstance<typeof gcs>;
+  let projectUtilsMock: sinon.SinonStubbedInstance<typeof projectUtils>;
+  let getProjectNumberMock: sinon.SinonStubbedInstance<typeof getProjectNumber>;
+
+  beforeEach(() => {
+    sandbox = sinon.createSandbox();
+    gcsMock = sandbox.stub(gcs);
+    projectUtilsMock = sandbox.stub(projectUtils);
+    getProjectNumberMock = sandbox.stub(getProjectNumber);
+
+    projectUtilsMock.needProjectId.returns(PROJECT_ID);
+    getProjectNumberMock.getProjectNumber.resolves(PROJECT_NUMBER);
+    gcsMock.upsertBucket.resolves(BUCKET_NAME);
+    gcsMock.uploadObject.resolves({
+      bucket: BUCKET_NAME,
+      object: "test-object",
+      generation: "1",
+    });
+  });
+
+  afterEach(() => {
+    sandbox.restore();
+  });
+
+  it("should throw an error if no app ID is provided", async () => {
+    await expect(command.runner()("filename", {})).to.be.rejectedWith(
+      FirebaseError,
+      "set --app <appId> to a valid Firebase application id",
+    );
+  });
+
+  it("should create the default cloud storage bucket", async () => {
+    await command.runner()(FILE_PATH, { app: "test-app" });
+    expect(gcsMock.upsertBucket).to.be.calledOnce;
+    const args = gcsMock.upsertBucket.firstCall.args;
+    expect(args[0].req.baseName).to.equal("firebasecrashlytics-sourcemaps-12345-us-central1");
+    expect(args[0].req.location).to.equal("US-CENTRAL1");
+  });
+
+  it("should create a custom cloud storage bucket", async () => {
+    const options = {
+      app: "test-app",
+      bucketLocation: "a-different-LoCaTiOn",
+    };
+    await command.runner()(FILE_PATH, options);
+    expect(gcsMock.upsertBucket).to.be.calledOnce;
+    const args = gcsMock.upsertBucket.firstCall.args;
+    expect(args[0].req.baseName).to.equal(
+      "firebasecrashlytics-sourcemaps-12345-a-different-location",
+    );
+    expect(args[0].req.location).to.equal("A-DIFFERENT-LOCATION");
+  });
+
+  it("should throw an error if the mapping file path is invalid", async () => {
+    expect(command.runner()("invalid/path", { app: "test-app" })).to.be.rejectedWith(
+      FirebaseError,
+      "provide a valid file path or directory",
+    );
+  });
+
+  it("should upload a single mapping file", async () => {
+    await command.runner()(FILE_PATH, { app: "test-app" });
+    expect(gcsMock.uploadObject).to.be.calledOnce;
+    expect(gcsMock.uploadObject).to.be.calledWith(sinon.match.any, BUCKET_NAME);
+    expect(gcsMock.uploadObject.firstCall.args[0].file).to.match(
+      /test-app-default-src-test-fixtures-mapping-files-mock_mapping\.js\.map\.zip/,
+    );
+  });
+
+  it("should find and upload mapping files in a directory", async () => {
+    await command.runner()(DIR_PATH, { app: "test-app" });
+    const expectedFiles = [
+      "test-app-default-src-test-fixtures-mapping-files-mock_mapping.js.map.zip",
+      "test-app-default-src-test-fixtures-mapping-files-subdir-subdir_mock_mapping.js.map.zip",
+    ];
+    const uploadedFiles = gcsMock.uploadObject
+      .getCalls()
+      .map((call) => call.args[0].file)
+      .sort();
+    expect(uploadedFiles).to.deep.equal(expectedFiles);
+  });
+});

src/commands/crashlytics-sourcemap-upload.ts

@@ -0,0 +1,152 @@
+import * as fs from "fs";
+import * as path from "path";
+import { statSync } from "fs-extra";
+import { readdirRecursive } from "../fsAsync";
+import { Command } from "../command";
+import { FirebaseError } from "../error";
+import { logLabeledBullet, logLabeledWarning } from "../utils";
+import { needProjectId } from "../projectUtils";
+import * as gcs from "../gcp/storage";
+import { getProjectNumber } from "../getProjectNumber";
+import { Options } from "../options";
+import { archiveFile } from "../archiveFile";
+
+interface CommandOptions extends Options {
+  app?: string;
+  bucketLocation?: string;
+  appVersion?: string;
+}
+
+export const command = new Command("crashlytics:sourcemap:upload <mappingFiles>")
+  .description("upload javascript source maps to de-minify stack traces")
+  .option("--app <appID>", "the app id of your Firebase app")
+  .option(
+    "--bucket-location <bucketLocation>",
+    'the location of the Google Cloud Storage bucket (default: "US-CENTRAL1"',
+  )
+  .option(
+    "--app-version <appVersion>",
+    "the version of your Firebase app (defaults to Git commit hash, if available)",
+  )
+  .action(async (mappingFiles: string, options: CommandOptions) => {
+    checkGoogleAppID(options);
+
+    // App version
+    const appVersion = getAppVersion();
+
+    // Get project identifiers
+    const projectId = needProjectId(options);
+    const projectNumber = await getProjectNumber(options);
+
+    // Upsert default GCS bucket
+    const bucketName = await upsertBucket(projectId, projectNumber, options);
+
+    // Find and upload mapping files
+    const rootDir = options.projectRoot ?? process.cwd();
+    const filePath = path.relative(rootDir, mappingFiles);
+    let fstat: fs.Stats;
+    try {
+      fstat = statSync(filePath);
+    } catch (e) {
+      throw new FirebaseError(
+        "provide a valid file path or directory to mapping file(s), e.g. app/build/outputs/app.js.map or app/build/outputs",
+      );
+    }
+    if (fstat.isFile()) {
+      await uploadMap(mappingFiles, bucketName, appVersion, options);
+    } else if (fstat.isDirectory()) {
+      logLabeledBullet("crashlytics", "Looking for mapping files in your directory...");
+      const files = (
+        await readdirRecursive({ path: filePath, ignore: ["node_modules", ".git"], maxDepth: 20 })
+      ).filter((f) => f.name.endsWith(".js.map"));
+      await Promise.all(files.map((f) => uploadMap(f.name, bucketName, appVersion, options)));
+    } else {
+      throw new FirebaseError(
+        "provide a valid file path or directory to mapping file(s), e.g. app/build/outputs/app.js.map or app/build/outputs",
+      );
+    }
+
+    // TODO: notify Firebase Telemetry service of the new mapping file
+  });
+
+function checkGoogleAppID(options: CommandOptions): void {
+  if (!options.app) {
+    throw new FirebaseError(
+      "set --app <appId> to a valid Firebase application id, e.g. 1:00000000:android:0000000",
+    );
+  }
+}
+
+function getAppVersion(): string {
+  // TODO: implement app version lookup
+  return "default";
+}
+
+async function upsertBucket(
+  projectId: string,
+  projectNumber: string,
+  options: CommandOptions,
+): Promise<string> {
+  let loc: string = "US-CENTRAL1";
+  if (options.bucketLocation) {
+    loc = (options.bucketLocation as string).toUpperCase();
+  } else {
+    logLabeledBullet(
+      "crashlytics",
+      "No Google Cloud Storage bucket location specified. Defaulting to US-CENTRAL1.",
+    );
+  }
+
+  const baseName = `firebasecrashlytics-sourcemaps-${projectNumber}-${loc.toLowerCase()}`;
+  return await gcs.upsertBucket({
+    product: "crashlytics",
+    createMessage: `Creating Cloud Storage bucket in ${loc} to store Crashlytics source maps at ${baseName}...`,
+    projectId,
+    req: {
+      baseName,
+      purposeLabel: `crashlytics-sourcemaps-${loc.toLowerCase()}`,
+      location: loc,
+      lifecycle: {
+        rule: [
+          {
+            action: {
+              type: "Delete",
+            },
+            condition: {
+              age: 30,
+            },
+          },
+        ],
+      },
+    },
+  });
+}
+
+async function uploadMap(
+  mappingFile: string,
+  bucketName: string,
+  appVersion: string,
+  options: CommandOptions,
+) {
+  logLabeledBullet("crashlytics", `Found mapping file ${mappingFile}...`);
+  try {
+    const filePath = path.relative(options.projectRoot ?? process.cwd(), mappingFile);
+    const tmpArchive = await archiveFile(filePath, { archivedFileName: "mapping.js.map" });
+    const gcsFile = `${options.app}-${appVersion}-${normalizeFileName(mappingFile)}.zip`;
+
+    const { bucket, object } = await gcs.uploadObject(
+      {
+        file: gcsFile,
+        stream: fs.createReadStream(tmpArchive),
+      },
+      bucketName,
+    );
+    logLabeledBullet("crashlytics", `Uploaded to gs://${bucket}/${object}`);
+  } catch (e) {
+    logLabeledWarning("crashlytics", `Failed to upload mapping file ${mappingFile}:\n${e}`);
+  }
+}
+
+function normalizeFileName(fileName: string): string {
+  return fileName.replaceAll(/\//g, "-");
+}

src/commands/index.ts

@@ -52,6 +52,8 @@ export function load(client: any): any {
   client.crashlytics.mappingfile = {};
   client.crashlytics.mappingfile.generateid = loadCommand("crashlytics-mappingfile-generateid");
   client.crashlytics.mappingfile.upload = loadCommand("crashlytics-mappingfile-upload");
+  client.crashlytics.sourcemap = {};
+  client.crashlytics.sourcemap.upload = loadCommand("crashlytics-sourcemap-upload");
   client.database = {};
   client.database.get = loadCommand("database-get");
   client.database.import = loadCommand("database-import");