From 91e5653a6f6b82555a266cb17ce6454a65f8411c Mon Sep 17 00:00:00 2001 From: James Grugett Date: Wed, 4 Jun 2025 20:47:19 -0700 Subject: [PATCH 01/11] Add deletion comments for every type of file --- backend/src/tools.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/tools.ts b/backend/src/tools.ts index bbbb7bcff..9cdd51c43 100644 --- a/backend/src/tools.ts +++ b/backend/src/tools.ts @@ -157,7 +157,7 @@ Block 3 // rest of code \`\`\` -**YOU MUST ALWAYS INCLUDE DELETION COMMENTS** when removing code blocks, functions, variables, or any other code elements. The fast-apply model cannot understand deletions without these explicit comments. +**YOU MUST ALWAYS INCLUDE DELETION COMMENTS** when removing code blocks, functions, variables, or any other code elements. The fast-apply model cannot understand deletions without these explicit comments. This applies to every type of file: source files, config files, markdown files, css files, json files, etc, etc. #### Additional Info From 1148a196f8f83ab5f0221c012b57b8662c983792 Mon Sep 17 00:00:00 2001 From: James Grugett Date: Wed, 4 Jun 2025 21:46:08 -0700 Subject: [PATCH 02/11] Add github action for e2e. Script to init script --- .github/workflows/e2e-cat-app-test.yml | 93 ++++++++++++++++++++++++++ evals/{ => e2e}/e2e-cat-app-script.ts | 8 +-- evals/e2e/seed-test-user.ts | 89 ++++++++++++++++++++++++ evals/package.json | 2 +- 4 files changed, 187 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/e2e-cat-app-test.yml rename evals/{ => e2e}/e2e-cat-app-script.ts (98%) create mode 100644 evals/e2e/seed-test-user.ts diff --git a/.github/workflows/e2e-cat-app-test.yml b/.github/workflows/e2e-cat-app-test.yml new file mode 100644 index 000000000..ea1a447b8 --- /dev/null +++ b/.github/workflows/e2e-cat-app-test.yml @@ -0,0 +1,93 @@ +name: E2E Cat App Test + +on: + push: + branches: ['main'] + pull_request: + branches: ['main'] + +env: + # Using the same local DB credentials as in docker-compose.yml + # These will be used by the PostgreSQL service and the application + POSTGRES_USER: manicode_user_local + POSTGRES_PASSWORD: secretpassword_local + POSTGRES_DB: manicode_db_local + # Ensure the application connects to the service container + DATABASE_URL: postgres://manicode_user_local:secretpassword_local@localhost:5432/manicode_db_local + # Set environment to local to avoid production database requirements for the script + NEXT_PUBLIC_CB_ENVIRONMENT: 'local' + # Disable BigQuery for testing, as seen in e2e-cat-app-script.ts + DISABLE_BIGQUERY: 'true' + # Port for the backend service started by the E2E script + PORT: 3001 + # Backend URL for the CLI started by the E2E script + CODEBUFF_BACKEND_URL: http://localhost:3001 + # NEXTAUTH_SECRET for the seeding script + NEXTAUTH_SECRET: ${{ secrets.NEXTAUTH_SECRET_E2E_TEST }} # Use a dedicated secret for this + +jobs: + e2e-test: + runs-on: ubuntu-latest + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: ${{ env.POSTGRES_USER }} + POSTGRES_PASSWORD: ${{ env.POSTGRES_PASSWORD }} + POSTGRES_DB: ${{ env.POSTGRES_DB }} + ports: + - 5432:5432 + # Options to ensure the service is healthy before proceeding + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Set up Bun + uses: oven-sh/setup-bun@v2 + with: + bun-version: '1.2.12' # Using version from ci.yml + + - name: Cache dependencies + uses: actions/cache@v3 + with: + path: | + node_modules + */node_modules + packages/*/node_modules + key: ${{ runner.os }}-deps-${{ hashFiles('**/bun.lockb') }} + restore-keys: | + ${{ runner.os }}-deps- + + - name: Install dependencies + run: bun install --frozen-lockfile + + - name: Wait for PostgreSQL to be ready + run: | + echo "Waiting for PostgreSQL to start..." + until pg_isready -h localhost -p 5432 -U ${{ env.POSTGRES_USER }}; do + sleep 1 + done + echo "PostgreSQL is ready." + + - name: Run database migrations + run: bun --cwd common db:migrate + env: + # Drizzle Kit needs the DATABASE_URL + DATABASE_URL: ${{ env.DATABASE_URL }} + + - name: Seed test user + run: bun evals/e2e/seed-test-user.ts + env: + DATABASE_URL: ${{ env.DATABASE_URL }} + NEXTAUTH_SECRET: ${{ env.NEXTAUTH_SECRET }} + + - name: Run E2E Cat App Test Script + run: bun evals/e2e/e2e-cat-app-script.ts + # The script has its own timeouts, but we can add a general timeout for the step + timeout-minutes: 5 # Adjust as needed, the script itself has a 2-minute task timeout + setup time diff --git a/evals/e2e-cat-app-script.ts b/evals/e2e/e2e-cat-app-script.ts similarity index 98% rename from evals/e2e-cat-app-script.ts rename to evals/e2e/e2e-cat-app-script.ts index 672000584..07027eabc 100644 --- a/evals/e2e-cat-app-script.ts +++ b/evals/e2e/e2e-cat-app-script.ts @@ -3,14 +3,14 @@ import { spawn, ChildProcess } from 'child_process' import fs from 'fs' import path from 'path' -import { sleep } from 'common/util/promise' +import { sleep } from '../../common/src/util/promise' // Corrected import path const BACKEND_PORT = 3001 const BACKEND_READY_TIMEOUT = 30000 // 30 seconds const CLI_READY_TIMEOUT = 10000 // 10 seconds const TASK_COMPLETION_TIMEOUT = 120000 // 2 minutes const TEST_DIR = 'cat-app-test' -const projectRoot = path.resolve(__dirname, '..') +const projectRoot = path.resolve(__dirname, '../..') // Corrected projectRoot definition interface ProcessInfo { process: ChildProcess @@ -176,7 +176,7 @@ export class E2ETestRunner { console.log('🚀 Starting backend server using package.json script...') const backendProcess = spawn('bun', ['run', 'start-server'], { - cwd: projectRoot, // Run from project root, not evals directory + cwd: projectRoot, // Run from actual project root, not evals directory detached: true, stdio: ['ignore', 'pipe', 'pipe'], env: { @@ -228,7 +228,7 @@ export class E2ETestRunner { console.log('🤖 Starting CLI using package.json script...') const cliProcess = spawn('bun', ['run', 'start-client'], { - cwd: path.join(process.cwd(), '..'), // Run from project root, not evals directory + cwd: projectRoot, // Corrected: Run from actual project root detached: true, stdio: ['pipe', 'pipe', 'pipe'], env: { diff --git a/evals/e2e/seed-test-user.ts b/evals/e2e/seed-test-user.ts new file mode 100644 index 000000000..9e41dabfa --- /dev/null +++ b/evals/e2e/seed-test-user.ts @@ -0,0 +1,89 @@ +#!/usr/bin/env bun +import fs from 'node:fs/promises' +import path from 'node:path' +import crypto from 'node:crypto' +import os from 'node:os' + +import db from '../../common/src/db' // Corrected import path +import * as schema from '../../common/src/db/schema' // Corrected import path +import { genAuthCode } from '../../common/src/util/credentials' // Corrected import path + +async function seedTestUser() { + console.log('🌱 Starting test user seeding...') + + const nextAuthSecret = process.env.NEXTAUTH_SECRET + if (!nextAuthSecret) { + console.error('❌ NEXTAUTH_SECRET environment variable is not set.') + process.exit(1) + } + + const userId = `test-user-${crypto.randomUUID()}` + const userEmail = `test-${crypto.randomBytes(8).toString('hex')}@example.com` + const userName = 'E2E Test User' + const fingerprintId = `test-fp-${crypto.randomUUID()}` + const sessionToken = `test-session-${crypto.randomUUID()}` + + // For the fingerprintHash, we need an expiry. Let's set it far in the future for the test. + const expiresAt = new Date(Date.now() + 365 * 24 * 60 * 60 * 1000) // 1 year + const fingerprintHash = genAuthCode(fingerprintId, expiresAt.getTime().toString(), nextAuthSecret) + + try { + // 1. Create User + await db.insert(schema.user).values({ + id: userId, + email: userEmail, + name: userName, + emailVerified: new Date(), // Mark as verified for simplicity + }) + console.log(`👤 Created user: ${userId} (${userEmail})`) + + // 2. Create Fingerprint + await db.insert(schema.fingerprint).values({ + id: fingerprintId, + sig_hash: fingerprintHash, // This hash links the fingerprint to the session/credentials + created_at: new Date(), + }) + console.log(`👆 Created fingerprint: ${fingerprintId}`) + + // 3. Create Session + await db.insert(schema.session).values({ + sessionToken: sessionToken, + userId: userId, + expires: expiresAt, + fingerprint_id: fingerprintId, + }) + console.log(`🔑 Created session: ${sessionToken} for user ${userId}`) + + // 4. Create credentials.json + const credentials = { + default: { + id: userId, + email: userEmail, + name: userName, + authToken: sessionToken, + fingerprintId: fingerprintId, + fingerprintHash: fingerprintHash, + }, + } + + // Determine credentials path (mimicking npm-app/src/credentials.ts logic for 'local' env) + const configDir = path.join(os.homedir(), '.config', 'manicode-local') + const credentialsPath = path.join(configDir, 'credentials.json') + + await fs.mkdir(configDir, { recursive: true }) + await fs.writeFile(credentialsPath, JSON.stringify(credentials, null, 2)) + console.log(`📝 Wrote credentials to: ${credentialsPath}`) + + console.log('✅ Test user seeding complete!') + } catch (error) { + console.error('❌ Error during test user seeding:', error) + process.exit(1) + } +} + +if (require.main === module) { + seedTestUser().catch((err) => { + console.error('Unhandled error in seedTestUser:', err) + process.exit(1) + }) +} diff --git a/evals/package.json b/evals/package.json index 46f3331e7..ac77019ae 100644 --- a/evals/package.json +++ b/evals/package.json @@ -7,7 +7,7 @@ "test:manifold": "bun test manifold.test.ts", "test:pglite": "bun test pglite-demo.test.ts", "test:swe-bench": "bun test swe-bench.test.ts", - "test:e2e-cat-app": "bun run e2e-cat-app-script.ts", + "test:e2e-cat-app": "bun run e2e/e2e-cat-app-script.ts", "typecheck": "tsc --noEmit", "gen-git-evals": "bun run git-evals/gen-evals.ts", "run-git-evals": "bun run git-evals/run-git-evals.ts", From 6e35ebfc8824acb3b496fef0af27c4de00cb3f56 Mon Sep 17 00:00:00 2001 From: James Grugett Date: Wed, 4 Jun 2025 21:53:01 -0700 Subject: [PATCH 03/11] Add env variables. Add interactive shell --- .github/workflows/e2e-cat-app-test.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/e2e-cat-app-test.yml b/.github/workflows/e2e-cat-app-test.yml index ea1a447b8..b0debdf12 100644 --- a/.github/workflows/e2e-cat-app-test.yml +++ b/.github/workflows/e2e-cat-app-test.yml @@ -67,6 +67,10 @@ jobs: - name: Install dependencies run: bun install --frozen-lockfile + - name: Set Environment Variables + run: | + echo "${{ secrets.ENV_LOCAL }}" > .env.local + - name: Wait for PostgreSQL to be ready run: | echo "Waiting for PostgreSQL to start..." @@ -91,3 +95,9 @@ jobs: run: bun evals/e2e/e2e-cat-app-script.ts # The script has its own timeouts, but we can add a general timeout for the step timeout-minutes: 5 # Adjust as needed, the script itself has a 2-minute task timeout + setup time + + + - name: Open interactive debug shell + if: ${{ failure() }} + uses: mxschmitt/action-tmate@v3 + timeout-minutes: 30 # optional guard \ No newline at end of file From 0d1c221f4794fb057561d5c0769451c5f5f3402a Mon Sep 17 00:00:00 2001 From: James Grugett Date: Wed, 4 Jun 2025 22:01:23 -0700 Subject: [PATCH 04/11] Remove next auth secret from github actions file --- .github/workflows/e2e-cat-app-test.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/e2e-cat-app-test.yml b/.github/workflows/e2e-cat-app-test.yml index b0debdf12..87f1acb13 100644 --- a/.github/workflows/e2e-cat-app-test.yml +++ b/.github/workflows/e2e-cat-app-test.yml @@ -22,8 +22,6 @@ env: PORT: 3001 # Backend URL for the CLI started by the E2E script CODEBUFF_BACKEND_URL: http://localhost:3001 - # NEXTAUTH_SECRET for the seeding script - NEXTAUTH_SECRET: ${{ secrets.NEXTAUTH_SECRET_E2E_TEST }} # Use a dedicated secret for this jobs: e2e-test: @@ -89,7 +87,6 @@ jobs: run: bun evals/e2e/seed-test-user.ts env: DATABASE_URL: ${{ env.DATABASE_URL }} - NEXTAUTH_SECRET: ${{ env.NEXTAUTH_SECRET }} - name: Run E2E Cat App Test Script run: bun evals/e2e/e2e-cat-app-script.ts From 021d4ae509b9d89db12a735495167ac20ad43005 Mon Sep 17 00:00:00 2001 From: James Grugett Date: Wed, 4 Jun 2025 22:03:56 -0700 Subject: [PATCH 05/11] Increase cli timeout --- evals/e2e/e2e-cat-app-script.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evals/e2e/e2e-cat-app-script.ts b/evals/e2e/e2e-cat-app-script.ts index 07027eabc..4dc633fd5 100644 --- a/evals/e2e/e2e-cat-app-script.ts +++ b/evals/e2e/e2e-cat-app-script.ts @@ -7,7 +7,7 @@ import { sleep } from '../../common/src/util/promise' // Corrected import path const BACKEND_PORT = 3001 const BACKEND_READY_TIMEOUT = 30000 // 30 seconds -const CLI_READY_TIMEOUT = 10000 // 10 seconds +const CLI_READY_TIMEOUT = 30000 // 30 seconds const TASK_COMPLETION_TIMEOUT = 120000 // 2 minutes const TEST_DIR = 'cat-app-test' const projectRoot = path.resolve(__dirname, '../..') // Corrected projectRoot definition From a9215252a65396cca0f3f457cee8297dcbdb822b Mon Sep 17 00:00:00 2001 From: James Grugett Date: Wed, 4 Jun 2025 22:08:35 -0700 Subject: [PATCH 06/11] Run build --- .github/workflows/e2e-cat-app-test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/e2e-cat-app-test.yml b/.github/workflows/e2e-cat-app-test.yml index 87f1acb13..635be2de1 100644 --- a/.github/workflows/e2e-cat-app-test.yml +++ b/.github/workflows/e2e-cat-app-test.yml @@ -65,6 +65,9 @@ jobs: - name: Install dependencies run: bun install --frozen-lockfile + - name: Build all workspaces + run: bun run build + - name: Set Environment Variables run: | echo "${{ secrets.ENV_LOCAL }}" > .env.local From 1f2d57151ad7b81db9a5c6d8ccaa411dfcc186be Mon Sep 17 00:00:00 2001 From: James Grugett Date: Wed, 4 Jun 2025 22:11:30 -0700 Subject: [PATCH 07/11] Set env vars first --- .github/workflows/e2e-cat-app-test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/e2e-cat-app-test.yml b/.github/workflows/e2e-cat-app-test.yml index 635be2de1..373e06834 100644 --- a/.github/workflows/e2e-cat-app-test.yml +++ b/.github/workflows/e2e-cat-app-test.yml @@ -62,16 +62,16 @@ jobs: restore-keys: | ${{ runner.os }}-deps- + - name: Set Environment Variables + run: | + echo "${{ secrets.ENV_LOCAL }}" > .env.local + - name: Install dependencies run: bun install --frozen-lockfile - name: Build all workspaces run: bun run build - - name: Set Environment Variables - run: | - echo "${{ secrets.ENV_LOCAL }}" > .env.local - - name: Wait for PostgreSQL to be ready run: | echo "Waiting for PostgreSQL to start..." From 56b1b3cad11090b564368220c2e3b90598783869 Mon Sep 17 00:00:00 2001 From: James Grugett Date: Wed, 4 Jun 2025 22:54:25 -0700 Subject: [PATCH 08/11] hacky build fix --- npm-app/project.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/npm-app/project.json b/npm-app/project.json index ce34452ed..25bf9e2ab 100644 --- a/npm-app/project.json +++ b/npm-app/project.json @@ -24,7 +24,8 @@ "mkdir -p dist/code-map", "cp -r ../packages/code-map/dist/* dist/code-map", "mkdir -p dist/utils/__tests__", - "cp -r src/utils/__tests__/*.txt dist/utils/__tests__" + "cp -r src/utils/__tests__/*.txt dist/utils/__tests__", + "sed -i '' 's/require(\"common\\/project-file-tree\")/require(\"..\\/common\\/project-file-tree\")/g' dist/workers/project-context.js" ], "cwd": "npm-app", "parallel": false From d2639b782b1ccfe3ae3e529ec4ae712e58641d50 Mon Sep 17 00:00:00 2001 From: James Grugett Date: Wed, 4 Jun 2025 23:11:05 -0700 Subject: [PATCH 09/11] fix hack --- npm-app/project.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/npm-app/project.json b/npm-app/project.json index 25bf9e2ab..62645d634 100644 --- a/npm-app/project.json +++ b/npm-app/project.json @@ -25,7 +25,7 @@ "cp -r ../packages/code-map/dist/* dist/code-map", "mkdir -p dist/utils/__tests__", "cp -r src/utils/__tests__/*.txt dist/utils/__tests__", - "sed -i '' 's/require(\"common\\/project-file-tree\")/require(\"..\\/common\\/project-file-tree\")/g' dist/workers/project-context.js" + "sed -i '' 's|require(\"common/project-file-tree\")|require(\"../common/project-file-tree\")|g' dist/workers/project-context.js" ], "cwd": "npm-app", "parallel": false From 6ca48f71135633b4db8e29108c156c785b9ca7b1 Mon Sep 17 00:00:00 2001 From: James Grugett Date: Fri, 6 Jun 2025 13:24:05 -0700 Subject: [PATCH 10/11] Try moving worker threads out of workers dir --- npm-app/project.json | 2 +- npm-app/src/{workers => }/checkpoint-worker.ts | 4 ++-- npm-app/src/checkpoints/checkpoint-manager.ts | 4 ++-- npm-app/src/{workers => }/project-context.ts | 4 ++-- npm-app/src/project-files.ts | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) rename npm-app/src/{workers => }/checkpoint-worker.ts (93%) rename npm-app/src/{workers => }/project-context.ts (81%) diff --git a/npm-app/project.json b/npm-app/project.json index 62645d634..e117a177c 100644 --- a/npm-app/project.json +++ b/npm-app/project.json @@ -25,7 +25,7 @@ "cp -r ../packages/code-map/dist/* dist/code-map", "mkdir -p dist/utils/__tests__", "cp -r src/utils/__tests__/*.txt dist/utils/__tests__", - "sed -i '' 's|require(\"common/project-file-tree\")|require(\"../common/project-file-tree\")|g' dist/workers/project-context.js" + "sed -i '' 's|require(\"common/project-file-tree\")|require(\"../common/project-file-tree\")|g' dist/project-context.js" ], "cwd": "npm-app", "parallel": false diff --git a/npm-app/src/workers/checkpoint-worker.ts b/npm-app/src/checkpoint-worker.ts similarity index 93% rename from npm-app/src/workers/checkpoint-worker.ts rename to npm-app/src/checkpoint-worker.ts index 8189b94b5..5d4546fd7 100644 --- a/npm-app/src/workers/checkpoint-worker.ts +++ b/npm-app/src/checkpoint-worker.ts @@ -1,7 +1,7 @@ import { parentPort as maybeParentPort } from 'worker_threads' -import { restoreFileState, storeFileState } from '../checkpoints/file-manager' -import { setProjectRoot } from '../project-files' +import { restoreFileState, storeFileState } from './checkpoints/file-manager' +import { setProjectRoot } from './project-files' /** * Message format for worker operations diff --git a/npm-app/src/checkpoints/checkpoint-manager.ts b/npm-app/src/checkpoints/checkpoint-manager.ts index 1d83f3a65..2ea2419d6 100644 --- a/npm-app/src/checkpoints/checkpoint-manager.ts +++ b/npm-app/src/checkpoints/checkpoint-manager.ts @@ -96,8 +96,8 @@ export class CheckpointManager { // NOTE: Uses the built workers/checkpoint-worker.js within dist. // So you need to run `bun run build` before running locally. const workerPath = __filename.endsWith('.ts') - ? join(__dirname, '../../dist', 'workers/checkpoint-worker.js') - : join(__dirname, '../workers/checkpoint-worker.js') + ? join(__dirname, '../../dist', 'checkpoint-worker.js') + : join(__dirname, '../checkpoint-worker.js') this.worker = new Worker(workerPath) } return this.worker diff --git a/npm-app/src/workers/project-context.ts b/npm-app/src/project-context.ts similarity index 81% rename from npm-app/src/workers/project-context.ts rename to npm-app/src/project-context.ts index 6e5151fe1..b98cfd2c0 100644 --- a/npm-app/src/workers/project-context.ts +++ b/npm-app/src/project-context.ts @@ -2,8 +2,8 @@ import { parentPort as maybeParentPort } from 'worker_threads' import { getAllFilePaths } from 'common/project-file-tree' -import { initializeCheckpointFileManager } from '../checkpoints/file-manager' -import { getProjectFileContext, setProjectRoot } from '../project-files' +import { initializeCheckpointFileManager } from './checkpoints/file-manager' +import { getProjectFileContext, setProjectRoot } from './project-files' if (maybeParentPort) { const parentPort = maybeParentPort diff --git a/npm-app/src/project-files.ts b/npm-app/src/project-files.ts index 3214738a1..cfc47e551 100644 --- a/npm-app/src/project-files.ts +++ b/npm-app/src/project-files.ts @@ -182,8 +182,8 @@ export function initProjectFileContextWithWorker( // NOTE: Uses the built worker-script-project-context.js within dist. // So you need to run `bun run build` before running locally. const workerPath = __filename.endsWith('.ts') - ? path.join(__dirname, '..', 'dist', 'workers/project-context.js') - : path.join(__dirname, 'workers/project-context.js') + ? path.join(__dirname, '..', 'dist', 'project-context.js') + : path.join(__dirname, 'project-context.js') const worker = new Worker(workerPath as any) worker.postMessage({ dir }) From bfd29c08ccc6168c12fc6404f0810d95c6c5bc6b Mon Sep 17 00:00:00 2001 From: James Grugett Date: Fri, 6 Jun 2025 13:28:20 -0700 Subject: [PATCH 11/11] revert sed thing --- npm-app/project.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/npm-app/project.json b/npm-app/project.json index e117a177c..ce34452ed 100644 --- a/npm-app/project.json +++ b/npm-app/project.json @@ -24,8 +24,7 @@ "mkdir -p dist/code-map", "cp -r ../packages/code-map/dist/* dist/code-map", "mkdir -p dist/utils/__tests__", - "cp -r src/utils/__tests__/*.txt dist/utils/__tests__", - "sed -i '' 's|require(\"common/project-file-tree\")|require(\"../common/project-file-tree\")|g' dist/project-context.js" + "cp -r src/utils/__tests__/*.txt dist/utils/__tests__" ], "cwd": "npm-app", "parallel": false