E2E test

.github/workflows/e2e-cat-app-test.yml

@@ -0,0 +1,103 @@
+name: E2E Cat App Test
+
+on:
+  push:
+    branches: ['main']
+  pull_request:
+    branches: ['main']
+
+env:
+  # Using the same local DB credentials as in docker-compose.yml
+  # These will be used by the PostgreSQL service and the application
+  POSTGRES_USER: manicode_user_local
+  POSTGRES_PASSWORD: secretpassword_local
+  POSTGRES_DB: manicode_db_local
+  # Ensure the application connects to the service container
+  DATABASE_URL: postgres://manicode_user_local:secretpassword_local@localhost:5432/manicode_db_local
+  # Set environment to local to avoid production database requirements for the script
+  NEXT_PUBLIC_CB_ENVIRONMENT: 'local'
+  # Disable BigQuery for testing, as seen in e2e-cat-app-script.ts
+  DISABLE_BIGQUERY: 'true'
+  # Port for the backend service started by the E2E script
+  PORT: 3001
+  # Backend URL for the CLI started by the E2E script
+  CODEBUFF_BACKEND_URL: http://localhost:3001
+
+jobs:
+  e2e-test:
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:16
+        env:
+          POSTGRES_USER: ${{ env.POSTGRES_USER }}
+          POSTGRES_PASSWORD: ${{ env.POSTGRES_PASSWORD }}
+          POSTGRES_DB: ${{ env.POSTGRES_DB }}
+        ports:
+          - 5432:5432
+        # Options to ensure the service is healthy before proceeding
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Set up Bun
+        uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: '1.2.12' # Using version from ci.yml
+
+      - name: Cache dependencies
+        uses: actions/cache@v3
+        with:
+          path: |
+            node_modules
+            */node_modules
+            packages/*/node_modules
+          key: ${{ runner.os }}-deps-${{ hashFiles('**/bun.lockb') }}
+          restore-keys: |
+            ${{ runner.os }}-deps-
+
+      - name: Set Environment Variables
+        run: |
+          echo "${{ secrets.ENV_LOCAL }}" > .env.local
+
+      - name: Install dependencies
+        run: bun install --frozen-lockfile
+
+      - name: Build all workspaces
+        run: bun run build
+
+      - name: Wait for PostgreSQL to be ready
+        run: |
+          echo "Waiting for PostgreSQL to start..."
+          until pg_isready -h localhost -p 5432 -U ${{ env.POSTGRES_USER }}; do
+            sleep 1
+          done
+          echo "PostgreSQL is ready."
+
+      - name: Run database migrations
+        run: bun --cwd common db:migrate
+        env:
+          # Drizzle Kit needs the DATABASE_URL
+          DATABASE_URL: ${{ env.DATABASE_URL }}
+
+      - name: Seed test user
+        run: bun evals/e2e/seed-test-user.ts
+        env:
+          DATABASE_URL: ${{ env.DATABASE_URL }}
+
+      - name: Run E2E Cat App Test Script
+        run: bun evals/e2e/e2e-cat-app-script.ts
+        # The script has its own timeouts, but we can add a general timeout for the step
+        timeout-minutes: 5 # Adjust as needed, the script itself has a 2-minute task timeout + setup time
+
+
+      - name: Open interactive debug shell
+        if: ${{ failure() }}
+        uses: mxschmitt/action-tmate@v3
+        timeout-minutes: 30 # optional guard

backend/src/tools.ts

@@ -157,7 +157,7 @@ Block 3
 // rest of code
 \`\`\`
 
-**YOU MUST ALWAYS INCLUDE DELETION COMMENTS** when removing code blocks, functions, variables, or any other code elements. The fast-apply model cannot understand deletions without these explicit comments.
+**YOU MUST ALWAYS INCLUDE DELETION COMMENTS** when removing code blocks, functions, variables, or any other code elements. The fast-apply model cannot understand deletions without these explicit comments. This applies to every type of file: source files, config files, markdown files, css files, json files, etc, etc.
 
 #### Additional Info
 

evals/e2e-cat-app-script.ts → evals/e2e/e2e-cat-app-script.ts

@@ -3,14 +3,14 @@
 import { spawn, ChildProcess } from 'child_process'
 import fs from 'fs'
 import path from 'path'
-import { sleep } from 'common/util/promise'
+import { sleep } from '../../common/src/util/promise' // Corrected import path
 
 const BACKEND_PORT = 3001
 const BACKEND_READY_TIMEOUT = 30000 // 30 seconds
-const CLI_READY_TIMEOUT = 10000 // 10 seconds
+const CLI_READY_TIMEOUT = 30000 // 30 seconds
 const TASK_COMPLETION_TIMEOUT = 120000 // 2 minutes
 const TEST_DIR = 'cat-app-test'
-const projectRoot = path.resolve(__dirname, '..')
+const projectRoot = path.resolve(__dirname, '../..') // Corrected projectRoot definition
 
 interface ProcessInfo {
   process: ChildProcess
@@ -176,7 +176,7 @@ export class E2ETestRunner {
     console.log('🚀 Starting backend server using package.json script...')
 
     const backendProcess = spawn('bun', ['run', 'start-server'], {
-      cwd: projectRoot, // Run from project root, not evals directory
+      cwd: projectRoot, // Run from actual project root, not evals directory
       detached: true,
       stdio: ['ignore', 'pipe', 'pipe'],
       env: {
@@ -228,7 +228,7 @@ export class E2ETestRunner {
     console.log('🤖 Starting CLI using package.json script...')
 
     const cliProcess = spawn('bun', ['run', 'start-client'], {
-      cwd: path.join(process.cwd(), '..'), // Run from project root, not evals directory
+      cwd: projectRoot, // Corrected: Run from actual project root
       detached: true,
       stdio: ['pipe', 'pipe', 'pipe'],
       env: {

evals/e2e/seed-test-user.ts

@@ -0,0 +1,89 @@
+#!/usr/bin/env bun
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import crypto from 'node:crypto'
+import os from 'node:os'
+
+import db from '../../common/src/db' // Corrected import path
+import * as schema from '../../common/src/db/schema' // Corrected import path
+import { genAuthCode } from '../../common/src/util/credentials' // Corrected import path
+
+async function seedTestUser() {
+  console.log('🌱 Starting test user seeding...')
+
+  const nextAuthSecret = process.env.NEXTAUTH_SECRET
+  if (!nextAuthSecret) {
+    console.error('❌ NEXTAUTH_SECRET environment variable is not set.')
+    process.exit(1)
+  }
+
+  const userId = `test-user-${crypto.randomUUID()}`
+  const userEmail = `test-${crypto.randomBytes(8).toString('hex')}@example.com`
+  const userName = 'E2E Test User'
+  const fingerprintId = `test-fp-${crypto.randomUUID()}`
+  const sessionToken = `test-session-${crypto.randomUUID()}`
+
+  // For the fingerprintHash, we need an expiry. Let's set it far in the future for the test.
+  const expiresAt = new Date(Date.now() + 365 * 24 * 60 * 60 * 1000) // 1 year
+  const fingerprintHash = genAuthCode(fingerprintId, expiresAt.getTime().toString(), nextAuthSecret)
+
+  try {
+    // 1. Create User
+    await db.insert(schema.user).values({
+      id: userId,
+      email: userEmail,
+      name: userName,
+      emailVerified: new Date(), // Mark as verified for simplicity
+    })
+    console.log(`👤 Created user: ${userId} (${userEmail})`)
+
+    // 2. Create Fingerprint
+    await db.insert(schema.fingerprint).values({
+      id: fingerprintId,
+      sig_hash: fingerprintHash, // This hash links the fingerprint to the session/credentials
+      created_at: new Date(),
+    })
+    console.log(`👆 Created fingerprint: ${fingerprintId}`)
+
+    // 3. Create Session
+    await db.insert(schema.session).values({
+      sessionToken: sessionToken,
+      userId: userId,
+      expires: expiresAt,
+      fingerprint_id: fingerprintId,
+    })
+    console.log(`🔑 Created session: ${sessionToken} for user ${userId}`)
+
+    // 4. Create credentials.json
+    const credentials = {
+      default: {
+        id: userId,
+        email: userEmail,
+        name: userName,
+        authToken: sessionToken,
+        fingerprintId: fingerprintId,
+        fingerprintHash: fingerprintHash,
+      },
+    }
+
+    // Determine credentials path (mimicking npm-app/src/credentials.ts logic for 'local' env)
+    const configDir = path.join(os.homedir(), '.config', 'manicode-local')
+    const credentialsPath = path.join(configDir, 'credentials.json')
+
+    await fs.mkdir(configDir, { recursive: true })
+    await fs.writeFile(credentialsPath, JSON.stringify(credentials, null, 2))
+    console.log(`📝 Wrote credentials to: ${credentialsPath}`)
+
+    console.log('✅ Test user seeding complete!')
+  } catch (error) {
+    console.error('❌ Error during test user seeding:', error)
+    process.exit(1)
+  }
+}
+
+if (require.main === module) {
+  seedTestUser().catch((err) => {
+    console.error('Unhandled error in seedTestUser:', err)
+    process.exit(1)
+  })
+}

evals/package.json

@@ -7,7 +7,7 @@
     "test:manifold": "bun test manifold.test.ts",
     "test:pglite": "bun test pglite-demo.test.ts",
     "test:swe-bench": "bun test swe-bench.test.ts",
-    "test:e2e-cat-app": "bun run e2e-cat-app-script.ts",
+    "test:e2e-cat-app": "bun run e2e/e2e-cat-app-script.ts",
     "typecheck": "tsc --noEmit",
     "gen-git-evals": "bun run git-evals/gen-evals.ts",
     "run-git-evals": "bun run git-evals/run-git-evals.ts",

npm-app/src/workers/checkpoint-worker.ts → npm-app/src/checkpoint-worker.ts

@@ -1,7 +1,7 @@
 import { parentPort as maybeParentPort } from 'worker_threads'
 
-import { restoreFileState, storeFileState } from '../checkpoints/file-manager'
-import { setProjectRoot } from '../project-files'
+import { restoreFileState, storeFileState } from './checkpoints/file-manager'
+import { setProjectRoot } from './project-files'
 
 /**
  * Message format for worker operations

npm-app/src/checkpoints/checkpoint-manager.ts

@@ -96,8 +96,8 @@ export class CheckpointManager {
       // NOTE: Uses the built workers/checkpoint-worker.js within dist.
       // So you need to run `bun run build` before running locally.
       const workerPath = __filename.endsWith('.ts')
-        ? join(__dirname, '../../dist', 'workers/checkpoint-worker.js')
-        : join(__dirname, '../workers/checkpoint-worker.js')
+        ? join(__dirname, '../../dist', 'checkpoint-worker.js')
+        : join(__dirname, '../checkpoint-worker.js')
       this.worker = new Worker(workerPath)
     }
     return this.worker

npm-app/src/workers/project-context.ts → npm-app/src/project-context.ts

@@ -2,8 +2,8 @@ import { parentPort as maybeParentPort } from 'worker_threads'
 
 import { getAllFilePaths } from 'common/project-file-tree'
 
-import { initializeCheckpointFileManager } from '../checkpoints/file-manager'
-import { getProjectFileContext, setProjectRoot } from '../project-files'
+import { initializeCheckpointFileManager } from './checkpoints/file-manager'
+import { getProjectFileContext, setProjectRoot } from './project-files'
 
 if (maybeParentPort) {
   const parentPort = maybeParentPort

npm-app/src/project-files.ts

@@ -182,8 +182,8 @@ export function initProjectFileContextWithWorker(
   // NOTE: Uses the built worker-script-project-context.js within dist.
   // So you need to run `bun run build` before running locally.
   const workerPath = __filename.endsWith('.ts')
-    ? path.join(__dirname, '..', 'dist', 'workers/project-context.js')
-    : path.join(__dirname, 'workers/project-context.js')
+    ? path.join(__dirname, '..', 'dist', 'project-context.js')
+    : path.join(__dirname, 'project-context.js')
   const worker = new Worker(workerPath as any)
 
   worker.postMessage({ dir })