feat: switch to new filepicker model (test top-2 filepicker)

Author: venkr

backend/src/find-files/request-files-prompt.ts

@@ -263,22 +263,17 @@ async function getRelevantFiles(
   const start = performance.now()
   let coreMessages = coreMessagesWithSystem(messagesWithPrompt, system)
 
-  if (costMode === 'experimental') {
-    coreMessages = coreMessages
-      .map((msg, i) => {
-        if (msg.role === 'assistant' && i !== coreMessages.length - 1) {
-          return castAssistantMessage(msg)
-        } else {
-          return msg
-        }
-      })
-      .filter((msg) => msg !== null)
-  }
-  // This finetunedModel is used for the promptFlashWithFallbacks call
-  const finetunedModel =
-    costMode === 'experimental'
-      ? finetunedVertexModels.ft_filepicker_010
-      : finetunedVertexModels.ft_filepicker_005
+  // Converts assistant messages to user messages for finetuned model
+  coreMessages = coreMessages
+    .map((msg, i) => {
+      if (msg.role === 'assistant' && i !== coreMessages.length - 1) {
+        return castAssistantMessage(msg)
+      } else {
+        return msg
+      }
+    })
+    .filter((msg) => msg !== null)
+  const finetunedModel = finetunedVertexModels.ft_filepicker_010
 
   let response = await promptFlashWithFallbacks(coreMessages, {
     clientSessionId,

common/src/constants.ts

@@ -197,6 +197,7 @@ export const finetunedVertexModels = {
   ft_filepicker_topk_002: '1694861989844615168',
   ft_filepicker_010: '3808739064941641728',
   ft_filepicker_010_epoch_2: '6231675664466968576',
+  ft_filepicker_topk_003: '1502192368286171136',
 } as const
 export const finetunedVertexModelNames: Record<string, string> = {
   [finetunedVertexModels.ft_filepicker_003]: 'ft_filepicker_003',
@@ -208,6 +209,7 @@ export const finetunedVertexModelNames: Record<string, string> = {
   [finetunedVertexModels.ft_filepicker_010]: 'ft_filepicker_010',
   [finetunedVertexModels.ft_filepicker_010_epoch_2]:
     'ft_filepicker_010_epoch_2',
+  [finetunedVertexModels.ft_filepicker_topk_003]: 'ft_filepicker_topk_003',
 }
 export type FinetunedVertexModel =
   (typeof finetunedVertexModels)[keyof typeof finetunedVertexModels]

scripts/ft-file-selection/README.md

@@ -105,7 +105,7 @@
 `ft_filepicker_010`
 - Gemini Endpoint ID: 3808739064941641728
 - Finetune date: June 2, 2025
-- Finetune file: ??? TODO
+- Finetune file: gemini-tune-data-048.jsonl
 - Finetune samples: 2648 messages
 - Tokens: 109M
 - Epochs: 3
@@ -121,6 +121,21 @@
   - ~$980 for finetuning ($3.00/1M tokens * 3 epochs * 109M tokens)
   - Reuses relabel inputs from ft_filepicker_008 - no additional labeling cost
 
+`ft_filepicker_topk_003`
+- Gemini Endpoint ID: 1502192368286171136
+- Finetune date: Jun 5, 2025
+- Finetune file: gemini-tune-data-048-top2.jsonl, available in the GCS bucket
+- Finetune samples: 2648 messages
+- Tokens: 109M
+- Epochs: 2
+- Base model: gemini-2.0-flash-001
+- Distilled model: claude-opus-4-20250514 
+- Notes: 
+  - Combines characteristics of `ft_filepicker_topk_002` and `ft_filepicker_010`, ie: uses top-2, with 3 lines of dashes at the end, but also blobbifies message history.
+- Est costs:
+  - ~$660 for finetuning ($3.00/1M tokens * 2 epochs * 109M tokens)
+  - Mostly reuses relabel inputs from other Claude runs - no additional cost anticipated.
+
 ## Scripts
 
 Contains a variety of scripts for inspecting and processing finetuning data from BigQuery.

scripts/ft-file-selection/print-recent-relabels.ts

@@ -22,6 +22,8 @@ User ID: ${relabel.user_id}
 Agent Step ID: ${relabel.agent_step_id}
 Created at: ${JSON.stringify(relabel.created_at)}
 Payload: ${JSON.stringify(relabel.payload).slice(0, 100)}...
+Model: ${relabel.model}
+Output: ${JSON.stringify(relabel.payload.output)}
 --------------------------------`)
     })
   } catch (error) {

scripts/ft-file-selection/relabel-for-offline-scoring.ts

@@ -25,7 +25,7 @@ import { isValidationSample } from './collect-tuning-data'
 const isProd = process.argv.includes('--prod')
 const DATASET = isProd ? 'codebuff_data' : 'codebuff_data_dev'
 const MAX_PARALLEL = 5 // Maximum number of traces to process in parallel for relabeling
-const LIMIT = 4000 // Total limit of traces to process
+const LIMIT = 400 // Total limit of traces to process
 const START_CURSOR = '2025-05-31T00:00:00.000Z' // User-provided start cursor or default
 
 const GROUND_TRUTH_MODEL = 'claude-opus-4-20250514-with-full-file-context-new'
@@ -37,6 +37,7 @@ const MODELS = [
   finetunedVertexModels.ft_filepicker_008,
   finetunedVertexModels.ft_filepicker_010,
   finetunedVertexModels.ft_filepicker_010_epoch_2,
+  finetunedVertexModels.ft_filepicker_topk_003,
 ] as const
 
 const modelDescriptions = {
@@ -52,6 +53,8 @@ const modelDescriptions = {
     'ft_filepicker_010: 109M tokens, 3 epochs, same as ft_filepicker_008 but with assistant messages converted to user messages',
   [finetunedVertexModels.ft_filepicker_010_epoch_2]:
     'ft_filepicker_010_epoch_2: 109M tokens, 2 epochs, same as ft_filepicker_008 but with assistant messages converted to user messages',
+  [finetunedVertexModels.ft_filepicker_topk_003]:
+    'ft_filepicker_topk_003: 109M tokens, 2 epochs, only uses top-2 files with 3 lines of dashes at the end',
 }
 
 async function getFilteredValidationBundles(): Promise<