feat(artifact): add file UID param in retrieval task (#1046)

Because

- Filename isn't a unique identifier in catalogs anymore

This commit

- Adds a file UID param in the retrieval task in the Instill Artifact
component.
- The filename param is kept temporarily for backwards compatibility
while clients update their recipes.
- Adds a file UID field in the retrieved chunk object.
- Removes the database schema version from the config.

Author: jvallesm

cmd/migration/main.go

@@ -107,7 +107,7 @@ func main() {
 	codeMigrator, cleanup := initCodeMigrator(ctx)
 	defer cleanup()
 
-	if err := runMigration(dsn, databaseConfig.Version, codeMigrator.Migrate); err != nil {
+	if err := runMigration(dsn, database.TargetSchemaVersion, codeMigrator.Migrate); err != nil {
 		log.With(zap.Error(err)).Fatal("Running migration")
 	}
 }

config/config.go

@@ -120,7 +120,6 @@ type DatabaseConfig struct {
 		ReplicationTimeFrame int    `koanf:"replicationtimeframe"` // in seconds
 	} `koanf:"replica"`
 	Name     string `koanf:"name"`
-	Version  uint   `koanf:"version"`
 	TimeZone string `koanf:"timezone"`
 	Pool     struct {
 		IdleConnections int           `koanf:"idleconnections"`

config/config.yaml

@@ -25,7 +25,6 @@ database:
   host: pg-sql
   port: 5432
   name: pipeline
-  version: 40
   timezone: Etc/UTC
   pool:
     idleconnections: 5

go.mod

@@ -45,7 +45,7 @@ require (
 	github.com/h2non/filetype v1.1.3
 	github.com/iancoleman/strcase v0.3.0
 	github.com/influxdata/influxdb-client-go/v2 v2.14.0
-	github.com/instill-ai/protogen-go v0.3.3-alpha.0.20250626141501-c8e22cc2e0b6
+	github.com/instill-ai/protogen-go v0.3.3-alpha.0.20250707160902-77023eb2f033
 	github.com/instill-ai/usage-client v0.4.0
 	github.com/instill-ai/x v0.8.0-alpha.0.20250522164415-9172edd336bb
 	github.com/itchyny/gojq v0.12.17

go.sum

@@ -460,8 +460,8 @@ github.com/influxdata/influxdb-client-go/v2 v2.14.0 h1:AjbBfJuq+QoaXNcrova8smSjw
 github.com/influxdata/influxdb-client-go/v2 v2.14.0/go.mod h1:Ahpm3QXKMJslpXl3IftVLVezreAUtBOTZssDrjZEFHI=
 github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf h1:7JTmneyiNEwVBOHSjoMxiWAqB992atOeepeFYegn5RU=
 github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo=
-github.com/instill-ai/protogen-go v0.3.3-alpha.0.20250626141501-c8e22cc2e0b6 h1:5o4fBRte53mNXcjengF2TtlGaY4vUiM49iWT2KoOa5Q=
-github.com/instill-ai/protogen-go v0.3.3-alpha.0.20250626141501-c8e22cc2e0b6/go.mod h1:bCnBosofpaUxKBuTTJM3/I3thAK37kvfBnKByjnLsl4=
+github.com/instill-ai/protogen-go v0.3.3-alpha.0.20250707160902-77023eb2f033 h1:jhP9Gz7tw57rTTHQ7WhHOWf7z/worMcfr/n3NAcjbD4=
+github.com/instill-ai/protogen-go v0.3.3-alpha.0.20250707160902-77023eb2f033/go.mod h1:bCnBosofpaUxKBuTTJM3/I3thAK37kvfBnKByjnLsl4=
 github.com/instill-ai/usage-client v0.4.0 h1:xf1hAlO4a8lZwZzz9bprZOJqU3ghIcIsavUUB7UURyg=
 github.com/instill-ai/usage-client v0.4.0/go.mod h1:zZ9LRoXps2u63ARYPAbR2YvqTib3dWJLObZn+9YqhF0=
 github.com/instill-ai/x v0.8.0-alpha.0.20250522164415-9172edd336bb h1:lrHTet3MB9ctNTvY/H8qcyiHd1vdTRqzUMkGsh5/Pp8=

pkg/component/data/instillartifact/v0/config/definition.yaml

@@ -16,6 +16,6 @@ description: Access files and perform RAG-based search and retrieval through cat
 tombstone: false
 type: COMPONENT_TYPE_DATA
 uid: 6ec46048-f82f-4452-ba19-79698af9186e
-version: 0.1.0
+version: 0.1.1
 sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/data/instillartifact/v0
 releaseStage: RELEASE_STAGE_ALPHA

pkg/component/data/instillartifact/v0/config/tasks.yaml

@@ -69,16 +69,22 @@ $defs:
         uiOrder: 2
         title: Text Content
         type: string
+      source-file-uid:
+        description: The UID of the source file.
+        uiOrder: 3
+        title: Source File UID
+        type: string
       source-file-name:
         description: The name of the source file.
-        uiOrder: 3
+        uiOrder: 4
         title: Source File Name
         type: string
     required:
       - chunk-uid
       - similarity-score
       - text-content
       - source-file-name
+      - source-file-uid
     title: Chunk
     type: object
   namespace:
@@ -582,11 +588,11 @@ TASK_RETRIEVE:
         uiOrder: 2
         type: integer
         title: Top K
-      filename:
-        description: File name to filter, empty for all.
+      file-uid:
+        description: Optional filter by file.
         uiOrder: 3
         type: string
-        title: Filename
+        title: File UID
       file-media-type:
         description: The media type to filter, empty for all.
         uiOrder: 4
@@ -606,6 +612,14 @@ TASK_RETRIEVE:
           - summary
           - augmented
         title: Content type
+      filename:
+        description: |-
+          File name to filter, empty for all. This field is deprecated and the
+          file UID should be used instead. The filename isn't unique by catalog
+          and therefore the filter might produce unexpected results.
+        uiOrder: 6
+        type: string
+        title: Filename
     required:
       - namespace
       - catalog-id

pkg/component/data/instillartifact/v0/io.go

@@ -164,12 +164,16 @@ type SearchChunksInput struct {
 	TextPrompt string `json:"text-prompt"`
 	// TopK for searching chunks
 	TopK uint32 `json:"top-k"`
-	// File name to filter
-	Filename string `json:"filename"`
+	// File filter
+	FileUID string `json:"file-uid"`
 	// The media type to filter
 	FileMediaType string `json:"file-media-type"`
 	// The content type to filter
-	ContetType string `json:"content-type"`
+	ContentType string `json:"content-type"`
+
+	// File name to filter
+	// Deprecated, use FileUID instead
+	Filename string `json:"filename"`
 }
 
 // SearchChunksOutput is the output for searching chunks
@@ -186,6 +190,8 @@ type SimilarityChunk struct {
 	SimilarityScore float32 `json:"similarity-score"`
 	// Text content of the chunk
 	TextContent string `json:"text-content"`
+	// Source file UID
+	SourceFileUID string `json:"source-file-uid"`
 	// Source file name
 	SourceFileName string `json:"source-file-name"`
 	// Content type

pkg/component/data/instillartifact/v0/task_query.go

@@ -39,15 +39,16 @@ func (e *execution) query(input *structpb.Struct) (*structpb.Struct, error) {
 
 	output := QueryOutput{
 		Answer: queryRes.Answer,
-		Chunks: []SimilarityChunk{},
+		Chunks: make([]SimilarityChunk, 0, len(queryRes.GetSimilarChunks())),
 	}
 
-	for _, chunkPB := range queryRes.SimilarChunks {
+	for _, chunkPB := range queryRes.GetSimilarChunks() {
 		output.Chunks = append(output.Chunks, SimilarityChunk{
-			ChunkUID:        chunkPB.ChunkUid,
-			SimilarityScore: chunkPB.SimilarityScore,
-			TextContent:     chunkPB.TextContent,
-			SourceFileName:  chunkPB.SourceFile,
+			ChunkUID:        chunkPB.GetChunkUid(),
+			SimilarityScore: chunkPB.GetSimilarityScore(),
+			TextContent:     chunkPB.GetTextContent(),
+			SourceFileName:  chunkPB.GetSourceFile(),
+			SourceFileUID:   chunkPB.GetChunkMetadata().GetOriginalFileUid(),
 		})
 	}
 

pkg/component/data/instillartifact/v0/task_search_chunks.go

@@ -10,11 +10,10 @@ import (
 
 	"github.com/instill-ai/pipeline-backend/pkg/component/base"
 
-	artifactPB "github.com/instill-ai/protogen-go/artifact/artifact/v1alpha"
+	artifactpb "github.com/instill-ai/protogen-go/artifact/artifact/v1alpha"
 )
 
 func (e *execution) searchChunks(input *structpb.Struct) (*structpb.Struct, error) {
-
 	inputStruct := SearchChunksInput{}
 	err := base.ConvertFromStructpb(input, &inputStruct)
 	if err != nil {
@@ -27,56 +26,60 @@ func (e *execution) searchChunks(input *structpb.Struct) (*structpb.Struct, erro
 	defer cancel()
 	ctx = metadata.NewOutgoingContext(ctx, getRequestMetadata(e.SystemVariables))
 
-	var fileMediaType artifactPB.FileMediaType
-	var contentType artifactPB.ContentType
+	var fileMediaType artifactpb.FileMediaType
+	var contentType artifactpb.ContentType
 
 	switch inputStruct.FileMediaType {
 	case "document":
-		fileMediaType = artifactPB.FileMediaType_FILE_MEDIA_TYPE_DOCUMENT
+		fileMediaType = artifactpb.FileMediaType_FILE_MEDIA_TYPE_DOCUMENT
 	case "image":
-		fileMediaType = artifactPB.FileMediaType_FILE_MEDIA_TYPE_IMAGE
+		fileMediaType = artifactpb.FileMediaType_FILE_MEDIA_TYPE_IMAGE
 	case "audio":
-		fileMediaType = artifactPB.FileMediaType_FILE_MEDIA_TYPE_AUDIO
+		fileMediaType = artifactpb.FileMediaType_FILE_MEDIA_TYPE_AUDIO
 	case "video":
-		fileMediaType = artifactPB.FileMediaType_FILE_MEDIA_TYPE_VIDEO
+		fileMediaType = artifactpb.FileMediaType_FILE_MEDIA_TYPE_VIDEO
 	default:
-		fileMediaType = artifactPB.FileMediaType_FILE_MEDIA_TYPE_UNSPECIFIED
+		fileMediaType = artifactpb.FileMediaType_FILE_MEDIA_TYPE_UNSPECIFIED
 	}
 
-	switch inputStruct.ContetType {
+	switch inputStruct.ContentType {
 	case "chunk":
-		contentType = artifactPB.ContentType_CONTENT_TYPE_CHUNK
+		contentType = artifactpb.ContentType_CONTENT_TYPE_CHUNK
 	case "summary":
-		contentType = artifactPB.ContentType_CONTENT_TYPE_SUMMARY
+		contentType = artifactpb.ContentType_CONTENT_TYPE_SUMMARY
 	case "augmented":
-		contentType = artifactPB.ContentType_CONTENT_TYPE_AUGMENTED
+		contentType = artifactpb.ContentType_CONTENT_TYPE_AUGMENTED
 	default:
-		contentType = artifactPB.ContentType_CONTENT_TYPE_UNSPECIFIED
+		contentType = artifactpb.ContentType_CONTENT_TYPE_UNSPECIFIED
 	}
 
-	searchRes, err := artifactClient.SimilarityChunksSearch(ctx, &artifactPB.SimilarityChunksSearchRequest{
+	searchRes, err := artifactClient.SimilarityChunksSearch(ctx, &artifactpb.SimilarityChunksSearchRequest{
 		NamespaceId:   inputStruct.Namespace,
 		CatalogId:     inputStruct.CatalogID,
 		TextPrompt:    inputStruct.TextPrompt,
 		TopK:          inputStruct.TopK,
-		FileName:      inputStruct.Filename,
+		FileUid:       inputStruct.FileUID,
 		FileMediaType: fileMediaType,
 		ContentType:   contentType,
+
+		// Deprecated: we keep using it for backwards compatibility.
+		FileName: inputStruct.Filename,
 	})
 	if err != nil {
 		return nil, fmt.Errorf("failed to search chunks: %w", err)
 	}
 
 	output := SearchChunksOutput{
-		Chunks: []SimilarityChunk{},
+		Chunks: make([]SimilarityChunk, 0, len(searchRes.GetSimilarChunks())),
 	}
 
-	for _, chunkPB := range searchRes.SimilarChunks {
+	for _, chunkPB := range searchRes.GetSimilarChunks() {
 		output.Chunks = append(output.Chunks, SimilarityChunk{
-			ChunkUID:        chunkPB.ChunkUid,
-			SimilarityScore: chunkPB.SimilarityScore,
-			TextContent:     chunkPB.TextContent,
-			SourceFileName:  chunkPB.SourceFile,
+			ChunkUID:        chunkPB.GetChunkUid(),
+			SimilarityScore: chunkPB.GetSimilarityScore(),
+			TextContent:     chunkPB.GetTextContent(),
+			SourceFileName:  chunkPB.GetSourceFile(),
+			SourceFileUID:   chunkPB.GetChunkMetadata().GetOriginalFileUid(),
 			ContentType:     chunkPB.GetChunkMetadata().GetContentType().String(),
 		})
 	}