fix(init): remove components from the database if they no longer exist in the definition list (#1049)

Because

- When a component is removed from the definition list, it should also
be removed from the definition index table.

This commit

- removes components from the database if they no longer exist in the
definition list.

Author: donch1989

Makefile

@@ -136,7 +136,7 @@ integration-test: ## Run integration test
 
 .PHONY: gen-mock
 gen-mock: ## Generate mock files
-	@go install github.com/gojuno/minimock/v3/cmd/minimock@v3.4.0
+	@go install github.com/gojuno/minimock/v3/cmd/minimock@v3.4.5
 	@go generate -run minimock ./...
 
 .PHONY: gen-component-doc

cmd/init/definitionupdater/updater.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/gofrs/uuid"
 	"github.com/launchdarkly/go-semver"
+	"go.uber.org/zap"
 
 	"github.com/instill-ai/pipeline-backend/pkg/datamodel"
 	"github.com/instill-ai/pipeline-backend/pkg/repository"
@@ -25,13 +26,34 @@ func UpdateComponentDefinitionIndex(ctx context.Context, repo repository.Reposit
 	defs := componentstore.Init(componentstore.InitParams{
 		Logger: logger,
 	}).ListDefinitions(nil, true)
-	for _, def := range defs {
 
+	// Create a map of UIDs from the current definitions for quick lookup
+	currentDefUIDs := make(map[string]bool)
+	for _, def := range defs {
+		currentDefUIDs[def.GetUid()] = true
 		if err := updateComponentDefinition(ctx, def, repo); err != nil {
 			return err
 		}
 	}
 
+	// Get all component definitions from the database
+	dbDefs, err := repo.ListAllComponentDefinitions(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to get component definitions from database: %w", err)
+	}
+
+	// Delete component definitions that don't exist in the current definitions
+	for _, dbDef := range dbDefs {
+		if !currentDefUIDs[dbDef.UID.String()] {
+			logger.Info("Deleting component definition that no longer exists",
+				zap.String("uid", dbDef.UID.String()),
+				zap.String("id", dbDef.ID))
+			if err := repo.DeleteComponentDefinition(ctx, dbDef.UID); err != nil {
+				return fmt.Errorf("failed to delete component definition %s: %w", dbDef.ID, err)
+			}
+		}
+	}
+
 	return nil
 }
 

pkg/component/ai/fireworksai/v0/fireworks_client_interface_mock_test.go

@@ -359,9 +359,10 @@ search the chunks in the catalog.
 | Namespace (required) | `namespace` | string | Fill in your namespace, you can get namespace through the tab of switching namespace. |
 | Text Prompt (required) | `text-prompt` | string | The prompt string to search the chunks. |
 | Top K | `top-k` | integer | The number of top chunks to return. The range is from 1~20, and default is 5. |
-| Filename | `filename` | string | File name to filter, empty for all. |
+| File UID | `file-uid` | string | Optional filter by file. |
 | File Media Type | `file-media-type` | string | The media type to filter, empty for all. <br/><details><summary><strong>Enum values</strong></summary><ul><li>`document`</li><li>`image`</li><li>`audio`</li><li>`video`</li></ul></details>  |
 | Content Type | `content-type` | string | The content type to filter, empty for all. <br/><details><summary><strong>Enum values</strong></summary><ul><li>`chunk`</li><li>`summary`</li><li>`augmented`</li></ul></details>  |
+| Filename | `filename` | string | File name to filter, empty for all. This field is deprecated and the file UID should be used instead. The filename isn't unique by catalog and therefore the filter might produce unexpected results. |
 </div>
 
 
@@ -388,6 +389,7 @@ search the chunks in the catalog.
 | Chunk UID | `chunk-uid` | string | The unique identifier of the chunk. |
 | Similarity | `similarity-score` | number | The similarity score of the chunk. |
 | Source File Name | `source-file-name` | string | The name of the source file. |
+| Source File UID | `source-file-uid` | string | The UID of the source file. |
 | Text Content | `text-content` | string | The text content of the chunk. |
 </div>
 </details>
@@ -433,6 +435,7 @@ Reply the questions based on the files in the catalog.
 | Chunk UID | `chunk-uid` | string | The unique identifier of the chunk. |
 | Similarity | `similarity-score` | number | The similarity score of the chunk. |
 | Source File Name | `source-file-name` | string | The name of the source file. |
+| Source File UID | `source-file-uid` | string | The UID of the source file. |
 | Text Content | `text-content` | string | The text content of the chunk. |
 </div>
 </details>