feat: temp workspace and cargo file

Author: rrrodzilla

Cargo.lock

@@ -4,7 +4,7 @@ version = "0.1.0"
 edition = "2024"
 
 [dependencies]
-rmcp = { version = "0.1.5", features = ["tower", "transport-io", "transport-sse-server"] }
+rmcp = { version = "0.1.5", features = ["tower", "transport-io", "transport-sse-server", "macros", "server"] } # Add macros, server, schemars
 tokio = { version = "1", features = ["macros", "rt-multi-thread"] }
 dotenvy = "0.15"
 serde = { version = "1", features = ["derive"] }
@@ -14,11 +14,12 @@ walkdir = "2.5.0"
 scraper = "0.23.1"
 ndarray = { version = "0.16.1", features = ["serde"] } # Enable serde feature
 async-openai = "0.28.0"
-async-trait = "0.1.88"
+# async-trait = "0.1.88" # Removed, likely no longer needed
 futures = "0.3"
 bincode = { version = "2.0.1", features = ["serde"] } # Enable serde integration
 tiktoken-rs = "0.6.0"
 cargo = "0.86.0"
 tempfile = "3.19.1"
 xdg = { version = "2.5.2", features = ["serde"] }
 anyhow = "1.0.97"
+schemars = "0.8.22"

Cargo.toml

@@ -1,5 +1,5 @@
 use scraper::{Html, Selector};
-use std::fs;
+use std::{fs::{self, File, create_dir_all}, io::Write}; // Added File, create_dir_all, and Write
 use cargo::core::resolver::features::CliFeatures;
 // use cargo::core::SourceId; // Removed unused import
 // use cargo::util::Filesystem; // Removed unused import
@@ -37,20 +37,47 @@ pub struct Document {
 /// Generates documentation for a given crate in a temporary directory,
 /// then loads and parses the HTML documents.
 /// Extracts text content from the main content area of rustdoc generated HTML.
-pub fn load_documents(crate_name: &str, _crate_version: &str) -> Result<Vec<Document>, DocLoaderError> { // Mark version as unused for now
-    println!("[DEBUG] load_documents called with crate_name: '{}', crate_version: '{}'", crate_name, _crate_version);
+pub fn load_documents(crate_name: &str, crate_version: &str) -> Result<Vec<Document>, DocLoaderError> { // Use crate_version
+    eprintln!("[DEBUG] load_documents called with crate_name: '{}', crate_version: '{}'", crate_name, crate_version);
     let mut documents = Vec::new();
 
     let temp_dir = tempdir().map_err(DocLoaderError::TempDirCreationFailed)?;
     let temp_dir_path = temp_dir.path();
+    let temp_manifest_path = temp_dir_path.join("Cargo.toml");
 
-    println!(
-        "Generating documentation for crate '{}' in temporary directory: {}",
+    eprintln!(
+        "Generating documentation for crate '{}' version '{}' in temporary directory: {}",
         crate_name,
+        crate_version,
         temp_dir_path.display()
     );
 
-    // Execute `cargo doc` using std::process::Command
+    // Create a temporary Cargo.toml
+    let cargo_toml_content = format!(
+        r#"[package]
+name = "temp-doc-crate"
+version = "0.1.0"
+edition = "2021"
+
+[lib] # Add an empty lib target to satisfy Cargo
+
+[dependencies]
+{} = "{}"
+"#,
+        crate_name, crate_version
+    );
+
+    // Create the src directory and an empty lib.rs file
+    let src_path = temp_dir_path.join("src");
+    create_dir_all(&src_path)?;
+    File::create(src_path.join("lib.rs"))?;
+    eprintln!("[DEBUG] Created empty src/lib.rs at: {}", src_path.join("lib.rs").display());
+
+    let mut temp_manifest_file = File::create(&temp_manifest_path)?;
+    temp_manifest_file.write_all(cargo_toml_content.as_bytes())?;
+    eprintln!("[DEBUG] Created temporary manifest at: {}", temp_manifest_path.display());
+
+
     // --- Use Cargo API ---
     let mut config = GlobalContext::default()?; // Make mutable
     // Configure context for quiet operation
@@ -67,17 +94,17 @@ pub fn load_documents(crate_name: &str, _crate_version: &str) -> Result<Vec<Docu
     )?;
     // config.shell().set_verbosity(Verbosity::Quiet); // Keep commented
 
-    let current_dir = std::env::current_dir()?;
-    let mut ws = Workspace::new(&current_dir.join("Cargo.toml"), &config)?; // Make ws mutable
-    println!("[DEBUG] Workspace target dir before set: {}", ws.target_dir().as_path_unlocked().display());
+    // Use the temporary manifest path for the Workspace
+    let mut ws = Workspace::new(&temp_manifest_path, &config)?; // Make ws mutable
+    eprintln!("[DEBUG] Workspace target dir before set: {}", ws.target_dir().as_path_unlocked().display());
     // Set target_dir directly on Workspace
     ws.set_target_dir(cargo::util::Filesystem::new(temp_dir_path.to_path_buf()));
-    println!("[DEBUG] Workspace target dir after set: {}", ws.target_dir().as_path_unlocked().display());
+    eprintln!("[DEBUG] Workspace target dir after set: {}", ws.target_dir().as_path_unlocked().display());
 
     // Create CompileOptions, relying on ::new for BuildConfig
     let mut compile_opts = CompileOptions::new(&config, cargo::core::compiler::CompileMode::Doc { deps: false, json: false })?;
     // Specify the package explicitly
-    let package_spec = crate_name.replace('-', "_"); // Just use name (with underscores)
+    let package_spec = crate_name.to_string(); // Just use name (with underscores)
     compile_opts.cli_features = CliFeatures::new_all(false); // Use new_all(false)
     compile_opts.spec = Packages::Packages(vec![package_spec.clone()]); // Clone spec
 
@@ -87,7 +114,7 @@ pub fn load_documents(crate_name: &str, _crate_version: &str) -> Result<Vec<Docu
         open_result: false, // Don't open in browser
         output_format: ops::OutputFormat::Html,
     };
-    println!("[DEBUG] package_spec for CompileOptions: '{}'", package_spec);
+    eprintln!("[DEBUG] package_spec for CompileOptions: '{}'", package_spec);
 
     ops::doc(&ws, &doc_opts).map_err(DocLoaderError::CargoLib)?; // Use ws
     // --- End Cargo API ---
@@ -97,49 +124,57 @@ pub fn load_documents(crate_name: &str, _crate_version: &str) -> Result<Vec<Docu
     let docs_path = temp_dir_path.join("doc").join(&crate_name_underscores);
 
     // Debug print relevant options before calling ops::doc
-    println!("[DEBUG] CompileOptions spec: {:?}", doc_opts.compile_opts.spec);
-    println!("[DEBUG] CompileOptions cli_features: {:?}", doc_opts.compile_opts.cli_features);
-    println!("[DEBUG] CompileOptions build_config mode: {:?}", doc_opts.compile_opts.build_config.mode);
-    println!("[DEBUG] DocOptions output_format: {:?}", doc_opts.output_format);
+    eprintln!("[DEBUG] CompileOptions spec: {:?}", doc_opts.compile_opts.spec);
+    eprintln!("[DEBUG] CompileOptions cli_features: {:?}", doc_opts.compile_opts.cli_features);
+    eprintln!("[DEBUG] CompileOptions build_config mode: {:?}", doc_opts.compile_opts.build_config.mode);
+    eprintln!("[DEBUG] DocOptions output_format: {:?}", doc_opts.output_format);
     if !docs_path.exists() || !docs_path.is_dir() {
          return Err(DocLoaderError::CargoLib(anyhow::anyhow!(
              "Generated documentation not found at expected path: {}. Check crate name and cargo doc output.",
              docs_path.display()
          )));
     }
-    println!("Generated documentation path: {}", docs_path.display());
+    eprintln!("Generated documentation path: {}", docs_path.display());
 
-    println!("[DEBUG] ops::doc called successfully.");
+    eprintln!("[DEBUG] ops::doc called successfully.");
 
     // Define the CSS selector for the main content area in rustdoc HTML
     // This might need adjustment based on the exact rustdoc version/theme
     let content_selector = Selector::parse("section#main-content.content")
         .map_err(|e| DocLoaderError::Selector(e.to_string()))?;
-    println!("[DEBUG] Calculated final docs_path: {}", docs_path.display());
+    eprintln!("[DEBUG] Calculated final docs_path: {}", docs_path.display());
 
-    println!("Starting document loading from: {}", docs_path.display());
-        println!("[DEBUG] docs_path does not exist or is not a directory.");
+    eprintln!("Starting document loading from: {}", docs_path.display());
+        eprintln!("[DEBUG] docs_path does not exist or is not a directory.");
 
-    for entry in WalkDir::new(docs_path)
+    for entry in WalkDir::new(&docs_path)
         .into_iter()
         .filter_map(Result::ok) // Ignore errors during iteration for now
         .filter(|e| !e.file_type().is_dir() && e.path().extension().is_some_and(|ext| ext == "html"))
     {
         let path = entry.path();
-        let path_str = path.to_string_lossy().to_string();
-        // println!("Processing file: {}", path.display()); // Uncommented
-
-        // println!("  Reading file content..."); // Added
-        let html_content = fs::read_to_string(path)?;
-        // println!("  Parsing HTML..."); // Added
+        // Calculate path relative to the docs_path root
+        let relative_path = path.strip_prefix(&docs_path).map_err(|e| {
+            // Provide more context in the error message
+            DocLoaderError::Io(std::io::Error::new(
+                std::io::ErrorKind::Other,
+                format!("Failed to strip prefix '{}' from path '{}': {}", docs_path.display(), path.display(), e)
+            ))
+        })?;
+        let path_str = relative_path.to_string_lossy().to_string(); // Use the relative path
+        // eprintln!("Processing file: {} (relative: {})", path.display(), path_str); // Updated debug log
+
+        // eprintln!("  Reading file content..."); // Added
+        let html_content = fs::read_to_string(path)?; // Still read from the absolute path
+        // eprintln!("  Parsing HTML..."); // Added
 
         // Parse the HTML document
         let document = Html::parse_document(&html_content);
 
         // Select the main content element
         if let Some(main_content_element) = document.select(&content_selector).next() {
             // Extract all text nodes within the main content
-            // println!("  Extracting text content..."); // Added
+            // eprintln!("  Extracting text content..."); // Added
             let text_content: String = main_content_element
                 .text()
                 .map(|s| s.trim())
@@ -148,20 +183,20 @@ pub fn load_documents(crate_name: &str, _crate_version: &str) -> Result<Vec<Docu
                 .join("\n"); // Join text nodes with newlines
 
             if !text_content.is_empty() {
-                // println!("  Extracted content ({} chars)", text_content.len()); // Uncommented and simplified
+                // eprintln!("  Extracted content ({} chars)", text_content.len()); // Uncommented and simplified
                 documents.push(Document {
                     path: path_str,
                     content: text_content,
                 });
             } else {
-                // println!("No text content found in main section for: {}", path.display()); // Verbose logging
+                // eprintln!("No text content found in main section for: {}", path.display()); // Verbose logging
             }
         } else {
-             // println!("'main-content' selector not found for: {}", path.display()); // Verbose logging
+             // eprintln!("'main-content' selector not found for: {}", path.display()); // Verbose logging
              // Optionally handle files without the main content selector differently
         }
     }
 
-    println!("Finished document loading. Found {} documents.", documents.len());
+    eprintln!("Finished document loading. Found {} documents.", documents.len());
     Ok(documents)
 }

src/doc_loader.rs

@@ -16,13 +16,15 @@ pub static OPENAI_CLIENT: OnceLock<OpenAIClient<OpenAIConfig>> = OnceLock::new()
 use bincode::{Encode, Decode};
 use serde::{Serialize, Deserialize};
 
-// Define a struct suitable for serialization with bincode
-#[derive(Serialize, Deserialize, Debug, Encode, Decode)] // Add Encode, Decode
-pub struct SerializableEmbedding {
+// Define a struct containing path, content, and embedding for caching
+#[derive(Serialize, Deserialize, Debug, Encode, Decode)]
+pub struct CachedDocumentEmbedding {
     pub path: String,
+    pub content: String, // Add the extracted document content
     pub vector: Vec<f32>,
 }
 
+
 /// Calculates the cosine similarity between two vectors.
 pub fn cosine_similarity(v1: ArrayView1<f32>, v2: ArrayView1<f32>) -> f32 {
     let dot_product = v1.dot(&v2);
@@ -42,7 +44,7 @@ pub async fn generate_embeddings(
     documents: &[Document],
     model: &str,
 ) -> Result<(Vec<(String, Array1<f32>)>, usize), ServerError> { // Return tuple: (embeddings, total_tokens)
-    // println!("Generating embeddings for {} documents...", documents.len());
+    // eprintln!("Generating embeddings for {} documents...", documents.len());
 
     // Get the tokenizer for the model and wrap in Arc
     let bpe = Arc::new(cl100k_base().map_err(|e| ServerError::Tiktoken(e.to_string()))?);
@@ -63,7 +65,7 @@ pub async fn generate_embeddings(
                 let token_count = bpe.encode_with_special_tokens(&doc.content).len();
 
                 if token_count > TOKEN_LIMIT {
-                    // println!(
+                    // eprintln!(
                     //     "    Skipping document {}: Actual tokens ({}) exceed limit ({}). Path: {}",
                     //     index + 1,
                     //     token_count,
@@ -82,14 +84,14 @@ pub async fn generate_embeddings(
                     .input(inputs)
                     .build()?; // Propagates OpenAIError
 
-                // println!(
+                // eprintln!(
                 //     "    Sending request for document {} ({} tokens)... Path: {}",
                 //     index + 1,
                 //     token_count, // Use correct variable name
                 //     doc.path
                 // );
                 let response = client.embeddings().create(request).await?; // Propagates OpenAIError
-                // println!("    Received response for document {}.", index + 1);
+                // eprintln!("    Received response for document {}.", index + 1);
 
                 if response.data.len() != 1 {
                     return Err(ServerError::OpenAI(
@@ -129,13 +131,13 @@ pub async fn generate_embeddings(
             Err(e) => {
                 // Log error but potentially continue? Or return the first error?
                 // For now, let's return the first error encountered.
-                println!("Error during concurrent embedding generation: {}", e);
+                eprintln!("Error during concurrent embedding generation: {}", e);
                 return Err(e);
             }
         }
     }
 
-    println!(
+    eprintln!(
         "Finished generating embeddings. Successfully processed {} documents ({} tokens).",
         embeddings_vec.len(), total_processed_tokens
     );

src/embeddings.rs

@@ -22,4 +22,6 @@ pub enum ServerError {
     Tiktoken(String),
     #[error("XDG Directory Error: {0}")]
     Xdg(String),
+    #[error("MCP Runtime Error: {0}")]
+    McpRuntime(String),
 }