Optimize total files calculation and remove unused code

Co-authored-by: Mte90 <403283+Mte90@users.noreply.github.com>

Author: Copilot

ai/analyzer.py

@@ -390,11 +390,13 @@ def analyze_local_path_sync(
 
         try:
             # Use batch update for efficiency - single database transaction
+            # Store total_files for performance (avoid re-scanning directory on every request)
             set_project_metadata_batch(database_path, {
                 "last_indexed_at": time.strftime("%Y-%m-%d %H:%M:%S"),
                 "last_index_duration": str(duration),
                 "files_indexed": str(file_count),
-                "files_skipped": str(skipped_count)
+                "files_skipped": str(skipped_count),
+                "total_files": str(total_files)  # Store total files found during indexing
             })
         except Exception:
             logger.exception("Failed to store indexing metadata")

ai/openai.py

@@ -106,240 +106,11 @@ def _retry_with_backoff(func, *args, **kwargs):
             delay = base_delay * (2 ** attempt)
             time.sleep(delay)
 
-
 class EmbeddingError(Exception):
     """Custom exception for embedding failures"""
     pass
 
 
-class EmbeddingClient:
-    """
-    Embedding client with detailed logging, retry logic, and configurable timeouts.
-    Provides better debugging for embedding API failures.
-    Uses OpenAI SDK for proper API compatibility.
-    """
-    def __init__(self,
-                 api_url: Optional[str] = None,
-                 api_key: Optional[str] = None,
-                 model: Optional[str] = None,
-                 timeout: float = 15.0,
-                 max_retries: int = 2,
-                 backoff: float = 1.5):
-        self.api_url = api_url or CFG.get("api_url")
-        self.api_key = api_key or CFG.get("api_key")
-        self.model = model or DEFAULT_EMBEDDING_MODEL or "text-embedding-3-small"
-        self.timeout = timeout
-        self.max_retries = max_retries
-        self.backoff = backoff
-        
-        # Use OpenAI SDK client instead of raw requests
-        # The SDK automatically handles the /embeddings path
-        self.client = _client
-
-    def _generate_curl_command(self, payload: Dict[str, Any]) -> str:
-        """
-        Generate a curl command for debugging purposes.
-        Masks the API key for security.
-        """
-        # Construct the full embeddings URL
-        base_url = self.api_url.rstrip('/')
-        if not base_url.endswith('/embeddings'):
-            url = f"{base_url}/embeddings"
-        else:
-            url = base_url
-        
-        # Start with basic curl command
-        curl_parts = ["curl", "-X", "POST", f"'{url}'"]
-        
-        # Add standard headers
-        headers = {
-            "Content-Type": "application/json",
-            "Authorization": f"Bearer <API_KEY_MASKED>"
-        }
-        
-        for key, value in headers.items():
-            curl_parts.append(f"-H '{key}: {value}'")
-        
-        # Add data payload
-        payload_json = json.dumps(payload)
-        # Escape single quotes in the JSON for shell compatibility
-        payload_json_escaped = payload_json.replace("'", "'\\''")
-        curl_parts.append(f"-d '{payload_json_escaped}'")
-        
-        return " \\\n  ".join(curl_parts)
-
-    def _save_curl_script(self, curl_command: str, request_id: str, file_path: str, chunk_index: int) -> Optional[str]:
-        """
-        Save curl command to a bash script in /tmp for debugging.
-        Returns the path to the generated script, or None if save failed.
-        """
-        try:
-            import tempfile
-            # Create a unique filename based on request_id
-            script_name = f"embedding_debug_{request_id[:8]}.sh"
-            script_path = os.path.join("/tmp", script_name)
-            
-            # Generate script content with shebang and comments
-            script_content = f"""#!/bin/bash
-# Embedding request debug script
-# Request ID: {request_id}
-# File: {file_path}
-# Chunk: {chunk_index}
-# Generated: {time.strftime('%Y-%m-%d %H:%M:%S')}
-
-{curl_command}
-"""
-            
-            with open(script_path, 'w') as f:
-                f.write(script_content)
-            
-            # Make the script executable
-            os.chmod(script_path, 0o755)
-            
-            return script_path
-        except Exception as e:
-            _embedding_logger.warning(f"Failed to save curl debug script: {e}")
-            return None
-
-
-    def _log_request_start(self, request_id: str, file_path: str, chunk_index: int, chunk_len: int):
-        _embedding_logger.debug(
-            "Embedding request START",
-            extra={
-                "request_id": request_id,
-                "file": file_path,
-                "chunk_index": chunk_index,
-                "chunk_length": chunk_len,
-                "model": self.model,
-                "api_url": self.api_url,
-                "timeout": self.timeout,
-            },
-        )
-
-    def _log_request_end(self, request_id: str, elapsed: float, status: Optional[int], response_body_preview: str):
-        _embedding_logger.debug(
-            "Embedding request END",
-            extra={
-                "request_id": request_id,
-                "elapsed_s": elapsed,
-                "status": status,
-                "response_preview": response_body_preview,
-            },
-        )
-
-    def embed_text(self, text: str, file_path: str = "<unknown>", chunk_index: int = 0) -> List[float]:
-        """
-        Embed a single chunk of text using OpenAI SDK. Returns the embedding vector.
-        Raises EmbeddingError on failure.
-        """
-        request_id = str(uuid.uuid4())
-        chunk_len = len(text)
-        self._log_request_start(request_id, file_path, chunk_index, chunk_len)
-
-        payload = {
-            "model": self.model,
-            "input": text,
-        }
-
-        attempt = 0
-        err_msg = ""
-        while True:
-            attempt += 1
-            start = time.perf_counter()
-            try:
-                # Use OpenAI SDK for embeddings
-                resp = self.client.embeddings.create(
-                    model=self.model,
-                    input=text,
-                    timeout=self.timeout
-                )
-                elapsed = time.perf_counter() - start
-
-                # Log successful response
-                self._log_request_end(request_id, elapsed, 200, "Success")
-
-                # Extract embedding from response
-                # The SDK returns a response object with a data list
-                if resp and hasattr(resp, 'data') and len(resp.data) > 0:
-                    embedding = resp.data[0].embedding
-                    if embedding and isinstance(embedding, list):
-                        return embedding
-                    else:
-                        raise EmbeddingError(f"Invalid embedding format in response")
-                else:
-                    raise EmbeddingError(f"Unexpected embedding response shape from SDK")
-
-            except Exception as e:
-                elapsed = time.perf_counter() - start
-                err_msg = f"Error after {elapsed:.2f}s: {e}"
-                
-                # Save debug information for timeout or API errors
-                script_path = None
-                if CFG.get("debug"):
-                    # Generate curl command for debugging
-                    curl_command = self._generate_curl_command(payload)
-                    script_path = self._save_curl_script(curl_command, request_id, file_path, chunk_index)
-                    if script_path:
-                        _embedding_logger.error(f"\nDebug script saved to: {script_path}")
-                        _embedding_logger.error(f"Run with: bash {script_path}")
-                    else:
-                        _embedding_logger.error(f"\nDebug with this curl command:")
-                        _embedding_logger.error(curl_command)
-                
-                _embedding_logger.error(
-                    "Embedding API Error",
-                    extra={
-                        "request_id": request_id,
-                        "error": str(e),
-                        "elapsed_s": elapsed,
-                        "attempt": attempt,
-                        "file": file_path,
-                        "chunk_index": chunk_index,
-                    }
-                )
-
-            # Retry logic
-            if attempt > self.max_retries:
-                _embedding_logger.error(
-                    "Max retries exceeded for embedding request",
-                    extra={"request_id": request_id, "file": file_path, "chunk_index": chunk_index, "attempts": attempt},
-                )
-                raise EmbeddingError(f"Failed to get embedding after {attempt} attempts. Last error: {err_msg}")
-
-            # Backoff and retry
-            sleep_for = self.backoff * (2 ** (attempt - 1))
-            _embedding_logger.info(
-                "Retrying embedding request",
-                extra={
-                    "request_id": request_id,
-                    "file": file_path,
-                    "chunk_index": chunk_index,
-                    "attempt": attempt,
-                    "sleep_s": sleep_for,
-                },
-            )
-            time.sleep(sleep_for)
-
-    def embed_multiple(self, chunks: List[str], file_path: str = "<unknown>") -> List[Dict[str, Any]]:
-        """
-        Embed a list of text chunks. Returns list of dicts: {"chunk_index": i, "embedding": [...]}.
-        This method logs progress and errors for each chunk.
-        """
-        results = []
-        for i, chunk in enumerate(chunks):
-            try:
-                emb = self.embed_text(chunk, file_path=file_path, chunk_index=i)
-                results.append({"chunk_index": i, "embedding": emb})
-            except EmbeddingError as e:
-                _embedding_logger.error(
-                    "Failed to embed chunk",
-                    extra={"file": file_path, "chunk_index": i, "error": str(e)},
-                )
-                # append a failure marker or skip depending on desired behavior
-                results.append({"chunk_index": i, "embedding": None, "error": str(e)})
-        return results
-
-
 def call_coding_api(prompt: str, model: Optional[str] = None, max_tokens: int = 1024):
     """
     Call a generative/coding model via the new OpenAI client.