Skip to content

Commit 9d2d1d6

Browse files
CopilotMte90
andauthored
Add progress logging for embedding indexing and PyCharm status bar integration (#5)
Co-authored-by: Mte90 <403283+Mte90@users.noreply.github.com> Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: Daniele Scasciafratte <mte90net@gmail.com>
1 parent c6d8a67 commit 9d2d1d6

File tree

5 files changed

+265
-4
lines changed

5 files changed

+265
-4
lines changed

.gitignore

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,6 @@ uv.lock
66
__pycache__
77
# Per-project databases
88
.picocode/
9+
# Gradle build artifacts
10+
ide-plugins/.gradle/
11+
ide-plugins/build/

ai/analyzer.py

Lines changed: 47 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import sqlite3
66
import importlib.resources
77
import hashlib
8+
import math
89
from pathlib import Path
910
from typing import Optional, Dict, Any, List
1011

@@ -44,6 +45,7 @@
4445
EMBEDDING_CONCURRENCY = 4
4546
# Increase batch size for parallel processing
4647
EMBEDDING_BATCH_SIZE = 16 # Process embeddings in batches for better throughput
48+
PROGRESS_LOG_INTERVAL = 10 # Log progress every N completed files
4749
_THREADPOOL_WORKERS = max(16, EMBEDDING_CONCURRENCY + 8)
4850
_EXECUTOR = concurrent.futures.ThreadPoolExecutor(max_workers=_THREADPOOL_WORKERS)
4951

@@ -333,11 +335,17 @@ def _process_file_sync(
333335
rel_path: str,
334336
cfg: Optional[Dict[str, Any]],
335337
incremental: bool = True,
338+
file_num: int = 0,
339+
total_files: int = 0,
336340
):
337341
"""
338342
Synchronous implementation of per-file processing.
339343
Intended to run on a ThreadPoolExecutor worker thread.
340344
Returns a dict: {"stored": bool, "embedded": bool, "skipped": bool}
345+
346+
Args:
347+
file_num: The current file number being processed (1-indexed)
348+
total_files: Total number of files to process
341349
"""
342350
try:
343351
# read file content
@@ -364,8 +372,11 @@ def _process_file_sync(
364372
logger.debug(f"Skipping unchanged file: {rel_path}")
365373
return {"stored": False, "embedded": False, "skipped": True}
366374

367-
# Log file processing
368-
logger.info(f"Processing file: {rel_path}")
375+
# Log file processing with progress
376+
if file_num > 0 and total_files > 0:
377+
logger.info(f"Processing file ({file_num}/{total_files}): {rel_path}")
378+
else:
379+
logger.info(f"Processing file: {rel_path}")
369380

370381
# store file (synchronous DB writer) with metadata
371382
try:
@@ -409,8 +420,13 @@ def _process_file_sync(
409420
chunk_tasks.append((idx, chunk_doc))
410421

411422
# Process embeddings in parallel batches for better throughput
412-
for batch_start in range(0, len(chunk_tasks), EMBEDDING_BATCH_SIZE):
423+
num_batches = math.ceil(len(chunk_tasks) / EMBEDDING_BATCH_SIZE)
424+
for batch_num, batch_start in enumerate(range(0, len(chunk_tasks), EMBEDDING_BATCH_SIZE), 1):
413425
batch = chunk_tasks[batch_start:batch_start + EMBEDDING_BATCH_SIZE]
426+
427+
# Log batch processing start
428+
logger.info(f"Generating embeddings for {rel_path}: batch {batch_num}/{num_batches} ({len(batch)} chunks)")
429+
414430
embedding_futures = []
415431

416432
for idx, chunk_doc in batch:
@@ -424,6 +440,7 @@ def _process_file_sync(
424440
raise
425441

426442
# Wait for batch to complete and store results
443+
saved_count = 0
427444
for idx, chunk_doc, future in embedding_futures:
428445
try:
429446
emb = future.result() # This will re-raise any exception from the worker
@@ -439,6 +456,7 @@ def _process_file_sync(
439456
try:
440457
_load_sqlite_vector_extension(conn2)
441458
_insert_chunk_vector_with_retry(conn2, fid, rel_path, idx, emb)
459+
saved_count += 1
442460
finally:
443461
conn2.close()
444462
embedded_any = True
@@ -454,6 +472,9 @@ def _process_file_sync(
454472
print(err_content)
455473
except Exception:
456474
logger.exception("Failed to write empty-embedding error to disk for %s chunk %d", rel_path, idx)
475+
476+
# Log batch completion
477+
logger.info(f"Saved {saved_count}/{len(batch)} embeddings for {rel_path} batch {batch_num}/{num_batches}")
457478

458479
return {"stored": True, "embedded": embedded_any, "skipped": False}
459480
except Exception:
@@ -514,14 +535,23 @@ def analyze_local_path_sync(
514535
continue
515536
file_paths.append({"full": full, "rel": rel})
516537

517-
logger.info(f"Found {len(file_paths)} files to process")
538+
total_files = len(file_paths)
539+
logger.info(f"Found {total_files} files to process")
540+
541+
# Thread-safe counters: [submitted_count, completed_count, lock]
542+
counters = [0, 0, threading.Lock()]
518543

519544
# Process files in chunks to avoid too many futures at once.
520545
CHUNK_SUBMIT = 256
521546
for chunk_start in range(0, len(file_paths), CHUNK_SUBMIT):
522547
chunk = file_paths[chunk_start : chunk_start + CHUNK_SUBMIT]
523548
futures = []
524549
for f in chunk:
550+
# Increment counter before starting file processing
551+
with counters[2]:
552+
counters[0] += 1
553+
file_num = counters[0]
554+
525555
fut = _EXECUTOR.submit(
526556
_process_file_sync,
527557
semaphore,
@@ -530,19 +560,32 @@ def analyze_local_path_sync(
530560
f["rel"],
531561
cfg,
532562
incremental,
563+
file_num,
564+
total_files,
533565
)
534566
futures.append(fut)
535567

536568
for fut in concurrent.futures.as_completed(futures):
537569
try:
538570
r = fut.result()
571+
572+
# Increment completed counter and check for periodic logging
573+
with counters[2]:
574+
counters[1] += 1
575+
completed_count = counters[1]
576+
should_log = completed_count % PROGRESS_LOG_INTERVAL == 0
577+
539578
if isinstance(r, dict):
540579
if r.get("stored"):
541580
file_count += 1
542581
if r.get("embedded"):
543582
emb_count += 1
544583
if r.get("skipped"):
545584
skipped_count += 1
585+
586+
# Log periodic progress updates (every 10 files)
587+
if should_log:
588+
logger.info(f"Progress: {completed_count}/{total_files} files processed ({file_count} stored, {emb_count} with embeddings, {skipped_count} skipped)")
546589
except Exception:
547590
logger.exception("A per-file task failed")
548591

ide-plugins/README.md

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ PyCharm/IntelliJ IDEA plugin for PicoCode RAG Assistant with per-project persist
99
- **Real-time Responses**: Streams responses from the coding model
1010
- **File Navigation**: Click on retrieved files to open them in the editor
1111
- **Progress Tracking**: Visual progress indicator during indexing
12+
- **Status Bar Integration**: Shows indexing status in the IDE status bar
1213

1314
## Building the Plugin
1415

@@ -33,6 +34,18 @@ The plugin ZIP will be in `build/distributions/`.
3334
3. Click "Index Project" to index your current project
3435
4. Ask questions in the query box and click "Query"
3536

37+
### Status Bar Widget
38+
39+
The status bar (bottom of the IDE) shows the current indexing status:
40+
41+
- **⚡ PicoCode: Indexing...** - Project is currently being indexed
42+
- **✓ PicoCode: N files** - Project is indexed and ready (shows file count)
43+
- **○ PicoCode: Not indexed** - Project created but not indexed yet
44+
- **✗ PicoCode: Error** - Indexing error occurred
45+
- **PicoCode** - Status unknown (server may not be running)
46+
47+
Hover over the status to see detailed information including file and embedding counts.
48+
3649
## Requirements
3750

3851
- PyCharm/IntelliJ IDEA 2023.1 or later
@@ -45,10 +58,12 @@ The plugin ZIP will be in `build/distributions/`.
4558
2. **API Communication**: HTTP REST API for project management and queries
4659
3. **Secure Storage**: API keys stored using IntelliJ's `PasswordSafe` API
4760
4. **File Navigation**: Uses IntelliJ's Open API to navigate to retrieved files
61+
5. **Status Polling**: Status bar widget polls `/api/projects/{id}` endpoint every 5 seconds
4862

4963
## API Endpoints Used
5064

5165
- `POST /api/projects` - Create/get project
66+
- `GET /api/projects/{id}` - Get project status and indexing stats
5267
- `POST /api/projects/index` - Start indexing
5368
- `POST /api/code` - Query with RAG context
5469
- `GET /api/projects` - List projects
Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,198 @@
1+
package com.picocode
2+
3+
import com.intellij.openapi.application.ApplicationManager
4+
import com.intellij.openapi.project.Project
5+
import com.intellij.openapi.wm.StatusBar
6+
import com.intellij.openapi.wm.StatusBarWidget
7+
import com.intellij.openapi.wm.StatusBarWidgetFactory
8+
import com.intellij.util.Consumer
9+
import com.google.gson.Gson
10+
import com.google.gson.JsonObject
11+
import java.awt.event.MouseEvent
12+
import java.net.HttpURLConnection
13+
import java.net.URL
14+
import java.util.concurrent.Executors
15+
import java.util.concurrent.TimeUnit
16+
17+
/**
18+
* Status bar widget that displays PicoCode indexing status
19+
*/
20+
class PicoCodeStatusBarWidget(private val project: Project) : StatusBarWidget,
21+
StatusBarWidget.TextPresentation {
22+
23+
companion object {
24+
const val ID = "PicoCodeStatusWidget"
25+
private const val DEFAULT_HOST = "localhost"
26+
private const val DEFAULT_PORT = 8000
27+
private const val POLLING_INTERVAL_SECONDS = 5L
28+
}
29+
30+
private val gson = Gson()
31+
private val executor = Executors.newSingleThreadScheduledExecutor()
32+
private var currentStatus: String = "Unknown"
33+
private var statusBar: StatusBar? = null
34+
private var projectId: String? = null
35+
private var indexingStats: IndexingStats? = null
36+
37+
data class IndexingStats(
38+
val fileCount: Int = 0,
39+
val embeddingCount: Int = 0,
40+
val isIndexed: Boolean = false
41+
)
42+
43+
init {
44+
// Start polling for status updates
45+
executor.scheduleAtFixedRate(
46+
{ updateStatus() },
47+
0,
48+
POLLING_INTERVAL_SECONDS,
49+
TimeUnit.SECONDS
50+
)
51+
}
52+
53+
override fun ID(): String = ID
54+
55+
override fun getPresentation(): StatusBarWidget.WidgetPresentation = this
56+
57+
override fun install(statusBar: StatusBar) {
58+
this.statusBar = statusBar
59+
}
60+
61+
override fun dispose() {
62+
executor.shutdown()
63+
try {
64+
executor.awaitTermination(5, TimeUnit.SECONDS)
65+
} catch (e: InterruptedException) {
66+
executor.shutdownNow()
67+
}
68+
}
69+
70+
override fun getText(): String {
71+
return when {
72+
currentStatus == "indexing" -> "⚡ PicoCode: Indexing..."
73+
currentStatus == "ready" && indexingStats?.isIndexed == true ->
74+
"✓ PicoCode: ${indexingStats?.fileCount ?: 0} files"
75+
currentStatus == "error" -> "✗ PicoCode: Error"
76+
currentStatus == "created" -> "○ PicoCode: Not indexed"
77+
else -> "PicoCode"
78+
}
79+
}
80+
81+
override fun getAlignment(): Float = 0.5f
82+
83+
override fun getTooltipText(): String? {
84+
return when {
85+
currentStatus == "indexing" -> "PicoCode is indexing your project..."
86+
currentStatus == "ready" && indexingStats != null ->
87+
"PicoCode: ${indexingStats?.fileCount} files, ${indexingStats?.embeddingCount} embeddings indexed"
88+
currentStatus == "error" -> "PicoCode indexing error occurred"
89+
currentStatus == "created" -> "PicoCode: Project created but not indexed yet"
90+
else -> "PicoCode status unknown - check if server is running"
91+
}
92+
}
93+
94+
override fun getClickConsumer(): Consumer<MouseEvent>? {
95+
return Consumer {
96+
// Optional: could open tool window or trigger re-indexing
97+
}
98+
}
99+
100+
private fun updateStatus() {
101+
ApplicationManager.getApplication().executeOnPooledThread {
102+
try {
103+
val projectPath = project.basePath ?: return@executeOnPooledThread
104+
105+
// Get or create project to get project ID
106+
if (projectId == null) {
107+
projectId = getOrCreateProject(projectPath)
108+
}
109+
110+
projectId?.let { id ->
111+
// Fetch project status
112+
val status = fetchProjectStatus(id)
113+
currentStatus = status.first
114+
indexingStats = status.second
115+
116+
// Update status bar on EDT
117+
ApplicationManager.getApplication().invokeLater {
118+
statusBar?.updateWidget(ID)
119+
}
120+
}
121+
} catch (e: Exception) {
122+
// Silently fail - don't spam logs if server is not running
123+
currentStatus = "Unavailable"
124+
}
125+
}
126+
}
127+
128+
private fun getOrCreateProject(projectPath: String): String? {
129+
return try {
130+
val url = URL("http://$DEFAULT_HOST:$DEFAULT_PORT/api/projects")
131+
val connection = url.openConnection() as HttpURLConnection
132+
connection.requestMethod = "POST"
133+
connection.setRequestProperty("Content-Type", "application/json")
134+
connection.doOutput = true
135+
136+
val body = gson.toJson(mapOf(
137+
"path" to projectPath,
138+
"name" to project.name
139+
))
140+
connection.outputStream.use { it.write(body.toByteArray()) }
141+
142+
val response = connection.inputStream.bufferedReader().readText()
143+
val json = gson.fromJson(response, JsonObject::class.java)
144+
json.get("id")?.asString
145+
} catch (e: Exception) {
146+
null
147+
}
148+
}
149+
150+
private fun fetchProjectStatus(projectId: String): Pair<String, IndexingStats?> {
151+
return try {
152+
val url = URL("http://$DEFAULT_HOST:$DEFAULT_PORT/api/projects/$projectId")
153+
val connection = url.openConnection() as HttpURLConnection
154+
connection.requestMethod = "GET"
155+
156+
val response = connection.inputStream.bufferedReader().readText()
157+
val json = gson.fromJson(response, JsonObject::class.java)
158+
159+
val status = json.get("status")?.asString ?: "unknown"
160+
val statsJson = json.getAsJsonObject("indexing_stats")
161+
162+
val stats = if (statsJson != null) {
163+
IndexingStats(
164+
fileCount = statsJson.get("file_count")?.asInt ?: 0,
165+
embeddingCount = statsJson.get("embedding_count")?.asInt ?: 0,
166+
isIndexed = statsJson.get("is_indexed")?.asBoolean ?: false
167+
)
168+
} else {
169+
null
170+
}
171+
172+
Pair(status, stats)
173+
} catch (e: Exception) {
174+
Pair("unavailable", null)
175+
}
176+
}
177+
}
178+
179+
/**
180+
* Factory for creating the status bar widget
181+
*/
182+
class PicoCodeStatusBarWidgetFactory : StatusBarWidgetFactory {
183+
override fun getId(): String = PicoCodeStatusBarWidget.ID
184+
185+
override fun getDisplayName(): String = "PicoCode Status"
186+
187+
override fun isAvailable(project: Project): Boolean = true
188+
189+
override fun createWidget(project: Project): StatusBarWidget {
190+
return PicoCodeStatusBarWidget(project)
191+
}
192+
193+
override fun disposeWidget(widget: StatusBarWidget) {
194+
// Disposal is handled by the widget itself
195+
}
196+
197+
override fun canBeEnabledOn(statusBar: StatusBar): Boolean = true
198+
}

0 commit comments

Comments
 (0)