#feat: Report progress of the crawling #64

nodes.py

@@ -56,12 +56,23 @@ class FetchRepo(Node):
             )
         else:
             print(f"Crawling directory: {prep_res['local_dir']}...")
+
+            def progress_callback(processed, total):
+                percentage = (processed / total) * 100 if total > 0 else 0
+                rounded_percentage = int(percentage)
+                if rounded_percentage > progress_callback.last_reported:
+                    progress_callback.last_reported = rounded_percentage
+                    print(f"\033[92mProgress: {processed}/{total} files ({rounded_percentage}%)\033[0m")
+
+            progress_callback.last_reported = -1
+
             result = crawl_local_files(
                 directory=prep_res["local_dir"],
                 include_patterns=prep_res["include_patterns"],
                 exclude_patterns=prep_res["exclude_patterns"],
                 max_file_size=prep_res["max_file_size"],
-                use_relative_paths=prep_res["use_relative_paths"]
+                use_relative_paths=prep_res["use_relative_paths"],
+                progress_callback=progress_callback
             )
 
         # Convert dict to list of tuples: [(path, content), ...]

utils/crawl_local_files.py

@@ -1,7 +1,7 @@
 import os
 import fnmatch
 
-def crawl_local_files(directory, include_patterns=None, exclude_patterns=None, max_file_size=None, use_relative_paths=True):
+def crawl_local_files(directory, include_patterns=None, exclude_patterns=None, max_file_size=None, use_relative_paths=True, progress_callback=None):
     """
     Crawl files in a local directory with similar interface as crawl_github_files.
     
@@ -11,6 +11,7 @@ def crawl_local_files(directory, include_patterns=None, exclude_patterns=None, m
         exclude_patterns (set): File patterns to exclude (e.g. {"tests/*"})
         max_file_size (int): Maximum file size in bytes
         use_relative_paths (bool): Whether to use paths relative to directory
+        progress_callback (callable): Function to report progress, takes (processed, total) as arguments
         
     Returns:
         dict: {"files": {filepath: content}}
@@ -19,11 +20,18 @@ def crawl_local_files(directory, include_patterns=None, exclude_patterns=None, m
         raise ValueError(f"Directory does not exist: {directory}")
         
     files_dict = {}
+    all_files = []
 
+    # Collect all files first to calculate total
     for root, _, files in os.walk(directory):
         for filename in files:
             filepath = os.path.join(root, filename)
+            all_files.append(filepath)
 
+    total_files = len(all_files)
+    processed_files = 0
+
+    for filepath in all_files:
         # Get path relative to directory if requested
         if use_relative_paths:
             relpath = os.path.relpath(filepath, directory)
@@ -49,10 +57,16 @@ def crawl_local_files(directory, include_patterns=None, exclude_patterns=None, m
                     break
 
         if not included or excluded:
+            processed_files += 1
+            if progress_callback:
+                progress_callback(processed_files, total_files)
             continue
 
         # Check file size
         if max_file_size and os.path.getsize(filepath) > max_file_size:
+            processed_files += 1
+            if progress_callback:
+                progress_callback(processed_files, total_files)
             continue
 
         try:
@@ -62,6 +76,10 @@ def crawl_local_files(directory, include_patterns=None, exclude_patterns=None, m
         except Exception as e:
             print(f"Warning: Could not read file {filepath}: {e}")
 
+        processed_files += 1
+        if progress_callback:
+            progress_callback(processed_files, total_files)
+
     return {"files": files_dict}
 
 if __name__ == "__main__":