Merge remote-tracking branch 'upstream/main' into crawling-progress

utils/crawl_local_files.py

@@ -1,10 +1,17 @@
 import os
 import fnmatch
+import pathspec
 
-def crawl_local_files(directory, include_patterns=None, exclude_patterns=None, max_file_size=None, use_relative_paths=True, progress_callback=None):
+def crawl_local_files(
+    directory,
+    include_patterns=None,
+    exclude_patterns=None,
+    max_file_size=None,
+    use_relative_paths=True,
+    progress_callback=None,
+):
     """
     Crawl files in a local directory with similar interface as crawl_github_files.
-    
     Args:
         directory (str): Path to local directory
         include_patterns (set): File patterns to include (e.g. {"*.py", "*.js"})
@@ -12,18 +19,48 @@ def crawl_local_files(directory, include_patterns=None, exclude_patterns=None, m
         max_file_size (int): Maximum file size in bytes
         use_relative_paths (bool): Whether to use paths relative to directory
         progress_callback (callable): Function to report progress, takes (processed, total) as arguments
-        
+
     Returns:
         dict: {"files": {filepath: content}}
     """
     if not os.path.isdir(directory):
         raise ValueError(f"Directory does not exist: {directory}")
-        
-    files_dict = {}
-    all_files = []
 
-    # Collect all files first to calculate total
-    for root, _, files in os.walk(directory):
+    files_dict = {}
+
+    # --- Load .gitignore ---
+    gitignore_path = os.path.join(directory, ".gitignore")
+    gitignore_spec = None
+    if os.path.exists(gitignore_path):
+        try:
+            with open(gitignore_path, "r", encoding="utf-8") as f:
+                gitignore_patterns = f.readlines()
+            gitignore_spec = pathspec.PathSpec.from_lines("gitwildmatch", gitignore_patterns)
+            print(f"Loaded .gitignore patterns from {gitignore_path}")
+        except Exception as e:
+            print(f"Warning: Could not read or parse .gitignore file {gitignore_path}: {e}")
+
+    all_files = []
+    for root, dirs, files in os.walk(directory):
+        # Filter directories using .gitignore and exclude_patterns early
+        excluded_dirs = set()
+        for d in dirs:
+            dirpath_rel = os.path.relpath(os.path.join(root, d), directory)
+
+            if gitignore_spec and gitignore_spec.match_file(dirpath_rel):
+                excluded_dirs.add(d)
+                continue
+
+            if exclude_patterns:
+                for pattern in exclude_patterns:
+                    if fnmatch.fnmatch(dirpath_rel, pattern) or fnmatch.fnmatch(d, pattern):
+                        excluded_dirs.add(d)
+                        break
+
+        for d in dirs.copy():
+            if d in excluded_dirs:
+                dirs.remove(d)
+
         for filename in files:
             filepath = os.path.join(root, filename)
             all_files.append(filepath)
@@ -32,13 +69,19 @@ def crawl_local_files(directory, include_patterns=None, exclude_patterns=None, m
     processed_files = 0
 
     for filepath in all_files:
-        # Get path relative to directory if requested
-        if use_relative_paths:
-            relpath = os.path.relpath(filepath, directory)
-        else:
-            relpath = filepath
+        relpath = os.path.relpath(filepath, directory) if use_relative_paths else filepath
+
+        # --- Exclusion check ---
+        excluded = False
+        if gitignore_spec and gitignore_spec.match_file(relpath):
+            excluded = True
+
+        if not excluded and exclude_patterns:
+            for pattern in exclude_patterns:
+                if fnmatch.fnmatch(relpath, pattern):
+                    excluded = True
+                    break
 
-        # Check if file matches any include pattern
         included = False
         if include_patterns:
             for pattern in include_patterns:
@@ -48,21 +91,12 @@ def crawl_local_files(directory, include_patterns=None, exclude_patterns=None, m
         else:
             included = True
 
-        # Check if file matches any exclude pattern
-        excluded = False
-        if exclude_patterns:
-            for pattern in exclude_patterns:
-                if fnmatch.fnmatch(relpath, pattern):
-                    excluded = True
-                    break
-
         if not included or excluded:
             processed_files += 1
             if progress_callback:
                 progress_callback(processed_files, total_files)
             continue
 
-        # Check file size
         if max_file_size and os.path.getsize(filepath) > max_file_size:
             processed_files += 1
             if progress_callback:
@@ -70,7 +104,7 @@ def crawl_local_files(directory, include_patterns=None, exclude_patterns=None, m
             continue
 
         try:
-            with open(filepath, 'r', encoding='utf-8') as f:
+            with open(filepath, "r", encoding="utf-8") as f:
                 content = f.read()
             files_dict[relpath] = content
         except Exception as e:
@@ -82,9 +116,20 @@ def crawl_local_files(directory, include_patterns=None, exclude_patterns=None, m
 
     return {"files": files_dict}
 
+
 if __name__ == "__main__":
     print("--- Crawling parent directory ('..') ---")
-    files_data = crawl_local_files("..", exclude_patterns={"*.pyc", "__pycache__/*", ".git/*", "output/*"})
+    files_data = crawl_local_files(
+        "..",
+        exclude_patterns={
+            "*.pyc",
+            "__pycache__/*",
+            ".venv/*",
+            ".git/*",
+            "docs/*",
+            "output/*",
+        },
+    )
     print(f"Found {len(files_data['files'])} files:")
     for path in files_data["files"]:
         print(f"  {path}")