add pathspec library and update crawl_local_files to ignore contents of .gitignore file

requirements.txt

@@ -5,3 +5,4 @@ gitpython>=3.1.0
 google-cloud-aiplatform>=1.25.0
 google-genai>=1.9.0
 python-dotenv>=1.0.0
+pathspec>=0.11.0

utils/crawl_local_files.py

@@ -1,5 +1,6 @@
 import os
 import fnmatch
+import pathspec
 
 def crawl_local_files(directory, include_patterns=None, exclude_patterns=None, max_file_size=None, use_relative_paths=True):
     """
@@ -20,7 +21,46 @@ def crawl_local_files(directory, include_patterns=None, exclude_patterns=None, m
 
     files_dict = {}
 
-    for root, _, files in os.walk(directory):
+    # --- Load .gitignore ---
+    gitignore_path = os.path.join(directory, '.gitignore')
+    gitignore_spec = None
+    if os.path.exists(gitignore_path):
+        try:
+            with open(gitignore_path, 'r', encoding='utf-8') as f:
+                gitignore_patterns = f.readlines()
+            gitignore_spec = pathspec.PathSpec.from_lines('gitwildmatch', gitignore_patterns)
+            print(f"Loaded .gitignore patterns from {gitignore_path}")
+        except Exception as e:
+            print(f"Warning: Could not read or parse .gitignore file {gitignore_path}: {e}")
+    # --- End Load .gitignore ---
+
+    for root, dirs, files in os.walk(directory):
+        # Filter directories using .gitignore and exclude_patterns early to avoid descending
+        # Need to process dirs list *in place* for os.walk to respect it
+        excluded_dirs = set()
+        for d in dirs:
+            dirpath_rel = os.path.relpath(os.path.join(root, d), directory)
+
+            # Check against .gitignore (important for directories)
+            if gitignore_spec and gitignore_spec.match_file(dirpath_rel):
+                excluded_dirs.add(d)
+                continue # Skip further checks if gitignored
+
+            # Check against standard exclude_patterns
+            if exclude_patterns:
+                for pattern in exclude_patterns:
+                    # Match pattern against full relative path or directory name itself
+                    if fnmatch.fnmatch(dirpath_rel, pattern) or fnmatch.fnmatch(d, pattern):
+                         excluded_dirs.add(d)
+                         break
+
+        # Modify dirs in-place: remove excluded ones
+        # Iterate over a copy (.copy()) because we are modifying the list during iteration
+        for d in dirs.copy():
+            if d in excluded_dirs:
+                dirs.remove(d)
+
+        # Now process files in the non-excluded directories
         for filename in files:
             filepath = os.path.join(root, filename)
 
@@ -30,7 +70,20 @@ def crawl_local_files(directory, include_patterns=None, exclude_patterns=None, m
             else:
                 relpath = filepath
 
-            # Check if file matches any include pattern
+            # --- Exclusion check ---
+            excluded = False
+            # 1. Check .gitignore first
+            if gitignore_spec and gitignore_spec.match_file(relpath):
+                excluded = True
+
+            # 2. Check standard exclude_patterns if not already excluded by .gitignore
+            if not excluded and exclude_patterns:
+                for pattern in exclude_patterns:
+                    if fnmatch.fnmatch(relpath, pattern):
+                        excluded = True
+                        break
+
+            # --- Inclusion Check (remains the same) ---
             included = False
             if include_patterns:
                 for pattern in include_patterns:
@@ -38,18 +91,13 @@ def crawl_local_files(directory, include_patterns=None, exclude_patterns=None, m
                         included = True
                         break
             else:
+                # If no include patterns, include everything *not excluded*
                 included = True
 
-            # Check if file matches any exclude pattern
-            excluded = False
-            if exclude_patterns:
-                for pattern in exclude_patterns:
-                    if fnmatch.fnmatch(relpath, pattern):
-                        excluded = True
-                        break
-                        
+            # Skip if not included or if excluded (by either method)
             if not included or excluded:
                 continue
+            # --- End Exclusion/Inclusion Logic ---
 
             # Check file size
             if max_file_size and os.path.getsize(filepath) > max_file_size: