Merge branch 'main' of https://github.com/redreamality/PocketFlow-Tutorial-Codebase-Knowledge

requirements.txt

@@ -5,3 +5,4 @@ gitpython>=3.1.0
 google-cloud-aiplatform>=1.25.0
 google-genai>=1.9.0
 python-dotenv>=1.0.0
+pathspec>=0.11.0

utils/crawl_local_files.py

@@ -1,13 +1,17 @@
 import os
 import fnmatch
-from tqdm import tqdm
+import pathspec
 
-def crawl_local_files(directory, include_patterns=None, exclude_patterns=None, max_file_size=None, use_relative_paths=True):
+
+def crawl_local_files(
+    directory,
+    include_patterns=None,
+    exclude_patterns=None,
+    max_file_size=None,
+    use_relative_paths=True,
+):
     """
     Crawl files in a local directory with similar interface as crawl_github_files.
-    Implements efficient folder-level filtering to skip entire directories that match exclude patterns,
-    significantly improving performance when excluding large directory trees.
-    
     Args:
         directory (str): Path to local directory
         include_patterns (set): File patterns to include (e.g. {"*.py", "*.js"})
@@ -23,78 +27,115 @@ def crawl_local_files(directory, include_patterns=None, exclude_patterns=None, m
 
     files_dict = {}
 
+    # --- Load .gitignore ---
+    gitignore_path = os.path.join(directory, ".gitignore")
+    gitignore_spec = None
+    if os.path.exists(gitignore_path):
+        try:
+            with open(gitignore_path, "r", encoding="utf-8") as f:
+                gitignore_patterns = f.readlines()
+            gitignore_spec = pathspec.PathSpec.from_lines(
+                "gitwildmatch", gitignore_patterns
+            )
+            print(f"Loaded .gitignore patterns from {gitignore_path}")
+        except Exception as e:
+            print(
+                f"Warning: Could not read or parse .gitignore file {gitignore_path}: {e}"
+            )
+    # --- End Load .gitignore ---
+
     for root, dirs, files in os.walk(directory):
-        print(f"root: {root}")
+        # Filter directories using .gitignore and exclude_patterns early to avoid descending
-        # Check if current directory should be excluded
+        # Need to process dirs list *in place* for os.walk to respect it
-        if exclude_patterns:
+        excluded_dirs = set()
+        for d in dirs:
+            dirpath_rel = os.path.relpath(os.path.join(root, d), directory)
+
+            # Check against .gitignore (important for directories)
+            if gitignore_spec and gitignore_spec.match_file(dirpath_rel):
+                excluded_dirs.add(d)
+                continue  # Skip further checks if gitignored
+
+            # Check against standard exclude_patterns
+            if exclude_patterns:
+                for pattern in exclude_patterns:
+                    # Match pattern against full relative path or directory name itself
+                    if fnmatch.fnmatch(dirpath_rel, pattern) or fnmatch.fnmatch(
+                        d, pattern
+                    ):
+                        excluded_dirs.add(d)
+                        break
+
+        # Modify dirs in-place: remove excluded ones
+        # Iterate over a copy (.copy()) because we are modifying the list during iteration
+        for d in dirs.copy():
+            if d in excluded_dirs:
+                dirs.remove(d)
+
+        # Now process files in the non-excluded directories
+        for filename in files:
+            filepath = os.path.join(root, filename)
+
             # Get path relative to directory if requested
-            rel_root = os.path.relpath(root, directory) if use_relative_paths else root
+            if use_relative_paths:
-            
+                relpath = os.path.relpath(filepath, directory)
-            # Handle the case where rel_root is the current directory
-            if rel_root == '.':
-                rel_root = ''
-            
-            # Check if directory matches any exclude pattern
-            for pattern in exclude_patterns:
-                # Normalize pattern to handle both forward and backward slashes
-                norm_pattern = pattern.replace("/", os.path.sep)
-                
-                # Check if the directory matches the pattern
-                if fnmatch.fnmatch(rel_root, norm_pattern) or \
-                   fnmatch.fnmatch(os.path.join(rel_root, ''), norm_pattern + os.path.sep):
-                    # Skip this directory and all subdirectories
-                    dirs[:] = []  # Clear dirs list to prevent further traversal
-                    print(f"Skipping directory: {rel_root} (matches pattern {pattern})")
-                    break
             else:
-            # print(root)
+                relpath = filepath
-                for filename in files:
-                    filepath = os.path.join(root, filename)
 
-                    # Get path relative to directory if requested
+            # --- Exclusion check ---
-                    if use_relative_paths:
+            excluded = False
-                        relpath = os.path.relpath(filepath, directory)
+            # 1. Check .gitignore first
-                    else:
+            if gitignore_spec and gitignore_spec.match_file(relpath):
-                        relpath = filepath
+                excluded = True
 
-                    # Check if file matches any include pattern
+            # 2. Check standard exclude_patterns if not already excluded by .gitignore
-                    included = False
+            if not excluded and exclude_patterns:
-                    if include_patterns:
+                for pattern in exclude_patterns:
-                        for pattern in include_patterns:
+                    if fnmatch.fnmatch(relpath, pattern):
-                            if fnmatch.fnmatch(relpath, pattern):
+                        excluded = True
-                                included = True
+                        break
-                                break
+
-                    else:
+            included = False
+            if include_patterns:
+                for pattern in include_patterns:
+                    if fnmatch.fnmatch(relpath, pattern):
                         included = True
+                        break
+            else:
+                # If no include patterns, include everything *not excluded*
+                included = True
 
-                    # Check if file matches any exclude pattern
+            # Skip if not included or if excluded (by either method)
-                    excluded = False
+            if not included or excluded:
-                    if exclude_patterns:
+                continue
-                        for pattern in exclude_patterns:
-                            if fnmatch.fnmatch(relpath, pattern) or fnmatch.fnmatch(relpath, pattern.replace("/", "\\")):
-                                print(relpath, pattern)
-                                excluded = True
-                                break
 
-                    if not included or excluded:
+            # Check file size
-                        continue
+            if max_file_size and os.path.getsize(filepath) > max_file_size:
+                continue
 
-                    # Check file size
+            try:
-                    if max_file_size and os.path.getsize(filepath) > max_file_size:
+                with open(filepath, "r", encoding="utf-8") as f:
-                        continue
+                    content = f.read()
-                        
+                files_dict[relpath] = content
-                    try:
+            except Exception as e:
-                        with open(filepath, 'r', encoding='utf-8') as f:
+                print(f"Warning: Could not read file {filepath}: {e}")
-                            content = f.read()
-                        files_dict[relpath] = content
-                    except Exception as e:
-                        print(f"Warning: Could not read file {filepath}: {e}")
 
     return {"files": files_dict}
 
+
 if __name__ == "__main__":
     print("--- Crawling parent directory ('..') ---")
-    files_data = crawl_local_files("..", exclude_patterns={"*.pyc", "__pycache__/*",".venv/*", ".git/*","docs/*", "output/*"})
+    files_data = crawl_local_files(
+        "..",
+        exclude_patterns={
+            "*.pyc",
+            "__pycache__/*",
+            ".venv/*",
+            ".git/*",
+            "docs/*",
+            "output/*",
+        },
+    )
     print(f"Found {len(files_data['files'])} files:")
     for path in files_data["files"]:
         print(f"  {path}")