mirror of
https://github.com/The-Pocket/PocketFlow-Tutorial-Codebase-Knowledge.git
synced 2026-08-29 08:34:31 +08:00
Merge branch 'main' of https://github.com/redreamality/PocketFlow-Tutorial-Codebase-Knowledge
This commit is contained in:
remy
@@ -5,3 +5,4 @@ gitpython>=3.1.0
|
||||
google-cloud-aiplatform>=1.25.0
|
||||
google-genai>=1.9.0
|
||||
python-dotenv>=1.0.0
|
||||
pathspec>=0.11.0
|
||||
|
||||
+104
-63
@@ -1,13 +1,17 @@
|
||||
import os
|
||||
import fnmatch
|
||||
from tqdm import tqdm
|
||||
import pathspec
|
||||
|
||||
def crawl_local_files(directory, include_patterns=None, exclude_patterns=None, max_file_size=None, use_relative_paths=True):
|
||||
|
||||
def crawl_local_files(
|
||||
directory,
|
||||
include_patterns=None,
|
||||
exclude_patterns=None,
|
||||
max_file_size=None,
|
||||
use_relative_paths=True,
|
||||
):
|
||||
"""
|
||||
Crawl files in a local directory with similar interface as crawl_github_files.
|
||||
Implements efficient folder-level filtering to skip entire directories that match exclude patterns,
|
||||
significantly improving performance when excluding large directory trees.
|
||||
|
||||
Args:
|
||||
directory (str): Path to local directory
|
||||
include_patterns (set): File patterns to include (e.g. {"*.py", "*.js"})
|
||||
@@ -23,78 +27,115 @@ def crawl_local_files(directory, include_patterns=None, exclude_patterns=None, m
|
||||
|
||||
files_dict = {}
|
||||
|
||||
# --- Load .gitignore ---
|
||||
gitignore_path = os.path.join(directory, ".gitignore")
|
||||
gitignore_spec = None
|
||||
if os.path.exists(gitignore_path):
|
||||
try:
|
||||
with open(gitignore_path, "r", encoding="utf-8") as f:
|
||||
gitignore_patterns = f.readlines()
|
||||
gitignore_spec = pathspec.PathSpec.from_lines(
|
||||
"gitwildmatch", gitignore_patterns
|
||||
)
|
||||
print(f"Loaded .gitignore patterns from {gitignore_path}")
|
||||
except Exception as e:
|
||||
print(
|
||||
f"Warning: Could not read or parse .gitignore file {gitignore_path}: {e}"
|
||||
)
|
||||
# --- End Load .gitignore ---
|
||||
|
||||
for root, dirs, files in os.walk(directory):
|
||||
print(f"root: {root}")
|
||||
# Check if current directory should be excluded
|
||||
if exclude_patterns:
|
||||
# Filter directories using .gitignore and exclude_patterns early to avoid descending
|
||||
# Need to process dirs list *in place* for os.walk to respect it
|
||||
excluded_dirs = set()
|
||||
for d in dirs:
|
||||
dirpath_rel = os.path.relpath(os.path.join(root, d), directory)
|
||||
|
||||
# Check against .gitignore (important for directories)
|
||||
if gitignore_spec and gitignore_spec.match_file(dirpath_rel):
|
||||
excluded_dirs.add(d)
|
||||
continue # Skip further checks if gitignored
|
||||
|
||||
# Check against standard exclude_patterns
|
||||
if exclude_patterns:
|
||||
for pattern in exclude_patterns:
|
||||
# Match pattern against full relative path or directory name itself
|
||||
if fnmatch.fnmatch(dirpath_rel, pattern) or fnmatch.fnmatch(
|
||||
d, pattern
|
||||
):
|
||||
excluded_dirs.add(d)
|
||||
break
|
||||
|
||||
# Modify dirs in-place: remove excluded ones
|
||||
# Iterate over a copy (.copy()) because we are modifying the list during iteration
|
||||
for d in dirs.copy():
|
||||
if d in excluded_dirs:
|
||||
dirs.remove(d)
|
||||
|
||||
# Now process files in the non-excluded directories
|
||||
for filename in files:
|
||||
filepath = os.path.join(root, filename)
|
||||
|
||||
# Get path relative to directory if requested
|
||||
rel_root = os.path.relpath(root, directory) if use_relative_paths else root
|
||||
|
||||
# Handle the case where rel_root is the current directory
|
||||
if rel_root == '.':
|
||||
rel_root = ''
|
||||
|
||||
# Check if directory matches any exclude pattern
|
||||
for pattern in exclude_patterns:
|
||||
# Normalize pattern to handle both forward and backward slashes
|
||||
norm_pattern = pattern.replace("/", os.path.sep)
|
||||
|
||||
# Check if the directory matches the pattern
|
||||
if fnmatch.fnmatch(rel_root, norm_pattern) or \
|
||||
fnmatch.fnmatch(os.path.join(rel_root, ''), norm_pattern + os.path.sep):
|
||||
# Skip this directory and all subdirectories
|
||||
dirs[:] = [] # Clear dirs list to prevent further traversal
|
||||
print(f"Skipping directory: {rel_root} (matches pattern {pattern})")
|
||||
break
|
||||
if use_relative_paths:
|
||||
relpath = os.path.relpath(filepath, directory)
|
||||
else:
|
||||
# print(root)
|
||||
for filename in files:
|
||||
filepath = os.path.join(root, filename)
|
||||
relpath = filepath
|
||||
|
||||
# Get path relative to directory if requested
|
||||
if use_relative_paths:
|
||||
relpath = os.path.relpath(filepath, directory)
|
||||
else:
|
||||
relpath = filepath
|
||||
# --- Exclusion check ---
|
||||
excluded = False
|
||||
# 1. Check .gitignore first
|
||||
if gitignore_spec and gitignore_spec.match_file(relpath):
|
||||
excluded = True
|
||||
|
||||
# Check if file matches any include pattern
|
||||
included = False
|
||||
if include_patterns:
|
||||
for pattern in include_patterns:
|
||||
if fnmatch.fnmatch(relpath, pattern):
|
||||
included = True
|
||||
break
|
||||
else:
|
||||
# 2. Check standard exclude_patterns if not already excluded by .gitignore
|
||||
if not excluded and exclude_patterns:
|
||||
for pattern in exclude_patterns:
|
||||
if fnmatch.fnmatch(relpath, pattern):
|
||||
excluded = True
|
||||
break
|
||||
|
||||
included = False
|
||||
if include_patterns:
|
||||
for pattern in include_patterns:
|
||||
if fnmatch.fnmatch(relpath, pattern):
|
||||
included = True
|
||||
break
|
||||
else:
|
||||
# If no include patterns, include everything *not excluded*
|
||||
included = True
|
||||
|
||||
# Check if file matches any exclude pattern
|
||||
excluded = False
|
||||
if exclude_patterns:
|
||||
for pattern in exclude_patterns:
|
||||
if fnmatch.fnmatch(relpath, pattern) or fnmatch.fnmatch(relpath, pattern.replace("/", "\\")):
|
||||
print(relpath, pattern)
|
||||
excluded = True
|
||||
break
|
||||
# Skip if not included or if excluded (by either method)
|
||||
if not included or excluded:
|
||||
continue
|
||||
|
||||
if not included or excluded:
|
||||
continue
|
||||
# Check file size
|
||||
if max_file_size and os.path.getsize(filepath) > max_file_size:
|
||||
continue
|
||||
|
||||
# Check file size
|
||||
if max_file_size and os.path.getsize(filepath) > max_file_size:
|
||||
continue
|
||||
|
||||
try:
|
||||
with open(filepath, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
files_dict[relpath] = content
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not read file {filepath}: {e}")
|
||||
try:
|
||||
with open(filepath, "r", encoding="utf-8") as f:
|
||||
content = f.read()
|
||||
files_dict[relpath] = content
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not read file {filepath}: {e}")
|
||||
|
||||
return {"files": files_dict}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("--- Crawling parent directory ('..') ---")
|
||||
files_data = crawl_local_files("..", exclude_patterns={"*.pyc", "__pycache__/*",".venv/*", ".git/*","docs/*", "output/*"})
|
||||
files_data = crawl_local_files(
|
||||
"..",
|
||||
exclude_patterns={
|
||||
"*.pyc",
|
||||
"__pycache__/*",
|
||||
".venv/*",
|
||||
".git/*",
|
||||
"docs/*",
|
||||
"output/*",
|
||||
},
|
||||
)
|
||||
print(f"Found {len(files_data['files'])} files:")
|
||||
for path in files_data["files"]:
|
||||
print(f" {path}")
|
||||
Reference in New Issue
Block a user