saberbud/PocketFlow-Tutorial-Codebase-Knowledge
1
0
Fork 0
mirror of https://github.com/The-Pocket/PocketFlow-Tutorial-Codebase-Knowledge.git synced 2026-08-30 17:10:32 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' of https://github.com/redreamality/PocketFlow-Tutorial-Codebase-Knowledge

Browse Source
This commit is contained in:
remy
2025-05-01 15:14:03 +10:00
parent 31e45f3137 a5991a664f
commit c27c79a2c5
2 changed files with 117 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files
requirements.txt
+2 -1
View File
@@ -4,4 +4,5 @@ requests>=2.28.0
gitpython>=3.1.0 gitpython>=3.1.0
google-cloud-aiplatform>=1.25.0 google-cloud-aiplatform>=1.25.0
google-genai>=1.9.0 google-genai>=1.9.0
python-dotenv>=1.0.0 python-dotenv>=1.0.0
pathspec>=0.11.0
utils/crawl_local_files.py
+115 -74
View File
@@ -1,100 +1,141 @@
import os import os
import fnmatch import fnmatch
from tqdm import tqdm import pathspec
def crawl_local_files(directory, include_patterns=None, exclude_patterns=None, max_file_size=None, use_relative_paths=True):
def crawl_local_files(
directory,
include_patterns=None,
exclude_patterns=None,
max_file_size=None,
use_relative_paths=True,
):
""" """
Crawl files in a local directory with similar interface as crawl_github_files. Crawl files in a local directory with similar interface as crawl_github_files.
Implements efficient folder-level filtering to skip entire directories that match exclude patterns,
significantly improving performance when excluding large directory trees.
Args: Args:
directory (str): Path to local directory directory (str): Path to local directory
include_patterns (set): File patterns to include (e.g. {"*.py", "*.js"}) include_patterns (set): File patterns to include (e.g. {"*.py", "*.js"})
exclude_patterns (set): File patterns to exclude (e.g. {"tests/*"}) exclude_patterns (set): File patterns to exclude (e.g. {"tests/*"})
max_file_size (int): Maximum file size in bytes max_file_size (int): Maximum file size in bytes
use_relative_paths (bool): Whether to use paths relative to directory use_relative_paths (bool): Whether to use paths relative to directory
Returns: Returns:
dict: {"files": {filepath: content}} dict: {"files": {filepath: content}}
""" """
if not os.path.isdir(directory): if not os.path.isdir(directory):
raise ValueError(f"Directory does not exist: {directory}") raise ValueError(f"Directory does not exist: {directory}")
files_dict = {} files_dict = {}
# --- Load .gitignore ---
gitignore_path = os.path.join(directory, ".gitignore")
gitignore_spec = None
if os.path.exists(gitignore_path):
try:
with open(gitignore_path, "r", encoding="utf-8") as f:
gitignore_patterns = f.readlines()
gitignore_spec = pathspec.PathSpec.from_lines(
"gitwildmatch", gitignore_patterns
)
print(f"Loaded .gitignore patterns from {gitignore_path}")
except Exception as e:
print(
f"Warning: Could not read or parse .gitignore file {gitignore_path}: {e}"
)
# --- End Load .gitignore ---
for root, dirs, files in os.walk(directory): for root, dirs, files in os.walk(directory):
print(f"root: {root}") # Filter directories using .gitignore and exclude_patterns early to avoid descending
# Check if current directory should be excluded # Need to process dirs list *in place* for os.walk to respect it
if exclude_patterns: excluded_dirs = set()
for d in dirs:
dirpath_rel = os.path.relpath(os.path.join(root, d), directory)
# Check against .gitignore (important for directories)
if gitignore_spec and gitignore_spec.match_file(dirpath_rel):
excluded_dirs.add(d)
continue # Skip further checks if gitignored
# Check against standard exclude_patterns
if exclude_patterns:
for pattern in exclude_patterns:
# Match pattern against full relative path or directory name itself
if fnmatch.fnmatch(dirpath_rel, pattern) or fnmatch.fnmatch(
d, pattern
):
excluded_dirs.add(d)
break
# Modify dirs in-place: remove excluded ones
# Iterate over a copy (.copy()) because we are modifying the list during iteration
for d in dirs.copy():
if d in excluded_dirs:
dirs.remove(d)
# Now process files in the non-excluded directories
for filename in files:
filepath = os.path.join(root, filename)
# Get path relative to directory if requested # Get path relative to directory if requested
rel_root = os.path.relpath(root, directory) if use_relative_paths else root if use_relative_paths:
relpath = os.path.relpath(filepath, directory)
# Handle the case where rel_root is the current directory else:
if rel_root == '.': relpath = filepath
rel_root = ''
# --- Exclusion check ---
# Check if directory matches any exclude pattern excluded = False
for pattern in exclude_patterns: # 1. Check .gitignore first
# Normalize pattern to handle both forward and backward slashes if gitignore_spec and gitignore_spec.match_file(relpath):
norm_pattern = pattern.replace("/", os.path.sep) excluded = True
# Check if the directory matches the pattern # 2. Check standard exclude_patterns if not already excluded by .gitignore
if fnmatch.fnmatch(rel_root, norm_pattern) or \ if not excluded and exclude_patterns:
fnmatch.fnmatch(os.path.join(rel_root, ''), norm_pattern + os.path.sep): for pattern in exclude_patterns:
# Skip this directory and all subdirectories if fnmatch.fnmatch(relpath, pattern):
dirs[:] = [] # Clear dirs list to prevent further traversal excluded = True
print(f"Skipping directory: {rel_root} (matches pattern {pattern})") break
break
else: included = False
# print(root) if include_patterns:
for filename in files: for pattern in include_patterns:
filepath = os.path.join(root, filename) if fnmatch.fnmatch(relpath, pattern):
# Get path relative to directory if requested
if use_relative_paths:
relpath = os.path.relpath(filepath, directory)
else:
relpath = filepath
# Check if file matches any include pattern
included = False
if include_patterns:
for pattern in include_patterns:
if fnmatch.fnmatch(relpath, pattern):
included = True
break
else:
included = True included = True
break
# Check if file matches any exclude pattern else:
excluded = False # If no include patterns, include everything *not excluded*
if exclude_patterns: included = True
for pattern in exclude_patterns:
if fnmatch.fnmatch(relpath, pattern) or fnmatch.fnmatch(relpath, pattern.replace("/", "\\")): # Skip if not included or if excluded (by either method)
print(relpath, pattern) if not included or excluded:
excluded = True continue
break
# Check file size
if not included or excluded: if max_file_size and os.path.getsize(filepath) > max_file_size:
continue continue
# Check file size try:
if max_file_size and os.path.getsize(filepath) > max_file_size: with open(filepath, "r", encoding="utf-8") as f:
continue content = f.read()
files_dict[relpath] = content
try: except Exception as e:
with open(filepath, 'r', encoding='utf-8') as f: print(f"Warning: Could not read file {filepath}: {e}")
content = f.read()
files_dict[relpath] = content
except Exception as e:
print(f"Warning: Could not read file {filepath}: {e}")
return {"files": files_dict} return {"files": files_dict}
if __name__ == "__main__": if __name__ == "__main__":
print("--- Crawling parent directory ('..') ---") print("--- Crawling parent directory ('..') ---")
files_data = crawl_local_files("..", exclude_patterns={"*.pyc", "__pycache__/*",".venv/*", ".git/*","docs/*", "output/*"}) files_data = crawl_local_files(
"..",
exclude_patterns={
"*.pyc",
"__pycache__/*",
".venv/*",
".git/*",
"docs/*",
"output/*",
},
)
print(f"Found {len(files_data['files'])} files:") print(f"Found {len(files_data['files'])} files:")
for path in files_data["files"]: for path in files_data["files"]:
print(f" {path}") print(f" {path}")