diff --git a/nodes.py b/nodes.py index e6ed955..67749e5 100644 --- a/nodes.py +++ b/nodes.py @@ -61,12 +61,23 @@ class FetchRepo(Node): ) else: print(f"Crawling directory: {prep_res['local_dir']}...") + + def progress_callback(processed, total): + percentage = (processed / total) * 100 if total > 0 else 0 + rounded_percentage = int(percentage) + if rounded_percentage > progress_callback.last_reported: + progress_callback.last_reported = rounded_percentage + print(f"\033[92mProgress: {processed}/{total} files ({rounded_percentage}%)\033[0m") + + progress_callback.last_reported = -1 + result = crawl_local_files( directory=prep_res["local_dir"], include_patterns=prep_res["include_patterns"], exclude_patterns=prep_res["exclude_patterns"], max_file_size=prep_res["max_file_size"], use_relative_paths=prep_res["use_relative_paths"], + progress_callback=progress_callback ) # Convert dict to list of tuples: [(path, content), ...] diff --git a/utils/crawl_local_files.py b/utils/crawl_local_files.py index 64fe132..722078f 100644 --- a/utils/crawl_local_files.py +++ b/utils/crawl_local_files.py @@ -2,13 +2,13 @@ import os import fnmatch import pathspec - def crawl_local_files( directory, include_patterns=None, exclude_patterns=None, max_file_size=None, use_relative_paths=True, + progress_callback=None, ): """ Crawl files in a local directory with similar interface as crawl_github_files. @@ -18,6 +18,7 @@ def crawl_local_files( exclude_patterns (set): File patterns to exclude (e.g. {"tests/*"}) max_file_size (int): Maximum file size in bytes use_relative_paths (bool): Whether to use paths relative to directory + progress_callback (callable): Function to report progress, takes (processed, total) as arguments Returns: dict: {"files": {filepath: content}} @@ -34,91 +35,84 @@ def crawl_local_files( try: with open(gitignore_path, "r", encoding="utf-8") as f: gitignore_patterns = f.readlines() - gitignore_spec = pathspec.PathSpec.from_lines( - "gitwildmatch", gitignore_patterns - ) + gitignore_spec = pathspec.PathSpec.from_lines("gitwildmatch", gitignore_patterns) print(f"Loaded .gitignore patterns from {gitignore_path}") except Exception as e: - print( - f"Warning: Could not read or parse .gitignore file {gitignore_path}: {e}" - ) - # --- End Load .gitignore --- + print(f"Warning: Could not read or parse .gitignore file {gitignore_path}: {e}") + all_files = [] for root, dirs, files in os.walk(directory): - # Filter directories using .gitignore and exclude_patterns early to avoid descending - # Need to process dirs list *in place* for os.walk to respect it + # Filter directories using .gitignore and exclude_patterns early excluded_dirs = set() for d in dirs: dirpath_rel = os.path.relpath(os.path.join(root, d), directory) - # Check against .gitignore (important for directories) if gitignore_spec and gitignore_spec.match_file(dirpath_rel): excluded_dirs.add(d) - continue # Skip further checks if gitignored + continue - # Check against standard exclude_patterns if exclude_patterns: for pattern in exclude_patterns: - # Match pattern against full relative path or directory name itself - if fnmatch.fnmatch(dirpath_rel, pattern) or fnmatch.fnmatch( - d, pattern - ): + if fnmatch.fnmatch(dirpath_rel, pattern) or fnmatch.fnmatch(d, pattern): excluded_dirs.add(d) break - # Modify dirs in-place: remove excluded ones - # Iterate over a copy (.copy()) because we are modifying the list during iteration for d in dirs.copy(): if d in excluded_dirs: dirs.remove(d) - # Now process files in the non-excluded directories for filename in files: filepath = os.path.join(root, filename) + all_files.append(filepath) - # Get path relative to directory if requested - if use_relative_paths: - relpath = os.path.relpath(filepath, directory) - else: - relpath = filepath + total_files = len(all_files) + processed_files = 0 - # --- Exclusion check --- - excluded = False - # 1. Check .gitignore first - if gitignore_spec and gitignore_spec.match_file(relpath): - excluded = True + for filepath in all_files: + relpath = os.path.relpath(filepath, directory) if use_relative_paths else filepath - # 2. Check standard exclude_patterns if not already excluded by .gitignore - if not excluded and exclude_patterns: - for pattern in exclude_patterns: - if fnmatch.fnmatch(relpath, pattern): - excluded = True - break + # --- Exclusion check --- + excluded = False + if gitignore_spec and gitignore_spec.match_file(relpath): + excluded = True - included = False - if include_patterns: - for pattern in include_patterns: - if fnmatch.fnmatch(relpath, pattern): - included = True - break - else: - # If no include patterns, include everything *not excluded* - included = True + if not excluded and exclude_patterns: + for pattern in exclude_patterns: + if fnmatch.fnmatch(relpath, pattern): + excluded = True + break - # Skip if not included or if excluded (by either method) - if not included or excluded: - continue + included = False + if include_patterns: + for pattern in include_patterns: + if fnmatch.fnmatch(relpath, pattern): + included = True + break + else: + included = True - # Check file size - if max_file_size and os.path.getsize(filepath) > max_file_size: - continue + if not included or excluded: + processed_files += 1 + if progress_callback: + progress_callback(processed_files, total_files) + continue - try: - with open(filepath, "r", encoding="utf-8") as f: - content = f.read() - files_dict[relpath] = content - except Exception as e: - print(f"Warning: Could not read file {filepath}: {e}") + if max_file_size and os.path.getsize(filepath) > max_file_size: + processed_files += 1 + if progress_callback: + progress_callback(processed_files, total_files) + continue + + try: + with open(filepath, "r", encoding="utf-8") as f: + content = f.read() + files_dict[relpath] = content + except Exception as e: + print(f"Warning: Could not read file {filepath}: {e}") + + processed_files += 1 + if progress_callback: + progress_callback(processed_files, total_files) return {"files": files_dict} @@ -138,4 +132,4 @@ if __name__ == "__main__": ) print(f"Found {len(files_data['files'])} files:") for path in files_data["files"]: - print(f" {path}") + print(f" {path}") \ No newline at end of file