diff --git a/nodes.py b/nodes.py index 67ab034..61ec11f 100644 --- a/nodes.py +++ b/nodes.py @@ -56,12 +56,23 @@ class FetchRepo(Node): ) else: print(f"Crawling directory: {prep_res['local_dir']}...") + + def progress_callback(processed, total): + percentage = (processed / total) * 100 if total > 0 else 0 + rounded_percentage = int(percentage) + if rounded_percentage > progress_callback.last_reported: + progress_callback.last_reported = rounded_percentage + print(f"\033[92mProgress: {processed}/{total} files ({rounded_percentage}%)\033[0m") + + progress_callback.last_reported = -1 + result = crawl_local_files( directory=prep_res["local_dir"], include_patterns=prep_res["include_patterns"], exclude_patterns=prep_res["exclude_patterns"], max_file_size=prep_res["max_file_size"], - use_relative_paths=prep_res["use_relative_paths"] + use_relative_paths=prep_res["use_relative_paths"], + progress_callback=progress_callback ) # Convert dict to list of tuples: [(path, content), ...] diff --git a/utils/crawl_local_files.py b/utils/crawl_local_files.py index 837215c..b3ac49b 100644 --- a/utils/crawl_local_files.py +++ b/utils/crawl_local_files.py @@ -1,7 +1,7 @@ import os import fnmatch -def crawl_local_files(directory, include_patterns=None, exclude_patterns=None, max_file_size=None, use_relative_paths=True): +def crawl_local_files(directory, include_patterns=None, exclude_patterns=None, max_file_size=None, use_relative_paths=True, progress_callback=None): """ Crawl files in a local directory with similar interface as crawl_github_files. @@ -11,6 +11,7 @@ def crawl_local_files(directory, include_patterns=None, exclude_patterns=None, m exclude_patterns (set): File patterns to exclude (e.g. {"tests/*"}) max_file_size (int): Maximum file size in bytes use_relative_paths (bool): Whether to use paths relative to directory + progress_callback (callable): Function to report progress, takes (processed, total) as arguments Returns: dict: {"files": {filepath: content}} @@ -19,49 +20,66 @@ def crawl_local_files(directory, include_patterns=None, exclude_patterns=None, m raise ValueError(f"Directory does not exist: {directory}") files_dict = {} - + all_files = [] + + # Collect all files first to calculate total for root, _, files in os.walk(directory): for filename in files: filepath = os.path.join(root, filename) - - # Get path relative to directory if requested - if use_relative_paths: - relpath = os.path.relpath(filepath, directory) - else: - relpath = filepath - - # Check if file matches any include pattern - included = False - if include_patterns: - for pattern in include_patterns: - if fnmatch.fnmatch(relpath, pattern): - included = True - break - else: - included = True - - # Check if file matches any exclude pattern - excluded = False - if exclude_patterns: - for pattern in exclude_patterns: - if fnmatch.fnmatch(relpath, pattern): - excluded = True - break - - if not included or excluded: - continue - - # Check file size - if max_file_size and os.path.getsize(filepath) > max_file_size: - continue - - try: - with open(filepath, 'r', encoding='utf-8') as f: - content = f.read() - files_dict[relpath] = content - except Exception as e: - print(f"Warning: Could not read file {filepath}: {e}") - + all_files.append(filepath) + + total_files = len(all_files) + processed_files = 0 + + for filepath in all_files: + # Get path relative to directory if requested + if use_relative_paths: + relpath = os.path.relpath(filepath, directory) + else: + relpath = filepath + + # Check if file matches any include pattern + included = False + if include_patterns: + for pattern in include_patterns: + if fnmatch.fnmatch(relpath, pattern): + included = True + break + else: + included = True + + # Check if file matches any exclude pattern + excluded = False + if exclude_patterns: + for pattern in exclude_patterns: + if fnmatch.fnmatch(relpath, pattern): + excluded = True + break + + if not included or excluded: + processed_files += 1 + if progress_callback: + progress_callback(processed_files, total_files) + continue + + # Check file size + if max_file_size and os.path.getsize(filepath) > max_file_size: + processed_files += 1 + if progress_callback: + progress_callback(processed_files, total_files) + continue + + try: + with open(filepath, 'r', encoding='utf-8') as f: + content = f.read() + files_dict[relpath] = content + except Exception as e: + print(f"Warning: Could not read file {filepath}: {e}") + + processed_files += 1 + if progress_callback: + progress_callback(processed_files, total_files) + return {"files": files_dict} if __name__ == "__main__":