diff --git a/.env.sample b/.env.sample index c26cdce..51bbc47 100644 --- a/.env.sample +++ b/.env.sample @@ -1,2 +1,5 @@ GEMINI_PROJECT_ID= -GITHUB_TOKEN= \ No newline at end of file +GEMINI_API_KEY= +GITHUB_TOKEN= +OPENROUTER_API_KEY = +OPENROUTER_MODEL = \ No newline at end of file diff --git a/.gitignore b/.gitignore index 7546330..08505f8 100644 --- a/.gitignore +++ b/.gitignore @@ -99,4 +99,11 @@ coverage/ llm_cache.json # Output files -output/ \ No newline at end of file +output/ + +# uv manage +pyproject.toml +uv.lock + +docs/*.pdf +docs/design-cn.md diff --git a/README.md b/README.md index c6d0ef1..15f476d 100644 --- a/README.md +++ b/README.md @@ -73,13 +73,16 @@ This is a tutorial project of [Pocket Flow](https://github.com/The-Pocket/Pocket ## 🚀 Getting Started 1. Clone this repository + ```bash + git clone https://github.com/The-Pocket/PocketFlow-Tutorial-Codebase-Knowledge + ``` -2. Install dependencies: +3. Install dependencies: ```bash pip install -r requirements.txt ``` -3. Set up LLM in [`utils/call_llm.py`](./utils/call_llm.py) by providing credentials. By default, you can use the AI Studio key with this client for Gemini Pro 2.5: +4. Set up LLM in [`utils/call_llm.py`](./utils/call_llm.py) by providing credentials. By default, you can use the AI Studio key with this client for Gemini Pro 2.5: ```python client = genai.Client( @@ -92,7 +95,7 @@ This is a tutorial project of [Pocket Flow](https://github.com/The-Pocket/Pocket python utils/call_llm.py ``` -4. Generate a complete codebase tutorial by running the main script: +5. Generate a complete codebase tutorial by running the main script: ```bash # Analyze a GitHub repository python main.py --repo https://github.com/username/repo --include "*.py" "*.js" --exclude "tests/*" --max-size 50000 @@ -112,6 +115,8 @@ This is a tutorial project of [Pocket Flow](https://github.com/The-Pocket/Pocket - `-e, --exclude` - Files to exclude (e.g., "tests/*" "docs/*") - `-s, --max-size` - Maximum file size in bytes (default: 100KB) - `--language` - Language for the generated tutorial (default: "english") + - `--max-abstractions` - Maximum number of abstractions to identify (default: 10) + - `--no-cache` - Disable LLM response caching (default: caching enabled) The application will crawl the repository, analyze the codebase structure, generate tutorial content in the specified language, and save the output in the specified directory (default: ./output). diff --git a/docs/_config.yml b/docs/_config.yml index dd32b3c..1c5479a 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -1,5 +1,5 @@ # Basic site settings -title: Codebase2Tutorial +title: Pocket Flow # Theme settings remote_theme: just-the-docs/just-the-docs diff --git a/main.py b/main.py index e0ccc82..bc881a1 100644 --- a/main.py +++ b/main.py @@ -14,8 +14,10 @@ DEFAULT_INCLUDE_PATTERNS = { } DEFAULT_EXCLUDE_PATTERNS = { + "assets/*", "data/*", "examples/*", "images/*", "public/*", "static/*", "temp/*", + "docs/*", "venv/*", ".venv/*", "*test*", "tests/*", "docs/*", "examples/*", "v1/*", - "dist/*", "build/*", "experimental/*", "deprecated/*", + "dist/*", "build/*", "experimental/*", "deprecated/*", "misc/*", "legacy/*", ".git/*", ".github/*", ".next/*", ".vscode/*", "obj/*", "bin/*", "node_modules/*", "*.log" } @@ -36,6 +38,10 @@ def main(): parser.add_argument("-s", "--max-size", type=int, default=100000, help="Maximum file size in bytes (default: 100000, about 100KB).") # Add language parameter for multi-language support parser.add_argument("--language", default="english", help="Language for the generated tutorial (default: english)") + # Add use_cache parameter to control LLM caching + parser.add_argument("--no-cache", action="store_true", help="Disable LLM response caching (default: caching enabled)") + # Add max_abstraction_num parameter to control the number of abstractions + parser.add_argument("--max-abstractions", type=int, default=10, help="Maximum number of abstractions to identify (default: 10)") args = parser.parse_args() @@ -61,6 +67,12 @@ def main(): # Add language for multi-language support "language": args.language, + + # Add use_cache flag (inverse of no-cache flag) + "use_cache": not args.no_cache, + + # Add max_abstraction_num parameter + "max_abstraction_num": args.max_abstractions, # Outputs will be populated by the nodes "files": [], @@ -73,6 +85,7 @@ def main(): # Display starting message with repository/directory and language print(f"Starting tutorial generation for: {args.repo or args.dir} in {args.language.capitalize()} language") + print(f"LLM caching: {'Disabled' if args.no_cache else 'Enabled'}") # Create the flow instance tutorial_flow = create_tutorial_flow() diff --git a/nodes.py b/nodes.py index 61ec11f..67749e5 100644 --- a/nodes.py +++ b/nodes.py @@ -1,19 +1,24 @@ import os +import re import yaml from pocketflow import Node, BatchNode from utils.crawl_github_files import crawl_github_files from utils.call_llm import call_llm from utils.crawl_local_files import crawl_local_files + # Helper to get content for specific file indices def get_content_for_indices(files_data, indices): content_map = {} for i in indices: if 0 <= i < len(files_data): path, content = files_data[i] - content_map[f"{i} # {path}"] = content # Use index + path as key for context + content_map[f"{i} # {path}"] = ( + content # Use index + path as key for context + ) return content_map + class FetchRepo(Node): def prep(self, shared): repo_url = shared.get("repo_url") @@ -23,7 +28,7 @@ class FetchRepo(Node): if not project_name: # Basic name derivation from URL or directory if repo_url: - project_name = repo_url.split('/')[-1].replace('.git', '') + project_name = repo_url.split("/")[-1].replace(".git", "") else: project_name = os.path.basename(os.path.abspath(local_dir)) shared["project_name"] = project_name @@ -40,7 +45,7 @@ class FetchRepo(Node): "include_patterns": include_patterns, "exclude_patterns": exclude_patterns, "max_file_size": max_file_size, - "use_relative_paths": True + "use_relative_paths": True, } def exec(self, prep_res): @@ -52,7 +57,7 @@ class FetchRepo(Node): include_patterns=prep_res["include_patterns"], exclude_patterns=prep_res["exclude_patterns"], max_file_size=prep_res["max_file_size"], - use_relative_paths=prep_res["use_relative_paths"] + use_relative_paths=prep_res["use_relative_paths"], ) else: print(f"Crawling directory: {prep_res['local_dir']}...") @@ -78,37 +83,58 @@ class FetchRepo(Node): # Convert dict to list of tuples: [(path, content), ...] files_list = list(result.get("files", {}).items()) if len(files_list) == 0: - raise(ValueError("Failed to fetch files")) + raise (ValueError("Failed to fetch files")) print(f"Fetched {len(files_list)} files.") return files_list def post(self, shared, prep_res, exec_res): - shared["files"] = exec_res # List of (path, content) tuples + shared["files"] = exec_res # List of (path, content) tuples + class IdentifyAbstractions(Node): def prep(self, shared): files_data = shared["files"] project_name = shared["project_name"] # Get project name - language = shared.get("language", "english") # Get language + language = shared.get("language", "english") # Get language + use_cache = shared.get("use_cache", True) # Get use_cache flag, default to True + max_abstraction_num = shared.get("max_abstraction_num", 10) # Get max_abstraction_num, default to 10 # Helper to create context from files, respecting limits (basic example) def create_llm_context(files_data): context = "" - file_info = [] # Store tuples of (index, path) + file_info = [] # Store tuples of (index, path) for i, (path, content) in enumerate(files_data): entry = f"--- File Index {i}: {path} ---\n{content}\n\n" context += entry file_info.append((i, path)) - return context, file_info # file_info is list of (index, path) + return context, file_info # file_info is list of (index, path) context, file_info = create_llm_context(files_data) # Format file info for the prompt (comment is just a hint for LLM) - file_listing_for_prompt = "\n".join([f"- {idx} # {path}" for idx, path in file_info]) - return context, file_listing_for_prompt, len(files_data), project_name, language # Return language + file_listing_for_prompt = "\n".join( + [f"- {idx} # {path}" for idx, path in file_info] + ) + return ( + context, + file_listing_for_prompt, + len(files_data), + project_name, + language, + use_cache, + max_abstraction_num, + ) # Return all parameters def exec(self, prep_res): - context, file_listing_for_prompt, file_count, project_name, language = prep_res # Unpack project name and language + ( + context, + file_listing_for_prompt, + file_count, + project_name, + language, + use_cache, + max_abstraction_num, + ) = prep_res # Unpack all parameters print(f"Identifying abstractions using LLM...") # Add language instruction and hints only if not English @@ -128,7 +154,7 @@ Codebase Context: {context} {language_instruction}Analyze the codebase context. -Identify the top 5-10 core most important abstractions to help those new to the codebase. +Identify the top 5-{max_abstraction_num} core most important abstractions to help those new to the codebase. For each abstraction, provide: 1. A concise `name`{name_lang_hint}. @@ -155,12 +181,13 @@ Format the output as a YAML list of dictionaries: Another core concept, similar to a blueprint for objects.{desc_lang_hint} file_indices: - 5 # path/to/another.js -# ... up to 10 abstractions +# ... up to {max_abstraction_num} abstractions ```""" - response = call_llm(prompt) + response = call_llm(prompt, use_cache=use_cache) # Pass use_cache parameter # --- Validation --- yaml_str = response.strip().split("```yaml")[1].split("```")[0].strip() + # add whitespace to fix llm generation error(except -) abstractions = yaml.safe_load(yaml_str) if not isinstance(abstractions, list): @@ -168,83 +195,117 @@ Format the output as a YAML list of dictionaries: validated_abstractions = [] for item in abstractions: - if not isinstance(item, dict) or not all(k in item for k in ["name", "description", "file_indices"]): + if not isinstance(item, dict) or not all( + k in item for k in ["name", "description", "file_indices"] + ): raise ValueError(f"Missing keys in abstraction item: {item}") if not isinstance(item["name"], str): - raise ValueError(f"Name is not a string in item: {item}") + raise ValueError(f"Name is not a string in item: {item}") if not isinstance(item["description"], str): - raise ValueError(f"Description is not a string in item: {item}") + raise ValueError(f"Description is not a string in item: {item}") if not isinstance(item["file_indices"], list): - raise ValueError(f"file_indices is not a list in item: {item}") + raise ValueError(f"file_indices is not a list in item: {item}") # Validate indices validated_indices = [] for idx_entry in item["file_indices"]: - try: - if isinstance(idx_entry, int): - idx = idx_entry - elif isinstance(idx_entry, str) and '#' in idx_entry: - idx = int(idx_entry.split('#')[0].strip()) - else: - idx = int(str(idx_entry).strip()) + try: + if isinstance(idx_entry, int): + idx = idx_entry + elif isinstance(idx_entry, str) and "#" in idx_entry: + idx = int(idx_entry.split("#")[0].strip()) + else: + idx = int(str(idx_entry).strip()) - if not (0 <= idx < file_count): - raise ValueError(f"Invalid file index {idx} found in item {item['name']}. Max index is {file_count - 1}.") - validated_indices.append(idx) - except (ValueError, TypeError): - raise ValueError(f"Could not parse index from entry: {idx_entry} in item {item['name']}") + if not (0 <= idx < file_count): + raise ValueError( + f"Invalid file index {idx} found in item {item['name']}. Max index is {file_count - 1}." + ) + validated_indices.append(idx) + except (ValueError, TypeError): + raise ValueError( + f"Could not parse index from entry: {idx_entry} in item {item['name']}" + ) item["files"] = sorted(list(set(validated_indices))) # Store only the required fields - validated_abstractions.append({ - "name": item["name"], # Potentially translated name - "description": item["description"], # Potentially translated description - "files": item["files"] - }) + validated_abstractions.append( + { + "name": item["name"], # Potentially translated name + "description": item[ + "description" + ], # Potentially translated description + "files": item["files"], + } + ) print(f"Identified {len(validated_abstractions)} abstractions.") return validated_abstractions def post(self, shared, prep_res, exec_res): - shared["abstractions"] = exec_res # List of {"name": str, "description": str, "files": [int]} + shared["abstractions"] = ( + exec_res # List of {"name": str, "description": str, "files": [int]} + ) + class AnalyzeRelationships(Node): def prep(self, shared): - abstractions = shared["abstractions"] # Now contains 'files' list of indices, name/description potentially translated + abstractions = shared[ + "abstractions" + ] # Now contains 'files' list of indices, name/description potentially translated files_data = shared["files"] project_name = shared["project_name"] # Get project name - language = shared.get("language", "english") # Get language + language = shared.get("language", "english") # Get language + use_cache = shared.get("use_cache", True) # Get use_cache flag, default to True + + # Get the actual number of abstractions directly + num_abstractions = len(abstractions) # Create context with abstraction names, indices, descriptions, and relevant file snippets - context = "Identified Abstractions:\n" + context = "Identified Abstractions:\\n" all_relevant_indices = set() abstraction_info_for_prompt = [] for i, abstr in enumerate(abstractions): # Use 'files' which contains indices directly - file_indices_str = ", ".join(map(str, abstr['files'])) + file_indices_str = ", ".join(map(str, abstr["files"])) # Abstraction name and description might be translated already - info_line = f"- Index {i}: {abstr['name']} (Relevant file indices: [{file_indices_str}])\n Description: {abstr['description']}" - context += info_line + "\n" - abstraction_info_for_prompt.append(f"{i} # {abstr['name']}") # Use potentially translated name here too - all_relevant_indices.update(abstr['files']) + info_line = f"- Index {i}: {abstr['name']} (Relevant file indices: [{file_indices_str}])\\n Description: {abstr['description']}" + context += info_line + "\\n" + abstraction_info_for_prompt.append( + f"{i} # {abstr['name']}" + ) # Use potentially translated name here too + all_relevant_indices.update(abstr["files"]) - context += "\nRelevant File Snippets (Referenced by Index and Path):\n" + context += "\\nRelevant File Snippets (Referenced by Index and Path):\\n" # Get content for relevant files using helper relevant_files_content_map = get_content_for_indices( - files_data, - sorted(list(all_relevant_indices)) + files_data, sorted(list(all_relevant_indices)) ) # Format file content for context - file_context_str = "\n\n".join( - f"--- File: {idx_path} ---\n{content}" + file_context_str = "\\n\\n".join( + f"--- File: {idx_path} ---\\n{content}" for idx_path, content in relevant_files_content_map.items() ) context += file_context_str - return context, "\n".join(abstraction_info_for_prompt), project_name, language # Return language + return ( + context, + "\n".join(abstraction_info_for_prompt), + num_abstractions, # Pass the actual count + project_name, + language, + use_cache, + ) # Return use_cache def exec(self, prep_res): - context, abstraction_listing, project_name, language = prep_res # Unpack project name and language + ( + context, + abstraction_listing, + num_abstractions, # Receive the actual count + project_name, + language, + use_cache, + ) = prep_res # Unpack use_cache print(f"Analyzing relationships using LLM...") # Add language instruction and hints only if not English @@ -254,7 +315,7 @@ class AnalyzeRelationships(Node): if language.lower() != "english": language_instruction = f"IMPORTANT: Generate the `summary` and relationship `label` fields in **{language.capitalize()}** language. Do NOT use English for these fields.\n\n" lang_hint = f" (in {language.capitalize()})" - list_lang_note = f" (Names might be in {language.capitalize()})" # Note for the input list + list_lang_note = f" (Names might be in {language.capitalize()})" # Note for the input list prompt = f""" Based on the following abstractions and relevant code snippets from the project `{project_name}`: @@ -294,90 +355,122 @@ relationships: Now, provide the YAML output: """ - response = call_llm(prompt) + response = call_llm(prompt, use_cache=use_cache) # --- Validation --- yaml_str = response.strip().split("```yaml")[1].split("```")[0].strip() relationships_data = yaml.safe_load(yaml_str) - if not isinstance(relationships_data, dict) or not all(k in relationships_data for k in ["summary", "relationships"]): - raise ValueError("LLM output is not a dict or missing keys ('summary', 'relationships')") + if not isinstance(relationships_data, dict) or not all( + k in relationships_data for k in ["summary", "relationships"] + ): + raise ValueError( + "LLM output is not a dict or missing keys ('summary', 'relationships')" + ) if not isinstance(relationships_data["summary"], str): - raise ValueError("summary is not a string") + raise ValueError("summary is not a string") if not isinstance(relationships_data["relationships"], list): - raise ValueError("relationships is not a list") + raise ValueError("relationships is not a list") # Validate relationships structure validated_relationships = [] - num_abstractions = len(abstraction_listing.split('\n')) for rel in relationships_data["relationships"]: - # Check for 'label' key - if not isinstance(rel, dict) or not all(k in rel for k in ["from_abstraction", "to_abstraction", "label"]): - raise ValueError(f"Missing keys (expected from_abstraction, to_abstraction, label) in relationship item: {rel}") - # Validate 'label' is a string - if not isinstance(rel["label"], str): - raise ValueError(f"Relationship label is not a string: {rel}") + # Check for 'label' key + if not isinstance(rel, dict) or not all( + k in rel for k in ["from_abstraction", "to_abstraction", "label"] + ): + raise ValueError( + f"Missing keys (expected from_abstraction, to_abstraction, label) in relationship item: {rel}" + ) + # Validate 'label' is a string + if not isinstance(rel["label"], str): + raise ValueError(f"Relationship label is not a string: {rel}") - # Validate indices - try: - from_idx = int(str(rel["from_abstraction"]).split('#')[0].strip()) - to_idx = int(str(rel["to_abstraction"]).split('#')[0].strip()) - if not (0 <= from_idx < num_abstractions and 0 <= to_idx < num_abstractions): - raise ValueError(f"Invalid index in relationship: from={from_idx}, to={to_idx}. Max index is {num_abstractions-1}.") - validated_relationships.append({ - "from": from_idx, - "to": to_idx, - "label": rel["label"] # Potentially translated label - }) - except (ValueError, TypeError): - raise ValueError(f"Could not parse indices from relationship: {rel}") + # Validate indices + try: + from_idx = int(str(rel["from_abstraction"]).split("#")[0].strip()) + to_idx = int(str(rel["to_abstraction"]).split("#")[0].strip()) + if not ( + 0 <= from_idx < num_abstractions and 0 <= to_idx < num_abstractions + ): + raise ValueError( + f"Invalid index in relationship: from={from_idx}, to={to_idx}. Max index is {num_abstractions-1}." + ) + validated_relationships.append( + { + "from": from_idx, + "to": to_idx, + "label": rel["label"], # Potentially translated label + } + ) + except (ValueError, TypeError): + raise ValueError(f"Could not parse indices from relationship: {rel}") print("Generated project summary and relationship details.") return { - "summary": relationships_data["summary"], # Potentially translated summary - "details": validated_relationships # Store validated, index-based relationships with potentially translated labels + "summary": relationships_data["summary"], # Potentially translated summary + "details": validated_relationships, # Store validated, index-based relationships with potentially translated labels } - def post(self, shared, prep_res, exec_res): # Structure is now {"summary": str, "details": [{"from": int, "to": int, "label": str}]} # Summary and label might be translated shared["relationships"] = exec_res + class OrderChapters(Node): def prep(self, shared): - abstractions = shared["abstractions"] # Name/description might be translated - relationships = shared["relationships"] # Summary/label might be translated + abstractions = shared["abstractions"] # Name/description might be translated + relationships = shared["relationships"] # Summary/label might be translated project_name = shared["project_name"] # Get project name - language = shared.get("language", "english") # Get language + language = shared.get("language", "english") # Get language + use_cache = shared.get("use_cache", True) # Get use_cache flag, default to True # Prepare context for the LLM abstraction_info_for_prompt = [] for i, a in enumerate(abstractions): - abstraction_info_for_prompt.append(f"- {i} # {a['name']}") # Use potentially translated name + abstraction_info_for_prompt.append( + f"- {i} # {a['name']}" + ) # Use potentially translated name abstraction_listing = "\n".join(abstraction_info_for_prompt) # Use potentially translated summary and labels summary_note = "" if language.lower() != "english": - summary_note = f" (Note: Project Summary might be in {language.capitalize()})" + summary_note = ( + f" (Note: Project Summary might be in {language.capitalize()})" + ) context = f"Project Summary{summary_note}:\n{relationships['summary']}\n\n" context += "Relationships (Indices refer to abstractions above):\n" - for rel in relationships['details']: - from_name = abstractions[rel['from']]['name'] - to_name = abstractions[rel['to']]['name'] - # Use potentially translated 'label' - context += f"- From {rel['from']} ({from_name}) to {rel['to']} ({to_name}): {rel['label']}\n" # Label might be translated + for rel in relationships["details"]: + from_name = abstractions[rel["from"]]["name"] + to_name = abstractions[rel["to"]]["name"] + # Use potentially translated 'label' + context += f"- From {rel['from']} ({from_name}) to {rel['to']} ({to_name}): {rel['label']}\n" # Label might be translated list_lang_note = "" if language.lower() != "english": - list_lang_note = f" (Names might be in {language.capitalize()})" + list_lang_note = f" (Names might be in {language.capitalize()})" - return abstraction_listing, context, len(abstractions), project_name, list_lang_note + return ( + abstraction_listing, + context, + len(abstractions), + project_name, + list_lang_note, + use_cache, + ) # Return use_cache def exec(self, prep_res): - abstraction_listing, context, num_abstractions, project_name, list_lang_note = prep_res + ( + abstraction_listing, + context, + num_abstractions, + project_name, + list_lang_note, + use_cache, + ) = prep_res # Unpack use_cache print("Determining chapter order using LLM...") # No language variation needed here in prompt instructions, just ordering based on structure # The input names might be translated, hence the note. @@ -417,60 +510,81 @@ Now, provide the YAML output: seen_indices = set() for entry in ordered_indices_raw: try: - if isinstance(entry, int): - idx = entry - elif isinstance(entry, str) and '#' in entry: - idx = int(entry.split('#')[0].strip()) - else: - idx = int(str(entry).strip()) + if isinstance(entry, int): + idx = entry + elif isinstance(entry, str) and "#" in entry: + idx = int(entry.split("#")[0].strip()) + else: + idx = int(str(entry).strip()) - if not (0 <= idx < num_abstractions): - raise ValueError(f"Invalid index {idx} in ordered list. Max index is {num_abstractions-1}.") - if idx in seen_indices: - raise ValueError(f"Duplicate index {idx} found in ordered list.") - ordered_indices.append(idx) - seen_indices.add(idx) + if not (0 <= idx < num_abstractions): + raise ValueError( + f"Invalid index {idx} in ordered list. Max index is {num_abstractions-1}." + ) + if idx in seen_indices: + raise ValueError(f"Duplicate index {idx} found in ordered list.") + ordered_indices.append(idx) + seen_indices.add(idx) except (ValueError, TypeError): - raise ValueError(f"Could not parse index from ordered list entry: {entry}") + raise ValueError( + f"Could not parse index from ordered list entry: {entry}" + ) # Check if all abstractions are included if len(ordered_indices) != num_abstractions: - raise ValueError(f"Ordered list length ({len(ordered_indices)}) does not match number of abstractions ({num_abstractions}). Missing indices: {set(range(num_abstractions)) - seen_indices}") + raise ValueError( + f"Ordered list length ({len(ordered_indices)}) does not match number of abstractions ({num_abstractions}). Missing indices: {set(range(num_abstractions)) - seen_indices}" + ) print(f"Determined chapter order (indices): {ordered_indices}") - return ordered_indices # Return the list of indices + return ordered_indices # Return the list of indices def post(self, shared, prep_res, exec_res): # exec_res is already the list of ordered indices - shared["chapter_order"] = exec_res # List of indices + shared["chapter_order"] = exec_res # List of indices + class WriteChapters(BatchNode): def prep(self, shared): - chapter_order = shared["chapter_order"] # List of indices - abstractions = shared["abstractions"] # List of dicts, name/desc potentially translated - files_data = shared["files"] - language = shared.get("language", "english") # Get language + chapter_order = shared["chapter_order"] # List of indices + abstractions = shared[ + "abstractions" + ] # List of {"name": str, "description": str, "files": [int]} + files_data = shared["files"] # List of (path, content) tuples + project_name = shared["project_name"] + language = shared.get("language", "english") + use_cache = shared.get("use_cache", True) # Get use_cache flag, default to True # Get already written chapters to provide context # We store them temporarily during the batch run, not in shared memory yet # The 'previous_chapters_summary' will be built progressively in the exec context - self.chapters_written_so_far = [] # Use instance variable for temporary storage across exec calls + self.chapters_written_so_far = ( + [] + ) # Use instance variable for temporary storage across exec calls # Create a complete list of all chapters all_chapters = [] - chapter_filenames = {} # Store chapter filename mapping for linking + chapter_filenames = {} # Store chapter filename mapping for linking for i, abstraction_index in enumerate(chapter_order): if 0 <= abstraction_index < len(abstractions): chapter_num = i + 1 - chapter_name = abstractions[abstraction_index]["name"] # Potentially translated name + chapter_name = abstractions[abstraction_index][ + "name" + ] # Potentially translated name # Create safe filename (from potentially translated name) - safe_name = "".join(c if c.isalnum() else '_' for c in chapter_name).lower() + safe_name = "".join( + c if c.isalnum() else "_" for c in chapter_name + ).lower() filename = f"{i+1:02d}_{safe_name}.md" # Format with link (using potentially translated name) all_chapters.append(f"{chapter_num}. [{chapter_name}]({filename})") # Store mapping of chapter index to filename for linking - chapter_filenames[abstraction_index] = {"num": chapter_num, "name": chapter_name, "filename": filename} + chapter_filenames[abstraction_index] = { + "num": chapter_num, + "name": chapter_name, + "filename": filename, + } # Create a formatted string with all chapters full_chapter_listing = "\n".join(all_chapters) @@ -478,47 +592,59 @@ class WriteChapters(BatchNode): items_to_process = [] for i, abstraction_index in enumerate(chapter_order): if 0 <= abstraction_index < len(abstractions): - abstraction_details = abstractions[abstraction_index] # Contains potentially translated name/desc + abstraction_details = abstractions[ + abstraction_index + ] # Contains potentially translated name/desc # Use 'files' (list of indices) directly related_file_indices = abstraction_details.get("files", []) # Get content using helper, passing indices - related_files_content_map = get_content_for_indices(files_data, related_file_indices) + related_files_content_map = get_content_for_indices( + files_data, related_file_indices + ) # Get previous chapter info for transitions (uses potentially translated name) prev_chapter = None if i > 0: - prev_idx = chapter_order[i-1] + prev_idx = chapter_order[i - 1] prev_chapter = chapter_filenames[prev_idx] # Get next chapter info for transitions (uses potentially translated name) next_chapter = None if i < len(chapter_order) - 1: - next_idx = chapter_order[i+1] + next_idx = chapter_order[i + 1] next_chapter = chapter_filenames[next_idx] - items_to_process.append({ - "chapter_num": i + 1, - "abstraction_index": abstraction_index, - "abstraction_details": abstraction_details, # Has potentially translated name/desc - "related_files_content_map": related_files_content_map, - "project_name": shared["project_name"], # Add project name - "full_chapter_listing": full_chapter_listing, # Add the full chapter listing (uses potentially translated names) - "chapter_filenames": chapter_filenames, # Add chapter filenames mapping (uses potentially translated names) - "prev_chapter": prev_chapter, # Add previous chapter info (uses potentially translated name) - "next_chapter": next_chapter, # Add next chapter info (uses potentially translated name) - "language": language, # Add language for multi-language support - # previous_chapters_summary will be added dynamically in exec - }) + items_to_process.append( + { + "chapter_num": i + 1, + "abstraction_index": abstraction_index, + "abstraction_details": abstraction_details, # Has potentially translated name/desc + "related_files_content_map": related_files_content_map, + "project_name": shared["project_name"], # Add project name + "full_chapter_listing": full_chapter_listing, # Add the full chapter listing (uses potentially translated names) + "chapter_filenames": chapter_filenames, # Add chapter filenames mapping (uses potentially translated names) + "prev_chapter": prev_chapter, # Add previous chapter info (uses potentially translated name) + "next_chapter": next_chapter, # Add next chapter info (uses potentially translated name) + "language": language, # Add language for multi-language support + # previous_chapters_summary will be added dynamically in exec + } + ) else: - print(f"Warning: Invalid abstraction index {abstraction_index} in chapter_order. Skipping.") + print( + f"Warning: Invalid abstraction index {abstraction_index} in chapter_order. Skipping." + ) print(f"Preparing to write {len(items_to_process)} chapters...") - return items_to_process # Iterable for BatchNode + return items_to_process # Iterable for BatchNode def exec(self, item): # This runs for each item prepared above - abstraction_name = item["abstraction_details"]["name"] # Potentially translated name - abstraction_description = item["abstraction_details"]["description"] # Potentially translated description + abstraction_name = item["abstraction_details"][ + "name" + ] # Potentially translated name + abstraction_description = item["abstraction_details"][ + "description" + ] # Potentially translated description chapter_num = item["chapter_num"] project_name = item.get("project_name") language = item.get("language", "english") @@ -553,10 +679,11 @@ class WriteChapters(BatchNode): instruction_lang_note = f" (in {lang_cap})" mermaid_lang_note = f" (Use {lang_cap} for labels/text if appropriate)" code_comment_note = f" (Translate to {lang_cap} if possible, otherwise keep minimal English for clarity)" - link_lang_note = f" (Use the {lang_cap} chapter title from the structure above)" + link_lang_note = ( + f" (Use the {lang_cap} chapter title from the structure above)" + ) tone_note = f" (appropriate for {lang_cap} readers)" - prompt = f""" {language_instruction}Write a very beginner-friendly tutorial chapter (in Markdown format) for the project `{project_name}` about the concept: "{abstraction_name}". This is Chapter {chapter_num}. @@ -585,7 +712,7 @@ Instructions for the chapter (Generate content in {language.capitalize()} unless - Explain how to use this abstraction to solve the use case{instruction_lang_note}. Give example inputs and outputs for code snippets (if the output isn't values, describe at a high level what will happen{instruction_lang_note}). -- Each code block should be BELOW 20 lines! If longer code blocks are needed, break them down into smaller pieces and walk through them one-by-one. Aggresively simplify the code to make it minimal. Use comments{code_comment_note} to skip non-important implementation details. Each code block should have a beginner friendly explanation right after it{instruction_lang_note}. +- Each code block should be BELOW 10 lines! If longer code blocks are needed, break them down into smaller pieces and walk through them one-by-one. Aggresively simplify the code to make it minimal. Use comments{code_comment_note} to skip non-important implementation details. Each code block should have a beginner friendly explanation right after it{instruction_lang_note}. - Describe the internal implementation to help understand what's under the hood{instruction_lang_note}. First provide a non-code or code-light walkthrough on what happens step-by-step when the abstraction is called{instruction_lang_note}. It's recommended to use a simple sequenceDiagram with a dummy example - keep it minimal with at most 5 participants to ensure clarity. If participant name has space, use: `participant QP as Query Processing`. {mermaid_lang_note}. @@ -607,20 +734,22 @@ Now, directly provide a super beginner-friendly Markdown output (DON'T need ```m """ chapter_content = call_llm(prompt) # Basic validation/cleanup - actual_heading = f"# Chapter {chapter_num}: {abstraction_name}" # Use potentially translated name + actual_heading = f"# Chapter {chapter_num}: {abstraction_name}" # Use potentially translated name if not chapter_content.strip().startswith(f"# Chapter {chapter_num}"): - # Add heading if missing or incorrect, trying to preserve content - lines = chapter_content.strip().split('\n') - if lines and lines[0].strip().startswith("#"): # If there's some heading, replace it - lines[0] = actual_heading - chapter_content = "\n".join(lines) - else: # Otherwise, prepend it - chapter_content = f"{actual_heading}\n\n{chapter_content}" + # Add heading if missing or incorrect, trying to preserve content + lines = chapter_content.strip().split("\n") + if lines and lines[0].strip().startswith( + "#" + ): # If there's some heading, replace it + lines[0] = actual_heading + chapter_content = "\n".join(lines) + else: # Otherwise, prepend it + chapter_content = f"{actual_heading}\n\n{chapter_content}" # Add the generated content to our temporary list for the next iteration's context self.chapters_written_so_far.append(chapter_content) - return chapter_content # Return the Markdown string (potentially translated) + return chapter_content # Return the Markdown string (potentially translated) def post(self, shared, prep_res, exec_res_list): # exec_res_list contains the generated Markdown for each chapter, in order @@ -629,19 +758,26 @@ Now, directly provide a super beginner-friendly Markdown output (DON'T need ```m del self.chapters_written_so_far print(f"Finished writing {len(exec_res_list)} chapters.") + class CombineTutorial(Node): def prep(self, shared): project_name = shared["project_name"] - output_base_dir = shared.get("output_dir", "output") # Default output dir + output_base_dir = shared.get("output_dir", "output") # Default output dir output_path = os.path.join(output_base_dir, project_name) repo_url = shared.get("repo_url") # Get the repository URL # language = shared.get("language", "english") # No longer needed for fixed strings # Get potentially translated data - relationships_data = shared["relationships"] # {"summary": str, "details": [{"from": int, "to": int, "label": str}]} -> summary/label potentially translated - chapter_order = shared["chapter_order"] # indices - abstractions = shared["abstractions"] # list of dicts -> name/description potentially translated - chapters_content = shared["chapters"] # list of strings -> content potentially translated + relationships_data = shared[ + "relationships" + ] # {"summary": str, "details": [{"from": int, "to": int, "label": str}]} -> summary/label potentially translated + chapter_order = shared["chapter_order"] # indices + abstractions = shared[ + "abstractions" + ] # list of dicts -> name/description potentially translated + chapters_content = shared[ + "chapters" + ] # list of strings -> content potentially translated # --- Generate Mermaid Diagram --- mermaid_lines = ["flowchart TD"] @@ -649,26 +785,32 @@ class CombineTutorial(Node): for i, abstr in enumerate(abstractions): node_id = f"A{i}" # Use potentially translated name, sanitize for Mermaid ID and label - sanitized_name = abstr['name'].replace('"', '') - node_label = sanitized_name # Using sanitized name only - mermaid_lines.append(f' {node_id}["{node_label}"]') # Node label uses potentially translated name + sanitized_name = abstr["name"].replace('"', "") + node_label = sanitized_name # Using sanitized name only + mermaid_lines.append( + f' {node_id}["{node_label}"]' + ) # Node label uses potentially translated name # Add edges for relationships using potentially translated labels - for rel in relationships_data['details']: + for rel in relationships_data["details"]: from_node_id = f"A{rel['from']}" to_node_id = f"A{rel['to']}" # Use potentially translated label, sanitize - edge_label = rel['label'].replace('"', '').replace('\n', ' ') # Basic sanitization + edge_label = ( + rel["label"].replace('"', "").replace("\n", " ") + ) # Basic sanitization max_label_len = 30 if len(edge_label) > max_label_len: - edge_label = edge_label[:max_label_len-3] + "..." - mermaid_lines.append(f' {from_node_id} -- "{edge_label}" --> {to_node_id}') # Edge label uses potentially translated label + edge_label = edge_label[: max_label_len - 3] + "..." + mermaid_lines.append( + f' {from_node_id} -- "{edge_label}" --> {to_node_id}' + ) # Edge label uses potentially translated label mermaid_diagram = "\n".join(mermaid_lines) # --- End Mermaid --- # --- Prepare index.md content --- index_content = f"# Tutorial: {project_name}\n\n" - index_content += f"{relationships_data['summary']}\n\n" # Use the potentially translated summary directly + index_content += f"{relationships_data['summary']}\n\n" # Use the potentially translated summary directly # Keep fixed strings in English index_content += f"**Source Repository:** [{repo_url}]({repo_url})\n\n" @@ -685,14 +827,18 @@ class CombineTutorial(Node): for i, abstraction_index in enumerate(chapter_order): # Ensure index is valid and we have content for it if 0 <= abstraction_index < len(abstractions) and i < len(chapters_content): - abstraction_name = abstractions[abstraction_index]["name"] # Potentially translated name + abstraction_name = abstractions[abstraction_index][ + "name" + ] # Potentially translated name # Sanitize potentially translated name for filename - safe_name = "".join(c if c.isalnum() else '_' for c in abstraction_name).lower() + safe_name = "".join( + c if c.isalnum() else "_" for c in abstraction_name + ).lower() filename = f"{i+1:02d}_{safe_name}.md" - index_content += f"{i+1}. [{abstraction_name}]({filename})\n" # Use potentially translated name in link text + index_content += f"{i+1}. [{abstraction_name}]({filename})\n" # Use potentially translated name in link text # Add attribution to chapter content (using English fixed string) - chapter_content = chapters_content[i] # Potentially translated content + chapter_content = chapters_content[i] # Potentially translated content if not chapter_content.endswith("\n\n"): chapter_content += "\n\n" # Keep fixed strings in English @@ -701,7 +847,9 @@ class CombineTutorial(Node): # Store filename and corresponding content chapter_files.append({"filename": filename, "content": chapter_content}) else: - print(f"Warning: Mismatch between chapter order, abstractions, or content at index {i} (abstraction index {abstraction_index}). Skipping file generation for this entry.") + print( + f"Warning: Mismatch between chapter order, abstractions, or content at index {i} (abstraction index {abstraction_index}). Skipping file generation for this entry." + ) # Add attribution to index content (using English fixed string) index_content += f"\n\n---\n\nGenerated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge)" @@ -709,7 +857,7 @@ class CombineTutorial(Node): return { "output_path": output_path, "index_content": index_content, - "chapter_files": chapter_files # List of {"filename": str, "content": str} + "chapter_files": chapter_files, # List of {"filename": str, "content": str} } def exec(self, prep_res): @@ -734,9 +882,8 @@ class CombineTutorial(Node): f.write(chapter_info["content"]) print(f" - Wrote {chapter_filepath}") - return output_path # Return the final path - + return output_path # Return the final path def post(self, shared, prep_res, exec_res): - shared["final_output_dir"] = exec_res # Store the output path + shared["final_output_dir"] = exec_res # Store the output path print(f"\nTutorial generation complete! Files are in: {exec_res}") diff --git a/requirements.txt b/requirements.txt index 06253bc..285df27 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,4 @@ gitpython>=3.1.0 google-cloud-aiplatform>=1.25.0 google-genai>=1.9.0 python-dotenv>=1.0.0 +pathspec>=0.11.0 diff --git a/utils/call_llm.py b/utils/call_llm.py index 0d794b4..6e13975 100644 --- a/utils/call_llm.py +++ b/utils/call_llm.py @@ -3,86 +3,93 @@ import os import logging import json from datetime import datetime +import requests # Configure logging log_directory = os.getenv("LOG_DIR", "logs") os.makedirs(log_directory, exist_ok=True) -log_file = os.path.join(log_directory, f"llm_calls_{datetime.now().strftime('%Y%m%d')}.log") +log_file = os.path.join( + log_directory, f"llm_calls_{datetime.now().strftime('%Y%m%d')}.log" +) # Set up logger logger = logging.getLogger("llm_logger") logger.setLevel(logging.INFO) logger.propagate = False # Prevent propagation to root logger file_handler = logging.FileHandler(log_file) -file_handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')) +file_handler.setFormatter( + logging.Formatter("%(asctime)s - %(levelname)s - %(message)s") +) logger.addHandler(file_handler) # Simple cache configuration cache_file = "llm_cache.json" + # By default, we Google Gemini 2.5 pro, as it shows great performance for code understanding def call_llm(prompt: str, use_cache: bool = True) -> str: # Log the prompt logger.info(f"PROMPT: {prompt}") - + # Check cache if enabled if use_cache: # Load cache from disk cache = {} if os.path.exists(cache_file): try: - with open(cache_file, 'r') as f: + with open(cache_file, "r") as f: cache = json.load(f) except: logger.warning(f"Failed to load cache, starting with empty cache") - + # Return from cache if exists if prompt in cache: logger.info(f"RESPONSE: {cache[prompt]}") return cache[prompt] - - # Call the LLM if not in cache or cache disabled - client = genai.Client( - vertexai=True, - # TODO: change to your own project id and location - project=os.getenv("GEMINI_PROJECT_ID", "your-project-id"), - location=os.getenv("GEMINI_LOCATION", "us-central1") - ) - # You can comment the previous line and use the AI Studio key instead: + + # # Call the LLM if not in cache or cache disabled # client = genai.Client( - # api_key=os.getenv("GEMINI_API_KEY", "your-api_key"), + # vertexai=True, + # # TODO: change to your own project id and location + # project=os.getenv("GEMINI_PROJECT_ID", "your-project-id"), + # location=os.getenv("GEMINI_LOCATION", "us-central1") # ) - model = os.getenv("GEMINI_MODEL", "gemini-2.5-pro-exp-03-25") - response = client.models.generate_content( - model=model, - contents=[prompt] + + # You can comment the previous line and use the AI Studio key instead: + client = genai.Client( + api_key=os.getenv("GEMINI_API_KEY", ""), ) - response_text = response.text + model = os.getenv("GEMINI_MODEL", "gemini-2.5-pro-exp-03-25") + # model = os.getenv("GEMINI_MODEL", "gemini-2.5-flash-preview-04-17") + response = client.models.generate_content(model=model, contents=[prompt]) + response_text = response.text + # Log the response logger.info(f"RESPONSE: {response_text}") - + # Update cache if enabled if use_cache: # Load cache again to avoid overwrites cache = {} if os.path.exists(cache_file): try: - with open(cache_file, 'r') as f: + with open(cache_file, "r") as f: cache = json.load(f) except: pass - + # Add to cache and save cache[prompt] = response_text try: - with open(cache_file, 'w') as f: + with open(cache_file, "w") as f: json.dump(cache, f) except Exception as e: logger.error(f"Failed to save cache: {e}") - + return response_text + # # Use Anthropic Claude 3.7 Sonnet Extended Thinking # def call_llm(prompt, use_cache: bool = True): # from anthropic import Anthropic @@ -101,7 +108,7 @@ def call_llm(prompt: str, use_cache: bool = True) -> str: # return response.content[1].text # # Use OpenAI o1 -# def call_llm(prompt, use_cache: bool = True): +# def call_llm(prompt, use_cache: bool = True): # from openai import OpenAI # client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY", "your-api-key")) # r = client.chat.completions.create( @@ -115,11 +122,86 @@ def call_llm(prompt: str, use_cache: bool = True) -> str: # ) # return r.choices[0].message.content +# Use OpenRouter API +# def call_llm(prompt: str, use_cache: bool = True) -> str: +# # Log the prompt +# logger.info(f"PROMPT: {prompt}") + +# # Check cache if enabled +# if use_cache: +# # Load cache from disk +# cache = {} +# if os.path.exists(cache_file): +# try: +# with open(cache_file, "r") as f: +# cache = json.load(f) +# except: +# logger.warning(f"Failed to load cache, starting with empty cache") + +# # Return from cache if exists +# if prompt in cache: +# logger.info(f"RESPONSE: {cache[prompt]}") +# return cache[prompt] + +# # OpenRouter API configuration +# api_key = os.getenv("OPENROUTER_API_KEY", "") +# model = os.getenv("OPENROUTER_MODEL", "google/gemini-2.0-flash-exp:free") + +# headers = { +# "Authorization": f"Bearer {api_key}", +# } + +# data = { +# "model": model, +# "messages": [{"role": "user", "content": prompt}] +# } + +# response = requests.post( +# "https://openrouter.ai/api/v1/chat/completions", +# headers=headers, +# json=data +# ) + +# if response.status_code != 200: +# error_msg = f"OpenRouter API call failed with status {response.status_code}: {response.text}" +# logger.error(error_msg) +# raise Exception(error_msg) +# try: +# response_text = response.json()["choices"][0]["message"]["content"] +# except Exception as e: +# error_msg = f"Failed to parse OpenRouter response: {e}; Response: {response.text}" +# logger.error(error_msg) +# raise Exception(error_msg) + + +# # Log the response +# logger.info(f"RESPONSE: {response_text}") + +# # Update cache if enabled +# if use_cache: +# # Load cache again to avoid overwrites +# cache = {} +# if os.path.exists(cache_file): +# try: +# with open(cache_file, "r") as f: +# cache = json.load(f) +# except: +# pass + +# # Add to cache and save +# cache[prompt] = response_text +# try: +# with open(cache_file, "w") as f: +# json.dump(cache, f) +# except Exception as e: +# logger.error(f"Failed to save cache: {e}") + +# return response_text + if __name__ == "__main__": test_prompt = "Hello, how are you?" - + # First call - should hit the API print("Making call...") response1 = call_llm(test_prompt, use_cache=False) print(f"Response: {response1}") - diff --git a/utils/crawl_local_files.py b/utils/crawl_local_files.py index b3ac49b..722078f 100644 --- a/utils/crawl_local_files.py +++ b/utils/crawl_local_files.py @@ -1,10 +1,17 @@ import os import fnmatch +import pathspec -def crawl_local_files(directory, include_patterns=None, exclude_patterns=None, max_file_size=None, use_relative_paths=True, progress_callback=None): +def crawl_local_files( + directory, + include_patterns=None, + exclude_patterns=None, + max_file_size=None, + use_relative_paths=True, + progress_callback=None, +): """ Crawl files in a local directory with similar interface as crawl_github_files. - Args: directory (str): Path to local directory include_patterns (set): File patterns to include (e.g. {"*.py", "*.js"}) @@ -12,18 +19,48 @@ def crawl_local_files(directory, include_patterns=None, exclude_patterns=None, m max_file_size (int): Maximum file size in bytes use_relative_paths (bool): Whether to use paths relative to directory progress_callback (callable): Function to report progress, takes (processed, total) as arguments - + Returns: dict: {"files": {filepath: content}} """ if not os.path.isdir(directory): raise ValueError(f"Directory does not exist: {directory}") - - files_dict = {} - all_files = [] - # Collect all files first to calculate total - for root, _, files in os.walk(directory): + files_dict = {} + + # --- Load .gitignore --- + gitignore_path = os.path.join(directory, ".gitignore") + gitignore_spec = None + if os.path.exists(gitignore_path): + try: + with open(gitignore_path, "r", encoding="utf-8") as f: + gitignore_patterns = f.readlines() + gitignore_spec = pathspec.PathSpec.from_lines("gitwildmatch", gitignore_patterns) + print(f"Loaded .gitignore patterns from {gitignore_path}") + except Exception as e: + print(f"Warning: Could not read or parse .gitignore file {gitignore_path}: {e}") + + all_files = [] + for root, dirs, files in os.walk(directory): + # Filter directories using .gitignore and exclude_patterns early + excluded_dirs = set() + for d in dirs: + dirpath_rel = os.path.relpath(os.path.join(root, d), directory) + + if gitignore_spec and gitignore_spec.match_file(dirpath_rel): + excluded_dirs.add(d) + continue + + if exclude_patterns: + for pattern in exclude_patterns: + if fnmatch.fnmatch(dirpath_rel, pattern) or fnmatch.fnmatch(d, pattern): + excluded_dirs.add(d) + break + + for d in dirs.copy(): + if d in excluded_dirs: + dirs.remove(d) + for filename in files: filepath = os.path.join(root, filename) all_files.append(filepath) @@ -32,13 +69,19 @@ def crawl_local_files(directory, include_patterns=None, exclude_patterns=None, m processed_files = 0 for filepath in all_files: - # Get path relative to directory if requested - if use_relative_paths: - relpath = os.path.relpath(filepath, directory) - else: - relpath = filepath + relpath = os.path.relpath(filepath, directory) if use_relative_paths else filepath + + # --- Exclusion check --- + excluded = False + if gitignore_spec and gitignore_spec.match_file(relpath): + excluded = True + + if not excluded and exclude_patterns: + for pattern in exclude_patterns: + if fnmatch.fnmatch(relpath, pattern): + excluded = True + break - # Check if file matches any include pattern included = False if include_patterns: for pattern in include_patterns: @@ -48,21 +91,12 @@ def crawl_local_files(directory, include_patterns=None, exclude_patterns=None, m else: included = True - # Check if file matches any exclude pattern - excluded = False - if exclude_patterns: - for pattern in exclude_patterns: - if fnmatch.fnmatch(relpath, pattern): - excluded = True - break - if not included or excluded: processed_files += 1 if progress_callback: progress_callback(processed_files, total_files) continue - # Check file size if max_file_size and os.path.getsize(filepath) > max_file_size: processed_files += 1 if progress_callback: @@ -70,7 +104,7 @@ def crawl_local_files(directory, include_patterns=None, exclude_patterns=None, m continue try: - with open(filepath, 'r', encoding='utf-8') as f: + with open(filepath, "r", encoding="utf-8") as f: content = f.read() files_dict[relpath] = content except Exception as e: @@ -82,9 +116,20 @@ def crawl_local_files(directory, include_patterns=None, exclude_patterns=None, m return {"files": files_dict} + if __name__ == "__main__": print("--- Crawling parent directory ('..') ---") - files_data = crawl_local_files("..", exclude_patterns={"*.pyc", "__pycache__/*", ".git/*", "output/*"}) + files_data = crawl_local_files( + "..", + exclude_patterns={ + "*.pyc", + "__pycache__/*", + ".venv/*", + ".git/*", + "docs/*", + "output/*", + }, + ) print(f"Found {len(files_data['files'])} files:") for path in files_data["files"]: print(f" {path}") \ No newline at end of file