diff --git a/nodes.py b/nodes.py index d2045f2..f6be367 100644 --- a/nodes.py +++ b/nodes.py @@ -5,7 +5,6 @@ from pocketflow import Node, BatchNode from utils.crawl_github_files import crawl_github_files from utils.call_llm import call_llm from utils.crawl_local_files import crawl_local_files -from utils.fix_yaml import add_indentation # Helper to get content for specific file indices @@ -14,9 +13,12 @@ def get_content_for_indices(files_data, indices): for i in indices: if 0 <= i < len(files_data): path, content = files_data[i] - content_map[f"{i} # {path}"] = content # Use index + path as key for context + content_map[f"{i} # {path}"] = ( + content # Use index + path as key for context + ) return content_map + class FetchRepo(Node): def prep(self, shared): repo_url = shared.get("repo_url") @@ -26,7 +28,7 @@ class FetchRepo(Node): if not project_name: # Basic name derivation from URL or directory if repo_url: - project_name = repo_url.split('/')[-1].replace('.git', '') + project_name = repo_url.split("/")[-1].replace(".git", "") else: project_name = os.path.basename(os.path.abspath(local_dir)) shared["project_name"] = project_name @@ -43,7 +45,7 @@ class FetchRepo(Node): "include_patterns": include_patterns, "exclude_patterns": exclude_patterns, "max_file_size": max_file_size, - "use_relative_paths": True + "use_relative_paths": True, } def exec(self, prep_res): @@ -55,7 +57,7 @@ class FetchRepo(Node): include_patterns=prep_res["include_patterns"], exclude_patterns=prep_res["exclude_patterns"], max_file_size=prep_res["max_file_size"], - use_relative_paths=prep_res["use_relative_paths"] + use_relative_paths=prep_res["use_relative_paths"], ) else: print(f"Crawling directory: {prep_res['local_dir']}...") @@ -64,44 +66,61 @@ class FetchRepo(Node): include_patterns=prep_res["include_patterns"], exclude_patterns=prep_res["exclude_patterns"], max_file_size=prep_res["max_file_size"], - use_relative_paths=prep_res["use_relative_paths"] + use_relative_paths=prep_res["use_relative_paths"], ) # Convert dict to list of tuples: [(path, content), ...] files_list = list(result.get("files", {}).items()) if len(files_list) == 0: - raise(ValueError("Failed to fetch files")) + raise (ValueError("Failed to fetch files")) print(f"Fetched {len(files_list)} files.") return files_list def post(self, shared, prep_res, exec_res): - shared["files"] = exec_res # List of (path, content) tuples + shared["files"] = exec_res # List of (path, content) tuples + class IdentifyAbstractions(Node): def prep(self, shared): files_data = shared["files"] project_name = shared["project_name"] # Get project name - language = shared.get("language", "english") # Get language + language = shared.get("language", "english") # Get language use_cache = shared.get("use_cache", True) # Get use_cache flag, default to True # Helper to create context from files, respecting limits (basic example) def create_llm_context(files_data): context = "" - file_info = [] # Store tuples of (index, path) + file_info = [] # Store tuples of (index, path) for i, (path, content) in enumerate(files_data): entry = f"--- File Index {i}: {path} ---\n{content}\n\n" context += entry file_info.append((i, path)) - return context, file_info # file_info is list of (index, path) + return context, file_info # file_info is list of (index, path) context, file_info = create_llm_context(files_data) # Format file info for the prompt (comment is just a hint for LLM) - file_listing_for_prompt = "\n".join([f"- {idx} # {path}" for idx, path in file_info]) - return context, file_listing_for_prompt, len(files_data), project_name, language, use_cache # Return use_cache + file_listing_for_prompt = "\n".join( + [f"- {idx} # {path}" for idx, path in file_info] + ) + return ( + context, + file_listing_for_prompt, + len(files_data), + project_name, + language, + use_cache, + ) # Return use_cache def exec(self, prep_res): - context, file_listing_for_prompt, file_count, project_name, language, use_cache = prep_res # Unpack use_cache + ( + context, + file_listing_for_prompt, + file_count, + project_name, + language, + use_cache, + ) = prep_res # Unpack use_cache print(f"Identifying abstractions using LLM...") # Add language instruction and hints only if not English @@ -155,7 +174,6 @@ Format the output as a YAML list of dictionaries: # --- Validation --- yaml_str = response.strip().split("```yaml")[1].split("```")[0].strip() # add whitespace to fix llm generation error(except -) - yaml_str = add_indentation(yaml_str) abstractions = yaml.safe_load(yaml_str) if not isinstance(abstractions, list): @@ -163,52 +181,67 @@ Format the output as a YAML list of dictionaries: validated_abstractions = [] for item in abstractions: - if not isinstance(item, dict) or not all(k in item for k in ["name", "description", "file_indices"]): + if not isinstance(item, dict) or not all( + k in item for k in ["name", "description", "file_indices"] + ): raise ValueError(f"Missing keys in abstraction item: {item}") if not isinstance(item["name"], str): - raise ValueError(f"Name is not a string in item: {item}") + raise ValueError(f"Name is not a string in item: {item}") if not isinstance(item["description"], str): - raise ValueError(f"Description is not a string in item: {item}") + raise ValueError(f"Description is not a string in item: {item}") if not isinstance(item["file_indices"], list): - raise ValueError(f"file_indices is not a list in item: {item}") + raise ValueError(f"file_indices is not a list in item: {item}") # Validate indices validated_indices = [] for idx_entry in item["file_indices"]: - try: - if isinstance(idx_entry, int): - idx = idx_entry - elif isinstance(idx_entry, str) and '#' in idx_entry: - idx = int(idx_entry.split('#')[0].strip()) - else: - idx = int(str(idx_entry).strip()) + try: + if isinstance(idx_entry, int): + idx = idx_entry + elif isinstance(idx_entry, str) and "#" in idx_entry: + idx = int(idx_entry.split("#")[0].strip()) + else: + idx = int(str(idx_entry).strip()) - if not (0 <= idx < file_count): - raise ValueError(f"Invalid file index {idx} found in item {item['name']}. Max index is {file_count - 1}.") - validated_indices.append(idx) - except (ValueError, TypeError): - raise ValueError(f"Could not parse index from entry: {idx_entry} in item {item['name']}") + if not (0 <= idx < file_count): + raise ValueError( + f"Invalid file index {idx} found in item {item['name']}. Max index is {file_count - 1}." + ) + validated_indices.append(idx) + except (ValueError, TypeError): + raise ValueError( + f"Could not parse index from entry: {idx_entry} in item {item['name']}" + ) item["files"] = sorted(list(set(validated_indices))) # Store only the required fields - validated_abstractions.append({ - "name": item["name"], # Potentially translated name - "description": item["description"], # Potentially translated description - "files": item["files"] - }) + validated_abstractions.append( + { + "name": item["name"], # Potentially translated name + "description": item[ + "description" + ], # Potentially translated description + "files": item["files"], + } + ) print(f"Identified {len(validated_abstractions)} abstractions.") return validated_abstractions def post(self, shared, prep_res, exec_res): - shared["abstractions"] = exec_res # List of {"name": str, "description": str, "files": [int]} + shared["abstractions"] = ( + exec_res # List of {"name": str, "description": str, "files": [int]} + ) + class AnalyzeRelationships(Node): def prep(self, shared): - abstractions = shared["abstractions"] # Now contains 'files' list of indices, name/description potentially translated + abstractions = shared[ + "abstractions" + ] # Now contains 'files' list of indices, name/description potentially translated files_data = shared["files"] project_name = shared["project_name"] # Get project name - language = shared.get("language", "english") # Get language + language = shared.get("language", "english") # Get language use_cache = shared.get("use_cache", True) # Get use_cache flag, default to True # Create context with abstraction names, indices, descriptions, and relevant file snippets @@ -217,18 +250,19 @@ class AnalyzeRelationships(Node): abstraction_info_for_prompt = [] for i, abstr in enumerate(abstractions): # Use 'files' which contains indices directly - file_indices_str = ", ".join(map(str, abstr['files'])) + file_indices_str = ", ".join(map(str, abstr["files"])) # Abstraction name and description might be translated already info_line = f"- Index {i}: {abstr['name']} (Relevant file indices: [{file_indices_str}])\n Description: {abstr['description']}" context += info_line + "\n" - abstraction_info_for_prompt.append(f"{i} # {abstr['name']}") # Use potentially translated name here too - all_relevant_indices.update(abstr['files']) + abstraction_info_for_prompt.append( + f"{i} # {abstr['name']}" + ) # Use potentially translated name here too + all_relevant_indices.update(abstr["files"]) context += "\nRelevant File Snippets (Referenced by Index and Path):\n" # Get content for relevant files using helper relevant_files_content_map = get_content_for_indices( - files_data, - sorted(list(all_relevant_indices)) + files_data, sorted(list(all_relevant_indices)) ) # Format file content for context file_context_str = "\n\n".join( @@ -237,10 +271,18 @@ class AnalyzeRelationships(Node): ) context += file_context_str - return context, "\n".join(abstraction_info_for_prompt), project_name, language, use_cache # Return use_cache + return ( + context, + "\n".join(abstraction_info_for_prompt), + project_name, + language, + use_cache, + ) # Return use_cache def exec(self, prep_res): - context, abstraction_listing, project_name, language, use_cache = prep_res # Unpack use_cache + context, abstraction_listing, project_name, language, use_cache = ( + prep_res # Unpack use_cache + ) print(f"Analyzing relationships using LLM...") # Add language instruction and hints only if not English @@ -250,7 +292,7 @@ class AnalyzeRelationships(Node): if language.lower() != "english": language_instruction = f"IMPORTANT: Generate the `summary` and relationship `label` fields in **{language.capitalize()}** language. Do NOT use English for these fields.\n\n" lang_hint = f" (in {language.capitalize()})" - list_lang_note = f" (Names might be in {language.capitalize()})" # Note for the input list + list_lang_note = f" (Names might be in {language.capitalize()})" # Note for the input list prompt = f""" Based on the following abstractions and relevant code snippets from the project `{project_name}`: @@ -296,85 +338,117 @@ Now, provide the YAML output: yaml_str = response.strip().split("```yaml")[1].split("```")[0].strip() relationships_data = yaml.safe_load(yaml_str) - if not isinstance(relationships_data, dict) or not all(k in relationships_data for k in ["summary", "relationships"]): - raise ValueError("LLM output is not a dict or missing keys ('summary', 'relationships')") + if not isinstance(relationships_data, dict) or not all( + k in relationships_data for k in ["summary", "relationships"] + ): + raise ValueError( + "LLM output is not a dict or missing keys ('summary', 'relationships')" + ) if not isinstance(relationships_data["summary"], str): - raise ValueError("summary is not a string") + raise ValueError("summary is not a string") if not isinstance(relationships_data["relationships"], list): - raise ValueError("relationships is not a list") + raise ValueError("relationships is not a list") # Validate relationships structure validated_relationships = [] - num_abstractions = len(abstraction_listing.split('\n')) + num_abstractions = len(abstraction_listing.split("\n")) for rel in relationships_data["relationships"]: - # Check for 'label' key - if not isinstance(rel, dict) or not all(k in rel for k in ["from_abstraction", "to_abstraction", "label"]): - raise ValueError(f"Missing keys (expected from_abstraction, to_abstraction, label) in relationship item: {rel}") - # Validate 'label' is a string - if not isinstance(rel["label"], str): - raise ValueError(f"Relationship label is not a string: {rel}") + # Check for 'label' key + if not isinstance(rel, dict) or not all( + k in rel for k in ["from_abstraction", "to_abstraction", "label"] + ): + raise ValueError( + f"Missing keys (expected from_abstraction, to_abstraction, label) in relationship item: {rel}" + ) + # Validate 'label' is a string + if not isinstance(rel["label"], str): + raise ValueError(f"Relationship label is not a string: {rel}") - # Validate indices - try: - from_idx = int(str(rel["from_abstraction"]).split('#')[0].strip()) - to_idx = int(str(rel["to_abstraction"]).split('#')[0].strip()) - if not (0 <= from_idx < num_abstractions and 0 <= to_idx < num_abstractions): - raise ValueError(f"Invalid index in relationship: from={from_idx}, to={to_idx}. Max index is {num_abstractions-1}.") - validated_relationships.append({ - "from": from_idx, - "to": to_idx, - "label": rel["label"] # Potentially translated label - }) - except (ValueError, TypeError): - raise ValueError(f"Could not parse indices from relationship: {rel}") + # Validate indices + try: + from_idx = int(str(rel["from_abstraction"]).split("#")[0].strip()) + to_idx = int(str(rel["to_abstraction"]).split("#")[0].strip()) + if not ( + 0 <= from_idx < num_abstractions and 0 <= to_idx < num_abstractions + ): + raise ValueError( + f"Invalid index in relationship: from={from_idx}, to={to_idx}. Max index is {num_abstractions-1}." + ) + validated_relationships.append( + { + "from": from_idx, + "to": to_idx, + "label": rel["label"], # Potentially translated label + } + ) + except (ValueError, TypeError): + raise ValueError(f"Could not parse indices from relationship: {rel}") print("Generated project summary and relationship details.") return { - "summary": relationships_data["summary"], # Potentially translated summary - "details": validated_relationships # Store validated, index-based relationships with potentially translated labels + "summary": relationships_data["summary"], # Potentially translated summary + "details": validated_relationships, # Store validated, index-based relationships with potentially translated labels } - def post(self, shared, prep_res, exec_res): # Structure is now {"summary": str, "details": [{"from": int, "to": int, "label": str}]} # Summary and label might be translated shared["relationships"] = exec_res + class OrderChapters(Node): def prep(self, shared): - abstractions = shared["abstractions"] # Name/description might be translated - relationships = shared["relationships"] # Summary/label might be translated + abstractions = shared["abstractions"] # Name/description might be translated + relationships = shared["relationships"] # Summary/label might be translated project_name = shared["project_name"] # Get project name - language = shared.get("language", "english") # Get language + language = shared.get("language", "english") # Get language use_cache = shared.get("use_cache", True) # Get use_cache flag, default to True # Prepare context for the LLM abstraction_info_for_prompt = [] for i, a in enumerate(abstractions): - abstraction_info_for_prompt.append(f"- {i} # {a['name']}") # Use potentially translated name + abstraction_info_for_prompt.append( + f"- {i} # {a['name']}" + ) # Use potentially translated name abstraction_listing = "\n".join(abstraction_info_for_prompt) # Use potentially translated summary and labels summary_note = "" if language.lower() != "english": - summary_note = f" (Note: Project Summary might be in {language.capitalize()})" + summary_note = ( + f" (Note: Project Summary might be in {language.capitalize()})" + ) context = f"Project Summary{summary_note}:\n{relationships['summary']}\n\n" context += "Relationships (Indices refer to abstractions above):\n" - for rel in relationships['details']: - from_name = abstractions[rel['from']]['name'] - to_name = abstractions[rel['to']]['name'] - # Use potentially translated 'label' - context += f"- From {rel['from']} ({from_name}) to {rel['to']} ({to_name}): {rel['label']}\n" # Label might be translated + for rel in relationships["details"]: + from_name = abstractions[rel["from"]]["name"] + to_name = abstractions[rel["to"]]["name"] + # Use potentially translated 'label' + context += f"- From {rel['from']} ({from_name}) to {rel['to']} ({to_name}): {rel['label']}\n" # Label might be translated list_lang_note = "" if language.lower() != "english": - list_lang_note = f" (Names might be in {language.capitalize()})" + list_lang_note = f" (Names might be in {language.capitalize()})" - return abstraction_listing, context, len(abstractions), project_name, list_lang_note, use_cache # Return use_cache + return ( + abstraction_listing, + context, + len(abstractions), + project_name, + list_lang_note, + use_cache, + ) # Return use_cache def exec(self, prep_res): - abstraction_listing, context, num_abstractions, project_name, list_lang_note, use_cache = prep_res # Unpack use_cache + ( + abstraction_listing, + context, + num_abstractions, + project_name, + list_lang_note, + use_cache, + ) = prep_res # Unpack use_cache print("Determining chapter order using LLM...") # No language variation needed here in prompt instructions, just ordering based on structure # The input names might be translated, hence the note. @@ -414,62 +488,81 @@ Now, provide the YAML output: seen_indices = set() for entry in ordered_indices_raw: try: - if isinstance(entry, int): - idx = entry - elif isinstance(entry, str) and '#' in entry: - idx = int(entry.split('#')[0].strip()) - else: - idx = int(str(entry).strip()) + if isinstance(entry, int): + idx = entry + elif isinstance(entry, str) and "#" in entry: + idx = int(entry.split("#")[0].strip()) + else: + idx = int(str(entry).strip()) - if not (0 <= idx < num_abstractions): - raise ValueError(f"Invalid index {idx} in ordered list. Max index is {num_abstractions-1}.") - if idx in seen_indices: - raise ValueError(f"Duplicate index {idx} found in ordered list.") - ordered_indices.append(idx) - seen_indices.add(idx) + if not (0 <= idx < num_abstractions): + raise ValueError( + f"Invalid index {idx} in ordered list. Max index is {num_abstractions-1}." + ) + if idx in seen_indices: + raise ValueError(f"Duplicate index {idx} found in ordered list.") + ordered_indices.append(idx) + seen_indices.add(idx) except (ValueError, TypeError): - raise ValueError(f"Could not parse index from ordered list entry: {entry}") + raise ValueError( + f"Could not parse index from ordered list entry: {entry}" + ) # Check if all abstractions are included if len(ordered_indices) != num_abstractions: - raise ValueError(f"Ordered list length ({len(ordered_indices)}) does not match number of abstractions ({num_abstractions}). Missing indices: {set(range(num_abstractions)) - seen_indices}") + raise ValueError( + f"Ordered list length ({len(ordered_indices)}) does not match number of abstractions ({num_abstractions}). Missing indices: {set(range(num_abstractions)) - seen_indices}" + ) print(f"Determined chapter order (indices): {ordered_indices}") - return ordered_indices # Return the list of indices + return ordered_indices # Return the list of indices def post(self, shared, prep_res, exec_res): # exec_res is already the list of ordered indices - shared["chapter_order"] = exec_res # List of indices + shared["chapter_order"] = exec_res # List of indices + class WriteChapters(BatchNode): def prep(self, shared): - chapter_order = shared["chapter_order"] # List of indices - abstractions = shared["abstractions"] # List of {"name": str, "description": str, "files": [int]} - files_data = shared["files"] # List of (path, content) tuples + chapter_order = shared["chapter_order"] # List of indices + abstractions = shared[ + "abstractions" + ] # List of {"name": str, "description": str, "files": [int]} + files_data = shared["files"] # List of (path, content) tuples project_name = shared["project_name"] language = shared.get("language", "english") use_cache = shared.get("use_cache", True) # Get use_cache flag, default to True - + # Get already written chapters to provide context # We store them temporarily during the batch run, not in shared memory yet # The 'previous_chapters_summary' will be built progressively in the exec context - self.chapters_written_so_far = [] # Use instance variable for temporary storage across exec calls + self.chapters_written_so_far = ( + [] + ) # Use instance variable for temporary storage across exec calls # Create a complete list of all chapters all_chapters = [] - chapter_filenames = {} # Store chapter filename mapping for linking + chapter_filenames = {} # Store chapter filename mapping for linking for i, abstraction_index in enumerate(chapter_order): if 0 <= abstraction_index < len(abstractions): chapter_num = i + 1 - chapter_name = abstractions[abstraction_index]["name"] # Potentially translated name + chapter_name = abstractions[abstraction_index][ + "name" + ] # Potentially translated name # Create safe filename (from potentially translated name) - safe_name = "".join(c if c.isalnum() else '_' for c in chapter_name).lower() + safe_name = "".join( + c if c.isalnum() else "_" for c in chapter_name + ).lower() filename = f"{i+1:02d}_{safe_name}.md" # Format with link (using potentially translated name) all_chapters.append(f"{chapter_num}. [{chapter_name}]({filename})") # Store mapping of chapter index to filename for linking - chapter_filenames[abstraction_index] = {"num": chapter_num, "name": chapter_name, "filename": filename} + chapter_filenames[abstraction_index] = { + "num": chapter_num, + "name": chapter_name, + "filename": filename, + } # Create a formatted string with all chapters full_chapter_listing = "\n".join(all_chapters) @@ -477,47 +570,59 @@ class WriteChapters(BatchNode): items_to_process = [] for i, abstraction_index in enumerate(chapter_order): if 0 <= abstraction_index < len(abstractions): - abstraction_details = abstractions[abstraction_index] # Contains potentially translated name/desc + abstraction_details = abstractions[ + abstraction_index + ] # Contains potentially translated name/desc # Use 'files' (list of indices) directly related_file_indices = abstraction_details.get("files", []) # Get content using helper, passing indices - related_files_content_map = get_content_for_indices(files_data, related_file_indices) + related_files_content_map = get_content_for_indices( + files_data, related_file_indices + ) # Get previous chapter info for transitions (uses potentially translated name) prev_chapter = None if i > 0: - prev_idx = chapter_order[i-1] + prev_idx = chapter_order[i - 1] prev_chapter = chapter_filenames[prev_idx] # Get next chapter info for transitions (uses potentially translated name) next_chapter = None if i < len(chapter_order) - 1: - next_idx = chapter_order[i+1] + next_idx = chapter_order[i + 1] next_chapter = chapter_filenames[next_idx] - items_to_process.append({ - "chapter_num": i + 1, - "abstraction_index": abstraction_index, - "abstraction_details": abstraction_details, # Has potentially translated name/desc - "related_files_content_map": related_files_content_map, - "project_name": shared["project_name"], # Add project name - "full_chapter_listing": full_chapter_listing, # Add the full chapter listing (uses potentially translated names) - "chapter_filenames": chapter_filenames, # Add chapter filenames mapping (uses potentially translated names) - "prev_chapter": prev_chapter, # Add previous chapter info (uses potentially translated name) - "next_chapter": next_chapter, # Add next chapter info (uses potentially translated name) - "language": language, # Add language for multi-language support - # previous_chapters_summary will be added dynamically in exec - }) + items_to_process.append( + { + "chapter_num": i + 1, + "abstraction_index": abstraction_index, + "abstraction_details": abstraction_details, # Has potentially translated name/desc + "related_files_content_map": related_files_content_map, + "project_name": shared["project_name"], # Add project name + "full_chapter_listing": full_chapter_listing, # Add the full chapter listing (uses potentially translated names) + "chapter_filenames": chapter_filenames, # Add chapter filenames mapping (uses potentially translated names) + "prev_chapter": prev_chapter, # Add previous chapter info (uses potentially translated name) + "next_chapter": next_chapter, # Add next chapter info (uses potentially translated name) + "language": language, # Add language for multi-language support + # previous_chapters_summary will be added dynamically in exec + } + ) else: - print(f"Warning: Invalid abstraction index {abstraction_index} in chapter_order. Skipping.") + print( + f"Warning: Invalid abstraction index {abstraction_index} in chapter_order. Skipping." + ) print(f"Preparing to write {len(items_to_process)} chapters...") - return items_to_process # Iterable for BatchNode + return items_to_process # Iterable for BatchNode def exec(self, item): # This runs for each item prepared above - abstraction_name = item["abstraction_details"]["name"] # Potentially translated name - abstraction_description = item["abstraction_details"]["description"] # Potentially translated description + abstraction_name = item["abstraction_details"][ + "name" + ] # Potentially translated name + abstraction_description = item["abstraction_details"][ + "description" + ] # Potentially translated description chapter_num = item["chapter_num"] project_name = item.get("project_name") language = item.get("language", "english") @@ -552,10 +657,11 @@ class WriteChapters(BatchNode): instruction_lang_note = f" (in {lang_cap})" mermaid_lang_note = f" (Use {lang_cap} for labels/text if appropriate)" code_comment_note = f" (Translate to {lang_cap} if possible, otherwise keep minimal English for clarity)" - link_lang_note = f" (Use the {lang_cap} chapter title from the structure above)" + link_lang_note = ( + f" (Use the {lang_cap} chapter title from the structure above)" + ) tone_note = f" (appropriate for {lang_cap} readers)" - prompt = f""" {language_instruction}Write a very beginner-friendly tutorial chapter (in Markdown format) for the project `{project_name}` about the concept: "{abstraction_name}". This is Chapter {chapter_num}. @@ -606,20 +712,22 @@ Now, directly provide a super beginner-friendly Markdown output (DON'T need ```m """ chapter_content = call_llm(prompt) # Basic validation/cleanup - actual_heading = f"# Chapter {chapter_num}: {abstraction_name}" # Use potentially translated name + actual_heading = f"# Chapter {chapter_num}: {abstraction_name}" # Use potentially translated name if not chapter_content.strip().startswith(f"# Chapter {chapter_num}"): - # Add heading if missing or incorrect, trying to preserve content - lines = chapter_content.strip().split('\n') - if lines and lines[0].strip().startswith("#"): # If there's some heading, replace it - lines[0] = actual_heading - chapter_content = "\n".join(lines) - else: # Otherwise, prepend it - chapter_content = f"{actual_heading}\n\n{chapter_content}" + # Add heading if missing or incorrect, trying to preserve content + lines = chapter_content.strip().split("\n") + if lines and lines[0].strip().startswith( + "#" + ): # If there's some heading, replace it + lines[0] = actual_heading + chapter_content = "\n".join(lines) + else: # Otherwise, prepend it + chapter_content = f"{actual_heading}\n\n{chapter_content}" # Add the generated content to our temporary list for the next iteration's context self.chapters_written_so_far.append(chapter_content) - return chapter_content # Return the Markdown string (potentially translated) + return chapter_content # Return the Markdown string (potentially translated) def post(self, shared, prep_res, exec_res_list): # exec_res_list contains the generated Markdown for each chapter, in order @@ -628,19 +736,26 @@ Now, directly provide a super beginner-friendly Markdown output (DON'T need ```m del self.chapters_written_so_far print(f"Finished writing {len(exec_res_list)} chapters.") + class CombineTutorial(Node): def prep(self, shared): project_name = shared["project_name"] - output_base_dir = shared.get("output_dir", "output") # Default output dir + output_base_dir = shared.get("output_dir", "output") # Default output dir output_path = os.path.join(output_base_dir, project_name) repo_url = shared.get("repo_url") # Get the repository URL # language = shared.get("language", "english") # No longer needed for fixed strings # Get potentially translated data - relationships_data = shared["relationships"] # {"summary": str, "details": [{"from": int, "to": int, "label": str}]} -> summary/label potentially translated - chapter_order = shared["chapter_order"] # indices - abstractions = shared["abstractions"] # list of dicts -> name/description potentially translated - chapters_content = shared["chapters"] # list of strings -> content potentially translated + relationships_data = shared[ + "relationships" + ] # {"summary": str, "details": [{"from": int, "to": int, "label": str}]} -> summary/label potentially translated + chapter_order = shared["chapter_order"] # indices + abstractions = shared[ + "abstractions" + ] # list of dicts -> name/description potentially translated + chapters_content = shared[ + "chapters" + ] # list of strings -> content potentially translated # --- Generate Mermaid Diagram --- mermaid_lines = ["flowchart TD"] @@ -648,26 +763,32 @@ class CombineTutorial(Node): for i, abstr in enumerate(abstractions): node_id = f"A{i}" # Use potentially translated name, sanitize for Mermaid ID and label - sanitized_name = abstr['name'].replace('"', '') - node_label = sanitized_name # Using sanitized name only - mermaid_lines.append(f' {node_id}["{node_label}"]') # Node label uses potentially translated name + sanitized_name = abstr["name"].replace('"', "") + node_label = sanitized_name # Using sanitized name only + mermaid_lines.append( + f' {node_id}["{node_label}"]' + ) # Node label uses potentially translated name # Add edges for relationships using potentially translated labels - for rel in relationships_data['details']: + for rel in relationships_data["details"]: from_node_id = f"A{rel['from']}" to_node_id = f"A{rel['to']}" # Use potentially translated label, sanitize - edge_label = rel['label'].replace('"', '').replace('\n', ' ') # Basic sanitization + edge_label = ( + rel["label"].replace('"', "").replace("\n", " ") + ) # Basic sanitization max_label_len = 30 if len(edge_label) > max_label_len: - edge_label = edge_label[:max_label_len-3] + "..." - mermaid_lines.append(f' {from_node_id} -- "{edge_label}" --> {to_node_id}') # Edge label uses potentially translated label + edge_label = edge_label[: max_label_len - 3] + "..." + mermaid_lines.append( + f' {from_node_id} -- "{edge_label}" --> {to_node_id}' + ) # Edge label uses potentially translated label mermaid_diagram = "\n".join(mermaid_lines) # --- End Mermaid --- # --- Prepare index.md content --- index_content = f"# Tutorial: {project_name}\n\n" - index_content += f"{relationships_data['summary']}\n\n" # Use the potentially translated summary directly + index_content += f"{relationships_data['summary']}\n\n" # Use the potentially translated summary directly # Keep fixed strings in English index_content += f"**Source Repository:** [{repo_url}]({repo_url})\n\n" @@ -684,14 +805,18 @@ class CombineTutorial(Node): for i, abstraction_index in enumerate(chapter_order): # Ensure index is valid and we have content for it if 0 <= abstraction_index < len(abstractions) and i < len(chapters_content): - abstraction_name = abstractions[abstraction_index]["name"] # Potentially translated name + abstraction_name = abstractions[abstraction_index][ + "name" + ] # Potentially translated name # Sanitize potentially translated name for filename - safe_name = "".join(c if c.isalnum() else '_' for c in abstraction_name).lower() + safe_name = "".join( + c if c.isalnum() else "_" for c in abstraction_name + ).lower() filename = f"{i+1:02d}_{safe_name}.md" - index_content += f"{i+1}. [{abstraction_name}]({filename})\n" # Use potentially translated name in link text + index_content += f"{i+1}. [{abstraction_name}]({filename})\n" # Use potentially translated name in link text # Add attribution to chapter content (using English fixed string) - chapter_content = chapters_content[i] # Potentially translated content + chapter_content = chapters_content[i] # Potentially translated content if not chapter_content.endswith("\n\n"): chapter_content += "\n\n" # Keep fixed strings in English @@ -700,7 +825,9 @@ class CombineTutorial(Node): # Store filename and corresponding content chapter_files.append({"filename": filename, "content": chapter_content}) else: - print(f"Warning: Mismatch between chapter order, abstractions, or content at index {i} (abstraction index {abstraction_index}). Skipping file generation for this entry.") + print( + f"Warning: Mismatch between chapter order, abstractions, or content at index {i} (abstraction index {abstraction_index}). Skipping file generation for this entry." + ) # Add attribution to index content (using English fixed string) index_content += f"\n\n---\n\nGenerated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge)" @@ -708,7 +835,7 @@ class CombineTutorial(Node): return { "output_path": output_path, "index_content": index_content, - "chapter_files": chapter_files # List of {"filename": str, "content": str} + "chapter_files": chapter_files, # List of {"filename": str, "content": str} } def exec(self, prep_res): @@ -733,9 +860,8 @@ class CombineTutorial(Node): f.write(chapter_info["content"]) print(f" - Wrote {chapter_filepath}") - return output_path # Return the final path - + return output_path # Return the final path def post(self, shared, prep_res, exec_res): - shared["final_output_dir"] = exec_res # Store the output path + shared["final_output_dir"] = exec_res # Store the output path print(f"\nTutorial generation complete! Files are in: {exec_res}") diff --git a/utils/fix_yaml.py b/utils/fix_yaml.py deleted file mode 100644 index 1e7f55d..0000000 --- a/utils/fix_yaml.py +++ /dev/null @@ -1,20 +0,0 @@ -import re - -def add_indentation(text): - # This pattern matches lines that don't start with a hyphen or whitespace - pattern = r'^(?![-\s])(.*)$' - - # Replace with 4 spaces followed by the captured content - result = re.sub(pattern, r' \1', text, flags=re.MULTILINE) - - return result - - -if __name__ == "__main__": - # Example usage - text = """This line will be indented - - This line won't be indented - This line won't be indented either - Another line that will be indented""" - - indented_text = add_indentation(text) \ No newline at end of file