mirror of
https://github.com/The-Pocket/PocketFlow-Tutorial-Codebase-Knowledge.git
synced 2026-08-30 09:00:32 +08:00
Merge branch 'feat/openrouter'
This commit is contained in:
remy
@@ -5,7 +5,6 @@ from pocketflow import Node, BatchNode
|
||||
from utils.crawl_github_files import crawl_github_files
|
||||
from utils.call_llm import call_llm
|
||||
from utils.crawl_local_files import crawl_local_files
|
||||
from utils.fix_yaml import add_indentation
|
||||
|
||||
|
||||
# Helper to get content for specific file indices
|
||||
@@ -14,9 +13,12 @@ def get_content_for_indices(files_data, indices):
|
||||
for i in indices:
|
||||
if 0 <= i < len(files_data):
|
||||
path, content = files_data[i]
|
||||
content_map[f"{i} # {path}"] = content # Use index + path as key for context
|
||||
content_map[f"{i} # {path}"] = (
|
||||
content # Use index + path as key for context
|
||||
)
|
||||
return content_map
|
||||
|
||||
|
||||
class FetchRepo(Node):
|
||||
def prep(self, shared):
|
||||
repo_url = shared.get("repo_url")
|
||||
@@ -26,7 +28,7 @@ class FetchRepo(Node):
|
||||
if not project_name:
|
||||
# Basic name derivation from URL or directory
|
||||
if repo_url:
|
||||
project_name = repo_url.split('/')[-1].replace('.git', '')
|
||||
project_name = repo_url.split("/")[-1].replace(".git", "")
|
||||
else:
|
||||
project_name = os.path.basename(os.path.abspath(local_dir))
|
||||
shared["project_name"] = project_name
|
||||
@@ -43,7 +45,7 @@ class FetchRepo(Node):
|
||||
"include_patterns": include_patterns,
|
||||
"exclude_patterns": exclude_patterns,
|
||||
"max_file_size": max_file_size,
|
||||
"use_relative_paths": True
|
||||
"use_relative_paths": True,
|
||||
}
|
||||
|
||||
def exec(self, prep_res):
|
||||
@@ -55,7 +57,7 @@ class FetchRepo(Node):
|
||||
include_patterns=prep_res["include_patterns"],
|
||||
exclude_patterns=prep_res["exclude_patterns"],
|
||||
max_file_size=prep_res["max_file_size"],
|
||||
use_relative_paths=prep_res["use_relative_paths"]
|
||||
use_relative_paths=prep_res["use_relative_paths"],
|
||||
)
|
||||
else:
|
||||
print(f"Crawling directory: {prep_res['local_dir']}...")
|
||||
@@ -64,44 +66,61 @@ class FetchRepo(Node):
|
||||
include_patterns=prep_res["include_patterns"],
|
||||
exclude_patterns=prep_res["exclude_patterns"],
|
||||
max_file_size=prep_res["max_file_size"],
|
||||
use_relative_paths=prep_res["use_relative_paths"]
|
||||
use_relative_paths=prep_res["use_relative_paths"],
|
||||
)
|
||||
|
||||
# Convert dict to list of tuples: [(path, content), ...]
|
||||
files_list = list(result.get("files", {}).items())
|
||||
if len(files_list) == 0:
|
||||
raise(ValueError("Failed to fetch files"))
|
||||
raise (ValueError("Failed to fetch files"))
|
||||
print(f"Fetched {len(files_list)} files.")
|
||||
return files_list
|
||||
|
||||
def post(self, shared, prep_res, exec_res):
|
||||
shared["files"] = exec_res # List of (path, content) tuples
|
||||
shared["files"] = exec_res # List of (path, content) tuples
|
||||
|
||||
|
||||
class IdentifyAbstractions(Node):
|
||||
def prep(self, shared):
|
||||
files_data = shared["files"]
|
||||
project_name = shared["project_name"] # Get project name
|
||||
language = shared.get("language", "english") # Get language
|
||||
language = shared.get("language", "english") # Get language
|
||||
use_cache = shared.get("use_cache", True) # Get use_cache flag, default to True
|
||||
|
||||
# Helper to create context from files, respecting limits (basic example)
|
||||
def create_llm_context(files_data):
|
||||
context = ""
|
||||
file_info = [] # Store tuples of (index, path)
|
||||
file_info = [] # Store tuples of (index, path)
|
||||
for i, (path, content) in enumerate(files_data):
|
||||
entry = f"--- File Index {i}: {path} ---\n{content}\n\n"
|
||||
context += entry
|
||||
file_info.append((i, path))
|
||||
|
||||
return context, file_info # file_info is list of (index, path)
|
||||
return context, file_info # file_info is list of (index, path)
|
||||
|
||||
context, file_info = create_llm_context(files_data)
|
||||
# Format file info for the prompt (comment is just a hint for LLM)
|
||||
file_listing_for_prompt = "\n".join([f"- {idx} # {path}" for idx, path in file_info])
|
||||
return context, file_listing_for_prompt, len(files_data), project_name, language, use_cache # Return use_cache
|
||||
file_listing_for_prompt = "\n".join(
|
||||
[f"- {idx} # {path}" for idx, path in file_info]
|
||||
)
|
||||
return (
|
||||
context,
|
||||
file_listing_for_prompt,
|
||||
len(files_data),
|
||||
project_name,
|
||||
language,
|
||||
use_cache,
|
||||
) # Return use_cache
|
||||
|
||||
def exec(self, prep_res):
|
||||
context, file_listing_for_prompt, file_count, project_name, language, use_cache = prep_res # Unpack use_cache
|
||||
(
|
||||
context,
|
||||
file_listing_for_prompt,
|
||||
file_count,
|
||||
project_name,
|
||||
language,
|
||||
use_cache,
|
||||
) = prep_res # Unpack use_cache
|
||||
print(f"Identifying abstractions using LLM...")
|
||||
|
||||
# Add language instruction and hints only if not English
|
||||
@@ -155,7 +174,6 @@ Format the output as a YAML list of dictionaries:
|
||||
# --- Validation ---
|
||||
yaml_str = response.strip().split("```yaml")[1].split("```")[0].strip()
|
||||
# add whitespace to fix llm generation error(except -)
|
||||
yaml_str = add_indentation(yaml_str)
|
||||
abstractions = yaml.safe_load(yaml_str)
|
||||
|
||||
if not isinstance(abstractions, list):
|
||||
@@ -163,52 +181,67 @@ Format the output as a YAML list of dictionaries:
|
||||
|
||||
validated_abstractions = []
|
||||
for item in abstractions:
|
||||
if not isinstance(item, dict) or not all(k in item for k in ["name", "description", "file_indices"]):
|
||||
if not isinstance(item, dict) or not all(
|
||||
k in item for k in ["name", "description", "file_indices"]
|
||||
):
|
||||
raise ValueError(f"Missing keys in abstraction item: {item}")
|
||||
if not isinstance(item["name"], str):
|
||||
raise ValueError(f"Name is not a string in item: {item}")
|
||||
raise ValueError(f"Name is not a string in item: {item}")
|
||||
if not isinstance(item["description"], str):
|
||||
raise ValueError(f"Description is not a string in item: {item}")
|
||||
raise ValueError(f"Description is not a string in item: {item}")
|
||||
if not isinstance(item["file_indices"], list):
|
||||
raise ValueError(f"file_indices is not a list in item: {item}")
|
||||
raise ValueError(f"file_indices is not a list in item: {item}")
|
||||
|
||||
# Validate indices
|
||||
validated_indices = []
|
||||
for idx_entry in item["file_indices"]:
|
||||
try:
|
||||
if isinstance(idx_entry, int):
|
||||
idx = idx_entry
|
||||
elif isinstance(idx_entry, str) and '#' in idx_entry:
|
||||
idx = int(idx_entry.split('#')[0].strip())
|
||||
else:
|
||||
idx = int(str(idx_entry).strip())
|
||||
try:
|
||||
if isinstance(idx_entry, int):
|
||||
idx = idx_entry
|
||||
elif isinstance(idx_entry, str) and "#" in idx_entry:
|
||||
idx = int(idx_entry.split("#")[0].strip())
|
||||
else:
|
||||
idx = int(str(idx_entry).strip())
|
||||
|
||||
if not (0 <= idx < file_count):
|
||||
raise ValueError(f"Invalid file index {idx} found in item {item['name']}. Max index is {file_count - 1}.")
|
||||
validated_indices.append(idx)
|
||||
except (ValueError, TypeError):
|
||||
raise ValueError(f"Could not parse index from entry: {idx_entry} in item {item['name']}")
|
||||
if not (0 <= idx < file_count):
|
||||
raise ValueError(
|
||||
f"Invalid file index {idx} found in item {item['name']}. Max index is {file_count - 1}."
|
||||
)
|
||||
validated_indices.append(idx)
|
||||
except (ValueError, TypeError):
|
||||
raise ValueError(
|
||||
f"Could not parse index from entry: {idx_entry} in item {item['name']}"
|
||||
)
|
||||
|
||||
item["files"] = sorted(list(set(validated_indices)))
|
||||
# Store only the required fields
|
||||
validated_abstractions.append({
|
||||
"name": item["name"], # Potentially translated name
|
||||
"description": item["description"], # Potentially translated description
|
||||
"files": item["files"]
|
||||
})
|
||||
validated_abstractions.append(
|
||||
{
|
||||
"name": item["name"], # Potentially translated name
|
||||
"description": item[
|
||||
"description"
|
||||
], # Potentially translated description
|
||||
"files": item["files"],
|
||||
}
|
||||
)
|
||||
|
||||
print(f"Identified {len(validated_abstractions)} abstractions.")
|
||||
return validated_abstractions
|
||||
|
||||
def post(self, shared, prep_res, exec_res):
|
||||
shared["abstractions"] = exec_res # List of {"name": str, "description": str, "files": [int]}
|
||||
shared["abstractions"] = (
|
||||
exec_res # List of {"name": str, "description": str, "files": [int]}
|
||||
)
|
||||
|
||||
|
||||
class AnalyzeRelationships(Node):
|
||||
def prep(self, shared):
|
||||
abstractions = shared["abstractions"] # Now contains 'files' list of indices, name/description potentially translated
|
||||
abstractions = shared[
|
||||
"abstractions"
|
||||
] # Now contains 'files' list of indices, name/description potentially translated
|
||||
files_data = shared["files"]
|
||||
project_name = shared["project_name"] # Get project name
|
||||
language = shared.get("language", "english") # Get language
|
||||
language = shared.get("language", "english") # Get language
|
||||
use_cache = shared.get("use_cache", True) # Get use_cache flag, default to True
|
||||
|
||||
# Create context with abstraction names, indices, descriptions, and relevant file snippets
|
||||
@@ -217,18 +250,19 @@ class AnalyzeRelationships(Node):
|
||||
abstraction_info_for_prompt = []
|
||||
for i, abstr in enumerate(abstractions):
|
||||
# Use 'files' which contains indices directly
|
||||
file_indices_str = ", ".join(map(str, abstr['files']))
|
||||
file_indices_str = ", ".join(map(str, abstr["files"]))
|
||||
# Abstraction name and description might be translated already
|
||||
info_line = f"- Index {i}: {abstr['name']} (Relevant file indices: [{file_indices_str}])\n Description: {abstr['description']}"
|
||||
context += info_line + "\n"
|
||||
abstraction_info_for_prompt.append(f"{i} # {abstr['name']}") # Use potentially translated name here too
|
||||
all_relevant_indices.update(abstr['files'])
|
||||
abstraction_info_for_prompt.append(
|
||||
f"{i} # {abstr['name']}"
|
||||
) # Use potentially translated name here too
|
||||
all_relevant_indices.update(abstr["files"])
|
||||
|
||||
context += "\nRelevant File Snippets (Referenced by Index and Path):\n"
|
||||
# Get content for relevant files using helper
|
||||
relevant_files_content_map = get_content_for_indices(
|
||||
files_data,
|
||||
sorted(list(all_relevant_indices))
|
||||
files_data, sorted(list(all_relevant_indices))
|
||||
)
|
||||
# Format file content for context
|
||||
file_context_str = "\n\n".join(
|
||||
@@ -237,10 +271,18 @@ class AnalyzeRelationships(Node):
|
||||
)
|
||||
context += file_context_str
|
||||
|
||||
return context, "\n".join(abstraction_info_for_prompt), project_name, language, use_cache # Return use_cache
|
||||
return (
|
||||
context,
|
||||
"\n".join(abstraction_info_for_prompt),
|
||||
project_name,
|
||||
language,
|
||||
use_cache,
|
||||
) # Return use_cache
|
||||
|
||||
def exec(self, prep_res):
|
||||
context, abstraction_listing, project_name, language, use_cache = prep_res # Unpack use_cache
|
||||
context, abstraction_listing, project_name, language, use_cache = (
|
||||
prep_res # Unpack use_cache
|
||||
)
|
||||
print(f"Analyzing relationships using LLM...")
|
||||
|
||||
# Add language instruction and hints only if not English
|
||||
@@ -250,7 +292,7 @@ class AnalyzeRelationships(Node):
|
||||
if language.lower() != "english":
|
||||
language_instruction = f"IMPORTANT: Generate the `summary` and relationship `label` fields in **{language.capitalize()}** language. Do NOT use English for these fields.\n\n"
|
||||
lang_hint = f" (in {language.capitalize()})"
|
||||
list_lang_note = f" (Names might be in {language.capitalize()})" # Note for the input list
|
||||
list_lang_note = f" (Names might be in {language.capitalize()})" # Note for the input list
|
||||
|
||||
prompt = f"""
|
||||
Based on the following abstractions and relevant code snippets from the project `{project_name}`:
|
||||
@@ -296,85 +338,117 @@ Now, provide the YAML output:
|
||||
yaml_str = response.strip().split("```yaml")[1].split("```")[0].strip()
|
||||
relationships_data = yaml.safe_load(yaml_str)
|
||||
|
||||
if not isinstance(relationships_data, dict) or not all(k in relationships_data for k in ["summary", "relationships"]):
|
||||
raise ValueError("LLM output is not a dict or missing keys ('summary', 'relationships')")
|
||||
if not isinstance(relationships_data, dict) or not all(
|
||||
k in relationships_data for k in ["summary", "relationships"]
|
||||
):
|
||||
raise ValueError(
|
||||
"LLM output is not a dict or missing keys ('summary', 'relationships')"
|
||||
)
|
||||
if not isinstance(relationships_data["summary"], str):
|
||||
raise ValueError("summary is not a string")
|
||||
raise ValueError("summary is not a string")
|
||||
if not isinstance(relationships_data["relationships"], list):
|
||||
raise ValueError("relationships is not a list")
|
||||
raise ValueError("relationships is not a list")
|
||||
|
||||
# Validate relationships structure
|
||||
validated_relationships = []
|
||||
num_abstractions = len(abstraction_listing.split('\n'))
|
||||
num_abstractions = len(abstraction_listing.split("\n"))
|
||||
for rel in relationships_data["relationships"]:
|
||||
# Check for 'label' key
|
||||
if not isinstance(rel, dict) or not all(k in rel for k in ["from_abstraction", "to_abstraction", "label"]):
|
||||
raise ValueError(f"Missing keys (expected from_abstraction, to_abstraction, label) in relationship item: {rel}")
|
||||
# Validate 'label' is a string
|
||||
if not isinstance(rel["label"], str):
|
||||
raise ValueError(f"Relationship label is not a string: {rel}")
|
||||
# Check for 'label' key
|
||||
if not isinstance(rel, dict) or not all(
|
||||
k in rel for k in ["from_abstraction", "to_abstraction", "label"]
|
||||
):
|
||||
raise ValueError(
|
||||
f"Missing keys (expected from_abstraction, to_abstraction, label) in relationship item: {rel}"
|
||||
)
|
||||
# Validate 'label' is a string
|
||||
if not isinstance(rel["label"], str):
|
||||
raise ValueError(f"Relationship label is not a string: {rel}")
|
||||
|
||||
# Validate indices
|
||||
try:
|
||||
from_idx = int(str(rel["from_abstraction"]).split('#')[0].strip())
|
||||
to_idx = int(str(rel["to_abstraction"]).split('#')[0].strip())
|
||||
if not (0 <= from_idx < num_abstractions and 0 <= to_idx < num_abstractions):
|
||||
raise ValueError(f"Invalid index in relationship: from={from_idx}, to={to_idx}. Max index is {num_abstractions-1}.")
|
||||
validated_relationships.append({
|
||||
"from": from_idx,
|
||||
"to": to_idx,
|
||||
"label": rel["label"] # Potentially translated label
|
||||
})
|
||||
except (ValueError, TypeError):
|
||||
raise ValueError(f"Could not parse indices from relationship: {rel}")
|
||||
# Validate indices
|
||||
try:
|
||||
from_idx = int(str(rel["from_abstraction"]).split("#")[0].strip())
|
||||
to_idx = int(str(rel["to_abstraction"]).split("#")[0].strip())
|
||||
if not (
|
||||
0 <= from_idx < num_abstractions and 0 <= to_idx < num_abstractions
|
||||
):
|
||||
raise ValueError(
|
||||
f"Invalid index in relationship: from={from_idx}, to={to_idx}. Max index is {num_abstractions-1}."
|
||||
)
|
||||
validated_relationships.append(
|
||||
{
|
||||
"from": from_idx,
|
||||
"to": to_idx,
|
||||
"label": rel["label"], # Potentially translated label
|
||||
}
|
||||
)
|
||||
except (ValueError, TypeError):
|
||||
raise ValueError(f"Could not parse indices from relationship: {rel}")
|
||||
|
||||
print("Generated project summary and relationship details.")
|
||||
return {
|
||||
"summary": relationships_data["summary"], # Potentially translated summary
|
||||
"details": validated_relationships # Store validated, index-based relationships with potentially translated labels
|
||||
"summary": relationships_data["summary"], # Potentially translated summary
|
||||
"details": validated_relationships, # Store validated, index-based relationships with potentially translated labels
|
||||
}
|
||||
|
||||
|
||||
def post(self, shared, prep_res, exec_res):
|
||||
# Structure is now {"summary": str, "details": [{"from": int, "to": int, "label": str}]}
|
||||
# Summary and label might be translated
|
||||
shared["relationships"] = exec_res
|
||||
|
||||
|
||||
class OrderChapters(Node):
|
||||
def prep(self, shared):
|
||||
abstractions = shared["abstractions"] # Name/description might be translated
|
||||
relationships = shared["relationships"] # Summary/label might be translated
|
||||
abstractions = shared["abstractions"] # Name/description might be translated
|
||||
relationships = shared["relationships"] # Summary/label might be translated
|
||||
project_name = shared["project_name"] # Get project name
|
||||
language = shared.get("language", "english") # Get language
|
||||
language = shared.get("language", "english") # Get language
|
||||
use_cache = shared.get("use_cache", True) # Get use_cache flag, default to True
|
||||
|
||||
# Prepare context for the LLM
|
||||
abstraction_info_for_prompt = []
|
||||
for i, a in enumerate(abstractions):
|
||||
abstraction_info_for_prompt.append(f"- {i} # {a['name']}") # Use potentially translated name
|
||||
abstraction_info_for_prompt.append(
|
||||
f"- {i} # {a['name']}"
|
||||
) # Use potentially translated name
|
||||
abstraction_listing = "\n".join(abstraction_info_for_prompt)
|
||||
|
||||
# Use potentially translated summary and labels
|
||||
summary_note = ""
|
||||
if language.lower() != "english":
|
||||
summary_note = f" (Note: Project Summary might be in {language.capitalize()})"
|
||||
summary_note = (
|
||||
f" (Note: Project Summary might be in {language.capitalize()})"
|
||||
)
|
||||
|
||||
context = f"Project Summary{summary_note}:\n{relationships['summary']}\n\n"
|
||||
context += "Relationships (Indices refer to abstractions above):\n"
|
||||
for rel in relationships['details']:
|
||||
from_name = abstractions[rel['from']]['name']
|
||||
to_name = abstractions[rel['to']]['name']
|
||||
# Use potentially translated 'label'
|
||||
context += f"- From {rel['from']} ({from_name}) to {rel['to']} ({to_name}): {rel['label']}\n" # Label might be translated
|
||||
for rel in relationships["details"]:
|
||||
from_name = abstractions[rel["from"]]["name"]
|
||||
to_name = abstractions[rel["to"]]["name"]
|
||||
# Use potentially translated 'label'
|
||||
context += f"- From {rel['from']} ({from_name}) to {rel['to']} ({to_name}): {rel['label']}\n" # Label might be translated
|
||||
|
||||
list_lang_note = ""
|
||||
if language.lower() != "english":
|
||||
list_lang_note = f" (Names might be in {language.capitalize()})"
|
||||
list_lang_note = f" (Names might be in {language.capitalize()})"
|
||||
|
||||
return abstraction_listing, context, len(abstractions), project_name, list_lang_note, use_cache # Return use_cache
|
||||
return (
|
||||
abstraction_listing,
|
||||
context,
|
||||
len(abstractions),
|
||||
project_name,
|
||||
list_lang_note,
|
||||
use_cache,
|
||||
) # Return use_cache
|
||||
|
||||
def exec(self, prep_res):
|
||||
abstraction_listing, context, num_abstractions, project_name, list_lang_note, use_cache = prep_res # Unpack use_cache
|
||||
(
|
||||
abstraction_listing,
|
||||
context,
|
||||
num_abstractions,
|
||||
project_name,
|
||||
list_lang_note,
|
||||
use_cache,
|
||||
) = prep_res # Unpack use_cache
|
||||
print("Determining chapter order using LLM...")
|
||||
# No language variation needed here in prompt instructions, just ordering based on structure
|
||||
# The input names might be translated, hence the note.
|
||||
@@ -414,62 +488,81 @@ Now, provide the YAML output:
|
||||
seen_indices = set()
|
||||
for entry in ordered_indices_raw:
|
||||
try:
|
||||
if isinstance(entry, int):
|
||||
idx = entry
|
||||
elif isinstance(entry, str) and '#' in entry:
|
||||
idx = int(entry.split('#')[0].strip())
|
||||
else:
|
||||
idx = int(str(entry).strip())
|
||||
if isinstance(entry, int):
|
||||
idx = entry
|
||||
elif isinstance(entry, str) and "#" in entry:
|
||||
idx = int(entry.split("#")[0].strip())
|
||||
else:
|
||||
idx = int(str(entry).strip())
|
||||
|
||||
if not (0 <= idx < num_abstractions):
|
||||
raise ValueError(f"Invalid index {idx} in ordered list. Max index is {num_abstractions-1}.")
|
||||
if idx in seen_indices:
|
||||
raise ValueError(f"Duplicate index {idx} found in ordered list.")
|
||||
ordered_indices.append(idx)
|
||||
seen_indices.add(idx)
|
||||
if not (0 <= idx < num_abstractions):
|
||||
raise ValueError(
|
||||
f"Invalid index {idx} in ordered list. Max index is {num_abstractions-1}."
|
||||
)
|
||||
if idx in seen_indices:
|
||||
raise ValueError(f"Duplicate index {idx} found in ordered list.")
|
||||
ordered_indices.append(idx)
|
||||
seen_indices.add(idx)
|
||||
|
||||
except (ValueError, TypeError):
|
||||
raise ValueError(f"Could not parse index from ordered list entry: {entry}")
|
||||
raise ValueError(
|
||||
f"Could not parse index from ordered list entry: {entry}"
|
||||
)
|
||||
|
||||
# Check if all abstractions are included
|
||||
if len(ordered_indices) != num_abstractions:
|
||||
raise ValueError(f"Ordered list length ({len(ordered_indices)}) does not match number of abstractions ({num_abstractions}). Missing indices: {set(range(num_abstractions)) - seen_indices}")
|
||||
raise ValueError(
|
||||
f"Ordered list length ({len(ordered_indices)}) does not match number of abstractions ({num_abstractions}). Missing indices: {set(range(num_abstractions)) - seen_indices}"
|
||||
)
|
||||
|
||||
print(f"Determined chapter order (indices): {ordered_indices}")
|
||||
return ordered_indices # Return the list of indices
|
||||
return ordered_indices # Return the list of indices
|
||||
|
||||
def post(self, shared, prep_res, exec_res):
|
||||
# exec_res is already the list of ordered indices
|
||||
shared["chapter_order"] = exec_res # List of indices
|
||||
shared["chapter_order"] = exec_res # List of indices
|
||||
|
||||
|
||||
class WriteChapters(BatchNode):
|
||||
def prep(self, shared):
|
||||
chapter_order = shared["chapter_order"] # List of indices
|
||||
abstractions = shared["abstractions"] # List of {"name": str, "description": str, "files": [int]}
|
||||
files_data = shared["files"] # List of (path, content) tuples
|
||||
chapter_order = shared["chapter_order"] # List of indices
|
||||
abstractions = shared[
|
||||
"abstractions"
|
||||
] # List of {"name": str, "description": str, "files": [int]}
|
||||
files_data = shared["files"] # List of (path, content) tuples
|
||||
project_name = shared["project_name"]
|
||||
language = shared.get("language", "english")
|
||||
use_cache = shared.get("use_cache", True) # Get use_cache flag, default to True
|
||||
|
||||
|
||||
# Get already written chapters to provide context
|
||||
# We store them temporarily during the batch run, not in shared memory yet
|
||||
# The 'previous_chapters_summary' will be built progressively in the exec context
|
||||
self.chapters_written_so_far = [] # Use instance variable for temporary storage across exec calls
|
||||
self.chapters_written_so_far = (
|
||||
[]
|
||||
) # Use instance variable for temporary storage across exec calls
|
||||
|
||||
# Create a complete list of all chapters
|
||||
all_chapters = []
|
||||
chapter_filenames = {} # Store chapter filename mapping for linking
|
||||
chapter_filenames = {} # Store chapter filename mapping for linking
|
||||
for i, abstraction_index in enumerate(chapter_order):
|
||||
if 0 <= abstraction_index < len(abstractions):
|
||||
chapter_num = i + 1
|
||||
chapter_name = abstractions[abstraction_index]["name"] # Potentially translated name
|
||||
chapter_name = abstractions[abstraction_index][
|
||||
"name"
|
||||
] # Potentially translated name
|
||||
# Create safe filename (from potentially translated name)
|
||||
safe_name = "".join(c if c.isalnum() else '_' for c in chapter_name).lower()
|
||||
safe_name = "".join(
|
||||
c if c.isalnum() else "_" for c in chapter_name
|
||||
).lower()
|
||||
filename = f"{i+1:02d}_{safe_name}.md"
|
||||
# Format with link (using potentially translated name)
|
||||
all_chapters.append(f"{chapter_num}. [{chapter_name}]({filename})")
|
||||
# Store mapping of chapter index to filename for linking
|
||||
chapter_filenames[abstraction_index] = {"num": chapter_num, "name": chapter_name, "filename": filename}
|
||||
chapter_filenames[abstraction_index] = {
|
||||
"num": chapter_num,
|
||||
"name": chapter_name,
|
||||
"filename": filename,
|
||||
}
|
||||
|
||||
# Create a formatted string with all chapters
|
||||
full_chapter_listing = "\n".join(all_chapters)
|
||||
@@ -477,47 +570,59 @@ class WriteChapters(BatchNode):
|
||||
items_to_process = []
|
||||
for i, abstraction_index in enumerate(chapter_order):
|
||||
if 0 <= abstraction_index < len(abstractions):
|
||||
abstraction_details = abstractions[abstraction_index] # Contains potentially translated name/desc
|
||||
abstraction_details = abstractions[
|
||||
abstraction_index
|
||||
] # Contains potentially translated name/desc
|
||||
# Use 'files' (list of indices) directly
|
||||
related_file_indices = abstraction_details.get("files", [])
|
||||
# Get content using helper, passing indices
|
||||
related_files_content_map = get_content_for_indices(files_data, related_file_indices)
|
||||
related_files_content_map = get_content_for_indices(
|
||||
files_data, related_file_indices
|
||||
)
|
||||
|
||||
# Get previous chapter info for transitions (uses potentially translated name)
|
||||
prev_chapter = None
|
||||
if i > 0:
|
||||
prev_idx = chapter_order[i-1]
|
||||
prev_idx = chapter_order[i - 1]
|
||||
prev_chapter = chapter_filenames[prev_idx]
|
||||
|
||||
# Get next chapter info for transitions (uses potentially translated name)
|
||||
next_chapter = None
|
||||
if i < len(chapter_order) - 1:
|
||||
next_idx = chapter_order[i+1]
|
||||
next_idx = chapter_order[i + 1]
|
||||
next_chapter = chapter_filenames[next_idx]
|
||||
|
||||
items_to_process.append({
|
||||
"chapter_num": i + 1,
|
||||
"abstraction_index": abstraction_index,
|
||||
"abstraction_details": abstraction_details, # Has potentially translated name/desc
|
||||
"related_files_content_map": related_files_content_map,
|
||||
"project_name": shared["project_name"], # Add project name
|
||||
"full_chapter_listing": full_chapter_listing, # Add the full chapter listing (uses potentially translated names)
|
||||
"chapter_filenames": chapter_filenames, # Add chapter filenames mapping (uses potentially translated names)
|
||||
"prev_chapter": prev_chapter, # Add previous chapter info (uses potentially translated name)
|
||||
"next_chapter": next_chapter, # Add next chapter info (uses potentially translated name)
|
||||
"language": language, # Add language for multi-language support
|
||||
# previous_chapters_summary will be added dynamically in exec
|
||||
})
|
||||
items_to_process.append(
|
||||
{
|
||||
"chapter_num": i + 1,
|
||||
"abstraction_index": abstraction_index,
|
||||
"abstraction_details": abstraction_details, # Has potentially translated name/desc
|
||||
"related_files_content_map": related_files_content_map,
|
||||
"project_name": shared["project_name"], # Add project name
|
||||
"full_chapter_listing": full_chapter_listing, # Add the full chapter listing (uses potentially translated names)
|
||||
"chapter_filenames": chapter_filenames, # Add chapter filenames mapping (uses potentially translated names)
|
||||
"prev_chapter": prev_chapter, # Add previous chapter info (uses potentially translated name)
|
||||
"next_chapter": next_chapter, # Add next chapter info (uses potentially translated name)
|
||||
"language": language, # Add language for multi-language support
|
||||
# previous_chapters_summary will be added dynamically in exec
|
||||
}
|
||||
)
|
||||
else:
|
||||
print(f"Warning: Invalid abstraction index {abstraction_index} in chapter_order. Skipping.")
|
||||
print(
|
||||
f"Warning: Invalid abstraction index {abstraction_index} in chapter_order. Skipping."
|
||||
)
|
||||
|
||||
print(f"Preparing to write {len(items_to_process)} chapters...")
|
||||
return items_to_process # Iterable for BatchNode
|
||||
return items_to_process # Iterable for BatchNode
|
||||
|
||||
def exec(self, item):
|
||||
# This runs for each item prepared above
|
||||
abstraction_name = item["abstraction_details"]["name"] # Potentially translated name
|
||||
abstraction_description = item["abstraction_details"]["description"] # Potentially translated description
|
||||
abstraction_name = item["abstraction_details"][
|
||||
"name"
|
||||
] # Potentially translated name
|
||||
abstraction_description = item["abstraction_details"][
|
||||
"description"
|
||||
] # Potentially translated description
|
||||
chapter_num = item["chapter_num"]
|
||||
project_name = item.get("project_name")
|
||||
language = item.get("language", "english")
|
||||
@@ -552,10 +657,11 @@ class WriteChapters(BatchNode):
|
||||
instruction_lang_note = f" (in {lang_cap})"
|
||||
mermaid_lang_note = f" (Use {lang_cap} for labels/text if appropriate)"
|
||||
code_comment_note = f" (Translate to {lang_cap} if possible, otherwise keep minimal English for clarity)"
|
||||
link_lang_note = f" (Use the {lang_cap} chapter title from the structure above)"
|
||||
link_lang_note = (
|
||||
f" (Use the {lang_cap} chapter title from the structure above)"
|
||||
)
|
||||
tone_note = f" (appropriate for {lang_cap} readers)"
|
||||
|
||||
|
||||
prompt = f"""
|
||||
{language_instruction}Write a very beginner-friendly tutorial chapter (in Markdown format) for the project `{project_name}` about the concept: "{abstraction_name}". This is Chapter {chapter_num}.
|
||||
|
||||
@@ -606,20 +712,22 @@ Now, directly provide a super beginner-friendly Markdown output (DON'T need ```m
|
||||
"""
|
||||
chapter_content = call_llm(prompt)
|
||||
# Basic validation/cleanup
|
||||
actual_heading = f"# Chapter {chapter_num}: {abstraction_name}" # Use potentially translated name
|
||||
actual_heading = f"# Chapter {chapter_num}: {abstraction_name}" # Use potentially translated name
|
||||
if not chapter_content.strip().startswith(f"# Chapter {chapter_num}"):
|
||||
# Add heading if missing or incorrect, trying to preserve content
|
||||
lines = chapter_content.strip().split('\n')
|
||||
if lines and lines[0].strip().startswith("#"): # If there's some heading, replace it
|
||||
lines[0] = actual_heading
|
||||
chapter_content = "\n".join(lines)
|
||||
else: # Otherwise, prepend it
|
||||
chapter_content = f"{actual_heading}\n\n{chapter_content}"
|
||||
# Add heading if missing or incorrect, trying to preserve content
|
||||
lines = chapter_content.strip().split("\n")
|
||||
if lines and lines[0].strip().startswith(
|
||||
"#"
|
||||
): # If there's some heading, replace it
|
||||
lines[0] = actual_heading
|
||||
chapter_content = "\n".join(lines)
|
||||
else: # Otherwise, prepend it
|
||||
chapter_content = f"{actual_heading}\n\n{chapter_content}"
|
||||
|
||||
# Add the generated content to our temporary list for the next iteration's context
|
||||
self.chapters_written_so_far.append(chapter_content)
|
||||
|
||||
return chapter_content # Return the Markdown string (potentially translated)
|
||||
return chapter_content # Return the Markdown string (potentially translated)
|
||||
|
||||
def post(self, shared, prep_res, exec_res_list):
|
||||
# exec_res_list contains the generated Markdown for each chapter, in order
|
||||
@@ -628,19 +736,26 @@ Now, directly provide a super beginner-friendly Markdown output (DON'T need ```m
|
||||
del self.chapters_written_so_far
|
||||
print(f"Finished writing {len(exec_res_list)} chapters.")
|
||||
|
||||
|
||||
class CombineTutorial(Node):
|
||||
def prep(self, shared):
|
||||
project_name = shared["project_name"]
|
||||
output_base_dir = shared.get("output_dir", "output") # Default output dir
|
||||
output_base_dir = shared.get("output_dir", "output") # Default output dir
|
||||
output_path = os.path.join(output_base_dir, project_name)
|
||||
repo_url = shared.get("repo_url") # Get the repository URL
|
||||
# language = shared.get("language", "english") # No longer needed for fixed strings
|
||||
|
||||
# Get potentially translated data
|
||||
relationships_data = shared["relationships"] # {"summary": str, "details": [{"from": int, "to": int, "label": str}]} -> summary/label potentially translated
|
||||
chapter_order = shared["chapter_order"] # indices
|
||||
abstractions = shared["abstractions"] # list of dicts -> name/description potentially translated
|
||||
chapters_content = shared["chapters"] # list of strings -> content potentially translated
|
||||
relationships_data = shared[
|
||||
"relationships"
|
||||
] # {"summary": str, "details": [{"from": int, "to": int, "label": str}]} -> summary/label potentially translated
|
||||
chapter_order = shared["chapter_order"] # indices
|
||||
abstractions = shared[
|
||||
"abstractions"
|
||||
] # list of dicts -> name/description potentially translated
|
||||
chapters_content = shared[
|
||||
"chapters"
|
||||
] # list of strings -> content potentially translated
|
||||
|
||||
# --- Generate Mermaid Diagram ---
|
||||
mermaid_lines = ["flowchart TD"]
|
||||
@@ -648,26 +763,32 @@ class CombineTutorial(Node):
|
||||
for i, abstr in enumerate(abstractions):
|
||||
node_id = f"A{i}"
|
||||
# Use potentially translated name, sanitize for Mermaid ID and label
|
||||
sanitized_name = abstr['name'].replace('"', '')
|
||||
node_label = sanitized_name # Using sanitized name only
|
||||
mermaid_lines.append(f' {node_id}["{node_label}"]') # Node label uses potentially translated name
|
||||
sanitized_name = abstr["name"].replace('"', "")
|
||||
node_label = sanitized_name # Using sanitized name only
|
||||
mermaid_lines.append(
|
||||
f' {node_id}["{node_label}"]'
|
||||
) # Node label uses potentially translated name
|
||||
# Add edges for relationships using potentially translated labels
|
||||
for rel in relationships_data['details']:
|
||||
for rel in relationships_data["details"]:
|
||||
from_node_id = f"A{rel['from']}"
|
||||
to_node_id = f"A{rel['to']}"
|
||||
# Use potentially translated label, sanitize
|
||||
edge_label = rel['label'].replace('"', '').replace('\n', ' ') # Basic sanitization
|
||||
edge_label = (
|
||||
rel["label"].replace('"', "").replace("\n", " ")
|
||||
) # Basic sanitization
|
||||
max_label_len = 30
|
||||
if len(edge_label) > max_label_len:
|
||||
edge_label = edge_label[:max_label_len-3] + "..."
|
||||
mermaid_lines.append(f' {from_node_id} -- "{edge_label}" --> {to_node_id}') # Edge label uses potentially translated label
|
||||
edge_label = edge_label[: max_label_len - 3] + "..."
|
||||
mermaid_lines.append(
|
||||
f' {from_node_id} -- "{edge_label}" --> {to_node_id}'
|
||||
) # Edge label uses potentially translated label
|
||||
|
||||
mermaid_diagram = "\n".join(mermaid_lines)
|
||||
# --- End Mermaid ---
|
||||
|
||||
# --- Prepare index.md content ---
|
||||
index_content = f"# Tutorial: {project_name}\n\n"
|
||||
index_content += f"{relationships_data['summary']}\n\n" # Use the potentially translated summary directly
|
||||
index_content += f"{relationships_data['summary']}\n\n" # Use the potentially translated summary directly
|
||||
# Keep fixed strings in English
|
||||
index_content += f"**Source Repository:** [{repo_url}]({repo_url})\n\n"
|
||||
|
||||
@@ -684,14 +805,18 @@ class CombineTutorial(Node):
|
||||
for i, abstraction_index in enumerate(chapter_order):
|
||||
# Ensure index is valid and we have content for it
|
||||
if 0 <= abstraction_index < len(abstractions) and i < len(chapters_content):
|
||||
abstraction_name = abstractions[abstraction_index]["name"] # Potentially translated name
|
||||
abstraction_name = abstractions[abstraction_index][
|
||||
"name"
|
||||
] # Potentially translated name
|
||||
# Sanitize potentially translated name for filename
|
||||
safe_name = "".join(c if c.isalnum() else '_' for c in abstraction_name).lower()
|
||||
safe_name = "".join(
|
||||
c if c.isalnum() else "_" for c in abstraction_name
|
||||
).lower()
|
||||
filename = f"{i+1:02d}_{safe_name}.md"
|
||||
index_content += f"{i+1}. [{abstraction_name}]({filename})\n" # Use potentially translated name in link text
|
||||
index_content += f"{i+1}. [{abstraction_name}]({filename})\n" # Use potentially translated name in link text
|
||||
|
||||
# Add attribution to chapter content (using English fixed string)
|
||||
chapter_content = chapters_content[i] # Potentially translated content
|
||||
chapter_content = chapters_content[i] # Potentially translated content
|
||||
if not chapter_content.endswith("\n\n"):
|
||||
chapter_content += "\n\n"
|
||||
# Keep fixed strings in English
|
||||
@@ -700,7 +825,9 @@ class CombineTutorial(Node):
|
||||
# Store filename and corresponding content
|
||||
chapter_files.append({"filename": filename, "content": chapter_content})
|
||||
else:
|
||||
print(f"Warning: Mismatch between chapter order, abstractions, or content at index {i} (abstraction index {abstraction_index}). Skipping file generation for this entry.")
|
||||
print(
|
||||
f"Warning: Mismatch between chapter order, abstractions, or content at index {i} (abstraction index {abstraction_index}). Skipping file generation for this entry."
|
||||
)
|
||||
|
||||
# Add attribution to index content (using English fixed string)
|
||||
index_content += f"\n\n---\n\nGenerated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge)"
|
||||
@@ -708,7 +835,7 @@ class CombineTutorial(Node):
|
||||
return {
|
||||
"output_path": output_path,
|
||||
"index_content": index_content,
|
||||
"chapter_files": chapter_files # List of {"filename": str, "content": str}
|
||||
"chapter_files": chapter_files, # List of {"filename": str, "content": str}
|
||||
}
|
||||
|
||||
def exec(self, prep_res):
|
||||
@@ -733,9 +860,8 @@ class CombineTutorial(Node):
|
||||
f.write(chapter_info["content"])
|
||||
print(f" - Wrote {chapter_filepath}")
|
||||
|
||||
return output_path # Return the final path
|
||||
|
||||
return output_path # Return the final path
|
||||
|
||||
def post(self, shared, prep_res, exec_res):
|
||||
shared["final_output_dir"] = exec_res # Store the output path
|
||||
shared["final_output_dir"] = exec_res # Store the output path
|
||||
print(f"\nTutorial generation complete! Files are in: {exec_res}")
|
||||
|
||||
@@ -1,20 +0,0 @@
|
||||
import re
|
||||
|
||||
def add_indentation(text):
|
||||
# This pattern matches lines that don't start with a hyphen or whitespace
|
||||
pattern = r'^(?![-\s])(.*)$'
|
||||
|
||||
# Replace with 4 spaces followed by the captured content
|
||||
result = re.sub(pattern, r' \1', text, flags=re.MULTILINE)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Example usage
|
||||
text = """This line will be indented
|
||||
- This line won't be indented
|
||||
This line won't be indented either
|
||||
Another line that will be indented"""
|
||||
|
||||
indented_text = add_indentation(text)
|
||||
Reference in New Issue
Block a user