fix: improve file filtering, add new utility,

- Improved the speed of file filtering in `crawl_local_files.py` with folder-level exclusion
- Added `fix_yaml.py` utility for YAML indentation fixes
- Updated `nodes.py` to support up to 20 core abstractions
- add option for no cache.
This commit is contained in:
remy
2025-04-30 19:00:50 +10:00
parent 4da93741fe
commit 98fa9fc0d5
8 changed files with 241 additions and 83 deletions
+9 -1
View File
@@ -14,8 +14,10 @@ DEFAULT_INCLUDE_PATTERNS = {
}
DEFAULT_EXCLUDE_PATTERNS = {
"assets/*", "data/*", "examples/*", "images/*", "public/*", "static/*", "temp/*",
"docs/*",
"venv/*", ".venv/*", "*test*", "tests/*", "docs/*", "examples/*", "v1/*",
"dist/*", "build/*", "experimental/*", "deprecated/*",
"dist/*", "build/*", "experimental/*", "deprecated/*", "misc/*",
"legacy/*", ".git/*", ".github/*", ".next/*", ".vscode/*", "obj/*", "bin/*", "node_modules/*", "*.log"
}
@@ -36,6 +38,8 @@ def main():
parser.add_argument("-s", "--max-size", type=int, default=100000, help="Maximum file size in bytes (default: 100000, about 100KB).")
# Add language parameter for multi-language support
parser.add_argument("--language", default="english", help="Language for the generated tutorial (default: english)")
# Add use_cache parameter to control LLM caching
parser.add_argument("--no-cache", action="store_true", help="Disable LLM response caching (default: caching enabled)")
args = parser.parse_args()
@@ -61,6 +65,9 @@ def main():
# Add language for multi-language support
"language": args.language,
# Add use_cache flag (inverse of no-cache flag)
"use_cache": not args.no_cache,
# Outputs will be populated by the nodes
"files": [],
@@ -73,6 +80,7 @@ def main():
# Display starting message with repository/directory and language
print(f"Starting tutorial generation for: {args.repo or args.dir} in {args.language.capitalize()} language")
print(f"LLM caching: {'Disabled' if args.no_cache else 'Enabled'}")
# Create the flow instance
tutorial_flow = create_tutorial_flow()