From 9e82cc8ed34e4516e67851f6519f9ee9973a5901 Mon Sep 17 00:00:00 2001
From: Ross Donald <rossdonald@gmail.com>
Date: Fri, 16 May 2025 10:36:18 +1000
Subject: [PATCH] fix: update file opening encoding to utf-8-sig to handle
 files with BOM

---
 utils/call_llm.py           | 14 +++++++-------
 utils/crawl_github_files.py |  2 +-
 utils/crawl_local_files.py  |  4 ++--
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/utils/call_llm.py b/utils/call_llm.py
index d0af597..81ef2a2 100644
--- a/utils/call_llm.py
+++ b/utils/call_llm.py
@@ -15,7 +15,7 @@ log_file = os.path.join(
 logger = logging.getLogger("llm_logger")
 logger.setLevel(logging.INFO)
 logger.propagate = False  # Prevent propagation to root logger
-file_handler = logging.FileHandler(log_file)
+file_handler = logging.FileHandler(log_file, encoding='utf-8')
 file_handler.setFormatter(
     logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
 )
@@ -36,7 +36,7 @@ def call_llm(prompt: str, use_cache: bool = True) -> str:
         cache = {}
         if os.path.exists(cache_file):
             try:
-                with open(cache_file, "r") as f:
+                with open(cache_file, "r", encoding="utf-8") as f:
                     cache = json.load(f)
             except:
                 logger.warning(f"Failed to load cache, starting with empty cache")
@@ -73,7 +73,7 @@ def call_llm(prompt: str, use_cache: bool = True) -> str:
         cache = {}
         if os.path.exists(cache_file):
             try:
-                with open(cache_file, "r") as f:
+                with open(cache_file, "r", encoding="utf-8") as f:
                     cache = json.load(f)
             except:
                 pass
@@ -81,7 +81,7 @@ def call_llm(prompt: str, use_cache: bool = True) -> str:
         # Add to cache and save
         cache[prompt] = response_text
         try:
-            with open(cache_file, "w") as f:
+            with open(cache_file, "w", encoding="utf-8") as f:
                 json.dump(cache, f)
         except Exception as e:
             logger.error(f"Failed to save cache: {e}")
@@ -161,7 +161,7 @@ def call_llm(prompt: str, use_cache: bool = True) -> str:
 #         cache = {}
 #         if os.path.exists(cache_file):
 #             try:
-#                 with open(cache_file, "r") as f:
+#                 with open(cache_file, "r", encoding="utf-8") as f:
 #                     cache = json.load(f)
 #             except:
 #                 logger.warning(f"Failed to load cache, starting with empty cache")
@@ -211,7 +211,7 @@ def call_llm(prompt: str, use_cache: bool = True) -> str:
 #         cache = {}
 #         if os.path.exists(cache_file):
 #             try:
-#                 with open(cache_file, "r") as f:
+#                 with open(cache_file, "r", encoding="utf-8") as f:
 #                     cache = json.load(f)
 #             except:
 #                 pass
@@ -219,7 +219,7 @@ def call_llm(prompt: str, use_cache: bool = True) -> str:
 #         # Add to cache and save
 #         cache[prompt] = response_text
 #         try:
-#             with open(cache_file, "w") as f:
+#             with open(cache_file, "w", encoding="utf-8") as f:
 #                 json.dump(cache, f)
 #         except Exception as e:
 #             logger.error(f"Failed to save cache: {e}")
diff --git a/utils/crawl_github_files.py b/utils/crawl_github_files.py
index b86802c..e7835bd 100644
--- a/utils/crawl_github_files.py
+++ b/utils/crawl_github_files.py
@@ -104,7 +104,7 @@ def crawl_github_files(
 
                     # Read content
                     try:
-                        with open(abs_path, "r", encoding="utf-8") as f:
+                        with open(abs_path, "r", encoding="utf-8-sig") as f:
                             content = f.read()
                         files[rel_path] = content
                         print(f"Added {rel_path} ({file_size} bytes)")
diff --git a/utils/crawl_local_files.py b/utils/crawl_local_files.py
index d6658b4..46fa9f1 100644
--- a/utils/crawl_local_files.py
+++ b/utils/crawl_local_files.py
@@ -32,7 +32,7 @@ def crawl_local_files(
     gitignore_spec = None
     if os.path.exists(gitignore_path):
         try:
-            with open(gitignore_path, "r", encoding="utf-8") as f:
+            with open(gitignore_path, "r", encoding="utf-8-sig") as f:
                 gitignore_patterns = f.readlines()
             gitignore_spec = pathspec.PathSpec.from_lines("gitwildmatch", gitignore_patterns)
             print(f"Loaded .gitignore patterns from {gitignore_path}")
@@ -113,7 +113,7 @@ def crawl_local_files(
 
         # --- File is being processed ---        
         try:
-            with open(filepath, "r", encoding="utf-8") as f:
+            with open(filepath, "r", encoding="utf-8-sig") as f:
                 content = f.read()
             files_dict[relpath] = content
         except Exception as e: