From fc08f1a814171dfd77d12ca61be9f362266b390f Mon Sep 17 00:00:00 2001 From: Jake Pullen Date: Sat, 16 May 2026 09:11:21 +0100 Subject: [PATCH] =?UTF-8?q?refactor:=20=F0=9F=94=A8=20Mostly=20working=20O?= =?UTF-8?q?llama=20migration,=20few=20tweaks=20left?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config.yaml | 24 ++++++++++++------------ load_ingestion_llms.sh | 4 ++-- src/embedding.py | 5 +++-- src/experts/retrieval_agent.py | 22 ++++++++++------------ src/ingest.py | 22 +++++++++++++++------- src/retrieve.py | 3 +-- 6 files changed, 43 insertions(+), 37 deletions(-) diff --git a/config.yaml b/config.yaml index 6864ca5..da91d6e 100644 --- a/config.yaml +++ b/config.yaml @@ -1,24 +1,24 @@ # --- Connection Settings --- api: - base_url: "http://framework.tawny-bellatrix.ts.net:1234" + base_url: "http://100.110.238.94:11434" api_version: "/v1/" # --- Model Settings --- models: - enrich: "lm_studio/qwen-" # will have an identifier, based on amount of active LLMs see ./load_ingestion_llms.sh - embedding: "text-embedding-qwen3-embedding-8b" - retrieval: "lm_studio/qwen/qwen3-30b-a3b-2507" - expansion: "lm_studio/qwen/qwen3-30b-a3b-2507" + enrich: "ollama/granite4.1:3b" + embedding: "qwen3-embedding:4b" + retrieval: "ollama/qwen3.6:latest" + expansion: "ollama/granite4.1:3b" # --- Ingestion Settings --- ingestion: data_dir: "/home/jake/DnD" db_path: "./data/" db_name: "dmv.db" - active_llms: 2 - parallel_requests_per_llm: 2 - chunk_size: 800 - chunk_overlap: 100 + active_llms: 1 + parallel_requests_per_llm: 6 + chunk_size: 1200 + chunk_overlap: 200 embedding_batch_size: 32 time_file_location: "./data/time_file.txt" @@ -29,7 +29,7 @@ ingestion_agent: Analyze the provided notes and extract a concise synopsis and relevant metadata. synopsis = A one-sentence summary of the document. tags = Relevant tags (NPCs, Locations, Items, Plot Points). - entities = a list of Key names of people, places, or factions. + entities = a list of names for people, places, or factions. "note -> synopsis:str, tags: list[str], entities: list[str]" retrieval_agent: @@ -41,7 +41,7 @@ retrieval_agent: expansion_agent: expansion_signature: | - You are a query expansion expert, specialised in Dungeons and Dragons. - Given a user's question, generate 3-5 similar but enhanced search queries that would help find more relevant information. + You are an expert Dungeon Master's assistant. + Given a user's question, generate 3-5 similar but enhanced search queries that would help find more relevant information in DnD notes. Each expanded query should be distinct and add different perspective to the original question. Return only the queries as a JSON list with key "queries".""" diff --git a/load_ingestion_llms.sh b/load_ingestion_llms.sh index 0caadc9..b62b842 100755 --- a/load_ingestion_llms.sh +++ b/load_ingestion_llms.sh @@ -1,5 +1,5 @@ -lms load qwen-4b-instruct-2507 --parallel 2 --identifier "qwen-0" --ttl 1800 -lms load qwen-4b-instruct-2507 --parallel 2 --identifier "qwen-1" --ttl 1800 +# lms load qwen-4b-instruct-2507 --parallel 2 --identifier "qwen-0" --ttl 1800 +# lms load qwen-4b-instruct-2507 --parallel 2 --identifier "qwen-1" --ttl 1800 # lms load qwen-4b-instruct-2507 --parallel 2 --identifier "qwen-2" --ttl 1800 # lms load qwen-4b-instruct-2507 --parallel 2 --identifier "qwen-3" --ttl 1800 # lms load qwen-4b-instruct-2507 --parallel 2 --identifier "qwen-4" --ttl 1800 diff --git a/src/embedding.py b/src/embedding.py index 291131d..a73c31f 100644 --- a/src/embedding.py +++ b/src/embedding.py @@ -10,7 +10,7 @@ API_VERSION = CFG["api"]["api_version"] class LocalLMEmbeddings(Embeddings): def __init__(self, model: str, base_url: str = API_BASE, batch_size: int = 32): - self.url = f"{base_url}/{API_VERSION}embeddings" + self.url = f"{base_url}/api/embed" self.model = model self.batch_size = batch_size @@ -22,10 +22,11 @@ class LocalLMEmbeddings(Embeddings): response = requests.post( self.url, json=payload, timeout=120 ) # Longer timeout for batches + # print(response) response.raise_for_status() data = response.json() # print(data) - return [item["embedding"] for item in data["data"]] + return data["embeddings"] except Exception as e: print(f"❌ Batch request failed: {e}") # Returning empty lists to maintain index integrity if needed, diff --git a/src/experts/retrieval_agent.py b/src/experts/retrieval_agent.py index 7e0f837..bd05abd 100644 --- a/src/experts/retrieval_agent.py +++ b/src/experts/retrieval_agent.py @@ -18,7 +18,7 @@ EXPANSION_CONFIG = CFG["expansion_agent"] def retrieve_from_turso(embedded_question, k=5): query = f""" - SELECT file_path, synopsis, tags, entities, chunk_data, + SELECT file_path, synopsis, tags, chunk_data, vector_distance_cos(embedding, vector32('{embedded_question}')) AS distance FROM notes ORDER BY distance ASC @@ -55,9 +55,7 @@ class DnDRAG(dspy.Module): base_url=API_BASE, # batch_size=1, ) - self.retrieval_lm = dspy.LM( - model=CFG["models"]["retrieval"], api_base=API_BASE + CFG["api"]["api_version"] - ) + self.retrieval_lm = dspy.LM(model=CFG["models"]["retrieval"], api_base=API_BASE) with dspy.context(lm=self.retrieval_lm, signature=ExpansionSignature): self.query_expander = dspy.Predict("question -> queries:list[str]") @@ -68,9 +66,9 @@ class DnDRAG(dspy.Module): print("Enhancing Question") with dspy.context(lm=self.retrieval_lm): expanded_queries = self.query_expander(question=question).queries - print("Enhanced Queries:") - for q in expanded_queries: - print(" ", q) + # print("Enhanced Queries:") + # for q in expanded_queries: + # print(" ", q) all_embeddings = self.embeddings_model.embed_documents([question] + expanded_queries) # print(all_embeddings) all_results = [] @@ -81,7 +79,7 @@ class DnDRAG(dspy.Module): seen = set() unique_results = [] for row in all_results: - key = (row[0], row[4]) + key = (row[0], row[3]) if key not in seen: seen.add(key) unique_results.append(row) @@ -91,18 +89,18 @@ class DnDRAG(dspy.Module): source = row[0] synopsis = row[1] tags = row[2] - entities = row[3] - content = row[4] - closeness = row[5] + # entities = row[3] + content = row[3] + closeness = row[4] context_parts.append(f""" --- Chunk {i + 1} from {source} --- synopsis: {synopsis}, tags: {tags}, -entities: {entities}, closeness: {closeness}, {content} """) + # entities: {entities}, context = "\n\n".join(context_parts) diff --git a/src/ingest.py b/src/ingest.py index b12b2b3..73501b0 100644 --- a/src/ingest.py +++ b/src/ingest.py @@ -83,7 +83,7 @@ def enrich_chunks(chunks: list) -> list: try: with dspy.context( - lm=dspy.LM(model=f"{MODEL_BASE}{lm_index}", api_base=API_BASE + API_VERSION), + lm=dspy.LM(model=f"{MODEL_BASE}", api_base=API_BASE), chat_template_kwargs={"enable_thinking": False}, ): response = IngestionAgent().ingest(note=chunk.page_content) @@ -140,10 +140,11 @@ def embed_chunks(chunks: List[Any], batch_size: int = EMBEDDING_BATCH_SIZE) -> L # Process chunks in batches for i in tqdm(range(0, total_chunks, batch_size), desc="Embedding batches"): batch = chunks[i : i + batch_size] - print(f"🚀 Processing batch {(i // batch_size) + 1} (Size: {len(batch)})...") + # print(f"🚀 Processing batch {(i // batch_size) + 1} (Size: {len(batch)})...") batch_content = [chunk.page_content for chunk in batch] try: batch_embeddings = embeddings_model.embed_documents(batch_content) + # print(len(batch_embeddings[0])) # Process each chunk in the batch for j, (chunk, embedding) in enumerate(zip(batch, batch_embeddings)): # Extract metadata @@ -233,8 +234,15 @@ def save_to_db(chunk_dicts): # SQL with named placeholders for clarity and safety insert_sql = """ INSERT INTO notes ( - file_path, file_name, chunk_data, synopsis, tags, entities, embedding, timestamp - ) VALUES (?, ?, ?, ?, ?, ?, vector32(?), ?) + file_path, + file_name, + chunk_data, + synopsis, + tags, + -- entities, + embedding, + timestamp + ) VALUES (?, ?, ?, ?, ?, vector32(?), ?) """ # Prepare batch data: convert each dict to a tuple in correct order @@ -250,7 +258,7 @@ def save_to_db(chunk_dicts): entry["chunk_data"], entry["synopsis"], ",".join(entry["tags"]), # Store as comma-separated string - ",".join(entry["entities"]), # Store as comma-separated string + # ",".join(entry["entities"]), # Store as comma-separated string embedding_str, entry["timestamp"], ) @@ -277,8 +285,8 @@ def create_db(): chunk_data TEXT NOT NULL, synopsis TEXT, tags TEXT, -- comma-separated - entities TEXT, -- comma-separated - embedding F32_BLOB(4096), + -- entities TEXT, -- comma-separated + embedding F32_BLOB(2560), timestamp DATETIME DEFAULT CURRENT_TIMESTAMP ) """) diff --git a/src/retrieve.py b/src/retrieve.py index 77e4743..fb01366 100644 --- a/src/retrieve.py +++ b/src/retrieve.py @@ -87,8 +87,7 @@ def main(): dspy.configure(verbose_errors=True) dspy.configure(callbacks=[CallbackHandler(logger)]) # 1. Setup the LLM - print("🚀 Initializing Qwen-8B via LM Studio...") - lm = dspy.LM(RETRIEVE_MODEL, api_base=API_BASE + API_VERSION) + lm = dspy.LM(RETRIEVE_MODEL, api_base=API_BASE) dspy.configure(lm=lm) # 2. Load the RAG System (only happens once!)