RAG and Vector Search Platform

from langchain.text_splitter import RecursiveCharacterTextSplitter

splitter = RecursiveCharacterTextSplitter(
    chunk_size=512,
    chunk_overlap=64,
    separators=["\n\n", "\n", ". ", " "]
)

def ingest_document(doc_id: str, content: str, metadata: dict):
    chunks = splitter.split_text(content)
    embeddings = embedding_model.encode(chunks)

    for i, (chunk, vector) in enumerate(zip(chunks, embeddings)):
        vector_db.upsert(
            id=f"{doc_id}_chunk_{i}",
            vector=vector,
            metadata={
                **metadata,
                "doc_id": doc_id,
                "chunk_index": i,
                "text": chunk
            }
        )

def answer_question(query: str, top_k: int = 5) -> dict:
    query_vector = embedding_model.encode(query)
    results = vector_db.search(
        vector=query_vector,
        top_k=top_k,
        filter={"category": "creator-help"}
    )

    context = "\n\n".join([
        f"[{r.metadata['doc_id']}]: {r.metadata['text']}"
        for r in results
    ])

    prompt = f"""Answer the question using only the provided context.
If the context does not contain the answer, say "I don't have information about that."

Context:
{context}

Question: {query}

Answer with citations to the source documents."""

    answer = llm.generate(prompt)
    return {
        "answer": answer,
        "sources": [r.metadata["doc_id"] for r in results],
        "retrieval_scores": [r.score for r in results]
    }

-- pgvector example
CREATE EXTENSION vector;

CREATE TABLE document_chunks (
  id          TEXT PRIMARY KEY,
  doc_id      TEXT NOT NULL,
  chunk_index INT NOT NULL,
  content     TEXT NOT NULL,
  embedding   vector(1536),  -- OpenAI ada-002 dimensions
  metadata    JSONB,
  created_at  TIMESTAMPTZ DEFAULT now()
);

CREATE INDEX idx_chunks_embedding ON document_chunks
  USING ivfflat (embedding vector_cosine_ops) WITH (lists = 100);