37 lines
1.5 KiB
Python
37 lines
1.5 KiB
Python
import os
|
|
from dotenv import load_dotenv
|
|
|
|
load_dotenv()
|
|
|
|
# ==== Embedding & Chunking params ====
|
|
MODEL_NAME = "text-embedding-3-large" # default embedding model
|
|
EMBEDDING_DIM = 3072 # dimension for the model above
|
|
CHUNK_SIZE = 700
|
|
OVERLAP = 200
|
|
BATCH_SIZE = 10 # items per embedding request
|
|
MAX_RETRIES = 3 # automatic retries for transient API errors
|
|
RETRY_BACKOFF_SEC = 2.0 # backoff between retries (exponential/backoff logic in client)
|
|
SPLIT_LENGTH = 128_000 # reserved for future splitting strategies
|
|
|
|
# ==== Storage ====
|
|
BASE_DIR = os.path.dirname(__file__)
|
|
DATA_DIR = os.path.join(BASE_DIR, "data")
|
|
EMB_DIR = os.path.join(DATA_DIR, "embeddings")
|
|
MANIFEST_DIR = os.path.join(DATA_DIR, "manifest")
|
|
os.makedirs(EMB_DIR, exist_ok=True)
|
|
os.makedirs(MANIFEST_DIR, exist_ok=True)
|
|
|
|
MANIFEST_PATH = os.path.join(MANIFEST_DIR, "manifest.jsonl")
|
|
|
|
# ==== Environment ====
|
|
# Use a neutral, public-friendly env var name. Keep legacy key for backward compatibility if needed.
|
|
OPENAI_API_KEY_ENV = "EMBEDDING_API_KEY"
|
|
|
|
# ==== Cost estimation (informative only) ====
|
|
AVG_CHARS_PER_TOKEN = 4.0 # rough average; depends on language/script
|
|
PRICE_PER_1K_TOKENS = 0.00013 # $ per 1K tokens for text-embedding-3-large (update if provider changes)
|
|
|
|
# ==== Canonical archive note ====
|
|
# Vectors are stored in Parquet (ZSTD compressed) as the canonical archive.
|
|
# The JSONL manifest holds metadata only (no vectors) for readability and version control.
|