Generating Embeddings

from sentence_transformers import SentenceTransformer

model = SentenceTransformer("all-MiniLM-L6-v2")
embeddings = model.encode(["hello world", "foo bar"]).tolist()

import numpy as np

arr = np.array(embeddings)
normed = arr / np.linalg.norm(arr, axis=1, keepdims=True)
normed_list = normed.tolist()

from openai import OpenAI
client = OpenAI()

def embed_batch(texts, batch=100):
    out = []
    for i in range(0, len(texts), batch):
        chunk = texts[i:i+batch]
        r = client.embeddings.create(model="text-embedding-3-small", input=chunk)
        out += [d.embedding for d in r.data]
    return out

import hashlib, json, redis

r = redis.Redis()

def cached_embed(text, model="text-embedding-3-small"):
    key = f"emb:{model}:{hashlib.sha256(text.encode()).hexdigest()}"
    if (v := r.get(key)): return json.loads(v)
    e = embed_one(text)
    r.set(key, json.dumps(e), ex=86400 * 30)
    return e

# OpenAI text-embedding-3 supports dimension truncation
r = client.embeddings.create(
    model="text-embedding-3-large",
    input="...",
    dimensions=512,  # truncate from 3072
)

from transformers import AutoTokenizer, AutoModel
import torch

tok = AutoTokenizer.from_pretrained("BAAI/bge-large-en-v1.5")
mdl = AutoModel.from_pretrained("BAAI/bge-large-en-v1.5").eval()

def embed(text):
    enc = tok(text, return_tensors="pt", truncation=True)
    with torch.no_grad():
        out = mdl(**enc)
    return out.last_hidden_state[:, 0].squeeze().tolist()

r = pc.inference.embed(
    model="multilingual-e5-large",
    inputs=["hello", "bonjour"],
    parameters={"input_type": "passage"},
)
vectors = [e["values"] for e in r.data]