Skip to content
Notifly Notifly

Provider changed the embeddings version

Embedding models at providers are regularly swapped under the same name (text-embedding-3-small updated without announcement). Old vectors in Qdrant become incompatible with new queries — search silently returns garbage.

A simple canary test: run a fixed set (query, expected_top_id) once an hour, alert if top-1 changed.

import os, json, requests
from openai import OpenAI


oai = OpenAI()
PAIRS = json.load(open("embed-canaries.json"))    # [{q, expected_id}, ...]
STATE = "/tmp/embed-canaries.json"


def handler(event, context):
    drift = []
    for c in PAIRS:
        v = oai.embeddings.create(model="text-embedding-3-small", input=c["q"]).data[0].embedding
        top = qdrant_search(v, top_k=1)[0]
        if top.id != c["expected_id"]:
            drift.append((c["q"], c["expected_id"], top.id))


    prev = json.load(open(STATE)) if os.path.exists(STATE) else []
    if drift and drift != prev:
        body = "\n".join(f"• {q[:50]} → ожидался {a}, стал {b}" for q, a, b in drift)
        push("🧭 Embeddings drift",
             f"Top-1 изменился на {len(drift)} канарейках:\n{body}",
             priority=9)
    json.dump(drift, open(STATE, "w"))
    return {"statusCode": 200}


def push(t, m, p):
    requests.post(f"{os.environ['NOTIFLY_URL']}/message",
                  params={"token": os.environ["NOTIFLY_TOKEN"]},
                  json={"title": t, "message": m, "priority": p}, timeout=5)
Section titled “Related recipes”