Skip to content
Notifly Notifly

Schema drift in tool-calls

Tool-calls in Claude / OpenAI go through JSON-Schema, but the model from time to time outputs something “almost valid”. Especially after a model change, a prompt update, or a quiet release from the provider. An early-detection alert:

import os, json, time, requests
from jsonschema import validate, ValidationError


W = "/tmp/schema-drift.json"


def observe(tool_name, args, schema):
    s = (json.load(open(W)) if os.path.exists(W) else {})
    s.setdefault(tool_name, {"ok": 0, "fail": 0, "examples": []})
    try:
        validate(args, schema)
        s[tool_name]["ok"] += 1
    except ValidationError as e:
        s[tool_name]["fail"] += 1
        s[tool_name]["examples"] = (s[tool_name]["examples"] + [str(e)[:300]])[-3:]
    json.dump(s, open(W, "w"))


    total = s[tool_name]["ok"] + s[tool_name]["fail"]
    if total > 100 and s[tool_name]["fail"] / total > 0.1:
        push(f"🪤 Schema-drift в {tool_name}",
             f"Невалидных вызовов: {s[tool_name]['fail']}/{total}\n\n"
             "Последние ошибки:\n" + "\n---\n".join(s[tool_name]["examples"]),
             priority=8)


def push(t, m, p):
    requests.post(f"{os.environ['NOTIFLY_URL']}/message",
                  params={"token": os.environ["NOTIFLY_TOKEN"]},
                  json={"title": t, "message": m, "priority": p}, timeout=5)
Section titled “Related recipes”