Handling Errors and Retries

import time, random

def with_backoff(fn, *args, max_tries=5, **kw):
    for i in range(max_tries):
        try:
            return fn(*args, **kw)
        except Exception as e:
            if i == max_tries - 1: raise
            wait = (2 ** i) + random.random()
            time.sleep(wait)

from pinecone.exceptions import PineconeApiException

try:
    index.upsert(vectors=batch)
except PineconeApiException as e:
    if e.status == 429:
        time.sleep(int(e.headers.get("Retry-After", "5")))
        index.upsert(vectors=batch)
    else: raise

from tenacity import retry, stop_after_attempt, wait_exponential

@retry(stop=stop_after_attempt(5),
       wait=wait_exponential(min=1, max=30))
def safe_query(qv):
    return index.query(vector=qv, top_k=10)

from pinecone import Pinecone
pc = Pinecone(api_key="...", additional_headers={"x-request-timeout": "30"})

import pybreaker
breaker = pybreaker.CircuitBreaker(fail_max=5, reset_timeout=30)

@breaker
def query(qv):
    return index.query(vector=qv, top_k=10)

failed = []
for b in batches:
    try: index.upsert(vectors=b)
    except Exception: failed.append(b)
# retry failed later
for b in failed: with_backoff(index.upsert, vectors=b)

try:
    index.query(vector=qv, top_k=10)
except PineconeApiException as e:
    log.error("pinecone_query_failed", extra={
        "status": e.status,
        "reason": e.reason,
        "body": getattr(e, "body", None),
    })
    raise

while not pc.describe_index("docs").status["ready"]:
    time.sleep(2)
index = pc.Index("docs")