Error Handling

import time, random, logging
from crawlbase import CrawlingAPI

api = CrawlingAPI({'token': 'YOUR_TOKEN'})
log = logging.getLogger('crawler')

TRANSIENT = {429, 500, 503, 522, 599}
TERMINAL  = {401, 402, 403, 404, 410, 422, 451}

def crawl(url, max_attempts=5, base=0.5, cap=30):
    for attempt in range(max_attempts):
        res = api.get(url)
        status = res['status_code']

        if status == 200 and res['pc_status'] == 200:
            return res

        if status in TERMINAL or res['pc_status'] in TERMINAL:
            log.warning(f'Terminal error {status}/{res['pc_status']} for {url}')
            raise PermanentFailure(url, status)

        # Transient — sleep with full jitter, then retry
        wait = min(cap, base * (2 ** attempt))
        wait = random.uniform(0, wait)
        log.info(f'Attempt {attempt+1} got {status}; sleeping {wait:.2f}s')
        time.sleep(wait)

    raise RuntimeError(f'Exhausted retries for {url}')

class PermanentFailure(Exception): pass

const { CrawlingAPI } = require('crawlbase');
const api = new CrawlingAPI({ token: process.env.CRAWLBASE_TOKEN });

const TRANSIENT = new Set([429, 500, 503, 522, 599]);
const TERMINAL  = new Set([401, 402, 403, 404, 410, 422, 451]);

async function crawl(url, { maxAttempts = 5, base = 500, cap = 30000 } = {}) {
  for (let attempt = 0; attempt < maxAttempts; attempt++) {
    const res = await api.get(url);
    const status = res.statusCode;

    if (status === 200 && res.pcStatus === 200) return res;

    if (TERMINAL.has(status) || TERMINAL.has(res.pcStatus)) {
      throw new Error(`Permanent failure \${status} for \${url}`);
    }

    const wait = Math.random() * Math.min(cap, base * 2 ** attempt);
    await new Promise(r => setTimeout(r, wait));
  }
  throw new Error(`Exhausted retries for \${url}`);
}