Webhook Delivery Reliability: Why Pull Beats Push for Mission-Critical Events

Timeline of a Push Webhook Failure:
10:00 AM - Deploy begins, server goes offline
10:05 AM - Stripe sends payment.succeeded webhook
10:06 AM - Webhook times out (no server to receive)
10:10 AM - Deploy completes, server back online
10:11 AM - Payment event is lost forever

Timeline with Pull (FetchHook):
10:00 AM - Deploy begins, server goes offline
10:05 AM - Stripe sends webhook -> FetchHook mailbox
10:10 AM - Deploy completes, server back online
10:11 AM - Server pulls from mailbox, event processed

Failure Scenario      | Push (Traditional) | Pull (FetchHook)
--------------------- | ------------------ | ----------------
Deploy downtime       | Event lost         | Queued, retrieved post-deploy
Network timeout       | Event lost/retry   | Queued, pulled when online
Container restart     | Event lost         | Queued in mailbox
Rate limit exceeded   | 429, event dropped | Buffered, consumed at own pace
Processing crash      | Event lost         | Remains in queue until ACKed

import requests
import time
import logging

API_KEY = "fh_live_xxx"
SOURCE_ID = "stash_abc123"

logging.basicConfig(level=logging.INFO)

def fetch_events():
    """Pull events from FetchHook mailbox."""
    response = requests.get(
        f"https://api.fetchhook.app/api/v1/{SOURCE_ID}",
        headers={"Authorization": f"Bearer {API_KEY}"}
    )
    response.raise_for_status()
    return response.json().get("events", [])

def process_event(event):
    """
    Process a single event with retry logic.
    Raises exception if processing fails.
    """
    event_id = event.get("id")
    event_type = event["payload"].get("type", "unknown")

    try:
        # Your mission-critical processing logic
        logging.info(f"Processing {event_type} (ID: {event_id})")

        # Example: charge a customer, update inventory, etc.
        # If this fails, exception is raised and event stays in queue

        logging.info(f"Successfully processed {event_id}")
        return True

    except Exception as e:
        logging.error(f"Failed to process {event_id}: {e}")
        raise  # Re-raise to prevent event from being marked as consumed

def reliable_consumer_loop():
    """Main loop with built-in reliability."""
    while True:
        try:
            events = fetch_events()

            for event in events:
                try:
                    process_event(event)
                except Exception:
                    # Event processing failed, will retry on next loop
                    # Event remains in FetchHook mailbox
                    continue

            if events:
                logging.info(f"Processed {len(events)} events")

            time.sleep(10)  # Poll every 10 seconds

        except Exception as e:
            logging.error(f"Consumer error: {e}")
            time.sleep(30)  # Back off on errors

if __name__ == "__main__":
    reliable_consumer_loop()

# Before deploy: Consumer running normally
# During deploy (5 min): Consumer offline, webhooks queue in FetchHook
# After deploy: Consumer starts, pulls queued events

# No code changes needed - the pattern handles it automatically
def post_deploy_sync():
    """Run this after deployment to catch up."""
    events = fetch_events()
    logging.info(f"Post-deploy: Processing {len(events)} queued events")

    for event in events:
        process_event(event)  # Catch up on missed events

# Consumer crashed at 10:00 AM
# Webhooks continued arriving -> FetchHook mailbox
# Consumer restarted at 10:30 AM

# On restart, consumer automatically pulls all queued events
def on_startup():
    """Automatic recovery on restart."""
    logging.info("Consumer started, checking for queued events...")

    # Pull all events since last successful run
    events = fetch_events()

    if events:
        logging.warning(f"Recovered {len(events)} events from downtime")
        for event in events:
            process_event(event)
    else:
        logging.info("No queued events, system healthy")

const axios = require('axios');

const API_KEY = process.env.FETCHHOOK_API_KEY;
const SOURCE_ID = 'stash_abc123';

let healthCheck = { healthy: true, lastSuccess: Date.now() };

async function fetchEvents() {
  const response = await axios.get(
    `https://api.fetchhook.app/api/v1/${SOURCE_ID}`,
    { headers: { Authorization: `Bearer ${API_KEY}` } }
  );
  return response.data.events || [];
}

async function processWithHealthCheck(event) {
  try {
    // Your processing logic
    await processEvent(event);

    // Mark as healthy after successful processing
    healthCheck.healthy = true;
    healthCheck.lastSuccess = Date.now();

  } catch (error) {
    healthCheck.healthy = false;
    console.error(`Processing failed: ${error.message}`);

    // Event stays in mailbox, will retry on next loop
    throw error;
  }
}

async function reliableConsumer() {
  while (true) {
    try {
      const events = await fetchEvents();

      for (const event of events) {
        await processWithHealthCheck(event);
      }

      // Adaptive polling based on health
      const delay = healthCheck.healthy ? 10000 : 30000;
      await new Promise(resolve => setTimeout(resolve, delay));

    } catch (error) {
      console.error('Consumer loop error:', error.message);
      await new Promise(resolve => setTimeout(resolve, 60000));
    }
  }
}

reliableConsumer();