Dead-Letter Queues: What Happens When Retries Keep Failing

Retrying a message that will never succeed isn’t resilience — it’s wasted work. A dead-letter queue is where a consumer puts a message once it’s given up, instead of retrying forever or silently dropping it.

Bounded retries, then dead-letter instead of retrying forever

class PoisonMessageError(Exception):
    pass

def process_payment(event):
    if event["amount"] < 0:
        raise PoisonMessageError(f"invalid amount: {event['amount']}")
    print(f"  processed payment for order {event['order_id']}: ${event['amount']}")

def consume_with_retry(event, max_retries=3):
    for attempt in range(1, max_retries + 1):
        try:
            process_payment(event)
            return True
        except PoisonMessageError as e:
            print(f"  attempt {attempt}/{max_retries} failed: {e}")
    return False

dead_letter_queue = []

events = [
    {"order_id": 1, "amount": 50},
    {"order_id": 2, "amount": -10},
    {"order_id": 3, "amount": 25},
]

for event in events:
    print(f"Processing order {event['order_id']}...")
    if not consume_with_retry(event):
        print(f"  order {event['order_id']} exhausted retries, routing to dead-letter queue")
        dead_letter_queue.append(event)

print(f"Dead-letter queue: {[e['order_id'] for e in dead_letter_queue]}")

$ python dlq_basic.py
Processing order 1...
  processed payment for order 1: $50
Processing order 2...
  attempt 1/3 failed: invalid amount: -10
  attempt 2/3 failed: invalid amount: -10
  attempt 3/3 failed: invalid amount: -10
  order 2 exhausted retries, routing to dead-letter queue
Processing order 3...
  processed payment for order 3: $25
Dead-letter queue: [2]

Order 2’s amount is negative, which process_payment always rejects — no number of retries changes that, so all 3 attempts fail identically. Orders 1 and 3 process normally and never touch the dead-letter queue at all; only the message that genuinely can’t succeed ends up there.

Real backoff between retries, not an instant loop

import time

class PoisonMessageError(Exception):
    pass

def process_payment(event):
    if event["amount"] < 0:
        raise PoisonMessageError(f"invalid amount: {event['amount']}")
    print(f"  processed payment for order {event['order_id']}: ${event['amount']}")

def consume_with_retry(event, max_retries=3):
    for attempt in range(1, max_retries + 1):
        try:
            process_payment(event)
            return True
        except PoisonMessageError as e:
            backoff = 0.2 * attempt
            start = time.perf_counter()
            time.sleep(backoff)
            elapsed = time.perf_counter() - start
            print(f"  attempt {attempt}/{max_retries} failed: {e} (backed off {elapsed:.1f}s)")
    return False

result = consume_with_retry({"order_id": 2, "amount": -10})
print(f"succeeded: {result}")

$ python dlq_backoff.py
  attempt 1/3 failed: invalid amount: -10 (backed off 0.2s)
  attempt 2/3 failed: invalid amount: -10 (backed off 0.4s)
  attempt 3/3 failed: invalid amount: -10 (backed off 0.6s)
succeeded: False

Each retry waits longer than the last (0.2 * attempt), and the printed elapsed time is measured with time.perf_counter() around the real time.sleep() call — it isn’t a number the code just claims, it’s the actual wall-clock time that attempt spent waiting. Growing the delay between retries (backoff) gives a genuinely transient problem more time to clear up between attempts, instead of hammering the dependency at full speed 3 times in a row.

Inspecting and replaying the dead-letter queue after a fix

dead_letter_queue = [
    {"order_id": 2, "amount": -10},
    {"order_id": 5, "amount": -30},
]

def inspect(dlq):
    print(f"{len(dlq)} message(s) in the dead-letter queue:")
    for msg in dlq:
        print(f"  order {msg['order_id']}: amount={msg['amount']}")

inspect(dead_letter_queue)

# Bug fix deployed: negative amounts are refunds, not errors — reprocess with the fix
def process_payment_fixed(event):
    if event["amount"] < 0:
        print(f"  processed refund for order {event['order_id']}: ${-event['amount']}")
    else:
        print(f"  processed payment for order {event['order_id']}: ${event['amount']}")

print("Replaying dead-letter queue after the fix...")
replayed = []
for msg in dead_letter_queue:
    process_payment_fixed(msg)
    replayed.append(msg)

for msg in replayed:
    dead_letter_queue.remove(msg)

print(f"Dead-letter queue after replay: {len(dead_letter_queue)} message(s) remaining")

$ python dlq_replay.py
2 message(s) in the dead-letter queue:
  order 2: amount=-10
  order 5: amount=-30
Replaying dead-letter queue after the fix...
  processed refund for order 2: $10
  processed refund for order 5: $30
Dead-letter queue after replay: 0 message(s) remaining

The two messages that were poison under the old logic (negative amounts always errored) turn out to be perfectly valid refunds once process_payment_fixed treats them that way. This is the entire point of dead-lettering instead of dropping: nothing was lost while the bug existed, and once it’s fixed, the exact same messages that failed before can be replayed through the corrected handler and finish successfully.

Practice

class PoisonMessageError(Exception):
    pass

def process_payment(event):
    if event["amount"] < 0:
        raise PoisonMessageError(f"invalid amount: {event['amount']}")
    print(f"  processed payment for order {event['order_id']}: ${event['amount']}")

def consume_with_retry(event, max_retries=1):
    for attempt in range(1, max_retries + 1):
        try:
            process_payment(event)
            return True
        except PoisonMessageError as e:
            print(f"  attempt {attempt}/{max_retries} failed: {e}")
    return False

event = {"order_id": 9, "amount": -5}
if not consume_with_retry(event, max_retries=1):
    print(f"order {event['order_id']} exhausted retries, routing to dead-letter queue")

$ python one_retry.py
  attempt 1/1 failed: invalid amount: -5
order 9 exhausted retries, routing to dead-letter queue

class TransientError(Exception):
    pass

attempt_count = {"value": 0}

def flaky_process(event):
    attempt_count["value"] += 1
    if attempt_count["value"] < 3:
        raise TransientError(f"downstream unavailable (attempt {attempt_count['value']})")
    print(f"  processed order {event['order_id']} on attempt {attempt_count['value']}")

def consume_with_retry(event, max_retries=3):
    for attempt in range(1, max_retries + 1):
        try:
            flaky_process(event)
            return True
        except TransientError as e:
            print(f"  attempt {attempt}/{max_retries} failed: {e}")
    return False

event = {"order_id": 12}
succeeded = consume_with_retry(event)
print(f"succeeded: {succeeded}")

$ python transient_recovers.py
  attempt 1/3 failed: downstream unavailable (attempt 1)
  attempt 2/3 failed: downstream unavailable (attempt 2)
  processed order 12 on attempt 3
succeeded: True

class PoisonMessageError(Exception):
    pass

dead_letter_queue = [
    {"order_id": 2, "amount": -10},   # now valid under the fixed logic
    {"order_id": 7, "amount": None},  # still broken - missing amount entirely
]

def process_payment_fixed(event):
    if event["amount"] is None:
        raise PoisonMessageError(f"order {event['order_id']} has no amount at all")
    if event["amount"] < 0:
        print(f"  processed refund for order {event['order_id']}: ${-event['amount']}")
    else:
        print(f"  processed payment for order {event['order_id']}: ${event['amount']}")

still_stuck = []
for msg in dead_letter_queue:
    try:
        process_payment_fixed(msg)
    except PoisonMessageError as e:
        print(f"  still failing: {e}")
        still_stuck.append(msg)

print(f"Remaining in dead-letter queue: {[m['order_id'] for m in still_stuck]}")

$ python partial_replay.py
  processed refund for order 2: $10
  still failing: order 7 has no amount at all
Remaining in dead-letter queue: [7]