Cache-Aside with Redis: Stale Reads, Races and Stampedes

Cache-aside is the most common way to put Redis in front of a database: read the cache, fall back to the database on a miss, and fill the cache on the way out. It’s simple to write, and it fails in ways that only show up under concurrency. This lesson triggers each of those failures on purpose.

Setup

Redis, plus a Python image with redis-py, on one Docker network:

# setup.sh - Redis and a Python image with redis-py, on one Docker network
docker network create redis-lab > /dev/null 2>&1
docker rm -f redis-lesson > /dev/null 2>&1
docker run -d --name redis-lesson --network redis-lab -p 6379:6379 redis:8.10.2-alpine > /dev/null
printf 'FROM python:3.13-slim\nRUN pip install --no-cache-dir redis==8.1.0\n' \
  | docker build -q -t redis-py - > /dev/null
sleep 1
docker exec redis-lesson redis-cli INFO server | grep redis_version
docker run --rm redis-py python -c \
  "import redis, sqlite3, sys; print('redis-py', redis.__version__, '| sqlite', sqlite3.sqlite_version, '| python', sys.version.split()[0])"

$ bash setup.sh
redis_version:8.10.2
redis-py 8.1.0 | sqlite 3.46.1 | python 3.13.16

Each program runs in that image, in the current directory. Define a py shortcut in your shell (in Git Bash on Windows, use "$(pwd -W)" in place of "$PWD"):

py() { docker run --rm --network redis-lab -v "$PWD":/app -w /app redis-py python "$@"; }

Every example imports this module. The “database” is SQLite with a deliberate 200 ms sleep in the loader, standing in for a slow query, and db_loads counts how often it was actually queried:

# shop.py - a slow "database" (SQLite) and the cache-aside read used by every example
import json, sqlite3, threading, time
import redis

r = redis.Redis(host="redis-lesson", decode_responses=True)
DB = "shop.db"
TTL = 300  # seconds
db_loads = 0  # how many times the slow database was actually queried
_count_lock = threading.Lock()


def setup_db():
    with sqlite3.connect(DB) as con:
        con.execute("DROP TABLE IF EXISTS product")
        con.execute("CREATE TABLE product (id INTEGER PRIMARY KEY, name TEXT, price INTEGER)")
        con.executemany("INSERT INTO product VALUES (?, ?, ?)",
                        [(1, "kettle", 30), (2, "toaster", 45), (3, "mug", 8)])
    r.flushall()


def load_product(pid):
    """Query the database. The sleep stands in for a slow query or a busy database."""
    global db_loads
    with _count_lock:
        db_loads += 1
    time.sleep(0.2)
    with sqlite3.connect(DB) as con:
        row = con.execute("SELECT id, name, price FROM product WHERE id = ?", (pid,)).fetchone()
    return None if row is None else {"id": row[0], "name": row[1], "price": row[2]}


def get_product(pid):
    """Cache-aside: try Redis, fall back to the database, then fill the cache."""
    key = f"product:{pid}"
    cached = r.get(key)
    if cached is not None:
        return json.loads(cached)
    product = load_product(pid)
    if product is not None:
        r.set(key, json.dumps(product), ex=TTL)
    return product


def set_price(pid, price):
    with sqlite3.connect(DB) as con:
        con.execute("UPDATE product SET price = ? WHERE id = ?", (price, pid))

get_product is the whole pattern in seven lines. Everything below is about what it gets wrong.

A miss, then hits

# miss_hit.py - the first read misses and goes to the database; the second is served by Redis
import time
import shop

shop.setup_db()
for attempt in ("first", "second", "third"):
    start = time.perf_counter()
    product = shop.get_product(1)
    ms = (time.perf_counter() - start) * 1000
    print(f"{attempt} read: {product}  {ms:6.1f} ms")

print("database loads:", shop.db_loads)
print("TTL left on product:1:", shop.r.ttl("product:1"), "s")

$ py miss_hit.py
first read: {'id': 1, 'name': 'kettle', 'price': 30}   213.0 ms
second read: {'id': 1, 'name': 'kettle', 'price': 30}     0.5 ms
third read: {'id': 1, 'name': 'kettle', 'price': 30}     0.5 ms
database loads: 1
TTL left on product:1: 300 s

The first read paid for the database, 213 to 218 ms across runs, almost all of it the deliberate sleep. The next two came from Redis in about a millisecond or less (0.3 to 1.3 ms), and the database was queried once. The 300-second TTL is the backstop for every problem below: whatever goes wrong, a cached value can’t outlive it.

The stale read

# stale.py - change the price in the database; the cache keeps serving the old one
import shop

shop.setup_db()
print("before:", shop.get_product(1))          # miss: loads price 30 and caches it

shop.set_price(1, 25)                           # the database now says 25
print("after: ", shop.get_product(1))          # hit: still 30

print("database price:", shop.load_product(1)["price"])
print("cache expires in", shop.r.ttl("product:1"), "s")

$ py stale.py
before: {'id': 1, 'name': 'kettle', 'price': 30}
after:  {'id': 1, 'name': 'kettle', 'price': 30}
database price: 25
cache expires in 300 s

Nothing told Redis the price changed, so customers see 30 for up to five more minutes. A TTL alone makes data eventually right; how stale is acceptable decides how short it has to be.

Invalidate on write

The fix is for the code that writes the database to delete the cached copy:

# invalidate.py - every write to the database deletes the cached copy
import shop


def update_price(pid, price):
    shop.set_price(pid, price)            # 1. commit to the database
    shop.r.delete(f"product:{pid}")       # 2. then drop the cached copy


shop.setup_db()
print("before:", shop.get_product(1))
update_price(1, 25)
print("after: ", shop.get_product(1))    # miss: reloads the new price
print("again: ", shop.get_product(1))    # hit
print("database loads:", shop.db_loads)

$ py invalidate.py
before: {'id': 1, 'name': 'kettle', 'price': 30}
after:  {'id': 1, 'name': 'kettle', 'price': 25}
again:  {'id': 1, 'name': 'kettle', 'price': 25}
database loads: 2

The read after the update misses and loads 25, and the one after that is a hit again. It costs one extra database load per write. Deleting rather than writing the new value into the cache is deliberate; Practice task 2 shows why.

The order of the two steps matters, which the next two examples show with threads. Each uses threading.Event to force one exact interleaving, so the race happens on every run instead of once in a thousand.

Race 1: delete first, then commit

# race_delete_first.py - delete the cache, then write the database: a reader slips in between
import json, threading
import shop

shop.setup_db()
shop.get_product(1)                         # price 30 is cached
deleted, reader_done = threading.Event(), threading.Event()


def writer():
    shop.r.delete("product:1")              # 1. drop the cached copy...
    deleted.set()
    reader_done.wait()                      # (the database write is slow to commit)
    shop.set_price(1, 25)                   # 2. ...then write the database


def reader():
    deleted.wait()                          # arrives between the delete and the commit
    print("reader got:", shop.get_product(1))   # miss: loads the OLD row and caches it
    reader_done.set()


threads = [threading.Thread(target=writer), threading.Thread(target=reader)]
for t in threads: t.start()
for t in threads: t.join()

print("database price:", shop.load_product(1)["price"])
print("cached price:  ", json.loads(shop.r.get("product:1"))["price"],
      "for another", shop.r.ttl("product:1"), "s")

$ py race_delete_first.py
reader got: {'id': 1, 'name': 'kettle', 'price': 30}
database price: 25
cached price:   30 for another 300 s

The delete ran, and the cache still ended up wrong, for the full TTL. The reader missed in the gap, read the row before the new price was committed, and wrote that old row back into the cache. On a busy key, any write that takes a few milliseconds to commit leaves a gap a reader can land in.

Commit first, then delete

The same reader, with the writer’s steps swapped:

# race_commit_first.py - write the database, then delete the cache: the same reader is only briefly stale
import threading
import shop

shop.setup_db()
shop.get_product(1)                         # price 30 is cached
committed, reader_done = threading.Event(), threading.Event()


def writer():
    shop.set_price(1, 25)                   # 1. write the database...
    committed.set()
    reader_done.wait()
    shop.r.delete("product:1")              # 2. ...then drop the cached copy


def reader():
    committed.wait()                        # arrives between the commit and the delete
    print("reader got:", shop.get_product(1))   # hit: the old price, for a moment
    reader_done.set()


threads = [threading.Thread(target=writer), threading.Thread(target=reader)]
for t in threads: t.start()
for t in threads: t.join()

print("cached copy left:", shop.r.get("product:1"))
print("next read:", shop.get_product(1))

$ py race_commit_first.py
reader got: {'id': 1, 'name': 'kettle', 'price': 30}
cached copy left: None
next read: {'id': 1, 'name': 'kettle', 'price': 25}

A reader in the gap still sees the old price, but only as a cache hit; it doesn’t write anything. The delete then removes the stale copy, and the next read is correct. The stale window is now as long as the time between the commit and the delete, not the whole TTL.

Race 2: the one commit-then-delete can’t close

There’s still one interleaving that goes wrong. The reader misses and loads the old row before the commit, the writer commits and deletes (there’s nothing to delete yet), and then the reader writes its old row into the cache. This version caches with a 2-second TTL to show what bounds it:

# race_remaining.py - the race commit-then-delete can't close, and the TTL that bounds it
import json, threading, time
import shop

shop.setup_db()                              # nothing cached yet
loaded, writer_done = threading.Event(), threading.Event()
SHORT_TTL = 2


def reader():
    product = shop.load_product(1)          # miss: reads price 30 from the database
    loaded.set()
    writer_done.wait()                      # (the reader is slow to write the cache)
    shop.r.set("product:1", json.dumps(product), ex=SHORT_TTL)


def writer():
    loaded.wait()
    shop.set_price(1, 25)                   # commit...
    shop.r.delete("product:1")              # ...then delete: nothing to delete yet
    writer_done.set()


threads = [threading.Thread(target=reader), threading.Thread(target=writer)]
for t in threads: t.start()
for t in threads: t.join()

print("database price:", shop.load_product(1)["price"])
print("cached price:  ", shop.get_product(1)["price"])
time.sleep(SHORT_TTL + 0.5)
print(f"after {SHORT_TTL + 0.5} s:  ", shop.get_product(1)["price"])

$ py race_remaining.py
database price: 25
cached price:   30
after 2.5 s:   25

The cache was wrong until the key expired, then correct again. This race needs a database read that starts before a write and a cache write that finishes after it, so it’s much rarer than race 1, but nothing in cache-aside prevents it. That’s why every cached key gets a TTL, and why the TTL is your answer to “how long can a customer see a wrong price?” If the answer must be “never”, cache-aside is the wrong pattern for that data.

The stampede

A popular key that isn’t cached, because it just expired or Redis restarted, gets every concurrent request at once:

# stampede.py - 50 requests arrive together for a key that isn't cached
import threading, time
import shop

shop.setup_db()                              # product:1 is not in the cache
start_line = threading.Barrier(50)


def request():
    start_line.wait()                        # all 50 start at the same moment
    shop.get_product(1)


start = time.perf_counter()
threads = [threading.Thread(target=request) for _ in range(50)]
for t in threads: t.start()
for t in threads: t.join()

print("requests:      50")
print("database loads:", shop.db_loads)
print(f"wall time:     {time.perf_counter() - start:.2f} s")

$ py stampede.py
requests:      50
database loads: 50
wall time:     0.53 s

All 50 requests missed before any of them had filled the cache, so the database ran the same query 50 times. On a real database that’s 50 times the load at the moment the cache stopped protecting it, which is how a cache expiry turns into a database outage.

Single-flight with SET NX PX

Let one request load the key and make the others wait for the cache. A short lock does it:

# stampede_lock.py - one request loads the key; the other 49 wait for the cache to fill
import json, threading, time, uuid
import shop


def get_product_once(pid):
    key, lock = f"product:{pid}", f"lock:product:{pid}"
    while True:
        cached = shop.r.get(key)
        if cached is not None:
            return json.loads(cached)
        token = str(uuid.uuid4())
        if shop.r.set(lock, token, nx=True, px=2000):     # only one caller wins
            try:
                cached = shop.r.get(key)                    # re-check: a winner may have
                if cached is not None:                      # just filled it and released
                    return json.loads(cached)
                product = shop.load_product(pid)
                shop.r.set(key, json.dumps(product), ex=shop.TTL)
                return product
            finally:
                if shop.r.get(lock) == token:              # release only our own lock
                    shop.r.delete(lock)
        time.sleep(0.01)                                    # lost: wait, then re-check the cache


shop.setup_db()
start_line = threading.Barrier(50)


def request():
    start_line.wait()
    get_product_once(1)


start = time.perf_counter()
threads = [threading.Thread(target=request) for _ in range(50)]
for t in threads: t.start()
for t in threads: t.join()

print("requests:      50")
print("database loads:", shop.db_loads)
print(f"wall time:     {time.perf_counter() - start:.2f} s")

$ py stampede_lock.py
requests:      50
database loads: 1
wall time:     0.41 s

One database load instead of 50. The wall time barely moved (0.33 to 0.51 s with the lock against 0.49 to 0.56 s without, across runs), because the other requests still wait for that one load. What changes is the database load.

The re-check after winning the lock matters. A request can miss the cache, then win the lock just after the first winner filled the cache and released it. Without the second GET, that request loads the row again.

SET NX PX and the token check are the same lock as in Distributed Locks with Redis. The token check and delete are two commands, so they aren’t atomic. That lesson does the release in a Lua script, which you’d use in production. The 2-second expiry stops a crashed loader from blocking the key forever. The waiting loop here has no limit; production code caps how long a request waits (a few hundred milliseconds, say) and then reads the database itself rather than hanging.

TTL jitter

Keys written together expire together. Warm a cache after a deploy and every key expires in the same second five minutes later, which is a stampede across the whole cache:

# jitter.py - keys cached together expire together, unless each TTL gets some random jitter
import collections, random
import shop


def expiry_spread(jitter):
    shop.r.flushall()
    for pid in range(1000):                  # e.g. a cache warmed by one deploy
        ttl = shop.TTL + (random.randint(0, 60) if jitter else 0)
        shop.r.set(f"product:{pid}", "{}", ex=ttl)
    per_second = collections.Counter(shop.r.ttl(f"product:{pid}") for pid in range(1000))
    return len(per_second), max(per_second.values())


for jitter in (False, True):
    seconds, worst = expiry_spread(jitter)
    print(f"jitter={jitter!s:5}  keys expire across {seconds:2} different seconds,"
          f" at most {worst:4} in the same second")

$ py jitter.py
jitter=False  keys expire across  1 different seconds, at most 1000 in the same second
jitter=True   keys expire across 61 different seconds, at most   28 in the same second

Without jitter, all 1,000 keys expire in the same second. Adding 0 to 60 random seconds spreads them over a minute, at most 23 to 30 per second across runs. The cost is that a key can be up to a minute staler, so size the jitter to a small fraction of the TTL.

Caching “not found”

get_product only caches products that exist. A request for a missing ID goes to the database every time:

# negative.py - lookups for a product that doesn't exist: cache the "not found" too
import json
import shop

MISSING = "__none__"


def get_product_neg(pid):
    key = f"product:{pid}"
    cached = shop.r.get(key)
    if cached == MISSING:
        return None                                  # a cached "not found"
    if cached is not None:
        return json.loads(cached)
    product = shop.load_product(pid)
    if product is None:
        shop.r.set(key, MISSING, ex=30)              # remember the miss, briefly
    else:
        shop.r.set(key, json.dumps(product), ex=shop.TTL)
    return product


shop.setup_db()
for _ in range(20):
    shop.get_product(999)                            # product 999 doesn't exist
print("get_product, 20 lookups:     database loads =", shop.db_loads)

shop.db_loads = 0
for _ in range(20):
    get_product_neg(999)
print("get_product_neg, 20 lookups: database loads =", shop.db_loads)

$ py negative.py
get_product, 20 lookups:     database loads = 20
get_product_neg, 20 lookups: database loads = 1

Twenty lookups for a missing product cost 20 database queries without negative caching and 1 with it. Missing IDs are exactly what a scraper or an attacker sends to get past a cache, so this matters more than it looks. The sentinel value __none__ must not be a value real data can take, and its TTL is short (30 s here) because a “not found” can become wrong the moment someone creates the record. Practice task 3 shows that case.

Gotchas

• Cache-aside doesn’t make Redis a source of truth. Every key can vanish (eviction, restart, expiry) and the code must still work by reading the database. If you run Redis with an eviction policy, see maxmemory and Eviction.
• Serialise with care. json.dumps drops types such as dates and decimals. Store a version field if the shape of cached objects can change between deploys.
• Watch the hit ratio. INFO stats reports keyspace_hits and keyspace_misses. A falling hit ratio is the early warning that the cache is too small or the TTL too short.
• This is one layer. A CDN caches whole responses in front of your app; cache-aside caches data inside it. See CDN and Edge Caching for the layer above.

Practice

1. A short TTL. Read product 1 every 0.3 s for 3 seconds, first with a 300-second TTL and then with a 1-second TTL. How many database loads does each cost?

# p1_short_ttl.py - a 1-second TTL: fresher data, more database loads
import time
import shop

shop.setup_db()
for ttl in (300, 1):
    shop.TTL = ttl
    shop.r.flushall()
    shop.db_loads = 0
    for _ in range(10):                      # one read every 0.3 s for 3 seconds
        shop.get_product(1)
        time.sleep(0.3)
    print(f"TTL {ttl:3} s: 10 reads -> {shop.db_loads} database loads")

$ py p1_short_ttl.py
TTL 300 s: 10 reads -> 1 database loads
TTL   1 s: 10 reads -> 3 database loads

2. SET instead of DEL on write. Two writers update the price: A commits 25, then B commits 20, but each also writes its new value into the cache, and A’s cache write lands last. What does the cache hold?

# p2_write_set.py - two writers that SET the cache instead of deleting it, finishing out of order
import json, threading
import shop

shop.setup_db()
shop.get_product(1)
a_committed, b_cached = threading.Event(), threading.Event()


def writer_a():
    shop.set_price(1, 25)                    # A commits first...
    a_committed.set()
    b_cached.wait()                          # ...but is slow to update the cache
    shop.r.set("product:1", json.dumps({"id": 1, "name": "kettle", "price": 25}), ex=shop.TTL)


def writer_b():
    a_committed.wait()
    shop.set_price(1, 20)                    # B commits second: 20 is the real price
    shop.r.set("product:1", json.dumps({"id": 1, "name": "kettle", "price": 20}), ex=shop.TTL)
    b_cached.set()


threads = [threading.Thread(target=writer_a), threading.Thread(target=writer_b)]
for t in threads: t.start()
for t in threads: t.join()

print("database price:", shop.load_product(1)["price"])
print("cached price:  ", shop.get_product(1)["price"])

$ py p2_write_set.py
database price: 20
cached price:   25

3. A cached “not found” hides a new product. Look up product 4 (missing, so the miss is cached), then create it. What does the next lookup return, and what fixes it?

# p3_negative_insert.py - a cached "not found" hides a product created afterwards
import json, sqlite3
import shop

MISSING = "__none__"


def get_product_neg(pid):
    key = f"product:{pid}"
    cached = shop.r.get(key)
    if cached == MISSING:
        return None
    if cached is not None:
        return json.loads(cached)
    product = shop.load_product(pid)
    shop.r.set(key, MISSING if product is None else json.dumps(product),
               ex=30 if product is None else shop.TTL)
    return product


def create_product(pid, name, price, invalidate):
    with sqlite3.connect(shop.DB) as con:
        con.execute("INSERT INTO product VALUES (?, ?, ?)", (pid, name, price))
    if invalidate:
        shop.r.delete(f"product:{pid}")      # clear any cached "not found"


shop.setup_db()
print("lookup 4:", get_product_neg(4))       # not found, and that is cached for 30 s
create_product(4, "teapot", 22, invalidate=False)
print("after insert, no invalidation:", get_product_neg(4))

shop.setup_db()
get_product_neg(4)
create_product(4, "teapot", 22, invalidate=True)
print("after insert, with invalidation:", get_product_neg(4))

$ py p3_negative_insert.py
lookup 4: None
after insert, no invalidation: None
after insert, with invalidation: {'id': 4, 'name': 'teapot', 'price': 22}