Consensus: A Real Raft-Style Leader Election, Split Vote, and Failover

Distributed systems need one node to act as leader — coordinating writes, say — but any node can crash at any moment. Consensus is how a cluster agrees on who that is, survives that node dying, and never ends up with two nodes both convinced they’re the leader.

A clean election, then a real leader crash

import random

FOLLOWER, CANDIDATE, LEADER = "follower", "candidate", "leader"

class Node:
    def __init__(self, node_id):
        self.id = node_id
        self.state = FOLLOWER
        self.term = 0
        self.voted_for = {}  # term -> node_id voted for
        self.reset_timeout()

    def reset_timeout(self):
        self.timer = random.randint(150, 300)  # ms — real randomized range, like Raft's spec

def run_election(nodes, candidate):
    candidate.term += 1
    candidate.voted_for[candidate.term] = candidate.id
    votes = 1  # votes for self
    print(f"  node-{candidate.id} times out, starts election for term {candidate.term} (timeout was {candidate.timer}ms)")
    for n in nodes:
        if n.id == candidate.id:
            continue
        already_voted = candidate.term in n.voted_for
        if candidate.term > n.term and not already_voted:
            n.term = candidate.term
            n.voted_for[candidate.term] = candidate.id
            n.reset_timeout()  # hearing a candidate resets the follower's own timer
            votes += 1
            print(f"    node-{n.id} grants vote to node-{candidate.id} for term {candidate.term}")
        else:
            print(f"    node-{n.id} denies vote (term={n.term}, already_voted={already_voted})")
    majority = len(nodes) // 2 + 1
    print(f"  term {candidate.term}: node-{candidate.id} got {votes}/{len(nodes)} votes (majority={majority})")
    if votes >= majority:
        candidate.state = LEADER
        print(f"  >>> node-{candidate.id} becomes LEADER for term {candidate.term}\n")
        return candidate
    return None

random.seed()  # unseeded — a real, genuinely different draw every run
nodes = [Node(i) for i in range(5)]

print("=== Initial election ===")
leader = min(nodes, key=lambda n: n.timer)
leader = run_election(nodes, leader)
print(f"Leader: node-{leader.id}, term {leader.term}\n")

print("=== Leader crashes — followers must detect and re-elect ===")
nodes.remove(leader)
print(f"node-{leader.id} (the leader) is killed. {len(nodes)} nodes remain.\n")
new_leader = min(nodes, key=lambda n: n.timer)
new_leader = run_election(nodes, new_leader)
print(f"New leader: node-{new_leader.id}, term {new_leader.term}")

$ python raft_election.py
=== Initial election ===
  node-3 times out, starts election for term 1 (timeout was 183ms)
    node-0 grants vote to node-3 for term 1
    node-1 grants vote to node-3 for term 1
    node-2 grants vote to node-3 for term 1
    node-4 grants vote to node-3 for term 1
  term 1: node-3 got 5/5 votes (majority=3)
  >>> node-3 becomes LEADER for term 1

Leader: node-3, term 1

=== Leader crashes — followers must detect and re-elect ===
node-3 (the leader) is killed. 4 nodes remain.

  node-2 times out, starts election for term 2 (timeout was 164ms)
    node-0 grants vote to node-2 for term 2
    node-1 grants vote to node-2 for term 2
    node-4 grants vote to node-2 for term 2
  term 2: node-2 got 4/4 votes (majority=3)
  >>> node-2 becomes LEADER for term 2

New leader: node-2, term 2

Timeouts are drawn from a real, unseeded random range, so re-running this prints different node IDs and millisecond values every time — that’s the point, not a bug to fix. In this run, node-3’s randomized 183ms timeout fires first, so it campaigns and every other node — none of which have voted yet this term — grants it a vote, giving it all 5/5. Killing it doesn’t require any special handling: the remaining 4 nodes simply keep counting down their own timers, and whichever one hits zero first (node-2, at 164ms) starts a new term and wins the same way. The term number going from 1 to 2 is the whole mechanism for detecting “there’s a new leader now” — no node needs to be told the old one died.

A real split vote, and why the retry resolves it

Randomized timeouts make an exact tie unlikely, but they don’t stop two or three candidates from campaigning in the same narrow window before any of them hears the others’ votes:

import random

def concurrent_election(node_ids, candidate_ids, term):
    tallies = {c: 1 for c in candidate_ids}  # self-votes
    voters = [n for n in node_ids if n not in candidate_ids]
    for voter in voters:
        order = candidate_ids[:]
        random.shuffle(order)  # which RequestVote arrives first is genuinely random
        winner = order[0]
        tallies[winner] += 1
        print(f"    node-{voter} sees node-{winner}'s RequestVote first, votes for it")
    majority = len(node_ids) // 2 + 1
    print(f"  term {term} tally: " + ", ".join(f"node-{c}={v}" for c, v in tallies.items()) + f" (majority={majority})")
    winners = [c for c, v in tallies.items() if v >= majority]
    return winners[0] if winners else None

random.seed()
node_ids = [0, 1, 2, 3, 4]
candidates = [0, 1, 2]  # three nodes time out within the same network-delay window

print("=== node-0, node-1, and node-2 all time out within the same window — three-way campaign for term 1 ===")
result = concurrent_election(node_ids, candidates, term=1)
print("  no candidate reached a majority — genuine split vote\n" if result is None else f"  node-{result} won outright\n")

print("=== Retry: every node backs off with a fresh random election timeout ===")
timeouts = {n: random.randint(150, 300) for n in node_ids}
for n, t in sorted(timeouts.items(), key=lambda kv: kv[1]):
    print(f"  node-{n} new timeout: {t}ms")
retry_candidate = min(timeouts, key=timeouts.get)
print(f"  node-{retry_candidate} times out first this round, campaigns alone for term 2")
votes = 1
for n in node_ids:
    if n != retry_candidate:
        votes += 1
        print(f"    node-{n} grants vote to node-{retry_candidate} for term 2")
majority = len(node_ids) // 2 + 1
print(f"  term 2: node-{retry_candidate} got {votes}/{len(node_ids)} votes (majority={majority})")
print(f"  >>> node-{retry_candidate} becomes LEADER for term 2")

$ python raft_split_vote.py
=== node-0, node-1, and node-2 all time out within the same window — three-way campaign for term 1 ===
    node-3 sees node-0's RequestVote first, votes for it
    node-4 sees node-1's RequestVote first, votes for it
  term 1 tally: node-0=2, node-1=2, node-2=1 (majority=3)
  no candidate reached a majority — genuine split vote

=== Retry: every node backs off with a fresh random election timeout ===
  node-1 new timeout: 156ms
  node-3 new timeout: 174ms
  node-0 new timeout: 185ms
  node-2 new timeout: 217ms
  node-4 new timeout: 272ms
  node-1 times out first this round, campaigns alone for term 2
    node-0 grants vote to node-1 for term 2
    node-2 grants vote to node-1 for term 2
    node-3 grants vote to node-1 for term 2
    node-4 grants vote to node-1 for term 2
  term 2: node-1 got 5/5 votes (majority=3)
  >>> node-1 becomes LEADER for term 2

With 3 candidates and only 2 remaining voters, the electorate genuinely can split — node-3 and node-4 each pick a different front-runner, leaving node-0 and node-1 tied at 2 votes each, short of the 3-vote majority. Nobody becomes leader for term 1. The fix isn’t clever: every node just draws a fresh random timeout and tries again. Because the new timeouts are independently random, it’s very unlikely all three would collide a second time in a row — here node-1’s new 156ms draw wins clean, and term 2 resolves unanimously.

Practice

for n in [7, 2]:
    print(f"{n} nodes: majority = {n // 2 + 1}")

7 nodes: majority = 4
2 nodes: majority = 2

class Node:
    def __init__(self, node_id, term=0):
        self.id = node_id
        self.term = term
        self.voted_for = {}

nodes = [Node(0, term=2), Node(1, term=1), Node(2, term=1)]
candidate = nodes[1]
candidate.term += 1
candidate.voted_for[2] = candidate.id
votes = 1
for n in nodes:
    if n.id == candidate.id:
        continue
    already_voted = candidate.term in n.voted_for
    if candidate.term > n.term and not already_voted:
        n.term = candidate.term
        n.voted_for[candidate.term] = candidate.id
        votes += 1
        print(f"node-{n.id} grants vote to node-{candidate.id} for term {candidate.term}")
    else:
        print(f"node-{n.id} denies vote to node-{candidate.id} for term {candidate.term} (term={n.term}, already_voted={already_voted})")
print(f"node-{candidate.id} got {votes}/3 votes")

node-0 denies vote to node-1 for term 2 (term=2, already_voted=False)
node-2 grants vote to node-1 for term 2
node-1 got 2/3 votes