Concurrency with pthreads in C

Every program here is built with -pthread and runs on Linux (glibc, 12 CPU cores). Threads run in whatever order the scheduler picks, so where the output order can vary, the lesson shows more than one run.

Creating and Joining Threads

pthread_create starts a function running on a new thread. pthread_join waits for it to finish and collects its return value:

#include <pthread.h>
#include <stdio.h>
#include <stdlib.h>

typedef struct { int id; int n; } Args;

void *worker(void *arg) {
    Args *a = arg;
    long sum = 0;
    for (int i = 1; i <= a->n; i++) sum += i;
    printf("thread %d finished\n", a->id);
    long *result = malloc(sizeof *result);   /* must outlive the thread */
    if (result) *result = sum;
    return result;
}

int main(void) {
    enum { N = 4 };
    pthread_t threads[N];
    Args args[N];
    for (int i = 0; i < N; i++) {
        args[i] = (Args){i, (i + 1) * 1000};
        if (pthread_create(&threads[i], NULL, worker, &args[i]) != 0) return 1;
    }
    for (int i = 0; i < N; i++) {
        void *ret;
        pthread_join(threads[i], &ret);        /* wait, then collect */
        printf("thread %d: sum 1..%d = %ld\n", i, args[i].n, *(long *)ret);
        free(ret);
    }
    return 0;
}

$ gcc -Wall -Wextra -std=c17 -pthread -o create create.c
$ ./create
thread 1 finished
thread 0 finished
thread 3 finished
thread 2 finished
thread 0: sum 1..1000 = 500500
thread 1: sum 1..2000 = 2001000
thread 2: sum 1..3000 = 4501500
thread 3: sum 1..4000 = 8002000
$ ./create
thread 0 finished
thread 1 finished
thread 0: sum 1..1000 = 500500
thread 1: sum 1..2000 = 2001000
thread 2 finished
thread 3 finished
thread 2: sum 1..3000 = 4501500
thread 3: sum 1..4000 = 8002000

The “finished” lines came out in a different order on each run, and in the first run thread 1 beat thread 0. The result lines are always in order 0 1 2 3, because main joins the threads in that order. Each thread gets its own Args. Passing the same struct to all four and changing it inside the loop would let threads read each other’s values. The result is heap-allocated because a pointer to the thread’s local sum would dangle once the thread exits.

Race Conditions

counter++ looks like one step but is really three: load, add, store. With eight threads doing it at once, updates get lost:

#include <pthread.h>
#include <stdio.h>

#define NUM_THREADS 8
#define INCREMENTS  100000

static long counter = 0;

void *increment(void *arg) {
    (void)arg;
    for (int i = 0; i < INCREMENTS; i++) counter++;   /* read, add, write */
    return NULL;
}

int main(void) {
    pthread_t t[NUM_THREADS];
    for (int i = 0; i < NUM_THREADS; i++) pthread_create(&t[i], NULL, increment, NULL);
    for (int i = 0; i < NUM_THREADS; i++) pthread_join(t[i], NULL);
    printf("expected %d, got %ld\n", NUM_THREADS * INCREMENTS, counter);
    return 0;
}

$ gcc -Wall -Wextra -std=c17 -pthread -o race race.c
$ ./race
expected 800000, got 198618
$ ./race
expected 800000, got 224371
$ ./race
expected 800000, got 180136
$ gcc -Wall -Wextra -std=c17 -pthread -O2 -o race_o2 race.c
$ ./race_o2
expected 800000, got 800000

Three runs lost about three-quarters of the increments, a different amount each time. Two threads read the same value, both add 1, and both write back the same number. The -O2 build printed the right answer, which is worse, because the bug is still there. objdump -d shows GCC turned the whole loop into a single addq $0x186a0 (add 100000) instruction, shrinking the race window until it rarely hits. A heavier workload or a different machine would expose it again. ThreadSanitizer finds the race regardless of luck:

$ gcc -Wall -Wextra -std=c17 -pthread -g -fsanitize=thread -o race_tsan race.c
$ ./race_tsan
==================
WARNING: ThreadSanitizer: data race (pid=2180)
  Write of size 8 at 0x000000404058 by thread T3:
    #0 increment /w/conc/race.c:11 (race_tsan+0x4011d6)
    #1 <null> <null> (libtsan.so.2+0x4bd49)

  Previous write of size 8 at 0x000000404058 by thread T1:
    #0 increment /w/conc/race.c:11 (race_tsan+0x4011d6)
    #1 <null> <null> (libtsan.so.2+0x4bd49)

  Location is global 'counter' of size 8 at 0x000000404058 (race_tsan+0x404058)
...

TSan reports two threads writing the same 8-byte global, counter, at race.c:11 with nothing ordering the writes. It catches the race on every run, however lucky the final count looks. (The full report goes on to list where each thread was created.) TSan slows a program down several times over, so it belongs in test builds, not production ones.

Fixing It: Mutex or Atomic

A mutex lets only one thread at a time into the critical section:

#include <pthread.h>
#include <stdio.h>

#define NUM_THREADS 8
#define INCREMENTS  100000

static long counter = 0;
static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER;

void *increment(void *arg) {
    (void)arg;
    for (int i = 0; i < INCREMENTS; i++) {
        pthread_mutex_lock(&lock);
        counter++;                     /* one thread at a time */
        pthread_mutex_unlock(&lock);
    }
    return NULL;
}

int main(void) {
    pthread_t t[NUM_THREADS];
    for (int i = 0; i < NUM_THREADS; i++) pthread_create(&t[i], NULL, increment, NULL);
    for (int i = 0; i < NUM_THREADS; i++) pthread_join(t[i], NULL);
    printf("expected %d, got %ld\n", NUM_THREADS * INCREMENTS, counter);
    return 0;
}

For a single variable, C11’s <stdatomic.h> does the same job with one hardware instruction and no lock:

#include <pthread.h>
#include <stdatomic.h>
#include <stdio.h>

#define NUM_THREADS 8
#define INCREMENTS  100000

static atomic_long counter = 0;

void *increment(void *arg) {
    (void)arg;
    for (int i = 0; i < INCREMENTS; i++) atomic_fetch_add(&counter, 1);
    return NULL;
}

int main(void) {
    pthread_t t[NUM_THREADS];
    for (int i = 0; i < NUM_THREADS; i++) pthread_create(&t[i], NULL, increment, NULL);
    for (int i = 0; i < NUM_THREADS; i++) pthread_join(t[i], NULL);
    printf("expected %d, got %ld\n", NUM_THREADS * INCREMENTS, (long)counter);
    return 0;
}

$ gcc -Wall -Wextra -std=c17 -pthread -o mutex mutex.c
$ ./mutex
expected 800000, got 800000
$ gcc -Wall -Wextra -std=c17 -pthread -o atomic atomic.c
$ ./atomic
expected 800000, got 800000
$ TIMEFORMAT='%R s'; for i in 1 2 3; do time ./mutex >/dev/null; time ./atomic >/dev/null; done
0.092 s
0.035 s
0.083 s
0.031 s
0.086 s
0.037 s

Both are correct every time. The atomic version ran in about 0.035 s against the mutex’s 0.085 s, consistently across three pairs of runs, because the mutex makes eight threads queue up 800,000 times. Atomics only cover single operations on single variables, though. “Move money from account A to B” changes two values that must stay consistent with each other, and that needs a mutex.

Condition Variables: Producer and Consumer

A condition variable lets a thread sleep until another thread changes some shared state. Here, two producers fill a 4-slot buffer and two consumers drain it:

#include <pthread.h>
#include <stdio.h>

#define CAP   4
#define ITEMS 1000                  /* per producer */

static int buf[CAP], count, head, tail;
static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER;
static pthread_cond_t not_full = PTHREAD_COND_INITIALIZER;
static pthread_cond_t not_empty = PTHREAD_COND_INITIALIZER;

void push(int v) {
    pthread_mutex_lock(&lock);
    while (count == CAP) pthread_cond_wait(&not_full, &lock);   /* sleep, lock released */
    buf[tail] = v; tail = (tail + 1) % CAP; count++;
    pthread_cond_signal(&not_empty);
    pthread_mutex_unlock(&lock);
}

int pop(void) {
    pthread_mutex_lock(&lock);
    while (count == 0) pthread_cond_wait(&not_empty, &lock);
    int v = buf[head]; head = (head + 1) % CAP; count--;
    pthread_cond_signal(&not_full);
    pthread_mutex_unlock(&lock);
    return v;
}

void *producer(void *arg) {
    for (int i = 1; i <= ITEMS; i++) push(i);
    (void)arg;
    return NULL;
}

void *consumer(void *arg) {
    long *total = arg;
    for (int i = 0; i < ITEMS; i++) *total += pop();
    return NULL;
}

int main(void) {
    pthread_t p1, p2, c1, c2;
    long t1 = 0, t2 = 0;
    pthread_create(&p1, NULL, producer, NULL);
    pthread_create(&p2, NULL, producer, NULL);
    pthread_create(&c1, NULL, consumer, &t1);
    pthread_create(&c2, NULL, consumer, &t2);
    pthread_join(p1, NULL); pthread_join(p2, NULL);
    pthread_join(c1, NULL); pthread_join(c2, NULL);
    printf("consumer 1: %ld, consumer 2: %ld, total %ld (expected %d)\n",
           t1, t2, t1 + t2, 2 * ITEMS * (ITEMS + 1) / 2);
    return 0;
}

$ gcc -Wall -Wextra -std=c17 -pthread -o prodcons prodcons.c
$ ./prodcons
consumer 1: 445924, consumer 2: 555076, total 1001000 (expected 1001000)
$ ./prodcons
consumer 1: 535779, consumer 2: 465221, total 1001000 (expected 1001000)

How the work split between the two consumers varied from run to run, but the total was exact both times: all 2,000 items passed through a 4-slot buffer with nothing lost or duplicated. pthread_cond_wait atomically releases the mutex and sleeps, then re-acquires the mutex before returning. That’s why a sleeping producer doesn’t block the consumers. The while (not if) around each wait is required, because a thread can wake up without the condition being true: another consumer may have grabbed the item first, or the wakeup may be spurious.

Semaphores: Limiting Concurrent Access

A counting semaphore starting at N lets at most N threads past sem_wait at once. This program tracks the peak number of threads holding a “connection”:

#define _POSIX_C_SOURCE 200809L          /* for nanosleep under -std=c17 */
#include <pthread.h>
#include <semaphore.h>
#include <stdatomic.h>
#include <stdio.h>
#include <time.h>

#define MAX_CONNECTIONS 3
#define NUM_THREADS     8

static sem_t pool;
static atomic_int in_use, peak;

void *use_connection(void *arg) {
    (void)arg;
    sem_wait(&pool);                          /* take a slot, or block */
    int now = atomic_fetch_add(&in_use, 1) + 1;
    int old = atomic_load(&peak);
    while (now > old && !atomic_compare_exchange_weak(&peak, &old, now)) { }
    nanosleep(&(struct timespec){0, 50000000}, NULL);   /* hold it for 50 ms */
    atomic_fetch_sub(&in_use, 1);
    sem_post(&pool);                          /* give the slot back */
    return NULL;
}

int main(void) {
    sem_init(&pool, 0, MAX_CONNECTIONS);      /* 3 slots available */
    pthread_t t[NUM_THREADS];
    for (int i = 0; i < NUM_THREADS; i++) pthread_create(&t[i], NULL, use_connection, NULL);
    for (int i = 0; i < NUM_THREADS; i++) pthread_join(t[i], NULL);
    printf("%d threads shared %d connections; peak concurrent use = %d\n",
           NUM_THREADS, MAX_CONNECTIONS, atomic_load(&peak));
    sem_destroy(&pool);
    return 0;
}

$ gcc -Wall -Wextra -std=c17 -pthread -o pool pool.c
$ time ./pool
8 threads shared 3 connections; peak concurrent use = 3

real	0m0.158s
user	0m0.003s
sys	0m0.000s

The peak never exceeded 3, and the timing shows why. Eight 50 ms jobs through 3 slots need ⌈8/3⌉ = 3 rounds, about 150 ms. With MAX_CONNECTIONS 8 the same run reports a peak of 8 in 0.059 s, a single round. The compare-and-swap loop records the maximum safely without a lock. usleep would be simpler than nanosleep, but it was removed from POSIX 2008 and isn’t declared under -std=c17. The _POSIX_C_SOURCE define is what makes nanosleep visible. (macOS doesn’t support unnamed sem_init semaphores. Use dispatch_semaphore there, or a mutex plus a condition variable.)

Thread-Local Storage

_Thread_local (C11) gives each thread its own copy of a variable, so per-thread state needs no locking:

#include <pthread.h>
#include <stdio.h>

static int shared_counter = 0;              /* one copy for everyone */
static _Thread_local int local_counter = 0; /* one copy per thread */

void *work(void *arg) {
    int n = *(int *)arg;
    for (int i = 0; i < n; i++) local_counter++;
    printf("thread asked for %d: local_counter = %d\n", n, local_counter);
    return NULL;
}

int main(void) {
    pthread_t t[3];
    int counts[] = {10, 20, 30};
    for (int i = 0; i < 3; i++) {
        pthread_create(&t[i], NULL, work, &counts[i]);
        pthread_join(t[i], NULL);            /* one at a time: tidy output */
    }
    printf("main thread: local_counter = %d, shared_counter = %d\n",
           local_counter, shared_counter);
    return 0;
}

$ gcc -Wall -Wextra -std=c17 -pthread -o tls tls.c
$ ./tls
thread asked for 10: local_counter = 10
thread asked for 20: local_counter = 20
thread asked for 30: local_counter = 30
main thread: local_counter = 0, shared_counter = 0

If local_counter were an ordinary global, the second thread would have printed 30 and the third 60. Instead each thread started from its own zero, and main’s copy was never touched. errno works exactly this way, which is why one thread’s failed call doesn’t clobber another thread’s errno.

Deadlock

Two threads that take the same two locks in opposite orders can each end up holding one lock and waiting forever for the other:

#include <pthread.h>
#include <stdio.h>

static pthread_mutex_t a = PTHREAD_MUTEX_INITIALIZER, b = PTHREAD_MUTEX_INITIALIZER;
static int moved;

void *transfer_ab(void *arg) {
    (void)arg;
    for (int i = 0; i < 100000; i++) {
        pthread_mutex_lock(&a); pthread_mutex_lock(&b);   /* always a, then b */
        moved++;
        pthread_mutex_unlock(&b); pthread_mutex_unlock(&a);
    }
    return NULL;
}

void *transfer_ba(void *arg) {
    (void)arg;
    for (int i = 0; i < 100000; i++) {
        pthread_mutex_lock(&b); pthread_mutex_lock(&a);   /* opposite order! */
        moved++;
        pthread_mutex_unlock(&b); pthread_mutex_unlock(&a);
    }
    return NULL;
}

int main(void) {
    pthread_t t1, t2;
    pthread_create(&t1, NULL, transfer_ab, NULL);
    pthread_create(&t2, NULL, transfer_ba, NULL);
    pthread_join(t1, NULL);
    pthread_join(t2, NULL);
    printf("moved = %d\n", moved);
    return 0;
}

$ gcc -Wall -Wextra -std=c17 -pthread -o deadlock deadlock.c
$ timeout 5 ./deadlock
$ echo $?
124

The program never printed anything: both threads were stuck, and timeout killed it after 5 seconds (status 124). A deadlock like this doesn’t happen on every run, only when the two threads interleave at the wrong moment. With 100,000 lock pairs per thread, it happened almost immediately here. The standard fix is a global lock order, which Practice 2 applies.

Practice

1. Split the sum 1..1,000,000 across 4 threads, each summing its own slice with no shared writes, and combine the results after joining.

#include <pthread.h>
#include <stdio.h>

#define N_THREADS 4
#define N         1000000

typedef struct { long start, end; long long sum; } Slice;

void *partial_sum(void *arg) {
    Slice *s = arg;
    for (long i = s->start; i < s->end; i++) s->sum += i;
    return NULL;
}

int main(void) {
    pthread_t t[N_THREADS];
    Slice slices[N_THREADS];
    long chunk = N / N_THREADS;
    for (int i = 0; i < N_THREADS; i++) {
        slices[i] = (Slice){i * chunk + 1, (i + 1) * chunk + 1, 0};
        pthread_create(&t[i], NULL, partial_sum, &slices[i]);
    }
    long long total = 0;
    for (int i = 0; i < N_THREADS; i++) {
        pthread_join(t[i], NULL);
        total += slices[i].sum;
    }
    printf("sum 1..%d = %lld (formula: %lld)\n", N, total, (long long)N * (N + 1) / 2);
    return 0;
}

$ gcc -Wall -Wextra -std=c17 -pthread -o p1 p1.c
$ ./p1
sum 1..1000000 = 500000500000 (formula: 500000500000)

2. Fix deadlock.c so it can never deadlock, without removing either lock.

#include <pthread.h>
#include <stdio.h>

static pthread_mutex_t a = PTHREAD_MUTEX_INITIALIZER, b = PTHREAD_MUTEX_INITIALIZER;
static int moved;

void *transfer_ab(void *arg) {
    (void)arg;
    for (int i = 0; i < 100000; i++) {
        pthread_mutex_lock(&a); pthread_mutex_lock(&b);   /* always a, then b */
        moved++;
        pthread_mutex_unlock(&b); pthread_mutex_unlock(&a);
    }
    return NULL;
}

void *transfer_ba(void *arg) {
    (void)arg;
    for (int i = 0; i < 100000; i++) {
        pthread_mutex_lock(&a); pthread_mutex_lock(&b);   /* same order: no deadlock */
        moved++;
        pthread_mutex_unlock(&b); pthread_mutex_unlock(&a);
    }
    return NULL;
}

int main(void) {
    pthread_t t1, t2;
    pthread_create(&t1, NULL, transfer_ab, NULL);
    pthread_create(&t2, NULL, transfer_ba, NULL);
    pthread_join(t1, NULL);
    pthread_join(t2, NULL);
    printf("moved = %d\n", moved);
    return 0;
}

$ gcc -Wall -Wextra -std=c17 -pthread -o p2 p2.c
$ ./p2
moved = 200000

3. Build race.c with -fsanitize=thread after switching it to the mutex version. What does ThreadSanitizer report?

$ gcc -Wall -Wextra -std=c17 -pthread -g -fsanitize=thread -o mutex_tsan mutex.c
$ ./mutex_tsan
expected 800000, got 800000