Python Performance

Measure First: timeit

Never optimize without measuring. timeit runs code thousands of times to give stable results.

import timeit

# Quick benchmark in code
result = timeit.timeit(
    stmt="sum(range(1000))",
    number=10_000
)
print(f"{result:.3f}s for 10,000 iterations")

# Compare two approaches
setup = "data = list(range(10_000))"

t1 = timeit.timeit("sum(data)", setup=setup, number=1000)
t2 = timeit.timeit("total = 0\nfor x in data:\n    total += x", setup=setup, number=1000)

print(f"sum(): {t1:.4f}s")
print(f"loop: {t2:.4f}s")

In IPython or Jupyter, use the magic:

%timeit sum(range(1_000_000))
# 13.1 ms ± 91.3 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)

cProfile: Finding Bottlenecks

import cProfile
import pstats

def slow_function():
    total = 0
    for i in range(100_000):
        total += sum(range(i % 100))
    return total

# Profile to file
cProfile.run("slow_function()", "profile_output")

# Analyze results
stats = pstats.Stats("profile_output")
stats.sort_stats("cumulative")
stats.print_stats(10)   # top 10 functions by cumulative time

Output looks like:

   ncalls  tottime  percall  cumtime  percall filename:lineno(function)
   100000    0.523    0.000    0.523    0.000 {built-in method builtins.sum}

line_profiler for Line-by-Line

pip install line_profiler

from line_profiler import LineProfiler

def process_items(items):
    results = []
    for item in items:
        cleaned = item.strip().lower()          # line 1
        words = cleaned.split()                 # line 2
        results.append(" ".join(sorted(words))) # line 3
    return results

profiler = LineProfiler()
profiler.add_function(process_items)
profiler.enable()
process_items(["hello world", "foo bar"] * 10_000)
profiler.disable()
profiler.print_stats()

lru_cache: Memoization

Cache expensive function results. The most impactful single-line optimization in Python.

from functools import lru_cache

# Without cache: O(2^n)
def fib(n):
    if n < 2:
        return n
    return fib(n-1) + fib(n-2)

# With cache: O(n)
@lru_cache(maxsize=None)
def fib_cached(n):
    if n < 2:
        return n
    return fib_cached(n-1) + fib_cached(n-2)

import timeit
print(timeit.timeit("fib(30)", globals=globals(), number=1))         # ~0.2s
print(timeit.timeit("fib_cached(30)", globals=globals(), number=1))  # ~0.00001s

# Inspect cache
fib_cached.cache_info()   # CacheInfo(hits=28, misses=31, maxsize=None, currsize=31)
fib_cached.cache_clear()  # invalidate

cache (Python 3.9+)

from functools import cache   # equivalent to lru_cache(maxsize=None)

@cache
def expensive_lookup(key: str) -> dict:
    return database.fetch(key)

NumPy Vectorization

Replace Python loops with NumPy operations — C speed with Python syntax.

import numpy as np
import timeit

data = list(range(1_000_000))
arr = np.array(data)

# Python loop
def python_sum_squares(lst):
    return sum(x**2 for x in lst)

# NumPy
def numpy_sum_squares(arr):
    return np.sum(arr ** 2)

t1 = timeit.timeit(lambda: python_sum_squares(data), number=10)
t2 = timeit.timeit(lambda: numpy_sum_squares(arr), number=10)

print(f"Python: {t1:.3f}s")   # ~2.0s
print(f"NumPy:  {t2:.3f}s")   # ~0.03s — ~60x faster

Avoiding NumPy Anti-Patterns

# Slow — Python loop over NumPy array
total = 0
for x in arr:
    total += x       # calls Python __add__ on each element

# Fast — vectorized
total = arr.sum()

# Slow — element-wise Python condition
result = np.array([x**2 if x > 0 else 0 for x in arr])

# Fast — NumPy where
result = np.where(arr > 0, arr**2, 0)

String Concatenation

import timeit

words = ["hello"] * 10_000

# Slow — O(n^2) due to immutable string copies
def concat_plus(words):
    result = ""
    for w in words:
        result += w
    return result

# Fast — O(n)
def concat_join(words):
    return "".join(words)

t1 = timeit.timeit(lambda: concat_plus(words), number=100)
t2 = timeit.timeit(lambda: concat_join(words), number=100)
print(f"Plus: {t1:.3f}s, Join: {t2:.4f}s")  # join is ~100x faster

List vs Generator

import sys

# List — all items in memory at once
squares_list = [x**2 for x in range(1_000_000)]
print(sys.getsizeof(squares_list))   # ~8.5 MB

# Generator — lazy, one item at a time
squares_gen = (x**2 for x in range(1_000_000))
print(sys.getsizeof(squares_gen))    # ~120 bytes

# If you only iterate once, use a generator
total = sum(x**2 for x in range(1_000_000))

Dict and Set Lookups

import timeit

data = list(range(100_000))
data_set = set(data)
data_dict = {x: True for x in data}

target = 99_999

# O(n) — scans the whole list
t1 = timeit.timeit(lambda: target in data, number=1000)

# O(1) — hash lookup
t2 = timeit.timeit(lambda: target in data_set, number=1000)

print(f"List: {t1:.4f}s")   # ~5s
print(f"Set:  {t2:.6f}s")   # ~0.0001s

Slots for Object-Heavy Code

import timeit

class SlowPoint:
    def __init__(self, x, y, z):
        self.x = x; self.y = y; self.z = z

class FastPoint:
    __slots__ = ("x", "y", "z")
    def __init__(self, x, y, z):
        self.x = x; self.y = y; self.z = z

t1 = timeit.timeit(lambda: SlowPoint(1, 2, 3), number=1_000_000)
t2 = timeit.timeit(lambda: FastPoint(1, 2, 3), number=1_000_000)
print(f"Without slots: {t1:.3f}s")
print(f"With slots:    {t2:.3f}s")   # typically 20-30% faster

Practical Optimization Checklist

1. Profile first — find the actual bottleneck
2. Replace Python loops with NumPy/Pandas operations
3. Add @lru_cache / @cache to pure functions called repeatedly
4. Use set for membership testing instead of list
5. Use "".join() instead of string + concatenation
6. Use generators instead of list comprehensions when you iterate once
7. Add __slots__ if creating millions of small objects
8. Consider multiprocessing for CPU-bound work
9. Consider asyncio for high-concurrency I/O
10. Profile again to confirm the improvement