TensorFlow RNNs and LSTMs

Real-World Scenario

A news platform wants to classify articles into topics (Politics, Technology, Sports, etc.) from headline text. An LSTM processes each headline word by word, building a representation that captures context — “Apple releases new iPhone” is Technology, not Food. With 50,000 labeled headlines, bidirectional LSTM achieves 91% accuracy.

Sequence Preprocessing

import numpy as np
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences

# Sample text classification data
texts = [
    "president signs new climate bill into law",
    "tech giant releases latest smartphone model",
    "local team wins championship in overtime",
    "stock markets reach record high on earnings",
    "scientists discover new exoplanet in habitable zone",
    "quarterback throws for three hundred yards in victory",
    "central bank raises interest rates by half point",
    "researchers develop faster machine learning algorithm",
]
labels = [0, 1, 2, 3, 4, 2, 3, 1]  # 0=Politics, 1=Tech, 2=Sports, 3=Finance, 4=Science

# Tokenize
tokenizer = Tokenizer(num_words=5000, oov_token="<OOV>")
tokenizer.fit_on_texts(texts)
sequences = tokenizer.texts_to_sequences(texts)

# Pad to uniform length
MAX_LEN = 20
X = pad_sequences(sequences, maxlen=MAX_LEN, padding="post", truncating="post")
y = np.array(labels)

VOCAB_SIZE = len(tokenizer.word_index) + 1
print(f"Vocabulary size: {VOCAB_SIZE}")
print(f"X shape: {X.shape}")   # (8, 20)

LSTM Text Classifier

import tensorflow as tf
from tensorflow import keras
import numpy as np

VOCAB_SIZE  = 5000
MAX_LEN     = 20
EMBED_DIM   = 64
LSTM_UNITS  = 128
N_CLASSES   = 5

# Bidirectional LSTM — reads the sequence in both directions
model = keras.Sequential([
    keras.layers.Embedding(VOCAB_SIZE, EMBED_DIM, mask_zero=True,
                           input_length=MAX_LEN),
    keras.layers.SpatialDropout1D(0.2),      # drops entire feature maps

    # Stack 2 LSTM layers — first must return sequences
    keras.layers.Bidirectional(
        keras.layers.LSTM(LSTM_UNITS, return_sequences=True, dropout=0.2)
    ),
    keras.layers.Bidirectional(
        keras.layers.LSTM(LSTM_UNITS // 2, dropout=0.2)  # last layer: no return_sequences
    ),

    keras.layers.Dense(64, activation="relu"),
    keras.layers.Dropout(0.3),
    keras.layers.Dense(N_CLASSES, activation="softmax"),
])

model.compile(
    optimizer=keras.optimizers.Adam(1e-3),
    loss="sparse_categorical_crossentropy",
    metrics=["accuracy"],
)
model.summary()

# Training with dummy data
X_train = np.random.randint(0, VOCAB_SIZE, (800, MAX_LEN))
y_train = np.random.randint(0, N_CLASSES, 800)
X_val   = np.random.randint(0, VOCAB_SIZE, (200, MAX_LEN))
y_val   = np.random.randint(0, N_CLASSES, 200)

history = model.fit(
    X_train, y_train,
    validation_data=(X_val, y_val),
    epochs=20,
    batch_size=32,
    callbacks=[
        keras.callbacks.EarlyStopping(patience=4, restore_best_weights=True),
        keras.callbacks.ReduceLROnPlateau(factor=0.5, patience=2, verbose=1),
    ],
    verbose=1,
)

Time-Series Forecasting with LSTM

import numpy as np
import tensorflow as tf
from tensorflow import keras

# Generate synthetic multivariate time series
# Features: temperature, humidity, pressure → predict temperature
rng  = np.random.default_rng(42)
N    = 2000

temperature = 20 + 10 * np.sin(np.arange(N) * 2 * np.pi / 365) + rng.normal(0, 1, N)
humidity    = 60 + 20 * np.cos(np.arange(N) * 2 * np.pi / 365) + rng.normal(0, 2, N)
pressure    = 1013 + rng.normal(0, 5, N)

data = np.stack([temperature, humidity, pressure], axis=1)  # (N, 3)

# Normalize
mean, std = data.mean(0), data.std(0)
data = (data - mean) / std

LOOKBACK   = 30   # days of history
HORIZON    = 1    # days to predict

def make_sequences(data: np.ndarray, lookback: int, horizon: int):
    X, y = [], []
    for i in range(len(data) - lookback - horizon + 1):
        X.append(data[i:i + lookback])              # (lookback, features)
        y.append(data[i + lookback, 0])             # predict temperature only
    return np.array(X), np.array(y)

X, y = make_sequences(data, LOOKBACK, HORIZON)
split = int(len(X) * 0.8)
X_train, X_val = X[:split], X[split:]
y_train, y_val = y[:split], y[split:]

# Functional API — cleaner for multi-input/output architectures
inputs = keras.Input(shape=(LOOKBACK, 3))
x = keras.layers.LSTM(64, return_sequences=True)(inputs)
x = keras.layers.Dropout(0.2)(x)
x = keras.layers.LSTM(32)(x)
x = keras.layers.Dense(16, activation="relu")(x)
outputs = keras.layers.Dense(1)(x)

model = keras.Model(inputs, outputs)
model.compile(
    optimizer=keras.optimizers.Adam(1e-3),
    loss="mse",
    metrics=["mae"],
)

history = model.fit(
    X_train, y_train,
    validation_data=(X_val, y_val),
    epochs=50,
    batch_size=64,
    callbacks=[
        keras.callbacks.EarlyStopping(patience=8, restore_best_weights=True),
        keras.callbacks.ReduceLROnPlateau(factor=0.5, patience=4),
    ],
    verbose=1,
)

# Evaluate
val_loss, val_mae = model.evaluate(X_val, y_val, verbose=0)
# Denormalize MAE back to original scale
real_mae = val_mae * std[0]
print(f"Validation MAE: {real_mae:.2f}°C")

Character-Level Text Generation with GRU

import tensorflow as tf
from tensorflow import keras
import numpy as np

TEXT = """
To be or not to be that is the question whether tis nobler in the mind to suffer
the slings and arrows of outrageous fortune or to take arms against a sea of troubles
and by opposing end them to die to sleep no more and by a sleep to say we end
""" * 10  # repeat for enough training data

chars       = sorted(set(TEXT))
char2idx    = {c: i for i, c in enumerate(chars)}
idx2char    = {i: c for c, i in char2idx.items()}
VOCAB_SIZE  = len(chars)

# Create input/target pairs
SEQ_LEN = 40
X_chars, y_chars = [], []
for i in range(0, len(TEXT) - SEQ_LEN - 1, 3):
    X_chars.append([char2idx[c] for c in TEXT[i:i + SEQ_LEN]])
    y_chars.append(char2idx[TEXT[i + SEQ_LEN]])

X_data = np.array(X_chars)
y_data = np.array(y_chars)

# GRU-based language model
model = keras.Sequential([
    keras.layers.Embedding(VOCAB_SIZE, 32, input_length=SEQ_LEN),
    keras.layers.GRU(128, return_sequences=True),
    keras.layers.GRU(64),
    keras.layers.Dense(VOCAB_SIZE, activation="softmax"),
])

model.compile(
    optimizer=keras.optimizers.Adam(1e-3),
    loss="sparse_categorical_crossentropy",
)

model.fit(X_data, y_data, epochs=30, batch_size=128, verbose=1)


def generate_text(model, seed: str, length: int = 200, temperature: float = 0.8) -> str:
    """Generate text by sampling from the model's output distribution."""
    generated = seed
    sequence  = [char2idx.get(c, 0) for c in seed[-SEQ_LEN:]]

    for _ in range(length):
        x = np.array([sequence[-SEQ_LEN:]])
        probs = model.predict(x, verbose=0)[0]

        # Temperature scaling: higher = more random, lower = more focused
        probs = np.log(probs + 1e-10) / temperature
        probs = np.exp(probs) / np.exp(probs).sum()

        next_idx = np.random.choice(len(probs), p=probs)
        generated += idx2char[next_idx]
        sequence.append(next_idx)

    return generated


print(generate_text(model, seed="to be or", length=200, temperature=0.7))