Introduction to LLM Engineering

What Is an LLM?

A large language model (LLM) is a neural network trained on massive text corpora to predict the next token given a sequence of tokens. Through scale and instruction tuning, these models develop the ability to follow instructions, reason, write code, summarize documents, and hold conversations.

The key models in production today:

• Claude (Anthropic) — claude-opus-4-8, claude-sonnet-4-6, claude-haiku-4-5
• GPT-4o (OpenAI) — strong general capability, multimodal
• Gemini (Google) — very large context windows
• Llama 3 (Meta) — open weights, self-hostable

How LLM APIs Work

Every LLM API follows the same core pattern: send a list of messages, receive a generated response.

Request:
  messages: [
    { role: "system",    content: "You are a helpful assistant." },
    { role: "user",      content: "Explain how a hash map works." },
  ]

Response:
  { role: "assistant",   content: "A hash map stores key-value pairs..." }

The system message sets the model’s persona and constraints. The user/assistant exchange is the conversation history.

Setting Up the Anthropic SDK

pip install anthropic
export ANTHROPIC_API_KEY="sk-ant-..."

import anthropic

client = anthropic.Anthropic()  # reads ANTHROPIC_API_KEY from environment

message = client.messages.create(
    model="claude-sonnet-4-6",
    max_tokens=1024,
    messages=[
        {"role": "user", "content": "What is the capital of France?"}
    ]
)

print(message.content[0].text)   # "The capital of France is Paris."
print(f"Input tokens:  {message.usage.input_tokens}")
print(f"Output tokens: {message.usage.output_tokens}")

System Prompts and Temperature

import anthropic

client = anthropic.Anthropic()

# System prompt sets persona and constraints
response = client.messages.create(
    model="claude-sonnet-4-6",
    max_tokens=2048,
    system="""You are a senior Python engineer at a fintech company.
When reviewing code:
- Be concise and direct
- Focus on security, correctness, and performance
- Suggest specific improvements with code examples
- Flag potential race conditions or data loss scenarios""",
    messages=[
        {"role": "user", "content": """Review this function:

def transfer_funds(from_account, to_account, amount):
    balance = db.get_balance(from_account)
    if balance >= amount:
        db.update_balance(from_account, balance - amount)
        db.update_balance(to_account, db.get_balance(to_account) + amount)
    return True
"""}
    ],
)
print(response.content[0].text)

Multi-Turn Conversations

import anthropic
from typing import List

client = anthropic.Anthropic()

def chat(messages: List[dict], user_input: str) -> tuple[str, List[dict]]:
    """Send a message, return the response and updated history."""
    messages.append({"role": "user", "content": user_input})

    response = client.messages.create(
        model="claude-sonnet-4-6",
        max_tokens=1024,
        system="You are a helpful Python tutor. Keep explanations concise with code examples.",
        messages=messages,
    )

    assistant_reply = response.content[0].text
    messages.append({"role": "assistant", "content": assistant_reply})
    return assistant_reply, messages


# Simulate a conversation
history = []
questions = [
    "What is a decorator in Python?",
    "Show me a real-world example using what you just explained.",
    "How would I unit test the decorator from your example?",
]

for question in questions:
    print(f"\nUser: {question}")
    reply, history = chat(history, question)
    print(f"Assistant: {reply[:200]}...")  # truncate for display

Streaming Responses

import anthropic

client = anthropic.Anthropic()

# Stream: print tokens as they arrive instead of waiting for the full response
with client.messages.stream(
    model="claude-sonnet-4-6",
    max_tokens=1024,
    messages=[
        {"role": "user", "content": "Explain gradient descent in 3 paragraphs."}
    ],
) as stream:
    for text_chunk in stream.text_stream:
        print(text_chunk, end="", flush=True)  # print without newline, flush buffer

final_message = stream.get_final_message()
print(f"\n\nTotal tokens: {final_message.usage.input_tokens + final_message.usage.output_tokens}")

Structured Outputs

import anthropic
import json

client = anthropic.Anthropic()

response = client.messages.create(
    model="claude-sonnet-4-6",
    max_tokens=1024,
    system="""Extract information from the user's text and return ONLY valid JSON 
matching this schema:
{
  "company": string,
  "revenue_usd_millions": number | null,
  "employee_count": number | null,
  "founded_year": number | null,
  "headquarters": string | null
}""",
    messages=[{
        "role": "user",
        "content": """Stripe was founded in 2010 by Patrick and John Collison. 
The company is headquartered in San Francisco and had revenue of approximately 
$14 billion in 2023. They employ around 8,000 people worldwide."""
    }]
)

data = json.loads(response.content[0].text)
print(json.dumps(data, indent=2))
# {
#   "company": "Stripe",
#   "revenue_usd_millions": 14000,
#   "employee_count": 8000,
#   "founded_year": 2010,
#   "headquarters": "San Francisco"
# }

Token Counting and Cost Estimation

import anthropic

client = anthropic.Anthropic()

messages = [{"role": "user", "content": "Summarize the history of machine learning."}]

# Count tokens before sending (no API cost for counting)
token_count = client.messages.count_tokens(
    model="claude-sonnet-4-6",
    messages=messages,
)
print(f"Input tokens: {token_count.input_tokens}")

# Claude Sonnet 4.6 pricing (verify current prices at anthropic.com)
INPUT_COST_PER_MILLION  = 3.00   # $3 per 1M input tokens
OUTPUT_COST_PER_MILLION = 15.00  # $15 per 1M output tokens

estimated_input_cost = token_count.input_tokens * INPUT_COST_PER_MILLION / 1_000_000
print(f"Estimated input cost: ${estimated_input_cost:.6f}")

Common Patterns

import anthropic
from functools import lru_cache

client = anthropic.Anthropic()

def llm_call(
    prompt: str,
    system: str = "You are a helpful assistant.",
    model: str = "claude-haiku-4-5-20251001",  # fast/cheap for simple tasks
    max_tokens: int = 512,
) -> str:
    """Minimal wrapper for one-shot LLM calls."""
    response = client.messages.create(
        model=model,
        max_tokens=max_tokens,
        system=system,
        messages=[{"role": "user", "content": prompt}],
    )
    return response.content[0].text


# Classification
def classify_sentiment(text: str) -> str:
    return llm_call(
        prompt=f"Classify the sentiment as POSITIVE, NEGATIVE, or NEUTRAL: {text}",
        system="Reply with only one word: POSITIVE, NEGATIVE, or NEUTRAL.",
    ).strip()


# Summarization
def summarize(text: str, max_words: int = 50) -> str:
    return llm_call(
        prompt=f"Summarize in under {max_words} words:\n\n{text}",
    )


# Entity extraction
def extract_emails(text: str) -> list[str]:
    import json
    result = llm_call(
        prompt=f"Extract all email addresses from this text as a JSON array: {text}",
        system="Return only a valid JSON array of strings, nothing else.",
    )
    return json.loads(result)


print(classify_sentiment("The product arrived on time and works perfectly!"))
print(extract_emails("Contact alice@company.com or bob@example.org for help."))

Practice

These exercises run offline and cost nothing. Wherever the lesson calls client.messages.create, the solutions call create() from the file below instead. It’s a local stand-in: Qwen2.5-0.5B-Instruct, a small open model run on your CPU with Hugging Face transformers (pip install torch transformers; the model downloads on first use). It takes the same inputs (system prompt, message list, max_tokens, temperature) and returns the reply plus input and output token counts, the numbers message.usage gives you. Its answers are a 0.5B model’s, not Claude’s, so expect weaker replies. The outputs below are real runs of this model.

# local_llm.py: a local stand-in for client.messages.create (no API key, no cost)
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
from transformers.utils import logging

logging.set_verbosity_error()
logging.disable_progress_bar()
MODEL = "Qwen/Qwen2.5-0.5B-Instruct"
tok = AutoTokenizer.from_pretrained(MODEL)
model = AutoModelForCausalLM.from_pretrained(MODEL)

def create(messages, system=None, max_tokens=256, temperature=0.0, seed=0):
    """Same inputs as messages.create; returns (reply_text, input_tokens, output_tokens)."""
    chat = ([{"role": "system", "content": system}] if system else []) + messages
    ids = tok.apply_chat_template(chat, add_generation_prompt=True, return_tensors="pt", return_dict=True)
    torch.manual_seed(seed)
    sample = temperature > 0
    out = model.generate(**ids, max_new_tokens=max_tokens, do_sample=sample,
                         temperature=temperature if sample else None,
                         top_p=None, top_k=None)
    new = out[0][ids["input_ids"].shape[1]:]
    return tok.decode(new, skip_special_tokens=True).strip(), ids["input_ids"].shape[1], len(new)

1. The first example prints input and output token counts. Ask “What is the capital of France?” with no system prompt, and then with system="Answer in one word.". Compare the input count to the question’s own token count, and print what the model actually read.

from local_llm import create, tok

question = "What is the capital of France?"
for system in [None, "Answer in one word."]:
    text, n_in, n_out = create([{"role": "user", "content": question}], system=system, max_tokens=40)
    print(f"system={system!r}")
    print(f"  question alone: {len(tok.encode(question))} tokens, input: {n_in}, output: {n_out}")
    print(f"  reply: {text}")

print("\nWhat the model actually read when system=None:")
print(tok.apply_chat_template([{"role": "user", "content": question}],
                              add_generation_prompt=True, tokenize=False))

$ python overhead.py
system=None
  question alone: 7 tokens, input: 36, output: 8
  reply: The capital of France is Paris.
system='Answer in one word.'
  question alone: 7 tokens, input: 25, output: 2
  reply: Paris

What the model actually read when system=None:
<|im_start|>system
You are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>
<|im_start|>user
What is the capital of France?<|im_end|>
<|im_start|>assistant

2. In the multi-turn example, print the input and output token counts for each turn instead of the reply. Use max_tokens=120.

from local_llm import create

SYSTEM = "You are a helpful Python tutor. Keep explanations concise with code examples."

def chat(messages, user_input):
    """The lesson's chat(), with create() standing in for client.messages.create."""
    messages.append({"role": "user", "content": user_input})
    reply, n_in, n_out = create(messages, system=SYSTEM, max_tokens=120)
    messages.append({"role": "assistant", "content": reply})
    return reply, messages, n_in, n_out

history = []
for question in [
    "What is a decorator in Python?",
    "Show me a real-world example using what you just explained.",
    "How would I unit test the decorator from your example?",
]:
    reply, history, n_in, n_out = chat(history, question)
    print(f"turn {len(history) // 2}: input={n_in:4d} tokens  output={n_out:3d}  history={len(history)} messages")

$ python history.py
turn 1: input=  34 tokens  output=120  history=2 messages
turn 2: input= 176 tokens  output=120  history=4 messages
turn 3: input= 317 tokens  output=120  history=6 messages

3. The section on system prompts and temperature never actually sets a temperature. Ask for a coffee-shop name at temperature=0.0 with seeds 1 and 2, then at temperature=1.0 with seeds 1, 2 and 3.

from local_llm import create

prompt = [{"role": "user", "content": "Suggest a name for a coffee shop. Reply with the name only."}]
for temperature, seed in [(0.0, 1), (0.0, 2), (1.0, 1), (1.0, 2), (1.0, 3)]:
    text, _, _ = create(prompt, max_tokens=15, temperature=temperature, seed=seed)
    print(f"temperature={temperature} seed={seed}: {text}")

$ python temperature.py
temperature=0.0 seed=1: Coffee Haven
temperature=0.0 seed=2: Coffee Haven
temperature=1.0 seed=1: Coffee Paradise
temperature=1.0 seed=2: Spiced Coffee Roost
temperature=1.0 seed=3: The Coffee Haven

4. Run the structured-outputs example’s system prompt and Stripe text through create(). Parse the reply with json.loads, catching json.JSONDecodeError, and check every value against the source text.

import json
from local_llm import create

SYSTEM = """Extract information from the user's text and return ONLY valid JSON
matching this schema:
{
  "company": string,
  "revenue_usd_millions": number | null,
  "employee_count": number | null,
  "founded_year": number | null,
  "headquarters": string | null
}"""
TEXT = """Stripe was founded in 2010 by Patrick and John Collison.
The company is headquartered in San Francisco and had revenue of approximately
$14 billion in 2023. They employ around 8,000 people worldwide."""

raw, _, _ = create([{"role": "user", "content": TEXT}], system=SYSTEM, max_tokens=150)
print("raw reply:", repr(raw))
try:
    print(json.dumps(json.loads(raw), indent=2))
except json.JSONDecodeError as e:
    print("json.loads failed:", e)

$ python extract.py
raw reply: '{\n  "company": "Stripe",\n  "revenue_usd_millions": 14,\n  "employee_count": 8000,\n  "founded_year": 2010,\n  "headquarters": "San Francisco"\n}'
{
  "company": "Stripe",
  "revenue_usd_millions": 14,
  "employee_count": 8000,
  "founded_year": 2010,
  "headquarters": "San Francisco"
}

5. Estimate what the lesson’s classify_sentiment wrapper costs at 10,000 reviews a day, at the lesson’s $3 / $15 per million tokens. Count tokens locally with tiktoken (pip install tiktoken), and include the system prompt in the input.

import tiktoken

enc = tiktoken.get_encoding("cl100k_base")    # OpenAI's tokenizer: a local estimate, not Claude's
INPUT_COST_PER_MILLION, OUTPUT_COST_PER_MILLION = 3.00, 15.00

system = "Reply with only one word: POSITIVE, NEGATIVE, or NEUTRAL."
review = "The product arrived on time and works perfectly!"
prompt = f"Classify the sentiment as POSITIVE, NEGATIVE, or NEUTRAL: {review}"

in_tokens = len(enc.encode(system)) + len(enc.encode(prompt))
out_tokens = len(enc.encode("POSITIVE"))
calls = 10_000
daily = calls * (in_tokens * INPUT_COST_PER_MILLION + out_tokens * OUTPUT_COST_PER_MILLION) / 1_000_000
print(f"per call: {in_tokens} input + {out_tokens} output tokens")
print(f"system prompt share of input: {len(enc.encode(system)) / in_tokens:.0%}")
print(f"{calls:,} calls/day: ${daily:.2f}/day, ${daily * 30:.2f}/month")

$ python daily_cost.py
per call: 42 input + 2 output tokens
system prompt share of input: 40%
10,000 calls/day: $1.56/day, $46.80/month