Pandas DataFrames and Series

Real-World Scenario

A data analyst at an e-commerce company receives a CSV of 500,000 customer orders. Before any analysis, they need to select relevant columns, filter to completed orders, add computed columns like order value, and fix data quality issues. These are DataFrame operations — the daily workflow of every data practitioner.

Selecting Columns

import pandas as pd

df = pd.DataFrame({
    "order_id":  [1001, 1002, 1003, 1004, 1005],
    "customer":  ["Alice", "Bob", "Carol", "Dave", "Eve"],
    "product":   ["Laptop", "Mouse", "Laptop", "Keyboard", "Monitor"],
    "quantity":  [1, 2, 1, 3, 1],
    "unit_price":[1200.0, 25.0, 1200.0, 45.0, 350.0],
    "status":    ["shipped", "delivered", "pending", "delivered", "shipped"],
})

# Single column — returns a Series
product_col = df["product"]
print(type(product_col))   # <class 'pandas.core.series.Series'>

# Multiple columns — returns a DataFrame (double brackets)
subset = df[["customer", "product", "unit_price"]]
print(subset.head())

# Attribute access — works for columns with valid Python identifiers
# Avoid for column names with spaces or matching a DataFrame method
print(df.status.value_counts())

Row Selection with loc and iloc

import pandas as pd

df = pd.DataFrame({
    "product":   ["Laptop", "Mouse", "Keyboard", "Monitor", "Webcam"],
    "category":  ["Electronics", "Accessories", "Accessories", "Electronics", "Accessories"],
    "price":     [1200.0, 25.0, 45.0, 350.0, 80.0],
    "in_stock":  [True, True, False, True, True],
}, index=["P001", "P002", "P003", "P004", "P005"])  # string index

# loc — label-based: [row_label, col_label]
print(df.loc["P001"])                     # entire row P001 as a Series
print(df.loc["P001", "price"])            # single value: 1200.0
print(df.loc["P001":"P003"])              # rows P001 through P003 inclusive
print(df.loc[["P001", "P004"], ["product", "price"]])  # specific rows & cols

# iloc — position-based: [row_int, col_int]
print(df.iloc[0])                         # first row
print(df.iloc[0, 2])                      # row 0, col 2 (price)
print(df.iloc[:3])                        # first 3 rows
print(df.iloc[:3, [0, 2]])               # first 3 rows, cols 0 and 2

Boolean Filtering

import pandas as pd

df = pd.DataFrame({
    "customer": ["Alice", "Bob", "Carol", "Dave", "Eve", "Frank"],
    "country":  ["US", "UK", "US", "DE", "US", "UK"],
    "revenue":  [4500, 1200, 8900, 3400, 6700, 2100],
    "orders":   [12, 3, 25, 8, 18, 5],
    "tier":     ["Gold", "Bronze", "Platinum", "Silver", "Gold", "Bronze"],
})

# Single condition
us_customers = df[df["country"] == "US"]
print(us_customers)

# Multiple conditions — use & (and), | (or), ~ (not)
high_value_us = df[(df["country"] == "US") & (df["revenue"] > 5000)]
print(high_value_us)

# isin — check membership in a list
english_speaking = df[df["country"].isin(["US", "UK"])]
print(english_speaking)

# String contains
gold_plus = df[df["tier"].isin(["Gold", "Platinum"])]

# query() — readable alternative for complex filters
result = df.query("country == 'US' and revenue > 5000")
print(result)

# query with a variable
min_revenue = 3000
result2 = df.query("revenue > @min_revenue")  # @ references a local variable

Adding and Modifying Columns

import pandas as pd
import numpy as np

df = pd.DataFrame({
    "product":    ["Laptop", "Mouse", "Keyboard", "Monitor"],
    "cost":       [800.0, 15.0, 30.0, 200.0],
    "quantity":   [50, 300, 200, 80],
    "category":   ["Electronics", "Accessories", "Accessories", "Electronics"],
})

# Add a computed column
df["price"] = df["cost"] * 1.5         # 50% markup
df["total_value"] = df["price"] * df["quantity"]

# Conditional column with np.where
df["high_value"] = np.where(df["total_value"] > 5000, True, False)

# Multiple conditions with np.select
conditions = [
    df["total_value"] > 10000,
    df["total_value"] > 5000,
    df["total_value"] > 1000,
]
choices = ["High", "Medium", "Low"]
df["value_tier"] = np.select(conditions, choices, default="Minimal")

# apply — custom function per row or per column
df["price_formatted"] = df["price"].apply(lambda x: f"${x:,.2f}")

# String operations via .str accessor
df["category_upper"] = df["category"].str.upper()
df["category_short"] = df["category"].str[:5]

print(df[["product", "price_formatted", "value_tier", "category_upper"]])

# Drop a column
df.drop(columns=["category_upper"], inplace=True)

# Rename columns
df.rename(columns={"cost": "unit_cost", "quantity": "units_available"}, inplace=True)

Sorting

import pandas as pd

df = pd.DataFrame({
    "name":    ["Alice", "Bob", "Carol", "Dave", "Eve"],
    "dept":    ["Eng", "Eng", "Sales", "Sales", "Eng"],
    "salary":  [95000, 88000, 72000, 76000, 105000],
    "years":   [5, 3, 7, 4, 8],
})

# Sort by a single column
print(df.sort_values("salary", ascending=False))

# Sort by multiple columns — dept ascending, salary descending within dept
print(df.sort_values(["dept", "salary"], ascending=[True, False]))

# Sort by index
df_shuffled = df.sample(frac=1, random_state=42)  # shuffle
print(df_shuffled.sort_index())  # restore original row order

Value Counts and Basic Exploration

import pandas as pd

df = pd.read_csv("orders.csv")  # assume a real dataset

# Value counts — frequency of each unique value
print(df["status"].value_counts())
print(df["status"].value_counts(normalize=True))  # as proportions

# Unique values
print(df["country"].unique())
print(df["country"].nunique())   # count of unique values

# Check for nulls
print(df.isnull().sum())         # null count per column
print(df.isnull().sum() / len(df))  # null rate per column

# Basic statistics
print(df["revenue"].describe())
print(df.describe(include="all"))  # includes string columns too