The framework for data journalism was established by Philip Meyer, a journalist for Knight-Ridder, Harvard Nieman Fellow and professor at UNC-Chapel Hill. In his book <i>The New Precision Journalism</i>, which outlines his ideas, Meyer encourages journalists to treat journalism "as if it were a science" by adopting the scientific method:
- Making observation(s) / formulating a questiom
- Researching the question / Collect, store and retrieve data
- Formulate a hypothesis
- Test the hypothesis, using both qualitative (interviews, documents etc.) and quantitative (data analysis etc.) methods
- Analyze the results and reduce them to the most important findings
- Present them to the audience

This process should be thought of as iterative, rather than sequential.

undefined

undefined

undefined

undefined

undefined

undefined

undefined

from typing import Any

import pandas as pd
import numpy as np
from rapidfuzz import fuzz
from itertools import combinations

formats = [
    '%m/%d/%Y', '%Y-%m-%d', '%B %d, %Y',
    '%d-%b-%y', '%m-%d-%Y', '%Y/%m/%d'
]

for fmt in formats:
    try:
        return pd.to_datetime(date_str, format=fmt)
    except:
        continue

# Fall back to pandas parser
try:
    return pd.to_datetime(date_str)
except:
    return None

elif per_thresh and (data_clean.isna().sum() / len(data_clean) * 100) >= per_thresh:
    return df.dropna(subset=[required_col]).reset_index(drop=True).copy()

else:
    return df

names = df[name_col].dropna().unique()

# Use combinations() to avoid nested loop and duplicate comparisons
dup_names: set[Any] = {
    name
    for name1, name2 in combinations(names, 2)
    if fuzz.ratio(str(name1).lower(), str(name2).lower()) >= threshold
    for name in (name1, name2)
}

df['has_similar_name'] = df[name_col].isin(dup_names)
return df

        # Load messy data — raw_data is a placeholder
        # Be sure to use the right reader for the filetype
        .read_csv('..data/raw/raw_data.csv')

        # DATA TYPE CORRECTIONS
        # Ensure proper types for analysis
        .assign(# Convert to numeric (handling errors)
                amount = lambda x: pd.to_numeric(x['amount'], errors='coerce'),
                
                # Convert to categorical (saves memory, enables ordering)
                status = lambda x: pd.to_Categorical(x['status'])) 
        
        .assign(
                # INCONSISTENT FORMATTING
                # Problem: Names in different formats
                # ie. "SMITH, JOHN" vs "John Smith" vs "smith john"
                name_clean = lambda x: standaridize_name(x['name']),
                
                # DATE INCONSISTENCIES
                # Problem: Dates in multiple formats
                # ie. "01/15/2024", "2024-01-15", "January 15, 2024", "15-Jan-24"
                parse_date = lambda x: parse_date(x['date']),
                
                # OUTLIERS
                # Identify potential data entry errors
                amount_outlier = lambda x: flag_outliers(x['amount']),
                
                )
        
        # Fuzzy duplicates (similar but not identical)
        # Use record linkage or manual review
        .pipe(find_similar_names, name_col='name_clean', threshold=85)

        # MISSING VALUES
        # Strategy depends on context
        # First check missing value patterns
        .pipe(handle_missing, thresh=None, per_thresh=None)

        # DUPLICATES — Find and handle duplicates
        .pipe(handle_duplicates, thresh=None)
        
        .reset_index(drop=True)
        .copy())

undefined

undefined

undefined

undefined

# Add skewness to the describe() output
stats['skewness'] = df[col].skew()

return stats.to_frame(name=col)

undefined

undefined

Args:
    df: Input DataFrame
    col: Column to calculate changes for
    periods: Number of rows to look back (1=previous row, 12=year-over-year for monthly)
"""
return df.assign(
    absolute_change=df[col].diff(periods),
    percent_change=df[col].pct_change(periods) * 100,
    direction=np.sign(df[col].diff(periods)).map({1: 'increased', -1: 'decreased', 0: 'unchanged'})
)

Args:
    amount: Value(s) to adjust
    from_year: Original year(s) of the amount
    to_year: Target year to adjust to
    country: ISO 2-letter country code (default 'US'). US uses BLS data via cpi package,
             others use World Bank CPI data (FP.CPI.TOTL indicator)
"""
if country == 'US':
    # Use cpi package for US (more accurate, from BLS)
    if isinstance(from_year, pd.Series):
        return pd.Series([cpi.inflate(amt, yr, to=to_year) 
                        for amt, yr in zip(amount, from_year)], index=amount.index)
    return cpi.inflate(amount, from_year, to=to_year)
else:
    # Use World Bank data for other countries
    cpi_data = wbdata.get_dataframe(
        {'FP.CPI.TOTL': 'cpi'}, 
        country=country
    )['cpi'].to_dict()
    
    from_cpi = pd.Series(from_year).map(cpi_data) if isinstance(from_year, pd.Series) else cpi_data[from_year]
    to_cpi = cpi_data[to_year]
    return amount * (to_cpi / from_cpi)

undefined

undefined

undefined

undefined

undefined

import plotly.express as px

fig = px.bar(
    data, 
    x=x_val, 
    y=y_val,
    text=desc,
    title=title,
    labels={'category': (x_lab if x_lab else x_val), 'value': (y_lab if y_lab else y_val)}
)

return fig

undefined

import pandas as pd
import datawrapper as dw

# Formatting options
value_label_format=dw.NumberFormat.ONE_DECIMAL,
show_value_labels=True,
value_label_alignment='left',
sort_bars=True,  # sort by value
reverse_order=False,

# Source attribution
source_name='Your Data Source',
source_url='https://example.com',
byline='Your Name',

# Optional: custom base color
base_color='#1d81a2'

undefined

undefined

undefined

undefined

Returns DataFrame with: latitude, longitude, state_fips, county_fips, 
tract, block, is_match, is_exact, returned_address, geocoded_address
"""
# Rename columns to expected format
col_map = {id_col: 'id', address_col: 'address', city_col: 'city'}
if state_col and state_col in df.columns:
    col_map[state_col] = 'state'
if zipcode_col and zipcode_col in df.columns:
    col_map[zipcode_col] = 'zipcode'

renamed_df = df.rename(columns=col_map)
records = renamed_df.to_dict('records')

# Small dataset: geocode directly
if len(records) <= chunk_size:
    results = censusbatchgeocoder.geocode(records)
    return pd.DataFrame(results)

# Large dataset: process in chunks to stay under 10,000 limit
all_results = []
for i in range(0, len(records), chunk_size):
    chunk = records[i:i + chunk_size]
    print(f"Geocoding rows {i:,} to {i + len(chunk):,} of {len(records):,}...")
    
    try:
        results = censusbatchgeocoder.geocode(chunk)
        all_results.extend(results)
    except Exception as e:
        print(f"Error on chunk starting at {i}: {e}")
        for record in chunk:
            all_results.append({**record, 'is_match': 'No_Match', 'latitude': None, 'longitude': None})

return pd.DataFrame(all_results)

undefined

import googlemaps
from typing import Optional

def geocode_address_google(address: str, api_key: str) -> Optional[dict]:
    """
    Geocode address using Google Maps API.
    Requires API key with Geocoding API enabled.
    """
    gmaps = googlemaps.Client(key=api_key)
    result = gmaps.geocode(address)
    
    if result:
        location = result[0]['geometry']['location']
        return {
            'formatted_address': result[0]['formatted_address'],
            'lat': location['lat'],
            'lon': location['lng'],
            'place_id': result[0]['place_id']
        }
    return None

results = []
for address in df[address_col]:
    try:
        result = gmaps.geocode(address)
        if result:
            loc = result[0]['geometry']['location']
            results.append({'lat': loc['lat'], 'lon': loc['lng']})
        else:
            results.append({'lat': None, 'lon': None})
    except Exception:
        results.append({'lat': None, 'lon': None})

return pd.concat([df, pd.DataFrame(results)], axis=1)

undefined

import geopandas as gpd
import pandas as pd
from shapely.geometry import Point

undefined

pip install folium mapclassify matplotlib

import geopandas as gpd

undefined

import geopandas as gpd
from lonboard import viz, Map, ScatterplotLayer, PolygonLayer

undefined

import datawrapper as dw
import pandas as pd

# Map configuration
basemap='us-states',           # Built-in US states boundaries
basemap_key='state',           # Column in your data with location codes
value_column='unemployment_rate',

# Styling
color_palette='YlOrRd',        # Color scheme
legend_title='Unemployment %',

# Attribution
source_name='Bureau of Labor Statistics',
source_url='https://www.bls.gov/',
byline='Your Name'

**Available Datawrapper basemaps include:**
- `us-states`, `us-counties`, `us-congressional-districts`
- `world`, `europe`, `africa`, `asia`
- Country-specific maps (e.g., `germany-states`, `uk-constituencies`)