# Quant Camp Day 2: data wrangling I ------------------------------------------
#
# This is the file we fill in together during the morning lecture. Save it
# inside your camp project as code/02-code.R and run it chunk by chunk.
#
# Today's pipeline:  raw data -> filter() keeps rows -> mutate() adds columns
#                    -> save it under a name -> the table you plot


# Packages ---------------------------------------------------------------------

library(tidyverse)   # wrangling + plotting
library(gapminder)   # country-year data on health and wealth
library(juanr)       # bot, climate
library(here)        # file paths that work on any computer


# 1. Pipes ---------------------------------------------------------------------

# the pipe |> means "and then": take the data, AND THEN filter it, AND THEN ...
# keyboard shortcut: Cmd/Ctrl + Shift + M

gapminder |>
  filter(continent == "Americas")


# 2. filter(): keep the rows you want ------------------------------------------

# keep rows that meet a rule. Notice the number of rows changing
gapminder
gapminder |> filter(continent == "Americas")

# chain rules: Americas AND life expectancy under 50
gapminder |>
  filter(continent == "Americas") |>
  filter(lifeExp < 50)

# exclude with !=
gapminder |> filter(continent != "Americas")

# a range: two rules in one filter()
gapminder |> filter(pop >= 20000000, pop <= 100000000)

# OR with | : Americas or Europe, then rich
gapminder |>
  filter(continent == "Americas" | continent == "Europe") |>
  filter(gdpPercap > 30000)

# filter() to FIND bad data: a made-up survey with two missing-value codes in age
set.seed(1990)
survey = tibble(
  id = 1:1000,
  age = round(rnorm(1000, 47, 16)),
  party = sample(c("Democrat", "Republican", "Independent"), 1000, replace = TRUE)
) |>
  mutate(age = if_else(age < 18, 18 + (18 - age), age),
         age = pmin(age, 95),
         age = case_when(id %% 97 == 0 ~ -99, id %% 151 == 0 ~ 999, TRUE ~ age))

# nobody is -99 or 999 years old. Write the rule for what should NOT exist:
summary(survey$age)
survey |> filter(age < 18 | age > 100)

# ...then flip it to keep what SHOULD exist, save as a new name, and count
survey_clean = survey |> filter(age >= 18, age <= 100)
nrow(survey)
nrow(survey_clean)

# read before you run: how many rows should come out? which countries?
gapminder |>
  filter(year == 2007) |>
  filter(lifeExp > 80)


# 🚨 Your turn: the (unusual) American voter -----------------------------------

# real voters from the Cooperative Election Study
bot
?bot

# distinct() shows you a variable's categories
bot |> distinct(pid7)

# 1. write a filter() that finds the most unusual voter you can think of.
#    how many rows come back?
# 2. ask the agent to find an even MORE unusual voter. Read its filter():
#    what did it decide "unusual" means?



# 3. Save what you made: objects -----------------------------------------------

# PREVIEW: R shows the result, then throws it away. Nothing is saved
gapminder |> filter(continent == "Americas")

# try to use it on the next line: "object not found"
# gap_americas |> ggplot(aes(x = gdpPercap, y = lifeExp)) + geom_point()

# SAVE AS: give it a name with = . Nothing prints; watch the Environment pane
gap_americas = gapminder |> filter(continent == "Americas")

# to see what you saved, type its name
gap_americas

# now you can use it
ggplot(gap_americas, aes(x = gdpPercap, y = lifeExp)) +
  geom_point()

# the original is untouched
gapminder

# SAVE (overwrite): same name on both sides. Risky with filter(): rows are gone
gapminder = gapminder |> filter(continent == "Americas")
nrow(gapminder)

# put the full dataset back
gapminder = gapminder::gapminder
nrow(gapminder)


# 🚨 Your turn: predict the environment ----------------------------------------

# BEFORE running these five lines, write down: after line 5, which objects
# exist in your Environment, and how many rows does each have?
# then run them one at a time and watch the Environment pane

gapminder |> filter(year == 2007)
gap_07 = gapminder |> filter(year == 2007)
gap_07_asia = gap_07 |> filter(continent == "Asia")
gap_07 = gap_07 |> filter(continent == "Europe")
gap_07_asia

# then ask the agent: "what is the difference between line 1 and line 2?"
# does its answer match yours?



# 4. mutate(): add new columns -------------------------------------------------

# murders in 2019 for the 100 biggest US cities (from Wikipedia)
# the file lives in data/ inside your project
murder_set = read_rds(here("data", "murder_wiki_2019.rds")) |>
  select(state, city, population, murder_total)

murder_set

# counts favor big cities. A RATE puts every city on the same footing:
#   murders / population * 100,000  =  murders per 100,000 residents

# mutate() adds a column. This is a PREVIEW: nothing saved yet
murder_set |>
  mutate(murder_capita = murder_total / population)

# two new columns at once; the second uses the first
murder_set |>
  mutate(per_capita = murder_total / population,
         rate = per_capita * 100000)

# the original is unchanged until you save
names(murder_set)

# SAVE (overwrite) is fine here: we are only ADDING columns, nothing is lost
murder_set = murder_set |>
  mutate(per_capita = murder_total / population,
         rate = per_capita * 100000)

murder_set

# the ten highest rates: not the biggest cities anymore
murder_set |>
  slice_max(rate, n = 10)

# overwriting a COLUMN with itself: the name no longer says what it holds
murder_set |>
  mutate(murder_total = murder_total / population) |>
  select(state, city, murder_total)


# The double-run trap ----------------------------------------------------------

# say I want population in thousands and overwrite the column with itself:
#   murder_set = murder_set |> mutate(population = population / 1000)
# run it ONCE: population in thousands. Run it AGAIN by accident: millions.
# no error, just wrong numbers. This is why a script runs top to bottom from a
# FRESH session: each line runs exactly once.

# Start fresh, every time you think you're done:
#   1. Restart R:  Session > Restart R   (Cmd/Ctrl + Shift + F10)
#   2. Run the script top to bottom      (Cmd/Ctrl + Shift + Enter)
# Once, today: Tools > Global Options > General
#   - uncheck "Restore .RData into workspace at startup"
#   - set "Save workspace to .RData on exit" to Never

# rules of thumb
#   removing rows  (filter)          -> Save As (new name), you may want them back
#   adding columns (mutate)          -> Save (overwrite) is fine
#   replacing a column with itself   -> stop and think; prefer a new column name


# 🚨 Your turn: climate change -------------------------------------------------

climate
?climate

# using climate, looking only at the USA, China, and India. SAVE each
# wrangling step under a name before you plot it
# 1. a grouped time series of co2 over time (color by country)
# 2. the same, per capita (check ?climate: what unit is co2 in?)
# 3. who is to "blame" for climate change? where should efforts focus?
# 4. ask the agent to mark the year China passed the US on the first plot.
#    read the code: how did it find the year? how many objects did it make?
# 5. restart R (Cmd/Ctrl + Shift + F10) and run your script top to bottom.
#    does it still work?
