# Quant Camp Day 3: data wrangling II -----------------------------------------
#
# This is the file we fill in together during the morning lecture. Save it
# inside your camp project as code/03-code.R and run it chunk by chunk.
#
# Today's three problems:
#   the values are messy      -> str_detect(), str_replace(), str_remove(), case_when()
#   the data is in two tables -> left_join(), count rows before and after, anti_join()
#   the data is the wrong shape -> long vs. wide


# Packages ---------------------------------------------------------------------

library(tidyverse)   # wrangling + plotting
library(gapminder)   # country-year data on health and wealth
library(juanr)       # bot, parties, elections, vdem, trade, climate
library(here)        # file paths that work on any computer


# 1. Look before you clean: count() --------------------------------------------

# count() shows every value in a column and how often it appears.
# to R, "Very liberal" and "Liberal" are two different words
bot |>
  count(ideo5)


# 2. Working with text: str_*() ------------------------------------------------

# str_detect(VARIABLE, PATTERN): is PATTERN in VARIABLE? TRUE or FALSE
# no two Green parties share a name, so == can't find them. A pattern can
parties |>
  filter(str_detect(party, "Green")) |>
  select(country, party, family)

# a pattern is a GUESS. Check it against a column you trust
parties |>
  filter(str_detect(party, "Green")) |>
  count(family, sort = TRUE)

# six aren't Green parties. Why? "Solidarity (Greenland)"

# str_replace(VARIABLE, OLD, NEW): swap OLD for NEW
gapminder |>
  mutate(country = str_replace(country, "Turkey", "Türkiye")) |>
  filter(country == "Türkiye")

# str_remove(VARIABLE, PATTERN): delete PATTERN
# collapse "Very liberal" into "Liberal" by removing "Very "
bot |>
  mutate(ideo3 = str_remove(ideo5, "Very ")) |>
  count(ideo5, ideo3)

# look closely: "liberal" (lowercase) is not "Liberal". Nothing collapsed.
# fix the capitalization, once for each side, then count again
bot |>
  mutate(ideo3 = str_remove(ideo5, "Very "),
         ideo3 = str_replace(ideo3, "liberal", "Liberal"),
         ideo3 = str_replace(ideo3, "conservative", "Conservative")) |>
  count(ideo3)

# count before, count after. You only know the cleaning worked if you look


# 3. Groups from numbers: case_when() ------------------------------------------

# case_when(CONDITION ~ LABEL) goes inside mutate(). If CONDITION is true,
# the row gets LABEL. Rows that match NO condition get NA

# two vote shares per county. We want one word: who won?
small_elections = elections |>
  select(name, state, hh_income, per_gop_2020, per_dem_2020)

# one condition: the Democratic counties come back NA
small_elections |>
  mutate(who_won = case_when(per_gop_2020 > per_dem_2020 ~ "Republicans"))

# two conditions, separated by a comma
small_elections |>
  mutate(who_won = case_when(
    per_gop_2020 > per_dem_2020 ~ "Republicans",
    per_dem_2020 > per_gop_2020 ~ "Democrats"))

# save it with a third condition for a tie, then count() the new column
elections_winner = small_elections |>
  mutate(who_won = case_when(
    per_gop_2020 > per_dem_2020 ~ "Republicans",
    per_dem_2020 > per_gop_2020 ~ "Democrats",
    per_gop_2020 == per_dem_2020 ~ "Tie"))

elections_winner |>
  count(who_won)

# plot it
ggplot(elections_winner, aes(x = per_gop_2020, y = per_dem_2020, color = who_won)) +
  geom_point() +
  labs(x = "GOP vote share", y = "Dem vote share", color = "2020 winner")

# three points are off the line. To find them, SORT: make the sum a column,
# then slice_min() grabs the smallest values
elections_winner |>
  mutate(total = per_gop_2020 + per_dem_2020) |>
  slice_min(total, n = 3)

# three rows with no name and no state. Yesterday's move: filter() them out


# 🚨 Your turn: three parties --------------------------------------------------

# pid7 has seven kinds of partisan
bot |>
  count(pid7)

# 1. make a new column, party3, with three values: "Democrat", "Republican",
#    and "Independent / other"
# 2. run count(pid7, party3). Did every value land where you wanted?
# 3. ask your agent to do the same for ideo5 (liberal, moderate, conservative).
#    read the case_when() it wrote. Where did it put "Not sure"? If there is a
#    line you don't recognize, ask it what that line does



# 4. Two tables: left_join() ---------------------------------------------------

# vdem has democracy scores by country-year; trade has exports by country-year
short_vdem = vdem |>
  select(country, year, v2x_libdem)

short_trade = trade |>
  select(country, year, exports)

# an IDENTIFIER uniquely picks out a row. Here: country AND year together

# the merge, in words: start with vdem, add an exports column, for each row
# look in trade for the same country and year, copy exports over; no match -> NA
merged_dem_trade = left_join(short_vdem, short_trade, by = c("country", "year"))
merged_dem_trade

# RULE: count rows before and after. A left_join() keeps the row count of the
# first dataset. If the number changes, stop
nrow(short_vdem)
nrow(merged_dem_trade)

# when the key is NOT an identifier: country alone appears in many years, so
# R matches every year to every year. No error. Only the count tells you
bad_merge = left_join(short_vdem, short_trade, by = "country")
nrow(bad_merge)

# with the merged data, plot both variables
ggplot(merged_dem_trade, aes(x = v2x_libdem, y = exports)) +
  geom_point() +
  labs(x = "V-Dem liberal democracy score (0-1)",
       y = "Exports (millions of 2011 USD)")

# row counts miss rows that had no match and got NA. anti_join() returns the
# rows in the first dataset with no match in the second
anti_join(short_vdem, short_trade, by = c("country", "year"))

# the pattern: the years don't line up. vdem goes back much further than trade


# 5. Democracy and emissions ---------------------------------------------------

# yesterday's climate data has CO2; vdem has democracy. Do democracies emit more?
climate_2015 = climate |>
  filter(year == 2015) |>
  select(country, year, population, co2)

vdem_2015 = vdem |>
  filter(year == 2015) |>
  select(country, year, v2x_polyarchy)

dem_co2 = left_join(climate_2015, vdem_2015, by = c("country", "year")) |>
  mutate(co2_pc = co2 * 1e6 / population) # co2 is in million tonnes: tonnes per person

# rows before and after match. Looks fine
nrow(climate_2015)
nrow(dem_co2)

ggplot(dem_co2, aes(x = v2x_polyarchy, y = co2_pc)) +
  geom_point() +
  scale_y_log10() +
  labs(x = "Electoral democracy (V-Dem, 0-1)",
       y = "CO2 per person (tonnes, log scale)")

# looks fine. Is the United States on this plot?


# 🚨 Your turn: who's missing? -------------------------------------------------

# 1. build dem_co2 as above. Confirm the row counts match
# 2. use anti_join() to list the countries in climate_2015 that have NO
#    democracy score. Find two that surprise you
# 3. ask your agent to fix the country names so those two merge, then re-run
#    your anti_join(). Read what it changed

# after the exercise we fix it together. Fill in the blanks:
# climate_2015 = climate_2015 |>
#   mutate(country = str_replace(country, "____", "____"),
#          country = str_replace(country, "____", "____"))
#
# dem_co2 = left_join(____, ____, by = c("country", "year")) |>
#   mutate(co2_pc = co2 * 1e6 / population)
#
# anti_join(climate_2015, vdem_2015, by = c("country", "year"))   # what's left?

# every merge: count rows before and after, then ask what didn't match



# 6. Long and wide -------------------------------------------------------------

# the same data, two shapes. What changes is WHAT A ROW IS

# long: one row per country-year
gap_long = gapminder |>
  select(country, year, lifeExp)
gap_long

# wide: one row per country, one column per year
gap_wide = gap_long |>
  pivot_wider(names_from = year, values_from = lifeExp)
gap_wide

# pivot_longer() goes wide to long. You don't need to memorize the arguments;
# you need to SAY the shape: "make this long, one row per country-year, with a
# year column and a lifeExp column"
gap_wide |>
  pivot_longer(cols = -country, names_to = "year", values_to = "lifeExp")

# then check that the row count makes sense: 142 countries x 12 years = 1,704
nrow(gap_long)
