#---------------------------------------------------------------------------#
# Nom : Webscraper.R                                                        #
# Description : Pseudocode for the obituary webscraper                      #
# Author(s) : Pietro Violo and Nadine Ouellette                             #
# Date : June 20 2025.                                                      #
# Modifications :                                                           #
#---------------------------------------------------------------------------#

options(scipen=999)
rm(list = ls())

#---------------------------------------------------------------------------#
# Package library                                                           #
#---------------------------------------------------------------------------#

library(tidyverse)
library(stringi)
library(rvest)
library(httr)

###############################################################################
# Ethical Note:
# This code originally included a web scraping routine to collect obituary 
# data from a public website. However, in accordance with ethical research 
# practices, the actual scraping targets (URLs and HTML selectors) have been 
# removed from the published version.
#
# Researchers intending to collect similar data are strongly encouraged to:
#   - Seek approval from a Research Ethics Board (REB) or equivalent.
#   - Obtain permission from the data source or verify that scraping is 
#     permitted under the site's terms of service.
#   - Minimize burden on servers (e.g., through respectful time delays).
#
# CSS Selectors:
# To locate the correct CSS selectors for scraping specific page elements
# (e.g., name, text, date, location), researchers can:
#   - Use the "Inspect Element" feature available in all major web browsers 
#     (right-click on a web element and select "Inspect").
#   - Use browser extensions like "SelectorGadget" (available for Chrome) 
#     to easily identify the most specific and relevant CSS selectors for 
#     any given content block.
###############################################################################

# Repeat the following for each given year of the archive
# Initialize empty dataframe to store results
df <- c()

# Define base URL of obituary listings (with pagination structure)
base_url <- "URL_TO_OBITUARY_LISTING_PAGE"

# Define total number of pages to scrape
nbofpages <- NUMBER_OF_PAGES

# Loop through all pages
for (n in 1:nbofpages) {
  
  # Construct full URL for the current page
  full_url <- paste(base_url, n, sep = "")
  
  # Request and parse the HTML content of the current listing page
  page <- read_html(GET(full_url))
  
  # Extract obituary names or titles (e.g., full names of deceased)
  obituary_names <- page %>%
    html_nodes("OBITUARY_NAME_SELECTOR") %>%
    html_text()
  
  # Extract links to individual obituary entries
  obituary_links <- page %>%
    html_nodes("OBITUARY_LINK_SELECTOR") %>%
    html_attr("href")
  
  # Optionally: filter out links that are irrelevant (e.g., not matching pattern)
  # For example, ones that have the word 'obituary' or 'necro' in them,
  # this is depending on the given website and might not be necessary
  obituary_links <- obituary_links[grepl("RELEVANT_LINK_PATTERN", obituary_links)]
  
  # Initialize containers for individual fields
  descriptions <- c()
  post_dates <- c()
  locations <- c()
  funeral_homes <- c()
  
  # Loop through each individual obituary page
  for (link in obituary_links) {
    
    # Access and parse the obituary detail page
    detail_page <- read_html(GET(link))
    
    # Extract obituary description or main body text
    body_text <- detail_page %>%
      html_nodes("DESCRIPTION_SELECTOR") %>%
      html_text() %>%
      paste(collapse = "&")
    
    # Extract posting/publication date
    posted_on <- detail_page %>%
      html_nodes("DATE_SELECTOR") %>%
      html_text() %>%
      paste(collapse = "&")
    
    # Extract funeral home name
    funeral_home <- detail_page %>%
      html_nodes("FUNERAL_HOME_SELECTOR") %>%
      html_text() %>%
      paste(collapse = "&")
    
    # Extract location of death or funeral
    location <- detail_page %>%
      html_nodes("LOCATION_SELECTOR") %>%
      html_text() %>%
      paste(collapse = "&")
    
    # Append extracted values to respective lists
    descriptions <- c(descriptions, body_text)
    post_dates <- c(post_dates, posted_on)
    locations <- c(locations, location)
    funeral_homes <- c(funeral_homes, funeral_home)
  }
  
  # Combine all fields into one data frame row per obituary
  batch_data <- cbind(obituary_names, descriptions, post_dates, locations, funeral_homes)
  
  # Append to the master dataset
  df <- rbind(df, batch_data)
  
  # Optional: display progress
  print(n)
  
  # Respectful delay to avoid overloading the server
  Sys.sleep(2)
}

# Convert to data.table format
df_table <- df %>% as.data.table()

# Assign readable column names
colnames(df_table) <- c("Name", "Description", "Post date", "Location", "Funeral home")



# Then, combine all years together and do Quebec and Ontario separate,
# naming them Quebec_obituaries.RData and Ontario_obituaries.RData, respectively.

