robotstxt::get_robotstxt("https://www.theguardian.com")

## # this is the robots.txt file for theguardian.com
## 
## User-agent: *
## Disallow: /sendarticle/
## Disallow: /Users/
## Disallow: /users/
## Disallow: /*/print$
## Disallow: /email/
## Disallow: /contactus/
## Disallow: /share/
## Disallow: /websearch
## Disallow: /*?commentpage=
## Disallow: /whsmiths/
## Disallow: /external/overture/
## Disallow: /discussion/report-abuse/*
## Disallow: /discussion/report-abuse-ajax/*
## Disallow: /discussion/comment-permalink/*
## Disallow: /discussion/report-abuse/*
## Disallow: /discussion/user-report-abuse/*
## Disallow: /discussion/handlers/*
## Disallow: /discussion/your-profile
## Disallow: /discussion/your-comments
## Disallow: /discussion/edit-profile
## Disallow: /discussion/search/comments
## Disallow: /discussion/*
## Disallow: /search
## Disallow: /music/artist/*
## Disallow: /music/album/*
## Disallow: /books/data/*
## Disallow: /settings/
## Disallow: /embed/
## Disallow: /*styles/js-on.css$
## Disallow: /sport/olympics/2008/events/*
## Disallow: /sport/olympics/2008/medals/*
## Disallow: /f/healthcheck
## Disallow: /sections
## Disallow: /top-stories
## Disallow: /most-read/sport
## Disallow: /articles
## Disallow: /podcasts
## Disallow: /global$
## Disallow: /*/feedarticle/*
## Disallow: /travel/2013/aug/22/been-there-readers-competition?*
## Disallow: /preference/*
## Disallow: /59666047/
## Disallow: /print/
## Disallow: /info/tech-feedback
## Disallow: /production-monitoring/
## 
## User-agent: Mediapartners-Google
## Disallow:
## 
## Sitemap: http://www.theguardian.com/sitemaps/news.xml
## Sitemap: http://www.theguardian.com/sitemaps/video.xml
## 
## User-agent: bingbot
## Crawl-delay: 1

robotstxt::paths_allowed("https://www.theguardian.com/lifeandstyle/series/experience")

## [1] TRUE

library("rvest")

xtract_titles <- function(node) {
  css <- 'span[class="js-headline-text"]'
  html_nodes(node, css) %>% html_text(trim = TRUE)
}


get_titles_from_page <- function(page_number){
  Sys.sleep(2)
  link <- paste0("https://www.theguardian.com/lifeandstyle/series/experience?page=", page_number)
  page_content <- read_html(link)
  xtract_titles(page_content)
}

experience_titles <- purrr::map(1:29, get_titles_from_page) %>% unlist()
save(experience_titles, file = "data/2017-10-02-guardian-experience.RData")

set.seed(1)
sample(experience_titles, 10)

##  [1] "Experience: pregnancy sickness nearly killed me"            
##  [2] "Experience: I was a sperm donor for my friends"             
##  [3] "Experience: I was attacked in my front garden"              
##  [4] "I was brought up in the exclusive brethren"                 
##  [5] "Experience: I am Dancing Man"                               
##  [6] "The boy who missed the mainstream"                          
##  [7] "I still can't explain what I saw"                           
##  [8] "Experience: My twin rewrote my childhood"                   
##  [9] "Experience: I've renewed my wedding vows more than 50 times"
## [10] "Experience: I talk with my eyes"

experience_titles <- stringr::str_replace(experience_titles, "^Experience: ", "")

library("tidytext")
library("rcorpora")

stopwords <- corpora("words/stopwords/en")$stopWords

words <- tibble::tibble(title = experience_titles) %>%
  unnest_tokens(word, title) %>%
  dplyr::filter(!word %in% stopwords) %>%
  dplyr::count(word, sort = TRUE) 

knitr::kable(words[1:20,])

afinn <- get_sentiments("afinn")

sentiment <- tibble::tibble(title = experience_titles) %>%
  dplyr::mutate(saved_title = title) %>%
  unnest_tokens(word, title) %>%
  dplyr::inner_join(afinn) %>%
  dplyr::group_by(saved_title) %>%
  dplyr::summarize(sentiment = sum(score)) %>%
  dplyr::filter(!is.na(sentiment))

knitr::kable(sentiment[1:10,])

library("ggplot2")
library("hrbrthemes")

ggplot(sentiment) +
  geom_bar(aes(sentiment)) +
  theme_ipsum_rc()

topics <- monkeylearn::monkeylearn_classify(experience_titles,
                                            classifier_id = "cl_5icAVzKR")

titles <- tibble::tibble(title = experience_titles,
                         text_md5 = purrr::map_chr(experience_titles, digest::digest, algo = "md5"))

titles <- dplyr::inner_join(titles, topics, by = "text_md5")

titles <- dplyr::filter(titles, probability > 0.5)
set.seed(1)
dplyr::sample_n(titles, size = 20) %>% 
  dplyr::select(title, label, probability) %>%
  knitr::kable()

titles %>%
  dplyr::group_by(label) %>%
  dplyr::summarise(n = n(),
                   some = toString(title[1:3])) %>%
  dplyr::arrange(dplyr::desc(n)) %>%
  head(n = 10) %>%
  knitr::kable()