title  (author)
- Highlight on Page 128 | Loc. 1962-68  | Added on Sunday, December 27, 2015, 03:09 PM
<empty line>
highlighted text
==========
title of the next highlighted book (author)
etc.

library(tidyverse)

## Loading tidyverse: ggplot2
## Loading tidyverse: tibble
## Loading tidyverse: tidyr
## Loading tidyverse: readr
## Loading tidyverse: purrr
## Loading tidyverse: dplyr

## Conflicts with tidy packages ----------------------------------------------

## filter(): dplyr, stats
## lag():    dplyr, stats

raw_text <- read_file("data/My Clippings.txt") # read in the text file
per_chunk <- unlist(strsplit(raw_text, "=========="))  # seperate into chunks
per_chunk[4]

## [1] "\r\nThe Clean Coder_ A Code of Conduct For Professional Programmers - Robert C. Martin (Robert C. Martin)\r\n- Highlight on Page 90 | Added on Monday, January 25, 2016, 04:06 PM\r\n\r\nK ATA In martial arts, a kata is a precise set of choreographed movements that simulates one side of a combat. The goal, which is asymptotically approached, is perfection. The artist strives to teach his body to make each movement perfectly and to assemble those movements into fluid enactment. Well-executed kata are beautiful to watch. Beautiful though they are, the purpose of learning a kata is not to perform it on stage. The purpose is to train your mind and body how to react in a particular combat situation. The goal is to make the perfected movements automatic and instinctive so that they are there when you need them. A programming kata is a precise set of choreographed keystrokes and mouse movements that simulates the solving of some programming problem. You aren’t actually solving the problem because you already know the solution. Rather, you are practicing the movements and decisions involved in solving the problem.\r\n"

# This function takes a chunk of character information
# and seperates it into lines. 
seperate_into_lines <- function(chunk){
    result <- stringr::str_split(chunk, "\r\n")
    unlist(result)
}
# result <- seperate_into_lines(per_chunk[100])  # testing if this works 
## you should put this into formal test frameworks such as testhat if you
## build a package. 



# Extract title sentance and remove author
# This function presumes that you already extracted the raw data into
# character chunks.
extract_title <- function(linechunk){
    # search for second line
    titleline <- linechunk[2]
    return <- gsub("\\(.*\\)", "", titleline) # it took me some 
            #time to work this regular expression out.
    stringr::str_trim(return, side = "both") # remove whitespace at ends
}
#extract_title(result) # testcase to see if it works for me.


# Extract the author from chunk, this function looks 
# very much like the one above, it uses the same logic.
extract_author <- function(linechunk){
    # search for second line
    titleline <- linechunk[2] # identical
    author <- stringr::str_extract(titleline, "\\(.*\\)") # extract piece
    return <- gsub("\\(|\\)", "", author)  # 
    stringr::str_trim(return, side = "both")
}
# extract_author(result)

# testset <- per_chunk[1:20]  # You would use this if you want the first 20 pieces.
set.seed(4579)  # if you do random stuff, it is wise to 
# set the seed so that others can reproduce your work.
testset <- per_chunk[base::sample(x =1:length(per_chunk),size = 20)] 
# unfortunately dplyr also has a function called sample. to specify that
# we want the 'normal' one I specify the name of the package followed by
# two ':'. 
testingframe <- data_frame(
    author = character(length = length(testset)),
    title = character(length(testset)))
for( i in seq_along(testset)){
    hold <- testset[i] %>% seperate_into_lines()
    testingframe$author[i] <- hold %>% extract_author()
    testingframe$title <- hold %>% extract_title()
}
testingframe

## # A tibble: 20 × 2
##                                     author
##                                      <chr>
## 1             Andrew Hunt and David Thomas
## 2             Andrew Hunt and David Thomas
## 3                            Alex Reinhart
## 4                              David Price
## 5            City Watch #1 Terry Pratchett
## 6            City Watch #1 Terry Pratchett
## 7                              Mark Manson
## 8                     Kim Stanley Robinson
## 9                     Kim Stanley Robinson
## 10                    Kim Stanley Robinson
## 11         Douglas DeCarlo, James P. Lewis
## 12                             David Price
## 13           City Watch #2 Terry Pratchett
## 14 Kenneth Knoblauch & Laurence T. Maloney
## 15                           Alex Reinhart
## 16            Andrew Hunt and David Thomas
## 17                        Robert C. Martin
## 18            Andrew Hunt and David Thomas
## 19            Andrew Hunt and David Thomas
## 20                             David Price
## # ... with 1 more variables: title <chr>

- Highlight on Page 132 | Loc. 2017-20  | Added on Saturday, August 20, 2016, 09:37 AM

# this function extracts all the pieces
# and subsequent functions will deal with the seperate stuff.
extract_type_location_date <- function(linechunk){
    meta_row <- linechunk[3]
    pieces <- stringr::str_split(meta_row, "\\|") # the literal character, 
    # the '|' has a special meaning in regexp.
    unlist(pieces)
}
# extract_type_location_date(result) # test function

# extract type from combined result.
# Here the use of the pipe `%>%` operator 
# makes the steps clear.
extract_type <- function(pieces){
    pieces[1] %>%  # select the first row
        stringr::str_extract( "- [[:alnum:]]{1,} ") %>% # extract at least one character.
        gsub("-", "", .) %>% # replace - with nothing, removing it
        stringr::str_trim( side = "both") # remove whitespace at both sides
}
# extract_type_location_date(result) %>% 
#     extract_type()


# extract page number by selecting first piece,
# trimming off of whitespace
# selecting a number, at least 1 times, followed by end of line.
extract_pagenumber <- function(pieces){
    pieces[1] %>%
        stringr::str_trim( side = "right") %>% # remove right end
        stringr::str_extract("[0-9]{1,}$") %>% 
        as.numeric()
}
# extract_type_location_date(result) %>%
#     extract_pagenumber()

# Extract locations. Just like above.
extract_locations <- function(pieces){
    pieces[2] %>% 
        stringr::str_trim( side = "both") %>% 
        stringr::str_extract("[0-9]{1,}-[0-9]{1,}$")
}
# extract_type_location_date(result) %>% 
#     extract_locations()

# Extract date and convert to standard time, not US centric.
# I use the strptime from the base package here. The time is 
# US-centric, but structured, so we can use the formatting from strptime.
# For example: %B is Full month name in the current locale
# and %I:%M %p means hours, minutes, am/pm. 
extract_date <- function(pieces){
    pieces[3] %>% 
        stringr::str_trim( side = "both") %>% 
        stringr::str_extract("[A-z]{3,} [0-9]{1,2}, [0-9]{4}, [0-9]{2}:[0-9]{2} [A-Z]{2}") %>% 
        strptime(format = "%B %e, %Y, %I:%M %p") 
}

# Extract the highlight part.
extract_highlights <- function(linechunk){
    linechunk[5]
}
# extract_highlights(result)

finalframe <- data_frame(
    author = character(length = length(testset)),
    title = character(length(testset)),
    location = character(length(testset)),
    pagenr = numeric(length(testset)),
    type = character(length(testset)),
    highlight = character(length(testset))
    )
# loop through all values 
for( i in seq_along(testset)){
    hold <- testset[i] %>% seperate_into_lines()
    finalframe$author[i] <- hold %>% extract_author()
    finalframe$title[i] <- hold %>% extract_title()
    finalframe$location[i] <- hold %>% extract_type_location_date() %>% extract_locations()
    finalframe$pagenr[i] <- hold %>% extract_type_location_date() %>% extract_pagenumber()
    finalframe$type[i] <- hold %>% extract_type_location_date() %>% extract_type()
    finalframe$highlight[i] <- hold %>% extract_highlights()
}
finalframe

## # A tibble: 20 × 6
##                                     author
##                                      <chr>
## 1             Andrew Hunt and David Thomas
## 2             Andrew Hunt and David Thomas
## 3                            Alex Reinhart
## 4                              David Price
## 5            City Watch #1 Terry Pratchett
## 6            City Watch #1 Terry Pratchett
## 7                              Mark Manson
## 8                     Kim Stanley Robinson
## 9                     Kim Stanley Robinson
## 10                    Kim Stanley Robinson
## 11         Douglas DeCarlo, James P. Lewis
## 12                             David Price
## 13           City Watch #2 Terry Pratchett
## 14 Kenneth Knoblauch & Laurence T. Maloney
## 15                           Alex Reinhart
## 16            Andrew Hunt and David Thomas
## 17                        Robert C. Martin
## 18            Andrew Hunt and David Thomas
## 19            Andrew Hunt and David Thomas
## 20                             David Price
## # ... with 5 more variables: title <chr>, location <chr>, pagenr <dbl>,
## #   type <chr>, highlight <chr>