Top 10 Most Starred Packages

Top 10 Most Downloaded Packages with Stars

Top 10 Packages by Stars per Download (frequently starred)

Bottom 10 Packages by Stars per Download (infrequently starred)

Full Script

Below and available via gist with data at: https://gist.github.com/StevenMMortimer/1b4b626d3d91240a77f969ae04b37114

# load packages & custom functions ---------------------------------------------

library(tidyverse)
library(httr)
library(cranlogs)
library(XML)
library(ggrepel)
library(scales)
library(knitr)
library(stringr)

gh_from_url <- function(x){
  if(!grepl(',', x)){
    x <- strsplit(x, " ")[[1]]
    x <- trimws(x[min(which(grepl(pattern='http://github.com|https://github.com|http://www.github.com', x, ignore.case=TRUE)))])    
  } else {
    x <- strsplit(x, ",")[[1]]
    x <- trimws(x[min(which(grepl(pattern='http://github.com|https://github.com|http://www.github.com', x, ignore.case=TRUE)))])    
  }
  x <- gsub("http://", "https://", tolower(x))
  x <- gsub("www\\.github\\.com", "github.com", x)
  x <- gsub("/$", "", x)
  x <- gsub("^github.com", "https://github.com", x)
  x <- gsub("/issues", "", x)
  x <- gsub("\\.git", "", x)
  return(x)
}

aut_maintainer_from_details <- function(x){
  x <- gsub("'|\"", "", x)
  if(grepl(',', x)){
    x <- strsplit(x, "\\],")[[1]]
    aut_cre_ind <- grepl(pattern='\\[aut, cre|\\[cre, aut|\\[cre', x, ignore.case=TRUE)
    if(any(aut_cre_ind)){
      x <- x[min(which(aut_cre_ind))]
      x <- gsub("\\[aut, cre|\\[cre, aut|\\[cre", "", x)
    }
    x <- strsplit(x, ",")[[1]][1]
    x <- trimws(gsub("\\]", "", x))
    x <- trimws(gsub(" \\[aut", "", x))
  }
  return(x)
}

gh_star_count <- function(url){
  stars  <- tryCatch({
    this_url <- gsub("https://github.com/", "https://api.github.com/repos/", url)
    req <- GET(this_url, gtoken)
    stop_for_status(req)
    cont <- content(req)
    cont$stargazers_count  
  }, error = function(e){
    return(NA_integer_)
  })
  return(stars)
}

# authenticate to github -------------------------------------------------------
# use Hadley's key and secret
myapp <- oauth_app("github",
                   key = "56b637a5baffac62cad9",
                   secret = "8e107541ae1791259e9987d544ca568633da2ebf")
github_token <- oauth2.0_token(oauth_endpoints("github"), myapp)
gtoken <- config(token = github_token)

# pull list of packages --------------------------------------------------------

# get list of currently available packages on CRAN
pkgs <- readHTMLTable(readLines(file.path('https://cran.rstudio.com/src/contrib')),
                      which = 1, 
                      stringsAsFactors = FALSE)[,-1]

# filter out lines that aren't really packages
pkgs <- pkgs %>% 
  filter(Size != "-", 
         grepl('tar.gz$', Name)) %>%
  mutate(Name = sub('^([a-zA-Z0-9\\.]*).*', '\\1', Name)) %>%
  distinct(Name, .keep_all = TRUE)

# get details for each package -------------------------------------------------

all_pkg_details <- NULL

# old fashioned looping! 
# WARNING: This takes awhile to complete
for(i in 1:nrow(pkgs)){
  
  if(i %% 100 == 0){
    message(sprintf("Processing package #%s out of %s", i, nrow(pkgs)))
  }
  
  pkg_details <- readHTMLTable(readLines(file.path(sprintf('https://cran.r-project.org/package=%s', pkgs[i,]$Name))), 
                               header=FALSE, 
                               which = 1, 
                               stringsAsFactors = FALSE)
  pkg_details <- pkg_details %>%
    mutate(V1 = gsub(":", "", V1)) %>%
    spread(V1, V2)
  
  this_url <- pkg_details$URL
  on_github <- FALSE
  this_github_url <- NA_character_
  gh_stars <- NA_integer_
  if(!is.null(this_url)){
    on_github <- grepl('http://github.com|https://github.com|http://www.github.com', this_url)
    if(on_github){
      this_github_url <- gh_from_url(this_url)
      gh_stars <- gh_star_count(this_github_url)
    } else {
      # check the BugReports URL as a backup (e.g. shiny package references GitHub this way)
      issues_on_github <- grepl('http://github.com|https://github.com|http://www.github.com', pkg_details$BugReports)
      if(length(issues_on_github) == 0 || !issues_on_github){
        this_github_url <- NA_character_
      } else {
        this_github_url <- gh_from_url(pkg_details$BugReports)
        gh_stars <- gh_star_count(this_github_url)  
        on_github <- TRUE
      }
    }
  } else {
    this_url <- NA_character_
  }
  
  downloads <- cran_downloads(pkgs[i,]$Name, from = "2014-01-01", to = "2018-06-15")
  
  all_pkg_details <- rbind(all_pkg_details, 
                           tibble(name = pkgs[i,]$Name, 
                                  published = pkg_details$Published,
                                  author = aut_maintainer_from_details(pkg_details$Author),
                                  url = this_url,
                                  github_ind = on_github, 
                                  github_url = this_github_url,
                                  downloads = sum(downloads$count),
                                  stars = gh_stars
                           )
  )
}

# basic summary stats ----------------------------------------------------------

# remove observations where the GitHub URL refers to a repository that 
# is not specific to R and therefore might have an inflated star count
all_pkg_details_clean <- all_pkg_details %>% 
  filter(!(name %in% c('xgboost', 'h2o', 'feather'))) %>% 
  mutate(downloads_per_star = downloads / stars, 
         downloads_per_star = ifelse(!is.finite(downloads_per_star), NA_real_, downloads_per_star))

# proportion of all packages listing github
sum(all_pkg_details$github_ind)
mean(all_pkg_details$github_ind)
# proportion of packages with stars
mean(!is.na(all_pkg_details$stars))

# typical number of stars per package
mean(all_pkg_details_clean$stars, na.rm=TRUE)
median(all_pkg_details_clean$stars, na.rm=TRUE)
max(all_pkg_details_clean$stars, na.rm=TRUE)

# typical number of downloads per package
mean(all_pkg_details_clean$downloads, na.rm=TRUE)
median(all_pkg_details_clean$downloads, na.rm=TRUE)

# percent of packages over 10 stars
mean(all_pkg_details_clean$stars > 10, na.rm=TRUE)

mean(all_pkg_details_clean$downloads_per_star, na.rm=TRUE)
median(all_pkg_details_clean$downloads_per_star, na.rm=TRUE)


# stars histogram --------------------------------------------------------------
ggplot(data=all_pkg_details_clean, mapping=aes(stars)) +
  geom_histogram(aes(fill=..count..), bins=60) +
  scale_x_continuous(trans = "log1p", breaks=c(0,1,2,3,10,100,1000,3000)) + 
  labs(x = "Stars", 
       y = "Count", 
       fill = "Count", 
       caption = "Source: api.github.com as of 6/16/18") + 
  ggtitle("Distribution of GitHub Stars on R Packages") + 
  theme_bw() + 
  theme(panel.grid.minor = element_blank(), 
        plot.caption=element_text(hjust = 0))

# stars to downloads scatterplot -----------------------------------------------
plot_dat <- all_pkg_details_clean
idx_label <- which(with(plot_dat, downloads > 10000000 | stars > 1000))
plot_dat$name2 <- plot_dat$name
plot_dat$name <- ""
plot_dat$name[idx_label] <- plot_dat$name2[idx_label]

ggplot(data=plot_dat, aes(stars, downloads, label = name)) +
  geom_point(color = ifelse(plot_dat$name == "", "grey50", "red")) +
  geom_text_repel(box.padding = .5) + 
  scale_y_continuous(labels = comma) + 
  scale_x_continuous(labels = comma) + 
  labs(x = "GitHub Stars", 
       y = "CRAN Downloads", 
       caption = "Sources:\napi.github.com as of 6/16/18\ncranlogs as of 1/1/14 - 6/15/18") + 
  ggtitle("Relationship Between CRAN Downloads and GitHub Stars") + 
  theme_bw() + 
  theme(plot.caption=element_text(hjust = 0))

# author stats -----------------------------------------------------------------

# summary by author
authors_detail <- all_pkg_details_clean %>% 
  group_by(author) %>% 
  summarize(downloads = sum(downloads, na.rm=TRUE), 
            stars = sum(stars, na.rm=TRUE)) %>% 
  mutate(downloads_per_star = downloads / stars, 
         downloads_per_star = ifelse(!is.finite(downloads_per_star), NA_real_, downloads_per_star)) %>% 
  arrange(desc(downloads))

# popular authors
pop_authors <- tibble(author = c('Hadley Wickham', 
                                 'Dirk Eddelbuettel', 
                                 'Yihui Xie', 
                                 'Winston Chang', 
                                 'Jennifer Bryan', 
                                 'JJ Allaire',
                                 'Jeroen Ooms', 
                                 'Scott Chamberlain', 
                                 'Jim Hester', 
                                 'Kirill Müller'), 
                      notable_packages = c('ggplot2, dplyr, httr',
                                           'Rcpp, BH', 
                                           'knitr, rmarkdown, bookdown', 
                                           'R6, shiny', 
                                           'readxl, gapminder, googlesheets', 
                                           'rstudioapi, reticulate, tensorflow', 
                                           'jsonlite, curl, openssl', 
                                           'geojsonio, taxize', 
                                           'devtools, memoise, readr', 
                                           'tibble, DBI')
)
author_stats <- pop_authors %>% 
  inner_join(., authors_detail, by='author') %>%
  select(author, notable_packages, downloads, stars, downloads_per_star) %>% 
  mutate(downloads_per_star = round(downloads_per_star, 1)) %>%
  rename_all(. %>% gsub("_", " ", .) %>% str_to_title)

# single author 
#all_pkg_details_clean %>% filter(author == 'Dirk Eddelbuettel') %>% arrange(desc(downloads))  

# top 10 lists -----------------------------------------------------------------

# Top 10 Most Starred Packages

top_starred <- all_pkg_details_clean %>% 
  select(name, author, downloads, stars, downloads_per_star) %>%
  arrange(desc(stars)) %>%
  slice(1:10) %>% 
  mutate(downloads_per_star = round(downloads_per_star, 1)) %>%
  rename_all(. %>% gsub("_", " ", .) %>% str_to_title)

# Top 10 Most Downloaded Packages with stars

top_downloaded <- all_pkg_details_clean %>% 
  filter(!is.na(stars)) %>%
  select(name, author, downloads, stars, downloads_per_star) %>%
  arrange(desc(downloads)) %>%
  slice(1:10) %>% 
  mutate(downloads_per_star = round(downloads_per_star, 1)) %>%
  rename_all(. %>% gsub("_", " ", .) %>% str_to_title)

# Bottom 10 Packages by Downloads per Star (frequently starred) 

frequently_starred <- all_pkg_details_clean %>% 
  filter(downloads > 100) %>% 
  select(name, author, downloads, stars, downloads_per_star) %>%
  arrange(downloads_per_star) %>%
  slice(1:10) %>% 
  mutate(downloads_per_star = round(downloads_per_star, 2)) %>%
  rename_all(. %>% gsub("_", " ", .) %>% str_to_title)

# Top 10 Packages by Downloads per Star (infrequently starred) 

infrequently_starred <- all_pkg_details_clean %>% 
  select(name, author, downloads, stars, downloads_per_star) %>%
  arrange(desc(downloads_per_star)) %>%
  slice(1:10) %>% 
  rename_all(. %>% gsub("_", " ", .) %>% str_to_title)