README.Rmd

---
output: github_document
---

<!-- README.md is generated from README.Rmd. Please edit that file -->

```{r setup, include = FALSE}
library(tidyverse)
library(vroom)
library(data.table)

knitr::opts_chunk$set(
  collapse = TRUE,
  comment = "#>"
)
```
# Data ingestion and manipulation 

## Download and unpack data

```{r}
if(!file.exists("flights.csv")) {
  download.file(
    "http://stat-computing.org/dataexpo/2009/2008.csv.bz2", 
    "flights.csv.bz2")
  R.utils::bunzip2(
    "flights.csv.bz2", 
    "flights.csv")
  unlink("flights.csv.bz2", force = TRUE)
  }
```

## Read data

### readr

```{r}
library(readr)

tr <- system.time(
  flights_readr <- read_csv("flights.csv")  
)

tr[[3]]
```

### data.table

```{r}
library(data.table)

tdt <- system.time(
  flights_dt <- fread("flights.csv")  
)

tdt[[3]]
```

### vroom

```{r}
tva <- system.time(
  flights_vroom_altrep <- vroom("flights.csv", altrep_opts = TRUE)
)

tva[[3]]
```

### Results

```{r}
library(tidyverse)

comparison <- tibble(
  readr = tr[[3]],
  `data.table` = tdt[[3]],
  vroom = tva[[3]]
)

comparison 
```

```{r, fig.width = 8, fig.height = 3}
comparison %>%
  gather() %>%
  ggplot(aes(key, value, fill = key)) +
  geom_col() +
  geom_label(aes(label = paste0(round(value), " secs")), fill = "white") +
  coord_flip() +
  labs(title = "File read times", x = "", y = "") +
  theme_minimal() +
  theme(legend.position = "none", axis.text.x = element_blank())
```

## Data manipulation

```{r}
flights_readr %>% 
    group_by(Month) %>% 
    summarise(avg_delay = mean(ArrDelay, na.rm = TRUE))
```

### Transformations

```{r}
mr <- system.time(
  flights_readr %>% 
    group_by(Month) %>% 
    summarise(avg_delay = mean(ArrDelay, na.rm = TRUE))
)
mva <- system.time(
  flights_vroom_altrep %>% 
    group_by(Month) %>% 
    summarise(avg_delay = mean(ArrDelay, na.rm = TRUE))
)
mdt <- system.time(
  flights_dt[!is.na(ArrDelay), .(avg_delay = mean(ArrDelay)), Month]
)
```

### Results

```{r}
comp <- tibble(
  readr = mr[[3]],
  `data.table` = mdt[[3]],
  vroom = mva[[3]]
)

comp
```

```{r, fig.width = 8, fig.height = 3}
comp %>%
  gather() %>%
  ggplot(aes(key, value, fill = key)) +
  geom_col() +
  geom_label(aes(label = paste0(round(value, 2), " secs")), fill = "white") +
  coord_flip() +
  labs(title = "Data manipulation times", x = "", y = "") +
  theme_minimal() +
  theme(legend.position = "none", axis.text.x = element_blank())
```