.Rhistory

column_data_list[[col_index]] <- column_data
# Searching for missing values
current_missing_value <- sum(is.na(column_data))
# Filling missing_values_list
missing_values_list[[col_index]] <- current_missing_value
# Increment the column index
col_index <- col_index + 1
}
# Adding Y data to the list
y_column_data <- train[["y"]]
column_data_list[["y"]] <- y_column_data
y_missing_values <- sum(is.na(y_column_data))
missing_values_list[["y"]] <- y_missing_values
# Creating a new dataframe for each data type
quali_nom <- metadata[metadata$Variavel.tipo=="Qualitativo nominal" & metadata$Variavel.cod != 'id',]
quali_ord <- metadata[metadata$Variavel.tipo=="Qualitativo ordinal", ]
quant_disc <- metadata[metadata$Variavel.tipo=="Quantitativo discreto", ]
quant_cont <- metadata[metadata$Variavel.tipo=="Quantitativo continua", ]
# Creating a list for each new dataframe
quali_nom_list <- list()
quali_ord_list <- list()
quant_disc_list <- list()
quant_cont_list <- list()
create_list_for_df <- function(df) {
col_index <- 1
# Temporary list
df_list <- list()
while (col_index <= length(col(df[1]))) {
column_name <- df$Variavel.cod[col_index]
# Access the column using the variable name
column_data <- train[[column_name]]
# Increment the list
df_list[[col_index]] <- column_data
# Counter
col_index <- col_index + 1
}
return(df_list)
}
# Populating lists
quali_ord_list <- create_list_for_df(quali_ord)
quali_nom_list <- create_list_for_df(quali_nom)
quant_cont_list <- create_list_for_df(quant_cont)
quant_disc_list <- create_list_for_df(quant_disc)
# Naming list vectors
names(quali_nom_list) <- quali_nom$Variavel.cod
names(quali_ord_list) <- quali_ord$Variavel.cod
names(quant_cont_list) <- quant_cont$Variavel.cod
names(quant_disc_list) <- quant_disc$Variavel.cod
## Summary Statistics:
quant_cont_summary <- list()
for(i in seq(length(quant_cont_list))) {
quant_cont_summary[[i]] <- summary(quant_cont_list[[i]])
}
# Removing y for this analysis
temp_quant_disc_list <- quant_disc_list
temp_quant_disc_list$y <- NULL
cor_quant_cont_y <- sapply(quant_cont_list, function(x) cor(x, train$y))
cor_quant_disc_y <- sapply(temp_quant_disc_list, function(x) cor(x, train$y))
cor_quali_ord_y <- sapply(quali_ord_list, function(x) cor(x, train$y))
cor_quali_nom_y <- sapply(quali_nom_list, function(x) cor(x, train$y))
# Summaries in relation to y:
summary(cor_quali_nom_y)
summary(cor_quali_ord_y)
summary(cor_quant_cont_y)
summary(cor_quant_disc_y)
# Pairs type by type:
# Continuos X Discrete
cor_matrix_quant_cont_disc <- cor(do.call(cbind, quant_cont_list), do.call(cbind, temp_quant_disc_list))
# Continuos X Nominal
cor_matrix_quant_cont_nom <- cor(do.call(cbind, quant_cont_list), do.call(cbind, quali_nom_list))
# Discrete X Nominal
cor_matrix_quant_disc_nom <- cor(do.call(cbind, quant_disc_list), do.call(cbind, quali_nom_list))
# Summaries type by type:
corrplot.mixed(cor_matrix_quant_cont_disc, tl.col="black", tl.pos="lt")
corrplot.mixed(cor_matrix_quant_cont_nom, tl.col="black", tl.pos="lt")
corrplot.mixed(cor_matrix_quant_disc_nom, tl.col="black", tl.pos="lt")
# Pairs in each type:
cor_matrix_quant_cont <- cor(do.call(cbind, quant_cont_list))
cor_matrix_quant_disc <- cor(do.call(cbind, quant_disc_list))
cor_matrix_quali_nom <- cor(do.call(cbind, quali_nom_list))
cor_matrix_quali_ord <- cor(do.call(cbind, quali_ord_list))
# Threshold
threshold <- 0.7
# Plotting
corrplot.mixed(cor_matrix_quant_cont, tl.col="black", tl.pos="lt")
corrplot.mixed(cor_matrix_quant_disc, tl.col="black", tl.pos="lt")
corrplot.mixed(cor_matrix_quali_nom, tl.col="black", tl.pos="lt")
## Utils
column_data_list_length <-length(column_data_list)
# Function to detect outliers using Tukey's fences
detect_outliers <- function(matrix_data) {
outliers <- list()
for (i in 1:column_data_list_length) {
matrix_values <- matrix_data[[i]]
# Calculate the lower and upper fences using Tukey's fences
q1 <- quantile(matrix_values, 0.25)
q3 <- quantile(matrix_values, 0.75)
iqr <- q3 - q1
lower_fence <- q1 - (1.5 * iqr)
upper_fence <- q3 + (1.5 * iqr)
# Identify outliers based on the fences
outliers[[i]] <- matrix_values[matrix_values < lower_fence | matrix_values > upper_fence]
}
return(outliers)
}
library(corrr)
library(corrr)
library(corrr)
library(corrr)
library("FactoMineR")
str(train)
head(train)
## PCA
train_without_id <- train
train_without_id[0]
train_without_id[[0]]
train_without_id
train_without_id['id']
train_without_id['id'] <- NULL
train_without_id['id']
# Normalization
train_normalized <- scale(train_without_id)
train_normalized
# Correlation matrix
corr_matrix <- cor(train_normalized)
ggcorrplot(corr_matrix)
library(ggcorrplot)
ggcorrplot(corr_matrix)
to.dataframe(quant_disc_list)]
to.dataframe(quant_disc_list)
data.frame(quant_cont_list)
# Dataframes:
quant_cont_df <- data.frame(quant_cont_list)
quant_disc_df <- data.frame(quant_disc_list)
quali_nom_df <- data.frame(quali_nom_list)
quali_ord_df <- data.frame(quali_nom_list)
quali_nom_normalized <- scale(quali_nom_df)
quali_ord_normalized <- scale(quali_ord_df)
quant_cont_normalized <- scale(quant_cont_df)
quant_disc_normalized <- scale(quant_disc_df)
corr_quali_nom_matrix <- cor(quali_nom_normalized)
corr_quali_ord_matrix <- cor(quali_ord_normalized)
cor_quant_cont_matrix <- cor(quant_cont_normalized)
cor_quant_disc_matrix <- cor(quant_disc_normalized)
corr_quant_cont_matrix <- cor(quant_cont_normalized)
corr_quant_disc_matrix <- cor(quant_disc_normalized)
ggcorrplot(corr_quali_nom_matrix)
ggcorrplot(corr_quali_ord_matrix)
ggcorrplot(corr_quant_cont_matrix)
ggcorrplot(corr_quant_disc_matrix)
# Applying PCA
data.pca <- princomp(corr_matrix)
summary(data.pca)
data.pca$loadings[, 1:14]
data.pca$loadings[, 1:2]
fviz_eig(data.pca, addlabels=TRUE)
# libraries
library(ggplot2)
library(corrplot)
library(outliers)
library(corrr)
library(ggcorrplot)
library("FactoMineR")
fviz_eig()
library(FactoMineR)
install.packages("FactoMineR")
install.packages("FactoMineR")
# libraries
library(ggplot2)
library(corrplot)
library(outliers)
library(corrr)
library(ggcorrplot)
library(FactoMineR)
# Importing data
metadata=read.csv("metadata.csv")
train=read.csv("train.csv")
# Initialize the column index
col_index<-1
# Initialize an empty list to store column data
column_data_list <- list()
# Missing Values List
missing_values_list <- list()
# Running through train data
while (col_index <= length(col(train))) {
# Create the variable name
column_name <- paste0("var", col_index)
# Access the column using the variable name
column_data <- train[[column_name]]
# Perform operations on the column
column_data_list[[col_index]] <- column_data
# Searching for missing values
current_missing_value <- sum(is.na(column_data))
# Filling missing_values_list
missing_values_list[[col_index]] <- current_missing_value
# Increment the column index
col_index <- col_index + 1
}
# Applying PCA
data.pca <- princomp(corr_matrix)
summary(data.pca)
fviz_eig(data.pca, addlabels=TRUE)
fviz_eig(data.pca, addlabels=TRUE)
library(factoextra)
fviz_eig(data.pca, addlabels=TRUE)
ggcorrplot(corr_matrix)
# PCA: Biplot
fviz_pca_var(data.pca, col.var = "black")
# PCA: Biplot
?fviz_pca_var()
# PCA: Biplot
fviz_pca_biplot(data.pca, col.var = "black")
# PCA: Biplot
fviz_pca_ind(data.pca, col.var = "black")
# Contribution of each variable
fviz_cos2(data.pca, choice = "var", axes = 1:14)
?fviz_cos2()
# Contribution of each variable
fviz_cos2(data.pca, choice = "var", choose.vars=30, axes = 1:14)
# Contribution of each variable
fviz_cos2(data.pca, choice = "var", choose.vars=10, axes = 1:14)
# Contribution of each variable
num_vars = 10
fviz_cos2(data.pca, choose.vars= num_vars, choice = "var", axes = 1:14)
# Contribution of each variable
num_vars = 10
fviz_cos2(data.pca, top = num_vars, choice = "var", axes = 1:14)
# Contribution of each variable
num_vars = 30
fviz_cos2(data.pca, top = num_vars, choice = "var", axes = 1:14)
# Contribution of each variable
num_vars = 25
fviz_cos2(data.pca, top = num_vars, choice = "var", axes = 1:14)
# Contribution of each variable
num_vars = 23 # One third of the total of vars[69/3]
fviz_cos2(data.pca, top = num_vars, choice = "var", axes = 1:14)
# 10 dimensions, since they add up to 87% of relevance
dim = 10
fviz_cos2(data.pca, top = num_vars, choice = "var", axes = 1:dim)
# Biplot & cos2
fviz_pca_var(data.pca, col.var = 'cos2',
gradient.cols = c('blue', 'purple', 'magenta'),
repel = TRUE)
# PCA: Biplot
fviz_pca_var(data.pca, col.var = "black", repel=TRUE)
# PCA: Biplot
fviz_pca_var(data.pca, col.var = "magenta", repel=TRUE)
# PCA: Biplot
fviz_pca_var(data.pca, col.var = "cyan", repel=TRUE)
# Biplot & cos2
fviz_pca_var(data.pca, col.var = 'cos2',
gradient.cols = c('blue', 'purple', 'magenta'),
repel = TRUE,
addEllipses=TRUE, ellipse.level=0.95)
fviz_pca_biplot(data.pca, col.var = "black", repel=TRUE,
addEllipses=TRUE, ellipse.level=0.95)
fviz_pca_ind(data.pca, col.var = "black", repel=TRUE,
addEllipses=TRUE, ellipse.level=0.95)
# Biplot & cos2
fviz_pca_biplot(data.pca, col.var = 'cos2',
gradient.cols = c('blue', 'purple', 'magenta'),
repel = TRUE)
# Biplot & cos2
fviz_pca_biplot(data.pca, col.var = 'cos2',
gradient.cols = c('blue', 'purple', 'magenta'),
repel = TRUE, axes=0:dim)
# Biplot & cos2
fviz_pca_biplot(data.pca, col.var = 'cos2',
gradient.cols = c('blue', 'purple', 'magenta'),
repel = TRUE, axes=c(1,2,3))
# Biplot & cos2
fviz_pca_biplot(data.pca, col.var = 'cos2',
gradient.cols = c('blue', 'purple', 'magenta'),
repel = TRUE, axes=2)
# Biplot & cos2
fviz_pca_var(data.pca, col.var = 'cos2',
gradient.cols = c('blue', 'purple', 'magenta'),
repel = TRUE)
fviz_cos2(data.pca, top = num_vars, choice = "var", axes = 1:2)
# Summaries type by type:
corrplot.mixed(cor_matrix_quant_cont_disc, tl.col="black", tl.pos="lt")
# Pairs in each type:
cor_matrix_quant_cont <- cor(do.call(cbind, quant_cont_list))
# Plotting
corrplot.mixed(cor_matrix_quant_cont, tl.col="black", tl.pos="lt")
## Summaries
summary(quali_nom_list)
## Summaries
summary(train_without_id)
?write.xlsx()
install.packages(xlsx)
install.packages("xlsx")
library(xlsx)
## Summaries
summary_train <- summary(train_without_id)
write.xlsx(summary_train, file = "train_summary.xlsx")
summary_train
library(openxlsx)
## Summaries
summary_train <- summary(train_without_id)
write.xlsx(summary_train, file = "train_summary.xlsx")
## Summaries
sd_train <- sapply(train_without_id, sd)
print(sd_train)
write.xlsx(sd_train, file = "sd_train.xlsx")
typeof(summary_train)
typeof(sd_train)
View(sd_train)
myv <- c(1.5, 2.7, 3.9, 4.2, 5.1)
write.xlsx(myv, file="myv.xlsx")
library(writexl)
write_xlsx(myv, path = "myv.xlsx")
write_xlsx(sd_train, path = "test.xlsx")
write_xlsx(data = list(myv = myv), path = "test.xlsx")
write_xlsx(list(myv = myv), path = "test.xlsx")
write_xlsx(data.frame(myv), path = "test.xlsx")
write_xlsx(data.frame(sd_train), path = "sd_train.xlsx")
summary_plot <- function(data, variables, summary_fun = mean, y_label = "Valor", error_bar = TRUE) {
# Calcula as estatísticas resumidas para cada variável
summary_values <- sapply(data[variables], summary_fun)
# Extrai as estatísticas relevantes (por exemplo, média e desvio padrão)
means <- summary_values["Mean", ]
sd <- summary_values["SD", ]
# Cria o gráfico de barras
barplot(means, ylim = c(min(0, min(means - sd)), max(means + sd) * 1.1),
xlab = "Variáveis", ylab = y_label, main = "Resumo das Variáveis",
col = "steelblue", border = "black", names.arg = variables)
'
# Adiciona barras de erro (opcional)
if (error_bar) {
arrows(x0 = seq_along(variables), y0 = means - sd, y1 = means + sd,
angle = 90, code = 3, length = 0.05)
}'
}
summary_plot(train_without_id)
train_without_id[1:]
cols(train_without_id)
col(train_without_id)
colnames(train_without_id)
summary_plot(train_without_id, colnames(train_without_id), 'mean')
summary_plot(train_without_id, colnames(train_without_id), 'mean')
summary_plot(train_without_id, colnames(train_without_id))
summary_plot <- function(data, variables, y_label = "Value", error_bar = TRUE) {
# Calculate the summary statistics for each variable
means <- sapply(data[, variables], mean)
sd <- sapply(data[, variables], sd)
# Create the bar plot
barplot(means, ylim = c(min(0, min(means - sd)), max(means + sd) * 1.1),
xlab = "Variables", ylab = y_label, main = "Summary of Variables",
col = "steelblue", border = "black", names.arg = variables)
# Add error bars (optional)
if (error_bar) {
arrows(x0 = seq_along(variables), y0 = means - sd, y1 = means + sd,
angle = 90, code = 3, length = 0.05)
}
}
summary_plot(train_without_id, colnames(train_without_id))
summary_plot <- function(data, variables, summary_fun = mean, y_label = "Value", error_bar = TRUE) {
# Calculate the summary statistics for each variable
summary_values <- summary(data[, variables])
# Extract the relevant summary statistics (e.g., mean and standard deviation)
means <- summary_values[, summary_fun]
sd <- summary_values[, "sd"]
# Create the bar plot
barplot(means, ylim = c(min(0, min(means - sd)), max(means + sd) * 1.1),
xlab = "Variables", ylab = y_label, main = "Summary of Variables",
col = "steelblue", border = "black", names.arg = variables)
# Add error bars (optional)
if (error_bar) {
arrows(x0 = seq_along(variables), y0 = means - sd, y1 = means + sd,
angle = 90, code = 3, length = 0.05)
}
}
summary_plot(train_without_id, colnames(train_without_id))
# Create the bar plot
barplot(means, ylim = c(min(0, min(means - sd)), max(means + sd) * 1.1),
xlab = "Variables", ylab = y_label, main = "Summary of Variables",
col = "steelblue", border = "black", names.arg = variables)
summary_plot <- function(data, variables, y_label = "Value", error_bar = TRUE) {
# Calculate the summary statistics for each variable
means <- sapply(data[, variables], mean)
sd <- sapply(data[, variables], sd)
# Create the bar plot
barplot(means, ylim = c(min(0, min(means - sd)), max(means + sd) * 1.1),
xlab = "Variables", ylab = y_label, main = "Summary of Variables",
col = "steelblue", border = "black", names.arg = variables)
# Add error bars (optional)
if (error_bar) {
arrows(x0 = seq_along(variables), y0 = means - sd, y1 = means + sd,
angle = 90, code = 3, length = 0.05)
}
}
summary_plot(train_without_id, colnames(train_without_id))
warnings()
<
summary_plot <- function(data, variables, y_label = "Value", error_bar = TRUE) {
# Calculate the summary statistics for each variable
means <- sapply(data[, variables], mean)
sd <- sapply(data[, variables], sd)
# Create the bar plot
barplot(means, ylim = c(min(0, min(means - sd)), max(means + sd) * 1.1),
xlab = "Variables", ylab = y_label, main = "Summary of Variables",
col = "steelblue", border = "black", names.arg = variables)
# Add error bars (optional)
if (error_bar) {
for (i in seq_along(variables)) {
if (!is.na(sd[i]) && sd[i] != 0) {
arrows(x0 = i, y0 = means[i] - sd[i], y1 = means[i] + sd[i],
angle = 90, code = 3, length = 0.05)
}
}
}
}
summary_plot(train_without_id, colnames(train_without_id))
summary_plot(train_without_id, colnames(train_without_id), mean)
summary_plot(train_without_id, colnames(train_without_id), 'mean')
summary_plot(train_without_id, colnames(train_without_id), 'ean')
summary_plot <- function(data, variables, y_label = "Value", error_bar = TRUE) {
# Calculate the summary statistics for each variable
means <- sapply(data[, variables], mean)
sd <- sapply(data[, variables], sd)
# Create the bar plot
barplot(y_label, ylim = c(min(0, min(means - sd)), max(means + sd) * 1.1),
xlab = "Variables", ylab = y_label, main = "Summary of Variables",
col = "steelblue", border = "black", names.arg = variables)
# Add error bars (optional)
if (error_bar) {
for (i in seq_along(variables)) {
if (!is.na(sd[i]) && sd[i] != 0) {
arrows(x0 = i, y0 = means[i] - sd[i], y1 = means[i] + sd[i],
angle = 90, code = 3, length = 0.05)
}
}
}
}
summary_plot(train_without_id, colnames(train_without_id), 'ean')
summary_plot(train_without_id, colnames(train_without_id), 'means')
summary_plot(train_without_id, colnames(train_without_id), 'means')
summary_plot <- function(data, variables, y_label, error_bar = TRUE) {
# Calculate the summary statistics for each variable
if (y_label == 'mean') {
y <- sapply(data[, variables], mean)
} ifelse(y_label == 'sd') {
summary_plot(train_without_id, colnames(train_without_id), 'mean')
summary_plot <- function(data, variables, y_label, error_bar = TRUE) {
# Calculate the summary statistics for each variable
y <- 'y'
print(y)
if (y_label == 'mean') {
y <- sapply(data[, variables], mean)
} ifelse(y_label == 'sd') {
summary_plot(train_without_id, colnames(train_without_id), 'mean')
summary_plot(train_without_id, colnames(train_without_id), 'mean', TRUE)
summary_plot(train_without_id, colnames(train_without_id))
summary_plot <- function(data, variables, y_label ='mean', error_bar = TRUE) {
# Calculate the summary statistics for each variable
y <- 'y'
print(y)
if (y_label == 'mean') {
y <- sapply(data[, variables], mean)
} ifelse(y_label == 'sd') {
summary_plot(train_without_id, colnames(train_without_id))
ggsummarystats(
train_without_id, x = "dose", y = "len",
ggfunc = ggbarplot, add = c("jitter", "median_iqr"), position = position_dodge(),
color = "supp", palette = "npg"
)
library(ggpubr)
install.packages('ggpubr')
library(ggpubr)
ggsummarystats(
train_without_id, x = "dose", y = "len",
ggfunc = ggbarplot, add = c("jitter", "median_iqr"), position = position_dodge(),
color = "supp", palette = "npg"
)
ggsummarystats(
train_without_id,
ggfunc = ggbarplot, add = c("jitter", "median_iqr"), position = position_dodge(),
color = "supp", palette = "npg"
)
plotsummary(train_without_id, trim = 0, types = c("stripes", "ecdf", "density", "boxplot"),
y.sizes = 4:1, design = "chessboard", main, mycols = "RB")
plotsummary(train_without_id, trim = 0, types = c("stripes", "ecdf", "density", "boxplot"),
y.sizes = 4:1, design = "chessboard", main, mycols = "RB")
library(aplpack)
plotsummary(train_without_id, trim = 0, types = c("stripes", "ecdf", "density", "boxplot"),
y.sizes = 4:1, design = "chessboard", main, mycols = "RB")
plotsummary(train_without_id)
print(rnorm(30))
# Removing y for this analysis
temp_quant_disc_list <- quant_disc_list
temp_quant_disc_list$y <- NULL
cor_quant_cont_y <- sapply(quant_cont_list, function(x) cor(x, train$y))
cor_quant_disc_y <- sapply(temp_quant_disc_list, function(x) cor(x, train$y))
cor_quali_ord_y <- sapply(quali_ord_list, function(x) cor(x, train$y))
cor_quali_nom_y <- sapply(quali_nom_list, function(x) cor(x, train$y))
# Summaries in relation to y:
summary(cor_quali_nom_y)
summary(cor_quali_ord_y)
summary(cor_quant_cont_y)
summary(cor_quant_disc_y)
# Pairs type by type:
# Continuos X Discrete
cor_matrix_quant_cont_disc <- cor(do.call(cbind, quant_cont_list), do.call(cbind, temp_quant_disc_list))
# Continuos X Nominal
cor_matrix_quant_cont_nom <- cor(do.call(cbind, quant_cont_list), do.call(cbind, quali_nom_list))
# Discrete X Nominal
cor_matrix_quant_disc_nom <- cor(do.call(cbind, quant_disc_list), do.call(cbind, quali_nom_list))
# Summaries type by type:
corrplot.mixed(cor_matrix_quant_cont_disc, tl.col="black", tl.pos="lt")
corrplot.mixed(cor_matrix_quant_cont_nom, tl.col="black", tl.pos="lt")
source("~/GitHub/eda-porto-seguro-data-challenge/main.R")