Default - 0. Preliminary Analysis
Nao Mimoto – Class Web Page
1. Load data
library(ISLR) # install.packages('ISLR', repos='https://cran.case.edu/')
head(Default) # see the data## default student balance income
## 1 No No 729.5265 44361.625
## 2 No Yes 817.1804 12106.135
## 3 No No 1073.5492 31767.139
## 4 No No 529.2506 35704.494
## 5 No No 785.6559 38463.496
## 6 No Yes 919.5885 7491.559## [1] 10000 4?Default # see explanation for variables
# A data frame with 10000 observations on the following 4 variables.
#
# 1 default (factor) A factor with levels No and Yes indicating whether the customer defaulted on their debt
# 2 student (factor) A factor with levels No and Yes indicating whether the customer is a student
# 3 balance (numeric) The average balance that the customer has remaining on their credit card after making their monthly payment
# 4 income (numeric) Income of customer
#
# column "default" is the respoinse variable.
dim(Default)## [1] 10000 4## default student balance income
## 1 No No 729.5265 44361.625
## 2 No Yes 817.1804 12106.135
## 3 No No 1073.5492 31767.139
## 4 No No 529.2506 35704.494
## 5 No No 785.6559 38463.496
## 6 No Yes 919.5885 7491.559## [1] "default" "student" "balance" "income"# Turn the data into tibble
library(tidyverse) # install.packages("tidyverse")
Default <- as_tibble(Default)
Default## # A tibble: 10,000 × 4
## default student balance income
## <fct> <fct> <dbl> <dbl>
## 1 No No 730. 44362.
## 2 No Yes 817. 12106.
## 3 No No 1074. 31767.
## 4 No No 529. 35704.
## 5 No No 786. 38463.
## 6 No Yes 920. 7492.
## 7 No No 826. 24905.
## 8 No Yes 809. 17600.
## 9 No No 1161. 37469.
## 10 No No 0 29275.
## # ℹ 9,990 more rows## [1] "tbl_df" "tbl" "data.frame"## # A tibble: 10,000 × 4
## default student balance income
## <fct> <fct> <dbl> <dbl>
## 1 No No 730. 44362.
## 2 No Yes 817. 12106.
## 3 No No 1074. 31767.
## 4 No No 529. 35704.
## 5 No No 786. 38463.
## 6 No Yes 920. 7492.
## 7 No No 826. 24905.
## 8 No Yes 809. 17600.
## 9 No No 1161. 37469.
## 10 No No 0 29275.
## 11 No Yes 0 21871.
## 12 No Yes 1221. 13269.
## 13 No No 237. 28252.
## 14 No No 607. 44995.
## 15 No No 1113. 23810.
## 16 No No 286. 45042.
## 17 No No 0 50265.
## 18 No Yes 528. 17637.
## 19 No No 486. 61566.
## 20 No No 1095. 26465.
## 21 No No 229. 50500.
## 22 No No 954. 32458.
## 23 No No 1056. 51318.
## 24 No No 642. 30466.
## 25 No No 773. 34353.
## 26 No No 855. 25211.
## 27 No No 643. 41474.
## 28 No No 1455. 32189.
## 29 No No 616. 39376.
## 30 No Yes 1120. 16556.
## 31 No No 495. 54385.
## 32 No Yes 449. 15799.
## 33 No Yes 585. 22430.
## 34 No No 914. 46907.
## 35 No Yes 1424. 22634.
## 36 No Yes 1500. 13191.
## 37 No No 692. 47807.
## 38 No No 351. 35087.
## 39 No No 743. 37865.
## 40 No No 653. 39490.
## 41 No No 872. 41788.
## 42 No No 837. 51472.
## 43 No No 1152. 42917.
## 44 No Yes 221. 16873.
## 45 No Yes 1690. 19053.
## 46 No No 409. 54207.
## 47 No No 1239. 50067.
## 48 No No 1228. 37409.
## 49 No No 821. 47747.
## 50 No No 857. 31688.
## 51 No No 564. 42641.
## 52 No Yes 1283. 13121.
## 53 No Yes 1506. 26557.
## 54 No Yes 904. 16882.
## 55 No No 0 49957.
## 56 No Yes 1294. 10464.
## 57 No Yes 1276. 15887.
## 58 No No 1537. 48767.
## 59 No No 1333. 39143.
## 60 No No 492. 33379.
## 61 No No 766. 46478.
## 62 No No 690. 63433.
## 63 No No 0 32482.
## 64 No No 1481. 36866.
## 65 No No 989. 45344.
## 66 No No 1302. 33695.
## 67 No No 1045. 54697.
## 68 No No 0 43658.
## 69 No No 866. 38363.
## 70 No No 691. 48140.
## 71 No No 598. 31578.
## 72 No No 429. 51311.
## 73 No No 1398. 39645.
## 74 No Yes 1578. 19886.
## 75 No No 857. 26656.
## 76 No Yes 752. 16211.
## 77 No Yes 775. 15194.
## 78 No No 728. 45132.
## 79 No No 77.0 28392.
## 80 No No 196. 41347.
## 81 No Yes 949. 14298.
## 82 No No 432. 39370.
## 83 No No 462. 46221.
## 84 No No 572. 33046.
## 85 No No 336. 51258.
## 86 No No 510. 60398.
## 87 No Yes 1005. 24039.
## 88 No Yes 162. 13242.
## 89 No No 933. 50537.
## 90 No Yes 893. 11906.
## 91 No No 1333. 44495.
## 92 No No 1404. 35043.
## 93 No Yes 1149. 16578.
## 94 No No 368. 57597.
## 95 No No 449. 45951.
## 96 No No 820. 51585.
## 97 No Yes 620. 15751.
## 98 No No 1048. 46417.
## 99 No No 244. 47194.
## 100 No No 187. 45431.
## # ℹ 9,900 more rows##
## No Yes
## 9667 333##
## No Yes
## 7056 2944Default2 <- Default %>%
rename(resp=default) %>% # Rename "default" column as "resp"
relocate(resp) # move "resp" columnm to 1st
Default2## # A tibble: 10,000 × 4
## resp student balance income
## <fct> <fct> <dbl> <dbl>
## 1 No No 730. 44362.
## 2 No Yes 817. 12106.
## 3 No No 1074. 31767.
## 4 No No 529. 35704.
## 5 No No 786. 38463.
## 6 No Yes 920. 7492.
## 7 No No 826. 24905.
## 8 No Yes 809. 17600.
## 9 No No 1161. 37469.
## 10 No No 0 29275.
## # ℹ 9,990 more rows2. Preliminary Plots
## # A tibble: 10,000 × 4
## default student balance income
## <fct> <fct> <dbl> <dbl>
## 1 No No 730. 44362.
## 2 No Yes 817. 12106.
## 3 No No 1074. 31767.
## 4 No No 529. 35704.
## 5 No No 786. 38463.
## 6 No Yes 920. 7492.
## 7 No No 826. 24905.
## 8 No Yes 809. 17600.
## 9 No No 1161. 37469.
## 10 No No 0 29275.
## # ℹ 9,990 more rows
plot(balance, income, col=ifelse((default=="Yes"), "red", "blue"),
pch=ifelse((default=="Yes"), 4, 3))
lines(balance[default=="Yes"], income[default=="Yes"], col="red",
pch=4, type="p")
3. Routine Exploratory Analysis (class of resp should be “dbl”)
## resp student balance income
## No :9667 No :7056 Min. : 0.0 Min. : 772
## Yes: 333 Yes:2944 1st Qu.: 481.7 1st Qu.:21340
## Median : 823.6 Median :34553
## Mean : 835.4 Mean :33517
## 3rd Qu.:1166.3 3rd Qu.:43808
## Max. :2654.3 Max. :73554## [1] 0## [1] 10000 4## [1] 10000 4 ##----------
##- Correlation Check
# Pick columns that are numeric
library(corrplot) # install.packages("corrplot")
Orig_num <- Orig %>% select_if(is.numeric)
cor(Orig_num)## balance income
## balance 1.0000000 -0.1522434
## income -0.1522434 1.0000000
##----------
##- Visualization
# pairs(Orig) equivalent
library(GGally) # install.packages("GGally")
GGally::ggpairs(Orig[, c(2:4, 1)], aes(color=resp, alpha=1))
###----------
###- Chi-sq test of association for each column
# Pick columns that has more than 1 unique value
list_cols <- Orig %>% summarise_all(function(x) length(unique(x))) %>%
gather() %>% filter(value>1) %>% pull(key)
# Apply chisq.test to those columns
ChiSq.pval <- Orig %>% select(list_cols) %>%
summarise_all(funs(chisq.test(., Orig$resp)$p.value))## Warning: Using an external vector in selections was deprecated in tidyselect 1.1.0.
## ℹ Please use `all_of()` or `any_of()` instead.
## # Was:
## data %>% select(list_cols)
##
## # Now:
## data %>% select(all_of(list_cols))
##
## See <https://tidyselect.r-lib.org/reference/faq-external-vector.html>.
## This warning is displayed once per session.
## Call `lifecycle::last_lifecycle_warnings()` to see where this warning was
## generated.## Warning: `funs()` was deprecated in dplyr 0.8.0.
## ℹ Please use a list of either functions or lambdas:
##
## # Simple named list: list(mean = mean, median = median)
##
## # Auto named with `tibble::lst()`: tibble::lst(mean, median)
##
## # Using lambdas list(~ mean(., trim = .2), ~ median(., na.rm = TRUE))
## Call `lifecycle::last_lifecycle_warnings()` to see where this warning was
## generated.## Warning: There were 2 warnings in `summarise()`.
## The first warning was:
## ℹ In argument: `balance = chisq.test(balance, Orig$resp)$p.value`.
## Caused by warning in `chisq.test()`:
## ! Chi-squared approximation may be incorrect
## ℹ Run `dplyr::last_dplyr_warnings()` to see the 1 remaining warning.## Warning: `funs()` was deprecated in dplyr 0.8.0.
## ℹ Please use a list of either functions or lambdas:
##
## # Simple named list: list(mean = mean, median = median)
##
## # Auto named with `tibble::lst()`: tibble::lst(mean, median)
##
## # Using lambdas list(~ mean(., trim = .2), ~ median(., na.rm = TRUE))
## There were 2 warnings in `summarise()`. The first warning was: ℹ In argument:
## `balance = chisq.test(balance, Orig$resp)$p.value`. Caused by warning in
## `chisq.test()`: ! Chi-squared approximation may be incorrect ℹ Run
## `dplyr::last_dplyr_warnings()` to see the 1 remaining warning.
## Call `lifecycle::last_lifecycle_warnings()` to see where this warning was
## generated.## # A tibble: 1 × 4
## resp student balance income
## <dbl> <dbl> <dbl> <dbl>
## 1 0 0.000500 0.000183 0.495
## [1] 1 2 3## [1] "resp" "student" "balance"## integer(0)## character(0)COCE ONLY
##--------------------------------------
## 1. Load data
library(ISLR) # install.packages('ISLR', repos='https://cran.case.edu/')
head(Default) # see the data
dim(Default)
?Default # see explanation for variables
# A data frame with 10000 observations on the following 4 variables.
#
# 1 default (factor) A factor with levels No and Yes indicating whether the customer defaulted on their debt
# 2 student (factor) A factor with levels No and Yes indicating whether the customer is a student
# 3 balance (numeric) The average balance that the customer has remaining on their credit card after making their monthly payment
# 4 income (numeric) Income of customer
#
# column "default" is the respoinse variable.
dim(Default)
head(Default) # see only first few rows
names(Default) # list of column names
# Turn the data into tibble
library(tidyverse) # install.packages("tidyverse")
Default <- as_tibble(Default)
Default
class(Default)
print(Default, n=100) # if you want to see more rows
# table of columns
table(Default$default) # Note that this is No/Yes instead of YES/no
table(Default$student)
Default2 <- Default %>%
rename(resp=default) %>% # Rename "default" column as "resp"
relocate(resp) # move "resp" columnm to 1st
Default2
##----------------------------------------
## 2. Preliminary Plots
Default
attach(Default)
# Back to Default data (not Default2)
plot(balance, default)
plot(default, balance, xlab="Default", ylab="Balance")
plot(income, default)
plot(default, income, xlab="Default", ylab="Income")
plot(student, default, ylab="Default", xlab="Student")
plot(default, student, xlab="Default", ylab="Student")
plot(balance, income, col=ifelse((default=="Yes"), "red", "blue"),
pch=ifelse((default=="Yes"), 4, 3))
lines(balance[default=="Yes"], income[default=="Yes"], col="red",
pch=4, type="p")
##----------------------------------------
## 3. Routine Exploratory Analysis (class of resp should be "dbl")
Orig <- Default2
#- Check for N/A in data. Remove if there's any.
summary(Orig)
sum(is.na(Orig))
# If there is na in the data, run below
dim(Orig)
Orig <- Orig %>% na.omit()
dim(Orig)
##----------
##- Correlation Check
# Pick columns that are numeric
library(corrplot) # install.packages("corrplot")
Orig_num <- Orig %>% select_if(is.numeric)
cor(Orig_num)
corrplot::corrplot(cor(Orig_num))
corrplot::corrplot(cor(Orig_num), method="number")
##----------
##- Visualization
# pairs(Orig) equivalent
library(GGally) # install.packages("GGally")
GGally::ggpairs(Orig[, c(2:4, 1)], aes(color=resp, alpha=1))
###----------
###- Chi-sq test of association for each column
# Pick columns that has more than 1 unique value
list_cols <- Orig %>% summarise_all(function(x) length(unique(x))) %>%
gather() %>% filter(value>1) %>% pull(key)
# Apply chisq.test to those columns
ChiSq.pval <- Orig %>% select(list_cols) %>%
summarise_all(funs(chisq.test(., Orig$resp)$p.value))
ChiSq.pval <- Orig %>% summarise_all(funs(chisq.test(., Orig$resp)$p.value))
ChiSq.pval
barplot(t(t(as.matrix(ChiSq.pval))), las=2, cex.names=1); abline(h=.05, col='red')
which(ChiSq.pval < .05) # Col num of variables w <.05 p-value
list_cols[which(ChiSq.pval < .05)] # Top list of 'important' variables
which(ChiSq.pval > .6) # Col num of variables w >.6 p-value
list_cols[which(ChiSq.pval > .6)] # Top list of 'unimportatnt' variables