SVM - Demo of Kernels
Nao Mimoto - Dept. of Statistics : The University of Akron – Class Web Page
1. Simulate data for SVM with linear kernel
##----------
## Simulate Data
set.seed(1)
size=200
x <- matrix(rnorm(size*2), ncol=2)
y <- c(rep(-1,100) , rep(1,100) )
x[y==1,] <- x[y==1,] + 1
sim.data=data.frame(x=x, y=as.factor(y))
plot(x, col=ifelse(y>0, "red", "blue"))
test = sample(1:200, 100)
sim.data.01 <- sim.data[test,] # Training data set
sim.data.TS <- sim.data[-test,] # Testing data set
# Visualize Training and Test Set
par(mfrow=c(1,2))
plot(sim.data.01[,1:2], col=ifelse(as.numeric(sim.data.01[,3])==1, "red", "blue"), main="Training Set")
plot(sim.data.TS[,1:2], col=ifelse(as.numeric(sim.data.01[,3])==1, "red", "blue"), main="Test Set")
##----------
## a. Analyze Training Set with SVM (cost is 10)
library (e1071) # install.packages("e1071")
svmfit=svm(y~., data=sim.data.01, kernel="linear", cost=10, scale =FALSE )
plot(svmfit, sim.data.01) # gives SVM classification plot
## [1] 1 5 8 12 25 26 27 28 30 31 40 44 45 49 50 59 62 63 70 71 79 80 81 84 89
## [26] 91 93 94 95 96 98 3 4 7 11 15 22 32 34 35 37 38 46 47 48 51 52 55 58 60
## [51] 67 68 75 76 82 83 86 87 88 90 99##
## Call:
## svm(formula = y ~ ., data = sim.data.01, kernel = "linear", cost = 10,
## scale = FALSE)
##
##
## Parameters:
## SVM-Type: C-classification
## SVM-Kernel: linear
## cost: 10
##
## Number of Support Vectors: 61
##
## ( 31 30 )
##
##
## Number of Classes: 2
##
## Levels:
## -1 1##----------
## b. Analyze Training Set with SVM (cost is 0.1)
library (e1071) # install.packages("e1071")
svmfit =svm(y~., data=sim.data.01, kernel="linear", cost=0.1, scale =FALSE )
plot(svmfit, sim.data.01)
## [1] 1 5 8 12 14 25 26 27 28 29 30 31 33 40 44 45 49 50 59
## [20] 62 63 70 71 72 74 79 80 81 84 89 91 93 94 95 96 98 3 4
## [39] 7 10 11 15 16 22 32 34 35 37 38 41 42 46 47 48 51 52 55
## [58] 58 60 67 68 75 76 82 83 86 87 88 90 99 100##----------
## c. Pick best Cost with 5-fold CV
library (e1071) # install.packages("e1071")
set.seed(1)
tune.out = tune(svm, y~., data=sim.data.01, kernel ="linear",
ranges=list(cost=c(0.001, 0.01, 0.1, 1, 5, 10, 100)),
tunecontrol = tune.control(cross = 5))
summary(tune.out)##
## Parameter tuning of 'svm':
##
## - sampling method: 5-fold cross validation
##
## - best parameters:
## cost
## 0.1
##
## - best performance: 0.24
##
## - Detailed performance results:
## cost error dispersion
## 1 1e-03 0.50 0.1695582
## 2 1e-02 0.34 0.1193734
## 3 1e-01 0.24 0.1294218
## 4 1e+00 0.24 0.1341641
## 5 5e+00 0.24 0.1341641
## 6 1e+01 0.25 0.1541104
## 7 1e+02 0.26 0.1474788##
## Call:
## best.tune(METHOD = svm, train.x = y ~ ., data = sim.data.01, ranges = list(cost = c(0.001,
## 0.01, 0.1, 1, 5, 10, 100)), tunecontrol = tune.control(cross = 5),
## kernel = "linear")
##
##
## Parameters:
## SVM-Type: C-classification
## SVM-Kernel: linear
## cost: 0.1
##
## Number of Support Vectors: 71
##
## ( 36 35 )
##
##
## Number of Classes: 2
##
## Levels:
## -1 1plot(bestmod, sim.data.01)
##----------
## Best Model
svmfit = svm(y~., data=sim.data.01, kernel="linear", cost=100)
plot(svmfit, sim.data.01) # Training Fit
Y.pred=predict(bestmod, sim.data.TS[,1:2]) # predict using bestmod
table(predict=Y.pred , truth=sim.data.TS$y )## truth
## predict -1 1
## -1 40 11
## 1 10 392. Simulate data for SVM with radial kernel
##----------
## Simulate Data
set.seed (1)
x = matrix(rnorm (200*2) , ncol=2)
x[1:100 ,] = x[1:100 ,] + 2
x[101:150 ,]= x[101:150 ,] - 2
y = c(rep (1 ,150) ,rep (2 ,50))-1
sim.data2 = data.frame(x=x, y=as.factor(y))
plot(x, col=ifelse(y>0, "red", "blue"))
set.seed (1)
test = sample(1:200, 100)
sim.data2.01 <- sim.data2[test,] # Training data set
sim.data2.TS <- sim.data2[-test,] # Testing data set
# Visualize Training and Test Set
par(mfrow=c(1,2))
plot(sim.data2.01[,1:2], col=ifelse(as.numeric(sim.data2.01[,3])==1, "red", "blue"), main="Training Set")
plot(sim.data2.TS[,1:2], col=ifelse(as.numeric(sim.data2.01[,3])==1, "red", "blue"), main="Test Set")
##----------
## a. Analyze Training Set with SVM (gamma=1, cost=1)
svmfit = svm(y~., data=sim.data2.01, kernel="radial", gamma=1, cost=1)
plot(svmfit , sim.data2.01)
##
## Call:
## svm(formula = y ~ ., data = sim.data2.01, kernel = "radial", gamma = 1,
## cost = 1)
##
##
## Parameters:
## SVM-Type: C-classification
## SVM-Kernel: radial
## cost: 1
##
## Number of Support Vectors: 41
##
## ( 22 19 )
##
##
## Number of Classes: 2
##
## Levels:
## 0 1##----------
## b. Analyze Training Set with SVM (gamma=1, cost=100000)
svmfit = svm(y~., data=sim.data2.01, kernel="radial", gamma=1, cost=10000)
plot(svmfit, sim.data2.01)
##----------
## c. Pick best Cost and Gamma with 10-fold CV
tune.out = tune(svm, y~., data=sim.data2.01, kernel ="radial",
ranges = list(cost=c(0.1 ,1 ,10 ,100 ,1000),
gamma=c(0.5,1,2,3,4)),
tunecontrol = tune.control(cross = 5))
summary(tune.out)##
## Parameter tuning of 'svm':
##
## - sampling method: 5-fold cross validation
##
## - best parameters:
## cost gamma
## 1 0.5
##
## - best performance: 0.1
##
## - Detailed performance results:
## cost gamma error dispersion
## 1 1e-01 0.5 0.29 0.06519202
## 2 1e+00 0.5 0.10 0.07071068
## 3 1e+01 0.5 0.12 0.10954451
## 4 1e+02 0.5 0.16 0.11937336
## 5 1e+03 0.5 0.19 0.10839742
## 6 1e-01 1.0 0.30 0.09354143
## 7 1e+00 1.0 0.11 0.08944272
## 8 1e+01 1.0 0.14 0.12942179
## 9 1e+02 1.0 0.18 0.11510864
## 10 1e+03 1.0 0.19 0.08215838
## 11 1e-01 2.0 0.31 0.08215838
## 12 1e+00 2.0 0.12 0.10954451
## 13 1e+01 2.0 0.17 0.10954451
## 14 1e+02 2.0 0.20 0.09354143
## 15 1e+03 2.0 0.22 0.09082951
## 16 1e-01 3.0 0.29 0.06519202
## 17 1e+00 3.0 0.13 0.13038405
## 18 1e+01 3.0 0.17 0.10954451
## 19 1e+02 3.0 0.22 0.12041595
## 20 1e+03 3.0 0.23 0.11510864
## 21 1e-01 4.0 0.29 0.06519202
## 22 1e+00 4.0 0.14 0.11937336
## 23 1e+01 4.0 0.19 0.12449900
## 24 1e+02 4.0 0.24 0.10246951
## 25 1e+03 4.0 0.25 0.11726039##
## Call:
## best.tune(METHOD = svm, train.x = y ~ ., data = sim.data2.01, ranges = list(cost = c(0.1,
## 1, 10, 100, 1000), gamma = c(0.5, 1, 2, 3, 4)), tunecontrol = tune.control(cross = 5),
## kernel = "radial")
##
##
## Parameters:
## SVM-Type: C-classification
## SVM-Kernel: radial
## cost: 1
##
## Number of Support Vectors: 41## [1] 0.5## Calculate Test CM from the best model
table(true=sim.data2.TS[, "y"],
pred=predict(tune.out$best.model, sim.data2.TS[, 1:2]))## pred
## true 0 1
## 0 72 7
## 1 1 20##----------
## Best Model
svmfit = svm(y~., data=sim.data2.01, kernel="radial", gamma=1, cost=100)
plot(svmfit, sim.data2.01) # Training Fit
CODE ONLY
##-------------------------------------------------
## 1. Simulate data for SVM with linear kernel
##----------
## Simulate Data
set.seed(1)
size=200
x <- matrix(rnorm(size*2), ncol=2)
y <- c(rep(-1,100) , rep(1,100) )
x[y==1,] <- x[y==1,] + 1
sim.data=data.frame(x=x, y=as.factor(y))
plot(x, col=ifelse(y>0, "red", "blue"))
test = sample(1:200, 100)
sim.data.01 <- sim.data[test,] # Training data set
sim.data.TS <- sim.data[-test,] # Testing data set
# Visualize Training and Test Set
par(mfrow=c(1,2))
plot(sim.data.01[,1:2], col=ifelse(as.numeric(sim.data.01[,3])==1, "red", "blue"), main="Training Set")
plot(sim.data.TS[,1:2], col=ifelse(as.numeric(sim.data.01[,3])==1, "red", "blue"), main="Test Set")
##----------
## a. Analyze Training Set with SVM (cost is 10)
library (e1071) # install.packages("e1071")
svmfit=svm(y~., data=sim.data.01, kernel="linear", cost=10, scale =FALSE )
plot(svmfit, sim.data.01) # gives SVM classification plot
svmfit$index # gives ix of support vectors
summary(svmfit)
##----------
## b. Analyze Training Set with SVM (cost is 0.1)
library (e1071) # install.packages("e1071")
svmfit =svm(y~., data=sim.data.01, kernel="linear", cost=0.1, scale =FALSE )
plot(svmfit, sim.data.01)
svmfit$index # support vectors are different now
##----------
## c. Pick best Cost with 5-fold CV
library (e1071) # install.packages("e1071")
set.seed(1)
tune.out = tune(svm, y~., data=sim.data.01, kernel ="linear",
ranges=list(cost=c(0.001, 0.01, 0.1, 1, 5, 10, 100)),
tunecontrol = tune.control(cross = 5))
summary(tune.out)
## Extract the model with best cost
bestmod <- tune.out$best.model
summary(bestmod)
plot(bestmod, sim.data.01)
##----------
## d. Apply Best model to Test Set
Y.pred=predict(bestmod, sim.data.TS[,1:2]) # predict using bestmod
table(predict=Y.pred , truth=sim.data.TS$y )
##-------------------------------------------------
## 2. Simulate data for SVM with radial kernel
##----------
## Simulate Data
set.seed (1)
x = matrix(rnorm (200*2) , ncol=2)
x[1:100 ,] = x[1:100 ,] + 2
x[101:150 ,]= x[101:150 ,] - 2
y = c(rep (1 ,150) ,rep (2 ,50))-1
sim.data2 = data.frame(x=x, y=as.factor(y))
plot(x, col=ifelse(y>0, "red", "blue"))
set.seed (1)
test = sample(1:200, 100)
sim.data2.01 <- sim.data2[test,] # Training data set
sim.data2.TS <- sim.data2[-test,] # Testing data set
# Visualize Training and Test Set
par(mfrow=c(1,2))
plot(sim.data2.01[,1:2], col=ifelse(as.numeric(sim.data2.01[,3])==1, "red", "blue"), main="Training Set")
plot(sim.data2.TS[,1:2], col=ifelse(as.numeric(sim.data2.01[,3])==1, "red", "blue"), main="Test Set")
##----------
## a. Analyze Training Set with SVM (gamma=1, cost=1)
svmfit = svm(y~., data=sim.data2.01, kernel="radial", gamma=1, cost=1)
plot(svmfit , sim.data2.01)
summary(svmfit)
##----------
## b. Analyze Training Set with SVM (gamma=1, cost=100000)
svmfit = svm(y~., data=sim.data2.01, kernel="radial", gamma=1, cost=10000)
plot(svmfit, sim.data2.01)
##----------
## c. Pick best Cost and Gamma with 10-fold CV
tune.out = tune(svm, y~., data=sim.data2.01, kernel ="radial",
ranges = list(cost=c(0.1 ,1 ,10 ,100 ,1000),
gamma=c(0.5,1,2,3,4)),
tunecontrol = tune.control(cross = 5))
summary(tune.out)
tune.out$best.model
tune.out$best.model$gamma
## Calculate Test CM from the best model
table(true=sim.data2.TS[, "y"],
pred=predict(tune.out$best.model, sim.data2.TS[, 1:2]))
##----------
## Best Model
svmfit = svm(y~., data=sim.data2.01, kernel="radial", gamma=1, cost=100)
plot(svmfit, sim.data2.01) # Training Fit
plot(svmfit, sim.data2.TS) # Test Fit