M09-Cross validating-naive-bayes
0 likes146 views
The document outlines the process of implementing cross-validation with a Naive Bayes model using a heart disease dataset in R. It details the steps for data preparation, model training, and evaluation metrics such as accuracy, sensitivity, and specificity. The results indicate an overall accuracy of 83.52% and highlights the effectiveness of cross-validation in assessing model performance.
![library('e1071')
file<-'c://Users/rk215/Data/heart.csv'
heart<-read.csv(file,head=T,sep=',',stringsAsFactors=F)
head(heart)
## age sex cp trestbps chol fbs restecg thalach exang oldpeak slope ca tha
l
## 1 63 1 3 145 233 1 0 150 0 2.3 0 0
1
## 2 37 1 2 130 250 0 1 187 0 3.5 0 0
2
## 3 41 0 1 130 204 0 0 172 0 1.4 2 0
2
## 4 56 1 1 120 236 0 1 178 0 0.8 2 0
2
## 5 57 0 0 120 354 0 1 163 1 0.6 2 0
2
## 6 57 1 0 140 192 0 1 148 0 0.4 1 0
1
## target
## 1 1
## 2 1
## 3 1
## 4 1
## 5 1
## 6 1
catheart<-heart[,c(2,3,6,7,9,11,12,13,14)]
set.seed(43)
trdidx<-sample(1:nrow(catheart),0.7*nrow(catheart),replace=F)
trcatheart<-catheart[trdidx,]
tstcatheart<-catheart[-trdidx,]
nb.model<-naiveBayes(target~.,data=trcatheart)
#str(nbtr.model)
object.size(nb.model) #11096
## 11096 bytes
nb.pred<-predict(nb.model,tstcatheart[,-c(9)],type='raw')
nb.class<-unlist(apply(round(nb.pred),1,which.max))-1
nb.tbl<-table(tstcatheart[[9]], nb.class)
nb.cfm<-caret::confusionMatrix(nb.tbl)
nb.cfm
Cross-Validation-naiveBayes file:///E:/users/rkannan/cuny/fall2020/fall2020/ML-Handbook/m09-cv-r...
3 of 7 11/23/2020, 5:27 PM](https://image.slidesharecdn.com/cross-validating-naive-bayes-201123222844/85/M09-Cross-validating-naive-bayes-3-320.jpg)
![## Confusion Matrix and Statistics
##
## nb.class
## 0 1
## 0 28 12
## 1 3 48
##
## Accuracy : 0.8352
## 95% CI : (0.7427, 0.9047)
## No Information Rate : 0.6593
## P-Value [Acc > NIR] : 0.0001482
##
## Kappa : 0.6571
##
## Mcnemar's Test P-Value : 0.0388671
##
## Sensitivity : 0.9032
## Specificity : 0.8000
## Pos Pred Value : 0.7000
## Neg Pred Value : 0.9412
## Prevalence : 0.3407
## Detection Rate : 0.3077
## Detection Prevalence : 0.4396
## Balanced Accuracy : 0.8516
##
## 'Positive' Class : 0
##
start_tm <- proc.time()
N<-nrow(trcatheart)
NF=10
folds<-split(1:N,cut(1:N, quantile(1:N, probs = seq(0, 1, by =1/NF))))
length(folds)
## [1] 10
lapply(folds,length)
Cross-Validation-naiveBayes file:///E:/users/rkannan/cuny/fall2020/fall2020/ML-Handbook/m09-cv-r...
4 of 7 11/23/2020, 5:27 PM](https://image.slidesharecdn.com/cross-validating-naive-bayes-201123222844/85/M09-Cross-validating-naive-bayes-4-320.jpg)
![## $`(1,22.1]`
## [1] 21
##
## $`(22.1,43.2]`
## [1] 21
##
## $`(43.2,64.3]`
## [1] 21
##
## $`(64.3,85.4]`
## [1] 21
##
## $`(85.4,106]`
## [1] 21
##
## $`(106,128]`
## [1] 21
##
## $`(128,149]`
## [1] 21
##
## $`(149,170]`
## [1] 21
##
## $`(170,191]`
## [1] 21
##
## $`(191,212]`
## [1] 22
ridx<-sample(1:nrow(trcatheart),nrow(trcatheart),replace=FALSE) # randomize
the data
cv_df<-do.call('rbind',lapply(folds,FUN=function(idx,data=trcatheart[ridx,])
{
m<-naiveBayes(target~.,data=data[-idx,]) # keep one fold for validation
p<-predict(m,data[idx,-c(9)],type='raw') # predict for that test fold
pc<-unlist(apply(round(p),1,which.max))-1
pred_tbl<-table(data[idx,c(9)],pc) #table(actual,predicted)
pred_cfm<-caret::confusionMatrix(pred_tbl)
list(fold=idx,m=m,cfm=pred_cfm) # store the fold, model,cfm
}
)) # lapply repeats over all folds
Cross-Validation-naiveBayes file:///E:/users/rkannan/cuny/fall2020/fall2020/ML-Handbook/m09-cv-r...
5 of 7 11/23/2020, 5:27 PM](https://image.slidesharecdn.com/cross-validating-naive-bayes-201123222844/85/M09-Cross-validating-naive-bayes-5-320.jpg)
![cv_df<-as.data.frame(cv_df)
tstcv.perf<-as.data.frame(do.call('rbind',lapply(cv_df$cfm,FUN=function(cfm)
c(cfm$overall,cfm$byClass))))
(cv.tst.perf<-apply(tstcv.perf[tstcv.perf$AccuracyPValue<0.01,-c(6:7)],2,mea
n))
## Accuracy Kappa AccuracyLower
## 0.8683983 0.7318000 0.6545460
## AccuracyUpper AccuracyNull Sensitivity
## 0.9700452 0.5666667 0.8373377
## Specificity Pos Pred Value Neg Pred Value
## 0.8900699 0.8924825 0.8629060
## Precision Recall F1
## 0.8924825 0.8373377 0.8583395
## Prevalence Detection Rate Detection Prevalence
## 0.4523810 0.3766234 0.4324675
## Balanced Accuracy
## 0.8637038
(cv.tst.perf.var<-apply(tstcv.perf[tstcv.perf$AccuracyPValue<0.01,-c(6:7)],
2,sd))
## Accuracy Kappa AccuracyLower
## 0.06018717 0.11323819 0.07464027
## AccuracyUpper AccuracyNull Sensitivity
## 0.02365967 0.07221786 0.06971453
## Specificity Pos Pred Value Neg Pred Value
## 0.12234605 0.10504433 0.07414154
## Precision Recall F1
## 0.10504433 0.06971453 0.04174328
## Prevalence Detection Rate Detection Prevalence
## 0.08908708 0.06878895 0.12277136
## Balanced Accuracy
## 0.05126200
Cross-Validation-naiveBayes file:///E:/users/rkannan/cuny/fall2020/fall2020/ML-Handbook/m09-cv-r...
6 of 7 11/23/2020, 5:27 PM](https://image.slidesharecdn.com/cross-validating-naive-bayes-201123222844/85/M09-Cross-validating-naive-bayes-6-320.jpg)
![tstcv_preds<-lapply(cv_df$m,FUN=function(M,D=tstcatheart[,-c(9)])predict(M,
D,type='raw'))
tstcv_cfm<-lapply(tstcv_preds,FUN=function(P,A=tstcatheart[[9]])
{pred_class<-unlist(apply(round(P),1,which.max))-1
pred_tbl<-table(pred_class,A)
pred_cfm<-caret::confusionMatrix(pred_tbl)
pred_cfm
})
tstcv.perf<-as.data.frame(do.call('rbind',lapply(tstcv_cfm,FUN=function(cfm)
c(cfm$overall,cfm$byClass))))
cv.tst.perf<-apply(tstcv.perf[tstcv.perf$AccuracyPValue<0.01,-c(6:7)],2,mea
n)
cv.tst.perf.var<-apply(tstcv.perf[tstcv.perf$AccuracyPValue<0.01,-c(6:7)],2,
sd)
###################
Cross-Validation-naiveBayes file:///E:/users/rkannan/cuny/fall2020/fall2020/ML-Handbook/m09-cv-r...
7 of 7 11/23/2020, 5:27 PM](https://image.slidesharecdn.com/cross-validating-naive-bayes-201123222844/85/M09-Cross-validating-naive-bayes-7-320.jpg)
M09-Cross validating-naive-bayes
- 1. Cross-Validation-naiveBayes file:///E:/users/rkannan/cuny/fall2020/fall2020/ML-Handbook/m09-cv-r... 1 of 7 11/23/2020, 5:27 PM
- 2. Cross-Validation-naiveBayes file:///E:/users/rkannan/cuny/fall2020/fall2020/ML-Handbook/m09-cv-r... 2 of 7 11/23/2020, 5:27 PM
- 3. library('e1071') file<-'c://Users/rk215/Data/heart.csv' heart<-read.csv(file,head=T,sep=',',stringsAsFactors=F) head(heart) ## age sex cp trestbps chol fbs restecg thalach exang oldpeak slope ca tha l ## 1 63 1 3 145 233 1 0 150 0 2.3 0 0 1 ## 2 37 1 2 130 250 0 1 187 0 3.5 0 0 2 ## 3 41 0 1 130 204 0 0 172 0 1.4 2 0 2 ## 4 56 1 1 120 236 0 1 178 0 0.8 2 0 2 ## 5 57 0 0 120 354 0 1 163 1 0.6 2 0 2 ## 6 57 1 0 140 192 0 1 148 0 0.4 1 0 1 ## target ## 1 1 ## 2 1 ## 3 1 ## 4 1 ## 5 1 ## 6 1 catheart<-heart[,c(2,3,6,7,9,11,12,13,14)] set.seed(43) trdidx<-sample(1:nrow(catheart),0.7*nrow(catheart),replace=F) trcatheart<-catheart[trdidx,] tstcatheart<-catheart[-trdidx,] nb.model<-naiveBayes(target~.,data=trcatheart) #str(nbtr.model) object.size(nb.model) #11096 ## 11096 bytes nb.pred<-predict(nb.model,tstcatheart[,-c(9)],type='raw') nb.class<-unlist(apply(round(nb.pred),1,which.max))-1 nb.tbl<-table(tstcatheart[[9]], nb.class) nb.cfm<-caret::confusionMatrix(nb.tbl) nb.cfm Cross-Validation-naiveBayes file:///E:/users/rkannan/cuny/fall2020/fall2020/ML-Handbook/m09-cv-r... 3 of 7 11/23/2020, 5:27 PM
- 4. ## Confusion Matrix and Statistics ## ## nb.class ## 0 1 ## 0 28 12 ## 1 3 48 ## ## Accuracy : 0.8352 ## 95% CI : (0.7427, 0.9047) ## No Information Rate : 0.6593 ## P-Value [Acc > NIR] : 0.0001482 ## ## Kappa : 0.6571 ## ## Mcnemar's Test P-Value : 0.0388671 ## ## Sensitivity : 0.9032 ## Specificity : 0.8000 ## Pos Pred Value : 0.7000 ## Neg Pred Value : 0.9412 ## Prevalence : 0.3407 ## Detection Rate : 0.3077 ## Detection Prevalence : 0.4396 ## Balanced Accuracy : 0.8516 ## ## 'Positive' Class : 0 ## start_tm <- proc.time() N<-nrow(trcatheart) NF=10 folds<-split(1:N,cut(1:N, quantile(1:N, probs = seq(0, 1, by =1/NF)))) length(folds) ## [1] 10 lapply(folds,length) Cross-Validation-naiveBayes file:///E:/users/rkannan/cuny/fall2020/fall2020/ML-Handbook/m09-cv-r... 4 of 7 11/23/2020, 5:27 PM
- 5. ## $`(1,22.1]` ## [1] 21 ## ## $`(22.1,43.2]` ## [1] 21 ## ## $`(43.2,64.3]` ## [1] 21 ## ## $`(64.3,85.4]` ## [1] 21 ## ## $`(85.4,106]` ## [1] 21 ## ## $`(106,128]` ## [1] 21 ## ## $`(128,149]` ## [1] 21 ## ## $`(149,170]` ## [1] 21 ## ## $`(170,191]` ## [1] 21 ## ## $`(191,212]` ## [1] 22 ridx<-sample(1:nrow(trcatheart),nrow(trcatheart),replace=FALSE) # randomize the data cv_df<-do.call('rbind',lapply(folds,FUN=function(idx,data=trcatheart[ridx,]) { m<-naiveBayes(target~.,data=data[-idx,]) # keep one fold for validation p<-predict(m,data[idx,-c(9)],type='raw') # predict for that test fold pc<-unlist(apply(round(p),1,which.max))-1 pred_tbl<-table(data[idx,c(9)],pc) #table(actual,predicted) pred_cfm<-caret::confusionMatrix(pred_tbl) list(fold=idx,m=m,cfm=pred_cfm) # store the fold, model,cfm } )) # lapply repeats over all folds Cross-Validation-naiveBayes file:///E:/users/rkannan/cuny/fall2020/fall2020/ML-Handbook/m09-cv-r... 5 of 7 11/23/2020, 5:27 PM
- 6. cv_df<-as.data.frame(cv_df) tstcv.perf<-as.data.frame(do.call('rbind',lapply(cv_df$cfm,FUN=function(cfm) c(cfm$overall,cfm$byClass)))) (cv.tst.perf<-apply(tstcv.perf[tstcv.perf$AccuracyPValue<0.01,-c(6:7)],2,mea n)) ## Accuracy Kappa AccuracyLower ## 0.8683983 0.7318000 0.6545460 ## AccuracyUpper AccuracyNull Sensitivity ## 0.9700452 0.5666667 0.8373377 ## Specificity Pos Pred Value Neg Pred Value ## 0.8900699 0.8924825 0.8629060 ## Precision Recall F1 ## 0.8924825 0.8373377 0.8583395 ## Prevalence Detection Rate Detection Prevalence ## 0.4523810 0.3766234 0.4324675 ## Balanced Accuracy ## 0.8637038 (cv.tst.perf.var<-apply(tstcv.perf[tstcv.perf$AccuracyPValue<0.01,-c(6:7)], 2,sd)) ## Accuracy Kappa AccuracyLower ## 0.06018717 0.11323819 0.07464027 ## AccuracyUpper AccuracyNull Sensitivity ## 0.02365967 0.07221786 0.06971453 ## Specificity Pos Pred Value Neg Pred Value ## 0.12234605 0.10504433 0.07414154 ## Precision Recall F1 ## 0.10504433 0.06971453 0.04174328 ## Prevalence Detection Rate Detection Prevalence ## 0.08908708 0.06878895 0.12277136 ## Balanced Accuracy ## 0.05126200 Cross-Validation-naiveBayes file:///E:/users/rkannan/cuny/fall2020/fall2020/ML-Handbook/m09-cv-r... 6 of 7 11/23/2020, 5:27 PM