From b2dac8e5b16f38eec2dcfd86ea23e81acf74fc00 Mon Sep 17 00:00:00 2001 From: charles-lang Date: Thu, 21 Nov 2019 15:51:55 -0500 Subject: [PATCH 1/2] finish --- Assignment7 - Answers.Rmd | 131 ------------ Assignment7.Rmd | 63 ++++-- Assignment7.html | 408 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 456 insertions(+), 146 deletions(-) delete mode 100644 Assignment7 - Answers.Rmd create mode 100644 Assignment7.html diff --git a/Assignment7 - Answers.Rmd b/Assignment7 - Answers.Rmd deleted file mode 100644 index 08d8673..0000000 --- a/Assignment7 - Answers.Rmd +++ /dev/null @@ -1,131 +0,0 @@ ---- -title: "Assignment 7" -author: "Charles Lang" -date: "12/1/2016" -output: html_document ---- -#Create Data -```{r} -id <- seq(1,1000,1) -D1 <- data.frame(id) - -post.test1 <- rnorm(500, 0.4,0.07) -post.test1 <- ifelse(post.test1 < 0, 0, post.test1) -post.test1 <- ifelse(post.test1 > 1, 1, post.test1) -post.test2 <- rnorm(500, 0.6,0.07) -post.test2 <- ifelse(post.test2 < 0, 0, post.test2) -post.test2 <- ifelse(post.test2 > 1, 1, post.test2) -#post.test3 <- rnorm(300, 0.75,0.07) -#post.test3 <- ifelse(post.test3 > 1, 1, post.test3) -D1$post.test.score <- round(c(post.test1,post.test2),2) - -D1$pre.test.score <- round(c(post.test1,post.test2), 2) - -D1$messages <- round(sample(c(150:200),1000, replace = TRUE)*D1$post.test.score) -D1$forum.posts <- round(sample(c(0:40),1000, replace = TRUE)*(D1$post.test.score) + 2) -D1$av.assignment.score <- round(D1$post.test.score*rnorm(1000, 0.5,0.1),2) -D1$level.up <- ifelse(D1$post.test.score > 0.45 & D1$av.assignment.score > 0.25, "yes", "no") - -``` - -#Visualize the distributions -```{r} -library(ggplot2) -library(dplyr) -library(tidyr) -#HINT: look up "facet" in the ggplot documentation -D2 <- select(D1, 1:7) -#Convert yes/no to 1/0 to avoid mixing variable types -D2$level.up <- ifelse(D2$level.up == "yes", 1,0) -D3 <- gather(D2, "measure", "score", 2:7) - -p1 <- ggplot(D3, aes(score)) + facet_wrap(~measure, scales = "free") -p1 + geom_histogram(stat = "count") - -#Visualize the relationships between variables -pairs(D2) -``` - -#Classification tree -```{r} -#Create a classification tree that predicts whether a student "levels up" in the online course using three variables of your choice (remember to set all controls to their minimums) - -library(rpart) -c.tree1 <- rpart(level.up ~ forum.posts + pre.test.score, method = "class", data = D1, control=rpart.control(minsplit=1, minbucket=1, cp=0.001)) - -printcp(c.tree1) -plot(c.tree1) -text(c.tree1) - -#Generate a probability value that represents the probability that a student levels up based your classification tree - -D1$pred <- predict(c.tree1, type = "prob")[,2]#Last class we used type = "class" which predicted the classification for us, this time we are using type = "prob" to see the probability that our previous classififcation was based on. - -#Now you can generate the ROC curve for your model. You will need to install the package ROCR to do this. - -library(ROCR) - -#Plot the curve -pred.detail <- prediction(D1$pred, D1$level.up) -plot(performance(pred.detail, "tpr", "fpr")) -abline(0, 1, lty = 2) - -#Calculate the Area Under the Curve -unlist(slot(performance(pred.detail,"auc"), "y.values"))#unlist liberates AUC value from the "performance" object created by ROCR - -#Now repeat this process, but using the variables you did not use for the previous model and compare the plots & results of your two models. Which one do you think was the better model? - -``` -#Thresholds -```{r} -#Look at the ROC plot for your first model. Based on this plot choose a probability threshold that balances capturing the most correct predictions against false positives. Then generate a new variable in your data set that classifies each student according to your chosen threshold. - -D1$threshold.pred1 <- ifelse(D1$pred >= 0.8, "yes", "no") - -D1$threshold.pred2 <- ifelse(D1$pred >= 0.95, "yes", "no") - -D1$threshold.pred3 <- ifelse(D1$pred >= 0.25, "yes", "no") - -#Now generate three diagnostics: - -accuracy.model1 <- mean(ifelse(D1$level.up == D1$threshold.pred1, 1, 0)) - -D1$truepos.model1 <- ifelse(D1$level.up == "yes" & D1$threshold.pred1 == "yes", 1, 0) -D1$falsepos.model1 <- ifelse(D1$level.up == "no" & D1$threshold.pred1 == "yes", 1,0) -D1$falseneg.model1 <- ifelse(D1$level.up == "yes" & D1$threshold.pred1 == "no", 1,0) - -precision.model1 <- sum(D1$truepos.model1)/(sum(D1$truepos.model1) + sum(D1$falsepos.model1)) - -recall.model1 <- sum(D1$truepos.model1)/(sum(D1$truepos.model1) + sum(D1$falseneg.model1)) - -#Finally, calculate Kappa for your model according to: - -#First generate the table of comparisons -table1 <- table(D1$level.up, D1$threshold.pred1) -table1 - -#Calculate kappa manually -po <- (586+247)/(586+247+14+153) -pe <- ((586 + 14)/(586+247+14+153))*((586 + 153)/(586+247+14+153)) + ((14 + 247)/(586+247+14+153))*((153+247)/(586+247+14+153)) - -kappa <- (po - pe)/(1 - pe) - -#Calculate OOB -library(psych) #You could also use the "irr" or "vcd" library versions -cohen.kappa(table1) - -#Now choose a different threshold value and repeat these diagnostics. What conclusions can you draw about your two thresholds? - -#Alternate kappa value - -library(irr) - -kappa2(D1[,c(7,9)], "unweighted") - -kappa2(D1[,c(7,10)], "unweighted") - -kappa2(D1[,c(7,11)], "unweighted") - -``` - - diff --git a/Assignment7.Rmd b/Assignment7.Rmd index 8da8079..d45eb61 100644 --- a/Assignment7.Rmd +++ b/Assignment7.Rmd @@ -1,7 +1,7 @@ --- title: "Assignment 7 - Answers" -author: "Charles Lang" -date: "11/30/2016" +author: "Beibei" +date: "12/09/2019" output: html_document --- @@ -11,24 +11,35 @@ In the following assignment you will be looking at data from an one level of an #Upload data ```{r} - +library(readr) +library(dplyr) +library(tidyr) +library(ggplot2) +f<-file.choose("~/Desktop/hudk4050/Assignment 7/online.data.csv") +D1 <- read.csv(f, header = TRUE) ``` #Visualization ```{r} #Start by creating histograms of the distributions for all variables (#HINT: look up "facet" in the ggplot documentation) - +D1$level.up <- ifelse(D1$level.up == "yes", 1,0) +D2 <- gather(D1, "measure", "score", 2:7) +p <- ggplot(D2, aes(score)) + facet_wrap(~measure, scales = "free") +p + geom_histogram() #Then visualize the relationships between variables - +pairs(D1) #Try to capture an intution about the data and the relationships ``` #Classification tree ```{r} #Create a classification tree that predicts whether a student "levels up" in the online course using three variables of your choice (As we did last time, set all controls to their minimums) - +library(rpart) +library(rpart.plot) +rp <- rpart(level.up ~ post.test.score + av.assignment.score + forum.posts, method = "class", data = D1, control = rpart.control(minsplit = 1, minbucket = 1, cp = 0.001)) #Plot and generate a CP table for your tree - +printcp(rp) +rpart.plot(rp) #Generate a probability value that represents the probability that a student levels up based your classification tree D1$pred <- predict(rp, type = "prob")[,2]#Last class we used type = "class" which predicted the classification for us, this time we are using type = "prob" to see the probability that our classififcation is based on. @@ -44,24 +55,34 @@ plot(performance(pred.detail, "tpr", "fpr")) abline(0, 1, lty = 2) #Calculate the Area Under the Curve -unlist(slot(performance(Pred2,"auc"), "y.values"))#Unlist liberates the AUC value from the "performance" object created by ROCR +unlist(slot(performance(pred.detail,"auc"), "y.values"))#Unlist liberates the AUC value from the "performance" object created by ROCR #Now repeat this process, but using the variables you did not use for the previous model and compare the plots & results of your two models. Which one do you think was the better model? Why? +rp2 <- rpart(level.up ~ pre.test.score + messages, method = "class", data = D1, control = rpart.control(minsplit = 1, minbucket = 1, cp = 0.001)) +printcp(rp2) +rpart.plot(rp2) +D1$pred2 <- predict(rp2, type = "prob")[,2] +pred.detail2 <- prediction(D1$pred2, D1$level.up) +plot(performance(pred.detail2, "tpr", "fpr")) +abline(0, 1, lty = 2) +unlist(slot(performance(pred.detail2,"auc"), "y.values")) +#I think the first one is a better model since the auc is 1, which means that there is 100% percent to distinguish between positive class and negative class. ``` ## Part III #Thresholds ```{r} #Look at the ROC plot for your first model. Based on this plot choose a probability threshold that balances capturing the most correct predictions against false positives. Then generate a new variable in your data set that classifies each student according to your chosen threshold. -threshold.pred1 <- +D1$threshold.pred1 <- ifelse(D1$pred2 >= 0.707, "yes", "no") #Now generate three diagnostics: -D1$accuracy.model1 <- - -D1$precision.model1 <- - -D1$recall.model1 <- +accuracy.model1 <- mean(ifelse(D1$level.up == D1$threshold.pred1,1,0)) +D1$truepos.model1 <- ifelse(D1$level.up == "yes"&D1$threshold.pred1 == "yes",1,0) +D1$falsepos.model1 <- ifelse(D1$level.up == "no" & D1$threshold.pred1 == "yes", 1,0) +D1$falseneg.model1 <- ifelse(D1$level.up == "yes" & D1$threshold.pred1 == "no", 1,0) +precision.model1 <- sum(D1$truepos.model1)/(sum(D1$truepos.model1) + sum(D1$falsepos.model1)) +recall.model1 <- sum(D1$truepos.model1)/(sum(D1$truepos.model1) + sum(D1$falseneg.model1)) #Finally, calculate Kappa for your model according to: @@ -75,7 +96,19 @@ matrix1 <- as.matrix(table1) kappa(matrix1, exact = TRUE)/kappa(matrix1) #Now choose a different threshold value and repeat these diagnostics. What conclusions can you draw about your two thresholds? - +D1$threshold.pred2 <- ifelse(D1$pred2 >= 0.606, "yes", "no") +accuracy.model2 <- mean(ifelse(D1$level.up == D1$threshold.pred2, 1, 0)) +D1$truepos.model2 <- ifelse(D1$level.up == "yes" & D1$threshold.pred2 == "yes", 1, 0) +D1$falsepos.model2 <- ifelse(D1$level.up == "no" & D1$threshold.pred2 == "yes", 1,0) +D1$falseneg.model2 <- ifelse(D1$level.up == "yes" & D1$threshold.pred2 == "no", 1,0) +precision.model2 <- sum(D1$truepos.model2)/(sum(D1$truepos.model2) + sum(D1$falsepos.model2)) +recall.model2 <- sum(D1$truepos.model2)/(sum(D1$truepos.model2) + sum(D1$falseneg.model2)) +table2 <- table(D1$level.up, D1$threshold.pred2) +matrix2 <- as.matrix(table2) +kappa(matrix2, exact = TRUE)/kappa(matrix2) +# The kappa value are very close with different thresholds. ``` +### To Submit Your Assignment +Please submit your assignment by first "knitting" your RMarkdown document into an html file and then commit, push and pull request both the RMarkdown file and the html file. diff --git a/Assignment7.html b/Assignment7.html new file mode 100644 index 0000000..a2a9a90 --- /dev/null +++ b/Assignment7.html @@ -0,0 +1,408 @@ + + + + + + + + + + + + + + + + +Assignment 7 - Answers + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + +

In the following assignment you will be looking at data from an one level of an online geography tutoring system used by 5th grade students. The game involves a pre-test of geography knowledge (pre.test), a series of assignments for which you have the average score (av.assignment.score), the number of messages sent by each student to other students about the assignments (messages), the number of forum posts students posted asking questions about the assignment (forum.posts), a post test at the end of the level (post.test) and whether or not the system allowed the students to go on to the next level (level.up).

+
+

Part I

+
+
+

Upload data

+
library(readr)
+library(dplyr)
+
## 
+## Attaching package: 'dplyr'
+
## The following objects are masked from 'package:stats':
+## 
+##     filter, lag
+
## The following objects are masked from 'package:base':
+## 
+##     intersect, setdiff, setequal, union
+
library(tidyr)
+library(ggplot2)
+f<-file.choose("~/Desktop/hudk4050/Assignment 7/online.data.csv")
+D1 <- read.csv(f, header = TRUE)
+
+
+

Visualization

+
#Start by creating histograms of the distributions for all variables (#HINT: look up "facet" in the ggplot documentation)
+D1$level.up <- ifelse(D1$level.up == "yes", 1,0)
+D2 <- gather(D1, "measure", "score", 2:7)
+p <- ggplot(D2, aes(score)) + facet_wrap(~measure, scales = "free")
+p + geom_histogram()
+
## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
+

+
#Then visualize the relationships between variables
+pairs(D1)
+

+
#Try to capture an intution about the data and the relationships
+
+
+

Classification tree

+
#Create a classification tree that predicts whether a student "levels up" in the online course using three variables of your choice (As we did last time, set all controls to their minimums)
+library(rpart)
+library(rpart.plot)
+rp <- rpart(level.up ~ post.test.score + av.assignment.score + forum.posts, method = "class", data = D1, control = rpart.control(minsplit = 1, minbucket = 1, cp = 0.001))
+#Plot and generate a CP table for your tree 
+printcp(rp)
+
## 
+## Classification tree:
+## rpart(formula = level.up ~ post.test.score + av.assignment.score + 
+##     forum.posts, data = D1, method = "class", control = rpart.control(minsplit = 1, 
+##     minbucket = 1, cp = 0.001))
+## 
+## Variables actually used in tree construction:
+## [1] av.assignment.score post.test.score    
+## 
+## Root node error: 400/1000 = 0.4
+## 
+## n= 1000 
+## 
+##      CP nsplit rel error xerror     xstd
+## 1 0.930      0      1.00   1.00 0.038730
+## 2 0.070      1      0.07   0.07 0.013042
+## 3 0.001      2      0.00   0.00 0.000000
+
rpart.plot(rp)
+

+
#Generate a probability value that represents the probability that a student levels up based your classification tree 
+
+D1$pred <- predict(rp, type = "prob")[,2]#Last class we used type = "class" which predicted the classification for us, this time we are using type = "prob" to see the probability that our classififcation is based on.
+
+

Part II

+
+
+
+

Now you can generate the ROC curve for your model. You will need to install the package ROCR to do this.

+
library(ROCR)
+
## Loading required package: gplots
+
## 
+## Attaching package: 'gplots'
+
## The following object is masked from 'package:stats':
+## 
+##     lowess
+
#Plot the curve
+pred.detail <- prediction(D1$pred, D1$level.up) 
+plot(performance(pred.detail, "tpr", "fpr"))
+abline(0, 1, lty = 2)
+

+
#Calculate the Area Under the Curve
+unlist(slot(performance(pred.detail,"auc"), "y.values"))#Unlist liberates the AUC value from the "performance" object created by ROCR
+
## [1] 1
+
#Now repeat this process, but using the variables you did not use for the previous model and compare the plots & results of your two models. Which one do you think was the better model? Why?
+rp2 <- rpart(level.up ~ pre.test.score + messages, method = "class", data = D1, control = rpart.control(minsplit = 1, minbucket = 1, cp = 0.001))
+printcp(rp2)
+
## 
+## Classification tree:
+## rpart(formula = level.up ~ pre.test.score + messages, data = D1, 
+##     method = "class", control = rpart.control(minsplit = 1, minbucket = 1, 
+##         cp = 0.001))
+## 
+## Variables actually used in tree construction:
+## [1] messages       pre.test.score
+## 
+## Root node error: 400/1000 = 0.4
+## 
+## n= 1000 
+## 
+##           CP nsplit rel error xerror     xstd
+## 1  0.5425000      0    1.0000 1.0000 0.038730
+## 2  0.0112500      1    0.4575 0.4625 0.030698
+## 3  0.0075000      3    0.4350 0.4875 0.031322
+## 4  0.0050000      4    0.4275 0.5025 0.031682
+## 5  0.0035000      5    0.4225 0.5100 0.031857
+## 6  0.0033333     21    0.3550 0.5075 0.031799
+## 7  0.0025000     24    0.3450 0.5075 0.031799
+## 8  0.0018750     52    0.2700 0.5250 0.032201
+## 9  0.0012500     56    0.2625 0.5650 0.033065
+## 10 0.0010714    117    0.1750 0.6050 0.033860
+## 11 0.0010000    124    0.1675 0.6150 0.034048
+
rpart.plot(rp2)
+
## Warning: labs do not fit even at cex 0.15, there may be some overplotting
+

+
D1$pred2 <- predict(rp2, type = "prob")[,2]
+pred.detail2 <- prediction(D1$pred2, D1$level.up) 
+plot(performance(pred.detail2, "tpr", "fpr"))
+abline(0, 1, lty = 2)
+

+
unlist(slot(performance(pred.detail2,"auc"), "y.values"))
+
## [1] 0.9622563
+
#I think the first one is a better model since the auc is 1, which means that there is 100% percent to distinguish between positive class and negative class.
+
+

Part III

+
+
+
+

Thresholds

+
#Look at the ROC plot for your first model. Based on this plot choose a probability threshold that balances capturing the most correct predictions against false positives. Then generate a new variable in your data set that classifies each student according to your chosen threshold.
+
+D1$threshold.pred1 <- ifelse(D1$pred2 >= 0.707, "yes", "no")
+
+#Now generate three diagnostics:
+
+accuracy.model1 <- mean(ifelse(D1$level.up == D1$threshold.pred1,1,0))
+D1$truepos.model1 <- ifelse(D1$level.up == "yes"&D1$threshold.pred1 == "yes",1,0)
+D1$falsepos.model1 <- ifelse(D1$level.up == "no" & D1$threshold.pred1 == "yes", 1,0)
+D1$falseneg.model1 <- ifelse(D1$level.up == "yes" & D1$threshold.pred1 == "no", 1,0)
+precision.model1 <- sum(D1$truepos.model1)/(sum(D1$truepos.model1) + sum(D1$falsepos.model1))
+recall.model1 <- sum(D1$truepos.model1)/(sum(D1$truepos.model1) + sum(D1$falseneg.model1))
+
+#Finally, calculate Kappa for your model according to:
+
+#First generate the table of comparisons
+table1 <- table(D1$level.up, D1$threshold.pred1)
+
+#Convert to matrix
+matrix1 <- as.matrix(table1)
+
+#Calculate kappa
+kappa(matrix1, exact = TRUE)/kappa(matrix1)
+
## [1] 1.101635
+
#Now choose a different threshold value and repeat these diagnostics. What conclusions can you draw about your two thresholds?
+D1$threshold.pred2 <- ifelse(D1$pred2 >= 0.606, "yes", "no")
+accuracy.model2 <- mean(ifelse(D1$level.up == D1$threshold.pred2, 1, 0))
+D1$truepos.model2 <- ifelse(D1$level.up == "yes" & D1$threshold.pred2 == "yes", 1, 0)
+D1$falsepos.model2 <- ifelse(D1$level.up == "no" & D1$threshold.pred2 == "yes", 1,0)
+D1$falseneg.model2 <- ifelse(D1$level.up == "yes" & D1$threshold.pred2 == "no", 1,0)
+precision.model2 <- sum(D1$truepos.model2)/(sum(D1$truepos.model2) + sum(D1$falsepos.model2))
+recall.model2 <- sum(D1$truepos.model2)/(sum(D1$truepos.model2) + sum(D1$falseneg.model2))
+table2 <- table(D1$level.up, D1$threshold.pred2)
+matrix2 <- as.matrix(table2)
+kappa(matrix2, exact = TRUE)/kappa(matrix2)
+
## [1] 1.093018
+
# The kappa value are very close with different thresholds.
+
+

To Submit Your Assignment

+

Please submit your assignment by first “knitting” your RMarkdown document into an html file and then commit, push and pull request both the RMarkdown file and the html file.

+
+
+ + + + +
+ + + + + + + + + + + + + + + From 08bfa519bd32cef2b7b27d29cd4d39825402c852 Mon Sep 17 00:00:00 2001 From: Beibei Date: Mon, 9 Dec 2019 21:48:26 -0500 Subject: [PATCH 2/2] done --- Assignment7.Rmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Assignment7.Rmd b/Assignment7.Rmd index d45eb61..398038c 100644 --- a/Assignment7.Rmd +++ b/Assignment7.Rmd @@ -1,6 +1,6 @@ --- title: "Assignment 7 - Answers" -author: "Beibei" +author: "Beibei Cao" date: "12/09/2019" output: html_document ---