RHadoop の紹介
Download as KEY, PDF9 likes3,203 views
This document provides examples of using SparkR to perform distributed computing tasks like word counting on HDFS files, distributed k-means clustering of large datasets, and saving/loading k-means models to/from HDFS. It shows how to use SparkR functions like mapreduce, to.dfs, from.dfs, and hdfs.write/hdfs.read to parallelize work across a cluster and handle large amounts of data.
![> small.ints = to.dfs(1:10)
> out = mapreduce(input = small.ints, map = function(k,v) keyval(k, k^2))
> res = from.dfs(out)
> colres <- do.call('rbind', lapply(res,"[[",2))
> t(colres)
[,1] [,2] [,3] [,4] [,5] [,6] [,7] [,8] [,9] [,10]
[1,] 1 4 9 16 25 36 49 64 81 100
> groups = to.dfs(rbinom(32, n = 50, prob = 0.4))
> out = mapreduce(input = groups, reduce = function(k,vv) keyval(k, length(vv)))
> res = from.dfs(out)
> colres <- do.call('rbind', lapply(res,"[[",2))
> t(colres)
[,1] [,2] [,3] [,4] [,5] [,6] [,7] [,8] [,9] [,10] [,11] [,12]
[1,] 2 7 3 1 1 12 2 8 8 1 4 1
10](https://image.slidesharecdn.com/rhadoop-111024071218-phpapp02/85/RHadoop-10-320.jpg)
![> wordcount = function(input, output = NULL, pattern = " ") {
+ mapreduce(input = input,
+ output = output,
+ textinputformat = rawtextinputformat,
+ map = function(k ,v) {
+ lapply(strsplit(x = v, split = pattern) [[1]],
+ function(w) keyval(w,1))
+ },
+ reduce = function(k, vv) {
+ keyval(k, sum(unlist(vv)))
+ },
+ combine = T)
+ }
> out <- wordcount(input="/user/hidekazu/the_social_network.txt")
> results <- from.dfs(out)
> results <- data.frame(word=unlist(lapply(results,"[[",1)),
count=unlist(lapply(results,"[[",2)))
> results <- (results[order(results$count, decreasing=TRUE), ])
> head(results)
word count
6313 the 1101
2381 a 700 11
26 and 637](https://image.slidesharecdn.com/rhadoop-111024071218-phpapp02/85/RHadoop-11-320.jpg)
![kmeans.iter =
function(points, distfun, ncenters = length(centers), centers = NULL) {
from.dfs(
mapreduce(input = points,
map = if (is.null(centers)) {
function(k,v)keyval(sample(1:ncenters,1),v)
} else {
function(k,v) {
distances = lapply(centers,
function(c) distfun(c,v))
keyval(centers[[which.min(distances)]],v)
}
},
reduce = function(k,vv) keyval(NULL,
apply(do.call(rbind, vv), 2, mean)))
)
}
13](https://image.slidesharecdn.com/rhadoop-111024071218-phpapp02/85/RHadoop-13-320.jpg)
![> model <- kmeans(iris[, 1:4], 3, nstart=10)
> modelfilename <- "my_smart_unique_name"
> modelfile <- hdfs.file(modelfilename, "w")
> hdfs.write(model, modelfile)
[1] TRUE
> hdfs.close(modelfile)
[1] TRUE
> modelfile = hdfs.file(modelfilename, "r")
> m <- hdfs.read(modelfile)
Warning message:
In function (h) : Closed unused DFS stream: my_smart_unique_name
> model <- unserialize(m)
> hdfs.close(modelfile)
[1] TRUE
> model
17](https://image.slidesharecdn.com/rhadoop-111024071218-phpapp02/85/RHadoop-18-320.jpg)
RHadoop の紹介
- 1. 1
- 2. 2
- 3. • • • • 3
- 4. • • • • 4
- 5. • • • • • 5
- 6. 6
- 7. • • • • • • • 7
- 8. R CMD INSTALL 'package filename' 8
- 9. 9
- 10. > small.ints = to.dfs(1:10) > out = mapreduce(input = small.ints, map = function(k,v) keyval(k, k^2)) > res = from.dfs(out) > colres <- do.call('rbind', lapply(res,"[[",2)) > t(colres) [,1] [,2] [,3] [,4] [,5] [,6] [,7] [,8] [,9] [,10] [1,] 1 4 9 16 25 36 49 64 81 100 > groups = to.dfs(rbinom(32, n = 50, prob = 0.4)) > out = mapreduce(input = groups, reduce = function(k,vv) keyval(k, length(vv))) > res = from.dfs(out) > colres <- do.call('rbind', lapply(res,"[[",2)) > t(colres) [,1] [,2] [,3] [,4] [,5] [,6] [,7] [,8] [,9] [,10] [,11] [,12] [1,] 2 7 3 1 1 12 2 8 8 1 4 1 10
- 11. > wordcount = function(input, output = NULL, pattern = " ") { + mapreduce(input = input, + output = output, + textinputformat = rawtextinputformat, + map = function(k ,v) { + lapply(strsplit(x = v, split = pattern) [[1]], + function(w) keyval(w,1)) + }, + reduce = function(k, vv) { + keyval(k, sum(unlist(vv))) + }, + combine = T) + } > out <- wordcount(input="/user/hidekazu/the_social_network.txt") > results <- from.dfs(out) > results <- data.frame(word=unlist(lapply(results,"[[",1)), count=unlist(lapply(results,"[[",2))) > results <- (results[order(results$count, decreasing=TRUE), ]) > head(results) word count 6313 the 1101 2381 a 700 11 26 and 637
- 12. 12
- 13. kmeans.iter = function(points, distfun, ncenters = length(centers), centers = NULL) { from.dfs( mapreduce(input = points, map = if (is.null(centers)) { function(k,v)keyval(sample(1:ncenters,1),v) } else { function(k,v) { distances = lapply(centers, function(c) distfun(c,v)) keyval(centers[[which.min(distances)]],v) } }, reduce = function(k,vv) keyval(NULL, apply(do.call(rbind, vv), 2, mean))) ) } 13
- 14. kmeans = function(points, ncenters, iterations = 10, distfun = function(a,b) norm(as.matrix(a-b), type = 'F')) { newCenters = kmeans.iter(points, distfun = distfun, ncenters = ncenters) for(i in 1:iterations) { newCenters = lapply(values(newCenters), unlist) newCenters = kmeans.iter(points, distfun, centers=newCenters) } newCenters } clustdata = lapply(1:10000, function(i) keyval(i, c(rnorm(1, mean = i%%3, sd = 0.01), rnorm(1, mean = i%%4, sd = 0.01)))) to.dfs(clustdata, "/tmp/clustdata") kmeans ("/tmp/clustdata", 12) 14
- 15. 15
- 16. 15
- 17. 16
- 18. > model <- kmeans(iris[, 1:4], 3, nstart=10) > modelfilename <- "my_smart_unique_name" > modelfile <- hdfs.file(modelfilename, "w") > hdfs.write(model, modelfile) [1] TRUE > hdfs.close(modelfile) [1] TRUE > modelfile = hdfs.file(modelfilename, "r") > m <- hdfs.read(modelfile) Warning message: In function (h) : Closed unused DFS stream: my_smart_unique_name > model <- unserialize(m) > hdfs.close(modelfile) [1] TRUE > model 17
- 19. 18
- 20. • • 19
- 21. • 20
- 22. • • • • • 21