RHadoop, R meets Hadoop

Scholarly Activity 05-09 change

50%

37.5%

25%

12.5%

0%

-12.5%

-25%

-37.5%
R SAS SPSS S-Plus Stata


50%

37.5%

25%

12.5%
Packages
0%

10000 -12.5%

-25%
1000 -37.5%

100

10

1
2002 2004 2006 2008 2010


50%

37.5%

25%

12.5%
Packages
0%

10000 -12.5%

-25%
1000 -37.5%

100

10
http://r4stats.com/popularity
1
2002 2004 2006 2008 2010

rh d f s

rhb
ase

rm
r

sapply(data, function)

mapreduce(data, map = function)

Rmr

Cascading,
Java, C++
Crunch

Rmr, Rhipe, Dumbo,
Rmr
Pydoop, Hadoopy

Cascading,
Java, C++
Crunch

Expose MR Hide MR

Rmr, Rhipe, Dumbo,
Rmr
Pydoop, Hadoopy

Cascading,
Java, C++
Crunch

Expose MR Hide MR
Hive, Pig

Rmr, Rhipe, Dumbo,
Rmr
Pydoop, Hadoopy

Cascading,
Java, C++
Crunch

Expose MR Hide MR
Hive, Pig

Rmr, Rhipe, Dumbo,
Rmr Cascalog,
Pydoop, Hadoopy Scalding, Scrunch

Cascading,
Java, C++
Crunch

mapreduce(input, output, map, reduce)

map = function(k, v) if (hash(k) %% 10 == 0)keyval(k, v)

reduce = function(k, vv) keyval(k, length(vv))

condition = function(x) x > 10


out = mapreduce(


out = mapreduce(
input = input,


out = mapreduce(
input = input,
map = function(k,v)


out = mapreduce(
input = input,
map = function(k,v)
if (condition(v)) keyval(k,v))

x = from.dfs(hdfs.object)

hdfs.object = to.dfs(x)

INSERT OVERWRITE TABLE pv_gender_sum
SELECT pv_users.gender, count (DISTINCT pv_users.userid)
FROM pv_users
GROUP BY pv_users.gender;

INSERT OVERWRITE TABLE pv_gender_sum
SELECT pv_users.gender, count (DISTINCT pv_users.userid)
FROM pv_users
GROUP BY pv_users.gender;

mapreduce(input =
mapreduce(input = "pv_users",
map = function(k, v) keyval(v['userid'], v['gender']),
reduce = function(uid, genders)
lapply(unique(genders), function(g) keyval(NULL, g)),
output = "pv_gender_sum",
map = function(x, gender) keyval(gender, 1)
reduce = function(gender,counts)
keyval(k,sum(unlist(counts)))

kmeans =
function(points, ncenters, iterations = 10,
distfun = function(a,b) norm(as.matrix(a-b), type = 'F')) {
newCenters = kmeans.iter(points, distfun, ncenters = ncenters)
for(i in 1:iterations) {
newCenters = kmeans.iter(points, distfun, centers = newCenters)}
newCenters}

kmeans =
function(points, ncenters, iterations = 10,
distfun = function(a,b) norm(as.matrix(a-b), type = 'F')) {
newCenters = kmeans.iter(points, distfun, ncenters = ncenters)
for(i in 1:iterations) {
newCenters = kmeans.iter(points, distfun, centers = newCenters)}
newCenters}

kmeans.iter =
function(points, distfun, ncenters = dim(centers)[1], centers = NULL) {
from.dfs(
mapreduce(
input = points,
map = if (is.null(centers)) {
function(k,v) keyval(sample(1:ncenters,1),v)}
else {
function(k,v) {
distances = apply(centers, 1, function(c) distfun(c,v))
keyval(centers[which.min(distances),], v)}},
reduce = function(k,vv) keyval(NULL, apply(do.call(rbind, vv), 2, mean))),
to.data.frame = T)}

#!/usr/bin/python
import sys
from math import fabs
from org.apache.pig.scripting import Pig

filename = "student.txt"
k = 4
tolerance = 0.01

MAX_SCORE = 4
MIN_SCORE = 0
MAX_ITERATION = 100

# initial centroid, equally divide the space
initial_centroids = ""
last_centroids = [None] * k
for i in range(k):
last_centroids[i] = MIN_SCORE + float(i)/k*(MAX_SCORE-MIN_SCORE)
initial_centroids = initial_centroids + str(last_centroids[i])
if i!=k-1:
initial_centroids = initial_centroids + ":"

P = Pig.compile("""register udf.jar
DEFINE find_centroid FindCentroid('$centroids');
raw = load 'student.txt' as (name:chararray, age:int, gpa:double);
centroided = foreach raw generate gpa, find_centroid(gpa) as centroid;
grouped = group centroided by centroid;
result = foreach grouped generate group, AVG(centroided.gpa);
store result into 'output';
""")

converged = False
iter_num = 0
while iter_num<MAX_ITERATION:
Q = P.bind({'centroids':initial_centroids})
results = Q.runSingle()

if results.isSuccessful() == "FAILED":
raise "Pig job failed"
iter = results.result("result").iterator()
centroids = [None] * k
distance_move = 0
# get new centroid of this iteration, caculate the moving distance with last iteration
for i in range(k):
tuple = iter.next()
centroids[i] = float(str(tuple.get(1)))
distance_move = distance_move + fabs(last_centroids[i]-centroids[i])
distance_move = distance_move / k;
Pig.fs("rmr output")
print("iteration " + str(iter_num))
print("average distance moved: " + str(distance_move))
if distance_move<tolerance:
sys.stdout.write("k-means converged at centroids: [")
sys.stdout.write(",".join(str(v) for v in centroids))
sys.stdout.write("]n")
converged = True
break
last_centroids = centroids[:]
initial_centroids = ""
for i in range(k):
initial_centroids = initial_centroids + str(last_centroids[i])
if i!=k-1:
initial_centroids = initial_centroids + ":"
iter_num += 1

if not converged:
print("not converge after " + str(iter_num) + " iterations")
sys.stdout.write("last centroids: [")
sys.stdout.write(",".join(str(v) for v in last_centroids))
sys.stdout.write("]n")

import java.io.IOException;

import org.apache.pig.EvalFunc;
import org.apache.pig.data.Tuple;

public class FindCentroid extends EvalFunc<Double> {
double[] centroids;
public FindCentroid(String initialCentroid) {
String[] centroidStrings = initialCentroid.split(":");
centroids = new double[centroidStrings.length];
for (int i=0;i<centroidStrings.length;i++)
centroids[i] = Double.parseDouble(centroidStrings[i]);
}
@Override
public Double exec(Tuple input) throws IOException {
double min_distance = Double.MAX_VALUE;
double closest_centroid = 0;
for (double centroid : centroids) {
double distance = Math.abs(centroid - (Double)input.get(0));
if (distance < min_distance) {
min_distance = distance;
closest_centroid = centroid;
}
}
return closest_centroid;
}

}

mapreduce(mapreduce(…

mapreduce(input = c(input1, input2), …)

mapreduce(mapreduce(…

mapreduce(input = c(input1, input2), …)

equijoin = function(
left.input, right.input, input,
output,
outer,
map.left, map.right,
reduce, reduce.all)

out1 = mapreduce(…)
mapreduce(input = out1, <xyz>)
mapreduce(input = out1, <abc>)

out1 = mapreduce(…)
mapreduce(input = out1, <xyz>)
mapreduce(input = out1, <abc>)

abstract.job = function(input, output, …) {
…
result = mapreduce(input = input,
output = output)
…
result}

input.format, output.format, format

combine

combine
reduce.on.data.frame

combine
local, hadoop backends

combine
backend.parameters

combine
backend.parameters
profiling

combine
backend.parameters
profiling
verbose

RHADOOP USER
ONE FAT CLUSTER AVE.
HYDROPOWER CITY, OR 0x0000

RHADOOP@
REVOLUTIONANALYTICS.COM

RHadoop, R meets Hadoop

More Related Content

What's hot (20)

Similar to RHadoop, R meets Hadoop (20)

More from Revolution Analytics (20)

Recently uploaded (20)

RHadoop, R meets Hadoop

Editor's Notes