Apache Spark for Library Developers with Erik Erlandson and William Benton
0 likes588 views
The document discusses the development of reusable open-source libraries for Apache Spark, specifically focusing on coding practices and build definitions in Scala. It emphasizes cross-building for Scala versions, managing dependencies, and writing efficient Spark code, including handling RDD caching and type variances. Additionally, it covers examples of data processing techniques like natural joins for data frames.
![#SAISDD6
def step(rdd: RDD[_]) = {
val wasUncached = rdd.storageLevel == StorageLevel.NONE
if (wasUncached) { rdd.cache() }
result = trainModel(rdd)
if (wasUncached) { rdd.unpersist() }
result
}
Caching when necessary
def step(rdd: RDD[_]) = {
val wasUncached = rdd.storageLevel == StorageLevel.NONE
if (wasUncached) { rdd.cache() }
result = trainModel(rdd)
if (wasUncached) { rdd.unpersist() }
result
}
rdd.cache()](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-27-320.jpg)
![#SAISDD6
def step(rdd: RDD[_]) = {
val wasUncached = rdd.storageLevel == StorageLevel.NONE
if (wasUncached) { rdd.cache() }
result = trainModel(rdd)
if (wasUncached) { rdd.unpersist() }
result
}
Caching when necessary
def step(rdd: RDD[_]) = {
val wasUncached = rdd.storageLevel == StorageLevel.NONE
if (wasUncached) { rdd.cache() }
result = trainModel(rdd)
if (wasUncached) { rdd.unpersist() }
result
}
rdd.cache()](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-28-320.jpg)
![#SAISDD6
def step(rdd: RDD[_]) = {
val wasUncached = rdd.storageLevel == StorageLevel.NONE
if (wasUncached) { rdd.cache() }
result = trainModel(rdd)
if (wasUncached) { rdd.unpersist() }
result
}
Caching when necessary
def step(rdd: RDD[_]) = {
val wasUncached = rdd.storageLevel == StorageLevel.NONE
if (wasUncached) { rdd.cache() }
result = trainModel(rdd)
if (wasUncached) { rdd.unpersist() }
result
}
rdd.cache()
rdd.unpersist()](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-29-320.jpg)
![#SAISDD6
def step(rdd: RDD[_]) = {
val wasUncached = rdd.storageLevel == StorageLevel.NONE
if (wasUncached) { rdd.cache() }
result = trainModel(rdd)
result
}
Caching when necessary
def step(rdd: RDD[_]) = {
val wasUncached = rdd.storageLevel == StorageLevel.NONE
if (wasUncached) { rdd.cache() }
result = trainModel(rdd)
if (wasUncached) { rdd.unpersist() }
result
}](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-30-320.jpg)
![#SAISDD6
def step(rdd: RDD[_]) = {
val wasUncached = rdd.storageLevel == StorageLevel.NONE
if (wasUncached) { rdd.cache() }
result = trainModel(rdd)
if (wasUncached) { rdd.unpersist() }
result
}
Caching when necessary
def step(rdd: RDD[_]) = {
val wasUncached = rdd.storageLevel == StorageLevel.NONE
if (wasUncached) { rdd.cache() }
result = trainModel(rdd)
if (wasUncached) { rdd.unpersist() }
result
}](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-31-320.jpg)
![#SAISDD6
local[*]](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-40-320.jpg)
![#SAISDD6
local[2]](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-45-320.jpg)
![#SAISDD6
The RDD is invariant
T <: U RDD[T] <: RDD[U]](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-49-320.jpg)
![#SAISDD6
The RDD is invariant
T <: U RDD[T] <: RDD[U]
dog animal](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-50-320.jpg)
![#SAISDD6
T <: U RDD[T] <: RDD[U]
trait HasUserId { val userid: Int }
case class Transaction(override val userid: Int,
timestamp: Int,
amount: Double)
extends HasUserId {}
def badKeyByUserId(r: RDD[HasUserId]) = r.map(x => (x.userid, x))
trait HasUserId { val userid: Int }
case class Transaction(override val userid: Int,
timestamp: Int,
amount: Double)
extends HasUserId {}
def badKeyByUserId(r: RDD[HasUserId]) = r.map(x => (x.userid, x))](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-51-320.jpg)
![#SAISDD6
T <: U RDD[T] <: RDD[U]
trait HasUserId { val userid: Int }
case class Transaction(override val userid: Int,
timestamp: Int,
amount: Double)
extends HasUserId {}
def badKeyByUserId(r: RDD[HasUserId]) = r.map(x => (x.userid, x))
trait HasUserId { val userid: Int }
case class Transaction(override val userid: Int,
timestamp: Int,
amount: Double)
extends HasUserId {}
def badKeyByUserId(r: RDD[HasUserId]) = r.map(x => (x.userid, x))
trait HasUserId { val userid: Int }
case class Transaction(override val userid: Int,
timestamp: Int,
amount: Double)
extends HasUserId {}
def badKeyByUserId(r: RDD[HasUserId]) = r.map(x => (x.userid, x))
trait HasUserId { val userid: Int }
case class Transaction(override val userid: Int,
timestamp: Int,
amount: Double)
extends HasUserId {}
def badKeyByUserId(r: RDD[HasUserId]) = r.map(x => (x.userid, x))](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-52-320.jpg)
![#SAISDD6
val xacts = spark.parallelize(Array(
Transaction(1, 1, 1.0),
Transaction(2, 2, 1.0)
))
badKeyByUserId(xacts)
<console>: error: type mismatch;
found : org.apache.spark.rdd.RDD[Transaction]
required: org.apache.spark.rdd.RDD[HasUserId]
Note: Transaction <: HasUserID, but class RDD is invariant in type T.
You may wish to define T as +T instead. (SLS 4.5)
badKeyByUserId(xacts)](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-53-320.jpg)
![#SAISDD6
val xacts = spark.parallelize(Array(
Transaction(1, 1, 1.0),
Transaction(2, 2, 1.0)
))
badKeyByUserId(xacts)
<console>: error: type mismatch;
found : org.apache.spark.rdd.RDD[Transaction]
required: org.apache.spark.rdd.RDD[HasUserId]
Note: Transaction <: HasUserID, but class RDD is invariant in type T.
You may wish to define T as +T instead. (SLS 4.5)
badKeyByUserId(xacts)](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-54-320.jpg)
![#SAISDD6
def natjoin(left: DataFrame, right: DataFrame): DataFrame = {
val lcols = left.columns
val rcols = right.columns
val ccols = lcols.toSet intersect rcols.toSet
if(ccols.isEmpty)
left.limit(0).crossJoin(right.limit(0))
else
left
.join(right, ccols.map {col => left(col) === right(col) }.reduce(_ && _))
.select(lcols.collect { case c if ccols.contains(c) => left(c) } ++
lcols.collect { case c if !ccols.contains(c) => left(c) } ++
rcols.collect { case c if !ccols.contains(c) => right(c) } : _*)
}
def natjoin(left: DataFrame, right: DataFrame): DataFrame = {
val lcols = left.columns
val rcols = right.columns
val ccols = lcols.toSet intersect rcols.toSet
if(ccols.isEmpty)
left.limit(0).crossJoin(right.limit(0))
else
left
.join(right, ccols.map {col => left(col) === right(col) }.reduce(_ && _))
.select(lcols.collect { case c if ccols.contains(c) => left(c) } ++
lcols.collect { case c if !ccols.contains(c) => left(c) } ++
rcols.collect { case c if !ccols.contains(c) => right(c) } : _*)
}
[left.a === right.a, left.b === right.b, …]](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-67-320.jpg)
![#SAISDD6
import json
from pyspark.sql.types import *
from pyspark.sql.functions import udf
def selectively_structure(fields):
resultType = StructType([StructField(f, StringType(), nullable=True)
for f in fields])
def impl(js):
try:
d = json.loads(js)
return [str(d.get(f)) for f in fields]
except:
return [None] * len(fields)
return udf(impl, resultType)
import json
from pyspark.sql.types import *
from pyspark.sql.functions import udf
def selectively_structure(fields):
resultType = StructType([StructField(f, StringType(), nullable=True)
for f in fields])
def impl(js):
try:
d = json.loads(js)
return [str(d.get(f)) for f in fields]
except:
return [None] * len(fields)
return udf(impl, resultType)](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-78-320.jpg)
![#SAISDD6
import json
from pyspark.sql.types import *
from pyspark.sql.functions import udf
def selectively_structure(fields):
resultType = StructType([StructField(f, StringType(), nullable=True)
for f in fields])
def impl(js):
try:
d = json.loads(js)
return [str(d.get(f)) for f in fields]
except:
return [None] * len(fields)
return udf(impl, resultType)
import json
from pyspark.sql.types import *
from pyspark.sql.functions import udf
def selectively_structure(fields):
resultType = StructType([StructField(f, StringType(), nullable=True)
for f in fields])
def impl(js):
try:
d = json.loads(js)
return [str(d.get(f)) for f in fields]
except:
return [None] * len(fields)
return udf(impl, resultType)](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-79-320.jpg)
![#SAISDD6
import json
from pyspark.sql.types import *
from pyspark.sql.functions import udf
def selectively_structure(fields):
resultType = StructType([StructField(f, StringType(), nullable=True)
for f in fields])
def impl(js):
try:
d = json.loads(js)
return [str(d.get(f)) for f in fields]
except:
return [None] * len(fields)
return udf(impl, resultType)
import json
from pyspark.sql.types import *
from pyspark.sql.functions import udf
def selectively_structure(fields):
resultType = StructType([StructField(f, StringType(), nullable=True)
for f in fields])
def impl(js):
try:
d = json.loads(js)
return [str(d.get(f)) for f in fields]
except:
return [None] * len(fields)
return udf(impl, resultType)](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-80-320.jpg)
![#SAISDD6
import json
from pyspark.sql.types import *
from pyspark.sql.functions import udf
def selectively_structure(fields):
resultType = StructType([StructField(f, StringType(), nullable=True)
for f in fields])
def impl(js):
try:
d = json.loads(js)
return [str(d.get(f)) for f in fields]
except:
return [None] * len(fields)
return udf(impl, resultType)
import json
from pyspark.sql.types import *
from pyspark.sql.functions import udf
def selectively_structure(fields):
resultType = StructType([StructField(f, StringType(), nullable=True)
for f in fields])
def impl(js):
try:
d = json.loads(js)
return [str(d.get(f)) for f in fields]
except:
return [None] * len(fields)
return udf(impl, resultType)](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-81-320.jpg)
![#SAISDD6
import json
from pyspark.sql.types import *
from pyspark.sql.functions import udf
def selectively_structure(fields):
resultType = StructType([StructField(f, StringType(), nullable=True)
for f in fields])
def impl(js):
try:
d = json.loads(js)
return [str(d.get(f)) for f in fields]
except:
return [None] * len(fields)
return udf(impl, resultType)
import json
from pyspark.sql.types import *
from pyspark.sql.functions import udf
def selectively_structure(fields):
resultType = StructType([StructField(f, StringType(), nullable=True)
for f in fields])
def impl(js):
try:
d = json.loads(js)
return [str(d.get(f)) for f in fields]
except:
return [None] * len(fields)
return udf(impl, resultType)](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-82-320.jpg)
![#SAISDD6
import json
from pyspark.sql.types import *
from pyspark.sql.functions import udf
def selectively_structure(fields):
resultType = StructType([StructField(f, StringType(), nullable=True)
for f in fields])
def impl(js):
try:
d = json.loads(js)
return [str(d.get(f)) for f in fields]
except:
return [None] * len(fields)
return udf(impl, resultType)
import json
from pyspark.sql.types import *
from pyspark.sql.functions import udf
def selectively_structure(fields):
resultType = StructType([StructField(f, StringType(), nullable=True)
for f in fields])
def impl(js):
try:
d = json.loads(js)
return [str(d.get(f)) for f in fields]
except:
return [None] * len(fields)
return udf(impl, resultType)
extract_bx = selectively_structure(["b", "x"])
structured_df = df.withColumn("result", extract_bx("json"))](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-83-320.jpg)
(implicit num: Numeric[N], dataTpe: TDigestUDAFDataType[N])
extends UserDefinedAggregateFunction {
def deterministic: Boolean = false
def inputSchema: StructType =
StructType(StructField("x", dataTpe.tpe) :: Nil)
def bufferSchema: StructType =
StructType(StructField("tdigest", TDigestUDT) :: Nil)
def dataType: DataType = TDigestUDT
case class TDigestUDAF[N](deltaV: Double, maxDiscreteV: Int)
(implicit num: Numeric[N], dataTpe: TDigestUDAFDataType[N])
extends UserDefinedAggregateFunction {
def deterministic: Boolean = false
def inputSchema: StructType =
StructType(StructField("x", dataTpe.tpe) :: Nil)
def bufferSchema: StructType =
StructType(StructField("tdigest", TDigestUDT) :: Nil)
def dataType: DataType = TDigestUDT](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-104-320.jpg)
(implicit num: Numeric[N], dataTpe: TDigestUDAFDataType[N])
extends UserDefinedAggregateFunction {
def deterministic: Boolean = false
def inputSchema: StructType =
StructType(StructField("x", dataTpe.tpe) :: Nil)
def bufferSchema: StructType =
StructType(StructField("tdigest", TDigestUDT) :: Nil)
def dataType: DataType = TDigestUDT
case class TDigestUDAF[N](deltaV: Double, maxDiscreteV: Int)
(implicit num: Numeric[N], dataTpe: TDigestUDAFDataType[N])
extends UserDefinedAggregateFunction {
def deterministic: Boolean = false
def inputSchema: StructType =
StructType(StructField("x", dataTpe.tpe) :: Nil)
def bufferSchema: StructType =
StructType(StructField("tdigest", TDigestUDT) :: Nil)
def dataType: DataType = TDigestUDT](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-105-320.jpg)
(implicit num: Numeric[N], dataTpe: TDigestUDAFDataType[N])
extends UserDefinedAggregateFunction {
def deterministic: Boolean = false
def inputSchema: StructType =
StructType(StructField("x", dataTpe.tpe) :: Nil)
def bufferSchema: StructType =
StructType(StructField("tdigest", TDigestUDT) :: Nil)
def dataType: DataType = TDigestUDT
case class TDigestUDAF[N](deltaV: Double, maxDiscreteV: Int)
(implicit num: Numeric[N], dataTpe: TDigestUDAFDataType[N])
extends UserDefinedAggregateFunction {
def deterministic: Boolean = false
def inputSchema: StructType =
StructType(StructField("x", dataTpe.tpe) :: Nil)
def bufferSchema: StructType =
StructType(StructField("tdigest", TDigestUDT) :: Nil)
def dataType: DataType = TDigestUDT](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-106-320.jpg)
(implicit num: Numeric[N], dataTpe: TDigestUDAFDataType[N])
extends UserDefinedAggregateFunction {
def deterministic: Boolean = false
def inputSchema: StructType =
StructType(StructField("x", dataTpe.tpe) :: Nil)
def bufferSchema: StructType =
StructType(StructField("tdigest", TDigestUDT) :: Nil)
def dataType: DataType = TDigestUDT
case class TDigestUDAF[N](deltaV: Double, maxDiscreteV: Int)
(implicit num: Numeric[N], dataTpe: TDigestUDAFDataType[N])
extends UserDefinedAggregateFunction {
def deterministic: Boolean = false
def inputSchema: StructType =
StructType(StructField("x", dataTpe.tpe) :: Nil)
def bufferSchema: StructType =
StructType(StructField("tdigest", TDigestUDT) :: Nil)
def dataType: DataType = TDigestUDT](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-107-320.jpg)
(implicit num: Numeric[N], dataTpe: TDigestUDAFDataType[N])
extends UserDefinedAggregateFunction {
def deterministic: Boolean = false
def inputSchema: StructType =
StructType(StructField("x", dataTpe.tpe) :: Nil)
def bufferSchema: StructType =
StructType(StructField("tdigest", TDigestUDT) :: Nil)
def dataType: DataType = TDigestUDT
case class TDigestUDAF[N](deltaV: Double, maxDiscreteV: Int)
(implicit num: Numeric[N], dataTpe: TDigestUDAFDataType[N])
extends UserDefinedAggregateFunction {
def deterministic: Boolean = false
def inputSchema: StructType =
StructType(StructField("x", dataTpe.tpe) :: Nil)
def bufferSchema: StructType =
StructType(StructField("tdigest", TDigestUDT) :: Nil)
def dataType: DataType = TDigestUDT](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-108-320.jpg)
(implicit num: Numeric[N], dataTpe: TDigestUDAFDataType[N])
extends UserDefinedAggregateFunction {
def deterministic: Boolean = false
def inputSchema: StructType =
StructType(StructField("x", dataTpe.tpe) :: Nil)
def bufferSchema: StructType =
StructType(StructField("tdigest", TDigestUDT) :: Nil)
def dataType: DataType = TDigestUDT
case class TDigestUDAF[N](deltaV: Double, maxDiscreteV: Int)
(implicit num: Numeric[N], dataTpe: TDigestUDAFDataType[N])
extends UserDefinedAggregateFunction {
def deterministic: Boolean = false
def inputSchema: StructType =
StructType(StructField("x", dataTpe.tpe) :: Nil)
def bufferSchema: StructType =
StructType(StructField("tdigest", TDigestUDT) :: Nil)
def dataType: DataType = TDigestUDT](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-109-320.jpg)
(implicit num: Numeric[N], dataTpe: TDigestUDAFDataType[N])
extends UserDefinedAggregateFunction {
def deterministic: Boolean = false
def inputSchema: StructType =
StructType(StructField("x", dataTpe.tpe) :: Nil)
def bufferSchema: StructType =
StructType(StructField("tdigest", TDigestUDT) :: Nil)
def dataType: DataType = TDigestUDT
case class TDigestUDAF[N](deltaV: Double, maxDiscreteV: Int)
(implicit num: Numeric[N], dataTpe: TDigestUDAFDataType[N])
extends UserDefinedAggregateFunction {
def deterministic: Boolean = false
def inputSchema: StructType =
StructType(StructField("x", dataTpe.tpe) :: Nil)
def bufferSchema: StructType =
StructType(StructField("tdigest", TDigestUDT) :: Nil)
def dataType: DataType = TDigestUDT](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-110-320.jpg)
(implicit num: Numeric[N], dataTpe: TDigestUDAFDataType[N])
extends UserDefinedAggregateFunction {
def deterministic: Boolean = false
def inputSchema: StructType =
StructType(StructField("x", dataTpe.tpe) :: Nil)
def bufferSchema: StructType =
StructType(StructField("tdigest", TDigestUDT) :: Nil)
def dataType: DataType = TDigestUDT
case class TDigestUDAF[N](deltaV: Double, maxDiscreteV: Int)
(implicit num: Numeric[N], dataTpe: TDigestUDAFDataType[N])
extends UserDefinedAggregateFunction {
def deterministic: Boolean = false
def inputSchema: StructType =
StructType(StructField("x", dataTpe.tpe) :: Nil)
def bufferSchema: StructType =
StructType(StructField("tdigest", TDigestUDT) :: Nil)
def dataType: DataType = TDigestUDT](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-111-320.jpg)
def initialize(buf: MutableAggregationBuffer): Unit = {
buf(0) = TDigestSQL(TDigest.empty(deltaV, maxDiscreteV))
}
def evaluate(buf: Row): Any = buf.getAs[TDigestSQL](0)](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-114-320.jpg)
def initialize(buf: MutableAggregationBuffer): Unit = {
buf(0) = TDigestSQL(TDigest.empty(deltaV, maxDiscreteV))
}
def evaluate(buf: Row): Any = buf.getAs[TDigestSQL](0)](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-115-320.jpg)
def initialize(buf: MutableAggregationBuffer): Unit = {
buf(0) = TDigestSQL(TDigest.empty(deltaV, maxDiscreteV))
}
def evaluate(buf: Row): Any = buf.getAs[TDigestSQL](0)](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-118-320.jpg)
def initialize(buf: MutableAggregationBuffer): Unit = {
buf(0) = TDigestSQL(TDigest.empty(deltaV, maxDiscreteV))
}
def evaluate(buf: Row): Any = buf.getAs[TDigestSQL](0)](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-119-320.jpg)
.tdigest +
num.toDouble(input.getAs[N](0)))
}
}
def merge(buf1: MutableAggregationBuffer, buf2: Row): Unit = {
buf1(0) = TDigestSQL(buf1.getAs[TDigestSQL](0).tdigest ++
buf2.getAs[TDigestSQL](0).tdigest)
}
def update(buf: MutableAggregationBuffer, input: Row): Unit = {
if (!input.isNullAt(0)) {
buf(0) = TDigestSQL(buf.getAs[TDigestSQL](0).tdigest +
num.toDouble(input.getAs[N](0)))
}
}
def merge(buf1: MutableAggregationBuffer, buf2: Row): Unit = {
buf1(0) = TDigestSQL(buf1.getAs[TDigestSQL](0).tdigest ++
buf2.getAs[TDigestSQL](0).tdigest)
}](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-122-320.jpg)
.tdigest +
num.toDouble(input.getAs[N](0)))
}
}
def merge(buf1: MutableAggregationBuffer, buf2: Row): Unit = {
buf1(0) = TDigestSQL(buf1.getAs[TDigestSQL](0).tdigest ++
buf2.getAs[TDigestSQL](0).tdigest)
}
def update(buf: MutableAggregationBuffer, input: Row): Unit = {
if (!input.isNullAt(0)) {
buf(0) = TDigestSQL(buf.getAs[TDigestSQL](0).tdigest +
num.toDouble(input.getAs[N](0)))
}
}
def merge(buf1: MutableAggregationBuffer, buf2: Row): Unit = {
buf1(0) = TDigestSQL(buf1.getAs[TDigestSQL](0).tdigest ++
buf2.getAs[TDigestSQL](0).tdigest)
}](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-123-320.jpg)
.tdigest +
num.toDouble(input.getAs[N](0)))
}
}
def merge(buf1: MutableAggregationBuffer, buf2: Row): Unit = {
buf1(0) = TDigestSQL(buf1.getAs[TDigestSQL](0).tdigest ++
buf2.getAs[TDigestSQL](0).tdigest)
}
def update(buf: MutableAggregationBuffer, input: Row): Unit = {
if (!input.isNullAt(0)) {
buf(0) = TDigestSQL(buf.getAs[TDigestSQL](0).tdigest +
num.toDouble(input.getAs[N](0)))
}
}
def merge(buf1: MutableAggregationBuffer, buf2: Row): Unit = {
buf1(0) = TDigestSQL(buf1.getAs[TDigestSQL](0).tdigest ++
buf2.getAs[TDigestSQL](0).tdigest)
}](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-124-320.jpg)
.tdigest +
num.toDouble(input.getAs[N](0)))
}
}
def merge(buf1: MutableAggregationBuffer, buf2: Row): Unit = {
buf1(0) = TDigestSQL(buf1.getAs[TDigestSQL](0).tdigest ++
buf2.getAs[TDigestSQL](0).tdigest)
}
def update(buf: MutableAggregationBuffer, input: Row): Unit = {
if (!input.isNullAt(0)) {
buf(0) = TDigestSQL(buf.getAs[TDigestSQL](0).tdigest +
num.toDouble(input.getAs[N](0)))
}
}
def merge(buf1: MutableAggregationBuffer, buf2: Row): Unit = {
buf1(0) = TDigestSQL(buf1.getAs[TDigestSQL](0).tdigest ++
buf2.getAs[TDigestSQL](0).tdigest)
}](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-125-320.jpg)
.tdigest +
num.toDouble(input.getAs[N](0)))
}
}
def merge(buf1: MutableAggregationBuffer, buf2: Row): Unit = {
buf1(0) = TDigestSQL(buf1.getAs[TDigestSQL](0).tdigest ++
buf2.getAs[TDigestSQL](0).tdigest)
}
def update(buf: MutableAggregationBuffer, input: Row): Unit = {
if (!input.isNullAt(0)) {
buf(0) = TDigestSQL(buf.getAs[TDigestSQL](0).tdigest +
num.toDouble(input.getAs[N](0)))
}
}
def merge(buf1: MutableAggregationBuffer, buf2: Row): Unit = {
buf1(0) = TDigestSQL(buf1.getAs[TDigestSQL](0).tdigest ++
buf2.getAs[TDigestSQL](0).tdigest)
}](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-126-320.jpg)
.tdigest +
num.toDouble(input.getAs[N](0)))
}
}
def merge(buf1: MutableAggregationBuffer, buf2: Row): Unit = {
buf1(0) = TDigestSQL(buf1.getAs[TDigestSQL](0).tdigest ++
buf2.getAs[TDigestSQL](0).tdigest)
}
def update(buf: MutableAggregationBuffer, input: Row): Unit = {
if (!input.isNullAt(0)) {
buf(0) = TDigestSQL(buf.getAs[TDigestSQL](0).tdigest +
num.toDouble(input.getAs[N](0)))
}
}
def merge(buf1: MutableAggregationBuffer, buf2: Row): Unit = {
buf1(0) = TDigestSQL(buf1.getAs[TDigestSQL](0).tdigest ++
buf2.getAs[TDigestSQL](0).tdigest)
}](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-127-320.jpg)
.tdigest +
num.toDouble(input.getAs[N](0)))
}
}
def merge(buf1: MutableAggregationBuffer, buf2: Row): Unit = {
buf1(0) = TDigestSQL(buf1.getAs[TDigestSQL](0).tdigest ++
buf2.getAs[TDigestSQL](0).tdigest)
}
def update(buf: MutableAggregationBuffer, input: Row): Unit = {
if (!input.isNullAt(0)) {
buf(0) = TDigestSQL(buf.getAs[TDigestSQL](0).tdigest +
num.toDouble(input.getAs[N](0)))
}
}
def merge(buf1: MutableAggregationBuffer, buf2: Row): Unit = {
buf1(0) = TDigestSQL(buf1.getAs[TDigestSQL](0).tdigest ++
buf2.getAs[TDigestSQL](0).tdigest)
}](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-128-320.jpg)
.tdigest +
num.toDouble(input.getAs[N](0)))
}
}
def merge(buf1: MutableAggregationBuffer, buf2: Row): Unit = {
buf1(0) = TDigestSQL(buf1.getAs[TDigestSQL](0).tdigest ++
buf2.getAs[TDigestSQL](0).tdigest)
}
def update(buf: MutableAggregationBuffer, input: Row): Unit = {
if (!input.isNullAt(0)) {
buf(0) = TDigestSQL(buf.getAs[TDigestSQL](0).tdigest +
num.toDouble(input.getAs[N](0)))
}
}
def merge(buf1: MutableAggregationBuffer, buf2: Row): Unit = {
buf1(0) = TDigestSQL(buf1.getAs[TDigestSQL](0).tdigest ++
buf2.getAs[TDigestSQL](0).tdigest)
}](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-131-320.jpg)
.tdigest +
num.toDouble(input.getAs[N](0)))
}
}
def merge(buf1: MutableAggregationBuffer, buf2: Row): Unit = {
buf1(0) = TDigestSQL(buf1.getAs[TDigestSQL](0).tdigest ++
buf2.getAs[TDigestSQL](0).tdigest)
}
def update(buf: MutableAggregationBuffer, input: Row): Unit = {
if (!input.isNullAt(0)) {
buf(0) = TDigestSQL(buf.getAs[TDigestSQL](0).tdigest +
num.toDouble(input.getAs[N](0)))
}
}
def merge(buf1: MutableAggregationBuffer, buf2: Row): Unit = {
buf1(0) = TDigestSQL(buf1.getAs[TDigestSQL](0).tdigest ++
buf2.getAs[TDigestSQL](0).tdigest)
}](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-132-320.jpg)
![#SAISDD6
User-defined types
package org.apache.spark.isarnproject.sketches.udt
@SQLUserDefinedType(udt = classOf[TDigestUDT])
case class TDigestSQL(tdigest: TDigest)
class TDigestUDT extends UserDefinedType[TDigestSQL] {
def userClass: Class[TDigestSQL] = classOf[TDigestSQL]
// ....
package org.apache.spark](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-134-320.jpg)
![#SAISDD6
package org.apache.spark.isarnproject.sketches.udt
@SQLUserDefinedType(udt = classOf[TDigestUDT])
case class TDigestSQL(tdigest: TDigest)
class TDigestUDT extends UserDefinedType[TDigestSQL] {
def userClass: Class[TDigestSQL] = classOf[TDigestSQL]
// ....
User-defined types
package org.apache.spark.isarnproject.sketches.udt
@SQLUserDefinedType(udt = classOf[TDigestUDT])
case class TDigestSQL(tdigest: TDigest)
class TDigestUDT extends UserDefinedType[TDigestSQL] {
def userClass: Class[TDigestSQL] = classOf[TDigestSQL]
// ....](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-135-320.jpg)
![#SAISDD6
package org.apache.spark.isarnproject.sketches.udt
@SQLUserDefinedType(udt = classOf[TDigestUDT])
case class TDigestSQL(tdigest: TDigest)
class TDigestUDT extends UserDefinedType[TDigestSQL] {
def userClass: Class[TDigestSQL] = classOf[TDigestSQL]
// ....
User-defined types
package org.apache.spark.isarnproject.sketches.udt
@SQLUserDefinedType(udt = classOf[TDigestUDT])
case class TDigestSQL(tdigest: TDigest)
class TDigestUDT extends UserDefinedType[TDigestSQL] {
def userClass: Class[TDigestSQL] = classOf[TDigestSQL]
// ....](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-136-320.jpg)
![#SAISDD6
Implementing custom types
class TDigestUDT extends UserDefinedType[TDigestSQL] {
def userClass: Class[TDigestSQL] = classOf[TDigestSQL]
override def pyUDT: String =
"isarnproject.sketches.udt.tdigest.TDigestUDT"
override def typeName: String = "tdigest"
def sqlType: DataType = StructType(
StructField("delta", DoubleType, false) ::
/* ... */
StructField("clustM", ArrayType(DoubleType, false), false) ::
Nil)
class TDigestUDT extends UserDefinedType[TDigestSQL] {
def userClass: Class[TDigestSQL] = classOf[TDigestSQL]
override def pyUDT: String =
"isarnproject.sketches.udt.tdigest.TDigestUDT"
override def typeName: String = "tdigest"
def sqlType: DataType = StructType(
StructField("delta", DoubleType, false) ::
/* ... */
StructField("clustM", ArrayType(DoubleType, false), false) ::
Nil)](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-137-320.jpg)
![#SAISDD6
class TDigestUDT extends UserDefinedType[TDigestSQL] {
def userClass: Class[TDigestSQL] = classOf[TDigestSQL]
override def pyUDT: String =
"isarnproject.sketches.udt.tdigest.TDigestUDT"
override def typeName: String = "tdigest"
def sqlType: DataType = StructType(
StructField("delta", DoubleType, false) ::
/* ... */
StructField("clustM", ArrayType(DoubleType, false), false) ::
Nil)
class TDigestUDT extends UserDefinedType[TDigestSQL] {
def userClass: Class[TDigestSQL] = classOf[TDigestSQL]
override def pyUDT: String =
"isarnproject.sketches.udt.tdigest.TDigestUDT"
override def typeName: String = "tdigest"
def sqlType: DataType = StructType(
StructField("delta", DoubleType, false) ::
/* ... */
StructField("clustM", ArrayType(DoubleType, false), false) ::
Nil)](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-138-320.jpg)
![#SAISDD6
class TDigestUDT extends UserDefinedType[TDigestSQL] {
def userClass: Class[TDigestSQL] = classOf[TDigestSQL]
override def pyUDT: String =
"isarnproject.sketches.udt.tdigest.TDigestUDT"
override def typeName: String = "tdigest"
def sqlType: DataType = StructType(
StructField("delta", DoubleType, false) ::
/* ... */
StructField("clustM", ArrayType(DoubleType, false), false) ::
Nil)
class TDigestUDT extends UserDefinedType[TDigestSQL] {
def userClass: Class[TDigestSQL] = classOf[TDigestSQL]
override def pyUDT: String =
"isarnproject.sketches.udt.tdigest.TDigestUDT"
override def typeName: String = "tdigest"
def sqlType: DataType = StructType(
StructField("delta", DoubleType, false) ::
/* ... */
StructField("clustM", ArrayType(DoubleType, false), false) ::
Nil)](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-139-320.jpg)
![#SAISDD6
class TDigestUDT extends UserDefinedType[TDigestSQL] {
def userClass: Class[TDigestSQL] = classOf[TDigestSQL]
override def pyUDT: String =
"isarnproject.sketches.udt.tdigest.TDigestUDT"
override def typeName: String = "tdigest"
def sqlType: DataType = StructType(
StructField("delta", DoubleType, false) ::
/* ... */
StructField("clustM", ArrayType(DoubleType, false), false) ::
Nil)
class TDigestUDT extends UserDefinedType[TDigestSQL] {
def userClass: Class[TDigestSQL] = classOf[TDigestSQL]
override def pyUDT: String =
"isarnproject.sketches.udt.tdigest.TDigestUDT"
override def typeName: String = "tdigest"
def sqlType: DataType = StructType(
StructField("delta", DoubleType, false) ::
/* ... */
StructField("clustM", ArrayType(DoubleType, false), false) ::
Nil)](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-140-320.jpg)
![#SAISDD6
def serialize(tdsql: TDigestSQL): Any = serializeTD(tdsql.tdigest)
private[sketches] def serializeTD(td: TDigest): InternalRow = {
val TDigest(delta, maxDiscrete, nclusters, clusters) = td
val row = new GenericInternalRow(5)
row.setDouble(0, delta)
row.setInt(1, maxDiscrete)
row.setInt(2, nclusters)
val clustX = clusters.keys.toArray
val clustM = clusters.values.toArray
row.update(3, UnsafeArrayData.fromPrimitiveArray(clustX))
row.update(4, UnsafeArrayData.fromPrimitiveArray(clustM))
row
}
def serialize(tdsql: TDigestSQL): Any = serializeTD(tdsql.tdigest)
private[sketches] def serializeTD(td: TDigest): InternalRow = {
val TDigest(delta, maxDiscrete, nclusters, clusters) = td
val row = new GenericInternalRow(5)
row.setDouble(0, delta)
row.setInt(1, maxDiscrete)
row.setInt(2, nclusters)
val clustX = clusters.keys.toArray
val clustM = clusters.values.toArray
row.update(3, UnsafeArrayData.fromPrimitiveArray(clustX))
row.update(4, UnsafeArrayData.fromPrimitiveArray(clustM))
row
}](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-141-320.jpg)
![#SAISDD6
def serialize(tdsql: TDigestSQL): Any = serializeTD(tdsql.tdigest)
private[sketches] def serializeTD(td: TDigest): InternalRow = {
val TDigest(delta, maxDiscrete, nclusters, clusters) = td
val row = new GenericInternalRow(5)
row.setDouble(0, delta)
row.setInt(1, maxDiscrete)
row.setInt(2, nclusters)
val clustX = clusters.keys.toArray
val clustM = clusters.values.toArray
row.update(3, UnsafeArrayData.fromPrimitiveArray(clustX))
row.update(4, UnsafeArrayData.fromPrimitiveArray(clustM))
row
}
def serialize(tdsql: TDigestSQL): Any = serializeTD(tdsql.tdigest)
private[sketches] def serializeTD(td: TDigest): InternalRow = {
val TDigest(delta, maxDiscrete, nclusters, clusters) = td
val row = new GenericInternalRow(5)
row.setDouble(0, delta)
row.setInt(1, maxDiscrete)
row.setInt(2, nclusters)
val clustX = clusters.keys.toArray
val clustM = clusters.values.toArray
row.update(3, UnsafeArrayData.fromPrimitiveArray(clustX))
row.update(4, UnsafeArrayData.fromPrimitiveArray(clustM))
row
}](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-142-320.jpg)
![#SAISDD6
def serialize(tdsql: TDigestSQL): Any = serializeTD(tdsql.tdigest)
private[sketches] def serializeTD(td: TDigest): InternalRow = {
val TDigest(delta, maxDiscrete, nclusters, clusters) = td
val row = new GenericInternalRow(5)
row.setDouble(0, delta)
row.setInt(1, maxDiscrete)
row.setInt(2, nclusters)
val clustX = clusters.keys.toArray
val clustM = clusters.values.toArray
row.update(3, UnsafeArrayData.fromPrimitiveArray(clustX))
row.update(4, UnsafeArrayData.fromPrimitiveArray(clustM))
row
}
def serialize(tdsql: TDigestSQL): Any = serializeTD(tdsql.tdigest)
private[sketches] def serializeTD(td: TDigest): InternalRow = {
val TDigest(delta, maxDiscrete, nclusters, clusters) = td
val row = new GenericInternalRow(5)
row.setDouble(0, delta)
row.setInt(1, maxDiscrete)
row.setInt(2, nclusters)
val clustX = clusters.keys.toArray
val clustM = clusters.values.toArray
row.update(3, UnsafeArrayData.fromPrimitiveArray(clustX))
row.update(4, UnsafeArrayData.fromPrimitiveArray(clustM))
row
}](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-143-320.jpg)
![#SAISDD6
def serialize(tdsql: TDigestSQL): Any = serializeTD(tdsql.tdigest)
private[sketches] def serializeTD(td: TDigest): InternalRow = {
val TDigest(delta, maxDiscrete, nclusters, clusters) = td
val row = new GenericInternalRow(5)
row.setDouble(0, delta)
row.setInt(1, maxDiscrete)
row.setInt(2, nclusters)
val clustX = clusters.keys.toArray
val clustM = clusters.values.toArray
row.update(3, UnsafeArrayData.fromPrimitiveArray(clustX))
row.update(4, UnsafeArrayData.fromPrimitiveArray(clustM))
row
}
def serialize(tdsql: TDigestSQL): Any = serializeTD(tdsql.tdigest)
private[sketches] def serializeTD(td: TDigest): InternalRow = {
val TDigest(delta, maxDiscrete, nclusters, clusters) = td
val row = new GenericInternalRow(5)
row.setDouble(0, delta)
row.setInt(1, maxDiscrete)
row.setInt(2, nclusters)
val clustX = clusters.keys.toArray
val clustM = clusters.values.toArray
row.update(3, UnsafeArrayData.fromPrimitiveArray(clustX))
row.update(4, UnsafeArrayData.fromPrimitiveArray(clustM))
row
}](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-144-320.jpg)
![#SAISDD6
def deserialize(td: Any): TDigestSQL = TDigestSQL(deserializeTD(td))
private[sketches] def deserializeTD(datum: Any): TDigest =
datum match { case row: InternalRow =>
val delta = row.getDouble(0)
val maxDiscrete = row.getInt(1)
val nclusters = row.getInt(2)
val clustX = row.getArray(3).toDoubleArray()
val clustM = row.getArray(4).toDoubleArray()
val clusters = clustX.zip(clustM)
.foldLeft(TDigestMap.empty) { case (td, e) => td + e }
TDigest(delta, maxDiscrete, nclusters, clusters)
}
def deserialize(td: Any): TDigestSQL = TDigestSQL(deserializeTD(td))
private[sketches] def deserializeTD(datum: Any): TDigest =
datum match { case row: InternalRow =>
val delta = row.getDouble(0)
val maxDiscrete = row.getInt(1)
val nclusters = row.getInt(2)
val clustX = row.getArray(3).toDoubleArray()
val clustM = row.getArray(4).toDoubleArray()
val clusters = clustX.zip(clustM)
.foldLeft(TDigestMap.empty) { case (td, e) => td + e }
TDigest(delta, maxDiscrete, nclusters, clusters)
}](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-145-320.jpg)
![#SAISDD6
def deserialize(td: Any): TDigestSQL = TDigestSQL(deserializeTD(td))
private[sketches] def deserializeTD(datum: Any): TDigest =
datum match { case row: InternalRow =>
val delta = row.getDouble(0)
val maxDiscrete = row.getInt(1)
val nclusters = row.getInt(2)
val clustX = row.getArray(3).toDoubleArray()
val clustM = row.getArray(4).toDoubleArray()
val clusters = clustX.zip(clustM)
.foldLeft(TDigestMap.empty) { case (td, e) => td + e }
TDigest(delta, maxDiscrete, nclusters, clusters)
}
def deserialize(td: Any): TDigestSQL = TDigestSQL(deserializeTD(td))
private[sketches] def deserializeTD(datum: Any): TDigest =
datum match { case row: InternalRow =>
val delta = row.getDouble(0)
val maxDiscrete = row.getInt(1)
val nclusters = row.getInt(2)
val clustX = row.getArray(3).toDoubleArray()
val clustM = row.getArray(4).toDoubleArray()
val clusters = clustX.zip(clustM)
.foldLeft(TDigestMap.empty) { case (td, e) => td + e }
TDigest(delta, maxDiscrete, nclusters, clusters)
}](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-146-320.jpg)
![#SAISDD6
def deserialize(td: Any): TDigestSQL = TDigestSQL(deserializeTD(td))
private[sketches] def deserializeTD(datum: Any): TDigest =
datum match { case row: InternalRow =>
val delta = row.getDouble(0)
val maxDiscrete = row.getInt(1)
val nclusters = row.getInt(2)
val clustX = row.getArray(3).toDoubleArray()
val clustM = row.getArray(4).toDoubleArray()
val clusters = clustX.zip(clustM)
.foldLeft(TDigestMap.empty) { case (td, e) => td + e }
TDigest(delta, maxDiscrete, nclusters, clusters)
}
def deserialize(td: Any): TDigestSQL = TDigestSQL(deserializeTD(td))
private[sketches] def deserializeTD(datum: Any): TDigest =
datum match { case row: InternalRow =>
val delta = row.getDouble(0)
val maxDiscrete = row.getInt(1)
val nclusters = row.getInt(2)
val clustX = row.getArray(3).toDoubleArray()
val clustM = row.getArray(4).toDoubleArray()
val clusters = clustX.zip(clustM)
.foldLeft(TDigestMap.empty) { case (td, e) => td + e }
TDigest(delta, maxDiscrete, nclusters, clusters)
}](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-147-320.jpg)
![#SAISDD6
[ ]](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-149-320.jpg)
![#SAISDD6
[ ]](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-150-320.jpg)
![#SAISDD6
[ ]](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-151-320.jpg)
![#SAISDD6
[ ]](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-152-320.jpg)
}
package org.isarnproject.sketches.udaf
object pythonBindings {
def tdigestDoubleUDAF(delta: Double, maxDiscrete: Int) =
TDigestUDAF[Double](delta, maxDiscrete)
}](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-156-320.jpg)
}
tdigestDoubleUDAF](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-157-320.jpg)
}
Double](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-158-320.jpg)
![#SAISDD6
from pyspark.sql.column import Column, _to_java_column, _to_seq
from pyspark.context import SparkContext
# one of these for each type parameter Double, Int, Long, etc
def tdigestDoubleUDAF(col, delta=0.5, maxDiscrete=0):
sc = SparkContext._active_spark_context
pb = sc._jvm.org.isarnproject.sketches.udaf.pythonBindings
tdapply = pb.tdigestDoubleUDAF(delta, maxDiscrete).apply
return Column(tdapply(_to_seq(sc, [col], _to_java_column)))
from pyspark.sql.column import Column, _to_java_column, _to_seq
from pyspark.context import SparkContext
# one of these for each type parameter Double, Int, Long, etc
def tdigestDoubleUDAF(col, delta=0.5, maxDiscrete=0):
sc = SparkContext._active_spark_context
pb = sc._jvm.org.isarnproject.sketches.udaf.pythonBindings
tdapply = pb.tdigestDoubleUDAF(delta, maxDiscrete).apply
return Column(tdapply(_to_seq(sc, [col], _to_java_column)))](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-159-320.jpg)
![#SAISDD6
from pyspark.sql.column import Column, _to_java_column, _to_seq
from pyspark.context import SparkContext
# one of these for each type parameter Double, Int, Long, etc
def tdigestDoubleUDAF(col, delta=0.5, maxDiscrete=0):
sc = SparkContext._active_spark_context
pb = sc._jvm.org.isarnproject.sketches.udaf.pythonBindings
tdapply = pb.tdigestDoubleUDAF(delta, maxDiscrete).apply
return Column(tdapply(_to_seq(sc, [col], _to_java_column)))
tdapply apply](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-160-320.jpg)
![#SAISDD6
from pyspark.sql.column import Column, _to_java_column, _to_seq
from pyspark.context import SparkContext
# one of these for each type parameter Double, Int, Long, etc
def tdigestDoubleUDAF(col, delta=0.5, maxDiscrete=0):
sc = SparkContext._active_spark_context
pb = sc._jvm.org.isarnproject.sketches.udaf.pythonBindings
tdapply = pb.tdigestDoubleUDAF(delta, maxDiscrete).apply
return Column(tdapply(_to_seq(sc, [col], _to_java_column)))tdapply(_to_seq(sc, [col], _to_java_column))](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-161-320.jpg)
![#SAISDD6
class TDigestUDT(UserDefinedType):
@classmethod
def sqlType(cls):
return StructType([
StructField("delta", DoubleType(), False),
StructField("maxDiscrete", IntegerType(), False),
StructField("nclusters", IntegerType(), False),
StructField("clustX", ArrayType(DoubleType(), False), False),
StructField("clustM", ArrayType(DoubleType(), False), False)])
class TDigestUDT(UserDefinedType):
@classmethod
def sqlType(cls):
return StructType([
StructField("delta", DoubleType(), False),
StructField("maxDiscrete", IntegerType(), False),
StructField("nclusters", IntegerType(), False),
StructField("clustX", ArrayType(DoubleType(), False), False),
StructField("clustM", ArrayType(DoubleType(), False), False)])
# ...](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-162-320.jpg)
![#SAISDD6
class TDigestUDT(UserDefinedType):
@classmethod
def sqlType(cls):
return StructType([
StructField("delta", DoubleType(), False),
StructField("maxDiscrete", IntegerType(), False),
StructField("nclusters", IntegerType(), False),
StructField("clustX", ArrayType(DoubleType(), False), False),
StructField("clustM", ArrayType(DoubleType(), False), False)])
class TDigestUDT(UserDefinedType):
@classmethod
def sqlType(cls):
return StructType([
StructField("delta", DoubleType(), False),
StructField("maxDiscrete", IntegerType(), False),
StructField("nclusters", IntegerType(), False),
StructField("clustX", ArrayType(DoubleType(), False), False),
StructField("clustM", ArrayType(DoubleType(), False), False)])
# ...](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-163-320.jpg)
![#SAISDD6
class TDigestUDT(UserDefinedType):
# ...
def serialize(self, obj):
return (obj.delta, obj.maxDiscrete, obj.nclusters,
[float(v) for v in obj.clustX],
[float(v) for v in obj.clustM])
def deserialize(self, datum):
return TDigest(datum[0], datum[1], datum[2], datum[3], datum[4])
class TDigestUDT(UserDefinedType):
# ...
def serialize(self, obj):
return (obj.delta, obj.maxDiscrete, obj.nclusters,
[float(v) for v in obj.clustX],
[float(v) for v in obj.clustM])
def deserialize(self, datum):
return TDigest(datum[0], datum[1], datum[2], datum[3], datum[4])](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-167-320.jpg)
![#SAISDD6
class TDigestUDT(UserDefinedType):
# ...
def serialize(self, obj):
return (obj.delta, obj.maxDiscrete, obj.nclusters,
[float(v) for v in obj.clustX],
[float(v) for v in obj.clustM])
def deserialize(self, datum):
return TDigest(datum[0], datum[1], datum[2], datum[3], datum[4])
class TDigestUDT(UserDefinedType):
# ...
def serialize(self, obj):
return (obj.delta, obj.maxDiscrete, obj.nclusters,
[float(v) for v in obj.clustX],
[float(v) for v in obj.clustM])
def deserialize(self, datum):
return TDigest(datum[0], datum[1], datum[2], datum[3], datum[4])](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-168-320.jpg)
![#SAISDD6
class TDigestUDT extends UserDefinedType[TDigestSQL] {
// ...
override def pyUDT: String =
“isarnproject.sketches.udt.tdigest.TDigestUDT"
}
class TDigestUDT extends UserDefinedType[TDigestSQL] {
// ...
override def pyUDT: String =
“isarnproject.sketches.udt.tdigest.TDigestUDT"
}](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-169-320.jpg)
![#SAISDD6
Cross-building for Python
lazy val compilePython = taskKey[Unit]("Compile python files")
compilePython := {
val s: TaskStreams = streams.value
s.log.info("compiling python...")
val stat = (Seq(pythonCMD, "-m", "compileall", "python/") !)
if (stat != 0) {
throw new IllegalStateException("python compile failed")
}
}
(packageBin in Compile) <<=
(packageBin in Compile).dependsOn(compilePython)
lazy val compilePython = taskKey[Unit]("Compile python files")
compilePython := {
val s: TaskStreams = streams.value
s.log.info("compiling python...")
val stat = (Seq(pythonCMD, "-m", "compileall", "python/") !)
if (stat != 0) {
throw new IllegalStateException("python compile failed")
}
}
(packageBin in Compile) <<=
(packageBin in Compile).dependsOn(compilePython)](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-174-320.jpg)
![#SAISDD6
lazy val compilePython = taskKey[Unit]("Compile python files")
compilePython := {
val s: TaskStreams = streams.value
s.log.info("compiling python...")
val stat = (Seq(pythonCMD, "-m", "compileall", "python/") !)
if (stat != 0) {
throw new IllegalStateException("python compile failed")
}
}
(packageBin in Compile) <<=
(packageBin in Compile).dependsOn(compilePython)
lazy val compilePython = taskKey[Unit]("Compile python files")
compilePython := {
val s: TaskStreams = streams.value
s.log.info("compiling python...")
val stat = (Seq(pythonCMD, "-m", "compileall", "python/") !)
if (stat != 0) {
throw new IllegalStateException("python compile failed")
}
}
(packageBin in Compile) <<=
(packageBin in Compile).dependsOn(compilePython)](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-175-320.jpg)
![#SAISDD6
lazy val compilePython = taskKey[Unit]("Compile python files")
compilePython := {
val s: TaskStreams = streams.value
s.log.info("compiling python...")
val stat = (Seq(pythonCMD, "-m", "compileall", "python/") !)
if (stat != 0) {
throw new IllegalStateException("python compile failed")
}
}
(packageBin in Compile) <<=
(packageBin in Compile).dependsOn(compilePython)
lazy val compilePython = taskKey[Unit]("Compile python files")
compilePython := {
val s: TaskStreams = streams.value
s.log.info("compiling python...")
if (stat != 0) {
throw new IllegalStateException("python compile failed")
}
}
(packageBin in Compile) <<=
(packageBin in Compile).dependsOn(compilePython)
val stat = (Seq(pythonCMD, "-m", "compileall", "python/") !)](https://image.slidesharecdn.com/07erikerlandsonwilliambenton-181010205559/85/Apache-Spark-for-Library-Developers-with-Erik-Erlandson-and-William-Benton-176-320.jpg)
Apache Spark for Library Developers with Erik Erlandson and William Benton
- 1. Apache Spark for library developers William Benton willb@redhat.com @willb Erik Erlandson eje@redhat.com @manyangled
- 2. About Will
- 3. #SAISDD6 The Silex and Isarn libraries Reusable open-source code that works with Spark, factored from internal apps. We’ve tracked Spark releases since Spark 1.3.0. See https://silex.radanalytics.io and http://isarnproject.org
- 5. #SAISDD6 Forecast Basic considerations for reusable Spark code Generic functions for parallel collections Extending data frames with custom aggregates Exposing JVM libraries to Python Sharing your work with the world
- 7. #SAISDD6
- 8. #SAISDD6
- 9. #SAISDD6
- 10. #SAISDD6
- 11. #SAISDD6
- 12. #SAISDD6
- 13. #SAISDD6
- 14. #SAISDD6
- 15. #SAISDD6
- 17. #SAISDD6 in your SBT build definition: Cross-building for Scala scalaVersion := "2.11.11" crossScalaVersions := Seq("2.10.6", "2.11.11") in your shell: $ sbt +compile $ sbt "++ 2.11.11" compile scalaVersion := "2.11.11" crossScalaVersions := Seq("2.10.6", "2.11.11")
- 18. #SAISDD6 in your SBT build definition: Cross-building for Scala scalaVersion := "2.11.11" crossScalaVersions := Seq("2.10.6", "2.11.11") in your shell: $ sbt +compile $ sbt "++ 2.11.11" compile $ sbt +compile # or test, package, publish, etc. $ sbt "++ 2.11.11" compile scalaVersion := "2.11.11" crossScalaVersions := Seq("2.10.6", "2.11.11")
- 19. #SAISDD6 in your SBT build definition: Cross-building for Scala scalaVersion := "2.11.11" crossScalaVersions := Seq("2.10.6", "2.11.11") in your shell: $ sbt +compile $ sbt "++ 2.11.11" compile $ sbt +compile # or test, package, publish, etc. $ sbt "++ 2.11.11" compile scalaVersion := "2.11.11" crossScalaVersions := Seq("2.10.6", "2.11.11")
- 20. #SAISDD6 in your SBT build definition: Bring-your-own Spark libraryDependencies ++= Seq( "org.apache.spark" %% "spark-core" % "2.3.0" % Provided, "org.apache.spark" %% "spark-sql" % "2.3.0" % Provided, "org.apache.spark" %% "spark-mllib" % "2.3.0" % Provided, "org.scalatest" %% "scalatest" % "2.2.4" % Test) libraryDependencies ++= Seq( "org.apache.spark" %% "spark-core" % "2.3.0" % Provided, "org.apache.spark" %% "spark-sql" % "2.3.0" % Provided, "org.apache.spark" %% "spark-mllib" % "2.3.0" % Provided, "org.scalatest" %% "scalatest" % "2.2.4" % Test)
- 21. #SAISDD6 in your SBT build definition: Bring-your-own Spark libraryDependencies ++= Seq( "org.apache.spark" %% "spark-core" % "2.3.0" % Provided, "org.apache.spark" %% "spark-sql" % "2.3.0" % Provided, "org.apache.spark" %% "spark-mllib" % "2.3.0" % Provided, "org.scalatest" %% "scalatest" % "2.2.4" % Test) libraryDependencies ++= Seq( "org.apache.spark" %% "spark-core" % "2.3.0" % Provided, "org.apache.spark" %% "spark-sql" % "2.3.0" % Provided, "org.apache.spark" %% "spark-mllib" % "2.3.0" % Provided, "org.scalatest" %% "scalatest" % "2.2.4" % Test)
- 22. #SAISDD6 in your SBT build definition: “Bring-your-own Spark” libraryDependencies ++= Seq( "org.apache.spark" %% "spark-core" % "2.3.0" % Provided, "org.apache.spark" %% "spark-sql" % "2.3.0" % Provided, "org.apache.spark" %% "spark-mllib" % "2.3.0" % Provided, "joda-time" % "joda-time" % "2.7", "org.scalatest" %% "scalatest" % "2.2.4" % Test) libraryDependencies ++= Seq( "org.apache.spark" %% "spark-core" % "2.3.0" % Provided, "org.apache.spark" %% "spark-sql" % "2.3.0" % Provided, "org.apache.spark" %% "spark-mllib" % "2.3.0" % Provided, "joda-time" % "joda-time" % "2.7", "org.scalatest" %% "scalatest" % "2.2.4" % Test)
- 23. #SAISDD6 in your SBT build definition: “Bring-your-own Spark” libraryDependencies ++= Seq( "org.apache.spark" %% "spark-core" % "2.3.0" % Provided, "org.apache.spark" %% "spark-sql" % "2.3.0" % Provided, "org.apache.spark" %% "spark-mllib" % "2.3.0" % Provided, "joda-time" % "joda-time" % "2.7", "org.scalatest" %% "scalatest" % "2.2.4" % Test) libraryDependencies ++= Seq( "org.apache.spark" %% "spark-core" % "2.3.0" % Provided, "org.apache.spark" %% "spark-sql" % "2.3.0" % Provided, "org.apache.spark" %% "spark-mllib" % "2.3.0" % Provided, "joda-time" % "joda-time" % "2.7", "org.scalatest" %% "scalatest" % "2.2.4" % Test)
- 24. #SAISDD6 Taking care with resources
- 25. #SAISDD6 Taking care with resources
- 26. #SAISDD6 Taking care with resources
- 27. #SAISDD6 def step(rdd: RDD[_]) = { val wasUncached = rdd.storageLevel == StorageLevel.NONE if (wasUncached) { rdd.cache() } result = trainModel(rdd) if (wasUncached) { rdd.unpersist() } result } Caching when necessary def step(rdd: RDD[_]) = { val wasUncached = rdd.storageLevel == StorageLevel.NONE if (wasUncached) { rdd.cache() } result = trainModel(rdd) if (wasUncached) { rdd.unpersist() } result } rdd.cache()
- 28. #SAISDD6 def step(rdd: RDD[_]) = { val wasUncached = rdd.storageLevel == StorageLevel.NONE if (wasUncached) { rdd.cache() } result = trainModel(rdd) if (wasUncached) { rdd.unpersist() } result } Caching when necessary def step(rdd: RDD[_]) = { val wasUncached = rdd.storageLevel == StorageLevel.NONE if (wasUncached) { rdd.cache() } result = trainModel(rdd) if (wasUncached) { rdd.unpersist() } result } rdd.cache()
- 29. #SAISDD6 def step(rdd: RDD[_]) = { val wasUncached = rdd.storageLevel == StorageLevel.NONE if (wasUncached) { rdd.cache() } result = trainModel(rdd) if (wasUncached) { rdd.unpersist() } result } Caching when necessary def step(rdd: RDD[_]) = { val wasUncached = rdd.storageLevel == StorageLevel.NONE if (wasUncached) { rdd.cache() } result = trainModel(rdd) if (wasUncached) { rdd.unpersist() } result } rdd.cache() rdd.unpersist()
- 30. #SAISDD6 def step(rdd: RDD[_]) = { val wasUncached = rdd.storageLevel == StorageLevel.NONE if (wasUncached) { rdd.cache() } result = trainModel(rdd) result } Caching when necessary def step(rdd: RDD[_]) = { val wasUncached = rdd.storageLevel == StorageLevel.NONE if (wasUncached) { rdd.cache() } result = trainModel(rdd) if (wasUncached) { rdd.unpersist() } result }
- 31. #SAISDD6 def step(rdd: RDD[_]) = { val wasUncached = rdd.storageLevel == StorageLevel.NONE if (wasUncached) { rdd.cache() } result = trainModel(rdd) if (wasUncached) { rdd.unpersist() } result } Caching when necessary def step(rdd: RDD[_]) = { val wasUncached = rdd.storageLevel == StorageLevel.NONE if (wasUncached) { rdd.cache() } result = trainModel(rdd) if (wasUncached) { rdd.unpersist() } result }
- 32. #SAISDD6 nextModel = modelFromState(newState) current.unpersist } var nextModel = initialModel for (int i = 0; i < iterations; i++) { val current = sc.broadcast(nextModel) val newState = current.unpersist sc.broadcast(nextModel)
- 33. #SAISDD6 nextModel = modelFromState(newState) current.unpersist } var nextModel = initialModel for (int i = 0; i < iterations; i++) { val current = sc.broadcast(nextModel) val newState = current.unpersist sc.broadcast(nextModel)
- 34. #SAISDD6 nextModel = modelFromState(newState) current.unpersist } var nextModel = initialModel for (int i = 0; i < iterations; i++) { val current = sc.broadcast(nextModel) val newState = current.unpersist sc.broadcast(nextModel)
- 35. #SAISDD6 Minding the JVM heap val mat = Array(Array(1.0, 2.0), Array(3.0, 4.0))
- 36. #SAISDD6 Minding the JVM heap val mat = Array(Array(1.0, 2.0), Array(3.0, 4.0)) class pointer flags size locks element pointer element pointer class pointer flags size locks 1.0 class pointer flags size locks 3.0 4.0 2.0
- 37. #SAISDD6 Minding the JVM heap val mat = Array(Array(1.0, 2.0), Array(3.0, 4.0)) class pointer flags size locks element pointer element pointer class pointer flags size locks 1.0 class pointer flags size locks 3.0 4.0 2.0 32 bytes of data…
- 38. #SAISDD6 Minding the JVM heap val mat = Array(Array(1.0, 2.0), Array(3.0, 4.0)) class pointer flags size locks element pointer element pointer class pointer flags size locks 1.0 class pointer flags size locks 3.0 4.0 2.0 …and 64 bytes of overhead! 32 bytes of data…
- 39. Continuous integration for Spark libraries and apps
- 42. #SAISDD6
- 43. #SAISDD6
- 44. #SAISDD6
- 46. #SAISDD6
- 47. #SAISDD6
- 48. Writing generic code for Spark’s parallel collections
- 49. #SAISDD6 The RDD is invariant T <: U RDD[T] <: RDD[U]
- 50. #SAISDD6 The RDD is invariant T <: U RDD[T] <: RDD[U] dog animal
- 51. #SAISDD6 T <: U RDD[T] <: RDD[U] trait HasUserId { val userid: Int } case class Transaction(override val userid: Int, timestamp: Int, amount: Double) extends HasUserId {} def badKeyByUserId(r: RDD[HasUserId]) = r.map(x => (x.userid, x)) trait HasUserId { val userid: Int } case class Transaction(override val userid: Int, timestamp: Int, amount: Double) extends HasUserId {} def badKeyByUserId(r: RDD[HasUserId]) = r.map(x => (x.userid, x))
- 52. #SAISDD6 T <: U RDD[T] <: RDD[U] trait HasUserId { val userid: Int } case class Transaction(override val userid: Int, timestamp: Int, amount: Double) extends HasUserId {} def badKeyByUserId(r: RDD[HasUserId]) = r.map(x => (x.userid, x)) trait HasUserId { val userid: Int } case class Transaction(override val userid: Int, timestamp: Int, amount: Double) extends HasUserId {} def badKeyByUserId(r: RDD[HasUserId]) = r.map(x => (x.userid, x)) trait HasUserId { val userid: Int } case class Transaction(override val userid: Int, timestamp: Int, amount: Double) extends HasUserId {} def badKeyByUserId(r: RDD[HasUserId]) = r.map(x => (x.userid, x)) trait HasUserId { val userid: Int } case class Transaction(override val userid: Int, timestamp: Int, amount: Double) extends HasUserId {} def badKeyByUserId(r: RDD[HasUserId]) = r.map(x => (x.userid, x))
- 53. #SAISDD6 val xacts = spark.parallelize(Array( Transaction(1, 1, 1.0), Transaction(2, 2, 1.0) )) badKeyByUserId(xacts) <console>: error: type mismatch; found : org.apache.spark.rdd.RDD[Transaction] required: org.apache.spark.rdd.RDD[HasUserId] Note: Transaction <: HasUserID, but class RDD is invariant in type T. You may wish to define T as +T instead. (SLS 4.5) badKeyByUserId(xacts)
- 54. #SAISDD6 val xacts = spark.parallelize(Array( Transaction(1, 1, 1.0), Transaction(2, 2, 1.0) )) badKeyByUserId(xacts) <console>: error: type mismatch; found : org.apache.spark.rdd.RDD[Transaction] required: org.apache.spark.rdd.RDD[HasUserId] Note: Transaction <: HasUserID, but class RDD is invariant in type T. You may wish to define T as +T instead. (SLS 4.5) badKeyByUserId(xacts)
- 55. #SAISDD6
- 56. #SAISDD6 An example: natural join A B C D E A EB X Y
- 57. #SAISDD6 An example: natural join A B C D E A EB X Y
- 58. #SAISDD6 An example: natural join A B C D E X Y
- 59. #SAISDD6 Ad-hoc natural join df1.join(df2, df1("a") === df2("a") && df1("b") === df2("b") && df1("e") === df2("e"))
- 60. #SAISDD6 = { val lcols = left.columns val rcols = right.columns val ccols = lcols.toSet intersect rcols.toSet if(ccols.isEmpty) left.limit(0).crossJoin(right.limit(0)) else left .join(right, ccols.map {col => left(col) === right(col) }.reduce(_ && _)) .select(lcols.collect { case c if ccols.contains(c) => left(c) } ++ lcols.collect { case c if !ccols.contains(c) => left(c) } ++ rcols.collect { case c if !ccols.contains(c) => right(c) } : _*) } def natjoin(left: DataFrame, right: DataFrame): DataFrame
- 61. #SAISDD6 def natjoin(left: DataFrame, right: DataFrame): DataFrame = { val lcols = left.columns val rcols = right.columns val ccols = lcols.toSet intersect rcols.toSet if(ccols.isEmpty) left.limit(0).crossJoin(right.limit(0)) else left .join(right, ccols.map {col => left(col) === right(col) }.reduce(_ && _)) .select(lcols.collect { case c if ccols.contains(c) => left(c) } ++ lcols.collect { case c if !ccols.contains(c) => left(c) } ++ rcols.collect { case c if !ccols.contains(c) => right(c) } : _*) } def natjoin(left: DataFrame, right: DataFrame): DataFrame = { val lcols = left.columns val rcols = right.columns val ccols = lcols.toSet intersect rcols.toSet if(ccols.isEmpty) left.limit(0).crossJoin(right.limit(0)) else left .join(right, ccols.map {col => left(col) === right(col) }.reduce(_ && _)) .select(lcols.collect { case c if ccols.contains(c) => left(c) } ++ lcols.collect { case c if !ccols.contains(c) => left(c) } ++ rcols.collect { case c if !ccols.contains(c) => right(c) } : _*) }
- 62. #SAISDD6 def natjoin(left: DataFrame, right: DataFrame): DataFrame = { val lcols = left.columns val rcols = right.columns val ccols = lcols.toSet intersect rcols.toSet if(ccols.isEmpty) left.limit(0).crossJoin(right.limit(0)) else left .join(right, ccols.map {col => left(col) === right(col) }.reduce(_ && _)) .select(lcols.collect { case c if ccols.contains(c) => left(c) } ++ lcols.collect { case c if !ccols.contains(c) => left(c) } ++ rcols.collect { case c if !ccols.contains(c) => right(c) } : _*) } def natjoin(left: DataFrame, right: DataFrame): DataFrame = { val lcols = left.columns val rcols = right.columns val ccols = lcols.toSet intersect rcols.toSet if(ccols.isEmpty) left.limit(0).crossJoin(right.limit(0)) else left .join(right, ccols.map {col => left(col) === right(col) }.reduce(_ && _)) .select(lcols.collect { case c if ccols.contains(c) => left(c) } ++ lcols.collect { case c if !ccols.contains(c) => left(c) } ++ rcols.collect { case c if !ccols.contains(c) => right(c) } : _*) } introspecting over column names
- 63. #SAISDD6 def natjoin(left: DataFrame, right: DataFrame): DataFrame = { val lcols = left.columns val rcols = right.columns val ccols = lcols.toSet intersect rcols.toSet if(ccols.isEmpty) left.limit(0).crossJoin(right.limit(0)) else left .join(right, ccols.map {col => left(col) === right(col) }.reduce(_ && _)) .select(lcols.collect { case c if ccols.contains(c) => left(c) } ++ lcols.collect { case c if !ccols.contains(c) => left(c) } ++ rcols.collect { case c if !ccols.contains(c) => right(c) } : _*) } def natjoin(left: DataFrame, right: DataFrame): DataFrame = { val lcols = left.columns val rcols = right.columns val ccols = lcols.toSet intersect rcols.toSet if(ccols.isEmpty) left.limit(0).crossJoin(right.limit(0)) else left .join(right, ccols.map {col => left(col) === right(col) }.reduce(_ && _)) .select(lcols.collect { case c if ccols.contains(c) => left(c) } ++ lcols.collect { case c if !ccols.contains(c) => left(c) } ++ rcols.collect { case c if !ccols.contains(c) => right(c) } : _*) }
- 64. #SAISDD6 def natjoin(left: DataFrame, right: DataFrame): DataFrame = { val lcols = left.columns val rcols = right.columns val ccols = lcols.toSet intersect rcols.toSet if(ccols.isEmpty) left.limit(0).crossJoin(right.limit(0)) else left .join(right, ccols.map {col => left(col) === right(col) }.reduce(_ && _)) .select(lcols.collect { case c if ccols.contains(c) => left(c) } ++ lcols.collect { case c if !ccols.contains(c) => left(c) } ++ rcols.collect { case c if !ccols.contains(c) => right(c) } : _*) } def natjoin(left: DataFrame, right: DataFrame): DataFrame = { val lcols = left.columns val rcols = right.columns val ccols = lcols.toSet intersect rcols.toSet if(ccols.isEmpty) left.limit(0).crossJoin(right.limit(0)) else left .join(right, ccols.map {col => left(col) === right(col) }.reduce(_ && _)) .select(lcols.collect { case c if ccols.contains(c) => left(c) } ++ lcols.collect { case c if !ccols.contains(c) => left(c) } ++ rcols.collect { case c if !ccols.contains(c) => right(c) } : _*) } dynamically constructing expressions
- 65. #SAISDD6 def natjoin(left: DataFrame, right: DataFrame): DataFrame = { val lcols = left.columns val rcols = right.columns val ccols = lcols.toSet intersect rcols.toSet if(ccols.isEmpty) left.limit(0).crossJoin(right.limit(0)) else left .join(right, ccols.map {col => left(col) === right(col) }.reduce(_ && _)) .select(lcols.collect { case c if ccols.contains(c) => left(c) } ++ lcols.collect { case c if !ccols.contains(c) => left(c) } ++ rcols.collect { case c if !ccols.contains(c) => right(c) } : _*) } def natjoin(left: DataFrame, right: DataFrame): DataFrame = { val lcols = left.columns val rcols = right.columns val ccols = lcols.toSet intersect rcols.toSet if(ccols.isEmpty) left.limit(0).crossJoin(right.limit(0)) else left .join(right, ccols.map {col => left(col) === right(col) }.reduce(_ && _)) .select(lcols.collect { case c if ccols.contains(c) => left(c) } ++ lcols.collect { case c if !ccols.contains(c) => left(c) } ++ rcols.collect { case c if !ccols.contains(c) => right(c) } : _*) } dynamically constructing expressions
- 66. #SAISDD6 def natjoin(left: DataFrame, right: DataFrame): DataFrame = { val lcols = left.columns val rcols = right.columns val ccols = lcols.toSet intersect rcols.toSet if(ccols.isEmpty) left.limit(0).crossJoin(right.limit(0)) else left .join(right, ccols.map {col => left(col) === right(col) }.reduce(_ && _)) .select(lcols.collect { case c if ccols.contains(c) => left(c) } ++ lcols.collect { case c if !ccols.contains(c) => left(c) } ++ rcols.collect { case c if !ccols.contains(c) => right(c) } : _*) } def natjoin(left: DataFrame, right: DataFrame): DataFrame = { val lcols = left.columns val rcols = right.columns val ccols = lcols.toSet intersect rcols.toSet if(ccols.isEmpty) left.limit(0).crossJoin(right.limit(0)) else left .join(right, ccols.map {col => left(col) === right(col) }.reduce(_ && _)) .select(lcols.collect { case c if ccols.contains(c) => left(c) } ++ lcols.collect { case c if !ccols.contains(c) => left(c) } ++ rcols.collect { case c if !ccols.contains(c) => right(c) } : _*) } dynamically constructing expressions
- 67. #SAISDD6 def natjoin(left: DataFrame, right: DataFrame): DataFrame = { val lcols = left.columns val rcols = right.columns val ccols = lcols.toSet intersect rcols.toSet if(ccols.isEmpty) left.limit(0).crossJoin(right.limit(0)) else left .join(right, ccols.map {col => left(col) === right(col) }.reduce(_ && _)) .select(lcols.collect { case c if ccols.contains(c) => left(c) } ++ lcols.collect { case c if !ccols.contains(c) => left(c) } ++ rcols.collect { case c if !ccols.contains(c) => right(c) } : _*) } def natjoin(left: DataFrame, right: DataFrame): DataFrame = { val lcols = left.columns val rcols = right.columns val ccols = lcols.toSet intersect rcols.toSet if(ccols.isEmpty) left.limit(0).crossJoin(right.limit(0)) else left .join(right, ccols.map {col => left(col) === right(col) }.reduce(_ && _)) .select(lcols.collect { case c if ccols.contains(c) => left(c) } ++ lcols.collect { case c if !ccols.contains(c) => left(c) } ++ rcols.collect { case c if !ccols.contains(c) => right(c) } : _*) } [left.a === right.a, left.b === right.b, …]
- 68. #SAISDD6 def natjoin(left: DataFrame, right: DataFrame): DataFrame = { val lcols = left.columns val rcols = right.columns val ccols = lcols.toSet intersect rcols.toSet if(ccols.isEmpty) left.limit(0).crossJoin(right.limit(0)) else left .join(right, ccols.map {col => left(col) === right(col) }.reduce(_ && _)) .select(lcols.collect { case c if ccols.contains(c) => left(c) } ++ lcols.collect { case c if !ccols.contains(c) => left(c) } ++ rcols.collect { case c if !ccols.contains(c) => right(c) } : _*) } def natjoin(left: DataFrame, right: DataFrame): DataFrame = { val lcols = left.columns val rcols = right.columns val ccols = lcols.toSet intersect rcols.toSet if(ccols.isEmpty) left.limit(0).crossJoin(right.limit(0)) else left .join(right, ccols.map {col => left(col) === right(col) }.reduce(_ && _)) .select(lcols.collect { case c if ccols.contains(c) => left(c) } ++ lcols.collect { case c if !ccols.contains(c) => left(c) } ++ rcols.collect { case c if !ccols.contains(c) => right(c) } : _*) } left.a === right.a && left.b === right.b && …
- 69. #SAISDD6 def natjoin(left: DataFrame, right: DataFrame): DataFrame = { val lcols = left.columns val rcols = right.columns val ccols = lcols.toSet intersect rcols.toSet if(ccols.isEmpty) left.limit(0).crossJoin(right.limit(0)) else left .join(right, ccols.map {col => left(col) === right(col) }.reduce(_ && _)) .select(lcols.collect { case c if ccols.contains(c) => left(c) } ++ lcols.collect { case c if !ccols.contains(c) => left(c) } ++ rcols.collect { case c if !ccols.contains(c) => right(c) } : _*) } def natjoin(left: DataFrame, right: DataFrame): DataFrame = { val lcols = left.columns val rcols = right.columns val ccols = lcols.toSet intersect rcols.toSet if(ccols.isEmpty) left.limit(0).crossJoin(right.limit(0)) else left .join(right, ccols.map {col => left(col) === right(col) }.reduce(_ && _)) .select(lcols.collect { case c if ccols.contains(c) => left(c) } ++ lcols.collect { case c if !ccols.contains(c) => left(c) } ++ rcols.collect { case c if !ccols.contains(c) => right(c) } : _*) } left.a === right.a && left.b === right.b && …
- 70. #SAISDD6 def natjoin(left: DataFrame, right: DataFrame): DataFrame = { val lcols = left.columns val rcols = right.columns val ccols = lcols.toSet intersect rcols.toSet if(ccols.isEmpty) left.limit(0).crossJoin(right.limit(0)) else left .join(right, ccols.map {col => left(col) === right(col) }.reduce(_ && _)) .select(lcols.collect { case c if ccols.contains(c) => left(c) } ++ lcols.collect { case c if !ccols.contains(c) => left(c) } ++ rcols.collect { case c if !ccols.contains(c) => right(c) } : _*) } def natjoin(left: DataFrame, right: DataFrame): DataFrame = { val lcols = left.columns val rcols = right.columns val ccols = lcols.toSet intersect rcols.toSet if(ccols.isEmpty) left.limit(0).crossJoin(right.limit(0)) else left .join(right, ccols.map {col => left(col) === right(col) }.reduce(_ && _)) .select(lcols.collect { case c if ccols.contains(c) => left(c) } ++ lcols.collect { case c if !ccols.contains(c) => left(c) } ++ rcols.collect { case c if !ccols.contains(c) => right(c) } : _*) }
- 71. #SAISDD6 def natjoin(left: DataFrame, right: DataFrame): DataFrame = { val lcols = left.columns val rcols = right.columns val ccols = lcols.toSet intersect rcols.toSet if(ccols.isEmpty) left.limit(0).crossJoin(right.limit(0)) else left .join(right, ccols.map {col => left(col) === right(col) }.reduce(_ && _)) .select(lcols.collect { case c if ccols.contains(c) => left(c) } ++ lcols.collect { case c if !ccols.contains(c) => left(c) } ++ rcols.collect { case c if !ccols.contains(c) => right(c) } : _*) } def natjoin(left: DataFrame, right: DataFrame): DataFrame = { val lcols = left.columns val rcols = right.columns val ccols = lcols.toSet intersect rcols.toSet if(ccols.isEmpty) left.limit(0).crossJoin(right.limit(0)) else left .join(right, ccols.map {col => left(col) === right(col) }.reduce(_ && _)) .select(lcols.collect { case c if ccols.contains(c) => left(c) } ++ lcols.collect { case c if !ccols.contains(c) => left(c) } ++ rcols.collect { case c if !ccols.contains(c) => right(c) } : _*) } dynamically constructing column lists
- 72. #SAISDD6 def natjoin(left: DataFrame, right: DataFrame): DataFrame = { val lcols = left.columns val rcols = right.columns val ccols = lcols.toSet intersect rcols.toSet if(ccols.isEmpty) left.limit(0).crossJoin(right.limit(0)) else left .join(right, ccols.map {col => left(col) === right(col) }.reduce(_ && _)) .select(lcols.collect { case c if ccols.contains(c) => left(c) } ++ lcols.collect { case c if !ccols.contains(c) => left(c) } ++ rcols.collect { case c if !ccols.contains(c) => right(c) } : _*) } def natjoin(left: DataFrame, right: DataFrame): DataFrame = { val lcols = left.columns val rcols = right.columns val ccols = lcols.toSet intersect rcols.toSet if(ccols.isEmpty) left.limit(0).crossJoin(right.limit(0)) else left .join(right, ccols.map {col => left(col) === right(col) }.reduce(_ && _)) .select(lcols.collect { case c if ccols.contains(c) => left(c) } ++ lcols.collect { case c if !ccols.contains(c) => left(c) } ++ rcols.collect { case c if !ccols.contains(c) => right(c) } : _*) } dynamically constructing column lists
- 73. #SAISDD6 case class DFWithNatJoin(df: DataFrame) extends NaturalJoining { def natjoin(other: DataFrame): DataFrame = super.natjoin(df, other) } object NaturalJoin extends NaturalJoining { object implicits { implicit def dfWithNatJoin(df: DataFrame) = DFWithNatJoin(df) } } import NaturalJoin.implicits._ df.natjoin(otherdf) case class DFWithNatJoin(df: DataFrame) extends NaturalJoining { def natjoin(other: DataFrame): DataFrame = super.natjoin(df, other) } object NaturalJoin extends NaturalJoining { object implicits { implicit def dfWithNatJoin(df: DataFrame) = DFWithNatJoin(df) } } import NaturalJoin.implicits._ df.natjoin(otherdf)
- 74. #SAISDD6 case class DFWithNatJoin(df: DataFrame) extends NaturalJoining { def natjoin(other: DataFrame): DataFrame = super.natjoin(df, other) } object NaturalJoin extends NaturalJoining { object implicits { implicit def dfWithNatJoin(df: DataFrame) = DFWithNatJoin(df) } } import NaturalJoin.implicits._ df.natjoin(otherdf) case class DFWithNatJoin(df: DataFrame) extends NaturalJoining { def natjoin(other: DataFrame): DataFrame = super.natjoin(df, other) } object NaturalJoin extends NaturalJoining { object implicits { implicit def dfWithNatJoin(df: DataFrame) = DFWithNatJoin(df) } } import NaturalJoin.implicits._ df.natjoin(otherdf)
- 75. #SAISDD6 case class DFWithNatJoin(df: DataFrame) extends NaturalJoining { def natjoin(other: DataFrame): DataFrame = super.natjoin(df, other) } object NaturalJoin extends NaturalJoining { object implicits { implicit def dfWithNatJoin(df: DataFrame) = DFWithNatJoin(df) } } import NaturalJoin.implicits._ df.natjoin(otherdf)
- 76. #SAISDD6 User-defined functions {"a": 1, "b": "wilma", ..., "x": "club"} {"a": 2, "b": "betty", ..., "x": "diamond"} {"a": 3, "b": "fred", ..., "x": "heart"} {"a": 4, "b": "barney", ..., "x": "spade"}
- 77. #SAISDD6 User-defined functions {"a": 1, "b": "wilma", ..., "x": "club"} {"a": 2, "b": "betty", ..., "x": "diamond"} {"a": 3, "b": "fred", ..., "x": "heart"} {"a": 4, "b": "barney", ..., "x": "spade"} wilma club betty diamond fred heart barney spade
- 78. #SAISDD6 import json from pyspark.sql.types import * from pyspark.sql.functions import udf def selectively_structure(fields): resultType = StructType([StructField(f, StringType(), nullable=True) for f in fields]) def impl(js): try: d = json.loads(js) return [str(d.get(f)) for f in fields] except: return [None] * len(fields) return udf(impl, resultType) import json from pyspark.sql.types import * from pyspark.sql.functions import udf def selectively_structure(fields): resultType = StructType([StructField(f, StringType(), nullable=True) for f in fields]) def impl(js): try: d = json.loads(js) return [str(d.get(f)) for f in fields] except: return [None] * len(fields) return udf(impl, resultType)
- 79. #SAISDD6 import json from pyspark.sql.types import * from pyspark.sql.functions import udf def selectively_structure(fields): resultType = StructType([StructField(f, StringType(), nullable=True) for f in fields]) def impl(js): try: d = json.loads(js) return [str(d.get(f)) for f in fields] except: return [None] * len(fields) return udf(impl, resultType) import json from pyspark.sql.types import * from pyspark.sql.functions import udf def selectively_structure(fields): resultType = StructType([StructField(f, StringType(), nullable=True) for f in fields]) def impl(js): try: d = json.loads(js) return [str(d.get(f)) for f in fields] except: return [None] * len(fields) return udf(impl, resultType)
- 80. #SAISDD6 import json from pyspark.sql.types import * from pyspark.sql.functions import udf def selectively_structure(fields): resultType = StructType([StructField(f, StringType(), nullable=True) for f in fields]) def impl(js): try: d = json.loads(js) return [str(d.get(f)) for f in fields] except: return [None] * len(fields) return udf(impl, resultType) import json from pyspark.sql.types import * from pyspark.sql.functions import udf def selectively_structure(fields): resultType = StructType([StructField(f, StringType(), nullable=True) for f in fields]) def impl(js): try: d = json.loads(js) return [str(d.get(f)) for f in fields] except: return [None] * len(fields) return udf(impl, resultType)
- 81. #SAISDD6 import json from pyspark.sql.types import * from pyspark.sql.functions import udf def selectively_structure(fields): resultType = StructType([StructField(f, StringType(), nullable=True) for f in fields]) def impl(js): try: d = json.loads(js) return [str(d.get(f)) for f in fields] except: return [None] * len(fields) return udf(impl, resultType) import json from pyspark.sql.types import * from pyspark.sql.functions import udf def selectively_structure(fields): resultType = StructType([StructField(f, StringType(), nullable=True) for f in fields]) def impl(js): try: d = json.loads(js) return [str(d.get(f)) for f in fields] except: return [None] * len(fields) return udf(impl, resultType)
- 82. #SAISDD6 import json from pyspark.sql.types import * from pyspark.sql.functions import udf def selectively_structure(fields): resultType = StructType([StructField(f, StringType(), nullable=True) for f in fields]) def impl(js): try: d = json.loads(js) return [str(d.get(f)) for f in fields] except: return [None] * len(fields) return udf(impl, resultType) import json from pyspark.sql.types import * from pyspark.sql.functions import udf def selectively_structure(fields): resultType = StructType([StructField(f, StringType(), nullable=True) for f in fields]) def impl(js): try: d = json.loads(js) return [str(d.get(f)) for f in fields] except: return [None] * len(fields) return udf(impl, resultType)
- 83. #SAISDD6 import json from pyspark.sql.types import * from pyspark.sql.functions import udf def selectively_structure(fields): resultType = StructType([StructField(f, StringType(), nullable=True) for f in fields]) def impl(js): try: d = json.loads(js) return [str(d.get(f)) for f in fields] except: return [None] * len(fields) return udf(impl, resultType) import json from pyspark.sql.types import * from pyspark.sql.functions import udf def selectively_structure(fields): resultType = StructType([StructField(f, StringType(), nullable=True) for f in fields]) def impl(js): try: d = json.loads(js) return [str(d.get(f)) for f in fields] except: return [None] * len(fields) return udf(impl, resultType) extract_bx = selectively_structure(["b", "x"]) structured_df = df.withColumn("result", extract_bx("json"))
- 87. #SAISDD6 Spark’s ML pipelines estimator.fit(df) model.transform(df)
- 88. #SAISDD6 Working with ML pipelines model.transform(df)
- 89. #SAISDD6 Working with ML pipelines model.transform(df)
- 92. #SAISDD6 Spark’s ML pipelines estimator.fit(df) model.transform(df)
- 93. #SAISDD6 Spark’s ML pipelines estimator.fit(df) model.transform(df) inputCol epochs seed outputCol
- 94. #SAISDD6
- 95. #SAISDD6 Forecast Basic considerations for reusable Spark code Generic functions for parallel collections Extending data frames with custom aggregates Exposing JVM libraries to Python Sharing your work with the world
- 96. About Erik
- 104. #SAISDD6 case class TDigestUDAF[N](deltaV: Double, maxDiscreteV: Int) (implicit num: Numeric[N], dataTpe: TDigestUDAFDataType[N]) extends UserDefinedAggregateFunction { def deterministic: Boolean = false def inputSchema: StructType = StructType(StructField("x", dataTpe.tpe) :: Nil) def bufferSchema: StructType = StructType(StructField("tdigest", TDigestUDT) :: Nil) def dataType: DataType = TDigestUDT case class TDigestUDAF[N](deltaV: Double, maxDiscreteV: Int) (implicit num: Numeric[N], dataTpe: TDigestUDAFDataType[N]) extends UserDefinedAggregateFunction { def deterministic: Boolean = false def inputSchema: StructType = StructType(StructField("x", dataTpe.tpe) :: Nil) def bufferSchema: StructType = StructType(StructField("tdigest", TDigestUDT) :: Nil) def dataType: DataType = TDigestUDT
- 105. #SAISDD6 case class TDigestUDAF[N](deltaV: Double, maxDiscreteV: Int) (implicit num: Numeric[N], dataTpe: TDigestUDAFDataType[N]) extends UserDefinedAggregateFunction { def deterministic: Boolean = false def inputSchema: StructType = StructType(StructField("x", dataTpe.tpe) :: Nil) def bufferSchema: StructType = StructType(StructField("tdigest", TDigestUDT) :: Nil) def dataType: DataType = TDigestUDT case class TDigestUDAF[N](deltaV: Double, maxDiscreteV: Int) (implicit num: Numeric[N], dataTpe: TDigestUDAFDataType[N]) extends UserDefinedAggregateFunction { def deterministic: Boolean = false def inputSchema: StructType = StructType(StructField("x", dataTpe.tpe) :: Nil) def bufferSchema: StructType = StructType(StructField("tdigest", TDigestUDT) :: Nil) def dataType: DataType = TDigestUDT
- 106. #SAISDD6 case class TDigestUDAF[N](deltaV: Double, maxDiscreteV: Int) (implicit num: Numeric[N], dataTpe: TDigestUDAFDataType[N]) extends UserDefinedAggregateFunction { def deterministic: Boolean = false def inputSchema: StructType = StructType(StructField("x", dataTpe.tpe) :: Nil) def bufferSchema: StructType = StructType(StructField("tdigest", TDigestUDT) :: Nil) def dataType: DataType = TDigestUDT case class TDigestUDAF[N](deltaV: Double, maxDiscreteV: Int) (implicit num: Numeric[N], dataTpe: TDigestUDAFDataType[N]) extends UserDefinedAggregateFunction { def deterministic: Boolean = false def inputSchema: StructType = StructType(StructField("x", dataTpe.tpe) :: Nil) def bufferSchema: StructType = StructType(StructField("tdigest", TDigestUDT) :: Nil) def dataType: DataType = TDigestUDT
- 107. #SAISDD6 case class TDigestUDAF[N](deltaV: Double, maxDiscreteV: Int) (implicit num: Numeric[N], dataTpe: TDigestUDAFDataType[N]) extends UserDefinedAggregateFunction { def deterministic: Boolean = false def inputSchema: StructType = StructType(StructField("x", dataTpe.tpe) :: Nil) def bufferSchema: StructType = StructType(StructField("tdigest", TDigestUDT) :: Nil) def dataType: DataType = TDigestUDT case class TDigestUDAF[N](deltaV: Double, maxDiscreteV: Int) (implicit num: Numeric[N], dataTpe: TDigestUDAFDataType[N]) extends UserDefinedAggregateFunction { def deterministic: Boolean = false def inputSchema: StructType = StructType(StructField("x", dataTpe.tpe) :: Nil) def bufferSchema: StructType = StructType(StructField("tdigest", TDigestUDT) :: Nil) def dataType: DataType = TDigestUDT
- 108. #SAISDD6 case class TDigestUDAF[N](deltaV: Double, maxDiscreteV: Int) (implicit num: Numeric[N], dataTpe: TDigestUDAFDataType[N]) extends UserDefinedAggregateFunction { def deterministic: Boolean = false def inputSchema: StructType = StructType(StructField("x", dataTpe.tpe) :: Nil) def bufferSchema: StructType = StructType(StructField("tdigest", TDigestUDT) :: Nil) def dataType: DataType = TDigestUDT case class TDigestUDAF[N](deltaV: Double, maxDiscreteV: Int) (implicit num: Numeric[N], dataTpe: TDigestUDAFDataType[N]) extends UserDefinedAggregateFunction { def deterministic: Boolean = false def inputSchema: StructType = StructType(StructField("x", dataTpe.tpe) :: Nil) def bufferSchema: StructType = StructType(StructField("tdigest", TDigestUDT) :: Nil) def dataType: DataType = TDigestUDT
- 109. #SAISDD6 case class TDigestUDAF[N](deltaV: Double, maxDiscreteV: Int) (implicit num: Numeric[N], dataTpe: TDigestUDAFDataType[N]) extends UserDefinedAggregateFunction { def deterministic: Boolean = false def inputSchema: StructType = StructType(StructField("x", dataTpe.tpe) :: Nil) def bufferSchema: StructType = StructType(StructField("tdigest", TDigestUDT) :: Nil) def dataType: DataType = TDigestUDT case class TDigestUDAF[N](deltaV: Double, maxDiscreteV: Int) (implicit num: Numeric[N], dataTpe: TDigestUDAFDataType[N]) extends UserDefinedAggregateFunction { def deterministic: Boolean = false def inputSchema: StructType = StructType(StructField("x", dataTpe.tpe) :: Nil) def bufferSchema: StructType = StructType(StructField("tdigest", TDigestUDT) :: Nil) def dataType: DataType = TDigestUDT
- 110. #SAISDD6 case class TDigestUDAF[N](deltaV: Double, maxDiscreteV: Int) (implicit num: Numeric[N], dataTpe: TDigestUDAFDataType[N]) extends UserDefinedAggregateFunction { def deterministic: Boolean = false def inputSchema: StructType = StructType(StructField("x", dataTpe.tpe) :: Nil) def bufferSchema: StructType = StructType(StructField("tdigest", TDigestUDT) :: Nil) def dataType: DataType = TDigestUDT case class TDigestUDAF[N](deltaV: Double, maxDiscreteV: Int) (implicit num: Numeric[N], dataTpe: TDigestUDAFDataType[N]) extends UserDefinedAggregateFunction { def deterministic: Boolean = false def inputSchema: StructType = StructType(StructField("x", dataTpe.tpe) :: Nil) def bufferSchema: StructType = StructType(StructField("tdigest", TDigestUDT) :: Nil) def dataType: DataType = TDigestUDT
- 111. #SAISDD6 case class TDigestUDAF[N](deltaV: Double, maxDiscreteV: Int) (implicit num: Numeric[N], dataTpe: TDigestUDAFDataType[N]) extends UserDefinedAggregateFunction { def deterministic: Boolean = false def inputSchema: StructType = StructType(StructField("x", dataTpe.tpe) :: Nil) def bufferSchema: StructType = StructType(StructField("tdigest", TDigestUDT) :: Nil) def dataType: DataType = TDigestUDT case class TDigestUDAF[N](deltaV: Double, maxDiscreteV: Int) (implicit num: Numeric[N], dataTpe: TDigestUDAFDataType[N]) extends UserDefinedAggregateFunction { def deterministic: Boolean = false def inputSchema: StructType = StructType(StructField("x", dataTpe.tpe) :: Nil) def bufferSchema: StructType = StructType(StructField("tdigest", TDigestUDT) :: Nil) def dataType: DataType = TDigestUDT
- 112. #SAISDD6 Four main functions: initialize initialize
- 113. #SAISDD6 Four main functions: initialize initialize
- 114. #SAISDD6 def initialize(buf: MutableAggregationBuffer): Unit = { buf(0) = TDigestSQL(TDigest.empty(deltaV, maxDiscreteV)) } def evaluate(buf: Row): Any = buf.getAs[TDigestSQL](0) def initialize(buf: MutableAggregationBuffer): Unit = { buf(0) = TDigestSQL(TDigest.empty(deltaV, maxDiscreteV)) } def evaluate(buf: Row): Any = buf.getAs[TDigestSQL](0)
- 115. #SAISDD6 def initialize(buf: MutableAggregationBuffer): Unit = { buf(0) = TDigestSQL(TDigest.empty(deltaV, maxDiscreteV)) } def evaluate(buf: Row): Any = buf.getAs[TDigestSQL](0) def initialize(buf: MutableAggregationBuffer): Unit = { buf(0) = TDigestSQL(TDigest.empty(deltaV, maxDiscreteV)) } def evaluate(buf: Row): Any = buf.getAs[TDigestSQL](0)
- 116. #SAISDD6 Four main functions: evaluate evaluate
- 117. #SAISDD6 Four main functions: evaluate evaluate
- 118. #SAISDD6 def initialize(buf: MutableAggregationBuffer): Unit = { buf(0) = TDigestSQL(TDigest.empty(deltaV, maxDiscreteV)) } def evaluate(buf: Row): Any = buf.getAs[TDigestSQL](0) def initialize(buf: MutableAggregationBuffer): Unit = { buf(0) = TDigestSQL(TDigest.empty(deltaV, maxDiscreteV)) } def evaluate(buf: Row): Any = buf.getAs[TDigestSQL](0)
- 119. #SAISDD6 def initialize(buf: MutableAggregationBuffer): Unit = { buf(0) = TDigestSQL(TDigest.empty(deltaV, maxDiscreteV)) } def evaluate(buf: Row): Any = buf.getAs[TDigestSQL](0) def initialize(buf: MutableAggregationBuffer): Unit = { buf(0) = TDigestSQL(TDigest.empty(deltaV, maxDiscreteV)) } def evaluate(buf: Row): Any = buf.getAs[TDigestSQL](0)
- 120. #SAISDD6 Four main functions: update update
- 121. #SAISDD6 Four main functions: update update
- 122. #SAISDD6 def update(buf: MutableAggregationBuffer, input: Row): Unit = { if (!input.isNullAt(0)) { buf(0) = TDigestSQL(buf.getAs[TDigestSQL](0).tdigest + num.toDouble(input.getAs[N](0))) } } def merge(buf1: MutableAggregationBuffer, buf2: Row): Unit = { buf1(0) = TDigestSQL(buf1.getAs[TDigestSQL](0).tdigest ++ buf2.getAs[TDigestSQL](0).tdigest) } def update(buf: MutableAggregationBuffer, input: Row): Unit = { if (!input.isNullAt(0)) { buf(0) = TDigestSQL(buf.getAs[TDigestSQL](0).tdigest + num.toDouble(input.getAs[N](0))) } } def merge(buf1: MutableAggregationBuffer, buf2: Row): Unit = { buf1(0) = TDigestSQL(buf1.getAs[TDigestSQL](0).tdigest ++ buf2.getAs[TDigestSQL](0).tdigest) }
- 123. #SAISDD6 def update(buf: MutableAggregationBuffer, input: Row): Unit = { if (!input.isNullAt(0)) { buf(0) = TDigestSQL(buf.getAs[TDigestSQL](0).tdigest + num.toDouble(input.getAs[N](0))) } } def merge(buf1: MutableAggregationBuffer, buf2: Row): Unit = { buf1(0) = TDigestSQL(buf1.getAs[TDigestSQL](0).tdigest ++ buf2.getAs[TDigestSQL](0).tdigest) } def update(buf: MutableAggregationBuffer, input: Row): Unit = { if (!input.isNullAt(0)) { buf(0) = TDigestSQL(buf.getAs[TDigestSQL](0).tdigest + num.toDouble(input.getAs[N](0))) } } def merge(buf1: MutableAggregationBuffer, buf2: Row): Unit = { buf1(0) = TDigestSQL(buf1.getAs[TDigestSQL](0).tdigest ++ buf2.getAs[TDigestSQL](0).tdigest) }
- 124. #SAISDD6 def update(buf: MutableAggregationBuffer, input: Row): Unit = { if (!input.isNullAt(0)) { buf(0) = TDigestSQL(buf.getAs[TDigestSQL](0).tdigest + num.toDouble(input.getAs[N](0))) } } def merge(buf1: MutableAggregationBuffer, buf2: Row): Unit = { buf1(0) = TDigestSQL(buf1.getAs[TDigestSQL](0).tdigest ++ buf2.getAs[TDigestSQL](0).tdigest) } def update(buf: MutableAggregationBuffer, input: Row): Unit = { if (!input.isNullAt(0)) { buf(0) = TDigestSQL(buf.getAs[TDigestSQL](0).tdigest + num.toDouble(input.getAs[N](0))) } } def merge(buf1: MutableAggregationBuffer, buf2: Row): Unit = { buf1(0) = TDigestSQL(buf1.getAs[TDigestSQL](0).tdigest ++ buf2.getAs[TDigestSQL](0).tdigest) }
- 125. #SAISDD6 def update(buf: MutableAggregationBuffer, input: Row): Unit = { if (!input.isNullAt(0)) { buf(0) = TDigestSQL(buf.getAs[TDigestSQL](0).tdigest + num.toDouble(input.getAs[N](0))) } } def merge(buf1: MutableAggregationBuffer, buf2: Row): Unit = { buf1(0) = TDigestSQL(buf1.getAs[TDigestSQL](0).tdigest ++ buf2.getAs[TDigestSQL](0).tdigest) } def update(buf: MutableAggregationBuffer, input: Row): Unit = { if (!input.isNullAt(0)) { buf(0) = TDigestSQL(buf.getAs[TDigestSQL](0).tdigest + num.toDouble(input.getAs[N](0))) } } def merge(buf1: MutableAggregationBuffer, buf2: Row): Unit = { buf1(0) = TDigestSQL(buf1.getAs[TDigestSQL](0).tdigest ++ buf2.getAs[TDigestSQL](0).tdigest) }
- 126. #SAISDD6 def update(buf: MutableAggregationBuffer, input: Row): Unit = { if (!input.isNullAt(0)) { buf(0) = TDigestSQL(buf.getAs[TDigestSQL](0).tdigest + num.toDouble(input.getAs[N](0))) } } def merge(buf1: MutableAggregationBuffer, buf2: Row): Unit = { buf1(0) = TDigestSQL(buf1.getAs[TDigestSQL](0).tdigest ++ buf2.getAs[TDigestSQL](0).tdigest) } def update(buf: MutableAggregationBuffer, input: Row): Unit = { if (!input.isNullAt(0)) { buf(0) = TDigestSQL(buf.getAs[TDigestSQL](0).tdigest + num.toDouble(input.getAs[N](0))) } } def merge(buf1: MutableAggregationBuffer, buf2: Row): Unit = { buf1(0) = TDigestSQL(buf1.getAs[TDigestSQL](0).tdigest ++ buf2.getAs[TDigestSQL](0).tdigest) }
- 127. #SAISDD6 def update(buf: MutableAggregationBuffer, input: Row): Unit = { if (!input.isNullAt(0)) { buf(0) = TDigestSQL(buf.getAs[TDigestSQL](0).tdigest + num.toDouble(input.getAs[N](0))) } } def merge(buf1: MutableAggregationBuffer, buf2: Row): Unit = { buf1(0) = TDigestSQL(buf1.getAs[TDigestSQL](0).tdigest ++ buf2.getAs[TDigestSQL](0).tdigest) } def update(buf: MutableAggregationBuffer, input: Row): Unit = { if (!input.isNullAt(0)) { buf(0) = TDigestSQL(buf.getAs[TDigestSQL](0).tdigest + num.toDouble(input.getAs[N](0))) } } def merge(buf1: MutableAggregationBuffer, buf2: Row): Unit = { buf1(0) = TDigestSQL(buf1.getAs[TDigestSQL](0).tdigest ++ buf2.getAs[TDigestSQL](0).tdigest) }
- 128. #SAISDD6 def update(buf: MutableAggregationBuffer, input: Row): Unit = { if (!input.isNullAt(0)) { buf(0) = TDigestSQL(buf.getAs[TDigestSQL](0).tdigest + num.toDouble(input.getAs[N](0))) } } def merge(buf1: MutableAggregationBuffer, buf2: Row): Unit = { buf1(0) = TDigestSQL(buf1.getAs[TDigestSQL](0).tdigest ++ buf2.getAs[TDigestSQL](0).tdigest) } def update(buf: MutableAggregationBuffer, input: Row): Unit = { if (!input.isNullAt(0)) { buf(0) = TDigestSQL(buf.getAs[TDigestSQL](0).tdigest + num.toDouble(input.getAs[N](0))) } } def merge(buf1: MutableAggregationBuffer, buf2: Row): Unit = { buf1(0) = TDigestSQL(buf1.getAs[TDigestSQL](0).tdigest ++ buf2.getAs[TDigestSQL](0).tdigest) }
- 129. #SAISDD6 Four main functions: merge 1 merge 2
- 130. #SAISDD6 Four main functions: merge merge 1 + 2
- 131. #SAISDD6 def update(buf: MutableAggregationBuffer, input: Row): Unit = { if (!input.isNullAt(0)) { buf(0) = TDigestSQL(buf.getAs[TDigestSQL](0).tdigest + num.toDouble(input.getAs[N](0))) } } def merge(buf1: MutableAggregationBuffer, buf2: Row): Unit = { buf1(0) = TDigestSQL(buf1.getAs[TDigestSQL](0).tdigest ++ buf2.getAs[TDigestSQL](0).tdigest) } def update(buf: MutableAggregationBuffer, input: Row): Unit = { if (!input.isNullAt(0)) { buf(0) = TDigestSQL(buf.getAs[TDigestSQL](0).tdigest + num.toDouble(input.getAs[N](0))) } } def merge(buf1: MutableAggregationBuffer, buf2: Row): Unit = { buf1(0) = TDigestSQL(buf1.getAs[TDigestSQL](0).tdigest ++ buf2.getAs[TDigestSQL](0).tdigest) }
- 132. #SAISDD6 def update(buf: MutableAggregationBuffer, input: Row): Unit = { if (!input.isNullAt(0)) { buf(0) = TDigestSQL(buf.getAs[TDigestSQL](0).tdigest + num.toDouble(input.getAs[N](0))) } } def merge(buf1: MutableAggregationBuffer, buf2: Row): Unit = { buf1(0) = TDigestSQL(buf1.getAs[TDigestSQL](0).tdigest ++ buf2.getAs[TDigestSQL](0).tdigest) } def update(buf: MutableAggregationBuffer, input: Row): Unit = { if (!input.isNullAt(0)) { buf(0) = TDigestSQL(buf.getAs[TDigestSQL](0).tdigest + num.toDouble(input.getAs[N](0))) } } def merge(buf1: MutableAggregationBuffer, buf2: Row): Unit = { buf1(0) = TDigestSQL(buf1.getAs[TDigestSQL](0).tdigest ++ buf2.getAs[TDigestSQL](0).tdigest) }
- 134. #SAISDD6 User-defined types package org.apache.spark.isarnproject.sketches.udt @SQLUserDefinedType(udt = classOf[TDigestUDT]) case class TDigestSQL(tdigest: TDigest) class TDigestUDT extends UserDefinedType[TDigestSQL] { def userClass: Class[TDigestSQL] = classOf[TDigestSQL] // .... package org.apache.spark
- 135. #SAISDD6 package org.apache.spark.isarnproject.sketches.udt @SQLUserDefinedType(udt = classOf[TDigestUDT]) case class TDigestSQL(tdigest: TDigest) class TDigestUDT extends UserDefinedType[TDigestSQL] { def userClass: Class[TDigestSQL] = classOf[TDigestSQL] // .... User-defined types package org.apache.spark.isarnproject.sketches.udt @SQLUserDefinedType(udt = classOf[TDigestUDT]) case class TDigestSQL(tdigest: TDigest) class TDigestUDT extends UserDefinedType[TDigestSQL] { def userClass: Class[TDigestSQL] = classOf[TDigestSQL] // ....
- 136. #SAISDD6 package org.apache.spark.isarnproject.sketches.udt @SQLUserDefinedType(udt = classOf[TDigestUDT]) case class TDigestSQL(tdigest: TDigest) class TDigestUDT extends UserDefinedType[TDigestSQL] { def userClass: Class[TDigestSQL] = classOf[TDigestSQL] // .... User-defined types package org.apache.spark.isarnproject.sketches.udt @SQLUserDefinedType(udt = classOf[TDigestUDT]) case class TDigestSQL(tdigest: TDigest) class TDigestUDT extends UserDefinedType[TDigestSQL] { def userClass: Class[TDigestSQL] = classOf[TDigestSQL] // ....
- 137. #SAISDD6 Implementing custom types class TDigestUDT extends UserDefinedType[TDigestSQL] { def userClass: Class[TDigestSQL] = classOf[TDigestSQL] override def pyUDT: String = "isarnproject.sketches.udt.tdigest.TDigestUDT" override def typeName: String = "tdigest" def sqlType: DataType = StructType( StructField("delta", DoubleType, false) :: /* ... */ StructField("clustM", ArrayType(DoubleType, false), false) :: Nil) class TDigestUDT extends UserDefinedType[TDigestSQL] { def userClass: Class[TDigestSQL] = classOf[TDigestSQL] override def pyUDT: String = "isarnproject.sketches.udt.tdigest.TDigestUDT" override def typeName: String = "tdigest" def sqlType: DataType = StructType( StructField("delta", DoubleType, false) :: /* ... */ StructField("clustM", ArrayType(DoubleType, false), false) :: Nil)
- 138. #SAISDD6 class TDigestUDT extends UserDefinedType[TDigestSQL] { def userClass: Class[TDigestSQL] = classOf[TDigestSQL] override def pyUDT: String = "isarnproject.sketches.udt.tdigest.TDigestUDT" override def typeName: String = "tdigest" def sqlType: DataType = StructType( StructField("delta", DoubleType, false) :: /* ... */ StructField("clustM", ArrayType(DoubleType, false), false) :: Nil) class TDigestUDT extends UserDefinedType[TDigestSQL] { def userClass: Class[TDigestSQL] = classOf[TDigestSQL] override def pyUDT: String = "isarnproject.sketches.udt.tdigest.TDigestUDT" override def typeName: String = "tdigest" def sqlType: DataType = StructType( StructField("delta", DoubleType, false) :: /* ... */ StructField("clustM", ArrayType(DoubleType, false), false) :: Nil)
- 139. #SAISDD6 class TDigestUDT extends UserDefinedType[TDigestSQL] { def userClass: Class[TDigestSQL] = classOf[TDigestSQL] override def pyUDT: String = "isarnproject.sketches.udt.tdigest.TDigestUDT" override def typeName: String = "tdigest" def sqlType: DataType = StructType( StructField("delta", DoubleType, false) :: /* ... */ StructField("clustM", ArrayType(DoubleType, false), false) :: Nil) class TDigestUDT extends UserDefinedType[TDigestSQL] { def userClass: Class[TDigestSQL] = classOf[TDigestSQL] override def pyUDT: String = "isarnproject.sketches.udt.tdigest.TDigestUDT" override def typeName: String = "tdigest" def sqlType: DataType = StructType( StructField("delta", DoubleType, false) :: /* ... */ StructField("clustM", ArrayType(DoubleType, false), false) :: Nil)
- 140. #SAISDD6 class TDigestUDT extends UserDefinedType[TDigestSQL] { def userClass: Class[TDigestSQL] = classOf[TDigestSQL] override def pyUDT: String = "isarnproject.sketches.udt.tdigest.TDigestUDT" override def typeName: String = "tdigest" def sqlType: DataType = StructType( StructField("delta", DoubleType, false) :: /* ... */ StructField("clustM", ArrayType(DoubleType, false), false) :: Nil) class TDigestUDT extends UserDefinedType[TDigestSQL] { def userClass: Class[TDigestSQL] = classOf[TDigestSQL] override def pyUDT: String = "isarnproject.sketches.udt.tdigest.TDigestUDT" override def typeName: String = "tdigest" def sqlType: DataType = StructType( StructField("delta", DoubleType, false) :: /* ... */ StructField("clustM", ArrayType(DoubleType, false), false) :: Nil)
- 141. #SAISDD6 def serialize(tdsql: TDigestSQL): Any = serializeTD(tdsql.tdigest) private[sketches] def serializeTD(td: TDigest): InternalRow = { val TDigest(delta, maxDiscrete, nclusters, clusters) = td val row = new GenericInternalRow(5) row.setDouble(0, delta) row.setInt(1, maxDiscrete) row.setInt(2, nclusters) val clustX = clusters.keys.toArray val clustM = clusters.values.toArray row.update(3, UnsafeArrayData.fromPrimitiveArray(clustX)) row.update(4, UnsafeArrayData.fromPrimitiveArray(clustM)) row } def serialize(tdsql: TDigestSQL): Any = serializeTD(tdsql.tdigest) private[sketches] def serializeTD(td: TDigest): InternalRow = { val TDigest(delta, maxDiscrete, nclusters, clusters) = td val row = new GenericInternalRow(5) row.setDouble(0, delta) row.setInt(1, maxDiscrete) row.setInt(2, nclusters) val clustX = clusters.keys.toArray val clustM = clusters.values.toArray row.update(3, UnsafeArrayData.fromPrimitiveArray(clustX)) row.update(4, UnsafeArrayData.fromPrimitiveArray(clustM)) row }
- 142. #SAISDD6 def serialize(tdsql: TDigestSQL): Any = serializeTD(tdsql.tdigest) private[sketches] def serializeTD(td: TDigest): InternalRow = { val TDigest(delta, maxDiscrete, nclusters, clusters) = td val row = new GenericInternalRow(5) row.setDouble(0, delta) row.setInt(1, maxDiscrete) row.setInt(2, nclusters) val clustX = clusters.keys.toArray val clustM = clusters.values.toArray row.update(3, UnsafeArrayData.fromPrimitiveArray(clustX)) row.update(4, UnsafeArrayData.fromPrimitiveArray(clustM)) row } def serialize(tdsql: TDigestSQL): Any = serializeTD(tdsql.tdigest) private[sketches] def serializeTD(td: TDigest): InternalRow = { val TDigest(delta, maxDiscrete, nclusters, clusters) = td val row = new GenericInternalRow(5) row.setDouble(0, delta) row.setInt(1, maxDiscrete) row.setInt(2, nclusters) val clustX = clusters.keys.toArray val clustM = clusters.values.toArray row.update(3, UnsafeArrayData.fromPrimitiveArray(clustX)) row.update(4, UnsafeArrayData.fromPrimitiveArray(clustM)) row }
- 143. #SAISDD6 def serialize(tdsql: TDigestSQL): Any = serializeTD(tdsql.tdigest) private[sketches] def serializeTD(td: TDigest): InternalRow = { val TDigest(delta, maxDiscrete, nclusters, clusters) = td val row = new GenericInternalRow(5) row.setDouble(0, delta) row.setInt(1, maxDiscrete) row.setInt(2, nclusters) val clustX = clusters.keys.toArray val clustM = clusters.values.toArray row.update(3, UnsafeArrayData.fromPrimitiveArray(clustX)) row.update(4, UnsafeArrayData.fromPrimitiveArray(clustM)) row } def serialize(tdsql: TDigestSQL): Any = serializeTD(tdsql.tdigest) private[sketches] def serializeTD(td: TDigest): InternalRow = { val TDigest(delta, maxDiscrete, nclusters, clusters) = td val row = new GenericInternalRow(5) row.setDouble(0, delta) row.setInt(1, maxDiscrete) row.setInt(2, nclusters) val clustX = clusters.keys.toArray val clustM = clusters.values.toArray row.update(3, UnsafeArrayData.fromPrimitiveArray(clustX)) row.update(4, UnsafeArrayData.fromPrimitiveArray(clustM)) row }
- 144. #SAISDD6 def serialize(tdsql: TDigestSQL): Any = serializeTD(tdsql.tdigest) private[sketches] def serializeTD(td: TDigest): InternalRow = { val TDigest(delta, maxDiscrete, nclusters, clusters) = td val row = new GenericInternalRow(5) row.setDouble(0, delta) row.setInt(1, maxDiscrete) row.setInt(2, nclusters) val clustX = clusters.keys.toArray val clustM = clusters.values.toArray row.update(3, UnsafeArrayData.fromPrimitiveArray(clustX)) row.update(4, UnsafeArrayData.fromPrimitiveArray(clustM)) row } def serialize(tdsql: TDigestSQL): Any = serializeTD(tdsql.tdigest) private[sketches] def serializeTD(td: TDigest): InternalRow = { val TDigest(delta, maxDiscrete, nclusters, clusters) = td val row = new GenericInternalRow(5) row.setDouble(0, delta) row.setInt(1, maxDiscrete) row.setInt(2, nclusters) val clustX = clusters.keys.toArray val clustM = clusters.values.toArray row.update(3, UnsafeArrayData.fromPrimitiveArray(clustX)) row.update(4, UnsafeArrayData.fromPrimitiveArray(clustM)) row }
- 145. #SAISDD6 def deserialize(td: Any): TDigestSQL = TDigestSQL(deserializeTD(td)) private[sketches] def deserializeTD(datum: Any): TDigest = datum match { case row: InternalRow => val delta = row.getDouble(0) val maxDiscrete = row.getInt(1) val nclusters = row.getInt(2) val clustX = row.getArray(3).toDoubleArray() val clustM = row.getArray(4).toDoubleArray() val clusters = clustX.zip(clustM) .foldLeft(TDigestMap.empty) { case (td, e) => td + e } TDigest(delta, maxDiscrete, nclusters, clusters) } def deserialize(td: Any): TDigestSQL = TDigestSQL(deserializeTD(td)) private[sketches] def deserializeTD(datum: Any): TDigest = datum match { case row: InternalRow => val delta = row.getDouble(0) val maxDiscrete = row.getInt(1) val nclusters = row.getInt(2) val clustX = row.getArray(3).toDoubleArray() val clustM = row.getArray(4).toDoubleArray() val clusters = clustX.zip(clustM) .foldLeft(TDigestMap.empty) { case (td, e) => td + e } TDigest(delta, maxDiscrete, nclusters, clusters) }
- 146. #SAISDD6 def deserialize(td: Any): TDigestSQL = TDigestSQL(deserializeTD(td)) private[sketches] def deserializeTD(datum: Any): TDigest = datum match { case row: InternalRow => val delta = row.getDouble(0) val maxDiscrete = row.getInt(1) val nclusters = row.getInt(2) val clustX = row.getArray(3).toDoubleArray() val clustM = row.getArray(4).toDoubleArray() val clusters = clustX.zip(clustM) .foldLeft(TDigestMap.empty) { case (td, e) => td + e } TDigest(delta, maxDiscrete, nclusters, clusters) } def deserialize(td: Any): TDigestSQL = TDigestSQL(deserializeTD(td)) private[sketches] def deserializeTD(datum: Any): TDigest = datum match { case row: InternalRow => val delta = row.getDouble(0) val maxDiscrete = row.getInt(1) val nclusters = row.getInt(2) val clustX = row.getArray(3).toDoubleArray() val clustM = row.getArray(4).toDoubleArray() val clusters = clustX.zip(clustM) .foldLeft(TDigestMap.empty) { case (td, e) => td + e } TDigest(delta, maxDiscrete, nclusters, clusters) }
- 147. #SAISDD6 def deserialize(td: Any): TDigestSQL = TDigestSQL(deserializeTD(td)) private[sketches] def deserializeTD(datum: Any): TDigest = datum match { case row: InternalRow => val delta = row.getDouble(0) val maxDiscrete = row.getInt(1) val nclusters = row.getInt(2) val clustX = row.getArray(3).toDoubleArray() val clustM = row.getArray(4).toDoubleArray() val clusters = clustX.zip(clustM) .foldLeft(TDigestMap.empty) { case (td, e) => td + e } TDigest(delta, maxDiscrete, nclusters, clusters) } def deserialize(td: Any): TDigestSQL = TDigestSQL(deserializeTD(td)) private[sketches] def deserializeTD(datum: Any): TDigest = datum match { case row: InternalRow => val delta = row.getDouble(0) val maxDiscrete = row.getInt(1) val nclusters = row.getInt(2) val clustX = row.getArray(3).toDoubleArray() val clustM = row.getArray(4).toDoubleArray() val clusters = clustX.zip(clustM) .foldLeft(TDigestMap.empty) { case (td, e) => td + e } TDigest(delta, maxDiscrete, nclusters, clusters) }
- 148. Extending PySpark with your Scala library
- 149. #SAISDD6 [ ]
- 150. #SAISDD6 [ ]
- 151. #SAISDD6 [ ]
- 152. #SAISDD6 [ ]
- 153. #SAISDD6 # class to access the active Spark context for Python from pyspark.context import SparkContext # gateway to the JVM from py4j sparkJVM = SparkContext._active_spark_context._jvm # use the gateway to access JVM objects and classes thisThing = sparkJVM.com.path.to.this.thing # class to access the active Spark context for Python from pyspark.context import SparkContext # gateway to the JVM from py4j sparkJVM = SparkContext._active_spark_context._jvm # use the gateway to access JVM objects and classes thisThing = sparkJVM.com.path.to.this.thing
- 154. #SAISDD6 # class to access the active Spark context for Python from pyspark.context import SparkContext # gateway to the JVM from py4j sparkJVM = SparkContext._active_spark_context._jvm # use the gateway to access JVM objects and classes thisThing = sparkJVM.com.path.to.this.thing # gateway to the JVM from py4j sparkJVM = SparkContext._active_spark_context._jvm
- 155. #SAISDD6 # class to access the active Spark context for Python from pyspark.context import SparkContext # gateway to the JVM from py4j sparkJVM = SparkContext._active_spark_context._jvm # use the gateway to access JVM objects and classes thisThing = sparkJVM.com.path.to.this.thing # use the gateway to access JVM objects and classes thisThing = sparkJVM.com.path.to.this.thing
- 156. #SAISDD6 A Python-friendly wrapper package org.isarnproject.sketches.udaf object pythonBindings { def tdigestDoubleUDAF(delta: Double, maxDiscrete: Int) = TDigestUDAF[Double](delta, maxDiscrete) } package org.isarnproject.sketches.udaf object pythonBindings { def tdigestDoubleUDAF(delta: Double, maxDiscrete: Int) = TDigestUDAF[Double](delta, maxDiscrete) }
- 157. #SAISDD6 package org.isarnproject.sketches.udaf object pythonBindings { def tdigestDoubleUDAF(delta: Double, maxDiscrete: Int) = TDigestUDAF[Double](delta, maxDiscrete) } tdigestDoubleUDAF
- 158. #SAISDD6 package org.isarnproject.sketches.udaf object pythonBindings { def tdigestDoubleUDAF(delta: Double, maxDiscrete: Int) = TDigestUDAF[Double](delta, maxDiscrete) } Double
- 159. #SAISDD6 from pyspark.sql.column import Column, _to_java_column, _to_seq from pyspark.context import SparkContext # one of these for each type parameter Double, Int, Long, etc def tdigestDoubleUDAF(col, delta=0.5, maxDiscrete=0): sc = SparkContext._active_spark_context pb = sc._jvm.org.isarnproject.sketches.udaf.pythonBindings tdapply = pb.tdigestDoubleUDAF(delta, maxDiscrete).apply return Column(tdapply(_to_seq(sc, [col], _to_java_column))) from pyspark.sql.column import Column, _to_java_column, _to_seq from pyspark.context import SparkContext # one of these for each type parameter Double, Int, Long, etc def tdigestDoubleUDAF(col, delta=0.5, maxDiscrete=0): sc = SparkContext._active_spark_context pb = sc._jvm.org.isarnproject.sketches.udaf.pythonBindings tdapply = pb.tdigestDoubleUDAF(delta, maxDiscrete).apply return Column(tdapply(_to_seq(sc, [col], _to_java_column)))
- 160. #SAISDD6 from pyspark.sql.column import Column, _to_java_column, _to_seq from pyspark.context import SparkContext # one of these for each type parameter Double, Int, Long, etc def tdigestDoubleUDAF(col, delta=0.5, maxDiscrete=0): sc = SparkContext._active_spark_context pb = sc._jvm.org.isarnproject.sketches.udaf.pythonBindings tdapply = pb.tdigestDoubleUDAF(delta, maxDiscrete).apply return Column(tdapply(_to_seq(sc, [col], _to_java_column))) tdapply apply
- 161. #SAISDD6 from pyspark.sql.column import Column, _to_java_column, _to_seq from pyspark.context import SparkContext # one of these for each type parameter Double, Int, Long, etc def tdigestDoubleUDAF(col, delta=0.5, maxDiscrete=0): sc = SparkContext._active_spark_context pb = sc._jvm.org.isarnproject.sketches.udaf.pythonBindings tdapply = pb.tdigestDoubleUDAF(delta, maxDiscrete).apply return Column(tdapply(_to_seq(sc, [col], _to_java_column)))tdapply(_to_seq(sc, [col], _to_java_column))
- 162. #SAISDD6 class TDigestUDT(UserDefinedType): @classmethod def sqlType(cls): return StructType([ StructField("delta", DoubleType(), False), StructField("maxDiscrete", IntegerType(), False), StructField("nclusters", IntegerType(), False), StructField("clustX", ArrayType(DoubleType(), False), False), StructField("clustM", ArrayType(DoubleType(), False), False)]) class TDigestUDT(UserDefinedType): @classmethod def sqlType(cls): return StructType([ StructField("delta", DoubleType(), False), StructField("maxDiscrete", IntegerType(), False), StructField("nclusters", IntegerType(), False), StructField("clustX", ArrayType(DoubleType(), False), False), StructField("clustM", ArrayType(DoubleType(), False), False)]) # ...
- 163. #SAISDD6 class TDigestUDT(UserDefinedType): @classmethod def sqlType(cls): return StructType([ StructField("delta", DoubleType(), False), StructField("maxDiscrete", IntegerType(), False), StructField("nclusters", IntegerType(), False), StructField("clustX", ArrayType(DoubleType(), False), False), StructField("clustM", ArrayType(DoubleType(), False), False)]) class TDigestUDT(UserDefinedType): @classmethod def sqlType(cls): return StructType([ StructField("delta", DoubleType(), False), StructField("maxDiscrete", IntegerType(), False), StructField("nclusters", IntegerType(), False), StructField("clustX", ArrayType(DoubleType(), False), False), StructField("clustM", ArrayType(DoubleType(), False), False)]) # ...
- 164. #SAISDD6 class TDigestUDT(UserDefinedType): # ... @classmethod def module(cls): return "isarnproject.sketches.udt.tdigest" @classmethod def scalaUDT(cls): return "org.apache.spark.isarnproject.sketches.udt.TDigestUDT" def simpleString(self): return "tdigest" class TDigestUDT(UserDefinedType): # ... @classmethod def module(cls): return "isarnproject.sketches.udt.tdigest" @classmethod def scalaUDT(cls): return "org.apache.spark.isarnproject.sketches.udt.TDigestUDT" def simpleString(self): return "tdigest"
- 165. #SAISDD6 class TDigestUDT(UserDefinedType): # ... @classmethod def module(cls): return "isarnproject.sketches.udt.tdigest" @classmethod def scalaUDT(cls): return "org.apache.spark.isarnproject.sketches.udt.TDigestUDT" def simpleString(self): return "tdigest" class TDigestUDT(UserDefinedType): # ... @classmethod def module(cls): return "isarnproject.sketches.udt.tdigest" @classmethod def scalaUDT(cls): return "org.apache.spark.isarnproject.sketches.udt.TDigestUDT" def simpleString(self): return "tdigest"
- 166. #SAISDD6 class TDigestUDT(UserDefinedType): # ... @classmethod def module(cls): return "isarnproject.sketches.udt.tdigest" @classmethod def scalaUDT(cls): return "org.apache.spark.isarnproject.sketches.udt.TDigestUDT" def simpleString(self): return "tdigest" class TDigestUDT(UserDefinedType): # ... @classmethod def module(cls): return "isarnproject.sketches.udt.tdigest" @classmethod def scalaUDT(cls): return "org.apache.spark.isarnproject.sketches.udt.TDigestUDT" def simpleString(self): return "tdigest"
- 167. #SAISDD6 class TDigestUDT(UserDefinedType): # ... def serialize(self, obj): return (obj.delta, obj.maxDiscrete, obj.nclusters, [float(v) for v in obj.clustX], [float(v) for v in obj.clustM]) def deserialize(self, datum): return TDigest(datum[0], datum[1], datum[2], datum[3], datum[4]) class TDigestUDT(UserDefinedType): # ... def serialize(self, obj): return (obj.delta, obj.maxDiscrete, obj.nclusters, [float(v) for v in obj.clustX], [float(v) for v in obj.clustM]) def deserialize(self, datum): return TDigest(datum[0], datum[1], datum[2], datum[3], datum[4])
- 168. #SAISDD6 class TDigestUDT(UserDefinedType): # ... def serialize(self, obj): return (obj.delta, obj.maxDiscrete, obj.nclusters, [float(v) for v in obj.clustX], [float(v) for v in obj.clustM]) def deserialize(self, datum): return TDigest(datum[0], datum[1], datum[2], datum[3], datum[4]) class TDigestUDT(UserDefinedType): # ... def serialize(self, obj): return (obj.delta, obj.maxDiscrete, obj.nclusters, [float(v) for v in obj.clustX], [float(v) for v in obj.clustM]) def deserialize(self, datum): return TDigest(datum[0], datum[1], datum[2], datum[3], datum[4])
- 169. #SAISDD6 class TDigestUDT extends UserDefinedType[TDigestSQL] { // ... override def pyUDT: String = “isarnproject.sketches.udt.tdigest.TDigestUDT" } class TDigestUDT extends UserDefinedType[TDigestSQL] { // ... override def pyUDT: String = “isarnproject.sketches.udt.tdigest.TDigestUDT" }
- 170. #SAISDD6 Python code in JAR files mappings in (Compile, packageBin) ++= Seq( (baseDirectory.value / "python" / "isarnproject" / "__init__.pyc") -> "isarnproject/__init__.pyc", (baseDirectory.value / "python" / "isarnproject" / "sketches" / "__init__.pyc") -> "isarnproject/sketches/__init__.pyc", (baseDirectory.value / "python" / "isarnproject" / "sketches" / "udaf" / "__init__.pyc") -> "isarnproject/sketches/udaf/__init__.pyc", (baseDirectory.value / "python" / "isarnproject" / "sketches" / "udaf" / "tdigest.pyc") -> "isarnproject/sketches/udaf/tdigest.pyc", (baseDirectory.value / "python" / "isarnproject" / "sketches" / "udt" / "__init__.pyc") -> "isarnproject/sketches/udt/__init__.pyc", (baseDirectory.value / "python" / "isarnproject" / "sketches" / "udt" / "tdigest.pyc") -> "isarnproject/sketches/udt/tdigest.pyc" ) mappings in (Compile, packageBin) ++= Seq( (baseDirectory.value / "python" / "isarnproject" / "__init__.pyc") -> "isarnproject/__init__.pyc", (baseDirectory.value / "python" / "isarnproject" / "sketches" / "__init__.pyc") -> "isarnproject/sketches/__init__.pyc", (baseDirectory.value / "python" / "isarnproject" / "sketches" / "udaf" / "__init__.pyc") -> "isarnproject/sketches/udaf/__init__.pyc", (baseDirectory.value / "python" / "isarnproject" / "sketches" / "udaf" / "tdigest.pyc") -> "isarnproject/sketches/udaf/tdigest.pyc", (baseDirectory.value / "python" / "isarnproject" / "sketches" / "udt" / "__init__.pyc") -> "isarnproject/sketches/udt/__init__.pyc", (baseDirectory.value / "python" / "isarnproject" / "sketches" / "udt" / "tdigest.pyc") -> "isarnproject/sketches/udt/tdigest.pyc" )
- 171. #SAISDD6 mappings in (Compile, packageBin) ++= Seq( (baseDirectory.value / "python" / "isarnproject" / "__init__.pyc") -> "isarnproject/__init__.pyc", (baseDirectory.value / "python" / "isarnproject" / "sketches" / "__init__.pyc") -> "isarnproject/sketches/__init__.pyc", (baseDirectory.value / "python" / "isarnproject" / "sketches" / "udaf" / "__init__.pyc") -> "isarnproject/sketches/udaf/__init__.pyc", (baseDirectory.value / "python" / "isarnproject" / "sketches" / "udaf" / "tdigest.pyc") -> "isarnproject/sketches/udaf/tdigest.pyc", (baseDirectory.value / "python" / "isarnproject" / "sketches" / "udt" / "__init__.pyc") -> "isarnproject/sketches/udt/__init__.pyc", (baseDirectory.value / "python" / "isarnproject" / "sketches" / "udt" / "tdigest.pyc") -> "isarnproject/sketches/udt/tdigest.pyc" ) mappings in (Compile, packageBin) ++= Seq( (baseDirectory.value / "python" / "isarnproject" / "__init__.pyc") -> "isarnproject/__init__.pyc", (baseDirectory.value / "python" / "isarnproject" / "sketches" / "__init__.pyc") -> "isarnproject/sketches/__init__.pyc", (baseDirectory.value / "python" / "isarnproject" / "sketches" / "udaf" / "__init__.pyc") -> "isarnproject/sketches/udaf/__init__.pyc", (baseDirectory.value / "python" / "isarnproject" / "sketches" / "udaf" / "tdigest.pyc") -> "isarnproject/sketches/udaf/tdigest.pyc", (baseDirectory.value / "python" / "isarnproject" / "sketches" / "udt" / "__init__.pyc") -> "isarnproject/sketches/udt/__init__.pyc", (baseDirectory.value / "python" / "isarnproject" / "sketches" / "udt" / "tdigest.pyc") -> "isarnproject/sketches/udt/tdigest.pyc" )
- 172. #SAISDD6 mappings in (Compile, packageBin) ++= Seq( (baseDirectory.value / "python" / "isarnproject" / "__init__.pyc") -> "isarnproject/__init__.pyc", (baseDirectory.value / "python" / "isarnproject" / "sketches" / "__init__.pyc") -> "isarnproject/sketches/__init__.pyc", (baseDirectory.value / "python" / "isarnproject" / "sketches" / "udaf" / "__init__.pyc") -> "isarnproject/sketches/udaf/__init__.pyc", (baseDirectory.value / "python" / "isarnproject" / "sketches" / "udaf" / "tdigest.pyc") -> "isarnproject/sketches/udaf/tdigest.pyc", (baseDirectory.value / "python" / "isarnproject" / "sketches" / "udt" / "__init__.pyc") -> "isarnproject/sketches/udt/__init__.pyc", (baseDirectory.value / "python" / "isarnproject" / "sketches" / "udt" / "tdigest.pyc") -> "isarnproject/sketches/udt/tdigest.pyc" ) mappings in (Compile, packageBin) ++= Seq( (baseDirectory.value / "python" / "isarnproject" / "__init__.pyc") -> "isarnproject/__init__.pyc", (baseDirectory.value / "python" / "isarnproject" / "sketches" / "__init__.pyc") -> "isarnproject/sketches/__init__.pyc", (baseDirectory.value / "python" / "isarnproject" / "sketches" / "udaf" / "__init__.pyc") -> "isarnproject/sketches/udaf/__init__.pyc", (baseDirectory.value / "python" / "isarnproject" / "sketches" / "udaf" / "tdigest.pyc") -> "isarnproject/sketches/udaf/tdigest.pyc", (baseDirectory.value / "python" / "isarnproject" / "sketches" / "udt" / "__init__.pyc") -> "isarnproject/sketches/udt/__init__.pyc", (baseDirectory.value / "python" / "isarnproject" / "sketches" / "udt" / "tdigest.pyc") -> "isarnproject/sketches/udt/tdigest.pyc" )
- 173. #SAISDD6 mappings in (Compile, packageBin) ++= Seq( (baseDirectory.value / "python" / "isarnproject" / "__init__.pyc") -> "isarnproject/__init__.pyc", (baseDirectory.value / "python" / "isarnproject" / "sketches" / "__init__.pyc") -> "isarnproject/sketches/__init__.pyc", (baseDirectory.value / "python" / "isarnproject" / "sketches" / "udaf" / "__init__.pyc") -> "isarnproject/sketches/udaf/__init__.pyc", (baseDirectory.value / "python" / "isarnproject" / "sketches" / "udaf" / "tdigest.pyc") -> "isarnproject/sketches/udaf/tdigest.pyc", (baseDirectory.value / "python" / "isarnproject" / "sketches" / "udt" / "__init__.pyc") -> "isarnproject/sketches/udt/__init__.pyc", (baseDirectory.value / "python" / "isarnproject" / "sketches" / "udt" / "tdigest.pyc") -> "isarnproject/sketches/udt/tdigest.pyc" ) mappings in (Compile, packageBin) ++= Seq( (baseDirectory.value / "python" / "isarnproject" / "__init__.pyc") -> "isarnproject/__init__.pyc", (baseDirectory.value / "python" / "isarnproject" / "sketches" / "__init__.pyc") -> "isarnproject/sketches/__init__.pyc", (baseDirectory.value / "python" / "isarnproject" / "sketches" / "udaf" / "__init__.pyc") -> "isarnproject/sketches/udaf/__init__.pyc", (baseDirectory.value / "python" / "isarnproject" / "sketches" / "udaf" / "tdigest.pyc") -> "isarnproject/sketches/udaf/tdigest.pyc", (baseDirectory.value / "python" / "isarnproject" / "sketches" / "udt" / "__init__.pyc") -> "isarnproject/sketches/udt/__init__.pyc", (baseDirectory.value / "python" / "isarnproject" / "sketches" / "udt" / "tdigest.pyc") -> "isarnproject/sketches/udt/tdigest.pyc" )
- 174. #SAISDD6 Cross-building for Python lazy val compilePython = taskKey[Unit]("Compile python files") compilePython := { val s: TaskStreams = streams.value s.log.info("compiling python...") val stat = (Seq(pythonCMD, "-m", "compileall", "python/") !) if (stat != 0) { throw new IllegalStateException("python compile failed") } } (packageBin in Compile) <<= (packageBin in Compile).dependsOn(compilePython) lazy val compilePython = taskKey[Unit]("Compile python files") compilePython := { val s: TaskStreams = streams.value s.log.info("compiling python...") val stat = (Seq(pythonCMD, "-m", "compileall", "python/") !) if (stat != 0) { throw new IllegalStateException("python compile failed") } } (packageBin in Compile) <<= (packageBin in Compile).dependsOn(compilePython)
- 175. #SAISDD6 lazy val compilePython = taskKey[Unit]("Compile python files") compilePython := { val s: TaskStreams = streams.value s.log.info("compiling python...") val stat = (Seq(pythonCMD, "-m", "compileall", "python/") !) if (stat != 0) { throw new IllegalStateException("python compile failed") } } (packageBin in Compile) <<= (packageBin in Compile).dependsOn(compilePython) lazy val compilePython = taskKey[Unit]("Compile python files") compilePython := { val s: TaskStreams = streams.value s.log.info("compiling python...") val stat = (Seq(pythonCMD, "-m", "compileall", "python/") !) if (stat != 0) { throw new IllegalStateException("python compile failed") } } (packageBin in Compile) <<= (packageBin in Compile).dependsOn(compilePython)
- 176. #SAISDD6 lazy val compilePython = taskKey[Unit]("Compile python files") compilePython := { val s: TaskStreams = streams.value s.log.info("compiling python...") val stat = (Seq(pythonCMD, "-m", "compileall", "python/") !) if (stat != 0) { throw new IllegalStateException("python compile failed") } } (packageBin in Compile) <<= (packageBin in Compile).dependsOn(compilePython) lazy val compilePython = taskKey[Unit]("Compile python files") compilePython := { val s: TaskStreams = streams.value s.log.info("compiling python...") if (stat != 0) { throw new IllegalStateException("python compile failed") } } (packageBin in Compile) <<= (packageBin in Compile).dependsOn(compilePython) val stat = (Seq(pythonCMD, "-m", "compileall", "python/") !)
- 177. #SAISDD6 Using versioned JAR files $ pyspark --packages 'org.isarnproject:isarn-sketches-spark_2.11:0.3.0-sp2.2-py2.7' $ pyspark --packages 'org.isarnproject:isarn-sketches-spark_2.11:0.3.0-sp2.2-py2.7'
- 178. #SAISDD6 Using versioned JAR files $ pyspark --packages 'org.isarnproject:isarn-sketches-spark_2.11:0.3.0-sp2.2-py2.7' $ pyspark --packages 'org.isarnproject:isarn-sketches-spark_2.11:0.3.0-sp2.2-py2.7'
- 179. #SAISDD6 Using versioned JAR files $ pyspark --packages 'org.isarnproject:isarn-sketches-spark_2.11:0.3.0-sp2.2-py2.7' $ pyspark --packages 'org.isarnproject:isarn-sketches-spark_2.11:0.3.0-sp2.2-py2.7'
- 180. Show your work: publishing results
- 181. #SAISDD6 Developing with git-flow $ brew install git-flow # macOS $ dnf install git-flow # Fedora $ yum install git-flow # CentOS $ apt-get install git-flow # Debian and friends (Search the internet for “git flow” to learn more!)
- 182. #SAISDD6 # Set up git-flow in this repository $ git flow init # Start work on my-awesome-feature; create # and switch to a feature branch $ git flow feature start my-awesome-feature $ ... # Finish work on my-awesome-feature; merge # feature/my-awesome-feature to develop $ git flow feature finish my-awesome-feature
- 183. #SAISDD6 # Start work on a release branch $ git flow release start 0.1.0 # Hack and bump version numbers $ ... # Finish work on v0.1.0; merge # release/0.1.0 to develop and master; # tag v0.1.0 $ git flow release finish 0.1.0
- 188. #SAISDD6 Maven Central Bintray not really easy to set up for library developers trivial trivial easy to set up for library users mostly yes, via sbt easy to publish yes, via sbt + plugins yes easy to resolve artifacts mostly
- 190. #SAISDD6
- 191. #SAISDD6
- 192. #SAISDD6
- 193. #SAISDD6
- 194. #SAISDD6 https://radanalytics.io eje@redhat.com • @manyangled willb@redhat.com • @willb KEEP IN TOUCH