Quill + Spark = Better Together

Alexander Ioﬀe

@deusaquilus
Quill + =
Better Together

So What’s The Difference?
• Abstraction

• Encapsulation

• Error Handling

• Good Control Flow

• Performance
Application Development
Languages
Data Retrieval  
Languages
• Natural Expression

• Possible Optimization

• Good Control Flow

• Performance

They Make Different Tradeoffs!
AbstractionPower
Possible Optimizations
Data Retrieval 
Languages
Application Development
Languages

Example Please???
CREATE VIEW HelloAmerican AS
SELECT 'Hello ' || t.firstName + ' ' || t.lastName + ' of ' || a.city
FROM Americans t
JOIN Addresses a on t.address_id == a.id
-- Hello John James of New York
CREATE VIEW HelloCanadian AS
SELECT 'Hello ' + t.name + ' ' + t.surname + ' of ' + a.city
FROM Canadians t
JOIN Addresses a on t.residence_id == a.id
-- Hello Jim Jones of Toronto
CREATE VIEW HelloYeti AS
SELECT 'Hello ' + t.gruntingSound + ' ' + t.roaringSound + ' of ' + a.city
FROM AbominableShowmen t
JOIN Addresses a on t.cave_id == a.id
-- Hello Aaargalah Gralala of Kholat Syakhl

CREATE FUNCTION AddressToSomeone (
@humanoidLivingSomewhere Table(called VARCHAR; alsoCalled VARCHAR; whereHeLives_id)
)
SELECT 'Hello ' || t.called || ' ' || t.alsoCalled || ' of ' || a.city
FROM @humanoidLivingSomewhere t
JOIN Addresses a on t.whereHeLives_id == a.id
CREATE VIEW AmericanClients
AS SELECT * from AddressToSomeone(
SELECT t.firstName as called, t.lastName as alsoCalled, a.address_id as whereHeLives_id
FROM Americans
)
CREATE VIEW CanadianClients
SELECT t.name as called, t.surname as alsoCalled, a.residence_id as whereHeLives_id
FROM Canadians
)
CREATE VIEW YetiClients
SELECT t.gruntingSound as called, t.roaringSound as alsoCalled,
a.cave_id as whereHeLives_id
FROM AbominableShowmen
)

CREATE FUNCTION concatName (
@called VARCHAR; @alsoCalled VARCHAR; @whereHeLives_id)
)
AS 'Hello ' || t.called || ' ' || t.alsoCalled || ' of ' || a.city
SELECT concatName(t.firstName, t.lastName, a.city)
FROM American t
SELECT concatName(t.name, t.surname, a.city)
FROM American t
SELECT concatName(t.gruntingSound, t.roaringSound, a.city)
FROM American t

CREATE FUNCTION concatName (
@called VARCHAR; @alsoCalled VARCHAR; @whereHeLives_id)
)
AS 'Hello ' || t.called || ' ' || t.alsoCalled || ' of ' || a.city
@humanoidLivingSomewhere Table(called VARCHAR; alsoCalled VARCHAR;
whereHeLives_id)
)
SELECT
'Hello ' ||t .called || ' ' || t.alsoCalled || ' of ' || a.city,
CASE
WHEN zd.zone_type = 'K' THEN 'StandardCategory'
WHEN zd.zone_type = 'N' AND rc.barbaz = 'GT' THEN 'NonStandardCategory'
ELSE 'UnknownCategory'
END as zoning_category1,
CASE
WHEN ru.kdd = 'IK' THEN 'Insanity'
WHEN zd.kdd = 'N' AND rc.barbaz = 'GTT' THEN 'MoreInsanity'
ELSE 'I_Dont_Even_Know_What_Goes_Here'
END as zoning_category2
JOIN Addresses a on t.whereHeLives_id = a.id
JOIN ResidenceUnit ru on a.rid = ru.id
JOIN ResidenceClass rc on ru.class_id = rc.class_id
JOIN ZoningDesignation zd on ru.zone_id = zd.rzid and zd.cid = rc.class_id
SELECT concatName(t.firstName, t.lastName, a.city)
FROM American t
SELECT concatName(t.name, t.surname, a.city)
FROM American t
SELECT concatName(t.gruntingSound, t.roaringSound, a.city)
FROM American t

)

)
WHERE a.current = true
DataFrame Can!
def addressToSomeone(df: DataFrame) = {
df.as("t")
.join(addresses.as("a"), $"whereHeLivesId" === $"id")
.select(
concat(lit("Hello "), $"t.called", lit(" "), $"t.alsoCalled",
lit(" of "), $"a.city"))
.filter($"a.current" === lit(true))
}

DataFrame Can!
addressToSomeone(
americans.select($"firstName" as "called", $"lastName" as "alsoCalled",
$"address_id" as "whereHeLives_id")
)
addressToSomeone(
canadians.select($"name" as "called", $"surname" as "alsoCalled",
$"residence_id" as "whereHeLives_id")
)
addressToSomeone(
yeti.select($"gruntSound" as "called", $"roarSound" as "alsoCalled",
$"cave_id" as "whereHeLives_id")
)
df.as("t")
.join(addresses.as("a"), $"id" === $"whereHeLives_id")
.select(
}

DataFrame Can…
addressToSomeone(
americans.select($"firstName" as "called", $"lastName" as "alsoCalled",
$"address_id" as "whereHeLives_id")
)
addressToSomeone(
canadians.select($"name" as "called", $"surname" as "alsoCalled",
$"residence_id" as "whereHeLives_id")
)
addressToSomeone(
yeti.select($"gruntSound" as "called", $"roarSound" as "alsoCalled",
$"cave_id" as "whereHeLives_id")
)
df.as("t")
.join(addresses.as("a"), $"whereHeLives_id" === $"id")
.select(
}

… Hurt!
def insaneJoin(df: DataFrame) =
df.as("t")
.join(addresses.as("a"), $"t.whereHeLives_id" === $"a.id")
.join(residenceUnit.as("ru"), $"a.rid" === $"ru.id")
.join(residenceClass.as("rc"), $"ru.class_id" === $"rc.class_id")
.join(zoningDesignation.as("zd"),
($"ru.zone_id" === "zd.rid") &&
($"zd.cid" === $"rc.class_id")
)
.select(
concat(
lit("Hello "), $"t.called", lit(" "), $"t.alsoCalled",
lit(" of "), $"a.city"),
when($"zd.zone_type" === lit("K"), "StandardCategory")
.when(($"zd.zone_type" === lit("N")) && ($"rc.barbaz" === lit("GT")),
"NonStandardCategory")
.otherwise("UnknownCategory")
.as("zoning_category1"),
when($"ru.kdd" === lit("IK"), "Insanity")
.when(($"zd.kdd" === lit("N")) && ($"rc.barbaz" === lit("GTT")),
"MoreInsanity")
.otherwise("I_Dont_Even_Know_What_Goes_Here")
.as("zoning_category2")
)

… Hurt!
def insaneJoin(df: DataFrame) =
df.as("t")
.join(addresses.as("a"), $"t.whereHeLives_id" === $"a.id")
.join(residenceUnit.as("ru"), $"a.rid" === $"ru.id")
.join(residenceClass.as("rc"), $"ru.class_id" === $"rc.class_id")
.join(zoningDesignation.as("zd"),
($"ru.zone_id" === "zd.rzid") &&
($"zd.cid" === $"rc.class_id")
)
.select(
concat(
lit("Hello "), $"t.called", lit(" "), $"t.alsoCalled",
lit(" of "), $"a.city"),
when($"zd.zone_type" === lit("K"), "StandardCategory")
.when(($"zd.zone_type" === lit("N")) && ($"rc.barbaz" === lit("GT")),
"NonStandardCategory")
.otherwise("UnknownCategory")
.as("zoning_category1"),
when($"ru.kdd" === lit("IK"), "Insanity")
.when(($"zd.kdd" === lit("N")) && ($"rc.barbaz" === lit("GTT")),
"MoreInsanity")
.otherwise("I_Dont_Even_Know_What_Goes_Here")
.as("zoning_category2")
)

case class HumanoidLivingSomewhere(
called:String, alsoCalled: String, whereHeLives_id:Int
)

Wait… Dataset?
def insaneJoin(humanoidsLivingSomewhere: Dataset[HumanoidLivingSomewhere]) =
humanoidsLivingSomewhere.as("t")
.joinWith(addresses.as("a"), $"whereHeLives_id" === $"id")
.joinWith(residenceUnit.as("ru"), $"a.rid" === $"ru.id")
.joinWith(residenceClass.as("rc"), $"ru.class_id" === $"rc.class_id")
.joinWith(zoningDesignation.as("zd"),
($"ru.zone_id" === "zd.rzid") &&
($"zd.cid" === $"ru.class_id")
)
.map { case ((((t, a), ru), rc), zd) => (
s"Hello ${t.called} ${t.alsoCalled} of ${a.city}",
if (zd.zone_type == "K") "StandardCategory"
else if (zd.zone_type == "N" && rc.barbaz == "GT")
"NonStandardCategory"
else
"UnknownCategory",
if (ru.kdd == "IK") "Insanity"
else if (zd.kdd == "N" && rc.barbaz == "GT")
"MoreInsanity"
else
"I_Dont_Even_Know_What_Goes_Here"
)
}

Wait… Dataset?
.joinWith(residenceUnit.as("ru"), $"_2.rid" === $"ru.id")
.joinWith(residenceClass.as("rc"), $"_2.class_id" === $"rc.class_id")
($"_1._2.zone_id" === "zd.rzid") &&
($"zd.cid" === $"_1._2.class_id")
)
else
"UnknownCategory",
"MoreInsanity"
else
)
}

Wait… Dataset?
($"_1._2.zone_id" === "zd.rzid") &&
($"zd.cid" === $"_1._2.class_id")
)
else
"UnknownCategory",
"MoreInsanity"
else
)
}
Dataset[(HumanoidLivingSomewhere, Address)]

Wait… Dataset?
($"_1._2.zone_id" === "zd.rzid") &&
($"zd.cid" === $"_1._2.class_id")
)
else
"UnknownCategory",
"MoreInsanity"
else
)
}
Dataset[
(((HumanoidLivingSomewhere, Address), ResidenceUnit), ResidenceClass)
]

($"_1._2.zone_id" === "zd.rzid") &&
($"zd.cid" === $"_1._2.class_id")
)
else
"UnknownCategory",
"MoreInsanity"
else
)
}

case class American(
firstName:String, lastName:String, address_id:Int,
irrelevantP1:String... irrelevantP100:String
)
case class Canadian(
name:String, surname:String, residence_id:Int,
)
case class Yeti(
gruntingSound:String, roaringSound:String, address_id:Int,
)
Say There's Stuff We Don't Care About
case class Address(
id:Int, street:String, city:String, current: Boolean
irrelevantA1:String... irrelevantA100:String
)
case class HumanoidLivingSomewhere(
called:String, alsoCalled: String, whereHeLives_id:Int
)

Let's Plug it In!
def addressToSomeone(humanoidLivingSomewhere: DataFrame) = {
humanoidLivingSomewhere.as("t")
.select(
concat(lit("Hello "), $"t.called", lit(" "), $"t.alsoCalled", lit(" of "), $"a.city"))
}
addressToSomeone(
americans.select(
$"firstName" as "called", $"lastName" as "alsoCalled", $"address_id" as "whereHeLives_id")
)
americans
.select($"firstName" as "called", $"lastName" as "alsoCalled", $"address_id" as "whereHeLives_id")
.as("t")
.select(

Explain This!
*(5) Project [concat(Hello , called, , alsoCalled, of , city)]
+- *(5) SortMergeJoin [whereHeLives_id], [id], Inner
+- Exchange hashpartitioning(whereHeLives_id)
+- *(1) Project [firstName AS called, ... AS whereHeLives_id]
+- *(1) FileScan parquet [firstName,lastName,address_id]
+- Exchange hashpartitioning(id)
+- *(3) FileScan parquet [id,city,current]
PushedFilters: [EqualTo(current,true)],
americans
.as("t")
.select(
.explain()

americans
.as("t")
.select(
.explain()
Explain This!

In English Please?
*(🤞) Gimme My Result! [concat(Hello , called, , alsoCalled, of , city)]
+- *(💂) We're Joining! Huzzah! [whereHeLives_id], [id], Inner
+- Join Key for the Left Side! (whereHeLives_id)
+- *(1) Rename these like I said! Pronto! [firstName as Called... ]
+- *(😇) I'm a smart format, load only: [firstName,lastName,address_id]
+- Join Key for the Right Side! (id)
+- *(😇) I'm a smart format, load only: [id,city,current]
Read only current addr. from the file! 😎: [EqualTo(current,true)],
americans
.as("t")
.select(
.explain()

How About Dataset?
def addressToSomeone(humanoidsLivingSomewhere: Dataset[HumanoidLivingSomewhere]) = {
humanoidsLivingSomewhere
.joinWith(addresses, $"id" === $"whereHeLivesId")
.filter(ta => ta._2.current == true)
.map { case (t, a) => s"Hello ${t.called} ${t.alsoCalled} of ${a.city}" }
}
val americanClients =
addressToSomeone(
americans.map(a => HumanoidLivingSomewhere(a.firstName, a.lastName, a.addressId))
)

Explain This... Please!
*(3) SerializeFromObject [UTF8String]
+- *(3) MapElements java.lang.String
+- DeserializeToObject newInstance(Tuple2)
+- SortMergeJoin [_1.whereHeLives_id], [_2.id], Inner
+- Exchange hashpartitioning(_1.whereHeLives_id)
+- *(1) Project [called, alsoCalled, whereHeLives_id]
+- *(1) SerializeFromObject [UTF8String]
+- *(1) MapElements HumanoidLivingSomewhere
+- DeserializeToObject newInstance(American)
+- FileScan parquet [firstName,lastName,address_id,irrelevantP1,irrelevantP2,i
+- Exchange hashpartitioning(_2.id)
+- FileScan parquet [id,street,city,current,irrelevantA1,irrelevantA2,irrelevantA3,irrel
PushedFilters: []
.explain()

*(🤮) Serialize Back Into a String Expensive!
+- *(3) Do the Outer Map that we Invoked
+- (🤮) Deserialize Tuple2 Expensive!
+- & We're Joining! Huzzah! [_1.whereHeLives_id], [_2.id], Inner
+- Join Key for the Left Side (_1.whereHeLives_id)
+- *(1) Project [called, alsoCalled, whereHeLives_id]
+- *(🤮) Serialize the Join Key. Expensive!
+- *(1) MapElements HumanoidLivingSomewhere
+- (🤮) Deserialize into a JVM Object (i.e. class American)
+- Scan All 'American' Columns Including 100 irrelevant ones!😱
+- Join Key for the Right Side (_2.id)
+- Scan All 'Address' Columns Including 100 irrelevant ones! 😱
We Need to Read The Entire Dataset! No Excluding Non-Current Addresses 😢
.explain()

americans.map(a =>
HumanoidLivingSomewhere(a.firstName, a.lastName, a.address_id)
)
.joinWith(addresses, $"whereHeLives_id" === $"id")
.filter { tup => tup._2.current == true }
What We See:
What Catalyst Sees:
americans.map(¯_(ツ)_/¯ : HumanoidLivingSomewhere)
.filter { ¯_(ツ)_/¯ : Boolean }
.map { ¯_(ツ)_/¯ : String }

americans.map(a =>
)
What We See:
What Catalyst Sees:
.map { ¯_(ツ)_/¯ : String }
Which Columns are
we using in here?
Which Columns are
we using in here?
Which Columns are
we using in here?
⏸

americans.map(a =>
)
What We See:
What Catalyst Sees:
.map { ¯_(ツ)_/¯ : String }
I Guess We Need 
All Of Them!
I Guess We Need 
All Of Them!
I Guess We Need 
All Of Them!
⏸

americans.map(a =>
)
What We See:
What Catalyst Sees:
.joinWith(addresses, ¯_(ツ)_/¯)
.map { ¯_(ツ)_/¯ : String }
⏸
What columns
am I joining by???

americans.map(a =>
)
What We See:
What Catalyst Sees:
americans.map( (🖼 American) => HumanoidLivingSomewhere 🖼 ) 🤮
.filter { (🖼 HumanoidLivingSomewhere) => Boolean } 🤮
.map { (🖼 HumanoidLivingSomewhere) => String 🖼 } 🤮
→ →
→
→ →

def addressToSomeone(humanoid: TypedDataset[HumanoidLivingSomewhere]) = {
val joined = humanoid
.joinInner(addresses) { humanoid('where) === addresses('id) }
joined.select(concat(
lit("Hello "), joined.colMany('_1, 'called), lit(" "),
joined.colMany('_1, 'alsoCalled), lit(" of "), joined.colMany('_2, 'city)))
}
What About Frameless?
addressToSomeone(
americans.select(americans('firstName), americans('lastName), americans('addressId))
.deserialized.map{ case (name, age, whereHeLives_id ) =>
HumanoidLivingSomewhere(
name.asInstanceOf[String],
age.asInstanceOf[String],
whereHeLives_id.asInstanceOf[Int])
}
)

}
addressToSomeone(
americans.select(americans('firstName), americans('lastName), americans('addressId))
.deserialized.map{ case (name, age, whereHeLives_id ) =>
HumanoidLivingSomewhere(
name.asInstanceOf[String],
age.asInstanceOf[String],
whereHeLives_id.asInstanceOf[Int])
}
)

}

val j1 = humanoid.joinInner(addresses) { humanoid('whereHeLives_id) === addresses('id) }
val j2 = j1.joinInner(residenceUnit) { j1.colMany('_2, 'rid) === residenceUnit('id) }
val j3 = j2.joinInner(residenceClass) { j2.colMany('_2, 'class_id) === residenceClass('class_id) }
val j4 = j3.joinInner(zoningDesignation) {
(j3.colMany('_1, '_2, 'zone_id) === zoningDesignation('rzid)) &&
(zoningDesignation('rzid) === j3.colMany('_1, '_, 'class_id))
}
type OUT = ((((HumanoidLivingSomewhere, Address), ResidenceUnit), ResidenceClass), ZoningDesignation)
j4.select(
concat(
lit("Hello "), j4.colMany('_1, '_1, '_1, '_1, 'called), lit(" "),
j4.colMany('_1, '_1, '_1, '_1, 'alsoCalled), lit(" of "), j4.colMany('_1, '_1, '_1, '_2, 'city)
),
when(j4.colMany('_2, 'zone_type) === "K", lit[String, OUT]("StandardCategory"))
.when((j4.colMany('_2, 'zone_type) === "N") && (j4.colMany('_1, '_2, 'barbaz) === "GT"),
lit("NonStandardCategory"))
.otherwise( lit("UnknownCategory")),
when(j4.colMany('_1, '_1, '_2, 'kdd) === "IK", lit[String, OUT]("Insanity"))
.when((j4.colMany('_2, 'kdd) === "N") && (j4.colMany('_1, '_2, 'barbaz) === "GTT"),
lit("MoreInsanity"))
.otherwise(lit("I_Dont_Even_Know_What_Goes_Here"))
)
}
What Sub-Tuple is class_id inside of?

}
j4.select(
concat(
),
)
}
What Sub-Tuple is class_id inside of?
Dataset[
(((HumanoidLivingSomewhere, Address), ResidenceUnit), ResidenceClass)
]

}
j4.select(
concat(
),
)
}
Also... What's This???

[error] found : frameless.TypedColumn[Nothing,String]
[error] required:
frameless.AbstractTypedColumn[((((org.ctl.complex.HumanoidLivingSomewhere,
org.ctl.complex.Address), org.ctl.complex.ResidenceUnit),
org.ctl.complex.ResidenceClass), org.ctl.complex.ZoningDesignation),String]
[error] Note: Nothing <: ((((org.ctl.complex.HumanoidLivingSomewhere,
org.ctl.complex.Address), org.ctl.complex.ResidenceUnit),
org.ctl.complex.ResidenceClass), org.ctl.complex.ZoningDesignation), but class
AbstractTypedColumn is invariant in type T.
[error] You may wish to define T as +T instead. (SLS 4.5)
[error] when(j4.colMany('_2, 'zone_type) === "K", lit("StandardCategory"))

DataFrame/SQL Untyped 😢
Column Pruning
Filter Pushdown 😎
Dataset Almost Typed 😕 Extra Serialization 🤮
Frameless Typed 😃
Very Complex if you
don't know Shapeless.

TypecheckingPower
Possible Optmizations
DataFrame/
Scala Code
(i.e. Dataset[T])
SQL

?
Scala Code SQL DataFrame
TypecheckingPower
Possible Optmizations

Scala Code SQL
Quill
DataFrame
quote {
scala-syntax-tree
}
AST
Macro

Scala Code SQL
Quill
quote {
scala-syntax-tree
}
AST
Macro

Query[R] SQL
Quill
quote { Query[R] }
AST
Macro
Quoted[Query[R]]

@humanoid
Table(
called VARCHAR;
alsoCalled VARCHAR;
whereHeLives_id
)
)
SELECT
'Hello ' ||
h.called || ' ' ||
h.alsoCalled || ' of ' ||
a.city
FROM
@humanoid h
JOIN
Addresses a
ON
h.whereHeLives_id == a.id
WHERE
a.current == true
case class Humanoid(
called:String,
alsoCalled: String,
whereHeLives_id:Int
)
val addressToSomeone = quote {
(humanoid: Query[?]) =>
for {
h <- humanoid
a <- addresses.join(a =>
a.id == h.whereHeLives_id
)
if (a.current == true)
} yield (
"Hello " +
h.called + " " +
h.alsoCalled + " of " +
a.city
)
}

@humanoid
Table(
called VARCHAR;
alsoCalled VARCHAR;
whereHeLives_id
)
)
SELECT
'Hello ' ||
h.called || ' ' ||
a.city
FROM
@humanoid h
JOIN
Addresses a
ON
WHERE
a.current == true
called:String,
alsoCalled: String,
whereHeLives_id:Int
)
(humanoid: Query[Humanoid]) =>
for {
h <- humanoid
)
} yield (
"Hello " +
h.called + " " +
a.city
)
}

@humanoid
Table(
called VARCHAR;
alsoCalled VARCHAR;
whereHeLives_id
)
)
SELECT
'Hello ' ||
h.called || ' ' ||
a.city
FROM
@humanoid h
JOIN
Addresses a
ON
WHERE
a.current == true
called:String,
alsoCalled: String,
whereHeLives_id:Int
)
for {
h <- humanoid
)
} yield ( )
"Hello " +
h.called + " " +
a.city
)
}
Query [Humanoid]
Humanoid
Query [Address]
Address
⏸

@humanoid
Table(
called VARCHAR;
alsoCalled VARCHAR;
whereHeLives_id
)
)
SELECT
'Hello ' ||
h.called || ' ' ||
a.city
FROM
@humanoid h
JOIN
Addresses a
ON
WHERE
a.current == true
called:String,
alsoCalled: String,
whereHeLives_id:Int
)
for {
h <- humanoid
a <- addresses if (
)
} yield (
"Hello " +
h.called + " " +
a.city
)
}

@humanoid
Table(
called VARCHAR;
alsoCalled VARCHAR;
whereHeLives_id
)
)
SELECT
'Hello ' ||
h.called || ' ' ||
a.city
FROM
@humanoid h
LEFT JOIN
Addresses a
ON
WHERE
a.current == true
called:String,
alsoCalled: String,
whereHeLives_id:Int
)
for {
h <- humanoid
a <- addresses.leftJoin(a =>
)
} yield (
"Hello " +
h.called + " " +
a.city
)
}

@humanoid
Table(
called VARCHAR;
alsoCalled VARCHAR;
whereHeLives_id
)
)
SELECT
'Hello ' ||
h.called || ' ' ||
a.city
FROM
@humanoid h
LEFT JOIN
Addresses a
ON
WHERE
a.current == true
called:String,
alsoCalled: String,
whereHeLives_id:Int
)
for {
h <- humanoid
)
} yield (
"Hello " +
h.called + " " +
a.city
)
}
Address

@humanoid
Table(
called VARCHAR;
alsoCalled VARCHAR;
whereHeLives_id
)
)
SELECT
'Hello ' ||
h.called || ' ' ||
a.city
FROM
@humanoid h
LEFT JOIN
Addresses a
ON
WHERE
a.current == true
called:String,
alsoCalled: String,
whereHeLives_id:Int
)
for {
h <- humanoid
)
} yield (
"Hello " +
h.called + " " +
a.city
)
}
Address
Option[Address] Address

called:String,
alsoCalled: String,
whereHeLives_id:Int
)
for {
h <- humanoid
)
} yield (
"Hello " +
h.called + " " +
a.city
)
}
@humanoid
Table(
called VARCHAR;
alsoCalled VARCHAR;
whereHeLives_id
)
)
SELECT
'Hello ' ||
h.called || ' ' ||
a.city
FROM
@humanoid h
JOIN
Addresses a
ON
WHERE
a.current == trueString

called:String,
alsoCalled: String,
whereHeLives_id:Int
)
for {
h <- humanoid
)
} yield (
"Hello " +
h.called + " " +
a.city
)
}
@humanoid
Table(
called VARCHAR;
alsoCalled VARCHAR;
whereHeLives_id
)
)
SELECT
'Hello ' ||
h.called || ' ' ||
a.city
FROM
@humanoid h
JOIN
Addresses a
ON
WHERE
a.current == true
Query[String]

called:String,
alsoCalled: String,
whereHeLives_id:Int
)
for {
h <- humanoid
)
} yield (
"Hello " +
h.called + " " +
a.city
)
}
@humanoid
Table(
called VARCHAR;
alsoCalled VARCHAR;
whereHeLives_id
)
)
SELECT
'Hello ' ||
h.called || ' ' ||
a.city
FROM
@humanoid h
JOIN
Addresses a
ON
WHERE
a.current == true
Query[Humanoid] => Query[String]
Query[String]

called:String,
alsoCalled: String,
whereHeLives_id:Int
)
for {
h <- humanoid
)
} yield (
"Hello " +
h.called + " " +
a.city
)
}
@humanoid
Table(
called VARCHAR;
alsoCalled VARCHAR;
whereHeLives_id
)
)
SELECT
'Hello ' ||
h.called || ' ' ||
a.city
FROM
@humanoid h
JOIN
Addresses a
ON
WHERE
a.current == true
Quoted[Query[Humanoid] => Query[String]]
Query[Humanoid] => Query[String]
Query[String]

called:String,
alsoCalled: String,
whereHeLives_id:Int
)
for {
h <- humanoid
)
} yield h
}
@humanoid
Table(
called VARCHAR;
alsoCalled VARCHAR;
whereHeLives_id
)
)
SELECT
h.*
FROM
@humanoid h
JOIN
Addresses a
ON
WHERE
a.current == true
Quoted[Query[Humanoid] => Query[Humanoid]]
Query[Humanoid] => Query[Humanoid]
Query[Humanoid]

called:String,
alsoCalled: String,
whereHeLives_id:Int
)
for {
h <- humanoid
)
} yield a
}
@humanoid
Table(
called VARCHAR;
alsoCalled VARCHAR;
whereHeLives_id
)
)
SELECT
a.*
FROM
@humanoid h
JOIN
Addresses a
ON
WHERE
a.current == true
Quoted[Query[Humanoid] => Query[Address]]
Query[Humanoid] => Query[Address]
Query[Address]

called:String,
alsoCalled: String,
whereHeLives_id:Int
)
for {
h <- humanoid
)
} yield (h, a)
}
@humanoid
Table(
called VARCHAR;
alsoCalled VARCHAR;
whereHeLives_id
)
)
SELECT
h.*, a.*
FROM
@humanoid h
JOIN
Addresses a
ON
WHERE
a.current == true
Quoted[Query[Humanoid] => Query[(Humanoid, Address)]]
Query[Humanoid] => Query[(Humanoid, Address)]
Query[(Humanoid, Address)]

called:String,
alsoCalled: String,
whereHeLives_id:Int
)
for {
h <- humanoid
)
} yield (Foobar(h, a))
}
@humanoid
Table(
called VARCHAR;
alsoCalled VARCHAR;
whereHeLives_id
)
)
SELECT
?? I don't understand objects ??
FROM
@humanoid h
JOIN
Addresses a
ON
WHERE
a.current == true
Quoted[Query[Humanoid] => Query[Foobar]]
Query[Humanoid] => Query[Foobar]
Query[Foobar]

@humanoid
Table(
called VARCHAR;
alsoCalled VARCHAR;
whereHeLives_id
)
)
SELECT
'Hello ' ||
h.called || ' ' ||
a.city
FROM
@humanoid h
JOIN
Addresses a
ON
WHERE
a.current == true
called:String,
alsoCalled: String,
whereHeLives_id:Int
)
for {
h <- humanoid
)
} yield (
"Hello " +
h.called + " " +
a.city
)
}
for {
h <- humanoid
)
} yield (
"Hello " +
h.called + " " +
a.city
)
▶

for {
h <- humanoid
)
} yield (
"Hello " +
h.called + " " +
a.city
)

for {
h <- humanoid
a <- addresses.join(a => a.id == h.whereHeLives_id)
} yield (
"Hello " +
h.called + " " +
a.city
)

for {
h <- humanoid
ru <-residenceUnit.join(ru => a.rid == ru.id)
} yield (
"Hello " +
h.called + " " +
a.city
)

for {
h <- humanoid
ru <-residenceUnit.join(ru => a.rid == ru.id) 
rc <- residenceClass.join(rc => ru.class_id == rc.class_id)
} yield (
"Hello " +
h.called + " " +
a.city
)

for {
h <- humanoid
zd <- zoningDesignation.join(zd =>
ru.zone_id == zd.rid && zd.cid == rc.class_id
)
} yield (
"Hello " +
h.called + " " +
a.city
)

for {
h <- humanoid
ru.zone_id == zd.r && zd.cid == rc.class_id
)
} yield (
"Hello " +
h.called + " " +
a.city,
else if (zd.zone_type == "N" && rc.barbaz == "GT") "NonStandard"
else "UnknownCategory"
)

for {
h <- humanoid
ru.zone_id == zd.r && zd.cid == rc.class_id
)
} yield (
"Hello " +
h.called + " " +
a.city,
else "UnknownCategory",
else if (zd.kdd == "N" && rc.barbaz == "GTT") "MoreInsanity"
else "I_Dont_Even_Know_What_Goes_Here"
)

for {
h <- humanoid
)
} yield (
"Hello " +
h.called + " " +
a.city,
)
[error] quillspark-examples/src/main/scala/Main.scala:107:28:
value rid is not a member of org.ctl.complex.ZoningDesignation
[error] ru.zone_id == zd.rid && zd.cid == rc.class_id
[error] ^

for {
h <- humanoid
)
} yield (
"Hello " +
h.called + " " +
a.city,
)
⏸

($"_1._2.zone_id" === "zd.rzid") &&
($"zd.cid" === $"_1._2.class_id")
)
else
"UnknownCategory",
"MoreInsanity"
else
)
}

for {
h <- humanoid
)
} yield (
"Hello " +
h.called + " " +
a.city,
)

for {
h <- humanoid
)
} yield (
"Hello " +
h.called + " " +
a.city,
else "UnknownCategory"
)

called:String,
alsoCalled: String,
whereHeLives_id:Int
)
}
@humanoid
Table(
called VARCHAR;
alsoCalled VARCHAR;
whereHeLives_id
)
)
SELECT
'Hello ' ||
h.called || ' ' ||
a.city
FROM
@humanoid h
JOIN
Addresses a
ON
WHERE
a.current == true
for {
h <- humanoid
)
} yield (
"Hello " +
h.called + " " +
a.city
)
▶

@humanoid
Table(
called VARCHAR;
alsoCalled VARCHAR;
whereHeLives_id
)
)
SELECT
'Hello ' ||
h.called || ' ' ||
a.city
FROM
@humanoid h
JOIN
Addresses a
ON
WHERE
a.current == true
▶
called:String,
alsoCalled: String,
whereHeLives_id:Int
)
for {
h <- humanoid
)
} yield (
"Hello " +
h.called + " " +
a.city
)
}

called:String,
alsoCalled: String,
whereHeLives_id:Int
)
for {
h <- humanoid
)
} yield (
"Hello " +
h.called + " " +
a.city
)
}
SELECT
'Hello ' ||
h.firstName || ' ' ||
h.lastName || ' of ' ||
a.city
FROM
Americans h
JOIN
Addresses a
ON
h.address_id == a.id
WHERE
a.current == true
quote { addressToSomeone(americans.map(am =>  
Humanoid(am.firstName, am.lastName, am.address_id))) }

SELECT
'Hello ' ||
a.city
FROM
Americans h
JOIN
Addresses a
ON
WHERE
a.current == true
called:String,
alsoCalled: String,
whereHeLives_id:Int
)
for {
h <- humanoid
)
} yield (
"Hello " +
h.called + " " +
a.city
)
}
quote {addressToSomeone(canadians.map(am =>  
Humanoid(am.name, am.surname, am.residence_id)))}
SELECT
'Hello ' ||
h.name || ' ' ||
h.surname || ' of ' ||
a.city
FROM
Canadians h
JOIN
Addresses a
ON
h.residence_id == a.id
WHERE
a.current == true

SELECT
'Hello ' ||
a.city
FROM
Americans h
JOIN
Addresses a
ON
WHERE
a.current == true
called:String,
alsoCalled: String,
whereHeLives_id:Int
)
for {
h <- humanoid
)
} yield (
"Hello " +
h.called + " " +
a.city
)
}
quote {addressToSomeone(yeti(am =>  
Humanoid(am.gruntingSound, am.roaringSound, am.cave_id)))}
SELECT
'Hello ' ||
h.name || ' ' ||
a.city
FROM
Canadians h
JOIN
Addresses a
ON
WHERE
a.current == true
SELECT
'Hello ' ||
h.gruntingSound || ' ' ||
h.roaringSound || ' of ' ||
a.city
FROM
Yeti h
JOIN
Addresses a
ON
h.cave_id == a.id
WHERE
a.current == true

SELECT
'Hello ' ||
a.city
FROM
Americans h
JOIN
Addresses a
ON
WHERE
a.current == true
called:String,
alsoCalled: String,
whereHeLives_id:Int
)
for {
h <- humanoid
)
} yield (
"Hello " +
h.called + " " +
a.city
)
}
SELECT
'Hello ' ||
h.name || ' ' ||
a.city
FROM
Canadians h
JOIN
Addresses a
ON
WHERE
a.current == true
SELECT
'Hello ' ||
a.city
FROM
Yeti h
JOIN
Addresses a
ON
h.cave_id == a.id
WHERE
a.current == true
▶
Humanoid(am.gruntingSound, am.roaringSound, am.cave_id)))}

SELECT
'Hello ' ||
a.city
FROM
Americans h
JOIN
Addresses a
ON
WHERE
a.current == true
SELECT
'Hello ' ||
h.name || ' ' ||
a.city
FROM
Canadians h
JOIN
Addresses a
ON
WHERE
a.current == true
SELECT
'Hello ' ||
a.city
FROM
Yeti h
JOIN
Addresses a
ON
h.cave_id == a.id
WHERE
a.current == true
DataFrame
Humanoid(am.gruntingSound,
am.roaringSound, am.cave_id)))}
quote {addressToSomeone(canadians.map(am =>  
Humanoid(am.name, am.surname,
am.residence_id)))}
quote { addressToSomeone(americans.map(am =>  
Humanoid(am.firstName, am.lastName,
am.address_id))) }
run(Query[String]) run(Query[String]) run(Query[String])

DataFrame 
Dataset[String]
SELECT
'Hello ' ||
a.city
FROM
Americans h
JOIN
Addresses a
ON
WHERE
a.current == true
SELECT
'Hello ' ||
h.name || ' ' ||
a.city
FROM
Canadians h
JOIN
Addresses a
ON
WHERE
a.current == true
SELECT
'Hello ' ||
a.city
FROM
Yeti h
JOIN
Addresses a
ON
h.cave_id == a.id
WHERE
a.current == true
run(Query[String]) run(Query[String]) run(Query[String])

DataFrame 
Dataset[Humanoid]
SELECT
h
FROM
Americans h
JOIN
Addresses a
ON
WHERE
a.current == true
SELECT
h
FROM
Canadians h
JOIN
Addresses a
ON
WHERE
a.current == true
SELECT
h
FROM
Yeti h
JOIN
Addresses a
ON
h.cave_id == a.id
WHERE
a.current == true
run(Query[Humanoid]) run(Query[Humanoid]) run(Query[Humanoid])

DataFrame 
Dataset[T]
run(Query[T]) run(Query[T]) run(Query[T])
SELECT
?
FROM
Americans h
JOIN
Addresses a
ON
WHERE
a.current == true
SELECT
?
FROM
Canadians h
JOIN
Addresses a
ON
WHERE
a.current == true
SELECT
?
FROM
Yeti h
JOIN
Addresses a
ON
h.cave_id == a.id
WHERE
a.current == true

Dataset[T] DataFrame
Easy! Just '.toDF'
Harder! '.[DoIReallyKnowItsThis?]'
run(Query[T])

val spark = SparkSession.builder()
.appName("SparkQuillExample")
.enableHiveSupport()
.getOrCreate()
implicit val sqlContext = spark.sqlContext
import sqlContext.implicits._
import QuillSparkContext._
val yetiDS = spark.read.parquet("output/yeti").as[Yeti]
val addressesDS = spark.read.parquet("output/addresses").as[Address]
Dataset[Yeti]
Dataset[Address]

val yeti = quote { liftQuery(yetiDS) }
val addresses = quote { liftQuery(addressesDS) }
Quoted[Query[Yeti]]
Quoted[Query[Address]]
Dataset[Yeti]
Dataset[Address]

(humanoids: Query[Humanoid]) =>
for {
h <- humanoids
if (a.current)
} yield "Hello " + h.called + " " + h.alsoCalled + " of " + a.city
}
val output = quote {
addressToSomeone(
yeti.map(am =>
Humanoid(am.firstName, am.lastName, am.address_id)
)
)
}

for {
h <- humanoids
if (a.current)
}
val yetiOfSomeplace: Dataset[String] = run(output)
addressToSomeone(
yeti.map(am =>
)
)
}

for {
h <- humanoids
if (a.current)
}
addressToSomeone(
yeti.map(am =>
)
)
}
Run This Query:
Then Give Me Back
My Dataset!!!
SELECT
'Hello ' ||
a.city
FROM
Yeti h
JOIN
Addresses a
ON
h.cave_id == a.id
WHERE
a.current == true

for {
h <- humanoids
if (a.current)
}
*(5) Project [concat(Hello , firstName, , lastName, of , city)]
+- *(5) SortMergeJoin [address_id], [id], Inner
+- Exchange hashpartitioning(address_id)
+- *(1) Project [firstName, lastName, address_id]
PushedFilters: [EqualTo(current,true)]
addressToSomeone(
yeti.map(am =>
)
)
}

for {
h <- humanoids
if (a.current)
}
addressToSomeone(
yeti.map(am =>
)
)
}
Optimized + No Serialization 😎
Optimized + No Serialization 😎

val addressToSomeone = quote { Quill Magic! }
▶
val output = quote { Quill Magic! }
val yetiDS: Dataset[Yeti] = parquet("output/yeti").as[Yeti]
val addressesDS: Dataset[Address] = parquet("output/addresses").as[Address]

▶
val addressToSomeone = quote { Quill Magic! }
val output = quote { Quill Magic! }
val yetiDS: Dataset[Yeti] = parquet("output/yeti").as[Yeti]
val addressesDS: Dataset[Address] = parquet("output/addresses").as[Address]

// Applicative Joins
yeti.join(addresses).on(_.caveId == _.id)
yeti.leftJoin(addresses).on(_.caveId == _.id)
// Implicit Joins
for {
y <- yeti
a <- addresses if (y.caveId == a.id)
} yield (y, a)
// Semi-Joins
val cavelessYeti = quote {
yeti.filter(y => !addresses.map(_.id).contains(y.caveId))
}
Some other stuff we can do...

// Group-By
orders.groupBy(_.sku).map {
case (sku, orders) => (sku, orders.map(_.price).avg)
}
// Concat-Map
val nodesChildren = quote {
(ns: Query[Node]) => ns.concatMap(n => n.children)
}
// Union/UnionAll
val americansAndCanadians = quote {
americans.map(_.firstName) unionAll canadians.map(_.surname)
}

// User Defined Aggregation Functions (UDAFs)
spark.udf.register("geomMean", new GeometricMean)
val geomMean = quote {
(q: Query[BigDecimal]) => infix"geomMean(${q})".as[BigDecimal]
}
orders.groupBy(_.sku).map {
case (sku, orders) => (sku, geomMean(orders.map(_.price)))
}
// Using Spark UDFs
spark.udf.register("businessLogicUdf", (str:String) => str + "-suffix")
val businessLogicUdf = quote {
(str: String) => infix"businessLogicUdf(${str})".as[String]
}
quote {
yeti.map(y => businessLogicUdf(y.gruntingSound))
}

https://getquill.io/

https://github.com/getquill/quill

https://gitter.im/getquill/quill
...Try It Out!
libraryDependencies ++= Seq(
"io.getquill" %% "quill-spark" % "3.4.10"
)
<dependency>
<groupId>io.getquill</groupId>
<artifactId>quill-spark_2.12</artifactId>
<version>3.4.10</version>
</dependency>

Quill + Spark = Better Together

More Related Content

What's hot (14)

Similar to Quill + Spark = Better Together (20)

Recently uploaded (20)

Quill + Spark = Better Together