Using Change Streams to Keep Up with Your Data

# M D B l o c a l
Aly Cabral, Product Manager
Using Change Streams to
Keep Up With Your Data

# M D B l o c a l
Alyson Cabral
PRODUCT MANAGER | MONGODB | @ ALY_CABRAL

# M D B l o c a l
“TIME IS MONEY.”
B E N J A M I N F R A N K L I N
WA R R E N B U F F E T

# M D B l o c a l
REAL-TIME IS AN EXPECTATION
@ A LY_C A B R A L

# M D B l o c a l
MongoDB
Action
Handler
@ A LY_C A B R A L

# M D B l o c a l
const changeStream=
db.collection(‘temperature’).watch();
c h a n g e S t r e a m . o n ( ' c h a n g e ' , f u n c t i o n ( c h a n g e ) {
c o n s o l e . l o g ( c h a n g e ) ;
} ) ;
@ A LY_C A B R A L

# M D B l o c a l
CHANGE STREAM
CHARACTERISTICS

# M D B l o c a l
5 OPERATION TYPES
INSERT
DELETE
REPLACE
UPDATE
INVALIDATE
@ A LY_C A B R A L

# M D B l o c a l
CHANGE STREAMS UTILIZE
COLLECTION ACCESS CONTROLS,
PRESENT A DEFINED API, AND ENABLE
SCALING ACROSS PRIMARIES AND
SECONDARIES.
@ A LY_C A B R A L

# M D B l o c a l
TOTAL ORDERING OF CHANGES ACROSS
SHARDS
mongos
3 1
2
Shard 1 Shard 2 Shard 3
@ A LY_C A B R A L

# M D B l o c a l
PS
CHANGES ARE DURABLE
P
S
@ A LY_C A B R A L

# M D B l o c a l
SP
CHANGE STREAMS ARE RESUMABLE
P
S
{_id: <resumeToken>,
operationType: ‘update’,
ns: {db: ‘data’, coll: ‘temperature’},
documentKey: {_id: ‘room12345’},
updateDescription: {updatedFields: {temperature: 70},
removedFields: [‘humidity’]}
}
var newChangeStream = coll.watch({ resumeAfter: <cachedResumeToken> });
@ A LY_C A B R A L

# M D B l o c a l
CHANGE STREAMS UTILIZE THE POWER
OF THE AGGREGATION FRAMEWORK
$match $project $addFields $replaceRoot $redact
var changeStream = coll.watch([{ $match: {$or: [{operationType: 'delete' }, {operationType:'replace'}]}}]);
@ A LY_C A B R A L

# M D B l o c a l
1. COLLECTION ACCESS CONTROLS
2. DEFINED API
3. ENABLE SCALING
4. TOTAL ORDERING
5. DURABLE
6. RESUMABLE
7. POWER OF AGGREGATION
@ A LY_C A B R A L

# M D B l o c a l
OPERATION TYPES

# M D B l o c a l
5 OPERATION TYPES
{ _id: (resumeToken),
operationType: ‘insert’,
ns: {db: ‘test’, coll: ‘foo’},
documentKey: {userName: ‘alice123’,
_id: ObjectId("58a4eb4a30c75625e00d2820")
},
fullDocument: {_id: ObjectId("58a4eb4a30c75625e00d2820"),
userName: ‘alice123’,
name: ‘Alice’
}
}
@ A LY_C A B R A L
INSERT

# M D B l o c a l
5 OPERATION TYPES
{_id: (resumeToken),
operationType: ‘delete’,
}
}
@ A LY_C A B R A L
DELETE

# M D B l o c a l
5 OPERATION
TYPES
@ A LY_C A B R A L
operationType: ‘replace’,
},
fullDocument: {_id: ObjectId("58a4eb4a30c75625e00d2820"),
name: ‘Alice’
}
}
REPLACE

# M D B l o c a l
UPDATE5 OPERATION TYPES
@ A LY_C A B R A L
},
updateDescription: {updatedFields: {email: ‘alice@10gen.com’},
removedFields: [‘phoneNumber’]}
}

# M D B l o c a l
5 OPERATION
TYPES
operationType: ‘invalidate’,
ns: {db: ‘test’, coll: ‘foo’}}
@ A LY_C A B R A L
INVALIDATE

# M D B l o c a l
_id: ObjectId("58a4eb4a30c75625e00d2820")},
updateDescription: {updatedFields: {email: ‘alice@10gen.com’},
removedFields: [‘phoneNumber’]}
}
UPDATE
var changeStream = coll.watch({ fullDocument: 'updateLookup' });
@ A LY_C A B R A L

# M D B l o c a l
var changeStream = coll.watch({ fullDocument: 'updateLookup' });
UPDATE fullDocument: updateLookup
_id: ObjectId("58a4eb4a30c75625e00d2820")},
updateDescription: {updatedFields: {email:
‘alice@10gen.com’},
removedFields: [‘phoneNumber’]},
fullDocument: { _id: ObjectId("58a4eb4a30c75625e00d2820"),
name: ‘Alice’,
email: ‘alice@10gen.com’,
team: ‘replication’}
}
@ A LY_C A B R A L

# M D B l o c a l
1 . S H A R D K E Y I S U S E D ( B Y T H E M O N G O S ) TO R O U T E
O P E R AT I O N S
2 . _ I D U N I Q U E N E S S I S E N F O R C E D B Y S H A R D
mongos
3 1
2
Shard 1 Shard 2 Shard 3
DEFINING THE documentKey
@ A LY_C A B R A L

# M D B l o c a l
BACK TO
THE USE CASE

# M D B l o c a l@ A LY_C A B R A L

# M D B l o c a l
Action Handler
documentKey: {_id: ‘greenRoom123’},
fullDocument: {_id: ‘greenRoom123’),
temperature: 75,
username: aly_cabral}
}
var changeStream = coll.watch([{
$match: {{ 'fullDocument.temperature': {$gte : 70 }}},
{fullDocument: 'updateLookup' }]);
@ A LY_C A B R A L

# M D B l o c a l
documentKey: {_id: ‘orangeRoom123’
fullDocument: {_id: ‘orangeRoom123’,
temperature: 76,
username: eliot_horowitz }
}
Action Handler
temperature: 75
username: aly_cabral
@ A LY_C A B R A L

# M D B l o c a l
Action Handler
documentKey: {_id: ‘greenRoom123’)},
updateDescription: {updatedFields: {temperature: 68}
fullDocument: {_id: ‘greenRoom123’,
temperature: 68,
}
temperature: 75
@ A LY_C A B R A L

# M D B l o c a l
IF YOUR APPLICATION REQUIRES STATE
ALWAYS MATCH ON UNCHANGING
FIELDS
@ A LY_C A B R A L

# M D B l o c a l
Action Handler
var changeStream = coll.watch([{ $match: {{ 'fullDocument.username': ‘aly_cabral’ }}},
temperature: 75
documentKey: {_id: ‘blueRoom123’},
fullDocument: {_id: ‘blueRoom123’,
temperature: 85,
}
@ A LY_C A B R A L

# M D B l o c a l
Action Handler
temperature: 75
temperature: 85
documentKey: {_id: ‘blueRoom123’}}
@ A LY_C A B R A L

# M D B l o c a l
Action Handler
documentKey: {_id: ‘blueRoom123’},
temperature: 71,
username: asya_kamsky}
}
temperature: 75
temperature: 85
@ A LY_C A B R A L

# M D B l o c a l
IF YOUR APPLICATION NEEDS TO NOTIFY ON DELETED
DATA
ALWAYS HANDLE DELETES AND
REPLACES APPROPRIATELY
@ A LY_C A B R A L

# M D B l o c a l
var changeStream = coll.watch([{ $match: {
{ $or: [ { 'fullDocument.username': ‘aly_cabral'
}, { operationType: 'delete' },
{ operationType: 'replace' }]}],
{fullDocument: 'updateLookup' });
Insert
@ A LY_C A B R A L

# M D B l o c a l
Insert Update
@ A LY_C A B R A L

# M D B l o c a l
Insert Update Delete
@ A LY_C A B R A L

# M D B l o c a l
In the update notification fullDocument: null
updateLookup
@ A LY_C A B R A L

# M D B l o c a l
In the update notification fullDocument: null
updateLookup
documentKey: {userName: ‘greenRoom123’},
updateDescription: {updatedFields: {temperature: 78}},
fullDocument: {}
}
@ A LY_C A B R A L

# M D B l o c a l
IF YOU NEED TO SEE EVERY CHANGE (EVEN OUTDATED
CHANGES) TO A DOCUMENT
HAVE MATCHING FIELDS IN THE
DOCUMENT KEY
@ A LY_C A B R A L

# M D B l o c a l
documentKey: {_id: ‘blueRoom123’,
username: “aly_cabral”},
temperature: 85,
username: “aly_cabral”}
}
Action Handler
var changeStream = coll.watch([{ $match: {{ ‘documentKey.username’: ‘aly_cabral’ }}},
temperature: 75
@ A LY_C A B R A L

# M D B l o c a l
Action Handler
temperature: 75
temperature: 85
username: “aly_cabral”)}}
@ A LY_C A B R A L

# M D B l o c a l
username: “aly_cabral”},
temperature: 85,
username: “aly_cabral”}
}
Action Handler
temperature: 75
temperature: 85
username: “aly_cabral”)}}
documentKey: {_id: ‘greenRoom123’,
username: ‘aly_cabral’},
fullDocument: {_id: ‘greenRoom123’,
temperature: 71,
username: “asya_kamsky”}
}
@ A LY_C A B R A L

# M D B l o c a l
UNDERLYING
TECHNOLOGY

# M D B l o c a l
P
S
S
S
S
@ A LY_C A B R A L

# M D B l o c a l
123456
@ A LY_C A B R A L

# M D B l o c a l
RESUMABLE
304 270 233 208 201 190
C A C H E D R E S U M E TO K E N : 2 0 8
N E W E S T O L D E S T
D E L E T E D
@ A LY_C A B R A L

# M D B l o c a l
304 270 233 208 201 190
C A C H E D R E S U M E TO K E N : 1 9 0
N E W E S T O L D E S T
D E L E T E D
NON-RESUMABLE
@ A LY_C A B R A L

# M D B l o c a l
YOU MADE IT!
@ A LY_C A B R A L

Using Change Streams to Keep Up with Your Data

More Related Content

What's hot (20)

Similar to Using Change Streams to Keep Up with Your Data (20)

More from MongoDB (20)

Using Change Streams to Keep Up with Your Data

Editor's Notes