[NodeConf.eu 2014] Scaling AB Testing on Netflix.com with Node.js
4 likes2,048 views
The document discusses the implementation of A/B testing and conditional dependency management on Netflix, showcasing 2,097,152 unique experiences across multiple tests. It details the architecture and strategy behind a large-scale A/B testing framework that optimizes product development through data-driven approaches. The findings emphasize efficient asset management and the importance of static analysis in enhancing deployment speeds.
![[NodeConf.eu 2014] Scaling AB Testing on Netflix.com with Node.js](https://image.slidesharecdn.com/nodeconfeu2014scalingabtestingonnetflix-140909054728-phpapp02/85/NodeConf-eu-2014-Scaling-AB-Testing-on-Netflix-com-with-Node-js-3-320.jpg)
![[NodeConf.eu 2014] Scaling AB Testing on Netflix.com with Node.js](https://image.slidesharecdn.com/nodeconfeu2014scalingabtestingonnetflix-140909054728-phpapp02/85/NodeConf-eu-2014-Scaling-AB-Testing-on-Netflix-com-with-Node-js-4-320.jpg)
![[NodeConf.eu 2014] Scaling AB Testing on Netflix.com with Node.js](https://image.slidesharecdn.com/nodeconfeu2014scalingabtestingonnetflix-140909054728-phpapp02/85/NodeConf-eu-2014-Scaling-AB-Testing-on-Netflix-com-with-Node-js-5-320.jpg)
![[NodeConf.eu 2014] Scaling AB Testing on Netflix.com with Node.js](https://image.slidesharecdn.com/nodeconfeu2014scalingabtestingonnetflix-140909054728-phpapp02/85/NodeConf-eu-2014-Scaling-AB-Testing-on-Netflix-com-with-Node-js-14-320.jpg)
![[NodeConf.eu 2014] Scaling AB Testing on Netflix.com with Node.js](https://image.slidesharecdn.com/nodeconfeu2014scalingabtestingonnetflix-140909054728-phpapp02/85/NodeConf-eu-2014-Scaling-AB-Testing-on-Netflix-com-with-Node-js-19-320.jpg)
![[NodeConf.eu 2014] Scaling AB Testing on Netflix.com with Node.js](https://image.slidesharecdn.com/nodeconfeu2014scalingabtestingonnetflix-140909054728-phpapp02/85/NodeConf-eu-2014-Scaling-AB-Testing-on-Netflix-com-with-Node-js-23-320.jpg)
!["app.js": {
"deps": [
"jquery",
"oldSearch.js",
"newSearch.js",
],
"depsFull": [
"jquery",
"oldSearchDep2.js",
"oldSearchDep1.js",
"oldSearch.js",
"newSearchDep2.js",
"newSearchDep1.js",
"newSearch.js"
],
"fileSize": "4.41 kB",
"fileSizeFull": "120.52 kB"
}](https://image.slidesharecdn.com/nodeconfeu2014scalingabtestingonnetflix-140909054728-phpapp02/85/NodeConf-eu-2014-Scaling-AB-Testing-on-Netflix-com-with-Node-js-32-320.jpg)
!["newSearch.js": {
"rule": "inNewSearch",
"deps": [
"jquery",
"newSearchDep2.js",
"newSearchDep1.js",
],
"depsFull": [
"jquery",
"newSearchSubDep3.js",
"newSearchSubDep2.js"
"newSearchSubDep1.js"
"newSearchDep2.js",
"newSearchDep1.js"
],
"fileSize": "10.41 kB",
"fileSizeFull": "40.52 kB"
}
nf-include-when](https://image.slidesharecdn.com/nodeconfeu2014scalingabtestingonnetflix-140909054728-phpapp02/85/NodeConf-eu-2014-Scaling-AB-Testing-on-Netflix-com-with-Node-js-33-320.jpg)
!["app.js": {
"deps": [
"jquery",
"oldSearch.js",
"newSearch.js",
],
"depsFull": [
"jquery",
"oldSearchDep2.js",
"oldSearchDep1.js",
"oldSearch.js",
"newSearchDep2.js",
"newSearchDep1.js",
"newSearch.js"
],
"fileSize": "4.41 kB",
"fileSizeFull": "120.52 kB"
}
Step 1:
Get the full dependency tree for the
requested package from the registry.](https://image.slidesharecdn.com/nodeconfeu2014scalingabtestingonnetflix-140909054728-phpapp02/85/NodeConf-eu-2014-Scaling-AB-Testing-on-Netflix-com-with-Node-js-36-320.jpg)
![[
"jquery", /* no rule */
"oldSearchDep2.js", /* no rule */
"oldSearchDep1.js", /* no rule */
"oldSearch.js", /* rules.notInNewSearch */
"newSearchDep2.js", /* no rule */
"newSearchDep1.js”, /* no rule */
"newSearch.js" /* rules.inNewSearch */
]
Step 2:
Determine which files have rules.](https://image.slidesharecdn.com/nodeconfeu2014scalingabtestingonnetflix-140909054728-phpapp02/85/NodeConf-eu-2014-Scaling-AB-Testing-on-Netflix-com-with-Node-js-37-320.jpg)
![[
"jquery", /* no rule */
"oldSearchDep2.js", /* no rule */
"oldSearchDep1.js", /* no rule */
"oldSearch.js", /* rules.notInNewSearch */
"newSearchDep2.js", /* no rule */
"newSearchDep1.js”, /* no rule */
"newSearch.js" /* rules.inNewSearch */
]
Step 3:
Run the rules. Filter out all deps that
resolved false.
✓](https://image.slidesharecdn.com/nodeconfeu2014scalingabtestingonnetflix-140909054728-phpapp02/85/NodeConf-eu-2014-Scaling-AB-Testing-on-Netflix-com-with-Node-js-38-320.jpg)
![[
"jquery", /* no rule */
"oldSearchDep2.js", /* no rule */
"oldSearchDep1.js", /* no rule */
"oldSearch.js", /* rules.notInNewSearch */
"newSearchDep2.js", /* no rule */
"newSearchDep1.js”, /* no rule */
"newSearch.js" /* rules.inNewSearch */
]
Step 4:
Filter out all extraneous
sub deps.
✓](https://image.slidesharecdn.com/nodeconfeu2014scalingabtestingonnetflix-140909054728-phpapp02/85/NodeConf-eu-2014-Scaling-AB-Testing-on-Netflix-com-with-Node-js-39-320.jpg)
![Step 5:
Concatenate the files.
[
"jquery", /* no rule */
"newSearchDep2.js", /* no rule */
"newSearchDep1.js”, /* no rule */
"newSearch.js" /* rules.inNewSearch */
]](https://image.slidesharecdn.com/nodeconfeu2014scalingabtestingonnetflix-140909054728-phpapp02/85/NodeConf-eu-2014-Scaling-AB-Testing-on-Netflix-com-with-Node-js-40-320.jpg)
!["account/bb/models/ratingHistoryModel.js": {
"rule": null,
"deps": [...],
"depsFull": [...],
"depsCount": {
"underscore": 2,
"backbone": 1,
"jquery": 2,
"common/requirejs-plugins.js": 4,
"requirejs-text": 4,
"utils/contextData.js": 1,
"common/nfNamespace.js": 1
},
"hash": "dd23b163",
"fileSize": "1.21 kB",
"fileSizeFull": "173.04 kB"
}
dependency
counting
dependency
pruning
file sizes](https://image.slidesharecdn.com/nodeconfeu2014scalingabtestingonnetflix-140909054728-phpapp02/85/NodeConf-eu-2014-Scaling-AB-Testing-on-Netflix-com-with-Node-js-45-320.jpg)
!["account/containerResponsive.css": {
"rule": null,
"deps": [...],
"depsFull": [...],
"depsCount": [...],
"hash": "65a431f3",
"fileSize": "709 B",
"fileSizeFull": "709 B",
"css": {
"selectors": 8,
"declarationBlocks": 6,
"declarations": 17,
"mediaQueries": 3
}
}
css
analysis](https://image.slidesharecdn.com/nodeconfeu2014scalingabtestingonnetflix-140909054728-phpapp02/85/NodeConf-eu-2014-Scaling-AB-Testing-on-Netflix-com-with-Node-js-47-320.jpg)
![[NodeConf.eu 2014] Scaling AB Testing on Netflix.com with Node.js](https://image.slidesharecdn.com/nodeconfeu2014scalingabtestingonnetflix-140909054728-phpapp02/85/NodeConf-eu-2014-Scaling-AB-Testing-on-Netflix-com-with-Node-js-52-320.jpg)
[NodeConf.eu 2014] Scaling AB Testing on Netflix.com with Node.js
- 1. Scaling A/B testing on Netflix.com with _________ Alex Liu @stinkydofu
- 2. data driven product development
- 6. Test 1 Test 2 Test 3 Test 4 Test 5 Test 6 Test 7 A B C D E F G A B C D E F G A B C D E F G A B C D E F G A B C D E F G A B C D E F G A B C D E F G
- 7. 2,097,152 unique experiences across seven tests
- 8. hundreds of new A/B tests per year
- 9. 43351892955034 94860861172181 85493567650… 72061153709996
- 10. 2105 566 685 templates CSS JS
- 11. 2.5M unique packages every week
- 13. Packaging
- 15. oldSearch app.js newSearch dep1 dep2 dep3 dep4 dep5 sub-dep sub-dep sub-dep sub-dep sub-dep sub-dep
- 16. oldSearch app.js newSearch dep1 dep2 dep3 dep4 dep5 sub-dep sub-dep sub-dep sub-dep sub-dep sub-dep
- 17. app.js import jquery from 'jquery'; import oldSearch from 'oldSearch'; import newSearch from 'newSearch'; export ...
- 18. oldSearch app.js newSearch dep1 dep2 dep3 dep4 dep5 sub-dep sub-dep sub-dep sub-dep sub-dep sub-dep
- 20. 685 files…? 2.5M packages…?
- 21. oldSearch app.js newSearch dep1 dep2 dep3 dep4 dep5 sub-dep sub-dep sub-dep sub-dep sub-dep sub-dep
- 24. build request
- 26. oldSearch.js /* * @includewhen rule.notInNewSearch */
- 27. newSearch.js /* * @includewhen rule.inNewSearch */
- 28. anatomy of a rule var Rule = require('nf-rule-infrastructure'), inNewSearch; inNewSearch = new Rule('inNewSearch', function(context, cb) { var test = context.abtests.get(1534); cb(test && test.cell(1)); }); module.exports = inNewSearch;
- 30. app.js import jquery from 'jquery'; import oldSearch from 'oldSearch'; import newSearch from 'newSearch'; export ...
- 31. app.js jquery oldSearch.js newSearch.js registry
- 32. "app.js": { "deps": [ "jquery", "oldSearch.js", "newSearch.js", ], "depsFull": [ "jquery", "oldSearchDep2.js", "oldSearchDep1.js", "oldSearch.js", "newSearchDep2.js", "newSearchDep1.js", "newSearch.js" ], "fileSize": "4.41 kB", "fileSizeFull": "120.52 kB" }
- 33. "newSearch.js": { "rule": "inNewSearch", "deps": [ "jquery", "newSearchDep2.js", "newSearchDep1.js", ], "depsFull": [ "jquery", "newSearchSubDep3.js", "newSearchSubDep2.js" "newSearchSubDep1.js" "newSearchDep2.js", "newSearchDep1.js" ], "fileSize": "10.41 kB", "fileSizeFull": "40.52 kB" } nf-include-when
- 35. var packager = require('nf-packager'), includeWhen = require('nf-include-when'), registries = require('nf-asset-registry'); function getScriptUrl() return packager.getPackageDefinition('app.js', registries, includeWhen); }
- 36. "app.js": { "deps": [ "jquery", "oldSearch.js", "newSearch.js", ], "depsFull": [ "jquery", "oldSearchDep2.js", "oldSearchDep1.js", "oldSearch.js", "newSearchDep2.js", "newSearchDep1.js", "newSearch.js" ], "fileSize": "4.41 kB", "fileSizeFull": "120.52 kB" } Step 1: Get the full dependency tree for the requested package from the registry.
- 37. [ "jquery", /* no rule */ "oldSearchDep2.js", /* no rule */ "oldSearchDep1.js", /* no rule */ "oldSearch.js", /* rules.notInNewSearch */ "newSearchDep2.js", /* no rule */ "newSearchDep1.js”, /* no rule */ "newSearch.js" /* rules.inNewSearch */ ] Step 2: Determine which files have rules.
- 38. [ "jquery", /* no rule */ "oldSearchDep2.js", /* no rule */ "oldSearchDep1.js", /* no rule */ "oldSearch.js", /* rules.notInNewSearch */ "newSearchDep2.js", /* no rule */ "newSearchDep1.js”, /* no rule */ "newSearch.js" /* rules.inNewSearch */ ] Step 3: Run the rules. Filter out all deps that resolved false. ✓
- 39. [ "jquery", /* no rule */ "oldSearchDep2.js", /* no rule */ "oldSearchDep1.js", /* no rule */ "oldSearch.js", /* rules.notInNewSearch */ "newSearchDep2.js", /* no rule */ "newSearchDep1.js”, /* no rule */ "newSearch.js" /* rules.inNewSearch */ ] Step 4: Filter out all extraneous sub deps. ✓
- 40. Step 5: Concatenate the files. [ "jquery", /* no rule */ "newSearchDep2.js", /* no rule */ "newSearchDep1.js”, /* no rule */ "newSearch.js" /* rules.inNewSearch */ ]
- 42. request registry packager rules
- 43. Bonus Round
- 44. be creative with the registry
- 45. "account/bb/models/ratingHistoryModel.js": { "rule": null, "deps": [...], "depsFull": [...], "depsCount": { "underscore": 2, "backbone": 1, "jquery": 2, "common/requirejs-plugins.js": 4, "requirejs-text": 4, "utils/contextData.js": 1, "common/nfNamespace.js": 1 }, "hash": "dd23b163", "fileSize": "1.21 kB", "fileSizeFull": "173.04 kB" } dependency counting dependency pruning file sizes
- 46. @import modules @import (reference) "/common/_nf_defs.less"; @import (reference) "/member/memberCore.less"; @import (reference) "/components/menu.less"; @import (reference) "/components/breadcrumbs.less";
- 47. "account/containerResponsive.css": { "rule": null, "deps": [...], "depsFull": [...], "depsCount": [...], "hash": "65a431f3", "fileSize": "709 B", "fileSizeFull": "709 B", "css": { "selectors": 8, "declarationBlocks": 6, "declarations": 17, "mediaQueries": 3 } } css analysis
- 48. the best part
- 49. "cache": { "account/pin.js": "define('account/pin.js', ['member/memberC…", "account/bb/models/changePlanModel.js": "define('account/b…", "account/bb/models/ratingHistoryModel.js": "define('account…", "account/bb/models/viewingActivityModel.js": "define('account…", "account/bb/views/changePlanView.js": "define('account/bb/vi…", "account/bb/views/changePlanView.js": "define('account/bb/vi…", "account/bb/views/emailSubView.js": "define('account/bb/views…", "account/bb/views/viewingActivityView.js": "define('account…", "common/UITracking.js": "define('common/UITracking.js, ['me…", "common/UITrackingOverlay.js": "define('common/UITrackingOve…", … … …
- 50. templates templates mappings javascript css mappings javascript css
- 51. templates mappings javascript css UI Bundle
- 53. deploy UI bundles anytime
- 54. never touch the file system
- 55. < 5ms package response times
- 56. Takeaways ▶ leverage the server ▶ static analysis FTW ▶ divide and conquer with modules
- 57. Our Learnings
- 58. learn by doing
- 59. fail fast move faster
- 60. “I have not failed. I’ve just found 10,000 ways that won’t work.” Thomas Edison
- 61. simplify
- 62. thanks! come find us! Alex Liu @stinkydofu Chris Saint-Amant @csaintamant Micah Ransdell @mjr578 Kris Baxter @kristoferbaxter