Finding a lost song with Node.js and async iterators - EnterJS 2021
0 likes93 views
The document discusses using the Last.fm API and Node.js async iterators to search a user's listening history when only a fragment of a song title is remembered. It describes making requests to retrieve the user's recent tracks paginated over time windows to avoid duplicate results. Code examples show iterating over the pages as a stream to expose the track data for processing.
![let page = 1
while (true) {
const query = querystring.stringify({
method: 'user.getrecenttracks',
user: 'loige',
api_key: process.env.API_KEY,
format: 'json',
page
})
const url = `https://ws.audioscrobbler.com/2.0/?${query}`
const response = await axios.get(url)
console.log(response.data)
if (page === Number(response.data.recenttracks['@attr'].totalPages)) {
break // it's the last page!
}
loige 25](https://image.slidesharecdn.com/finding-a-lost-song-with-node-js-and-async-iterators-enterjs-210930184331/85/Finding-a-lost-song-with-Node-js-and-async-iterators-EnterJS-2021-36-320.jpg)
![let page = 1
while (true) {
const query = querystring.stringify({
method: 'user.getrecenttracks',
user: 'loige',
api_key: process.env.API_KEY,
format: 'json',
page
})
const url = `https://ws.audioscrobbler.com/2.0/?${query}`
const response = await axios.get(url)
console.log(response.data)
if (page === Number(response.data.recenttracks['@attr'].totalPages)) {
break // it's the last page!
}
loige 25](https://image.slidesharecdn.com/finding-a-lost-song-with-node-js-and-async-iterators-enterjs-210930184331/85/Finding-a-lost-song-with-Node-js-and-async-iterators-EnterJS-2021-37-320.jpg)
![let page = 1
while (true) {
const query = querystring.stringify({
method: 'user.getrecenttracks',
user: 'loige',
api_key: process.env.API_KEY,
format: 'json',
page
})
const url = `https://ws.audioscrobbler.com/2.0/?${query}`
const response = await axios.get(url)
console.log(response.data)
if (page === Number(response.data.recenttracks['@attr'].totalPages)) {
break // it's the last page!
}
loige 25](https://image.slidesharecdn.com/finding-a-lost-song-with-node-js-and-async-iterators-enterjs-210930184331/85/Finding-a-lost-song-with-Node-js-and-async-iterators-EnterJS-2021-38-320.jpg)
![let page = 1
while (true) {
const query = querystring.stringify({
method: 'user.getrecenttracks',
user: 'loige',
api_key: process.env.API_KEY,
format: 'json',
page
})
const url = `https://ws.audioscrobbler.com/2.0/?${query}`
const response = await axios.get(url)
console.log(response.data)
if (page === Number(response.data.recenttracks['@attr'].totalPages)) {
break // it's the last page!
}
loige 25](https://image.slidesharecdn.com/finding-a-lost-song-with-node-js-and-async-iterators-enterjs-210930184331/85/Finding-a-lost-song-with-Node-js-and-async-iterators-EnterJS-2021-39-320.jpg)
![let page = 1
while (true) {
const query = querystring.stringify({
method: 'user.getrecenttracks',
user: 'loige',
api_key: process.env.API_KEY,
format: 'json',
page
})
const url = `https://ws.audioscrobbler.com/2.0/?${query}`
const response = await axios.get(url)
console.log(response.data)
if (page === Number(response.data.recenttracks['@attr'].totalPages)) {
break // it's the last page!
}
loige 25](https://image.slidesharecdn.com/finding-a-lost-song-with-node-js-and-async-iterators-enterjs-210930184331/85/Finding-a-lost-song-with-Node-js-and-async-iterators-EnterJS-2021-40-320.jpg)
![let page = 1
while (true) {
const query = querystring.stringify({
method: 'user.getrecenttracks',
user: 'loige',
api_key: process.env.API_KEY,
format: 'json',
page
})
const url = `https://ws.audioscrobbler.com/2.0/?${query}`
const response = await axios.get(url)
console.log(response.data)
if (page === Number(response.data.recenttracks['@attr'].totalPages)) {
break // it's the last page!
}
loige 25](https://image.slidesharecdn.com/finding-a-lost-song-with-node-js-and-async-iterators-enterjs-210930184331/85/Finding-a-lost-song-with-Node-js-and-async-iterators-EnterJS-2021-41-320.jpg)
![// ...
for (const track of response.data.recenttracks.track) {
console.log(
track.date?.['#text'],
`${track.artist['#text']} - ${track.name}`
)
}
console.log('--- end page ---')
// ...
loige 28](https://image.slidesharecdn.com/finding-a-lost-song-with-node-js-and-async-iterators-enterjs-210930184331/85/Finding-a-lost-song-with-Node-js-and-async-iterators-EnterJS-2021-45-320.jpg)
![let to
while (true) {
const query = querystring.stringify({
method: 'user.getrecenttracks',
user: 'loige',
api_key: process.env.API_KEY,
format: 'json',
limit: '10',
to
})
const url = `https://ws.audioscrobbler.com/2.0/?${query}`
const response = await axios.get(url)
const tracks = response.data.recenttracks.track
console.log(
`--- ↓ page to ${to}`,
`remaining pages: ${response.data.recenttracks['@attr'].totalPages} ---`
)
for (const track of tracks) {
console.log(track.date?.uts, `${track.artist['#text']} - ${track.name}`)
}
if (response.data.recenttracks['@attr'].totalPages <= 1) {
break // it's the last page!
}
const lastTrackInPage = tracks[tracks.length - 1]
to = lastTrackInPage.date.uts
} loige 35](https://image.slidesharecdn.com/finding-a-lost-song-with-node-js-and-async-iterators-enterjs-210930184331/85/Finding-a-lost-song-with-Node-js-and-async-iterators-EnterJS-2021-65-320.jpg)
![let to
while (true) {
const query = querystring.stringify({
method: 'user.getrecenttracks',
user: 'loige',
api_key: process.env.API_KEY,
format: 'json',
limit: '10',
to
})
const url = `https://ws.audioscrobbler.com/2.0/?${query}`
const response = await axios.get(url)
const tracks = response.data.recenttracks.track
console.log(
`--- ↓ page to ${to}`,
`remaining pages: ${response.data.recenttracks['@attr'].totalPages} ---`
)
for (const track of tracks) {
console.log(track.date?.uts, `${track.artist['#text']} - ${track.name}`)
}
if (response.data.recenttracks['@attr'].totalPages <= 1) {
break // it's the last page!
}
const lastTrackInPage = tracks[tracks.length - 1]
to = lastTrackInPage.date.uts
} loige 35](https://image.slidesharecdn.com/finding-a-lost-song-with-node-js-and-async-iterators-enterjs-210930184331/85/Finding-a-lost-song-with-Node-js-and-async-iterators-EnterJS-2021-66-320.jpg)
![let to
while (true) {
const query = querystring.stringify({
method: 'user.getrecenttracks',
user: 'loige',
api_key: process.env.API_KEY,
format: 'json',
limit: '10',
to
})
const url = `https://ws.audioscrobbler.com/2.0/?${query}`
const response = await axios.get(url)
const tracks = response.data.recenttracks.track
console.log(
`--- ↓ page to ${to}`,
`remaining pages: ${response.data.recenttracks['@attr'].totalPages} ---`
)
for (const track of tracks) {
console.log(track.date?.uts, `${track.artist['#text']} - ${track.name}`)
}
if (response.data.recenttracks['@attr'].totalPages <= 1) {
break // it's the last page!
}
const lastTrackInPage = tracks[tracks.length - 1]
to = lastTrackInPage.date.uts
} loige 35](https://image.slidesharecdn.com/finding-a-lost-song-with-node-js-and-async-iterators-enterjs-210930184331/85/Finding-a-lost-song-with-Node-js-and-async-iterators-EnterJS-2021-67-320.jpg)
![let to
while (true) {
const query = querystring.stringify({
method: 'user.getrecenttracks',
user: 'loige',
api_key: process.env.API_KEY,
format: 'json',
limit: '10',
to
})
const url = `https://ws.audioscrobbler.com/2.0/?${query}`
const response = await axios.get(url)
const tracks = response.data.recenttracks.track
console.log(
`--- ↓ page to ${to}`,
`remaining pages: ${response.data.recenttracks['@attr'].totalPages} ---`
)
for (const track of tracks) {
console.log(track.date?.uts, `${track.artist['#text']} - ${track.name}`)
}
if (response.data.recenttracks['@attr'].totalPages <= 1) {
break // it's the last page!
}
const lastTrackInPage = tracks[tracks.length - 1]
to = lastTrackInPage.date.uts
} loige 35](https://image.slidesharecdn.com/finding-a-lost-song-with-node-js-and-async-iterators-enterjs-210930184331/85/Finding-a-lost-song-with-Node-js-and-async-iterators-EnterJS-2021-68-320.jpg)
![let to
while (true) {
const query = querystring.stringify({
method: 'user.getrecenttracks',
user: 'loige',
api_key: process.env.API_KEY,
format: 'json',
limit: '10',
to
})
const url = `https://ws.audioscrobbler.com/2.0/?${query}`
const response = await axios.get(url)
const tracks = response.data.recenttracks.track
console.log(
`--- ↓ page to ${to}`,
`remaining pages: ${response.data.recenttracks['@attr'].totalPages} ---`
)
for (const track of tracks) {
console.log(track.date?.uts, `${track.artist['#text']} - ${track.name}`)
}
if (response.data.recenttracks['@attr'].totalPages <= 1) {
break // it's the last page!
}
const lastTrackInPage = tracks[tracks.length - 1]
to = lastTrackInPage.date.uts
} loige 35](https://image.slidesharecdn.com/finding-a-lost-song-with-node-js-and-async-iterators-enterjs-210930184331/85/Finding-a-lost-song-with-Node-js-and-async-iterators-EnterJS-2021-69-320.jpg)
![function createCountdown (from) {
let nextVal = from
return {
[Symbol.iterator]: () => ({
next () {
if (nextVal < 0) {
return { done: true }
}
return { done: false, value: nextVal-- }
}
})
}
}
loige 54](https://image.slidesharecdn.com/finding-a-lost-song-with-node-js-and-async-iterators-enterjs-210930184331/85/Finding-a-lost-song-with-Node-js-and-async-iterators-EnterJS-2021-95-320.jpg)
![function createCountdown (from) {
let nextVal = from
return {
[Symbol.iterator]: () => ({
next () {
if (nextVal < 0) {
return { done: true }
}
return { done: false, value: nextVal-- }
}
})
}
}
loige 54](https://image.slidesharecdn.com/finding-a-lost-song-with-node-js-and-async-iterators-enterjs-210930184331/85/Finding-a-lost-song-with-Node-js-and-async-iterators-EnterJS-2021-96-320.jpg)
![function createCountdown (from) {
let nextVal = from
return {
[Symbol.iterator]: () => ({
next () {
if (nextVal < 0) {
return { done: true }
}
return { done: false, value: nextVal-- }
}
})
}
}
loige 54](https://image.slidesharecdn.com/finding-a-lost-song-with-node-js-and-async-iterators-enterjs-210930184331/85/Finding-a-lost-song-with-Node-js-and-async-iterators-EnterJS-2021-97-320.jpg)
![function createCountdown (from) {
return {
[Symbol.iterator]: function * () {
for (let i = from; i >= 0; i--) {
yield i
}
}
}
}
loige 55](https://image.slidesharecdn.com/finding-a-lost-song-with-node-js-and-async-iterators-enterjs-210930184331/85/Finding-a-lost-song-with-Node-js-and-async-iterators-EnterJS-2021-98-320.jpg)
![import { setTimeout } from 'timers/promises'
function createAsyncCountdown (from, delay = 1000) {
return {
[Symbol.asyncIterator]: async function * () {
for (let i = from; i >= 0; i--) {
await setTimeout(delay)
yield i
}
}
}
}
loige 64](https://image.slidesharecdn.com/finding-a-lost-song-with-node-js-and-async-iterators-enterjs-210930184331/85/Finding-a-lost-song-with-Node-js-and-async-iterators-EnterJS-2021-112-320.jpg)
![import { setTimeout } from 'timers/promises'
function createAsyncCountdown (from, delay = 1000) {
return {
[Symbol.asyncIterator]: async function * () {
for (let i = from; i >= 0; i--) {
await setTimeout(delay)
yield i
}
}
}
}
loige 64](https://image.slidesharecdn.com/finding-a-lost-song-with-node-js-and-async-iterators-enterjs-210930184331/85/Finding-a-lost-song-with-Node-js-and-async-iterators-EnterJS-2021-113-320.jpg)
![import querystring from 'querystring'
import axios from 'axios'
async function * createLastFmRecentTracks (apiKey, user) {
let to
while (true) {
const query = querystring.stringify({
method: 'user.getrecenttracks',
user,
api_key: apiKey,
format: 'json',
to
})
const url = `https://ws.audioscrobbler.com/2.0/?${query}`
const response = await axios.get(url)
const tracks = response.data.recenttracks.track
yield tracks
if (response.data.recenttracks['@attr'].totalPages <= 1) {
break // it's the last page!
}
const lastTrackInPage = tracks[tracks.length - 1]
to = lastTrackInPage.date.uts
}
} loige 69](https://image.slidesharecdn.com/finding-a-lost-song-with-node-js-and-async-iterators-enterjs-210930184331/85/Finding-a-lost-song-with-Node-js-and-async-iterators-EnterJS-2021-119-320.jpg)
![import querystring from 'querystring'
import axios from 'axios'
async function * createLastFmRecentTracks (apiKey, user) {
let to
while (true) {
const query = querystring.stringify({
method: 'user.getrecenttracks',
user,
api_key: apiKey,
format: 'json',
to
})
const url = `https://ws.audioscrobbler.com/2.0/?${query}`
const response = await axios.get(url)
const tracks = response.data.recenttracks.track
yield tracks
if (response.data.recenttracks['@attr'].totalPages <= 1) {
break // it's the last page!
}
const lastTrackInPage = tracks[tracks.length - 1]
to = lastTrackInPage.date.uts
}
} loige 69](https://image.slidesharecdn.com/finding-a-lost-song-with-node-js-and-async-iterators-enterjs-210930184331/85/Finding-a-lost-song-with-Node-js-and-async-iterators-EnterJS-2021-120-320.jpg)
![import querystring from 'querystring'
import axios from 'axios'
async function * createLastFmRecentTracks (apiKey, user) {
let to
while (true) {
const query = querystring.stringify({
method: 'user.getrecenttracks',
user,
api_key: apiKey,
format: 'json',
to
})
const url = `https://ws.audioscrobbler.com/2.0/?${query}`
const response = await axios.get(url)
const tracks = response.data.recenttracks.track
yield tracks
if (response.data.recenttracks['@attr'].totalPages <= 1) {
break // it's the last page!
}
const lastTrackInPage = tracks[tracks.length - 1]
to = lastTrackInPage.date.uts
}
} loige 69](https://image.slidesharecdn.com/finding-a-lost-song-with-node-js-and-async-iterators-enterjs-210930184331/85/Finding-a-lost-song-with-Node-js-and-async-iterators-EnterJS-2021-121-320.jpg)
![async function main () {
const recentTracks = createLastFmRecentTracks(
process.env.API_KEY,
'loige'
)
for await (const page of recentTracks) {
for (const track of page) {
if (track.name.toLowerCase().includes('dark')) {
console.log(`${track.artist['#text']} - ${track.name}`)
}
}
}
}
loige 72](https://image.slidesharecdn.com/finding-a-lost-song-with-node-js-and-async-iterators-enterjs-210930184331/85/Finding-a-lost-song-with-Node-js-and-async-iterators-EnterJS-2021-124-320.jpg)
Finding a lost song with Node.js and async iterators - EnterJS 2021
- 1. Finding a lost song with Node.js & async iterators Luciano Mammino ( ) @loige 2021-09-30 loige.link/enter-iterators 1
- 3. Photo by on Darius Bashar Unsplash A random song you haven't listened to in years pops into your head... The Ohrwurm! 👂🐛 3
- 4. It doesn't matter what you do all day... It keeps coming back to you! 🐛 Photo by on Attentie Attentie Unsplash 4
- 5. And now you want to listen to it! Photo by on Volodymyr Hryshchenko Unsplash 5
- 6. But, what if you can't remember the title or the author?! Photo by on Tachina Lee Unsplash 6
- 7. THERE MUST BE A WAY TO REMEMBER! Photo by on Marius Niveri Unsplash 7
- 8. Today, I'll tell you how I solved this problem using - Last.fm API - Node.js - Async Iterators Photo by on Quinton Coetzee Unsplash 8
- 9. Let me introduce myself first... 9
- 10. Let me introduce myself first... I'm Luciano ( 🍕🍝) 👋 9
- 11. Let me introduce myself first... I'm Luciano ( 🍕🍝) 👋 Senior Architect @ fourTheorem (Dublin ) 9
- 12. Let me introduce myself first... I'm Luciano ( 🍕🍝) 👋 Senior Architect @ fourTheorem (Dublin ) nodejsdp.link Co-Author of Node.js Design Patterns 👉 9
- 13. Let me introduce myself first... I'm Luciano ( 🍕🍝) 👋 Senior Architect @ fourTheorem (Dublin ) nodejsdp.link Co-Author of Node.js Design Patterns 👉 Connect with me: (blog) (twitter) (twitch) (github) loige.co @loige loige lmammino 9
- 14. We are business focused technologists that deliver. | | Accelerated Serverless AI as a Service Platform Modernisation We are hiring: do you want to ? work with us loige 10
- 15. So, there was this ohrwurm... 🐛 loige 11
- 16. I could only remember some random parts and the word "dark" (probably in the title) loige 12
- 17. 13
- 18. 14
- 19. loige 15
- 20. Luciano - scrobbling since 12 Feb 2007 loige 15
- 21. Luciano - scrobbling since 12 Feb 2007 loige ~250k scrobbles... that song must be there! 15
- 22. loige 16
- 23. loige ~5k pages of history & no search functionality! 😓 16
- 24. loige But there's an API! https://www.last.fm/api 17
- 25. loige 18
- 26. loige Let's give it a shot curl "http://ws.audioscrobbler.com/2.0/? method=user.getrecenttracks&user=loige&api_key =${API_KEY}&format=json" | jq . 19
- 27. loige 20
- 28. It works! 🥳 Now let's do this with JavaScript loige 21
- 29. import querystring from 'querystring' import axios from 'axios' const query = querystring.stringify({ method: 'user.getrecenttracks', user: 'loige', api_key: process.env.API_KEY, format: 'json' }) const url = `https://ws.audioscrobbler.com/2.0/?${query}` const response = await axios.get(url) console.log(response.data) loige 22
- 30. loige 23
- 31. loige We are getting a "paginated" response with 50 tracks per page 23
- 32. loige We are getting a "paginated" response with 50 tracks per page but there are 51 here! 🤔 23
- 33. loige We are getting a "paginated" response with 50 tracks per page but there are 51 here! 🤔 (let's ignore this for now...) 23
- 34. loige We are getting a "paginated" response with 50 tracks per page but there are 51 here! 🤔 How do we fetch the next pages? (let's ignore this for now...) 23
- 35. loige 24
- 36. let page = 1 while (true) { const query = querystring.stringify({ method: 'user.getrecenttracks', user: 'loige', api_key: process.env.API_KEY, format: 'json', page }) const url = `https://ws.audioscrobbler.com/2.0/?${query}` const response = await axios.get(url) console.log(response.data) if (page === Number(response.data.recenttracks['@attr'].totalPages)) { break // it's the last page! } loige 25
- 37. let page = 1 while (true) { const query = querystring.stringify({ method: 'user.getrecenttracks', user: 'loige', api_key: process.env.API_KEY, format: 'json', page }) const url = `https://ws.audioscrobbler.com/2.0/?${query}` const response = await axios.get(url) console.log(response.data) if (page === Number(response.data.recenttracks['@attr'].totalPages)) { break // it's the last page! } loige 25
- 38. let page = 1 while (true) { const query = querystring.stringify({ method: 'user.getrecenttracks', user: 'loige', api_key: process.env.API_KEY, format: 'json', page }) const url = `https://ws.audioscrobbler.com/2.0/?${query}` const response = await axios.get(url) console.log(response.data) if (page === Number(response.data.recenttracks['@attr'].totalPages)) { break // it's the last page! } loige 25
- 39. let page = 1 while (true) { const query = querystring.stringify({ method: 'user.getrecenttracks', user: 'loige', api_key: process.env.API_KEY, format: 'json', page }) const url = `https://ws.audioscrobbler.com/2.0/?${query}` const response = await axios.get(url) console.log(response.data) if (page === Number(response.data.recenttracks['@attr'].totalPages)) { break // it's the last page! } loige 25
- 40. let page = 1 while (true) { const query = querystring.stringify({ method: 'user.getrecenttracks', user: 'loige', api_key: process.env.API_KEY, format: 'json', page }) const url = `https://ws.audioscrobbler.com/2.0/?${query}` const response = await axios.get(url) console.log(response.data) if (page === Number(response.data.recenttracks['@attr'].totalPages)) { break // it's the last page! } loige 25
- 41. let page = 1 while (true) { const query = querystring.stringify({ method: 'user.getrecenttracks', user: 'loige', api_key: process.env.API_KEY, format: 'json', page }) const url = `https://ws.audioscrobbler.com/2.0/?${query}` const response = await axios.get(url) console.log(response.data) if (page === Number(response.data.recenttracks['@attr'].totalPages)) { break // it's the last page! } loige 25
- 42. loige 26
- 43. loige 26
- 44. Seems good! Let's look at the tracks... loige 27
- 45. // ... for (const track of response.data.recenttracks.track) { console.log( track.date?.['#text'], `${track.artist['#text']} - ${track.name}` ) } console.log('--- end page ---') // ... loige 28
- 46. loige 29
- 47. loige * Note that page size here is 10 tracks per page 29
- 48. loige * Note that page size here is 10 tracks per page Every page has a song with undefined time... This is the song I am currently listening to! It appears at the top of every page. 29
- 49. loige * Note that page size here is 10 tracks per page Sometimes there are duplicated tracks between pages... 😨 29
- 50. The "sliding windows" problem 😩 loige 30
- 51. loige ... tracks (newest to oldest) 31
- 52. loige ... tracks (newest to oldest) 31 Page1 Page2
- 53. loige ... tracks (newest to oldest) 31 Page1 Page2 ...
- 54. loige ... tracks (newest to oldest) 31 Page1 Page2 ... new track
- 55. loige ... tracks (newest to oldest) 31 Page1 Page2 ... Page1 Page2 new track
- 56. loige ... tracks (newest to oldest) 31 Page1 Page2 ... Page1 Page2 new track moved from page 1 to page 2
- 57. loige 32
- 58. Time based windows 😎 loige 33
- 59. loige ...* tracks (newest to oldest) 34 * we are done when we get an empty page (or num pages is 1) to ... from
- 60. loige ...* tracks (newest to oldest) 34 Page1 * we are done when we get an empty page (or num pages is 1) to ... from
- 61. loige ...* tracks (newest to oldest) 34 Page1 t1 * we are done when we get an empty page (or num pages is 1) to ... from
- 62. loige ...* tracks (newest to oldest) 34 Page1 before t1 (page 1 "to" t1) t1 * we are done when we get an empty page (or num pages is 1) to ... from
- 63. loige ...* tracks (newest to oldest) 34 Page1 before t1 (page 1 "to" t1) t1 t2 * we are done when we get an empty page (or num pages is 1) to ... from
- 64. loige ...* tracks (newest to oldest) 34 Page1 before t1 (page 1 "to" t1) t1 t2 before t2 (page 1 "to" t2) * we are done when we get an empty page (or num pages is 1) to ... from
- 65. let to while (true) { const query = querystring.stringify({ method: 'user.getrecenttracks', user: 'loige', api_key: process.env.API_KEY, format: 'json', limit: '10', to }) const url = `https://ws.audioscrobbler.com/2.0/?${query}` const response = await axios.get(url) const tracks = response.data.recenttracks.track console.log( `--- ↓ page to ${to}`, `remaining pages: ${response.data.recenttracks['@attr'].totalPages} ---` ) for (const track of tracks) { console.log(track.date?.uts, `${track.artist['#text']} - ${track.name}`) } if (response.data.recenttracks['@attr'].totalPages <= 1) { break // it's the last page! } const lastTrackInPage = tracks[tracks.length - 1] to = lastTrackInPage.date.uts } loige 35
- 66. let to while (true) { const query = querystring.stringify({ method: 'user.getrecenttracks', user: 'loige', api_key: process.env.API_KEY, format: 'json', limit: '10', to }) const url = `https://ws.audioscrobbler.com/2.0/?${query}` const response = await axios.get(url) const tracks = response.data.recenttracks.track console.log( `--- ↓ page to ${to}`, `remaining pages: ${response.data.recenttracks['@attr'].totalPages} ---` ) for (const track of tracks) { console.log(track.date?.uts, `${track.artist['#text']} - ${track.name}`) } if (response.data.recenttracks['@attr'].totalPages <= 1) { break // it's the last page! } const lastTrackInPage = tracks[tracks.length - 1] to = lastTrackInPage.date.uts } loige 35
- 67. let to while (true) { const query = querystring.stringify({ method: 'user.getrecenttracks', user: 'loige', api_key: process.env.API_KEY, format: 'json', limit: '10', to }) const url = `https://ws.audioscrobbler.com/2.0/?${query}` const response = await axios.get(url) const tracks = response.data.recenttracks.track console.log( `--- ↓ page to ${to}`, `remaining pages: ${response.data.recenttracks['@attr'].totalPages} ---` ) for (const track of tracks) { console.log(track.date?.uts, `${track.artist['#text']} - ${track.name}`) } if (response.data.recenttracks['@attr'].totalPages <= 1) { break // it's the last page! } const lastTrackInPage = tracks[tracks.length - 1] to = lastTrackInPage.date.uts } loige 35
- 68. let to while (true) { const query = querystring.stringify({ method: 'user.getrecenttracks', user: 'loige', api_key: process.env.API_KEY, format: 'json', limit: '10', to }) const url = `https://ws.audioscrobbler.com/2.0/?${query}` const response = await axios.get(url) const tracks = response.data.recenttracks.track console.log( `--- ↓ page to ${to}`, `remaining pages: ${response.data.recenttracks['@attr'].totalPages} ---` ) for (const track of tracks) { console.log(track.date?.uts, `${track.artist['#text']} - ${track.name}`) } if (response.data.recenttracks['@attr'].totalPages <= 1) { break // it's the last page! } const lastTrackInPage = tracks[tracks.length - 1] to = lastTrackInPage.date.uts } loige 35
- 69. let to while (true) { const query = querystring.stringify({ method: 'user.getrecenttracks', user: 'loige', api_key: process.env.API_KEY, format: 'json', limit: '10', to }) const url = `https://ws.audioscrobbler.com/2.0/?${query}` const response = await axios.get(url) const tracks = response.data.recenttracks.track console.log( `--- ↓ page to ${to}`, `remaining pages: ${response.data.recenttracks['@attr'].totalPages} ---` ) for (const track of tracks) { console.log(track.date?.uts, `${track.artist['#text']} - ${track.name}`) } if (response.data.recenttracks['@attr'].totalPages <= 1) { break // it's the last page! } const lastTrackInPage = tracks[tracks.length - 1] to = lastTrackInPage.date.uts } loige 35
- 70. loige 36
- 71. loige The track of the last timestamp becomes the boundary for the next page 36
- 72. We have a working solution! 🎉 Can we generalise it? loige 37
- 73. We know how to iterate over every page/track. How do we expose this information? loige 38
- 74. const reader = LastFmRecentTracks({ apikey: process.env.API_KEY, user: 'loige' }) // callbacks reader.readPages( (page) => { /* ... */ }, // on page (err) => { /* ... */} // on completion (or error) ) loige 39
- 75. const reader = LastFmRecentTracks({ apikey: process.env.API_KEY, user: 'loige' }) // event emitter reader.read() reader.on('page', (page) => { /* ... */ }) reader.on('completed', (err) => { /* ... */ }) loige 40
- 76. const reader = LastFmRecentTracks({ apikey: process.env.API_KEY, user: 'loige' }) // streams <3 reader.pipe(/* transform or writable stream here */) reader.on('end', () => { /* ... */ }) reader.on('error', () => { /* ... */ }) loige 41
- 77. import { pipeline } from 'stream' const reader = LastFmRecentTracks({ apikey: process.env.API_KEY, user: 'loige' }) // streams pipeline <3 <3 pipeline( reader, yourProcessingStream, (err) => { // handle completion or err } ) loige 42
- 78. const reader = LastFmRecentTracks({ apikey: process.env.API_KEY, user: 'loige' }) // ASYNC ITERATORS! for await (const page of reader) { /* ... */ } // ... do more stuff when all the data is consumed loige 43
- 79. const reader = LastFmRecentTracks({ apikey: process.env.API_KEY, user: 'loige' }) // ASYNC ITERATORS WITH ERROR HANDLING! try { for await (const page of reader) { /* ... */ } } catch (err) { // handle errors } // ... do more stuff when all the data is consumed loige 44
- 80. How can we build an async iterator? 🧐 loige 45
- 81. Meet the iteration protocols! loige loige.co/javascript-iterator-patterns 46
- 82. The iterator protocol An object is an iterator if it has a next() method. Every time you call it, it returns an object with the keys done (boolean) and value. loige 47
- 83. function createCountdown (from) { let nextVal = from return { next () { if (nextVal < 0) { return { done: true } } return { done: false, value: nextVal-- } } } } loige 48
- 84. function createCountdown (from) { let nextVal = from return { next () { if (nextVal < 0) { return { done: true } } return { done: false, value: nextVal-- } } } } loige 48
- 85. function createCountdown (from) { let nextVal = from return { next () { if (nextVal < 0) { return { done: true } } return { done: false, value: nextVal-- } } } } loige 48
- 86. function createCountdown (from) { let nextVal = from return { next () { if (nextVal < 0) { return { done: true } } return { done: false, value: nextVal-- } } } } loige 48
- 87. function createCountdown (from) { let nextVal = from return { next () { if (nextVal < 0) { return { done: true } } return { done: false, value: nextVal-- } } } } loige 48
- 88. const countdown = createCountdown(3) console.log(countdown.next()) // { done: false, value: 3 } console.log(countdown.next()) // { done: false, value: 2 } console.log(countdown.next()) // { done: false, value: 1 } console.log(countdown.next()) // { done: false, value: 0 } console.log(countdown.next()) // { done: true } loige 49
- 90. function * createCountdown (from) { for (let i = from; i >= 0; i--) { yield i } } loige 51
- 91. function * createCountdown (from) { for (let i = from; i >= 0; i--) { yield i } } loige 51
- 92. function * createCountdown (from) { for (let i = from; i >= 0; i--) { yield i } } loige 51
- 93. const countdown = createCountdown(3) console.log(countdown.next()) // { done: false, value: 3 } console.log(countdown.next()) // { done: false, value: 2 } console.log(countdown.next()) // { done: false, value: 1 } console.log(countdown.next()) // { done: false, value: 0 } console.log(countdown.next()) // { done: true, value: undefined } loige 52
- 94. The iterable protocol An object is iterable if it implements the @@iterator* method, a zero-argument function that returns an iterator. loige *Symbol.iterator 53
- 95. function createCountdown (from) { let nextVal = from return { [Symbol.iterator]: () => ({ next () { if (nextVal < 0) { return { done: true } } return { done: false, value: nextVal-- } } }) } } loige 54
- 96. function createCountdown (from) { let nextVal = from return { [Symbol.iterator]: () => ({ next () { if (nextVal < 0) { return { done: true } } return { done: false, value: nextVal-- } } }) } } loige 54
- 97. function createCountdown (from) { let nextVal = from return { [Symbol.iterator]: () => ({ next () { if (nextVal < 0) { return { done: true } } return { done: false, value: nextVal-- } } }) } } loige 54
- 98. function createCountdown (from) { return { [Symbol.iterator]: function * () { for (let i = from; i >= 0; i--) { yield i } } } } loige 55
- 99. const countdown = createCountdown(3) for (const value of countdown) { console.log(value) } // 3 // 2 // 1 // 0 loige 56
- 100. OK. So far this is all synchronous iteration. What about async? 🙄 loige 57
- 101. The async iterator protocol An object is an async iterator if it has a next() method. Every time you call it, it returns a promise that resolves to an object with the keys done (boolean) and value. loige 58
- 102. import { setTimeout } from 'timers/promises' function createAsyncCountdown (from, delay = 1000) { let nextVal = from return { async next () { await setTimeout(delay) if (nextVal < 0) { return { done: true } } return { done: false, value: nextVal-- } } } } loige 59
- 103. import { setTimeout } from 'timers/promises' function createAsyncCountdown (from, delay = 1000) { let nextVal = from return { async next () { await setTimeout(delay) if (nextVal < 0) { return { done: true } } return { done: false, value: nextVal-- } } } } loige 59
- 104. import { setTimeout } from 'timers/promises' function createAsyncCountdown (from, delay = 1000) { let nextVal = from return { async next () { await setTimeout(delay) if (nextVal < 0) { return { done: true } } return { done: false, value: nextVal-- } } } } loige 59
- 105. const countdown = createAsyncCountdown(3) console.log(await countdown.next()) // { done: false, value: 3 } console.log(await countdown.next()) // { done: false, value: 2 } console.log(await countdown.next()) // { done: false, value: 1 } console.log(await countdown.next()) // { done: false, value: 0 } console.log(await countdown.next()) // { done: true } loige 60
- 106. const countdown = createAsyncCountdown(3) console.log(await countdown.next()) // { done: false, value: 3 } console.log(await countdown.next()) // { done: false, value: 2 } console.log(await countdown.next()) // { done: false, value: 1 } console.log(await countdown.next()) // { done: false, value: 0 } console.log(await countdown.next()) // { done: true } loige 60
- 107. loige 61
- 108. import { setTimeout } from 'timers/promises' // async generators "produce" async iterators! async function * createAsyncCountdown (from, delay = 1000) { for (let i = from; i >= 0; i--) { await setTimeout(delay) yield i } } loige 62
- 109. import { setTimeout } from 'timers/promises' // async generators "produce" async iterators! async function * createAsyncCountdown (from, delay = 1000) { for (let i = from; i >= 0; i--) { await setTimeout(delay) yield i } } loige 62
- 110. import { setTimeout } from 'timers/promises' // async generators "produce" async iterators! async function * createAsyncCountdown (from, delay = 1000) { for (let i = from; i >= 0; i--) { await setTimeout(delay) yield i } } loige 62
- 111. The async iterable protocol An object is an async iterable if it implements the @@asyncIterator* method, a zero-argument function that returns an async iterator. loige *Symbol.asyncIterator 63
- 112. import { setTimeout } from 'timers/promises' function createAsyncCountdown (from, delay = 1000) { return { [Symbol.asyncIterator]: async function * () { for (let i = from; i >= 0; i--) { await setTimeout(delay) yield i } } } } loige 64
- 113. import { setTimeout } from 'timers/promises' function createAsyncCountdown (from, delay = 1000) { return { [Symbol.asyncIterator]: async function * () { for (let i = from; i >= 0; i--) { await setTimeout(delay) yield i } } } } loige 64
- 114. HOT TIP 🔥 With async generators we can create objects that are both async iterators and async iterables! loige (We don't need to specify Symbol.asyncIterator explicitly!) 65
- 115. import { setTimeout } from 'timers/promises' // async generators "produce" async iterators // (and iterables!) async function * createAsyncCountdown (from, delay = 1000) { for (let i = from; i >= 0; i--) { await setTimeout(delay) yield i } } loige 66
- 116. const countdown = createAsyncCountdown(3) for await (const value of countdown) { console.log(value) } loige 67
- 117. const countdown = createAsyncCountdown(3) for await (const value of countdown) { console.log(value) } loige 67
- 118. Now we know how to make our LastFmRecentTracks an Async Iterable 🤩 loige 68
- 119. import querystring from 'querystring' import axios from 'axios' async function * createLastFmRecentTracks (apiKey, user) { let to while (true) { const query = querystring.stringify({ method: 'user.getrecenttracks', user, api_key: apiKey, format: 'json', to }) const url = `https://ws.audioscrobbler.com/2.0/?${query}` const response = await axios.get(url) const tracks = response.data.recenttracks.track yield tracks if (response.data.recenttracks['@attr'].totalPages <= 1) { break // it's the last page! } const lastTrackInPage = tracks[tracks.length - 1] to = lastTrackInPage.date.uts } } loige 69
- 120. import querystring from 'querystring' import axios from 'axios' async function * createLastFmRecentTracks (apiKey, user) { let to while (true) { const query = querystring.stringify({ method: 'user.getrecenttracks', user, api_key: apiKey, format: 'json', to }) const url = `https://ws.audioscrobbler.com/2.0/?${query}` const response = await axios.get(url) const tracks = response.data.recenttracks.track yield tracks if (response.data.recenttracks['@attr'].totalPages <= 1) { break // it's the last page! } const lastTrackInPage = tracks[tracks.length - 1] to = lastTrackInPage.date.uts } } loige 69
- 121. import querystring from 'querystring' import axios from 'axios' async function * createLastFmRecentTracks (apiKey, user) { let to while (true) { const query = querystring.stringify({ method: 'user.getrecenttracks', user, api_key: apiKey, format: 'json', to }) const url = `https://ws.audioscrobbler.com/2.0/?${query}` const response = await axios.get(url) const tracks = response.data.recenttracks.track yield tracks if (response.data.recenttracks['@attr'].totalPages <= 1) { break // it's the last page! } const lastTrackInPage = tracks[tracks.length - 1] to = lastTrackInPage.date.uts } } loige 69
- 122. const recentTracks = createLastFmRecentTracks( process.env.API_KEY, 'loige' ) for await (const page of recentTracks) { console.log(page) } loige 70
- 123. Let's search for all the songs that contain the word "dark" in their title! 🧐 loige 71
- 124. async function main () { const recentTracks = createLastFmRecentTracks( process.env.API_KEY, 'loige' ) for await (const page of recentTracks) { for (const track of page) { if (track.name.toLowerCase().includes('dark')) { console.log(`${track.artist['#text']} - ${track.name}`) } } } } loige 72
- 125. loige 73
- 126. loige OMG! This is the song! 😱 ...from 8 years ago! 73
- 127. For a more serious package that allows you to fetch data from Last.fm: loige npm install scrobbles 74
- 128. Cover picture by on Thanks to Jacek Spera, , , , for reviews and suggestions. Daniel Fontenele Unsplash @eoins @pelger @gbinside @ManuEomm - loige.link/enter-iterators loige.link/async-it-code for await (const _ of createAsyncCountdown(1_000_000)) { console.log("THANK YOU! 😍") } loige nodejsdp.link 75