11 "import pandas as pd\n",
12 "import numpy as np\n",
13 "import matplotlib\n",
14 "import matplotlib.pyplot as plt\n",
15 "%matplotlib inline \n",
16 "import urllib.request\n",
17 "import urllib.parse\n",
18 "import urllib.error\n",
21 "import configparser\n",
22 "from bs4 import BeautifulSoup\n",
25 "from datetime import datetime\n",
27 "import collections\n",
39 "# Open a connection to the Mongo server\n",
40 "client = pymongo.MongoClient('mongodb://localhost:27017/')"
51 "# Create a database and a collections within it.\n",
52 "songs_db = client.songs\n",
53 "albums = songs_db.albums\n",
54 "tracks = songs_db.tracks\n",
55 "genius_tracks = songs_db.gtracks"
68 "['app_name', 'consumer_key', 'consumer_secret', 'token']"
73 "output_type": "execute_result"
77 "config = configparser.ConfigParser()\n",
78 "config.read('secrets.ini')\n",
79 "[k for k in config['discogs']]"
84 "execution_count": null,
90 "def get_audio_features(track_ids, auth_type, auth_token):\n",
91 " url = 'https://api.spotify.com/v1/audio-features?ids={ids}'.format(ids=','.join(track_ids))\n",
92 " headers = {'Authorization': auth_type + ' ' + auth_token}\n",
93 " request = urllib.request.Request(url, headers=headers, method='GET')\n",
95 " for _ in range(10):\n",
97 " with urllib.request.urlopen(request) as f:\n",
98 " response = json.loads(f.read().decode('utf-8'))\n",
99 " for track in response['audio_features']:\n",
100 " tracks.update_one({'_id': track['id']}, {'$set': track})\n",
102 " except urllib.error.HTTPError as e:\n",
103 " print(\"Rate limited. Pausing for\", e.info()['Retry-After'])\n",
104 " time.sleep(int(e.info()['Retry-After']) + 0.5)\n",
110 "execution_count": null,
116 "def get_artists(artist_name):\n",
117 " query = urllib.parse.urlencode({'q': artist_name, 'type': 'artist'})\n",
118 " request = 'https://api.spotify.com/v1/search?{}'.format(query)\n",
119 " with urllib.request.urlopen(request) as f:\n",
120 " response = json.loads(f.read().decode('utf-8'))\n",
122 " for artist in response['artists']['items']:\n",
123 " if artist['name'].lower() == artist_name.lower():\n",
124 " this_artist = {'name': artist['name'], 'id': artist['id']}\n",
125 " if artist['images']:\n",
126 " this_artist['image'] = artist['images'][0]['url']\n",
127 " artists += [this_artist]\n",
133 "execution_count": null,
139 "/database/search?q={query}&{?type,title,release_title,credit,artist,anv,label,genre,style,country,year,format,catno,barcode,track,submitter,contributor}"
144 "execution_count": 45,
150 "def get_artist(artist_name):\n",
151 " query = urllib.parse.urlencode({'q': artist_name, 'type': 'artist'})\n",
152 " # query = urllib.parse.urlencode({'q': artist_name})\n",
153 " url = 'https://api.discogs.com/database/search?{}'.format(query)\n",
154 " headers = {'Authorization': 'Discogs token=' + config['discogs']['token']}\n",
157 " request = urllib.request.Request(url, headers=headers, method='GET')\n",
158 " with urllib.request.urlopen(request) as f:\n",
159 " response = json.loads(f.read().decode('utf-8'))\n",
160 " artists += response['results']\n",
161 " if 'next' in response['pagination']['urls']:\n",
162 " url = response['pagination']['urls']['next']\n",
170 "execution_count": 44,
177 "output_type": "stream",
179 "{'next': 'https://api.discogs.com/database/search?q=Nirvana&per_page=50&type=artist&page=2', 'last': 'https://api.discogs.com/database/search?q=Nirvana&per_page=50&type=artist&page=5'}\n",
180 "{'first': 'https://api.discogs.com/database/search?q=Nirvana&per_page=50&type=artist&page=1', 'next': 'https://api.discogs.com/database/search?q=Nirvana&per_page=50&type=artist&page=3', 'last': 'https://api.discogs.com/database/search?q=Nirvana&per_page=50&type=artist&page=5', 'prev': 'https://api.discogs.com/database/search?q=Nirvana&per_page=50&type=artist&page=1'}\n",
181 "{'first': 'https://api.discogs.com/database/search?q=Nirvana&per_page=50&type=artist&page=1', 'next': 'https://api.discogs.com/database/search?q=Nirvana&per_page=50&type=artist&page=4', 'last': 'https://api.discogs.com/database/search?q=Nirvana&per_page=50&type=artist&page=5', 'prev': 'https://api.discogs.com/database/search?q=Nirvana&per_page=50&type=artist&page=2'}\n",
182 "{'first': 'https://api.discogs.com/database/search?q=Nirvana&per_page=50&type=artist&page=1', 'next': 'https://api.discogs.com/database/search?q=Nirvana&per_page=50&type=artist&page=5', 'last': 'https://api.discogs.com/database/search?q=Nirvana&per_page=50&type=artist&page=5', 'prev': 'https://api.discogs.com/database/search?q=Nirvana&per_page=50&type=artist&page=3'}\n",
183 "{'first': 'https://api.discogs.com/database/search?q=Nirvana&per_page=50&type=artist&page=1', 'prev': 'https://api.discogs.com/database/search?q=Nirvana&per_page=50&type=artist&page=4'}\n"
192 "execution_count": 44,
194 "output_type": "execute_result"
198 "nivs = get_artist('Nirvana')\n",
204 "execution_count": 47,
213 " 'resource_url': 'https://api.discogs.com/artists/125246',\n",
214 " 'thumb': 'https://api-img.discogs.com/6AS7RIgqIBFEuGusd3tG_z2J2rs=/150x150/smart/filters:strip_icc():format(jpeg):mode_rgb():quality(40)/discogs-images/A-125246-1105986304.jpg.jpg',\n",
215 " 'title': 'Nirvana',\n",
216 " 'type': 'artist',\n",
217 " 'uri': '/artist/125246-Nirvana'},\n",
219 " 'resource_url': 'https://api.discogs.com/artists/307513',\n",
220 " 'thumb': 'https://api-img.discogs.com/5skyOqEGgSOnt9lhPHhjP4uCACE=/150x150/smart/filters:strip_icc():format(jpeg):mode_rgb():quality(40)/discogs-images/A-307513-1270236942.jpeg.jpg',\n",
221 " 'title': 'Nirvana (2)',\n",
222 " 'type': 'artist',\n",
223 " 'uri': '/artist/307513-Nirvana-2'},\n",
224 " {'id': 1087206,\n",
225 " 'resource_url': 'https://api.discogs.com/artists/1087206',\n",
226 " 'thumb': 'https://api-img.discogs.com/Wgp-oaSc03_RK9skZYyCp3YrQOU=/150x150/smart/filters:strip_icc():format(jpeg):mode_rgb():quality(40)/discogs-images/A-1087206-1266869411.jpeg.jpg',\n",
227 " 'title': 'Nirvana 2002',\n",
228 " 'type': 'artist',\n",
229 " 'uri': '/artist/1087206-Nirvana-2002'},\n",
230 " {'id': 1082359,\n",
231 " 'resource_url': 'https://api.discogs.com/artists/1082359',\n",
233 " 'title': 'Nirvana (6)',\n",
234 " 'type': 'artist',\n",
235 " 'uri': '/artist/1082359-Nirvana-6'},\n",
236 " {'id': 2744248,\n",
237 " 'resource_url': 'https://api.discogs.com/artists/2744248',\n",
239 " 'title': 'Nirvana (10)',\n",
240 " 'type': 'artist',\n",
241 " 'uri': '/artist/2744248-Nirvana-10'},\n",
242 " {'id': 1176738,\n",
243 " 'resource_url': 'https://api.discogs.com/artists/1176738',\n",
244 " 'thumb': 'https://api-img.discogs.com/NV8RiufOYHnOPR1oA6tGpIy3mog=/150x150/smart/filters:strip_icc():format(jpeg):mode_rgb():quality(40)/discogs-images/A-1176738-1317705549.jpeg.jpg',\n",
245 " 'title': 'Nirvana (7)',\n",
246 " 'type': 'artist',\n",
247 " 'uri': '/artist/1176738-Nirvana-7'},\n",
249 " 'resource_url': 'https://api.discogs.com/artists/457609',\n",
251 " 'title': 'Nirvana (4)',\n",
252 " 'type': 'artist',\n",
253 " 'uri': '/artist/457609-Nirvana-4'},\n",
254 " {'id': 1389061,\n",
255 " 'resource_url': 'https://api.discogs.com/artists/1389061',\n",
257 " 'title': 'Nirvana (8)',\n",
258 " 'type': 'artist',\n",
259 " 'uri': '/artist/1389061-Nirvana-8'},\n",
261 " 'resource_url': 'https://api.discogs.com/artists/816495',\n",
263 " 'title': 'Nirvana (5)',\n",
264 " 'type': 'artist',\n",
265 " 'uri': '/artist/816495-Nirvana-5'},\n",
267 " 'resource_url': 'https://api.discogs.com/artists/965460',\n",
268 " 'thumb': 'https://api-img.discogs.com/tnmDXyMMViUQYCPPRkrINumoAP0=/150x150/smart/filters:strip_icc():format(jpeg):mode_rgb():quality(40)/discogs-images/A-965460-1370853336-6662.jpeg.jpg',\n",
269 " 'title': 'The Nirvana Devils',\n",
270 " 'type': 'artist',\n",
271 " 'uri': '/artist/965460-The-Nirvana-Devils'},\n",
273 " 'resource_url': 'https://api.discogs.com/artists/319676',\n",
275 " 'title': 'Nirvana (3)',\n",
276 " 'type': 'artist',\n",
277 " 'uri': '/artist/319676-Nirvana-3'},\n",
278 " {'id': 2414323,\n",
279 " 'resource_url': 'https://api.discogs.com/artists/2414323',\n",
281 " 'title': 'El Nirvana',\n",
282 " 'type': 'artist',\n",
283 " 'uri': '/artist/2414323-El-Nirvana'},\n",
285 " 'resource_url': 'https://api.discogs.com/artists/934885',\n",
287 " 'title': 'Nirvana Bros.',\n",
288 " 'type': 'artist',\n",
289 " 'uri': '/artist/934885-Nirvana-Bros'},\n",
290 " {'id': 2103552,\n",
291 " 'resource_url': 'https://api.discogs.com/artists/2103552',\n",
293 " 'title': 'Nirvana (9)',\n",
294 " 'type': 'artist',\n",
295 " 'uri': '/artist/2103552-Nirvana-9'},\n",
297 " 'resource_url': 'https://api.discogs.com/artists/45870',\n",
299 " 'title': 'The Nirvana Sitar & String Group',\n",
300 " 'type': 'artist',\n",
301 " 'uri': '/artist/45870-The-Nirvana-Sitar-String-Group'},\n",
302 " {'id': 4634135,\n",
303 " 'resource_url': 'https://api.discogs.com/artists/4634135',\n",
305 " 'title': 'Nirvana (13)',\n",
306 " 'type': 'artist',\n",
307 " 'uri': '/artist/4634135-Nirvana-13'},\n",
309 " 'resource_url': 'https://api.discogs.com/artists/520870',\n",
311 " 'title': 'The Attainment Of Nirvana',\n",
312 " 'type': 'artist',\n",
313 " 'uri': '/artist/520870-The-Attainment-Of-Nirvana'},\n",
314 " {'id': 5145389,\n",
315 " 'resource_url': 'https://api.discogs.com/artists/5145389',\n",
316 " 'thumb': 'https://api-img.discogs.com/YSxmWcLnQ716R0Om_psIRruIyZ4=/150x150/smart/filters:strip_icc():format(jpeg):mode_rgb():quality(40)/discogs-images/A-5145389-1468099375-2943.jpeg.jpg',\n",
317 " 'title': 'Nirvana (14)',\n",
318 " 'type': 'artist',\n",
319 " 'uri': '/artist/5145389-Nirvana-14'},\n",
320 " {'id': 3465490,\n",
321 " 'resource_url': 'https://api.discogs.com/artists/3465490',\n",
322 " 'thumb': 'https://api-img.discogs.com/-oS5YYSnPEdE5qT4sh1_jyY2h_s=/150x150/smart/filters:strip_icc():format(jpeg):mode_rgb():quality(40)/discogs-images/A-3465490-1379316060-1118.jpeg.jpg',\n",
323 " 'title': 'Approaching Nirvana',\n",
324 " 'type': 'artist',\n",
325 " 'uri': '/artist/3465490-Approaching-Nirvana'},\n",
326 " {'id': 5047262,\n",
327 " 'resource_url': 'https://api.discogs.com/artists/5047262',\n",
329 " 'title': 'Nirvana Kelly',\n",
330 " 'type': 'artist',\n",
331 " 'uri': '/artist/5047262-Nirvana-Kelly'},\n",
332 " {'id': 1907189,\n",
333 " 'resource_url': 'https://api.discogs.com/artists/1907189',\n",
335 " 'title': 'Paulette \"Nirvana\" Buckley',\n",
336 " 'type': 'artist',\n",
337 " 'uri': '/artist/1907189-Paulette-Nirvana-Buckley'},\n",
338 " {'id': 1286267,\n",
339 " 'resource_url': 'https://api.discogs.com/artists/1286267',\n",
341 " 'title': 'The Nirvana',\n",
342 " 'type': 'artist',\n",
343 " 'uri': '/artist/1286267-The-Nirvana'},\n",
344 " {'id': 2888206,\n",
345 " 'resource_url': 'https://api.discogs.com/artists/2888206',\n",
347 " 'title': 'Nada Nirvana',\n",
348 " 'type': 'artist',\n",
349 " 'uri': '/artist/2888206-Nada-Nirvana'},\n",
350 " {'id': 1459803,\n",
351 " 'resource_url': 'https://api.discogs.com/artists/1459803',\n",
352 " 'thumb': 'https://api-img.discogs.com/HW8koSJDiyjCmFZHsTSBpbIiD2A=/150x150/smart/filters:strip_icc():format(jpeg):mode_rgb():quality(40)/discogs-images/A-1459803-1420753729-4099.jpeg.jpg',\n",
353 " 'title': 'Nirvana Savoury',\n",
354 " 'type': 'artist',\n",
355 " 'uri': '/artist/1459803-Nirvana-Savoury'},\n",
356 " {'id': 4809078,\n",
357 " 'resource_url': 'https://api.discogs.com/artists/4809078',\n",
359 " 'title': 'The Nirvana Ensemble',\n",
360 " 'type': 'artist',\n",
361 " 'uri': '/artist/4809078-The-Nirvana-Ensemble'},\n",
363 " 'resource_url': 'https://api.discogs.com/artists/501548',\n",
365 " 'title': 'Nirvana 1 Way',\n",
366 " 'type': 'artist',\n",
367 " 'uri': '/artist/501548-Nirvana-1-Way'},\n",
368 " {'id': 2100392,\n",
369 " 'resource_url': 'https://api.discogs.com/artists/2100392',\n",
371 " 'title': 'Last Casino In Nirvana',\n",
372 " 'type': 'artist',\n",
373 " 'uri': '/artist/2100392-Last-Casino-In-Nirvana'},\n",
374 " {'id': 3995371,\n",
375 " 'resource_url': 'https://api.discogs.com/artists/3995371',\n",
377 " 'title': 'Nirvana (12)',\n",
378 " 'type': 'artist',\n",
379 " 'uri': '/artist/3995371-Nirvana-12'},\n",
380 " {'id': 2728399,\n",
381 " 'resource_url': 'https://api.discogs.com/artists/2728399',\n",
383 " 'title': 'The Nirvana (2)',\n",
384 " 'type': 'artist',\n",
385 " 'uri': '/artist/2728399-The-Nirvana-2'},\n",
386 " {'id': 3105291,\n",
387 " 'resource_url': 'https://api.discogs.com/artists/3105291',\n",
388 " 'thumb': 'https://api-img.discogs.com/uGA8CJEGkje1yNUt7kwEIDF1TBQ=/150x150/smart/filters:strip_icc():format(jpeg):mode_rgb():quality(40)/discogs-images/A-3105291-1358422778-8324.jpeg.jpg',\n",
389 " 'title': 'Nirvana (11)',\n",
390 " 'type': 'artist',\n",
391 " 'uri': '/artist/3105291-Nirvana-11'},\n",
392 " {'id': 2852124,\n",
393 " 'resource_url': 'https://api.discogs.com/artists/2852124',\n",
395 " 'title': 'The Nirvana Banana',\n",
396 " 'type': 'artist',\n",
397 " 'uri': '/artist/2852124-The-Nirvana-Banana'},\n",
398 " {'id': 3114787,\n",
399 " 'resource_url': 'https://api.discogs.com/artists/3114787',\n",
401 " 'title': 'Nirvana Pan Jali',\n",
402 " 'type': 'artist',\n",
403 " 'uri': '/artist/3114787-Nirvana-Pan-Jali'},\n",
404 " {'id': 3574712,\n",
405 " 'resource_url': 'https://api.discogs.com/artists/3574712',\n",
407 " 'title': 'Ricky Nirvana',\n",
408 " 'type': 'artist',\n",
409 " 'uri': '/artist/3574712-Ricky-Nirvana'},\n",
410 " {'id': 1285360,\n",
411 " 'resource_url': 'https://api.discogs.com/artists/1285360',\n",
413 " 'title': 'Nirvana Banana',\n",
414 " 'type': 'artist',\n",
415 " 'uri': '/artist/1285360-Nirvana-Banana'},\n",
416 " {'id': 2966881,\n",
417 " 'resource_url': 'https://api.discogs.com/artists/2966881',\n",
419 " 'title': 'Weed Nirvana',\n",
420 " 'type': 'artist',\n",
421 " 'uri': '/artist/2966881-Weed-Nirvana'},\n",
422 " {'id': 4562747,\n",
423 " 'resource_url': 'https://api.discogs.com/artists/4562747',\n",
425 " 'title': \"Grass (The Real '60s UK Nirvana)\",\n",
426 " 'type': 'artist',\n",
427 " 'uri': '/artist/4562747-Grass-The-Real-60s-UK-Nirvana'},\n",
428 " {'id': 3758745,\n",
429 " 'resource_url': 'https://api.discogs.com/artists/3758745',\n",
431 " 'title': 'Nihil Nirvana',\n",
432 " 'type': 'artist',\n",
433 " 'uri': '/artist/3758745-Nihil-Nirvana'},\n",
434 " {'id': 4143675,\n",
435 " 'resource_url': 'https://api.discogs.com/artists/4143675',\n",
436 " 'thumb': 'https://api-img.discogs.com/3rc6erC0JTTNXR05sF9T7eF60Qc=/150x150/smart/filters:strip_icc():format(jpeg):mode_rgb():quality(40)/discogs-images/A-4143675-1486326311-8056.jpeg.jpg',\n",
437 " 'title': 'Nirvanass',\n",
438 " 'type': 'artist',\n",
439 " 'uri': '/artist/4143675-Nirvanass'},\n",
440 " {'id': 5375335,\n",
441 " 'resource_url': 'https://api.discogs.com/artists/5375335',\n",
443 " 'title': 'Nirvana Heire',\n",
444 " 'type': 'artist',\n",
445 " 'uri': '/artist/5375335-Nirvana-Heire'},\n",
446 " {'id': 3221382,\n",
447 " 'resource_url': 'https://api.discogs.com/artists/3221382',\n",
449 " 'title': 'Nirvana Groove',\n",
450 " 'type': 'artist',\n",
451 " 'uri': '/artist/3221382-Nirvana-Groove'},\n",
452 " {'id': 3875208,\n",
453 " 'resource_url': 'https://api.discogs.com/artists/3875208',\n",
455 " 'title': 'Nirvana Lucie',\n",
456 " 'type': 'artist',\n",
457 " 'uri': '/artist/3875208-Nirvana-Lucie'},\n",
458 " {'id': 2337392,\n",
459 " 'resource_url': 'https://api.discogs.com/artists/2337392',\n",
461 " 'title': 'Nirvana Curve',\n",
462 " 'type': 'artist',\n",
463 " 'uri': '/artist/2337392-Nirvana-Curve'},\n",
464 " {'id': 3801487,\n",
465 " 'resource_url': 'https://api.discogs.com/artists/3801487',\n",
467 " 'title': 'Nirvana Gherbaz Duraković',\n",
468 " 'type': 'artist',\n",
469 " 'uri': '/artist/3801487-Nirvana-Gherbaz-Duraković'},\n",
470 " {'id': 3189817,\n",
471 " 'resource_url': 'https://api.discogs.com/artists/3189817',\n",
473 " 'title': 'Takis Nirvanas And The 4 Chambers',\n",
474 " 'type': 'artist',\n",
475 " 'uri': '/artist/3189817-Takis-Nirvanas-And-The-4-Chambers'},\n",
476 " {'id': 5305718,\n",
477 " 'resource_url': 'https://api.discogs.com/artists/5305718',\n",
479 " 'title': 'Nirvana Teen Spirit',\n",
480 " 'type': 'artist',\n",
481 " 'uri': '/artist/5305718-Nirvana-Teen-Spirit'},\n",
482 " {'id': 5515837,\n",
483 " 'resource_url': 'https://api.discogs.com/artists/5515837',\n",
484 " 'thumb': 'https://api-img.discogs.com/images/warning.png',\n",
485 " 'title': 'Bon Jovi, Nirvana, Michael Jackson, INXS, ZZTOP, Extreme, Boston, Garth Brooks, Midnight Oil, Huey Lewis & The News',\n",
486 " 'type': 'artist',\n",
487 " 'uri': '/artist/5515837-Bon-Jovi-Nirvana-Michael-Jackson-INXS-ZZTOP-Extreme-Boston-Garth-Brooks-Midnight-Oil-Huey-Lewis-The-News'}]"
490 "execution_count": 47,
492 "output_type": "execute_result"
496 "[n for n in nivs if 'Nirvana' in n['title']]"
501 "execution_count": 48,
512 "execution_count": 48,
514 "output_type": "execute_result"
518 "tbs = get_artist('The Beatles')\n",
524 "execution_count": 56,
535 "execution_count": 56,
537 "output_type": "execute_result"
541 "atbs = [a for a in tbs if re.match('^The Beatles(\\s\\(\\d+\\))?$', a['title'])]\n",
547 "execution_count": 57,
556 " 'resource_url': 'https://api.discogs.com/artists/82730',\n",
557 " 'thumb': 'https://api-img.discogs.com/v1NIz7CyzwLHJsnSGIjg6sCL5FI=/150x150/smart/filters:strip_icc():format(jpeg):mode_rgb():quality(40)/discogs-images/A-82730-1449581547-9306.jpeg.jpg',\n",
558 " 'title': 'The Beatles',\n",
559 " 'type': 'artist',\n",
560 " 'uri': '/artist/82730-The-Beatles'},\n",
561 " {'id': 2517607,\n",
562 " 'resource_url': 'https://api.discogs.com/artists/2517607',\n",
564 " 'title': 'The Beatles (2)',\n",
565 " 'type': 'artist',\n",
566 " 'uri': '/artist/2517607-The-Beatles-2'},\n",
567 " {'id': 4290435,\n",
568 " 'resource_url': 'https://api.discogs.com/artists/4290435',\n",
570 " 'title': 'The Beatles (3)',\n",
571 " 'type': 'artist',\n",
572 " 'uri': '/artist/4290435-The-Beatles-3'}]"
575 "execution_count": 57,
577 "output_type": "execute_result"
586 "execution_count": null,
596 "display_name": "Python 3",
597 "language": "python",
605 "file_extension": ".py",
606 "mimetype": "text/x-python",
608 "nbconvert_exporter": "python",
609 "pygments_lexer": "ipython3",