From f639221ca116f25e4fb9912cadce213e43217221 Mon Sep 17 00:00:00 2001 From: Stephan Lensky Date: Sat, 11 May 2019 21:25:39 -0400 Subject: [PATCH 1/4] Fix playlist and cover art download --- .gitignore | 1 + spotify_ripper/post_actions.py | 2 +- spotify_ripper/ripper.py | 28 +++++++++----- ...rom_playlist.py => spotipy_integration.py} | 38 ++++++++++++++----- spotify_ripper/web.py | 15 ++------ 5 files changed, 53 insertions(+), 31 deletions(-) rename spotify_ripper/{remove_all_from_playlist.py => spotipy_integration.py} (50%) diff --git a/.gitignore b/.gitignore index dd40d3f..01042ac 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ develop-eggs .installed.cfg lib lib64 +.cache-* # Installer logs pip-log.txt diff --git a/spotify_ripper/post_actions.py b/spotify_ripper/post_actions.py index 0ec49c3..97630e6 100644 --- a/spotify_ripper/post_actions.py +++ b/spotify_ripper/post_actions.py @@ -9,7 +9,7 @@ import spotify import codecs import shutil -from spotify_ripper.remove_all_from_playlist import remove_all_from_playlist +from spotify_ripper.spotipy_integration import remove_all_from_playlist class PostActions(object): diff --git a/spotify_ripper/ripper.py b/spotify_ripper/ripper.py index 113d263..49adf42 100644 --- a/spotify_ripper/ripper.py +++ b/spotify_ripper/ripper.py @@ -12,7 +12,7 @@ from spotify_ripper.sync import Sync from spotify_ripper.eventloop import EventLoop from datetime import datetime -from spotify_ripper.remove_all_from_playlist import get_playlist_tracks +from spotify_ripper.spotipy_integration import get_playlist_tracks, init_spotipy import os import sys import time @@ -189,12 +189,16 @@ def run(self): self.ripper_continue.wait() if self.abort.is_set(): return - #set session to provate + + # set session to private self.session.social.private_session = True # list of spotify URIs uris = args.uri + # init spotipy api + init_spotipy(self.session.user.canonical_name) + def get_tracks_from_uri(uri): self.current_playlist = None self.current_album = None @@ -225,11 +229,16 @@ def get_tracks_from_uri(uri): # calculate total size and time all_tracks = [] + all_uris = {} for uri in uris: - tracks = list(get_tracks_from_uri(uri)) + if uri not in all_uris: + all_uris[uri] = list(get_tracks_from_uri(uri)) + tracks = all_uris[uri] # TODO: remove dependency on current_album, ... for idx, track in enumerate(tracks): + print('Loading track {}/{}...'.format(idx, len(tracks)), end='\r') + sys.stdout.flush() # ignore local tracks if track.is_local: @@ -237,6 +246,7 @@ def get_tracks_from_uri(uri): audio_file = self.format_track_path(idx, track) all_tracks.append((track, audio_file)) + print('Loading track {}/{}...'.format(len(tracks), len(tracks))) self.progress.calc_total(all_tracks) @@ -250,7 +260,9 @@ def get_tracks_from_uri(uri): if self.abort.is_set(): break - tracks = list(get_tracks_from_uri(uri)) + if uri not in all_uris: + all_uris[uri] = list(get_tracks_from_uri(uri)) + tracks = all_uris[uri] if args.playlist_sync and self.current_playlist: self.sync = Sync(args, self) @@ -269,8 +281,8 @@ def get_tracks_from_uri(uri): track.load(args.timeout) if track.availability != 1 or track.is_local: print( - Fore.RED + 'Track is not available, ' - 'skipping...' + Fore.RESET) + Fore.RED + 'Track {} - {} is not available, skipping...' + .format(track.artists[0].name, track.name) + Fore.RESET) self.post.log_failure(track) continue @@ -363,7 +375,7 @@ def get_tracks_from_uri(uri): self.post.create_playlist_wpl(tracks) # actually removing the tracks from playlist - self.post.remove_tracks_from_playlist() + # self.post.remove_tracks_from_playlist() # remove libspotify's offline storage cache self.post.remove_offline_cache() @@ -427,7 +439,6 @@ def load_link(self, uri): track = link.as_track() return iter([track]) elif link.type == spotify.LinkType.PLAYLIST: - print('get playlist tracks') self.playlist_uri = uri tracks = get_playlist_tracks(self.session.user.canonical_name, uri) track_list = tracks.get('items') @@ -438,7 +449,6 @@ def load_link(self, uri): tracksIter = iter(uriList) for i in tracksIter: trackList.append(self.session.get_link(i).as_track()) - print('Loading playlist...') return iter(trackList) elif link.type == spotify.LinkType.STARRED: link_user = link.as_user() diff --git a/spotify_ripper/remove_all_from_playlist.py b/spotify_ripper/spotipy_integration.py similarity index 50% rename from spotify_ripper/remove_all_from_playlist.py rename to spotify_ripper/spotipy_integration.py index 5fc15de..8ce1069 100644 --- a/spotify_ripper/remove_all_from_playlist.py +++ b/spotify_ripper/spotipy_integration.py @@ -1,3 +1,5 @@ +import sys + import spotipy.util as util import spotipy.client import os @@ -15,7 +17,17 @@ spotInstance = None +def init_spotipy(username): + global token + token = util.prompt_for_user_token(username, scope) + + global spotInstance + spotInstance = spotipy.Spotify(auth=token) + spotInstance.trace = False + + def remove_all_from_playlist(username, playlistURI): + print('hello remove_all') tracks = get_playlist_tracks(username, playlistURI) track_ids = [] @@ -30,16 +42,22 @@ def get_playlist_tracks(username, playlistURI): global rPlaylistID p1, p2, p3, p4, rPlaylistID = playlistURI.split(':', 5) - global token - token = util.prompt_for_user_token(username, scope) - - global spotInstance - spotInstance = spotipy.Spotify(auth=token) - spotInstance.trace = False + # the spotify api limits the number of tracks which can be retrieved from a playlist in a single request + # make multiple requests to collect all playlist tracks before continuing + print('Collecting tracks from playlist', end='\r') + tracks = spotInstance.user_playlist(username, rPlaylistID, fields='tracks,next')['tracks'] + sys.stdout.flush() + print('Collecting tracks from playlist ({}/{})'.format(len(tracks['items']), tracks['total']), end='\r') + paged_tracks = tracks + while paged_tracks['next']: + paged_tracks = spotInstance.next(paged_tracks) + tracks['items'].extend(paged_tracks['items']) + sys.stdout.flush() + print('Collecting tracks from playlist ({}/{})'.format(len(tracks['items']), tracks['total']), end='\r') + print('Collecting tracks from playlist ({}/{})'.format(len(tracks['items']), tracks['total'])) - print('Getting Results') - results = spotInstance.user_playlist(username, rPlaylistID, fields="tracks,next") + return tracks - tracks = results['tracks'] - return tracks +def get_track_json(track_uri): + return spotInstance.track(track_uri) diff --git a/spotify_ripper/web.py b/spotify_ripper/web.py index 11dc062..9520f47 100644 --- a/spotify_ripper/web.py +++ b/spotify_ripper/web.py @@ -4,6 +4,7 @@ from colorama import Fore from spotify_ripper.utils import * +from spotify_ripper.spotipy_integration import get_track_json import os import time import spotify @@ -233,12 +234,7 @@ def sanity_check_date(val): self.cache_result("charts", uri, charts_obj) return charts_obj - def get_large_coverart(self, uri): - def get_track_json(track_id): - url = self.api_url('tracks/' + track_id) - return self.request_json(url, "track") - def get_image_data(url): response = self.request_url(url, "cover art") return response.content @@ -248,18 +244,15 @@ def get_image_data(url): if cached_result is not None: return get_image_data(cached_result) - # extract album id from uri - uri_tokens = uri.split(':') - if len(uri_tokens) != 3: - return None - - track = get_track_json(uri_tokens[2]) + track = get_track_json(uri) if track is None: + print(Fore.RED + "Failed to retrieve track information, cover art cannot be set" + Fore.RESET) return None try: images = track['album']['images'] except KeyError: + print(Fore.RED + "Album art not found in track JSON, cover art cannot be set" + Fore.RESET) return None for image in images: From 9fc461ec30743b8880ccea9154b10d41f05d2241 Mon Sep 17 00:00:00 2001 From: Stephan Lensky Date: Sat, 11 May 2019 21:38:07 -0400 Subject: [PATCH 2/4] Remove debug print statement --- spotify_ripper/spotipy_integration.py | 1 - 1 file changed, 1 deletion(-) diff --git a/spotify_ripper/spotipy_integration.py b/spotify_ripper/spotipy_integration.py index 8ce1069..f91ecd0 100644 --- a/spotify_ripper/spotipy_integration.py +++ b/spotify_ripper/spotipy_integration.py @@ -27,7 +27,6 @@ def init_spotipy(username): def remove_all_from_playlist(username, playlistURI): - print('hello remove_all') tracks = get_playlist_tracks(username, playlistURI) track_ids = [] From 639189d375044e18906fd903270a372f9425635e Mon Sep 17 00:00:00 2001 From: Stephan Lensky Date: Sun, 12 May 2019 03:43:54 -0400 Subject: [PATCH 3/4] .cache-* files are temp data created by spotipy --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 01042ac..8d1f29b 100644 --- a/.gitignore +++ b/.gitignore @@ -17,7 +17,6 @@ develop-eggs .installed.cfg lib lib64 -.cache-* # Installer logs pip-log.txt @@ -62,3 +61,4 @@ spotify_appkey.key *.m4a tmp/* *~ +.cache-* From 72d06df06334a7f389bc0186b9f23fca3b5939bc Mon Sep 17 00:00:00 2001 From: Stephan Lensky Date: Sun, 12 May 2019 03:49:50 -0400 Subject: [PATCH 4/4] Reauthorize token before making spotipy requests --- spotify_ripper/spotipy_integration.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/spotify_ripper/spotipy_integration.py b/spotify_ripper/spotipy_integration.py index f91ecd0..3d4f37b 100644 --- a/spotify_ripper/spotipy_integration.py +++ b/spotify_ripper/spotipy_integration.py @@ -2,6 +2,7 @@ import spotipy.util as util import spotipy.client +import spotipy.oauth2 as oauth2 import os redirect_uri = 'http://www.purple.com' @@ -15,9 +16,13 @@ token = None spotInstance = None +spotAuthUsername = None def init_spotipy(username): + global spotAuthUsername + spotAuthUsername = username + global token token = util.prompt_for_user_token(username, scope) @@ -26,7 +31,17 @@ def init_spotipy(username): spotInstance.trace = False +def refresh_access_token(): + global token + token = util.prompt_for_user_token(spotAuthUsername, scope) + + global spotInstance + spotInstance = spotipy.Spotify(auth=token) + spotInstance.trace = False + + def remove_all_from_playlist(username, playlistURI): + refresh_access_token() tracks = get_playlist_tracks(username, playlistURI) track_ids = [] @@ -38,6 +53,7 @@ def remove_all_from_playlist(username, playlistURI): def get_playlist_tracks(username, playlistURI): + refresh_access_token() global rPlaylistID p1, p2, p3, p4, rPlaylistID = playlistURI.split(':', 5) @@ -59,4 +75,5 @@ def get_playlist_tracks(username, playlistURI): def get_track_json(track_uri): + refresh_access_token() return spotInstance.track(track_uri)