From e64746f277e24f30c13aee9dcb9954a4607c5a79 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sun, 2 Apr 2017 17:02:41 +0200 Subject: [PATCH] Huge music overhaul - Fixes #254 --- resources/lib/itemtypes.py | 346 +++++++++--------- resources/lib/library_sync/__init__.py | 1 + resources/lib/library_sync/fanart.py | 88 +++++ resources/lib/library_sync/get_metadata.py | 140 +++++++ .../lib/library_sync/process_metadata.py | 104 ++++++ resources/lib/library_sync/sync_info.py | 81 ++++ resources/lib/librarysync.py | 341 ++--------------- 7 files changed, 623 insertions(+), 478 deletions(-) create mode 100644 resources/lib/library_sync/__init__.py create mode 100644 resources/lib/library_sync/fanart.py create mode 100644 resources/lib/library_sync/get_metadata.py create mode 100644 resources/lib/library_sync/process_metadata.py create mode 100644 resources/lib/library_sync/sync_info.py diff --git a/resources/lib/itemtypes.py b/resources/lib/itemtypes.py index 2e994043..5fa46dc9 100644 --- a/resources/lib/itemtypes.py +++ b/resources/lib/itemtypes.py @@ -1296,7 +1296,7 @@ class Music(Items): name, sortname = API.getTitle() # musicBrainzId = API.getProvider('MusicBrainzArtist') musicBrainzId = None - genres = API.joinList(API.getGenres()) + genres = ' / '.join(API.getGenres()) bio = API.getPlot() # Associate artwork @@ -1335,31 +1335,32 @@ class Music(Items): # Process the artist if v.KODIVERSION >= 16: - query = ' '.join(( - - "UPDATE artist", - "SET strGenres = ?, strBiography = ?, strImage = ?, strFanart = ?,", - "lastScraped = ?", - "WHERE idArtist = ?" - )) + query = ''' + UPDATE artist + SET strGenres = ?, strBiography = ?, strImage = ?, + strFanart = ?, lastScraped = ? + WHERE idArtist = ? + ''' kodicursor.execute(query, (genres, bio, thumb, fanart, lastScraped, artistid)) else: - query = ' '.join(( - - "UPDATE artist", - "SET strGenres = ?, strBiography = ?, strImage = ?, strFanart = ?,", - "lastScraped = ?, dateAdded = ?", - "WHERE idArtist = ?" - )) + query = ''' + UPDATE artist + SET strGenres = ?, strBiography = ?, strImage = ?, + strFanart = ?, lastScraped = ?, dateAdded = ? + WHERE idArtist = ? + ''' kodicursor.execute(query, (genres, bio, thumb, fanart, lastScraped, dateadded, artistid)) # Update artwork - artwork.addArtwork(artworks, artistid, "artist", kodicursor) + artwork.addArtwork(artworks, artistid, v.KODI_TYPE_ARTIST, kodicursor) @CatchExceptions(warnuser=True) - def add_updateAlbum(self, item, viewtag=None, viewid=None): + def add_updateAlbum(self, item, viewtag=None, viewid=None, children=None): + """ + children: list of child xml's, so in this case songs + """ kodicursor = self.kodicursor plex_db = self.plex_db artwork = self.artwork @@ -1387,21 +1388,21 @@ class Music(Items): # musicBrainzId = API.getProvider('MusicBrainzAlbum') musicBrainzId = None year = API.getYear() - genres = API.getGenres() - genre = API.joinList(genres) + self.genres = API.getGenres() + self.genre = ' / '.join(self.genres) bio = API.getPlot() rating = userdata['UserRating'] studio = API.getMusicStudio() - # artists = item['AlbumArtists'] - # if not artists: - # artists = item['ArtistItems'] - # artistname = [] - # for artist in artists: - # artistname.append(artist['Name']) artistname = item.attrib.get('parentTitle') if not artistname: artistname = item.attrib.get('originalTitle') - + # See if we have a compilation - Plex does NOT feature a compilation + # flag for albums + self.compilation = 0 + for child in children: + if child.attrib.get('originalTitle') is not None: + self.compilation = 1 + break # Associate artwork artworks = API.getAllArtwork(parentInfo=True) thumb = artworks['Primary'] @@ -1433,56 +1434,54 @@ class Music(Items): # Process the album info if v.KODIVERSION >= 17: # Kodi Krypton - query = ' '.join(( - - "UPDATE album", - "SET strArtists = ?, iYear = ?, strGenres = ?, strReview = ?, strImage = ?,", - "iUserrating = ?, lastScraped = ?, strReleaseType = ?, " - "strLabel = ? ", - "WHERE idAlbum = ?" - )) - kodicursor.execute(query, (artistname, year, genre, bio, thumb, - rating, lastScraped, "album", studio, - albumid)) + query = ''' + UPDATE album + SET strArtists = ?, iYear = ?, strGenres = ?, strReview = ?, + strImage = ?, iUserrating = ?, lastScraped = ?, + strReleaseType = ?, strLabel = ?, bCompilation = ? + WHERE idAlbum = ? + ''' + kodicursor.execute(query, (artistname, year, self.genre, bio, + thumb, rating, lastScraped, + v.KODI_TYPE_ALBUM, studio, + self.compilation, albumid)) elif v.KODIVERSION == 16: # Kodi Jarvis - query = ' '.join(( - - "UPDATE album", - "SET strArtists = ?, iYear = ?, strGenres = ?, strReview = ?, strImage = ?,", - "iRating = ?, lastScraped = ?, strReleaseType = ?, " - "strLabel = ? ", - "WHERE idAlbum = ?" - )) - kodicursor.execute(query, (artistname, year, genre, bio, thumb, - rating, lastScraped, "album", studio, - albumid)) + query = ''' + UPDATE album + SET strArtists = ?, iYear = ?, strGenres = ?, strReview = ?, + strImage = ?, iRating = ?, lastScraped = ?, + strReleaseType = ?, strLabel = ?, bCompilation = ? + WHERE idAlbum = ? + ''' + kodicursor.execute(query, (artistname, year, self.genre, bio, + thumb, rating, lastScraped, + v.KODI_TYPE_ALBUM, studio, + self.compilation, albumid)) elif v.KODIVERSION == 15: # Kodi Isengard - query = ' '.join(( - - "UPDATE album", - "SET strArtists = ?, iYear = ?, strGenres = ?, strReview = ?, strImage = ?,", - "iRating = ?, lastScraped = ?, dateAdded = ?, " - "strReleaseType = ?, strLabel = ? ", - "WHERE idAlbum = ?" - )) - kodicursor.execute(query, (artistname, year, genre, bio, thumb, - rating, lastScraped, dateadded, - "album", studio, albumid)) + query = ''' + UPDATE album + SET strArtists = ?, iYear = ?, strGenres = ?, strReview = ?, + strImage = ?, iRating = ?, lastScraped = ?, dateAdded = ?, + strReleaseType = ?, strLabel = ? + WHERE idAlbum = ? + ''' + kodicursor.execute(query, (artistname, year, self.genre, bio, + thumb, rating, lastScraped, dateadded, + v.KODI_TYPE_ALBUM, studio, albumid)) else: # Kodi Helix - query = ' '.join(( - - "UPDATE album", - "SET strArtists = ?, iYear = ?, strGenres = ?, strReview = ?, strImage = ?,", - "iRating = ?, lastScraped = ?, dateAdded = ?, " - "strLabel = ? ", - "WHERE idAlbum = ?" - )) - kodicursor.execute(query, (artistname, year, genre, bio, thumb, - rating, lastScraped, dateadded, studio, - albumid)) + query = ''' + UPDATE album + SET strArtists = ?, iYear = ?, strGenres = ?, strReview = ?, + strImage = ?, iRating = ?, lastScraped = ?, dateAdded = ?, + strLabel = ? + WHERE idAlbum = ? + ''' + kodicursor.execute(query, (artistname, year, self.genre, bio, + thumb, rating, lastScraped, dateadded, + studio, albumid)) # Associate the parentid for plex reference parentId = item.attrib.get('parentRatingKey') @@ -1496,7 +1495,7 @@ class Music(Items): artist = GetPlexMetadata(parentId) # Item may not be an artist, verification necessary. if artist is not None and artist != 401: - if artist[0].attrib.get('type') == "artist": + if artist[0].attrib.get('type') == v.PLEX_TYPE_ARTIST: # Update with the parentId, for remove reference plex_db.addReference(parentId, v.PLEX_TYPE_ARTIST, @@ -1530,29 +1529,26 @@ class Music(Items): % (artistname, artistid)) # Add artist to album - query = ( - ''' + query = ''' INSERT OR REPLACE INTO album_artist(idArtist, idAlbum, strArtist) - VALUES (?, ?, ?) - ''' - ) + ''' kodicursor.execute(query, (artistid, albumid, artistname)) # Update discography - query = ( - ''' + query = ''' INSERT OR REPLACE INTO discography(idArtist, strAlbum, strYear) - VALUES (?, ?, ?) - ''' - ) + ''' kodicursor.execute(query, (artistid, name, year)) # Update plex reference with parentid plex_db.updateParentId(artistId, albumid) # Add genres - self.kodi_db.addMusicGenres(albumid, genres, "album") + self.kodi_db.addMusicGenres(albumid, self.genres, v.KODI_TYPE_ALBUM) # Update artwork - artwork.addArtwork(artworks, albumid, "album", kodicursor) + artwork.addArtwork(artworks, albumid, v.KODI_TYPE_ALBUM, kodicursor) + # Add all children - all tracks + for child in children: + self.add_updateSong(child, viewtag, viewid) @CatchExceptions(warnuser=True) def add_updateSong(self, item, viewtag=None, viewid=None): @@ -1592,9 +1588,22 @@ class Music(Items): title, sorttitle = API.getTitle() # musicBrainzId = API.getProvider('MusicBrainzTrackId') musicBrainzId = None - genres = API.getGenres() - genre = API.joinList(genres) - artists = item.attrib.get('grandparentTitle') + try: + genres = self.genres + genre = self.genre + except AttributeError: + # No parent album - hence no genre information from Plex + genres = None + genre = None + try: + if self.compilation == 0: + artists = item.attrib.get('grandparentTitle') + else: + artists = item.attrib.get('originalTitle') + except AttributeError: + # compilation not set + artists = item.attrib.get('originalTitle', + item.attrib.get('grandparentTitle')) tracknumber = int(item.attrib.get('index', 0)) disc = int(item.attrib.get('parentIndex', 1)) if disc == 1: @@ -1604,9 +1613,13 @@ class Music(Items): year = API.getYear() resume, duration = API.getRuntime() rating = userdata['UserRating'] - - hasEmbeddedCover = False comment = None + # Moods + moods = [] + for entry in item: + if entry.tag == 'Mood': + moods.append(entry.attrib['tag']) + mood = ' / '.join(moods) # GET THE FILE AND PATH ##### doIndirect = not self.directpath @@ -1644,16 +1657,18 @@ class Music(Items): kodicursor.execute(query, (path, '123', pathid)) # Update the song entry - query = ' '.join(( - "UPDATE song", - "SET idAlbum = ?, strArtists = ?, strGenres = ?, strTitle = ?, iTrack = ?,", - "iDuration = ?, iYear = ?, strFilename = ?, iTimesPlayed = ?, lastplayed = ?,", - "rating = ?, comment = ?", - "WHERE idSong = ?" - )) + query = ''' + UPDATE song + SET idAlbum = ?, strArtists = ?, strGenres = ?, strTitle = ?, + iTrack = ?, iDuration = ?, iYear = ?, strFilename = ?, + iTimesPlayed = ?, lastplayed = ?, rating = ?, comment = ?, + mood = ? + WHERE idSong = ? + ''' kodicursor.execute(query, (albumid, artists, genre, title, track, duration, year, filename, playcount, - dateplayed, rating, comment, songid)) + dateplayed, rating, comment, mood, + songid)) # Update the checksum in plex table plex_db.updateReference(itemid, checksum) @@ -1676,7 +1691,9 @@ class Music(Items): if album_name: log.info("Creating virtual music album for song: %s." % itemid) - albumid = self.kodi_db.addAlbum(album_name, API.getProvider('MusicBrainzAlbum')) + albumid = self.kodi_db.addAlbum( + album_name, + API.getProvider('MusicBrainzAlbum')) plex_db.addReference("%salbum%s" % (itemid, albumid), v.PLEX_TYPE_ALBUM, albumid, @@ -1704,54 +1721,51 @@ class Music(Items): except TypeError: # No album found, create a single's album log.info("Failed to add album. Creating singles.") - kodicursor.execute("select coalesce(max(idAlbum),0) from album") + kodicursor.execute( + "select coalesce(max(idAlbum),0) from album") albumid = kodicursor.fetchone()[0] + 1 if v.KODIVERSION >= 16: # Kodi Jarvis - query = ( - ''' - INSERT INTO album(idAlbum, strGenres, iYear, strReleaseType) - + query = ''' + INSERT INTO album( + idAlbum, strGenres, iYear, strReleaseType) VALUES (?, ?, ?, ?) - ''' - ) - kodicursor.execute(query, (albumid, genre, year, "single")) + ''' + kodicursor.execute(query, + (albumid, genre, year, "single")) elif v.KODIVERSION == 15: # Kodi Isengard - query = ( - ''' - INSERT INTO album(idAlbum, strGenres, iYear, dateAdded, strReleaseType) - + query = ''' + INSERT INTO album( + idAlbum, strGenres, iYear, dateAdded, + strReleaseType) VALUES (?, ?, ?, ?, ?) - ''' - ) - kodicursor.execute(query, (albumid, genre, year, dateadded, "single")) + ''' + kodicursor.execute(query, (albumid, genre, year, + dateadded, "single")) else: # Kodi Helix - query = ( - ''' - INSERT INTO album(idAlbum, strGenres, iYear, dateAdded) - + query = ''' + INSERT INTO album( + idAlbum, strGenres, iYear, dateAdded) VALUES (?, ?, ?, ?) - ''' - ) - kodicursor.execute(query, (albumid, genre, year, dateadded)) + ''' + kodicursor.execute(query, (albumid, genre, year, + dateadded)) # Create the song entry - query = ( - ''' + query = ''' INSERT INTO song( - idSong, idAlbum, idPath, strArtists, strGenres, strTitle, iTrack, - iDuration, iYear, strFileName, strMusicBrainzTrackID, iTimesPlayed, lastplayed, - rating, iStartOffset, iEndOffset) - - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + idSong, idAlbum, idPath, strArtists, strGenres, strTitle, + iTrack, iDuration, iYear, strFileName, + strMusicBrainzTrackID, iTimesPlayed, lastplayed, + rating, iStartOffset, iEndOffset, mood) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ''' - ) kodicursor.execute( query, (songid, albumid, pathid, artists, genre, title, track, duration, year, filename, musicBrainzId, playcount, - dateplayed, rating, 0, 0)) + dateplayed, rating, 0, 0, mood)) # Create the reference in plex table plex_db.addReference(itemid, @@ -1764,14 +1778,11 @@ class Music(Items): view_id=viewid) # Link song to album - query = ( - ''' + query = ''' INSERT OR REPLACE INTO albuminfosong( idAlbumInfoSong, idAlbumInfo, iTrack, strTitle, iDuration) - VALUES (?, ?, ?, ?, ?) - ''' - ) + ''' kodicursor.execute(query, (songid, albumid, track, title, duration)) # Link song to artists @@ -1799,29 +1810,27 @@ class Music(Items): finally: if v.KODIVERSION >= 17: # Kodi Krypton - query = ( - ''' - INSERT OR REPLACE INTO song_artist(idArtist, idSong, idRole, iOrder, strArtist) + query = ''' + INSERT OR REPLACE INTO song_artist( + idArtist, idSong, idRole, iOrder, strArtist) VALUES (?, ?, ?, ?, ?) - ''' - ) - kodicursor.execute(query,(artistid, songid, 1, index, artist_name)) + ''' + kodicursor.execute(query, (artistid, songid, 1, index, + artist_name)) # May want to look into only doing this once? - query = ( - ''' + query = ''' INSERT OR REPLACE INTO role(idRole, strRole) VALUES (?, ?) - ''' - ) + ''' kodicursor.execute(query, (1, 'Composer')) else: - query = ( - ''' - INSERT OR REPLACE INTO song_artist(idArtist, idSong, iOrder, strArtist) + query = ''' + INSERT OR REPLACE INTO song_artist( + idArtist, idSong, iOrder, strArtist) VALUES (?, ?, ?, ?) - ''' - ) - kodicursor.execute(query, (artistid, songid, index, artist_name)) + ''' + kodicursor.execute(query, (artistid, songid, index, + artist_name)) # Verify if album artist exists album_artists = [] @@ -1843,31 +1852,28 @@ class Music(Items): artist_edb = plex_db.getItem_byId(artist_eid) artistid = artist_edb[0] finally: - query = ( - ''' - INSERT OR REPLACE INTO album_artist(idArtist, idAlbum, strArtist) + query = ''' + INSERT OR REPLACE INTO album_artist( + idArtist, idAlbum, strArtist) VALUES (?, ?, ?) - ''' - ) + ''' kodicursor.execute(query, (artistid, albumid, artist_name)) # Update discography if item.get('Album'): - query = ( - ''' - INSERT OR REPLACE INTO discography(idArtist, strAlbum, strYear) + query = ''' + INSERT OR REPLACE INTO discography( + idArtist, strAlbum, strYear) VALUES (?, ?, ?) - ''' - ) + ''' kodicursor.execute(query, (artistid, item['Album'], 0)) # else: if False: album_artists = " / ".join(album_artists) - query = ' '.join(( - - "SELECT strArtists", - "FROM album", - "WHERE idAlbum = ?" - )) + query = ''' + SELECT strArtists + FROM album + WHERE idAlbum = ? + ''' kodicursor.execute(query, (albumid,)) result = kodicursor.fetchone() if result and result[0] != album_artists: @@ -1886,18 +1892,16 @@ class Music(Items): kodicursor.execute(query, (album_artists, albumid)) # Add genres - self.kodi_db.addMusicGenres(songid, genres, "song") + if genres: + self.kodi_db.addMusicGenres(songid, genres, v.KODI_TYPE_SONG) # Update artwork allart = API.getAllArtwork(parentInfo=True) - if hasEmbeddedCover: - allart["Primary"] = "image://music@" + artwork.single_urlencode( playurl ) - artwork.addArtwork(allart, songid, "song", kodicursor) + artwork.addArtwork(allart, songid, v.KODI_TYPE_SONG, kodicursor) - # if item.get('AlbumId') is None: if item.get('parentKey') is None: # Update album artwork - artwork.addArtwork(allart, albumid, "album", kodicursor) + artwork.addArtwork(allart, albumid, v.KODI_TYPE_ALBUM, kodicursor) def remove(self, itemid): # Remove kodiid, fileid, pathid, plex reference diff --git a/resources/lib/library_sync/__init__.py b/resources/lib/library_sync/__init__.py new file mode 100644 index 00000000..b93054b3 --- /dev/null +++ b/resources/lib/library_sync/__init__.py @@ -0,0 +1 @@ +# Dummy file to make this directory a package. diff --git a/resources/lib/library_sync/fanart.py b/resources/lib/library_sync/fanart.py new file mode 100644 index 00000000..7f9fc074 --- /dev/null +++ b/resources/lib/library_sync/fanart.py @@ -0,0 +1,88 @@ +# -*- coding: utf-8 -*- +from logging import getLogger +from threading import Thread +from Queue import Empty + +from xbmc import sleep + +from utils import ThreadMethodsAdditionalStop, ThreadMethods, window, \ + ThreadMethodsAdditionalSuspend +import plexdb_functions as plexdb +import itemtypes +import variables as v + +############################################################################### + +log = getLogger("PLEX."+__name__) + +############################################################################### + + +@ThreadMethodsAdditionalSuspend('suspend_LibraryThread') +@ThreadMethodsAdditionalStop('plex_shouldStop') +@ThreadMethods +class Process_Fanart_Thread(Thread): + """ + Threaded download of additional fanart in the background + + Input: + queue Queue.Queue() object that you will need to fill with + dicts of the following form: + { + 'plex_id': the Plex id as a string + 'plex_type': the Plex media type, e.g. 'movie' + 'refresh': True/False if True, will overwrite any 3rd party + fanart. If False, will only get missing + } + """ + def __init__(self, queue): + self.queue = queue + Thread.__init__(self) + + def run(self): + """ + Catch all exceptions and log them + """ + try: + self.__run() + except Exception as e: + log.error('Exception %s' % e) + import traceback + log.error("Traceback:\n%s" % traceback.format_exc()) + + def __run(self): + """ + Do the work + """ + log.debug("---===### Starting FanartSync ###===---") + threadStopped = self.threadStopped + threadSuspended = self.threadSuspended + queue = self.queue + while not threadStopped(): + # In the event the server goes offline + while threadSuspended() or window('plex_dbScan'): + # Set in service.py + if threadStopped(): + # Abort was requested while waiting. We should exit + log.info("---===### Stopped FanartSync ###===---") + return + sleep(1000) + # grabs Plex item from queue + try: + item = queue.get(block=False) + except Empty: + sleep(200) + continue + + log.debug('Get additional fanart for Plex id %s' % item['plex_id']) + with getattr(itemtypes, + v.ITEMTYPE_FROM_PLEXTYPE[item['plex_type']])() as cls: + result = cls.getfanart(item['plex_id'], + refresh=item['refresh']) + if result is True: + log.debug('Done getting fanart for Plex id %s' + % item['plex_id']) + with plexdb.Get_Plex_DB() as plex_db: + plex_db.set_fanart_synched(item['plex_id']) + queue.task_done() + log.debug("---===### Stopped FanartSync ###===---") diff --git a/resources/lib/library_sync/get_metadata.py b/resources/lib/library_sync/get_metadata.py new file mode 100644 index 00000000..5fd25859 --- /dev/null +++ b/resources/lib/library_sync/get_metadata.py @@ -0,0 +1,140 @@ +# -*- coding: utf-8 -*- +from logging import getLogger +from threading import Thread +from Queue import Empty + +from xbmc import sleep + +from utils import ThreadMethodsAdditionalStop, ThreadMethods, window +from PlexFunctions import GetPlexMetadata, GetAllPlexChildren +import sync_info + +############################################################################### + +log = getLogger("PLEX."+__name__) + +############################################################################### + + +@ThreadMethodsAdditionalStop('suspend_LibraryThread') +@ThreadMethods +class Threaded_Get_Metadata(Thread): + """ + Threaded download of Plex XML metadata for a certain library item. + Fills the out_queue with the downloaded etree XML objects + + Input: + queue Queue.Queue() object that you'll need to fill up + with Plex itemIds + out_queue Queue() object where this thread will store + the downloaded metadata XMLs as etree objects + """ + def __init__(self, queue, out_queue): + self.queue = queue + self.out_queue = out_queue + Thread.__init__(self) + + def terminate_now(self): + """ + Needed to terminate this thread, because there might be items left in + the queue which could cause other threads to hang + """ + while not self.queue.empty(): + # Still try because remaining item might have been taken + try: + self.queue.get(block=False) + except Empty: + sleep(10) + continue + else: + self.queue.task_done() + if self.threadStopped(): + # Shutdown from outside requested; purge out_queue as well + while not self.out_queue.empty(): + # Still try because remaining item might have been taken + try: + self.out_queue.get(block=False) + except Empty: + sleep(10) + continue + else: + self.out_queue.task_done() + + def run(self): + """ + Catch all exceptions and log them + """ + try: + self.__run() + except Exception as e: + log.error('Exception %s' % e) + import traceback + log.error("Traceback:\n%s" % traceback.format_exc()) + + def __run(self): + """ + Do the work + """ + log.debug('Starting get metadata thread') + # cache local variables because it's faster + queue = self.queue + out_queue = self.out_queue + threadStopped = self.threadStopped + while threadStopped() is False: + # grabs Plex item from queue + try: + item = queue.get(block=False) + # Empty queue + except Empty: + sleep(20) + continue + # Download Metadata + xml = GetPlexMetadata(item['itemId']) + if xml is None: + # Did not receive a valid XML - skip that item for now + log.error("Could not get metadata for %s. Skipping that item " + "for now" % item['itemId']) + # Increase BOTH counters - since metadata won't be processed + with sync_info.LOCK: + sync_info.GET_METADATA_COUNT += 1 + sync_info.PROCESS_METADATA_COUNT += 1 + queue.task_done() + continue + elif xml == 401: + log.error('HTTP 401 returned by PMS. Too much strain? ' + 'Cancelling sync for now') + window('plex_scancrashed', value='401') + # Kill remaining items in queue (for main thread to cont.) + queue.task_done() + break + + item['XML'] = xml + if item.get('get_children') is True: + children_xml = GetAllPlexChildren(item['itemId']) + try: + children_xml[0].attrib + except (TypeError, IndexError, AttributeError): + log.error('Could not get children for Plex id %s' + % item['itemId']) + else: + item['children'] = [] + for child in children_xml: + child_xml = GetPlexMetadata(child.attrib['ratingKey']) + try: + child_xml[0].attrib + except (TypeError, IndexError, AttributeError): + log.error('Could not get child for Plex id %s' + % child.attrib['ratingKey']) + else: + item['children'].append(child_xml[0]) + + # place item into out queue + out_queue.put(item) + # Keep track of where we are at + with sync_info.LOCK: + sync_info.GET_METADATA_COUNT += 1 + # signals to queue job is done + queue.task_done() + # Empty queue in case PKC was shut down (main thread hangs otherwise) + self.terminate_now() + log.debug('Get metadata thread terminated') diff --git a/resources/lib/library_sync/process_metadata.py b/resources/lib/library_sync/process_metadata.py new file mode 100644 index 00000000..e6765b41 --- /dev/null +++ b/resources/lib/library_sync/process_metadata.py @@ -0,0 +1,104 @@ +# -*- coding: utf-8 -*- +from logging import getLogger +from threading import Thread +from Queue import Empty + +from xbmc import sleep + +from utils import ThreadMethodsAdditionalStop, ThreadMethods +import itemtypes +import sync_info + +############################################################################### + +log = getLogger("PLEX."+__name__) + +############################################################################### + + +@ThreadMethodsAdditionalStop('suspend_LibraryThread') +@ThreadMethods +class Threaded_Process_Metadata(Thread): + """ + Not yet implemented for more than 1 thread - if ever. Only to be called by + ONE thread! + Processes the XML metadata in the queue + + Input: + queue: Queue.Queue() object that you'll need to fill up with + the downloaded XML eTree objects + item_type: as used to call functions in itemtypes.py e.g. 'Movies' => + itemtypes.Movies() + """ + def __init__(self, queue, item_type): + self.queue = queue + self.item_type = item_type + Thread.__init__(self) + + def terminate_now(self): + """ + Needed to terminate this thread, because there might be items left in + the queue which could cause other threads to hang + """ + while not self.queue.empty(): + # Still try because remaining item might have been taken + try: + self.queue.get(block=False) + except Empty: + sleep(10) + continue + else: + self.queue.task_done() + + def run(self): + """ + Catch all exceptions and log them + """ + try: + self.__run() + except Exception as e: + log.error('Exception %s' % e) + import traceback + log.error("Traceback:\n%s" % traceback.format_exc()) + + def __run(self): + """ + Do the work + """ + log.debug('Processing thread started') + # Constructs the method name, e.g. itemtypes.Movies + item_fct = getattr(itemtypes, self.item_type) + # cache local variables because it's faster + queue = self.queue + threadStopped = self.threadStopped + with item_fct() as item_class: + while threadStopped() is False: + # grabs item from queue + try: + item = queue.get(block=False) + except Empty: + sleep(20) + continue + # Do the work + item_method = getattr(item_class, item['method']) + if item.get('children') is not None: + item_method(item['XML'][0], + viewtag=item['viewName'], + viewid=item['viewId'], + children=item['children']) + else: + item_method(item['XML'][0], + viewtag=item['viewName'], + viewid=item['viewId']) + # Keep track of where we are at + try: + log.debug('found child: %s' + % item['children'].attrib) + except: + pass + with sync_info.LOCK: + sync_info.PROCESS_METADATA_COUNT += 1 + sync_info.PROCESSING_VIEW_NAME = item['title'] + queue.task_done() + self.terminate_now() + log.debug('Processing thread terminated') diff --git a/resources/lib/library_sync/sync_info.py b/resources/lib/library_sync/sync_info.py new file mode 100644 index 00000000..df14e433 --- /dev/null +++ b/resources/lib/library_sync/sync_info.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- +from logging import getLogger +from threading import Thread, Lock + +from xbmc import sleep + +from utils import ThreadMethodsAdditionalStop, ThreadMethods, language as lang + +############################################################################### + +log = getLogger("PLEX."+__name__) + +GET_METADATA_COUNT = 0 +PROCESS_METADATA_COUNT = 0 +PROCESSING_VIEW_NAME = '' +LOCK = Lock() + +############################################################################### + + +@ThreadMethodsAdditionalStop('suspend_LibraryThread') +@ThreadMethods +class Threaded_Show_Sync_Info(Thread): + """ + Threaded class to show the Kodi statusbar of the metadata download. + + Input: + dialog xbmcgui.DialogProgressBG() object to show progress + total: Total number of items to get + """ + def __init__(self, dialog, total, item_type): + self.total = total + self.dialog = dialog + self.item_type = item_type + Thread.__init__(self) + + def run(self): + """ + Catch all exceptions and log them + """ + try: + self.__run() + except Exception as e: + log.error('Exception %s' % e) + import traceback + log.error("Traceback:\n%s" % traceback.format_exc()) + + def __run(self): + """ + Do the work + """ + log.debug('Show sync info thread started') + # cache local variables because it's faster + total = self.total + dialog = self.dialog + threadStopped = self.threadStopped + dialog.create("%s: Sync %s: %s items" + % (lang(29999), self.item_type, str(total)), + "Starting") + + total = 2 * total + totalProgress = 0 + while threadStopped() is False: + with LOCK: + get_progress = GET_METADATA_COUNT + process_progress = PROCESS_METADATA_COUNT + viewName = PROCESSING_VIEW_NAME + totalProgress = get_progress + process_progress + try: + percentage = int(float(totalProgress) / float(total)*100.0) + except ZeroDivisionError: + percentage = 0 + dialog.update(percentage, + message="%s downloaded. %s processed: %s" + % (get_progress, + process_progress, + viewName)) + # Sleep for x milliseconds + sleep(200) + dialog.close() + log.debug('Show sync info thread terminated') diff --git a/resources/lib/librarysync.py b/resources/lib/librarysync.py index 2db99558..c22074f7 100644 --- a/resources/lib/librarysync.py +++ b/resources/lib/librarysync.py @@ -3,7 +3,7 @@ ############################################################################### import logging -from threading import Thread, Lock +from threading import Thread import Queue from random import shuffle @@ -28,6 +28,10 @@ import variables as v from PlexFunctions import GetPlexMetadata, GetAllPlexLeaves, scrobble, \ GetPlexSectionResults, GetAllPlexChildren, GetPMSStatus import PlexAPI +from library_sync.get_metadata import Threaded_Get_Metadata +from library_sync.process_metadata import Threaded_Process_Metadata +import library_sync.sync_info as sync_info +from library_sync.fanart import Process_Fanart_Thread ############################################################################### @@ -36,282 +40,6 @@ log = logging.getLogger("PLEX."+__name__) ############################################################################### -@ThreadMethodsAdditionalStop('suspend_LibraryThread') -@ThreadMethods -class ThreadedGetMetadata(Thread): - """ - Threaded download of Plex XML metadata for a certain library item. - Fills the out_queue with the downloaded etree XML objects - - Input: - queue Queue.Queue() object that you'll need to fill up - with Plex itemIds - out_queue Queue() object where this thread will store - the downloaded metadata XMLs as etree objects - lock Lock(), used for counting where we are - """ - def __init__(self, queue, out_queue, lock, processlock): - self.queue = queue - self.out_queue = out_queue - self.lock = lock - self.processlock = processlock - Thread.__init__(self) - - def terminateNow(self): - while not self.queue.empty(): - # Still try because remaining item might have been taken - try: - self.queue.get(block=False) - except Queue.Empty: - xbmc.sleep(10) - continue - else: - self.queue.task_done() - if self.threadStopped(): - # Shutdown from outside requested; purge out_queue as well - while not self.out_queue.empty(): - # Still try because remaining item might have been taken - try: - self.out_queue.get(block=False) - except Queue.Empty: - xbmc.sleep(10) - continue - else: - self.out_queue.task_done() - - def run(self): - # cache local variables because it's faster - queue = self.queue - out_queue = self.out_queue - lock = self.lock - processlock = self.processlock - threadStopped = self.threadStopped - global getMetadataCount - global processMetadataCount - while threadStopped() is False: - # grabs Plex item from queue - try: - updateItem = queue.get(block=False) - # Empty queue - except Queue.Empty: - xbmc.sleep(10) - continue - # Download Metadata - plexXML = GetPlexMetadata(updateItem['itemId']) - if plexXML is None: - # Did not receive a valid XML - skip that item for now - log.warn("Could not get metadata for %s. Skipping that item " - "for now" % updateItem['itemId']) - # Increase BOTH counters - since metadata won't be processed - with lock: - getMetadataCount += 1 - with processlock: - processMetadataCount += 1 - queue.task_done() - continue - elif plexXML == 401: - log.warn('HTTP 401 returned by PMS. Too much strain? ' - 'Cancelling sync for now') - window('plex_scancrashed', value='401') - # Kill remaining items in queue (for main thread to cont.) - queue.task_done() - break - - updateItem['XML'] = plexXML - # place item into out queue - out_queue.put(updateItem) - # Keep track of where we are at - with lock: - getMetadataCount += 1 - # signals to queue job is done - queue.task_done() - # Empty queue in case PKC was shut down (main thread hangs otherwise) - self.terminateNow() - log.debug('Download thread terminated') - - -@ThreadMethodsAdditionalStop('suspend_LibraryThread') -@ThreadMethods -class ThreadedProcessMetadata(Thread): - """ - Not yet implemented - if ever. Only to be called by ONE thread! - Processes the XML metadata in the queue - - Input: - queue: Queue.Queue() object that you'll need to fill up with - the downloaded XML eTree objects - itemType: as used to call functions in itemtypes.py - e.g. 'Movies' => itemtypes.Movies() - lock: Lock(), used for counting where we are - """ - def __init__(self, queue, itemType, lock): - self.queue = queue - self.lock = lock - self.itemType = itemType - Thread.__init__(self) - - def terminateNow(self): - while not self.queue.empty(): - # Still try because remaining item might have been taken - try: - self.queue.get(block=False) - except Queue.Empty: - xbmc.sleep(10) - continue - else: - self.queue.task_done() - - def run(self): - # Constructs the method name, e.g. itemtypes.Movies - itemFkt = getattr(itemtypes, self.itemType) - # cache local variables because it's faster - queue = self.queue - lock = self.lock - threadStopped = self.threadStopped - global processMetadataCount - global processingViewName - with itemFkt() as item: - while threadStopped() is False: - # grabs item from queue - try: - updateItem = queue.get(block=False) - except Queue.Empty: - xbmc.sleep(10) - continue - # Do the work - plexitem = updateItem['XML'] - method = updateItem['method'] - viewName = updateItem['viewName'] - viewId = updateItem['viewId'] - title = updateItem['title'] - itemSubFkt = getattr(item, method) - # Get the one child entry in the xml and process - for child in plexitem: - itemSubFkt(child, - viewtag=viewName, - viewid=viewId) - # Keep track of where we are at - with lock: - processMetadataCount += 1 - processingViewName = title - # signals to queue job is done - queue.task_done() - # Empty queue in case PKC was shut down (main thread hangs otherwise) - self.terminateNow() - log.debug('Processing thread terminated') - - -@ThreadMethodsAdditionalStop('suspend_LibraryThread') -@ThreadMethods -class ThreadedShowSyncInfo(Thread): - """ - Threaded class to show the Kodi statusbar of the metadata download. - - Input: - dialog xbmcgui.DialogProgressBG() object to show progress - locks = [downloadLock, processLock] Locks() to the other threads - total: Total number of items to get - """ - def __init__(self, dialog, locks, total, itemType): - self.locks = locks - self.total = total - self.dialog = dialog - self.itemType = itemType - Thread.__init__(self) - - def run(self): - # cache local variables because it's faster - total = self.total - dialog = self.dialog - threadStopped = self.threadStopped - downloadLock = self.locks[0] - processLock = self.locks[1] - dialog.create("%s: Sync %s: %s items" - % (lang(29999), self.itemType, str(total)), - "Starting") - global getMetadataCount - global processMetadataCount - global processingViewName - total = 2 * total - totalProgress = 0 - while threadStopped() is False: - with downloadLock: - getMetadataProgress = getMetadataCount - with processLock: - processMetadataProgress = processMetadataCount - viewName = processingViewName - totalProgress = getMetadataProgress + processMetadataProgress - try: - percentage = int(float(totalProgress) / float(total)*100.0) - except ZeroDivisionError: - percentage = 0 - dialog.update(percentage, - message="%s downloaded. %s processed: %s" - % (getMetadataProgress, - processMetadataProgress, - viewName)) - # Sleep for x milliseconds - xbmc.sleep(200) - dialog.close() - log.debug('Dialog Infobox thread terminated') - - -@ThreadMethodsAdditionalSuspend('suspend_LibraryThread') -@ThreadMethodsAdditionalStop('plex_shouldStop') -@ThreadMethods -class ProcessFanartThread(Thread): - """ - Threaded download of additional fanart in the background - - Input: - queue Queue.Queue() object that you will need to fill with - dicts of the following form: - { - 'plex_id': the Plex id as a string - 'plex_type': the Plex media type, e.g. 'movie' - 'refresh': True/False if True, will overwrite any 3rd party - fanart. If False, will only get missing - } - """ - def __init__(self, queue): - self.queue = queue - Thread.__init__(self) - - def run(self): - threadStopped = self.threadStopped - threadSuspended = self.threadSuspended - queue = self.queue - log.info("---===### Starting FanartSync ###===---") - while not threadStopped(): - # In the event the server goes offline - while threadSuspended() or window('plex_dbScan'): - # Set in service.py - if threadStopped(): - # Abort was requested while waiting. We should exit - log.info("---===### Stopped FanartSync ###===---") - return - xbmc.sleep(1000) - # grabs Plex item from queue - try: - item = queue.get(block=False) - except Queue.Empty: - xbmc.sleep(200) - continue - - log.debug('Get additional fanart for Plex id %s' % item['plex_id']) - with getattr(itemtypes, - v.ITEMTYPE_FROM_PLEXTYPE[item['plex_type']])() as cls: - result = cls.getfanart(item['plex_id'], - refresh=item['refresh']) - if result is True: - log.debug('Done getting fanart for Plex id %s' - % item['plex_id']) - with plexdb.Get_Plex_DB() as plex_db: - plex_db.set_fanart_synched(item['plex_id']) - queue.task_done() - log.info("---===### Stopped FanartSync ###===---") - - @ThreadMethodsAdditionalSuspend('suspend_LibraryThread') @ThreadMethodsAdditionalStop('plex_shouldStop') @ThreadMethods @@ -330,7 +58,7 @@ class LibrarySync(Thread): self.sessionKeys = [] self.fanartqueue = Queue.Queue() if settings('FanartTV') == 'true': - self.fanartthread = ProcessFanartThread(self.fanartqueue) + self.fanartthread = Process_Fanart_Thread(self.fanartqueue) # How long should we wait at least to process new/changed PMS items? self.saftyMargin = int(settings('backgroundsync_saftyMargin')) @@ -700,8 +428,8 @@ class LibrarySync(Thread): viewid=folderid, delete=True) # Added new playlist - if (foldername not in playlists and - mediatype in (v.PLEX_TYPE_MOVIE, v.PLEX_TYPE_SHOW)): + if (foldername not in playlists and mediatype in + (v.PLEX_TYPE_MOVIE, v.PLEX_TYPE_SHOW)): playlistXSP(mediatype, foldername, folderid, @@ -726,8 +454,8 @@ class LibrarySync(Thread): else: # Validate the playlist exists or recreate it if mediatype != v.PLEX_TYPE_ARTIST: - if (foldername not in playlists and - mediatype in (v.PLEX_TYPE_MOVIE, v.PLEX_TYPE_SHOW)): + if (foldername not in playlists and mediatype in + (v.PLEX_TYPE_MOVIE, v.PLEX_TYPE_SHOW)): playlistXSP(mediatype, foldername, folderid, @@ -777,7 +505,8 @@ class LibrarySync(Thread): for view in sections: itemType = view.attrib['type'] - if itemType in (v.PLEX_TYPE_MOVIE, v.PLEX_TYPE_SHOW, v.PLEX_TYPE_PHOTO): # NOT artist for now + if (itemType in + (v.PLEX_TYPE_MOVIE, v.PLEX_TYPE_SHOW, v.PLEX_TYPE_PHOTO)): self.sorted_views.append(view.attrib['title']) log.debug('Sorted views: %s' % self.sorted_views) @@ -859,7 +588,8 @@ class LibrarySync(Thread): with itemtypes.Music() as music: music.remove(item['plex_id']) - def GetUpdatelist(self, xml, itemType, method, viewName, viewId): + def GetUpdatelist(self, xml, itemType, method, viewName, viewId, + get_children=False): """ THIS METHOD NEEDS TO BE FAST! => e.g. no API calls @@ -872,6 +602,8 @@ class LibrarySync(Thread): see itemtypes.py viewName: Name of the Plex view (e.g. 'My TV shows') viewId: Id/Key of Plex library (e.g. '1') + get_children: will get Plex children of the item if True, + e.g. for music albums Output: self.updatelist, self.allPlexElementsId self.updatelist APPENDED(!!) list itemids (Plex Keys as @@ -906,7 +638,8 @@ class LibrarySync(Thread): 'viewName': viewName, 'viewId': viewId, 'title': item.attrib.get('title', 'Missing Title'), - 'mediaType': item.attrib.get('type') + 'mediaType': item.attrib.get('type'), + 'get_children': get_children }) self.just_processed[itemId] = now return @@ -932,7 +665,8 @@ class LibrarySync(Thread): 'viewName': viewName, 'viewId': viewId, 'title': item.attrib.get('title', 'Missing Title'), - 'mediaType': item.attrib.get('type') + 'mediaType': item.attrib.get('type'), + 'get_children': get_children }) self.just_processed[itemId] = now else: @@ -951,7 +685,8 @@ class LibrarySync(Thread): 'viewName': viewName, 'viewId': viewId, 'title': item.attrib.get('title', 'Missing Title'), - 'mediaType': item.attrib.get('type') + 'mediaType': item.attrib.get('type'), + 'get_children': get_children }) self.just_processed[itemId] = now @@ -976,49 +711,38 @@ class LibrarySync(Thread): log.info("Starting sync threads") getMetadataQueue = Queue.Queue() processMetadataQueue = Queue.Queue(maxsize=100) - getMetadataLock = Lock() - processMetadataLock = Lock() # To keep track - global getMetadataCount - getMetadataCount = 0 - global processMetadataCount - processMetadataCount = 0 - global processingViewName - processingViewName = '' + sync_info.GET_METADATA_COUNT = 0 + sync_info.PROCESS_METADATA_COUNT = 0 + sync_info.PROCESSING_VIEW_NAME = '' # Populate queue: GetMetadata for updateItem in self.updatelist: getMetadataQueue.put(updateItem) # Spawn GetMetadata threads for downloading threads = [] for i in range(min(self.syncThreadNumber, itemNumber)): - thread = ThreadedGetMetadata(getMetadataQueue, - processMetadataQueue, - getMetadataLock, - processMetadataLock) + thread = Threaded_Get_Metadata(getMetadataQueue, + processMetadataQueue) thread.setDaemon(True) thread.start() threads.append(thread) log.info("%s download threads spawned" % len(threads)) # Spawn one more thread to process Metadata, once downloaded - thread = ThreadedProcessMetadata(processMetadataQueue, - itemType, - processMetadataLock) + thread = Threaded_Process_Metadata(processMetadataQueue, + itemType) thread.setDaemon(True) thread.start() threads.append(thread) - log.info("Processing thread spawned") # Start one thread to show sync progress ONLY for new PMS items if self.new_items_only is True and window('dbSyncIndicator') == 'true': dialog = xbmcgui.DialogProgressBG() - thread = ThreadedShowSyncInfo( + thread = sync_info.Threaded_Show_Sync_Info( dialog, - [getMetadataLock, processMetadataLock], itemNumber, itemType) thread.setDaemon(True) thread.start() threads.append(thread) - log.info("Kodi Infobox thread spawned") # Wait until finished getMetadataQueue.join() @@ -1322,6 +1046,8 @@ class LibrarySync(Thread): return True def ProcessMusic(self, views, kind, urlArgs, method): + # For albums, we need to look at the album's songs simultaneously + get_children = True if kind == v.PLEX_TYPE_ALBUM else False # Get a list of items already existing in Kodi db if self.compare: with plexdb.Get_Plex_DB() as plex_db: @@ -1347,7 +1073,8 @@ class LibrarySync(Thread): 'Music', method, view['name'], - view['id']) + view['id'], + get_children=get_children) if self.compare: # Manual sync, process deletes with itemtypes.Music() as Music: