From 48a530b49af3a05b1277b698dc7fb319cf24963f Mon Sep 17 00:00:00 2001 From: croneter Date: Sun, 4 Nov 2018 16:53:42 +0100 Subject: [PATCH] More hacking --- resources/lib/itemtypes/common.py | 43 +++-- resources/lib/library_sync/full_sync.py | 157 ++++++++---------- .../lib/library_sync/process_metadata.py | 3 +- resources/lib/library_sync/time.py | 2 +- resources/lib/plex_functions.py | 23 ++- resources/lib/utils.py | 8 + 6 files changed, 117 insertions(+), 119 deletions(-) diff --git a/resources/lib/itemtypes/common.py b/resources/lib/itemtypes/common.py index 7a352ba0..e1e481fc 100644 --- a/resources/lib/itemtypes/common.py +++ b/resources/lib/itemtypes/common.py @@ -80,34 +80,29 @@ class ItemBase(object): kodi_type, self.kodicursor) - def updateUserdata(self, xml): + def update_userdata(self, xml_element, plex_type): """ Updates the Kodi watched state of the item from PMS. Also retrieves Plex resume points for movies in progress. - - viewtag and viewid only serve as dummies """ - for mediaitem in xml: - api = API(mediaitem) - # Get key and db entry on the Kodi db side - db_item = self.plexdb.getItem_byId(api.plex_id()) - try: - fileid = db_item[1] - except TypeError: - continue - # Grab the user's viewcount, resume points etc. from PMS' answer - userdata = api.userdata() - # Write to Kodi DB - self.kodi_db.set_resume(fileid, - userdata['Resume'], - userdata['Runtime'], - userdata['PlayCount'], - userdata['LastPlayedDate'], - api.plex_type()) - if v.KODIVERSION >= 17: - self.kodi_db.update_userrating(db_item[0], - db_item[4], - userdata['UserRating']) + api = API(xml_element) + # Get key and db entry on the Kodi db side + db_item = self.plexdb.item_by_id(api.plex_id(), plex_type) + if not db_item: + LOG.error('Item not yet synced: %s', xml_element.attrib) + return + # Grab the user's viewcount, resume points etc. from PMS' answer + userdata = api.userdata() + # Write to Kodi DB + self.kodi_db.set_resume(db_item['kodi_fileid'], + userdata['Resume'], + userdata['Runtime'], + userdata['PlayCount'], + userdata['LastPlayedDate'], + plex_type) + self.kodi_db.update_userrating(db_item['kodi_id'], + db_item['kodi_type'], + userdata['UserRating']) def update_playstate(self, mark_played, view_count, resume, duration, kodi_fileid, lastViewedAt, plex_type): diff --git a/resources/lib/library_sync/full_sync.py b/resources/lib/library_sync/full_sync.py index 9baea92f..3ca0247b 100644 --- a/resources/lib/library_sync/full_sync.py +++ b/resources/lib/library_sync/full_sync.py @@ -9,7 +9,6 @@ from .. import utils, backgroundthread, variables as v, state from .. import plex_functions as PF, itemtypes from ..plex_db import PlexDB - if (v.PLATFORM != 'Microsoft UWP' and utils.settings('enablePlaylistSync') == 'true'): # Xbox cannot use watchdog, a dependency for PKC playlist features @@ -18,7 +17,6 @@ if (v.PLATFORM != 'Microsoft UWP' and else: PLAYLIST_SYNC_ENABLED = False - LOG = getLogger('PLEX.sync.full_sync') @@ -33,84 +31,63 @@ class FullSync(backgroundthread.KillableThread, common.libsync_mixin): self.show_dialog = show_dialog self.queue = None self.process_thread = None - self.last_sync = None + self.current_sync = None self.plexdb = None self.plex_type = None + self.section_type = None self.processing_thread = None self.install_sync_done = utils.settings('SyncInstallRunDone') == 'true' super(FullSync, self).__init__() - def plex_update_watched(self, viewId, item_class, lastViewedAt=None, - updatedAt=None): - """ - YET to implement - - Updates plex elements' view status ('watched' or 'unwatched') and - also updates resume times. - This is done by downloading one XML for ALL elements with viewId - """ - if self.new_items_only is False: - # Only do this once for fullsync: the first run where new items are - # added to Kodi - return - xml = PF.GetAllPlexLeaves(viewId, - lastViewedAt=lastViewedAt, - updatedAt=updatedAt) - # Return if there are no items in PMS reply - it's faster - try: - xml[0].attrib - except (TypeError, AttributeError, IndexError): - LOG.error('Error updating watch status. Could not get viewId: ' - '%s of item_class %s with lastViewedAt: %s, updatedAt: ' - '%s', viewId, item_class, lastViewedAt, updatedAt) - return - - if item_class in ('Movies', 'TVShows'): - self.update_kodi_video_library = True - elif item_class == 'Music': - self.update_kodi_music_library = True - with getattr(itemtypes, item_class)() as itemtype: - itemtype.updateUserdata(xml) - def process_item(self, xml_item): """ Processes a single library item """ plex_id = int(xml_item.get('ratingKey')) - if self.new_items_only: - if self.plexdb.is_recorded(plex_id, self.plex_type): - return - else: - if self.plexdb.checksum(plex_id, self.plex_type) == \ - int('%s%s' % (plex_id, - xml_item.get('updatedAt'))): - self.plexdb.update_last_sync(plex_id, - self.plex_type, - self.last_sync) - return + if not self.repair and self.plexdb.checksum(plex_id, self.plex_type) == \ + int('%s%s' % (plex_id, + xml_item.get('updatedAt'))): + # Already got EXACTLY this item in our DB + self.plexdb.update_last_sync(plex_id, + self.plex_type, + self.current_sync) + return task = GetMetadataTask() task.setup(self.queue, plex_id, self.get_children) backgroundthread.BGThreader.addTask(task) def process_delete(self): """ - Removes all the items that have NOT been updated (last_sync timestamp) - is different + Removes all the items that have NOT been updated (last_sync timestamp + is different) """ - with self.context(self.last_sync) as c: + with self.context(self.current_sync) as c: for plex_id in self.plexdb.plex_id_by_last_sync(self.plex_type, - self.last_sync): + self.current_sync): if self.isCanceled(): return c.remove(plex_id, plex_type=self.plex_type) + @utils.log_time + def process_playstate(self, iterator): + """ + Updates the playstate (resume point, number of views, userrating, last + played date, etc.) for all elements in the (xml-)iterator + """ + with self.context(self.current_sync) as c: + for xml_item in iterator: + if self.isCanceled(): + return False + c.update_userdata(xml_item, self.plex_type) + @utils.log_time def process_kind(self): """ """ - LOG.debug('Start processing %ss', self.plex_type) + successful = True + LOG.debug('Start processing %ss', self.section_type) sects = (x for x in sections.SECTIONS - if x['plex_type'] == self.plex_type) + if x['plex_type'] == self.section_type) for section in sects: LOG.debug('Processing library section %s', section) if self.isCanceled(): @@ -118,15 +95,15 @@ class FullSync(backgroundthread.KillableThread, common.libsync_mixin): if not self.install_sync_done: state.PATH_VERIFIED = False try: + # Sync new, updated and deleted items iterator = PF.SectionItems(section['section_id'], - {'type': self.plex_type}) + plex_type=self.plex_type) # Tell the processing thread about this new section queue_info = process_metadata.InitNewSection( self.context, utils.cast(int, iterator.get('totalSize', 0)), iterator.get('librarySectionTitle'), - section['section_id'], - section['plex_type']) + section['section_id']) self.queue.put(queue_info) with PlexDB() as self.plexdb: for xml_item in iterator: @@ -135,39 +112,58 @@ class FullSync(backgroundthread.KillableThread, common.libsync_mixin): self.process_item(xml_item) except RuntimeError: LOG.error('Could not entirely process section %s', section) + successful = False continue + LOG.debug('Waiting for processing thread to finish section') self.queue.join() + try: + # Sync playstate of every item + iterator = PF.SectionItems(section['section_id'], + plex_type=self.plex_type) + # Tell the processing thread that we're syncing playstate + queue_info = process_metadata.InitNewSection( + self.context, + utils.cast(int, iterator.get('totalSize', 0)), + iterator.get('librarySectionTitle'), + section['section_id']) + self.queue.put(queue_info) + self.process_playstate(iterator) + except RuntimeError: + LOG.error('Could not process playstate for section %s', section) + successful = False + continue + LOG.debug('Done processing playstate for section') LOG.debug('Finished processing %ss', self.plex_type) - return True + return successful def full_library_sync(self): """ """ kinds = [ - (v.PLEX_TYPE_MOVIE, itemtypes.Movie, False), - (v.PLEX_TYPE_SHOW, itemtypes.Show, False), - (v.PLEX_TYPE_SEASON, itemtypes.Season, False), - (v.PLEX_TYPE_EPISODE, itemtypes.Episode, False) + (v.PLEX_TYPE_MOVIE, v.PLEX_TYPE_MOVIE, itemtypes.Movie, False), + (v.PLEX_TYPE_SHOW, v.PLEX_TYPE_SHOW, itemtypes.Show, False), + (v.PLEX_TYPE_SEASON, v.PLEX_TYPE_SHOW, itemtypes.Season, False), + (v.PLEX_TYPE_EPISODE, v.PLEX_TYPE_SHOW, itemtypes.Episode, False) ] if state.ENABLE_MUSIC: kinds.extend([ - (v.PLEX_TYPE_ARTIST, itemtypes.Artist, False), - (v.PLEX_TYPE_ALBUM, itemtypes.Album, True), - (v.PLEX_TYPE_SONG, itemtypes.Song, False) + (v.PLEX_TYPE_ARTIST, v.PLEX_TYPE_ARTIST, itemtypes.Artist, False), + (v.PLEX_TYPE_ALBUM, v.PLEX_TYPE_ARTIST, itemtypes.Album, True), + (v.PLEX_TYPE_SONG, v.PLEX_TYPE_ARTIST, itemtypes.Song, False) ]) with PlexDB() as self.plexdb: for kind in kinds: # Setup our variables self.plex_type = kind[0] - self.context = kind[1] - self.get_children = kind[2] + self.section_type = kind[1] + self.context = kind[2] + self.get_children = kind[3] # Now do the heavy lifting if self.isCanceled() or not self.process_kind(): return False - if self.new_items_only: - # Delete movies that are not on Plex anymore - do this only once - self.process_delete() + # Delete movies that are not on Plex anymore + self.process_delete() return True @utils.log_time @@ -175,7 +171,7 @@ class FullSync(backgroundthread.KillableThread, common.libsync_mixin): if self.isCanceled(): return successful = False - self.last_sync = utils.unix_timestamp() + self.current_sync = utils.unix_timestamp() # Delete playlist and video node files from Kodi utils.delete_playlists() utils.delete_nodes() @@ -184,26 +180,18 @@ class FullSync(backgroundthread.KillableThread, common.libsync_mixin): return try: # Fire up our single processing thread - self.queue = backgroundthread.Queue.Queue(maxsize=200) + self.queue = backgroundthread.Queue.Queue(maxsize=400) self.processing_thread = process_metadata.ProcessMetadata( - self.queue, self.last_sync, self.show_dialog) + self.queue, self.current_sync, self.show_dialog) self.processing_thread.start() # Actual syncing - do only new items first - LOG.info('Running fullsync for **NEW** items with repair=%s', + LOG.info('Running full_library_sync with repair=%s', self.repair) - self.new_items_only = True - # This will also update playstates and userratings! - if not self.full_library_sync(): - return - if self.isCanceled(): - return - # This will NOT update playstates and userratings! - LOG.info('Running fullsync for **CHANGED** items with repair=%s', - self.repair) - self.new_items_only = False if not self.full_library_sync(): return + # Tell the processing thread to exit with one last element None + self.queue.put(None) if self.isCanceled(): return if PLAYLIST_SYNC_ENABLED and not playlists.full_sync(): @@ -212,10 +200,7 @@ class FullSync(backgroundthread.KillableThread, common.libsync_mixin): except: utils.ERROR(txt='full_sync.py crashed', notify=True) finally: - # Last element will kill the processing thread (if not already - # done so, e.g. quitting Kodi) - self.queue.put(None) - # This will block until the processing thread exits + # This will block until the processing thread really exits LOG.debug('Waiting for processing thread to exit') self.processing_thread.join() if self.callback: diff --git a/resources/lib/library_sync/process_metadata.py b/resources/lib/library_sync/process_metadata.py index 8be4e0f9..ea278129 100644 --- a/resources/lib/library_sync/process_metadata.py +++ b/resources/lib/library_sync/process_metadata.py @@ -17,12 +17,11 @@ class InitNewSection(object): context: itemtypes.Movie, itemtypes.Episode, etc. """ def __init__(self, context, total_number_of_items, section_name, - section_id, plex_type): + section_id): self.context = context self.total = total_number_of_items self.name = section_name self.id = section_id - self.plex_type = plex_type class ProcessMetadata(backgroundthread.KillableThread, common.libsync_mixin): diff --git a/resources/lib/library_sync/time.py b/resources/lib/library_sync/time.py index 9f587cfa..25963217 100644 --- a/resources/lib/library_sync/time.py +++ b/resources/lib/library_sync/time.py @@ -47,7 +47,7 @@ def sync_pms_time(): continue library_id = section.attrib['key'] try: - iterator = PF.SectionItems(library_id, {'type': plex_type}) + iterator = PF.SectionItems(library_id, plex_type=plex_type) for item in iterator: if item.get('viewCount'): # Don't want to mess with items that have playcount>0 diff --git a/resources/lib/plex_functions.py b/resources/lib/plex_functions.py index 2000fd4c..812475f4 100644 --- a/resources/lib/plex_functions.py +++ b/resources/lib/plex_functions.py @@ -534,8 +534,18 @@ class DownloadGen(object): Yields XML etree children or raises RuntimeError """ - def __init__(self, url, args=None): + def __init__(self, url, plex_type=None, last_viewed_at=None, + updated_at=None, args=None): self._args = args or {} + url += '?' + if plex_type: + url = '%stype=%s&' % (url, v.PLEX_TYPE_NUMBER_FROM_PLEX_TYPE[plex_type]) + if last_viewed_at: + url = '%slastViewedAt>=%s&' % (url, last_viewed_at) + if updated_at: + url = '%supdatedAt>=%s&' % (url, updated_at) + if url.endswith('?') or url.endswith('&'): + url = url[:-1] self._url = url self._pos = 0 self._exhausted = False @@ -549,6 +559,7 @@ class DownloadGen(object): 'sort': 'id', # Entries are sorted by plex_id 'excludeAllLeaves': 1 # PMS wont attach a first summary child }) + LOG.debug('DownloadGen url: %s, args: %s', self._url, self._args) self.xml = DU().downloadUrl(self._url, parameters=self._args) try: self.xml.attrib @@ -588,11 +599,11 @@ class SectionItems(DownloadGen): """ Iterator object to get all items of a Plex library section """ - def __init__(self, section_id, args=None): - if args and 'type' in args: - args['type'] = v.PLEX_TYPE_NUMBER_FROM_PLEX_TYPE[args['type']] - super(SectionItems, self).__init__( - '{server}/library/sections/%s/all' % section_id, args) + def __init__(self, section_id, plex_type=None, last_viewed_at=None, + updated_at=None, args=None): + url = '{server}/library/sections/%s/all' % section_id + super(SectionItems, self).__init__(url, plex_type, last_viewed_at, + updated_at, args) class Children(DownloadGen): diff --git a/resources/lib/utils.py b/resources/lib/utils.py index 80a1fb0a..d656515c 100644 --- a/resources/lib/utils.py +++ b/resources/lib/utils.py @@ -446,6 +446,14 @@ def unix_date_to_kodi(stamp): return localdate +def kodi_time_to_plex(stamp): + """ + Returns a Kodi timestamp (int/float) in Plex time (subtracting the + KODI_PLEX_TIME_OFFSET) + """ + return stamp - state.KODI_PLEX_TIME_OFFSET + + def unix_timestamp(seconds_into_the_future=None): """ Returns a Unix time stamp (seconds passed since January 1 1970) for NOW as