More hacking

resources/lib/itemtypes/common.py

@@ -80,34 +80,29 @@ class ItemBase(object):
                                     kodi_type,
                                     self.kodicursor)
 
-    def updateUserdata(self, xml):
+    def update_userdata(self, xml_element, plex_type):
         """
         Updates the Kodi watched state of the item from PMS. Also retrieves
         Plex resume points for movies in progress.
-
-        viewtag and viewid only serve as dummies
         """
-        for mediaitem in xml:
+        api = API(xml_element)
-            api = API(mediaitem)
+        # Get key and db entry on the Kodi db side
-            # Get key and db entry on the Kodi db side
+        db_item = self.plexdb.item_by_id(api.plex_id(), plex_type)
-            db_item = self.plexdb.getItem_byId(api.plex_id())
+        if not db_item:
-            try:
+            LOG.error('Item not yet synced: %s', xml_element.attrib)
-                fileid = db_item[1]
+            return
-            except TypeError:
+        # Grab the user's viewcount, resume points etc. from PMS' answer
-                continue
+        userdata = api.userdata()
-            # Grab the user's viewcount, resume points etc. from PMS' answer
+        # Write to Kodi DB
-            userdata = api.userdata()
+        self.kodi_db.set_resume(db_item['kodi_fileid'],
-            # Write to Kodi DB
+                                userdata['Resume'],
-            self.kodi_db.set_resume(fileid,
+                                userdata['Runtime'],
-                                    userdata['Resume'],
+                                userdata['PlayCount'],
-                                    userdata['Runtime'],
+                                userdata['LastPlayedDate'],
-                                    userdata['PlayCount'],
+                                plex_type)
-                                    userdata['LastPlayedDate'],
+        self.kodi_db.update_userrating(db_item['kodi_id'],
-                                    api.plex_type())
+                                       db_item['kodi_type'],
-            if v.KODIVERSION >= 17:
+                                       userdata['UserRating'])
-                self.kodi_db.update_userrating(db_item[0],
-                                               db_item[4],
-                                               userdata['UserRating'])
 
     def update_playstate(self, mark_played, view_count, resume, duration,
                          kodi_fileid, lastViewedAt, plex_type):

resources/lib/library_sync/full_sync.py

@@ -9,7 +9,6 @@ from .. import utils, backgroundthread, variables as v, state
 from .. import plex_functions as PF, itemtypes
 from ..plex_db import PlexDB
 
-
 if (v.PLATFORM != 'Microsoft UWP' and
         utils.settings('enablePlaylistSync') == 'true'):
     # Xbox cannot use watchdog, a dependency for PKC playlist features
@@ -18,7 +17,6 @@ if (v.PLATFORM != 'Microsoft UWP' and
 else:
     PLAYLIST_SYNC_ENABLED = False
 
-
 LOG = getLogger('PLEX.sync.full_sync')
 
 
@@ -33,84 +31,63 @@ class FullSync(backgroundthread.KillableThread, common.libsync_mixin):
         self.show_dialog = show_dialog
         self.queue = None
         self.process_thread = None
-        self.last_sync = None
+        self.current_sync = None
         self.plexdb = None
         self.plex_type = None
+        self.section_type = None
         self.processing_thread = None
         self.install_sync_done = utils.settings('SyncInstallRunDone') == 'true'
         super(FullSync, self).__init__()
 
-    def plex_update_watched(self, viewId, item_class, lastViewedAt=None,
-                            updatedAt=None):
-        """
-        YET to implement
-
-        Updates plex elements' view status ('watched' or 'unwatched') and
-        also updates resume times.
-        This is done by downloading one XML for ALL elements with viewId
-        """
-        if self.new_items_only is False:
-            # Only do this once for fullsync: the first run where new items are
-            # added to Kodi
-            return
-        xml = PF.GetAllPlexLeaves(viewId,
-                                  lastViewedAt=lastViewedAt,
-                                  updatedAt=updatedAt)
-        # Return if there are no items in PMS reply - it's faster
-        try:
-            xml[0].attrib
-        except (TypeError, AttributeError, IndexError):
-            LOG.error('Error updating watch status. Could not get viewId: '
-                      '%s of item_class %s with lastViewedAt: %s, updatedAt: '
-                      '%s', viewId, item_class, lastViewedAt, updatedAt)
-            return
-
-        if item_class in ('Movies', 'TVShows'):
-            self.update_kodi_video_library = True
-        elif item_class == 'Music':
-            self.update_kodi_music_library = True
-        with getattr(itemtypes, item_class)() as itemtype:
-            itemtype.updateUserdata(xml)
-
     def process_item(self, xml_item):
         """
         Processes a single library item
         """
         plex_id = int(xml_item.get('ratingKey'))
-        if self.new_items_only:
+        if not self.repair and self.plexdb.checksum(plex_id, self.plex_type) == \
-            if self.plexdb.is_recorded(plex_id, self.plex_type):
+                int('%s%s' % (plex_id,
-                return
+                              xml_item.get('updatedAt'))):
-        else:
+            # Already got EXACTLY this item in our DB
-            if self.plexdb.checksum(plex_id, self.plex_type) == \
+            self.plexdb.update_last_sync(plex_id,
-                    int('%s%s' % (plex_id,
+                                         self.plex_type,
-                                  xml_item.get('updatedAt'))):
+                                         self.current_sync)
-                self.plexdb.update_last_sync(plex_id,
+            return
-                                             self.plex_type,
-                                             self.last_sync)
-                return
         task = GetMetadataTask()
         task.setup(self.queue, plex_id, self.get_children)
         backgroundthread.BGThreader.addTask(task)
 
     def process_delete(self):
         """
-        Removes all the items that have NOT been updated (last_sync timestamp)
+        Removes all the items that have NOT been updated (last_sync timestamp
-        is different
+        is different)
         """
-        with self.context(self.last_sync) as c:
+        with self.context(self.current_sync) as c:
             for plex_id in self.plexdb.plex_id_by_last_sync(self.plex_type,
-                                                            self.last_sync):
+                                                            self.current_sync):
                 if self.isCanceled():
                     return
                 c.remove(plex_id, plex_type=self.plex_type)
 
+    @utils.log_time
+    def process_playstate(self, iterator):
+        """
+        Updates the playstate (resume point, number of views, userrating, last
+        played date, etc.) for all elements in the (xml-)iterator
+        """
+        with self.context(self.current_sync) as c:
+            for xml_item in iterator:
+                if self.isCanceled():
+                    return False
+                c.update_userdata(xml_item, self.plex_type)
+
     @utils.log_time
     def process_kind(self):
         """
         """
-        LOG.debug('Start processing %ss', self.plex_type)
+        successful = True
+        LOG.debug('Start processing %ss', self.section_type)
         sects = (x for x in sections.SECTIONS
-                 if x['plex_type'] == self.plex_type)
+                 if x['plex_type'] == self.section_type)
         for section in sects:
             LOG.debug('Processing library section %s', section)
             if self.isCanceled():
@@ -118,15 +95,15 @@ class FullSync(backgroundthread.KillableThread, common.libsync_mixin):
             if not self.install_sync_done:
                 state.PATH_VERIFIED = False
             try:
+                # Sync new, updated and deleted items
                 iterator = PF.SectionItems(section['section_id'],
-                                           {'type': self.plex_type})
+                                           plex_type=self.plex_type)
                 # Tell the processing thread about this new section
                 queue_info = process_metadata.InitNewSection(
                     self.context,
                     utils.cast(int, iterator.get('totalSize', 0)),
                     iterator.get('librarySectionTitle'),
-                    section['section_id'],
+                    section['section_id'])
-                    section['plex_type'])
                 self.queue.put(queue_info)
                 with PlexDB() as self.plexdb:
                     for xml_item in iterator:
@@ -135,39 +112,58 @@ class FullSync(backgroundthread.KillableThread, common.libsync_mixin):
                         self.process_item(xml_item)
             except RuntimeError:
                 LOG.error('Could not entirely process section %s', section)
+                successful = False
                 continue
+            LOG.debug('Waiting for processing thread to finish section')
             self.queue.join()
+            try:
+                # Sync playstate of every item
+                iterator = PF.SectionItems(section['section_id'],
+                                           plex_type=self.plex_type)
+                # Tell the processing thread that we're syncing playstate
+                queue_info = process_metadata.InitNewSection(
+                    self.context,
+                    utils.cast(int, iterator.get('totalSize', 0)),
+                    iterator.get('librarySectionTitle'),
+                    section['section_id'])
+                self.queue.put(queue_info)
+                self.process_playstate(iterator)
+            except RuntimeError:
+                LOG.error('Could not process playstate for section %s', section)
+                successful = False
+                continue
+            LOG.debug('Done processing playstate for section')
 
         LOG.debug('Finished processing %ss', self.plex_type)
-        return True
+        return successful
 
     def full_library_sync(self):
         """
         """
         kinds = [
-            (v.PLEX_TYPE_MOVIE, itemtypes.Movie, False),
+            (v.PLEX_TYPE_MOVIE, v.PLEX_TYPE_MOVIE, itemtypes.Movie, False),
-            (v.PLEX_TYPE_SHOW, itemtypes.Show, False),
+            (v.PLEX_TYPE_SHOW, v.PLEX_TYPE_SHOW, itemtypes.Show, False),
-            (v.PLEX_TYPE_SEASON, itemtypes.Season, False),
+            (v.PLEX_TYPE_SEASON, v.PLEX_TYPE_SHOW, itemtypes.Season, False),
-            (v.PLEX_TYPE_EPISODE, itemtypes.Episode, False)
+            (v.PLEX_TYPE_EPISODE, v.PLEX_TYPE_SHOW, itemtypes.Episode, False)
         ]
         if state.ENABLE_MUSIC:
             kinds.extend([
-                (v.PLEX_TYPE_ARTIST, itemtypes.Artist, False),
+                (v.PLEX_TYPE_ARTIST, v.PLEX_TYPE_ARTIST, itemtypes.Artist, False),
-                (v.PLEX_TYPE_ALBUM, itemtypes.Album, True),
+                (v.PLEX_TYPE_ALBUM, v.PLEX_TYPE_ARTIST, itemtypes.Album, True),
-                (v.PLEX_TYPE_SONG, itemtypes.Song, False)
+                (v.PLEX_TYPE_SONG, v.PLEX_TYPE_ARTIST, itemtypes.Song, False)
             ])
         with PlexDB() as self.plexdb:
             for kind in kinds:
                 # Setup our variables
                 self.plex_type = kind[0]
-                self.context = kind[1]
+                self.section_type = kind[1]
-                self.get_children = kind[2]
+                self.context = kind[2]
+                self.get_children = kind[3]
                 # Now do the heavy lifting
                 if self.isCanceled() or not self.process_kind():
                     return False
-                if self.new_items_only:
+                # Delete movies that are not on Plex anymore
-                    # Delete movies that are not on Plex anymore - do this only once
+                self.process_delete()
-                    self.process_delete()
         return True
 
     @utils.log_time
@@ -175,7 +171,7 @@ class FullSync(backgroundthread.KillableThread, common.libsync_mixin):
         if self.isCanceled():
             return
         successful = False
-        self.last_sync = utils.unix_timestamp()
+        self.current_sync = utils.unix_timestamp()
         # Delete playlist and video node files from Kodi
         utils.delete_playlists()
         utils.delete_nodes()
@@ -184,26 +180,18 @@ class FullSync(backgroundthread.KillableThread, common.libsync_mixin):
             return
         try:
             # Fire up our single processing thread
-            self.queue = backgroundthread.Queue.Queue(maxsize=200)
+            self.queue = backgroundthread.Queue.Queue(maxsize=400)
             self.processing_thread = process_metadata.ProcessMetadata(
-                self.queue, self.last_sync, self.show_dialog)
+                self.queue, self.current_sync, self.show_dialog)
             self.processing_thread.start()
 
             # Actual syncing - do only new items first
-            LOG.info('Running fullsync for **NEW** items with repair=%s',
+            LOG.info('Running full_library_sync with repair=%s',
                      self.repair)
-            self.new_items_only = True
-            # This will also update playstates and userratings!
-            if not self.full_library_sync():
-                return
-            if self.isCanceled():
-                return
-            # This will NOT update playstates and userratings!
-            LOG.info('Running fullsync for **CHANGED** items with repair=%s',
-                     self.repair)
-            self.new_items_only = False
             if not self.full_library_sync():
                 return
+            # Tell the processing thread to exit with one last element None
+            self.queue.put(None)
             if self.isCanceled():
                 return
             if PLAYLIST_SYNC_ENABLED and not playlists.full_sync():
@@ -212,10 +200,7 @@ class FullSync(backgroundthread.KillableThread, common.libsync_mixin):
         except:
             utils.ERROR(txt='full_sync.py crashed', notify=True)
         finally:
-            # Last element will kill the processing thread (if not already
+            # This will block until the processing thread really exits
-            # done so, e.g. quitting Kodi)
-            self.queue.put(None)
-            # This will block until the processing thread exits
             LOG.debug('Waiting for processing thread to exit')
             self.processing_thread.join()
             if self.callback:

resources/lib/library_sync/process_metadata.py

@@ -17,12 +17,11 @@ class InitNewSection(object):
     context: itemtypes.Movie, itemtypes.Episode, etc.
     """
     def __init__(self, context, total_number_of_items, section_name,
-                 section_id, plex_type):
+                 section_id):
         self.context = context
         self.total = total_number_of_items
         self.name = section_name
         self.id = section_id
-        self.plex_type = plex_type
 
 
 class ProcessMetadata(backgroundthread.KillableThread, common.libsync_mixin):

resources/lib/library_sync/time.py

@@ -47,7 +47,7 @@ def sync_pms_time():
                 continue
             library_id = section.attrib['key']
             try:
-                iterator = PF.SectionItems(library_id, {'type': plex_type})
+                iterator = PF.SectionItems(library_id, plex_type=plex_type)
                 for item in iterator:
                     if item.get('viewCount'):
                         # Don't want to mess with items that have playcount>0

resources/lib/plex_functions.py

@@ -534,8 +534,18 @@ class DownloadGen(object):
 
     Yields XML etree children or raises RuntimeError
     """
-    def __init__(self, url, args=None):
+    def __init__(self, url, plex_type=None, last_viewed_at=None,
+                 updated_at=None, args=None):
         self._args = args or {}
+        url += '?'
+        if plex_type:
+            url = '%stype=%s&' % (url, v.PLEX_TYPE_NUMBER_FROM_PLEX_TYPE[plex_type])
+        if last_viewed_at:
+            url = '%slastViewedAt>=%s&' % (url, last_viewed_at)
+        if updated_at:
+            url = '%supdatedAt>=%s&' % (url, updated_at)
+        if url.endswith('?') or url.endswith('&'):
+            url = url[:-1]
         self._url = url
         self._pos = 0
         self._exhausted = False
@@ -549,6 +559,7 @@ class DownloadGen(object):
             'sort': 'id',  # Entries are sorted by plex_id
             'excludeAllLeaves': 1  # PMS wont attach a first summary child
         })
+        LOG.debug('DownloadGen url: %s, args: %s', self._url, self._args)
         self.xml = DU().downloadUrl(self._url, parameters=self._args)
         try:
             self.xml.attrib
@@ -588,11 +599,11 @@ class SectionItems(DownloadGen):
     """
     Iterator object to get all items of a Plex library section
     """
-    def __init__(self, section_id, args=None):
+    def __init__(self, section_id, plex_type=None, last_viewed_at=None,
-        if args and 'type' in args:
+                 updated_at=None, args=None):
-            args['type'] = v.PLEX_TYPE_NUMBER_FROM_PLEX_TYPE[args['type']]
+        url = '{server}/library/sections/%s/all' % section_id
-        super(SectionItems, self).__init__(
+        super(SectionItems, self).__init__(url, plex_type, last_viewed_at,
-            '{server}/library/sections/%s/all' % section_id, args)
+                                           updated_at, args)
 
 
 class Children(DownloadGen):

resources/lib/utils.py

@@ -446,6 +446,14 @@ def unix_date_to_kodi(stamp):
     return localdate
 
 
+def kodi_time_to_plex(stamp):
+    """
+    Returns a Kodi timestamp (int/float) in Plex time (subtracting the
+    KODI_PLEX_TIME_OFFSET)
+    """
+    return stamp - state.KODI_PLEX_TIME_OFFSET
+
+
 def unix_timestamp(seconds_into_the_future=None):
     """
     Returns a Unix time stamp (seconds passed since January 1 1970) for NOW as