From bafd3545f4a965c97744608c43650c2694924102 Mon Sep 17 00:00:00 2001 From: croneter Date: Sun, 24 Nov 2019 14:01:42 +0100 Subject: [PATCH 01/50] Improve sync resiliance when certain items are not to be synced to Kodi --- resources/lib/itemtypes/movies.py | 4 ++-- resources/lib/itemtypes/music.py | 14 ++++++++++++-- resources/lib/itemtypes/tvshows.py | 12 ++++++------ 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/resources/lib/itemtypes/movies.py b/resources/lib/itemtypes/movies.py index f9193c88..54758444 100644 --- a/resources/lib/itemtypes/movies.py +++ b/resources/lib/itemtypes/movies.py @@ -20,10 +20,10 @@ class Movie(ItemBase): Process single movie """ api = API(xml) - if not self.sync_this_item(api.library_section_id()): + if not self.sync_this_item(section_id or api.library_section_id()): LOG.debug('Skipping sync of %s %s: %s - section %s not synched to ' 'Kodi', api.plex_type, api.plex_id, api.title(), - api.library_section_id()) + section_id or api.library_section_id()) return plex_id = api.plex_id movie = self.plexdb.movie(plex_id) diff --git a/resources/lib/itemtypes/music.py b/resources/lib/itemtypes/music.py index 61b16003..f609829a 100644 --- a/resources/lib/itemtypes/music.py +++ b/resources/lib/itemtypes/music.py @@ -159,10 +159,10 @@ class Artist(MusicMixin, ItemBase): Process a single artist """ api = API(xml) - if not self.sync_this_item(api.library_section_id()): + if not self.sync_this_item(section_id or api.library_section_id()): LOG.debug('Skipping sync of %s %s: %s - section %s not synched to ' 'Kodi', api.plex_type, api.plex_id, api.title(), - api.library_section_id()) + section_id or api.library_section_id()) return plex_id = api.plex_id artist = self.plexdb.artist(plex_id) @@ -225,6 +225,11 @@ class Album(MusicMixin, ItemBase): avoid infinite loops """ api = API(xml) + if not self.sync_this_item(section_id or api.library_section_id()): + LOG.debug('Skipping sync of %s %s: %s - section %s not synched to ' + 'Kodi', api.plex_type, api.plex_id, api.title(), + section_id or api.library_section_id()) + return plex_id = api.plex_id album = self.plexdb.album(plex_id) if album: @@ -387,6 +392,11 @@ class Song(MusicMixin, ItemBase): Process single song/track """ api = API(xml) + if not self.sync_this_item(section_id or api.library_section_id()): + LOG.debug('Skipping sync of %s %s: %s - section %s not synched to ' + 'Kodi', api.plex_type, api.plex_id, api.title(), + section_id or api.library_section_id()) + return plex_id = api.plex_id song = self.plexdb.song(plex_id) if song: diff --git a/resources/lib/itemtypes/tvshows.py b/resources/lib/itemtypes/tvshows.py index 61d8d050..f52a03f1 100644 --- a/resources/lib/itemtypes/tvshows.py +++ b/resources/lib/itemtypes/tvshows.py @@ -148,10 +148,10 @@ class Show(TvShowMixin, ItemBase): Process a single show """ api = API(xml) - if not self.sync_this_item(api.library_section_id()): + if not self.sync_this_item(section_id or api.library_section_id()): LOG.debug('Skipping sync of %s %s: %s - section %s not synched to ' 'Kodi', api.plex_type, api.plex_id, api.title(), - api.library_section_id()) + section_id or api.library_section_id()) return plex_id = api.plex_id show = self.plexdb.show(plex_id) @@ -303,10 +303,10 @@ class Season(TvShowMixin, ItemBase): Process a single season of a certain tv show """ api = API(xml) - if not self.sync_this_item(api.library_section_id()): + if not self.sync_this_item(section_id or api.library_section_id()): LOG.debug('Skipping sync of %s %s: %s - section %s not synched to ' 'Kodi', api.plex_type, api.plex_id, api.title(), - api.library_section_id()) + section_id or api.library_section_id()) return plex_id = api.plex_id season = self.plexdb.season(plex_id) @@ -372,10 +372,10 @@ class Episode(TvShowMixin, ItemBase): Process single episode """ api = API(xml) - if not self.sync_this_item(api.library_section_id()): + if not self.sync_this_item(section_id or api.library_section_id()): LOG.debug('Skipping sync of %s %s: %s - section %s not synched to ' 'Kodi', api.plex_type, api.plex_id, api.title(), - api.library_section_id()) + section_id or api.library_section_id()) return plex_id = api.plex_id episode = self.plexdb.episode(plex_id) From 56324b1e881d0552f4abc2b0cbb9ba5f8c396cc6 Mon Sep 17 00:00:00 2001 From: croneter Date: Mon, 25 Nov 2019 08:00:43 +0100 Subject: [PATCH 02/50] Fix OperationalError when resetting PKC --- resources/lib/migration.py | 3 ++- resources/lib/playlists/__init__.py | 36 +++++++++++++++++++---------- resources/lib/playlists/db.py | 17 ++++++++++++++ resources/lib/playlists/kodi_pl.py | 15 ++++++++++++ resources/lib/plex_db/playlists.py | 13 +++++++++++ resources/lib/utils.py | 27 ++-------------------- 6 files changed, 73 insertions(+), 38 deletions(-) diff --git a/resources/lib/migration.py b/resources/lib/migration.py index ae04e96a..13ec7729 100644 --- a/resources/lib/migration.py +++ b/resources/lib/migration.py @@ -64,7 +64,8 @@ def check_migration(): if not utils.compare_version(last_migration, '2.9.3'): LOG.info('Migrating to version 2.9.2') # Re-sync all playlists to Kodi - utils.wipe_synched_playlists() + from .playlists import remove_synced_playlists + remove_synced_playlists() if not utils.compare_version(last_migration, '2.9.7'): LOG.info('Migrating to version 2.9.6') diff --git a/resources/lib/playlists/__init__.py b/resources/lib/playlists/__init__.py index b3658ba0..d865087f 100644 --- a/resources/lib/playlists/__init__.py +++ b/resources/lib/playlists/__init__.py @@ -68,6 +68,18 @@ def kodi_playlist_monitor(): return observer +def remove_synced_playlists(): + """ + Deletes all synched playlists on the Kodi side, not on the Plex side + """ + LOG.info('Removing all playlists that we synced to Kodi') + with app.APP.lock_playlists: + paths = db.get_all_kodi_playlist_paths() + kodi_pl.delete_kodi_playlists(paths) + db.wipe_table() + LOG.info('Done removing all synced playlists') + + def websocket(plex_id, status): """ Call this function to process websocket messages from the PMS @@ -370,19 +382,19 @@ class PlaylistEventhandler(events.FileSystemEventHandler): """ path = event.dest_path if event.event_type == events.EVENT_TYPE_MOVED \ else event.src_path - if not sync_kodi_playlist(path): - return - if path in kodi_pl.IGNORE_KODI_PLAYLIST_CHANGE: - LOG.debug('Ignoring event %s', event) - kodi_pl.IGNORE_KODI_PLAYLIST_CHANGE.remove(path) - return - _method_map = { - events.EVENT_TYPE_MODIFIED: self.on_modified, - events.EVENT_TYPE_MOVED: self.on_moved, - events.EVENT_TYPE_CREATED: self.on_created, - events.EVENT_TYPE_DELETED: self.on_deleted, - } with app.APP.lock_playlists: + if not sync_kodi_playlist(path): + return + if path in kodi_pl.IGNORE_KODI_PLAYLIST_CHANGE: + LOG.debug('Ignoring event %s', event) + kodi_pl.IGNORE_KODI_PLAYLIST_CHANGE.remove(path) + return + _method_map = { + events.EVENT_TYPE_MODIFIED: self.on_modified, + events.EVENT_TYPE_MOVED: self.on_moved, + events.EVENT_TYPE_CREATED: self.on_created, + events.EVENT_TYPE_DELETED: self.on_deleted, + } _method_map[event.event_type](event) def on_created(self, event): diff --git a/resources/lib/playlists/db.py b/resources/lib/playlists/db.py index 6b63c39a..673fc02a 100644 --- a/resources/lib/playlists/db.py +++ b/resources/lib/playlists/db.py @@ -57,6 +57,23 @@ def get_playlist(path=None, plex_id=None): return playlist +def get_all_kodi_playlist_paths(): + """ + Returns a list with all paths for the playlists on the Kodi side + """ + with PlexDB() as plexdb: + paths = list(plexdb.all_kodi_paths()) + return paths + + +def wipe_table(): + """ + Deletes all playlists entries in the Plex DB + """ + with PlexDB() as plexdb: + plexdb.wipe_playlists() + + def _m3u_iterator(text): """ Yields e.g. plugin://plugin.video.plexkodiconnect.movies/?plex_id=xxx diff --git a/resources/lib/playlists/kodi_pl.py b/resources/lib/playlists/kodi_pl.py index 4b040c25..8753326d 100644 --- a/resources/lib/playlists/kodi_pl.py +++ b/resources/lib/playlists/kodi_pl.py @@ -96,6 +96,21 @@ def delete(playlist): db.update_playlist(playlist, delete=True) +def delete_kodi_playlists(playlist_paths): + """ + Deletes all the the playlist files passed in; WILL IGNORE THIS CHANGE ON + THE PLEX SIDE! + """ + for path in playlist_paths: + try: + path_ops.remove(path) + # Ensure we're not deleting the playlists on the Plex side later + IGNORE_KODI_PLAYLIST_CHANGE.append(path) + LOG.info('Removed playlist %s', path) + except (OSError, IOError): + LOG.warn('Could not remove playlist %s', path) + + def _write_playlist_to_file(playlist, xml): """ Feed with playlist Playlist. Will write the playlist to a m3u file diff --git a/resources/lib/plex_db/playlists.py b/resources/lib/plex_db/playlists.py index 586e745a..68575bb8 100644 --- a/resources/lib/plex_db/playlists.py +++ b/resources/lib/plex_db/playlists.py @@ -81,3 +81,16 @@ class Playlists(object): playlist.kodi_type = answ[4] playlist.kodi_hash = answ[5] return playlist + + def all_kodi_paths(self): + """ + Returns a generator for all kodi_paths of all synched playlists + """ + self.cursor.execute('SELECT kodi_path FROM playlists') + return (x[0] for x in self.cursor) + + def wipe_playlists(self): + """ + Deletes all entries in the playlists table + """ + self.cursor.execute('DELETE FROM playlists') diff --git a/resources/lib/utils.py b/resources/lib/utils.py index 0b025128..3bf26dc2 100644 --- a/resources/lib/utils.py +++ b/resources/lib/utils.py @@ -525,29 +525,6 @@ def delete_temporary_subtitles(): root, file, err) -def wipe_synched_playlists(): - """ - Deletes all synched playlist files on the Kodi side; resets the Plex table - listing all synched Plex playlists - """ - from . import plex_db - try: - with plex_db.PlexDB() as plexdb: - plexdb.cursor.execute('SELECT kodi_path FROM playlists') - playlist_paths = [x[0] for x in plexdb.cursor] - except OperationalError: - # Plex DB completely empty yet - playlist_paths = [] - for path in playlist_paths: - try: - path_ops.remove(path) - LOG.info('Removed playlist %s', path) - except (OSError, IOError): - LOG.warn('Could not remove playlist %s', path) - # Now wipe our database - plex_db.wipe(table='playlists') - - def wipe_database(reboot=True): """ Deletes all Plex playlists as well as video nodes, then clears Kodi as well @@ -557,10 +534,10 @@ def wipe_database(reboot=True): LOG.warn('Start wiping') from .library_sync.sections import delete_files from . import kodi_db, plex_db + from .playlists import remove_synced_playlists # Clean up the playlists and video nodes delete_files() - # Wipe all synched playlists - wipe_synched_playlists() + remove_synced_playlists() try: with plex_db.PlexDB() as plexdb: if plexdb.songs_have_been_synced(): From e01e50a6500c922db08519c119a84a6aa036eec4 Mon Sep 17 00:00:00 2001 From: croneter Date: Sat, 30 Nov 2019 12:50:36 +0100 Subject: [PATCH 03/50] Make sure bool is returned instead of an int --- resources/lib/app/application.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resources/lib/app/application.py b/resources/lib/app/application.py index 9d3fc0d0..03955049 100644 --- a/resources/lib/app/application.py +++ b/resources/lib/app/application.py @@ -54,11 +54,11 @@ class App(object): @property def is_playing(self): - return self.player.isPlaying() + return self.player.isPlaying() == 1 @property def is_playing_video(self): - return self.player.isPlayingVideo() + return self.player.isPlayingVideo() == 1 def register_fanart_thread(self, thread): self.fanart_thread = thread From c85e1e2bd07799d1262afc897213ba6c9d967320 Mon Sep 17 00:00:00 2001 From: croneter Date: Thu, 28 Nov 2019 17:49:48 +0100 Subject: [PATCH 04/50] Optimize threads by using events instead of a polling mechanism. Fixes PKC become unresponsive, e.g. when switching users --- resources/lib/app/application.py | 21 +-- resources/lib/artwork.py | 18 ++- resources/lib/backgroundthread.py | 158 ++++++++------------- resources/lib/library_sync/common.py | 28 ---- resources/lib/library_sync/fanart.py | 63 ++++---- resources/lib/library_sync/full_sync.py | 33 +++-- resources/lib/library_sync/get_metadata.py | 19 ++- resources/lib/library_sync/sections.py | 10 +- resources/lib/playlists/__init__.py | 10 +- resources/lib/playqueue.py | 13 +- resources/lib/plex_companion.py | 9 +- resources/lib/service_entry.py | 4 +- resources/lib/sync.py | 21 +-- resources/lib/websocket_client.py | 28 ++-- resources/lib/windows/userselect.py | 2 +- 15 files changed, 194 insertions(+), 243 deletions(-) diff --git a/resources/lib/app/application.py b/resources/lib/app/application.py index 9d3fc0d0..d2e919d3 100644 --- a/resources/lib/app/application.py +++ b/resources/lib/app/application.py @@ -124,19 +124,16 @@ class App(object): if block: while True: for thread in self.threads: - if not thread.suspend_reached: + if not thread.is_suspended(): LOG.debug('Waiting for thread to suspend: %s', thread) # Send suspend signal again in case self.threads # changed - thread.suspend() - if self.monitor.waitForAbort(0.1): - return True - break + thread.suspend(block=True) else: break return xbmc.abortRequested - def resume_threads(self, block=True): + def resume_threads(self): """ Resume all thread activity with or without blocking. Returns True only if PKC shutdown requested @@ -144,16 +141,6 @@ class App(object): LOG.debug('Resuming threads: %s', self.threads) for thread in self.threads: thread.resume() - if block: - while True: - for thread in self.threads: - if thread.suspend_reached: - LOG.debug('Waiting for thread to resume: %s', thread) - if self.monitor.waitForAbort(0.1): - return True - break - else: - break return xbmc.abortRequested def stop_threads(self, block=True): @@ -163,7 +150,7 @@ class App(object): """ LOG.debug('Killing threads: %s', self.threads) for thread in self.threads: - thread.abort() + thread.cancel() if block: while self.threads: LOG.debug('Waiting for threads to exit: %s', self.threads) diff --git a/resources/lib/artwork.py b/resources/lib/artwork.py index 47fdfd45..113d8a32 100644 --- a/resources/lib/artwork.py +++ b/resources/lib/artwork.py @@ -33,8 +33,8 @@ class ImageCachingThread(backgroundthread.KillableThread): if not utils.settings('imageSyncDuringPlayback') == 'true': self.suspend_points.append((app.APP, 'is_playing_video')) - def isSuspended(self): - return any(getattr(obj, txt) for obj, txt in self.suspend_points) + def should_suspend(self): + return any(getattr(obj, attrib) for obj, attrib in self.suspend_points) @staticmethod def _url_generator(kind, kodi_type): @@ -73,18 +73,26 @@ class ImageCachingThread(backgroundthread.KillableThread): app.APP.deregister_caching_thread(self) LOG.info("---===### Stopped ImageCachingThread ###===---") - def _run(self): + def _loop(self): kinds = [KodiVideoDB] if app.SYNC.enable_music: kinds.append(KodiMusicDB) for kind in kinds: for kodi_type in ('poster', 'fanart'): for url in self._url_generator(kind, kodi_type): - if self.wait_while_suspended(): - return + if self.should_suspend() or self.should_cancel(): + return False cache_url(url) # Toggles Image caching completed to Yes utils.settings('plex_status_image_caching', value=utils.lang(107)) + return True + + def _run(self): + while True: + if self._loop(): + break + if self.wait_while_suspended(): + break def cache_url(url): diff --git a/resources/lib/backgroundthread.py b/resources/lib/backgroundthread.py index 7823fffd..45f7602f 100644 --- a/resources/lib/backgroundthread.py +++ b/resources/lib/backgroundthread.py @@ -13,131 +13,95 @@ LOG = getLogger('PLEX.threads') class KillableThread(threading.Thread): - '''A thread class that supports raising exception in the thread from - another thread. - ''' - # def _get_my_tid(self): - # """determines this (self's) thread id - - # CAREFUL : this function is executed in the context of the caller - # thread, to get the identity of the thread represented by this - # instance. - # """ - # if not self.isAlive(): - # raise threading.ThreadError("the thread is not active") - - # return self.ident - - # def _raiseExc(self, exctype): - # """Raises the given exception type in the context of this thread. - - # If the thread is busy in a system call (time.sleep(), - # socket.accept(), ...), the exception is simply ignored. - - # If you are sure that your exception should terminate the thread, - # one way to ensure that it works is: - - # t = ThreadWithExc( ... ) - # ... - # t.raiseExc( SomeException ) - # while t.isAlive(): - # time.sleep( 0.1 ) - # t.raiseExc( SomeException ) - - # If the exception is to be caught by the thread, you need a way to - # check that your thread has caught it. - - # CAREFUL : this function is executed in the context of the - # caller thread, to raise an excpetion in the context of the - # thread represented by this instance. - # """ - # _async_raise(self._get_my_tid(), exctype) - - def kill(self, force_and_wait=False): - pass - # try: - # self._raiseExc(KillThreadException) - - # if force_and_wait: - # time.sleep(0.1) - # while self.isAlive(): - # self._raiseExc(KillThreadException) - # time.sleep(0.1) - # except threading.ThreadError: - # pass - - # def onKilled(self): - # pass - - # def run(self): - # try: - # self._Thread__target(*self._Thread__args, **self._Thread__kwargs) - # except KillThreadException: - # self.onKilled() - def __init__(self, group=None, target=None, name=None, args=(), kwargs={}): self._canceled = False - # Set to True to set the thread to suspended self._suspended = False - # Thread will return True only if suspended state is reached - self.suspend_reached = False + self._is_not_suspended = threading.Event() + self._is_not_suspended.set() + self._suspension_reached = threading.Event() + self._is_not_asleep = threading.Event() + self._is_not_asleep.set() + self.suspension_timeout = None super(KillableThread, self).__init__(group, target, name, args, kwargs) - def isCanceled(self): + def should_cancel(self): """ - Returns True if the thread is stopped + Returns True if the thread should be stopped immediately """ - if self._canceled or xbmc.abortRequested: - return True - return False + return self._canceled or app.APP.stop_pkc - def abort(self): + def cancel(self): """ - Call to stop this thread + Call from another thread to stop this current thread """ self._canceled = True + # Make sure thread is running in order to exit quickly + self._is_not_suspended.set() + self._is_not_asleep.set() - def suspend(self, block=False): + def should_suspend(self): """ - Call to suspend this thread + Returns True if the current thread should be suspended immediately """ + return self._suspended + + def suspend(self, block=False, timeout=None): + """ + Call from another thread to suspend the current thread. Provide a + timeout [float] in seconds optionally. block=True will block the caller + until the thread-to-be-suspended is indeed suspended + Will wake a thread that is asleep! + """ + self.suspension_timeout = timeout self._suspended = True + self._is_not_suspended.clear() + # Make sure thread wakes up in order to suspend + self._is_not_asleep.set() if block: - while not self.suspend_reached: - LOG.debug('Waiting for thread to suspend: %s', self) - if app.APP.monitor.waitForAbort(0.1): - return + self._suspension_reached.wait() def resume(self): """ - Call to revive a suspended thread back to life + Call from another thread to revive a suspended or asleep current thread + back to life """ self._suspended = False + self._is_not_suspended.set() + self._is_not_asleep.set() def wait_while_suspended(self): """ Blocks until thread is not suspended anymore or the thread should - exit. - Returns True only if the thread should exit (=isCanceled()) + exit or for a period of self.suspension_timeout (set by the caller of + suspend()) + Returns the value of should_cancel() """ - while self.isSuspended(): - try: - self.suspend_reached = True - # Set in service.py - if self.isCanceled(): - # Abort was requested while waiting. We should exit - return True - if app.APP.monitor.waitForAbort(0.1): - return True - finally: - self.suspend_reached = False - return self.isCanceled() + self._suspension_reached.set() + self._is_not_suspended.wait(self.suspension_timeout) + self._suspension_reached.clear() + return self.should_cancel() - def isSuspended(self): + def is_suspended(self): """ - Returns True if the thread is suspended + Check from another thread whether the current thread is suspended """ - return self._suspended + return self._suspension_reached.is_set() + + def sleep(self, timeout): + """ + Only call from the current thread in order to sleep for a period of + timeout [float, seconds]. Will unblock immediately if thread should + cancel (should_cancel()) or the thread should_suspend + """ + self._is_not_asleep.clear() + self._is_not_asleep.wait(timeout) + self._is_not_asleep.set() + + def is_asleep(self): + """ + Check from another thread whether the current thread is asleep + """ + return not self._is_not_asleep.is_set() class OrderedQueue(Queue.PriorityQueue, object): @@ -239,7 +203,7 @@ class Task(object): def cancel(self): self._canceled = True - def isCanceled(self): + def should_cancel(self): return self._canceled or xbmc.abortRequested def isValid(self): diff --git a/resources/lib/library_sync/common.py b/resources/lib/library_sync/common.py index 5497dbf8..c2481956 100644 --- a/resources/lib/library_sync/common.py +++ b/resources/lib/library_sync/common.py @@ -9,34 +9,6 @@ PLAYLIST_SYNC_ENABLED = (v.DEVICE != 'Microsoft UWP' and utils.settings('enablePlaylistSync') == 'true') -class fullsync_mixin(object): - def __init__(self): - self._canceled = False - - def abort(self): - """Hit method to terminate the thread""" - self._canceled = True - # Let's NOT suspend sync threads but immediately terminate them - suspend = abort - - @property - def suspend_reached(self): - """Since we're not suspending, we'll never set it to True""" - return False - - @suspend_reached.setter - def suspend_reached(self): - pass - - def resume(self): - """Obsolete since we're not suspending""" - pass - - def isCanceled(self): - """Check whether we should exit this thread""" - return self._canceled - - def update_kodi_library(video=True, music=True): """ Updates the Kodi library and thus refreshes the Kodi views and widgets diff --git a/resources/lib/library_sync/fanart.py b/resources/lib/library_sync/fanart.py index 7a4b59ff..9ad26a7a 100644 --- a/resources/lib/library_sync/fanart.py +++ b/resources/lib/library_sync/fanart.py @@ -27,48 +27,51 @@ class FanartThread(backgroundthread.KillableThread): self.refresh = refresh super(FanartThread, self).__init__() - def isSuspended(self): + def should_suspend(self): return self._suspended or app.APP.is_playing_video def run(self): LOG.info('Starting FanartThread') app.APP.register_fanart_thread(self) try: - self._run_internal() + self._run() except Exception: utils.ERROR(notify=True) finally: app.APP.deregister_fanart_thread(self) - def _run_internal(self): + def _loop(self): + for typus in SUPPORTED_TYPES: + offset = 0 + while True: + with PlexDB() as plexdb: + # Keep DB connection open only for a short period of time! + if self.refresh: + batch = list(plexdb.every_plex_id(typus, + offset, + BATCH_SIZE)) + else: + batch = list(plexdb.missing_fanart(typus, + offset, + BATCH_SIZE)) + for plex_id in batch: + # Do the actual, time-consuming processing + if self.should_suspend() or self.should_cancel(): + return False + process_fanart(plex_id, typus, self.refresh) + if len(batch) < BATCH_SIZE: + break + offset += BATCH_SIZE + return True + + def _run(self): finished = False - try: - for typus in SUPPORTED_TYPES: - offset = 0 - while True: - with PlexDB() as plexdb: - # Keep DB connection open only for a short period of time! - if self.refresh: - batch = list(plexdb.every_plex_id(typus, - offset, - BATCH_SIZE)) - else: - batch = list(plexdb.missing_fanart(typus, - offset, - BATCH_SIZE)) - for plex_id in batch: - # Do the actual, time-consuming processing - if self.wait_while_suspended(): - return - process_fanart(plex_id, typus, self.refresh) - if len(batch) < BATCH_SIZE: - break - offset += BATCH_SIZE - else: - finished = True - finally: - LOG.info('FanartThread finished: %s', finished) - self.callback(finished) + while not finished: + finished = self._loop() + if self.wait_while_suspended(): + break + LOG.info('FanartThread finished: %s', finished) + self.callback(finished) class FanartTask(backgroundthread.Task): diff --git a/resources/lib/library_sync/full_sync.py b/resources/lib/library_sync/full_sync.py index e3f4f1eb..609d45b6 100644 --- a/resources/lib/library_sync/full_sync.py +++ b/resources/lib/library_sync/full_sync.py @@ -39,7 +39,7 @@ class InitNewSection(object): self.plex_type = plex_type -class FullSync(common.fullsync_mixin): +class FullSync(backgroundthread.KillableThread): def __init__(self, repair, callback, show_dialog): """ repair=True: force sync EVERY item @@ -75,6 +75,12 @@ class FullSync(common.fullsync_mixin): worker_count=self.worker_count) super(FullSync, self).__init__() + def suspend(self, block=False, timeout=None): + """ + Let's NOT suspend sync threads but immediately terminate them + """ + self.cancel() + def update_progressbar(self): if self.dialog: try: @@ -112,7 +118,7 @@ class FullSync(common.fullsync_mixin): if not self.section: _, self.section = self.queue.get() self.queue.task_done() - while not self.isCanceled() and self.item_count > 0: + while not self.should_cancel() and self.item_count > 0: section = self.section if not section: break @@ -124,12 +130,12 @@ class FullSync(common.fullsync_mixin): self.section_type_text = utils.lang( v.TRANSLATION_FROM_PLEXTYPE[section.plex_type]) with section.context(self.current_sync) as context: - while not self.isCanceled() and self.item_count > 0: + while not self.should_cancel() and self.item_count > 0: try: _, item = self.queue.get(block=False) except backgroundthread.Queue.Empty: if self.threader.threader.working(): - app.APP.monitor.waitForAbort(0.02) + self.sleep(0.02) continue else: # Try again, in case a thread just finished @@ -187,7 +193,7 @@ class FullSync(common.fullsync_mixin): # Check Plex DB to see what we need to add/update with PlexDB() as self.plexdb: for last, xml_item in loop: - if self.isCanceled(): + if self.should_cancel(): return False self.process_item(xml_item) if self.item_count == BATCH_SIZE: @@ -227,7 +233,7 @@ class FullSync(common.fullsync_mixin): while True: with section.context(self.current_sync) as itemtype: for i, (last, xml_item) in enumerate(loop): - if self.isCanceled(): + if self.should_cancel(): return False if not itemtype.update_userdata(xml_item, section.plex_type): # Somehow did not sync this item yet @@ -256,7 +262,7 @@ class FullSync(common.fullsync_mixin): for kind in kinds: for section in (x for x in app.SYNC.sections if x.section_type == kind[1]): - if self.isCanceled(): + if self.should_cancel(): LOG.debug('Need to exit now') return if not section.sync_to_kodi: @@ -332,7 +338,7 @@ class FullSync(common.fullsync_mixin): self.get_children = section.get_children self.queue = section.Queue() # Now do the heavy lifting - if self.isCanceled() or not self.addupdate_section(section): + if self.should_cancel() or not self.addupdate_section(section): return False if self.section_success: # Need to check because a thread might have missed to get @@ -391,7 +397,7 @@ class FullSync(common.fullsync_mixin): self.context = section.context self.get_children = section.get_children # Now do the heavy lifting - if self.isCanceled() or not self.playstate_per_section(section): + if self.should_cancel() or not self.playstate_per_section(section): return False # Delete movies that are not on Plex anymore @@ -416,7 +422,7 @@ class FullSync(common.fullsync_mixin): self.current_sync, BATCH_SIZE)) for plex_id in plex_ids: - if self.isCanceled(): + if self.should_cancel(): return False ctx.remove(plex_id, plex_type) if len(plex_ids) < BATCH_SIZE: @@ -436,7 +442,7 @@ class FullSync(common.fullsync_mixin): def _run(self): self.current_sync = timing.plex_now() # Get latest Plex libraries and build playlist and video node files - if self.isCanceled() or not sections.sync_from_pms(self): + if self.should_cancel() or not sections.sync_from_pms(self): return self.successful = True try: @@ -447,7 +453,7 @@ class FullSync(common.fullsync_mixin): # Actual syncing - do only new items first LOG.info('Running full_library_sync with repair=%s', self.repair) - if self.isCanceled() or not self.full_library_sync(): + if self.should_cancel() or not self.full_library_sync(): self.successful = False return finally: @@ -457,7 +463,7 @@ class FullSync(common.fullsync_mixin): if self.threader: self.threader.shutdown() self.threader = None - if not self.successful and not self.isCanceled(): + if not self.successful and not self.should_cancel(): # "ERROR in library sync" utils.dialog('notification', heading='{plex}', @@ -468,4 +474,5 @@ class FullSync(common.fullsync_mixin): def start(show_dialog, repair=False, callback=None): + # Call run() and NOT start in order to not spawn another thread FullSync(repair, callback, show_dialog).run() diff --git a/resources/lib/library_sync/get_metadata.py b/resources/lib/library_sync/get_metadata.py index 6cd653c2..427169e2 100644 --- a/resources/lib/library_sync/get_metadata.py +++ b/resources/lib/library_sync/get_metadata.py @@ -2,7 +2,6 @@ from __future__ import absolute_import, division, unicode_literals from logging import getLogger -from . import common from ..plex_api import API from .. import plex_functions as PF, backgroundthread, utils, variables as v @@ -27,7 +26,7 @@ def reset_collections(): COLLECTION_XMLS = {} -class GetMetadataTask(common.fullsync_mixin, backgroundthread.Task): +class GetMetadataTask(backgroundthread.Task): """ Threaded download of Plex XML metadata for a certain library item. Fills the queue with the downloaded etree XML objects @@ -45,6 +44,12 @@ class GetMetadataTask(common.fullsync_mixin, backgroundthread.Task): self.count = count super(GetMetadataTask, self).__init__() + def suspend(self, block=False, timeout=None): + """ + Let's NOT suspend sync threads but immediately terminate them + """ + self.cancel() + def _collections(self, item): global COLLECTION_MATCH, COLLECTION_XMLS api = API(item['xml'][0]) @@ -59,7 +64,7 @@ class GetMetadataTask(common.fullsync_mixin, backgroundthread.Task): utils.cast(int, x.get('ratingKey'))) for x in COLLECTION_MATCH] item['children'] = {} for plex_set_id, set_name in api.collections(): - if self.isCanceled(): + if self.should_cancel(): return if plex_set_id not in COLLECTION_XMLS: # Get Plex metadata for collections - a pain @@ -84,7 +89,7 @@ class GetMetadataTask(common.fullsync_mixin, backgroundthread.Task): """ Do the work """ - if self.isCanceled(): + if self.should_cancel(): return # Download Metadata item = { @@ -101,7 +106,7 @@ class GetMetadataTask(common.fullsync_mixin, backgroundthread.Task): 'Cancelling sync for now') utils.window('plex_scancrashed', value='401') return - if not self.isCanceled() and self.plex_type == v.PLEX_TYPE_MOVIE: + if not self.should_cancel() and self.plex_type == v.PLEX_TYPE_MOVIE: # Check for collections/sets collections = False for child in item['xml'][0]: @@ -112,7 +117,7 @@ class GetMetadataTask(common.fullsync_mixin, backgroundthread.Task): global LOCK with LOCK: self._collections(item) - if not self.isCanceled() and self.get_children: + if not self.should_cancel() and self.get_children: children_xml = PF.GetAllPlexChildren(self.plex_id) try: children_xml[0].attrib @@ -121,5 +126,5 @@ class GetMetadataTask(common.fullsync_mixin, backgroundthread.Task): self.plex_id) else: item['children'] = children_xml - if not self.isCanceled(): + if not self.should_cancel(): self.queue.put((self.count, item)) diff --git a/resources/lib/library_sync/sections.py b/resources/lib/library_sync/sections.py index a1b78f23..451673b3 100644 --- a/resources/lib/library_sync/sections.py +++ b/resources/lib/library_sync/sections.py @@ -16,7 +16,7 @@ LOG = getLogger('PLEX.sync.sections') BATCH_SIZE = 500 # Need a way to interrupt our synching process -IS_CANCELED = None +SHOULD_CANCEL = None LIBRARY_PATH = path_ops.translate_path('special://profile/library/video/') # The video library might not yet exist for this user - create it @@ -490,7 +490,7 @@ def _delete_kodi_db_items(section): with kodi_context(texture_db=True) as kodidb: typus = context(None, plexdb=plexdb, kodidb=kodidb) for plex_id in plex_ids: - if IS_CANCELED(): + if SHOULD_CANCEL(): return False typus.remove(plex_id) if len(plex_ids) < BATCH_SIZE: @@ -582,13 +582,13 @@ def sync_from_pms(parent_self, pick_libraries=False): pick_libraries=True will prompt the user the select the libraries he wants to sync """ - global IS_CANCELED + global SHOULD_CANCEL LOG.info('Starting synching sections from the PMS') - IS_CANCELED = parent_self.isCanceled + SHOULD_CANCEL = parent_self.should_cancel try: return _sync_from_pms(pick_libraries) finally: - IS_CANCELED = None + SHOULD_CANCEL = None LOG.info('Done synching sections from the PMS: %s', app.SYNC.sections) diff --git a/resources/lib/playlists/__init__.py b/resources/lib/playlists/__init__.py index b3658ba0..3553c655 100644 --- a/resources/lib/playlists/__init__.py +++ b/resources/lib/playlists/__init__.py @@ -38,7 +38,7 @@ SUPPORTED_FILETYPES = ( ############################################################################### -def isCanceled(): +def should_cancel(): return app.APP.stop_pkc or app.SYNC.stop_sync @@ -167,7 +167,7 @@ def _full_sync(): # before. If yes, make sure that hashes are identical. If not, sync it. old_plex_ids = db.plex_playlist_ids() for xml_playlist in xml: - if isCanceled(): + if should_cancel(): return False api = API(xml_playlist) try: @@ -199,7 +199,7 @@ def _full_sync(): LOG.info('Could not recreate playlist %s', api.plex_id) # Get rid of old Plex playlists that were deleted on the Plex side for plex_id in old_plex_ids: - if isCanceled(): + if should_cancel(): return False playlist = db.get_playlist(plex_id=plex_id) LOG.debug('Removing outdated Plex playlist from Kodi: %s', playlist) @@ -213,7 +213,7 @@ def _full_sync(): old_kodi_paths = db.kodi_playlist_paths() for root, _, files in path_ops.walk(v.PLAYLIST_PATH): for f in files: - if isCanceled(): + if should_cancel(): return False path = path_ops.path.join(root, f) try: @@ -244,7 +244,7 @@ def _full_sync(): except PlaylistError: LOG.info('Skipping Kodi playlist %s', path) for kodi_path in old_kodi_paths: - if isCanceled(): + if should_cancel(): return False playlist = db.get_playlist(path=kodi_path) if not playlist: diff --git a/resources/lib/playqueue.py b/resources/lib/playqueue.py index 7e78bee9..45fb1d50 100644 --- a/resources/lib/playqueue.py +++ b/resources/lib/playqueue.py @@ -120,7 +120,7 @@ class PlayqueueMonitor(backgroundthread.KillableThread): # Ignore new media added by other addons continue for j, old_item in enumerate(old): - if self.isCanceled(): + if self.should_suspend() or self.should_cancel(): # Chances are that we got an empty Kodi playlist due to # Kodi exit return @@ -189,7 +189,7 @@ class PlayqueueMonitor(backgroundthread.KillableThread): for j in range(i, len(index)): index[j] += 1 for i in reversed(index): - if self.isCanceled(): + if self.should_suspend() or self.should_cancel(): # Chances are that we got an empty Kodi playlist due to # Kodi exit return @@ -212,9 +212,10 @@ class PlayqueueMonitor(backgroundthread.KillableThread): LOG.info("----===## PlayqueueMonitor stopped ##===----") def _run(self): - while not self.isCanceled(): - if self.wait_while_suspended(): - return + while not self.should_cancel(): + if self.should_suspend(): + if self.wait_while_suspended(): + return with app.APP.lock_playqueues: for playqueue in PLAYQUEUES: kodi_pl = js.playlist_get_items(playqueue.playlistid) @@ -228,4 +229,4 @@ class PlayqueueMonitor(backgroundthread.KillableThread): # compare old and new playqueue self._compare_playqueues(playqueue, kodi_pl) playqueue.old_kodi_pl = list(kodi_pl) - app.APP.monitor.waitForAbort(0.2) + self.sleep(0.2) diff --git a/resources/lib/plex_companion.py b/resources/lib/plex_companion.py index 036ac864..53a73c4c 100644 --- a/resources/lib/plex_companion.py +++ b/resources/lib/plex_companion.py @@ -312,12 +312,13 @@ class PlexCompanion(backgroundthread.KillableThread): if httpd: thread = Thread(target=httpd.handle_request) - while not self.isCanceled(): + while not self.should_cancel(): # If we are not authorized, sleep # Otherwise, we trigger a download which leads to a # re-authorizations - if self.wait_while_suspended(): - break + if self.should_suspend(): + if self.wait_while_suspended(): + break try: message_count += 1 if httpd: @@ -356,6 +357,6 @@ class PlexCompanion(backgroundthread.KillableThread): app.APP.companion_queue.task_done() # Don't sleep continue - app.APP.monitor.waitForAbort(0.05) + self.sleep(0.05) subscription_manager.signal_stop() client.stop_all() diff --git a/resources/lib/service_entry.py b/resources/lib/service_entry.py index b7b10aed..6d0ac9df 100644 --- a/resources/lib/service_entry.py +++ b/resources/lib/service_entry.py @@ -101,7 +101,7 @@ class Service(object): self._init_done = True @staticmethod - def isCanceled(): + def should_cancel(): return xbmc.abortRequested or app.APP.stop_pkc def on_connection_check(self, result): @@ -437,7 +437,7 @@ class Service(object): self.playqueue = playqueue.PlayqueueMonitor() # Main PKC program loop - while not self.isCanceled(): + while not self.should_cancel(): # Check for PKC commands from other Python instances plex_command = utils.window('plexkodiconnect.command') diff --git a/resources/lib/sync.py b/resources/lib/sync.py index 87efc477..6ed3eae7 100644 --- a/resources/lib/sync.py +++ b/resources/lib/sync.py @@ -38,7 +38,9 @@ class Sync(backgroundthread.KillableThread): self.start_library_sync(show_dialog=True, repair=app.SYNC.run_lib_scan == 'repair', block=True) - if not self.sync_successful and not self.isSuspended() and not self.isCanceled(): + if (not self.sync_successful and + not self.should_suspend() and + not self.should_cancel()): # ERROR in library sync LOG.warn('Triggered full/repair sync has not been successful') elif app.SYNC.run_lib_scan == 'fanart': @@ -112,7 +114,7 @@ class Sync(backgroundthread.KillableThread): LOG.info('Not synching Plex artwork - not caching') return if self.image_cache_thread and self.image_cache_thread.is_alive(): - self.image_cache_thread.abort() + self.image_cache_thread.cancel() self.image_cache_thread.join() self.image_cache_thread = artwork.ImageCachingThread() self.image_cache_thread.start() @@ -163,10 +165,11 @@ class Sync(backgroundthread.KillableThread): utils.init_dbs() - while not self.isCanceled(): + while not self.should_cancel(): # In the event the server goes offline - if self.wait_while_suspended(): - return + if self.should_suspend(): + if self.wait_while_suspended(): + return if not install_sync_done: # Very FIRST sync ever upon installation or reset of Kodi DB LOG.info('Initial start-up full sync starting') @@ -188,7 +191,7 @@ class Sync(backgroundthread.KillableThread): self.start_image_cache_thread() else: LOG.error('Initial start-up full sync unsuccessful') - app.APP.monitor.waitForAbort(1) + self.sleep(1) xbmc.executebuiltin('InhibitIdleShutdown(false)') elif not initial_sync_done: @@ -205,7 +208,7 @@ class Sync(backgroundthread.KillableThread): self.start_image_cache_thread() else: LOG.info('Startup sync has not yet been successful') - app.APP.monitor.waitForAbort(1) + self.sleep(1) # Currently no db scan, so we could start a new scan else: @@ -240,9 +243,9 @@ class Sync(backgroundthread.KillableThread): library_sync.store_websocket_message(message) queue.task_done() # Sleep just a bit - app.APP.monitor.waitForAbort(0.01) + self.sleep(0.01) continue - app.APP.monitor.waitForAbort(0.1) + self.sleep(0.1) # Shut down playlist monitoring if playlist_monitor: playlist_monitor.stop() diff --git a/resources/lib/websocket_client.py b/resources/lib/websocket_client.py index e999e669..8ff767b1 100644 --- a/resources/lib/websocket_client.py +++ b/resources/lib/websocket_client.py @@ -19,7 +19,7 @@ class WebSocket(backgroundthread.KillableThread): def __init__(self): self.ws = None self.redirect_uri = None - self.sleeptime = 0 + self.sleeptime = 0.0 super(WebSocket, self).__init__() def process(self, opcode, message): @@ -46,15 +46,15 @@ class WebSocket(backgroundthread.KillableThread): def getUri(self): raise NotImplementedError - def __sleep(self): + def _sleep_cycle(self): """ Sleeps for 2^self.sleeptime where sleeping period will be doubled with each unsuccessful connection attempt. Will sleep at most 64 seconds """ - app.APP.monitor.waitForAbort(2**self.sleeptime) + self.sleep(2 ** self.sleeptime) if self.sleeptime < 6: - self.sleeptime += 1 + self.sleeptime += 1.0 def run(self): LOG.info("----===## Starting %s ##===----", self.__class__.__name__) @@ -69,9 +69,9 @@ class WebSocket(backgroundthread.KillableThread): LOG.info("##===---- %s Stopped ----===##", self.__class__.__name__) def _run(self): - while not self.isCanceled(): + while not self.should_cancel(): # In the event the server goes offline - if self.isSuspended(): + if self.should_suspend(): # Set in service.py if self.ws is not None: self.ws.close() @@ -99,11 +99,11 @@ class WebSocket(backgroundthread.KillableThread): # Server is probably offline LOG.debug("%s: IOError connecting", self.__class__.__name__) self.ws = None - self.__sleep() + self._sleep_cycle() except websocket.WebSocketTimeoutException: LOG.debug("%s: WebSocketTimeoutException", self.__class__.__name__) self.ws = None - self.__sleep() + self._sleep_cycle() except websocket.WebsocketRedirect as e: LOG.debug('301 redirect detected: %s', e) self.redirect_uri = e.headers.get('location', @@ -111,11 +111,11 @@ class WebSocket(backgroundthread.KillableThread): if self.redirect_uri: self.redirect_uri = self.redirect_uri.decode('utf-8') self.ws = None - self.__sleep() + self._sleep_cycle() except websocket.WebSocketException as e: LOG.debug('%s: WebSocketException: %s', self.__class__.__name__, e) self.ws = None - self.__sleep() + self._sleep_cycle() except Exception as e: LOG.error('%s: Unknown exception encountered when ' 'connecting: %s', self.__class__.__name__, e) @@ -123,9 +123,9 @@ class WebSocket(backgroundthread.KillableThread): LOG.error("%s: Traceback:\n%s", self.__class__.__name__, traceback.format_exc()) self.ws = None - self.__sleep() + self._sleep_cycle() else: - self.sleeptime = 0 + self.sleeptime = 0.0 except Exception as e: LOG.error("%s: Unknown exception encountered: %s", self.__class__.__name__, e) @@ -141,7 +141,7 @@ class PMS_Websocket(WebSocket): """ Websocket connection with the PMS for Plex Companion """ - def isSuspended(self): + def should_suspend(self): """ Returns True if the thread is suspended """ @@ -206,7 +206,7 @@ class Alexa_Websocket(WebSocket): """ Websocket connection to talk to Amazon Alexa. """ - def isSuspended(self): + def should_suspend(self): """ Overwrite method since we need to check for plex token """ diff --git a/resources/lib/windows/userselect.py b/resources/lib/windows/userselect.py index d68ac1ad..84427741 100644 --- a/resources/lib/windows/userselect.py +++ b/resources/lib/windows/userselect.py @@ -24,7 +24,7 @@ class UserThumbTask(backgroundthread.Task): def run(self): for user in self.users: - if self.isCanceled(): + if self.should_cancel(): return thumb, back = user.thumb, '' self.callback(user, thumb, back) From 8f86f43a93d704b57b0e2853e28d64708b011e7a Mon Sep 17 00:00:00 2001 From: croneter Date: Sun, 24 Nov 2019 09:33:16 +0100 Subject: [PATCH 05/50] Rewire library sync to speed it up and fix sync getting stuck in rare cases --- resources/lib/backgroundthread.py | 226 +++++++-- resources/lib/library_sync/common.py | 14 + .../lib/library_sync/fill_metadata_queue.py | 118 +++++ resources/lib/library_sync/full_sync.py | 450 ++++++------------ resources/lib/library_sync/get_metadata.py | 181 +++---- .../lib/library_sync/process_metadata.py | 104 ++++ resources/lib/library_sync/sections.py | 57 ++- 7 files changed, 693 insertions(+), 457 deletions(-) create mode 100644 resources/lib/library_sync/fill_metadata_queue.py create mode 100644 resources/lib/library_sync/process_metadata.py diff --git a/resources/lib/backgroundthread.py b/resources/lib/backgroundthread.py index 45f7602f..682dcd8f 100644 --- a/resources/lib/backgroundthread.py +++ b/resources/lib/backgroundthread.py @@ -2,12 +2,15 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import, division, unicode_literals from logging import getLogger +from time import time as _time import threading import Queue import heapq +from collections import deque + import xbmc -from . import utils, app +from . import utils, app, variables as v LOG = getLogger('PLEX.threads') @@ -36,8 +39,8 @@ class KillableThread(threading.Thread): """ self._canceled = True # Make sure thread is running in order to exit quickly - self._is_not_suspended.set() self._is_not_asleep.set() + self._is_not_suspended.set() def should_suspend(self): """ @@ -66,8 +69,8 @@ class KillableThread(threading.Thread): back to life """ self._suspended = False - self._is_not_suspended.set() self._is_not_asleep.set() + self._is_not_suspended.set() def wait_while_suspended(self): """ @@ -104,6 +107,166 @@ class KillableThread(threading.Thread): return not self._is_not_asleep.is_set() +class ProcessingQueue(Queue.Queue, object): + """ + Queue of queues that processes a queue completely before moving on to the + next queue. There's one queue per Section(). You need to initialize each + section with add_section(section) first. + Put tuples (count, item) into this queue, with count being the respective + position of the item in the queue, starting with 0 (zero). + (None, None) is the sentinel for a single queue being exhausted, added by + put_sentinel() + """ + def _init(self, maxsize): + self.queue = deque() + self._sections = deque() + self._queues = deque() + self._current_section = None + self._current_queue = None + self._counter = 0 + + def _qsize(self): + return self._current_queue._qsize() if self._current_queue else 0 + + def total_size(self): + """ + Return the approximate total size of all queues (not reliable!) + """ + self.mutex.acquire() + n = sum(q._qsize() for q in self._queues) if self._queues else 0 + self.mutex.release() + return n + + def put(self, item, block=True, timeout=None): + """Put an item into the queue. + + If optional args 'block' is true and 'timeout' is None (the default), + block if necessary until a free slot is available. If 'timeout' is + a non-negative number, it blocks at most 'timeout' seconds and raises + the Full exception if no free slot was available within that time. + Otherwise ('block' is false), put an item on the queue if a free slot + is immediately available, else raise the Full exception ('timeout' + is ignored in that case). + """ + self.not_full.acquire() + try: + if self.maxsize > 0: + if not block: + # Use >= instead of == due to OrderedQueue! + if self._qsize() >= self.maxsize: + raise Queue.Full + elif timeout is None: + while self._qsize() >= self.maxsize: + self.not_full.wait() + elif timeout < 0: + raise ValueError("'timeout' must be a non-negative number") + else: + endtime = _time() + timeout + while self._qsize() >= self.maxsize: + remaining = endtime - _time() + if remaining <= 0.0: + raise Queue.Full + self.not_full.wait(remaining) + if self._put(item) == 0: + # Only notify one waiting thread if this item is put into the + # current queue + self.not_empty.notify() + else: + # Be sure to signal not_empty only once! + self._unlock_after_section_change() + self.unfinished_tasks += 1 + finally: + self.not_full.release() + + def _put(self, item): + """ + Returns the index of the section in whose subqueue we need to put the + item into + """ + for i, section in enumerate(self._sections): + if item[1]['section'] == section: + self._queues[i]._put(item) + break + else: + raise RuntimeError('Could not find section for item %s' % item[1]) + return i + + def _unlock_after_section_change(self): + """ + Ugly work-around if we expected more items to be synced, but we had + to lower our section.number_of_items because PKC decided that nothing + changed and we don't need to sync the respective item(s). + get() thus might block indefinitely + """ + while (self._current_section and + self._counter == self._current_section.number_of_items): + LOG.debug('Signaling completion of current section') + self._init_next_section() + if self._current_queue and self._current_queue._qsize(): + LOG.debug('Signaling not_empty') + self.not_empty.notify() + + def put_sentinel(self, section): + """ + Adds a new empty section as a sentinel. Call with an empty Section() + object. + Once the get()-method returns None, you've received the sentinel and + you've thus exhausted the queue + """ + self.not_empty.acquire() + try: + section.number_of_items = 1 + self._add_section(section) + # Add the actual sentinel to the queue we just added + self._queues[-1]._put((None, None)) + self.unfinished_tasks += 1 + if len(self._queues) == 1: + # queue was already exhausted! + self._switch_queues() + self._counter = 0 + self.not_empty.notify() + else: + self._unlock_after_section_change() + finally: + self.not_empty.release() + + def add_section(self, section): + """ + Be sure to add all sections first before starting to pop items off this + queue or adding them to the queue + """ + self.mutex.acquire() + try: + self._add_section(section) + finally: + self.mutex.release() + + def _add_section(self, section): + self._sections.append(section) + self._queues.append( + OrderedQueue() if section.plex_type == v.PLEX_TYPE_ALBUM + else Queue.Queue()) + if self._current_section is None: + self._switch_queues() + + def _init_next_section(self): + self._sections.popleft() + self._queues.popleft() + self._counter = 0 + self._switch_queues() + + def _switch_queues(self): + self._current_section = self._sections[0] if self._sections else None + self._current_queue = self._queues[0] if self._queues else None + + def _get(self): + item = self._current_queue._get() + self._counter += 1 + if self._counter == self._current_section.number_of_items: + self._init_next_section() + return item[1] + + class OrderedQueue(Queue.PriorityQueue, object): """ Queue that enforces an order on the items it returns. An item you push @@ -111,58 +274,21 @@ class OrderedQueue(Queue.PriorityQueue, object): (index, item) where index=-1 is the item that will be returned first. The Queue will block until index=-1, 0, 1, 2, 3, ... is then made available + + maxsize will be rather fuzzy, as _qsize returns 0 if we're still waiting + for the next smalles index. put() thus might not block always when it + should. """ def __init__(self, maxsize=0): + self.next_index = 0 super(OrderedQueue, self).__init__(maxsize) - self.smallest = -1 - self.not_next_item = threading.Condition(self.mutex) - def _put(self, item, heappush=heapq.heappush): - heappush(self.queue, item) - if item[0] == self.smallest: - self.not_next_item.notify() + def _qsize(self, len=len): + return len(self.queue) if self.queue[0][0] == self.next_index else 0 - def get(self, block=True, timeout=None): - """Remove and return an item from the queue. - - If optional args 'block' is true and 'timeout' is None (the default), - block if necessary until an item is available. If 'timeout' is - a non-negative number, it blocks at most 'timeout' seconds and raises - the Empty exception if no item was available within that time. - Otherwise ('block' is false), return an item if one is immediately - available, else raise the Empty exception ('timeout' is ignored - in that case). - """ - self.not_empty.acquire() - try: - if not block: - if not self._qsize() or self.queue[0][0] != self.smallest: - raise Queue.Empty - elif timeout is None: - while not self._qsize(): - self.not_empty.wait() - while self.queue[0][0] != self.smallest: - self.not_next_item.wait() - elif timeout < 0: - raise ValueError("'timeout' must be a non-negative number") - else: - endtime = Queue._time() + timeout - while not self._qsize(): - remaining = endtime - Queue._time() - if remaining <= 0.0: - raise Queue.Empty - self.not_empty.wait(remaining) - while self.queue[0][0] != self.smallest: - remaining = endtime - Queue._time() - if remaining <= 0.0: - raise Queue.Empty - self.not_next_item.wait(remaining) - item = self._get() - self.smallest += 1 - self.not_full.notify() - return item - finally: - self.not_empty.release() + def _get(self, heappop=heapq.heappop): + self.next_index += 1 + return heappop(self.queue) class Tasks(list): diff --git a/resources/lib/library_sync/common.py b/resources/lib/library_sync/common.py index c2481956..d6aae66f 100644 --- a/resources/lib/library_sync/common.py +++ b/resources/lib/library_sync/common.py @@ -9,6 +9,20 @@ PLAYLIST_SYNC_ENABLED = (v.DEVICE != 'Microsoft UWP' and utils.settings('enablePlaylistSync') == 'true') +class LibrarySyncMixin(object): + def suspend(self, block=False, timeout=None): + """ + Let's NOT suspend sync threads but immediately terminate them + """ + self.cancel() + + def wait_while_suspended(self): + """ + Return immediately + """ + return self.should_cancel() + + def update_kodi_library(video=True, music=True): """ Updates the Kodi library and thus refreshes the Kodi views and widgets diff --git a/resources/lib/library_sync/fill_metadata_queue.py b/resources/lib/library_sync/fill_metadata_queue.py new file mode 100644 index 00000000..c92f7a96 --- /dev/null +++ b/resources/lib/library_sync/fill_metadata_queue.py @@ -0,0 +1,118 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, division, unicode_literals +from logging import getLogger +import Queue +from collections import deque + +from . import common +from ..plex_db import PlexDB +from .. import backgroundthread, app + +LOG = getLogger('PLEX.sync.fill_metadata_queue') + + +def batch_sizes(): + """ + Increase batch sizes in order to get download threads for an items xml + metadata started soon. Corresponds to batch sizes when downloading lists + of items from the PMS ('limitindex' in the PKC settings) + """ + for i in (50, 100, 200, 400): + yield i + while True: + yield 1000 + + +class FillMetadataQueue(common.LibrarySyncMixin, + backgroundthread.KillableThread, ): + """ + Threaded download of Plex XML metadata for a certain library item. + Fills the queue with the downloaded etree XML objects + + Input: + queue Queue.Queue() object where this thread will store + the downloaded metadata XMLs as etree objects + """ + def __init__(self, repair, section_queue, get_metadata_queue): + self.repair = repair + self.section_queue = section_queue + self.get_metadata_queue = get_metadata_queue + self.count = 0 + self.batch_size = batch_sizes() + super(FillMetadataQueue, self).__init__() + + def _loop(self, section, items): + while items and not self.should_cancel(): + try: + with PlexDB(lock=False) as plexdb: + while items and not self.should_cancel(): + last, plex_id, checksum = items.popleft() + if (not self.repair and + plexdb.checksum(plex_id, section.plex_type) == checksum): + continue + if last: + # We might have received LESS items from the PMS + # than anticipated. Ensures that our queues finish + section.number_of_items = self.count + 1 + self.get_metadata_queue.put((self.count, plex_id, section), + block=False) + self.count += 1 + except Queue.Full: + # Close the DB for speed! + LOG.debug('Queue full') + self.sleep(5) + while not self.should_cancel(): + try: + self.get_metadata_queue.put((self.count, plex_id, section), + block=False) + except Queue.Full: + LOG.debug('Queue fuller') + self.sleep(2) + else: + self.count += 1 + break + + def _process_section(self, section): + # Initialize only once to avoid loosing the last value before we're + # breaking the for loop + iterator = common.tag_last(section.iterator) + last = True + self.count = 0 + while not self.should_cancel(): + batch_size = next(self.batch_size) + LOG.debug('Process batch of size %s with count %s for section %s', + batch_size, self.count, section) + # Iterator will block for download - let's not do that when the + # DB connection is open + items = deque() + for i, (last, xml) in enumerate(iterator): + plex_id = int(xml.get('ratingKey')) + checksum = int('{}{}'.format( + plex_id, + xml.get('updatedAt', + xml.get('addedAt', '1541572987')))) + items.append((last, plex_id, checksum)) + if i == batch_size: + break + self._loop(section, items) + if last: + break + + def run(self): + LOG.debug('Starting %s thread', self.__class__.__name__) + app.APP.register_thread(self) + try: + while not self.should_cancel(): + section = self.section_queue.get() + self.section_queue.task_done() + if section is None: + break + self._process_section(section) + except Exception: + from .. import utils + utils.ERROR(notify=True) + finally: + # Signal the download metadata threads to stop with a sentinel + self.get_metadata_queue.put(None) + app.APP.deregister_thread(self) + LOG.debug('##===---- %s Stopped ----===##', self.__class__.__name__) diff --git a/resources/lib/library_sync/full_sync.py b/resources/lib/library_sync/full_sync.py index 609d45b6..87c6bafd 100644 --- a/resources/lib/library_sync/full_sync.py +++ b/resources/lib/library_sync/full_sync.py @@ -3,15 +3,15 @@ from __future__ import absolute_import, division, unicode_literals from logging import getLogger import Queue -import copy import xbmcgui -from .get_metadata import GetMetadataTask, reset_collections +from .get_metadata import GetMetadataThread +from .fill_metadata_queue import FillMetadataQueue +from .process_metadata import ProcessMetadataThread from . import common, sections from .. import utils, timing, backgroundthread, variables as v, app from .. import plex_functions as PF, itemtypes -from ..plex_db import PlexDB if common.PLAYLIST_SYNC_ENABLED: from .. import playlists @@ -19,222 +19,107 @@ if common.PLAYLIST_SYNC_ENABLED: LOG = getLogger('PLEX.sync.full_sync') # How many items will be put through the processing chain at once? -BATCH_SIZE = 2000 +BATCH_SIZE = 250 +# Size of queue for xmls to be downloaded from PMS for/and before processing +QUEUE_BUFFER = 50 +# Max number of xmls held in memory +MAX_QUEUE_SIZE = 500 # Safety margin to filter PMS items - how many seconds to look into the past? UPDATED_AT_SAFETY = 60 * 5 LAST_VIEWED_AT_SAFETY = 60 * 5 -class InitNewSection(object): - """ - Throw this into the queue used for ProcessMetadata to tell it which - Plex library section we're looking at - """ - def __init__(self, context, total_number_of_items, section_name, - section_id, plex_type): - self.context = context - self.total = total_number_of_items - self.name = section_name - self.id = section_id - self.plex_type = plex_type - - -class FullSync(backgroundthread.KillableThread): +class FullSync(common.LibrarySyncMixin, backgroundthread.KillableThread): def __init__(self, repair, callback, show_dialog): """ repair=True: force sync EVERY item """ self.repair = repair self.callback = callback - self.queue = None - self.process_thread = None - self.current_sync = None - self.plexdb = None - self.plex_type = None - self.section_type = None - self.worker_count = int(utils.settings('syncThreadNumber')) - self.item_count = 0 # For progress dialog self.show_dialog = show_dialog self.show_dialog_userdata = utils.settings('playstate_sync_indicator') == 'true' - self.dialog = None - self.total = 0 - self.current = 0 - self.processed = 0 - self.title = '' - self.section = None - self.section_name = None - self.section_type_text = None - self.context = None - self.get_children = None - self.successful = None - self.section_success = None + if self.show_dialog: + self.dialog = xbmcgui.DialogProgressBG() + self.dialog.create(utils.lang(39714)) + else: + self.dialog = None + + self.section_queue = Queue.Queue() + self.get_metadata_queue = Queue.Queue(maxsize=5000) + self.processing_queue = backgroundthread.ProcessingQueue(maxsize=500) + self.current_time = timing.plex_now() + self.last_section = sections.Section() + + self.successful = True self.install_sync_done = utils.settings('SyncInstallRunDone') == 'true' - self.threader = backgroundthread.ThreaderManager( - worker=backgroundthread.NonstoppingBackgroundWorker, - worker_count=self.worker_count) + self.threads = [ + GetMetadataThread(self.get_metadata_queue, self.processing_queue) + for _ in range(int(utils.settings('syncThreadNumber'))) + ] + for t in self.threads: + t.start() super(FullSync, self).__init__() - def suspend(self, block=False, timeout=None): - """ - Let's NOT suspend sync threads but immediately terminate them - """ - self.cancel() - - def update_progressbar(self): - if self.dialog: - try: - progress = int(float(self.current) / float(self.total) * 100.0) - except ZeroDivisionError: - progress = 0 - self.dialog.update(progress, - '%s (%s)' % (self.section_name, self.section_type_text), - '%s %s/%s' - % (self.title, self.current, self.total)) - if app.APP.is_playing_video: - self.dialog.close() - self.dialog = None - - def process_item(self, xml_item): - """ - Processes a single library item - """ - plex_id = int(xml_item.get('ratingKey')) - if not self.repair and self.plexdb.checksum(plex_id, self.plex_type) == \ - int('%s%s' % (plex_id, - xml_item.get('updatedAt', - xml_item.get('addedAt', 1541572987)))): + def update_progressbar(self, section, title, current): + if not self.dialog: return - self.threader.addTask(GetMetadataTask(self.queue, - plex_id, - self.plex_type, - self.get_children, - self.item_count)) - self.item_count += 1 - - def update_library(self): - LOG.debug('Writing changes to Kodi library now') - i = 0 - if not self.section: - _, self.section = self.queue.get() - self.queue.task_done() - while not self.should_cancel() and self.item_count > 0: - section = self.section - if not section: - break - LOG.debug('Start or continue processing section %s (%ss)', - section.name, section.plex_type) - self.processed = 0 - self.total = section.total - self.section_name = section.name - self.section_type_text = utils.lang( - v.TRANSLATION_FROM_PLEXTYPE[section.plex_type]) - with section.context(self.current_sync) as context: - while not self.should_cancel() and self.item_count > 0: - try: - _, item = self.queue.get(block=False) - except backgroundthread.Queue.Empty: - if self.threader.threader.working(): - self.sleep(0.02) - continue - else: - # Try again, in case a thread just finished - i += 1 - if i == 3: - break - continue - i = 0 - self.queue.task_done() - if isinstance(item, dict): - context.add_update(item['xml'][0], - section_name=section.name, - section_id=section.id, - children=item['children']) - self.title = item['xml'][0].get('title') - self.processed += 1 - elif isinstance(item, InitNewSection) or item is None: - self.section = item - break - else: - raise ValueError('Unknown type %s' % type(item)) - self.item_count -= 1 - self.current += 1 - self.update_progressbar() - if self.processed == 500: - self.processed = 0 - context.commit() - LOG.debug('Done writing changes to Kodi library') - - @utils.log_time - def addupdate_section(self, section): - LOG.debug('Processing library section for new or changed items %s', - section) - if not self.install_sync_done: - app.SYNC.path_verified = False + current += 1 try: - # Sync new, updated and deleted items - iterator = section.iterator - # Tell the processing thread about this new section - queue_info = InitNewSection(section.context, - iterator.total, - iterator.get('librarySectionTitle', - iterator.get('title1')), - section.section_id, - section.plex_type) - self.queue.put((-1, queue_info)) - last = True - # To keep track of the item-number in order to kill while loops - self.item_count = 0 - self.current = 0 - # Initialize only once to avoid loosing the last value before - # we're breaking the for loop - loop = common.tag_last(iterator) - while True: - # Check Plex DB to see what we need to add/update - with PlexDB() as self.plexdb: - for last, xml_item in loop: - if self.should_cancel(): - return False - self.process_item(xml_item) - if self.item_count == BATCH_SIZE: - break - # Make sure Plex DB above is closed before adding/updating! - self.update_library() - if last: - break - reset_collections() - return True - except RuntimeError: - LOG.error('Could not entirely process section %s', section) - return False + progress = int(float(current) / float(section.number_of_items) * 100.0) + except ZeroDivisionError: + progress = 0 + self.dialog.update(progress, + '%s (%s)' % (section.name, section.section_type_text), + '%s %s/%s' + % (title, current, section.number_of_items)) + if app.APP.is_playing_video: + self.dialog.close() + self.dialog = None @utils.log_time + def processing_loop_new_and_changed_items(self): + LOG.debug('Start working') + scanner_thread = FillMetadataQueue(self.repair, + self.section_queue, + self.get_metadata_queue) + scanner_thread.start() + process_thread = ProcessMetadataThread(self.current_time, + self.processing_queue, + self.update_progressbar) + process_thread.start() + LOG.debug('Waiting for scanner thread to finish up') + scanner_thread.join() + LOG.debug('Waiting for metadata download threads to finish up') + for t in self.threads: + t.join() + LOG.debug('Download metadata threads finished') + # Sentinel for the process_thread once we added everything else + self.processing_queue.put_sentinel(sections.Section()) + process_thread.join() + self.successful = process_thread.successful + LOG.debug('threads finished work. successful: %s', self.successful) + + @utils.log_time + def processing_loop_playstates(self): + while not self.should_cancel(): + section = self.section_queue.get() + self.section_queue.task_done() + if section is None: + break + self.playstate_per_section(section) + def playstate_per_section(self, section): LOG.debug('Processing %s playstates for library section %s', - section.iterator.total, section) + section.number_of_items, section) try: - # Sync new, updated and deleted items iterator = section.iterator - # Tell the processing thread about this new section - queue_info = InitNewSection(section.context, - iterator.total, - section.name, - section.section_id, - section.plex_type) - self.queue.put((-1, queue_info)) - self.total = iterator.total - self.section_name = section.name - self.section_type_text = utils.lang( - v.TRANSLATION_FROM_PLEXTYPE[section.plex_type]) - self.current = 0 - + iterator = common.tag_last(iterator) last = True - loop = common.tag_last(iterator) - while True: - with section.context(self.current_sync) as itemtype: - for i, (last, xml_item) in enumerate(loop): - if self.should_cancel(): - return False + while not self.should_cancel(): + with section.context(self.current_time) as itemtype: + for last, xml_item in iterator: + section.count += 1 if not itemtype.update_userdata(xml_item, section.plex_type): # Somehow did not sync this item yet itemtype.add_update(xml_item, @@ -242,22 +127,21 @@ class FullSync(backgroundthread.KillableThread): section_id=section.section_id) itemtype.plexdb.update_last_sync(int(xml_item.attrib['ratingKey']), section.plex_type, - self.current_sync) - self.current += 1 - self.update_progressbar() - if (i + 1) % (10 * BATCH_SIZE) == 0: + self.current_time) + self.update_progressbar(section, '', section.count) + if section.count % (10 * BATCH_SIZE) == 0: break if last: break - return True except RuntimeError: LOG.error('Could not entirely process section %s', section) - return False + self.successful = False - def threaded_get_iterators(self, kinds, queue, all_items=False): + def threaded_get_iterators(self, kinds, queue, all_items): """ Getting iterators is costly, so let's do it asynchronously """ + LOG.debug('Start threaded_get_iterators') try: for kind in kinds: for section in (x for x in app.SYNC.sections @@ -268,86 +152,58 @@ class FullSync(backgroundthread.KillableThread): if not section.sync_to_kodi: LOG.info('User chose to not sync section %s', section) continue - element = copy.deepcopy(section) - element.plex_type = kind[0] - element.section_type = element.plex_type - element.context = kind[2] - element.get_children = kind[3] - element.Queue = kind[4] + section = sections.get_sync_section(section, + plex_type=kind[0]) if self.repair or all_items: updated_at = None else: updated_at = section.last_sync - UPDATED_AT_SAFETY \ if section.last_sync else None try: - element.iterator = PF.get_section_iterator( + section.iterator = PF.get_section_iterator( section.section_id, - plex_type=element.plex_type, + plex_type=section.plex_type, updated_at=updated_at, last_viewed_at=None) except RuntimeError: - LOG.warn('Sync at least partially unsuccessful') - self.successful = False - self.section_success = False + LOG.error('Sync at least partially unsuccessful!') + LOG.error('Error getting section iterator %s', section) else: - queue.put(element) + section.number_of_items = section.iterator.total + if section.number_of_items > 0: + self.processing_queue.add_section(section) + queue.put(section) + LOG.debug('Put section in queue with %s items: %s', + section.number_of_items, section) except Exception: utils.ERROR(notify=True) finally: queue.put(None) + LOG.debug('Exiting threaded_get_iterators') def full_library_sync(self): - """ - """ - # structure: - # (plex_type, - # section_type, - # context for itemtype, - # download children items, e.g. songs for a specific album?, - # Queue) kinds = [ - (v.PLEX_TYPE_MOVIE, v.PLEX_TYPE_MOVIE, itemtypes.Movie, False, Queue.Queue), - (v.PLEX_TYPE_SHOW, v.PLEX_TYPE_SHOW, itemtypes.Show, False, Queue.Queue), - (v.PLEX_TYPE_SEASON, v.PLEX_TYPE_SHOW, itemtypes.Season, False, Queue.Queue), - (v.PLEX_TYPE_EPISODE, v.PLEX_TYPE_SHOW, itemtypes.Episode, False, Queue.Queue) + (v.PLEX_TYPE_MOVIE, v.PLEX_TYPE_MOVIE), + (v.PLEX_TYPE_SHOW, v.PLEX_TYPE_SHOW), + (v.PLEX_TYPE_SEASON, v.PLEX_TYPE_SHOW), + (v.PLEX_TYPE_EPISODE, v.PLEX_TYPE_SHOW) ] if app.SYNC.enable_music: kinds.extend([ - (v.PLEX_TYPE_ARTIST, v.PLEX_TYPE_ARTIST, itemtypes.Artist, False, Queue.Queue), - (v.PLEX_TYPE_ALBUM, v.PLEX_TYPE_ARTIST, itemtypes.Album, True, backgroundthread.OrderedQueue), + (v.PLEX_TYPE_ARTIST, v.PLEX_TYPE_ARTIST), + (v.PLEX_TYPE_ALBUM, v.PLEX_TYPE_ARTIST), ]) # ADD NEW ITEMS # Already start setting up the iterators. We need to enforce # syncing e.g. show before season before episode - iterator_queue = Queue.Queue() - task = backgroundthread.FunctionAsTask(self.threaded_get_iterators, - None, - kinds, - iterator_queue) - backgroundthread.BGThreader.addTask(task) - while True: - self.section_success = True - section = iterator_queue.get() - iterator_queue.task_done() - if section is None: - break - # Setup our variables - self.plex_type = section.plex_type - self.section_type = section.section_type - self.context = section.context - self.get_children = section.get_children - self.queue = section.Queue() - # Now do the heavy lifting - if self.should_cancel() or not self.addupdate_section(section): - return False - if self.section_success: - # Need to check because a thread might have missed to get - # some items from the PMS - with PlexDB() as plexdb: - # Set the new time mark for the next delta sync - plexdb.update_section_last_sync(section.section_id, - self.current_sync) + backgroundthread.KillableThread( + target=self.threaded_get_iterators, + args=(kinds, self.section_queue, False)).start() + # Do the heavy lifting + self.processing_loop_new_and_changed_items() common.update_kodi_library(video=True, music=True) + if self.should_cancel() or not self.successful: + return # Sync Plex playlists to Kodi and vice-versa if common.PLAYLIST_SYNC_ENABLED: @@ -357,48 +213,29 @@ class FullSync(backgroundthread.KillableThread): self.dialog = xbmcgui.DialogProgressBG() # "Synching playlists" self.dialog.create(utils.lang(39715)) - if not playlists.full_sync(): - return False + if not playlists.full_sync() or self.should_cancel(): + return # SYNC PLAYSTATE of ALL items (otherwise we won't pick up on items that # were set to unwatched). Also mark all items on the PMS to be able # to delete the ones still in Kodi - LOG.info('Start synching playstate and userdata for every item') - # In order to not delete all your songs again + LOG.debug('Start synching playstate and userdata for every item') if app.SYNC.enable_music: - # We don't need to enforce the album order now - kinds.pop(5) + # In order to not delete all your songs again kinds.extend([ - (v.PLEX_TYPE_ALBUM, v.PLEX_TYPE_ARTIST, itemtypes.Album, True, Queue.Queue), - (v.PLEX_TYPE_SONG, v.PLEX_TYPE_ARTIST, itemtypes.Song, True, Queue.Queue), + (v.PLEX_TYPE_SONG, v.PLEX_TYPE_ARTIST), ]) # Make sure we're not showing an item's title in the sync dialog - self.title = '' - self.threader.shutdown() - self.threader = None if not self.show_dialog_userdata and self.dialog: # Close the progress indicator dialog self.dialog.close() self.dialog = None - task = backgroundthread.FunctionAsTask(self.threaded_get_iterators, - None, - kinds, - iterator_queue, - all_items=True) - backgroundthread.BGThreader.addTask(task) - while True: - section = iterator_queue.get() - iterator_queue.task_done() - if section is None: - break - # Setup our variables - self.plex_type = section.plex_type - self.section_type = section.section_type - self.context = section.context - self.get_children = section.get_children - # Now do the heavy lifting - if self.should_cancel() or not self.playstate_per_section(section): - return False + backgroundthread.KillableThread( + target=self.threaded_get_iterators, + args=(kinds, self.section_queue, True)).start() + self.processing_loop_playstates() + if self.should_cancel() or not self.successful: + return # Delete movies that are not on Plex anymore LOG.debug('Looking for items to delete') @@ -417,60 +254,49 @@ class FullSync(backgroundthread.KillableThread): for plex_type, context in kinds: # Delete movies that are not on Plex anymore while True: - with context(self.current_sync) as ctx: - plex_ids = list(ctx.plexdb.plex_id_by_last_sync(plex_type, - self.current_sync, - BATCH_SIZE)) + with context(self.current_time) as ctx: + plex_ids = list( + ctx.plexdb.plex_id_by_last_sync(plex_type, + self.current_time, + BATCH_SIZE)) for plex_id in plex_ids: if self.should_cancel(): - return False + return ctx.remove(plex_id, plex_type) if len(plex_ids) < BATCH_SIZE: break - LOG.debug('Done deleting') - return True + LOG.debug('Done looking for items to delete') def run(self): app.APP.register_thread(self) + LOG.info('Running library sync with repair=%s', self.repair) try: - self._run() + self.run_full_library_sync() finally: app.APP.deregister_thread(self) - LOG.info('Done full_sync') + LOG.info('Library sync done. successful: %s', self.successful) @utils.log_time - def _run(self): - self.current_sync = timing.plex_now() - # Get latest Plex libraries and build playlist and video node files - if self.should_cancel() or not sections.sync_from_pms(self): - return - self.successful = True + def run_full_library_sync(self): try: - if self.show_dialog: - self.dialog = xbmcgui.DialogProgressBG() - self.dialog.create(utils.lang(39714)) - - # Actual syncing - do only new items first - LOG.info('Running full_library_sync with repair=%s', - self.repair) - if self.should_cancel() or not self.full_library_sync(): + # Get latest Plex libraries and build playlist and video node files + if self.should_cancel() or not sections.sync_from_pms(self): + return + if self.should_cancel(): self.successful = False return + self.full_library_sync() finally: common.update_kodi_library(video=True, music=True) if self.dialog: self.dialog.close() - if self.threader: - self.threader.shutdown() - self.threader = None if not self.successful and not self.should_cancel(): # "ERROR in library sync" utils.dialog('notification', heading='{plex}', message=utils.lang(39410), icon='{error}') - if self.callback: - self.callback(self.successful) + self.callback(self.successful) def start(show_dialog, repair=False, callback=None): diff --git a/resources/lib/library_sync/get_metadata.py b/resources/lib/library_sync/get_metadata.py index 427169e2..972e222c 100644 --- a/resources/lib/library_sync/get_metadata.py +++ b/resources/lib/library_sync/get_metadata.py @@ -2,73 +2,46 @@ from __future__ import absolute_import, division, unicode_literals from logging import getLogger +from . import common from ..plex_api import API -from .. import plex_functions as PF, backgroundthread, utils, variables as v - - -LOG = getLogger("PLEX." + __name__) +from .. import backgroundthread, plex_functions as PF, utils, variables as v +from .. import app +LOG = getLogger('PLEX.sync.get_metadata') LOCK = backgroundthread.threading.Lock() -# List of tuples: (collection index [as in an item's metadata with "Collection -# id"], collection plex id) -COLLECTION_MATCH = None -# Dict with entries of the form : -COLLECTION_XMLS = {} -def reset_collections(): - """ - Collections seem unique to Plex sections - """ - global LOCK, COLLECTION_MATCH, COLLECTION_XMLS - with LOCK: - COLLECTION_MATCH = None - COLLECTION_XMLS = {} - - -class GetMetadataTask(backgroundthread.Task): +class GetMetadataThread(common.LibrarySyncMixin, + backgroundthread.KillableThread): """ Threaded download of Plex XML metadata for a certain library item. Fills the queue with the downloaded etree XML objects - - Input: - queue Queue.Queue() object where this thread will store - the downloaded metadata XMLs as etree objects """ - def __init__(self, queue, plex_id, plex_type, get_children=False, - count=None): - self.queue = queue - self.plex_id = plex_id - self.plex_type = plex_type - self.get_children = get_children - self.count = count - super(GetMetadataTask, self).__init__() - - def suspend(self, block=False, timeout=None): - """ - Let's NOT suspend sync threads but immediately terminate them - """ - self.cancel() + def __init__(self, get_metadata_queue, processing_queue): + self.get_metadata_queue = get_metadata_queue + self.processing_queue = processing_queue + super(GetMetadataThread, self).__init__() def _collections(self, item): - global COLLECTION_MATCH, COLLECTION_XMLS api = API(item['xml'][0]) - if COLLECTION_MATCH is None: - COLLECTION_MATCH = PF.collections(api.library_section_id()) - if COLLECTION_MATCH is None: + collection_match = item['section'].collection_match + collection_xmls = item['section'].collection_xmls + if collection_match is None: + collection_match = PF.collections(api.library_section_id()) + if collection_match is None: LOG.error('Could not download collections') return # Extract what we need to know - COLLECTION_MATCH = \ + collection_match = \ [(utils.cast(int, x.get('index')), - utils.cast(int, x.get('ratingKey'))) for x in COLLECTION_MATCH] + utils.cast(int, x.get('ratingKey'))) for x in collection_match] item['children'] = {} for plex_set_id, set_name in api.collections(): if self.should_cancel(): return - if plex_set_id not in COLLECTION_XMLS: + if plex_set_id not in collection_xmls: # Get Plex metadata for collections - a pain - for index, collection_plex_id in COLLECTION_MATCH: + for index, collection_plex_id in collection_match: if index == plex_set_id: collection_xml = PF.GetPlexMetadata(collection_plex_id) try: @@ -77,54 +50,84 @@ class GetMetadataTask(backgroundthread.Task): LOG.error('Could not get collection %s %s', collection_plex_id, set_name) continue - COLLECTION_XMLS[plex_set_id] = collection_xml + collection_xmls[plex_set_id] = collection_xml break else: LOG.error('Did not find Plex collection %s %s', plex_set_id, set_name) continue - item['children'][plex_set_id] = COLLECTION_XMLS[plex_set_id] + item['children'][plex_set_id] = collection_xmls[plex_set_id] + + def _process_abort(self, count, section): + # Make sure other threads will also receive sentinel + self.get_metadata_queue.put(None) + if count is not None: + self._process_skipped_item(count, section) + + def _process_skipped_item(self, count, section): + section.sync_successful = False + # Add a "dummy" item so we're not skipping a beat + self.processing_queue.put((count, {'section': section, 'xml': None})) def run(self): - """ - Do the work - """ - if self.should_cancel(): - return - # Download Metadata - item = { - 'xml': PF.GetPlexMetadata(self.plex_id), - 'children': None - } - if item['xml'] is None: - # Did not receive a valid XML - skip that item for now - LOG.error("Could not get metadata for %s. Skipping that item " - "for now", self.plex_id) - return - elif item['xml'] == 401: - LOG.error('HTTP 401 returned by PMS. Too much strain? ' - 'Cancelling sync for now') - utils.window('plex_scancrashed', value='401') - return - if not self.should_cancel() and self.plex_type == v.PLEX_TYPE_MOVIE: - # Check for collections/sets - collections = False - for child in item['xml'][0]: - if child.tag == 'Collection': - collections = True - break - if collections: - global LOCK - with LOCK: - self._collections(item) - if not self.should_cancel() and self.get_children: - children_xml = PF.GetAllPlexChildren(self.plex_id) + LOG.debug('Starting %s thread', self.__class__.__name__) + app.APP.register_thread(self) + try: + self._run() + finally: + app.APP.deregister_thread(self) + LOG.debug('##===---- %s Stopped ----===##', self.__class__.__name__) + + def _run(self): + while True: + item = self.get_metadata_queue.get() try: - children_xml[0].attrib - except (TypeError, IndexError, AttributeError): - LOG.error('Could not get children for Plex id %s', - self.plex_id) - else: - item['children'] = children_xml - if not self.should_cancel(): - self.queue.put((self.count, item)) + if item is None or self.should_cancel(): + self._process_abort(item[0] if item else None, + item[2] if item else None) + break + count, plex_id, section = item + item = { + 'xml': PF.GetPlexMetadata(plex_id), # This will block + 'children': None, + 'section': section + } + if item['xml'] is None: + # Did not receive a valid XML - skip that item for now + LOG.error("Could not get metadata for %s. Skipping item " + "for now", plex_id) + self._process_skipped_item(count, section) + continue + elif item['xml'] == 401: + LOG.error('HTTP 401 returned by PMS. Too much strain? ' + 'Cancelling sync for now') + utils.window('plex_scancrashed', value='401') + self._process_abort(count, section) + break + if section.plex_type == v.PLEX_TYPE_MOVIE: + # Check for collections/sets + collections = False + for child in item['xml'][0]: + if child.tag == 'Collection': + collections = True + break + if collections: + with LOCK: + self._collections(item) + if section.get_children: + if self.should_cancel(): + self._process_abort(count, section) + break + children_xml = PF.GetAllPlexChildren(plex_id) # Will block + try: + children_xml[0].attrib + except (TypeError, IndexError, AttributeError): + LOG.error('Could not get children for Plex id %s', + plex_id) + self._process_skipped_item(count, section) + continue + else: + item['children'] = children_xml + self.processing_queue.put((count, item)) + finally: + self.get_metadata_queue.task_done() diff --git a/resources/lib/library_sync/process_metadata.py b/resources/lib/library_sync/process_metadata.py new file mode 100644 index 00000000..abc70fdd --- /dev/null +++ b/resources/lib/library_sync/process_metadata.py @@ -0,0 +1,104 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, division, unicode_literals +from logging import getLogger + +from . import common, sections +from ..plex_db import PlexDB +from .. import backgroundthread, app + +LOG = getLogger('PLEX.sync.process_metadata') + +COMMIT_TO_DB_EVERY_X_ITEMS = 500 + + +class ProcessMetadataThread(common.LibrarySyncMixin, + backgroundthread.KillableThread): + """ + Invoke once in order to process the received PMS metadata xmls + """ + def __init__(self, current_time, processing_queue, update_progressbar): + self.current_time = current_time + self.processing_queue = processing_queue + self.update_progressbar = update_progressbar + self.last_section = sections.Section() + self.successful = True + super(ProcessMetadataThread, self).__init__() + + def start_section(self, section): + if section != self.last_section: + if self.last_section: + self.finish_last_section() + LOG.debug('Start or continue processing section %s', section) + self.last_section = section + # Warn the user for this new section if we cannot access a file + app.SYNC.path_verified = False + else: + LOG.debug('Resume processing section %s', section) + + def finish_last_section(self): + if (not self.should_cancel() and self.last_section and + self.last_section.sync_successful): + # Check for should_cancel() because we cannot be sure that we + # processed every item of the section + with PlexDB() as plexdb: + # Set the new time mark for the next delta sync + plexdb.update_section_last_sync(self.last_section.section_id, + self.current_time) + LOG.info('Finished processing section successfully: %s', + self.last_section) + elif self.last_section and not self.last_section.sync_successful: + LOG.warn('Sync not successful for section %s', self.last_section) + self.successful = False + + def _get(self): + item = {'xml': None} + while not self.should_cancel() and item and item['xml'] is None: + item = self.processing_queue.get() + self.processing_queue.task_done() + return item + + def run(self): + LOG.debug('Starting %s thread', self.__class__.__name__) + app.APP.register_thread(self) + try: + self._run() + except Exception: + from .. import utils + utils.ERROR(notify=True) + finally: + app.APP.deregister_thread(self) + LOG.debug('##===---- %s Stopped ----===##', self.__class__.__name__) + + def _run(self): + # There are 2 sentinels: None for aborting/ending this thread, the dict + # {'section': section, 'xml': None} for skipped/invalid items + item = self._get() + if item: + section = item['section'] + processed = 0 + self.start_section(section) + while not self.should_cancel(): + if item is None: + break + elif item['section'] != section: + # We received an entirely new section + self.start_section(item['section']) + section = item['section'] + with section.context(self.current_time) as context: + while not self.should_cancel(): + if item is None or item['section'] != section: + break + self.update_progressbar(section, + item['xml'][0].get('title'), + section.count) + context.add_update(item['xml'][0], + section_name=section.name, + section_id=section.section_id, + children=item['children']) + processed += 1 + section.count += 1 + if processed == COMMIT_TO_DB_EVERY_X_ITEMS: + processed = 0 + context.commit() + item = self._get() + self.finish_last_section() diff --git a/resources/lib/library_sync/sections.py b/resources/lib/library_sync/sections.py index 451673b3..da97e455 100644 --- a/resources/lib/library_sync/sections.py +++ b/resources/lib/library_sync/sections.py @@ -54,6 +54,8 @@ class Section(object): self.content = None # unicode # Setting the section_type WILL re_set sync_to_kodi! self._section_type = None # unicode + # E.g. "season" or "movie" (translated) + self.section_type_text = None # Do we sync all items of this section to the Kodi DB? # This will be set with section_type!! self.sync_to_kodi = None # bool @@ -77,13 +79,9 @@ class Section(object): self.order = None # Original PMS xml for this section, including children self.xml = None - # Attributes that will be initialized later by full_sync.py - self.iterator = None - self.context = None - self.get_children = None # A section_type encompasses possible several plex_types! E.g. shows # contain shows, seasons, episodes - self.plex_type = None + self._plex_type = None if xml_element is not None: self.from_xml(xml_element) elif section_db_element: @@ -106,9 +104,14 @@ class Section(object): self.section_type is not None) def __eq__(self, section): + """ + Sections compare equal if their section_id, name and plex_type (first + prio) OR section_type (if there is no plex_type is set) compare equal + """ return (self.section_id == section.section_id and self.name == section.name and - self.section_type == section.section_type) + (self.plex_type == section.plex_type if self.plex_type else + self.section_type == section.section_type)) def __ne__(self, section): return not self == section @@ -140,6 +143,15 @@ class Section(object): else: self.sync_to_kodi = True + @property + def plex_type(self): + return self._plex_type + + @plex_type.setter + def plex_type(self, value): + self._plex_type = value + self.section_type_text = utils.lang(v.TRANSLATION_FROM_PLEXTYPE[value]) + @property def index(self): return self._index @@ -431,6 +443,39 @@ class Section(object): self.remove_from_plex() +def _get_children(plex_type): + if plex_type == v.PLEX_TYPE_ALBUM: + return True + else: + return False + + +def get_sync_section(section, plex_type): + """ + Deep-copies section and adds certain arguments in order to prep section + for the library sync + """ + section = copy.deepcopy(section) + section.plex_type = plex_type + section.context = itemtypes.ITEMTYPE_FROM_PLEXTYPE[plex_type] + section.get_children = _get_children(plex_type) + # Some more init stuff + # Has sync for this section been successful? + section.sync_successful = True + # List of tuples: (collection index [as in an item's metadata with + # "Collection id"], collection plex id) + section.collection_match = None + # Dict with entries of the form : + section.collection_xmls = {} + # Keep count during sync + section.count = 0 + # Total number of items that we need to sync + section.number_of_items = 0 + # Iterator to get one sync item after the other + section.iterator = None + return section + + def force_full_sync(): """ Resets the sync timestamp for all sections to 0, thus forcing a subsequent From a87dfa0a7a73a62e58865ef650ac6110f8f7fccc Mon Sep 17 00:00:00 2001 From: croneter Date: Fri, 6 Dec 2019 08:24:13 +0100 Subject: [PATCH 06/50] Don't use a dedicated thread to get section generators --- resources/lib/library_sync/full_sync.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/resources/lib/library_sync/full_sync.py b/resources/lib/library_sync/full_sync.py index 87c6bafd..d5706643 100644 --- a/resources/lib/library_sync/full_sync.py +++ b/resources/lib/library_sync/full_sync.py @@ -137,11 +137,11 @@ class FullSync(common.LibrarySyncMixin, backgroundthread.KillableThread): LOG.error('Could not entirely process section %s', section) self.successful = False - def threaded_get_iterators(self, kinds, queue, all_items): + def get_generators(self, kinds, queue, all_items): """ Getting iterators is costly, so let's do it asynchronously """ - LOG.debug('Start threaded_get_iterators') + LOG.debug('Start get_generators') try: for kind in kinds: for section in (x for x in app.SYNC.sections @@ -179,7 +179,7 @@ class FullSync(common.LibrarySyncMixin, backgroundthread.KillableThread): utils.ERROR(notify=True) finally: queue.put(None) - LOG.debug('Exiting threaded_get_iterators') + LOG.debug('Exiting get_generators') def full_library_sync(self): kinds = [ @@ -194,11 +194,8 @@ class FullSync(common.LibrarySyncMixin, backgroundthread.KillableThread): (v.PLEX_TYPE_ALBUM, v.PLEX_TYPE_ARTIST), ]) # ADD NEW ITEMS - # Already start setting up the iterators. We need to enforce - # syncing e.g. show before season before episode - backgroundthread.KillableThread( - target=self.threaded_get_iterators, - args=(kinds, self.section_queue, False)).start() + # We need to enforce syncing e.g. show before season before episode + self.get_generators(kinds, self.section_queue, False) # Do the heavy lifting self.processing_loop_new_and_changed_items() common.update_kodi_library(video=True, music=True) @@ -230,9 +227,7 @@ class FullSync(common.LibrarySyncMixin, backgroundthread.KillableThread): # Close the progress indicator dialog self.dialog.close() self.dialog = None - backgroundthread.KillableThread( - target=self.threaded_get_iterators, - args=(kinds, self.section_queue, True)).start() + self.get_generators(kinds, self.section_queue, True) self.processing_loop_playstates() if self.should_cancel() or not self.successful: return From 2744b9da7e9a594c794cf0463b0d4b63e15b679f Mon Sep 17 00:00:00 2001 From: croneter Date: Fri, 6 Dec 2019 08:54:21 +0100 Subject: [PATCH 07/50] Copy entire plex.db to avoid db-locks entirely --- resources/lib/db.py | 2 + .../lib/library_sync/fill_metadata_queue.py | 87 ++++--------------- resources/lib/library_sync/full_sync.py | 19 +++- resources/lib/plex_db/common.py | 7 +- resources/lib/variables.py | 1 + 5 files changed, 40 insertions(+), 76 deletions(-) diff --git a/resources/lib/db.py b/resources/lib/db.py index 30f91315..fc622ae5 100644 --- a/resources/lib/db.py +++ b/resources/lib/db.py @@ -74,6 +74,8 @@ def connect(media_type=None, wal_mode=True): """ if media_type == "plex": db_path = v.DB_PLEX_PATH + elif media_type == 'plex-copy': + db_path = v.DB_PLEX_COPY_PATH elif media_type == "music": db_path = v.DB_MUSIC_PATH elif media_type == "texture": diff --git a/resources/lib/library_sync/fill_metadata_queue.py b/resources/lib/library_sync/fill_metadata_queue.py index c92f7a96..7ed361b0 100644 --- a/resources/lib/library_sync/fill_metadata_queue.py +++ b/resources/lib/library_sync/fill_metadata_queue.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import, division, unicode_literals from logging import getLogger -import Queue -from collections import deque from . import common from ..plex_db import PlexDB @@ -11,92 +9,43 @@ from .. import backgroundthread, app LOG = getLogger('PLEX.sync.fill_metadata_queue') -def batch_sizes(): - """ - Increase batch sizes in order to get download threads for an items xml - metadata started soon. Corresponds to batch sizes when downloading lists - of items from the PMS ('limitindex' in the PKC settings) - """ - for i in (50, 100, 200, 400): - yield i - while True: - yield 1000 - - class FillMetadataQueue(common.LibrarySyncMixin, backgroundthread.KillableThread, ): """ Threaded download of Plex XML metadata for a certain library item. - Fills the queue with the downloaded etree XML objects - - Input: - queue Queue.Queue() object where this thread will store - the downloaded metadata XMLs as etree objects + Fills the queue with the downloaded etree XML objects. Will use a COPIED + plex.db file (plex-copy.db) in order to read much faster without the + writing thread stalling """ def __init__(self, repair, section_queue, get_metadata_queue): self.repair = repair self.section_queue = section_queue self.get_metadata_queue = get_metadata_queue - self.count = 0 - self.batch_size = batch_sizes() super(FillMetadataQueue, self).__init__() - def _loop(self, section, items): - while items and not self.should_cancel(): - try: - with PlexDB(lock=False) as plexdb: - while items and not self.should_cancel(): - last, plex_id, checksum = items.popleft() - if (not self.repair and - plexdb.checksum(plex_id, section.plex_type) == checksum): - continue - if last: - # We might have received LESS items from the PMS - # than anticipated. Ensures that our queues finish - section.number_of_items = self.count + 1 - self.get_metadata_queue.put((self.count, plex_id, section), - block=False) - self.count += 1 - except Queue.Full: - # Close the DB for speed! - LOG.debug('Queue full') - self.sleep(5) - while not self.should_cancel(): - try: - self.get_metadata_queue.put((self.count, plex_id, section), - block=False) - except Queue.Full: - LOG.debug('Queue fuller') - self.sleep(2) - else: - self.count += 1 - break - def _process_section(self, section): # Initialize only once to avoid loosing the last value before we're # breaking the for loop - iterator = common.tag_last(section.iterator) - last = True - self.count = 0 - while not self.should_cancel(): - batch_size = next(self.batch_size) - LOG.debug('Process batch of size %s with count %s for section %s', - batch_size, self.count, section) - # Iterator will block for download - let's not do that when the - # DB connection is open - items = deque() - for i, (last, xml) in enumerate(iterator): + LOG.debug('Process section %s with %s items', + section, section.number_of_items) + count = 0 + with PlexDB(lock=False, copy=True) as plexdb: + for xml in section.iterator: + if self.should_cancel(): + break plex_id = int(xml.get('ratingKey')) checksum = int('{}{}'.format( plex_id, xml.get('updatedAt', xml.get('addedAt', '1541572987')))) - items.append((last, plex_id, checksum)) - if i == batch_size: - break - self._loop(section, items) - if last: - break + if (not self.repair and + plexdb.checksum(plex_id, section.plex_type) == checksum): + continue + self.get_metadata_queue.put((count, plex_id, section)) + count += 1 + # We might have received LESS items from the PMS than anticipated. + # Ensures that our queues finish + section.number_of_items = count def run(self): LOG.debug('Starting %s thread', self.__class__.__name__) diff --git a/resources/lib/library_sync/full_sync.py b/resources/lib/library_sync/full_sync.py index d5706643..0312ec21 100644 --- a/resources/lib/library_sync/full_sync.py +++ b/resources/lib/library_sync/full_sync.py @@ -11,7 +11,7 @@ from .fill_metadata_queue import FillMetadataQueue from .process_metadata import ProcessMetadataThread from . import common, sections from .. import utils, timing, backgroundthread, variables as v, app -from .. import plex_functions as PF, itemtypes +from .. import plex_functions as PF, itemtypes, path_ops if common.PLAYLIST_SYNC_ENABLED: from .. import playlists @@ -77,6 +77,16 @@ class FullSync(common.LibrarySyncMixin, backgroundthread.KillableThread): self.dialog.close() self.dialog = None + @staticmethod + def copy_plex_db(): + """ + Takes the current plex.db file and copies it to plex-copy.db + This will allow us to have "concurrent" connections during adding/ + updating items, increasing sync speed tremendously. + Using the same DB with e.g. WAL mode did not really work out... + """ + path_ops.copyfile(v.DB_PLEX_PATH, v.DB_PLEX_COPY_PATH) + @utils.log_time def processing_loop_new_and_changed_items(self): LOG.debug('Start working') @@ -267,6 +277,9 @@ class FullSync(common.LibrarySyncMixin, backgroundthread.KillableThread): LOG.info('Running library sync with repair=%s', self.repair) try: self.run_full_library_sync() + except Exception: + utils.ERROR(notify=True) + self.successful = False finally: app.APP.deregister_thread(self) LOG.info('Library sync done. successful: %s', self.successful) @@ -277,9 +290,7 @@ class FullSync(common.LibrarySyncMixin, backgroundthread.KillableThread): # Get latest Plex libraries and build playlist and video node files if self.should_cancel() or not sections.sync_from_pms(self): return - if self.should_cancel(): - self.successful = False - return + self.copy_plex_db() self.full_library_sync() finally: common.update_kodi_library(video=True, music=True) diff --git a/resources/lib/plex_db/common.py b/resources/lib/plex_db/common.py index f150fa38..6a70cb42 100644 --- a/resources/lib/plex_db/common.py +++ b/resources/lib/plex_db/common.py @@ -20,18 +20,19 @@ SUPPORTED_KODI_TYPES = ( class PlexDBBase(object): """ - Plex database methods used for all types of items + Plex database methods used for all types of items. """ - def __init__(self, plexconn=None, lock=True): + def __init__(self, plexconn=None, lock=True, copy=False): # Allows us to use this class with a cursor instead of context mgr self.plexconn = plexconn self.cursor = self.plexconn.cursor() if self.plexconn else None self.lock = lock + self.copy = copy def __enter__(self): if self.lock: PLEXDB_LOCK.acquire() - self.plexconn = db.connect('plex') + self.plexconn = db.connect('plex-copy' if self.copy else 'plex') self.cursor = self.plexconn.cursor() return self diff --git a/resources/lib/variables.py b/resources/lib/variables.py index b36ba29c..2ccc241d 100644 --- a/resources/lib/variables.py +++ b/resources/lib/variables.py @@ -127,6 +127,7 @@ DB_MUSIC_PATH = None DB_TEXTURE_VERSION = None DB_TEXTURE_PATH = None DB_PLEX_PATH = try_decode(xbmc.translatePath("special://database/plex.db")) +DB_PLEX_COPY_PATH = try_decode(xbmc.translatePath("special://database/plex-copy.db")) EXTERNAL_SUBTITLE_TEMP_PATH = try_decode(xbmc.translatePath( "special://profile/addon_data/%s/temp/" % ADDON_ID)) From e257e5426e2bf558c9a4c8fa6476b554a3f6f1d9 Mon Sep 17 00:00:00 2001 From: croneter Date: Fri, 6 Dec 2019 11:56:16 +0100 Subject: [PATCH 08/50] Optimize adding values to Kodi databases by not using sqlite COALESCE command --- resources/lib/itemtypes/movies.py | 68 +++++------- resources/lib/itemtypes/tvshows.py | 135 ++++++++++-------------- resources/lib/kodi_db/music.py | 36 +++---- resources/lib/kodi_db/video.py | 162 +++++++++-------------------- 4 files changed, 141 insertions(+), 260 deletions(-) diff --git a/resources/lib/itemtypes/movies.py b/resources/lib/itemtypes/movies.py index 54758444..8f6f0263 100644 --- a/resources/lib/itemtypes/movies.py +++ b/resources/lib/itemtypes/movies.py @@ -74,31 +74,21 @@ class Movie(ItemBase): api.date_created()) if file_id != old_kodi_fileid: self.kodidb.remove_file(old_kodi_fileid) - rating_id = self.kodidb.get_ratingid(kodi_id, - v.KODI_TYPE_MOVIE) - self.kodidb.update_ratings(kodi_id, - v.KODI_TYPE_MOVIE, - "default", - api.rating(), - api.votecount(), - rating_id) - # update new uniqueid Kodi 17 + rating_id = self.kodidb.update_ratings(kodi_id, + v.KODI_TYPE_MOVIE, + "default", + api.rating(), + api.votecount()) if api.provider('imdb') is not None: - uniqueid = self.kodidb.get_uniqueid(kodi_id, - v.KODI_TYPE_MOVIE) - self.kodidb.update_uniqueid(kodi_id, - v.KODI_TYPE_MOVIE, - api.provider('imdb'), - "imdb", - uniqueid) + uniqueid = self.kodidb.update_uniqueid(kodi_id, + v.KODI_TYPE_MOVIE, + 'imdb', + api.provider('imdb')) elif api.provider('tmdb') is not None: - uniqueid = self.kodidb.get_uniqueid(kodi_id, - v.KODI_TYPE_MOVIE) - self.kodidb.update_uniqueid(kodi_id, - v.KODI_TYPE_MOVIE, - api.provider('tmdb'), - "tmdb", - uniqueid) + uniqueid = self.kodidb.update_uniqueid(kodi_id, + v.KODI_TYPE_MOVIE, + 'tmdb', + api.provider('tmdb')) else: self.kodidb.remove_uniqueid(kodi_id, v.KODI_TYPE_MOVIE) uniqueid = -1 @@ -114,27 +104,21 @@ class Movie(ItemBase): file_id = self.kodidb.add_file(filename, kodi_pathid, api.date_created()) - rating_id = self.kodidb.add_ratingid() - self.kodidb.add_ratings(rating_id, - kodi_id, - v.KODI_TYPE_MOVIE, - "default", - api.rating(), - api.votecount()) + rating_id = self.kodidb.add_ratings(kodi_id, + v.KODI_TYPE_MOVIE, + "default", + api.rating(), + api.votecount()) if api.provider('imdb') is not None: - uniqueid = self.kodidb.add_uniqueid_id() - self.kodidb.add_uniqueid(uniqueid, - kodi_id, - v.KODI_TYPE_MOVIE, - api.provider('imdb'), - "imdb") + uniqueid = self.kodidb.add_uniqueid(kodi_id, + v.KODI_TYPE_MOVIE, + api.provider('imdb'), + "imdb") elif api.provider('tmdb') is not None: - uniqueid = self.kodidb.add_uniqueid_id() - self.kodidb.add_uniqueid(uniqueid, - kodi_id, - v.KODI_TYPE_MOVIE, - api.provider('tmdb'), - "tmdb") + uniqueid = self.kodidb.add_uniqueid(kodi_id, + v.KODI_TYPE_MOVIE, + api.provider('tmdb'), + "tmdb") else: uniqueid = -1 self.kodidb.add_people(kodi_id, diff --git a/resources/lib/itemtypes/tvshows.py b/resources/lib/itemtypes/tvshows.py index f52a03f1..4a61091d 100644 --- a/resources/lib/itemtypes/tvshows.py +++ b/resources/lib/itemtypes/tvshows.py @@ -189,29 +189,21 @@ class Show(TvShowMixin, ItemBase): if update_item: LOG.info("UPDATE tvshow plex_id: %s - %s", plex_id, api.title()) # update new ratings Kodi 17 - rating_id = self.kodidb.get_ratingid(kodi_id, v.KODI_TYPE_SHOW) - self.kodidb.update_ratings(kodi_id, - v.KODI_TYPE_SHOW, - "default", - api.rating(), - api.votecount(), - rating_id) + rating_id = self.kodidb.update_ratings(kodi_id, + v.KODI_TYPE_SHOW, + "default", + api.rating(), + api.votecount()) if api.provider('tvdb') is not None: - uniqueid = self.kodidb.get_uniqueid(kodi_id, - v.KODI_TYPE_SHOW) - self.kodidb.update_uniqueid(kodi_id, - v.KODI_TYPE_SHOW, - api.provider('tvdb'), - 'tvdb', - uniqueid) + uniqueid = self.kodidb.update_uniqueid(kodi_id, + v.KODI_TYPE_SHOW, + 'tvdb', + api.provider('tvdb')) elif api.provider('tmdb') is not None: - uniqueid = self.kodidb.get_uniqueid(kodi_id, - v.KODI_TYPE_SHOW) - self.kodidb.update_uniqueid(kodi_id, - v.KODI_TYPE_SHOW, - api.provider('tmdb'), - 'tmdb', - uniqueid) + uniqueid = self.kodidb.update_uniqueid(kodi_id, + v.KODI_TYPE_SHOW, + 'tmdb', + api.provider('tmdb')) else: self.kodidb.remove_uniqueid(kodi_id, v.KODI_TYPE_SHOW) uniqueid = -1 @@ -239,27 +231,21 @@ class Show(TvShowMixin, ItemBase): LOG.info("ADD tvshow plex_id: %s - %s", plex_id, api.title()) # Link the path self.kodidb.add_showlinkpath(kodi_id, kodi_pathid) - rating_id = self.kodidb.get_ratingid(kodi_id, v.KODI_TYPE_SHOW) - self.kodidb.add_ratings(rating_id, - kodi_id, - v.KODI_TYPE_SHOW, - "default", - api.rating(), - api.votecount()) + rating_id = self.kodidb.add_ratings(kodi_id, + v.KODI_TYPE_SHOW, + "default", + api.rating(), + api.votecount()) if api.provider('tvdb'): - uniqueid = self.kodidb.add_uniqueid_id() - self.kodidb.add_uniqueid(uniqueid, - kodi_id, - v.KODI_TYPE_SHOW, - api.provider('tvdb'), - 'tvdb') + uniqueid = self.kodidb.add_uniqueid(kodi_id, + v.KODI_TYPE_SHOW, + api.provider('tvdb'), + 'tvdb') if api.provider('tmdb'): - uniqueid = self.kodidb.add_uniqueid_id() - self.kodidb.add_uniqueid(uniqueid, - kodi_id, - v.KODI_TYPE_SHOW, - api.provider('tmdb'), - 'tmdb') + uniqueid = self.kodidb.add_uniqueid(kodi_id, + v.KODI_TYPE_SHOW, + api.provider('tmdb'), + 'tmdb') else: uniqueid = -1 self.kodidb.add_people(kodi_id, @@ -489,30 +475,21 @@ class Episode(TvShowMixin, ItemBase): self.kodidb.remove_file(old_kodi_fileid) if not app.SYNC.direct_paths: self.kodidb.remove_file(old_kodi_fileid_2) - ratingid = self.kodidb.get_ratingid(kodi_id, - v.KODI_TYPE_EPISODE) - self.kodidb.update_ratings(kodi_id, - v.KODI_TYPE_EPISODE, - "default", - api.rating(), - api.votecount(), - ratingid) + ratingid = self.kodidb.update_ratings(kodi_id, + v.KODI_TYPE_EPISODE, + "default", + api.rating(), + api.votecount()) if api.provider('tvdb'): - uniqueid = self.kodidb.get_uniqueid(kodi_id, - v.KODI_TYPE_EPISODE) - self.kodidb.update_uniqueid(kodi_id, - v.KODI_TYPE_EPISODE, - api.provider('tvdb'), - "tvdb", - uniqueid) + uniqueid = self.kodidb.update_uniqueid(kodi_id, + v.KODI_TYPE_EPISODE, + 'tvdb', + api.provider('tvdb')) elif api.provider('tmdb'): - uniqueid = self.kodidb.get_uniqueid(kodi_id, - v.KODI_TYPE_EPISODE) - self.kodidb.update_uniqueid(kodi_id, - v.KODI_TYPE_EPISODE, - api.provider('tmdb'), - "tmdb", - uniqueid) + uniqueid = self.kodidb.update_uniqueid(kodi_id, + v.KODI_TYPE_EPISODE, + 'tmdb', + api.provider('tmdb')) else: self.kodidb.remove_uniqueid(kodi_id, v.KODI_TYPE_EPISODE) uniqueid = -1 @@ -537,6 +514,7 @@ class Episode(TvShowMixin, ItemBase): airs_before_episode, playurl, kodi_pathid, + uniqueid, kodi_fileid, # and NOT kodi_fileid_2 parent_id, api.userrating(), @@ -577,27 +555,21 @@ class Episode(TvShowMixin, ItemBase): else: kodi_fileid_2 = None - rating_id = self.kodidb.add_ratingid() - self.kodidb.add_ratings(rating_id, - kodi_id, - v.KODI_TYPE_EPISODE, - "default", - api.rating(), - api.votecount()) + rating_id = self.kodidb.add_ratings(kodi_id, + v.KODI_TYPE_EPISODE, + "default", + api.rating(), + api.votecount()) if api.provider('tvdb'): - uniqueid = self.kodidb.add_uniqueid_id() - self.kodidb.add_uniqueid(uniqueid, - kodi_id, - v.KODI_TYPE_EPISODE, - api.provider('tvdb'), - "tvdb") + uniqueid = self.kodidb.add_uniqueid(kodi_id, + v.KODI_TYPE_EPISODE, + api.provider('tvdb'), + "tvdb") elif api.provider('tmdb'): - uniqueid = self.kodidb.add_uniqueid_id() - self.kodidb.add_uniqueid(uniqueid, - kodi_id, - v.KODI_TYPE_EPISODE, - api.provider('tmdb'), - "tmdb") + uniqueid = self.kodidb.add_uniqueid(kodi_id, + v.KODI_TYPE_EPISODE, + api.provider('tmdb'), + "tmdb") else: uniqueid = -1 self.kodidb.add_people(kodi_id, @@ -624,6 +596,7 @@ class Episode(TvShowMixin, ItemBase): airs_before_episode, playurl, kodi_pathid, + uniqueid, parent_id, api.userrating()) self.kodidb.set_resume(kodi_fileid, diff --git a/resources/lib/kodi_db/music.py b/resources/lib/kodi_db/music.py index 77ede150..50993806 100644 --- a/resources/lib/kodi_db/music.py +++ b/resources/lib/kodi_db/music.py @@ -25,13 +25,9 @@ class KodiMusicDB(common.KodiDBBase): try: pathid = self.cursor.fetchone()[0] except TypeError: - self.cursor.execute("SELECT COALESCE(MAX(idPath),0) FROM path") - pathid = self.cursor.fetchone()[0] + 1 - self.cursor.execute(''' - INSERT INTO path(idPath, strPath, strHash) - VALUES (?, ?, ?) - ''', - (pathid, path, '123')) + self.cursor.execute('INSERT INTO path(strPath, strHash) VALUES (?, ?)', + (path, '123')) + pathid = self.cursor.lastrowid return pathid @db.catch_operationalerrors @@ -382,10 +378,9 @@ class KodiMusicDB(common.KodiDBBase): genreid = self.cursor.fetchone()[0] except TypeError: # Create the genre - self.cursor.execute('SELECT COALESCE(MAX(idGenre),0) FROM genre') - genreid = self.cursor.fetchone()[0] + 1 - self.cursor.execute('INSERT INTO genre(idGenre, strGenre) VALUES(?, ?)', - (genreid, genre)) + self.cursor.execute('INSERT INTO genre(strGenre) VALUES(?)', + (genre, )) + genreid = self.cursor.lastrowid self.cursor.execute(''' INSERT OR REPLACE INTO album_genre( idGenre, @@ -403,10 +398,9 @@ class KodiMusicDB(common.KodiDBBase): genreid = self.cursor.fetchone()[0] except TypeError: # Create the genre - self.cursor.execute('SELECT COALESCE(MAX(idGenre),0) FROM genre') - genreid = self.cursor.fetchone()[0] + 1 - self.cursor.execute('INSERT INTO genre(idGenre, strGenre) values(?, ?)', - (genreid, genre)) + self.cursor.execute('INSERT INTO genre(strGenre) VALUES (?)', + (genre, )) + genreid = self.cursor.lastrowid self.cursor.execute(''' INSERT OR REPLACE INTO song_genre( idGenre, @@ -550,15 +544,11 @@ class KodiMusicDB(common.KodiDBBase): except TypeError: # Krypton has a dummy first entry idArtist: 1 strArtist: # [Missing Tag] strMusicBrainzArtistID: Artist Tag Missing - self.cursor.execute('SELECT COALESCE(MAX(idArtist),1) FROM artist') - artistid = self.cursor.fetchone()[0] + 1 self.cursor.execute(''' - INSERT INTO artist( - idArtist, - strArtist, - strMusicBrainzArtistID) - VALUES (?, ?, ?) - ''', (artistid, name, musicbrainz)) + INSERT INTO artist(strArtist, strMusicBrainzArtistID) + VALUES (?, ?) + ''', (name, musicbrainz)) + artistid = self.cursor.lastrowid else: if artistname != name: self.cursor.execute('UPDATE artist SET strArtist = ? WHERE idArtist = ?', diff --git a/resources/lib/kodi_db/video.py b/resources/lib/kodi_db/video.py index 6d9e1692..415ebcb7 100644 --- a/resources/lib/kodi_db/video.py +++ b/resources/lib/kodi_db/video.py @@ -40,19 +40,15 @@ class KodiVideoDB(common.KodiDBBase): """ path_id = self.get_path(MOVIE_PATH) if path_id is None: - self.cursor.execute("SELECT COALESCE(MAX(idPath),0) FROM path") - path_id = self.cursor.fetchone()[0] + 1 query = ''' - INSERT INTO path(idPath, - strPath, + INSERT INTO path(strPath, strContent, strScraper, noUpdate, exclude) - VALUES (?, ?, ?, ?, ?, ?) + VALUES (?, ?, ?, ?, ?) ''' - self.cursor.execute(query, (path_id, - MOVIE_PATH, + self.cursor.execute(query, (MOVIE_PATH, 'movies', 'metadata.local', 1, @@ -60,19 +56,15 @@ class KodiVideoDB(common.KodiDBBase): # And TV shows path_id = self.get_path(SHOW_PATH) if path_id is None: - self.cursor.execute("SELECT COALESCE(MAX(idPath),0) FROM path") - path_id = self.cursor.fetchone()[0] + 1 query = ''' - INSERT INTO path(idPath, - strPath, + INSERT INTO path(strPath, strContent, strScraper, noUpdate, exclude) - VALUES (?, ?, ?, ?, ?, ?) + VALUES (?, ?, ?, ?, ?) ''' - self.cursor.execute(query, (path_id, - SHOW_PATH, + self.cursor.execute(query, (SHOW_PATH, 'tvshows', 'metadata.local', 1, @@ -89,13 +81,12 @@ class KodiVideoDB(common.KodiDBBase): path_ops.decode_path(path_ops.path.pardir))) pathid = self.get_path(parentpath) if pathid is None: - self.cursor.execute("SELECT COALESCE(MAX(idPath),0) FROM path") - pathid = self.cursor.fetchone()[0] + 1 self.cursor.execute(''' - INSERT INTO path(idPath, strPath, dateAdded) - VALUES (?, ?, ?) + INSERT INTO path(strPath, dateAdded) + VALUES (?, ?) ''', - (pathid, parentpath, timing.kodi_now())) + (parentpath, timing.kodi_now())) + pathid = self.cursor.lastrowid if parentpath != path: # In case we end up having media in the filesystem root, C:\ parent_id = self.parent_path_id(parentpath) @@ -127,21 +118,19 @@ class KodiVideoDB(common.KodiDBBase): try: pathid = self.cursor.fetchone()[0] except TypeError: - self.cursor.execute("SELECT COALESCE(MAX(idPath),0) FROM path") - pathid = self.cursor.fetchone()[0] + 1 self.cursor.execute(''' INSERT INTO path( - idPath, strPath, dateAdded, idParentPath, strContent, strScraper, noUpdate) - VALUES (?, ?, ?, ?, ?, ?, ?) + VALUES (?, ?, ?, ?, ?, ?) ''', - (pathid, path, date_added, id_parent_path, - content, scraper, 1)) + (path, date_added, id_parent_path, content, + scraper, 1)) + pathid = self.cursor.lastrowid return pathid def get_path(self, path): @@ -161,18 +150,12 @@ class KodiVideoDB(common.KodiDBBase): Adds the filename [unicode] to the table files if not already added and returns the idFile. """ - self.cursor.execute('SELECT COALESCE(MAX(idFile), 0) FROM files') - file_id = self.cursor.fetchone()[0] + 1 self.cursor.execute(''' - INSERT INTO files( - idFile, - idPath, - strFilename, - dateAdded) - VALUES (?, ?, ?, ?) + INSERT INTO files(idPath, strFilename, dateAdded) + VALUES (?, ?, ?) ''', - (file_id, path_id, filename, date_added)) - return file_id + (path_id, filename, date_added)) + return self.cursor.lastrowid def modify_file(self, filename, path_id, date_added): self.cursor.execute('SELECT idFile FROM files WHERE idPath = ? AND strFilename = ?', @@ -261,11 +244,9 @@ class KodiVideoDB(common.KodiDBBase): try: entry_id = self.cursor.fetchone()[0] except TypeError: - self.cursor.execute('SELECT COALESCE(MAX(%s), %s) FROM %s' - % (key, first_id - 1, table)) - entry_id = self.cursor.fetchone()[0] + 1 - self.cursor.execute('INSERT INTO %s(%s, name) values(?, ?)' - % (table, key), (entry_id, entry)) + self.cursor.execute('INSERT INTO %s(name) VALUES(?)' % table, + (entry, )) + entry_id = self.cursor.lastrowid finally: entry_ids.append(entry_id) # Now process the ids obtained from the names @@ -458,10 +439,8 @@ class KodiVideoDB(common.KodiDBBase): @db.catch_operationalerrors def _new_actor_id(self, name, art_url): # Not yet in actor DB, add person - self.cursor.execute('SELECT COALESCE(MAX(actor_id), 0) FROM actor') - actor_id = self.cursor.fetchone()[0] + 1 - self.cursor.execute('INSERT INTO actor(actor_id, name) VALUES (?, ?)', - (actor_id, name)) + self.cursor.execute('INSERT INTO actor(name) VALUES (?)', (name, )) + actor_id = self.cursor.lastrowid if art_url: self.add_art(art_url, actor_id, 'actor', 'thumb') return actor_id @@ -649,12 +628,8 @@ class KodiVideoDB(common.KodiDBBase): (playcount or None, dateplayed, file_id)) # Set the resume bookmark if resume_seconds: - self.cursor.execute( - 'SELECT COALESCE(MAX(idBookmark), 0) FROM bookmark') - bookmark_id = self.cursor.fetchone()[0] + 1 self.cursor.execute(''' INSERT INTO bookmark( - idBookmark, idFile, timeInSeconds, totalTimeInSeconds, @@ -662,9 +637,8 @@ class KodiVideoDB(common.KodiDBBase): player, playerState, type) - VALUES (?, ?, ?, ?, ?, ?, ?, ?) - ''', (bookmark_id, - file_id, + VALUES (?, ?, ?, ?, ?, ?, ?) + ''', (file_id, resume_seconds, total_seconds, '', @@ -682,10 +656,8 @@ class KodiVideoDB(common.KodiDBBase): try: tag_id = self.cursor.fetchone()[0] except TypeError: - self.cursor.execute("SELECT COALESCE(MAX(tag_id), 0) FROM tag") - tag_id = self.cursor.fetchone()[0] + 1 - self.cursor.execute('INSERT INTO tag(tag_id, name) VALUES(?, ?)', - (tag_id, name)) + self.cursor.execute('INSERT INTO tag(name) VALUES(?)', (name, )) + tag_id = self.cursor.lastrowid return tag_id @db.catch_operationalerrors @@ -717,10 +689,8 @@ class KodiVideoDB(common.KodiDBBase): try: setid = self.cursor.fetchone()[0] except TypeError: - self.cursor.execute("SELECT COALESCE(MAX(idSet), 0) FROM sets") - setid = self.cursor.fetchone()[0] + 1 - self.cursor.execute('INSERT INTO sets(idSet, strSet) VALUES(?, ?)', - (setid, set_name)) + self.cursor.execute('INSERT INTO sets(strSet) VALUES(?)', (set_name, )) + setid = self.cursor.lastrowid return setid @db.catch_operationalerrors @@ -768,19 +738,14 @@ class KodiVideoDB(common.KodiDBBase): Adds a TV show season to the Kodi video DB or simply returns the ID, if there already is an entry in the DB """ - self.cursor.execute("SELECT COALESCE(MAX(idSeason),0) FROM seasons") - seasonid = self.cursor.fetchone()[0] + 1 - self.cursor.execute(''' - INSERT INTO seasons(idSeason, idShow, season) - VALUES (?, ?, ?) - ''', (seasonid, showid, seasonnumber)) - return seasonid + self.cursor.execute('INSERT INTO seasons(idShow, season) VALUES (?, ?)', + (showid, seasonnumber)) + return self.cursor.lastrowid @db.catch_operationalerrors def add_uniqueid(self, *args): """ Feed with: - uniqueid_id: int media_id: int media_type: string value: string @@ -788,39 +753,24 @@ class KodiVideoDB(common.KodiDBBase): """ self.cursor.execute(''' INSERT INTO uniqueid( - uniqueid_id, media_id, media_type, value, type) - VALUES (?, ?, ?, ?, ?) + VALUES (?, ?, ?, ?) ''', (args)) - - def add_uniqueid_id(self): - self.cursor.execute('SELECT COALESCE(MAX(uniqueid_id), 0) FROM uniqueid') - return self.cursor.fetchone()[0] + 1 - - def get_uniqueid(self, kodi_id, kodi_type): - """ - Returns the uniqueid_id - """ - self.cursor.execute('SELECT uniqueid_id FROM uniqueid WHERE media_id = ? AND media_type =?', - (kodi_id, kodi_type)) - try: - return self.cursor.fetchone()[0] - except TypeError: - return self.add_uniqueid_id() + return self.cursor.lastrowid @db.catch_operationalerrors def update_uniqueid(self, *args): """ - Pass in media_id, media_type, value, type, uniqueid_id + Pass in value, media_id, media_type, type """ self.cursor.execute(''' - UPDATE uniqueid - SET media_id = ?, media_type = ?, value = ?, type = ? - WHERE uniqueid_id = ? + INSERT OR REPLACE INTO uniqueid(media_id, media_type, type, value) + VALUES(?, ?, ?, ?) ''', (args)) + return self.cursor.lastrowid @db.catch_operationalerrors def remove_uniqueid(self, kodi_id, kodi_type): @@ -830,54 +780,36 @@ class KodiVideoDB(common.KodiDBBase): self.cursor.execute('DELETE FROM uniqueid WHERE media_id = ? AND media_type = ?', (kodi_id, kodi_type)) - def add_ratingid(self): - self.cursor.execute('SELECT COALESCE(MAX(rating_id),0) FROM rating') - return self.cursor.fetchone()[0] + 1 - - def get_ratingid(self, kodi_id, kodi_type): - """ - Create if needed and return the unique rating_id from rating table - """ - self.cursor.execute('SELECT rating_id FROM rating WHERE media_id = ? AND media_type = ?', - (kodi_id, kodi_type)) - try: - return self.cursor.fetchone()[0] - except TypeError: - return self.add_ratingid() - @db.catch_operationalerrors def update_ratings(self, *args): """ Feed with media_id, media_type, rating_type, rating, votes, rating_id """ self.cursor.execute(''' - UPDATE rating - SET media_id = ?, - media_type = ?, - rating_type = ?, - rating = ?, - votes = ? - WHERE rating_id = ? + INSERT OR REPLACE INTO + rating(media_id, media_type, rating_type, rating, votes) + VALUES (?, ?, ?, ?, ?) ''', (args)) + return self.cursor.lastrowid @db.catch_operationalerrors def add_ratings(self, *args): """ feed with: - rating_id, media_id, media_type, rating_type, rating, votes + media_id, media_type, rating_type, rating, votes rating_type = 'default' """ self.cursor.execute(''' INSERT INTO rating( - rating_id, media_id, media_type, rating_type, rating, votes) - VALUES (?, ?, ?, ?, ?, ?) + VALUES (?, ?, ?, ?, ?) ''', (args)) + return self.cursor.lastrowid @db.catch_operationalerrors def remove_ratings(self, kodi_id, kodi_type): @@ -917,10 +849,11 @@ class KodiVideoDB(common.KodiDBBase): c16, c18, c19, + c20, idSeason, userrating) VALUES - (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ''', (args)) @db.catch_operationalerrors @@ -942,6 +875,7 @@ class KodiVideoDB(common.KodiDBBase): c16 = ?, c18 = ?, c19 = ?, + c20 = ?, idFile=?, idSeason = ?, userrating = ? From 9080ca89b9d1c9d7d352fc3cdb59e97f40b918f0 Mon Sep 17 00:00:00 2001 From: croneter Date: Fri, 6 Dec 2019 08:59:01 +0100 Subject: [PATCH 09/50] Don't use WAL mode for sqlite connections, it is not making any difference --- resources/lib/db.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/resources/lib/db.py b/resources/lib/db.py index 30f91315..c7e0e0cb 100644 --- a/resources/lib/db.py +++ b/resources/lib/db.py @@ -58,9 +58,10 @@ def _initial_db_connection_setup(conn, wal_mode): before. Also start a transaction """ if wal_mode: - conn.execute('PRAGMA journal_mode=WAL;') - conn.execute('PRAGMA cache_size = -8000;') - conn.execute('PRAGMA synchronous=NORMAL;') + pass + # conn.execute('PRAGMA journal_mode=WAL;') + # conn.execute('PRAGMA cache_size = -8000;') + # conn.execute('PRAGMA synchronous=NORMAL;') conn.execute('BEGIN') From 2f1cae5026d9d93f99e6643f4e067f9b5d687354 Mon Sep 17 00:00:00 2001 From: croneter Date: Wed, 4 Dec 2019 07:51:39 +0100 Subject: [PATCH 10/50] Beta version bump 2.10.5 --- README.md | 2 +- addon.xml | 13 +++++++++++-- changelog.txt | 9 +++++++++ 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 60b05038..4d746977 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ [![stable version](https://img.shields.io/badge/stable_version-2.10.4-blue.svg?maxAge=60&style=flat) ](https://github.com/croneter/binary_repo/raw/master/stable/repository.plexkodiconnect/repository.plexkodiconnect-1.0.2.zip) -[![beta version](https://img.shields.io/badge/beta_version-2.10.4-red.svg?maxAge=60&style=flat) ](https://github.com/croneter/binary_repo/raw/master/beta/repository.plexkodiconnectbeta/repository.plexkodiconnectbeta-1.0.2.zip) +[![beta version](https://img.shields.io/badge/beta_version-2.10.5-red.svg?maxAge=60&style=flat) ](https://github.com/croneter/binary_repo/raw/master/beta/repository.plexkodiconnectbeta/repository.plexkodiconnectbeta-1.0.2.zip) [![Installation](https://img.shields.io/badge/wiki-installation-brightgreen.svg?maxAge=60&style=flat)](https://github.com/croneter/PlexKodiConnect/wiki/Installation) [![FAQ](https://img.shields.io/badge/wiki-FAQ-brightgreen.svg?maxAge=60&style=flat)](https://github.com/croneter/PlexKodiConnect/wiki/faq) diff --git a/addon.xml b/addon.xml index 11c552df..81ef3bce 100644 --- a/addon.xml +++ b/addon.xml @@ -1,5 +1,5 @@ - + @@ -83,7 +83,16 @@ Natūralioji „Plex“ integracija į „Kodi“ Prijunkite „Kodi“ prie „Plex Medija Serverio“. Šiame papildinyje daroma prielaida, kad valdote visus savo vaizdo įrašus naudodami „Plex“ (ir nė vieno su „Kodi“). Galite prarasti jau saugomus „Kodi“ vaizdo įrašų ir muzikos duomenų bazių duomenis (kadangi šis papildinys juos tiesiogiai pakeičia). Naudokite savo pačių rizika! Naudokite savo pačių rizika - version 2.10.4: + version 2.10.5 (beta only): +- Rewire library sync to speed it up and fix sync getting stuck in rare cases +- Optimize threads by using events instead of a polling mechanism. Fixes PKC becoming unresponsive, e.g. when switching users +- Optimize adding values to Kodi databases by not using sqlite COALESCE command +- Fix OperationalError when resetting PKC +- Improve sync resiliance when certain items are not to be synced to Kodi or PKC skipped an item in the past +- Make sure bool is returned instead of an int +- Don't use WAL mode for sqlite connections, it is not making any difference + +version 2.10.4: - version 2.10.3 for everyone - Fix to correctly wipe Kodi databases diff --git a/changelog.txt b/changelog.txt index 07ee3840..c401c21b 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,12 @@ +version 2.10.5 (beta only): +- Rewire library sync to speed it up and fix sync getting stuck in rare cases +- Optimize threads by using events instead of a polling mechanism. Fixes PKC becoming unresponsive, e.g. when switching users +- Optimize adding values to Kodi databases by not using sqlite COALESCE command +- Fix OperationalError when resetting PKC +- Improve sync resiliance when certain items are not to be synced to Kodi or PKC skipped an item in the past +- Make sure bool is returned instead of an int +- Don't use WAL mode for sqlite connections, it is not making any difference + version 2.10.4: - version 2.10.3 for everyone - Fix to correctly wipe Kodi databases From ab998f79410bb2f8c152f858ff4c9661d7cd99bc Mon Sep 17 00:00:00 2001 From: croneter Date: Sun, 8 Dec 2019 16:24:14 +0100 Subject: [PATCH 11/50] Fix IndexError --- resources/lib/backgroundthread.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/resources/lib/backgroundthread.py b/resources/lib/backgroundthread.py index 682dcd8f..60acc59e 100644 --- a/resources/lib/backgroundthread.py +++ b/resources/lib/backgroundthread.py @@ -284,7 +284,10 @@ class OrderedQueue(Queue.PriorityQueue, object): super(OrderedQueue, self).__init__(maxsize) def _qsize(self, len=len): - return len(self.queue) if self.queue[0][0] == self.next_index else 0 + try: + return len(self.queue) if self.queue[0][0] == self.next_index else 0 + except IndexError: + return 0 def _get(self, heappop=heapq.heappop): self.next_index += 1 From f9755cc39c71fd33ff476fccb95361a4cd48e6b0 Mon Sep 17 00:00:00 2001 From: croneter Date: Sun, 8 Dec 2019 16:31:19 +0100 Subject: [PATCH 12/50] Fix AttributeError if user enters an invalid pin code --- resources/lib/windows/userselect.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/resources/lib/windows/userselect.py b/resources/lib/windows/userselect.py index 84427741..f6e5ba7b 100644 --- a/resources/lib/windows/userselect.py +++ b/resources/lib/windows/userselect.py @@ -169,13 +169,13 @@ class UserSelectWindow(kodigui.BaseWindow): utils.settings('plexToken'), utils.settings('plex_machineIdentifier')) if self.user.authToken is None: - self.user = None item.setProperty('pin', item.dataSource.title) item.setProperty('editing.pin', '') # 'Error': 'Login failed with plex.tv for user' utils.messageDialog(utils.lang(30135), - '%s %s' % (utils.lang(39229), - self.user.username)) + '{}{}'.format(utils.lang(39229), + self.user.username)) + self.user = None return self.doClose() From 80181873d1e6a2b7b3762d9ba03e484afd30fd88 Mon Sep 17 00:00:00 2001 From: croneter Date: Sun, 8 Dec 2019 16:41:43 +0100 Subject: [PATCH 13/50] Fix OperationalError when starting with a fresh PKC installation --- resources/lib/playlists/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/resources/lib/playlists/__init__.py b/resources/lib/playlists/__init__.py index 7463557d..1e2ff6bc 100644 --- a/resources/lib/playlists/__init__.py +++ b/resources/lib/playlists/__init__.py @@ -13,6 +13,7 @@ """ from __future__ import absolute_import, division, unicode_literals from logging import getLogger +from sqlite3 import OperationalError from .common import Playlist, PlaylistError, PlaylistObserver, \ kodi_playlist_hash @@ -74,7 +75,11 @@ def remove_synced_playlists(): """ LOG.info('Removing all playlists that we synced to Kodi') with app.APP.lock_playlists: - paths = db.get_all_kodi_playlist_paths() + try: + paths = db.get_all_kodi_playlist_paths() + except OperationalError: + LOG.info('Playlists table has not yet been set-up') + return kodi_pl.delete_kodi_playlists(paths) db.wipe_table() LOG.info('Done removing all synced playlists') From 0c337d8aaeb56d62baf8621f28747660c5bdd5dd Mon Sep 17 00:00:00 2001 From: croneter Date: Sun, 8 Dec 2019 16:47:00 +0100 Subject: [PATCH 14/50] Beta version bump 2.10.6 --- README.md | 2 +- addon.xml | 9 +++++++-- changelog.txt | 5 +++++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 4d746977..4706bb6f 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ [![stable version](https://img.shields.io/badge/stable_version-2.10.4-blue.svg?maxAge=60&style=flat) ](https://github.com/croneter/binary_repo/raw/master/stable/repository.plexkodiconnect/repository.plexkodiconnect-1.0.2.zip) -[![beta version](https://img.shields.io/badge/beta_version-2.10.5-red.svg?maxAge=60&style=flat) ](https://github.com/croneter/binary_repo/raw/master/beta/repository.plexkodiconnectbeta/repository.plexkodiconnectbeta-1.0.2.zip) +[![beta version](https://img.shields.io/badge/beta_version-2.10.6-red.svg?maxAge=60&style=flat) ](https://github.com/croneter/binary_repo/raw/master/beta/repository.plexkodiconnectbeta/repository.plexkodiconnectbeta-1.0.2.zip) [![Installation](https://img.shields.io/badge/wiki-installation-brightgreen.svg?maxAge=60&style=flat)](https://github.com/croneter/PlexKodiConnect/wiki/Installation) [![FAQ](https://img.shields.io/badge/wiki-FAQ-brightgreen.svg?maxAge=60&style=flat)](https://github.com/croneter/PlexKodiConnect/wiki/faq) diff --git a/addon.xml b/addon.xml index 81ef3bce..0192ed9a 100644 --- a/addon.xml +++ b/addon.xml @@ -1,5 +1,5 @@ - + @@ -83,7 +83,12 @@ Natūralioji „Plex“ integracija į „Kodi“ Prijunkite „Kodi“ prie „Plex Medija Serverio“. Šiame papildinyje daroma prielaida, kad valdote visus savo vaizdo įrašus naudodami „Plex“ (ir nė vieno su „Kodi“). Galite prarasti jau saugomus „Kodi“ vaizdo įrašų ir muzikos duomenų bazių duomenis (kadangi šis papildinys juos tiesiogiai pakeičia). Naudokite savo pačių rizika! Naudokite savo pačių rizika - version 2.10.5 (beta only): + version 2.10.6 (beta only): +- Fix AttributeError if user enters an invalid pin code +- Fix OperationalError when starting with a fresh PKC installation +- Fix IndexError + +version 2.10.5 (beta only): - Rewire library sync to speed it up and fix sync getting stuck in rare cases - Optimize threads by using events instead of a polling mechanism. Fixes PKC becoming unresponsive, e.g. when switching users - Optimize adding values to Kodi databases by not using sqlite COALESCE command diff --git a/changelog.txt b/changelog.txt index c401c21b..0410e9f7 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,8 @@ +version 2.10.6 (beta only): +- Fix AttributeError if user enters an invalid pin code +- Fix OperationalError when starting with a fresh PKC installation +- Fix IndexError + version 2.10.5 (beta only): - Rewire library sync to speed it up and fix sync getting stuck in rare cases - Optimize threads by using events instead of a polling mechanism. Fixes PKC becoming unresponsive, e.g. when switching users From ed3301a5236bf6afe28a1623087aa75c30610aa1 Mon Sep 17 00:00:00 2001 From: croneter Date: Tue, 10 Dec 2019 08:15:10 +0100 Subject: [PATCH 15/50] Fix PKC not starting up on iOS --- resources/lib/variables.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/resources/lib/variables.py b/resources/lib/variables.py index 2ccc241d..3cba3bc2 100644 --- a/resources/lib/variables.py +++ b/resources/lib/variables.py @@ -69,7 +69,14 @@ elif xbmc.getCondVisibility('system.platform.android'): else: DEVICE = "Unknown" -MODEL = platform.release() or 'Unknown' +try: + MODEL = platform.release() or 'Unknown' +except IOError: + # E.g. iOS + # It seems that Kodi doesn't allow python to spawn subprocesses in order to + # determine the system name + # See https://github.com/psf/requests/issues/4434 + MODEL = 'Unknown' DEVICENAME = try_decode(_ADDON.getSetting('deviceName')) if not DEVICENAME: From 9182e0ad76ebed5c9a45a6b62d333f26b6557d49 Mon Sep 17 00:00:00 2001 From: croneter Date: Fri, 13 Dec 2019 07:38:52 +0100 Subject: [PATCH 16/50] Fix PKC becoming unresponsive e.g. when switching the PMS --- resources/lib/app/application.py | 3 +++ resources/lib/backgroundthread.py | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/resources/lib/app/application.py b/resources/lib/app/application.py index 533a7594..5738913d 100644 --- a/resources/lib/app/application.py +++ b/resources/lib/app/application.py @@ -65,6 +65,7 @@ class App(object): self.threads.append(thread) def deregister_fanart_thread(self, thread): + self.fanart_thread.unblock_callers() self.fanart_thread = None self.threads.remove(thread) @@ -85,6 +86,7 @@ class App(object): self.threads.append(thread) def deregister_caching_thread(self, thread): + self.caching_thread.unblock_callers() self.caching_thread = None self.threads.remove(thread) @@ -111,6 +113,7 @@ class App(object): """ Sync thread has done it's work and is e.g. about to die """ + thread.unblock_callers() self.threads.remove(thread) def suspend_threads(self, block=True): diff --git a/resources/lib/backgroundthread.py b/resources/lib/backgroundthread.py index 60acc59e..e4eb44d7 100644 --- a/resources/lib/backgroundthread.py +++ b/resources/lib/backgroundthread.py @@ -106,6 +106,13 @@ class KillableThread(threading.Thread): """ return not self._is_not_asleep.is_set() + def unblock_callers(self): + """ + Ensures that any other thread that requested this thread's suspension + is released + """ + self._suspension_reached.set() + class ProcessingQueue(Queue.Queue, object): """ From 3000bfcd7d25688ce07ff1a1a015fdeb867f7849 Mon Sep 17 00:00:00 2001 From: croneter Date: Tue, 10 Dec 2019 17:26:00 +0100 Subject: [PATCH 17/50] Always use sqlite WAL mode (did not switch back to normal journal mode automatically anyway) --- resources/lib/db.py | 17 ++++++----------- resources/lib/kodi_db/__init__.py | 6 +++--- resources/lib/kodi_db/common.py | 9 +++------ 3 files changed, 12 insertions(+), 20 deletions(-) diff --git a/resources/lib/db.py b/resources/lib/db.py index 1ea7e3fe..2c701040 100644 --- a/resources/lib/db.py +++ b/resources/lib/db.py @@ -52,26 +52,21 @@ def catch_operationalerrors(method): return wrapper -def _initial_db_connection_setup(conn, wal_mode): +def _initial_db_connection_setup(conn): """ Set-up DB e.g. for WAL journal mode, if that hasn't already been done before. Also start a transaction """ - if wal_mode: - pass - # conn.execute('PRAGMA journal_mode=WAL;') - # conn.execute('PRAGMA cache_size = -8000;') - # conn.execute('PRAGMA synchronous=NORMAL;') + conn.execute('PRAGMA journal_mode = WAL;') + conn.execute('PRAGMA cache_size = -8000;') + conn.execute('PRAGMA synchronous = NORMAL;') conn.execute('BEGIN') -def connect(media_type=None, wal_mode=True): +def connect(media_type=None): """ Open a connection to the Kodi database. media_type: 'video' (standard if not passed), 'plex', 'music', 'texture' - Pass wal_mode=False if you want the standard (and slower) sqlite - journal_mode, e.g. when wiping entire tables. Useful if you do NOT want - concurrent access to DB for both PKC and Kodi """ if media_type == "plex": db_path = v.DB_PLEX_PATH @@ -87,7 +82,7 @@ def connect(media_type=None, wal_mode=True): attempts = DB_WRITE_ATTEMPTS while True: try: - _initial_db_connection_setup(conn, wal_mode) + _initial_db_connection_setup(conn) except sqlite3.OperationalError as err: if 'database is locked' not in err: # Not an error we want to catch, so reraise it diff --git a/resources/lib/kodi_db/__init__.py b/resources/lib/kodi_db/__init__.py index 200d83ed..a7946736 100644 --- a/resources/lib/kodi_db/__init__.py +++ b/resources/lib/kodi_db/__init__.py @@ -62,7 +62,7 @@ def setup_kodi_default_entries(): def reset_cached_images(): LOG.info('Resetting cached artwork') LOG.debug('Resetting the Kodi texture DB') - with KodiTextureDB(wal_mode=False) as kodidb: + with KodiTextureDB() as kodidb: kodidb.wipe() LOG.debug('Deleting all cached image files') path = path_ops.translate_path('special://thumbnails/') @@ -91,11 +91,11 @@ def wipe_dbs(music=True): """ LOG.warn('Wiping Kodi databases!') LOG.info('Wiping Kodi video database') - with KodiVideoDB(wal_mode=False) as kodidb: + with KodiVideoDB() as kodidb: kodidb.wipe() if music: LOG.info('Wiping Kodi music database') - with KodiMusicDB(wal_mode=False) as kodidb: + with KodiMusicDB() as kodidb: kodidb.wipe() reset_cached_images() setup_kodi_default_entries() diff --git a/resources/lib/kodi_db/common.py b/resources/lib/kodi_db/common.py index 2a331290..e362ca6c 100644 --- a/resources/lib/kodi_db/common.py +++ b/resources/lib/kodi_db/common.py @@ -15,11 +15,9 @@ class KodiDBBase(object): Kodi database methods used for all types of items """ def __init__(self, texture_db=False, kodiconn=None, artconn=None, - lock=True, wal_mode=True): + lock=True): """ Allows direct use with a cursor instead of context mgr - Pass wal_mode=False if you want the standard sqlite journal_mode, e.g. - when wiping entire tables """ self._texture_db = texture_db self.lock = lock @@ -27,14 +25,13 @@ class KodiDBBase(object): self.cursor = self.kodiconn.cursor() if self.kodiconn else None self.artconn = artconn self.artcursor = self.artconn.cursor() if self.artconn else None - self.wal_mode = wal_mode def __enter__(self): if self.lock: KODIDB_LOCK.acquire() - self.kodiconn = db.connect(self.db_kind, self.wal_mode) + self.kodiconn = db.connect(self.db_kind) self.cursor = self.kodiconn.cursor() - self.artconn = db.connect('texture', self.wal_mode) if self._texture_db \ + self.artconn = db.connect('texture') if self._texture_db \ else None self.artcursor = self.artconn.cursor() if self._texture_db else None return self From 70b7a4451493aef47127b5286e63c24304faeb65 Mon Sep 17 00:00:00 2001 From: croneter Date: Thu, 12 Dec 2019 17:29:46 +0100 Subject: [PATCH 18/50] Avoid duplicate code --- resources/lib/library_sync/common.py | 15 +++++++++ .../lib/library_sync/fill_metadata_queue.py | 31 +++++++------------ resources/lib/library_sync/full_sync.py | 14 +-------- resources/lib/library_sync/get_metadata.py | 10 ------ .../lib/library_sync/process_metadata.py | 12 ------- 5 files changed, 27 insertions(+), 55 deletions(-) diff --git a/resources/lib/library_sync/common.py b/resources/lib/library_sync/common.py index d6aae66f..a0f43285 100644 --- a/resources/lib/library_sync/common.py +++ b/resources/lib/library_sync/common.py @@ -1,10 +1,13 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- from __future__ import absolute_import, division, unicode_literals +from logging import getLogger import xbmc from .. import utils, app, variables as v +LOG = getLogger('PLEX.sync') + PLAYLIST_SYNC_ENABLED = (v.DEVICE != 'Microsoft UWP' and utils.settings('enablePlaylistSync') == 'true') @@ -22,6 +25,18 @@ class LibrarySyncMixin(object): """ return self.should_cancel() + def run(self): + app.APP.register_thread(self) + LOG.debug('##===--- Starting %s ---===##', self.__class__.__name__) + try: + self._run() + except Exception as err: + LOG.error('Exception encountered: %s', err) + utils.ERROR(notify=True) + finally: + app.APP.deregister_thread(self) + LOG.debug('##===--- %s Stopped ---===##', self.__class__.__name__) + def update_kodi_library(video=True, music=True): """ diff --git a/resources/lib/library_sync/fill_metadata_queue.py b/resources/lib/library_sync/fill_metadata_queue.py index 7ed361b0..ceb05f06 100644 --- a/resources/lib/library_sync/fill_metadata_queue.py +++ b/resources/lib/library_sync/fill_metadata_queue.py @@ -4,13 +4,13 @@ from logging import getLogger from . import common from ..plex_db import PlexDB -from .. import backgroundthread, app +from .. import backgroundthread LOG = getLogger('PLEX.sync.fill_metadata_queue') class FillMetadataQueue(common.LibrarySyncMixin, - backgroundthread.KillableThread, ): + backgroundthread.KillableThread): """ Threaded download of Plex XML metadata for a certain library item. Fills the queue with the downloaded etree XML objects. Will use a COPIED @@ -47,21 +47,12 @@ class FillMetadataQueue(common.LibrarySyncMixin, # Ensures that our queues finish section.number_of_items = count - def run(self): - LOG.debug('Starting %s thread', self.__class__.__name__) - app.APP.register_thread(self) - try: - while not self.should_cancel(): - section = self.section_queue.get() - self.section_queue.task_done() - if section is None: - break - self._process_section(section) - except Exception: - from .. import utils - utils.ERROR(notify=True) - finally: - # Signal the download metadata threads to stop with a sentinel - self.get_metadata_queue.put(None) - app.APP.deregister_thread(self) - LOG.debug('##===---- %s Stopped ----===##', self.__class__.__name__) + def _run(self): + while not self.should_cancel(): + section = self.section_queue.get() + self.section_queue.task_done() + if section is None: + break + self._process_section(section) + # Signal the download metadata threads to stop with a sentinel + self.get_metadata_queue.put(None) diff --git a/resources/lib/library_sync/full_sync.py b/resources/lib/library_sync/full_sync.py index 0312ec21..692f620c 100644 --- a/resources/lib/library_sync/full_sync.py +++ b/resources/lib/library_sync/full_sync.py @@ -272,20 +272,8 @@ class FullSync(common.LibrarySyncMixin, backgroundthread.KillableThread): break LOG.debug('Done looking for items to delete') - def run(self): - app.APP.register_thread(self) - LOG.info('Running library sync with repair=%s', self.repair) - try: - self.run_full_library_sync() - except Exception: - utils.ERROR(notify=True) - self.successful = False - finally: - app.APP.deregister_thread(self) - LOG.info('Library sync done. successful: %s', self.successful) - @utils.log_time - def run_full_library_sync(self): + def _run(self): try: # Get latest Plex libraries and build playlist and video node files if self.should_cancel() or not sections.sync_from_pms(self): diff --git a/resources/lib/library_sync/get_metadata.py b/resources/lib/library_sync/get_metadata.py index 972e222c..e21d2c27 100644 --- a/resources/lib/library_sync/get_metadata.py +++ b/resources/lib/library_sync/get_metadata.py @@ -5,7 +5,6 @@ from logging import getLogger from . import common from ..plex_api import API from .. import backgroundthread, plex_functions as PF, utils, variables as v -from .. import app LOG = getLogger('PLEX.sync.get_metadata') LOCK = backgroundthread.threading.Lock() @@ -69,15 +68,6 @@ class GetMetadataThread(common.LibrarySyncMixin, # Add a "dummy" item so we're not skipping a beat self.processing_queue.put((count, {'section': section, 'xml': None})) - def run(self): - LOG.debug('Starting %s thread', self.__class__.__name__) - app.APP.register_thread(self) - try: - self._run() - finally: - app.APP.deregister_thread(self) - LOG.debug('##===---- %s Stopped ----===##', self.__class__.__name__) - def _run(self): while True: item = self.get_metadata_queue.get() diff --git a/resources/lib/library_sync/process_metadata.py b/resources/lib/library_sync/process_metadata.py index abc70fdd..cddcd20f 100644 --- a/resources/lib/library_sync/process_metadata.py +++ b/resources/lib/library_sync/process_metadata.py @@ -57,18 +57,6 @@ class ProcessMetadataThread(common.LibrarySyncMixin, self.processing_queue.task_done() return item - def run(self): - LOG.debug('Starting %s thread', self.__class__.__name__) - app.APP.register_thread(self) - try: - self._run() - except Exception: - from .. import utils - utils.ERROR(notify=True) - finally: - app.APP.deregister_thread(self) - LOG.debug('##===---- %s Stopped ----===##', self.__class__.__name__) - def _run(self): # There are 2 sentinels: None for aborting/ending this thread, the dict # {'section': section, 'xml': None} for skipped/invalid items From 6d39adbd8cdf5ebc443b5c332aaed43bffeceff7 Mon Sep 17 00:00:00 2001 From: croneter Date: Fri, 13 Dec 2019 07:45:38 +0100 Subject: [PATCH 19/50] Use sqlite isolation_level=None in order to use autocommit mode and thus avoid sqlite auto-committing --- resources/lib/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/lib/db.py b/resources/lib/db.py index 2c701040..ca998941 100644 --- a/resources/lib/db.py +++ b/resources/lib/db.py @@ -78,7 +78,7 @@ def connect(media_type=None): db_path = v.DB_TEXTURE_PATH else: db_path = v.DB_VIDEO_PATH - conn = sqlite3.connect(db_path, timeout=30.0) + conn = sqlite3.connect(db_path, timeout=30.0, isolation_level=None) attempts = DB_WRITE_ATTEMPTS while True: try: From b4e132af85db5c1693e5abfdb44648b5b1129a3b Mon Sep 17 00:00:00 2001 From: croneter Date: Fri, 13 Dec 2019 12:42:22 +0100 Subject: [PATCH 20/50] Optimize code --- resources/lib/kodi_db/video.py | 47 ++++++++++++---------------------- 1 file changed, 16 insertions(+), 31 deletions(-) diff --git a/resources/lib/kodi_db/video.py b/resources/lib/kodi_db/video.py index 415ebcb7..2466e5e3 100644 --- a/resources/lib/kodi_db/video.py +++ b/resources/lib/kodi_db/video.py @@ -38,37 +38,22 @@ class KodiVideoDB(common.KodiDBBase): For some reason, Kodi ignores this if done via itemtypes while e.g. adding or updating items. (addPath method does NOT work) """ - path_id = self.get_path(MOVIE_PATH) - if path_id is None: - query = ''' - INSERT INTO path(strPath, - strContent, - strScraper, - noUpdate, - exclude) - VALUES (?, ?, ?, ?, ?) - ''' - self.cursor.execute(query, (MOVIE_PATH, - 'movies', - 'metadata.local', - 1, - 0)) - # And TV shows - path_id = self.get_path(SHOW_PATH) - if path_id is None: - query = ''' - INSERT INTO path(strPath, - strContent, - strScraper, - noUpdate, - exclude) - VALUES (?, ?, ?, ?, ?) - ''' - self.cursor.execute(query, (SHOW_PATH, - 'tvshows', - 'metadata.local', - 1, - 0)) + for path, kind in ((MOVIE_PATH, 'movies'), (SHOW_PATH, 'tvshows')): + path_id = self.get_path(path) + if path_id is None: + query = ''' + INSERT INTO path(strPath, + strContent, + strScraper, + noUpdate, + exclude) + VALUES (?, ?, ?, ?, ?) + ''' + self.cursor.execute(query, (path, + kind, + 'metadata.local', + 1, + 0)) @db.catch_operationalerrors def parent_path_id(self, path): From a715b3a473007de42bb97f6db81b390537110348 Mon Sep 17 00:00:00 2001 From: croneter Date: Fri, 13 Dec 2019 12:46:26 +0100 Subject: [PATCH 21/50] raise exception instead of returning None if PKC needs to exit and we're trying to connect to a DB --- resources/lib/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/lib/db.py b/resources/lib/db.py index ca998941..53db56c7 100644 --- a/resources/lib/db.py +++ b/resources/lib/db.py @@ -93,7 +93,7 @@ def connect(media_type=None): raise LockedDatabase('Database is locked') if app.APP.monitor.waitForAbort(0.05): # PKC needs to quit - return + raise LockedDatabase('Database was locked and we need to exit') else: break return conn From 0d537f108efaae325c4f8904b3f971376da13117 Mon Sep 17 00:00:00 2001 From: croneter Date: Fri, 13 Dec 2019 12:47:56 +0100 Subject: [PATCH 22/50] Lower timeout for a DB connection from 30s to 10s --- resources/lib/db.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/resources/lib/db.py b/resources/lib/db.py index 53db56c7..28ffc7c7 100644 --- a/resources/lib/db.py +++ b/resources/lib/db.py @@ -6,6 +6,7 @@ from functools import wraps from . import variables as v, app DB_WRITE_ATTEMPTS = 100 +DB_CONNECTION_TIMEOUT = 10 class LockedDatabase(Exception): @@ -78,7 +79,9 @@ def connect(media_type=None): db_path = v.DB_TEXTURE_PATH else: db_path = v.DB_VIDEO_PATH - conn = sqlite3.connect(db_path, timeout=30.0, isolation_level=None) + conn = sqlite3.connect(db_path, + timeout=DB_CONNECTION_TIMEOUT, + isolation_level=None) attempts = DB_WRITE_ATTEMPTS while True: try: From 654748218e49eb1f37e908883f6c925f69b73be9 Mon Sep 17 00:00:00 2001 From: croneter Date: Fri, 13 Dec 2019 13:00:35 +0100 Subject: [PATCH 23/50] Get section iterators in a dedicated thread to gain speed --- resources/lib/library_sync/full_sync.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/resources/lib/library_sync/full_sync.py b/resources/lib/library_sync/full_sync.py index 692f620c..4e2571d7 100644 --- a/resources/lib/library_sync/full_sync.py +++ b/resources/lib/library_sync/full_sync.py @@ -147,11 +147,11 @@ class FullSync(common.LibrarySyncMixin, backgroundthread.KillableThread): LOG.error('Could not entirely process section %s', section) self.successful = False - def get_generators(self, kinds, queue, all_items): + def threaded_get_generators(self, kinds, queue, all_items): """ - Getting iterators is costly, so let's do it asynchronously + Getting iterators is costly, so let's do it in a dedicated thread """ - LOG.debug('Start get_generators') + LOG.debug('Start threaded_get_generators') try: for kind in kinds: for section in (x for x in app.SYNC.sections @@ -189,7 +189,7 @@ class FullSync(common.LibrarySyncMixin, backgroundthread.KillableThread): utils.ERROR(notify=True) finally: queue.put(None) - LOG.debug('Exiting get_generators') + LOG.debug('Exiting threaded_get_generators') def full_library_sync(self): kinds = [ @@ -205,7 +205,10 @@ class FullSync(common.LibrarySyncMixin, backgroundthread.KillableThread): ]) # ADD NEW ITEMS # We need to enforce syncing e.g. show before season before episode - self.get_generators(kinds, self.section_queue, False) + thread = backgroundthread.KillableThread( + target=self.threaded_get_generators, + args=(kinds, self.section_queue, False)) + thread.start() # Do the heavy lifting self.processing_loop_new_and_changed_items() common.update_kodi_library(video=True, music=True) @@ -237,7 +240,10 @@ class FullSync(common.LibrarySyncMixin, backgroundthread.KillableThread): # Close the progress indicator dialog self.dialog.close() self.dialog = None - self.get_generators(kinds, self.section_queue, True) + thread = backgroundthread.KillableThread( + target=self.threaded_get_generators, + args=(kinds, self.section_queue, True)) + thread.start() self.processing_loop_playstates() if self.should_cancel() or not self.successful: return From 136af95351d26f028a25e58dc3bc4d3efb9206e3 Mon Sep 17 00:00:00 2001 From: croneter Date: Fri, 13 Dec 2019 13:09:57 +0100 Subject: [PATCH 24/50] Speed up and simplify sync of playstates --- resources/lib/library_sync/full_sync.py | 35 +++++++++++-------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/resources/lib/library_sync/full_sync.py b/resources/lib/library_sync/full_sync.py index 4e2571d7..652b99e3 100644 --- a/resources/lib/library_sync/full_sync.py +++ b/resources/lib/library_sync/full_sync.py @@ -20,6 +20,7 @@ if common.PLAYLIST_SYNC_ENABLED: LOG = getLogger('PLEX.sync.full_sync') # How many items will be put through the processing chain at once? BATCH_SIZE = 250 +PLAYSTATE_BATCH_SIZE = 5000 # Size of queue for xmls to be downloaded from PMS for/and before processing QUEUE_BUFFER = 50 # Max number of xmls held in memory @@ -123,26 +124,20 @@ class FullSync(common.LibrarySyncMixin, backgroundthread.KillableThread): LOG.debug('Processing %s playstates for library section %s', section.number_of_items, section) try: - iterator = section.iterator - iterator = common.tag_last(iterator) - last = True - while not self.should_cancel(): - with section.context(self.current_time) as itemtype: - for last, xml_item in iterator: - section.count += 1 - if not itemtype.update_userdata(xml_item, section.plex_type): - # Somehow did not sync this item yet - itemtype.add_update(xml_item, - section_name=section.name, - section_id=section.section_id) - itemtype.plexdb.update_last_sync(int(xml_item.attrib['ratingKey']), - section.plex_type, - self.current_time) - self.update_progressbar(section, '', section.count) - if section.count % (10 * BATCH_SIZE) == 0: - break - if last: - break + with section.context(self.current_time) as context: + for xml in section.iterator: + section.count += 1 + if not context.update_userdata(xml, section.plex_type): + # Somehow did not sync this item yet + context.add_update(xml, + section_name=section.name, + section_id=section.section_id) + context.plexdb.update_last_sync(int(xml.attrib['ratingKey']), + section.plex_type, + self.current_time) + self.update_progressbar(section, '', section.count) + if section.count % PLAYSTATE_BATCH_SIZE == 0: + context.commit() except RuntimeError: LOG.error('Could not entirely process section %s', section) self.successful = False From b55b22efb0e69b3dc1c0607260a63fae8428fd59 Mon Sep 17 00:00:00 2001 From: croneter Date: Fri, 13 Dec 2019 13:12:16 +0100 Subject: [PATCH 25/50] Clarify variables --- resources/lib/library_sync/full_sync.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/resources/lib/library_sync/full_sync.py b/resources/lib/library_sync/full_sync.py index 652b99e3..82f53f34 100644 --- a/resources/lib/library_sync/full_sync.py +++ b/resources/lib/library_sync/full_sync.py @@ -18,13 +18,13 @@ if common.PLAYLIST_SYNC_ENABLED: LOG = getLogger('PLEX.sync.full_sync') -# How many items will be put through the processing chain at once? -BATCH_SIZE = 250 +DELETION_BATCH_SIZE = 250 PLAYSTATE_BATCH_SIZE = 5000 -# Size of queue for xmls to be downloaded from PMS for/and before processing -QUEUE_BUFFER = 50 + +# Max. number of plex_ids held in memory for later processing +BACKLOG_QUEUE_SIZE = 10000 # Max number of xmls held in memory -MAX_QUEUE_SIZE = 500 +XML_QUEUE_SIZE = 500 # Safety margin to filter PMS items - how many seconds to look into the past? UPDATED_AT_SAFETY = 60 * 5 LAST_VIEWED_AT_SAFETY = 60 * 5 @@ -47,8 +47,8 @@ class FullSync(common.LibrarySyncMixin, backgroundthread.KillableThread): self.dialog = None self.section_queue = Queue.Queue() - self.get_metadata_queue = Queue.Queue(maxsize=5000) - self.processing_queue = backgroundthread.ProcessingQueue(maxsize=500) + self.get_metadata_queue = Queue.Queue(maxsize=BACKLOG_QUEUE_SIZE) + self.processing_queue = backgroundthread.ProcessingQueue(maxsize=XML_QUEUE_SIZE) self.current_time = timing.plex_now() self.last_section = sections.Section() @@ -264,12 +264,12 @@ class FullSync(common.LibrarySyncMixin, backgroundthread.KillableThread): plex_ids = list( ctx.plexdb.plex_id_by_last_sync(plex_type, self.current_time, - BATCH_SIZE)) + DELETION_BATCH_SIZE)) for plex_id in plex_ids: if self.should_cancel(): return ctx.remove(plex_id, plex_type) - if len(plex_ids) < BATCH_SIZE: + if len(plex_ids) < DELETION_BATCH_SIZE: break LOG.debug('Done looking for items to delete') From 6510d5e399dfe9f3c503a373e6c4a78726b54f1f Mon Sep 17 00:00:00 2001 From: croneter Date: Fri, 13 Dec 2019 13:19:15 +0100 Subject: [PATCH 26/50] Fix display of item numbers during playstate sync --- resources/lib/library_sync/full_sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/lib/library_sync/full_sync.py b/resources/lib/library_sync/full_sync.py index 82f53f34..d885d8d2 100644 --- a/resources/lib/library_sync/full_sync.py +++ b/resources/lib/library_sync/full_sync.py @@ -135,7 +135,7 @@ class FullSync(common.LibrarySyncMixin, backgroundthread.KillableThread): context.plexdb.update_last_sync(int(xml.attrib['ratingKey']), section.plex_type, self.current_time) - self.update_progressbar(section, '', section.count) + self.update_progressbar(section, '', section.count - 1) if section.count % PLAYSTATE_BATCH_SIZE == 0: context.commit() except RuntimeError: From 58a86d34f1781df4baca3fa5c11b8d3c5b8d9058 Mon Sep 17 00:00:00 2001 From: croneter Date: Fri, 13 Dec 2019 13:45:34 +0100 Subject: [PATCH 27/50] Clarify class description --- resources/lib/library_sync/fill_metadata_queue.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/resources/lib/library_sync/fill_metadata_queue.py b/resources/lib/library_sync/fill_metadata_queue.py index ceb05f06..64665611 100644 --- a/resources/lib/library_sync/fill_metadata_queue.py +++ b/resources/lib/library_sync/fill_metadata_queue.py @@ -12,10 +12,9 @@ LOG = getLogger('PLEX.sync.fill_metadata_queue') class FillMetadataQueue(common.LibrarySyncMixin, backgroundthread.KillableThread): """ - Threaded download of Plex XML metadata for a certain library item. - Fills the queue with the downloaded etree XML objects. Will use a COPIED - plex.db file (plex-copy.db) in order to read much faster without the - writing thread stalling + Determines which plex_ids we need to sync and puts these ids in a separate + queue. Will use a COPIED plex.db file (plex-copy.db) in order to read much + faster without the writing thread stalling """ def __init__(self, repair, section_queue, get_metadata_queue): self.repair = repair From b611a66ff51b5fef33d963a6c2c7e41a813c7e7f Mon Sep 17 00:00:00 2001 From: croneter Date: Fri, 13 Dec 2019 13:55:56 +0100 Subject: [PATCH 28/50] Fix sync getting stuck --- resources/lib/library_sync/fill_metadata_queue.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/resources/lib/library_sync/fill_metadata_queue.py b/resources/lib/library_sync/fill_metadata_queue.py index 64665611..ca6c2d4e 100644 --- a/resources/lib/library_sync/fill_metadata_queue.py +++ b/resources/lib/library_sync/fill_metadata_queue.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import, division, unicode_literals from logging import getLogger +from Queue import Empty from . import common from ..plex_db import PlexDB @@ -8,6 +9,8 @@ from .. import backgroundthread LOG = getLogger('PLEX.sync.fill_metadata_queue') +QUEUE_TIMEOUT = 10 # seconds + class FillMetadataQueue(common.LibrarySyncMixin, backgroundthread.KillableThread): @@ -40,7 +43,14 @@ class FillMetadataQueue(common.LibrarySyncMixin, if (not self.repair and plexdb.checksum(plex_id, section.plex_type) == checksum): continue - self.get_metadata_queue.put((count, plex_id, section)) + try: + self.get_metadata_queue.put((count, plex_id, section), + timeout=QUEUE_TIMEOUT) + except Empty: + LOG.error('Putting %s in get_metadata_queue timed out - ' + 'aborting sync now', plex_id) + section.sync_successful = False + break count += 1 # We might have received LESS items from the PMS than anticipated. # Ensures that our queues finish From 8502c00d89a545c0386038a4ae4f512dbe779bde Mon Sep 17 00:00:00 2001 From: croneter Date: Fri, 13 Dec 2019 14:02:22 +0100 Subject: [PATCH 29/50] Beta version bump 2.10.7 --- README.md | 2 +- addon.xml | 9 +++++++-- changelog.txt | 5 +++++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 4706bb6f..3e46d376 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ [![stable version](https://img.shields.io/badge/stable_version-2.10.4-blue.svg?maxAge=60&style=flat) ](https://github.com/croneter/binary_repo/raw/master/stable/repository.plexkodiconnect/repository.plexkodiconnect-1.0.2.zip) -[![beta version](https://img.shields.io/badge/beta_version-2.10.6-red.svg?maxAge=60&style=flat) ](https://github.com/croneter/binary_repo/raw/master/beta/repository.plexkodiconnectbeta/repository.plexkodiconnectbeta-1.0.2.zip) +[![beta version](https://img.shields.io/badge/beta_version-2.10.7-red.svg?maxAge=60&style=flat) ](https://github.com/croneter/binary_repo/raw/master/beta/repository.plexkodiconnectbeta/repository.plexkodiconnectbeta-1.0.2.zip) [![Installation](https://img.shields.io/badge/wiki-installation-brightgreen.svg?maxAge=60&style=flat)](https://github.com/croneter/PlexKodiConnect/wiki/Installation) [![FAQ](https://img.shields.io/badge/wiki-FAQ-brightgreen.svg?maxAge=60&style=flat)](https://github.com/croneter/PlexKodiConnect/wiki/faq) diff --git a/addon.xml b/addon.xml index 0192ed9a..28aef47a 100644 --- a/addon.xml +++ b/addon.xml @@ -1,5 +1,5 @@ - + @@ -83,7 +83,12 @@ Natūralioji „Plex“ integracija į „Kodi“ Prijunkite „Kodi“ prie „Plex Medija Serverio“. Šiame papildinyje daroma prielaida, kad valdote visus savo vaizdo įrašus naudodami „Plex“ (ir nė vieno su „Kodi“). Galite prarasti jau saugomus „Kodi“ vaizdo įrašų ir muzikos duomenų bazių duomenis (kadangi šis papildinys juos tiesiogiai pakeičia). Naudokite savo pačių rizika! Naudokite savo pačių rizika - version 2.10.6 (beta only): + version 2.10.7 (beta only): +- Fix PKC not starting up on iOS +- Optimize the new sync process and fix some bugs that were introduced +- Fix PKC becoming unresponsive e.g. when switching the PMS + +version 2.10.6 (beta only): - Fix AttributeError if user enters an invalid pin code - Fix OperationalError when starting with a fresh PKC installation - Fix IndexError diff --git a/changelog.txt b/changelog.txt index 0410e9f7..b0213510 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,8 @@ +version 2.10.7 (beta only): +- Fix PKC not starting up on iOS +- Optimize the new sync process and fix some bugs that were introduced +- Fix PKC becoming unresponsive e.g. when switching the PMS + version 2.10.6 (beta only): - Fix AttributeError if user enters an invalid pin code - Fix OperationalError when starting with a fresh PKC installation From a1e6cdcf2991acf0d5a2e5f0b5121ad598fa1f3f Mon Sep 17 00:00:00 2001 From: croneter Date: Sun, 15 Dec 2019 07:36:50 +0100 Subject: [PATCH 30/50] Attempt to fix broken pipe error --- resources/lib/plex_companion.py | 2 +- resources/lib/plexbmchelper/listener.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/resources/lib/plex_companion.py b/resources/lib/plex_companion.py index 53a73c4c..ad9ae747 100644 --- a/resources/lib/plex_companion.py +++ b/resources/lib/plex_companion.py @@ -293,7 +293,7 @@ class PlexCompanion(backgroundthread.KillableThread): subscription_manager, ('', v.COMPANION_PORT), listener.MyHandler) - httpd.timeout = 0.95 + httpd.timeout = 10.0 break except Exception: LOG.error("Unable to start PlexCompanion. Traceback:") diff --git a/resources/lib/plexbmchelper/listener.py b/resources/lib/plexbmchelper/listener.py index 00b0cb4b..a5c6ecd1 100644 --- a/resources/lib/plexbmchelper/listener.py +++ b/resources/lib/plexbmchelper/listener.py @@ -37,6 +37,7 @@ RESOURCES_XML = ('%s\n' v.PLATFORM, v.PLATFORM_VERSION) + class MyHandler(BaseHTTPRequestHandler): """ BaseHTTPRequestHandler implementation of Plex Companion listener From fe857cb6091cd87d7ff4d751b53278a8b9ceea84 Mon Sep 17 00:00:00 2001 From: croneter Date: Tue, 17 Dec 2019 16:01:35 +0100 Subject: [PATCH 31/50] Improve thread pool management to render PKC snappier --- resources/lib/backgroundthread.py | 54 +++++++++++++++++++------------ resources/lib/service_entry.py | 1 + 2 files changed, 34 insertions(+), 21 deletions(-) diff --git a/resources/lib/backgroundthread.py b/resources/lib/backgroundthread.py index e4eb44d7..835ea16f 100644 --- a/resources/lib/backgroundthread.py +++ b/resources/lib/backgroundthread.py @@ -12,6 +12,7 @@ import xbmc from . import utils, app, variables as v +WORKER_COUNT = 3 LOG = getLogger('PLEX.threads') @@ -346,6 +347,11 @@ class Task(object): return not self.finished and not self._canceled +class ShutdownSentinel(Task): + def run(self): + pass + + class FunctionAsTask(Task): def __init__(self, function, callback, *args, **kwargs): self._function = function @@ -372,7 +378,7 @@ class MutablePriorityQueue(Queue.PriorityQueue): lowest = self.queue and min(self.queue) or None except Exception: lowest = None - utils.ERROR() + utils.ERROR(notify=True) finally: self.mutex.release() return lowest @@ -393,7 +399,7 @@ class BackgroundWorker(object): try: task._run() except Exception: - utils.ERROR() + utils.ERROR(notify=True) def abort(self): self._abort = True @@ -423,13 +429,13 @@ class BackgroundWorker(object): except Queue.Empty: LOG.debug('(%s): Idle', self.name) - def shutdown(self): + def shutdown(self, block=True): self.abort() if self._task: self._task.cancel() - if self._thread and self._thread.isAlive(): + if block and self._thread and self._thread.isAlive(): LOG.debug('thread (%s): Waiting...', self.name) self._thread.join() LOG.debug('thread (%s): Done', self.name) @@ -444,16 +450,17 @@ class NonstoppingBackgroundWorker(BackgroundWorker): super(NonstoppingBackgroundWorker, self).__init__(queue, name) def _queueLoop(self): + LOG.debug('Starting Worker %s', self.name) while not self.aborted(): - try: - self._task = self._queue.get_nowait() - self._working = True - self._runTask(self._task) - self._working = False - self._queue.task_done() - self._task = None - except Queue.Empty: - app.APP.monitor.waitForAbort(0.05) + self._task = self._queue.get() + if self._task is ShutdownSentinel: + break + self._working = True + self._runTask(self._task) + self._working = False + self._queue.task_done() + self._task = None + LOG.debug('Exiting Worker %s', self.name) def working(self): return self._working @@ -465,7 +472,10 @@ class BackgroundThreader: self._queue = MutablePriorityQueue() self._abort = False self.priority = -1 - self.workers = [worker(self._queue, 'queue.{0}:worker.{1}'.format(self.name, x)) for x in range(worker_count)] + self.workers = [ + worker(self._queue, 'queue.{0}:worker.{1}'.format(self.name, x)) + for x in range(worker_count) + ] def _nextPriority(self): self.priority += 1 @@ -480,11 +490,11 @@ class BackgroundThreader: def aborted(self): return self._abort or xbmc.abortRequested - def shutdown(self): + def shutdown(self, block=True): self.abort() - + self.addTasksToFront([ShutdownSentinel() for _ in self.workers]) for w in self.workers: - w.shutdown() + w.shutdown(block) def addTask(self, task): task.priority = self._nextPriority() @@ -537,7 +547,9 @@ class BackgroundThreader: class ThreaderManager: - def __init__(self, worker=BackgroundWorker, worker_count=6): + def __init__(self, + worker=NonstoppingBackgroundWorker, + worker_count=WORKER_COUNT): self.index = 0 self.abandoned = [] self._workerhandler = worker @@ -557,10 +569,10 @@ class ThreaderManager: self.threader = BackgroundThreader(name=str(self.index), worker=self._workerhandler) - def shutdown(self): - self.threader.shutdown() + def shutdown(self, block=True): + self.threader.shutdown(block) for a in self.abandoned: - a.shutdown() + a.shutdown(block) BGThreader = ThreaderManager() diff --git a/resources/lib/service_entry.py b/resources/lib/service_entry.py index 6d0ac9df..ac65266e 100644 --- a/resources/lib/service_entry.py +++ b/resources/lib/service_entry.py @@ -544,6 +544,7 @@ class Service(object): # Tell all threads to terminate (e.g. several lib sync threads) LOG.debug('Aborting all threads') app.APP.stop_pkc = True + backgroundthread.BGThreader.shutdown(block=False) # Load/Reset PKC entirely - important for user/Kodi profile switch # Clear video nodes properties library_sync.clear_window_vars() From 0255551ea983161ea5e666afd15581891459299c Mon Sep 17 00:00:00 2001 From: croneter Date: Tue, 17 Dec 2019 16:10:24 +0100 Subject: [PATCH 32/50] Don't spin up 2 separate threads but use the thread pool --- resources/lib/library_sync/full_sync.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/resources/lib/library_sync/full_sync.py b/resources/lib/library_sync/full_sync.py index d885d8d2..82039dbd 100644 --- a/resources/lib/library_sync/full_sync.py +++ b/resources/lib/library_sync/full_sync.py @@ -10,7 +10,7 @@ from .get_metadata import GetMetadataThread from .fill_metadata_queue import FillMetadataQueue from .process_metadata import ProcessMetadataThread from . import common, sections -from .. import utils, timing, backgroundthread, variables as v, app +from .. import utils, timing, backgroundthread as bg, variables as v, app from .. import plex_functions as PF, itemtypes, path_ops if common.PLAYLIST_SYNC_ENABLED: @@ -30,7 +30,7 @@ UPDATED_AT_SAFETY = 60 * 5 LAST_VIEWED_AT_SAFETY = 60 * 5 -class FullSync(common.LibrarySyncMixin, backgroundthread.KillableThread): +class FullSync(common.LibrarySyncMixin, bg.KillableThread): def __init__(self, repair, callback, show_dialog): """ repair=True: force sync EVERY item @@ -48,7 +48,7 @@ class FullSync(common.LibrarySyncMixin, backgroundthread.KillableThread): self.section_queue = Queue.Queue() self.get_metadata_queue = Queue.Queue(maxsize=BACKLOG_QUEUE_SIZE) - self.processing_queue = backgroundthread.ProcessingQueue(maxsize=XML_QUEUE_SIZE) + self.processing_queue = bg.ProcessingQueue(maxsize=XML_QUEUE_SIZE) self.current_time = timing.plex_now() self.last_section = sections.Section() @@ -200,10 +200,9 @@ class FullSync(common.LibrarySyncMixin, backgroundthread.KillableThread): ]) # ADD NEW ITEMS # We need to enforce syncing e.g. show before season before episode - thread = backgroundthread.KillableThread( - target=self.threaded_get_generators, - args=(kinds, self.section_queue, False)) - thread.start() + bg.FunctionAsTask(self.threaded_get_generators, + None, + kinds, self.section_queue, False).start() # Do the heavy lifting self.processing_loop_new_and_changed_items() common.update_kodi_library(video=True, music=True) @@ -235,10 +234,9 @@ class FullSync(common.LibrarySyncMixin, backgroundthread.KillableThread): # Close the progress indicator dialog self.dialog.close() self.dialog = None - thread = backgroundthread.KillableThread( - target=self.threaded_get_generators, - args=(kinds, self.section_queue, True)) - thread.start() + bg.FunctionAsTask(self.threaded_get_generators, + None, + kinds, self.section_queue, True).start() self.processing_loop_playstates() if self.should_cancel() or not self.successful: return From 23ac39a8606a15e7987ca0a366b17d536e878217 Mon Sep 17 00:00:00 2001 From: croneter Date: Fri, 31 Jan 2020 21:07:27 +0100 Subject: [PATCH 33/50] Fix DirectPaths when a video's folder name is identical to a video's filename (you will need to manually reset the Kodi database) --- resources/lib/itemtypes/movies.py | 4 ++-- resources/lib/itemtypes/music.py | 4 ++-- resources/lib/itemtypes/tvshows.py | 4 ++-- resources/lib/utils.py | 9 +++++++++ 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/resources/lib/itemtypes/movies.py b/resources/lib/itemtypes/movies.py index 8f6f0263..971da6e6 100644 --- a/resources/lib/itemtypes/movies.py +++ b/resources/lib/itemtypes/movies.py @@ -5,7 +5,7 @@ from logging import getLogger from .common import ItemBase from ..plex_api import API -from .. import app, variables as v, plex_functions as PF +from .. import app, variables as v, plex_functions as PF, utils LOG = getLogger('PLEX.movies') @@ -54,7 +54,7 @@ class Movie(ItemBase): else: # Network share filename = playurl.rsplit("/", 1)[1] - path = playurl.replace(filename, "") + path = utils.rreplace(playurl, filename, "", 1) kodi_pathid = self.kodidb.add_path(path, content='movies', scraper='metadata.local') diff --git a/resources/lib/itemtypes/music.py b/resources/lib/itemtypes/music.py index f609829a..15b98bfa 100644 --- a/resources/lib/itemtypes/music.py +++ b/resources/lib/itemtypes/music.py @@ -7,7 +7,7 @@ from .common import ItemBase from ..plex_api import API from ..plex_db import PlexDB, PLEXDB_LOCK from ..kodi_db import KodiMusicDB, KODIDB_LOCK -from .. import plex_functions as PF, db, timing, app, variables as v +from .. import plex_functions as PF, db, timing, app, variables as v, utils LOG = getLogger('PLEX.music') @@ -539,7 +539,7 @@ class Song(MusicMixin, ItemBase): else: # Network share filename = playurl.rsplit("/", 1)[1] - path = playurl.replace(filename, "") + path = utils.rreplace(playurl, filename, "", 1) if do_indirect: # Plex works a bit differently path = "%s%s" % (app.CONN.server, xml[0][0].get('key')) diff --git a/resources/lib/itemtypes/tvshows.py b/resources/lib/itemtypes/tvshows.py index 4a61091d..50c25e49 100644 --- a/resources/lib/itemtypes/tvshows.py +++ b/resources/lib/itemtypes/tvshows.py @@ -5,7 +5,7 @@ from logging import getLogger from .common import ItemBase, process_path from ..plex_api import API -from .. import plex_functions as PF, app, variables as v +from .. import plex_functions as PF, app, variables as v, utils LOG = getLogger('PLEX.tvshows') @@ -434,7 +434,7 @@ class Episode(TvShowMixin, ItemBase): else: # Network share filename = playurl.rsplit("/", 1)[1] - path = playurl.replace(filename, "") + path = utils.rreplace(playurl, filename, "", 1) parent_path_id = self.kodidb.parent_path_id(path) kodi_pathid = self.kodidb.add_path(path, id_parent_path=parent_path_id) diff --git a/resources/lib/utils.py b/resources/lib/utils.py index 3bf26dc2..2da4a378 100644 --- a/resources/lib/utils.py +++ b/resources/lib/utils.py @@ -249,6 +249,15 @@ def ERROR(txt='', hide_tb=False, notify=False, cancel_sync=False): return short +def rreplace(s, old, new, occurrence=-1): + """ + Replaces the string old [str, unicode] with new from the RIGHT given a + string s. + """ + li = s.rsplit(old, occurrence) + return new.join(li) + + class AttributeDict(dict): """ Turns an etree xml response's xml.attrib into an object with attributes From 2d20f0436eaaf01873c50fa3b37d2b5992c8b8fe Mon Sep 17 00:00:00 2001 From: croneter Date: Fri, 20 Dec 2019 14:15:35 +0100 Subject: [PATCH 34/50] Beta version bump 2.10.8 --- README.md | 2 +- addon.xml | 9 +++++++-- changelog.txt | 5 +++++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 3e46d376..6307ffe8 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ [![stable version](https://img.shields.io/badge/stable_version-2.10.4-blue.svg?maxAge=60&style=flat) ](https://github.com/croneter/binary_repo/raw/master/stable/repository.plexkodiconnect/repository.plexkodiconnect-1.0.2.zip) -[![beta version](https://img.shields.io/badge/beta_version-2.10.7-red.svg?maxAge=60&style=flat) ](https://github.com/croneter/binary_repo/raw/master/beta/repository.plexkodiconnectbeta/repository.plexkodiconnectbeta-1.0.2.zip) +[![beta version](https://img.shields.io/badge/beta_version-2.10.8-red.svg?maxAge=60&style=flat) ](https://github.com/croneter/binary_repo/raw/master/beta/repository.plexkodiconnectbeta/repository.plexkodiconnectbeta-1.0.2.zip) [![Installation](https://img.shields.io/badge/wiki-installation-brightgreen.svg?maxAge=60&style=flat)](https://github.com/croneter/PlexKodiConnect/wiki/Installation) [![FAQ](https://img.shields.io/badge/wiki-FAQ-brightgreen.svg?maxAge=60&style=flat)](https://github.com/croneter/PlexKodiConnect/wiki/faq) diff --git a/addon.xml b/addon.xml index 28aef47a..28ec3619 100644 --- a/addon.xml +++ b/addon.xml @@ -1,5 +1,5 @@ - + @@ -83,7 +83,12 @@ Natūralioji „Plex“ integracija į „Kodi“ Prijunkite „Kodi“ prie „Plex Medija Serverio“. Šiame papildinyje daroma prielaida, kad valdote visus savo vaizdo įrašus naudodami „Plex“ (ir nė vieno su „Kodi“). Galite prarasti jau saugomus „Kodi“ vaizdo įrašų ir muzikos duomenų bazių duomenis (kadangi šis papildinys juos tiesiogiai pakeičia). Naudokite savo pačių rizika! Naudokite savo pačių rizika - version 2.10.7 (beta only): + version 2.10.8 (beta only): +- Improve thread pool management to render PKC snappier +- Attempt to fix broken pipe error +- Fix DirectPaths when a video's folder name is identical to a video's filename (you will need to manually reset the Kodi database) + +version 2.10.7 (beta only): - Fix PKC not starting up on iOS - Optimize the new sync process and fix some bugs that were introduced - Fix PKC becoming unresponsive e.g. when switching the PMS diff --git a/changelog.txt b/changelog.txt index b0213510..b75a4904 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,8 @@ +version 2.10.8 (beta only): +- Improve thread pool management to render PKC snappier +- Attempt to fix broken pipe error +- Fix DirectPaths when a video's folder name is identical to a video's filename (you will need to manually reset the Kodi database) + version 2.10.7 (beta only): - Fix PKC not starting up on iOS - Optimize the new sync process and fix some bugs that were introduced From 25172c2f57e9d93d5fec062f4b91f79aff0bae41 Mon Sep 17 00:00:00 2001 From: croneter Date: Sun, 2 Feb 2020 14:09:10 +0100 Subject: [PATCH 35/50] Other Kodi add-ons can now search for Plex items using plugin://plugin.video.plexkodiconnect?mode=search&query= --- default.py | 6 ++++-- resources/lib/entrypoint.py | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/default.py b/default.py index a05dc5d4..149f145e 100644 --- a/default.py +++ b/default.py @@ -50,7 +50,8 @@ class Main(): plex_type=params.get('plex_type'), section_id=params.get('section_id'), synched=params.get('synched') != 'false', - prompt=params.get('prompt')) + prompt=params.get('prompt'), + query=params.get('query')) elif mode == 'show_section': entrypoint.show_section(params.get('section_index')) @@ -66,7 +67,8 @@ class Main(): entrypoint.browse_plex(key='/hubs/search', args={'includeCollections': 1, 'includeExternalMedia': 1}, - prompt=utils.lang(137)) + prompt=utils.lang(137), + query=params.get('query')) elif mode == 'route_to_extras': # Hack so we can store this path in the Kodi DB diff --git a/resources/lib/entrypoint.py b/resources/lib/entrypoint.py index 8880501e..ee25c2b1 100644 --- a/resources/lib/entrypoint.py +++ b/resources/lib/entrypoint.py @@ -468,7 +468,7 @@ def watchlater(): def browse_plex(key=None, plex_type=None, section_id=None, synched=True, - args=None, prompt=None): + args=None, prompt=None, query=None): """ Lists the content of a Plex folder, e.g. channels. Either pass in key (to be used directly for PMS url {server}) or the section_id @@ -483,7 +483,9 @@ def browse_plex(key=None, plex_type=None, section_id=None, synched=True, return app.init(entrypoint=True) args = args or {} - if prompt: + if query: + args['query'] = query + elif prompt: prompt = utils.dialog('input', prompt) if prompt is None: # User cancelled From 821993224556dc5de8e0dec356b48cf9ce62dd46 Mon Sep 17 00:00:00 2001 From: croneter Date: Sun, 2 Feb 2020 14:18:17 +0100 Subject: [PATCH 36/50] Beta version bump 2.10.9 --- README.md | 2 +- addon.xml | 7 +++++-- changelog.txt | 3 +++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 6307ffe8..3ff42861 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ [![stable version](https://img.shields.io/badge/stable_version-2.10.4-blue.svg?maxAge=60&style=flat) ](https://github.com/croneter/binary_repo/raw/master/stable/repository.plexkodiconnect/repository.plexkodiconnect-1.0.2.zip) -[![beta version](https://img.shields.io/badge/beta_version-2.10.8-red.svg?maxAge=60&style=flat) ](https://github.com/croneter/binary_repo/raw/master/beta/repository.plexkodiconnectbeta/repository.plexkodiconnectbeta-1.0.2.zip) +[![beta version](https://img.shields.io/badge/beta_version-2.10.9-red.svg?maxAge=60&style=flat) ](https://github.com/croneter/binary_repo/raw/master/beta/repository.plexkodiconnectbeta/repository.plexkodiconnectbeta-1.0.2.zip) [![Installation](https://img.shields.io/badge/wiki-installation-brightgreen.svg?maxAge=60&style=flat)](https://github.com/croneter/PlexKodiConnect/wiki/Installation) [![FAQ](https://img.shields.io/badge/wiki-FAQ-brightgreen.svg?maxAge=60&style=flat)](https://github.com/croneter/PlexKodiConnect/wiki/faq) diff --git a/addon.xml b/addon.xml index 28ec3619..c664d391 100644 --- a/addon.xml +++ b/addon.xml @@ -1,5 +1,5 @@ - + @@ -83,7 +83,10 @@ Natūralioji „Plex“ integracija į „Kodi“ Prijunkite „Kodi“ prie „Plex Medija Serverio“. Šiame papildinyje daroma prielaida, kad valdote visus savo vaizdo įrašus naudodami „Plex“ (ir nė vieno su „Kodi“). Galite prarasti jau saugomus „Kodi“ vaizdo įrašų ir muzikos duomenų bazių duomenis (kadangi šis papildinys juos tiesiogiai pakeičia). Naudokite savo pačių rizika! Naudokite savo pačių rizika - version 2.10.8 (beta only): + version 2.10.9 (beta only): +- Other Kodi add-ons can now search for Plex items using plugin://plugin.video.plexkodiconnect?mode=search&query=YOUR SEARCH STRING HERE + +version 2.10.8 (beta only): - Improve thread pool management to render PKC snappier - Attempt to fix broken pipe error - Fix DirectPaths when a video's folder name is identical to a video's filename (you will need to manually reset the Kodi database) diff --git a/changelog.txt b/changelog.txt index b75a4904..0eedd152 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,6 @@ +version 2.10.9 (beta only): +- Other Kodi add-ons can now search for Plex items using plugin://plugin.video.plexkodiconnect?mode=search&query=YOUR SEARCH STRING HERE + version 2.10.8 (beta only): - Improve thread pool management to render PKC snappier - Attempt to fix broken pipe error From d116bbdfe9c61939672e7fa2de0221894cf2fc71 Mon Sep 17 00:00:00 2001 From: croneter Date: Fri, 14 Feb 2020 13:40:46 +0100 Subject: [PATCH 37/50] Rename method --- resources/lib/backgroundthread.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/resources/lib/backgroundthread.py b/resources/lib/backgroundthread.py index 835ea16f..f10c491b 100644 --- a/resources/lib/backgroundthread.py +++ b/resources/lib/backgroundthread.py @@ -230,7 +230,7 @@ class ProcessingQueue(Queue.Queue, object): self.unfinished_tasks += 1 if len(self._queues) == 1: # queue was already exhausted! - self._switch_queues() + self._activate_next_section() self._counter = 0 self.not_empty.notify() else: @@ -255,15 +255,15 @@ class ProcessingQueue(Queue.Queue, object): OrderedQueue() if section.plex_type == v.PLEX_TYPE_ALBUM else Queue.Queue()) if self._current_section is None: - self._switch_queues() + self._activate_next_section() def _init_next_section(self): self._sections.popleft() self._queues.popleft() self._counter = 0 - self._switch_queues() + self._activate_next_section() - def _switch_queues(self): + def _activate_next_section(self): self._current_section = self._sections[0] if self._sections else None self._current_queue = self._queues[0] if self._queues else None From b69070275fd93dbf5588c344a28515d861bd5e50 Mon Sep 17 00:00:00 2001 From: croneter Date: Fri, 14 Feb 2020 13:41:28 +0100 Subject: [PATCH 38/50] Make sure OrdererQueue returns the correct queue size --- resources/lib/backgroundthread.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/resources/lib/backgroundthread.py b/resources/lib/backgroundthread.py index f10c491b..1a5f5ca2 100644 --- a/resources/lib/backgroundthread.py +++ b/resources/lib/backgroundthread.py @@ -136,14 +136,8 @@ class ProcessingQueue(Queue.Queue, object): def _qsize(self): return self._current_queue._qsize() if self._current_queue else 0 - def total_size(self): - """ - Return the approximate total size of all queues (not reliable!) - """ - self.mutex.acquire() - n = sum(q._qsize() for q in self._queues) if self._queues else 0 - self.mutex.release() - return n + def _total_qsize(self): + return sum(q._qsize() for q in self._queues) if self._queues else 0 def put(self, item, block=True, timeout=None): """Put an item into the queue. @@ -160,17 +154,16 @@ class ProcessingQueue(Queue.Queue, object): try: if self.maxsize > 0: if not block: - # Use >= instead of == due to OrderedQueue! - if self._qsize() >= self.maxsize: + if self._total_qsize() == self.maxsize: raise Queue.Full elif timeout is None: - while self._qsize() >= self.maxsize: + while self._total_qsize() == self.maxsize: self.not_full.wait() elif timeout < 0: raise ValueError("'timeout' must be a non-negative number") else: endtime = _time() + timeout - while self._qsize() >= self.maxsize: + while self._total_qsize() == self.maxsize: remaining = endtime - _time() if remaining <= 0.0: raise Queue.Full From ddd356deda99b7c27b560c7b8b446a2ec3cc9868 Mon Sep 17 00:00:00 2001 From: croneter Date: Fri, 14 Feb 2020 15:09:53 +0100 Subject: [PATCH 39/50] Refactor code --- resources/lib/library_sync/full_sync.py | 60 +++++++++++++------------ 1 file changed, 31 insertions(+), 29 deletions(-) diff --git a/resources/lib/library_sync/full_sync.py b/resources/lib/library_sync/full_sync.py index 82039dbd..3305f188 100644 --- a/resources/lib/library_sync/full_sync.py +++ b/resources/lib/library_sync/full_sync.py @@ -35,6 +35,7 @@ class FullSync(common.LibrarySyncMixin, bg.KillableThread): """ repair=True: force sync EVERY item """ + self.successful = True self.repair = repair self.callback = callback # For progress dialog @@ -45,21 +46,9 @@ class FullSync(common.LibrarySyncMixin, bg.KillableThread): self.dialog.create(utils.lang(39714)) else: self.dialog = None - - self.section_queue = Queue.Queue() - self.get_metadata_queue = Queue.Queue(maxsize=BACKLOG_QUEUE_SIZE) - self.processing_queue = bg.ProcessingQueue(maxsize=XML_QUEUE_SIZE) self.current_time = timing.plex_now() self.last_section = sections.Section() - - self.successful = True self.install_sync_done = utils.settings('SyncInstallRunDone') == 'true' - self.threads = [ - GetMetadataThread(self.get_metadata_queue, self.processing_queue) - for _ in range(int(utils.settings('syncThreadNumber'))) - ] - for t in self.threads: - t.start() super(FullSync, self).__init__() def update_progressbar(self, section, title, current): @@ -89,33 +78,42 @@ class FullSync(common.LibrarySyncMixin, bg.KillableThread): path_ops.copyfile(v.DB_PLEX_PATH, v.DB_PLEX_COPY_PATH) @utils.log_time - def processing_loop_new_and_changed_items(self): + def processing_loop_new_and_changed_items(self, section_queue, + processing_queue): LOG.debug('Start working') + get_metadata_queue = Queue.Queue(maxsize=BACKLOG_QUEUE_SIZE) scanner_thread = FillMetadataQueue(self.repair, - self.section_queue, - self.get_metadata_queue) + section_queue, + get_metadata_queue) scanner_thread.start() + metadata_threads = [ + GetMetadataThread(get_metadata_queue, processing_queue) + for _ in range(int(utils.settings('syncThreadNumber'))) + ] + for t in metadata_threads: + t.start() process_thread = ProcessMetadataThread(self.current_time, - self.processing_queue, + processing_queue, self.update_progressbar) process_thread.start() LOG.debug('Waiting for scanner thread to finish up') scanner_thread.join() LOG.debug('Waiting for metadata download threads to finish up') - for t in self.threads: + for t in metadata_threads: t.join() LOG.debug('Download metadata threads finished') # Sentinel for the process_thread once we added everything else - self.processing_queue.put_sentinel(sections.Section()) + processing_queue.put_sentinel(sections.Section()) + LOG.debug('Put sentinel into queue, waiting for processing thread') process_thread.join() self.successful = process_thread.successful LOG.debug('threads finished work. successful: %s', self.successful) @utils.log_time - def processing_loop_playstates(self): + def processing_loop_playstates(self, section_queue): while not self.should_cancel(): - section = self.section_queue.get() - self.section_queue.task_done() + section = section_queue.get() + section_queue.task_done() if section is None: break self.playstate_per_section(section) @@ -142,7 +140,8 @@ class FullSync(common.LibrarySyncMixin, bg.KillableThread): LOG.error('Could not entirely process section %s', section) self.successful = False - def threaded_get_generators(self, kinds, queue, all_items): + def threaded_get_generators(self, kinds, section_queue, processing_queue, + all_items): """ Getting iterators is costly, so let's do it in a dedicated thread """ @@ -176,17 +175,19 @@ class FullSync(common.LibrarySyncMixin, bg.KillableThread): else: section.number_of_items = section.iterator.total if section.number_of_items > 0: - self.processing_queue.add_section(section) - queue.put(section) + processing_queue.add_section(section) + section_queue.put(section) LOG.debug('Put section in queue with %s items: %s', section.number_of_items, section) except Exception: utils.ERROR(notify=True) finally: - queue.put(None) + section_queue.put(None) LOG.debug('Exiting threaded_get_generators') def full_library_sync(self): + section_queue = Queue.Queue() + processing_queue = bg.ProcessingQueue(maxsize=XML_QUEUE_SIZE) kinds = [ (v.PLEX_TYPE_MOVIE, v.PLEX_TYPE_MOVIE), (v.PLEX_TYPE_SHOW, v.PLEX_TYPE_SHOW), @@ -202,9 +203,10 @@ class FullSync(common.LibrarySyncMixin, bg.KillableThread): # We need to enforce syncing e.g. show before season before episode bg.FunctionAsTask(self.threaded_get_generators, None, - kinds, self.section_queue, False).start() + kinds, section_queue, processing_queue, False).start() # Do the heavy lifting - self.processing_loop_new_and_changed_items() + self.processing_loop_new_and_changed_items(section_queue, + processing_queue) common.update_kodi_library(video=True, music=True) if self.should_cancel() or not self.successful: return @@ -236,8 +238,8 @@ class FullSync(common.LibrarySyncMixin, bg.KillableThread): self.dialog = None bg.FunctionAsTask(self.threaded_get_generators, None, - kinds, self.section_queue, True).start() - self.processing_loop_playstates() + kinds, section_queue, processing_queue, True).start() + self.processing_loop_playstates(section_queue) if self.should_cancel() or not self.successful: return From 9a0ce533ee83623868d69f973de6fd567f55f3f0 Mon Sep 17 00:00:00 2001 From: croneter Date: Fri, 14 Feb 2020 15:11:20 +0100 Subject: [PATCH 40/50] Rename method --- resources/lib/library_sync/full_sync.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/resources/lib/library_sync/full_sync.py b/resources/lib/library_sync/full_sync.py index 3305f188..b2e7e39c 100644 --- a/resources/lib/library_sync/full_sync.py +++ b/resources/lib/library_sync/full_sync.py @@ -78,8 +78,7 @@ class FullSync(common.LibrarySyncMixin, bg.KillableThread): path_ops.copyfile(v.DB_PLEX_PATH, v.DB_PLEX_COPY_PATH) @utils.log_time - def processing_loop_new_and_changed_items(self, section_queue, - processing_queue): + def process_new_and_changed_items(self, section_queue, processing_queue): LOG.debug('Start working') get_metadata_queue = Queue.Queue(maxsize=BACKLOG_QUEUE_SIZE) scanner_thread = FillMetadataQueue(self.repair, @@ -205,8 +204,7 @@ class FullSync(common.LibrarySyncMixin, bg.KillableThread): None, kinds, section_queue, processing_queue, False).start() # Do the heavy lifting - self.processing_loop_new_and_changed_items(section_queue, - processing_queue) + self.process_new_and_changed_items(section_queue, processing_queue) common.update_kodi_library(video=True, music=True) if self.should_cancel() or not self.successful: return From a4a0b075bf9d1046a942621673b70327288297aa Mon Sep 17 00:00:00 2001 From: croneter Date: Sat, 15 Feb 2020 17:11:31 +0100 Subject: [PATCH 41/50] Increase logging for the number of items we actually process --- resources/lib/library_sync/fill_metadata_queue.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/resources/lib/library_sync/fill_metadata_queue.py b/resources/lib/library_sync/fill_metadata_queue.py index ca6c2d4e..9dedcf32 100644 --- a/resources/lib/library_sync/fill_metadata_queue.py +++ b/resources/lib/library_sync/fill_metadata_queue.py @@ -54,6 +54,8 @@ class FillMetadataQueue(common.LibrarySyncMixin, count += 1 # We might have received LESS items from the PMS than anticipated. # Ensures that our queues finish + LOG.debug('Expected to process %s items, actually processed %s for ' + 'section %s', section.number_of_items, count, section) section.number_of_items = count def _run(self): From 73ffb706f8ac8e637f1ab8a1cfcd045c3c6342c6 Mon Sep 17 00:00:00 2001 From: croneter Date: Sat, 15 Feb 2020 17:12:46 +0100 Subject: [PATCH 42/50] Make sure we're receiving valid item from the processing queue in case we should be aborting sync --- resources/lib/library_sync/process_metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/lib/library_sync/process_metadata.py b/resources/lib/library_sync/process_metadata.py index cddcd20f..0d573965 100644 --- a/resources/lib/library_sync/process_metadata.py +++ b/resources/lib/library_sync/process_metadata.py @@ -52,7 +52,7 @@ class ProcessMetadataThread(common.LibrarySyncMixin, def _get(self): item = {'xml': None} - while not self.should_cancel() and item and item['xml'] is None: + while item and item['xml'] is None: item = self.processing_queue.get() self.processing_queue.task_done() return item From 51d1538f95d616bc5c999413f810a4bcda849acd Mon Sep 17 00:00:00 2001 From: croneter Date: Sat, 15 Feb 2020 17:46:48 +0100 Subject: [PATCH 43/50] Rewire the ProcessingQueue to ensure that we can exhaust it completely and don't get stuck --- resources/lib/backgroundthread.py | 82 ++++++++++--------------- resources/lib/library_sync/full_sync.py | 6 +- 2 files changed, 34 insertions(+), 54 deletions(-) diff --git a/resources/lib/backgroundthread.py b/resources/lib/backgroundthread.py index 1a5f5ca2..fa45883d 100644 --- a/resources/lib/backgroundthread.py +++ b/resources/lib/backgroundthread.py @@ -123,7 +123,7 @@ class ProcessingQueue(Queue.Queue, object): Put tuples (count, item) into this queue, with count being the respective position of the item in the queue, starting with 0 (zero). (None, None) is the sentinel for a single queue being exhausted, added by - put_sentinel() + add_sentinel() """ def _init(self, maxsize): self.queue = deque() @@ -131,6 +131,7 @@ class ProcessingQueue(Queue.Queue, object): self._queues = deque() self._current_section = None self._current_queue = None + # Item-index for the currently active queue self._counter = 0 def _qsize(self): @@ -140,15 +141,9 @@ class ProcessingQueue(Queue.Queue, object): return sum(q._qsize() for q in self._queues) if self._queues else 0 def put(self, item, block=True, timeout=None): - """Put an item into the queue. - - If optional args 'block' is true and 'timeout' is None (the default), - block if necessary until a free slot is available. If 'timeout' is - a non-negative number, it blocks at most 'timeout' seconds and raises - the Full exception if no free slot was available within that time. - Otherwise ('block' is false), put an item on the queue if a free slot - is immediately available, else raise the Full exception ('timeout' - is ignored in that case). + """ + PKC customization of Queue.put. item needs to be the tuple + (count [int], {'section': [Section], 'xml': [etree xml]}) """ self.not_full.acquire() try: @@ -168,73 +163,46 @@ class ProcessingQueue(Queue.Queue, object): if remaining <= 0.0: raise Queue.Full self.not_full.wait(remaining) - if self._put(item) == 0: - # Only notify one waiting thread if this item is put into the - # current queue - self.not_empty.notify() - else: - # Be sure to signal not_empty only once! - self._unlock_after_section_change() + self._put(item) self.unfinished_tasks += 1 + self.not_empty.notify() finally: self.not_full.release() def _put(self, item): - """ - Returns the index of the section in whose subqueue we need to put the - item into - """ for i, section in enumerate(self._sections): if item[1]['section'] == section: self._queues[i]._put(item) break else: raise RuntimeError('Could not find section for item %s' % item[1]) - return i - def _unlock_after_section_change(self): - """ - Ugly work-around if we expected more items to be synced, but we had - to lower our section.number_of_items because PKC decided that nothing - changed and we don't need to sync the respective item(s). - get() thus might block indefinitely - """ - while (self._current_section and - self._counter == self._current_section.number_of_items): - LOG.debug('Signaling completion of current section') - self._init_next_section() - if self._current_queue and self._current_queue._qsize(): - LOG.debug('Signaling not_empty') - self.not_empty.notify() - - def put_sentinel(self, section): + def add_sentinel(self, section): """ Adds a new empty section as a sentinel. Call with an empty Section() - object. + object. Call this method immediately after having added all sections + with add_section(). Once the get()-method returns None, you've received the sentinel and you've thus exhausted the queue """ - self.not_empty.acquire() + self.not_full.acquire() try: section.number_of_items = 1 self._add_section(section) # Add the actual sentinel to the queue we just added self._queues[-1]._put((None, None)) self.unfinished_tasks += 1 - if len(self._queues) == 1: - # queue was already exhausted! - self._activate_next_section() - self._counter = 0 - self.not_empty.notify() - else: - self._unlock_after_section_change() + self.not_empty.notify() finally: - self.not_empty.release() + self.not_full.release() def add_section(self, section): """ - Be sure to add all sections first before starting to pop items off this - queue or adding them to the queue + Add a new Section() to this Queue. Each section will be entirely + processed before moving on to the next section. + + Be sure to set section.number_of_items correctly as it will signal + when processing is completely done for a specific section! """ self.mutex.acquire() try: @@ -251,12 +219,24 @@ class ProcessingQueue(Queue.Queue, object): self._activate_next_section() def _init_next_section(self): + """ + Call only when a section has been completely exhausted + """ + # Might have some items left if we lowered section.number_of_items + leftover = self._current_queue._qsize() + if leftover: + LOG.warn('Still have %s items in the current queue', leftover) + self.unfinished_tasks -= leftover + if self.unfinished_tasks == 0: + self.all_tasks_done.notify_all() + elif self.unfinished_tasks < 0: + raise RuntimeError('Got negative number of unfinished_tasks') self._sections.popleft() self._queues.popleft() - self._counter = 0 self._activate_next_section() def _activate_next_section(self): + self._counter = 0 self._current_section = self._sections[0] if self._sections else None self._current_queue = self._queues[0] if self._queues else None diff --git a/resources/lib/library_sync/full_sync.py b/resources/lib/library_sync/full_sync.py index b2e7e39c..5a9ac400 100644 --- a/resources/lib/library_sync/full_sync.py +++ b/resources/lib/library_sync/full_sync.py @@ -101,9 +101,6 @@ class FullSync(common.LibrarySyncMixin, bg.KillableThread): for t in metadata_threads: t.join() LOG.debug('Download metadata threads finished') - # Sentinel for the process_thread once we added everything else - processing_queue.put_sentinel(sections.Section()) - LOG.debug('Put sentinel into queue, waiting for processing thread') process_thread.join() self.successful = process_thread.successful LOG.debug('threads finished work. successful: %s', self.successful) @@ -181,7 +178,10 @@ class FullSync(common.LibrarySyncMixin, bg.KillableThread): except Exception: utils.ERROR(notify=True) finally: + # Sentinel for the section queue section_queue.put(None) + # Sentinel for the process_thread once we added everything else + processing_queue.add_sentinel(sections.Section()) LOG.debug('Exiting threaded_get_generators') def full_library_sync(self): From 6f553e5c9440d5a113014ed9bfe880bd74dcb05a Mon Sep 17 00:00:00 2001 From: croneter Date: Sat, 15 Feb 2020 18:40:14 +0100 Subject: [PATCH 44/50] Fix PKC background sync not working in some cases --- resources/lib/library_sync/websocket.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/lib/library_sync/websocket.py b/resources/lib/library_sync/websocket.py index f68233c1..a899362d 100644 --- a/resources/lib/library_sync/websocket.py +++ b/resources/lib/library_sync/websocket.py @@ -125,7 +125,7 @@ def process_new_item_message(message): with itemtypes.ITEMTYPE_FROM_PLEXTYPE[plex_type](timing.unix_timestamp()) as typus: typus.add_update(xml[0], section_name=xml.get('librarySectionTitle'), - section_id=xml.get('librarySectionID')) + section_id=utils.cast(int, xml.get('librarySectionID'))) cache_artwork(message['plex_id'], plex_type) return True, plex_type in v.PLEX_VIDEOTYPES, plex_type in v.PLEX_AUDIOTYPES From 8a65a86cb2ed36d57f7548889960baa4099735ea Mon Sep 17 00:00:00 2001 From: croneter Date: Sat, 15 Feb 2020 18:42:26 +0100 Subject: [PATCH 45/50] Beta version bump 2.10.10 --- README.md | 2 +- addon.xml | 8 ++++++-- changelog.txt | 4 ++++ 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 3ff42861..cb8d94ad 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ [![stable version](https://img.shields.io/badge/stable_version-2.10.4-blue.svg?maxAge=60&style=flat) ](https://github.com/croneter/binary_repo/raw/master/stable/repository.plexkodiconnect/repository.plexkodiconnect-1.0.2.zip) -[![beta version](https://img.shields.io/badge/beta_version-2.10.9-red.svg?maxAge=60&style=flat) ](https://github.com/croneter/binary_repo/raw/master/beta/repository.plexkodiconnectbeta/repository.plexkodiconnectbeta-1.0.2.zip) +[![beta version](https://img.shields.io/badge/beta_version-2.10.10-red.svg?maxAge=60&style=flat) ](https://github.com/croneter/binary_repo/raw/master/beta/repository.plexkodiconnectbeta/repository.plexkodiconnectbeta-1.0.2.zip) [![Installation](https://img.shields.io/badge/wiki-installation-brightgreen.svg?maxAge=60&style=flat)](https://github.com/croneter/PlexKodiConnect/wiki/Installation) [![FAQ](https://img.shields.io/badge/wiki-FAQ-brightgreen.svg?maxAge=60&style=flat)](https://github.com/croneter/PlexKodiConnect/wiki/faq) diff --git a/addon.xml b/addon.xml index c664d391..1d5e2131 100644 --- a/addon.xml +++ b/addon.xml @@ -1,5 +1,5 @@ - + @@ -83,7 +83,11 @@ Natūralioji „Plex“ integracija į „Kodi“ Prijunkite „Kodi“ prie „Plex Medija Serverio“. Šiame papildinyje daroma prielaida, kad valdote visus savo vaizdo įrašus naudodami „Plex“ (ir nė vieno su „Kodi“). Galite prarasti jau saugomus „Kodi“ vaizdo įrašų ir muzikos duomenų bazių duomenis (kadangi šis papildinys juos tiesiogiai pakeičia). Naudokite savo pačių rizika! Naudokite savo pačių rizika - version 2.10.9 (beta only): + version 2.10.10 (beta only): +- Fix rare but annoying bug where PKC becomes unresponsive during sync +- Fix PKC background sync not working in some cases + +version 2.10.9 (beta only): - Other Kodi add-ons can now search for Plex items using plugin://plugin.video.plexkodiconnect?mode=search&query=YOUR SEARCH STRING HERE version 2.10.8 (beta only): diff --git a/changelog.txt b/changelog.txt index 0eedd152..1a18ed49 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,7 @@ +version 2.10.10 (beta only): +- Fix rare but annoying bug where PKC becomes unresponsive during sync +- Fix PKC background sync not working in some cases + version 2.10.9 (beta only): - Other Kodi add-ons can now search for Plex items using plugin://plugin.video.plexkodiconnect?mode=search&query=YOUR SEARCH STRING HERE From fbf484cde41f190048a57dbb4f70f3b45175eb05 Mon Sep 17 00:00:00 2001 From: croneter Date: Sat, 15 Feb 2020 18:45:06 +0100 Subject: [PATCH 46/50] Beta version bump 2.10.10 --- README.md | 2 +- addon.xml | 8 ++++++-- changelog.txt | 4 ++++ 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 3ff42861..cb8d94ad 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ [![stable version](https://img.shields.io/badge/stable_version-2.10.4-blue.svg?maxAge=60&style=flat) ](https://github.com/croneter/binary_repo/raw/master/stable/repository.plexkodiconnect/repository.plexkodiconnect-1.0.2.zip) -[![beta version](https://img.shields.io/badge/beta_version-2.10.9-red.svg?maxAge=60&style=flat) ](https://github.com/croneter/binary_repo/raw/master/beta/repository.plexkodiconnectbeta/repository.plexkodiconnectbeta-1.0.2.zip) +[![beta version](https://img.shields.io/badge/beta_version-2.10.10-red.svg?maxAge=60&style=flat) ](https://github.com/croneter/binary_repo/raw/master/beta/repository.plexkodiconnectbeta/repository.plexkodiconnectbeta-1.0.2.zip) [![Installation](https://img.shields.io/badge/wiki-installation-brightgreen.svg?maxAge=60&style=flat)](https://github.com/croneter/PlexKodiConnect/wiki/Installation) [![FAQ](https://img.shields.io/badge/wiki-FAQ-brightgreen.svg?maxAge=60&style=flat)](https://github.com/croneter/PlexKodiConnect/wiki/faq) diff --git a/addon.xml b/addon.xml index c664d391..1d5e2131 100644 --- a/addon.xml +++ b/addon.xml @@ -1,5 +1,5 @@ - + @@ -83,7 +83,11 @@ Natūralioji „Plex“ integracija į „Kodi“ Prijunkite „Kodi“ prie „Plex Medija Serverio“. Šiame papildinyje daroma prielaida, kad valdote visus savo vaizdo įrašus naudodami „Plex“ (ir nė vieno su „Kodi“). Galite prarasti jau saugomus „Kodi“ vaizdo įrašų ir muzikos duomenų bazių duomenis (kadangi šis papildinys juos tiesiogiai pakeičia). Naudokite savo pačių rizika! Naudokite savo pačių rizika - version 2.10.9 (beta only): + version 2.10.10 (beta only): +- Fix rare but annoying bug where PKC becomes unresponsive during sync +- Fix PKC background sync not working in some cases + +version 2.10.9 (beta only): - Other Kodi add-ons can now search for Plex items using plugin://plugin.video.plexkodiconnect?mode=search&query=YOUR SEARCH STRING HERE version 2.10.8 (beta only): diff --git a/changelog.txt b/changelog.txt index 0eedd152..1a18ed49 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,7 @@ +version 2.10.10 (beta only): +- Fix rare but annoying bug where PKC becomes unresponsive during sync +- Fix PKC background sync not working in some cases + version 2.10.9 (beta only): - Other Kodi add-ons can now search for Plex items using plugin://plugin.video.plexkodiconnect?mode=search&query=YOUR SEARCH STRING HERE From 64af58172b22ef03fe0793ea183a24bbcba99e76 Mon Sep 17 00:00:00 2001 From: croneter Date: Mon, 17 Feb 2020 07:26:28 +0100 Subject: [PATCH 47/50] Fix yet another rare but annoying bug where PKC becomes unresponsive during sync --- .../lib/library_sync/fill_metadata_queue.py | 17 +++++++++++++---- resources/lib/library_sync/full_sync.py | 13 +++++-------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/resources/lib/library_sync/fill_metadata_queue.py b/resources/lib/library_sync/fill_metadata_queue.py index 9dedcf32..37e8eef0 100644 --- a/resources/lib/library_sync/fill_metadata_queue.py +++ b/resources/lib/library_sync/fill_metadata_queue.py @@ -3,7 +3,7 @@ from __future__ import absolute_import, division, unicode_literals from logging import getLogger from Queue import Empty -from . import common +from . import common, sections from ..plex_db import PlexDB from .. import backgroundthread @@ -19,10 +19,12 @@ class FillMetadataQueue(common.LibrarySyncMixin, queue. Will use a COPIED plex.db file (plex-copy.db) in order to read much faster without the writing thread stalling """ - def __init__(self, repair, section_queue, get_metadata_queue): + def __init__(self, repair, section_queue, get_metadata_queue, + processing_queue): self.repair = repair self.section_queue = section_queue self.get_metadata_queue = get_metadata_queue + self.processing_queue = processing_queue super(FillMetadataQueue, self).__init__() def _process_section(self, section): @@ -31,6 +33,7 @@ class FillMetadataQueue(common.LibrarySyncMixin, LOG.debug('Process section %s with %s items', section, section.number_of_items) count = 0 + do_process_section = False with PlexDB(lock=False, copy=True) as plexdb: for xml in section.iterator: if self.should_cancel(): @@ -52,10 +55,14 @@ class FillMetadataQueue(common.LibrarySyncMixin, section.sync_successful = False break count += 1 + if not do_process_section: + do_process_section = True + self.processing_queue.add_section(section) + LOG.debug('Put section in queue with %s items: %s', + section.number_of_items, section) # We might have received LESS items from the PMS than anticipated. # Ensures that our queues finish - LOG.debug('Expected to process %s items, actually processed %s for ' - 'section %s', section.number_of_items, count, section) + LOG.debug('%s items to process for section %s', count, section) section.number_of_items = count def _run(self): @@ -67,3 +74,5 @@ class FillMetadataQueue(common.LibrarySyncMixin, self._process_section(section) # Signal the download metadata threads to stop with a sentinel self.get_metadata_queue.put(None) + # Sentinel for the process_thread once we added everything else + self.processing_queue.add_sentinel(sections.Section()) diff --git a/resources/lib/library_sync/full_sync.py b/resources/lib/library_sync/full_sync.py index 5a9ac400..cdb85e30 100644 --- a/resources/lib/library_sync/full_sync.py +++ b/resources/lib/library_sync/full_sync.py @@ -83,7 +83,8 @@ class FullSync(common.LibrarySyncMixin, bg.KillableThread): get_metadata_queue = Queue.Queue(maxsize=BACKLOG_QUEUE_SIZE) scanner_thread = FillMetadataQueue(self.repair, section_queue, - get_metadata_queue) + get_metadata_queue, + processing_queue) scanner_thread.start() metadata_threads = [ GetMetadataThread(get_metadata_queue, processing_queue) @@ -136,8 +137,7 @@ class FullSync(common.LibrarySyncMixin, bg.KillableThread): LOG.error('Could not entirely process section %s', section) self.successful = False - def threaded_get_generators(self, kinds, section_queue, processing_queue, - all_items): + def threaded_get_generators(self, kinds, section_queue, all_items): """ Getting iterators is costly, so let's do it in a dedicated thread """ @@ -171,7 +171,6 @@ class FullSync(common.LibrarySyncMixin, bg.KillableThread): else: section.number_of_items = section.iterator.total if section.number_of_items > 0: - processing_queue.add_section(section) section_queue.put(section) LOG.debug('Put section in queue with %s items: %s', section.number_of_items, section) @@ -180,8 +179,6 @@ class FullSync(common.LibrarySyncMixin, bg.KillableThread): finally: # Sentinel for the section queue section_queue.put(None) - # Sentinel for the process_thread once we added everything else - processing_queue.add_sentinel(sections.Section()) LOG.debug('Exiting threaded_get_generators') def full_library_sync(self): @@ -202,7 +199,7 @@ class FullSync(common.LibrarySyncMixin, bg.KillableThread): # We need to enforce syncing e.g. show before season before episode bg.FunctionAsTask(self.threaded_get_generators, None, - kinds, section_queue, processing_queue, False).start() + kinds, section_queue, False).start() # Do the heavy lifting self.process_new_and_changed_items(section_queue, processing_queue) common.update_kodi_library(video=True, music=True) @@ -236,7 +233,7 @@ class FullSync(common.LibrarySyncMixin, bg.KillableThread): self.dialog = None bg.FunctionAsTask(self.threaded_get_generators, None, - kinds, section_queue, processing_queue, True).start() + kinds, section_queue, True).start() self.processing_loop_playstates(section_queue) if self.should_cancel() or not self.successful: return From a67d39609e71e6b4529e3ea57601929482297408 Mon Sep 17 00:00:00 2001 From: croneter Date: Mon, 17 Feb 2020 18:59:28 +0100 Subject: [PATCH 48/50] Get rid of obsolete code --- resources/lib/backgroundthread.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/resources/lib/backgroundthread.py b/resources/lib/backgroundthread.py index fa45883d..c74fa83e 100644 --- a/resources/lib/backgroundthread.py +++ b/resources/lib/backgroundthread.py @@ -222,15 +222,6 @@ class ProcessingQueue(Queue.Queue, object): """ Call only when a section has been completely exhausted """ - # Might have some items left if we lowered section.number_of_items - leftover = self._current_queue._qsize() - if leftover: - LOG.warn('Still have %s items in the current queue', leftover) - self.unfinished_tasks -= leftover - if self.unfinished_tasks == 0: - self.all_tasks_done.notify_all() - elif self.unfinished_tasks < 0: - raise RuntimeError('Got negative number of unfinished_tasks') self._sections.popleft() self._queues.popleft() self._activate_next_section() From 94b4ed52d6cc2fb3374a060054bf80f4d88e5630 Mon Sep 17 00:00:00 2001 From: croneter Date: Mon, 17 Feb 2020 19:18:15 +0100 Subject: [PATCH 49/50] Beta version bump 2.10.11 --- README.md | 2 +- addon.xml | 7 +++++-- changelog.txt | 3 +++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index cb8d94ad..2c08b00a 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ [![stable version](https://img.shields.io/badge/stable_version-2.10.4-blue.svg?maxAge=60&style=flat) ](https://github.com/croneter/binary_repo/raw/master/stable/repository.plexkodiconnect/repository.plexkodiconnect-1.0.2.zip) -[![beta version](https://img.shields.io/badge/beta_version-2.10.10-red.svg?maxAge=60&style=flat) ](https://github.com/croneter/binary_repo/raw/master/beta/repository.plexkodiconnectbeta/repository.plexkodiconnectbeta-1.0.2.zip) +[![beta version](https://img.shields.io/badge/beta_version-2.10.11-red.svg?maxAge=60&style=flat) ](https://github.com/croneter/binary_repo/raw/master/beta/repository.plexkodiconnectbeta/repository.plexkodiconnectbeta-1.0.2.zip) [![Installation](https://img.shields.io/badge/wiki-installation-brightgreen.svg?maxAge=60&style=flat)](https://github.com/croneter/PlexKodiConnect/wiki/Installation) [![FAQ](https://img.shields.io/badge/wiki-FAQ-brightgreen.svg?maxAge=60&style=flat)](https://github.com/croneter/PlexKodiConnect/wiki/faq) diff --git a/addon.xml b/addon.xml index 1d5e2131..880d29b7 100644 --- a/addon.xml +++ b/addon.xml @@ -1,5 +1,5 @@ - + @@ -83,7 +83,10 @@ Natūralioji „Plex“ integracija į „Kodi“ Prijunkite „Kodi“ prie „Plex Medija Serverio“. Šiame papildinyje daroma prielaida, kad valdote visus savo vaizdo įrašus naudodami „Plex“ (ir nė vieno su „Kodi“). Galite prarasti jau saugomus „Kodi“ vaizdo įrašų ir muzikos duomenų bazių duomenis (kadangi šis papildinys juos tiesiogiai pakeičia). Naudokite savo pačių rizika! Naudokite savo pačių rizika - version 2.10.10 (beta only): + version 2.10.11 (beta only): +- Fix yet another rare but annoying bug where PKC becomes unresponsive during sync + +version 2.10.10 (beta only): - Fix rare but annoying bug where PKC becomes unresponsive during sync - Fix PKC background sync not working in some cases diff --git a/changelog.txt b/changelog.txt index 1a18ed49..a7afb00b 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,6 @@ +version 2.10.11 (beta only): +- Fix yet another rare but annoying bug where PKC becomes unresponsive during sync + version 2.10.10 (beta only): - Fix rare but annoying bug where PKC becomes unresponsive during sync - Fix PKC background sync not working in some cases From 0f914c52ab64f77d55eefe67a9114b35be7cd06b Mon Sep 17 00:00:00 2001 From: croneter Date: Wed, 19 Feb 2020 15:51:11 +0100 Subject: [PATCH 50/50] Stable and beta version bump 2.10.12 --- README.md | 4 ++-- addon.xml | 7 +++++-- changelog.txt | 3 +++ 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 2c08b00a..e02c1298 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ -[![stable version](https://img.shields.io/badge/stable_version-2.10.4-blue.svg?maxAge=60&style=flat) ](https://github.com/croneter/binary_repo/raw/master/stable/repository.plexkodiconnect/repository.plexkodiconnect-1.0.2.zip) -[![beta version](https://img.shields.io/badge/beta_version-2.10.11-red.svg?maxAge=60&style=flat) ](https://github.com/croneter/binary_repo/raw/master/beta/repository.plexkodiconnectbeta/repository.plexkodiconnectbeta-1.0.2.zip) +[![stable version](https://img.shields.io/badge/stable_version-2.10.12-blue.svg?maxAge=60&style=flat) ](https://github.com/croneter/binary_repo/raw/master/stable/repository.plexkodiconnect/repository.plexkodiconnect-1.0.2.zip) +[![beta version](https://img.shields.io/badge/beta_version-2.10.12-red.svg?maxAge=60&style=flat) ](https://github.com/croneter/binary_repo/raw/master/beta/repository.plexkodiconnectbeta/repository.plexkodiconnectbeta-1.0.2.zip) [![Installation](https://img.shields.io/badge/wiki-installation-brightgreen.svg?maxAge=60&style=flat)](https://github.com/croneter/PlexKodiConnect/wiki/Installation) [![FAQ](https://img.shields.io/badge/wiki-FAQ-brightgreen.svg?maxAge=60&style=flat)](https://github.com/croneter/PlexKodiConnect/wiki/faq) diff --git a/addon.xml b/addon.xml index 880d29b7..4e2944c4 100644 --- a/addon.xml +++ b/addon.xml @@ -1,5 +1,5 @@ - + @@ -83,7 +83,10 @@ Natūralioji „Plex“ integracija į „Kodi“ Prijunkite „Kodi“ prie „Plex Medija Serverio“. Šiame papildinyje daroma prielaida, kad valdote visus savo vaizdo įrašus naudodami „Plex“ (ir nė vieno su „Kodi“). Galite prarasti jau saugomus „Kodi“ vaizdo įrašų ir muzikos duomenų bazių duomenis (kadangi šis papildinys juos tiesiogiai pakeičia). Naudokite savo pačių rizika! Naudokite savo pačių rizika - version 2.10.11 (beta only): + version 2.10.12: +- versions 2.10.5-11 for everyone + +version 2.10.11 (beta only): - Fix yet another rare but annoying bug where PKC becomes unresponsive during sync version 2.10.10 (beta only): diff --git a/changelog.txt b/changelog.txt index a7afb00b..132f35a4 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,6 @@ +version 2.10.12: +- versions 2.10.5-11 for everyone + version 2.10.11 (beta only): - Fix yet another rare but annoying bug where PKC becomes unresponsive during sync