diff --git a/README.md b/README.md index 60b05038..e02c1298 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ -[![stable version](https://img.shields.io/badge/stable_version-2.10.4-blue.svg?maxAge=60&style=flat) ](https://github.com/croneter/binary_repo/raw/master/stable/repository.plexkodiconnect/repository.plexkodiconnect-1.0.2.zip) -[![beta version](https://img.shields.io/badge/beta_version-2.10.4-red.svg?maxAge=60&style=flat) ](https://github.com/croneter/binary_repo/raw/master/beta/repository.plexkodiconnectbeta/repository.plexkodiconnectbeta-1.0.2.zip) +[![stable version](https://img.shields.io/badge/stable_version-2.10.12-blue.svg?maxAge=60&style=flat) ](https://github.com/croneter/binary_repo/raw/master/stable/repository.plexkodiconnect/repository.plexkodiconnect-1.0.2.zip) +[![beta version](https://img.shields.io/badge/beta_version-2.10.12-red.svg?maxAge=60&style=flat) ](https://github.com/croneter/binary_repo/raw/master/beta/repository.plexkodiconnectbeta/repository.plexkodiconnectbeta-1.0.2.zip) [![Installation](https://img.shields.io/badge/wiki-installation-brightgreen.svg?maxAge=60&style=flat)](https://github.com/croneter/PlexKodiConnect/wiki/Installation) [![FAQ](https://img.shields.io/badge/wiki-FAQ-brightgreen.svg?maxAge=60&style=flat)](https://github.com/croneter/PlexKodiConnect/wiki/faq) diff --git a/addon.xml b/addon.xml index 11c552df..4e2944c4 100644 --- a/addon.xml +++ b/addon.xml @@ -1,5 +1,5 @@ - + @@ -83,7 +83,44 @@ Natūralioji „Plex“ integracija į „Kodi“ Prijunkite „Kodi“ prie „Plex Medija Serverio“. Šiame papildinyje daroma prielaida, kad valdote visus savo vaizdo įrašus naudodami „Plex“ (ir nė vieno su „Kodi“). Galite prarasti jau saugomus „Kodi“ vaizdo įrašų ir muzikos duomenų bazių duomenis (kadangi šis papildinys juos tiesiogiai pakeičia). Naudokite savo pačių rizika! Naudokite savo pačių rizika - version 2.10.4: + version 2.10.12: +- versions 2.10.5-11 for everyone + +version 2.10.11 (beta only): +- Fix yet another rare but annoying bug where PKC becomes unresponsive during sync + +version 2.10.10 (beta only): +- Fix rare but annoying bug where PKC becomes unresponsive during sync +- Fix PKC background sync not working in some cases + +version 2.10.9 (beta only): +- Other Kodi add-ons can now search for Plex items using plugin://plugin.video.plexkodiconnect?mode=search&query=YOUR SEARCH STRING HERE + +version 2.10.8 (beta only): +- Improve thread pool management to render PKC snappier +- Attempt to fix broken pipe error +- Fix DirectPaths when a video's folder name is identical to a video's filename (you will need to manually reset the Kodi database) + +version 2.10.7 (beta only): +- Fix PKC not starting up on iOS +- Optimize the new sync process and fix some bugs that were introduced +- Fix PKC becoming unresponsive e.g. when switching the PMS + +version 2.10.6 (beta only): +- Fix AttributeError if user enters an invalid pin code +- Fix OperationalError when starting with a fresh PKC installation +- Fix IndexError + +version 2.10.5 (beta only): +- Rewire library sync to speed it up and fix sync getting stuck in rare cases +- Optimize threads by using events instead of a polling mechanism. Fixes PKC becoming unresponsive, e.g. when switching users +- Optimize adding values to Kodi databases by not using sqlite COALESCE command +- Fix OperationalError when resetting PKC +- Improve sync resiliance when certain items are not to be synced to Kodi or PKC skipped an item in the past +- Make sure bool is returned instead of an int +- Don't use WAL mode for sqlite connections, it is not making any difference + +version 2.10.4: - version 2.10.3 for everyone - Fix to correctly wipe Kodi databases diff --git a/changelog.txt b/changelog.txt index 07ee3840..132f35a4 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,40 @@ +version 2.10.12: +- versions 2.10.5-11 for everyone + +version 2.10.11 (beta only): +- Fix yet another rare but annoying bug where PKC becomes unresponsive during sync + +version 2.10.10 (beta only): +- Fix rare but annoying bug where PKC becomes unresponsive during sync +- Fix PKC background sync not working in some cases + +version 2.10.9 (beta only): +- Other Kodi add-ons can now search for Plex items using plugin://plugin.video.plexkodiconnect?mode=search&query=YOUR SEARCH STRING HERE + +version 2.10.8 (beta only): +- Improve thread pool management to render PKC snappier +- Attempt to fix broken pipe error +- Fix DirectPaths when a video's folder name is identical to a video's filename (you will need to manually reset the Kodi database) + +version 2.10.7 (beta only): +- Fix PKC not starting up on iOS +- Optimize the new sync process and fix some bugs that were introduced +- Fix PKC becoming unresponsive e.g. when switching the PMS + +version 2.10.6 (beta only): +- Fix AttributeError if user enters an invalid pin code +- Fix OperationalError when starting with a fresh PKC installation +- Fix IndexError + +version 2.10.5 (beta only): +- Rewire library sync to speed it up and fix sync getting stuck in rare cases +- Optimize threads by using events instead of a polling mechanism. Fixes PKC becoming unresponsive, e.g. when switching users +- Optimize adding values to Kodi databases by not using sqlite COALESCE command +- Fix OperationalError when resetting PKC +- Improve sync resiliance when certain items are not to be synced to Kodi or PKC skipped an item in the past +- Make sure bool is returned instead of an int +- Don't use WAL mode for sqlite connections, it is not making any difference + version 2.10.4: - version 2.10.3 for everyone - Fix to correctly wipe Kodi databases diff --git a/default.py b/default.py index a05dc5d4..149f145e 100644 --- a/default.py +++ b/default.py @@ -50,7 +50,8 @@ class Main(): plex_type=params.get('plex_type'), section_id=params.get('section_id'), synched=params.get('synched') != 'false', - prompt=params.get('prompt')) + prompt=params.get('prompt'), + query=params.get('query')) elif mode == 'show_section': entrypoint.show_section(params.get('section_index')) @@ -66,7 +67,8 @@ class Main(): entrypoint.browse_plex(key='/hubs/search', args={'includeCollections': 1, 'includeExternalMedia': 1}, - prompt=utils.lang(137)) + prompt=utils.lang(137), + query=params.get('query')) elif mode == 'route_to_extras': # Hack so we can store this path in the Kodi DB diff --git a/resources/lib/app/application.py b/resources/lib/app/application.py index 9d3fc0d0..5738913d 100644 --- a/resources/lib/app/application.py +++ b/resources/lib/app/application.py @@ -54,17 +54,18 @@ class App(object): @property def is_playing(self): - return self.player.isPlaying() + return self.player.isPlaying() == 1 @property def is_playing_video(self): - return self.player.isPlayingVideo() + return self.player.isPlayingVideo() == 1 def register_fanart_thread(self, thread): self.fanart_thread = thread self.threads.append(thread) def deregister_fanart_thread(self, thread): + self.fanart_thread.unblock_callers() self.fanart_thread = None self.threads.remove(thread) @@ -85,6 +86,7 @@ class App(object): self.threads.append(thread) def deregister_caching_thread(self, thread): + self.caching_thread.unblock_callers() self.caching_thread = None self.threads.remove(thread) @@ -111,6 +113,7 @@ class App(object): """ Sync thread has done it's work and is e.g. about to die """ + thread.unblock_callers() self.threads.remove(thread) def suspend_threads(self, block=True): @@ -124,19 +127,16 @@ class App(object): if block: while True: for thread in self.threads: - if not thread.suspend_reached: + if not thread.is_suspended(): LOG.debug('Waiting for thread to suspend: %s', thread) # Send suspend signal again in case self.threads # changed - thread.suspend() - if self.monitor.waitForAbort(0.1): - return True - break + thread.suspend(block=True) else: break return xbmc.abortRequested - def resume_threads(self, block=True): + def resume_threads(self): """ Resume all thread activity with or without blocking. Returns True only if PKC shutdown requested @@ -144,16 +144,6 @@ class App(object): LOG.debug('Resuming threads: %s', self.threads) for thread in self.threads: thread.resume() - if block: - while True: - for thread in self.threads: - if thread.suspend_reached: - LOG.debug('Waiting for thread to resume: %s', thread) - if self.monitor.waitForAbort(0.1): - return True - break - else: - break return xbmc.abortRequested def stop_threads(self, block=True): @@ -163,7 +153,7 @@ class App(object): """ LOG.debug('Killing threads: %s', self.threads) for thread in self.threads: - thread.abort() + thread.cancel() if block: while self.threads: LOG.debug('Waiting for threads to exit: %s', self.threads) diff --git a/resources/lib/artwork.py b/resources/lib/artwork.py index 47fdfd45..113d8a32 100644 --- a/resources/lib/artwork.py +++ b/resources/lib/artwork.py @@ -33,8 +33,8 @@ class ImageCachingThread(backgroundthread.KillableThread): if not utils.settings('imageSyncDuringPlayback') == 'true': self.suspend_points.append((app.APP, 'is_playing_video')) - def isSuspended(self): - return any(getattr(obj, txt) for obj, txt in self.suspend_points) + def should_suspend(self): + return any(getattr(obj, attrib) for obj, attrib in self.suspend_points) @staticmethod def _url_generator(kind, kodi_type): @@ -73,18 +73,26 @@ class ImageCachingThread(backgroundthread.KillableThread): app.APP.deregister_caching_thread(self) LOG.info("---===### Stopped ImageCachingThread ###===---") - def _run(self): + def _loop(self): kinds = [KodiVideoDB] if app.SYNC.enable_music: kinds.append(KodiMusicDB) for kind in kinds: for kodi_type in ('poster', 'fanart'): for url in self._url_generator(kind, kodi_type): - if self.wait_while_suspended(): - return + if self.should_suspend() or self.should_cancel(): + return False cache_url(url) # Toggles Image caching completed to Yes utils.settings('plex_status_image_caching', value=utils.lang(107)) + return True + + def _run(self): + while True: + if self._loop(): + break + if self.wait_while_suspended(): + break def cache_url(url): diff --git a/resources/lib/backgroundthread.py b/resources/lib/backgroundthread.py index 7823fffd..c74fa83e 100644 --- a/resources/lib/backgroundthread.py +++ b/resources/lib/backgroundthread.py @@ -2,142 +2,241 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import, division, unicode_literals from logging import getLogger +from time import time as _time import threading import Queue import heapq +from collections import deque + import xbmc -from . import utils, app +from . import utils, app, variables as v +WORKER_COUNT = 3 LOG = getLogger('PLEX.threads') class KillableThread(threading.Thread): - '''A thread class that supports raising exception in the thread from - another thread. - ''' - # def _get_my_tid(self): - # """determines this (self's) thread id - - # CAREFUL : this function is executed in the context of the caller - # thread, to get the identity of the thread represented by this - # instance. - # """ - # if not self.isAlive(): - # raise threading.ThreadError("the thread is not active") - - # return self.ident - - # def _raiseExc(self, exctype): - # """Raises the given exception type in the context of this thread. - - # If the thread is busy in a system call (time.sleep(), - # socket.accept(), ...), the exception is simply ignored. - - # If you are sure that your exception should terminate the thread, - # one way to ensure that it works is: - - # t = ThreadWithExc( ... ) - # ... - # t.raiseExc( SomeException ) - # while t.isAlive(): - # time.sleep( 0.1 ) - # t.raiseExc( SomeException ) - - # If the exception is to be caught by the thread, you need a way to - # check that your thread has caught it. - - # CAREFUL : this function is executed in the context of the - # caller thread, to raise an excpetion in the context of the - # thread represented by this instance. - # """ - # _async_raise(self._get_my_tid(), exctype) - - def kill(self, force_and_wait=False): - pass - # try: - # self._raiseExc(KillThreadException) - - # if force_and_wait: - # time.sleep(0.1) - # while self.isAlive(): - # self._raiseExc(KillThreadException) - # time.sleep(0.1) - # except threading.ThreadError: - # pass - - # def onKilled(self): - # pass - - # def run(self): - # try: - # self._Thread__target(*self._Thread__args, **self._Thread__kwargs) - # except KillThreadException: - # self.onKilled() - def __init__(self, group=None, target=None, name=None, args=(), kwargs={}): self._canceled = False - # Set to True to set the thread to suspended self._suspended = False - # Thread will return True only if suspended state is reached - self.suspend_reached = False + self._is_not_suspended = threading.Event() + self._is_not_suspended.set() + self._suspension_reached = threading.Event() + self._is_not_asleep = threading.Event() + self._is_not_asleep.set() + self.suspension_timeout = None super(KillableThread, self).__init__(group, target, name, args, kwargs) - def isCanceled(self): + def should_cancel(self): """ - Returns True if the thread is stopped + Returns True if the thread should be stopped immediately """ - if self._canceled or xbmc.abortRequested: - return True - return False + return self._canceled or app.APP.stop_pkc - def abort(self): + def cancel(self): """ - Call to stop this thread + Call from another thread to stop this current thread """ self._canceled = True + # Make sure thread is running in order to exit quickly + self._is_not_asleep.set() + self._is_not_suspended.set() - def suspend(self, block=False): + def should_suspend(self): """ - Call to suspend this thread + Returns True if the current thread should be suspended immediately """ + return self._suspended + + def suspend(self, block=False, timeout=None): + """ + Call from another thread to suspend the current thread. Provide a + timeout [float] in seconds optionally. block=True will block the caller + until the thread-to-be-suspended is indeed suspended + Will wake a thread that is asleep! + """ + self.suspension_timeout = timeout self._suspended = True + self._is_not_suspended.clear() + # Make sure thread wakes up in order to suspend + self._is_not_asleep.set() if block: - while not self.suspend_reached: - LOG.debug('Waiting for thread to suspend: %s', self) - if app.APP.monitor.waitForAbort(0.1): - return + self._suspension_reached.wait() def resume(self): """ - Call to revive a suspended thread back to life + Call from another thread to revive a suspended or asleep current thread + back to life """ self._suspended = False + self._is_not_asleep.set() + self._is_not_suspended.set() def wait_while_suspended(self): """ Blocks until thread is not suspended anymore or the thread should - exit. - Returns True only if the thread should exit (=isCanceled()) + exit or for a period of self.suspension_timeout (set by the caller of + suspend()) + Returns the value of should_cancel() """ - while self.isSuspended(): - try: - self.suspend_reached = True - # Set in service.py - if self.isCanceled(): - # Abort was requested while waiting. We should exit - return True - if app.APP.monitor.waitForAbort(0.1): - return True - finally: - self.suspend_reached = False - return self.isCanceled() + self._suspension_reached.set() + self._is_not_suspended.wait(self.suspension_timeout) + self._suspension_reached.clear() + return self.should_cancel() - def isSuspended(self): + def is_suspended(self): """ - Returns True if the thread is suspended + Check from another thread whether the current thread is suspended """ - return self._suspended + return self._suspension_reached.is_set() + + def sleep(self, timeout): + """ + Only call from the current thread in order to sleep for a period of + timeout [float, seconds]. Will unblock immediately if thread should + cancel (should_cancel()) or the thread should_suspend + """ + self._is_not_asleep.clear() + self._is_not_asleep.wait(timeout) + self._is_not_asleep.set() + + def is_asleep(self): + """ + Check from another thread whether the current thread is asleep + """ + return not self._is_not_asleep.is_set() + + def unblock_callers(self): + """ + Ensures that any other thread that requested this thread's suspension + is released + """ + self._suspension_reached.set() + + +class ProcessingQueue(Queue.Queue, object): + """ + Queue of queues that processes a queue completely before moving on to the + next queue. There's one queue per Section(). You need to initialize each + section with add_section(section) first. + Put tuples (count, item) into this queue, with count being the respective + position of the item in the queue, starting with 0 (zero). + (None, None) is the sentinel for a single queue being exhausted, added by + add_sentinel() + """ + def _init(self, maxsize): + self.queue = deque() + self._sections = deque() + self._queues = deque() + self._current_section = None + self._current_queue = None + # Item-index for the currently active queue + self._counter = 0 + + def _qsize(self): + return self._current_queue._qsize() if self._current_queue else 0 + + def _total_qsize(self): + return sum(q._qsize() for q in self._queues) if self._queues else 0 + + def put(self, item, block=True, timeout=None): + """ + PKC customization of Queue.put. item needs to be the tuple + (count [int], {'section': [Section], 'xml': [etree xml]}) + """ + self.not_full.acquire() + try: + if self.maxsize > 0: + if not block: + if self._total_qsize() == self.maxsize: + raise Queue.Full + elif timeout is None: + while self._total_qsize() == self.maxsize: + self.not_full.wait() + elif timeout < 0: + raise ValueError("'timeout' must be a non-negative number") + else: + endtime = _time() + timeout + while self._total_qsize() == self.maxsize: + remaining = endtime - _time() + if remaining <= 0.0: + raise Queue.Full + self.not_full.wait(remaining) + self._put(item) + self.unfinished_tasks += 1 + self.not_empty.notify() + finally: + self.not_full.release() + + def _put(self, item): + for i, section in enumerate(self._sections): + if item[1]['section'] == section: + self._queues[i]._put(item) + break + else: + raise RuntimeError('Could not find section for item %s' % item[1]) + + def add_sentinel(self, section): + """ + Adds a new empty section as a sentinel. Call with an empty Section() + object. Call this method immediately after having added all sections + with add_section(). + Once the get()-method returns None, you've received the sentinel and + you've thus exhausted the queue + """ + self.not_full.acquire() + try: + section.number_of_items = 1 + self._add_section(section) + # Add the actual sentinel to the queue we just added + self._queues[-1]._put((None, None)) + self.unfinished_tasks += 1 + self.not_empty.notify() + finally: + self.not_full.release() + + def add_section(self, section): + """ + Add a new Section() to this Queue. Each section will be entirely + processed before moving on to the next section. + + Be sure to set section.number_of_items correctly as it will signal + when processing is completely done for a specific section! + """ + self.mutex.acquire() + try: + self._add_section(section) + finally: + self.mutex.release() + + def _add_section(self, section): + self._sections.append(section) + self._queues.append( + OrderedQueue() if section.plex_type == v.PLEX_TYPE_ALBUM + else Queue.Queue()) + if self._current_section is None: + self._activate_next_section() + + def _init_next_section(self): + """ + Call only when a section has been completely exhausted + """ + self._sections.popleft() + self._queues.popleft() + self._activate_next_section() + + def _activate_next_section(self): + self._counter = 0 + self._current_section = self._sections[0] if self._sections else None + self._current_queue = self._queues[0] if self._queues else None + + def _get(self): + item = self._current_queue._get() + self._counter += 1 + if self._counter == self._current_section.number_of_items: + self._init_next_section() + return item[1] class OrderedQueue(Queue.PriorityQueue, object): @@ -147,58 +246,24 @@ class OrderedQueue(Queue.PriorityQueue, object): (index, item) where index=-1 is the item that will be returned first. The Queue will block until index=-1, 0, 1, 2, 3, ... is then made available + + maxsize will be rather fuzzy, as _qsize returns 0 if we're still waiting + for the next smalles index. put() thus might not block always when it + should. """ def __init__(self, maxsize=0): + self.next_index = 0 super(OrderedQueue, self).__init__(maxsize) - self.smallest = -1 - self.not_next_item = threading.Condition(self.mutex) - def _put(self, item, heappush=heapq.heappush): - heappush(self.queue, item) - if item[0] == self.smallest: - self.not_next_item.notify() - - def get(self, block=True, timeout=None): - """Remove and return an item from the queue. - - If optional args 'block' is true and 'timeout' is None (the default), - block if necessary until an item is available. If 'timeout' is - a non-negative number, it blocks at most 'timeout' seconds and raises - the Empty exception if no item was available within that time. - Otherwise ('block' is false), return an item if one is immediately - available, else raise the Empty exception ('timeout' is ignored - in that case). - """ - self.not_empty.acquire() + def _qsize(self, len=len): try: - if not block: - if not self._qsize() or self.queue[0][0] != self.smallest: - raise Queue.Empty - elif timeout is None: - while not self._qsize(): - self.not_empty.wait() - while self.queue[0][0] != self.smallest: - self.not_next_item.wait() - elif timeout < 0: - raise ValueError("'timeout' must be a non-negative number") - else: - endtime = Queue._time() + timeout - while not self._qsize(): - remaining = endtime - Queue._time() - if remaining <= 0.0: - raise Queue.Empty - self.not_empty.wait(remaining) - while self.queue[0][0] != self.smallest: - remaining = endtime - Queue._time() - if remaining <= 0.0: - raise Queue.Empty - self.not_next_item.wait(remaining) - item = self._get() - self.smallest += 1 - self.not_full.notify() - return item - finally: - self.not_empty.release() + return len(self.queue) if self.queue[0][0] == self.next_index else 0 + except IndexError: + return 0 + + def _get(self, heappop=heapq.heappop): + self.next_index += 1 + return heappop(self.queue) class Tasks(list): @@ -239,13 +304,18 @@ class Task(object): def cancel(self): self._canceled = True - def isCanceled(self): + def should_cancel(self): return self._canceled or xbmc.abortRequested def isValid(self): return not self.finished and not self._canceled +class ShutdownSentinel(Task): + def run(self): + pass + + class FunctionAsTask(Task): def __init__(self, function, callback, *args, **kwargs): self._function = function @@ -272,7 +342,7 @@ class MutablePriorityQueue(Queue.PriorityQueue): lowest = self.queue and min(self.queue) or None except Exception: lowest = None - utils.ERROR() + utils.ERROR(notify=True) finally: self.mutex.release() return lowest @@ -293,7 +363,7 @@ class BackgroundWorker(object): try: task._run() except Exception: - utils.ERROR() + utils.ERROR(notify=True) def abort(self): self._abort = True @@ -323,13 +393,13 @@ class BackgroundWorker(object): except Queue.Empty: LOG.debug('(%s): Idle', self.name) - def shutdown(self): + def shutdown(self, block=True): self.abort() if self._task: self._task.cancel() - if self._thread and self._thread.isAlive(): + if block and self._thread and self._thread.isAlive(): LOG.debug('thread (%s): Waiting...', self.name) self._thread.join() LOG.debug('thread (%s): Done', self.name) @@ -344,16 +414,17 @@ class NonstoppingBackgroundWorker(BackgroundWorker): super(NonstoppingBackgroundWorker, self).__init__(queue, name) def _queueLoop(self): + LOG.debug('Starting Worker %s', self.name) while not self.aborted(): - try: - self._task = self._queue.get_nowait() - self._working = True - self._runTask(self._task) - self._working = False - self._queue.task_done() - self._task = None - except Queue.Empty: - app.APP.monitor.waitForAbort(0.05) + self._task = self._queue.get() + if self._task is ShutdownSentinel: + break + self._working = True + self._runTask(self._task) + self._working = False + self._queue.task_done() + self._task = None + LOG.debug('Exiting Worker %s', self.name) def working(self): return self._working @@ -365,7 +436,10 @@ class BackgroundThreader: self._queue = MutablePriorityQueue() self._abort = False self.priority = -1 - self.workers = [worker(self._queue, 'queue.{0}:worker.{1}'.format(self.name, x)) for x in range(worker_count)] + self.workers = [ + worker(self._queue, 'queue.{0}:worker.{1}'.format(self.name, x)) + for x in range(worker_count) + ] def _nextPriority(self): self.priority += 1 @@ -380,11 +454,11 @@ class BackgroundThreader: def aborted(self): return self._abort or xbmc.abortRequested - def shutdown(self): + def shutdown(self, block=True): self.abort() - + self.addTasksToFront([ShutdownSentinel() for _ in self.workers]) for w in self.workers: - w.shutdown() + w.shutdown(block) def addTask(self, task): task.priority = self._nextPriority() @@ -437,7 +511,9 @@ class BackgroundThreader: class ThreaderManager: - def __init__(self, worker=BackgroundWorker, worker_count=6): + def __init__(self, + worker=NonstoppingBackgroundWorker, + worker_count=WORKER_COUNT): self.index = 0 self.abandoned = [] self._workerhandler = worker @@ -457,10 +533,10 @@ class ThreaderManager: self.threader = BackgroundThreader(name=str(self.index), worker=self._workerhandler) - def shutdown(self): - self.threader.shutdown() + def shutdown(self, block=True): + self.threader.shutdown(block) for a in self.abandoned: - a.shutdown() + a.shutdown(block) BGThreader = ThreaderManager() diff --git a/resources/lib/db.py b/resources/lib/db.py index 30f91315..28ffc7c7 100644 --- a/resources/lib/db.py +++ b/resources/lib/db.py @@ -6,6 +6,7 @@ from functools import wraps from . import variables as v, app DB_WRITE_ATTEMPTS = 100 +DB_CONNECTION_TIMEOUT = 10 class LockedDatabase(Exception): @@ -52,39 +53,39 @@ def catch_operationalerrors(method): return wrapper -def _initial_db_connection_setup(conn, wal_mode): +def _initial_db_connection_setup(conn): """ Set-up DB e.g. for WAL journal mode, if that hasn't already been done before. Also start a transaction """ - if wal_mode: - conn.execute('PRAGMA journal_mode=WAL;') - conn.execute('PRAGMA cache_size = -8000;') - conn.execute('PRAGMA synchronous=NORMAL;') + conn.execute('PRAGMA journal_mode = WAL;') + conn.execute('PRAGMA cache_size = -8000;') + conn.execute('PRAGMA synchronous = NORMAL;') conn.execute('BEGIN') -def connect(media_type=None, wal_mode=True): +def connect(media_type=None): """ Open a connection to the Kodi database. media_type: 'video' (standard if not passed), 'plex', 'music', 'texture' - Pass wal_mode=False if you want the standard (and slower) sqlite - journal_mode, e.g. when wiping entire tables. Useful if you do NOT want - concurrent access to DB for both PKC and Kodi """ if media_type == "plex": db_path = v.DB_PLEX_PATH + elif media_type == 'plex-copy': + db_path = v.DB_PLEX_COPY_PATH elif media_type == "music": db_path = v.DB_MUSIC_PATH elif media_type == "texture": db_path = v.DB_TEXTURE_PATH else: db_path = v.DB_VIDEO_PATH - conn = sqlite3.connect(db_path, timeout=30.0) + conn = sqlite3.connect(db_path, + timeout=DB_CONNECTION_TIMEOUT, + isolation_level=None) attempts = DB_WRITE_ATTEMPTS while True: try: - _initial_db_connection_setup(conn, wal_mode) + _initial_db_connection_setup(conn) except sqlite3.OperationalError as err: if 'database is locked' not in err: # Not an error we want to catch, so reraise it @@ -95,7 +96,7 @@ def connect(media_type=None, wal_mode=True): raise LockedDatabase('Database is locked') if app.APP.monitor.waitForAbort(0.05): # PKC needs to quit - return + raise LockedDatabase('Database was locked and we need to exit') else: break return conn diff --git a/resources/lib/entrypoint.py b/resources/lib/entrypoint.py index 8880501e..ee25c2b1 100644 --- a/resources/lib/entrypoint.py +++ b/resources/lib/entrypoint.py @@ -468,7 +468,7 @@ def watchlater(): def browse_plex(key=None, plex_type=None, section_id=None, synched=True, - args=None, prompt=None): + args=None, prompt=None, query=None): """ Lists the content of a Plex folder, e.g. channels. Either pass in key (to be used directly for PMS url {server}) or the section_id @@ -483,7 +483,9 @@ def browse_plex(key=None, plex_type=None, section_id=None, synched=True, return app.init(entrypoint=True) args = args or {} - if prompt: + if query: + args['query'] = query + elif prompt: prompt = utils.dialog('input', prompt) if prompt is None: # User cancelled diff --git a/resources/lib/itemtypes/movies.py b/resources/lib/itemtypes/movies.py index f9193c88..971da6e6 100644 --- a/resources/lib/itemtypes/movies.py +++ b/resources/lib/itemtypes/movies.py @@ -5,7 +5,7 @@ from logging import getLogger from .common import ItemBase from ..plex_api import API -from .. import app, variables as v, plex_functions as PF +from .. import app, variables as v, plex_functions as PF, utils LOG = getLogger('PLEX.movies') @@ -20,10 +20,10 @@ class Movie(ItemBase): Process single movie """ api = API(xml) - if not self.sync_this_item(api.library_section_id()): + if not self.sync_this_item(section_id or api.library_section_id()): LOG.debug('Skipping sync of %s %s: %s - section %s not synched to ' 'Kodi', api.plex_type, api.plex_id, api.title(), - api.library_section_id()) + section_id or api.library_section_id()) return plex_id = api.plex_id movie = self.plexdb.movie(plex_id) @@ -54,7 +54,7 @@ class Movie(ItemBase): else: # Network share filename = playurl.rsplit("/", 1)[1] - path = playurl.replace(filename, "") + path = utils.rreplace(playurl, filename, "", 1) kodi_pathid = self.kodidb.add_path(path, content='movies', scraper='metadata.local') @@ -74,31 +74,21 @@ class Movie(ItemBase): api.date_created()) if file_id != old_kodi_fileid: self.kodidb.remove_file(old_kodi_fileid) - rating_id = self.kodidb.get_ratingid(kodi_id, - v.KODI_TYPE_MOVIE) - self.kodidb.update_ratings(kodi_id, - v.KODI_TYPE_MOVIE, - "default", - api.rating(), - api.votecount(), - rating_id) - # update new uniqueid Kodi 17 + rating_id = self.kodidb.update_ratings(kodi_id, + v.KODI_TYPE_MOVIE, + "default", + api.rating(), + api.votecount()) if api.provider('imdb') is not None: - uniqueid = self.kodidb.get_uniqueid(kodi_id, - v.KODI_TYPE_MOVIE) - self.kodidb.update_uniqueid(kodi_id, - v.KODI_TYPE_MOVIE, - api.provider('imdb'), - "imdb", - uniqueid) + uniqueid = self.kodidb.update_uniqueid(kodi_id, + v.KODI_TYPE_MOVIE, + 'imdb', + api.provider('imdb')) elif api.provider('tmdb') is not None: - uniqueid = self.kodidb.get_uniqueid(kodi_id, - v.KODI_TYPE_MOVIE) - self.kodidb.update_uniqueid(kodi_id, - v.KODI_TYPE_MOVIE, - api.provider('tmdb'), - "tmdb", - uniqueid) + uniqueid = self.kodidb.update_uniqueid(kodi_id, + v.KODI_TYPE_MOVIE, + 'tmdb', + api.provider('tmdb')) else: self.kodidb.remove_uniqueid(kodi_id, v.KODI_TYPE_MOVIE) uniqueid = -1 @@ -114,27 +104,21 @@ class Movie(ItemBase): file_id = self.kodidb.add_file(filename, kodi_pathid, api.date_created()) - rating_id = self.kodidb.add_ratingid() - self.kodidb.add_ratings(rating_id, - kodi_id, - v.KODI_TYPE_MOVIE, - "default", - api.rating(), - api.votecount()) + rating_id = self.kodidb.add_ratings(kodi_id, + v.KODI_TYPE_MOVIE, + "default", + api.rating(), + api.votecount()) if api.provider('imdb') is not None: - uniqueid = self.kodidb.add_uniqueid_id() - self.kodidb.add_uniqueid(uniqueid, - kodi_id, - v.KODI_TYPE_MOVIE, - api.provider('imdb'), - "imdb") + uniqueid = self.kodidb.add_uniqueid(kodi_id, + v.KODI_TYPE_MOVIE, + api.provider('imdb'), + "imdb") elif api.provider('tmdb') is not None: - uniqueid = self.kodidb.add_uniqueid_id() - self.kodidb.add_uniqueid(uniqueid, - kodi_id, - v.KODI_TYPE_MOVIE, - api.provider('tmdb'), - "tmdb") + uniqueid = self.kodidb.add_uniqueid(kodi_id, + v.KODI_TYPE_MOVIE, + api.provider('tmdb'), + "tmdb") else: uniqueid = -1 self.kodidb.add_people(kodi_id, diff --git a/resources/lib/itemtypes/music.py b/resources/lib/itemtypes/music.py index 61b16003..15b98bfa 100644 --- a/resources/lib/itemtypes/music.py +++ b/resources/lib/itemtypes/music.py @@ -7,7 +7,7 @@ from .common import ItemBase from ..plex_api import API from ..plex_db import PlexDB, PLEXDB_LOCK from ..kodi_db import KodiMusicDB, KODIDB_LOCK -from .. import plex_functions as PF, db, timing, app, variables as v +from .. import plex_functions as PF, db, timing, app, variables as v, utils LOG = getLogger('PLEX.music') @@ -159,10 +159,10 @@ class Artist(MusicMixin, ItemBase): Process a single artist """ api = API(xml) - if not self.sync_this_item(api.library_section_id()): + if not self.sync_this_item(section_id or api.library_section_id()): LOG.debug('Skipping sync of %s %s: %s - section %s not synched to ' 'Kodi', api.plex_type, api.plex_id, api.title(), - api.library_section_id()) + section_id or api.library_section_id()) return plex_id = api.plex_id artist = self.plexdb.artist(plex_id) @@ -225,6 +225,11 @@ class Album(MusicMixin, ItemBase): avoid infinite loops """ api = API(xml) + if not self.sync_this_item(section_id or api.library_section_id()): + LOG.debug('Skipping sync of %s %s: %s - section %s not synched to ' + 'Kodi', api.plex_type, api.plex_id, api.title(), + section_id or api.library_section_id()) + return plex_id = api.plex_id album = self.plexdb.album(plex_id) if album: @@ -387,6 +392,11 @@ class Song(MusicMixin, ItemBase): Process single song/track """ api = API(xml) + if not self.sync_this_item(section_id or api.library_section_id()): + LOG.debug('Skipping sync of %s %s: %s - section %s not synched to ' + 'Kodi', api.plex_type, api.plex_id, api.title(), + section_id or api.library_section_id()) + return plex_id = api.plex_id song = self.plexdb.song(plex_id) if song: @@ -529,7 +539,7 @@ class Song(MusicMixin, ItemBase): else: # Network share filename = playurl.rsplit("/", 1)[1] - path = playurl.replace(filename, "") + path = utils.rreplace(playurl, filename, "", 1) if do_indirect: # Plex works a bit differently path = "%s%s" % (app.CONN.server, xml[0][0].get('key')) diff --git a/resources/lib/itemtypes/tvshows.py b/resources/lib/itemtypes/tvshows.py index 61d8d050..50c25e49 100644 --- a/resources/lib/itemtypes/tvshows.py +++ b/resources/lib/itemtypes/tvshows.py @@ -5,7 +5,7 @@ from logging import getLogger from .common import ItemBase, process_path from ..plex_api import API -from .. import plex_functions as PF, app, variables as v +from .. import plex_functions as PF, app, variables as v, utils LOG = getLogger('PLEX.tvshows') @@ -148,10 +148,10 @@ class Show(TvShowMixin, ItemBase): Process a single show """ api = API(xml) - if not self.sync_this_item(api.library_section_id()): + if not self.sync_this_item(section_id or api.library_section_id()): LOG.debug('Skipping sync of %s %s: %s - section %s not synched to ' 'Kodi', api.plex_type, api.plex_id, api.title(), - api.library_section_id()) + section_id or api.library_section_id()) return plex_id = api.plex_id show = self.plexdb.show(plex_id) @@ -189,29 +189,21 @@ class Show(TvShowMixin, ItemBase): if update_item: LOG.info("UPDATE tvshow plex_id: %s - %s", plex_id, api.title()) # update new ratings Kodi 17 - rating_id = self.kodidb.get_ratingid(kodi_id, v.KODI_TYPE_SHOW) - self.kodidb.update_ratings(kodi_id, - v.KODI_TYPE_SHOW, - "default", - api.rating(), - api.votecount(), - rating_id) + rating_id = self.kodidb.update_ratings(kodi_id, + v.KODI_TYPE_SHOW, + "default", + api.rating(), + api.votecount()) if api.provider('tvdb') is not None: - uniqueid = self.kodidb.get_uniqueid(kodi_id, - v.KODI_TYPE_SHOW) - self.kodidb.update_uniqueid(kodi_id, - v.KODI_TYPE_SHOW, - api.provider('tvdb'), - 'tvdb', - uniqueid) + uniqueid = self.kodidb.update_uniqueid(kodi_id, + v.KODI_TYPE_SHOW, + 'tvdb', + api.provider('tvdb')) elif api.provider('tmdb') is not None: - uniqueid = self.kodidb.get_uniqueid(kodi_id, - v.KODI_TYPE_SHOW) - self.kodidb.update_uniqueid(kodi_id, - v.KODI_TYPE_SHOW, - api.provider('tmdb'), - 'tmdb', - uniqueid) + uniqueid = self.kodidb.update_uniqueid(kodi_id, + v.KODI_TYPE_SHOW, + 'tmdb', + api.provider('tmdb')) else: self.kodidb.remove_uniqueid(kodi_id, v.KODI_TYPE_SHOW) uniqueid = -1 @@ -239,27 +231,21 @@ class Show(TvShowMixin, ItemBase): LOG.info("ADD tvshow plex_id: %s - %s", plex_id, api.title()) # Link the path self.kodidb.add_showlinkpath(kodi_id, kodi_pathid) - rating_id = self.kodidb.get_ratingid(kodi_id, v.KODI_TYPE_SHOW) - self.kodidb.add_ratings(rating_id, - kodi_id, - v.KODI_TYPE_SHOW, - "default", - api.rating(), - api.votecount()) + rating_id = self.kodidb.add_ratings(kodi_id, + v.KODI_TYPE_SHOW, + "default", + api.rating(), + api.votecount()) if api.provider('tvdb'): - uniqueid = self.kodidb.add_uniqueid_id() - self.kodidb.add_uniqueid(uniqueid, - kodi_id, - v.KODI_TYPE_SHOW, - api.provider('tvdb'), - 'tvdb') + uniqueid = self.kodidb.add_uniqueid(kodi_id, + v.KODI_TYPE_SHOW, + api.provider('tvdb'), + 'tvdb') if api.provider('tmdb'): - uniqueid = self.kodidb.add_uniqueid_id() - self.kodidb.add_uniqueid(uniqueid, - kodi_id, - v.KODI_TYPE_SHOW, - api.provider('tmdb'), - 'tmdb') + uniqueid = self.kodidb.add_uniqueid(kodi_id, + v.KODI_TYPE_SHOW, + api.provider('tmdb'), + 'tmdb') else: uniqueid = -1 self.kodidb.add_people(kodi_id, @@ -303,10 +289,10 @@ class Season(TvShowMixin, ItemBase): Process a single season of a certain tv show """ api = API(xml) - if not self.sync_this_item(api.library_section_id()): + if not self.sync_this_item(section_id or api.library_section_id()): LOG.debug('Skipping sync of %s %s: %s - section %s not synched to ' 'Kodi', api.plex_type, api.plex_id, api.title(), - api.library_section_id()) + section_id or api.library_section_id()) return plex_id = api.plex_id season = self.plexdb.season(plex_id) @@ -372,10 +358,10 @@ class Episode(TvShowMixin, ItemBase): Process single episode """ api = API(xml) - if not self.sync_this_item(api.library_section_id()): + if not self.sync_this_item(section_id or api.library_section_id()): LOG.debug('Skipping sync of %s %s: %s - section %s not synched to ' 'Kodi', api.plex_type, api.plex_id, api.title(), - api.library_section_id()) + section_id or api.library_section_id()) return plex_id = api.plex_id episode = self.plexdb.episode(plex_id) @@ -448,7 +434,7 @@ class Episode(TvShowMixin, ItemBase): else: # Network share filename = playurl.rsplit("/", 1)[1] - path = playurl.replace(filename, "") + path = utils.rreplace(playurl, filename, "", 1) parent_path_id = self.kodidb.parent_path_id(path) kodi_pathid = self.kodidb.add_path(path, id_parent_path=parent_path_id) @@ -489,30 +475,21 @@ class Episode(TvShowMixin, ItemBase): self.kodidb.remove_file(old_kodi_fileid) if not app.SYNC.direct_paths: self.kodidb.remove_file(old_kodi_fileid_2) - ratingid = self.kodidb.get_ratingid(kodi_id, - v.KODI_TYPE_EPISODE) - self.kodidb.update_ratings(kodi_id, - v.KODI_TYPE_EPISODE, - "default", - api.rating(), - api.votecount(), - ratingid) + ratingid = self.kodidb.update_ratings(kodi_id, + v.KODI_TYPE_EPISODE, + "default", + api.rating(), + api.votecount()) if api.provider('tvdb'): - uniqueid = self.kodidb.get_uniqueid(kodi_id, - v.KODI_TYPE_EPISODE) - self.kodidb.update_uniqueid(kodi_id, - v.KODI_TYPE_EPISODE, - api.provider('tvdb'), - "tvdb", - uniqueid) + uniqueid = self.kodidb.update_uniqueid(kodi_id, + v.KODI_TYPE_EPISODE, + 'tvdb', + api.provider('tvdb')) elif api.provider('tmdb'): - uniqueid = self.kodidb.get_uniqueid(kodi_id, - v.KODI_TYPE_EPISODE) - self.kodidb.update_uniqueid(kodi_id, - v.KODI_TYPE_EPISODE, - api.provider('tmdb'), - "tmdb", - uniqueid) + uniqueid = self.kodidb.update_uniqueid(kodi_id, + v.KODI_TYPE_EPISODE, + 'tmdb', + api.provider('tmdb')) else: self.kodidb.remove_uniqueid(kodi_id, v.KODI_TYPE_EPISODE) uniqueid = -1 @@ -537,6 +514,7 @@ class Episode(TvShowMixin, ItemBase): airs_before_episode, playurl, kodi_pathid, + uniqueid, kodi_fileid, # and NOT kodi_fileid_2 parent_id, api.userrating(), @@ -577,27 +555,21 @@ class Episode(TvShowMixin, ItemBase): else: kodi_fileid_2 = None - rating_id = self.kodidb.add_ratingid() - self.kodidb.add_ratings(rating_id, - kodi_id, - v.KODI_TYPE_EPISODE, - "default", - api.rating(), - api.votecount()) + rating_id = self.kodidb.add_ratings(kodi_id, + v.KODI_TYPE_EPISODE, + "default", + api.rating(), + api.votecount()) if api.provider('tvdb'): - uniqueid = self.kodidb.add_uniqueid_id() - self.kodidb.add_uniqueid(uniqueid, - kodi_id, - v.KODI_TYPE_EPISODE, - api.provider('tvdb'), - "tvdb") + uniqueid = self.kodidb.add_uniqueid(kodi_id, + v.KODI_TYPE_EPISODE, + api.provider('tvdb'), + "tvdb") elif api.provider('tmdb'): - uniqueid = self.kodidb.add_uniqueid_id() - self.kodidb.add_uniqueid(uniqueid, - kodi_id, - v.KODI_TYPE_EPISODE, - api.provider('tmdb'), - "tmdb") + uniqueid = self.kodidb.add_uniqueid(kodi_id, + v.KODI_TYPE_EPISODE, + api.provider('tmdb'), + "tmdb") else: uniqueid = -1 self.kodidb.add_people(kodi_id, @@ -624,6 +596,7 @@ class Episode(TvShowMixin, ItemBase): airs_before_episode, playurl, kodi_pathid, + uniqueid, parent_id, api.userrating()) self.kodidb.set_resume(kodi_fileid, diff --git a/resources/lib/kodi_db/__init__.py b/resources/lib/kodi_db/__init__.py index 200d83ed..a7946736 100644 --- a/resources/lib/kodi_db/__init__.py +++ b/resources/lib/kodi_db/__init__.py @@ -62,7 +62,7 @@ def setup_kodi_default_entries(): def reset_cached_images(): LOG.info('Resetting cached artwork') LOG.debug('Resetting the Kodi texture DB') - with KodiTextureDB(wal_mode=False) as kodidb: + with KodiTextureDB() as kodidb: kodidb.wipe() LOG.debug('Deleting all cached image files') path = path_ops.translate_path('special://thumbnails/') @@ -91,11 +91,11 @@ def wipe_dbs(music=True): """ LOG.warn('Wiping Kodi databases!') LOG.info('Wiping Kodi video database') - with KodiVideoDB(wal_mode=False) as kodidb: + with KodiVideoDB() as kodidb: kodidb.wipe() if music: LOG.info('Wiping Kodi music database') - with KodiMusicDB(wal_mode=False) as kodidb: + with KodiMusicDB() as kodidb: kodidb.wipe() reset_cached_images() setup_kodi_default_entries() diff --git a/resources/lib/kodi_db/common.py b/resources/lib/kodi_db/common.py index 2a331290..e362ca6c 100644 --- a/resources/lib/kodi_db/common.py +++ b/resources/lib/kodi_db/common.py @@ -15,11 +15,9 @@ class KodiDBBase(object): Kodi database methods used for all types of items """ def __init__(self, texture_db=False, kodiconn=None, artconn=None, - lock=True, wal_mode=True): + lock=True): """ Allows direct use with a cursor instead of context mgr - Pass wal_mode=False if you want the standard sqlite journal_mode, e.g. - when wiping entire tables """ self._texture_db = texture_db self.lock = lock @@ -27,14 +25,13 @@ class KodiDBBase(object): self.cursor = self.kodiconn.cursor() if self.kodiconn else None self.artconn = artconn self.artcursor = self.artconn.cursor() if self.artconn else None - self.wal_mode = wal_mode def __enter__(self): if self.lock: KODIDB_LOCK.acquire() - self.kodiconn = db.connect(self.db_kind, self.wal_mode) + self.kodiconn = db.connect(self.db_kind) self.cursor = self.kodiconn.cursor() - self.artconn = db.connect('texture', self.wal_mode) if self._texture_db \ + self.artconn = db.connect('texture') if self._texture_db \ else None self.artcursor = self.artconn.cursor() if self._texture_db else None return self diff --git a/resources/lib/kodi_db/music.py b/resources/lib/kodi_db/music.py index 77ede150..50993806 100644 --- a/resources/lib/kodi_db/music.py +++ b/resources/lib/kodi_db/music.py @@ -25,13 +25,9 @@ class KodiMusicDB(common.KodiDBBase): try: pathid = self.cursor.fetchone()[0] except TypeError: - self.cursor.execute("SELECT COALESCE(MAX(idPath),0) FROM path") - pathid = self.cursor.fetchone()[0] + 1 - self.cursor.execute(''' - INSERT INTO path(idPath, strPath, strHash) - VALUES (?, ?, ?) - ''', - (pathid, path, '123')) + self.cursor.execute('INSERT INTO path(strPath, strHash) VALUES (?, ?)', + (path, '123')) + pathid = self.cursor.lastrowid return pathid @db.catch_operationalerrors @@ -382,10 +378,9 @@ class KodiMusicDB(common.KodiDBBase): genreid = self.cursor.fetchone()[0] except TypeError: # Create the genre - self.cursor.execute('SELECT COALESCE(MAX(idGenre),0) FROM genre') - genreid = self.cursor.fetchone()[0] + 1 - self.cursor.execute('INSERT INTO genre(idGenre, strGenre) VALUES(?, ?)', - (genreid, genre)) + self.cursor.execute('INSERT INTO genre(strGenre) VALUES(?)', + (genre, )) + genreid = self.cursor.lastrowid self.cursor.execute(''' INSERT OR REPLACE INTO album_genre( idGenre, @@ -403,10 +398,9 @@ class KodiMusicDB(common.KodiDBBase): genreid = self.cursor.fetchone()[0] except TypeError: # Create the genre - self.cursor.execute('SELECT COALESCE(MAX(idGenre),0) FROM genre') - genreid = self.cursor.fetchone()[0] + 1 - self.cursor.execute('INSERT INTO genre(idGenre, strGenre) values(?, ?)', - (genreid, genre)) + self.cursor.execute('INSERT INTO genre(strGenre) VALUES (?)', + (genre, )) + genreid = self.cursor.lastrowid self.cursor.execute(''' INSERT OR REPLACE INTO song_genre( idGenre, @@ -550,15 +544,11 @@ class KodiMusicDB(common.KodiDBBase): except TypeError: # Krypton has a dummy first entry idArtist: 1 strArtist: # [Missing Tag] strMusicBrainzArtistID: Artist Tag Missing - self.cursor.execute('SELECT COALESCE(MAX(idArtist),1) FROM artist') - artistid = self.cursor.fetchone()[0] + 1 self.cursor.execute(''' - INSERT INTO artist( - idArtist, - strArtist, - strMusicBrainzArtistID) - VALUES (?, ?, ?) - ''', (artistid, name, musicbrainz)) + INSERT INTO artist(strArtist, strMusicBrainzArtistID) + VALUES (?, ?) + ''', (name, musicbrainz)) + artistid = self.cursor.lastrowid else: if artistname != name: self.cursor.execute('UPDATE artist SET strArtist = ? WHERE idArtist = ?', diff --git a/resources/lib/kodi_db/video.py b/resources/lib/kodi_db/video.py index 6d9e1692..2466e5e3 100644 --- a/resources/lib/kodi_db/video.py +++ b/resources/lib/kodi_db/video.py @@ -38,45 +38,22 @@ class KodiVideoDB(common.KodiDBBase): For some reason, Kodi ignores this if done via itemtypes while e.g. adding or updating items. (addPath method does NOT work) """ - path_id = self.get_path(MOVIE_PATH) - if path_id is None: - self.cursor.execute("SELECT COALESCE(MAX(idPath),0) FROM path") - path_id = self.cursor.fetchone()[0] + 1 - query = ''' - INSERT INTO path(idPath, - strPath, - strContent, - strScraper, - noUpdate, - exclude) - VALUES (?, ?, ?, ?, ?, ?) - ''' - self.cursor.execute(query, (path_id, - MOVIE_PATH, - 'movies', - 'metadata.local', - 1, - 0)) - # And TV shows - path_id = self.get_path(SHOW_PATH) - if path_id is None: - self.cursor.execute("SELECT COALESCE(MAX(idPath),0) FROM path") - path_id = self.cursor.fetchone()[0] + 1 - query = ''' - INSERT INTO path(idPath, - strPath, - strContent, - strScraper, - noUpdate, - exclude) - VALUES (?, ?, ?, ?, ?, ?) - ''' - self.cursor.execute(query, (path_id, - SHOW_PATH, - 'tvshows', - 'metadata.local', - 1, - 0)) + for path, kind in ((MOVIE_PATH, 'movies'), (SHOW_PATH, 'tvshows')): + path_id = self.get_path(path) + if path_id is None: + query = ''' + INSERT INTO path(strPath, + strContent, + strScraper, + noUpdate, + exclude) + VALUES (?, ?, ?, ?, ?) + ''' + self.cursor.execute(query, (path, + kind, + 'metadata.local', + 1, + 0)) @db.catch_operationalerrors def parent_path_id(self, path): @@ -89,13 +66,12 @@ class KodiVideoDB(common.KodiDBBase): path_ops.decode_path(path_ops.path.pardir))) pathid = self.get_path(parentpath) if pathid is None: - self.cursor.execute("SELECT COALESCE(MAX(idPath),0) FROM path") - pathid = self.cursor.fetchone()[0] + 1 self.cursor.execute(''' - INSERT INTO path(idPath, strPath, dateAdded) - VALUES (?, ?, ?) + INSERT INTO path(strPath, dateAdded) + VALUES (?, ?) ''', - (pathid, parentpath, timing.kodi_now())) + (parentpath, timing.kodi_now())) + pathid = self.cursor.lastrowid if parentpath != path: # In case we end up having media in the filesystem root, C:\ parent_id = self.parent_path_id(parentpath) @@ -127,21 +103,19 @@ class KodiVideoDB(common.KodiDBBase): try: pathid = self.cursor.fetchone()[0] except TypeError: - self.cursor.execute("SELECT COALESCE(MAX(idPath),0) FROM path") - pathid = self.cursor.fetchone()[0] + 1 self.cursor.execute(''' INSERT INTO path( - idPath, strPath, dateAdded, idParentPath, strContent, strScraper, noUpdate) - VALUES (?, ?, ?, ?, ?, ?, ?) + VALUES (?, ?, ?, ?, ?, ?) ''', - (pathid, path, date_added, id_parent_path, - content, scraper, 1)) + (path, date_added, id_parent_path, content, + scraper, 1)) + pathid = self.cursor.lastrowid return pathid def get_path(self, path): @@ -161,18 +135,12 @@ class KodiVideoDB(common.KodiDBBase): Adds the filename [unicode] to the table files if not already added and returns the idFile. """ - self.cursor.execute('SELECT COALESCE(MAX(idFile), 0) FROM files') - file_id = self.cursor.fetchone()[0] + 1 self.cursor.execute(''' - INSERT INTO files( - idFile, - idPath, - strFilename, - dateAdded) - VALUES (?, ?, ?, ?) + INSERT INTO files(idPath, strFilename, dateAdded) + VALUES (?, ?, ?) ''', - (file_id, path_id, filename, date_added)) - return file_id + (path_id, filename, date_added)) + return self.cursor.lastrowid def modify_file(self, filename, path_id, date_added): self.cursor.execute('SELECT idFile FROM files WHERE idPath = ? AND strFilename = ?', @@ -261,11 +229,9 @@ class KodiVideoDB(common.KodiDBBase): try: entry_id = self.cursor.fetchone()[0] except TypeError: - self.cursor.execute('SELECT COALESCE(MAX(%s), %s) FROM %s' - % (key, first_id - 1, table)) - entry_id = self.cursor.fetchone()[0] + 1 - self.cursor.execute('INSERT INTO %s(%s, name) values(?, ?)' - % (table, key), (entry_id, entry)) + self.cursor.execute('INSERT INTO %s(name) VALUES(?)' % table, + (entry, )) + entry_id = self.cursor.lastrowid finally: entry_ids.append(entry_id) # Now process the ids obtained from the names @@ -458,10 +424,8 @@ class KodiVideoDB(common.KodiDBBase): @db.catch_operationalerrors def _new_actor_id(self, name, art_url): # Not yet in actor DB, add person - self.cursor.execute('SELECT COALESCE(MAX(actor_id), 0) FROM actor') - actor_id = self.cursor.fetchone()[0] + 1 - self.cursor.execute('INSERT INTO actor(actor_id, name) VALUES (?, ?)', - (actor_id, name)) + self.cursor.execute('INSERT INTO actor(name) VALUES (?)', (name, )) + actor_id = self.cursor.lastrowid if art_url: self.add_art(art_url, actor_id, 'actor', 'thumb') return actor_id @@ -649,12 +613,8 @@ class KodiVideoDB(common.KodiDBBase): (playcount or None, dateplayed, file_id)) # Set the resume bookmark if resume_seconds: - self.cursor.execute( - 'SELECT COALESCE(MAX(idBookmark), 0) FROM bookmark') - bookmark_id = self.cursor.fetchone()[0] + 1 self.cursor.execute(''' INSERT INTO bookmark( - idBookmark, idFile, timeInSeconds, totalTimeInSeconds, @@ -662,9 +622,8 @@ class KodiVideoDB(common.KodiDBBase): player, playerState, type) - VALUES (?, ?, ?, ?, ?, ?, ?, ?) - ''', (bookmark_id, - file_id, + VALUES (?, ?, ?, ?, ?, ?, ?) + ''', (file_id, resume_seconds, total_seconds, '', @@ -682,10 +641,8 @@ class KodiVideoDB(common.KodiDBBase): try: tag_id = self.cursor.fetchone()[0] except TypeError: - self.cursor.execute("SELECT COALESCE(MAX(tag_id), 0) FROM tag") - tag_id = self.cursor.fetchone()[0] + 1 - self.cursor.execute('INSERT INTO tag(tag_id, name) VALUES(?, ?)', - (tag_id, name)) + self.cursor.execute('INSERT INTO tag(name) VALUES(?)', (name, )) + tag_id = self.cursor.lastrowid return tag_id @db.catch_operationalerrors @@ -717,10 +674,8 @@ class KodiVideoDB(common.KodiDBBase): try: setid = self.cursor.fetchone()[0] except TypeError: - self.cursor.execute("SELECT COALESCE(MAX(idSet), 0) FROM sets") - setid = self.cursor.fetchone()[0] + 1 - self.cursor.execute('INSERT INTO sets(idSet, strSet) VALUES(?, ?)', - (setid, set_name)) + self.cursor.execute('INSERT INTO sets(strSet) VALUES(?)', (set_name, )) + setid = self.cursor.lastrowid return setid @db.catch_operationalerrors @@ -768,19 +723,14 @@ class KodiVideoDB(common.KodiDBBase): Adds a TV show season to the Kodi video DB or simply returns the ID, if there already is an entry in the DB """ - self.cursor.execute("SELECT COALESCE(MAX(idSeason),0) FROM seasons") - seasonid = self.cursor.fetchone()[0] + 1 - self.cursor.execute(''' - INSERT INTO seasons(idSeason, idShow, season) - VALUES (?, ?, ?) - ''', (seasonid, showid, seasonnumber)) - return seasonid + self.cursor.execute('INSERT INTO seasons(idShow, season) VALUES (?, ?)', + (showid, seasonnumber)) + return self.cursor.lastrowid @db.catch_operationalerrors def add_uniqueid(self, *args): """ Feed with: - uniqueid_id: int media_id: int media_type: string value: string @@ -788,39 +738,24 @@ class KodiVideoDB(common.KodiDBBase): """ self.cursor.execute(''' INSERT INTO uniqueid( - uniqueid_id, media_id, media_type, value, type) - VALUES (?, ?, ?, ?, ?) + VALUES (?, ?, ?, ?) ''', (args)) - - def add_uniqueid_id(self): - self.cursor.execute('SELECT COALESCE(MAX(uniqueid_id), 0) FROM uniqueid') - return self.cursor.fetchone()[0] + 1 - - def get_uniqueid(self, kodi_id, kodi_type): - """ - Returns the uniqueid_id - """ - self.cursor.execute('SELECT uniqueid_id FROM uniqueid WHERE media_id = ? AND media_type =?', - (kodi_id, kodi_type)) - try: - return self.cursor.fetchone()[0] - except TypeError: - return self.add_uniqueid_id() + return self.cursor.lastrowid @db.catch_operationalerrors def update_uniqueid(self, *args): """ - Pass in media_id, media_type, value, type, uniqueid_id + Pass in value, media_id, media_type, type """ self.cursor.execute(''' - UPDATE uniqueid - SET media_id = ?, media_type = ?, value = ?, type = ? - WHERE uniqueid_id = ? + INSERT OR REPLACE INTO uniqueid(media_id, media_type, type, value) + VALUES(?, ?, ?, ?) ''', (args)) + return self.cursor.lastrowid @db.catch_operationalerrors def remove_uniqueid(self, kodi_id, kodi_type): @@ -830,54 +765,36 @@ class KodiVideoDB(common.KodiDBBase): self.cursor.execute('DELETE FROM uniqueid WHERE media_id = ? AND media_type = ?', (kodi_id, kodi_type)) - def add_ratingid(self): - self.cursor.execute('SELECT COALESCE(MAX(rating_id),0) FROM rating') - return self.cursor.fetchone()[0] + 1 - - def get_ratingid(self, kodi_id, kodi_type): - """ - Create if needed and return the unique rating_id from rating table - """ - self.cursor.execute('SELECT rating_id FROM rating WHERE media_id = ? AND media_type = ?', - (kodi_id, kodi_type)) - try: - return self.cursor.fetchone()[0] - except TypeError: - return self.add_ratingid() - @db.catch_operationalerrors def update_ratings(self, *args): """ Feed with media_id, media_type, rating_type, rating, votes, rating_id """ self.cursor.execute(''' - UPDATE rating - SET media_id = ?, - media_type = ?, - rating_type = ?, - rating = ?, - votes = ? - WHERE rating_id = ? + INSERT OR REPLACE INTO + rating(media_id, media_type, rating_type, rating, votes) + VALUES (?, ?, ?, ?, ?) ''', (args)) + return self.cursor.lastrowid @db.catch_operationalerrors def add_ratings(self, *args): """ feed with: - rating_id, media_id, media_type, rating_type, rating, votes + media_id, media_type, rating_type, rating, votes rating_type = 'default' """ self.cursor.execute(''' INSERT INTO rating( - rating_id, media_id, media_type, rating_type, rating, votes) - VALUES (?, ?, ?, ?, ?, ?) + VALUES (?, ?, ?, ?, ?) ''', (args)) + return self.cursor.lastrowid @db.catch_operationalerrors def remove_ratings(self, kodi_id, kodi_type): @@ -917,10 +834,11 @@ class KodiVideoDB(common.KodiDBBase): c16, c18, c19, + c20, idSeason, userrating) VALUES - (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ''', (args)) @db.catch_operationalerrors @@ -942,6 +860,7 @@ class KodiVideoDB(common.KodiDBBase): c16 = ?, c18 = ?, c19 = ?, + c20 = ?, idFile=?, idSeason = ?, userrating = ? diff --git a/resources/lib/library_sync/common.py b/resources/lib/library_sync/common.py index 5497dbf8..a0f43285 100644 --- a/resources/lib/library_sync/common.py +++ b/resources/lib/library_sync/common.py @@ -1,40 +1,41 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- from __future__ import absolute_import, division, unicode_literals +from logging import getLogger import xbmc from .. import utils, app, variables as v +LOG = getLogger('PLEX.sync') + PLAYLIST_SYNC_ENABLED = (v.DEVICE != 'Microsoft UWP' and utils.settings('enablePlaylistSync') == 'true') -class fullsync_mixin(object): - def __init__(self): - self._canceled = False +class LibrarySyncMixin(object): + def suspend(self, block=False, timeout=None): + """ + Let's NOT suspend sync threads but immediately terminate them + """ + self.cancel() - def abort(self): - """Hit method to terminate the thread""" - self._canceled = True - # Let's NOT suspend sync threads but immediately terminate them - suspend = abort + def wait_while_suspended(self): + """ + Return immediately + """ + return self.should_cancel() - @property - def suspend_reached(self): - """Since we're not suspending, we'll never set it to True""" - return False - - @suspend_reached.setter - def suspend_reached(self): - pass - - def resume(self): - """Obsolete since we're not suspending""" - pass - - def isCanceled(self): - """Check whether we should exit this thread""" - return self._canceled + def run(self): + app.APP.register_thread(self) + LOG.debug('##===--- Starting %s ---===##', self.__class__.__name__) + try: + self._run() + except Exception as err: + LOG.error('Exception encountered: %s', err) + utils.ERROR(notify=True) + finally: + app.APP.deregister_thread(self) + LOG.debug('##===--- %s Stopped ---===##', self.__class__.__name__) def update_kodi_library(video=True, music=True): diff --git a/resources/lib/library_sync/fanart.py b/resources/lib/library_sync/fanart.py index 7a4b59ff..9ad26a7a 100644 --- a/resources/lib/library_sync/fanart.py +++ b/resources/lib/library_sync/fanart.py @@ -27,48 +27,51 @@ class FanartThread(backgroundthread.KillableThread): self.refresh = refresh super(FanartThread, self).__init__() - def isSuspended(self): + def should_suspend(self): return self._suspended or app.APP.is_playing_video def run(self): LOG.info('Starting FanartThread') app.APP.register_fanart_thread(self) try: - self._run_internal() + self._run() except Exception: utils.ERROR(notify=True) finally: app.APP.deregister_fanart_thread(self) - def _run_internal(self): + def _loop(self): + for typus in SUPPORTED_TYPES: + offset = 0 + while True: + with PlexDB() as plexdb: + # Keep DB connection open only for a short period of time! + if self.refresh: + batch = list(plexdb.every_plex_id(typus, + offset, + BATCH_SIZE)) + else: + batch = list(plexdb.missing_fanart(typus, + offset, + BATCH_SIZE)) + for plex_id in batch: + # Do the actual, time-consuming processing + if self.should_suspend() or self.should_cancel(): + return False + process_fanart(plex_id, typus, self.refresh) + if len(batch) < BATCH_SIZE: + break + offset += BATCH_SIZE + return True + + def _run(self): finished = False - try: - for typus in SUPPORTED_TYPES: - offset = 0 - while True: - with PlexDB() as plexdb: - # Keep DB connection open only for a short period of time! - if self.refresh: - batch = list(plexdb.every_plex_id(typus, - offset, - BATCH_SIZE)) - else: - batch = list(plexdb.missing_fanart(typus, - offset, - BATCH_SIZE)) - for plex_id in batch: - # Do the actual, time-consuming processing - if self.wait_while_suspended(): - return - process_fanart(plex_id, typus, self.refresh) - if len(batch) < BATCH_SIZE: - break - offset += BATCH_SIZE - else: - finished = True - finally: - LOG.info('FanartThread finished: %s', finished) - self.callback(finished) + while not finished: + finished = self._loop() + if self.wait_while_suspended(): + break + LOG.info('FanartThread finished: %s', finished) + self.callback(finished) class FanartTask(backgroundthread.Task): diff --git a/resources/lib/library_sync/fill_metadata_queue.py b/resources/lib/library_sync/fill_metadata_queue.py new file mode 100644 index 00000000..37e8eef0 --- /dev/null +++ b/resources/lib/library_sync/fill_metadata_queue.py @@ -0,0 +1,78 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, division, unicode_literals +from logging import getLogger +from Queue import Empty + +from . import common, sections +from ..plex_db import PlexDB +from .. import backgroundthread + +LOG = getLogger('PLEX.sync.fill_metadata_queue') + +QUEUE_TIMEOUT = 10 # seconds + + +class FillMetadataQueue(common.LibrarySyncMixin, + backgroundthread.KillableThread): + """ + Determines which plex_ids we need to sync and puts these ids in a separate + queue. Will use a COPIED plex.db file (plex-copy.db) in order to read much + faster without the writing thread stalling + """ + def __init__(self, repair, section_queue, get_metadata_queue, + processing_queue): + self.repair = repair + self.section_queue = section_queue + self.get_metadata_queue = get_metadata_queue + self.processing_queue = processing_queue + super(FillMetadataQueue, self).__init__() + + def _process_section(self, section): + # Initialize only once to avoid loosing the last value before we're + # breaking the for loop + LOG.debug('Process section %s with %s items', + section, section.number_of_items) + count = 0 + do_process_section = False + with PlexDB(lock=False, copy=True) as plexdb: + for xml in section.iterator: + if self.should_cancel(): + break + plex_id = int(xml.get('ratingKey')) + checksum = int('{}{}'.format( + plex_id, + xml.get('updatedAt', + xml.get('addedAt', '1541572987')))) + if (not self.repair and + plexdb.checksum(plex_id, section.plex_type) == checksum): + continue + try: + self.get_metadata_queue.put((count, plex_id, section), + timeout=QUEUE_TIMEOUT) + except Empty: + LOG.error('Putting %s in get_metadata_queue timed out - ' + 'aborting sync now', plex_id) + section.sync_successful = False + break + count += 1 + if not do_process_section: + do_process_section = True + self.processing_queue.add_section(section) + LOG.debug('Put section in queue with %s items: %s', + section.number_of_items, section) + # We might have received LESS items from the PMS than anticipated. + # Ensures that our queues finish + LOG.debug('%s items to process for section %s', count, section) + section.number_of_items = count + + def _run(self): + while not self.should_cancel(): + section = self.section_queue.get() + self.section_queue.task_done() + if section is None: + break + self._process_section(section) + # Signal the download metadata threads to stop with a sentinel + self.get_metadata_queue.put(None) + # Sentinel for the process_thread once we added everything else + self.processing_queue.add_sentinel(sections.Section()) diff --git a/resources/lib/library_sync/full_sync.py b/resources/lib/library_sync/full_sync.py index e3f4f1eb..cdb85e30 100644 --- a/resources/lib/library_sync/full_sync.py +++ b/resources/lib/library_sync/full_sync.py @@ -3,345 +3,208 @@ from __future__ import absolute_import, division, unicode_literals from logging import getLogger import Queue -import copy import xbmcgui -from .get_metadata import GetMetadataTask, reset_collections +from .get_metadata import GetMetadataThread +from .fill_metadata_queue import FillMetadataQueue +from .process_metadata import ProcessMetadataThread from . import common, sections -from .. import utils, timing, backgroundthread, variables as v, app -from .. import plex_functions as PF, itemtypes -from ..plex_db import PlexDB +from .. import utils, timing, backgroundthread as bg, variables as v, app +from .. import plex_functions as PF, itemtypes, path_ops if common.PLAYLIST_SYNC_ENABLED: from .. import playlists LOG = getLogger('PLEX.sync.full_sync') -# How many items will be put through the processing chain at once? -BATCH_SIZE = 2000 +DELETION_BATCH_SIZE = 250 +PLAYSTATE_BATCH_SIZE = 5000 + +# Max. number of plex_ids held in memory for later processing +BACKLOG_QUEUE_SIZE = 10000 +# Max number of xmls held in memory +XML_QUEUE_SIZE = 500 # Safety margin to filter PMS items - how many seconds to look into the past? UPDATED_AT_SAFETY = 60 * 5 LAST_VIEWED_AT_SAFETY = 60 * 5 -class InitNewSection(object): - """ - Throw this into the queue used for ProcessMetadata to tell it which - Plex library section we're looking at - """ - def __init__(self, context, total_number_of_items, section_name, - section_id, plex_type): - self.context = context - self.total = total_number_of_items - self.name = section_name - self.id = section_id - self.plex_type = plex_type - - -class FullSync(common.fullsync_mixin): +class FullSync(common.LibrarySyncMixin, bg.KillableThread): def __init__(self, repair, callback, show_dialog): """ repair=True: force sync EVERY item """ + self.successful = True self.repair = repair self.callback = callback - self.queue = None - self.process_thread = None - self.current_sync = None - self.plexdb = None - self.plex_type = None - self.section_type = None - self.worker_count = int(utils.settings('syncThreadNumber')) - self.item_count = 0 # For progress dialog self.show_dialog = show_dialog self.show_dialog_userdata = utils.settings('playstate_sync_indicator') == 'true' - self.dialog = None - self.total = 0 - self.current = 0 - self.processed = 0 - self.title = '' - self.section = None - self.section_name = None - self.section_type_text = None - self.context = None - self.get_children = None - self.successful = None - self.section_success = None + if self.show_dialog: + self.dialog = xbmcgui.DialogProgressBG() + self.dialog.create(utils.lang(39714)) + else: + self.dialog = None + self.current_time = timing.plex_now() + self.last_section = sections.Section() self.install_sync_done = utils.settings('SyncInstallRunDone') == 'true' - self.threader = backgroundthread.ThreaderManager( - worker=backgroundthread.NonstoppingBackgroundWorker, - worker_count=self.worker_count) super(FullSync, self).__init__() - def update_progressbar(self): - if self.dialog: - try: - progress = int(float(self.current) / float(self.total) * 100.0) - except ZeroDivisionError: - progress = 0 - self.dialog.update(progress, - '%s (%s)' % (self.section_name, self.section_type_text), - '%s %s/%s' - % (self.title, self.current, self.total)) - if app.APP.is_playing_video: - self.dialog.close() - self.dialog = None - - def process_item(self, xml_item): - """ - Processes a single library item - """ - plex_id = int(xml_item.get('ratingKey')) - if not self.repair and self.plexdb.checksum(plex_id, self.plex_type) == \ - int('%s%s' % (plex_id, - xml_item.get('updatedAt', - xml_item.get('addedAt', 1541572987)))): + def update_progressbar(self, section, title, current): + if not self.dialog: return - self.threader.addTask(GetMetadataTask(self.queue, - plex_id, - self.plex_type, - self.get_children, - self.item_count)) - self.item_count += 1 - - def update_library(self): - LOG.debug('Writing changes to Kodi library now') - i = 0 - if not self.section: - _, self.section = self.queue.get() - self.queue.task_done() - while not self.isCanceled() and self.item_count > 0: - section = self.section - if not section: - break - LOG.debug('Start or continue processing section %s (%ss)', - section.name, section.plex_type) - self.processed = 0 - self.total = section.total - self.section_name = section.name - self.section_type_text = utils.lang( - v.TRANSLATION_FROM_PLEXTYPE[section.plex_type]) - with section.context(self.current_sync) as context: - while not self.isCanceled() and self.item_count > 0: - try: - _, item = self.queue.get(block=False) - except backgroundthread.Queue.Empty: - if self.threader.threader.working(): - app.APP.monitor.waitForAbort(0.02) - continue - else: - # Try again, in case a thread just finished - i += 1 - if i == 3: - break - continue - i = 0 - self.queue.task_done() - if isinstance(item, dict): - context.add_update(item['xml'][0], - section_name=section.name, - section_id=section.id, - children=item['children']) - self.title = item['xml'][0].get('title') - self.processed += 1 - elif isinstance(item, InitNewSection) or item is None: - self.section = item - break - else: - raise ValueError('Unknown type %s' % type(item)) - self.item_count -= 1 - self.current += 1 - self.update_progressbar() - if self.processed == 500: - self.processed = 0 - context.commit() - LOG.debug('Done writing changes to Kodi library') - - @utils.log_time - def addupdate_section(self, section): - LOG.debug('Processing library section for new or changed items %s', - section) - if not self.install_sync_done: - app.SYNC.path_verified = False + current += 1 try: - # Sync new, updated and deleted items - iterator = section.iterator - # Tell the processing thread about this new section - queue_info = InitNewSection(section.context, - iterator.total, - iterator.get('librarySectionTitle', - iterator.get('title1')), - section.section_id, - section.plex_type) - self.queue.put((-1, queue_info)) - last = True - # To keep track of the item-number in order to kill while loops - self.item_count = 0 - self.current = 0 - # Initialize only once to avoid loosing the last value before - # we're breaking the for loop - loop = common.tag_last(iterator) - while True: - # Check Plex DB to see what we need to add/update - with PlexDB() as self.plexdb: - for last, xml_item in loop: - if self.isCanceled(): - return False - self.process_item(xml_item) - if self.item_count == BATCH_SIZE: - break - # Make sure Plex DB above is closed before adding/updating! - self.update_library() - if last: - break - reset_collections() - return True - except RuntimeError: - LOG.error('Could not entirely process section %s', section) - return False + progress = int(float(current) / float(section.number_of_items) * 100.0) + except ZeroDivisionError: + progress = 0 + self.dialog.update(progress, + '%s (%s)' % (section.name, section.section_type_text), + '%s %s/%s' + % (title, current, section.number_of_items)) + if app.APP.is_playing_video: + self.dialog.close() + self.dialog = None + + @staticmethod + def copy_plex_db(): + """ + Takes the current plex.db file and copies it to plex-copy.db + This will allow us to have "concurrent" connections during adding/ + updating items, increasing sync speed tremendously. + Using the same DB with e.g. WAL mode did not really work out... + """ + path_ops.copyfile(v.DB_PLEX_PATH, v.DB_PLEX_COPY_PATH) @utils.log_time + def process_new_and_changed_items(self, section_queue, processing_queue): + LOG.debug('Start working') + get_metadata_queue = Queue.Queue(maxsize=BACKLOG_QUEUE_SIZE) + scanner_thread = FillMetadataQueue(self.repair, + section_queue, + get_metadata_queue, + processing_queue) + scanner_thread.start() + metadata_threads = [ + GetMetadataThread(get_metadata_queue, processing_queue) + for _ in range(int(utils.settings('syncThreadNumber'))) + ] + for t in metadata_threads: + t.start() + process_thread = ProcessMetadataThread(self.current_time, + processing_queue, + self.update_progressbar) + process_thread.start() + LOG.debug('Waiting for scanner thread to finish up') + scanner_thread.join() + LOG.debug('Waiting for metadata download threads to finish up') + for t in metadata_threads: + t.join() + LOG.debug('Download metadata threads finished') + process_thread.join() + self.successful = process_thread.successful + LOG.debug('threads finished work. successful: %s', self.successful) + + @utils.log_time + def processing_loop_playstates(self, section_queue): + while not self.should_cancel(): + section = section_queue.get() + section_queue.task_done() + if section is None: + break + self.playstate_per_section(section) + def playstate_per_section(self, section): LOG.debug('Processing %s playstates for library section %s', - section.iterator.total, section) + section.number_of_items, section) try: - # Sync new, updated and deleted items - iterator = section.iterator - # Tell the processing thread about this new section - queue_info = InitNewSection(section.context, - iterator.total, - section.name, - section.section_id, - section.plex_type) - self.queue.put((-1, queue_info)) - self.total = iterator.total - self.section_name = section.name - self.section_type_text = utils.lang( - v.TRANSLATION_FROM_PLEXTYPE[section.plex_type]) - self.current = 0 - - last = True - loop = common.tag_last(iterator) - while True: - with section.context(self.current_sync) as itemtype: - for i, (last, xml_item) in enumerate(loop): - if self.isCanceled(): - return False - if not itemtype.update_userdata(xml_item, section.plex_type): - # Somehow did not sync this item yet - itemtype.add_update(xml_item, - section_name=section.name, - section_id=section.section_id) - itemtype.plexdb.update_last_sync(int(xml_item.attrib['ratingKey']), - section.plex_type, - self.current_sync) - self.current += 1 - self.update_progressbar() - if (i + 1) % (10 * BATCH_SIZE) == 0: - break - if last: - break - return True + with section.context(self.current_time) as context: + for xml in section.iterator: + section.count += 1 + if not context.update_userdata(xml, section.plex_type): + # Somehow did not sync this item yet + context.add_update(xml, + section_name=section.name, + section_id=section.section_id) + context.plexdb.update_last_sync(int(xml.attrib['ratingKey']), + section.plex_type, + self.current_time) + self.update_progressbar(section, '', section.count - 1) + if section.count % PLAYSTATE_BATCH_SIZE == 0: + context.commit() except RuntimeError: LOG.error('Could not entirely process section %s', section) - return False + self.successful = False - def threaded_get_iterators(self, kinds, queue, all_items=False): + def threaded_get_generators(self, kinds, section_queue, all_items): """ - Getting iterators is costly, so let's do it asynchronously + Getting iterators is costly, so let's do it in a dedicated thread """ + LOG.debug('Start threaded_get_generators') try: for kind in kinds: for section in (x for x in app.SYNC.sections if x.section_type == kind[1]): - if self.isCanceled(): + if self.should_cancel(): LOG.debug('Need to exit now') return if not section.sync_to_kodi: LOG.info('User chose to not sync section %s', section) continue - element = copy.deepcopy(section) - element.plex_type = kind[0] - element.section_type = element.plex_type - element.context = kind[2] - element.get_children = kind[3] - element.Queue = kind[4] + section = sections.get_sync_section(section, + plex_type=kind[0]) if self.repair or all_items: updated_at = None else: updated_at = section.last_sync - UPDATED_AT_SAFETY \ if section.last_sync else None try: - element.iterator = PF.get_section_iterator( + section.iterator = PF.get_section_iterator( section.section_id, - plex_type=element.plex_type, + plex_type=section.plex_type, updated_at=updated_at, last_viewed_at=None) except RuntimeError: - LOG.warn('Sync at least partially unsuccessful') - self.successful = False - self.section_success = False + LOG.error('Sync at least partially unsuccessful!') + LOG.error('Error getting section iterator %s', section) else: - queue.put(element) + section.number_of_items = section.iterator.total + if section.number_of_items > 0: + section_queue.put(section) + LOG.debug('Put section in queue with %s items: %s', + section.number_of_items, section) except Exception: utils.ERROR(notify=True) finally: - queue.put(None) + # Sentinel for the section queue + section_queue.put(None) + LOG.debug('Exiting threaded_get_generators') def full_library_sync(self): - """ - """ - # structure: - # (plex_type, - # section_type, - # context for itemtype, - # download children items, e.g. songs for a specific album?, - # Queue) + section_queue = Queue.Queue() + processing_queue = bg.ProcessingQueue(maxsize=XML_QUEUE_SIZE) kinds = [ - (v.PLEX_TYPE_MOVIE, v.PLEX_TYPE_MOVIE, itemtypes.Movie, False, Queue.Queue), - (v.PLEX_TYPE_SHOW, v.PLEX_TYPE_SHOW, itemtypes.Show, False, Queue.Queue), - (v.PLEX_TYPE_SEASON, v.PLEX_TYPE_SHOW, itemtypes.Season, False, Queue.Queue), - (v.PLEX_TYPE_EPISODE, v.PLEX_TYPE_SHOW, itemtypes.Episode, False, Queue.Queue) + (v.PLEX_TYPE_MOVIE, v.PLEX_TYPE_MOVIE), + (v.PLEX_TYPE_SHOW, v.PLEX_TYPE_SHOW), + (v.PLEX_TYPE_SEASON, v.PLEX_TYPE_SHOW), + (v.PLEX_TYPE_EPISODE, v.PLEX_TYPE_SHOW) ] if app.SYNC.enable_music: kinds.extend([ - (v.PLEX_TYPE_ARTIST, v.PLEX_TYPE_ARTIST, itemtypes.Artist, False, Queue.Queue), - (v.PLEX_TYPE_ALBUM, v.PLEX_TYPE_ARTIST, itemtypes.Album, True, backgroundthread.OrderedQueue), + (v.PLEX_TYPE_ARTIST, v.PLEX_TYPE_ARTIST), + (v.PLEX_TYPE_ALBUM, v.PLEX_TYPE_ARTIST), ]) # ADD NEW ITEMS - # Already start setting up the iterators. We need to enforce - # syncing e.g. show before season before episode - iterator_queue = Queue.Queue() - task = backgroundthread.FunctionAsTask(self.threaded_get_iterators, - None, - kinds, - iterator_queue) - backgroundthread.BGThreader.addTask(task) - while True: - self.section_success = True - section = iterator_queue.get() - iterator_queue.task_done() - if section is None: - break - # Setup our variables - self.plex_type = section.plex_type - self.section_type = section.section_type - self.context = section.context - self.get_children = section.get_children - self.queue = section.Queue() - # Now do the heavy lifting - if self.isCanceled() or not self.addupdate_section(section): - return False - if self.section_success: - # Need to check because a thread might have missed to get - # some items from the PMS - with PlexDB() as plexdb: - # Set the new time mark for the next delta sync - plexdb.update_section_last_sync(section.section_id, - self.current_sync) + # We need to enforce syncing e.g. show before season before episode + bg.FunctionAsTask(self.threaded_get_generators, + None, + kinds, section_queue, False).start() + # Do the heavy lifting + self.process_new_and_changed_items(section_queue, processing_queue) common.update_kodi_library(video=True, music=True) + if self.should_cancel() or not self.successful: + return # Sync Plex playlists to Kodi and vice-versa if common.PLAYLIST_SYNC_ENABLED: @@ -351,48 +214,29 @@ class FullSync(common.fullsync_mixin): self.dialog = xbmcgui.DialogProgressBG() # "Synching playlists" self.dialog.create(utils.lang(39715)) - if not playlists.full_sync(): - return False + if not playlists.full_sync() or self.should_cancel(): + return # SYNC PLAYSTATE of ALL items (otherwise we won't pick up on items that # were set to unwatched). Also mark all items on the PMS to be able # to delete the ones still in Kodi - LOG.info('Start synching playstate and userdata for every item') - # In order to not delete all your songs again + LOG.debug('Start synching playstate and userdata for every item') if app.SYNC.enable_music: - # We don't need to enforce the album order now - kinds.pop(5) + # In order to not delete all your songs again kinds.extend([ - (v.PLEX_TYPE_ALBUM, v.PLEX_TYPE_ARTIST, itemtypes.Album, True, Queue.Queue), - (v.PLEX_TYPE_SONG, v.PLEX_TYPE_ARTIST, itemtypes.Song, True, Queue.Queue), + (v.PLEX_TYPE_SONG, v.PLEX_TYPE_ARTIST), ]) # Make sure we're not showing an item's title in the sync dialog - self.title = '' - self.threader.shutdown() - self.threader = None if not self.show_dialog_userdata and self.dialog: # Close the progress indicator dialog self.dialog.close() self.dialog = None - task = backgroundthread.FunctionAsTask(self.threaded_get_iterators, - None, - kinds, - iterator_queue, - all_items=True) - backgroundthread.BGThreader.addTask(task) - while True: - section = iterator_queue.get() - iterator_queue.task_done() - if section is None: - break - # Setup our variables - self.plex_type = section.plex_type - self.section_type = section.section_type - self.context = section.context - self.get_children = section.get_children - # Now do the heavy lifting - if self.isCanceled() or not self.playstate_per_section(section): - return False + bg.FunctionAsTask(self.threaded_get_generators, + None, + kinds, section_queue, True).start() + self.processing_loop_playstates(section_queue) + if self.should_cancel() or not self.successful: + return # Delete movies that are not on Plex anymore LOG.debug('Looking for items to delete') @@ -411,61 +255,40 @@ class FullSync(common.fullsync_mixin): for plex_type, context in kinds: # Delete movies that are not on Plex anymore while True: - with context(self.current_sync) as ctx: - plex_ids = list(ctx.plexdb.plex_id_by_last_sync(plex_type, - self.current_sync, - BATCH_SIZE)) + with context(self.current_time) as ctx: + plex_ids = list( + ctx.plexdb.plex_id_by_last_sync(plex_type, + self.current_time, + DELETION_BATCH_SIZE)) for plex_id in plex_ids: - if self.isCanceled(): - return False + if self.should_cancel(): + return ctx.remove(plex_id, plex_type) - if len(plex_ids) < BATCH_SIZE: + if len(plex_ids) < DELETION_BATCH_SIZE: break - LOG.debug('Done deleting') - return True - - def run(self): - app.APP.register_thread(self) - try: - self._run() - finally: - app.APP.deregister_thread(self) - LOG.info('Done full_sync') + LOG.debug('Done looking for items to delete') @utils.log_time def _run(self): - self.current_sync = timing.plex_now() - # Get latest Plex libraries and build playlist and video node files - if self.isCanceled() or not sections.sync_from_pms(self): - return - self.successful = True try: - if self.show_dialog: - self.dialog = xbmcgui.DialogProgressBG() - self.dialog.create(utils.lang(39714)) - - # Actual syncing - do only new items first - LOG.info('Running full_library_sync with repair=%s', - self.repair) - if self.isCanceled() or not self.full_library_sync(): - self.successful = False + # Get latest Plex libraries and build playlist and video node files + if self.should_cancel() or not sections.sync_from_pms(self): return + self.copy_plex_db() + self.full_library_sync() finally: common.update_kodi_library(video=True, music=True) if self.dialog: self.dialog.close() - if self.threader: - self.threader.shutdown() - self.threader = None - if not self.successful and not self.isCanceled(): + if not self.successful and not self.should_cancel(): # "ERROR in library sync" utils.dialog('notification', heading='{plex}', message=utils.lang(39410), icon='{error}') - if self.callback: - self.callback(self.successful) + self.callback(self.successful) def start(show_dialog, repair=False, callback=None): + # Call run() and NOT start in order to not spawn another thread FullSync(repair, callback, show_dialog).run() diff --git a/resources/lib/library_sync/get_metadata.py b/resources/lib/library_sync/get_metadata.py index 6cd653c2..e21d2c27 100644 --- a/resources/lib/library_sync/get_metadata.py +++ b/resources/lib/library_sync/get_metadata.py @@ -4,66 +4,43 @@ from logging import getLogger from . import common from ..plex_api import API -from .. import plex_functions as PF, backgroundthread, utils, variables as v - - -LOG = getLogger("PLEX." + __name__) +from .. import backgroundthread, plex_functions as PF, utils, variables as v +LOG = getLogger('PLEX.sync.get_metadata') LOCK = backgroundthread.threading.Lock() -# List of tuples: (collection index [as in an item's metadata with "Collection -# id"], collection plex id) -COLLECTION_MATCH = None -# Dict with entries of the form : -COLLECTION_XMLS = {} -def reset_collections(): - """ - Collections seem unique to Plex sections - """ - global LOCK, COLLECTION_MATCH, COLLECTION_XMLS - with LOCK: - COLLECTION_MATCH = None - COLLECTION_XMLS = {} - - -class GetMetadataTask(common.fullsync_mixin, backgroundthread.Task): +class GetMetadataThread(common.LibrarySyncMixin, + backgroundthread.KillableThread): """ Threaded download of Plex XML metadata for a certain library item. Fills the queue with the downloaded etree XML objects - - Input: - queue Queue.Queue() object where this thread will store - the downloaded metadata XMLs as etree objects """ - def __init__(self, queue, plex_id, plex_type, get_children=False, - count=None): - self.queue = queue - self.plex_id = plex_id - self.plex_type = plex_type - self.get_children = get_children - self.count = count - super(GetMetadataTask, self).__init__() + def __init__(self, get_metadata_queue, processing_queue): + self.get_metadata_queue = get_metadata_queue + self.processing_queue = processing_queue + super(GetMetadataThread, self).__init__() def _collections(self, item): - global COLLECTION_MATCH, COLLECTION_XMLS api = API(item['xml'][0]) - if COLLECTION_MATCH is None: - COLLECTION_MATCH = PF.collections(api.library_section_id()) - if COLLECTION_MATCH is None: + collection_match = item['section'].collection_match + collection_xmls = item['section'].collection_xmls + if collection_match is None: + collection_match = PF.collections(api.library_section_id()) + if collection_match is None: LOG.error('Could not download collections') return # Extract what we need to know - COLLECTION_MATCH = \ + collection_match = \ [(utils.cast(int, x.get('index')), - utils.cast(int, x.get('ratingKey'))) for x in COLLECTION_MATCH] + utils.cast(int, x.get('ratingKey'))) for x in collection_match] item['children'] = {} for plex_set_id, set_name in api.collections(): - if self.isCanceled(): + if self.should_cancel(): return - if plex_set_id not in COLLECTION_XMLS: + if plex_set_id not in collection_xmls: # Get Plex metadata for collections - a pain - for index, collection_plex_id in COLLECTION_MATCH: + for index, collection_plex_id in collection_match: if index == plex_set_id: collection_xml = PF.GetPlexMetadata(collection_plex_id) try: @@ -72,54 +49,75 @@ class GetMetadataTask(common.fullsync_mixin, backgroundthread.Task): LOG.error('Could not get collection %s %s', collection_plex_id, set_name) continue - COLLECTION_XMLS[plex_set_id] = collection_xml + collection_xmls[plex_set_id] = collection_xml break else: LOG.error('Did not find Plex collection %s %s', plex_set_id, set_name) continue - item['children'][plex_set_id] = COLLECTION_XMLS[plex_set_id] + item['children'][plex_set_id] = collection_xmls[plex_set_id] - def run(self): - """ - Do the work - """ - if self.isCanceled(): - return - # Download Metadata - item = { - 'xml': PF.GetPlexMetadata(self.plex_id), - 'children': None - } - if item['xml'] is None: - # Did not receive a valid XML - skip that item for now - LOG.error("Could not get metadata for %s. Skipping that item " - "for now", self.plex_id) - return - elif item['xml'] == 401: - LOG.error('HTTP 401 returned by PMS. Too much strain? ' - 'Cancelling sync for now') - utils.window('plex_scancrashed', value='401') - return - if not self.isCanceled() and self.plex_type == v.PLEX_TYPE_MOVIE: - # Check for collections/sets - collections = False - for child in item['xml'][0]: - if child.tag == 'Collection': - collections = True - break - if collections: - global LOCK - with LOCK: - self._collections(item) - if not self.isCanceled() and self.get_children: - children_xml = PF.GetAllPlexChildren(self.plex_id) + def _process_abort(self, count, section): + # Make sure other threads will also receive sentinel + self.get_metadata_queue.put(None) + if count is not None: + self._process_skipped_item(count, section) + + def _process_skipped_item(self, count, section): + section.sync_successful = False + # Add a "dummy" item so we're not skipping a beat + self.processing_queue.put((count, {'section': section, 'xml': None})) + + def _run(self): + while True: + item = self.get_metadata_queue.get() try: - children_xml[0].attrib - except (TypeError, IndexError, AttributeError): - LOG.error('Could not get children for Plex id %s', - self.plex_id) - else: - item['children'] = children_xml - if not self.isCanceled(): - self.queue.put((self.count, item)) + if item is None or self.should_cancel(): + self._process_abort(item[0] if item else None, + item[2] if item else None) + break + count, plex_id, section = item + item = { + 'xml': PF.GetPlexMetadata(plex_id), # This will block + 'children': None, + 'section': section + } + if item['xml'] is None: + # Did not receive a valid XML - skip that item for now + LOG.error("Could not get metadata for %s. Skipping item " + "for now", plex_id) + self._process_skipped_item(count, section) + continue + elif item['xml'] == 401: + LOG.error('HTTP 401 returned by PMS. Too much strain? ' + 'Cancelling sync for now') + utils.window('plex_scancrashed', value='401') + self._process_abort(count, section) + break + if section.plex_type == v.PLEX_TYPE_MOVIE: + # Check for collections/sets + collections = False + for child in item['xml'][0]: + if child.tag == 'Collection': + collections = True + break + if collections: + with LOCK: + self._collections(item) + if section.get_children: + if self.should_cancel(): + self._process_abort(count, section) + break + children_xml = PF.GetAllPlexChildren(plex_id) # Will block + try: + children_xml[0].attrib + except (TypeError, IndexError, AttributeError): + LOG.error('Could not get children for Plex id %s', + plex_id) + self._process_skipped_item(count, section) + continue + else: + item['children'] = children_xml + self.processing_queue.put((count, item)) + finally: + self.get_metadata_queue.task_done() diff --git a/resources/lib/library_sync/process_metadata.py b/resources/lib/library_sync/process_metadata.py new file mode 100644 index 00000000..0d573965 --- /dev/null +++ b/resources/lib/library_sync/process_metadata.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, division, unicode_literals +from logging import getLogger + +from . import common, sections +from ..plex_db import PlexDB +from .. import backgroundthread, app + +LOG = getLogger('PLEX.sync.process_metadata') + +COMMIT_TO_DB_EVERY_X_ITEMS = 500 + + +class ProcessMetadataThread(common.LibrarySyncMixin, + backgroundthread.KillableThread): + """ + Invoke once in order to process the received PMS metadata xmls + """ + def __init__(self, current_time, processing_queue, update_progressbar): + self.current_time = current_time + self.processing_queue = processing_queue + self.update_progressbar = update_progressbar + self.last_section = sections.Section() + self.successful = True + super(ProcessMetadataThread, self).__init__() + + def start_section(self, section): + if section != self.last_section: + if self.last_section: + self.finish_last_section() + LOG.debug('Start or continue processing section %s', section) + self.last_section = section + # Warn the user for this new section if we cannot access a file + app.SYNC.path_verified = False + else: + LOG.debug('Resume processing section %s', section) + + def finish_last_section(self): + if (not self.should_cancel() and self.last_section and + self.last_section.sync_successful): + # Check for should_cancel() because we cannot be sure that we + # processed every item of the section + with PlexDB() as plexdb: + # Set the new time mark for the next delta sync + plexdb.update_section_last_sync(self.last_section.section_id, + self.current_time) + LOG.info('Finished processing section successfully: %s', + self.last_section) + elif self.last_section and not self.last_section.sync_successful: + LOG.warn('Sync not successful for section %s', self.last_section) + self.successful = False + + def _get(self): + item = {'xml': None} + while item and item['xml'] is None: + item = self.processing_queue.get() + self.processing_queue.task_done() + return item + + def _run(self): + # There are 2 sentinels: None for aborting/ending this thread, the dict + # {'section': section, 'xml': None} for skipped/invalid items + item = self._get() + if item: + section = item['section'] + processed = 0 + self.start_section(section) + while not self.should_cancel(): + if item is None: + break + elif item['section'] != section: + # We received an entirely new section + self.start_section(item['section']) + section = item['section'] + with section.context(self.current_time) as context: + while not self.should_cancel(): + if item is None or item['section'] != section: + break + self.update_progressbar(section, + item['xml'][0].get('title'), + section.count) + context.add_update(item['xml'][0], + section_name=section.name, + section_id=section.section_id, + children=item['children']) + processed += 1 + section.count += 1 + if processed == COMMIT_TO_DB_EVERY_X_ITEMS: + processed = 0 + context.commit() + item = self._get() + self.finish_last_section() diff --git a/resources/lib/library_sync/sections.py b/resources/lib/library_sync/sections.py index a1b78f23..da97e455 100644 --- a/resources/lib/library_sync/sections.py +++ b/resources/lib/library_sync/sections.py @@ -16,7 +16,7 @@ LOG = getLogger('PLEX.sync.sections') BATCH_SIZE = 500 # Need a way to interrupt our synching process -IS_CANCELED = None +SHOULD_CANCEL = None LIBRARY_PATH = path_ops.translate_path('special://profile/library/video/') # The video library might not yet exist for this user - create it @@ -54,6 +54,8 @@ class Section(object): self.content = None # unicode # Setting the section_type WILL re_set sync_to_kodi! self._section_type = None # unicode + # E.g. "season" or "movie" (translated) + self.section_type_text = None # Do we sync all items of this section to the Kodi DB? # This will be set with section_type!! self.sync_to_kodi = None # bool @@ -77,13 +79,9 @@ class Section(object): self.order = None # Original PMS xml for this section, including children self.xml = None - # Attributes that will be initialized later by full_sync.py - self.iterator = None - self.context = None - self.get_children = None # A section_type encompasses possible several plex_types! E.g. shows # contain shows, seasons, episodes - self.plex_type = None + self._plex_type = None if xml_element is not None: self.from_xml(xml_element) elif section_db_element: @@ -106,9 +104,14 @@ class Section(object): self.section_type is not None) def __eq__(self, section): + """ + Sections compare equal if their section_id, name and plex_type (first + prio) OR section_type (if there is no plex_type is set) compare equal + """ return (self.section_id == section.section_id and self.name == section.name and - self.section_type == section.section_type) + (self.plex_type == section.plex_type if self.plex_type else + self.section_type == section.section_type)) def __ne__(self, section): return not self == section @@ -140,6 +143,15 @@ class Section(object): else: self.sync_to_kodi = True + @property + def plex_type(self): + return self._plex_type + + @plex_type.setter + def plex_type(self, value): + self._plex_type = value + self.section_type_text = utils.lang(v.TRANSLATION_FROM_PLEXTYPE[value]) + @property def index(self): return self._index @@ -431,6 +443,39 @@ class Section(object): self.remove_from_plex() +def _get_children(plex_type): + if plex_type == v.PLEX_TYPE_ALBUM: + return True + else: + return False + + +def get_sync_section(section, plex_type): + """ + Deep-copies section and adds certain arguments in order to prep section + for the library sync + """ + section = copy.deepcopy(section) + section.plex_type = plex_type + section.context = itemtypes.ITEMTYPE_FROM_PLEXTYPE[plex_type] + section.get_children = _get_children(plex_type) + # Some more init stuff + # Has sync for this section been successful? + section.sync_successful = True + # List of tuples: (collection index [as in an item's metadata with + # "Collection id"], collection plex id) + section.collection_match = None + # Dict with entries of the form : + section.collection_xmls = {} + # Keep count during sync + section.count = 0 + # Total number of items that we need to sync + section.number_of_items = 0 + # Iterator to get one sync item after the other + section.iterator = None + return section + + def force_full_sync(): """ Resets the sync timestamp for all sections to 0, thus forcing a subsequent @@ -490,7 +535,7 @@ def _delete_kodi_db_items(section): with kodi_context(texture_db=True) as kodidb: typus = context(None, plexdb=plexdb, kodidb=kodidb) for plex_id in plex_ids: - if IS_CANCELED(): + if SHOULD_CANCEL(): return False typus.remove(plex_id) if len(plex_ids) < BATCH_SIZE: @@ -582,13 +627,13 @@ def sync_from_pms(parent_self, pick_libraries=False): pick_libraries=True will prompt the user the select the libraries he wants to sync """ - global IS_CANCELED + global SHOULD_CANCEL LOG.info('Starting synching sections from the PMS') - IS_CANCELED = parent_self.isCanceled + SHOULD_CANCEL = parent_self.should_cancel try: return _sync_from_pms(pick_libraries) finally: - IS_CANCELED = None + SHOULD_CANCEL = None LOG.info('Done synching sections from the PMS: %s', app.SYNC.sections) diff --git a/resources/lib/library_sync/websocket.py b/resources/lib/library_sync/websocket.py index f68233c1..a899362d 100644 --- a/resources/lib/library_sync/websocket.py +++ b/resources/lib/library_sync/websocket.py @@ -125,7 +125,7 @@ def process_new_item_message(message): with itemtypes.ITEMTYPE_FROM_PLEXTYPE[plex_type](timing.unix_timestamp()) as typus: typus.add_update(xml[0], section_name=xml.get('librarySectionTitle'), - section_id=xml.get('librarySectionID')) + section_id=utils.cast(int, xml.get('librarySectionID'))) cache_artwork(message['plex_id'], plex_type) return True, plex_type in v.PLEX_VIDEOTYPES, plex_type in v.PLEX_AUDIOTYPES diff --git a/resources/lib/migration.py b/resources/lib/migration.py index ae04e96a..13ec7729 100644 --- a/resources/lib/migration.py +++ b/resources/lib/migration.py @@ -64,7 +64,8 @@ def check_migration(): if not utils.compare_version(last_migration, '2.9.3'): LOG.info('Migrating to version 2.9.2') # Re-sync all playlists to Kodi - utils.wipe_synched_playlists() + from .playlists import remove_synced_playlists + remove_synced_playlists() if not utils.compare_version(last_migration, '2.9.7'): LOG.info('Migrating to version 2.9.6') diff --git a/resources/lib/playlists/__init__.py b/resources/lib/playlists/__init__.py index b3658ba0..1e2ff6bc 100644 --- a/resources/lib/playlists/__init__.py +++ b/resources/lib/playlists/__init__.py @@ -13,6 +13,7 @@ """ from __future__ import absolute_import, division, unicode_literals from logging import getLogger +from sqlite3 import OperationalError from .common import Playlist, PlaylistError, PlaylistObserver, \ kodi_playlist_hash @@ -38,7 +39,7 @@ SUPPORTED_FILETYPES = ( ############################################################################### -def isCanceled(): +def should_cancel(): return app.APP.stop_pkc or app.SYNC.stop_sync @@ -68,6 +69,22 @@ def kodi_playlist_monitor(): return observer +def remove_synced_playlists(): + """ + Deletes all synched playlists on the Kodi side, not on the Plex side + """ + LOG.info('Removing all playlists that we synced to Kodi') + with app.APP.lock_playlists: + try: + paths = db.get_all_kodi_playlist_paths() + except OperationalError: + LOG.info('Playlists table has not yet been set-up') + return + kodi_pl.delete_kodi_playlists(paths) + db.wipe_table() + LOG.info('Done removing all synced playlists') + + def websocket(plex_id, status): """ Call this function to process websocket messages from the PMS @@ -167,7 +184,7 @@ def _full_sync(): # before. If yes, make sure that hashes are identical. If not, sync it. old_plex_ids = db.plex_playlist_ids() for xml_playlist in xml: - if isCanceled(): + if should_cancel(): return False api = API(xml_playlist) try: @@ -199,7 +216,7 @@ def _full_sync(): LOG.info('Could not recreate playlist %s', api.plex_id) # Get rid of old Plex playlists that were deleted on the Plex side for plex_id in old_plex_ids: - if isCanceled(): + if should_cancel(): return False playlist = db.get_playlist(plex_id=plex_id) LOG.debug('Removing outdated Plex playlist from Kodi: %s', playlist) @@ -213,7 +230,7 @@ def _full_sync(): old_kodi_paths = db.kodi_playlist_paths() for root, _, files in path_ops.walk(v.PLAYLIST_PATH): for f in files: - if isCanceled(): + if should_cancel(): return False path = path_ops.path.join(root, f) try: @@ -244,7 +261,7 @@ def _full_sync(): except PlaylistError: LOG.info('Skipping Kodi playlist %s', path) for kodi_path in old_kodi_paths: - if isCanceled(): + if should_cancel(): return False playlist = db.get_playlist(path=kodi_path) if not playlist: @@ -370,19 +387,19 @@ class PlaylistEventhandler(events.FileSystemEventHandler): """ path = event.dest_path if event.event_type == events.EVENT_TYPE_MOVED \ else event.src_path - if not sync_kodi_playlist(path): - return - if path in kodi_pl.IGNORE_KODI_PLAYLIST_CHANGE: - LOG.debug('Ignoring event %s', event) - kodi_pl.IGNORE_KODI_PLAYLIST_CHANGE.remove(path) - return - _method_map = { - events.EVENT_TYPE_MODIFIED: self.on_modified, - events.EVENT_TYPE_MOVED: self.on_moved, - events.EVENT_TYPE_CREATED: self.on_created, - events.EVENT_TYPE_DELETED: self.on_deleted, - } with app.APP.lock_playlists: + if not sync_kodi_playlist(path): + return + if path in kodi_pl.IGNORE_KODI_PLAYLIST_CHANGE: + LOG.debug('Ignoring event %s', event) + kodi_pl.IGNORE_KODI_PLAYLIST_CHANGE.remove(path) + return + _method_map = { + events.EVENT_TYPE_MODIFIED: self.on_modified, + events.EVENT_TYPE_MOVED: self.on_moved, + events.EVENT_TYPE_CREATED: self.on_created, + events.EVENT_TYPE_DELETED: self.on_deleted, + } _method_map[event.event_type](event) def on_created(self, event): diff --git a/resources/lib/playlists/db.py b/resources/lib/playlists/db.py index 6b63c39a..673fc02a 100644 --- a/resources/lib/playlists/db.py +++ b/resources/lib/playlists/db.py @@ -57,6 +57,23 @@ def get_playlist(path=None, plex_id=None): return playlist +def get_all_kodi_playlist_paths(): + """ + Returns a list with all paths for the playlists on the Kodi side + """ + with PlexDB() as plexdb: + paths = list(plexdb.all_kodi_paths()) + return paths + + +def wipe_table(): + """ + Deletes all playlists entries in the Plex DB + """ + with PlexDB() as plexdb: + plexdb.wipe_playlists() + + def _m3u_iterator(text): """ Yields e.g. plugin://plugin.video.plexkodiconnect.movies/?plex_id=xxx diff --git a/resources/lib/playlists/kodi_pl.py b/resources/lib/playlists/kodi_pl.py index 4b040c25..8753326d 100644 --- a/resources/lib/playlists/kodi_pl.py +++ b/resources/lib/playlists/kodi_pl.py @@ -96,6 +96,21 @@ def delete(playlist): db.update_playlist(playlist, delete=True) +def delete_kodi_playlists(playlist_paths): + """ + Deletes all the the playlist files passed in; WILL IGNORE THIS CHANGE ON + THE PLEX SIDE! + """ + for path in playlist_paths: + try: + path_ops.remove(path) + # Ensure we're not deleting the playlists on the Plex side later + IGNORE_KODI_PLAYLIST_CHANGE.append(path) + LOG.info('Removed playlist %s', path) + except (OSError, IOError): + LOG.warn('Could not remove playlist %s', path) + + def _write_playlist_to_file(playlist, xml): """ Feed with playlist Playlist. Will write the playlist to a m3u file diff --git a/resources/lib/playqueue.py b/resources/lib/playqueue.py index 7e78bee9..45fb1d50 100644 --- a/resources/lib/playqueue.py +++ b/resources/lib/playqueue.py @@ -120,7 +120,7 @@ class PlayqueueMonitor(backgroundthread.KillableThread): # Ignore new media added by other addons continue for j, old_item in enumerate(old): - if self.isCanceled(): + if self.should_suspend() or self.should_cancel(): # Chances are that we got an empty Kodi playlist due to # Kodi exit return @@ -189,7 +189,7 @@ class PlayqueueMonitor(backgroundthread.KillableThread): for j in range(i, len(index)): index[j] += 1 for i in reversed(index): - if self.isCanceled(): + if self.should_suspend() or self.should_cancel(): # Chances are that we got an empty Kodi playlist due to # Kodi exit return @@ -212,9 +212,10 @@ class PlayqueueMonitor(backgroundthread.KillableThread): LOG.info("----===## PlayqueueMonitor stopped ##===----") def _run(self): - while not self.isCanceled(): - if self.wait_while_suspended(): - return + while not self.should_cancel(): + if self.should_suspend(): + if self.wait_while_suspended(): + return with app.APP.lock_playqueues: for playqueue in PLAYQUEUES: kodi_pl = js.playlist_get_items(playqueue.playlistid) @@ -228,4 +229,4 @@ class PlayqueueMonitor(backgroundthread.KillableThread): # compare old and new playqueue self._compare_playqueues(playqueue, kodi_pl) playqueue.old_kodi_pl = list(kodi_pl) - app.APP.monitor.waitForAbort(0.2) + self.sleep(0.2) diff --git a/resources/lib/plex_companion.py b/resources/lib/plex_companion.py index 036ac864..ad9ae747 100644 --- a/resources/lib/plex_companion.py +++ b/resources/lib/plex_companion.py @@ -293,7 +293,7 @@ class PlexCompanion(backgroundthread.KillableThread): subscription_manager, ('', v.COMPANION_PORT), listener.MyHandler) - httpd.timeout = 0.95 + httpd.timeout = 10.0 break except Exception: LOG.error("Unable to start PlexCompanion. Traceback:") @@ -312,12 +312,13 @@ class PlexCompanion(backgroundthread.KillableThread): if httpd: thread = Thread(target=httpd.handle_request) - while not self.isCanceled(): + while not self.should_cancel(): # If we are not authorized, sleep # Otherwise, we trigger a download which leads to a # re-authorizations - if self.wait_while_suspended(): - break + if self.should_suspend(): + if self.wait_while_suspended(): + break try: message_count += 1 if httpd: @@ -356,6 +357,6 @@ class PlexCompanion(backgroundthread.KillableThread): app.APP.companion_queue.task_done() # Don't sleep continue - app.APP.monitor.waitForAbort(0.05) + self.sleep(0.05) subscription_manager.signal_stop() client.stop_all() diff --git a/resources/lib/plex_db/common.py b/resources/lib/plex_db/common.py index f150fa38..6a70cb42 100644 --- a/resources/lib/plex_db/common.py +++ b/resources/lib/plex_db/common.py @@ -20,18 +20,19 @@ SUPPORTED_KODI_TYPES = ( class PlexDBBase(object): """ - Plex database methods used for all types of items + Plex database methods used for all types of items. """ - def __init__(self, plexconn=None, lock=True): + def __init__(self, plexconn=None, lock=True, copy=False): # Allows us to use this class with a cursor instead of context mgr self.plexconn = plexconn self.cursor = self.plexconn.cursor() if self.plexconn else None self.lock = lock + self.copy = copy def __enter__(self): if self.lock: PLEXDB_LOCK.acquire() - self.plexconn = db.connect('plex') + self.plexconn = db.connect('plex-copy' if self.copy else 'plex') self.cursor = self.plexconn.cursor() return self diff --git a/resources/lib/plex_db/playlists.py b/resources/lib/plex_db/playlists.py index 586e745a..68575bb8 100644 --- a/resources/lib/plex_db/playlists.py +++ b/resources/lib/plex_db/playlists.py @@ -81,3 +81,16 @@ class Playlists(object): playlist.kodi_type = answ[4] playlist.kodi_hash = answ[5] return playlist + + def all_kodi_paths(self): + """ + Returns a generator for all kodi_paths of all synched playlists + """ + self.cursor.execute('SELECT kodi_path FROM playlists') + return (x[0] for x in self.cursor) + + def wipe_playlists(self): + """ + Deletes all entries in the playlists table + """ + self.cursor.execute('DELETE FROM playlists') diff --git a/resources/lib/plexbmchelper/listener.py b/resources/lib/plexbmchelper/listener.py index 00b0cb4b..a5c6ecd1 100644 --- a/resources/lib/plexbmchelper/listener.py +++ b/resources/lib/plexbmchelper/listener.py @@ -37,6 +37,7 @@ RESOURCES_XML = ('%s\n' v.PLATFORM, v.PLATFORM_VERSION) + class MyHandler(BaseHTTPRequestHandler): """ BaseHTTPRequestHandler implementation of Plex Companion listener diff --git a/resources/lib/service_entry.py b/resources/lib/service_entry.py index b7b10aed..ac65266e 100644 --- a/resources/lib/service_entry.py +++ b/resources/lib/service_entry.py @@ -101,7 +101,7 @@ class Service(object): self._init_done = True @staticmethod - def isCanceled(): + def should_cancel(): return xbmc.abortRequested or app.APP.stop_pkc def on_connection_check(self, result): @@ -437,7 +437,7 @@ class Service(object): self.playqueue = playqueue.PlayqueueMonitor() # Main PKC program loop - while not self.isCanceled(): + while not self.should_cancel(): # Check for PKC commands from other Python instances plex_command = utils.window('plexkodiconnect.command') @@ -544,6 +544,7 @@ class Service(object): # Tell all threads to terminate (e.g. several lib sync threads) LOG.debug('Aborting all threads') app.APP.stop_pkc = True + backgroundthread.BGThreader.shutdown(block=False) # Load/Reset PKC entirely - important for user/Kodi profile switch # Clear video nodes properties library_sync.clear_window_vars() diff --git a/resources/lib/sync.py b/resources/lib/sync.py index 87efc477..6ed3eae7 100644 --- a/resources/lib/sync.py +++ b/resources/lib/sync.py @@ -38,7 +38,9 @@ class Sync(backgroundthread.KillableThread): self.start_library_sync(show_dialog=True, repair=app.SYNC.run_lib_scan == 'repair', block=True) - if not self.sync_successful and not self.isSuspended() and not self.isCanceled(): + if (not self.sync_successful and + not self.should_suspend() and + not self.should_cancel()): # ERROR in library sync LOG.warn('Triggered full/repair sync has not been successful') elif app.SYNC.run_lib_scan == 'fanart': @@ -112,7 +114,7 @@ class Sync(backgroundthread.KillableThread): LOG.info('Not synching Plex artwork - not caching') return if self.image_cache_thread and self.image_cache_thread.is_alive(): - self.image_cache_thread.abort() + self.image_cache_thread.cancel() self.image_cache_thread.join() self.image_cache_thread = artwork.ImageCachingThread() self.image_cache_thread.start() @@ -163,10 +165,11 @@ class Sync(backgroundthread.KillableThread): utils.init_dbs() - while not self.isCanceled(): + while not self.should_cancel(): # In the event the server goes offline - if self.wait_while_suspended(): - return + if self.should_suspend(): + if self.wait_while_suspended(): + return if not install_sync_done: # Very FIRST sync ever upon installation or reset of Kodi DB LOG.info('Initial start-up full sync starting') @@ -188,7 +191,7 @@ class Sync(backgroundthread.KillableThread): self.start_image_cache_thread() else: LOG.error('Initial start-up full sync unsuccessful') - app.APP.monitor.waitForAbort(1) + self.sleep(1) xbmc.executebuiltin('InhibitIdleShutdown(false)') elif not initial_sync_done: @@ -205,7 +208,7 @@ class Sync(backgroundthread.KillableThread): self.start_image_cache_thread() else: LOG.info('Startup sync has not yet been successful') - app.APP.monitor.waitForAbort(1) + self.sleep(1) # Currently no db scan, so we could start a new scan else: @@ -240,9 +243,9 @@ class Sync(backgroundthread.KillableThread): library_sync.store_websocket_message(message) queue.task_done() # Sleep just a bit - app.APP.monitor.waitForAbort(0.01) + self.sleep(0.01) continue - app.APP.monitor.waitForAbort(0.1) + self.sleep(0.1) # Shut down playlist monitoring if playlist_monitor: playlist_monitor.stop() diff --git a/resources/lib/utils.py b/resources/lib/utils.py index 0b025128..2da4a378 100644 --- a/resources/lib/utils.py +++ b/resources/lib/utils.py @@ -249,6 +249,15 @@ def ERROR(txt='', hide_tb=False, notify=False, cancel_sync=False): return short +def rreplace(s, old, new, occurrence=-1): + """ + Replaces the string old [str, unicode] with new from the RIGHT given a + string s. + """ + li = s.rsplit(old, occurrence) + return new.join(li) + + class AttributeDict(dict): """ Turns an etree xml response's xml.attrib into an object with attributes @@ -525,29 +534,6 @@ def delete_temporary_subtitles(): root, file, err) -def wipe_synched_playlists(): - """ - Deletes all synched playlist files on the Kodi side; resets the Plex table - listing all synched Plex playlists - """ - from . import plex_db - try: - with plex_db.PlexDB() as plexdb: - plexdb.cursor.execute('SELECT kodi_path FROM playlists') - playlist_paths = [x[0] for x in plexdb.cursor] - except OperationalError: - # Plex DB completely empty yet - playlist_paths = [] - for path in playlist_paths: - try: - path_ops.remove(path) - LOG.info('Removed playlist %s', path) - except (OSError, IOError): - LOG.warn('Could not remove playlist %s', path) - # Now wipe our database - plex_db.wipe(table='playlists') - - def wipe_database(reboot=True): """ Deletes all Plex playlists as well as video nodes, then clears Kodi as well @@ -557,10 +543,10 @@ def wipe_database(reboot=True): LOG.warn('Start wiping') from .library_sync.sections import delete_files from . import kodi_db, plex_db + from .playlists import remove_synced_playlists # Clean up the playlists and video nodes delete_files() - # Wipe all synched playlists - wipe_synched_playlists() + remove_synced_playlists() try: with plex_db.PlexDB() as plexdb: if plexdb.songs_have_been_synced(): diff --git a/resources/lib/variables.py b/resources/lib/variables.py index b36ba29c..3cba3bc2 100644 --- a/resources/lib/variables.py +++ b/resources/lib/variables.py @@ -69,7 +69,14 @@ elif xbmc.getCondVisibility('system.platform.android'): else: DEVICE = "Unknown" -MODEL = platform.release() or 'Unknown' +try: + MODEL = platform.release() or 'Unknown' +except IOError: + # E.g. iOS + # It seems that Kodi doesn't allow python to spawn subprocesses in order to + # determine the system name + # See https://github.com/psf/requests/issues/4434 + MODEL = 'Unknown' DEVICENAME = try_decode(_ADDON.getSetting('deviceName')) if not DEVICENAME: @@ -127,6 +134,7 @@ DB_MUSIC_PATH = None DB_TEXTURE_VERSION = None DB_TEXTURE_PATH = None DB_PLEX_PATH = try_decode(xbmc.translatePath("special://database/plex.db")) +DB_PLEX_COPY_PATH = try_decode(xbmc.translatePath("special://database/plex-copy.db")) EXTERNAL_SUBTITLE_TEMP_PATH = try_decode(xbmc.translatePath( "special://profile/addon_data/%s/temp/" % ADDON_ID)) diff --git a/resources/lib/websocket_client.py b/resources/lib/websocket_client.py index e999e669..8ff767b1 100644 --- a/resources/lib/websocket_client.py +++ b/resources/lib/websocket_client.py @@ -19,7 +19,7 @@ class WebSocket(backgroundthread.KillableThread): def __init__(self): self.ws = None self.redirect_uri = None - self.sleeptime = 0 + self.sleeptime = 0.0 super(WebSocket, self).__init__() def process(self, opcode, message): @@ -46,15 +46,15 @@ class WebSocket(backgroundthread.KillableThread): def getUri(self): raise NotImplementedError - def __sleep(self): + def _sleep_cycle(self): """ Sleeps for 2^self.sleeptime where sleeping period will be doubled with each unsuccessful connection attempt. Will sleep at most 64 seconds """ - app.APP.monitor.waitForAbort(2**self.sleeptime) + self.sleep(2 ** self.sleeptime) if self.sleeptime < 6: - self.sleeptime += 1 + self.sleeptime += 1.0 def run(self): LOG.info("----===## Starting %s ##===----", self.__class__.__name__) @@ -69,9 +69,9 @@ class WebSocket(backgroundthread.KillableThread): LOG.info("##===---- %s Stopped ----===##", self.__class__.__name__) def _run(self): - while not self.isCanceled(): + while not self.should_cancel(): # In the event the server goes offline - if self.isSuspended(): + if self.should_suspend(): # Set in service.py if self.ws is not None: self.ws.close() @@ -99,11 +99,11 @@ class WebSocket(backgroundthread.KillableThread): # Server is probably offline LOG.debug("%s: IOError connecting", self.__class__.__name__) self.ws = None - self.__sleep() + self._sleep_cycle() except websocket.WebSocketTimeoutException: LOG.debug("%s: WebSocketTimeoutException", self.__class__.__name__) self.ws = None - self.__sleep() + self._sleep_cycle() except websocket.WebsocketRedirect as e: LOG.debug('301 redirect detected: %s', e) self.redirect_uri = e.headers.get('location', @@ -111,11 +111,11 @@ class WebSocket(backgroundthread.KillableThread): if self.redirect_uri: self.redirect_uri = self.redirect_uri.decode('utf-8') self.ws = None - self.__sleep() + self._sleep_cycle() except websocket.WebSocketException as e: LOG.debug('%s: WebSocketException: %s', self.__class__.__name__, e) self.ws = None - self.__sleep() + self._sleep_cycle() except Exception as e: LOG.error('%s: Unknown exception encountered when ' 'connecting: %s', self.__class__.__name__, e) @@ -123,9 +123,9 @@ class WebSocket(backgroundthread.KillableThread): LOG.error("%s: Traceback:\n%s", self.__class__.__name__, traceback.format_exc()) self.ws = None - self.__sleep() + self._sleep_cycle() else: - self.sleeptime = 0 + self.sleeptime = 0.0 except Exception as e: LOG.error("%s: Unknown exception encountered: %s", self.__class__.__name__, e) @@ -141,7 +141,7 @@ class PMS_Websocket(WebSocket): """ Websocket connection with the PMS for Plex Companion """ - def isSuspended(self): + def should_suspend(self): """ Returns True if the thread is suspended """ @@ -206,7 +206,7 @@ class Alexa_Websocket(WebSocket): """ Websocket connection to talk to Amazon Alexa. """ - def isSuspended(self): + def should_suspend(self): """ Overwrite method since we need to check for plex token """ diff --git a/resources/lib/windows/userselect.py b/resources/lib/windows/userselect.py index d68ac1ad..f6e5ba7b 100644 --- a/resources/lib/windows/userselect.py +++ b/resources/lib/windows/userselect.py @@ -24,7 +24,7 @@ class UserThumbTask(backgroundthread.Task): def run(self): for user in self.users: - if self.isCanceled(): + if self.should_cancel(): return thumb, back = user.thumb, '' self.callback(user, thumb, back) @@ -169,13 +169,13 @@ class UserSelectWindow(kodigui.BaseWindow): utils.settings('plexToken'), utils.settings('plex_machineIdentifier')) if self.user.authToken is None: - self.user = None item.setProperty('pin', item.dataSource.title) item.setProperty('editing.pin', '') # 'Error': 'Login failed with plex.tv for user' utils.messageDialog(utils.lang(30135), - '%s %s' % (utils.lang(39229), - self.user.username)) + '{}{}'.format(utils.lang(39229), + self.user.username)) + self.user = None return self.doClose()