Troplo/PlexKodiConnect
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Attempt to fix items getting deleted on second sync

Browse source
This commit is contained in:
croneter 2018-12-01 18:43:42 +01:00
parent a776f940de
commit 459bd72299
4 changed files with 78 additions and 60 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
resources/lib/itemtypes/common.py
View file

@ -76,6 +76,7 @@ class ItemBase(object):
self.plexconn.close() self.plexconn.close()
self.kodiconn.close() self.kodiconn.close()
self.artconn.close() self.artconn.close()
LOG.debug('Committed and closed DBs')
return self return self
def commit(self): def commit(self):

32
resources/lib/library_sync/full_sync.py
View file

@ -11,7 +11,7 @@ from StringIO import StringIO
from .get_metadata import GetMetadataTask, reset_collections from .get_metadata import GetMetadataTask, reset_collections
from .process_metadata import InitNewSection, UpdateLastSync, ProcessMetadata, \ from .process_metadata import InitNewSection, UpdateLastSync, ProcessMetadata, \
DeleteItem, UpdateUserdata DeleteItem, UpdateUserdata, StopSection
from . import common, sections from . import common, sections
from .. import utils, timing, backgroundthread, variables as v, app from .. import utils, timing, backgroundthread, variables as v, app
from .. import plex_functions as PF, itemtypes from .. import plex_functions as PF, itemtypes
@ -38,6 +38,7 @@ class FullSync(common.libsync_mixin):
self.callback = callback self.callback = callback
self.show_dialog = show_dialog self.show_dialog = show_dialog
self.queue = None self.queue = None
self.out_queue = None
self.process_thread = None self.process_thread = None
self.current_sync = None self.current_sync = None
self.plexdb = None self.plexdb = None
@ -71,10 +72,11 @@ class FullSync(common.libsync_mixin):
Removes all the items that have NOT been updated (last_sync timestamp Removes all the items that have NOT been updated (last_sync timestamp
is different) is different)
""" """
for plex_id in self.plexdb.plex_id_by_last_sync(self.plex_type, for plex_id, last_sync in self.plexdb.plex_id_by_last_sync(self.plex_type,
self.current_sync): self.current_sync):
if self.isCanceled(): if self.isCanceled():
return return
LOG.debug('process_delete %s, %s', plex_id, last_sync)
self.queue.put(DeleteItem(plex_id)) self.queue.put(DeleteItem(plex_id))
@utils.log_time @utils.log_time
@ -112,12 +114,6 @@ class FullSync(common.libsync_mixin):
except RuntimeError: except RuntimeError:
LOG.error('Could not entirely process section %s', section) LOG.error('Could not entirely process section %s', section)
return False return False
LOG.debug('Waiting for download threads to finish')
while self.threader.threader.working():
app.APP.monitor.waitForAbort(0.1)
LOG.debug('Waiting for processing thread to finish section')
self.queue.join()
reset_collections()
try: try:
# Sync playstate of every item # Sync playstate of every item
iterator = section['iterator_2'] iterator = section['iterator_2']
@ -128,9 +124,23 @@ class FullSync(common.libsync_mixin):
section['section_id'], section['section_id'],
section['plex_type']) section['plex_type'])
self.queue.put(queue_info) self.queue.put(queue_info)
# We cannot safely use queue.join(), so use another queue
# This will block!
LOG.debug('Waiting for processing to finish process_item part')
self.out_queue.get()
self.out_queue.task_done()
LOG.debug('process_item part done')
reset_collections()
# Delete movies that are not on Plex anymore
self.process_delete()
self.out_queue.get()
self.out_queue.task_done()
if section['plex_type'] != v.PLEX_TYPE_ARTIST: if section['plex_type'] != v.PLEX_TYPE_ARTIST:
self.process_playstate(iterator) self.process_playstate(iterator)
self.queue.join() self.queue.put(StopSection())
# This will block until processing thread has reached StopSection
self.out_queue.get()
self.out_queue.task_done()
except RuntimeError: except RuntimeError:
LOG.error('Could not process playstate for section %s', section) LOG.error('Could not process playstate for section %s', section)
return False return False
@ -196,8 +206,6 @@ class FullSync(common.libsync_mixin):
# Now do the heavy lifting # Now do the heavy lifting
if self.isCanceled() or not self.process_section(section): if self.isCanceled() or not self.process_section(section):
return False return False
# Delete movies that are not on Plex anymore
self.process_delete()
iterator_queue.task_done() iterator_queue.task_done()
return True return True
@ -218,7 +226,9 @@ class FullSync(common.libsync_mixin):
try: try:
# Fire up our single processing thread # Fire up our single processing thread
self.queue = backgroundthread.Queue.Queue(maxsize=1000) self.queue = backgroundthread.Queue.Queue(maxsize=1000)
self.out_queue = backgroundthread.Queue.Queue()
self.processing_thread = ProcessMetadata(self.queue, self.processing_thread = ProcessMetadata(self.queue,
self.out_queue,
self.current_sync, self.current_sync,
self.show_dialog) self.show_dialog)
self.processing_thread.start() self.processing_thread.start()

27
resources/lib/library_sync/process_metadata.py
View file

@ -29,6 +29,10 @@ class InitNewSection(object):
self.plex_type = plex_type self.plex_type = plex_type
class StopSection(object):
pass
class UpdateUserdata(object): class UpdateUserdata(object):
def __init__(self, xml_item): def __init__(self, xml_item):
self.xml_item = xml_item self.xml_item = xml_item
@ -49,16 +53,11 @@ class ProcessMetadata(common.libsync_mixin, backgroundthread.KillableThread):
Not yet implemented for more than 1 thread - if ever. Only to be called by Not yet implemented for more than 1 thread - if ever. Only to be called by
ONE thread! ONE thread!
Processes the XML metadata in the queue Processes the XML metadata in the queue
Input:
queue: Queue.Queue() object that you'll need to fill up with
the downloaded XML eTree objects
item_class: as used to call functions in itemtypes.py e.g. 'Movies' =>
itemtypes.Movies()
""" """
def __init__(self, queue, last_sync, show_dialog): def __init__(self, queue, out_queue, last_sync, show_dialog):
self._canceled = False self._canceled = False
self.queue = queue self.queue = queue
self.out_queue = out_queue
self.last_sync = last_sync self.last_sync = last_sync
self.show_dialog = show_dialog self.show_dialog = show_dialog
self.total = 0 self.total = 0
@ -105,11 +104,13 @@ class ProcessMetadata(common.libsync_mixin, backgroundthread.KillableThread):
# Init with the very first library section. This will block! # Init with the very first library section. This will block!
section = self.queue.get() section = self.queue.get()
self.queue.task_done() self.queue.task_done()
if section is None:
return
while not self.isCanceled(): while not self.isCanceled():
if section is None: if section is None:
break break
if isinstance(section, StopSection):
section = self.queue.get()
continue
else:
LOG.debug('Start processing section %s (%ss)', LOG.debug('Start processing section %s (%ss)',
section.name, section.plex_type) section.name, section.plex_type)
self.current = 1 self.current = 1
@ -132,12 +133,16 @@ class ProcessMetadata(common.libsync_mixin, backgroundthread.KillableThread):
self.title = item['xml'][0].get('title') self.title = item['xml'][0].get('title')
self.processed += 1 self.processed += 1
elif isinstance(item, UpdateLastSync): elif isinstance(item, UpdateLastSync):
LOG.debug('UpdateLastSync to %s', self.last_sync)
context.plexdb.update_last_sync(item.plex_id, context.plexdb.update_last_sync(item.plex_id,
section.plex_type, section.plex_type,
self.last_sync) self.last_sync)
LOG.debug('Item now: %s', context.plexdb.movie(item.plex_id))
elif isinstance(item, UpdateUserdata): elif isinstance(item, UpdateUserdata):
context.update_userdata(item.xml_item, section.plex_type) context.update_userdata(item.xml_item, section.plex_type)
elif isinstance(item, InitNewSection) or item is None: elif (isinstance(item, InitNewSection) or
isinstance(item, StopSection) or
item is None):
section = item section = item
self.queue.task_done() self.queue.task_done()
break break
@ -149,6 +154,8 @@ class ProcessMetadata(common.libsync_mixin, backgroundthread.KillableThread):
self.processed = 0 self.processed = 0
context.commit() context.commit()
self.queue.task_done() self.queue.task_done()
# Tell the main sync thread that we're done with this section
self.out_queue.put(None)
profile.disable() profile.disable()
string_io = StringIO() string_io = StringIO()
stats = Stats(profile, stream=string_io).sort_stats('cumulative') stats = Stats(profile, stream=string_io).sort_stats('cumulative')

4
resources/lib/plex_db/common.py
View file

@ -93,8 +93,8 @@ class PlexDBBase(object):
""" """
Returns an iterator for all items where the last_sync is NOT identical Returns an iterator for all items where the last_sync is NOT identical
""" """
return (x[0] for x in return (x for x in
self.cursor.execute('SELECT plex_id FROM %s WHERE last_sync <> ?' % plex_type, self.cursor.execute('SELECT plex_id, last_sync FROM %s WHERE last_sync <> ?' % plex_type,
(last_sync, ))) (last_sync, )))
def checksum(self, plex_id, plex_type): def checksum(self, plex_id, plex_type):