Troplo/PlexKodiConnect
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Attempt to fix items getting deleted on second sync

Browse source
This commit is contained in:
croneter 2018-12-01 18:43:42 +01:00
parent a776f940de
commit 459bd72299
4 changed files with 78 additions and 60 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
resources/lib/itemtypes/common.py
View file

@ -76,6 +76,7 @@ class ItemBase(object):
self.plexconn.close()
self.kodiconn.close()
self.artconn.close()
LOG.debug('Committed and closed DBs')
return self
def commit(self):

32
resources/lib/library_sync/full_sync.py
View file

@ -11,7 +11,7 @@ from StringIO import StringIO
from .get_metadata import GetMetadataTask, reset_collections
from .process_metadata import InitNewSection, UpdateLastSync, ProcessMetadata, \
DeleteItem, UpdateUserdata
DeleteItem, UpdateUserdata, StopSection
from . import common, sections
from .. import utils, timing, backgroundthread, variables as v, app
from .. import plex_functions as PF, itemtypes
@ -38,6 +38,7 @@ class FullSync(common.libsync_mixin):
self.callback = callback
self.show_dialog = show_dialog
self.queue = None
self.out_queue = None
self.process_thread = None
self.current_sync = None
self.plexdb = None
@ -71,10 +72,11 @@ class FullSync(common.libsync_mixin):
Removes all the items that have NOT been updated (last_sync timestamp
is different)
"""
for plex_id in self.plexdb.plex_id_by_last_sync(self.plex_type,
for plex_id, last_sync in self.plexdb.plex_id_by_last_sync(self.plex_type,
self.current_sync):
if self.isCanceled():
return
LOG.debug('process_delete %s, %s', plex_id, last_sync)
self.queue.put(DeleteItem(plex_id))
@utils.log_time
@ -112,12 +114,6 @@ class FullSync(common.libsync_mixin):
except RuntimeError:
LOG.error('Could not entirely process section %s', section)
return False
LOG.debug('Waiting for download threads to finish')
while self.threader.threader.working():
app.APP.monitor.waitForAbort(0.1)
LOG.debug('Waiting for processing thread to finish section')
self.queue.join()
reset_collections()
try:
# Sync playstate of every item
iterator = section['iterator_2']
@ -128,9 +124,23 @@ class FullSync(common.libsync_mixin):
section['section_id'],
section['plex_type'])
self.queue.put(queue_info)
# We cannot safely use queue.join(), so use another queue
# This will block!
LOG.debug('Waiting for processing to finish process_item part')
self.out_queue.get()
self.out_queue.task_done()
LOG.debug('process_item part done')
reset_collections()
# Delete movies that are not on Plex anymore
self.process_delete()
self.out_queue.get()
self.out_queue.task_done()
if section['plex_type'] != v.PLEX_TYPE_ARTIST:
self.process_playstate(iterator)
self.queue.join()
self.queue.put(StopSection())
# This will block until processing thread has reached StopSection
self.out_queue.get()
self.out_queue.task_done()
except RuntimeError:
LOG.error('Could not process playstate for section %s', section)
return False
@ -196,8 +206,6 @@ class FullSync(common.libsync_mixin):
# Now do the heavy lifting
if self.isCanceled() or not self.process_section(section):
return False
# Delete movies that are not on Plex anymore
self.process_delete()
iterator_queue.task_done()
return True
@ -218,7 +226,9 @@ class FullSync(common.libsync_mixin):
try:
# Fire up our single processing thread
self.queue = backgroundthread.Queue.Queue(maxsize=1000)
self.out_queue = backgroundthread.Queue.Queue()
self.processing_thread = ProcessMetadata(self.queue,
self.out_queue,
self.current_sync,
self.show_dialog)
self.processing_thread.start()

101
resources/lib/library_sync/process_metadata.py
View file

@ -29,6 +29,10 @@ class InitNewSection(object):
self.plex_type = plex_type
class StopSection(object):
pass
class UpdateUserdata(object):
def __init__(self, xml_item):
self.xml_item = xml_item
@ -49,16 +53,11 @@ class ProcessMetadata(common.libsync_mixin, backgroundthread.KillableThread):
Not yet implemented for more than 1 thread - if ever. Only to be called by
ONE thread!
Processes the XML metadata in the queue
Input:
queue: Queue.Queue() object that you'll need to fill up with
the downloaded XML eTree objects
item_class: as used to call functions in itemtypes.py e.g. 'Movies' =>
itemtypes.Movies()
"""
def __init__(self, queue, last_sync, show_dialog):
def __init__(self, queue, out_queue, last_sync, show_dialog):
self._canceled = False
self.queue = queue
self.out_queue = out_queue
self.last_sync = last_sync
self.show_dialog = show_dialog
self.total = 0
@ -105,50 +104,58 @@ class ProcessMetadata(common.libsync_mixin, backgroundthread.KillableThread):
# Init with the very first library section. This will block!
section = self.queue.get()
self.queue.task_done()
if section is None:
return
while not self.isCanceled():
if section is None:
break
LOG.debug('Start processing section %s (%ss)',
section.name, section.plex_type)
self.current = 1
self.processed = 0
self.total = section.total
self.section_name = section.name
self.section_type_text = utils.lang(
v.TRANSLATION_FROM_PLEXTYPE[section.plex_type])
profile = Profile()
profile.enable()
with section.context(self.last_sync) as context:
while not self.isCanceled():
# grabs item from queue. This will block!
item = self.queue.get()
if isinstance(item, dict):
context.add_update(item['xml'][0],
section_name=section.name,
section_id=section.id,
children=item['children'])
self.title = item['xml'][0].get('title')
self.processed += 1
elif isinstance(item, UpdateLastSync):
context.plexdb.update_last_sync(item.plex_id,
section.plex_type,
self.last_sync)
elif isinstance(item, UpdateUserdata):
context.update_userdata(item.xml_item, section.plex_type)
elif isinstance(item, InitNewSection) or item is None:
section = item
if isinstance(section, StopSection):
section = self.queue.get()
continue
else:
LOG.debug('Start processing section %s (%ss)',
section.name, section.plex_type)
self.current = 1
self.processed = 0
self.total = section.total
self.section_name = section.name
self.section_type_text = utils.lang(
v.TRANSLATION_FROM_PLEXTYPE[section.plex_type])
profile = Profile()
profile.enable()
with section.context(self.last_sync) as context:
while not self.isCanceled():
# grabs item from queue. This will block!
item = self.queue.get()
if isinstance(item, dict):
context.add_update(item['xml'][0],
section_name=section.name,
section_id=section.id,
children=item['children'])
self.title = item['xml'][0].get('title')
self.processed += 1
elif isinstance(item, UpdateLastSync):
LOG.debug('UpdateLastSync to %s', self.last_sync)
context.plexdb.update_last_sync(item.plex_id,
section.plex_type,
self.last_sync)
LOG.debug('Item now: %s', context.plexdb.movie(item.plex_id))
elif isinstance(item, UpdateUserdata):
context.update_userdata(item.xml_item, section.plex_type)
elif (isinstance(item, InitNewSection) or
isinstance(item, StopSection) or
item is None):
section = item
self.queue.task_done()
break
else:
context.remove(item.plex_id, plex_type=section.plex_type)
self.update_progressbar()
self.current += 1
if self.processed == 500:
self.processed = 0
context.commit()
self.queue.task_done()
break
else:
context.remove(item.plex_id, plex_type=section.plex_type)
self.update_progressbar()
self.current += 1
if self.processed == 500:
self.processed = 0
context.commit()
self.queue.task_done()
# Tell the main sync thread that we're done with this section
self.out_queue.put(None)
profile.disable()
string_io = StringIO()
stats = Stats(profile, stream=string_io).sort_stats('cumulative')

4
resources/lib/plex_db/common.py
View file

@ -93,8 +93,8 @@ class PlexDBBase(object):
"""
Returns an iterator for all items where the last_sync is NOT identical
"""
return (x[0] for x in
self.cursor.execute('SELECT plex_id FROM %s WHERE last_sync <> ?' % plex_type,
return (x for x in
self.cursor.execute('SELECT plex_id, last_sync FROM %s WHERE last_sync <> ?' % plex_type,
(last_sync, )))
def checksum(self, plex_id, plex_type):