Troplo/PlexKodiConnect
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Get section overview xml asynchronously

Browse source
This commit is contained in:
croneter 2018-12-01 11:14:12 +01:00
parent 41bbdbc206
commit 07cf25b324
1 changed files with 102 additions and 65 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

167
resources/lib/library_sync/full_sync.py
View file

@ -2,6 +2,12 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
from __future__ import absolute_import, division, unicode_literals from __future__ import absolute_import, division, unicode_literals
from logging import getLogger from logging import getLogger
import Queue
import copy
from cProfile import Profile
from pstats import Stats
from StringIO import StringIO
from .get_metadata import GetMetadataTask, reset_collections from .get_metadata import GetMetadataTask, reset_collections
from .process_metadata import InitNewSection, UpdateLastSync, ProcessMetadata, \ from .process_metadata import InitNewSection, UpdateLastSync, ProcessMetadata, \
@ -84,67 +90,78 @@ class FullSync(common.libsync_mixin):
self.queue.put(UpdateUserdata(xml_item)) self.queue.put(UpdateUserdata(xml_item))
@utils.log_time @utils.log_time
def process_kind(self): def process_section(self, section):
""" LOG.debug('Processing library section %s', section)
""" if self.isCanceled():
successful = True return False
LOG.debug('Start processing %ss', self.section_type) if not self.install_sync_done:
sects = (x for x in sections.SECTIONS app.SYNC.path_verified = False
if x['plex_type'] == self.section_type) try:
for section in sects: # Sync new, updated and deleted items
LOG.debug('Processing library section %s', section) self.section_initiated = True
if self.isCanceled(): iterator = section['iterator_1']
return False # Tell the processing thread about this new section
if not self.install_sync_done: queue_info = InitNewSection(section['context'],
app.SYNC.path_verified = False iterator.total,
try: iterator.get('librarySectionTitle'),
# Sync new, updated and deleted items section['section_id'],
self.section_initiated = True section['plex_type'])
iterator = PF.SectionItems(section['section_id'], self.queue.put(queue_info)
plex_type=self.plex_type) for xml_item in iterator:
# Tell the processing thread about this new section if self.isCanceled():
queue_info = InitNewSection(self.context, return False
iterator.total, self.process_item(xml_item)
iterator.get('librarySectionTitle'), except RuntimeError:
section['section_id'], LOG.error('Could not entirely process section %s', section)
self.plex_type) return False
self.queue.put(queue_info) LOG.debug('Waiting for download threads to finish')
for xml_item in iterator: while self.threader.threader.working():
if self.isCanceled(): app.APP.monitor.waitForAbort(0.1)
return False LOG.debug('Waiting for processing thread to finish section')
self.process_item(xml_item) self.queue.join()
except RuntimeError: reset_collections()
LOG.error('Could not entirely process section %s', section) try:
successful = False # Sync playstate of every item
continue iterator = section['iterator_2']
LOG.debug('Waiting for download threads to finish') # Tell the processing thread that we're syncing playstate
while self.threader.threader.working(): queue_info = InitNewSection(section['context'],
app.APP.monitor.waitForAbort(0.1) iterator.total,
LOG.debug('Waiting for processing thread to finish section') iterator.get('librarySectionTitle'),
section['section_id'],
section['plex_type'])
self.queue.put(queue_info)
if section['plex_type'] != v.PLEX_TYPE_ARTIST:
self.process_playstate(iterator)
self.queue.join() self.queue.join()
reset_collections() except RuntimeError:
try: LOG.error('Could not process playstate for section %s', section)
# Sync playstate of every item return False
iterator = PF.SectionItems(section['section_id'], LOG.debug('Done processing playstate for section')
plex_type=self.plex_type) return True
# Tell the processing thread that we're syncing playstate
queue_info = InitNewSection(self.context,
iterator.total,
iterator.get('librarySectionTitle'),
section['section_id'],
self.plex_type)
self.queue.put(queue_info)
if self.plex_type != v.PLEX_TYPE_ARTIST:
self.process_playstate(iterator)
self.queue.join()
except RuntimeError:
LOG.error('Could not process playstate for section %s', section)
successful = False
continue
LOG.debug('Done processing playstate for section')
LOG.debug('Finished processing %ss', self.plex_type) def threaded_get_iterators(self, kinds, queue):
return successful """
PF.SectionItems is costly, so let's do it asynchronous
"""
try:
for kind in kinds:
for section in (x for x in sections.SECTIONS
if x['plex_type'] == kind[1]):
if self.isCanceled():
return
element = copy.deepcopy(section)
element['section_type'] = element['plex_type']
element['plex_type'] = kind[0]
element['element_type'] = kind[1]
element['context'] = kind[2]
element['get_children'] = kind[3]
element['iterator_1'] = PF.SectionItems(section['section_id'],
plex_type=kind[0])
element['iterator_2'] = PF.SectionItems(section['section_id'],
plex_type=kind[0])
queue.put(element)
finally:
queue.put(None)
def full_library_sync(self): def full_library_sync(self):
""" """
@ -160,16 +177,27 @@ class FullSync(common.libsync_mixin):
(v.PLEX_TYPE_ARTIST, v.PLEX_TYPE_ARTIST, itemtypes.Artist, False), (v.PLEX_TYPE_ARTIST, v.PLEX_TYPE_ARTIST, itemtypes.Artist, False),
(v.PLEX_TYPE_ALBUM, v.PLEX_TYPE_ARTIST, itemtypes.Album, True), (v.PLEX_TYPE_ALBUM, v.PLEX_TYPE_ARTIST, itemtypes.Album, True),
]) ])
# Already start setting up the iterators. We need to enforce
# syncing e.g. show before season before episode
iterator_queue = Queue.Queue()
task = backgroundthread.FunctionAsTask(self.threaded_get_iterators,
None,
kinds,
iterator_queue)
backgroundthread.BGThreader.addTask(task)
with PlexDB() as self.plexdb: with PlexDB() as self.plexdb:
for kind in kinds: while True:
section = iterator_queue.get()
if section is None:
break
self.section_initiated = False self.section_initiated = False
# Setup our variables # Setup our variables
self.plex_type = kind[0] self.plex_type = section['plex_type']
self.section_type = kind[1] self.section_type = section['section_type']
self.context = kind[2] self.context = section['context']
self.get_children = kind[3] self.get_children = section['get_children']
# Now do the heavy lifting # Now do the heavy lifting
if self.isCanceled() or not self.process_kind(): if self.isCanceled() or not self.process_section(section):
return False return False
# Delete movies that are not on Plex anymore # Delete movies that are not on Plex anymore
if not self.section_initiated: if not self.section_initiated:
@ -181,10 +209,13 @@ class FullSync(common.libsync_mixin):
self.plex_type) self.plex_type)
self.queue.put(queue_info) self.queue.put(queue_info)
self.process_delete() self.process_delete()
iterator_queue.task_done()
return True return True
@utils.log_time @utils.log_time
def run(self): def run(self):
profile = Profile()
profile.enable()
if self.isCanceled(): if self.isCanceled():
return return
successful = False successful = False
@ -226,6 +257,12 @@ class FullSync(common.libsync_mixin):
if self.callback: if self.callback:
self.callback(successful) self.callback(successful)
LOG.info('Done full_sync') LOG.info('Done full_sync')
profile.disable()
string_io = StringIO()
stats = Stats(profile, stream=string_io).sort_stats('cumulative')
stats.print_stats()
LOG.info('cProfile result: ')
LOG.info(string_io.getvalue())
def start(show_dialog, repair=False, callback=None): def start(show_dialog, repair=False, callback=None):