From e6692a901235f6e78f8ed162af80178a307717d5 Mon Sep 17 00:00:00 2001 From: croneter Date: Sat, 20 Oct 2018 14:49:04 +0200 Subject: [PATCH] Rewire llibrary sync, part 1 --- resources/lib/initialsetup.py | 2 +- resources/lib/itemtypes/__init__.py | 7 + resources/lib/itemtypes/common.py | 138 +++++ resources/lib/itemtypes/tvshows.py | 548 ++++++++++++++++++ resources/lib/library_sync/common.py | 29 + resources/lib/library_sync/full_sync.py | 223 +++++++ resources/lib/library_sync/get_metadata.py | 129 +---- .../lib/library_sync/process_metadata.py | 104 ++-- resources/lib/library_sync/sections.py | 238 ++++++++ .../lib/{ => library_sync}/videonodes.py | 0 resources/lib/librarysync.py | 37 +- resources/lib/plex_api.py | 31 +- resources/lib/plex_functions.py | 76 ++- resources/lib/plexdb_functions.py | 209 +++---- resources/lib/utils.py | 24 + 15 files changed, 1449 insertions(+), 346 deletions(-) create mode 100644 resources/lib/itemtypes/__init__.py create mode 100644 resources/lib/itemtypes/common.py create mode 100644 resources/lib/itemtypes/tvshows.py create mode 100644 resources/lib/library_sync/common.py create mode 100644 resources/lib/library_sync/full_sync.py create mode 100644 resources/lib/library_sync/sections.py rename resources/lib/{ => library_sync}/videonodes.py (100%) diff --git a/resources/lib/initialsetup.py b/resources/lib/initialsetup.py index 70f83170..21a0d3e8 100644 --- a/resources/lib/initialsetup.py +++ b/resources/lib/initialsetup.py @@ -11,7 +11,6 @@ from .utils import etree from . import path_ops from . import migration from .downloadutils import DownloadUtils as DU -from . import videonodes from . import userclient from . import clientinfo from . import plex_functions as PF @@ -54,6 +53,7 @@ def reload_pkc(): for prop in WINDOW_PROPERTIES: utils.window(prop, clear=True) # Clear video nodes properties + from .librarysync import videonodes videonodes.VideoNodes().clearProperties() # Initializing diff --git a/resources/lib/itemtypes/__init__.py b/resources/lib/itemtypes/__init__.py new file mode 100644 index 00000000..bd99cd60 --- /dev/null +++ b/resources/lib/itemtypes/__init__.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +from __future__ import absolute_import, division, unicode_literals +from .tvshows import Show, Season, Episode + +# Note: always use same order of URL arguments, NOT urlencode: +# plex_id=&plex_type=&mode=play diff --git a/resources/lib/itemtypes/common.py b/resources/lib/itemtypes/common.py new file mode 100644 index 00000000..2b80869c --- /dev/null +++ b/resources/lib/itemtypes/common.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +from __future__ import absolute_import, division, unicode_literals +from logging import getLogger +from ntpath import dirname + +from . import artwork +from . import utils +from . import plexdb_functions as plexdb +from . import kodidb_functions as kodidb +from .plex_api import API +from . import variables as v +############################################################################### + +LOG = getLogger('PLEX.itemtypes.common') + +# Note: always use same order of URL arguments, NOT urlencode: +# plex_id=&plex_type=&mode=play + +############################################################################### + + +def process_path(playurl): + """ + Do NOT use os.path since we have paths that might not apply to the current + OS! + """ + if '\\' in playurl: + # Local path + path = '%s\\' % playurl + toplevelpath = '%s\\' % dirname(dirname(path)) + else: + # Network path + path = '%s/' % playurl + toplevelpath = '%s/' % dirname(dirname(path)) + return path, toplevelpath + + +class ItemBase(object): + """ + Items to be called with "with Items() as xxx:" to ensure that __enter__ + method is called (opens db connections) + + Input: + kodiType: optional argument; e.g. 'video' or 'music' + """ + def __init__(self, plex_db=None, kodi_db=None): + self.artwork = artwork.Artwork() + self.plexconn = None + self.plexcursor = plex_db.plexcursor if plex_db else None + self.kodiconn = None + self.kodicursor = kodi_db.cursor if kodi_db else None + self.plex_db = plex_db + self.kodi_db = kodi_db + + def __enter__(self): + """ + Open DB connections and cursors + """ + self.plexconn = utils.kodi_sql('plex') + self.plexcursor = self.plexconn.cursor() + self.kodiconn = utils.kodi_sql('video') + self.kodicursor = self.kodiconn.cursor() + self.plex_db = plexdb.Plex_DB_Functions(self.plexcursor) + self.kodi_db = kodidb.KodiDBMethods(self.kodicursor) + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + """ + Make sure DB changes are committed and connection to DB is closed. + """ + self.plexconn.commit() + self.kodiconn.commit() + self.plexconn.close() + self.kodiconn.close() + return self + + def set_fanart(self, artworks, kodi_id, kodi_type): + """ + Writes artworks [dict containing only set artworks] to the Kodi art DB + """ + self.artwork.modify_artwork(artworks, + kodi_id, + kodi_type, + self.kodicursor) + + def updateUserdata(self, xml): + """ + Updates the Kodi watched state of the item from PMS. Also retrieves + Plex resume points for movies in progress. + + viewtag and viewid only serve as dummies + """ + for mediaitem in xml: + api = API(mediaitem) + # Get key and db entry on the Kodi db side + db_item = self.plex_db.getItem_byId(api.plex_id()) + try: + fileid = db_item[1] + except TypeError: + continue + # Grab the user's viewcount, resume points etc. from PMS' answer + userdata = api.userdata() + # Write to Kodi DB + self.kodi_db.set_resume(fileid, + userdata['Resume'], + userdata['Runtime'], + userdata['PlayCount'], + userdata['LastPlayedDate'], + api.plex_type()) + if v.KODIVERSION >= 17: + self.kodi_db.update_userrating(db_item[0], + db_item[4], + userdata['UserRating']) + + def updatePlaystate(self, mark_played, view_count, resume, duration, + file_id, lastViewedAt, plex_type): + """ + Use with websockets, not xml + """ + # If the playback was stopped, check whether we need to increment the + # playcount. PMS won't tell us the playcount via websockets + LOG.debug('Playstate file_id %s: viewcount: %s, resume: %s, type: %s', + file_id, view_count, resume, plex_type) + if mark_played: + LOG.info('Marking as completely watched in Kodi') + try: + view_count += 1 + except TypeError: + view_count = 1 + resume = 0 + # Do the actual update + self.kodi_db.set_resume(file_id, + resume, + duration, + view_count, + lastViewedAt, + plex_type) diff --git a/resources/lib/itemtypes/tvshows.py b/resources/lib/itemtypes/tvshows.py new file mode 100644 index 00000000..9872e615 --- /dev/null +++ b/resources/lib/itemtypes/tvshows.py @@ -0,0 +1,548 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +from __future__ import absolute_import, division, unicode_literals +from logging import getLogger + +from .common import ItemBase, process_path +from ..plex_api import API +from .. import state, variables as v + +LOG = getLogger('PLEX.tvshows') + + +class TvShowMixin(object): + def remove(self, plex_id): + """ + Remove the entire TV shows object (show, season or episode) including + all associated entries from the Kodi DB. + """ + entry = self.plex_db.getItem_byId(plex_id) + if entry is None: + LOG.debug('Cannot delete plex_id %s - not found in DB', plex_id) + return + kodi_id = entry[0] + file_id = entry[1] + parent_id = entry[3] + kodi_type = entry[4] + LOG.debug("Removing %s with kodi_id: %s file_id: %s parent_id: %s", + kodi_type, kodi_id, file_id, parent_id) + + # Remove the plex reference + self.plex_db.removeItem(plex_id) + + # EPISODE ##### + if kodi_type == v.KODI_TYPE_EPISODE: + # Delete episode, verify season and tvshow + self.remove_episode(kodi_id, file_id) + # Season verification + season = self.plex_db.getItem_byKodiId(parent_id, + v.KODI_TYPE_SEASON) + if season is not None: + if not self.plex_db.getItem_byParentId(parent_id, + v.KODI_TYPE_EPISODE): + # No episode left for season - so delete the season + self.remove_season(parent_id) + self.plex_db.removeItem(season[0]) + show = self.plex_db.getItem_byKodiId(season[1], + v.KODI_TYPE_SHOW) + if show is not None: + if not self.plex_db.getItem_byParentId(season[1], + v.KODI_TYPE_SEASON): + # No seasons for show left - so delete entire show + self.remove_show(season[1]) + self.plex_db.removeItem(show[0]) + else: + LOG.error('No show found in Plex DB for season %s', season) + else: + LOG.error('No season found in Plex DB!') + # SEASON ##### + elif kodi_type == v.KODI_TYPE_SEASON: + # Remove episodes, season, verify tvshow + for episode in self.plex_db.getItem_byParentId( + kodi_id, v.KODI_TYPE_EPISODE): + self.remove_episode(episode[1], episode[2]) + self.plex_db.removeItem(episode[0]) + # Remove season + self.remove_season(kodi_id) + # Show verification + if not self.plex_db.getItem_byParentId(parent_id, + v.KODI_TYPE_SEASON): + # There's no other season left, delete the show + self.remove_show(parent_id) + self.plex_db.removeItem_byKodiId(parent_id, v.KODI_TYPE_SHOW) + # TVSHOW ##### + elif kodi_type == v.KODI_TYPE_SHOW: + # Remove episodes, seasons and the tvshow itself + for season in self.plex_db.getItem_byParentId(kodi_id, + v.KODI_TYPE_SEASON): + for episode in self.plex_db.getItem_byParentId( + season[1], v.KODI_TYPE_EPISODE): + self.remove_episode(episode[1], episode[2]) + self.plex_db.removeItem(episode[0]) + self.remove_season(season[1]) + self.plex_db.removeItem(season[0]) + self.remove_show(kodi_id) + + LOG.debug("Deleted %s %s from Kodi database", kodi_type, plex_id) + + def remove_show(self, kodi_id): + """ + Remove a TV show, and only the show, no seasons or episodes + """ + self.kodi_db.modify_genres(kodi_id, v.KODI_TYPE_SHOW) + self.kodi_db.modify_studios(kodi_id, v.KODI_TYPE_SHOW) + self.kodi_db.modify_tags(kodi_id, v.KODI_TYPE_SHOW) + self.artwork.delete_artwork(kodi_id, + v.KODI_TYPE_SHOW, + self.kodicursor) + self.kodicursor.execute("DELETE FROM tvshow WHERE idShow = ?", + (kodi_id,)) + if v.KODIVERSION >= 17: + self.kodi_db.remove_uniqueid(kodi_id, v.KODI_TYPE_SHOW) + self.kodi_db.remove_ratings(kodi_id, v.KODI_TYPE_SHOW) + LOG.debug("Removed tvshow: %s", kodi_id) + + def remove_season(self, kodi_id): + """ + Remove a season, and only a season, not the show or episodes + """ + self.artwork.delete_artwork(kodi_id, + v.KODI_TYPE_SEASON, + self.kodicursor) + self.kodicursor.execute("DELETE FROM seasons WHERE idSeason = ?", + (kodi_id,)) + LOG.debug("Removed season: %s", kodi_id) + + def remove_episode(self, kodi_id, file_id): + """ + Remove an episode, and episode only from the Kodi DB (not Plex DB) + """ + self.kodi_db.modify_people(kodi_id, v.KODI_TYPE_EPISODE) + self.kodi_db.remove_file(file_id, plex_type=v.PLEX_TYPE_EPISODE) + self.artwork.delete_artwork(kodi_id, + v.KODI_TYPE_EPISODE, + self.kodicursor) + self.kodicursor.execute("DELETE FROM episode WHERE idEpisode = ?", + (kodi_id,)) + if v.KODIVERSION >= 17: + self.kodi_db.remove_uniqueid(kodi_id, v.KODI_TYPE_EPISODE) + self.kodi_db.remove_ratings(kodi_id, v.KODI_TYPE_EPISODE) + LOG.debug("Removed episode: %s", kodi_id) + + +class Show(ItemBase, TvShowMixin): + """ + For Plex library-type TV shows + """ + def add_update(self, xml, viewtag=None, viewid=None): + """ + Process a single show + """ + api = API(xml) + update_item = True + plex_id = api.plex_id() + LOG.debug('Adding show with plex_id %s', plex_id) + if not plex_id: + LOG.error("Cannot parse XML data for TV show") + return + update_item = True + entry = self.plex_db.getItem_byId(plex_id) + try: + kodi_id = entry[0] + path_id = entry[2] + except TypeError: + update_item = False + query = 'SELECT COALESCE(MAX(idShow), 0) FROM tvshow' + self.kodicursor.execute(query) + kodi_id = self.kodicursor.fetchone()[0] + 1 + else: + # Verification the item is still in Kodi + query = 'SELECT * FROM tvshow WHERE idShow = ?' + self.kodicursor.execute(query, (kodi_id,)) + try: + self.kodicursor.fetchone()[0] + except TypeError: + # item is not found, let's recreate it. + update_item = False + LOG.info("idShow: %s missing from Kodi, repairing the entry.", + kodi_id) + + genres = api.genre_list() + genre = api.list_to_string(genres) + studios = api.music_studio_list() + studio = api.list_to_string(studios) + + # GET THE FILE AND PATH ##### + if state.DIRECT_PATHS: + # Direct paths is set the Kodi way + playurl = api.validate_playurl(api.tv_show_path(), + api.plex_type(), + folder=True) + if playurl is None: + return + path, toplevelpath = process_path(playurl) + toppathid = self.kodi_db.add_video_path( + toplevelpath, + content='tvshows', + scraper='metadata.local') + else: + # Set plugin path + toplevelpath = "plugin://%s.tvshows/" % v.ADDON_ID + path = "%s%s/" % (toplevelpath, plex_id) + # Do NOT set a parent id because addon-path cannot be "stacked" + toppathid = None + + path_id = self.kodi_db.add_video_path(path, + date_added=api.date_created(), + id_parent_path=toppathid) + # UPDATE THE TVSHOW ##### + if update_item: + LOG.info("UPDATE tvshow plex_id: %s - Title: %s", + plex_id, api.title()) + # Add reference is idempotent; the call here updates also fileid + # and path_id when item is moved or renamed + self.plex_db.addReference(plex_id, + v.PLEX_TYPE_SHOW, + kodi_id, + v.KODI_TYPE_SHOW, + kodi_pathid=path_id, + checksum=api.checksum(), + view_id=viewid) + # update new ratings Kodi 17 + rating_id = self.kodi_db.get_ratingid(kodi_id, v.KODI_TYPE_SHOW) + self.kodi_db.update_ratings(kodi_id, + v.KODI_TYPE_SHOW, + "default", + api.audience_rating(), + api.votecount(), + rating_id) + # update new uniqueid Kodi 17 + if api.provider('tvdb') is not None: + uniqueid = self.kodi_db.get_uniqueid(kodi_id, + v.KODI_TYPE_SHOW) + self.kodi_db.update_uniqueid(kodi_id, + v.KODI_TYPE_SHOW, + api.provider('tvdb'), + "unknown", + uniqueid) + else: + self.kodi_db.remove_uniqueid(kodi_id, v.KODI_TYPE_SHOW) + uniqueid = -1 + # Update the tvshow entry + query = ''' + UPDATE tvshow + SET c00 = ?, c01 = ?, c04 = ?, c05 = ?, c08 = ?, c09 = ?, + c12 = ?, c13 = ?, c14 = ?, c15 = ? + WHERE idShow = ? + ''' + self.kodicursor.execute( + query, (api.title(), api.plot(), rating_id, + api.premiere_date(), genre, api.title(), uniqueid, + api.content_rating(), studio, api.sorttitle(), + kodi_id)) + # OR ADD THE TVSHOW ##### + else: + LOG.info("ADD tvshow plex_id: %s - Title: %s", + plex_id, api.title()) + # Link the path + query = "INSERT INTO tvshowlinkpath(idShow, idPath) values (?, ?)" + self.kodicursor.execute(query, (kodi_id, path_id)) + # Create the reference in plex table + self.plex_db.addReference(plex_id, + v.PLEX_TYPE_SHOW, + kodi_id, + v.KODI_TYPE_SHOW, + kodi_pathid=path_id, + checksum=api.checksum(), + view_id=viewid) + rating_id = self.kodi_db.get_ratingid(kodi_id, v.KODI_TYPE_SHOW) + self.kodi_db.add_ratings(rating_id, + kodi_id, + v.KODI_TYPE_SHOW, + "default", + api.audience_rating(), + api.votecount()) + if api.provider('tvdb') is not None: + uniqueid = self.kodi_db.get_uniqueid(kodi_id, + v.KODI_TYPE_SHOW) + self.kodi_db.add_uniqueid(uniqueid, + kodi_id, + v.KODI_TYPE_SHOW, + api.provider('tvdb'), + "unknown") + else: + uniqueid = -1 + # Create the tvshow entry + query = ''' + INSERT INTO tvshow( + idShow, c00, c01, c04, c05, c08, c09, c12, c13, c14, + c15) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ''' + self.kodicursor.execute( + query, (kodi_id, api.title(), api.plot(), rating_id, + api.premiere_date(), genre, api.title(), uniqueid, + api.content_rating(), studio, api.sorttitle())) + + self.kodi_db.modify_people(kodi_id, + v.KODI_TYPE_SHOW, + api.people_list()) + self.kodi_db.modify_genres(kodi_id, v.KODI_TYPE_SHOW, genres) + self.artwork.modify_artwork(api.artwork(), + kodi_id, + v.KODI_TYPE_SHOW, + self.kodicursor) + # Process studios + self.kodi_db.modify_studios(kodi_id, v.KODI_TYPE_SHOW, studios) + # Process tags: view, PMS collection tags + tags = [viewtag] + tags.extend([i for _, i in api.collection_list()]) + self.kodi_db.modify_tags(kodi_id, v.KODI_TYPE_SHOW, tags) + + +class Season(ItemBase, TvShowMixin): + def add_update(self, xml, viewtag=None, viewid=None): + """ + Process a single season of a certain tv show + """ + api = API(xml) + plex_id = api.plex_id() + LOG.debug('Adding season with plex_id %s', plex_id) + if not plex_id: + LOG.error('Error getting plex_id for season, skipping') + return + entry = self.plex_db.getItem_byId(api.parent_plex_id()) + try: + show_id = entry[0] + except TypeError: + LOG.error('Could not find parent tv show for season %s. ' + 'Skipping season for now.', plex_id) + return + kodi_id = self.kodi_db.add_season(show_id, api.season_number()) + # Check whether Season already exists + entry = self.plex_db.getItem_byId(plex_id) + update_item = False if entry is None else True + self.artwork.modify_artwork(api.artwork(), + kodi_id, + v.KODI_TYPE_SEASON, + self.kodicursor) + if update_item: + self.plex_db.updateReference(plex_id, api.checksum()) + else: + self.plex_db.addReference(plex_id, + v.PLEX_TYPE_SEASON, + kodi_id, + v.KODI_TYPE_SEASON, + parent_id=show_id, + view_id=viewid, + checksum=api.checksum()) + + +class Episode(ItemBase, TvShowMixin): + def add_update(self, xml, viewtag=None, viewid=None): + """ + Process single episode + """ + api = API(xml) + update_item = True + plex_id = api.plex_id() + LOG.debug('Adding episode with plex_id %s', plex_id) + if not plex_id: + LOG.error('Error getting plex_id for episode, skipping') + return + entry = self.plex_db.getItem_byId(plex_id) + try: + kodi_id = entry[0] + old_file_id = entry[1] + path_id = entry[2] + except TypeError: + update_item = False + query = 'SELECT COALESCE(MAX(idEpisode), 0) FROM episode' + self.kodicursor.execute(query) + kodi_id = self.kodicursor.fetchone()[0] + 1 + else: + # Verification the item is still in Kodi + query = 'SELECT * FROM episode WHERE idEpisode = ?' + self.kodicursor.execute(query, (kodi_id, )) + try: + self.kodicursor.fetchone()[0] + except TypeError: + # item is not found, let's recreate it. + update_item = False + LOG.info('idEpisode %s missing from Kodi, repairing entry.', + kodi_id) + + peoples = api.people() + director = api.list_to_string(peoples['Director']) + writer = api.list_to_string(peoples['Writer']) + userdata = api.userdata() + series_id, _, season, episode = api.episode_data() + + if season is None: + season = -1 + if episode is None: + episode = -1 + airs_before_season = "-1" + airs_before_episode = "-1" + + # Get season id + show = self.plex_db.getItem_byId(series_id) + try: + show_id = show[0] + except TypeError: + LOG.error("Parent tvshow now found, skip item") + return False + season_id = self.kodi_db.add_season(show_id, season) + + # GET THE FILE AND PATH ##### + do_indirect = not state.DIRECT_PATHS + if state.DIRECT_PATHS: + playurl = api.file_path(force_first_media=True) + if playurl is None: + do_indirect = True + else: + playurl = api.validate_playurl(playurl, v.PLEX_TYPE_EPISODE) + if "\\" in playurl: + # Local path + filename = playurl.rsplit("\\", 1)[1] + else: + # Network share + filename = playurl.rsplit("/", 1)[1] + path = playurl.replace(filename, "") + parent_path_id = self.kodi_db.parent_path_id(path) + path_id = self.kodi_db.add_video_path( + path, id_parent_path=parent_path_id) + if do_indirect: + # Set plugin path - do NOT use "intermediate" paths for the show + # as with direct paths! + filename = api.file_name(force_first_media=True) + path = 'plugin://%s.tvshows/%s/' % (v.ADDON_ID, series_id) + filename = ('%s?plex_id=%s&plex_type=%s&mode=play&filename=%s' + % (path, plex_id, v.PLEX_TYPE_EPISODE, filename)) + playurl = filename + # Root path tvshows/ already saved in Kodi DB + path_id = self.kodi_db.add_video_path(path) + + # add/retrieve path_id and fileid + # if the path or file already exists, the calls return current value + file_id = self.kodi_db.add_file(filename, path_id, api.date_created()) + + # UPDATE THE EPISODE ##### + if update_item: + LOG.info("UPDATE episode plex_id: %s, Title: %s", + plex_id, api.title()) + if file_id != old_file_id: + self.kodi_db.remove_file(old_file_id) + ratingid = self.kodi_db.get_ratingid(kodi_id, + v.KODI_TYPE_EPISODE) + self.kodi_db.update_ratings(kodi_id, + v.KODI_TYPE_EPISODE, + "default", + userdata['Rating'], + api.votecount(), + ratingid) + # update new uniqueid Kodi 17 + uniqueid = self.kodi_db.get_uniqueid(kodi_id, + v.KODI_TYPE_EPISODE) + self.kodi_db.update_uniqueid(kodi_id, + v.KODI_TYPE_EPISODE, + api.provider('tvdb'), + "tvdb", + uniqueid) + query = ''' + UPDATE episode + SET c00 = ?, c01 = ?, c03 = ?, c04 = ?, c05 = ?, c09 = ?, + c10 = ?, c12 = ?, c13 = ?, c14 = ?, c15 = ?, c16 = ?, + c18 = ?, c19 = ?, idFile=?, idSeason = ?, + userrating = ? + WHERE idEpisode = ? + ''' + self.kodicursor.execute( + query, (api.title(), api.plot(), ratingid, writer, + api.premiere_date(), api.runtime(), director, season, + episode, api.title(), airs_before_season, + airs_before_episode, playurl, path_id, file_id, + season_id, userdata['UserRating'], kodi_id)) + # Update parentid reference + self.plex_db.updateParentId(plex_id, season_id) + + # OR ADD THE EPISODE ##### + else: + LOG.info("ADD episode plex_id: %s - Title: %s", + plex_id, api.title()) + # Create the episode entry + rating_id = self.kodi_db.get_ratingid(kodi_id, + v.KODI_TYPE_EPISODE) + self.kodi_db.add_ratings(rating_id, + kodi_id, + v.KODI_TYPE_EPISODE, + "default", + userdata['Rating'], + api.votecount()) + # add new uniqueid Kodi 17 + uniqueid = self.kodi_db.get_uniqueid(kodi_id, + v.KODI_TYPE_EPISODE) + self.kodi_db.add_uniqueid(uniqueid, + kodi_id, + v.KODI_TYPE_EPISODE, + api.provider('tvdb'), + "tvdb") + query = ''' + INSERT INTO episode( idEpisode, idFile, c00, c01, c03, c04, + c05, c09, c10, c12, c13, c14, idShow, c15, c16, c18, + c19, idSeason, userrating) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, + ?, ?) + ''' + self.kodicursor.execute( + query, (kodi_id, file_id, api.title(), api.plot(), rating_id, + writer, api.premiere_date(), api.runtime(), director, + season, episode, api.title(), show_id, + airs_before_season, airs_before_episode, playurl, + path_id, season_id, userdata['UserRating'])) + + # Create or update the reference in plex table Add reference is + # idempotent; the call here updates also file_id and path_id when item + # is moved or renamed + self.plex_db.addReference(plex_id, + v.PLEX_TYPE_EPISODE, + kodi_id, + v.KODI_TYPE_EPISODE, + kodi_file_id=file_id, + kodi_pathid=path_id, + parent_id=season_id, + checksum=api.checksum(), + view_id=viewid) + self.kodi_db.modify_people(kodi_id, + v.KODI_TYPE_EPISODE, + api.people_list()) + self.artwork.modify_artwork(api.artwork(), + kodi_id, + v.KODI_TYPE_EPISODE, + self.kodicursor) + streams = api.mediastreams() + self.kodi_db.modify_streams(file_id, streams, api.runtime()) + self.kodi_db.set_resume(file_id, + api.resume_point(), + api.runtime(), + userdata['PlayCount'], + userdata['LastPlayedDate'], + None) # Do send None, we check here + if not state.DIRECT_PATHS: + # need to set a SECOND file entry for a path without plex show id + filename = api.file_name(force_first_media=True) + path = 'plugin://%s.tvshows/' % v.ADDON_ID + # Filename is exactly the same, WITH plex show id! + filename = ('%s%s/?plex_id=%s&plex_type=%s&mode=play&filename=%s' + % (path, series_id, plex_id, v.PLEX_TYPE_EPISODE, + filename)) + path_id = self.kodi_db.add_video_path(path) + file_id = self.kodi_db.add_file(filename, + path_id, + api.date_created()) + self.kodi_db.set_resume(file_id, + api.resume_point(), + api.runtime(), + userdata['PlayCount'], + userdata['LastPlayedDate'], + None) # Do send None - 2nd entry diff --git a/resources/lib/library_sync/common.py b/resources/lib/library_sync/common.py new file mode 100644 index 00000000..4995f23c --- /dev/null +++ b/resources/lib/library_sync/common.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +from __future__ import absolute_import, division, unicode_literals +import xbmc + +from .. import state + + +class libsync_mixin(object): + def isCanceled(self): + return (super(libsync_mixin, self).isCanceled() or + state.SUSPEND_LIBRARY_THREAD or + state.SUSPEND_SYNC) + + +def update_kodi_library(video=True, music=True): + """ + Updates the Kodi library and thus refreshes the Kodi views and widgets + """ + if xbmc.getCondVisibility('Container.Content(musicvideos)') or \ + xbmc.getCondVisibility('Window.IsMedia'): + # Prevent cursor from moving + xbmc.executebuiltin('Container.Refresh') + else: + # Update widgets + if video: + xbmc.executebuiltin('UpdateLibrary(video)') + if music: + xbmc.executebuiltin('UpdateLibrary(music)') diff --git a/resources/lib/library_sync/full_sync.py b/resources/lib/library_sync/full_sync.py new file mode 100644 index 00000000..249c07fa --- /dev/null +++ b/resources/lib/library_sync/full_sync.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +from __future__ import absolute_import, division, unicode_literals +from logging import getLogger +import threading +import Queue + +from . import common, process_metadata, sections +from .get_metadata import GetMetadataTask +from .. import utils, backgroundthread, playlists, variables as v, state +from .. import plex_functions as PF, itemtypes + +LOG = getLogger('PLEX.library_sync.full_sync') +DOWNLOAD_QUEUE = Queue.Queue(maxsize=500) +PROCESS_QUEUE = Queue.Queue(maxsize=100) +FANARTQUEUE = Queue.Queue() +THREADS = [] + + +def start(repair, callback): + """ + """ + # backgroundthread.BGThreader.addTask(FullSync().setup(repair, callback)) + FullSync(repair, callback).start() + + +class FullSync(threading.Thread, common.libsync_mixin): + def __init__(self, repair, callback): + """ + repair=True: force sync EVERY item + """ + self.repair = repair + self.callback = callback + super(FullSync, self).__init__() + + def process_item(self, xml_item, section): + """ + Processes a single library item + """ + plex_id = xml_item.get('ratingKey') + if plex_id is None: + # Skipping items 'title=All episodes' without a 'ratingKey' + return + if self.new_items_only: + if self.plex_db.check_plexid(plex_id) is None: + backgroundthread.BGThreader.addTask( + GetMetadataTask().setup(PROCESS_QUEUE, + plex_id, + section)) + else: + if self.plex_db.check_checksum( + 'K%s%s' % (plex_id, xml_item.get('updatedAt', ''))) is None: + pass + + def plex_movies(self): + """ + Syncs movies + """ + LOG.debug('Processing Plex movies') + sections = (x for x in sections.SECTIONS + if x['kodi_type'] == v.KODI_TYPE_MOVIE) + self.queue = Queue.Queue(maxsize=200) + for section in sections: + LOG.debug('Processing library section %s', section) + if self.isCanceled(): + return False + if not self.install_sync_done: + state.PATH_VERIFIED = False + try: + iterator = PF.PlexSectionItems(section['id']) + t = process_metadata.ProcessMetadata( + self.queue, + itemtypes.Movie, + utils.cast(int, iterator.get('totalSize', 0))) + for xml_item in PF.plex_section_items_generator(section['id']): + if self.isCanceled(): + return False + self.process_item(xml_item, section) + except RuntimeError: + LOG.error('Could not entirely process section %s', section) + return False + + + + # Populate self.updatelist and self.all_plex_ids + self.get_updatelist(all_plexmovies, + item_class, + 'add_update', + view['name'], + view['id']) + self.process_updatelist(item_class) + # Update viewstate for EVERY item + sections = (x for x in sections.SECTIONS + if x['kodi_type'] == v.KODI_TYPE_MOVIE) + for view in sections: + if self.isCanceled(): + return False + self.plex_update_watched(view['id'], item_class) + + # PROCESS DELETES ##### + if not self.repair: + # Manual sync, process deletes + with itemtypes.Movies() as movie_db: + for kodimovie in self.all_kodi_ids: + if kodimovie not in self.all_plex_ids: + movie_db.remove(kodimovie) + LOG.info("%s sync is finished.", item_class) + return True + + def full_library_sync(self, new_items_only=False): + """ + """ + process = [self.plex_movies, self.plex_tv_show] + if state.ENABLE_MUSIC: + process.append(self.plex_music) + + # Do the processing + for kind in process: + if self.isCanceled() or not kind(): + return False + + # Let kodi update the views in any case, since we're doing a full sync + common.update_kodi_library(video=True, music=state.ENABLE_MUSIC) + + if utils.window('plex_scancrashed') == 'true': + # Show warning if itemtypes.py crashed at some point + utils.messageDialog(utils.lang(29999), utils.lang(39408)) + utils.window('plex_scancrashed', clear=True) + elif utils.window('plex_scancrashed') == '401': + utils.window('plex_scancrashed', clear=True) + if state.PMS_STATUS not in ('401', 'Auth'): + # Plex server had too much and returned ERROR + utils.messageDialog(utils.lang(29999), utils.lang(39409)) + return True + + @utils.log_time + def run(self): + successful = False + try: + if self.isCanceled(): + return + LOG.info('Running fullsync for NEW PMS items with repair=%s', + self.repair) + if not sections.sync_from_pms(): + return + if self.isCanceled(): + return + # This will also update playstates and userratings! + if self.full_library_sync(new_items_only=True) is False: + return + if self.isCanceled(): + return + # This will NOT update playstates and userratings! + LOG.info('Running fullsync for CHANGED PMS items with repair=%s', + self.repair) + if not self.full_library_sync(): + return + if self.isCanceled(): + return + if PLAYLIST_SYNC_ENABLED and not playlists.full_sync(): + return + successful = True + except: + utils.ERROR(txt='full_sync.py crashed', notify=True) + finally: + self.callback(successful) + + +def process_updatelist(item_class, show_sync_info=True): + """ + Downloads all XMLs for item_class (e.g. Movies, TV-Shows). Processes + them by then calling item_classs.() + + Input: + item_class: 'Movies', 'TVShows' (itemtypes.py classes) + """ + search_fanart = (item_class in ('Movies', 'TVShows') and + utils.settings('FanartTV') == 'true') + LOG.debug("Starting sync threads") + # Spawn GetMetadata threads for downloading + for _ in range(state.SYNC_THREAD_NUMBER): + thread = get_metadata.ThreadedGetMetadata(DOWNLOAD_QUEUE, + PROCESS_QUEUE) + thread.start() + THREADS.append(thread) + LOG.debug("%s download threads spawned", state.SYNC_THREAD_NUMBER) + # Spawn one more thread to process Metadata, once downloaded + thread = process_metadata.ThreadedProcessMetadata(PROCESS_QUEUE, + item_class) + thread.start() + THREADS.append(thread) + # Start one thread to show sync progress ONLY for new PMS items + if show_sync_info: + sync_info.GET_METADATA_COUNT = 0 + sync_info.PROCESS_METADATA_COUNT = 0 + sync_info.PROCESSING_VIEW_NAME = '' + thread = sync_info.ThreadedShowSyncInfo(item_number, item_class) + thread.start() + THREADS.append(thread) + # Process items we need to download + for _ in generator: + DOWNLOAD_QUEUE.put(self.updatelist.pop(0)) + if search_fanart: + pass + # Wait until finished + DOWNLOAD_QUEUE.join() + PROCESS_QUEUE.join() + # Kill threads + LOG.debug("Waiting to kill threads") + for thread in THREADS: + # Threads might already have quit by themselves (e.g. Kodi exit) + try: + thread.stop() + except AttributeError: + pass + LOG.debug("Stop sent to all threads") + # Wait till threads are indeed dead + for thread in threads: + try: + thread.join(1.0) + except AttributeError: + pass + LOG.debug("Sync threads finished") diff --git a/resources/lib/library_sync/get_metadata.py b/resources/lib/library_sync/get_metadata.py index e7f2ce16..c42f16db 100644 --- a/resources/lib/library_sync/get_metadata.py +++ b/resources/lib/library_sync/get_metadata.py @@ -1,13 +1,9 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import, division, unicode_literals from logging import getLogger -from threading import Thread -from Queue import Empty -from xbmc import sleep -from .. import utils -from .. import plex_functions as PF -from . import sync_info +from . import common +from .. import plex_functions as PF, backgroundthread, utils ############################################################################### @@ -16,107 +12,46 @@ LOG = getLogger("PLEX." + __name__) ############################################################################### -@utils.thread_methods(add_stops=['SUSPEND_LIBRARY_THREAD', - 'STOP_SYNC', - 'SUSPEND_SYNC']) -class ThreadedGetMetadata(Thread): +class GetMetadataTask(backgroundthread.Task, common.libsync_mixin): """ Threaded download of Plex XML metadata for a certain library item. - Fills the out_queue with the downloaded etree XML objects + Fills the queue with the downloaded etree XML objects Input: - queue Queue.Queue() object that you'll need to fill up - with plex_ids - out_queue Queue() object where this thread will store + queue Queue.Queue() object where this thread will store the downloaded metadata XMLs as etree objects """ - def __init__(self, queue, out_queue): + def setup(self, queue, plex_id, get_children=False): self.queue = queue - self.out_queue = out_queue - Thread.__init__(self) - - def terminate_now(self): - """ - Needed to terminate this thread, because there might be items left in - the queue which could cause other threads to hang - """ - while not self.queue.empty(): - # Still try because remaining item might have been taken - try: - self.queue.get(block=False) - except Empty: - sleep(10) - continue - else: - self.queue.task_done() - if self.stopped(): - # Shutdown from outside requested; purge out_queue as well - while not self.out_queue.empty(): - # Still try because remaining item might have been taken - try: - self.out_queue.get(block=False) - except Empty: - sleep(10) - continue - else: - self.out_queue.task_done() + self.plex_id = plex_id + self.get_children = get_children def run(self): """ Do the work """ - LOG.debug('Starting get metadata thread') - # cache local variables because it's faster - queue = self.queue - out_queue = self.out_queue - stopped = self.stopped - while stopped() is False: - # grabs Plex item from queue + if self.isCanceled(): + return + # Download Metadata + xml = PF.GetPlexMetadata(self.plex_id) + if xml is None: + # Did not receive a valid XML - skip that item for now + LOG.error("Could not get metadata for %s. Skipping that item " + "for now", self.plex_id) + continue + elif xml == 401: + LOG.error('HTTP 401 returned by PMS. Too much strain? ' + 'Cancelling sync for now') + utils.window('plex_scancrashed', value='401') + return + xml.children = None + if not self.isCanceled() and self.get_children: + children_xml = PF.GetAllPlexChildren(self.plex_id) try: - item = queue.get(block=False) - # Empty queue - except Empty: - sleep(20) - continue - # Download Metadata - xml = PF.GetPlexMetadata(item['plex_id']) - if xml is None: - # Did not receive a valid XML - skip that item for now - LOG.error("Could not get metadata for %s. Skipping that item " - "for now", item['plex_id']) - # Increase BOTH counters - since metadata won't be processed - with sync_info.LOCK: - sync_info.GET_METADATA_COUNT += 1 - sync_info.PROCESS_METADATA_COUNT += 1 - queue.task_done() - continue - elif xml == 401: - LOG.error('HTTP 401 returned by PMS. Too much strain? ' - 'Cancelling sync for now') - utils.window('plex_scancrashed', value='401') - # Kill remaining items in queue (for main thread to cont.) - queue.task_done() - break - - item['xml'] = xml - if item.get('get_children') is True: - children_xml = PF.GetAllPlexChildren(item['plex_id']) - try: - children_xml[0].attrib - except (TypeError, IndexError, AttributeError): - LOG.error('Could not get children for Plex id %s', - item['plex_id']) - item['children'] = [] - else: - item['children'] = children_xml - - # place item into out queue - out_queue.put(item) - # Keep track of where we are at - with sync_info.LOCK: - sync_info.GET_METADATA_COUNT += 1 - # signals to queue job is done - queue.task_done() - # Empty queue in case PKC was shut down (main thread hangs otherwise) - self.terminate_now() - LOG.debug('Get metadata thread terminated') + children_xml[0].attrib + except (TypeError, IndexError, AttributeError): + LOG.error('Could not get children for Plex id %s', + self.plex_id) + else: + xml.children = children_xml + self.queue.put(xml) diff --git a/resources/lib/library_sync/process_metadata.py b/resources/lib/library_sync/process_metadata.py index c0258730..512dcf95 100644 --- a/resources/lib/library_sync/process_metadata.py +++ b/resources/lib/library_sync/process_metadata.py @@ -1,24 +1,16 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import, division, unicode_literals from logging import getLogger -from threading import Thread -from Queue import Empty -from xbmc import sleep +import xbmc +import xbmcgui -from .. import utils -from .. import itemtypes -from . import sync_info +from . import common +from .. import utils, backgroundthread -############################################################################### -LOG = getLogger("PLEX." + __name__) - -############################################################################### +LOG = getLogger('PLEX.library_sync.process_metadata') -@utils.thread_methods(add_stops=['SUSPEND_LIBRARY_THREAD', - 'STOP_SYNC', - 'SUSPEND_SYNC']) -class ThreadedProcessMetadata(Thread): +class ProcessMetadata(backgroundthread.KillableThread, common.libsync_mixin): """ Not yet implemented for more than 1 thread - if ever. Only to be called by ONE thread! @@ -30,59 +22,59 @@ class ThreadedProcessMetadata(Thread): item_class: as used to call functions in itemtypes.py e.g. 'Movies' => itemtypes.Movies() """ - def __init__(self, queue, item_class): + def __init__(self, queue, context, total_number_of_items): self.queue = queue - self.item_class = item_class - Thread.__init__(self) + self.context = context + self.total = total_number_of_items + self.current = 0 + self.title = None + super(ProcessMetadata, self).__init__() - def terminate_now(self): + def update_dialog(self): """ - Needed to terminate this thread, because there might be items left in - the queue which could cause other threads to hang """ - while not self.queue.empty(): - # Still try because remaining item might have been taken - try: - self.queue.get(block=False) - except Empty: - sleep(10) - continue - else: - self.queue.task_done() + try: + progress = int(float(self.current) / float(self.total) * 100.0) + except ZeroDivisionError: + progress = 0 + self.dialog.update(progress, + utils.lang(29999), + '%s/%s: %s' + % (self.current, self.total, self.title)) def run(self): """ Do the work """ LOG.debug('Processing thread started') - # Constructs the method name, e.g. itemtypes.Movies - item_fct = getattr(itemtypes, self.item_class) - # cache local variables because it's faster - queue = self.queue - stopped = self.stopped - with item_fct() as item_class: - while stopped() is False: + self.dialog = xbmcgui.DialogProgressBG() + self.dialog.create(utils.lang(39714)) + with self.context() as context: + while self.isCanceled() is False: # grabs item from queue try: - item = queue.get(block=False) - except Empty: - sleep(20) + xml = self.queue.get(block=False) + except backgroundthread.Queue.Empty: + xbmc.sleep(10) continue - # Do the work - item_method = getattr(item_class, item['method']) - if item.get('children') is not None: - item_method(item['xml'][0], - viewtag=item['view_name'], - viewid=item['view_id'], - children=item['children']) - else: - item_method(item['xml'][0], - viewtag=item['view_name'], - viewid=item['view_id']) - # Keep track of where we are at - with sync_info.LOCK: - sync_info.PROCESS_METADATA_COUNT += 1 - sync_info.PROCESSING_VIEW_NAME = item['title'] - queue.task_done() - self.terminate_now() + self.queue.task_done() + if xml is None: + break + try: + if xml.children is not None: + context.add_update(xml[0], + viewtag=xml['view_name'], + viewid=xml['view_id'], + children=xml['children']) + else: + context.add_update(xml[0], + viewtag=xml['view_name'], + viewid=xml['view_id']) + except: + utils.ERROR(txt='process_metadata crashed', notify=True) + self.current += 1 + if self.current % 20 == 0: + self.title = utils.cast(unicode, xml[0].get('title')) + self.update_dialog() + self.dialog.close() LOG.debug('Processing thread terminated') diff --git a/resources/lib/library_sync/sections.py b/resources/lib/library_sync/sections.py new file mode 100644 index 00000000..398acba2 --- /dev/null +++ b/resources/lib/library_sync/sections.py @@ -0,0 +1,238 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +from __future__ import absolute_import, division, unicode_literals +from logging import getLogger +import copy + +from . import common, videonodes +from ..utils import cast +from .. import plexdb_functions as plexdb, kodidb_functions as kodidb +from .. import itemtypes +from .. import PlexFunctions as PF, music, utils, state, variables as v + +LOG = getLogger('PLEX.library_sync.sections') + +VNODES = videonodes.VideoNodes() +PLAYLISTS = {} +NODES = {} +SECTIONS = [] + + +def sync_from_pms(): + """ + Sync the Plex library sections + """ + sections = PF.get_plex_sections() + try: + sections.attrib + except AttributeError: + LOG.error("Error download PMS sections, abort") + return False + if state.DIRECT_PATHS is True and state.ENABLE_MUSIC is True: + # Will reboot Kodi is new library detected + music.excludefromscan_music_folders(xml=sections) + + global PLAYLISTS, NODES, SECTIONS + SECTIONS = [] + NODES = { + v.PLEX_TYPE_MOVIE: [], + v.PLEX_TYPE_SHOW: [], + v.PLEX_TYPE_ARTIST: [], + v.PLEX_TYPE_PHOTO: [] + } + PLAYLISTS = copy.deepcopy(NODES) + sorted_sections = [] + + for section in sections: + if (section.attrib['type'] in + (v.PLEX_TYPE_MOVIE, v.PLEX_TYPE_SHOW, v.PLEX_TYPE_PHOTO, + v.PLEX_TYPE_ARTIST)): + sorted_sections.append(cast(unicode, + section.attrib['title'])) + LOG.debug('Sorted sections: %s', sorted_sections) + totalnodes = len(sorted_sections) + + VNODES.clearProperties() + + with plexdb.Get_Plex_DB() as plex_db: + # Backup old sections to delete them later, if needed (at the end + # of this method, only unused sections will be left in old_sections) + old_sections = plex_db.sections() + with kodidb.GetKodiDB('video') as kodi_db: + for section in sections: + _process_section(section, + kodi_db, + plex_db, + sorted_sections, + old_sections, + totalnodes) + + if old_sections: + # Section has been deleted on the PMS + delete_sections(old_sections) + # update sections for all: + with plexdb.Get_Plex_DB() as plex_db: + SECTIONS = plex_db.list_section_info() + utils.window('Plex.nodes.total', str(totalnodes)) + LOG.info("Finished processing library sections: %s", SECTIONS) + return True + + +def _process_section(section_xml, kodi_db, plex_db, sorted_sections, + old_sections, totalnodes): + folder = section_xml.attrib + plex_type = cast(unicode, folder['type']) + # Only process supported formats + if plex_type not in (v.PLEX_TYPE_MOVIE, v.PLEX_TYPE_SHOW, + v.PLEX_TYPE_ARTIST, v.PLEX_TYPE_PHOTO): + LOG.error('Unsupported Plex section type: %s', folder) + return totalnodes + section_id = cast(int, folder['key']) + section_name = cast(unicode, folder['title']) + global PLAYLISTS, NODES + # Prevent duplicate for nodes of the same type + nodes = NODES[plex_type] + # Prevent duplicate for playlists of the same type + playlists = PLAYLISTS[plex_type] + # Get current media folders from plex database + section = plex_db.section_by_id(section_id) + try: + current_sectionname = section[1] + current_sectiontype = section[2] + current_tagid = section[3] + except TypeError: + LOG.info('Creating section id: %s in Plex database.', section_id) + tagid = kodi_db.create_tag(section_name) + # Create playlist for the video library + if (section_name not in playlists and + plex_type in (v.PLEX_TYPE_MOVIE, v.PLEX_TYPE_SHOW)): + utils.playlist_xsp(plex_type, section_name, section_id) + playlists.append(section_name) + # Create the video node + if section_name not in nodes: + VNODES.viewNode(sorted_sections.index(section_name), + section_name, + plex_type, + None, + section_id) + nodes.append(section_name) + totalnodes += 1 + # Add view to plex database + plex_db.add_section(section_id, section_name, plex_type, tagid) + else: + LOG.info('Found library section id %s, name %s, type %s, tagid %s', + section_id, current_sectionname, current_sectiontype, + current_tagid) + # Remove views that are still valid to delete rest later + try: + old_sections.remove(section_id) + except ValueError: + # View was just created, nothing to remove + pass + + # View was modified, update with latest info + if current_sectionname != section_name: + LOG.info('section id: %s new sectionname: %s', + section_id, section_name) + tagid = kodi_db.create_tag(section_name) + + # Update view with new info + plex_db.update_section(section_name, tagid, section_id) + + if plex_db.section_id_by_name(current_sectionname) is None: + # The tag could be a combined view. Ensure there's + # no other tags with the same name before deleting + # playlist. + utils.playlist_xsp(plex_type, + current_sectionname, + section_id, + current_sectiontype, + True) + # Delete video node + if plex_type != "musicvideos": + VNODES.viewNode( + indexnumber=sorted_sections.index(section_name), + tagname=current_sectionname, + mediatype=plex_type, + viewtype=None, + viewid=section_id, + delete=True) + # Added new playlist + if section_name not in playlists and plex_type in v.KODI_VIDEOTYPES: + utils.playlist_xsp(plex_type, + section_name, + section_id) + playlists.append(section_name) + # Add new video node + if section_name not in nodes and plex_type != "musicvideos": + VNODES.viewNode(sorted_sections.index(section_name), + section_name, + plex_type, + None, + section_id) + nodes.append(section_name) + totalnodes += 1 + # Update items with new tag + for item in plex_db.kodi_id_by_section(section_id): + # Remove the "s" from viewtype for tags + kodi_db.update_tag( + current_tagid, tagid, item[0], current_sectiontype[:-1]) + else: + # Validate the playlist exists or recreate it + if (section_name not in playlists and plex_type in + (v.PLEX_TYPE_MOVIE, v.PLEX_TYPE_SHOW)): + utils.playlist_xsp(plex_type, + section_name, + section_id) + playlists.append(section_name) + # Create the video node if not already exists + if section_name not in nodes and plex_type != "musicvideos": + VNODES.viewNode(sorted_sections.index(section_name), + section_name, + plex_type, + None, + section_id) + nodes.append(section_name) + totalnodes += 1 + return totalnodes + + +def delete_sections(old_sections): + """ + Deletes all elements for a Plex section that has been deleted. (e.g. all + TV shows, Seasons and Episodes of a Show section) + """ + utils.dialog('notification', + heading='{plex}', + message=utils.lang(30052), + icon='{plex}', + sound=False) + video_library_update = False + music_library_update = False + with plexdb.Get_Plex_DB() as plex_db: + old_sections = [plex_db.section_by_id(x) for x in old_sections] + LOG.info("Removing entire Plex library sections: %s", old_sections) + with kodidb.GetKodiDB() as kodi_db: + for section in old_sections: + if section[2] == v.KODI_TYPE_MOVIE: + video_library_update = True + context = itemtypes.Movie(plex_db=plex_db, + kodi_db=kodi_db) + elif section[2] == v.KODI_TYPE_SHOW: + video_library_update = True + context = itemtypes.Show(plex_db=plex_db, + kodi_db=kodi_db) + elif section[2] == v.KODI_TYPE_ARTIST: + music_library_update = True + context = itemtypes.Artist(plex_db=plex_db, + kodi_db=kodi_db) + elif section[2] == v.KODI_TYPE_PHOTO: + # not synced + plex_db.remove_section(section[0]) + continue + for plex_id in plex_db.plexid_by_section(section[0]): + context.remove(plex_id) + # Only remove Plex entry if we've removed all items first + plex_db.remove_section(section[0]) + common.update_kodi_library(video=video_library_update, + music=music_library_update) diff --git a/resources/lib/videonodes.py b/resources/lib/library_sync/videonodes.py similarity index 100% rename from resources/lib/videonodes.py rename to resources/lib/library_sync/videonodes.py diff --git a/resources/lib/librarysync.py b/resources/lib/librarysync.py index 7358c44b..58946ad2 100644 --- a/resources/lib/librarysync.py +++ b/resources/lib/librarysync.py @@ -16,7 +16,6 @@ from . import itemtypes from . import plexdb_functions as plexdb from . import kodidb_functions as kodidb from . import artwork -from . import videonodes from . import plex_functions as PF from .plex_api import API from .library_sync import get_metadata, process_metadata, fanart, sync_info @@ -39,25 +38,6 @@ LOG = getLogger('PLEX.librarysync') ############################################################################### -def update_library(video=True, music=True): - """ - Updates the Kodi library and thus refreshes the Kodi views and widgets - """ - if xbmc.getCondVisibility('Container.Content(musicvideos)') or \ - xbmc.getCondVisibility('Window.IsMedia'): - # Prevent cursor from moving - LOG.debug("Refreshing container") - xbmc.executebuiltin('Container.Refresh') - else: - # Update widgets - if video: - LOG.debug("Doing Kodi Video Lib update") - xbmc.executebuiltin('UpdateLibrary(video)') - if music: - LOG.debug("Doing Kodi Music Lib update") - xbmc.executebuiltin('UpdateLibrary(music)') - - @utils.thread_methods(add_suspends=['SUSPEND_LIBRARY_THREAD', 'STOP_SYNC']) class LibrarySync(Thread): """ @@ -236,8 +216,8 @@ class LibrarySync(Thread): # Create the tables for the plex database plex_db.plexcursor.execute(''' CREATE TABLE IF NOT EXISTS plex( - plex_id TEXT UNIQUE, - view_id TEXT, + plex_id INTEGER PRIMARY KEY ASC, + section_id INTEGER, plex_type TEXT, kodi_type TEXT, kodi_id INTEGER, @@ -245,13 +225,14 @@ class LibrarySync(Thread): kodi_pathid INTEGER, parent_id INTEGER, checksum INTEGER, - fanart_synced INTEGER) + fanart_synced INTEGER, + last_sync INTEGER) ''') plex_db.plexcursor.execute(''' - CREATE TABLE IF NOT EXISTS view( - view_id TEXT UNIQUE, - view_name TEXT, - kodi_type TEXT, + CREATE TABLE IF NOT EXISTS sections( + section_id INTEGER PRIMARY KEY, + section_name TEXT, + plex_type TEXT, kodi_tagid INTEGER, sync_to_kodi INTEGER) ''') @@ -260,7 +241,7 @@ class LibrarySync(Thread): ''') plex_db.plexcursor.execute(''' CREATE TABLE IF NOT EXISTS playlists( - plex_id TEXT UNIQUE, + plex_id PRIMARY KEY, plex_name TEXT, plex_updatedat TEXT, kodi_path TEXT, diff --git a/resources/lib/plex_api.py b/resources/lib/plex_api.py index ed1b0dad..450c29df 100644 --- a/resources/lib/plex_api.py +++ b/resources/lib/plex_api.py @@ -118,9 +118,12 @@ class API(object): def plex_id(self): """ - Returns the Plex ratingKey such as '246922' as Unicode or None + Returns the Plex ratingKey such as 246922 as an integer or None """ - return _unicode_or_none(self.item.get('ratingKey')) + try: + return int(self.item.get('ratingKey')) + except TypeError, ValueError: + pass def path(self, force_first_media=True, force_addon=False, direct_paths=None): @@ -463,23 +466,31 @@ class API(object): provider = None return provider + def votecount(self): + """ + Not implemented by Plex yet + """ + pass + def title(self): """ Returns the title of the element as unicode or 'Missing Title Name' """ return utils.try_decode(self.item.get('title', 'Missing Title Name')) - def titles(self): + def title(self): """ - Returns an item's name/title or "Missing Title Name". - Output is the tuple - title, sorttitle + Returns an item's name/title or "Missing Title". + """ + return self.item.get('title', 'Missing Title') - sorttitle = title, if no sorttitle is found + def sorttitle(self): + """ + Returns an item's sorting name/title or the title itself if not found + "Missing Title" if both are not present """ - title = self.item.get('title', 'Missing Title Name') - sorttitle = self.item.get('titleSort', title) - return title, sorttitle + return self.item.get('titleSort', + self.item.get('title','Missing Title')) def plot(self): """ diff --git a/resources/lib/plex_functions.py b/resources/lib/plex_functions.py index 40d29716..db4664ec 100644 --- a/resources/lib/plex_functions.py +++ b/resources/lib/plex_functions.py @@ -496,7 +496,8 @@ def GetPlexMetadata(key): def plex_children_generator(key): """ """ - yield download_generator('{server}/library/metadata/%s/children' % key) + for entry in download_generator('{server}/library/metadata/%s/children' % key): + yield entry def GetAllPlexChildren(key): @@ -526,38 +527,67 @@ def GetPlexSectionResults(viewId, args=None): return DownloadChunks(url) -def download_generator(url): +class DownloadGen(object): """ - Generator to yield XML children piece-wise. - PMS XML is downloaded chunks of CONTAINERSIZE. + Special iterator object that will yield all child xmls piece-wise. It also + saves the original xml.attrib. Yields XML etree children or raises RuntimeError """ - pos = 0 - error_counter = 0 - while error_counter < 3: + def __init__(self, url): + self._url = url + self._pos = 0 + self._exhausted = False + self._download_chunk() + self.attrib = deepcopy(self.xml.attrib) + + def _download_chunk(self): args = { 'X-Plex-Container-Size': CONTAINERSIZE, - 'X-Plex-Container-Start': pos + 'X-Plex-Container-Start': self._pos } - xmlpart = DU().downloadUrl(url, parameters=args) - # If something went wrong - skip in the hope that it works next time + self.xml = DU().downloadUrl(self._url, parameters=args) try: - xmlpart.attrib + self.xml.attrib except AttributeError: LOG.error('Error while downloading chunks: %s, args: %s', - url, args) - error_counter += 1 - continue - for child in xmlpart: - yield child - # Done as soon as we don't receive a full complement of items - if len(xmlpart) < CONTAINERSIZE: - break - pos += CONTAINERSIZE - else: - LOG.error('Fatal error while downloading chunks for %s', url) - raise RuntimeError('Error while downloading chunks for %s' % url) + self._url, args) + raise RuntimeError('Error while downloading chunks for %s' + % self._url) + + def __iter__(self): + return self + + def next(self): + return self.__next__() + + def __next__(self): + if len(self.xml): + child = self.xml[0] + self.xml.remove(child) + return child + elif self._exhausted: + raise StopIteration + else: + self._pos += CONTAINERSIZE + self._download_chunk() + if not len(self.xml): + raise StopIteration + if len(self.xml) < CONTAINERSIZE: + self._exhausted = True + return self.__next__() + + def get(self, key, default=None): + return self.attrib.get(key, default) + + +class PlexSectionItems(DownloadGen): + """ + Iterator object to get all items of a Plex library section + """ + def __init__(self, section_id): + super(PlexSectionItems, self).__init__( + '{server}/library/sections/%s/all' % section_id) def DownloadChunks(url): diff --git a/resources/lib/plexdb_functions.py b/resources/lib/plexdb_functions.py index 49408479..2bf2d912 100644 --- a/resources/lib/plexdb_functions.py +++ b/resources/lib/plexdb_functions.py @@ -29,163 +29,96 @@ class Plex_DB_Functions(): def __init__(self, plexcursor): self.plexcursor = plexcursor - def getViews(self): + def sections(self): """ - Returns a list of view_id + Returns a list of section Plex ids for all sections """ - views = [] - query = ''' - SELECT view_id - FROM view - ''' - self.plexcursor.execute(query) - rows = self.plexcursor.fetchall() - for row in rows: - views.append(row[0]) - return views + self.plexcursor.execute('SELECT section_id FROM sections') + return [x[0] for x in self.plexcursor] - def getAllViewInfo(self): + def list_section_info(self): """ Returns a list of dicts for all Plex libraries: { - 'id': view_id, - 'name': view_name, - 'itemtype': kodi_type + 'section_id' + 'section_name' + 'plex_type' 'kodi_tagid' 'sync_to_kodi' } """ - plexcursor = self.plexcursor - views = [] - query = '''SELECT * FROM view''' - plexcursor.execute(query) - rows = plexcursor.fetchall() - for row in rows: - views.append({'id': row[0], - 'name': row[1], - 'itemtype': row[2], - 'kodi_tagid': row[3], - 'sync_to_kodi': row[4]}) - return views + self.plexcursor.execute('SELECT * FROM sections') + return [{'section_id': x[0], + 'section_name': x[1], + 'plex_type': x[2], + 'kodi_tagid': x[3], + 'sync_to_kodi': x[4]} for x in self.plexcursor] - def getView_byId(self, view_id): + def section_by_id(self, section_id): """ - Returns tuple (view_name, kodi_type, kodi_tagid) for view_id + Returns tuple (section_id, section_name, plex_type, kodi_tagid, + sync_to_kodi) for section_id + """ + self.plexcursor.execute('SELECT * FROM sections WHERE section_id = ? LIMIT 1', + (section_id, )) + return self.plexcursor.fetchone() + + def section_id_by_name(self, section_name): + """ + Returns the section_id for section_name (or None) """ query = ''' - SELECT view_name, kodi_type, kodi_tagid - FROM view - WHERE view_id = ? + SELECT section_id FROM sections + WHERE section_name = ? + LIMIT 1 ''' - self.plexcursor.execute(query, (view_id,)) - view = self.plexcursor.fetchone() - return view - - def getView_byType(self, kodi_type): - """ - Returns a list of dicts for kodi_type: - {'id': view_id, 'name': view_name, 'itemtype': kodi_type} - """ - views = [] - query = ''' - SELECT view_id, view_name, kodi_type - FROM view - WHERE kodi_type = ? - ''' - self.plexcursor.execute(query, (kodi_type,)) - rows = self.plexcursor.fetchall() - for row in rows: - views.append({ - 'id': row[0], - 'name': row[1], - 'itemtype': row[2] - }) - return views - - def getView_byName(self, view_name): - """ - Returns the view_id for view_name (or None) - """ - query = ''' - SELECT view_id - FROM view - WHERE view_name = ? - ''' - self.plexcursor.execute(query, (view_name,)) + self.plexcursor.execute(query, (section_name,)) try: - view = self.plexcursor.fetchone()[0] + section = self.plexcursor.fetchone()[0] except TypeError: - view = None - return view + section = None + return section - def addView(self, view_id, view_name, kodi_type, kodi_tagid, sync=True): + def add_section(self, section_id, section_name, plex_type, kodi_tagid, + sync_to_kodi=True): """ - Appends an entry to the view table - + Appends a Plex section to the Plex sections table sync=False: Plex library won't be synced to Kodi """ query = ''' - INSERT INTO view( - view_id, view_name, kodi_type, kodi_tagid, sync_to_kodi) + INSERT INTO sections( + section_id, section_name, plex_type, kodi_tagid, sync_to_kodi) VALUES (?, ?, ?, ?, ?) ''' self.plexcursor.execute(query, - (view_id, - view_name, - kodi_type, + (section_id, + section_name, + plex_type, kodi_tagid, - 1 if sync is True else 0)) + sync_to_kodi)) - def updateView(self, view_name, kodi_tagid, view_id): + def update_section(self, section_name, kodi_tagid, section_id): """ - Updates the view_id with view_name and kodi_tagid + Updates the section_id with section_name and kodi_tagid """ query = ''' - UPDATE view - SET view_name = ?, kodi_tagid = ? - WHERE view_id = ? + UPDATE sections + SET section_name = ?, kodi_tagid = ? + WHERE section_id = ? ''' - self.plexcursor.execute(query, (view_name, kodi_tagid, view_id)) + self.plexcursor.execute(query, (section_name, kodi_tagid, section_id)) - def removeView(self, view_id): - query = ''' - DELETE FROM view - WHERE view_id = ? - ''' - self.plexcursor.execute(query, (view_id,)) + def remove_section(self, section_id): + self.plexcursor.execute('DELETE FROM sections WHERE section_id = ?', + (section_id, )) - def get_items_by_viewid(self, view_id): + def plexid_by_section(self, section_id): """ - Returns a list for view_id with one item like this: - { - 'plex_id': xxx - 'kodi_type': xxx - } + Returns an iterator for the plex_id for section_id """ - query = '''SELECT plex_id, kodi_type FROM plex WHERE view_id = ?''' - self.plexcursor.execute(query, (view_id, )) - rows = self.plexcursor.fetchall() - res = [] - for row in rows: - res.append({'plex_id': row[0], 'kodi_type': row[1]}) - return res - - def getItem_byFileId(self, kodi_fileid, kodi_type): - """ - Returns plex_id for kodi_fileid and kodi_type - - None if not found - """ - query = ''' - SELECT plex_id FROM plex WHERE kodi_fileid = ? AND kodi_type = ? - LIMIT 1 - ''' - self.plexcursor.execute(query, (kodi_fileid, kodi_type)) - try: - item = self.plexcursor.fetchone()[0] - except TypeError: - item = None - return item + self.plexcursor.execute('SELECT plex_id FROM plex WHERE section_id = ?', + (section_id, )) + return (x[0] for x in self.plexcursor) def getItem_byId(self, plex_id): """ @@ -215,17 +148,13 @@ class Plex_DB_Functions(): self.plexcursor.execute(query, (plex_id + "%",)) return self.plexcursor.fetchall() - def getItem_byView(self, view_id): + def kodi_id_by_section(self, section_id): """ - Returns kodi_id for view_id + Returns an iterator! Returns kodi_id for section_id """ - query = ''' - SELECT kodi_id - FROM plex - WHERE view_id = ? - ''' - self.plexcursor.execute(query, (view_id,)) - return self.plexcursor.fetchall() + self.plexcursor.execute('SELECT kodi_id FROM plex WHERE section_id = ?', + (section_id, )) + return self.plexcursor def getItem_byKodiId(self, kodi_id, kodi_type): """ @@ -267,6 +196,24 @@ class Plex_DB_Functions(): self.plexcursor.execute(query, (parent_id, kodi_type,)) return self.plexcursor.fetchall() + def check_plexid(self, plex_id): + """ + FAST method to check whether plex_id has already been safed in db. + Returns None if not yet in plex DB + """ + self.plexcursor.execute('SELECT plex_id FROM plex WHERE plex_id = ? LIMIT 1', + (plex_id, )) + return self.plexcursor.fetchone() + + def check_checksum(self, checksum): + """ + FAST method to check whether checksum has already been safed in db. + Returns None if not yet in plex DB + """ + self.plexcursor.execute('SELECT checksum FROM plex WHERE checksum = ? LIMIT 1', + (checksum, )) + return self.plexcursor.fetchone() + def checksum(self, plex_type): """ Returns a list of tuples (plex_id, checksum) for plex_type diff --git a/resources/lib/utils.py b/resources/lib/utils.py index e9637d60..53eb9617 100644 --- a/resources/lib/utils.py +++ b/resources/lib/utils.py @@ -314,6 +314,30 @@ def kodi_time_to_millis(time): return ret +def cast(func, value): + """ + Cast the specified value to the specified type (returned by func). Currently this + only support int, float, bool. Should be extended if needed. + Parameters: + func (func): Calback function to used cast to type (int, bool, float). + value (any): value to be cast and returned. + """ + if value is not None: + if func == bool: + return bool(int(value)) + elif func == unicode: + return value.decode('utf-8') + elif func == str: + return value.encode('utf-8') + elif func in (int, float): + try: + return func(value) + except ValueError: + return float('nan') + return func(value) + return value + + def try_encode(input_str, encoding='utf-8'): """ Will try to encode input_str (in unicode) to encoding. This possibly