From e38f99f0881e49f2b5efcaff5dcb29b45e2a13a9 Mon Sep 17 00:00:00 2001 From: Croneter Date: Tue, 1 May 2018 14:48:49 +0200 Subject: [PATCH] Playlist sync support, part 2 --- resources/lib/PlexAPI.py | 13 +++ resources/lib/librarysync.py | 12 +- resources/lib/playlist_func.py | 9 +- resources/lib/playlists.py | 187 ++++++++++++++++++++++++------ resources/lib/plexdb_functions.py | 54 +++++++++ resources/lib/utils.py | 67 ++++++++--- resources/lib/variables.py | 21 +++- 7 files changed, 297 insertions(+), 66 deletions(-) diff --git a/resources/lib/PlexAPI.py b/resources/lib/PlexAPI.py index bafb067a..bc6e8f70 100644 --- a/resources/lib/PlexAPI.py +++ b/resources/lib/PlexAPI.py @@ -85,6 +85,19 @@ class API(object): """ return self.item.get('type') + def playlist_type(self): + """ + Returns the playlist type ('video', 'audio') or None + """ + return self.item.get('playlistType') + + def updated_at(self): + """ + Returns the last time this item was updated as unicode, e.g. + '1524739868', or None + """ + return self.item.get('updatedAt') + def checksum(self): """ Returns a string, not int. diff --git a/resources/lib/librarysync.py b/resources/lib/librarysync.py index 69f930d8..c011459f 100644 --- a/resources/lib/librarysync.py +++ b/resources/lib/librarysync.py @@ -1527,7 +1527,7 @@ class LibrarySync(Thread): last_time_sync = utils.unix_timestamp() window('plex_dbScan', clear=True) state.DB_SCAN = False - kodi_playlist_monitor = None + playlist_monitor = None while not self.stopped(): # In the event the server goes offline @@ -1554,7 +1554,8 @@ class LibrarySync(Thread): kodi_db_version_checked = True last_sync = utils.unix_timestamp() self.fanartthread.start() - kodi_playlist_monitor = playlists.kodi_playlist_monitor() + if playlists.full_sync(): + playlist_monitor = playlists.kodi_playlist_monitor() else: LOG.error('Initial start-up full sync unsuccessful') xbmc.executebuiltin('InhibitIdleShutdown(false)') @@ -1600,11 +1601,12 @@ class LibrarySync(Thread): LOG.info('Done initial sync on Kodi startup') artwork.Artwork().cache_major_artwork() self.fanartthread.start() + if playlists.full_sync(): + playlist_monitor = playlists.kodi_playlist_monitor() else: LOG.info('Startup sync has not yet been successful') window('plex_dbScan', clear=True) state.DB_SCAN = False - kodi_playlist_monitor = playlists.kodi_playlist_monitor() # Currently no db scan, so we can start a new scan elif state.DB_SCAN is False: @@ -1665,8 +1667,8 @@ class LibrarySync(Thread): continue xbmc.sleep(100) # Shut down playlist monitoring - if kodi_playlist_monitor: - kodi_playlist_monitor.stop() + if playlist_monitor: + playlist_monitor.stop() # doUtils could still have a session open due to interrupted sync try: DU().stopSession() diff --git a/resources/lib/playlist_func.py b/resources/lib/playlist_func.py index 17fdde42..e84dc589 100644 --- a/resources/lib/playlist_func.py +++ b/resources/lib/playlist_func.py @@ -139,9 +139,9 @@ class Playlist_Object(PlaylistObjectBaseclase): except ValueError: raise PlaylistError('Invalid path: %s' % path) if path.startswith(v.PLAYLIST_PATH_VIDEO): - self.type = 'video' + self.type = v.KODI_TYPE_VIDEO_PLAYLIST elif path.startswith(v.PLAYLIST_PATH_MUSIC): - self.type = 'music' + self.type = v.KODI_TYPE_AUDIO_PLAYLIST else: raise PlaylistError('Playlist type not supported: %s' % path) if not self.plex_name: @@ -670,6 +670,7 @@ def get_all_playlists(): try: xml.attrib except (AttributeError, TypeError): + LOG.error('Could not download a list of all playlists') xml = None return xml @@ -682,9 +683,7 @@ def get_PMS_playlist(playlist, playlist_id=None): Returns None if something went wrong """ playlist_id = playlist_id if playlist_id else playlist.id - xml = DU().downloadUrl( - "{server}/%ss/%s" % (playlist.kind, playlist_id), - headerOptions={'Accept': 'application/xml'}) + xml = DU().downloadUrl("{server}/%ss/%s" % (playlist.kind, playlist_id)) try: xml.attrib['%sID' % playlist.kind] except (AttributeError, KeyError): diff --git a/resources/lib/playlists.py b/resources/lib/playlists.py index 514f6ed3..4dc92729 100644 --- a/resources/lib/playlists.py +++ b/resources/lib/playlists.py @@ -2,7 +2,8 @@ from logging import getLogger import os import sys -import re + +from xbmcvfs import exists import watchdog import playlist_func as PL @@ -11,15 +12,12 @@ import kodidb_functions as kodidb import plexdb_functions as plexdb import utils import variables as v +import state ############################################################################### LOG = getLogger("PLEX." + __name__) -# Our PKC playlists. Keys: ID [int] of plex DB table playlists. Values: -# playlist_func.Playlist_Object() -PLAYLISTS = {} - # Which playlist formates are supported by PKC? SUPPORTED_FILETYPES = ( 'm3u', @@ -28,12 +26,11 @@ SUPPORTED_FILETYPES = ( # 'cue', ) +# m3u files do not have encoding specified DEFAULT_ENCODING = sys.getdefaultencoding() -REGEX_PLEX_ID = re.compile(r'''plex_id=(\d+)''') - -def create_plex_playlist(playlist): +def create_plex_playlist(playlist=None, path=None): """ Adds the playlist [Playlist_Object] to the PMS. If playlist.plex_id is not None the existing Plex playlist will be overwritten; otherwise a new @@ -43,6 +40,9 @@ def create_plex_playlist(playlist): Returns None or raises PL.PlaylistError """ + if not playlist: + playlist = PL.Playlist_Object() + playlist.kodi_path = path LOG.info('Creating Plex playlist from Kodi file: %s', playlist.kodi_path) plex_ids = _playlist_file_to_plex_ids(playlist) for pos, plex_id in enumerate(plex_ids): @@ -64,40 +64,50 @@ def delete_plex_playlist(playlist): LOG.info('Deleting playlist %s from the PMS', playlist.plex_name) try: PL.delete_playlist_from_pms(playlist) - except PL.PlaylistError as err: - LOG.error('Could not delete Plex playlist: %s', err.strerror) + except PL.PlaylistError: + pass else: update_plex_table(playlist, delete=True) -def create_kodi_playlist(plex_id): +def create_kodi_playlist(plex_id=None): """ Creates a new Kodi playlist file. Will also add (or modify an existing) Plex playlist table entry. Assumes that the Plex playlist is indeed new. A NEW Kodi playlist will be - created in any case (not replaced) + created in any case (not replaced). Thus make sure that the "same" playlist + is deleted from both disk and the Plex database. Returns the playlist or raises PL.PlaylistError """ - LOG.info('Creating new Kodi playlist from Plex playlist %s', plex_id) - playlist = PL.Playlist_Object() - playlist.id = plex_id - xml = PL.get_PMS_playlist(playlist) + xml = PL.get_PMS_playlist(playlist_id=plex_id) if not xml: - LOG.error('Could not create Kodi playlist for %s', plex_id) + LOG.error('Could not get Plex playlist %s', plex_id) return - PL.get_playlist_details_from_xml(playlist, xml) - if xml.get('playlistType') == 'audio': - playlist.type = 'music' - elif xml.get('playlistType') == 'video': - playlist.type = 'video' - else: - raise RuntimeError('Plex playlist type unknown: %s' - % xml.get('playlistType')) - playlist.plex_name = xml.get('title') - name = utils.slugify(playlist.plex_name) - playlist.kodi_path = os.join(v.PLAYLIST_PATH, - playlist.type, - '%s.m3u8' % name) + api = API(xml) + playlist = PL.Playlist_Object() + playlist.id = api.plex_id() + playlist.type = v.KODI_PLAYLIST_TYPE_FROM_PLEX[api.playlist_type()] + playlist.plex_name = api.title() + playlist.plex_updatedat = api.updated_at() + LOG.info('Creating new Kodi playlist from Plex playlist %s: %s', + playlist.id, playlist.plex_name) + name = utils.valid_filename(playlist.plex_name) + path = os.join(v.PLAYLIST_PATH, playlist.type, '%s.m3u8' % name) + while exists(path) or playlist_object_from_db(path=path): + # In case the Plex playlist names are not unique + occurance = utils.REGEX_FILE_NUMBERING.search(path) + if not occurance: + path = os.join(v.PLAYLIST_PATH, + playlist.type, + '%s_01.m3u8' % name[:min(len(name), 247)]) + else: + occurance = int(occurance.group(1)) + 1 + path = os.join(v.PLAYLIST_PATH, + playlist.type, + '%s_%02d.m3u8' % (name[:min(len(name), 247)], + occurance)) + LOG.debug('Kodi playlist path: %s', path) + playlist.kodi_path = path # Derive filename close to Plex playlist name _write_playlist_to_file(playlist, xml) update_plex_table(playlist, update_kodi_hash=True) @@ -114,9 +124,10 @@ def delete_kodi_playlist(playlist): """ try: os.remove(playlist.kodi_path) - except OSError as err: + except (OSError, IOError) as err: LOG.error('Could not delete Kodi playlist file %s. Error:\n %s: %s', playlist.kodi_path, err.errno, err.strerror) + raise PL.PlaylistError('Could not delete %s' % playlist.kodi_path) else: update_plex_table(playlist, delete=True) @@ -181,7 +192,7 @@ def m3u_to_plex_ids(playlist): playlist.kodi_path) text = text.decode('ISO-8859-1') for entry in _m3u_iterator(text): - plex_id = REGEX_PLEX_ID.search(entry) + plex_id = utils.REGEX_PLEX_ID.search(entry) if plex_id: plex_id = plex_id.group(1) plex_ids.append(plex_id) @@ -210,8 +221,14 @@ def _write_playlist_to_file(playlist, xml): % (api.runtime(), api.title(), api.path())) text += '\n' text = text.encode('utf-8') - with open(playlist.kodi_path, 'wb') as f: - f.write(text) + try: + with open(playlist.kodi_path, 'wb') as f: + f.write(text) + except (OSError, IOError) as err: + LOG.error('Could not write Kodi playlist file: %s', playlist.kodi_path) + LOG.error('Error message %s: %s', err.errno, err.strerror) + raise PL.PlaylistError('Cannot write Kodi playlist to path %s' + % playlist.kodi_path) def change_plex_playlist_name(playlist, new_name): @@ -233,24 +250,118 @@ def plex_id_from_playlist_path(path): return plex_id -def playlist_object_from_db(path=None): +def playlist_object_from_db(path=None, kodi_hash=None, plex_id=None): """ - Returns the playlist as a Playlist_Object for path [unicode] from the Plex - playlists table or None if not found. + Returns the playlist as a Playlist_Object for either the plex_id, path or + kodi_hash. kodi_hash will be more reliable as it includes path and file + content. """ playlist = PL.Playlist_Object() + with plexdb.Get_Plex_DB() as plex_db: + playlist = plex_db.retrieve_playlist(playlist, plex_id, path, kodi_hash) return playlist +def _kodi_playlist_identical(xml_element): + """ + Feed with one playlist xml element from the PMS. Will return True if PKC + already synced this playlist, False if not or if the Play playlist has + changed in the meantime + """ + pass + + def full_sync(): """ Full sync of playlists between Kodi and Plex. Returns True is successful, False otherwise """ + LOG.info('Starting playlist full sync') + # Get all Plex playlists xml = PL.get_all_playlists() if not xml: return False - for entry in xml: + # For each playlist, check Plex database to see whether we already synced + # before. If yes, make sure that hashes are identical. If not, sync it. + with plexdb.Get_Plex_DB() as plex_db: + old_plex_ids = plex_db.plex_ids_all_playlists() + for xml_playlist in xml: + api = API(xml_playlist) + if (not state.ENABLE_MUSIC and + api.playlist_type() == v.PLEX_TYPE_AUDIO_PLAYLIST): + continue + playlist = playlist_object_from_db(plex_id=api.plex_id()) + try: + if not playlist: + LOG.debug('New Plex playlist %s discovered: %s', + api.plex_id(), api.title()) + create_kodi_playlist(api.plex_id()) + continue + elif playlist.plex_updatedat != api.updated_at(): + LOG.debug('Detected changed Plex playlist %s: %s', + api.plex_id(), api.title()) + delete_kodi_playlist(playlist) + create_kodi_playlist(api.plex_id()) + except PL.PlaylistError: + LOG.info('Skipping playlist %s: %s', api.plex_id(), api.title()) + old_plex_ids.remove(api.plex_id()) + # Get rid of old Plex playlists that were deleted on the Plex side + for plex_id in old_plex_ids: + playlist = playlist_object_from_db(plex_id=api.plex_id()) + if playlist: + LOG.debug('Removing outdated Plex playlist %s from %s', + playlist.plex_name, playlist.kodi_path) + try: + delete_kodi_playlist(playlist) + except PL.PlaylistError: + pass + # Look at all supported Kodi playlists. Check whether they are in the DB. + with plexdb.Get_Plex_DB() as plex_db: + old_kodi_hashes = plex_db.kodi_hashes_all_playlists() + master_paths = [v.PLAYLIST_PATH_VIDEO] + if state.ENABLE_MUSIC: + master_paths.append(v.PLAYLIST_PATH_MUSIC) + for master_path in master_paths: + for root, _, files in os.walk(master_path): + for file in files: + try: + extension = file.rsplit('.', 1)[1] + except IndexError: + continue + if extension not in SUPPORTED_FILETYPES: + continue + path = os.path.join(root, file) + kodi_hash = utils.generate_file_md5(path) + playlist = playlist_object_from_db(kodi_hash=kodi_hash) + playlist_2 = playlist_object_from_db(path=path) + if playlist: + # Nothing changed at all - neither path nor content + old_kodi_hashes.remove(kodi_hash) + continue + try: + if playlist_2: + LOG.debug('Changed Kodi playlist %s detected: %s', + playlist_2.plex_name, path) + playlist = PL.Playlist_Object() + playlist.id = playlist_2.id + playlist.kodi_path = path + playlist.plex_name = playlist_2.plex_name + delete_plex_playlist(playlist_2) + create_plex_playlist(playlist) + else: + LOG.debug('New Kodi playlist detected: %s', path) + # Make sure that we delete any playlist with other hash + create_plex_playlist(path=path) + except PL.PlaylistError: + LOG.info('Skipping Kodi playlist %s', path) + for kodi_hash in old_kodi_hashes: + playlist = playlist_object_from_db(kodi_hash=kodi_hash) + if playlist: + try: + delete_plex_playlist(playlist) + except PL.PlaylistError: + pass + return True class PlaylistEventhandler(watchdog.events.FileSystemEventHandler): diff --git a/resources/lib/plexdb_functions.py b/resources/lib/plexdb_functions.py index 9d1f8727..b31ef8a6 100644 --- a/resources/lib/plexdb_functions.py +++ b/resources/lib/plexdb_functions.py @@ -406,6 +406,60 @@ class Plex_DB_Functions(): plex_id = None return plex_id + def plex_ids_all_playlists(self): + """ + Returns a list of all Plex ids of playlists. + """ + answ = [] + self.plexcursor.execute('SELECT plex_id FROM playlists') + for entry in self.plexcursor.fetchall(): + answ.append(entry[0]) + return answ + + def kodi_hashes_all_playlists(self): + """ + Returns a list of all Kodi hashes of playlists. + """ + answ = [] + self.plexcursor.execute('SELECT kodi_hash FROM playlists') + for entry in self.plexcursor.fetchall(): + answ.append(entry[0]) + return answ + + def retrieve_playlist(self, playlist, plex_id=None, path=None, + kodi_hash=None): + """ + Returns a complete Playlist_Object (empty one passed in via playlist) + for the entry with plex_id. Or None if not found + """ + query = ''' + SELECT plex_id, plex_name, plex_updatedat, kodi_path, kodi_type, + kodi_hash + FROM playlists + WHERE %s = ? + LIMIT 1 + ''' + if plex_id: + query = query % 'plex_id' + var = plex_id + elif kodi_hash: + query = query % 'kodi_hash' + var = kodi_hash + else: + query = query % 'kodi_path' + var = path + self.plexcursor.execute(query, (var, )) + answ = self.plexcursor.fetchone() + if not answ: + return + playlist.plex_id = answ[0] + playlist.plex_name = answ[1] + playlist.plex_updatedat = answ[2] + playlist.kodi_path = answ[3] + playlist.kodi_type = answ[4] + playlist.kodi_hash = answ[5] + return playlist + def insert_playlist_entry(self, playlist): """ Inserts or modifies an existing entry in the Plex playlists table. diff --git a/resources/lib/utils.py b/resources/lib/utils.py index 94475f32..1d9fc928 100644 --- a/resources/lib/utils.py +++ b/resources/lib/utils.py @@ -4,6 +4,7 @@ Various functions and decorators for PKC """ ############################################################################### from logging import getLogger +import os from cProfile import Profile from pstats import Stats from sqlite3 import connect, OperationalError @@ -13,11 +14,11 @@ from time import localtime, strftime from unicodedata import normalize import xml.etree.ElementTree as etree from functools import wraps, partial -from os.path import join -from os import remove, walk, makedirs from shutil import rmtree from urllib import quote_plus import hashlib +import re +import unicodedata import xbmc import xbmcaddon @@ -35,6 +36,10 @@ WINDOW = xbmcgui.Window(10000) ADDON = xbmcaddon.Addon(id='plugin.video.plexkodiconnect') EPOCH = datetime.utcfromtimestamp(0) +REGEX_PLEX_ID = re.compile(r'''plex_id=(\d+)''') +REGEX_FILE_NUMBERING = re.compile(r'''_(\d+)\.\w+$''') + + ############################################################################### # Main methods @@ -111,7 +116,7 @@ def exists_dir(path): if v.KODIVERSION >= 17: answ = exists(try_encode(path)) else: - dummyfile = join(try_decode(path), 'dummyfile.txt') + dummyfile = os.path.join(try_decode(path), 'dummyfile.txt') try: with open(dummyfile, 'w') as filer: filer.write('text') @@ -285,6 +290,39 @@ def slugify(text): return unicode(normalize('NFKD', text).encode('ascii', 'ignore')) +def valid_filename(text): + """ + Return a valid filename after passing it in [unicode]. + """ + # Get rid of all whitespace except a normal space + text = re.sub(r'(?! )\s', '', text) + # ASCII characters 0 to 31 (non-printable, just in case) + text = re.sub(u'[\x00-\x1f]', '', text) + if v.PLATFORM == 'Windows': + # Whitespace at the end of the filename is illegal + text = text.strip() + # Dot at the end of a filename is illegal + text = re.sub(r'\.+$', '', text) + # Illegal Windows characters + text = re.sub(r'[/\\:*?"<>|\^]', '', text) + elif v.PLATFORM == 'MacOSX': + # Colon is illegal + text = re.sub(r':', '', text) + # Files cannot begin with a dot + text = re.sub(r'^\.+', '', text) + else: + # Linux + text = re.sub(r'/', '', text) + if not os.path.supports_unicode_filenames: + text = unicodedata.normalize('NFKD', text) + text = text.encode('ascii', 'ignore') + text = text.decode('ascii') + # Ensure that filename length is at most 255 chars (including 4 chars for + # filename extension and 1 dot to separate the extension) + text = text[:min(len(text), 250)] + return text + + def escape_html(string): """ Escapes the following: @@ -478,7 +516,7 @@ def reset(): addon = xbmcaddon.Addon() addondir = try_decode(xbmc.translatePath(addon.getAddonInfo('profile'))) LOG.info("Deleting: settings.xml") - remove("%ssettings.xml" % addondir) + os.remove("%ssettings.xml" % addondir) reboot_kodi() @@ -630,9 +668,9 @@ class XmlKodiSetting(object): top_element=None): self.filename = filename if path is None: - self.path = join(v.KODI_PROFILE, filename) + self.path = os.path.join(v.KODI_PROFILE, filename) else: - self.path = join(path, filename) + self.path = os.path.join(path, filename) self.force_create = force_create self.top_element = top_element self.tree = None @@ -929,13 +967,13 @@ def playlist_xsp(mediatype, tagname, viewid, viewtype="", delete=False): # Create the playlist directory if not exists(try_encode(path)): LOG.info("Creating directory: %s", path) - makedirs(path) + os.makedirs(path) # Only add the playlist if it doesn't already exists if exists(try_encode(xsppath)): LOG.info('Path %s does exist', xsppath) if delete: - remove(xsppath) + os.remove(xsppath) LOG.info("Successfully removed playlist: %s.", tagname) return @@ -966,29 +1004,32 @@ def delete_playlists(): Clean up the playlists """ path = try_decode(xbmc.translatePath("special://profile/playlists/video/")) - for root, _, files in walk(path): + for root, _, files in os.walk(path): for file in files: if file.startswith('Plex'): - remove(join(root, file)) + os.remove(os.path.join(root, file)) def delete_nodes(): """ Clean up video nodes """ path = try_decode(xbmc.translatePath("special://profile/library/video/")) - for root, dirs, _ in walk(path): + for root, dirs, _ in os.walk(path): for directory in dirs: if directory.startswith('Plex-'): - rmtree(join(root, directory)) + rmtree(os.path.join(root, directory)) break def generate_file_md5(path): """ - Generates the md5 hash value for the file located at path [unicode] + Generates the md5 hash value for the file located at path [unicode]. + The hash includes the path and is thus different for the same file for + different filenames. Returns a unique string containing only hexadecimal digits """ m = hashlib.md5() + m.update(path.encode('utf-8')) with open(path, 'rb') as f: while True: piece = f.read(32768) diff --git a/resources/lib/variables.py b/resources/lib/variables.py index 7de618b8..0f89ef08 100644 --- a/resources/lib/variables.py +++ b/resources/lib/variables.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from os.path import join +import os import xbmc from xbmcaddon import Addon @@ -41,10 +41,6 @@ KODILANGUAGE = xbmc.getLanguage(xbmc.ISO_639_1) KODIVERSION = int(xbmc.getInfoLabel("System.BuildVersion")[:2]) KODILONGVERSION = xbmc.getInfoLabel('System.BuildVersion') KODI_PROFILE = try_decode(xbmc.translatePath("special://profile")) -PLAYLIST_PATH = join(KODI_PROFILE, 'playlist') -PLAYLIST_PATH_MIXED = join(PLAYLIST_PATH, 'mixed') -PLAYLIST_PATH_VIDEO = join(PLAYLIST_PATH, 'video') -PLAYLIST_PATH_MUSIC = join(PLAYLIST_PATH, 'music') if xbmc.getCondVisibility('system.platform.osx'): PLATFORM = "MacOSX" @@ -127,6 +123,21 @@ EXTERNAL_SUBTITLE_TEMP_PATH = try_decode(xbmc.translatePath( # Multiply Plex time by this factor to receive Kodi time PLEX_TO_KODI_TIMEFACTOR = 1.0 / 1000.0 +# Playlist stuff +PLAYLIST_PATH = os.path.join(KODI_PROFILE, 'playlist') +PLAYLIST_PATH_MIXED = os.path.join(PLAYLIST_PATH, 'mixed') +PLAYLIST_PATH_VIDEO = os.path.join(PLAYLIST_PATH, 'video') +PLAYLIST_PATH_MUSIC = os.path.join(PLAYLIST_PATH, 'music') + +PLEX_TYPE_AUDIO_PLAYLIST = 'audio' +PLEX_TYPE_VIDEO_PLAYLIST = 'video' +KODI_TYPE_AUDIO_PLAYLIST = 'music' +KODI_TYPE_VIDEO_PLAYLIST = 'video' +KODI_PLAYLIST_TYPE_FROM_PLEX = { + PLEX_TYPE_AUDIO_PLAYLIST: KODI_TYPE_AUDIO_PLAYLIST, + PLEX_TYPE_VIDEO_PLAYLIST: KODI_TYPE_VIDEO_PLAYLIST +} + # All the Plex types as communicated in the PMS xml replies PLEX_TYPE_VIDEO = 'video'