diff --git a/README.md b/README.md index 285114e0..60b05038 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ -[![stable version](https://img.shields.io/badge/stable_version-2.10.2-blue.svg?maxAge=60&style=flat) ](https://github.com/croneter/binary_repo/raw/master/stable/repository.plexkodiconnect/repository.plexkodiconnect-1.0.2.zip) -[![beta version](https://img.shields.io/badge/beta_version-2.10.2-red.svg?maxAge=60&style=flat) ](https://github.com/croneter/binary_repo/raw/master/beta/repository.plexkodiconnectbeta/repository.plexkodiconnectbeta-1.0.2.zip) +[![stable version](https://img.shields.io/badge/stable_version-2.10.4-blue.svg?maxAge=60&style=flat) ](https://github.com/croneter/binary_repo/raw/master/stable/repository.plexkodiconnect/repository.plexkodiconnect-1.0.2.zip) +[![beta version](https://img.shields.io/badge/beta_version-2.10.4-red.svg?maxAge=60&style=flat) ](https://github.com/croneter/binary_repo/raw/master/beta/repository.plexkodiconnectbeta/repository.plexkodiconnectbeta-1.0.2.zip) [![Installation](https://img.shields.io/badge/wiki-installation-brightgreen.svg?maxAge=60&style=flat)](https://github.com/croneter/PlexKodiConnect/wiki/Installation) [![FAQ](https://img.shields.io/badge/wiki-FAQ-brightgreen.svg?maxAge=60&style=flat)](https://github.com/croneter/PlexKodiConnect/wiki/faq) diff --git a/addon.xml b/addon.xml index 5106f555..11c552df 100644 --- a/addon.xml +++ b/addon.xml @@ -1,5 +1,5 @@ - + @@ -83,7 +83,20 @@ Natūralioji „Plex“ integracija į „Kodi“ Prijunkite „Kodi“ prie „Plex Medija Serverio“. Šiame papildinyje daroma prielaida, kad valdote visus savo vaizdo įrašus naudodami „Plex“ (ir nė vieno su „Kodi“). Galite prarasti jau saugomus „Kodi“ vaizdo įrašų ir muzikos duomenų bazių duomenis (kadangi šis papildinys juos tiesiogiai pakeičia). Naudokite savo pačių rizika! Naudokite savo pačių rizika - version 2.10.2: + version 2.10.4: +- version 2.10.3 for everyone +- Fix to correctly wipe Kodi databases + +version 2.10.3 (beta only): +- Fix a couple of issues with music when using direct paths: correctly escape music paths for Kodi regex matching +- Fix Recently Added Albums sort order (you will have to reset the Kodi database manually) +- Fix database being locked in rare cases +- Increase batch size for library sync from 500 to 2000 to increase sync speed +- Optimize some code +- Fix KeyError when using Plex search capabilities +- Check faster for available Plex Media Server to connect to + +version 2.10.2: - Fix Kodi playback jumping to the beginning of a video that just started - Fix transcoding quality degenerating quickly while playing with a new setting to deactivate auto quality for transcoding (applicable e.g. for Chromecast) - Update translations diff --git a/changelog.txt b/changelog.txt index ac015308..07ee3840 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,16 @@ +version 2.10.4: +- version 2.10.3 for everyone +- Fix to correctly wipe Kodi databases + +version 2.10.3 (beta only): +- Fix a couple of issues with music when using direct paths: correctly escape music paths for Kodi regex matching +- Fix Recently Added Albums sort order (you will have to reset the Kodi database manually) +- Fix database being locked in rare cases +- Increase batch size for library sync from 500 to 2000 to increase sync speed +- Optimize some code +- Fix KeyError when using Plex search capabilities +- Check faster for available Plex Media Server to connect to + version 2.10.2: - Fix Kodi playback jumping to the beginning of a video that just started - Fix transcoding quality degenerating quickly while playing with a new setting to deactivate auto quality for transcoding (applicable e.g. for Chromecast) diff --git a/resources/lib/backgroundthread.py b/resources/lib/backgroundthread.py index 80f099a0..7823fffd 100644 --- a/resources/lib/backgroundthread.py +++ b/resources/lib/backgroundthread.py @@ -140,6 +140,67 @@ class KillableThread(threading.Thread): return self._suspended +class OrderedQueue(Queue.PriorityQueue, object): + """ + Queue that enforces an order on the items it returns. An item you push + onto the queue must be a tuple + (index, item) + where index=-1 is the item that will be returned first. The Queue will block + until index=-1, 0, 1, 2, 3, ... is then made available + """ + def __init__(self, maxsize=0): + super(OrderedQueue, self).__init__(maxsize) + self.smallest = -1 + self.not_next_item = threading.Condition(self.mutex) + + def _put(self, item, heappush=heapq.heappush): + heappush(self.queue, item) + if item[0] == self.smallest: + self.not_next_item.notify() + + def get(self, block=True, timeout=None): + """Remove and return an item from the queue. + + If optional args 'block' is true and 'timeout' is None (the default), + block if necessary until an item is available. If 'timeout' is + a non-negative number, it blocks at most 'timeout' seconds and raises + the Empty exception if no item was available within that time. + Otherwise ('block' is false), return an item if one is immediately + available, else raise the Empty exception ('timeout' is ignored + in that case). + """ + self.not_empty.acquire() + try: + if not block: + if not self._qsize() or self.queue[0][0] != self.smallest: + raise Queue.Empty + elif timeout is None: + while not self._qsize(): + self.not_empty.wait() + while self.queue[0][0] != self.smallest: + self.not_next_item.wait() + elif timeout < 0: + raise ValueError("'timeout' must be a non-negative number") + else: + endtime = Queue._time() + timeout + while not self._qsize(): + remaining = endtime - Queue._time() + if remaining <= 0.0: + raise Queue.Empty + self.not_empty.wait(remaining) + while self.queue[0][0] != self.smallest: + remaining = endtime - Queue._time() + if remaining <= 0.0: + raise Queue.Empty + self.not_next_item.wait(remaining) + item = self._get() + self.smallest += 1 + self.not_full.notify() + return item + finally: + self.not_empty.release() + + class Tasks(list): def add(self, task): for t in self: diff --git a/resources/lib/db.py b/resources/lib/db.py new file mode 100644 index 00000000..30f91315 --- /dev/null +++ b/resources/lib/db.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +import sqlite3 +from functools import wraps + +from . import variables as v, app + +DB_WRITE_ATTEMPTS = 100 + + +class LockedDatabase(Exception): + """ + Dedicated class to make sure we're not silently catching locked DBs. + """ + pass + + +def catch_operationalerrors(method): + """ + sqlite.OperationalError is raised immediately if another DB connection + is open, reading something that we're trying to change + + So let's catch it and try again + + Also see https://github.com/mattn/go-sqlite3/issues/274 + """ + @wraps(method) + def wrapper(self, *args, **kwargs): + attempts = DB_WRITE_ATTEMPTS + while True: + try: + return method(self, *args, **kwargs) + except sqlite3.OperationalError as err: + if 'database is locked' not in err: + # Not an error we want to catch, so reraise it + raise + attempts -= 1 + if attempts == 0: + # Reraise in order to NOT catch nested OperationalErrors + raise LockedDatabase('Database is locked') + # Need to close the transactions and begin new ones + self.kodiconn.commit() + if self.artconn: + self.artconn.commit() + if app.APP.monitor.waitForAbort(0.1): + # PKC needs to quit + return + # Start new transactions + self.kodiconn.execute('BEGIN') + if self.artconn: + self.artconn.execute('BEGIN') + return wrapper + + +def _initial_db_connection_setup(conn, wal_mode): + """ + Set-up DB e.g. for WAL journal mode, if that hasn't already been done + before. Also start a transaction + """ + if wal_mode: + conn.execute('PRAGMA journal_mode=WAL;') + conn.execute('PRAGMA cache_size = -8000;') + conn.execute('PRAGMA synchronous=NORMAL;') + conn.execute('BEGIN') + + +def connect(media_type=None, wal_mode=True): + """ + Open a connection to the Kodi database. + media_type: 'video' (standard if not passed), 'plex', 'music', 'texture' + Pass wal_mode=False if you want the standard (and slower) sqlite + journal_mode, e.g. when wiping entire tables. Useful if you do NOT want + concurrent access to DB for both PKC and Kodi + """ + if media_type == "plex": + db_path = v.DB_PLEX_PATH + elif media_type == "music": + db_path = v.DB_MUSIC_PATH + elif media_type == "texture": + db_path = v.DB_TEXTURE_PATH + else: + db_path = v.DB_VIDEO_PATH + conn = sqlite3.connect(db_path, timeout=30.0) + attempts = DB_WRITE_ATTEMPTS + while True: + try: + _initial_db_connection_setup(conn, wal_mode) + except sqlite3.OperationalError as err: + if 'database is locked' not in err: + # Not an error we want to catch, so reraise it + raise + attempts -= 1 + if attempts == 0: + # Reraise in order to NOT catch nested OperationalErrors + raise LockedDatabase('Database is locked') + if app.APP.monitor.waitForAbort(0.05): + # PKC needs to quit + return + else: + break + return conn diff --git a/resources/lib/itemtypes/common.py b/resources/lib/itemtypes/common.py index 8c4f5d18..3188a112 100644 --- a/resources/lib/itemtypes/common.py +++ b/resources/lib/itemtypes/common.py @@ -6,7 +6,7 @@ from ntpath import dirname from ..plex_db import PlexDB, PLEXDB_LOCK from ..kodi_db import KodiVideoDB, KODIDB_LOCK -from .. import utils, timing, app +from .. import db, timing, app LOG = getLogger('PLEX.itemtypes.common') @@ -57,11 +57,11 @@ class ItemBase(object): if self.lock: PLEXDB_LOCK.acquire() KODIDB_LOCK.acquire() - self.plexconn = utils.kodi_sql('plex') + self.plexconn = db.connect('plex') self.plexcursor = self.plexconn.cursor() - self.kodiconn = utils.kodi_sql('video') + self.kodiconn = db.connect('video') self.kodicursor = self.kodiconn.cursor() - self.artconn = utils.kodi_sql('texture') + self.artconn = db.connect('texture') self.artcursor = self.artconn.cursor() self.plexdb = PlexDB(plexconn=self.plexconn, lock=False) self.kodidb = KodiVideoDB(texture_db=True, diff --git a/resources/lib/itemtypes/music.py b/resources/lib/itemtypes/music.py index f747ea84..61b16003 100644 --- a/resources/lib/itemtypes/music.py +++ b/resources/lib/itemtypes/music.py @@ -7,7 +7,7 @@ from .common import ItemBase from ..plex_api import API from ..plex_db import PlexDB, PLEXDB_LOCK from ..kodi_db import KodiMusicDB, KODIDB_LOCK -from .. import plex_functions as PF, utils, timing, app, variables as v +from .. import plex_functions as PF, db, timing, app, variables as v LOG = getLogger('PLEX.music') @@ -20,11 +20,11 @@ class MusicMixin(object): if self.lock: PLEXDB_LOCK.acquire() KODIDB_LOCK.acquire() - self.plexconn = utils.kodi_sql('plex') + self.plexconn = db.connect('plex') self.plexcursor = self.plexconn.cursor() - self.kodiconn = utils.kodi_sql('music') + self.kodiconn = db.connect('music') self.kodicursor = self.kodiconn.cursor() - self.artconn = utils.kodi_sql('texture') + self.artconn = db.connect('texture') self.artcursor = self.artconn.cursor() self.plexdb = PlexDB(plexconn=self.plexconn, lock=False) self.kodidb = KodiMusicDB(texture_db=True, diff --git a/resources/lib/kodi_db/__init__.py b/resources/lib/kodi_db/__init__.py index 052f7827..200d83ed 100644 --- a/resources/lib/kodi_db/__init__.py +++ b/resources/lib/kodi_db/__init__.py @@ -8,7 +8,7 @@ from .video import KodiVideoDB from .music import KodiMusicDB from .texture import KodiTextureDB -from .. import path_ops, utils, timing, variables as v +from .. import path_ops, utils, variables as v LOG = getLogger('PLEX.kodi_db') @@ -56,35 +56,15 @@ def setup_kodi_default_entries(): """ if utils.settings('enableMusic') == 'true': with KodiMusicDB() as kodidb: - kodidb.cursor.execute(''' - INSERT OR REPLACE INTO artist( - idArtist, - strArtist, - strMusicBrainzArtistID) - VALUES (?, ?, ?) - ''', (1, '[Missing Tag]', 'Artist Tag Missing')) - kodidb.cursor.execute(''' - INSERT OR REPLACE INTO role( - idRole, - strRole) - VALUES (?, ?) - ''', (1, 'Artist')) - if v.KODIVERSION >= 18: - kodidb.cursor.execute('DELETE FROM versiontagscan') - kodidb.cursor.execute(''' - INSERT INTO versiontagscan( - idVersion, - iNeedsScan, - lastscanned) - VALUES (?, ?, ?) - ''', (v.DB_MUSIC_VERSION, - 0, - timing.kodi_now())) + kodidb.setup_kodi_default_entries() def reset_cached_images(): LOG.info('Resetting cached artwork') - # Remove all existing textures first + LOG.debug('Resetting the Kodi texture DB') + with KodiTextureDB(wal_mode=False) as kodidb: + kodidb.wipe() + LOG.debug('Deleting all cached image files') path = path_ops.translate_path('special://thumbnails/') if path_ops.exists(path): path_ops.rmtree(path, ignore_errors=True) @@ -95,13 +75,10 @@ def reset_cached_images(): new_path = path_ops.translate_path('special://thumbnails/%s' % path) try: path_ops.makedirs(path_ops.encode_path(new_path)) - except OSError: - pass - with KodiTextureDB() as kodidb: - for row in kodidb.cursor.execute('SELECT tbl_name FROM sqlite_master WHERE type=?', - ('table', )): - if row[0] != 'version': - kodidb.cursor.execute("DELETE FROM %s" % row[0]) + except OSError as err: + LOG.warn('Could not create thumbnail directory %s: %s', + new_path, err) + LOG.info('Done resetting cached artwork') def wipe_dbs(music=True): @@ -109,28 +86,18 @@ def wipe_dbs(music=True): Completely resets the Kodi databases 'video', 'texture' and 'music' (if music sync is enabled) - DO NOT use context menu as we need to connect without WAL mode - if Kodi - is still accessing the DB + We need to connect without sqlite WAL mode as Kodi might still be accessing + the dbs and we need to prevent that """ - from sqlite3 import connect LOG.warn('Wiping Kodi databases!') - kinds = [v.DB_VIDEO_PATH, v.DB_TEXTURE_PATH] + LOG.info('Wiping Kodi video database') + with KodiVideoDB(wal_mode=False) as kodidb: + kodidb.wipe() if music: - kinds.append(v.DB_MUSIC_PATH) - for path in kinds: - conn = connect(path, timeout=30.0) - cursor = conn.cursor() - cursor.execute("SELECT name FROM sqlite_master WHERE type = 'table'") - tables = cursor.fetchall() - tables = [i[0] for i in tables] - if 'version' in tables: - tables.remove('version') - if 'versiontagscan' in tables: - tables.remove('versiontagscan') - for table in tables: - cursor.execute('DELETE FROM %s' % table) - conn.commit() - conn.close() + LOG.info('Wiping Kodi music database') + with KodiMusicDB(wal_mode=False) as kodidb: + kodidb.wipe() + reset_cached_images() setup_kodi_default_entries() # Delete SQLITE wal files import xbmc @@ -140,6 +107,14 @@ def wipe_dbs(music=True): xbmc.executebuiltin('UpdateLibrary(music)') +def create_kodi_db_indicees(): + """ + Index the "actors" because we got a TON - speed up SELECT and WHEN + """ + with KodiVideoDB() as kodidb: + kodidb.create_kodi_db_indicees() + + KODIDB_FROM_PLEXTYPE = { v.PLEX_TYPE_MOVIE: KodiVideoDB, v.PLEX_TYPE_SHOW: KodiVideoDB, diff --git a/resources/lib/kodi_db/common.py b/resources/lib/kodi_db/common.py index 510d8559..2a331290 100644 --- a/resources/lib/kodi_db/common.py +++ b/resources/lib/kodi_db/common.py @@ -2,65 +2,24 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import, division, unicode_literals from threading import Lock -from functools import wraps -from .. import utils, path_ops, app +from .. import db, path_ops KODIDB_LOCK = Lock() -DB_WRITE_ATTEMPTS = 100 - - -class LockedKodiDatabase(Exception): - """ - Dedicated class to make sure we're not silently catching locked DBs. - """ - pass - - -def catch_operationalerrors(method): - """ - sqlite.OperationalError is raised immediately if another DB connection - is open, reading something that we're trying to change - - So let's catch it and try again - - Also see https://github.com/mattn/go-sqlite3/issues/274 - """ - @wraps(method) - def wrapper(self, *args, **kwargs): - attempts = DB_WRITE_ATTEMPTS - while True: - try: - return method(self, *args, **kwargs) - except utils.OperationalError as err: - if 'database is locked' not in err: - # Not an error we want to catch, so reraise it - raise - attempts -= 1 - if attempts == 0: - # Reraise in order to NOT catch nested OperationalErrors - raise LockedKodiDatabase('Kodi database locked') - # Need to close the transactions and begin new ones - self.kodiconn.commit() - if self.artconn: - self.artconn.commit() - if app.APP.monitor.waitForAbort(0.1): - # PKC needs to quit - return - # Start new transactions - self.kodiconn.execute('BEGIN') - if self.artconn: - self.artconn.execute('BEGIN') - return wrapper +# Names of tables we generally leave untouched and e.g. don't wipe +UNTOUCHED_TABLES = ('version', 'versiontagscan') class KodiDBBase(object): """ Kodi database methods used for all types of items """ - def __init__(self, texture_db=False, kodiconn=None, artconn=None, lock=True): + def __init__(self, texture_db=False, kodiconn=None, artconn=None, + lock=True, wal_mode=True): """ Allows direct use with a cursor instead of context mgr + Pass wal_mode=False if you want the standard sqlite journal_mode, e.g. + when wiping entire tables """ self._texture_db = texture_db self.lock = lock @@ -68,13 +27,15 @@ class KodiDBBase(object): self.cursor = self.kodiconn.cursor() if self.kodiconn else None self.artconn = artconn self.artcursor = self.artconn.cursor() if self.artconn else None + self.wal_mode = wal_mode def __enter__(self): if self.lock: KODIDB_LOCK.acquire() - self.kodiconn = utils.kodi_sql(self.db_kind) + self.kodiconn = db.connect(self.db_kind, self.wal_mode) self.cursor = self.kodiconn.cursor() - self.artconn = utils.kodi_sql('texture') if self._texture_db else None + self.artconn = db.connect('texture', self.wal_mode) if self._texture_db \ + else None self.artcursor = self.artconn.cursor() if self._texture_db else None return self @@ -110,7 +71,7 @@ class KodiDBBase(object): for kodi_art, url in artworks.iteritems(): self.add_art(url, kodi_id, kodi_type, kodi_art) - @catch_operationalerrors + @db.catch_operationalerrors def add_art(self, url, kodi_id, kodi_type, kodi_art): """ Adds or modifies the artwork of kind kodi_art (e.g. 'poster') in the @@ -129,7 +90,7 @@ class KodiDBBase(object): for kodi_art, url in artworks.iteritems(): self.modify_art(url, kodi_id, kodi_type, kodi_art) - @catch_operationalerrors + @db.catch_operationalerrors def modify_art(self, url, kodi_id, kodi_type, kodi_art): """ Adds or modifies the artwork of kind kodi_art (e.g. 'poster') in the @@ -166,7 +127,7 @@ class KodiDBBase(object): for row in self.cursor.fetchall(): self.delete_cached_artwork(row[0]) - @catch_operationalerrors + @db.catch_operationalerrors def delete_cached_artwork(self, url): try: self.artcursor.execute("SELECT cachedurl FROM texture WHERE url = ? LIMIT 1", @@ -182,3 +143,16 @@ class KodiDBBase(object): if path_ops.exists(path): path_ops.rmtree(path, ignore_errors=True) self.artcursor.execute("DELETE FROM texture WHERE url = ?", (url, )) + + @db.catch_operationalerrors + def wipe(self): + """ + Completely wipes the corresponding Kodi database + """ + self.cursor.execute("SELECT name FROM sqlite_master WHERE type = 'table'") + tables = [i[0] for i in self.cursor.fetchall()] + for table in UNTOUCHED_TABLES: + if table in tables: + tables.remove(table) + for table in tables: + self.cursor.execute('DELETE FROM %s' % table) diff --git a/resources/lib/kodi_db/music.py b/resources/lib/kodi_db/music.py index 37f5c4a9..77ede150 100644 --- a/resources/lib/kodi_db/music.py +++ b/resources/lib/kodi_db/music.py @@ -4,7 +4,7 @@ from __future__ import absolute_import, division, unicode_literals from logging import getLogger from . import common -from .. import variables as v, app +from .. import db, variables as v, app, timing LOG = getLogger('PLEX.kodi_db.music') @@ -12,7 +12,7 @@ LOG = getLogger('PLEX.kodi_db.music') class KodiMusicDB(common.KodiDBBase): db_kind = 'music' - @common.catch_operationalerrors + @db.catch_operationalerrors def add_path(self, path): """ Add the path (unicode) to the music DB, if it does not exist already. @@ -34,7 +34,38 @@ class KodiMusicDB(common.KodiDBBase): (pathid, path, '123')) return pathid - @common.catch_operationalerrors + @db.catch_operationalerrors + def setup_kodi_default_entries(self): + """ + Makes sure that we retain the Kodi standard databases. E.g. that there + is a dummy artist with ID 1 + """ + self.cursor.execute(''' + INSERT OR REPLACE INTO artist( + idArtist, + strArtist, + strMusicBrainzArtistID) + VALUES (?, ?, ?) + ''', (1, '[Missing Tag]', 'Artist Tag Missing')) + self.cursor.execute(''' + INSERT OR REPLACE INTO role( + idRole, + strRole) + VALUES (?, ?) + ''', (1, 'Artist')) + if v.KODIVERSION >= 18: + self.cursor.execute('DELETE FROM versiontagscan') + self.cursor.execute(''' + INSERT INTO versiontagscan( + idVersion, + iNeedsScan, + lastscanned) + VALUES (?, ?, ?) + ''', (v.DB_MUSIC_VERSION, + 0, + timing.kodi_now())) + + @db.catch_operationalerrors def update_path(self, path, kodi_pathid): self.cursor.execute(''' UPDATE path @@ -62,7 +93,7 @@ class KodiMusicDB(common.KodiDBBase): return return song_ids[0][0] - @common.catch_operationalerrors + @db.catch_operationalerrors def delete_song_from_song_artist(self, song_id): """ Deletes son from song_artist table and possibly orphaned roles @@ -79,7 +110,7 @@ class KodiMusicDB(common.KodiDBBase): self.cursor.execute('DELETE FROM song_artist WHERE idSong = ?', (song_id, )) - @common.catch_operationalerrors + @db.catch_operationalerrors def delete_album_from_discography(self, album_id): """ Removes the album with id album_id from the table discography @@ -99,7 +130,7 @@ class KodiMusicDB(common.KodiDBBase): self.cursor.execute('DELETE FROM discography WHERE idArtist = ? AND strAlbum = ? AND strYear = ?', (artist[0], name, year)) - @common.catch_operationalerrors + @db.catch_operationalerrors def delete_song_from_song_genre(self, song_id): """ Deletes the one entry with id song_id from the song_genre table. @@ -120,7 +151,7 @@ class KodiMusicDB(common.KodiDBBase): if not self.cursor.fetchone(): self.delete_genre(genre[0]) - @common.catch_operationalerrors + @db.catch_operationalerrors def delete_genre(self, genre_id): """ Dedicated method in order to catch OperationalErrors correctly @@ -128,7 +159,7 @@ class KodiMusicDB(common.KodiDBBase): self.cursor.execute('DELETE FROM genre WHERE idGenre = ?', (genre_id, )) - @common.catch_operationalerrors + @db.catch_operationalerrors def delete_album_from_album_genre(self, album_id): """ Deletes the one entry with id album_id from the album_genre table. @@ -153,7 +184,7 @@ class KodiMusicDB(common.KodiDBBase): self.cursor.execute('SELECT COALESCE(MAX(idAlbum), 0) FROM album') return self.cursor.fetchone()[0] + 1 - @common.catch_operationalerrors + @db.catch_operationalerrors def add_album_17(self, *args): """ strReleaseType: 'album' or 'single' @@ -196,7 +227,7 @@ class KodiMusicDB(common.KodiDBBase): VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ''', (args)) - @common.catch_operationalerrors + @db.catch_operationalerrors def update_album_17(self, *args): if app.SYNC.artwork: self.cursor.execute(''' @@ -234,7 +265,7 @@ class KodiMusicDB(common.KodiDBBase): WHERE idAlbum = ? ''', (args)) - @common.catch_operationalerrors + @db.catch_operationalerrors def add_album(self, *args): """ strReleaseType: 'album' or 'single' @@ -277,7 +308,7 @@ class KodiMusicDB(common.KodiDBBase): VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ''', (args)) - @common.catch_operationalerrors + @db.catch_operationalerrors def update_album(self, *args): if app.SYNC.artwork: self.cursor.execute(''' @@ -315,7 +346,7 @@ class KodiMusicDB(common.KodiDBBase): WHERE idAlbum = ? ''', (args)) - @common.catch_operationalerrors + @db.catch_operationalerrors def add_albumartist(self, artist_id, kodi_id, artistname): self.cursor.execute(''' INSERT OR REPLACE INTO album_artist( @@ -325,7 +356,7 @@ class KodiMusicDB(common.KodiDBBase): VALUES (?, ?, ?) ''', (artist_id, kodi_id, artistname)) - @common.catch_operationalerrors + @db.catch_operationalerrors def add_discography(self, artist_id, albumname, year): self.cursor.execute(''' INSERT OR REPLACE INTO discography( @@ -335,7 +366,7 @@ class KodiMusicDB(common.KodiDBBase): VALUES (?, ?, ?) ''', (artist_id, albumname, year)) - @common.catch_operationalerrors + @db.catch_operationalerrors def add_music_genres(self, kodiid, genres, mediatype): """ Adds a list of genres (list of unicode) for a certain Kodi item @@ -388,7 +419,7 @@ class KodiMusicDB(common.KodiDBBase): self.cursor.execute('SELECT COALESCE(MAX(idSong),0) FROM song') return self.cursor.fetchone()[0] + 1 - @common.catch_operationalerrors + @db.catch_operationalerrors def add_song(self, *args): self.cursor.execute(''' INSERT INTO song( @@ -413,7 +444,7 @@ class KodiMusicDB(common.KodiDBBase): VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ''', (args)) - @common.catch_operationalerrors + @db.catch_operationalerrors def add_song_17(self, *args): self.cursor.execute(''' INSERT INTO song( @@ -438,7 +469,7 @@ class KodiMusicDB(common.KodiDBBase): VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ''', (args)) - @common.catch_operationalerrors + @db.catch_operationalerrors def update_song(self, *args): self.cursor.execute(''' UPDATE song @@ -459,7 +490,7 @@ class KodiMusicDB(common.KodiDBBase): WHERE idSong = ? ''', (args)) - @common.catch_operationalerrors + @db.catch_operationalerrors def set_playcount(self, *args): self.cursor.execute(''' UPDATE song @@ -468,7 +499,7 @@ class KodiMusicDB(common.KodiDBBase): WHERE idSong = ? ''', (args)) - @common.catch_operationalerrors + @db.catch_operationalerrors def update_song_17(self, *args): self.cursor.execute(''' UPDATE song @@ -497,7 +528,7 @@ class KodiMusicDB(common.KodiDBBase): except TypeError: pass - @common.catch_operationalerrors + @db.catch_operationalerrors def add_artist(self, name, musicbrainz): """ Adds a single artist's name to the db @@ -534,7 +565,7 @@ class KodiMusicDB(common.KodiDBBase): (name, artistid,)) return artistid - @common.catch_operationalerrors + @db.catch_operationalerrors def update_artist(self, *args): if app.SYNC.artwork: self.cursor.execute(''' @@ -557,15 +588,15 @@ class KodiMusicDB(common.KodiDBBase): WHERE idArtist = ? ''', (args)) - @common.catch_operationalerrors + @db.catch_operationalerrors def remove_song(self, kodi_id): self.cursor.execute('DELETE FROM song WHERE idSong = ?', (kodi_id, )) - @common.catch_operationalerrors + @db.catch_operationalerrors def remove_path(self, path_id): self.cursor.execute('DELETE FROM path WHERE idPath = ?', (path_id, )) - @common.catch_operationalerrors + @db.catch_operationalerrors def add_song_artist(self, artist_id, song_id, artist_name): self.cursor.execute(''' INSERT OR REPLACE INTO song_artist( @@ -577,7 +608,7 @@ class KodiMusicDB(common.KodiDBBase): VALUES (?, ?, ?, ?, ?) ''', (artist_id, song_id, 1, 0, artist_name)) - @common.catch_operationalerrors + @db.catch_operationalerrors def add_albuminfosong(self, song_id, album_id, track_no, track_title, runtime): """ @@ -593,7 +624,7 @@ class KodiMusicDB(common.KodiDBBase): VALUES (?, ?, ?, ?, ?) ''', (song_id, album_id, track_no, track_title, runtime)) - @common.catch_operationalerrors + @db.catch_operationalerrors def update_userrating(self, kodi_id, kodi_type, userrating): """ Updates userrating for songs and albums @@ -610,7 +641,7 @@ class KodiMusicDB(common.KodiDBBase): % (kodi_type, column), (userrating, identifier, kodi_id)) - @common.catch_operationalerrors + @db.catch_operationalerrors def remove_albuminfosong(self, kodi_id): """ Kodi 17 only @@ -618,7 +649,7 @@ class KodiMusicDB(common.KodiDBBase): self.cursor.execute('DELETE FROM albuminfosong WHERE idAlbumInfoSong = ?', (kodi_id, )) - @common.catch_operationalerrors + @db.catch_operationalerrors def remove_album(self, kodi_id): if v.KODIVERSION < 18: self.cursor.execute('DELETE FROM albuminfosong WHERE idAlbumInfo = ?', @@ -627,7 +658,7 @@ class KodiMusicDB(common.KodiDBBase): (kodi_id, )) self.cursor.execute('DELETE FROM album WHERE idAlbum = ?', (kodi_id, )) - @common.catch_operationalerrors + @db.catch_operationalerrors def remove_artist(self, kodi_id): self.cursor.execute('DELETE FROM album_artist WHERE idArtist = ?', (kodi_id, )) diff --git a/resources/lib/kodi_db/video.py b/resources/lib/kodi_db/video.py index 2c4d409a..6d9e1692 100644 --- a/resources/lib/kodi_db/video.py +++ b/resources/lib/kodi_db/video.py @@ -5,7 +5,7 @@ from logging import getLogger from sqlite3 import IntegrityError from . import common -from .. import path_ops, timing, variables as v +from .. import db, path_ops, timing, variables as v LOG = getLogger('PLEX.kodi_db.video') @@ -16,7 +16,19 @@ SHOW_PATH = 'plugin://%s.tvshows/' % v.ADDON_ID class KodiVideoDB(common.KodiDBBase): db_kind = 'video' - @common.catch_operationalerrors + @db.catch_operationalerrors + def create_kodi_db_indicees(self): + """ + Index the "actors" because we got a TON - speed up SELECT and WHEN + """ + commands = ( + 'CREATE UNIQUE INDEX IF NOT EXISTS ix_actor_2 ON actor (actor_id);', + 'CREATE UNIQUE INDEX IF NOT EXISTS ix_files_2 ON files (idFile);', + ) + for cmd in commands: + self.cursor.execute(cmd) + + @db.catch_operationalerrors def setup_path_table(self): """ Use with Kodi video DB @@ -66,7 +78,7 @@ class KodiVideoDB(common.KodiDBBase): 1, 0)) - @common.catch_operationalerrors + @db.catch_operationalerrors def parent_path_id(self, path): """ Video DB: Adds all subdirectories to path table while setting a "trail" @@ -90,7 +102,7 @@ class KodiVideoDB(common.KodiDBBase): self.update_parentpath_id(parent_id, pathid) return pathid - @common.catch_operationalerrors + @db.catch_operationalerrors def update_parentpath_id(self, parent_id, pathid): """ Dedicated method in order to catch OperationalErrors correctly @@ -98,7 +110,7 @@ class KodiVideoDB(common.KodiDBBase): self.cursor.execute('UPDATE path SET idParentPath = ? WHERE idPath = ?', (parent_id, pathid)) - @common.catch_operationalerrors + @db.catch_operationalerrors def add_path(self, path, date_added=None, id_parent_path=None, content=None, scraper=None): """ @@ -143,7 +155,7 @@ class KodiVideoDB(common.KodiDBBase): except TypeError: pass - @common.catch_operationalerrors + @db.catch_operationalerrors def add_file(self, filename, path_id, date_added): """ Adds the filename [unicode] to the table files if not already added @@ -201,7 +213,7 @@ class KodiVideoDB(common.KodiDBBase): except TypeError: pass - @common.catch_operationalerrors + @db.catch_operationalerrors def remove_file(self, file_id, remove_orphans=True): """ Removes the entry for file_id from the files table. Will also delete @@ -237,7 +249,7 @@ class KodiVideoDB(common.KodiDBBase): ''' self.cursor.execute(query, (path_id, MOVIE_PATH, SHOW_PATH)) - @common.catch_operationalerrors + @db.catch_operationalerrors def _modify_link_and_table(self, kodi_id, kodi_type, entries, link_table, table, key, first_id=None): first_id = first_id if first_id is not None else 1 @@ -343,7 +355,7 @@ class KodiVideoDB(common.KodiDBBase): for kind, people_list in people.iteritems(): self._add_people_kind(kodi_id, kodi_type, kind, people_list) - @common.catch_operationalerrors + @db.catch_operationalerrors def _add_people_kind(self, kodi_id, kodi_type, kind, people_list): # Save new people to Kodi DB by iterating over the remaining entries if kind == 'actor': @@ -388,7 +400,7 @@ class KodiVideoDB(common.KodiDBBase): 'writer': []}).iteritems(): self._modify_people_kind(kodi_id, kodi_type, kind, people_list) - @common.catch_operationalerrors + @db.catch_operationalerrors def _modify_people_kind(self, kodi_id, kodi_type, kind, people_list): # Get the people already saved in the DB for this specific item if kind == 'actor': @@ -443,7 +455,7 @@ class KodiVideoDB(common.KodiDBBase): # Save new people to Kodi DB by iterating over the remaining entries self._add_people_kind(kodi_id, kodi_type, kind, people_list) - @common.catch_operationalerrors + @db.catch_operationalerrors def _new_actor_id(self, name, art_url): # Not yet in actor DB, add person self.cursor.execute('SELECT COALESCE(MAX(actor_id), 0) FROM actor') @@ -503,7 +515,7 @@ class KodiVideoDB(common.KodiDBBase): (kodi_id, kodi_type)) return dict(self.cursor.fetchall()) - @common.catch_operationalerrors + @db.catch_operationalerrors def modify_streams(self, fileid, streamdetails=None, runtime=None): """ Leave streamdetails and runtime empty to delete all stream entries for @@ -622,7 +634,7 @@ class KodiVideoDB(common.KodiDBBase): except TypeError: pass - @common.catch_operationalerrors + @db.catch_operationalerrors def set_resume(self, file_id, resume_seconds, total_seconds, playcount, dateplayed): """ @@ -660,7 +672,7 @@ class KodiVideoDB(common.KodiDBBase): '', 1)) - @common.catch_operationalerrors + @db.catch_operationalerrors def create_tag(self, name): """ Will create a new tag if needed and return the tag_id @@ -676,7 +688,7 @@ class KodiVideoDB(common.KodiDBBase): (tag_id, name)) return tag_id - @common.catch_operationalerrors + @db.catch_operationalerrors def update_tag(self, oldtag, newtag, kodiid, mediatype): """ Updates the tag_id by replaying oldtag with newtag @@ -695,7 +707,7 @@ class KodiVideoDB(common.KodiDBBase): WHERE media_id = ? AND media_type = ? AND tag_id = ? ''', (kodiid, mediatype, oldtag,)) - @common.catch_operationalerrors + @db.catch_operationalerrors def create_collection(self, set_name): """ Returns the collection/set id for set_name [unicode] @@ -711,7 +723,7 @@ class KodiVideoDB(common.KodiDBBase): (setid, set_name)) return setid - @common.catch_operationalerrors + @db.catch_operationalerrors def assign_collection(self, setid, movieid): """ Assign the movie to one set/collection @@ -719,7 +731,7 @@ class KodiVideoDB(common.KodiDBBase): self.cursor.execute('UPDATE movie SET idSet = ? WHERE idMovie = ?', (setid, movieid,)) - @common.catch_operationalerrors + @db.catch_operationalerrors def remove_from_set(self, movieid): """ Remove the movie with movieid [int] from an associated movie set, movie @@ -739,7 +751,7 @@ class KodiVideoDB(common.KodiDBBase): except TypeError: pass - @common.catch_operationalerrors + @db.catch_operationalerrors def delete_possibly_empty_set(self, set_id): """ Checks whether there are other movies in the set set_id. If not, @@ -750,7 +762,7 @@ class KodiVideoDB(common.KodiDBBase): if self.cursor.fetchone() is None: self.cursor.execute('DELETE FROM sets WHERE idSet = ?', (set_id,)) - @common.catch_operationalerrors + @db.catch_operationalerrors def add_season(self, showid, seasonnumber): """ Adds a TV show season to the Kodi video DB or simply returns the ID, @@ -764,7 +776,7 @@ class KodiVideoDB(common.KodiDBBase): ''', (seasonid, showid, seasonnumber)) return seasonid - @common.catch_operationalerrors + @db.catch_operationalerrors def add_uniqueid(self, *args): """ Feed with: @@ -799,7 +811,7 @@ class KodiVideoDB(common.KodiDBBase): except TypeError: return self.add_uniqueid_id() - @common.catch_operationalerrors + @db.catch_operationalerrors def update_uniqueid(self, *args): """ Pass in media_id, media_type, value, type, uniqueid_id @@ -810,7 +822,7 @@ class KodiVideoDB(common.KodiDBBase): WHERE uniqueid_id = ? ''', (args)) - @common.catch_operationalerrors + @db.catch_operationalerrors def remove_uniqueid(self, kodi_id, kodi_type): """ Deletes the entry from the uniqueid table for the item @@ -833,7 +845,7 @@ class KodiVideoDB(common.KodiDBBase): except TypeError: return self.add_ratingid() - @common.catch_operationalerrors + @db.catch_operationalerrors def update_ratings(self, *args): """ Feed with media_id, media_type, rating_type, rating, votes, rating_id @@ -848,7 +860,7 @@ class KodiVideoDB(common.KodiDBBase): WHERE rating_id = ? ''', (args)) - @common.catch_operationalerrors + @db.catch_operationalerrors def add_ratings(self, *args): """ feed with: @@ -867,7 +879,7 @@ class KodiVideoDB(common.KodiDBBase): VALUES (?, ?, ?, ?, ?, ?) ''', (args)) - @common.catch_operationalerrors + @db.catch_operationalerrors def remove_ratings(self, kodi_id, kodi_type): """ Removes all ratings from the rating table for the item @@ -883,7 +895,7 @@ class KodiVideoDB(common.KodiDBBase): self.cursor.execute('SELECT COALESCE(MAX(idEpisode), 0) FROM episode') return self.cursor.fetchone()[0] + 1 - @common.catch_operationalerrors + @db.catch_operationalerrors def add_episode(self, *args): self.cursor.execute( ''' @@ -911,7 +923,7 @@ class KodiVideoDB(common.KodiDBBase): (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ''', (args)) - @common.catch_operationalerrors + @db.catch_operationalerrors def update_episode(self, *args): self.cursor.execute( ''' @@ -936,7 +948,7 @@ class KodiVideoDB(common.KodiDBBase): WHERE idEpisode = ? ''', (args)) - @common.catch_operationalerrors + @db.catch_operationalerrors def add_show(self, *args): self.cursor.execute( ''' @@ -955,7 +967,7 @@ class KodiVideoDB(common.KodiDBBase): VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ''', (args)) - @common.catch_operationalerrors + @db.catch_operationalerrors def update_show(self, *args): self.cursor.execute( ''' @@ -973,21 +985,21 @@ class KodiVideoDB(common.KodiDBBase): WHERE idShow = ? ''', (args)) - @common.catch_operationalerrors + @db.catch_operationalerrors def add_showlinkpath(self, kodi_id, kodi_pathid): self.cursor.execute('INSERT INTO tvshowlinkpath(idShow, idPath) VALUES (?, ?)', (kodi_id, kodi_pathid)) - @common.catch_operationalerrors + @db.catch_operationalerrors def remove_show(self, kodi_id): self.cursor.execute('DELETE FROM tvshow WHERE idShow = ?', (kodi_id,)) - @common.catch_operationalerrors + @db.catch_operationalerrors def remove_season(self, kodi_id): self.cursor.execute('DELETE FROM seasons WHERE idSeason = ?', (kodi_id,)) - @common.catch_operationalerrors + @db.catch_operationalerrors def remove_episode(self, kodi_id): self.cursor.execute('DELETE FROM episode WHERE idEpisode = ?', (kodi_id,)) @@ -996,7 +1008,7 @@ class KodiVideoDB(common.KodiDBBase): self.cursor.execute('SELECT COALESCE(MAX(idMovie), 0) FROM movie') return self.cursor.fetchone()[0] + 1 - @common.catch_operationalerrors + @db.catch_operationalerrors def add_movie(self, *args): self.cursor.execute( ''' @@ -1030,11 +1042,11 @@ class KodiVideoDB(common.KodiDBBase): ?, ?, ?, ?) ''', (args)) - @common.catch_operationalerrors + @db.catch_operationalerrors def remove_movie(self, kodi_id): self.cursor.execute('DELETE FROM movie WHERE idMovie = ?', (kodi_id,)) - @common.catch_operationalerrors + @db.catch_operationalerrors def update_userrating(self, kodi_id, kodi_type, userrating): """ Updates userrating diff --git a/resources/lib/library_sync/full_sync.py b/resources/lib/library_sync/full_sync.py index b0fe1f3f..e3f4f1eb 100644 --- a/resources/lib/library_sync/full_sync.py +++ b/resources/lib/library_sync/full_sync.py @@ -19,7 +19,7 @@ if common.PLAYLIST_SYNC_ENABLED: LOG = getLogger('PLEX.sync.full_sync') # How many items will be put through the processing chain at once? -BATCH_SIZE = 500 +BATCH_SIZE = 2000 # Safety margin to filter PMS items - how many seconds to look into the past? UPDATED_AT_SAFETY = 60 * 5 LAST_VIEWED_AT_SAFETY = 60 * 5 @@ -102,14 +102,15 @@ class FullSync(common.fullsync_mixin): self.threader.addTask(GetMetadataTask(self.queue, plex_id, self.plex_type, - self.get_children)) + self.get_children, + self.item_count)) self.item_count += 1 def update_library(self): LOG.debug('Writing changes to Kodi library now') i = 0 if not self.section: - self.section = self.queue.get() + _, self.section = self.queue.get() self.queue.task_done() while not self.isCanceled() and self.item_count > 0: section = self.section @@ -125,7 +126,7 @@ class FullSync(common.fullsync_mixin): with section.context(self.current_sync) as context: while not self.isCanceled() and self.item_count > 0: try: - item = self.queue.get(block=False) + _, item = self.queue.get(block=False) except backgroundthread.Queue.Empty: if self.threader.threader.working(): app.APP.monitor.waitForAbort(0.02) @@ -174,7 +175,7 @@ class FullSync(common.fullsync_mixin): iterator.get('title1')), section.section_id, section.plex_type) - self.queue.put(queue_info) + self.queue.put((-1, queue_info)) last = True # To keep track of the item-number in order to kill while loops self.item_count = 0 @@ -191,12 +192,10 @@ class FullSync(common.fullsync_mixin): self.process_item(xml_item) if self.item_count == BATCH_SIZE: break - # Make sure Plex DB above is closed before adding/updating - if self.item_count == BATCH_SIZE: - self.update_library() + # Make sure Plex DB above is closed before adding/updating! + self.update_library() if last: break - self.update_library() reset_collections() return True except RuntimeError: @@ -216,7 +215,7 @@ class FullSync(common.fullsync_mixin): section.name, section.section_id, section.plex_type) - self.queue.put(queue_info) + self.queue.put((-1, queue_info)) self.total = iterator.total self.section_name = section.name self.section_type_text = utils.lang( @@ -251,7 +250,7 @@ class FullSync(common.fullsync_mixin): def threaded_get_iterators(self, kinds, queue, all_items=False): """ - PF.SectionItems is costly, so let's do it asynchronous + Getting iterators is costly, so let's do it asynchronously """ try: for kind in kinds: @@ -268,16 +267,18 @@ class FullSync(common.fullsync_mixin): element.section_type = element.plex_type element.context = kind[2] element.get_children = kind[3] + element.Queue = kind[4] if self.repair or all_items: updated_at = None else: updated_at = section.last_sync - UPDATED_AT_SAFETY \ if section.last_sync else None try: - element.iterator = PF.SectionItems(section.section_id, - plex_type=element.plex_type, - updated_at=updated_at, - last_viewed_at=None) + element.iterator = PF.get_section_iterator( + section.section_id, + plex_type=element.plex_type, + updated_at=updated_at, + last_viewed_at=None) except RuntimeError: LOG.warn('Sync at least partially unsuccessful') self.successful = False @@ -292,16 +293,22 @@ class FullSync(common.fullsync_mixin): def full_library_sync(self): """ """ + # structure: + # (plex_type, + # section_type, + # context for itemtype, + # download children items, e.g. songs for a specific album?, + # Queue) kinds = [ - (v.PLEX_TYPE_MOVIE, v.PLEX_TYPE_MOVIE, itemtypes.Movie, False), - (v.PLEX_TYPE_SHOW, v.PLEX_TYPE_SHOW, itemtypes.Show, False), - (v.PLEX_TYPE_SEASON, v.PLEX_TYPE_SHOW, itemtypes.Season, False), - (v.PLEX_TYPE_EPISODE, v.PLEX_TYPE_SHOW, itemtypes.Episode, False) + (v.PLEX_TYPE_MOVIE, v.PLEX_TYPE_MOVIE, itemtypes.Movie, False, Queue.Queue), + (v.PLEX_TYPE_SHOW, v.PLEX_TYPE_SHOW, itemtypes.Show, False, Queue.Queue), + (v.PLEX_TYPE_SEASON, v.PLEX_TYPE_SHOW, itemtypes.Season, False, Queue.Queue), + (v.PLEX_TYPE_EPISODE, v.PLEX_TYPE_SHOW, itemtypes.Episode, False, Queue.Queue) ] if app.SYNC.enable_music: kinds.extend([ - (v.PLEX_TYPE_ARTIST, v.PLEX_TYPE_ARTIST, itemtypes.Artist, False), - (v.PLEX_TYPE_ALBUM, v.PLEX_TYPE_ARTIST, itemtypes.Album, True), + (v.PLEX_TYPE_ARTIST, v.PLEX_TYPE_ARTIST, itemtypes.Artist, False, Queue.Queue), + (v.PLEX_TYPE_ALBUM, v.PLEX_TYPE_ARTIST, itemtypes.Album, True, backgroundthread.OrderedQueue), ]) # ADD NEW ITEMS # Already start setting up the iterators. We need to enforce @@ -323,6 +330,7 @@ class FullSync(common.fullsync_mixin): self.section_type = section.section_type self.context = section.context self.get_children = section.get_children + self.queue = section.Queue() # Now do the heavy lifting if self.isCanceled() or not self.addupdate_section(section): return False @@ -352,8 +360,11 @@ class FullSync(common.fullsync_mixin): LOG.info('Start synching playstate and userdata for every item') # In order to not delete all your songs again if app.SYNC.enable_music: + # We don't need to enforce the album order now + kinds.pop(5) kinds.extend([ - (v.PLEX_TYPE_SONG, v.PLEX_TYPE_ARTIST, itemtypes.Song, True), + (v.PLEX_TYPE_ALBUM, v.PLEX_TYPE_ARTIST, itemtypes.Album, True, Queue.Queue), + (v.PLEX_TYPE_SONG, v.PLEX_TYPE_ARTIST, itemtypes.Song, True, Queue.Queue), ]) # Make sure we're not showing an item's title in the sync dialog self.title = '' @@ -429,7 +440,6 @@ class FullSync(common.fullsync_mixin): return self.successful = True try: - self.queue = backgroundthread.Queue.Queue() if self.show_dialog: self.dialog = xbmcgui.DialogProgressBG() self.dialog.create(utils.lang(39714)) diff --git a/resources/lib/library_sync/get_metadata.py b/resources/lib/library_sync/get_metadata.py index 5e623de9..6cd653c2 100644 --- a/resources/lib/library_sync/get_metadata.py +++ b/resources/lib/library_sync/get_metadata.py @@ -36,11 +36,13 @@ class GetMetadataTask(common.fullsync_mixin, backgroundthread.Task): queue Queue.Queue() object where this thread will store the downloaded metadata XMLs as etree objects """ - def __init__(self, queue, plex_id, plex_type, get_children=False): + def __init__(self, queue, plex_id, plex_type, get_children=False, + count=None): self.queue = queue self.plex_id = plex_id self.plex_type = plex_type self.get_children = get_children + self.count = count super(GetMetadataTask, self).__init__() def _collections(self, item): @@ -120,4 +122,4 @@ class GetMetadataTask(common.fullsync_mixin, backgroundthread.Task): else: item['children'] = children_xml if not self.isCanceled(): - self.queue.put(item) + self.queue.put((self.count, item)) diff --git a/resources/lib/music.py b/resources/lib/music.py index cf24f7f0..af638674 100644 --- a/resources/lib/music.py +++ b/resources/lib/music.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import, division, unicode_literals from logging import getLogger +import re from .plex_api.media import Media from . import utils @@ -85,7 +86,7 @@ def _turn_to_regex(path): else: if not path.endswith('\\'): path = '%s\\' % path - # Need to escape backslashes - path = path.replace('\\', '\\\\') + # Escape all characters that could cause problems + path = re.escape(path) # Beginning of path only needs to be similar return '^%s' % path diff --git a/resources/lib/plex_db/common.py b/resources/lib/plex_db/common.py index c2764cad..f150fa38 100644 --- a/resources/lib/plex_db/common.py +++ b/resources/lib/plex_db/common.py @@ -3,7 +3,7 @@ from __future__ import absolute_import, division, unicode_literals from threading import Lock -from .. import utils, variables as v +from .. import db, variables as v PLEXDB_LOCK = Lock() @@ -31,7 +31,7 @@ class PlexDBBase(object): def __enter__(self): if self.lock: PLEXDB_LOCK.acquire() - self.plexconn = utils.kodi_sql('plex') + self.plexconn = db.connect('plex') self.cursor = self.plexconn.cursor() return self diff --git a/resources/lib/plex_functions.py b/resources/lib/plex_functions.py index ea571e90..52e403d7 100644 --- a/resources/lib/plex_functions.py +++ b/resources/lib/plex_functions.py @@ -427,7 +427,7 @@ def _poke_pms(pms, queue): authenticate=False, headerOptions={'X-Plex-Token': pms['token']}, verifySSL=True if v.KODIVERSION >= 18 else False, - timeout=10) + timeout=(3.0, 5.0)) try: xml.attrib['machineIdentifier'] except (AttributeError, KeyError): @@ -557,23 +557,7 @@ def GetAllPlexChildren(key): return DownloadChunks("{server}/library/metadata/%s/children" % key) -def GetPlexSectionResults(viewId, args=None): - """ - Returns a list (XML API dump) of all Plex items in the Plex - section with key = viewId. - - Input: - args: optional dict to be urlencoded - - Returns None if something went wrong - """ - url = "{server}/library/sections/%s/all" % viewId - if args: - url = utils.extend_url(url, args) - return DownloadChunks(url) - - -class DownloadChunk(backgroundthread.Task): +class ThreadedDownloadChunk(backgroundthread.Task): """ This task will also be executed while library sync is suspended! """ @@ -581,7 +565,7 @@ class DownloadChunk(backgroundthread.Task): self.url = url self.args = args self.callback = callback - super(DownloadChunk, self).__init__() + super(ThreadedDownloadChunk, self).__init__() def run(self): xml = DU().downloadUrl(self.url, parameters=self.args) @@ -601,14 +585,15 @@ class DownloadGen(object): Yields XML etree children or raises RuntimeError at the end """ - def __init__(self, url, plex_type=None, last_viewed_at=None, - updated_at=None, args=None): + def __init__(self, url, plex_type, last_viewed_at, updated_at, args, + downloader): + self._downloader = downloader self.successful = True - self.args = args or {} + self.xml = None + self.args = args self.args.update({ - 'X-Plex-Container-Size': CONTAINERSIZE, - 'sort': 'id', # Entries are sorted by plex_id - 'excludeAllLeaves': 1 # PMS wont attach a first summary child + 'X-Plex-Container-Start': 0, + 'X-Plex-Container-Size': CONTAINERSIZE }) url += '?' if plex_type: @@ -618,8 +603,8 @@ class DownloadGen(object): if updated_at: url = '%supdatedAt>=%s&' % (url, updated_at) self.url = url[:-1] - self._download_chunk(start=0) - self.attrib = deepcopy(self.xml.attrib) + _blocking_download_chunk(self.url, self.args, 0, self.set_xml) + self.attrib = self.xml.attrib self.current = 0 self.total = int(self.attrib['totalSize']) self.cache_factor = 10 @@ -629,34 +614,24 @@ class DownloadGen(object): self.total + CONTAINERSIZE - self.total % CONTAINERSIZE) for pos in range(CONTAINERSIZE, end, CONTAINERSIZE): self.pending_counter.append(None) - self._download_chunk(start=pos) + self._downloader(self.url, self.args, pos, self.on_chunk_downloaded) - def _download_chunk(self, start): - self.args['X-Plex-Container-Start'] = start - if start == 0: - # We need the result NOW - self.xml = DU().downloadUrl(self.url, parameters=self.args) - try: - self.xml.attrib - except AttributeError: - LOG.error('Error while downloading chunks: %s, args: %s', - self.url, self.args) - raise RuntimeError('Error while downloading chunks for %s' - % self.url) - else: - task = DownloadChunk(self.url, - deepcopy(self.args), # Beware! - self.on_chunk_downloaded) - backgroundthread.BGThreader.addTask(task) + def set_xml(self, xml): + self.xml = xml def on_chunk_downloaded(self, xml): if xml is not None: - for child in xml: - self.xml.append(child) + self.xml.extend(xml) else: self.successful = False self.pending_counter.pop() + def get(self, key, default=None): + """ + Mimick etree xml's way to access xml.attrib via xml.get(key, default) + """ + return self.attrib.get(key, default) + def __iter__(self): return self @@ -669,8 +644,11 @@ class DownloadGen(object): if (self.current % CONTAINERSIZE == 0 and self.current <= self.total - (self.cache_factor - 1) * CONTAINERSIZE): self.pending_counter.append(None) - self._download_chunk( - start=self.current + (self.cache_factor - 1) * CONTAINERSIZE) + self._downloader( + self.url, + self.args, + self.current + (self.cache_factor - 1) * CONTAINERSIZE, + self.on_chunk_downloaded) return child except IndexError: if not self.pending_counter and not len(self.xml): @@ -679,46 +657,67 @@ class DownloadGen(object): else: raise StopIteration() LOG.debug('Waiting for download to finish') - app.APP.monitor.waitForAbort(0.1) + if app.APP.monitor.waitForAbort(0.1): + raise StopIteration('PKC needs to exit now') next = __next__ - def get(self, key, default=None): - return self.attrib.get(key, default) + +def _blocking_download_chunk(url, args, start, callback): + """ + callback will be called with the downloaded xml (fragment) + """ + args['X-Plex-Container-Start'] = start + xml = DU().downloadUrl(url, parameters=args) + try: + xml.attrib + except AttributeError: + LOG.error('Error while downloading chunks: %s, args: %s', + url, args) + raise RuntimeError('Error while downloading chunks for %s' + % url) + callback(xml) -class SectionItems(DownloadGen): - """ - Iterator object to get all items of a Plex library section - """ - def __init__(self, section_id, plex_type=None, last_viewed_at=None, - updated_at=None, args=None): - if plex_type in (v.PLEX_TYPE_EPISODE, v.PLEX_TYPE_SONG): - # Annoying Plex bug. You won't get all episodes otherwise - url = '{server}/library/sections/%s/allLeaves' % section_id - plex_type = None - else: - url = '{server}/library/sections/%s/all' % section_id - super(SectionItems, self).__init__(url, plex_type, last_viewed_at, - updated_at, args) +def _async_download_chunk(url, args, start, callback): + args['X-Plex-Container-Start'] = start + task = ThreadedDownloadChunk(url, + deepcopy(args), # Beware! + callback) + backgroundthread.BGThreader.addTask(task) -class Children(DownloadGen): - """ - Iterator object to get all items of a Plex library section - """ - def __init__(self, plex_id): - super(Children, self).__init__( - '{server}/library/metadata/%s/children' % plex_id) - - -class Leaves(DownloadGen): - """ - Iterator object to get all items of a Plex library section - """ - def __init__(self, section_id): - super(Leaves, self).__init__( - '{server}/library/sections/%s/allLeaves' % section_id) +def get_section_iterator(section_id, plex_type=None, last_viewed_at=None, + updated_at=None, args=None): + args = args or {} + args.update({ + 'checkFiles': 0, + 'includeExtras': 0, # Trailers and Extras => Extras + 'includeReviews': 0, + 'includeRelated': 0, # Similar movies => Video -> Related + 'skipRefresh': 1, # don't scan + 'excludeAllLeaves': 1 # PMS wont attach a first summary child + }) + if plex_type == v.PLEX_TYPE_ALBUM: + # Kodi sorts Newest Albums by their position within the Kodi music + # database - great... + downloader = _blocking_download_chunk + args['sort'] = 'addedAt:asc' + else: + downloader = _async_download_chunk + args['sort'] = 'id' # Entries are sorted by plex_id + if plex_type in (v.PLEX_TYPE_EPISODE, v.PLEX_TYPE_SONG): + # Annoying Plex bug. You won't get all episodes otherwise + url = '{server}/library/sections/%s/allLeaves' % section_id + plex_type = None + else: + url = '{server}/library/sections/%s/all' % section_id + return DownloadGen(url, + plex_type, + last_viewed_at, + updated_at, + args, + downloader) def DownloadChunks(url): @@ -767,37 +766,6 @@ def DownloadChunks(url): return xml -def GetAllPlexLeaves(viewId, lastViewedAt=None, updatedAt=None): - """ - Returns a list (raw XML API dump) of all Plex subitems for the key. - (e.g. /library/sections/2/allLeaves pointing to all TV shows) - - Input: - viewId Id of Plex library, e.g. '2' - lastViewedAt Unix timestamp; only retrieves PMS items viewed - since that point of time until now. - updatedAt Unix timestamp; only retrieves PMS items updated - by the PMS since that point of time until now. - - If lastViewedAt and updatedAt=None, ALL PMS items are returned. - - Warning: lastViewedAt and updatedAt are combined with AND by the PMS! - - Relevant "master time": PMS server. I guess this COULD lead to problems, - e.g. when server and client are in different time zones. - """ - args = [] - url = "{server}/library/sections/%s/allLeaves" % viewId - - if lastViewedAt: - args.append('lastViewedAt>=%s' % lastViewedAt) - if updatedAt: - args.append('updatedAt>=%s' % updatedAt) - if args: - url += '?' + '&'.join(args) - return DownloadChunks(url) - - def GetPlexOnDeck(viewId): """ """ diff --git a/resources/lib/utils.py b/resources/lib/utils.py index 555cb85a..0b025128 100644 --- a/resources/lib/utils.py +++ b/resources/lib/utils.py @@ -5,7 +5,7 @@ Various functions and decorators for PKC """ from __future__ import absolute_import, division, unicode_literals from logging import getLogger -from sqlite3 import connect, OperationalError +from sqlite3 import OperationalError from datetime import datetime from unicodedata import normalize from threading import Lock @@ -525,50 +525,6 @@ def delete_temporary_subtitles(): root, file, err) -def kodi_sql(media_type=None): - """ - Open a connection to the Kodi database. - media_type: 'video' (standard if not passed), 'plex', 'music', 'texture' - """ - if media_type == "plex": - db_path = v.DB_PLEX_PATH - elif media_type == "music": - db_path = v.DB_MUSIC_PATH - elif media_type == "texture": - db_path = v.DB_TEXTURE_PATH - else: - db_path = v.DB_VIDEO_PATH - conn = connect(db_path, timeout=30.0) - conn.execute('PRAGMA journal_mode=WAL;') - conn.execute('PRAGMA cache_size = -8000;') - conn.execute('PRAGMA synchronous=NORMAL;') - conn.execute('BEGIN') - # Use transactions - return conn - - -def create_kodi_db_indicees(): - """ - Index the "actors" because we got a TON - speed up SELECT and WHEN - """ - conn = kodi_sql('video') - cursor = conn.cursor() - commands = ( - 'CREATE UNIQUE INDEX IF NOT EXISTS ix_actor_2 ON actor (actor_id);', - 'CREATE UNIQUE INDEX IF NOT EXISTS ix_files_2 ON files (idFile);', - ) - for cmd in commands: - cursor.execute(cmd) - # Already used in Kodi >=17: CREATE UNIQUE INDEX ix_actor_1 ON actor (name) - # try: - # cursor.execute('CREATE UNIQUE INDEX ix_pkc_actor_index ON actor (name);') - # except OperationalError: - # # Index already exists - # pass - conn.commit() - conn.close() - - def wipe_synched_playlists(): """ Deletes all synched playlist files on the Kodi side; resets the Plex table @@ -617,12 +573,10 @@ def wipe_database(reboot=True): # Plex DB completely empty yet. Wipe existing Kodi music only if we # expect to sync Plex music music = settings('enableMusic') == 'true' + LOG.info("Resetting all cached artwork.") kodi_db.wipe_dbs(music) plex_db.wipe() - LOG.info("Resetting all cached artwork.") - # Remove all cached artwork - kodi_db.reset_cached_images() # reset the install run flag settings('SyncInstallRunDone', value="false") settings('sections_asked_for_machine_identifier', value='') @@ -644,7 +598,7 @@ def init_dbs(): # Ensure that Plex DB is set-up plex_db.initialize() # Hack to speed up look-ups for actors (giant table!) - create_kodi_db_indicees() + kodi_db.create_kodi_db_indicees() kodi_db.setup_kodi_default_entries() with kodi_db.KodiVideoDB() as kodidb: # Setup the paths for addon-paths (even when using direct paths) diff --git a/resources/lib/variables.py b/resources/lib/variables.py index e12d07d3..b36ba29c 100644 --- a/resources/lib/variables.py +++ b/resources/lib/variables.py @@ -444,6 +444,7 @@ CONTENT_FROM_PLEX_TYPE = { PLEX_TYPE_VIDEO: CONTENT_TYPE_VIDEO, PLEX_TYPE_PLAYLIST: CONTENT_TYPE_PLAYLIST, PLEX_TYPE_CHANNEL: CONTENT_TYPE_FILE, + PLEX_TYPE_TAG: CONTENT_TYPE_FILE, 'mixed': CONTENT_TYPE_SHOW, None: CONTENT_TYPE_FILE }