Merge pull request #1066 from croneter/beta-version

Bump master

README.md

@@ -1,5 +1,5 @@
-[![stable version](https://img.shields.io/badge/stable_version-2.10.2-blue.svg?maxAge=60&style=flat) ](https://github.com/croneter/binary_repo/raw/master/stable/repository.plexkodiconnect/repository.plexkodiconnect-1.0.2.zip) 
+[![stable version](https://img.shields.io/badge/stable_version-2.10.4-blue.svg?maxAge=60&style=flat) ](https://github.com/croneter/binary_repo/raw/master/stable/repository.plexkodiconnect/repository.plexkodiconnect-1.0.2.zip) 
-[![beta version](https://img.shields.io/badge/beta_version-2.10.2-red.svg?maxAge=60&style=flat) ](https://github.com/croneter/binary_repo/raw/master/beta/repository.plexkodiconnectbeta/repository.plexkodiconnectbeta-1.0.2.zip)
+[![beta version](https://img.shields.io/badge/beta_version-2.10.4-red.svg?maxAge=60&style=flat) ](https://github.com/croneter/binary_repo/raw/master/beta/repository.plexkodiconnectbeta/repository.plexkodiconnectbeta-1.0.2.zip)
 
 [![Installation](https://img.shields.io/badge/wiki-installation-brightgreen.svg?maxAge=60&style=flat)](https://github.com/croneter/PlexKodiConnect/wiki/Installation)
 [![FAQ](https://img.shields.io/badge/wiki-FAQ-brightgreen.svg?maxAge=60&style=flat)](https://github.com/croneter/PlexKodiConnect/wiki/faq)

addon.xml

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<addon  id="plugin.video.plexkodiconnect" name="PlexKodiConnect" version="2.10.2" provider-name="croneter">
+<addon  id="plugin.video.plexkodiconnect" name="PlexKodiConnect" version="2.10.4" provider-name="croneter">
   <requires>
     <import addon="xbmc.python" version="2.1.0"/>
     <import addon="script.module.requests" version="2.9.1" />
@@ -83,7 +83,20 @@
     <summary lang="lt_LT">Natūralioji „Plex“ integracija į „Kodi“</summary>
     <description lang="lt_LT">Prijunkite „Kodi“ prie „Plex Medija Serverio“. Šiame papildinyje daroma prielaida, kad valdote visus savo vaizdo įrašus naudodami „Plex“ (ir nė vieno su „Kodi“). Galite prarasti jau saugomus „Kodi“ vaizdo įrašų ir muzikos duomenų bazių duomenis (kadangi šis papildinys juos tiesiogiai pakeičia). Naudokite savo pačių rizika!</description>
     <disclaimer lang="lt_LT">Naudokite savo pačių rizika</disclaimer>
-    <news>version 2.10.2:
+    <news>version 2.10.4:
+- version 2.10.3 for everyone
+- Fix to correctly wipe Kodi databases
+
+version 2.10.3 (beta only):
+- Fix a couple of issues with music when using direct paths: correctly escape music paths for Kodi regex matching
+- Fix Recently Added Albums sort order (you will have to reset the Kodi database manually)
+- Fix database being locked in rare cases
+- Increase batch size for library sync from 500 to 2000 to increase sync speed
+- Optimize some code
+- Fix KeyError when using Plex search capabilities
+- Check faster for available Plex Media Server to connect to
+
+version 2.10.2:
 - Fix Kodi playback jumping to the beginning of a video that just started
 - Fix transcoding quality degenerating quickly while playing with a new setting to deactivate auto quality for transcoding (applicable e.g. for Chromecast) 
 - Update translations

changelog.txt

@@ -1,3 +1,16 @@
+version 2.10.4:
+- version 2.10.3 for everyone
+- Fix to correctly wipe Kodi databases
+
+version 2.10.3 (beta only):
+- Fix a couple of issues with music when using direct paths: correctly escape music paths for Kodi regex matching
+- Fix Recently Added Albums sort order (you will have to reset the Kodi database manually)
+- Fix database being locked in rare cases
+- Increase batch size for library sync from 500 to 2000 to increase sync speed
+- Optimize some code
+- Fix KeyError when using Plex search capabilities
+- Check faster for available Plex Media Server to connect to
+
 version 2.10.2:
 - Fix Kodi playback jumping to the beginning of a video that just started
 - Fix transcoding quality degenerating quickly while playing with a new setting to deactivate auto quality for transcoding (applicable e.g. for Chromecast) 

resources/lib/backgroundthread.py

@@ -140,6 +140,67 @@ class KillableThread(threading.Thread):
         return self._suspended
 
 
+class OrderedQueue(Queue.PriorityQueue, object):
+    """
+    Queue that enforces an order on the items it returns. An item you push
+    onto the queue must be a tuple
+        (index, item)
+    where index=-1 is the item that will be returned first. The Queue will block
+    until index=-1, 0, 1, 2, 3, ... is then made available
+    """
+    def __init__(self, maxsize=0):
+        super(OrderedQueue, self).__init__(maxsize)
+        self.smallest = -1
+        self.not_next_item = threading.Condition(self.mutex)
+
+    def _put(self, item, heappush=heapq.heappush):
+        heappush(self.queue, item)
+        if item[0] == self.smallest:
+            self.not_next_item.notify()
+
+    def get(self, block=True, timeout=None):
+        """Remove and return an item from the queue.
+
+        If optional args 'block' is true and 'timeout' is None (the default),
+        block if necessary until an item is available. If 'timeout' is
+        a non-negative number, it blocks at most 'timeout' seconds and raises
+        the Empty exception if no item was available within that time.
+        Otherwise ('block' is false), return an item if one is immediately
+        available, else raise the Empty exception ('timeout' is ignored
+        in that case).
+        """
+        self.not_empty.acquire()
+        try:
+            if not block:
+                if not self._qsize() or self.queue[0][0] != self.smallest:
+                    raise Queue.Empty
+            elif timeout is None:
+                while not self._qsize():
+                    self.not_empty.wait()
+                while self.queue[0][0] != self.smallest:
+                    self.not_next_item.wait()
+            elif timeout < 0:
+                raise ValueError("'timeout' must be a non-negative number")
+            else:
+                endtime = Queue._time() + timeout
+                while not self._qsize():
+                    remaining = endtime - Queue._time()
+                    if remaining <= 0.0:
+                        raise Queue.Empty
+                    self.not_empty.wait(remaining)
+                while self.queue[0][0] != self.smallest:
+                    remaining = endtime - Queue._time()
+                    if remaining <= 0.0:
+                        raise Queue.Empty
+                    self.not_next_item.wait(remaining)
+            item = self._get()
+            self.smallest += 1
+            self.not_full.notify()
+            return item
+        finally:
+            self.not_empty.release()
+
+
 class Tasks(list):
     def add(self, task):
         for t in self:

resources/lib/db.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import sqlite3
+from functools import wraps
+
+from . import variables as v, app
+
+DB_WRITE_ATTEMPTS = 100
+
+
+class LockedDatabase(Exception):
+    """
+    Dedicated class to make sure we're not silently catching locked DBs.
+    """
+    pass
+
+
+def catch_operationalerrors(method):
+    """
+    sqlite.OperationalError is raised immediately if another DB connection
+    is open, reading something that we're trying to change
+
+    So let's catch it and try again
+
+    Also see https://github.com/mattn/go-sqlite3/issues/274
+    """
+    @wraps(method)
+    def wrapper(self, *args, **kwargs):
+        attempts = DB_WRITE_ATTEMPTS
+        while True:
+            try:
+                return method(self, *args, **kwargs)
+            except sqlite3.OperationalError as err:
+                if 'database is locked' not in err:
+                    # Not an error we want to catch, so reraise it
+                    raise
+                attempts -= 1
+                if attempts == 0:
+                    # Reraise in order to NOT catch nested OperationalErrors
+                    raise LockedDatabase('Database is locked')
+                # Need to close the transactions and begin new ones
+                self.kodiconn.commit()
+                if self.artconn:
+                    self.artconn.commit()
+                if app.APP.monitor.waitForAbort(0.1):
+                    # PKC needs to quit
+                    return
+                # Start new transactions
+                self.kodiconn.execute('BEGIN')
+                if self.artconn:
+                    self.artconn.execute('BEGIN')
+    return wrapper
+
+
+def _initial_db_connection_setup(conn, wal_mode):
+    """
+    Set-up DB e.g. for WAL journal mode, if that hasn't already been done
+    before. Also start a transaction
+    """
+    if wal_mode:
+        conn.execute('PRAGMA journal_mode=WAL;')
+        conn.execute('PRAGMA cache_size = -8000;')
+        conn.execute('PRAGMA synchronous=NORMAL;')
+    conn.execute('BEGIN')
+
+
+def connect(media_type=None, wal_mode=True):
+    """
+    Open a connection to the Kodi database.
+        media_type: 'video' (standard if not passed), 'plex', 'music', 'texture'
+    Pass wal_mode=False if you want the standard (and slower) sqlite
+    journal_mode, e.g. when wiping entire tables. Useful if you do NOT want
+    concurrent access to DB for both PKC and Kodi
+    """
+    if media_type == "plex":
+        db_path = v.DB_PLEX_PATH
+    elif media_type == "music":
+        db_path = v.DB_MUSIC_PATH
+    elif media_type == "texture":
+        db_path = v.DB_TEXTURE_PATH
+    else:
+        db_path = v.DB_VIDEO_PATH
+    conn = sqlite3.connect(db_path, timeout=30.0)
+    attempts = DB_WRITE_ATTEMPTS
+    while True:
+        try:
+            _initial_db_connection_setup(conn, wal_mode)
+        except sqlite3.OperationalError as err:
+            if 'database is locked' not in err:
+                # Not an error we want to catch, so reraise it
+                raise
+            attempts -= 1
+            if attempts == 0:
+                # Reraise in order to NOT catch nested OperationalErrors
+                raise LockedDatabase('Database is locked')
+            if app.APP.monitor.waitForAbort(0.05):
+                # PKC needs to quit
+                return
+        else:
+            break
+    return conn

resources/lib/itemtypes/common.py

@@ -6,7 +6,7 @@ from ntpath import dirname
 
 from ..plex_db import PlexDB, PLEXDB_LOCK
 from ..kodi_db import KodiVideoDB, KODIDB_LOCK
-from .. import utils, timing, app
+from .. import db, timing, app
 
 LOG = getLogger('PLEX.itemtypes.common')
 
@@ -57,11 +57,11 @@ class ItemBase(object):
         if self.lock:
             PLEXDB_LOCK.acquire()
             KODIDB_LOCK.acquire()
-        self.plexconn = utils.kodi_sql('plex')
+        self.plexconn = db.connect('plex')
         self.plexcursor = self.plexconn.cursor()
-        self.kodiconn = utils.kodi_sql('video')
+        self.kodiconn = db.connect('video')
         self.kodicursor = self.kodiconn.cursor()
-        self.artconn = utils.kodi_sql('texture')
+        self.artconn = db.connect('texture')
         self.artcursor = self.artconn.cursor()
         self.plexdb = PlexDB(plexconn=self.plexconn, lock=False)
         self.kodidb = KodiVideoDB(texture_db=True,

resources/lib/itemtypes/music.py

@@ -7,7 +7,7 @@ from .common import ItemBase
 from ..plex_api import API
 from ..plex_db import PlexDB, PLEXDB_LOCK
 from ..kodi_db import KodiMusicDB, KODIDB_LOCK
-from .. import plex_functions as PF, utils, timing, app, variables as v
+from .. import plex_functions as PF, db, timing, app, variables as v
 
 LOG = getLogger('PLEX.music')
 
@@ -20,11 +20,11 @@ class MusicMixin(object):
         if self.lock:
             PLEXDB_LOCK.acquire()
             KODIDB_LOCK.acquire()
-        self.plexconn = utils.kodi_sql('plex')
+        self.plexconn = db.connect('plex')
         self.plexcursor = self.plexconn.cursor()
-        self.kodiconn = utils.kodi_sql('music')
+        self.kodiconn = db.connect('music')
         self.kodicursor = self.kodiconn.cursor()
-        self.artconn = utils.kodi_sql('texture')
+        self.artconn = db.connect('texture')
         self.artcursor = self.artconn.cursor()
         self.plexdb = PlexDB(plexconn=self.plexconn, lock=False)
         self.kodidb = KodiMusicDB(texture_db=True,

resources/lib/kodi_db/__init__.py

@@ -8,7 +8,7 @@ from .video import KodiVideoDB
 from .music import KodiMusicDB
 from .texture import KodiTextureDB
 
-from .. import path_ops, utils, timing, variables as v
+from .. import path_ops, utils, variables as v
 
 LOG = getLogger('PLEX.kodi_db')
 
@@ -56,35 +56,15 @@ def setup_kodi_default_entries():
     """
     if utils.settings('enableMusic') == 'true':
         with KodiMusicDB() as kodidb:
-            kodidb.cursor.execute('''
+            kodidb.setup_kodi_default_entries()
-                INSERT OR REPLACE INTO artist(
-                    idArtist,
-                    strArtist,
-                    strMusicBrainzArtistID)
-                VALUES (?, ?, ?)
-            ''', (1, '[Missing Tag]', 'Artist Tag Missing'))
-            kodidb.cursor.execute('''
-                INSERT OR REPLACE INTO role(
-                    idRole,
-                    strRole)
-                VALUES (?, ?)
-            ''', (1, 'Artist'))
-            if v.KODIVERSION >= 18:
-                kodidb.cursor.execute('DELETE FROM versiontagscan')
-                kodidb.cursor.execute('''
-                    INSERT INTO versiontagscan(
-                        idVersion,
-                        iNeedsScan,
-                        lastscanned)
-                    VALUES (?, ?, ?)
-                ''', (v.DB_MUSIC_VERSION,
-                      0,
-                      timing.kodi_now()))
 
 
 def reset_cached_images():
     LOG.info('Resetting cached artwork')
-    # Remove all existing textures first
+    LOG.debug('Resetting the Kodi texture DB')
+    with KodiTextureDB(wal_mode=False) as kodidb:
+        kodidb.wipe()
+    LOG.debug('Deleting all cached image files')
     path = path_ops.translate_path('special://thumbnails/')
     if path_ops.exists(path):
         path_ops.rmtree(path, ignore_errors=True)
@@ -95,13 +75,10 @@ def reset_cached_images():
             new_path = path_ops.translate_path('special://thumbnails/%s' % path)
             try:
                 path_ops.makedirs(path_ops.encode_path(new_path))
-            except OSError:
+            except OSError as err:
-                pass
+                LOG.warn('Could not create thumbnail directory %s: %s',
-    with KodiTextureDB() as kodidb:
+                         new_path, err)
-        for row in kodidb.cursor.execute('SELECT tbl_name FROM sqlite_master WHERE type=?',
+    LOG.info('Done resetting cached artwork')
-                                         ('table', )):
-            if row[0] != 'version':
-                kodidb.cursor.execute("DELETE FROM %s" % row[0])
 
 
 def wipe_dbs(music=True):
@@ -109,28 +86,18 @@ def wipe_dbs(music=True):
     Completely resets the Kodi databases 'video', 'texture' and 'music' (if
     music sync is enabled)
 
-    DO NOT use context menu as we need to connect without WAL mode - if Kodi
+    We need to connect without sqlite WAL mode as Kodi might still be accessing
-    is still accessing the DB
+    the dbs and we need to prevent that
     """
-    from sqlite3 import connect
     LOG.warn('Wiping Kodi databases!')
-    kinds = [v.DB_VIDEO_PATH, v.DB_TEXTURE_PATH]
+    LOG.info('Wiping Kodi video database')
+    with KodiVideoDB(wal_mode=False) as kodidb:
+        kodidb.wipe()
     if music:
-        kinds.append(v.DB_MUSIC_PATH)
+        LOG.info('Wiping Kodi music database')
-    for path in kinds:
+        with KodiMusicDB(wal_mode=False) as kodidb:
-        conn = connect(path, timeout=30.0)
+            kodidb.wipe()
-        cursor = conn.cursor()
+    reset_cached_images()
-        cursor.execute("SELECT name FROM sqlite_master WHERE type = 'table'")
-        tables = cursor.fetchall()
-        tables = [i[0] for i in tables]
-        if 'version' in tables:
-            tables.remove('version')
-        if 'versiontagscan' in tables:
-            tables.remove('versiontagscan')
-        for table in tables:
-            cursor.execute('DELETE FROM %s' % table)
-        conn.commit()
-        conn.close()
     setup_kodi_default_entries()
     # Delete SQLITE wal files
     import xbmc
@@ -140,6 +107,14 @@ def wipe_dbs(music=True):
         xbmc.executebuiltin('UpdateLibrary(music)')
 
 
+def create_kodi_db_indicees():
+    """
+    Index the "actors" because we got a TON - speed up SELECT and WHEN
+    """
+    with KodiVideoDB() as kodidb:
+        kodidb.create_kodi_db_indicees()
+
+
 KODIDB_FROM_PLEXTYPE = {
     v.PLEX_TYPE_MOVIE: KodiVideoDB,
     v.PLEX_TYPE_SHOW: KodiVideoDB,

resources/lib/kodi_db/common.py

@@ -2,65 +2,24 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import, division, unicode_literals
 from threading import Lock
-from functools import wraps
 
-from .. import utils, path_ops, app
+from .. import db, path_ops
 
 KODIDB_LOCK = Lock()
-DB_WRITE_ATTEMPTS = 100
+# Names of tables we generally leave untouched and e.g. don't wipe
-
+UNTOUCHED_TABLES = ('version', 'versiontagscan')
-
-class LockedKodiDatabase(Exception):
-    """
-    Dedicated class to make sure we're not silently catching locked DBs.
-    """
-    pass
-
-
-def catch_operationalerrors(method):
-    """
-    sqlite.OperationalError is raised immediately if another DB connection
-    is open, reading something that we're trying to change
-
-    So let's catch it and try again
-
-    Also see https://github.com/mattn/go-sqlite3/issues/274
-    """
-    @wraps(method)
-    def wrapper(self, *args, **kwargs):
-        attempts = DB_WRITE_ATTEMPTS
-        while True:
-            try:
-                return method(self, *args, **kwargs)
-            except utils.OperationalError as err:
-                if 'database is locked' not in err:
-                    # Not an error we want to catch, so reraise it
-                    raise
-                attempts -= 1
-                if attempts == 0:
-                    # Reraise in order to NOT catch nested OperationalErrors
-                    raise LockedKodiDatabase('Kodi database locked')
-                # Need to close the transactions and begin new ones
-                self.kodiconn.commit()
-                if self.artconn:
-                    self.artconn.commit()
-                if app.APP.monitor.waitForAbort(0.1):
-                    # PKC needs to quit
-                    return
-                # Start new transactions
-                self.kodiconn.execute('BEGIN')
-                if self.artconn:
-                    self.artconn.execute('BEGIN')
-    return wrapper
 
 
 class KodiDBBase(object):
     """
     Kodi database methods used for all types of items
     """
-    def __init__(self, texture_db=False, kodiconn=None, artconn=None, lock=True):
+    def __init__(self, texture_db=False, kodiconn=None, artconn=None,
+                 lock=True, wal_mode=True):
         """
         Allows direct use with a cursor instead of context mgr
+        Pass wal_mode=False if you want the standard sqlite journal_mode, e.g.
+        when wiping entire tables
         """
         self._texture_db = texture_db
         self.lock = lock
@@ -68,13 +27,15 @@ class KodiDBBase(object):
         self.cursor = self.kodiconn.cursor() if self.kodiconn else None
         self.artconn = artconn
         self.artcursor = self.artconn.cursor() if self.artconn else None
+        self.wal_mode = wal_mode
 
     def __enter__(self):
         if self.lock:
             KODIDB_LOCK.acquire()
-        self.kodiconn = utils.kodi_sql(self.db_kind)
+        self.kodiconn = db.connect(self.db_kind, self.wal_mode)
         self.cursor = self.kodiconn.cursor()
-        self.artconn = utils.kodi_sql('texture') if self._texture_db else None
+        self.artconn = db.connect('texture', self.wal_mode) if self._texture_db \
+            else None
         self.artcursor = self.artconn.cursor() if self._texture_db else None
         return self
 
@@ -110,7 +71,7 @@ class KodiDBBase(object):
         for kodi_art, url in artworks.iteritems():
             self.add_art(url, kodi_id, kodi_type, kodi_art)
 
-    @catch_operationalerrors
+    @db.catch_operationalerrors
     def add_art(self, url, kodi_id, kodi_type, kodi_art):
         """
         Adds or modifies the artwork of kind kodi_art (e.g. 'poster') in the
@@ -129,7 +90,7 @@ class KodiDBBase(object):
         for kodi_art, url in artworks.iteritems():
             self.modify_art(url, kodi_id, kodi_type, kodi_art)
 
-    @catch_operationalerrors
+    @db.catch_operationalerrors
     def modify_art(self, url, kodi_id, kodi_type, kodi_art):
         """
         Adds or modifies the artwork of kind kodi_art (e.g. 'poster') in the
@@ -166,7 +127,7 @@ class KodiDBBase(object):
         for row in self.cursor.fetchall():
             self.delete_cached_artwork(row[0])
 
-    @catch_operationalerrors
+    @db.catch_operationalerrors
     def delete_cached_artwork(self, url):
         try:
             self.artcursor.execute("SELECT cachedurl FROM texture WHERE url = ? LIMIT 1",
@@ -182,3 +143,16 @@ class KodiDBBase(object):
             if path_ops.exists(path):
                 path_ops.rmtree(path, ignore_errors=True)
             self.artcursor.execute("DELETE FROM texture WHERE url = ?", (url, ))
+
+    @db.catch_operationalerrors
+    def wipe(self):
+        """
+        Completely wipes the corresponding Kodi database
+        """
+        self.cursor.execute("SELECT name FROM sqlite_master WHERE type = 'table'")
+        tables = [i[0] for i in self.cursor.fetchall()]
+        for table in UNTOUCHED_TABLES:
+            if table in tables:
+                tables.remove(table)
+        for table in tables:
+            self.cursor.execute('DELETE FROM %s' % table)

resources/lib/kodi_db/music.py

@@ -4,7 +4,7 @@ from __future__ import absolute_import, division, unicode_literals
 from logging import getLogger
 
 from . import common
-from .. import variables as v, app
+from .. import db, variables as v, app, timing
 
 LOG = getLogger('PLEX.kodi_db.music')
 
@@ -12,7 +12,7 @@ LOG = getLogger('PLEX.kodi_db.music')
 class KodiMusicDB(common.KodiDBBase):
     db_kind = 'music'
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def add_path(self, path):
         """
         Add the path (unicode) to the music DB, if it does not exist already.
@@ -34,7 +34,38 @@ class KodiMusicDB(common.KodiDBBase):
                                 (pathid, path, '123'))
         return pathid
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
+    def setup_kodi_default_entries(self):
+        """
+        Makes sure that we retain the Kodi standard databases. E.g. that there
+        is a dummy artist with ID 1
+        """
+        self.cursor.execute('''
+            INSERT OR REPLACE INTO artist(
+                idArtist,
+                strArtist,
+                strMusicBrainzArtistID)
+            VALUES (?, ?, ?)
+        ''', (1, '[Missing Tag]', 'Artist Tag Missing'))
+        self.cursor.execute('''
+            INSERT OR REPLACE INTO role(
+                idRole,
+                strRole)
+            VALUES (?, ?)
+        ''', (1, 'Artist'))
+        if v.KODIVERSION >= 18:
+            self.cursor.execute('DELETE FROM versiontagscan')
+            self.cursor.execute('''
+                INSERT INTO versiontagscan(
+                    idVersion,
+                    iNeedsScan,
+                    lastscanned)
+                VALUES (?, ?, ?)
+            ''', (v.DB_MUSIC_VERSION,
+                  0,
+                  timing.kodi_now()))
+
+    @db.catch_operationalerrors
     def update_path(self, path, kodi_pathid):
         self.cursor.execute('''
             UPDATE path
@@ -62,7 +93,7 @@ class KodiMusicDB(common.KodiDBBase):
             return
         return song_ids[0][0]
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def delete_song_from_song_artist(self, song_id):
         """
         Deletes son from song_artist table and possibly orphaned roles
@@ -79,7 +110,7 @@ class KodiMusicDB(common.KodiDBBase):
         self.cursor.execute('DELETE FROM song_artist WHERE idSong = ?',
                             (song_id, ))
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def delete_album_from_discography(self, album_id):
         """
         Removes the album with id album_id from the table discography
@@ -99,7 +130,7 @@ class KodiMusicDB(common.KodiDBBase):
         self.cursor.execute('DELETE FROM discography WHERE idArtist = ? AND strAlbum = ? AND strYear = ?',
                             (artist[0], name, year))
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def delete_song_from_song_genre(self, song_id):
         """
         Deletes the one entry with id song_id from the song_genre table.
@@ -120,7 +151,7 @@ class KodiMusicDB(common.KodiDBBase):
                 if not self.cursor.fetchone():
                     self.delete_genre(genre[0])
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def delete_genre(self, genre_id):
         """
         Dedicated method in order to catch OperationalErrors correctly
@@ -128,7 +159,7 @@ class KodiMusicDB(common.KodiDBBase):
         self.cursor.execute('DELETE FROM genre WHERE idGenre = ?',
                             (genre_id, ))
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def delete_album_from_album_genre(self, album_id):
         """
         Deletes the one entry with id album_id from the album_genre table.
@@ -153,7 +184,7 @@ class KodiMusicDB(common.KodiDBBase):
         self.cursor.execute('SELECT COALESCE(MAX(idAlbum), 0) FROM album')
         return self.cursor.fetchone()[0] + 1
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def add_album_17(self, *args):
         """
         strReleaseType: 'album' or 'single'
@@ -196,7 +227,7 @@ class KodiMusicDB(common.KodiDBBase):
                 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
             ''', (args))
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def update_album_17(self, *args):
         if app.SYNC.artwork:
             self.cursor.execute('''
@@ -234,7 +265,7 @@ class KodiMusicDB(common.KodiDBBase):
                 WHERE idAlbum = ?
             ''', (args))
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def add_album(self, *args):
         """
         strReleaseType: 'album' or 'single'
@@ -277,7 +308,7 @@ class KodiMusicDB(common.KodiDBBase):
                 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
             ''', (args))
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def update_album(self, *args):
         if app.SYNC.artwork:
             self.cursor.execute('''
@@ -315,7 +346,7 @@ class KodiMusicDB(common.KodiDBBase):
                 WHERE idAlbum = ?
             ''', (args))
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def add_albumartist(self, artist_id, kodi_id, artistname):
         self.cursor.execute('''
             INSERT OR REPLACE INTO album_artist(
@@ -325,7 +356,7 @@ class KodiMusicDB(common.KodiDBBase):
             VALUES (?, ?, ?)
         ''', (artist_id, kodi_id, artistname))
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def add_discography(self, artist_id, albumname, year):
         self.cursor.execute('''
             INSERT OR REPLACE INTO discography(
@@ -335,7 +366,7 @@ class KodiMusicDB(common.KodiDBBase):
             VALUES (?, ?, ?)
         ''', (artist_id, albumname, year))
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def add_music_genres(self, kodiid, genres, mediatype):
         """
         Adds a list of genres (list of unicode) for a certain Kodi item
@@ -388,7 +419,7 @@ class KodiMusicDB(common.KodiDBBase):
         self.cursor.execute('SELECT COALESCE(MAX(idSong),0) FROM song')
         return self.cursor.fetchone()[0] + 1
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def add_song(self, *args):
         self.cursor.execute('''
             INSERT INTO song(
@@ -413,7 +444,7 @@ class KodiMusicDB(common.KodiDBBase):
             VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
         ''', (args))
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def add_song_17(self, *args):
         self.cursor.execute('''
             INSERT INTO song(
@@ -438,7 +469,7 @@ class KodiMusicDB(common.KodiDBBase):
             VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
         ''', (args))
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def update_song(self, *args):
         self.cursor.execute('''
             UPDATE song
@@ -459,7 +490,7 @@ class KodiMusicDB(common.KodiDBBase):
             WHERE idSong = ?
         ''', (args))
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def set_playcount(self, *args):
         self.cursor.execute('''
             UPDATE song
@@ -468,7 +499,7 @@ class KodiMusicDB(common.KodiDBBase):
             WHERE idSong = ?
         ''', (args))
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def update_song_17(self, *args):
         self.cursor.execute('''
             UPDATE song
@@ -497,7 +528,7 @@ class KodiMusicDB(common.KodiDBBase):
         except TypeError:
             pass
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def add_artist(self, name, musicbrainz):
         """
         Adds a single artist's name to the db
@@ -534,7 +565,7 @@ class KodiMusicDB(common.KodiDBBase):
                                     (name, artistid,))
         return artistid
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def update_artist(self, *args):
         if app.SYNC.artwork:
             self.cursor.execute('''
@@ -557,15 +588,15 @@ class KodiMusicDB(common.KodiDBBase):
                 WHERE idArtist = ?
             ''', (args))
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def remove_song(self, kodi_id):
         self.cursor.execute('DELETE FROM song WHERE idSong = ?', (kodi_id, ))
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def remove_path(self, path_id):
         self.cursor.execute('DELETE FROM path WHERE idPath = ?', (path_id, ))
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def add_song_artist(self, artist_id, song_id, artist_name):
         self.cursor.execute('''
             INSERT OR REPLACE INTO song_artist(
@@ -577,7 +608,7 @@ class KodiMusicDB(common.KodiDBBase):
             VALUES (?, ?, ?, ?, ?)
         ''', (artist_id, song_id, 1, 0, artist_name))
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def add_albuminfosong(self, song_id, album_id, track_no, track_title,
                           runtime):
         """
@@ -593,7 +624,7 @@ class KodiMusicDB(common.KodiDBBase):
             VALUES (?, ?, ?, ?, ?)
         ''', (song_id, album_id, track_no, track_title, runtime))
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def update_userrating(self, kodi_id, kodi_type, userrating):
         """
         Updates userrating for songs and albums
@@ -610,7 +641,7 @@ class KodiMusicDB(common.KodiDBBase):
                             % (kodi_type, column),
                             (userrating, identifier, kodi_id))
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def remove_albuminfosong(self, kodi_id):
         """
         Kodi 17 only
@@ -618,7 +649,7 @@ class KodiMusicDB(common.KodiDBBase):
         self.cursor.execute('DELETE FROM albuminfosong WHERE idAlbumInfoSong = ?',
                             (kodi_id, ))
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def remove_album(self, kodi_id):
         if v.KODIVERSION < 18:
             self.cursor.execute('DELETE FROM albuminfosong WHERE idAlbumInfo = ?',
@@ -627,7 +658,7 @@ class KodiMusicDB(common.KodiDBBase):
                             (kodi_id, ))
         self.cursor.execute('DELETE FROM album WHERE idAlbum = ?', (kodi_id, ))
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def remove_artist(self, kodi_id):
         self.cursor.execute('DELETE FROM album_artist WHERE idArtist = ?',
                             (kodi_id, ))

resources/lib/kodi_db/video.py

@@ -5,7 +5,7 @@ from logging import getLogger
 from sqlite3 import IntegrityError
 
 from . import common
-from .. import path_ops, timing, variables as v
+from .. import db, path_ops, timing, variables as v
 
 LOG = getLogger('PLEX.kodi_db.video')
 
@@ -16,7 +16,19 @@ SHOW_PATH = 'plugin://%s.tvshows/' % v.ADDON_ID
 class KodiVideoDB(common.KodiDBBase):
     db_kind = 'video'
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
+    def create_kodi_db_indicees(self):
+        """
+        Index the "actors" because we got a TON - speed up SELECT and WHEN
+        """
+        commands = (
+            'CREATE UNIQUE INDEX IF NOT EXISTS ix_actor_2 ON actor (actor_id);',
+            'CREATE UNIQUE INDEX IF NOT EXISTS ix_files_2 ON files (idFile);',
+        )
+        for cmd in commands:
+            self.cursor.execute(cmd)
+
+    @db.catch_operationalerrors
     def setup_path_table(self):
         """
         Use with Kodi video DB
@@ -66,7 +78,7 @@ class KodiVideoDB(common.KodiDBBase):
                                         1,
                                         0))
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def parent_path_id(self, path):
         """
         Video DB: Adds all subdirectories to path table while setting a "trail"
@@ -90,7 +102,7 @@ class KodiVideoDB(common.KodiDBBase):
                 self.update_parentpath_id(parent_id, pathid)
         return pathid
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def update_parentpath_id(self, parent_id, pathid):
         """
         Dedicated method in order to catch OperationalErrors correctly
@@ -98,7 +110,7 @@ class KodiVideoDB(common.KodiDBBase):
         self.cursor.execute('UPDATE path SET idParentPath = ? WHERE idPath = ?',
                             (parent_id, pathid))
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def add_path(self, path, date_added=None, id_parent_path=None,
                  content=None, scraper=None):
         """
@@ -143,7 +155,7 @@ class KodiVideoDB(common.KodiDBBase):
         except TypeError:
             pass
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def add_file(self, filename, path_id, date_added):
         """
         Adds the filename [unicode] to the table files if not already added
@@ -201,7 +213,7 @@ class KodiVideoDB(common.KodiDBBase):
         except TypeError:
             pass
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def remove_file(self, file_id, remove_orphans=True):
         """
         Removes the entry for file_id from the files table. Will also delete
@@ -237,7 +249,7 @@ class KodiVideoDB(common.KodiDBBase):
                 '''
                 self.cursor.execute(query, (path_id, MOVIE_PATH, SHOW_PATH))
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def _modify_link_and_table(self, kodi_id, kodi_type, entries, link_table,
                                table, key, first_id=None):
         first_id = first_id if first_id is not None else 1
@@ -343,7 +355,7 @@ class KodiVideoDB(common.KodiDBBase):
         for kind, people_list in people.iteritems():
             self._add_people_kind(kodi_id, kodi_type, kind, people_list)
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def _add_people_kind(self, kodi_id, kodi_type, kind, people_list):
         # Save new people to Kodi DB by iterating over the remaining entries
         if kind == 'actor':
@@ -388,7 +400,7 @@ class KodiVideoDB(common.KodiDBBase):
                                    'writer': []}).iteritems():
             self._modify_people_kind(kodi_id, kodi_type, kind, people_list)
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def _modify_people_kind(self, kodi_id, kodi_type, kind, people_list):
         # Get the people already saved in the DB for this specific item
         if kind == 'actor':
@@ -443,7 +455,7 @@ class KodiVideoDB(common.KodiDBBase):
         # Save new people to Kodi DB by iterating over the remaining entries
         self._add_people_kind(kodi_id, kodi_type, kind, people_list)
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def _new_actor_id(self, name, art_url):
         # Not yet in actor DB, add person
         self.cursor.execute('SELECT COALESCE(MAX(actor_id), 0) FROM actor')
@@ -503,7 +515,7 @@ class KodiVideoDB(common.KodiDBBase):
                             (kodi_id, kodi_type))
         return dict(self.cursor.fetchall())
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def modify_streams(self, fileid, streamdetails=None, runtime=None):
         """
         Leave streamdetails and runtime empty to delete all stream entries for
@@ -622,7 +634,7 @@ class KodiVideoDB(common.KodiDBBase):
         except TypeError:
             pass
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def set_resume(self, file_id, resume_seconds, total_seconds, playcount,
                    dateplayed):
         """
@@ -660,7 +672,7 @@ class KodiVideoDB(common.KodiDBBase):
                   '',
                   1))
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def create_tag(self, name):
         """
         Will create a new tag if needed and return the tag_id
@@ -676,7 +688,7 @@ class KodiVideoDB(common.KodiDBBase):
                                 (tag_id, name))
         return tag_id
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def update_tag(self, oldtag, newtag, kodiid, mediatype):
         """
         Updates the tag_id by replaying oldtag with newtag
@@ -695,7 +707,7 @@ class KodiVideoDB(common.KodiDBBase):
                 WHERE media_id = ? AND media_type = ? AND tag_id = ?
             ''', (kodiid, mediatype, oldtag,))
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def create_collection(self, set_name):
         """
         Returns the collection/set id for set_name [unicode]
@@ -711,7 +723,7 @@ class KodiVideoDB(common.KodiDBBase):
                                 (setid, set_name))
         return setid
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def assign_collection(self, setid, movieid):
         """
         Assign the movie to one set/collection
@@ -719,7 +731,7 @@ class KodiVideoDB(common.KodiDBBase):
         self.cursor.execute('UPDATE movie SET idSet = ? WHERE idMovie = ?',
                             (setid, movieid,))
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def remove_from_set(self, movieid):
         """
         Remove the movie with movieid [int] from an associated movie set, movie
@@ -739,7 +751,7 @@ class KodiVideoDB(common.KodiDBBase):
         except TypeError:
             pass
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def delete_possibly_empty_set(self, set_id):
         """
         Checks whether there are other movies in the set set_id. If not,
@@ -750,7 +762,7 @@ class KodiVideoDB(common.KodiDBBase):
         if self.cursor.fetchone() is None:
             self.cursor.execute('DELETE FROM sets WHERE idSet = ?', (set_id,))
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def add_season(self, showid, seasonnumber):
         """
         Adds a TV show season to the Kodi video DB or simply returns the ID,
@@ -764,7 +776,7 @@ class KodiVideoDB(common.KodiDBBase):
         ''', (seasonid, showid, seasonnumber))
         return seasonid
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def add_uniqueid(self, *args):
         """
         Feed with:
@@ -799,7 +811,7 @@ class KodiVideoDB(common.KodiDBBase):
         except TypeError:
             return self.add_uniqueid_id()
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def update_uniqueid(self, *args):
         """
         Pass in media_id, media_type, value, type, uniqueid_id
@@ -810,7 +822,7 @@ class KodiVideoDB(common.KodiDBBase):
             WHERE uniqueid_id = ?
         ''', (args))
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def remove_uniqueid(self, kodi_id, kodi_type):
         """
         Deletes the entry from the uniqueid table for the item
@@ -833,7 +845,7 @@ class KodiVideoDB(common.KodiDBBase):
         except TypeError:
             return self.add_ratingid()
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def update_ratings(self, *args):
         """
         Feed with media_id, media_type, rating_type, rating, votes, rating_id
@@ -848,7 +860,7 @@ class KodiVideoDB(common.KodiDBBase):
             WHERE rating_id = ?
         ''', (args))
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def add_ratings(self, *args):
         """
         feed with:
@@ -867,7 +879,7 @@ class KodiVideoDB(common.KodiDBBase):
             VALUES (?, ?, ?, ?, ?, ?)
         ''', (args))
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def remove_ratings(self, kodi_id, kodi_type):
         """
         Removes all ratings from the rating table for the item
@@ -883,7 +895,7 @@ class KodiVideoDB(common.KodiDBBase):
         self.cursor.execute('SELECT COALESCE(MAX(idEpisode), 0) FROM episode')
         return self.cursor.fetchone()[0] + 1
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def add_episode(self, *args):
         self.cursor.execute(
             '''
@@ -911,7 +923,7 @@ class KodiVideoDB(common.KodiDBBase):
                     (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
             ''', (args))
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def update_episode(self, *args):
         self.cursor.execute(
             '''
@@ -936,7 +948,7 @@ class KodiVideoDB(common.KodiDBBase):
                 WHERE idEpisode = ?
             ''', (args))
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def add_show(self, *args):
         self.cursor.execute(
             '''
@@ -955,7 +967,7 @@ class KodiVideoDB(common.KodiDBBase):
                 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
             ''', (args))
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def update_show(self, *args):
         self.cursor.execute(
             '''
@@ -973,21 +985,21 @@ class KodiVideoDB(common.KodiDBBase):
                 WHERE idShow = ?
             ''', (args))
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def add_showlinkpath(self, kodi_id, kodi_pathid):
         self.cursor.execute('INSERT INTO tvshowlinkpath(idShow, idPath) VALUES (?, ?)',
                             (kodi_id, kodi_pathid))
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def remove_show(self, kodi_id):
         self.cursor.execute('DELETE FROM tvshow WHERE idShow = ?', (kodi_id,))
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def remove_season(self, kodi_id):
         self.cursor.execute('DELETE FROM seasons WHERE idSeason = ?',
                             (kodi_id,))
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def remove_episode(self, kodi_id):
         self.cursor.execute('DELETE FROM episode WHERE idEpisode = ?',
                             (kodi_id,))
@@ -996,7 +1008,7 @@ class KodiVideoDB(common.KodiDBBase):
         self.cursor.execute('SELECT COALESCE(MAX(idMovie), 0) FROM movie')
         return self.cursor.fetchone()[0] + 1
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def add_movie(self, *args):
         self.cursor.execute(
             '''
@@ -1030,11 +1042,11 @@ class KodiVideoDB(common.KodiDBBase):
                  ?, ?, ?, ?)
         ''', (args))
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def remove_movie(self, kodi_id):
         self.cursor.execute('DELETE FROM movie WHERE idMovie = ?', (kodi_id,))
 
-    @common.catch_operationalerrors
+    @db.catch_operationalerrors
     def update_userrating(self, kodi_id, kodi_type, userrating):
         """
         Updates userrating

resources/lib/library_sync/full_sync.py

@@ -19,7 +19,7 @@ if common.PLAYLIST_SYNC_ENABLED:
 
 LOG = getLogger('PLEX.sync.full_sync')
 # How many items will be put through the processing chain at once?
-BATCH_SIZE = 500
+BATCH_SIZE = 2000
 # Safety margin to filter PMS items - how many seconds to look into the past?
 UPDATED_AT_SAFETY = 60 * 5
 LAST_VIEWED_AT_SAFETY = 60 * 5
@@ -102,14 +102,15 @@ class FullSync(common.fullsync_mixin):
         self.threader.addTask(GetMetadataTask(self.queue,
                                               plex_id,
                                               self.plex_type,
-                                              self.get_children))
+                                              self.get_children,
+                                              self.item_count))
         self.item_count += 1
 
     def update_library(self):
         LOG.debug('Writing changes to Kodi library now')
         i = 0
         if not self.section:
-            self.section = self.queue.get()
+            _, self.section = self.queue.get()
             self.queue.task_done()
         while not self.isCanceled() and self.item_count > 0:
             section = self.section
@@ -125,7 +126,7 @@ class FullSync(common.fullsync_mixin):
             with section.context(self.current_sync) as context:
                 while not self.isCanceled() and self.item_count > 0:
                     try:
-                        item = self.queue.get(block=False)
+                        _, item = self.queue.get(block=False)
                     except backgroundthread.Queue.Empty:
                         if self.threader.threader.working():
                             app.APP.monitor.waitForAbort(0.02)
@@ -174,7 +175,7 @@ class FullSync(common.fullsync_mixin):
                                                      iterator.get('title1')),
                                         section.section_id,
                                         section.plex_type)
-            self.queue.put(queue_info)
+            self.queue.put((-1, queue_info))
             last = True
             # To keep track of the item-number in order to kill while loops
             self.item_count = 0
@@ -191,12 +192,10 @@ class FullSync(common.fullsync_mixin):
                         self.process_item(xml_item)
                         if self.item_count == BATCH_SIZE:
                             break
-                # Make sure Plex DB above is closed before adding/updating
+                # Make sure Plex DB above is closed before adding/updating!
-                if self.item_count == BATCH_SIZE:
                 self.update_library()
                 if last:
                     break
-            self.update_library()
             reset_collections()
             return True
         except RuntimeError:
@@ -216,7 +215,7 @@ class FullSync(common.fullsync_mixin):
                                         section.name,
                                         section.section_id,
                                         section.plex_type)
-            self.queue.put(queue_info)
+            self.queue.put((-1, queue_info))
             self.total = iterator.total
             self.section_name = section.name
             self.section_type_text = utils.lang(
@@ -251,7 +250,7 @@ class FullSync(common.fullsync_mixin):
 
     def threaded_get_iterators(self, kinds, queue, all_items=False):
         """
-        PF.SectionItems is costly, so let's do it asynchronous
+        Getting iterators is costly, so let's do it asynchronously
         """
         try:
             for kind in kinds:
@@ -268,13 +267,15 @@ class FullSync(common.fullsync_mixin):
                     element.section_type = element.plex_type
                     element.context = kind[2]
                     element.get_children = kind[3]
+                    element.Queue = kind[4]
                     if self.repair or all_items:
                         updated_at = None
                     else:
                         updated_at = section.last_sync - UPDATED_AT_SAFETY \
                             if section.last_sync else None
                     try:
-                        element.iterator = PF.SectionItems(section.section_id,
+                        element.iterator = PF.get_section_iterator(
+                            section.section_id,
                             plex_type=element.plex_type,
                             updated_at=updated_at,
                             last_viewed_at=None)
@@ -292,16 +293,22 @@ class FullSync(common.fullsync_mixin):
     def full_library_sync(self):
         """
         """
+        # structure:
+        #  (plex_type,
+        #   section_type,
+        #   context for itemtype,
+        #   download children items, e.g. songs for a specific album?,
+        #   Queue)
         kinds = [
-            (v.PLEX_TYPE_MOVIE, v.PLEX_TYPE_MOVIE, itemtypes.Movie, False),
+            (v.PLEX_TYPE_MOVIE, v.PLEX_TYPE_MOVIE, itemtypes.Movie, False, Queue.Queue),
-            (v.PLEX_TYPE_SHOW, v.PLEX_TYPE_SHOW, itemtypes.Show, False),
+            (v.PLEX_TYPE_SHOW, v.PLEX_TYPE_SHOW, itemtypes.Show, False, Queue.Queue),
-            (v.PLEX_TYPE_SEASON, v.PLEX_TYPE_SHOW, itemtypes.Season, False),
+            (v.PLEX_TYPE_SEASON, v.PLEX_TYPE_SHOW, itemtypes.Season, False, Queue.Queue),
-            (v.PLEX_TYPE_EPISODE, v.PLEX_TYPE_SHOW, itemtypes.Episode, False)
+            (v.PLEX_TYPE_EPISODE, v.PLEX_TYPE_SHOW, itemtypes.Episode, False, Queue.Queue)
         ]
         if app.SYNC.enable_music:
             kinds.extend([
-                (v.PLEX_TYPE_ARTIST, v.PLEX_TYPE_ARTIST, itemtypes.Artist, False),
+                (v.PLEX_TYPE_ARTIST, v.PLEX_TYPE_ARTIST, itemtypes.Artist, False, Queue.Queue),
-                (v.PLEX_TYPE_ALBUM, v.PLEX_TYPE_ARTIST, itemtypes.Album, True),
+                (v.PLEX_TYPE_ALBUM, v.PLEX_TYPE_ARTIST, itemtypes.Album, True, backgroundthread.OrderedQueue),
             ])
         # ADD NEW ITEMS
         # Already start setting up the iterators. We need to enforce
@@ -323,6 +330,7 @@ class FullSync(common.fullsync_mixin):
             self.section_type = section.section_type
             self.context = section.context
             self.get_children = section.get_children
+            self.queue = section.Queue()
             # Now do the heavy lifting
             if self.isCanceled() or not self.addupdate_section(section):
                 return False
@@ -352,8 +360,11 @@ class FullSync(common.fullsync_mixin):
         LOG.info('Start synching playstate and userdata for every item')
         # In order to not delete all your songs again
         if app.SYNC.enable_music:
+            # We don't need to enforce the album order now
+            kinds.pop(5)
             kinds.extend([
-                (v.PLEX_TYPE_SONG, v.PLEX_TYPE_ARTIST, itemtypes.Song, True),
+                (v.PLEX_TYPE_ALBUM, v.PLEX_TYPE_ARTIST, itemtypes.Album, True, Queue.Queue),
+                (v.PLEX_TYPE_SONG, v.PLEX_TYPE_ARTIST, itemtypes.Song, True, Queue.Queue),
             ])
         # Make sure we're not showing an item's title in the sync dialog
         self.title = ''
@@ -429,7 +440,6 @@ class FullSync(common.fullsync_mixin):
             return
         self.successful = True
         try:
-            self.queue = backgroundthread.Queue.Queue()
             if self.show_dialog:
                 self.dialog = xbmcgui.DialogProgressBG()
                 self.dialog.create(utils.lang(39714))

resources/lib/library_sync/get_metadata.py

@@ -36,11 +36,13 @@ class GetMetadataTask(common.fullsync_mixin, backgroundthread.Task):
         queue               Queue.Queue() object where this thread will store
                             the downloaded metadata XMLs as etree objects
     """
-    def __init__(self, queue, plex_id, plex_type, get_children=False):
+    def __init__(self, queue, plex_id, plex_type, get_children=False,
+                 count=None):
         self.queue = queue
         self.plex_id = plex_id
         self.plex_type = plex_type
         self.get_children = get_children
+        self.count = count
         super(GetMetadataTask, self).__init__()
 
     def _collections(self, item):
@@ -120,4 +122,4 @@ class GetMetadataTask(common.fullsync_mixin, backgroundthread.Task):
             else:
                 item['children'] = children_xml
         if not self.isCanceled():
-            self.queue.put(item)
+            self.queue.put((self.count, item))

resources/lib/music.py

@@ -2,6 +2,7 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import, division, unicode_literals
 from logging import getLogger
+import re
 
 from .plex_api.media import Media
 from . import utils
@@ -85,7 +86,7 @@ def _turn_to_regex(path):
     else:
         if not path.endswith('\\'):
             path = '%s\\' % path
-    # Need to escape backslashes
+    # Escape all characters that could cause problems
-    path = path.replace('\\', '\\\\')
+    path = re.escape(path)
     # Beginning of path only needs to be similar
     return '^%s' % path

resources/lib/plex_db/common.py

@@ -3,7 +3,7 @@
 from __future__ import absolute_import, division, unicode_literals
 from threading import Lock
 
-from .. import utils, variables as v
+from .. import db, variables as v
 
 PLEXDB_LOCK = Lock()
 
@@ -31,7 +31,7 @@ class PlexDBBase(object):
     def __enter__(self):
         if self.lock:
             PLEXDB_LOCK.acquire()
-        self.plexconn = utils.kodi_sql('plex')
+        self.plexconn = db.connect('plex')
         self.cursor = self.plexconn.cursor()
         return self
 

resources/lib/plex_functions.py

@@ -427,7 +427,7 @@ def _poke_pms(pms, queue):
                            authenticate=False,
                            headerOptions={'X-Plex-Token': pms['token']},
                            verifySSL=True if v.KODIVERSION >= 18 else False,
-                           timeout=10)
+                           timeout=(3.0, 5.0))
     try:
         xml.attrib['machineIdentifier']
     except (AttributeError, KeyError):
@@ -557,23 +557,7 @@ def GetAllPlexChildren(key):
     return DownloadChunks("{server}/library/metadata/%s/children" % key)
 
 
-def GetPlexSectionResults(viewId, args=None):
+class ThreadedDownloadChunk(backgroundthread.Task):
-    """
-    Returns a list (XML API dump) of all Plex items in the Plex
-    section with key = viewId.
-
-    Input:
-        args:       optional dict to be urlencoded
-
-    Returns None if something went wrong
-    """
-    url = "{server}/library/sections/%s/all" % viewId
-    if args:
-        url = utils.extend_url(url, args)
-    return DownloadChunks(url)
-
-
-class DownloadChunk(backgroundthread.Task):
     """
     This task will also be executed while library sync is suspended!
     """
@@ -581,7 +565,7 @@ class DownloadChunk(backgroundthread.Task):
         self.url = url
         self.args = args
         self.callback = callback
-        super(DownloadChunk, self).__init__()
+        super(ThreadedDownloadChunk, self).__init__()
 
     def run(self):
         xml = DU().downloadUrl(self.url, parameters=self.args)
@@ -601,14 +585,15 @@ class DownloadGen(object):
 
     Yields XML etree children or raises RuntimeError at the end
     """
-    def __init__(self, url, plex_type=None, last_viewed_at=None,
+    def __init__(self, url, plex_type, last_viewed_at, updated_at, args,
-                 updated_at=None, args=None):
+                 downloader):
+        self._downloader = downloader
         self.successful = True
-        self.args = args or {}
+        self.xml = None
+        self.args = args
         self.args.update({
-            'X-Plex-Container-Size': CONTAINERSIZE,
+            'X-Plex-Container-Start': 0,
-            'sort': 'id',  # Entries are sorted by plex_id
+            'X-Plex-Container-Size': CONTAINERSIZE
-            'excludeAllLeaves': 1  # PMS wont attach a first summary child
         })
         url += '?'
         if plex_type:
@@ -618,8 +603,8 @@ class DownloadGen(object):
         if updated_at:
             url = '%supdatedAt>=%s&' % (url, updated_at)
         self.url = url[:-1]
-        self._download_chunk(start=0)
+        _blocking_download_chunk(self.url, self.args, 0, self.set_xml)
-        self.attrib = deepcopy(self.xml.attrib)
+        self.attrib = self.xml.attrib
         self.current = 0
         self.total = int(self.attrib['totalSize'])
         self.cache_factor = 10
@@ -629,34 +614,24 @@ class DownloadGen(object):
                   self.total + CONTAINERSIZE - self.total % CONTAINERSIZE)
         for pos in range(CONTAINERSIZE, end, CONTAINERSIZE):
             self.pending_counter.append(None)
-            self._download_chunk(start=pos)
+            self._downloader(self.url, self.args, pos, self.on_chunk_downloaded)
 
-    def _download_chunk(self, start):
+    def set_xml(self, xml):
-        self.args['X-Plex-Container-Start'] = start
+        self.xml = xml
-        if start == 0:
-            # We need the result NOW
-            self.xml = DU().downloadUrl(self.url, parameters=self.args)
-            try:
-                self.xml.attrib
-            except AttributeError:
-                LOG.error('Error while downloading chunks: %s, args: %s',
-                          self.url, self.args)
-                raise RuntimeError('Error while downloading chunks for %s'
-                                   % self.url)
-        else:
-            task = DownloadChunk(self.url,
-                                 deepcopy(self.args),  # Beware!
-                                 self.on_chunk_downloaded)
-            backgroundthread.BGThreader.addTask(task)
 
     def on_chunk_downloaded(self, xml):
         if xml is not None:
-            for child in xml:
+            self.xml.extend(xml)
-                self.xml.append(child)
         else:
             self.successful = False
         self.pending_counter.pop()
 
+    def get(self, key, default=None):
+        """
+        Mimick etree xml's way to access xml.attrib via xml.get(key, default)
+        """
+        return self.attrib.get(key, default)
+
     def __iter__(self):
         return self
 
@@ -669,8 +644,11 @@ class DownloadGen(object):
                 if (self.current % CONTAINERSIZE == 0 and
                         self.current <= self.total - (self.cache_factor - 1) * CONTAINERSIZE):
                     self.pending_counter.append(None)
-                    self._download_chunk(
+                    self._downloader(
-                        start=self.current + (self.cache_factor - 1) * CONTAINERSIZE)
+                        self.url,
+                        self.args,
+                        self.current + (self.cache_factor - 1) * CONTAINERSIZE,
+                        self.on_chunk_downloaded)
                 return child
             except IndexError:
                 if not self.pending_counter and not len(self.xml):
@@ -679,46 +657,67 @@ class DownloadGen(object):
                     else:
                         raise StopIteration()
             LOG.debug('Waiting for download to finish')
-            app.APP.monitor.waitForAbort(0.1)
+            if app.APP.monitor.waitForAbort(0.1):
+                raise StopIteration('PKC needs to exit now')
 
     next = __next__
 
-    def get(self, key, default=None):
+
-        return self.attrib.get(key, default)
+def _blocking_download_chunk(url, args, start, callback):
+    """
+    callback will be called with the downloaded xml (fragment)
+    """
+    args['X-Plex-Container-Start'] = start
+    xml = DU().downloadUrl(url, parameters=args)
+    try:
+        xml.attrib
+    except AttributeError:
+        LOG.error('Error while downloading chunks: %s, args: %s',
+                  url, args)
+        raise RuntimeError('Error while downloading chunks for %s'
+                           % url)
+    callback(xml)
 
 
-class SectionItems(DownloadGen):
+def _async_download_chunk(url, args, start, callback):
-    """
+    args['X-Plex-Container-Start'] = start
-    Iterator object to get all items of a Plex library section
+    task = ThreadedDownloadChunk(url,
-    """
+                                 deepcopy(args),  # Beware!
-    def __init__(self, section_id, plex_type=None, last_viewed_at=None,
+                                 callback)
+    backgroundthread.BGThreader.addTask(task)
+
+
+def get_section_iterator(section_id, plex_type=None, last_viewed_at=None,
                          updated_at=None, args=None):
+    args = args or {}
+    args.update({
+        'checkFiles': 0,
+        'includeExtras': 0,         # Trailers and Extras => Extras
+        'includeReviews': 0,
+        'includeRelated': 0,        # Similar movies => Video -> Related
+        'skipRefresh': 1,  # don't scan
+        'excludeAllLeaves': 1  # PMS wont attach a first summary child
+    })
+    if plex_type == v.PLEX_TYPE_ALBUM:
+        # Kodi sorts Newest Albums by their position within the Kodi music
+        # database - great...
+        downloader = _blocking_download_chunk
+        args['sort'] = 'addedAt:asc'
+    else:
+        downloader = _async_download_chunk
+        args['sort'] = 'id'  # Entries are sorted by plex_id
     if plex_type in (v.PLEX_TYPE_EPISODE, v.PLEX_TYPE_SONG):
         # Annoying Plex bug. You won't get all episodes otherwise
         url = '{server}/library/sections/%s/allLeaves' % section_id
         plex_type = None
     else:
         url = '{server}/library/sections/%s/all' % section_id
-        super(SectionItems, self).__init__(url, plex_type, last_viewed_at,
+    return DownloadGen(url,
-                                           updated_at, args)
+                       plex_type,
-
+                       last_viewed_at,
-
+                       updated_at,
-class Children(DownloadGen):
+                       args,
-    """
+                       downloader)
-    Iterator object to get all items of a Plex library section
-    """
-    def __init__(self, plex_id):
-        super(Children, self).__init__(
-            '{server}/library/metadata/%s/children' % plex_id)
-
-
-class Leaves(DownloadGen):
-    """
-    Iterator object to get all items of a Plex library section
-    """
-    def __init__(self, section_id):
-        super(Leaves, self).__init__(
-            '{server}/library/sections/%s/allLeaves' % section_id)
 
 
 def DownloadChunks(url):
@@ -767,37 +766,6 @@ def DownloadChunks(url):
     return xml
 
 
-def GetAllPlexLeaves(viewId, lastViewedAt=None, updatedAt=None):
-    """
-    Returns a list (raw XML API dump) of all Plex subitems for the key.
-    (e.g. /library/sections/2/allLeaves pointing to all TV shows)
-
-    Input:
-        viewId              Id of Plex library, e.g. '2'
-        lastViewedAt        Unix timestamp; only retrieves PMS items viewed
-                            since that point of time until now.
-        updatedAt           Unix timestamp; only retrieves PMS items updated
-                            by the PMS since that point of time until now.
-
-    If lastViewedAt and updatedAt=None, ALL PMS items are returned.
-
-    Warning: lastViewedAt and updatedAt are combined with AND by the PMS!
-
-    Relevant "master time": PMS server. I guess this COULD lead to problems,
-    e.g. when server and client are in different time zones.
-    """
-    args = []
-    url = "{server}/library/sections/%s/allLeaves" % viewId
-
-    if lastViewedAt:
-        args.append('lastViewedAt>=%s' % lastViewedAt)
-    if updatedAt:
-        args.append('updatedAt>=%s' % updatedAt)
-    if args:
-        url += '?' + '&'.join(args)
-    return DownloadChunks(url)
-
-
 def GetPlexOnDeck(viewId):
     """
     """

resources/lib/utils.py

@@ -5,7 +5,7 @@ Various functions and decorators for PKC
 """
 from __future__ import absolute_import, division, unicode_literals
 from logging import getLogger
-from sqlite3 import connect, OperationalError
+from sqlite3 import OperationalError
 from datetime import datetime
 from unicodedata import normalize
 from threading import Lock
@@ -525,50 +525,6 @@ def delete_temporary_subtitles():
                           root, file, err)
 
 
-def kodi_sql(media_type=None):
-    """
-    Open a connection to the Kodi database.
-        media_type: 'video' (standard if not passed), 'plex', 'music', 'texture'
-    """
-    if media_type == "plex":
-        db_path = v.DB_PLEX_PATH
-    elif media_type == "music":
-        db_path = v.DB_MUSIC_PATH
-    elif media_type == "texture":
-        db_path = v.DB_TEXTURE_PATH
-    else:
-        db_path = v.DB_VIDEO_PATH
-    conn = connect(db_path, timeout=30.0)
-    conn.execute('PRAGMA journal_mode=WAL;')
-    conn.execute('PRAGMA cache_size = -8000;')
-    conn.execute('PRAGMA synchronous=NORMAL;')
-    conn.execute('BEGIN')
-    # Use transactions
-    return conn
-
-
-def create_kodi_db_indicees():
-    """
-    Index the "actors" because we got a TON - speed up SELECT and WHEN
-    """
-    conn = kodi_sql('video')
-    cursor = conn.cursor()
-    commands = (
-        'CREATE UNIQUE INDEX IF NOT EXISTS ix_actor_2 ON actor (actor_id);',
-        'CREATE UNIQUE INDEX IF NOT EXISTS ix_files_2 ON files (idFile);',
-    )
-    for cmd in commands:
-        cursor.execute(cmd)
-    # Already used in Kodi >=17: CREATE UNIQUE INDEX ix_actor_1 ON actor (name)
-    # try:
-    #     cursor.execute('CREATE UNIQUE INDEX ix_pkc_actor_index ON actor (name);')
-    # except OperationalError:
-    #     # Index already exists
-    #     pass
-    conn.commit()
-    conn.close()
-
-
 def wipe_synched_playlists():
     """
     Deletes all synched playlist files on the Kodi side; resets the Plex table
@@ -617,12 +573,10 @@ def wipe_database(reboot=True):
         # Plex DB completely empty yet. Wipe existing Kodi music only if we
         # expect to sync Plex music
         music = settings('enableMusic') == 'true'
+    LOG.info("Resetting all cached artwork.")
     kodi_db.wipe_dbs(music)
     plex_db.wipe()
 
-    LOG.info("Resetting all cached artwork.")
-    # Remove all cached artwork
-    kodi_db.reset_cached_images()
     # reset the install run flag
     settings('SyncInstallRunDone', value="false")
     settings('sections_asked_for_machine_identifier', value='')
@@ -644,7 +598,7 @@ def init_dbs():
     # Ensure that Plex DB is set-up
     plex_db.initialize()
     # Hack to speed up look-ups for actors (giant table!)
-    create_kodi_db_indicees()
+    kodi_db.create_kodi_db_indicees()
     kodi_db.setup_kodi_default_entries()
     with kodi_db.KodiVideoDB() as kodidb:
         # Setup the paths for addon-paths (even when using direct paths)

resources/lib/variables.py

@@ -444,6 +444,7 @@ CONTENT_FROM_PLEX_TYPE = {
     PLEX_TYPE_VIDEO: CONTENT_TYPE_VIDEO,
     PLEX_TYPE_PLAYLIST: CONTENT_TYPE_PLAYLIST,
     PLEX_TYPE_CHANNEL: CONTENT_TYPE_FILE,
+    PLEX_TYPE_TAG: CONTENT_TYPE_FILE,
     'mixed': CONTENT_TYPE_SHOW,
     None: CONTENT_TYPE_FILE
 }