Reduce Python DB overhead

resources/lib/kodimonitor.py

@@ -599,16 +599,10 @@ def _clean_file_table():
     This function tries for at most 5 seconds to clean the file table.
     """
     LOG.debug('Start cleaning Kodi files table')
-    i = 0
+    xbmc.sleep(2000)
-    while i < 100 and not state.STOP_PKC:
+    with kodidb.GetKodiDB('video') as kodi_db_1:
-        with kodidb.GetKodiDB('video') as kodi_db:
+        with kodidb.GetKodiDB('video') as kodi_db_2:
-            files = kodi_db.obsolete_file_ids()
+            for file_id in kodi_db_1.obsolete_file_ids():
-        if files:
+                LOG.debug('Removing obsolete Kodi file_id %s', file_id)
-            break
+                kodi_db_2.remove_file(file_id, remove_orphans=False)
-        i += 1
-        xbmc.sleep(50)
-    with kodidb.GetKodiDB('video') as kodi_db:
-        for file_id in files:
-            LOG.debug('Removing obsolete Kodi file_id %s', file_id)
-            kodi_db.remove_file(file_id[0], remove_orphans=False)
     LOG.debug('Done cleaning up Kodi file table')

resources/lib/plex_db/common.py

@@ -36,8 +36,8 @@ class PlexDBBase(object):
         """
         FAST method to check whether a plex_id has already been recorded
         """
-        query = 'SELECT plex_id FROM %s WHERE plex_id = ?' % plex_type
+        self.cursor.execute('SELECT plex_id FROM %s WHERE plex_id = ?' % plex_type,
-        self.cursor.execute(query, (plex_id, ))
+                            (plex_id, ))
         return self.cursor.fetchone() is not None
 
     def item_by_id(self, plex_id, plex_type=None):
@@ -80,9 +80,9 @@ class PlexDBBase(object):
         """
         if kodi_type not in SUPPORTED_KODI_TYPES:
             return
-        query = ('SELECT * from %s WHERE kodi_id = ? LIMIT 1'
+        self.cursor.execute('SELECT * from %s WHERE kodi_id = ? LIMIT 1'
-                 % v.PLEX_TYPE_FROM_KODI_TYPE[kodi_type])
+                            % v.PLEX_TYPE_FROM_KODI_TYPE[kodi_type],
-        self.cursor.execute(query, (kodi_id, ))
+                            (kodi_id, ))
         method = getattr(self, 'entry_to_%s' % v.PLEX_TYPE_FROM_KODI_TYPE[kodi_type])
         return method(self.cursor.fetchone())
 
@@ -90,16 +90,16 @@ class PlexDBBase(object):
         """
         Returns an iterator for all items where the last_sync is NOT identical
         """
-        query = 'SELECT plex_id FROM %s WHERE last_sync <> ?' % plex_type
+        return (x[0] for x in
-        self.cursor.execute(query, (last_sync, ))
+                self.cursor.execute('SELECT plex_id FROM %s WHERE last_sync <> ?' % plex_type,
-        return (x[0] for x in self.cursor)
+                                    (last_sync, )))
 
     def checksum(self, plex_id, plex_type):
         """
         Returns the checksum for plex_id
         """
-        query = 'SELECT checksum FROM %s WHERE plex_id = ?' % plex_type
+        self.cursor.execute('SELECT checksum FROM %s WHERE plex_id = ? LIMIT 1' % plex_type,
-        self.cursor.execute(query, (plex_id, ))
+                            (plex_id, ))
         try:
             return self.cursor.fetchone()[0]
         except TypeError:
@@ -109,39 +109,36 @@ class PlexDBBase(object):
         """
         Sets a new timestamp for plex_id
         """
-        query = 'UPDATE %s SET last_sync = ? WHERE plex_id = ?' % plex_type
+        self.cursor.execute('UPDATE %s SET last_sync = ? WHERE plex_id = ?' % plex_type,
-        self.cursor.execute(query, (last_sync, plex_id))
+                            (last_sync, plex_id))
 
     def remove(self, plex_id, plex_type):
         """
         Removes the item from our Plex db
         """
-        query = 'DELETE FROM ? WHERE plex_id = ?' % plex_type
+        self.cursor.execute('DELETE FROM ? WHERE plex_id = ?' % plex_type, (plex_id, ))
-        self.cursor.execute(query, (plex_id, ))
 
     def every_plex_id(self, plex_type):
         """
         Returns an iterator for plex_type for every single plex_id
         """
-        query = 'SELECT plex_id from %s' % plex_type
+        return (x[0] for x in
-        self.cursor.execute(query)
+                self.cursor.execute('SELECT plex_id from %s' % plex_type))
-        return (x[0] for x in self.cursor)
 
     def missing_fanart(self, plex_type):
         """
         Returns an iterator for plex_type for all plex_id, where fanart_synced
         has not yet been set to 1
         """
-        query = 'SELECT plex_id from %s WHERE fanart_synced = 0' % plex_type
+        return (x[0] for x in
-        self.cursor.execute(query)
+                self.cursor.execute('SELECT plex_id from %s WHERE fanart_synced = 0' % plex_type))
-        return (x[0] for x in self.cursor)
 
     def set_fanart_synced(self, plex_id, plex_type):
         """
         Toggles fanart_synced to 1 for plex_id
         """
-        query = 'UPDATE %s SET fanart_synced = 1 WHERE plex_id = ?' % plex_type
+        self.cursor.execute('UPDATE %s SET fanart_synced = 1 WHERE plex_id = ?' % plex_type,
-        self.cursor.execute(query, (plex_id, ))
+                            (plex_id, ))
 
 
 def initialize():
@@ -258,11 +255,8 @@ def wipe():
     """
     Completely resets the Plex database
     """
-    query = "SELECT name FROM sqlite_master WHERE type = 'table'"
     with PlexDBBase() as plexdb:
-        plexdb.cursor.execute(query)
+        plexdb.cursor.execute("SELECT name FROM sqlite_master WHERE type = 'table'")
-        tables = plexdb.cursor.fetchall()
+        tables = [i[0] for i in plexdb.cursor.fetchall()]
-        tables = [i[0] for i in tables]
         for table in tables:
-            delete_query = 'DROP table IF EXISTS %s' % table
+            plexdb.cursor.execute('DROP table IF EXISTS %s' % table)
-            plexdb.cursor.execute(delete_query)

resources/lib/plex_db/tvshows.py

@@ -10,7 +10,8 @@ class TVShows(object):
         """
         Appends or replaces tv show entry into the plex table
         """
-        query = '''
+        self.cursor.execute(
+            '''
             INSERT OR REPLACE INTO show(
                 plex_id,
                 checksum,
@@ -20,9 +21,7 @@ class TVShows(object):
                 fanart_synced,
                 last_sync)
             VALUES (?, ?, ?, ?, ?, ?, ?)
-        '''
+            ''',
-        self.cursor.execute(
-            query,
             (plex_id,
              checksum,
              section_id,
@@ -36,7 +35,8 @@ class TVShows(object):
         """
         Appends or replaces an entry into the plex table
         """
-        query = '''
+        self.cursor.execute(
+            '''
             INSERT OR REPLACE INTO season(
                 plex_id,
                 checksum,
@@ -47,9 +47,7 @@ class TVShows(object):
                 fanart_synced,
                 last_sync)
             VALUES (?, ?, ?, ?, ?, ?, ?, ?)
-        '''
+            ''',
-        self.cursor.execute(
-            query,
             (plex_id,
              checksum,
              section_id,
@@ -65,7 +63,8 @@ class TVShows(object):
         """
         Appends or replaces an entry into the plex table
         """
-        query = '''
+        self.cursor.execute(
+            '''
             INSERT OR REPLACE INTO episode(
                 plex_id,
                 checksum,
@@ -80,9 +79,7 @@ class TVShows(object):
                 fanart_synced,
                 last_sync)
             VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-            '''
+            ''',
-        self.cursor.execute(
-            query,
             (plex_id,
              checksum,
              section_id,
@@ -236,24 +233,24 @@ class TVShows(object):
         Returns an iterator for all episodes that have a parent season_id with
         a value of plex_id
         """
-        self.cursor.execute('SELECT * FROM episode WHERE season_id = ?',
+        return (self.entry_to_episode(x) for x in
-                            (plex_id, ))
+                self.cursor.execute('SELECT * FROM episode WHERE season_id = ?',
-        return (self.entry_to_episode(x) for x in self.cursor)
+                                    (plex_id, )))
 
     def episode_by_show(self, plex_id):
         """
         Returns an iterator for all episodes that have a grandparent show_id
         with a value of plex_id
         """
-        self.cursor.execute('SELECT * FROM episode WHERE show_id = ?',
+        return (self.entry_to_episode(x) for x in
-                            (plex_id, ))
+                self.cursor.execute('SELECT * FROM episode WHERE show_id = ?',
-        return (self.entry_to_episode(x) for x in self.cursor)
+                                    (plex_id, )))
 
     def season_by_show(self, plex_id):
         """
         Returns an iterator for all seasons that have a parent show_id
         with a value of plex_id
         """
-        self.cursor.execute('SELECT * FROM season WHERE show_id = ?',
+        return (self.entry_to_season(x) for x in
-                            (plex_id, ))
+                self.cursor.execute('SELECT * FROM season WHERE show_id = ?',
-        return (self.entry_to_season(x) for x in self.cursor)
+                                    (plex_id, )))

resources/lib/sync.py

@@ -235,7 +235,7 @@ class Sync(backgroundthread.KillableThread):
         # Ensure that Plex DB is set-up
         plex_db.initialize()
         # Hack to speed up look-ups for actors (giant table!)
-        utils.create_actor_db_index()
+        utils.create_kodi_db_indicees()
         with kodidb.GetKodiDB('video') as kodi_db:
             # Setup the paths for addon-paths (even when using direct paths)
             kodi_db.setup_path_table()

resources/lib/utils.py

@@ -493,20 +493,24 @@ def kodi_sql(media_type=None):
     return conn
 
 
-def create_actor_db_index():
+def create_kodi_db_indicees():
     """
     Index the "actors" because we got a TON - speed up SELECT and WHEN
     """
+    return
     conn = kodi_sql('video')
     cursor = conn.cursor()
     try:
-        cursor.execute("""
+        cursor.execute('CREATE UNIQUE INDEX ix_files_2 ON files (idFile);')
-            CREATE UNIQUE INDEX index_name
-            ON actor (name);
-        """)
     except OperationalError:
         # Index already exists
         pass
+    # Already used in Kodi >=17: CREATE UNIQUE INDEX ix_actor_1 ON actor (name)
+    # try:
+    #     cursor.execute('CREATE UNIQUE INDEX ix_pkc_actor_index ON actor (name);')
+    # except OperationalError:
+    #     # Index already exists
+    #     pass
     conn.commit()
     conn.close()