Try sqlite3 in-memory fullsync, might be faster

resources/lib/itemtypes.py

@@ -12,7 +12,7 @@ import xbmcgui
 
 import artwork
 from utils import tryEncode, tryDecode, settings, window, kodiSQL, \
-    CatchExceptions
+    CatchExceptions, copy_database
 import embydb_functions as embydb
 import kodidb_functions as kodidb
 
@@ -33,23 +33,35 @@ class Items(object):
 
     Input:
         kodiType:       optional argument; e.g. 'video' or 'music'
+        inmemory:       Do the work in-memory (copying the Kodi DB around!)
     """
 
-    def __init__(self):
+    def __init__(self, inmemory=False):
         self.kodiversion = int(xbmc.getInfoLabel("System.BuildVersion")[:2])
         self.directpath = window('useDirectPaths') == 'true'
 
         self.artwork = artwork.Artwork()
         self.userid = window('currUserId')
         self.server = window('pms_server')
+        self.inmemory = inmemory
 
     def __enter__(self):
         """
         Open DB connections and cursors
         """
-        self.embyconn = kodiSQL('emby')
+        if self.inmemory is True:
+            # Copy the databases into memory first
+            log.info('Processing in-memory!')
+            embyconn = kodiSQL('emby')
+            self.embyconn = copy_database(embyconn, dest_conn=':memory:')
+            embyconn.close()
+            kodiconn = kodiSQL('video')
+            self.kodiconn = copy_database(kodiconn, dest_conn=':memory:')
+            kodiconn.close()
+        else:
+            self.embyconn = kodiSQL('emby')
+            self.kodiconn = kodiSQL('video')
         self.embycursor = self.embyconn.cursor()
-        self.kodiconn = kodiSQL('video')
         self.kodicursor = self.kodiconn.cursor()
         self.emby_db = embydb.Embydb_Functions(self.embycursor)
         self.kodi_db = kodidb.Kodidb_Functions(self.kodicursor)
@@ -61,6 +73,15 @@ class Items(object):
         """
         self.embyconn.commit()
         self.kodiconn.commit()
+        if self.inmemory is True:
+            # Copy the in-memory db back to disk
+            embyconn = kodiSQL('emby')
+            self.embyconn = copy_database(self.embyconn, embyconn)
+            embyconn.close()
+            kodiconn = kodiSQL('video')
+            self.kodiconn = copy_database(self.kodiconn, kodiconn)
+            kodiconn.close()
+            log.info('Processed in-memory finished')
         self.embyconn.close()
         self.kodiconn.close()
         return self

resources/lib/librarysync.py

@@ -143,11 +143,13 @@ class ThreadedProcessMetadata(Thread):
         itemType:   as used to call functions in itemtypes.py
                     e.g. 'Movies' => itemtypes.Movies()
         lock:       Lock(), used for counting where we are
+        inmemory:   Do the work in-memory (copying the Kodi DB around!)
     """
-    def __init__(self, queue, itemType, lock):
+    def __init__(self, queue, itemType, lock, inmemory=False):
         self.queue = queue
         self.lock = lock
         self.itemType = itemType
+        self.inmemory = inmemory
         Thread.__init__(self)
 
     def terminateNow(self):
@@ -170,7 +172,7 @@ class ThreadedProcessMetadata(Thread):
         threadStopped = self.threadStopped
         global processMetadataCount
         global processingViewName
-        with itemFkt() as item:
+        with itemFkt(inmemory=self.inmemory) as item:
             while threadStopped() is False:
                 # grabs item from queue
                 try:
@@ -985,14 +987,17 @@ class LibrarySync(Thread):
             thread.start()
             threads.append(thread)
         log.info("%s download threads spawned" % len(threads))
+        # Process the Kodi DB in-memory if we got a lot of items
+        inmemory = True if itemNumber > 50 else False
         # Spawn one more thread to process Metadata, once downloaded
         thread = ThreadedProcessMetadata(processMetadataQueue,
                                          itemType,
-                                         processMetadataLock)
+                                         processMetadataLock,
+                                         inmemory=inmemory)
         thread.setDaemon(True)
         thread.start()
         threads.append(thread)
-        log.info("Processing thread spawned")
+        log.info("Processing thread spawned with inmemory=%s" % inmemory)
         # Start one thread to show sync progress ONLY for new PMS items
         if self.new_items_only is True and window('dbSyncIndicator') == 'true':
             dialog = xbmcgui.DialogProgressBG()

resources/lib/utils.py

@@ -212,6 +212,20 @@ def kodiSQL(media_type="video"):
     connection = sqlite3.connect(dbPath, timeout=15.0)
     return connection
 
+
+def copy_database(source_conn, dest_conn=':memory:'):
+    '''
+    Returns a connection to a new copy of an existing database. Raises an
+    sqlite3.OperationalError if the destination already exists.
+
+    dest_conn=':memory:'      causes the destination db to reside in-memory
+    '''
+    if dest_conn == ':memory:':
+        dest_conn = sqlite3.connect(dest_conn, timeout=15.0)
+    dest_conn.executescript(''.join(source_conn.iterdump()))
+    return dest_conn
+
+
 def getKodiVideoDBPath():
 
     dbVersion = {