Troplo/PlexKodiConnect
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Reduce Python DB overhead

Browse source
This commit is contained in:
croneter 2018-11-08 15:15:52 +01:00
parent 2319d2fc9c
commit 150229061b
6 changed files with 335 additions and 442 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

604
resources/lib/kodidb_functions.py
View file

File diff suppressed because it is too large Load diff

16
resources/lib/kodimonitor.py
View file

@ -599,16 +599,10 @@ def _clean_file_table():
This function tries for at most 5 seconds to clean the file table.
"""
LOG.debug('Start cleaning Kodi files table')
i = 0
while i < 100 and not state.STOP_PKC:
with kodidb.GetKodiDB('video') as kodi_db:
files = kodi_db.obsolete_file_ids()
if files:
break
i += 1
xbmc.sleep(50)
with kodidb.GetKodiDB('video') as kodi_db:
for file_id in files:
xbmc.sleep(2000)
with kodidb.GetKodiDB('video') as kodi_db_1:
with kodidb.GetKodiDB('video') as kodi_db_2:
for file_id in kodi_db_1.obsolete_file_ids():
LOG.debug('Removing obsolete Kodi file_id %s', file_id)
kodi_db.remove_file(file_id[0], remove_orphans=False)
kodi_db_2.remove_file(file_id, remove_orphans=False)
LOG.debug('Done cleaning up Kodi file table')

50
resources/lib/plex_db/common.py
View file

@ -36,8 +36,8 @@ class PlexDBBase(object):
"""
FAST method to check whether a plex_id has already been recorded
"""
query = 'SELECT plex_id FROM %s WHERE plex_id = ?' % plex_type
self.cursor.execute(query, (plex_id, ))
self.cursor.execute('SELECT plex_id FROM %s WHERE plex_id = ?' % plex_type,
(plex_id, ))
return self.cursor.fetchone() is not None
def item_by_id(self, plex_id, plex_type=None):
@ -80,9 +80,9 @@ class PlexDBBase(object):
"""
if kodi_type not in SUPPORTED_KODI_TYPES:
return
query = ('SELECT * from %s WHERE kodi_id = ? LIMIT 1'
% v.PLEX_TYPE_FROM_KODI_TYPE[kodi_type])
self.cursor.execute(query, (kodi_id, ))
self.cursor.execute('SELECT * from %s WHERE kodi_id = ? LIMIT 1'
% v.PLEX_TYPE_FROM_KODI_TYPE[kodi_type],
(kodi_id, ))
method = getattr(self, 'entry_to_%s' % v.PLEX_TYPE_FROM_KODI_TYPE[kodi_type])
return method(self.cursor.fetchone())
@ -90,16 +90,16 @@ class PlexDBBase(object):
"""
Returns an iterator for all items where the last_sync is NOT identical
"""
query = 'SELECT plex_id FROM %s WHERE last_sync <> ?' % plex_type
self.cursor.execute(query, (last_sync, ))
return (x[0] for x in self.cursor)
return (x[0] for x in
self.cursor.execute('SELECT plex_id FROM %s WHERE last_sync <> ?' % plex_type,
(last_sync, )))
def checksum(self, plex_id, plex_type):
"""
Returns the checksum for plex_id
"""
query = 'SELECT checksum FROM %s WHERE plex_id = ?' % plex_type
self.cursor.execute(query, (plex_id, ))
self.cursor.execute('SELECT checksum FROM %s WHERE plex_id = ? LIMIT 1' % plex_type,
(plex_id, ))
try:
return self.cursor.fetchone()[0]
except TypeError:
@ -109,39 +109,36 @@ class PlexDBBase(object):
"""
Sets a new timestamp for plex_id
"""
query = 'UPDATE %s SET last_sync = ? WHERE plex_id = ?' % plex_type
self.cursor.execute(query, (last_sync, plex_id))
self.cursor.execute('UPDATE %s SET last_sync = ? WHERE plex_id = ?' % plex_type,
(last_sync, plex_id))
def remove(self, plex_id, plex_type):
"""
Removes the item from our Plex db
"""
query = 'DELETE FROM ? WHERE plex_id = ?' % plex_type
self.cursor.execute(query, (plex_id, ))
self.cursor.execute('DELETE FROM ? WHERE plex_id = ?' % plex_type, (plex_id, ))
def every_plex_id(self, plex_type):
"""
Returns an iterator for plex_type for every single plex_id
"""
query = 'SELECT plex_id from %s' % plex_type
self.cursor.execute(query)
return (x[0] for x in self.cursor)
return (x[0] for x in
self.cursor.execute('SELECT plex_id from %s' % plex_type))
def missing_fanart(self, plex_type):
"""
Returns an iterator for plex_type for all plex_id, where fanart_synced
has not yet been set to 1
"""
query = 'SELECT plex_id from %s WHERE fanart_synced = 0' % plex_type
self.cursor.execute(query)
return (x[0] for x in self.cursor)
return (x[0] for x in
self.cursor.execute('SELECT plex_id from %s WHERE fanart_synced = 0' % plex_type))
def set_fanart_synced(self, plex_id, plex_type):
"""
Toggles fanart_synced to 1 for plex_id
"""
query = 'UPDATE %s SET fanart_synced = 1 WHERE plex_id = ?' % plex_type
self.cursor.execute(query, (plex_id, ))
self.cursor.execute('UPDATE %s SET fanart_synced = 1 WHERE plex_id = ?' % plex_type,
(plex_id, ))
def initialize():
@ -258,11 +255,8 @@ def wipe():
"""
Completely resets the Plex database
"""
query = "SELECT name FROM sqlite_master WHERE type = 'table'"
with PlexDBBase() as plexdb:
plexdb.cursor.execute(query)
tables = plexdb.cursor.fetchall()
tables = [i[0] for i in tables]
plexdb.cursor.execute("SELECT name FROM sqlite_master WHERE type = 'table'")
tables = [i[0] for i in plexdb.cursor.fetchall()]
for table in tables:
delete_query = 'DROP table IF EXISTS %s' % table
plexdb.cursor.execute(delete_query)
plexdb.cursor.execute('DROP table IF EXISTS %s' % table)

33
resources/lib/plex_db/tvshows.py
View file

@ -10,7 +10,8 @@ class TVShows(object):
"""
Appends or replaces tv show entry into the plex table
"""
query = '''
self.cursor.execute(
'''
INSERT OR REPLACE INTO show(
plex_id,
checksum,
@ -20,9 +21,7 @@ class TVShows(object):
fanart_synced,
last_sync)
VALUES (?, ?, ?, ?, ?, ?, ?)
'''
self.cursor.execute(
query,
''',
(plex_id,
checksum,
section_id,
@ -36,7 +35,8 @@ class TVShows(object):
"""
Appends or replaces an entry into the plex table
"""
query = '''
self.cursor.execute(
'''
INSERT OR REPLACE INTO season(
plex_id,
checksum,
@ -47,9 +47,7 @@ class TVShows(object):
fanart_synced,
last_sync)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
'''
self.cursor.execute(
query,
''',
(plex_id,
checksum,
section_id,
@ -65,7 +63,8 @@ class TVShows(object):
"""
Appends or replaces an entry into the plex table
"""
query = '''
self.cursor.execute(
'''
INSERT OR REPLACE INTO episode(
plex_id,
checksum,
@ -80,9 +79,7 @@ class TVShows(object):
fanart_synced,
last_sync)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
'''
self.cursor.execute(
query,
''',
(plex_id,
checksum,
section_id,
@ -236,24 +233,24 @@ class TVShows(object):
Returns an iterator for all episodes that have a parent season_id with
a value of plex_id
"""
return (self.entry_to_episode(x) for x in
self.cursor.execute('SELECT * FROM episode WHERE season_id = ?',
(plex_id, ))
return (self.entry_to_episode(x) for x in self.cursor)
(plex_id, )))
def episode_by_show(self, plex_id):
"""
Returns an iterator for all episodes that have a grandparent show_id
with a value of plex_id
"""
return (self.entry_to_episode(x) for x in
self.cursor.execute('SELECT * FROM episode WHERE show_id = ?',
(plex_id, ))
return (self.entry_to_episode(x) for x in self.cursor)
(plex_id, )))
def season_by_show(self, plex_id):
"""
Returns an iterator for all seasons that have a parent show_id
with a value of plex_id
"""
return (self.entry_to_season(x) for x in
self.cursor.execute('SELECT * FROM season WHERE show_id = ?',
(plex_id, ))
return (self.entry_to_season(x) for x in self.cursor)
(plex_id, )))

2
resources/lib/sync.py
View file

@ -235,7 +235,7 @@ class Sync(backgroundthread.KillableThread):
# Ensure that Plex DB is set-up
plex_db.initialize()
# Hack to speed up look-ups for actors (giant table!)
utils.create_actor_db_index()
utils.create_kodi_db_indicees()
with kodidb.GetKodiDB('video') as kodi_db:
# Setup the paths for addon-paths (even when using direct paths)
kodi_db.setup_path_table()

14
resources/lib/utils.py
View file

@ -493,20 +493,24 @@ def kodi_sql(media_type=None):
return conn
def create_actor_db_index():
def create_kodi_db_indicees():
"""
Index the "actors" because we got a TON - speed up SELECT and WHEN
"""
return
conn = kodi_sql('video')
cursor = conn.cursor()
try:
cursor.execute("""
CREATE UNIQUE INDEX index_name
ON actor (name);
""")
cursor.execute('CREATE UNIQUE INDEX ix_files_2 ON files (idFile);')
except OperationalError:
# Index already exists
pass
# Already used in Kodi >=17: CREATE UNIQUE INDEX ix_actor_1 ON actor (name)
# try:
# cursor.execute('CREATE UNIQUE INDEX ix_pkc_actor_index ON actor (name);')
# except OperationalError:
# # Index already exists
# pass
conn.commit()
conn.close()