Troplo/PlexKodiConnect
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Increase database resiliance with sqlite WAL mode

Browse source
This commit is contained in:
croneter 2019-01-04 18:02:58 +01:00
parent 1f35caba54
commit 3d4ba1e165
9 changed files with 103 additions and 73 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
resources/lib/itemtypes/common.py
View file

@ -4,8 +4,8 @@ from __future__ import absolute_import, division, unicode_literals
from logging import getLogger
from ntpath import dirname
from ..plex_db import PlexDB
from ..kodi_db import KodiVideoDB
from ..plex_db import PlexDB, PLEXDB_LOCK
from ..kodi_db import KodiVideoDB, KODIDB_LOCK
from .. import utils, timing
LOG = getLogger('PLEX.itemtypes.common')
@ -38,8 +38,9 @@ class ItemBase(object):
Input:
kodiType: optional argument; e.g. 'video' or 'music'
"""
def __init__(self, last_sync, plexdb=None, kodidb=None):
def __init__(self, last_sync, plexdb=None, kodidb=None, lock=True):
self.last_sync = last_sync
self.lock = lock
self.plexconn = None
self.plexcursor = plexdb.cursor if plexdb else None
self.kodiconn = None
@ -53,13 +54,16 @@ class ItemBase(object):
"""
Open DB connections and cursors
"""
if self.lock:
PLEXDB_LOCK.acquire()
KODIDB_LOCK.acquire()
self.plexconn = utils.kodi_sql('plex')
self.plexcursor = self.plexconn.cursor()
self.kodiconn = utils.kodi_sql('video')
self.kodicursor = self.kodiconn.cursor()
self.artconn = utils.kodi_sql('texture')
self.artcursor = self.artconn.cursor()
self.plexdb = PlexDB(self.plexcursor)
self.plexdb = PlexDB(cursor=self.plexcursor)
self.kodidb = KodiVideoDB(texture_db=True,
cursor=self.kodicursor,
artcursor=self.artcursor)
@ -69,16 +73,21 @@ class ItemBase(object):
"""
Make sure DB changes are committed and connection to DB is closed.
"""
try:
if exc_type:
# re-raise any exception
return False
self.plexconn.commit()
self.artconn.commit()
self.kodiconn.commit()
return self
finally:
self.plexconn.close()
self.kodiconn.close()
self.artconn.close()
return self
if self.lock:
PLEXDB_LOCK.release()
KODIDB_LOCK.release()
def commit(self):
self.plexconn.commit()

7
resources/lib/itemtypes/music.py
View file

@ -5,8 +5,8 @@ from logging import getLogger
from .common import ItemBase
from ..plex_api import API
from ..plex_db import PlexDB
from ..kodi_db import KodiMusicDB
from ..plex_db import PlexDB, PLEXDB_LOCK
from ..kodi_db import KodiMusicDB, KODIDB_LOCK
from .. import plex_functions as PF, utils, timing, app, variables as v
LOG = getLogger('PLEX.music')
@ -17,6 +17,9 @@ class MusicMixin(object):
"""
Overwrite to use the Kodi music DB instead of the video DB
"""
if self.lock:
PLEXDB_LOCK.acquire()
KODIDB_LOCK.acquire()
self.plexconn = utils.kodi_sql('plex')
self.plexcursor = self.plexconn.cursor()
self.kodiconn = utils.kodi_sql('music')

1
resources/lib/kodi_db/__init__.py
View file

@ -3,6 +3,7 @@
from __future__ import absolute_import, division, unicode_literals
from logging import getLogger
from .common import KODIDB_LOCK
from .video import KodiVideoDB
from .music import KodiMusicDB
from .texture import KodiTextureDB

41
resources/lib/kodi_db/common.py
View file

@ -1,15 +1,34 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import absolute_import, division, unicode_literals
from threading import Lock
from functools import wraps
from .. import utils, path_ops
from .. import utils, path_ops, app
KODIDB_LOCK = Lock()
def catch_operationalerrors(method):
@wraps(method)
def wrapped(*args, **kwargs):
attempts = 3
while True:
try:
return method(*args, **kwargs)
except utils.OperationalError:
app.APP.monitor.waitForAbort(0.01)
attempts -= 1
if attempts == 0:
raise
return wrapped
class KodiDBBase(object):
"""
Kodi database methods used for all types of items
"""
def __init__(self, texture_db=False, cursor=None, artcursor=None):
def __init__(self, texture_db=False, cursor=None, artcursor=None, lock=True):
"""
Allows direct use with a cursor instead of context mgr
"""
@ -17,8 +36,11 @@ class KodiDBBase(object):
self.cursor = cursor
self.artconn = None
self.artcursor = artcursor
self.lock = lock
def __enter__(self):
if self.lock:
KODIDB_LOCK.acquire()
self.kodiconn = utils.kodi_sql(self.db_kind)
self.cursor = self.kodiconn.cursor()
if self._texture_db:
@ -27,14 +49,19 @@ class KodiDBBase(object):
return self
def __exit__(self, e_typ, e_val, trcbak):
try:
if e_typ:
# re-raise any exception
return False
self.kodiconn.commit()
self.kodiconn.close()
if self.artconn:
self.artconn.commit()
finally:
self.kodiconn.close()
if self.artconn:
self.artconn.close()
if self.lock:
KODIDB_LOCK.release()
def art_urls(self, kodi_id, kodi_type):
return (x[0] for x in
@ -53,6 +80,7 @@ class KodiDBBase(object):
for kodi_art, url in artworks.iteritems():
self.add_art(url, kodi_id, kodi_type, kodi_art)
@catch_operationalerrors
def add_art(self, url, kodi_id, kodi_type, kodi_art):
"""
Adds or modifies the artwork of kind kodi_art (e.g. 'poster') in the
@ -71,6 +99,7 @@ class KodiDBBase(object):
for kodi_art, url in artworks.iteritems():
self.modify_art(url, kodi_id, kodi_type, kodi_art)
@catch_operationalerrors
def modify_art(self, url, kodi_id, kodi_type, kodi_art):
"""
Adds or modifies the artwork of kind kodi_art (e.g. 'poster') in the
@ -102,10 +131,12 @@ class KodiDBBase(object):
''', (url, kodi_id, kodi_type, kodi_art))
def delete_artwork(self, kodi_id, kodi_type):
for row in self.cursor.execute('SELECT url FROM art WHERE media_id = ? AND media_type = ?',
(kodi_id, kodi_type, )):
self.cursor.execute('SELECT url FROM art WHERE media_id = ? AND media_type = ?',
(kodi_id, kodi_type, ))
for row in self.cursor.fetchall():
self.delete_cached_artwork(row[0])
@catch_operationalerrors
def delete_cached_artwork(self, url):
try:
self.artcursor.execute("SELECT cachedurl FROM texture WHERE url = ? LIMIT 1",

10
resources/lib/library_sync/fanart.py
View file

@ -153,15 +153,7 @@ def process_fanart(plex_id, plex_type, refresh=False):
setid,
v.KODI_TYPE_SET)
done = True
except utils.OperationalError:
# We were not fast enough when a sync started
pass
finally:
if done is True and not suspends():
try:
with PlexDB() as plexdb:
plexdb.set_fanart_synced(plex_id,
plex_type)
except utils.OperationalError:
# We were not fast enough when a sync started
pass
plexdb.set_fanart_synced(plex_id, plex_type)

20
resources/lib/library_sync/websocket.py
View file

@ -127,27 +127,11 @@ def process_new_item_message(message):
LOG.error('Could not download metadata for %s', message['plex_id'])
return False, False, False
LOG.debug("Processing new/updated PMS item: %s", message['plex_id'])
attempts = 3
while True:
try:
with itemtypes.ITEMTYPE_FROM_PLEXTYPE[plex_type](timing.unix_timestamp()) as typus:
typus.add_update(xml[0],
section_name=xml.get('librarySectionTitle'),
section_id=xml.get('librarySectionID'))
cache_artwork(message['plex_id'], plex_type)
except utils.OperationalError:
# Since parallel caching of artwork might invalidade the current
# WAL snapshot of the db, sqlite immediatly throws
# OperationalError, NOT after waiting for a duraton of timeout
# See https://github.com/mattn/go-sqlite3/issues/274#issuecomment-211759641
LOG.debug('sqlite OperationalError encountered, trying again')
attempts -= 1
if attempts == 0:
LOG.error('Repeatedly could not process message %s', message)
return False, False, False
continue
else:
break
return True, plex_type in v.PLEX_VIDEOTYPES, plex_type in v.PLEX_AUDIOTYPES
@ -349,9 +333,9 @@ def process_playing(data):
else:
mark_played = False
LOG.debug('Update playstate for user %s for %s with plex id %s to '
'viewCount %s, resume %s, mark_played %s',
'viewCount %s, resume %s, mark_played %s for item %s',
app.ACCOUNT.plex_username, session['kodi_type'], plex_id,
session['viewCount'], resume, mark_played)
session['viewCount'], resume, mark_played, PLAYSTATE_SESSIONS[session_key])
func = itemtypes.ITEMTYPE_FROM_KODITYPE[session['kodi_type']]
with func(None) as fkt:
fkt.update_playstate(mark_played,

10
resources/lib/plex_api.py
View file

@ -929,7 +929,7 @@ class API(object):
artworks[kodi_artwork] = art
if not full_artwork:
return artworks
with PlexDB() as plexdb:
with PlexDB(lock=False) as plexdb:
db_item = plexdb.item_by_id(self.plex_id(),
v.PLEX_TYPE_EPISODE)
if db_item:
@ -938,12 +938,12 @@ class API(object):
else:
return artworks
# Grab artwork from the season
with KodiVideoDB() as kodidb:
with KodiVideoDB(lock=False) as kodidb:
season_art = kodidb.get_art(season_id, v.KODI_TYPE_SEASON)
for kodi_art in season_art:
artworks['season.%s' % kodi_art] = season_art[kodi_art]
# Grab more artwork from the show
with KodiVideoDB() as kodidb:
with KodiVideoDB(lock=False) as kodidb:
show_art = kodidb.get_art(show_id, v.KODI_TYPE_SHOW)
for kodi_art in show_art:
artworks['tvshow.%s' % kodi_art] = show_art[kodi_art]
@ -952,10 +952,10 @@ class API(object):
if kodi_id:
# in Kodi database, potentially with additional e.g. clearart
if self.plex_type() in v.PLEX_VIDEOTYPES:
with KodiVideoDB() as kodidb:
with KodiVideoDB(lock=False) as kodidb:
return kodidb.get_art(kodi_id, kodi_type)
else:
with KodiMusicDB() as kodidb:
with KodiMusicDB(lock=False) as kodidb:
return kodidb.get_art(kodi_id, kodi_type)
# Grab artwork from Plex

2
resources/lib/plex_db/__init__.py
View file

@ -2,7 +2,7 @@
# -*- coding: utf-8 -*-
from __future__ import absolute_import, division, unicode_literals
from .common import PlexDBBase, initialize, wipe
from .common import PlexDBBase, initialize, wipe, PLEXDB_LOCK
from .tvshows import TVShows
from .movies import Movies
from .music import Music

12
resources/lib/plex_db/common.py
View file

@ -1,9 +1,12 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import absolute_import, division, unicode_literals
from threading import Lock
from .. import utils, variables as v
PLEXDB_LOCK = Lock()
SUPPORTED_KODI_TYPES = (
v.KODI_TYPE_MOVIE,
v.KODI_TYPE_SHOW,
@ -19,21 +22,28 @@ class PlexDBBase(object):
"""
Plex database methods used for all types of items
"""
def __init__(self, cursor=None):
def __init__(self, cursor=None, lock=True):
# Allows us to use this class with a cursor instead of context mgr
self.cursor = cursor
self.lock = lock
def __enter__(self):
if self.lock:
PLEXDB_LOCK.acquire()
self.plexconn = utils.kodi_sql('plex')
self.cursor = self.plexconn.cursor()
return self
def __exit__(self, e_typ, e_val, trcbak):
try:
if e_typ:
# re-raise any exception
return False
self.plexconn.commit()
finally:
self.plexconn.close()
if self.lock:
PLEXDB_LOCK.release()
def is_recorded(self, plex_id, plex_type):
"""