Increase database resiliance with sqlite WAL mode

This commit is contained in:
croneter 2019-01-04 18:02:58 +01:00
parent 1f35caba54
commit 3d4ba1e165
9 changed files with 103 additions and 73 deletions

View file

@ -4,8 +4,8 @@ from __future__ import absolute_import, division, unicode_literals
from logging import getLogger from logging import getLogger
from ntpath import dirname from ntpath import dirname
from ..plex_db import PlexDB from ..plex_db import PlexDB, PLEXDB_LOCK
from ..kodi_db import KodiVideoDB from ..kodi_db import KodiVideoDB, KODIDB_LOCK
from .. import utils, timing from .. import utils, timing
LOG = getLogger('PLEX.itemtypes.common') LOG = getLogger('PLEX.itemtypes.common')
@ -38,8 +38,9 @@ class ItemBase(object):
Input: Input:
kodiType: optional argument; e.g. 'video' or 'music' kodiType: optional argument; e.g. 'video' or 'music'
""" """
def __init__(self, last_sync, plexdb=None, kodidb=None): def __init__(self, last_sync, plexdb=None, kodidb=None, lock=True):
self.last_sync = last_sync self.last_sync = last_sync
self.lock = lock
self.plexconn = None self.plexconn = None
self.plexcursor = plexdb.cursor if plexdb else None self.plexcursor = plexdb.cursor if plexdb else None
self.kodiconn = None self.kodiconn = None
@ -53,13 +54,16 @@ class ItemBase(object):
""" """
Open DB connections and cursors Open DB connections and cursors
""" """
if self.lock:
PLEXDB_LOCK.acquire()
KODIDB_LOCK.acquire()
self.plexconn = utils.kodi_sql('plex') self.plexconn = utils.kodi_sql('plex')
self.plexcursor = self.plexconn.cursor() self.plexcursor = self.plexconn.cursor()
self.kodiconn = utils.kodi_sql('video') self.kodiconn = utils.kodi_sql('video')
self.kodicursor = self.kodiconn.cursor() self.kodicursor = self.kodiconn.cursor()
self.artconn = utils.kodi_sql('texture') self.artconn = utils.kodi_sql('texture')
self.artcursor = self.artconn.cursor() self.artcursor = self.artconn.cursor()
self.plexdb = PlexDB(self.plexcursor) self.plexdb = PlexDB(cursor=self.plexcursor)
self.kodidb = KodiVideoDB(texture_db=True, self.kodidb = KodiVideoDB(texture_db=True,
cursor=self.kodicursor, cursor=self.kodicursor,
artcursor=self.artcursor) artcursor=self.artcursor)
@ -69,16 +73,21 @@ class ItemBase(object):
""" """
Make sure DB changes are committed and connection to DB is closed. Make sure DB changes are committed and connection to DB is closed.
""" """
try:
if exc_type: if exc_type:
# re-raise any exception # re-raise any exception
return False return False
self.plexconn.commit() self.plexconn.commit()
self.artconn.commit() self.artconn.commit()
self.kodiconn.commit() self.kodiconn.commit()
return self
finally:
self.plexconn.close() self.plexconn.close()
self.kodiconn.close() self.kodiconn.close()
self.artconn.close() self.artconn.close()
return self if self.lock:
PLEXDB_LOCK.release()
KODIDB_LOCK.release()
def commit(self): def commit(self):
self.plexconn.commit() self.plexconn.commit()

View file

@ -5,8 +5,8 @@ from logging import getLogger
from .common import ItemBase from .common import ItemBase
from ..plex_api import API from ..plex_api import API
from ..plex_db import PlexDB from ..plex_db import PlexDB, PLEXDB_LOCK
from ..kodi_db import KodiMusicDB from ..kodi_db import KodiMusicDB, KODIDB_LOCK
from .. import plex_functions as PF, utils, timing, app, variables as v from .. import plex_functions as PF, utils, timing, app, variables as v
LOG = getLogger('PLEX.music') LOG = getLogger('PLEX.music')
@ -17,6 +17,9 @@ class MusicMixin(object):
""" """
Overwrite to use the Kodi music DB instead of the video DB Overwrite to use the Kodi music DB instead of the video DB
""" """
if self.lock:
PLEXDB_LOCK.acquire()
KODIDB_LOCK.acquire()
self.plexconn = utils.kodi_sql('plex') self.plexconn = utils.kodi_sql('plex')
self.plexcursor = self.plexconn.cursor() self.plexcursor = self.plexconn.cursor()
self.kodiconn = utils.kodi_sql('music') self.kodiconn = utils.kodi_sql('music')

View file

@ -3,6 +3,7 @@
from __future__ import absolute_import, division, unicode_literals from __future__ import absolute_import, division, unicode_literals
from logging import getLogger from logging import getLogger
from .common import KODIDB_LOCK
from .video import KodiVideoDB from .video import KodiVideoDB
from .music import KodiMusicDB from .music import KodiMusicDB
from .texture import KodiTextureDB from .texture import KodiTextureDB

View file

@ -1,15 +1,34 @@
#!/usr/bin/env python #!/usr/bin/env python
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
from __future__ import absolute_import, division, unicode_literals from __future__ import absolute_import, division, unicode_literals
from threading import Lock
from functools import wraps
from .. import utils, path_ops from .. import utils, path_ops, app
KODIDB_LOCK = Lock()
def catch_operationalerrors(method):
@wraps(method)
def wrapped(*args, **kwargs):
attempts = 3
while True:
try:
return method(*args, **kwargs)
except utils.OperationalError:
app.APP.monitor.waitForAbort(0.01)
attempts -= 1
if attempts == 0:
raise
return wrapped
class KodiDBBase(object): class KodiDBBase(object):
""" """
Kodi database methods used for all types of items Kodi database methods used for all types of items
""" """
def __init__(self, texture_db=False, cursor=None, artcursor=None): def __init__(self, texture_db=False, cursor=None, artcursor=None, lock=True):
""" """
Allows direct use with a cursor instead of context mgr Allows direct use with a cursor instead of context mgr
""" """
@ -17,8 +36,11 @@ class KodiDBBase(object):
self.cursor = cursor self.cursor = cursor
self.artconn = None self.artconn = None
self.artcursor = artcursor self.artcursor = artcursor
self.lock = lock
def __enter__(self): def __enter__(self):
if self.lock:
KODIDB_LOCK.acquire()
self.kodiconn = utils.kodi_sql(self.db_kind) self.kodiconn = utils.kodi_sql(self.db_kind)
self.cursor = self.kodiconn.cursor() self.cursor = self.kodiconn.cursor()
if self._texture_db: if self._texture_db:
@ -27,14 +49,19 @@ class KodiDBBase(object):
return self return self
def __exit__(self, e_typ, e_val, trcbak): def __exit__(self, e_typ, e_val, trcbak):
try:
if e_typ: if e_typ:
# re-raise any exception # re-raise any exception
return False return False
self.kodiconn.commit() self.kodiconn.commit()
self.kodiconn.close()
if self.artconn: if self.artconn:
self.artconn.commit() self.artconn.commit()
finally:
self.kodiconn.close()
if self.artconn:
self.artconn.close() self.artconn.close()
if self.lock:
KODIDB_LOCK.release()
def art_urls(self, kodi_id, kodi_type): def art_urls(self, kodi_id, kodi_type):
return (x[0] for x in return (x[0] for x in
@ -53,6 +80,7 @@ class KodiDBBase(object):
for kodi_art, url in artworks.iteritems(): for kodi_art, url in artworks.iteritems():
self.add_art(url, kodi_id, kodi_type, kodi_art) self.add_art(url, kodi_id, kodi_type, kodi_art)
@catch_operationalerrors
def add_art(self, url, kodi_id, kodi_type, kodi_art): def add_art(self, url, kodi_id, kodi_type, kodi_art):
""" """
Adds or modifies the artwork of kind kodi_art (e.g. 'poster') in the Adds or modifies the artwork of kind kodi_art (e.g. 'poster') in the
@ -71,6 +99,7 @@ class KodiDBBase(object):
for kodi_art, url in artworks.iteritems(): for kodi_art, url in artworks.iteritems():
self.modify_art(url, kodi_id, kodi_type, kodi_art) self.modify_art(url, kodi_id, kodi_type, kodi_art)
@catch_operationalerrors
def modify_art(self, url, kodi_id, kodi_type, kodi_art): def modify_art(self, url, kodi_id, kodi_type, kodi_art):
""" """
Adds or modifies the artwork of kind kodi_art (e.g. 'poster') in the Adds or modifies the artwork of kind kodi_art (e.g. 'poster') in the
@ -102,10 +131,12 @@ class KodiDBBase(object):
''', (url, kodi_id, kodi_type, kodi_art)) ''', (url, kodi_id, kodi_type, kodi_art))
def delete_artwork(self, kodi_id, kodi_type): def delete_artwork(self, kodi_id, kodi_type):
for row in self.cursor.execute('SELECT url FROM art WHERE media_id = ? AND media_type = ?', self.cursor.execute('SELECT url FROM art WHERE media_id = ? AND media_type = ?',
(kodi_id, kodi_type, )): (kodi_id, kodi_type, ))
for row in self.cursor.fetchall():
self.delete_cached_artwork(row[0]) self.delete_cached_artwork(row[0])
@catch_operationalerrors
def delete_cached_artwork(self, url): def delete_cached_artwork(self, url):
try: try:
self.artcursor.execute("SELECT cachedurl FROM texture WHERE url = ? LIMIT 1", self.artcursor.execute("SELECT cachedurl FROM texture WHERE url = ? LIMIT 1",

View file

@ -153,15 +153,7 @@ def process_fanart(plex_id, plex_type, refresh=False):
setid, setid,
v.KODI_TYPE_SET) v.KODI_TYPE_SET)
done = True done = True
except utils.OperationalError:
# We were not fast enough when a sync started
pass
finally: finally:
if done is True and not suspends(): if done is True and not suspends():
try:
with PlexDB() as plexdb: with PlexDB() as plexdb:
plexdb.set_fanart_synced(plex_id, plexdb.set_fanart_synced(plex_id, plex_type)
plex_type)
except utils.OperationalError:
# We were not fast enough when a sync started
pass

View file

@ -127,27 +127,11 @@ def process_new_item_message(message):
LOG.error('Could not download metadata for %s', message['plex_id']) LOG.error('Could not download metadata for %s', message['plex_id'])
return False, False, False return False, False, False
LOG.debug("Processing new/updated PMS item: %s", message['plex_id']) LOG.debug("Processing new/updated PMS item: %s", message['plex_id'])
attempts = 3
while True:
try:
with itemtypes.ITEMTYPE_FROM_PLEXTYPE[plex_type](timing.unix_timestamp()) as typus: with itemtypes.ITEMTYPE_FROM_PLEXTYPE[plex_type](timing.unix_timestamp()) as typus:
typus.add_update(xml[0], typus.add_update(xml[0],
section_name=xml.get('librarySectionTitle'), section_name=xml.get('librarySectionTitle'),
section_id=xml.get('librarySectionID')) section_id=xml.get('librarySectionID'))
cache_artwork(message['plex_id'], plex_type) cache_artwork(message['plex_id'], plex_type)
except utils.OperationalError:
# Since parallel caching of artwork might invalidade the current
# WAL snapshot of the db, sqlite immediatly throws
# OperationalError, NOT after waiting for a duraton of timeout
# See https://github.com/mattn/go-sqlite3/issues/274#issuecomment-211759641
LOG.debug('sqlite OperationalError encountered, trying again')
attempts -= 1
if attempts == 0:
LOG.error('Repeatedly could not process message %s', message)
return False, False, False
continue
else:
break
return True, plex_type in v.PLEX_VIDEOTYPES, plex_type in v.PLEX_AUDIOTYPES return True, plex_type in v.PLEX_VIDEOTYPES, plex_type in v.PLEX_AUDIOTYPES
@ -349,9 +333,9 @@ def process_playing(data):
else: else:
mark_played = False mark_played = False
LOG.debug('Update playstate for user %s for %s with plex id %s to ' LOG.debug('Update playstate for user %s for %s with plex id %s to '
'viewCount %s, resume %s, mark_played %s', 'viewCount %s, resume %s, mark_played %s for item %s',
app.ACCOUNT.plex_username, session['kodi_type'], plex_id, app.ACCOUNT.plex_username, session['kodi_type'], plex_id,
session['viewCount'], resume, mark_played) session['viewCount'], resume, mark_played, PLAYSTATE_SESSIONS[session_key])
func = itemtypes.ITEMTYPE_FROM_KODITYPE[session['kodi_type']] func = itemtypes.ITEMTYPE_FROM_KODITYPE[session['kodi_type']]
with func(None) as fkt: with func(None) as fkt:
fkt.update_playstate(mark_played, fkt.update_playstate(mark_played,

View file

@ -929,7 +929,7 @@ class API(object):
artworks[kodi_artwork] = art artworks[kodi_artwork] = art
if not full_artwork: if not full_artwork:
return artworks return artworks
with PlexDB() as plexdb: with PlexDB(lock=False) as plexdb:
db_item = plexdb.item_by_id(self.plex_id(), db_item = plexdb.item_by_id(self.plex_id(),
v.PLEX_TYPE_EPISODE) v.PLEX_TYPE_EPISODE)
if db_item: if db_item:
@ -938,12 +938,12 @@ class API(object):
else: else:
return artworks return artworks
# Grab artwork from the season # Grab artwork from the season
with KodiVideoDB() as kodidb: with KodiVideoDB(lock=False) as kodidb:
season_art = kodidb.get_art(season_id, v.KODI_TYPE_SEASON) season_art = kodidb.get_art(season_id, v.KODI_TYPE_SEASON)
for kodi_art in season_art: for kodi_art in season_art:
artworks['season.%s' % kodi_art] = season_art[kodi_art] artworks['season.%s' % kodi_art] = season_art[kodi_art]
# Grab more artwork from the show # Grab more artwork from the show
with KodiVideoDB() as kodidb: with KodiVideoDB(lock=False) as kodidb:
show_art = kodidb.get_art(show_id, v.KODI_TYPE_SHOW) show_art = kodidb.get_art(show_id, v.KODI_TYPE_SHOW)
for kodi_art in show_art: for kodi_art in show_art:
artworks['tvshow.%s' % kodi_art] = show_art[kodi_art] artworks['tvshow.%s' % kodi_art] = show_art[kodi_art]
@ -952,10 +952,10 @@ class API(object):
if kodi_id: if kodi_id:
# in Kodi database, potentially with additional e.g. clearart # in Kodi database, potentially with additional e.g. clearart
if self.plex_type() in v.PLEX_VIDEOTYPES: if self.plex_type() in v.PLEX_VIDEOTYPES:
with KodiVideoDB() as kodidb: with KodiVideoDB(lock=False) as kodidb:
return kodidb.get_art(kodi_id, kodi_type) return kodidb.get_art(kodi_id, kodi_type)
else: else:
with KodiMusicDB() as kodidb: with KodiMusicDB(lock=False) as kodidb:
return kodidb.get_art(kodi_id, kodi_type) return kodidb.get_art(kodi_id, kodi_type)
# Grab artwork from Plex # Grab artwork from Plex

View file

@ -2,7 +2,7 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
from __future__ import absolute_import, division, unicode_literals from __future__ import absolute_import, division, unicode_literals
from .common import PlexDBBase, initialize, wipe from .common import PlexDBBase, initialize, wipe, PLEXDB_LOCK
from .tvshows import TVShows from .tvshows import TVShows
from .movies import Movies from .movies import Movies
from .music import Music from .music import Music

View file

@ -1,9 +1,12 @@
#!/usr/bin/env python #!/usr/bin/env python
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
from __future__ import absolute_import, division, unicode_literals from __future__ import absolute_import, division, unicode_literals
from threading import Lock
from .. import utils, variables as v from .. import utils, variables as v
PLEXDB_LOCK = Lock()
SUPPORTED_KODI_TYPES = ( SUPPORTED_KODI_TYPES = (
v.KODI_TYPE_MOVIE, v.KODI_TYPE_MOVIE,
v.KODI_TYPE_SHOW, v.KODI_TYPE_SHOW,
@ -19,21 +22,28 @@ class PlexDBBase(object):
""" """
Plex database methods used for all types of items Plex database methods used for all types of items
""" """
def __init__(self, cursor=None): def __init__(self, cursor=None, lock=True):
# Allows us to use this class with a cursor instead of context mgr # Allows us to use this class with a cursor instead of context mgr
self.cursor = cursor self.cursor = cursor
self.lock = lock
def __enter__(self): def __enter__(self):
if self.lock:
PLEXDB_LOCK.acquire()
self.plexconn = utils.kodi_sql('plex') self.plexconn = utils.kodi_sql('plex')
self.cursor = self.plexconn.cursor() self.cursor = self.plexconn.cursor()
return self return self
def __exit__(self, e_typ, e_val, trcbak): def __exit__(self, e_typ, e_val, trcbak):
try:
if e_typ: if e_typ:
# re-raise any exception # re-raise any exception
return False return False
self.plexconn.commit() self.plexconn.commit()
finally:
self.plexconn.close() self.plexconn.close()
if self.lock:
PLEXDB_LOCK.release()
def is_recorded(self, plex_id, plex_type): def is_recorded(self, plex_id, plex_type):
""" """