Troplo/PlexKodiConnect
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge pull request #704 from croneter/fix-slow-sync

Browse source 
Greatly speed up sync for episodes, especially for large libraries
This commit is contained in:
croneter 2019-02-05 18:59:16 +01:00 committed by GitHub
commit 950a2de0f5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 105 additions and 113 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
resources/lib/itemtypes/common.py
View file

@ -110,7 +110,7 @@ class ItemBase(object):
kodi_type)
def update_playstate(self, mark_played, view_count, resume, duration,
kodi_fileid, lastViewedAt, plex_type):
kodi_fileid, kodi_fileid_2, lastViewedAt):
"""
Use with websockets, not xml
"""
@ -128,5 +128,11 @@ class ItemBase(object):
resume,
duration,
view_count,
timing.plex_date_to_kodi(lastViewedAt),
plex_type)
timing.plex_date_to_kodi(lastViewedAt))
if kodi_fileid_2:
# Our dirty hack for episodes
self.kodidb.set_resume(kodi_fileid_2,
resume,
duration,
view_count,
timing.plex_date_to_kodi(lastViewedAt))

6
resources/lib/itemtypes/movies.py
View file

@ -214,8 +214,7 @@ class Movie(ItemBase):
resume,
runtime,
playcount,
dateplayed,
v.PLEX_TYPE_MOVIE)
dateplayed)
self.plexdb.add_movie(plex_id=plex_id,
checksum=api.checksum(),
section_id=section_id,
@ -279,8 +278,7 @@ class Movie(ItemBase):
userdata['Resume'],
userdata['Runtime'],
userdata['PlayCount'],
userdata['LastPlayedDate'],
plex_type)
userdata['LastPlayedDate'])
self.kodidb.update_userrating(db_item['kodi_id'],
db_item['kodi_type'],
userdata['UserRating'])

103
resources/lib/itemtypes/tvshows.py
View file

@ -32,8 +32,13 @@ class TvShowMixin(object):
userdata['Resume'],
userdata['Runtime'],
userdata['PlayCount'],
userdata['LastPlayedDate'],
plex_type)
userdata['LastPlayedDate'])
if db_item['kodi_fileid_2']:
self.kodidb.set_resume(db_item['kodi_fileid_2'],
userdata['Resume'],
userdata['Runtime'],
userdata['PlayCount'],
userdata['LastPlayedDate'])
return True
def remove(self, plex_id, plex_type=None):
@ -54,7 +59,7 @@ class TvShowMixin(object):
# EPISODE #####
if db_item['plex_type'] == v.PLEX_TYPE_EPISODE:
# Delete episode, verify season and tvshow
self.remove_episode(db_item['kodi_id'], db_item['kodi_fileid'])
self.remove_episode(db_item)
# Season verification
if (db_item['season_id'] and
not self.plexdb.season_has_episodes(db_item['season_id'])):
@ -72,7 +77,7 @@ class TvShowMixin(object):
# Remove episodes, season, verify tvshow
episodes = list(self.plexdb.episode_by_season(db_item['plex_id']))
for episode in episodes:
self.remove_episode(episode['kodi_id'], episode['kodi_fileid'])
self.remove_episode(episode)
self.plexdb.remove(episode['plex_id'], v.PLEX_TYPE_EPISODE)
# Remove season
self.remove_season(db_item['kodi_id'])
@ -91,8 +96,7 @@ class TvShowMixin(object):
self.plexdb.remove(season['plex_id'], v.PLEX_TYPE_SEASON)
episodes = list(self.plexdb.episode_by_show(db_item['plex_id']))
for episode in episodes:
self.remove_episode(episode['kodi_id'],
episode['kodi_fileid'])
self.remove_episode(episode)
self.plexdb.remove(episode['plex_id'], v.PLEX_TYPE_EPISODE)
self.remove_show(db_item['kodi_id'])
@ -120,17 +124,19 @@ class TvShowMixin(object):
self.kodidb.remove_season(kodi_id)
LOG.debug("Removed season: %s", kodi_id)
def remove_episode(self, kodi_id, file_id):
def remove_episode(self, db_item):
"""
Remove an episode, and episode only from the Kodi DB (not Plex DB)
"""
self.kodidb.modify_people(kodi_id, v.KODI_TYPE_EPISODE)
self.kodidb.remove_file(file_id, plex_type=v.PLEX_TYPE_EPISODE)
self.kodidb.delete_artwork(kodi_id, v.KODI_TYPE_EPISODE)
self.kodidb.remove_episode(kodi_id)
self.kodidb.remove_uniqueid(kodi_id, v.KODI_TYPE_EPISODE)
self.kodidb.remove_ratings(kodi_id, v.KODI_TYPE_EPISODE)
LOG.debug("Removed episode: %s", kodi_id)
self.kodidb.modify_people(db_item['kodi_id'], v.KODI_TYPE_EPISODE)
self.kodidb.remove_file(db_item['kodi_fileid'])
if db_item['kodi_fileid_2']:
self.kodidb.remove_file(db_item['kodi_fileid_2'])
self.kodidb.delete_artwork(db_item['kodi_id'], v.KODI_TYPE_EPISODE)
self.kodidb.remove_episode(db_item['kodi_id'])
self.kodidb.remove_uniqueid(db_item['kodi_id'], v.KODI_TYPE_EPISODE)
self.kodidb.remove_ratings(db_item['kodi_id'], v.KODI_TYPE_EPISODE)
LOG.debug("Removed episode: %s", db_item['kodi_id'])
class Show(TvShowMixin, ItemBase):
@ -367,6 +373,7 @@ class Episode(TvShowMixin, ItemBase):
update_item = True
kodi_id = episode['kodi_id']
old_kodi_fileid = episode['kodi_fileid']
old_kodi_fileid_2 = episode['kodi_fileid_2']
kodi_pathid = episode['kodi_pathid']
peoples = api.people()
@ -452,6 +459,15 @@ class Episode(TvShowMixin, ItemBase):
playurl = filename
# Root path tvshows/ already saved in Kodi DB
kodi_pathid = self.kodidb.add_path(path)
if not app.SYNC.direct_paths:
# need to set a 2nd file entry for a path without plex show id
# This fixes e.g. context menu and widgets working as they
# should
# A dirty hack, really
path_2 = 'plugin://%s.tvshows/' % v.ADDON_ID
# filename_2 is exactly the same as filename
# so WITH plex show id!
kodi_pathid_2 = self.kodidb.add_path(path_2)
# UPDATE THE EPISODE #####
if update_item:
@ -459,9 +475,17 @@ class Episode(TvShowMixin, ItemBase):
kodi_fileid = self.kodidb.modify_file(filename,
kodi_pathid,
api.date_created())
if not app.SYNC.direct_paths:
kodi_fileid_2 = self.kodidb.modify_file(filename,
kodi_pathid_2,
api.date_created())
else:
kodi_fileid_2 = None
if kodi_fileid != old_kodi_fileid:
self.kodidb.remove_file(old_kodi_fileid)
if not app.SYNC.direct_paths:
self.kodidb.remove_file(old_kodi_fileid_2)
ratingid = self.kodidb.get_ratingid(kodi_id,
v.KODI_TYPE_EPISODE)
self.kodidb.update_ratings(kodi_id,
@ -502,7 +526,7 @@ class Episode(TvShowMixin, ItemBase):
airs_before_episode,
playurl,
kodi_pathid,
kodi_fileid,
kodi_fileid, # and NOT kodi_fileid_2
parent_id,
userdata['UserRating'],
kodi_id)
@ -510,8 +534,13 @@ class Episode(TvShowMixin, ItemBase):
api.resume_point(),
api.runtime(),
userdata['PlayCount'],
userdata['LastPlayedDate'],
v.PLEX_TYPE_EPISODE)
userdata['LastPlayedDate'])
if not app.SYNC.direct_paths:
self.kodidb.set_resume(kodi_fileid_2,
api.resume_point(),
api.runtime(),
userdata['PlayCount'],
userdata['LastPlayedDate'])
self.plexdb.add_episode(plex_id=plex_id,
checksum=api.checksum(),
section_id=section_id,
@ -521,6 +550,7 @@ class Episode(TvShowMixin, ItemBase):
parent_id=parent_id,
kodi_id=kodi_id,
kodi_fileid=kodi_fileid,
kodi_fileid_2=kodi_fileid_2,
kodi_pathid=kodi_pathid,
last_sync=self.last_sync)
# OR ADD THE EPISODE #####
@ -529,6 +559,12 @@ class Episode(TvShowMixin, ItemBase):
kodi_fileid = self.kodidb.add_file(filename,
kodi_pathid,
api.date_created())
if not app.SYNC.direct_paths:
kodi_fileid_2 = self.kodidb.add_file(filename,
kodi_pathid_2,
api.date_created())
else:
kodi_fileid_2 = None
rating_id = self.kodidb.add_ratingid()
self.kodidb.add_ratings(rating_id,
@ -552,7 +588,7 @@ class Episode(TvShowMixin, ItemBase):
kodi_id,
v.KODI_TYPE_EPISODE)
self.kodidb.add_episode(kodi_id,
kodi_fileid,
kodi_fileid, # and NOT kodi_fileid_2
api.title(),
api.plot(),
rating_id,
@ -574,8 +610,13 @@ class Episode(TvShowMixin, ItemBase):
api.resume_point(),
api.runtime(),
userdata['PlayCount'],
userdata['LastPlayedDate'],
None) # Do send None to avoid episode loop
userdata['LastPlayedDate'])
if not app.SYNC.direct_paths:
self.kodidb.set_resume(kodi_fileid_2,
api.resume_point(),
api.runtime(),
userdata['PlayCount'],
userdata['LastPlayedDate'])
self.plexdb.add_episode(plex_id=plex_id,
checksum=api.checksum(),
section_id=section_id,
@ -585,26 +626,10 @@ class Episode(TvShowMixin, ItemBase):
parent_id=parent_id,
kodi_id=kodi_id,
kodi_fileid=kodi_fileid,
kodi_fileid_2=kodi_fileid_2,
kodi_pathid=kodi_pathid,
last_sync=self.last_sync)
if not app.SYNC.direct_paths:
# need to set a SECOND file entry for a path without plex show id
filename = api.file_name(force_first_media=True)
path = 'plugin://%s.tvshows/' % v.ADDON_ID
# Filename is exactly the same, WITH plex show id!
filename = ('%s%s/?plex_id=%s&plex_type=%s&mode=play&filename=%s'
% (path, show_id, plex_id, v.PLEX_TYPE_EPISODE,
filename))
kodi_pathid = self.kodidb.add_path(path)
second_kodi_fileid = self.kodidb.add_file(filename,
kodi_pathid,
api.date_created())
self.kodidb.set_resume(second_kodi_fileid,
api.resume_point(),
api.runtime(),
userdata['PlayCount'],
userdata['LastPlayedDate'],
None) # Do send None - 2nd entry
self.kodidb.modify_streams(kodi_fileid,
self.kodidb.modify_streams(kodi_fileid, # and NOT kodi_fileid_2
api.mediastreams(),
api.runtime())

40
resources/lib/kodi_db/video.py
View file

@ -199,29 +199,13 @@ class KodiVideoDB(common.KodiDBBase):
pass
@common.catch_operationalerrors
def remove_file(self, file_id, remove_orphans=True, plex_type=None):
def remove_file(self, file_id, remove_orphans=True):
"""
Removes the entry for file_id from the files table. Will also delete
entries from the associated tables: bookmark, settings, streamdetails.
If remove_orphans is true, this method will delete any orphaned path
entries in the Kodi path table
Passing plex_type = v.PLEX_TYPE_EPISODE deletes any secondary files for
add-on paths
"""
if not app.SYNC.direct_paths and plex_type == v.PLEX_TYPE_EPISODE:
# Hack for the 2 entries for episodes for addon paths
self.cursor.execute('SELECT strFilename FROM files WHERE idFile = ? LIMIT 1',
(file_id, ))
filename = self.cursor.fetchone()
if not filename:
LOG.error('Could not find file_id %s', file_id)
return
for new_id in self.cursor.execute('SELECT idFile FROM files WHERE strFilename = ? LIMIT 2',
(filename[0], )):
self.remove_file(new_id[0], remove_orphans=remove_orphans)
return
self.cursor.execute('SELECT idPath FROM files WHERE idFile = ? LIMIT 1',
(file_id,))
try:
@ -612,31 +596,11 @@ class KodiVideoDB(common.KodiDBBase):
@common.catch_operationalerrors
def set_resume(self, file_id, resume_seconds, total_seconds, playcount,
dateplayed, plex_type):
dateplayed):
"""
Adds a resume marker for a video library item. Will even set 2,
considering add-on path widget hacks.
"""
if not app.SYNC.direct_paths and plex_type == v.PLEX_TYPE_EPISODE:
# Need to make sure to set a SECOND bookmark entry for another,
# second file_id that points to the path .tvshows instead of
# .tvshows/<plex show id/!
self.cursor.execute('SELECT strFilename FROM files WHERE idFile = ? LIMIT 1',
(file_id, ))
try:
filename = self.cursor.fetchone()[0]
except TypeError:
LOG.error('Did not get a filename, aborting for file_id %s',
file_id)
return
self.cursor.execute('SELECT idFile FROM files WHERE strFilename = ? LIMIT 2',
(filename, ))
file_ids = self.cursor.fetchall()
for new_id in file_ids:
self.set_resume(new_id[0], resume_seconds, total_seconds,
playcount, dateplayed, None)
return
# Delete existing resume point
self.cursor.execute('DELETE FROM bookmark WHERE idFile = ?', (file_id,))
# Set watched count

10
resources/lib/kodimonitor.py
View file

@ -493,8 +493,14 @@ def _record_playstate(status, ended):
time,
totaltime,
playcount,
last_played,
status['plex_type'])
last_played)
if 'kodi_fileid_2' in db_item and db_item['kodi_fileid_2']:
# Dirty hack for our episodes
kodidb.set_resume(db_item['kodi_fileid_2'],
time,
totaltime,
playcount,
last_played)
# Hack to force "in progress" widget to appear if it wasn't visible before
if (app.APP.force_reload_skin and
xbmc.getCondVisibility('Window.IsVisible(Home.xml)')):

4
resources/lib/library_sync/full_sync.py
View file

@ -19,7 +19,7 @@ if common.PLAYLIST_SYNC_ENABLED:
LOG = getLogger('PLEX.sync.full_sync')
# How many items will be put through the processing chain at once?
BATCH_SIZE = 200
BATCH_SIZE = 500
# Safety margin to filter PMS items - how many seconds to look into the past?
UPDATED_AT_SAFETY = 60 * 5
LAST_VIEWED_AT_SAFETY = 60 * 5
@ -240,7 +240,7 @@ class FullSync(common.fullsync_mixin):
self.current_sync)
self.current += 1
self.update_progressbar()
if (i + 1) % BATCH_SIZE == 0:
if (i + 1) % (10 * BATCH_SIZE) == 0:
break
if last:
break

2
resources/lib/library_sync/sections.py
View file

@ -13,7 +13,7 @@ from .. import plex_functions as PF, music, utils, variables as v, app
LOG = getLogger('PLEX.sync.sections')
BATCH_SIZE = 200
BATCH_SIZE = 500
VNODES = videonodes.VideoNodes()
PLAYLISTS = {}
NODES = {}

12
resources/lib/library_sync/websocket.py
View file

@ -285,10 +285,14 @@ def process_playing(data):
PLAYSTATE_SESSIONS)
# Attach Kodi info to the session
try:
PLAYSTATE_SESSIONS[session_key]['file_id'] = typus['kodi_fileid']
PLAYSTATE_SESSIONS[session_key]['kodi_fileid'] = typus['kodi_fileid']
except KeyError:
# media type without file - no need to do anything
continue
if typus['plex_type'] == v.PLEX_TYPE_EPISODE:
PLAYSTATE_SESSIONS[session_key]['kodi_fileid_2'] = typus['kodi_fileid_2']
else:
PLAYSTATE_SESSIONS[session_key]['kodi_fileid_2'] = None
PLAYSTATE_SESSIONS[session_key]['kodi_id'] = typus['kodi_id']
PLAYSTATE_SESSIONS[session_key]['kodi_type'] = typus['kodi_type']
session = PLAYSTATE_SESSIONS[session_key]
@ -357,9 +361,9 @@ def process_playing(data):
session['viewCount'],
resume,
session['duration'],
session['file_id'],
timing.unix_timestamp(),
v.PLEX_TYPE_FROM_KODI_TYPE[session['kodi_type']])
session['kodi_fileid'],
session['kodi_fileid_2'],
timing.unix_timestamp())
def cache_artwork(plex_id, plex_type, kodi_id=None, kodi_type=None):

1
resources/lib/plex_db/common.py
View file

@ -239,6 +239,7 @@ def initialize():
parent_id INTEGER,
kodi_id INTEGER,
kodi_fileid INTEGER,
kodi_fileid_2 INTEGER,
kodi_pathid INTEGER,
fanart_synced INTEGER,
last_sync INTEGER)

28
resources/lib/plex_db/tvshows.py
View file

@ -59,7 +59,7 @@ class TVShows(object):
def add_episode(self, plex_id, checksum, section_id, show_id,
grandparent_id, season_id, parent_id, kodi_id, kodi_fileid,
kodi_pathid, last_sync):
kodi_fileid_2, kodi_pathid, last_sync):
"""
Appends or replaces an entry into the plex table
"""
@ -75,10 +75,11 @@ class TVShows(object):
parent_id,
kodi_id,
kodi_fileid,
kodi_fileid_2,
kodi_pathid,
fanart_synced,
last_sync)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
''',
(plex_id,
checksum,
@ -89,6 +90,7 @@ class TVShows(object):
parent_id,
kodi_id,
kodi_fileid,
kodi_fileid_2,
kodi_pathid,
0,
last_sync))
@ -129,21 +131,6 @@ class TVShows(object):
return self.entry_to_season(self.cursor.fetchone())
def episode(self, plex_id):
"""
Returns the show info as a tuple for the TV show with plex_id:
plex_id INTEGER PRIMARY KEY,
checksum INTEGER UNIQUE,
section_id INTEGER,
show_id INTEGER, # plex_id of the parent show
grandparent_id INTEGER, # kodi_id of the parent show
season_id INTEGER, # plex_id of the parent season
parent_id INTEGER, # kodi_id of the parent season
kodi_id INTEGER,
kodi_fileid INTEGER,
kodi_pathid INTEGER,
fanart_synced INTEGER,
last_sync INTEGER
"""
if plex_id is None:
return
self.cursor.execute('SELECT * FROM episode WHERE plex_id = ? LIMIT 1',
@ -166,9 +153,10 @@ class TVShows(object):
'parent_id': entry[6],
'kodi_id': entry[7],
'kodi_fileid': entry[8],
'kodi_pathid': entry[9],
'fanart_synced': entry[10],
'last_sync': entry[11]
'kodi_fileid_2': entry[9],
'kodi_pathid': entry[10],
'fanart_synced': entry[11],
'last_sync': entry[12]
}
@staticmethod