Troplo/PlexKodiConnect
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Huge music overhaul

Browse source 
- Fixes #254
This commit is contained in:
tomkat83 2017-04-02 17:02:41 +02:00
parent 95fd016bd7
commit e64746f277
7 changed files with 623 additions and 478 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

346
resources/lib/itemtypes.py
View file

@ -1296,7 +1296,7 @@ class Music(Items):
name, sortname = API.getTitle()
# musicBrainzId = API.getProvider('MusicBrainzArtist')
musicBrainzId = None
genres = API.joinList(API.getGenres())
genres = ' / '.join(API.getGenres())
bio = API.getPlot()
# Associate artwork
@ -1335,31 +1335,32 @@ class Music(Items):
# Process the artist
if v.KODIVERSION >= 16:
query = ' '.join((
"UPDATE artist",
"SET strGenres = ?, strBiography = ?, strImage = ?, strFanart = ?,",
"lastScraped = ?",
"WHERE idArtist = ?"
))
query = '''
UPDATE artist
SET strGenres = ?, strBiography = ?, strImage = ?,
strFanart = ?, lastScraped = ?
WHERE idArtist = ?
'''
kodicursor.execute(query, (genres, bio, thumb, fanart,
lastScraped, artistid))
else:
query = ' '.join((
"UPDATE artist",
"SET strGenres = ?, strBiography = ?, strImage = ?, strFanart = ?,",
"lastScraped = ?, dateAdded = ?",
"WHERE idArtist = ?"
))
query = '''
UPDATE artist
SET strGenres = ?, strBiography = ?, strImage = ?,
strFanart = ?, lastScraped = ?, dateAdded = ?
WHERE idArtist = ?
'''
kodicursor.execute(query, (genres, bio, thumb, fanart, lastScraped,
dateadded, artistid))
# Update artwork
artwork.addArtwork(artworks, artistid, "artist", kodicursor)
artwork.addArtwork(artworks, artistid, v.KODI_TYPE_ARTIST, kodicursor)
@CatchExceptions(warnuser=True)
def add_updateAlbum(self, item, viewtag=None, viewid=None):
def add_updateAlbum(self, item, viewtag=None, viewid=None, children=None):
"""
children: list of child xml's, so in this case songs
"""
kodicursor = self.kodicursor
plex_db = self.plex_db
artwork = self.artwork
@ -1387,21 +1388,21 @@ class Music(Items):
# musicBrainzId = API.getProvider('MusicBrainzAlbum')
musicBrainzId = None
year = API.getYear()
genres = API.getGenres()
genre = API.joinList(genres)
self.genres = API.getGenres()
self.genre = ' / '.join(self.genres)
bio = API.getPlot()
rating = userdata['UserRating']
studio = API.getMusicStudio()
# artists = item['AlbumArtists']
# if not artists:
# artists = item['ArtistItems']
# artistname = []
# for artist in artists:
# artistname.append(artist['Name'])
artistname = item.attrib.get('parentTitle')
if not artistname:
artistname = item.attrib.get('originalTitle')
# See if we have a compilation - Plex does NOT feature a compilation
# flag for albums
self.compilation = 0
for child in children:
if child.attrib.get('originalTitle') is not None:
self.compilation = 1
break
# Associate artwork
artworks = API.getAllArtwork(parentInfo=True)
thumb = artworks['Primary']
@ -1433,56 +1434,54 @@ class Music(Items):
# Process the album info
if v.KODIVERSION >= 17:
# Kodi Krypton
query = ' '.join((
"UPDATE album",
"SET strArtists = ?, iYear = ?, strGenres = ?, strReview = ?, strImage = ?,",
"iUserrating = ?, lastScraped = ?, strReleaseType = ?, "
"strLabel = ? ",
"WHERE idAlbum = ?"
))
kodicursor.execute(query, (artistname, year, genre, bio, thumb,
rating, lastScraped, "album", studio,
albumid))
query = '''
UPDATE album
SET strArtists = ?, iYear = ?, strGenres = ?, strReview = ?,
strImage = ?, iUserrating = ?, lastScraped = ?,
strReleaseType = ?, strLabel = ?, bCompilation = ?
WHERE idAlbum = ?
'''
kodicursor.execute(query, (artistname, year, self.genre, bio,
thumb, rating, lastScraped,
v.KODI_TYPE_ALBUM, studio,
self.compilation, albumid))
elif v.KODIVERSION == 16:
# Kodi Jarvis
query = ' '.join((
"UPDATE album",
"SET strArtists = ?, iYear = ?, strGenres = ?, strReview = ?, strImage = ?,",
"iRating = ?, lastScraped = ?, strReleaseType = ?, "
"strLabel = ? ",
"WHERE idAlbum = ?"
))
kodicursor.execute(query, (artistname, year, genre, bio, thumb,
rating, lastScraped, "album", studio,
albumid))
query = '''
UPDATE album
SET strArtists = ?, iYear = ?, strGenres = ?, strReview = ?,
strImage = ?, iRating = ?, lastScraped = ?,
strReleaseType = ?, strLabel = ?, bCompilation = ?
WHERE idAlbum = ?
'''
kodicursor.execute(query, (artistname, year, self.genre, bio,
thumb, rating, lastScraped,
v.KODI_TYPE_ALBUM, studio,
self.compilation, albumid))
elif v.KODIVERSION == 15:
# Kodi Isengard
query = ' '.join((
"UPDATE album",
"SET strArtists = ?, iYear = ?, strGenres = ?, strReview = ?, strImage = ?,",
"iRating = ?, lastScraped = ?, dateAdded = ?, "
"strReleaseType = ?, strLabel = ? ",
"WHERE idAlbum = ?"
))
kodicursor.execute(query, (artistname, year, genre, bio, thumb,
rating, lastScraped, dateadded,
"album", studio, albumid))
query = '''
UPDATE album
SET strArtists = ?, iYear = ?, strGenres = ?, strReview = ?,
strImage = ?, iRating = ?, lastScraped = ?, dateAdded = ?,
strReleaseType = ?, strLabel = ?
WHERE idAlbum = ?
'''
kodicursor.execute(query, (artistname, year, self.genre, bio,
thumb, rating, lastScraped, dateadded,
v.KODI_TYPE_ALBUM, studio, albumid))
else:
# Kodi Helix
query = ' '.join((
"UPDATE album",
"SET strArtists = ?, iYear = ?, strGenres = ?, strReview = ?, strImage = ?,",
"iRating = ?, lastScraped = ?, dateAdded = ?, "
"strLabel = ? ",
"WHERE idAlbum = ?"
))
kodicursor.execute(query, (artistname, year, genre, bio, thumb,
rating, lastScraped, dateadded, studio,
albumid))
query = '''
UPDATE album
SET strArtists = ?, iYear = ?, strGenres = ?, strReview = ?,
strImage = ?, iRating = ?, lastScraped = ?, dateAdded = ?,
strLabel = ?
WHERE idAlbum = ?
'''
kodicursor.execute(query, (artistname, year, self.genre, bio,
thumb, rating, lastScraped, dateadded,
studio, albumid))
# Associate the parentid for plex reference
parentId = item.attrib.get('parentRatingKey')
@ -1496,7 +1495,7 @@ class Music(Items):
artist = GetPlexMetadata(parentId)
# Item may not be an artist, verification necessary.
if artist is not None and artist != 401:
if artist[0].attrib.get('type') == "artist":
if artist[0].attrib.get('type') == v.PLEX_TYPE_ARTIST:
# Update with the parentId, for remove reference
plex_db.addReference(parentId,
v.PLEX_TYPE_ARTIST,
@ -1530,29 +1529,26 @@ class Music(Items):
% (artistname, artistid))
# Add artist to album
query = (
'''
query = '''
INSERT OR REPLACE INTO album_artist(idArtist, idAlbum, strArtist)
VALUES (?, ?, ?)
'''
)
'''
kodicursor.execute(query, (artistid, albumid, artistname))
# Update discography
query = (
'''
query = '''
INSERT OR REPLACE INTO discography(idArtist, strAlbum, strYear)
VALUES (?, ?, ?)
'''
)
'''
kodicursor.execute(query, (artistid, name, year))
# Update plex reference with parentid
plex_db.updateParentId(artistId, albumid)
# Add genres
self.kodi_db.addMusicGenres(albumid, genres, "album")
self.kodi_db.addMusicGenres(albumid, self.genres, v.KODI_TYPE_ALBUM)
# Update artwork
artwork.addArtwork(artworks, albumid, "album", kodicursor)
artwork.addArtwork(artworks, albumid, v.KODI_TYPE_ALBUM, kodicursor)
# Add all children - all tracks
for child in children:
self.add_updateSong(child, viewtag, viewid)
@CatchExceptions(warnuser=True)
def add_updateSong(self, item, viewtag=None, viewid=None):
@ -1592,9 +1588,22 @@ class Music(Items):
title, sorttitle = API.getTitle()
# musicBrainzId = API.getProvider('MusicBrainzTrackId')
musicBrainzId = None
genres = API.getGenres()
genre = API.joinList(genres)
artists = item.attrib.get('grandparentTitle')
try:
genres = self.genres
genre = self.genre
except AttributeError:
# No parent album - hence no genre information from Plex
genres = None
genre = None
try:
if self.compilation == 0:
artists = item.attrib.get('grandparentTitle')
else:
artists = item.attrib.get('originalTitle')
except AttributeError:
# compilation not set
artists = item.attrib.get('originalTitle',
item.attrib.get('grandparentTitle'))
tracknumber = int(item.attrib.get('index', 0))
disc = int(item.attrib.get('parentIndex', 1))
if disc == 1:
@ -1604,9 +1613,13 @@ class Music(Items):
year = API.getYear()
resume, duration = API.getRuntime()
rating = userdata['UserRating']
hasEmbeddedCover = False
comment = None
# Moods
moods = []
for entry in item:
if entry.tag == 'Mood':
moods.append(entry.attrib['tag'])
mood = ' / '.join(moods)
# GET THE FILE AND PATH #####
doIndirect = not self.directpath
@ -1644,16 +1657,18 @@ class Music(Items):
kodicursor.execute(query, (path, '123', pathid))
# Update the song entry
query = ' '.join((
"UPDATE song",
"SET idAlbum = ?, strArtists = ?, strGenres = ?, strTitle = ?, iTrack = ?,",
"iDuration = ?, iYear = ?, strFilename = ?, iTimesPlayed = ?, lastplayed = ?,",
"rating = ?, comment = ?",
"WHERE idSong = ?"
))
query = '''
UPDATE song
SET idAlbum = ?, strArtists = ?, strGenres = ?, strTitle = ?,
iTrack = ?, iDuration = ?, iYear = ?, strFilename = ?,
iTimesPlayed = ?, lastplayed = ?, rating = ?, comment = ?,
mood = ?
WHERE idSong = ?
'''
kodicursor.execute(query, (albumid, artists, genre, title, track,
duration, year, filename, playcount,
dateplayed, rating, comment, songid))
dateplayed, rating, comment, mood,
songid))
# Update the checksum in plex table
plex_db.updateReference(itemid, checksum)
@ -1676,7 +1691,9 @@ class Music(Items):
if album_name:
log.info("Creating virtual music album for song: %s."
% itemid)
albumid = self.kodi_db.addAlbum(album_name, API.getProvider('MusicBrainzAlbum'))
albumid = self.kodi_db.addAlbum(
album_name,
API.getProvider('MusicBrainzAlbum'))
plex_db.addReference("%salbum%s" % (itemid, albumid),
v.PLEX_TYPE_ALBUM,
albumid,
@ -1704,54 +1721,51 @@ class Music(Items):
except TypeError:
# No album found, create a single's album
log.info("Failed to add album. Creating singles.")
kodicursor.execute("select coalesce(max(idAlbum),0) from album")
kodicursor.execute(
"select coalesce(max(idAlbum),0) from album")
albumid = kodicursor.fetchone()[0] + 1
if v.KODIVERSION >= 16:
# Kodi Jarvis
query = (
'''
INSERT INTO album(idAlbum, strGenres, iYear, strReleaseType)
query = '''
INSERT INTO album(
idAlbum, strGenres, iYear, strReleaseType)
VALUES (?, ?, ?, ?)
'''
)
kodicursor.execute(query, (albumid, genre, year, "single"))
'''
kodicursor.execute(query,
(albumid, genre, year, "single"))
elif v.KODIVERSION == 15:
# Kodi Isengard
query = (
'''
INSERT INTO album(idAlbum, strGenres, iYear, dateAdded, strReleaseType)
query = '''
INSERT INTO album(
idAlbum, strGenres, iYear, dateAdded,
strReleaseType)
VALUES (?, ?, ?, ?, ?)
'''
)
kodicursor.execute(query, (albumid, genre, year, dateadded, "single"))
'''
kodicursor.execute(query, (albumid, genre, year,
dateadded, "single"))
else:
# Kodi Helix
query = (
'''
INSERT INTO album(idAlbum, strGenres, iYear, dateAdded)
query = '''
INSERT INTO album(
idAlbum, strGenres, iYear, dateAdded)
VALUES (?, ?, ?, ?)
'''
)
kodicursor.execute(query, (albumid, genre, year, dateadded))
'''
kodicursor.execute(query, (albumid, genre, year,
dateadded))
# Create the song entry
query = (
'''
query = '''
INSERT INTO song(
idSong, idAlbum, idPath, strArtists, strGenres, strTitle, iTrack,
iDuration, iYear, strFileName, strMusicBrainzTrackID, iTimesPlayed, lastplayed,
rating, iStartOffset, iEndOffset)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
idSong, idAlbum, idPath, strArtists, strGenres, strTitle,
iTrack, iDuration, iYear, strFileName,
strMusicBrainzTrackID, iTimesPlayed, lastplayed,
rating, iStartOffset, iEndOffset, mood)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
'''
)
kodicursor.execute(
query, (songid, albumid, pathid, artists, genre, title, track,
duration, year, filename, musicBrainzId, playcount,
dateplayed, rating, 0, 0))
dateplayed, rating, 0, 0, mood))
# Create the reference in plex table
plex_db.addReference(itemid,
@ -1764,14 +1778,11 @@ class Music(Items):
view_id=viewid)
# Link song to album
query = (
'''
query = '''
INSERT OR REPLACE INTO albuminfosong(
idAlbumInfoSong, idAlbumInfo, iTrack, strTitle, iDuration)
VALUES (?, ?, ?, ?, ?)
'''
)
'''
kodicursor.execute(query, (songid, albumid, track, title, duration))
# Link song to artists
@ -1799,29 +1810,27 @@ class Music(Items):
finally:
if v.KODIVERSION >= 17:
# Kodi Krypton
query = (
'''
INSERT OR REPLACE INTO song_artist(idArtist, idSong, idRole, iOrder, strArtist)
query = '''
INSERT OR REPLACE INTO song_artist(
idArtist, idSong, idRole, iOrder, strArtist)
VALUES (?, ?, ?, ?, ?)
'''
)
kodicursor.execute(query,(artistid, songid, 1, index, artist_name))
'''
kodicursor.execute(query, (artistid, songid, 1, index,
artist_name))
# May want to look into only doing this once?
query = (
'''
query = '''
INSERT OR REPLACE INTO role(idRole, strRole)
VALUES (?, ?)
'''
)
'''
kodicursor.execute(query, (1, 'Composer'))
else:
query = (
'''
INSERT OR REPLACE INTO song_artist(idArtist, idSong, iOrder, strArtist)
query = '''
INSERT OR REPLACE INTO song_artist(
idArtist, idSong, iOrder, strArtist)
VALUES (?, ?, ?, ?)
'''
)
kodicursor.execute(query, (artistid, songid, index, artist_name))
'''
kodicursor.execute(query, (artistid, songid, index,
artist_name))
# Verify if album artist exists
album_artists = []
@ -1843,31 +1852,28 @@ class Music(Items):
artist_edb = plex_db.getItem_byId(artist_eid)
artistid = artist_edb[0]
finally:
query = (
'''
INSERT OR REPLACE INTO album_artist(idArtist, idAlbum, strArtist)
query = '''
INSERT OR REPLACE INTO album_artist(
idArtist, idAlbum, strArtist)
VALUES (?, ?, ?)
'''
)
'''
kodicursor.execute(query, (artistid, albumid, artist_name))
# Update discography
if item.get('Album'):
query = (
'''
INSERT OR REPLACE INTO discography(idArtist, strAlbum, strYear)
query = '''
INSERT OR REPLACE INTO discography(
idArtist, strAlbum, strYear)
VALUES (?, ?, ?)
'''
)
'''
kodicursor.execute(query, (artistid, item['Album'], 0))
# else:
if False:
album_artists = " / ".join(album_artists)
query = ' '.join((
"SELECT strArtists",
"FROM album",
"WHERE idAlbum = ?"
))
query = '''
SELECT strArtists
FROM album
WHERE idAlbum = ?
'''
kodicursor.execute(query, (albumid,))
result = kodicursor.fetchone()
if result and result[0] != album_artists:
@ -1886,18 +1892,16 @@ class Music(Items):
kodicursor.execute(query, (album_artists, albumid))
# Add genres
self.kodi_db.addMusicGenres(songid, genres, "song")
if genres:
self.kodi_db.addMusicGenres(songid, genres, v.KODI_TYPE_SONG)
# Update artwork
allart = API.getAllArtwork(parentInfo=True)
if hasEmbeddedCover:
allart["Primary"] = "image://music@" + artwork.single_urlencode( playurl )
artwork.addArtwork(allart, songid, "song", kodicursor)
artwork.addArtwork(allart, songid, v.KODI_TYPE_SONG, kodicursor)
# if item.get('AlbumId') is None:
if item.get('parentKey') is None:
# Update album artwork
artwork.addArtwork(allart, albumid, "album", kodicursor)
artwork.addArtwork(allart, albumid, v.KODI_TYPE_ALBUM, kodicursor)
def remove(self, itemid):
# Remove kodiid, fileid, pathid, plex reference

1
resources/lib/library_sync/__init__.py Normal file
View file

@ -0,0 +1 @@
# Dummy file to make this directory a package.

88
resources/lib/library_sync/fanart.py Normal file
View file

@ -0,0 +1,88 @@
# -*- coding: utf-8 -*-
from logging import getLogger
from threading import Thread
from Queue import Empty
from xbmc import sleep
from utils import ThreadMethodsAdditionalStop, ThreadMethods, window, \
ThreadMethodsAdditionalSuspend
import plexdb_functions as plexdb
import itemtypes
import variables as v
###############################################################################
log = getLogger("PLEX."+__name__)
###############################################################################
@ThreadMethodsAdditionalSuspend('suspend_LibraryThread')
@ThreadMethodsAdditionalStop('plex_shouldStop')
@ThreadMethods
class Process_Fanart_Thread(Thread):
"""
Threaded download of additional fanart in the background
Input:
queue Queue.Queue() object that you will need to fill with
dicts of the following form:
{
'plex_id': the Plex id as a string
'plex_type': the Plex media type, e.g. 'movie'
'refresh': True/False if True, will overwrite any 3rd party
fanart. If False, will only get missing
}
"""
def __init__(self, queue):
self.queue = queue
Thread.__init__(self)
def run(self):
"""
Catch all exceptions and log them
"""
try:
self.__run()
except Exception as e:
log.error('Exception %s' % e)
import traceback
log.error("Traceback:\n%s" % traceback.format_exc())
def __run(self):
"""
Do the work
"""
log.debug("---===### Starting FanartSync ###===---")
threadStopped = self.threadStopped
threadSuspended = self.threadSuspended
queue = self.queue
while not threadStopped():
# In the event the server goes offline
while threadSuspended() or window('plex_dbScan'):
# Set in service.py
if threadStopped():
# Abort was requested while waiting. We should exit
log.info("---===### Stopped FanartSync ###===---")
return
sleep(1000)
# grabs Plex item from queue
try:
item = queue.get(block=False)
except Empty:
sleep(200)
continue
log.debug('Get additional fanart for Plex id %s' % item['plex_id'])
with getattr(itemtypes,
v.ITEMTYPE_FROM_PLEXTYPE[item['plex_type']])() as cls:
result = cls.getfanart(item['plex_id'],
refresh=item['refresh'])
if result is True:
log.debug('Done getting fanart for Plex id %s'
% item['plex_id'])
with plexdb.Get_Plex_DB() as plex_db:
plex_db.set_fanart_synched(item['plex_id'])
queue.task_done()
log.debug("---===### Stopped FanartSync ###===---")

140
resources/lib/library_sync/get_metadata.py Normal file
View file

@ -0,0 +1,140 @@
# -*- coding: utf-8 -*-
from logging import getLogger
from threading import Thread
from Queue import Empty
from xbmc import sleep
from utils import ThreadMethodsAdditionalStop, ThreadMethods, window
from PlexFunctions import GetPlexMetadata, GetAllPlexChildren
import sync_info
###############################################################################
log = getLogger("PLEX."+__name__)
###############################################################################
@ThreadMethodsAdditionalStop('suspend_LibraryThread')
@ThreadMethods
class Threaded_Get_Metadata(Thread):
"""
Threaded download of Plex XML metadata for a certain library item.
Fills the out_queue with the downloaded etree XML objects
Input:
queue Queue.Queue() object that you'll need to fill up
with Plex itemIds
out_queue Queue() object where this thread will store
the downloaded metadata XMLs as etree objects
"""
def __init__(self, queue, out_queue):
self.queue = queue
self.out_queue = out_queue
Thread.__init__(self)
def terminate_now(self):
"""
Needed to terminate this thread, because there might be items left in
the queue which could cause other threads to hang
"""
while not self.queue.empty():
# Still try because remaining item might have been taken
try:
self.queue.get(block=False)
except Empty:
sleep(10)
continue
else:
self.queue.task_done()
if self.threadStopped():
# Shutdown from outside requested; purge out_queue as well
while not self.out_queue.empty():
# Still try because remaining item might have been taken
try:
self.out_queue.get(block=False)
except Empty:
sleep(10)
continue
else:
self.out_queue.task_done()
def run(self):
"""
Catch all exceptions and log them
"""
try:
self.__run()
except Exception as e:
log.error('Exception %s' % e)
import traceback
log.error("Traceback:\n%s" % traceback.format_exc())
def __run(self):
"""
Do the work
"""
log.debug('Starting get metadata thread')
# cache local variables because it's faster
queue = self.queue
out_queue = self.out_queue
threadStopped = self.threadStopped
while threadStopped() is False:
# grabs Plex item from queue
try:
item = queue.get(block=False)
# Empty queue
except Empty:
sleep(20)
continue
# Download Metadata
xml = GetPlexMetadata(item['itemId'])
if xml is None:
# Did not receive a valid XML - skip that item for now
log.error("Could not get metadata for %s. Skipping that item "
"for now" % item['itemId'])
# Increase BOTH counters - since metadata won't be processed
with sync_info.LOCK:
sync_info.GET_METADATA_COUNT += 1
sync_info.PROCESS_METADATA_COUNT += 1
queue.task_done()
continue
elif xml == 401:
log.error('HTTP 401 returned by PMS. Too much strain? '
'Cancelling sync for now')
window('plex_scancrashed', value='401')
# Kill remaining items in queue (for main thread to cont.)
queue.task_done()
break
item['XML'] = xml
if item.get('get_children') is True:
children_xml = GetAllPlexChildren(item['itemId'])
try:
children_xml[0].attrib
except (TypeError, IndexError, AttributeError):
log.error('Could not get children for Plex id %s'
% item['itemId'])
else:
item['children'] = []
for child in children_xml:
child_xml = GetPlexMetadata(child.attrib['ratingKey'])
try:
child_xml[0].attrib
except (TypeError, IndexError, AttributeError):
log.error('Could not get child for Plex id %s'
% child.attrib['ratingKey'])
else:
item['children'].append(child_xml[0])
# place item into out queue
out_queue.put(item)
# Keep track of where we are at
with sync_info.LOCK:
sync_info.GET_METADATA_COUNT += 1
# signals to queue job is done
queue.task_done()
# Empty queue in case PKC was shut down (main thread hangs otherwise)
self.terminate_now()
log.debug('Get metadata thread terminated')

104
resources/lib/library_sync/process_metadata.py Normal file
View file

@ -0,0 +1,104 @@
# -*- coding: utf-8 -*-
from logging import getLogger
from threading import Thread
from Queue import Empty
from xbmc import sleep
from utils import ThreadMethodsAdditionalStop, ThreadMethods
import itemtypes
import sync_info
###############################################################################
log = getLogger("PLEX."+__name__)
###############################################################################
@ThreadMethodsAdditionalStop('suspend_LibraryThread')
@ThreadMethods
class Threaded_Process_Metadata(Thread):
"""
Not yet implemented for more than 1 thread - if ever. Only to be called by
ONE thread!
Processes the XML metadata in the queue
Input:
queue: Queue.Queue() object that you'll need to fill up with
the downloaded XML eTree objects
item_type: as used to call functions in itemtypes.py e.g. 'Movies' =>
itemtypes.Movies()
"""
def __init__(self, queue, item_type):
self.queue = queue
self.item_type = item_type
Thread.__init__(self)
def terminate_now(self):
"""
Needed to terminate this thread, because there might be items left in
the queue which could cause other threads to hang
"""
while not self.queue.empty():
# Still try because remaining item might have been taken
try:
self.queue.get(block=False)
except Empty:
sleep(10)
continue
else:
self.queue.task_done()
def run(self):
"""
Catch all exceptions and log them
"""
try:
self.__run()
except Exception as e:
log.error('Exception %s' % e)
import traceback
log.error("Traceback:\n%s" % traceback.format_exc())
def __run(self):
"""
Do the work
"""
log.debug('Processing thread started')
# Constructs the method name, e.g. itemtypes.Movies
item_fct = getattr(itemtypes, self.item_type)
# cache local variables because it's faster
queue = self.queue
threadStopped = self.threadStopped
with item_fct() as item_class:
while threadStopped() is False:
# grabs item from queue
try:
item = queue.get(block=False)
except Empty:
sleep(20)
continue
# Do the work
item_method = getattr(item_class, item['method'])
if item.get('children') is not None:
item_method(item['XML'][0],
viewtag=item['viewName'],
viewid=item['viewId'],
children=item['children'])
else:
item_method(item['XML'][0],
viewtag=item['viewName'],
viewid=item['viewId'])
# Keep track of where we are at
try:
log.debug('found child: %s'
% item['children'].attrib)
except:
pass
with sync_info.LOCK:
sync_info.PROCESS_METADATA_COUNT += 1
sync_info.PROCESSING_VIEW_NAME = item['title']
queue.task_done()
self.terminate_now()
log.debug('Processing thread terminated')

81
resources/lib/library_sync/sync_info.py Normal file
View file

@ -0,0 +1,81 @@
# -*- coding: utf-8 -*-
from logging import getLogger
from threading import Thread, Lock
from xbmc import sleep
from utils import ThreadMethodsAdditionalStop, ThreadMethods, language as lang
###############################################################################
log = getLogger("PLEX."+__name__)
GET_METADATA_COUNT = 0
PROCESS_METADATA_COUNT = 0
PROCESSING_VIEW_NAME = ''
LOCK = Lock()
###############################################################################
@ThreadMethodsAdditionalStop('suspend_LibraryThread')
@ThreadMethods
class Threaded_Show_Sync_Info(Thread):
"""
Threaded class to show the Kodi statusbar of the metadata download.
Input:
dialog xbmcgui.DialogProgressBG() object to show progress
total: Total number of items to get
"""
def __init__(self, dialog, total, item_type):
self.total = total
self.dialog = dialog
self.item_type = item_type
Thread.__init__(self)
def run(self):
"""
Catch all exceptions and log them
"""
try:
self.__run()
except Exception as e:
log.error('Exception %s' % e)
import traceback
log.error("Traceback:\n%s" % traceback.format_exc())
def __run(self):
"""
Do the work
"""
log.debug('Show sync info thread started')
# cache local variables because it's faster
total = self.total
dialog = self.dialog
threadStopped = self.threadStopped
dialog.create("%s: Sync %s: %s items"
% (lang(29999), self.item_type, str(total)),
"Starting")
total = 2 * total
totalProgress = 0
while threadStopped() is False:
with LOCK:
get_progress = GET_METADATA_COUNT
process_progress = PROCESS_METADATA_COUNT
viewName = PROCESSING_VIEW_NAME
totalProgress = get_progress + process_progress
try:
percentage = int(float(totalProgress) / float(total)*100.0)
except ZeroDivisionError:
percentage = 0
dialog.update(percentage,
message="%s downloaded. %s processed: %s"
% (get_progress,
process_progress,
viewName))
# Sleep for x milliseconds
sleep(200)
dialog.close()
log.debug('Show sync info thread terminated')

341
resources/lib/librarysync.py
View file

@ -3,7 +3,7 @@
###############################################################################
import logging
from threading import Thread, Lock
from threading import Thread
import Queue
from random import shuffle
@ -28,6 +28,10 @@ import variables as v
from PlexFunctions import GetPlexMetadata, GetAllPlexLeaves, scrobble, \
GetPlexSectionResults, GetAllPlexChildren, GetPMSStatus
import PlexAPI
from library_sync.get_metadata import Threaded_Get_Metadata
from library_sync.process_metadata import Threaded_Process_Metadata
import library_sync.sync_info as sync_info
from library_sync.fanart import Process_Fanart_Thread
###############################################################################
@ -36,282 +40,6 @@ log = logging.getLogger("PLEX."+__name__)
###############################################################################
@ThreadMethodsAdditionalStop('suspend_LibraryThread')
@ThreadMethods
class ThreadedGetMetadata(Thread):
"""
Threaded download of Plex XML metadata for a certain library item.
Fills the out_queue with the downloaded etree XML objects
Input:
queue Queue.Queue() object that you'll need to fill up
with Plex itemIds
out_queue Queue() object where this thread will store
the downloaded metadata XMLs as etree objects
lock Lock(), used for counting where we are
"""
def __init__(self, queue, out_queue, lock, processlock):
self.queue = queue
self.out_queue = out_queue
self.lock = lock
self.processlock = processlock
Thread.__init__(self)
def terminateNow(self):
while not self.queue.empty():
# Still try because remaining item might have been taken
try:
self.queue.get(block=False)
except Queue.Empty:
xbmc.sleep(10)
continue
else:
self.queue.task_done()
if self.threadStopped():
# Shutdown from outside requested; purge out_queue as well
while not self.out_queue.empty():
# Still try because remaining item might have been taken
try:
self.out_queue.get(block=False)
except Queue.Empty:
xbmc.sleep(10)
continue
else:
self.out_queue.task_done()
def run(self):
# cache local variables because it's faster
queue = self.queue
out_queue = self.out_queue
lock = self.lock
processlock = self.processlock
threadStopped = self.threadStopped
global getMetadataCount
global processMetadataCount
while threadStopped() is False:
# grabs Plex item from queue
try:
updateItem = queue.get(block=False)
# Empty queue
except Queue.Empty:
xbmc.sleep(10)
continue
# Download Metadata
plexXML = GetPlexMetadata(updateItem['itemId'])
if plexXML is None:
# Did not receive a valid XML - skip that item for now
log.warn("Could not get metadata for %s. Skipping that item "
"for now" % updateItem['itemId'])
# Increase BOTH counters - since metadata won't be processed
with lock:
getMetadataCount += 1
with processlock:
processMetadataCount += 1
queue.task_done()
continue
elif plexXML == 401:
log.warn('HTTP 401 returned by PMS. Too much strain? '
'Cancelling sync for now')
window('plex_scancrashed', value='401')
# Kill remaining items in queue (for main thread to cont.)
queue.task_done()
break
updateItem['XML'] = plexXML
# place item into out queue
out_queue.put(updateItem)
# Keep track of where we are at
with lock:
getMetadataCount += 1
# signals to queue job is done
queue.task_done()
# Empty queue in case PKC was shut down (main thread hangs otherwise)
self.terminateNow()
log.debug('Download thread terminated')
@ThreadMethodsAdditionalStop('suspend_LibraryThread')
@ThreadMethods
class ThreadedProcessMetadata(Thread):
"""
Not yet implemented - if ever. Only to be called by ONE thread!
Processes the XML metadata in the queue
Input:
queue: Queue.Queue() object that you'll need to fill up with
the downloaded XML eTree objects
itemType: as used to call functions in itemtypes.py
e.g. 'Movies' => itemtypes.Movies()
lock: Lock(), used for counting where we are
"""
def __init__(self, queue, itemType, lock):
self.queue = queue
self.lock = lock
self.itemType = itemType
Thread.__init__(self)
def terminateNow(self):
while not self.queue.empty():
# Still try because remaining item might have been taken
try:
self.queue.get(block=False)
except Queue.Empty:
xbmc.sleep(10)
continue
else:
self.queue.task_done()
def run(self):
# Constructs the method name, e.g. itemtypes.Movies
itemFkt = getattr(itemtypes, self.itemType)
# cache local variables because it's faster
queue = self.queue
lock = self.lock
threadStopped = self.threadStopped
global processMetadataCount
global processingViewName
with itemFkt() as item:
while threadStopped() is False:
# grabs item from queue
try:
updateItem = queue.get(block=False)
except Queue.Empty:
xbmc.sleep(10)
continue
# Do the work
plexitem = updateItem['XML']
method = updateItem['method']
viewName = updateItem['viewName']
viewId = updateItem['viewId']
title = updateItem['title']
itemSubFkt = getattr(item, method)
# Get the one child entry in the xml and process
for child in plexitem:
itemSubFkt(child,
viewtag=viewName,
viewid=viewId)
# Keep track of where we are at
with lock:
processMetadataCount += 1
processingViewName = title
# signals to queue job is done
queue.task_done()
# Empty queue in case PKC was shut down (main thread hangs otherwise)
self.terminateNow()
log.debug('Processing thread terminated')
@ThreadMethodsAdditionalStop('suspend_LibraryThread')
@ThreadMethods
class ThreadedShowSyncInfo(Thread):
"""
Threaded class to show the Kodi statusbar of the metadata download.
Input:
dialog xbmcgui.DialogProgressBG() object to show progress
locks = [downloadLock, processLock] Locks() to the other threads
total: Total number of items to get
"""
def __init__(self, dialog, locks, total, itemType):
self.locks = locks
self.total = total
self.dialog = dialog
self.itemType = itemType
Thread.__init__(self)
def run(self):
# cache local variables because it's faster
total = self.total
dialog = self.dialog
threadStopped = self.threadStopped
downloadLock = self.locks[0]
processLock = self.locks[1]
dialog.create("%s: Sync %s: %s items"
% (lang(29999), self.itemType, str(total)),
"Starting")
global getMetadataCount
global processMetadataCount
global processingViewName
total = 2 * total
totalProgress = 0
while threadStopped() is False:
with downloadLock:
getMetadataProgress = getMetadataCount
with processLock:
processMetadataProgress = processMetadataCount
viewName = processingViewName
totalProgress = getMetadataProgress + processMetadataProgress
try:
percentage = int(float(totalProgress) / float(total)*100.0)
except ZeroDivisionError:
percentage = 0
dialog.update(percentage,
message="%s downloaded. %s processed: %s"
% (getMetadataProgress,
processMetadataProgress,
viewName))
# Sleep for x milliseconds
xbmc.sleep(200)
dialog.close()
log.debug('Dialog Infobox thread terminated')
@ThreadMethodsAdditionalSuspend('suspend_LibraryThread')
@ThreadMethodsAdditionalStop('plex_shouldStop')
@ThreadMethods
class ProcessFanartThread(Thread):
"""
Threaded download of additional fanart in the background
Input:
queue Queue.Queue() object that you will need to fill with
dicts of the following form:
{
'plex_id': the Plex id as a string
'plex_type': the Plex media type, e.g. 'movie'
'refresh': True/False if True, will overwrite any 3rd party
fanart. If False, will only get missing
}
"""
def __init__(self, queue):
self.queue = queue
Thread.__init__(self)
def run(self):
threadStopped = self.threadStopped
threadSuspended = self.threadSuspended
queue = self.queue
log.info("---===### Starting FanartSync ###===---")
while not threadStopped():
# In the event the server goes offline
while threadSuspended() or window('plex_dbScan'):
# Set in service.py
if threadStopped():
# Abort was requested while waiting. We should exit
log.info("---===### Stopped FanartSync ###===---")
return
xbmc.sleep(1000)
# grabs Plex item from queue
try:
item = queue.get(block=False)
except Queue.Empty:
xbmc.sleep(200)
continue
log.debug('Get additional fanart for Plex id %s' % item['plex_id'])
with getattr(itemtypes,
v.ITEMTYPE_FROM_PLEXTYPE[item['plex_type']])() as cls:
result = cls.getfanart(item['plex_id'],
refresh=item['refresh'])
if result is True:
log.debug('Done getting fanart for Plex id %s'
% item['plex_id'])
with plexdb.Get_Plex_DB() as plex_db:
plex_db.set_fanart_synched(item['plex_id'])
queue.task_done()
log.info("---===### Stopped FanartSync ###===---")
@ThreadMethodsAdditionalSuspend('suspend_LibraryThread')
@ThreadMethodsAdditionalStop('plex_shouldStop')
@ThreadMethods
@ -330,7 +58,7 @@ class LibrarySync(Thread):
self.sessionKeys = []
self.fanartqueue = Queue.Queue()
if settings('FanartTV') == 'true':
self.fanartthread = ProcessFanartThread(self.fanartqueue)
self.fanartthread = Process_Fanart_Thread(self.fanartqueue)
# How long should we wait at least to process new/changed PMS items?
self.saftyMargin = int(settings('backgroundsync_saftyMargin'))
@ -700,8 +428,8 @@ class LibrarySync(Thread):
viewid=folderid,
delete=True)
# Added new playlist
if (foldername not in playlists and
mediatype in (v.PLEX_TYPE_MOVIE, v.PLEX_TYPE_SHOW)):
if (foldername not in playlists and mediatype in
(v.PLEX_TYPE_MOVIE, v.PLEX_TYPE_SHOW)):
playlistXSP(mediatype,
foldername,
folderid,
@ -726,8 +454,8 @@ class LibrarySync(Thread):
else:
# Validate the playlist exists or recreate it
if mediatype != v.PLEX_TYPE_ARTIST:
if (foldername not in playlists and
mediatype in (v.PLEX_TYPE_MOVIE, v.PLEX_TYPE_SHOW)):
if (foldername not in playlists and mediatype in
(v.PLEX_TYPE_MOVIE, v.PLEX_TYPE_SHOW)):
playlistXSP(mediatype,
foldername,
folderid,
@ -777,7 +505,8 @@ class LibrarySync(Thread):
for view in sections:
itemType = view.attrib['type']
if itemType in (v.PLEX_TYPE_MOVIE, v.PLEX_TYPE_SHOW, v.PLEX_TYPE_PHOTO): # NOT artist for now
if (itemType in
(v.PLEX_TYPE_MOVIE, v.PLEX_TYPE_SHOW, v.PLEX_TYPE_PHOTO)):
self.sorted_views.append(view.attrib['title'])
log.debug('Sorted views: %s' % self.sorted_views)
@ -859,7 +588,8 @@ class LibrarySync(Thread):
with itemtypes.Music() as music:
music.remove(item['plex_id'])
def GetUpdatelist(self, xml, itemType, method, viewName, viewId):
def GetUpdatelist(self, xml, itemType, method, viewName, viewId,
get_children=False):
"""
THIS METHOD NEEDS TO BE FAST! => e.g. no API calls
@ -872,6 +602,8 @@ class LibrarySync(Thread):
see itemtypes.py
viewName: Name of the Plex view (e.g. 'My TV shows')
viewId: Id/Key of Plex library (e.g. '1')
get_children: will get Plex children of the item if True,
e.g. for music albums
Output: self.updatelist, self.allPlexElementsId
self.updatelist APPENDED(!!) list itemids (Plex Keys as
@ -906,7 +638,8 @@ class LibrarySync(Thread):
'viewName': viewName,
'viewId': viewId,
'title': item.attrib.get('title', 'Missing Title'),
'mediaType': item.attrib.get('type')
'mediaType': item.attrib.get('type'),
'get_children': get_children
})
self.just_processed[itemId] = now
return
@ -932,7 +665,8 @@ class LibrarySync(Thread):
'viewName': viewName,
'viewId': viewId,
'title': item.attrib.get('title', 'Missing Title'),
'mediaType': item.attrib.get('type')
'mediaType': item.attrib.get('type'),
'get_children': get_children
})
self.just_processed[itemId] = now
else:
@ -951,7 +685,8 @@ class LibrarySync(Thread):
'viewName': viewName,
'viewId': viewId,
'title': item.attrib.get('title', 'Missing Title'),
'mediaType': item.attrib.get('type')
'mediaType': item.attrib.get('type'),
'get_children': get_children
})
self.just_processed[itemId] = now
@ -976,49 +711,38 @@ class LibrarySync(Thread):
log.info("Starting sync threads")
getMetadataQueue = Queue.Queue()
processMetadataQueue = Queue.Queue(maxsize=100)
getMetadataLock = Lock()
processMetadataLock = Lock()
# To keep track
global getMetadataCount
getMetadataCount = 0
global processMetadataCount
processMetadataCount = 0
global processingViewName
processingViewName = ''
sync_info.GET_METADATA_COUNT = 0
sync_info.PROCESS_METADATA_COUNT = 0
sync_info.PROCESSING_VIEW_NAME = ''
# Populate queue: GetMetadata
for updateItem in self.updatelist:
getMetadataQueue.put(updateItem)
# Spawn GetMetadata threads for downloading
threads = []
for i in range(min(self.syncThreadNumber, itemNumber)):
thread = ThreadedGetMetadata(getMetadataQueue,
processMetadataQueue,
getMetadataLock,
processMetadataLock)
thread = Threaded_Get_Metadata(getMetadataQueue,
processMetadataQueue)
thread.setDaemon(True)
thread.start()
threads.append(thread)
log.info("%s download threads spawned" % len(threads))
# Spawn one more thread to process Metadata, once downloaded
thread = ThreadedProcessMetadata(processMetadataQueue,
itemType,
processMetadataLock)
thread = Threaded_Process_Metadata(processMetadataQueue,
itemType)
thread.setDaemon(True)
thread.start()
threads.append(thread)
log.info("Processing thread spawned")
# Start one thread to show sync progress ONLY for new PMS items
if self.new_items_only is True and window('dbSyncIndicator') == 'true':
dialog = xbmcgui.DialogProgressBG()
thread = ThreadedShowSyncInfo(
thread = sync_info.Threaded_Show_Sync_Info(
dialog,
[getMetadataLock, processMetadataLock],
itemNumber,
itemType)
thread.setDaemon(True)
thread.start()
threads.append(thread)
log.info("Kodi Infobox thread spawned")
# Wait until finished
getMetadataQueue.join()
@ -1322,6 +1046,8 @@ class LibrarySync(Thread):
return True
def ProcessMusic(self, views, kind, urlArgs, method):
# For albums, we need to look at the album's songs simultaneously
get_children = True if kind == v.PLEX_TYPE_ALBUM else False
# Get a list of items already existing in Kodi db
if self.compare:
with plexdb.Get_Plex_DB() as plex_db:
@ -1347,7 +1073,8 @@ class LibrarySync(Thread):
'Music',
method,
view['name'],
view['id'])
view['id'],
get_children=get_children)
if self.compare:
# Manual sync, process deletes
with itemtypes.Music() as Music: