From 26fa1ff909e2473306e07c252ec1b0a73b8e59fb Mon Sep 17 00:00:00 2001 From: croneter Date: Thu, 1 Aug 2019 13:56:50 +0200 Subject: [PATCH 1/2] Use file size and last modification time to compare Kodi playlist files instead of slow MD5 hash of file --- resources/lib/playlists/__init__.py | 11 ++++++----- resources/lib/playlists/common.py | 18 ++++++++++++++++++ resources/lib/playlists/kodi_pl.py | 4 ++-- resources/lib/utils.py | 18 ------------------ 4 files changed, 26 insertions(+), 25 deletions(-) diff --git a/resources/lib/playlists/__init__.py b/resources/lib/playlists/__init__.py index 54c90550..b3658ba0 100644 --- a/resources/lib/playlists/__init__.py +++ b/resources/lib/playlists/__init__.py @@ -14,7 +14,8 @@ from __future__ import absolute_import, division, unicode_literals from logging import getLogger -from .common import Playlist, PlaylistError, PlaylistObserver +from .common import Playlist, PlaylistError, PlaylistObserver, \ + kodi_playlist_hash from . import pms, db, kodi_pl, plex_pl from ..watchdog import events @@ -221,7 +222,7 @@ def _full_sync(): pass if not sync_kodi_playlist(path): continue - kodi_hash = utils.generate_file_md5(path) + kodi_hash = kodi_playlist_hash(path) playlist = db.get_playlist(path=path) if playlist and playlist.kodi_hash == kodi_hash: continue @@ -387,7 +388,7 @@ class PlaylistEventhandler(events.FileSystemEventHandler): def on_created(self, event): LOG.debug('on_created: %s', event.src_path) old_playlist = db.get_playlist(path=event.src_path) - kodi_hash = utils.generate_file_md5(event.src_path) + kodi_hash = kodi_playlist_hash(event.src_path) if old_playlist and old_playlist.kodi_hash == kodi_hash: LOG.debug('Playlist already in DB - skipping') return @@ -406,7 +407,7 @@ class PlaylistEventhandler(events.FileSystemEventHandler): def on_modified(self, event): LOG.debug('on_modified: %s', event.src_path) old_playlist = db.get_playlist(path=event.src_path) - kodi_hash = utils.generate_file_md5(event.src_path) + kodi_hash = kodi_playlist_hash(event.src_path) if old_playlist and old_playlist.kodi_hash == kodi_hash: LOG.debug('Nothing modified, playlist already in DB - skipping') return @@ -425,7 +426,7 @@ class PlaylistEventhandler(events.FileSystemEventHandler): def on_moved(self, event): LOG.debug('on_moved: %s to %s', event.src_path, event.dest_path) - kodi_hash = utils.generate_file_md5(event.dest_path) + kodi_hash = kodi_playlist_hash(event.dest_path) # First check whether we don't already have destination playlist in # our DB. Just in case.... old_playlist = db.get_playlist(path=event.dest_path) diff --git a/resources/lib/playlists/common.py b/resources/lib/playlists/common.py index ce173475..5b348cd4 100644 --- a/resources/lib/playlists/common.py +++ b/resources/lib/playlists/common.py @@ -4,6 +4,8 @@ from __future__ import absolute_import, division, unicode_literals from logging import getLogger import Queue import time +import os +import hashlib from ..watchdog import events from ..watchdog.observers import Observer @@ -121,6 +123,22 @@ class Playlist(object): self._kodi_path = path +def kodi_playlist_hash(path): + """ + Returns a md5 hash [unicode] using os.stat() st_size and st_mtime for the + playlist located at path [unicode] + (size of file in bytes and time of most recent content modification) + + There are probably way more efficient ways out there to do this + """ + stat = os.stat(path_ops.encode_path(path)) + # stat.st_size is of type long; stat.st_mtime is of type float - hash both + m = hashlib.md5() + m.update(repr(stat.st_size)) + m.update(repr(stat.st_mtime)) + return m.hexdigest().decode('utf-8') + + class PlaylistQueue(OrderedSetQueue): """ OrderedSetQueue that drops all directory events immediately diff --git a/resources/lib/playlists/kodi_pl.py b/resources/lib/playlists/kodi_pl.py index 147f935b..4b040c25 100644 --- a/resources/lib/playlists/kodi_pl.py +++ b/resources/lib/playlists/kodi_pl.py @@ -7,7 +7,7 @@ from __future__ import absolute_import, division, unicode_literals from logging import getLogger import re -from .common import Playlist, PlaylistError +from .common import Playlist, PlaylistError, kodi_playlist_hash from . import db, pms from ..plex_api import API @@ -71,7 +71,7 @@ def create(plex_id): except Exception: IGNORE_KODI_PLAYLIST_CHANGE.remove(playlist.kodi_path) raise - playlist.kodi_hash = utils.generate_file_md5(path) + playlist.kodi_hash = kodi_playlist_hash(path) db.update_playlist(playlist) LOG.debug('Created Kodi playlist based on Plex playlist: %s', playlist) diff --git a/resources/lib/utils.py b/resources/lib/utils.py index be8aadcc..c84d5e25 100644 --- a/resources/lib/utils.py +++ b/resources/lib/utils.py @@ -17,7 +17,6 @@ import xml.etree.ElementTree as etree from . import defused_etree from xml.etree.ElementTree import ParseError from functools import wraps -import hashlib import re import gc try: @@ -931,23 +930,6 @@ class XmlKodiSetting(object): return element -def generate_file_md5(path): - """ - Generates the md5 hash value for the file located at path [unicode]. - The hash does not include the path and filename and is thus identical for - a file that was moved/changed name. - Returns a unique unicode containing only hexadecimal digits - """ - m = hashlib.md5() - with open(path_ops.encode_path(path), 'rb') as f: - while True: - piece = f.read(32768) - if not piece: - break - m.update(piece) - return m.hexdigest().decode('utf-8') - - def process_method_on_list(method_to_run, items): """ helper method that processes a method on each item with pooling if the From 7d8802467f7598e6b42faea517bca28d2524de8f Mon Sep 17 00:00:00 2001 From: croneter Date: Thu, 1 Aug 2019 14:22:44 +0200 Subject: [PATCH 2/2] Ensure playlists are freshly synched on PKC version bump --- resources/lib/migration.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/resources/lib/migration.py b/resources/lib/migration.py index d88ca2d4..14600572 100644 --- a/resources/lib/migration.py +++ b/resources/lib/migration.py @@ -57,4 +57,9 @@ def check_migration(): sections.clear_window_vars() sections.delete_videonode_files() + if not utils.compare_version(last_migration, '2.9.3'): + LOG.info('Migrating to version 2.9.2') + # Re-sync all playlists to Kodi + utils.wipe_synched_playlists() + utils.settings('last_migrated_PKC_version', value=v.ADDON_VERSION)