From b3b41240a18362a231759c337dff847cca1fce26 Mon Sep 17 00:00:00 2001 From: Thomas Adamcik Date: Sun, 19 May 2013 23:47:53 +0200 Subject: [PATCH 1/7] core: Add last_modified to track model --- mopidy/models.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mopidy/models.py b/mopidy/models.py index d138b490..b5ec6a5f 100644 --- a/mopidy/models.py +++ b/mopidy/models.py @@ -231,6 +231,8 @@ class Track(ImmutableObject): :type bitrate: integer :param musicbrainz_id: MusicBrainz ID :type musicbrainz_id: string + :param last_modified: integer representing last modifcation time + :type last_modified: integer """ #: The track URI. Read-only. @@ -263,6 +265,10 @@ class Track(ImmutableObject): #: The MusicBrainz ID of the track. Read-only. musicbrainz_id = None + #: Integer representing when the track was last modified, exact meaning + #: depends on source of track. + last_modified = 0 + def __init__(self, *args, **kwargs): self.__dict__['artists'] = frozenset(kwargs.pop('artists', [])) super(Track, self).__init__(*args, **kwargs) From 855d03c81e8808daddcb219c7609a02b9df0528d Mon Sep 17 00:00:00 2001 From: Thomas Adamcik Date: Mon, 20 May 2013 00:52:07 +0200 Subject: [PATCH 2/7] local: start reading mtime from tag caches --- mopidy/backends/local/translator.py | 3 +++ tests/backends/local/translator_test.py | 14 +++++++++----- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/mopidy/backends/local/translator.py b/mopidy/backends/local/translator.py index d2ee0d45..4ae10af2 100644 --- a/mopidy/backends/local/translator.py +++ b/mopidy/backends/local/translator.py @@ -112,6 +112,9 @@ def _convert_mpd_data(data, tracks, music_dir): else: track_kwargs['track_no'] = int(data['track']) + if 'mtime' in data: + track_kwargs['last_modified'] = int(data['mtime']) + if 'artist' in data: artist_kwargs['name'] = data['artist'] albumartist_kwargs['name'] = data['artist'] diff --git a/tests/backends/local/translator_test.py b/tests/backends/local/translator_test.py index 096d9a0d..4f958232 100644 --- a/tests/backends/local/translator_test.py +++ b/tests/backends/local/translator_test.py @@ -101,7 +101,8 @@ def generate_track(path, ident): uri = path_to_uri(path_to_data_dir(path)) track = Track( uri=uri, name='trackname', artists=expected_artists, - album=expected_albums[0], track_no=1, date='2006', length=4000) + album=expected_albums[0], track_no=1, date='2006', length=4000, + last_modified=1272319626) expected_tracks.append(track) @@ -128,7 +129,8 @@ class MPDTagCacheToTracksTest(unittest.TestCase): uri = path_to_uri(path_to_data_dir('song1.mp3')) track = Track( uri=uri, name='trackname', artists=expected_artists, track_no=1, - album=expected_albums[0], date='2006', length=4000) + album=expected_albums[0], date='2006', length=4000, + last_modified=1272319626) self.assertEqual(set([track]), tracks) def test_advanced_cache(self): @@ -144,7 +146,8 @@ class MPDTagCacheToTracksTest(unittest.TestCase): artists = [Artist(name='æøå')] album = Album(name='æøå', artists=artists) track = Track( - uri=uri, name='æøå', artists=artists, album=album, length=4000) + uri=uri, name='æøå', artists=artists, album=album, length=4000, + last_modified=1272319626) self.assertEqual(track, list(tracks)[0]) @@ -157,7 +160,8 @@ class MPDTagCacheToTracksTest(unittest.TestCase): tracks = parse_mpd_tag_cache( path_to_data_dir('blank_tag_cache'), path_to_data_dir('')) uri = path_to_uri(path_to_data_dir('song1.mp3')) - self.assertEqual(set([Track(uri=uri, length=4000)]), tracks) + expected = Track(uri=uri, length=4000, last_modified=1272319626) + self.assertEqual(set([expected]), tracks) def test_musicbrainz_tagcache(self): tracks = parse_mpd_tag_cache( @@ -184,5 +188,5 @@ class MPDTagCacheToTracksTest(unittest.TestCase): album = expected_albums[0].copy(artists=[artist]) track = Track( uri=uri, name='trackname', artists=expected_artists, track_no=1, - album=album, date='2006', length=4000) + album=album, date='2006', length=4000, last_modified=1272319626) self.assertEqual(track, list(tracks)[0]) From 802e6ad5ed90be39f9e64ee872825e6d47287ba1 Mon Sep 17 00:00:00 2001 From: Thomas Adamcik Date: Mon, 20 May 2013 01:00:46 +0200 Subject: [PATCH 3/7] scanner: Add last_modified to scanner data --- mopidy/scanner.py | 5 ++++- tests/scanner_test.py | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/mopidy/scanner.py b/mopidy/scanner.py index 41b42347..01d5397a 100644 --- a/mopidy/scanner.py +++ b/mopidy/scanner.py @@ -141,6 +141,7 @@ def translator(data): album_kwargs['artists'] = [Artist(**albumartist_kwargs)] track_kwargs['uri'] = data['uri'] + track_kwargs['last_modified'] = int(data['mtime']) track_kwargs['length'] = data[gst.TAG_DURATION] track_kwargs['album'] = Album(**album_kwargs) track_kwargs['artists'] = [Artist(**artist_kwargs)] @@ -195,7 +196,9 @@ class Scanner(object): if message.structure.get_name() != 'handoff': return - self.data['uri'] = unicode(self.uribin.get_property('uri')) + uri = unicode(self.uribin.get_property('uri')) + self.data['uri'] = uri + self.data['mtime'] = os.path.getmtime(path.uri_to_path(uri)) self.data[gst.TAG_DURATION] = self.get_duration() try: diff --git a/tests/scanner_test.py b/tests/scanner_test.py index 75fc60fd..ef68346e 100644 --- a/tests/scanner_test.py +++ b/tests/scanner_test.py @@ -32,6 +32,7 @@ class TranslatorTest(unittest.TestCase): 'musicbrainz-albumid': 'mbalbumid', 'musicbrainz-artistid': 'mbartistid', 'musicbrainz-albumartistid': 'mbalbumartistid', + 'mtime': 1234, } self.album = { @@ -57,6 +58,7 @@ class TranslatorTest(unittest.TestCase): 'track_no': 1, 'length': 4531, 'musicbrainz_id': 'mbtrackid', + 'last_modified': 1234, } def build_track(self): From c59f488b1100f4b637a1b8ab1a61a0bde060c37f Mon Sep 17 00:00:00 2001 From: Thomas Adamcik Date: Mon, 20 May 2013 01:19:41 +0200 Subject: [PATCH 4/7] scanner: Pass in files to scan --- mopidy/scanner.py | 13 +++++++++---- tests/scanner_test.py | 6 ++++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/mopidy/scanner.py b/mopidy/scanner.py index 01d5397a..65cdc13e 100644 --- a/mopidy/scanner.py +++ b/mopidy/scanner.py @@ -58,6 +58,10 @@ def main(): tracks = [] + def find(base_directory): + for p in path.find_files(base_directory): + yield path.path_to_uri(p) + def store(data): track = translator(data) tracks.append(track) @@ -69,7 +73,8 @@ def main(): logging.info('Scanning %s', config['local']['media_dir']) - scanner = Scanner(config['local']['media_dir'], store, debug) + uris = find(config['local']['media_dir']) + scanner = Scanner(uris, store, debug) try: scanner.start() except KeyboardInterrupt: @@ -150,9 +155,9 @@ def translator(data): class Scanner(object): - def __init__(self, base_dir, data_callback, error_callback=None): + def __init__(self, uris, data_callback, error_callback=None): self.data = {} - self.files = path.find_files(base_dir) + self.uris = iter(uris) self.data_callback = data_callback self.error_callback = error_callback self.loop = gobject.MainLoop() @@ -237,7 +242,7 @@ class Scanner(object): def next_uri(self): self.data = {} try: - uri = path.path_to_uri(self.files.next()) + uri = next(self.uris) except StopIteration: self.stop() return False diff --git a/tests/scanner_test.py b/tests/scanner_test.py index ef68346e..c9671523 100644 --- a/tests/scanner_test.py +++ b/tests/scanner_test.py @@ -4,6 +4,7 @@ import unittest from mopidy.scanner import Scanner, translator from mopidy.models import Track, Artist, Album +from mopidy.utils import path as path_lib from tests import path_to_data_dir @@ -143,8 +144,9 @@ class ScannerTest(unittest.TestCase): self.data = {} def scan(self, path): - scanner = Scanner( - path_to_data_dir(path), self.data_callback, self.error_callback) + paths = path_lib.find_files(path_to_data_dir(path)) + uris = (path_lib.path_to_uri(p) for p in paths) + scanner = Scanner(uris, self.data_callback, self.error_callback) scanner.start() def check(self, name, key, value): From cb6634db8c6adf6443106acee7858db7c0d2bfbf Mon Sep 17 00:00:00 2001 From: Thomas Adamcik Date: Mon, 20 May 2013 01:24:40 +0200 Subject: [PATCH 5/7] scanner: Load existing tracks from local library. --- mopidy/scanner.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/mopidy/scanner.py b/mopidy/scanner.py index 65cdc13e..af66a5f4 100644 --- a/mopidy/scanner.py +++ b/mopidy/scanner.py @@ -27,6 +27,7 @@ pygst.require('0.10') import gst from mopidy import config as config_lib, ext +from mopidy.audio import dummy as dummy_audio from mopidy.frontends.mpd import translator as mpd_translator from mopidy.models import Track, Artist, Album from mopidy.utils import log, path, versioning @@ -45,9 +46,9 @@ def main(): log.setup_root_logger() log.setup_console_logging(logging_config, args.verbosity_level) - extensions = ext.load_extensions() + extensions = dict((e.ext_name, e) for e in ext.load_extensions()) config, errors = config_lib.load( - config_files, extensions, config_overrides) + config_files, extensions.values(), config_overrides) log.setup_log_levels(config) if not config['local']['media_dir']: @@ -56,7 +57,13 @@ def main(): # TODO: missing error checking and other default setup code. - tracks = [] + audio = dummy_audio.DummyAudio() + local_backend_class = extensions['local'].get_backend_classes() + local_backend = local_backend_class[0](config, audio) + + tracks = {} + for track in local_backend.library.search().tracks: + tracks[track.uri] = track def find(base_directory): for p in path.find_files(base_directory): @@ -64,7 +71,7 @@ def main(): def store(data): track = translator(data) - tracks.append(track) + tracks[track.uri] = track logging.debug('Added %s', track.uri) def debug(uri, error, debug): @@ -83,7 +90,7 @@ def main(): logging.info('Done scanning; writing tag cache...') for row in mpd_translator.tracks_to_tag_cache_format( - tracks, config['local']['media_dir']): + tracks.values(), config['local']['media_dir']): if len(row) == 1: print ('%s' % row).encode('utf-8') else: From afb46b23de6e994782f815a4b72fa6fc2672c13f Mon Sep 17 00:00:00 2001 From: Thomas Adamcik Date: Mon, 20 May 2013 01:45:52 +0200 Subject: [PATCH 6/7] scanner: Respect mtime when picking files to scan. Speedup scanning by checking what files exist, and which have been updated. --- mopidy/scanner.py | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/mopidy/scanner.py b/mopidy/scanner.py index af66a5f4..42c30861 100644 --- a/mopidy/scanner.py +++ b/mopidy/scanner.py @@ -61,13 +61,31 @@ def main(): local_backend_class = extensions['local'].get_backend_classes() local_backend = local_backend_class[0](config, audio) - tracks = {} + tracks = {} # Current lib. + update = [] # Paths to rescan for updates/adds. + remove = [] # Paths to delete from lib. + for track in local_backend.library.search().tracks: tracks[track.uri] = track - def find(base_directory): - for p in path.find_files(base_directory): - yield path.path_to_uri(p) + logging.info('Checking %d files from library.', len(tracks)) + for track in tracks.itervalues(): + try: + stat = os.stat(path.uri_to_path(track.uri)) + if int(stat.st_mtime) > track.last_modified: + update.append(track.uri) + except OSError: + remove.append(track.uri) + + logging.info('Removing %d files from library.', len(remove)) + for uri in remove: + del tracks[uri] + + logging.info('Checking %s for changes.', config['local']['media_dir']) + for p in path.find_files(config['local']['media_dir']): + uri = path.path_to_uri(p) + if uri not in tracks: + update.append(uri) def store(data): track = translator(data) @@ -78,10 +96,9 @@ def main(): logging.warning('Failed %s: %s', uri, error) logging.debug('Debug info for %s: %s', uri, debug) - logging.info('Scanning %s', config['local']['media_dir']) - uris = find(config['local']['media_dir']) - scanner = Scanner(uris, store, debug) + logging.info('Scanning %d files.', len(update)) + scanner = Scanner(update, store, debug) try: scanner.start() except KeyboardInterrupt: From b4411ec877b2a82b77125ad14aaf4a1abbacfe98 Mon Sep 17 00:00:00 2001 From: Thomas Adamcik Date: Mon, 20 May 2013 15:28:41 +0200 Subject: [PATCH 7/7] scanner: Update based on review comments. --- mopidy/models.py | 5 +++-- mopidy/scanner.py | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/mopidy/models.py b/mopidy/models.py index b5ec6a5f..fe390ddf 100644 --- a/mopidy/models.py +++ b/mopidy/models.py @@ -231,7 +231,7 @@ class Track(ImmutableObject): :type bitrate: integer :param musicbrainz_id: MusicBrainz ID :type musicbrainz_id: string - :param last_modified: integer representing last modifcation time + :param last_modified: Represents last modification time :type last_modified: integer """ @@ -266,7 +266,8 @@ class Track(ImmutableObject): musicbrainz_id = None #: Integer representing when the track was last modified, exact meaning - #: depends on source of track. + #: depends on source of track. For local files this is the mtime, for other + #: backends it could be a timestamp or simply a version counter. last_modified = 0 def __init__(self, *args, **kwargs): diff --git a/mopidy/scanner.py b/mopidy/scanner.py index 42c30861..7fd7b541 100644 --- a/mopidy/scanner.py +++ b/mopidy/scanner.py @@ -58,8 +58,8 @@ def main(): # TODO: missing error checking and other default setup code. audio = dummy_audio.DummyAudio() - local_backend_class = extensions['local'].get_backend_classes() - local_backend = local_backend_class[0](config, audio) + local_backend_classes = extensions['local'].get_backend_classes() + local_backend = local_backend_classes[0](config, audio) tracks = {} # Current lib. update = [] # Paths to rescan for updates/adds. @@ -97,7 +97,7 @@ def main(): logging.debug('Debug info for %s: %s', uri, debug) - logging.info('Scanning %d files.', len(update)) + logging.info('Scanning %d new/changed files.', len(update)) scanner = Scanner(update, store, debug) try: scanner.start()