From 8e343e1a4010675a7246f4afa23b827e25a1d3b0 Mon Sep 17 00:00:00 2001 From: Stein Magnus Jodal Date: Sat, 2 Nov 2013 23:37:58 +0100 Subject: [PATCH 01/18] Set version to 0.16.1 --- mopidy/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mopidy/__init__.py b/mopidy/__init__.py index b632fc0a..d41e656f 100644 --- a/mopidy/__init__.py +++ b/mopidy/__init__.py @@ -21,4 +21,4 @@ if (isinstance(pykka.__version__, basestring) warnings.filterwarnings('ignore', 'could not open display') -__version__ = '0.17.0a1' +__version__ = '0.16.1' From 80a112c31641a4b62f02acda18612b6c7cf95013 Mon Sep 17 00:00:00 2001 From: Stein Magnus Jodal Date: Sat, 2 Nov 2013 23:38:06 +0100 Subject: [PATCH 02/18] docs: Update changelog for v0.16.1 --- docs/changelog.rst | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 0e560f3d..59d8524a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,16 +4,28 @@ Changelog This changelog is used to track all major changes to Mopidy. -v0.17.0 (UNRELEASED) + +v0.16.1 (2013-11-02) ==================== +This is very small release to get Mopidy's Debian package ready for inclusion +in Debian. + +**Commands** + +- Fix removal of last dir level in paths to dependencies in + ``mopidy --show-deps`` output. + +- Add manpages for all commands. + **Local backend** -- Fix search filtering by track number. +- Fix search filtering by track number that was added in 0.16.0. **MPD frontend** -- Add support for ``list "albumartist" ...``. +- Add support for ``list "albumartist" ...`` which was missed when ``find`` and + ``search`` learned to handle ``albumartist`` in 0.16.0. v0.16.0 (2013-10-27) From 4b920e66d40f001797f4fa720a74e853b4ae728b Mon Sep 17 00:00:00 2001 From: Stein Magnus Jodal Date: Sun, 3 Nov 2013 12:24:56 +0100 Subject: [PATCH 03/18] docs: Add refs to fixed issues --- docs/changelog.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index bcb8e32a..44ca89f7 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -11,7 +11,7 @@ v0.17.0 (UNRELEASED) **Core** - The search field ``track`` has been renamed to ``track_name`` to avoid - confusion with ``track_no``. + confusion with ``track_no``. (Fixes: :issue:`535`) **Local backend** @@ -40,7 +40,7 @@ in Debian. **MPD frontend** - Add support for ``list "albumartist" ...`` which was missed when ``find`` and - ``search`` learned to handle ``albumartist`` in 0.16.0. + ``search`` learned to handle ``albumartist`` in 0.16.0. (Fixes: :issue:`553`) v0.16.0 (2013-10-27) From 0ab1aacbc5b57d84cda296f5ff66e9e33a67eb1d Mon Sep 17 00:00:00 2001 From: Stein Magnus Jodal Date: Mon, 4 Nov 2013 23:36:02 +0100 Subject: [PATCH 04/18] docs: Remove redundant Read The Docs config --- docs/conf.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 81b0d41b..77ee897e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -8,11 +8,6 @@ import os import sys -# -- Read The Docs configuration ---------------------------------------------- - -RTD_NEW_THEME = True - - # -- Workarounds to have autodoc generate API docs ---------------------------- sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) From d6ab78a86cfaf311369c769618c371fe8e4b1e0d Mon Sep 17 00:00:00 2001 From: Thomas Adamcik Date: Wed, 6 Nov 2013 13:25:26 +0100 Subject: [PATCH 05/18] audio: Re-add improved version of python based scanner. - Unlike the old python version we do not wait for the first audio handoff, we only progress until the PAUSED state. This ensure we don't block on empty files. - Instead of using the signal watch and running the main loop we simply poll the messages from the bus directly allowing for a synchronous code flow. - Between each file the pipeline is always returned to NULL, this is done as we found that gst 0.10 will slow down as the uribin does not cleanup the children it creates for handling each file. This issue is not present in 1.0. - This also works around a segfault that was likely caused by a race condition that seems to trigger in the 0.10 version of the pbutils discoverer. - This version of the scanner also fixes the per track slow down, and works out to be considerably faster than even the built in discoverer from 1.0. - Finally this removes the WMA hack as I kan no longer find any evidence of it being needed. --- mopidy/audio/scan.py | 88 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 mopidy/audio/scan.py diff --git a/mopidy/audio/scan.py b/mopidy/audio/scan.py new file mode 100644 index 00000000..a1c15e2a --- /dev/null +++ b/mopidy/audio/scan.py @@ -0,0 +1,88 @@ +from __future__ import unicode_literals + +import pygst +pygst.require('0.10') +import gst +import gobject + +import time + +from mopidy import exceptions + + +class Scanner(object): + def __init__(self, timeout=1000): + self.timeout_ms = timeout + + sink = gst.element_factory_make('fakesink') + + audio_caps = gst.Caps(b'audio/x-raw-int; audio/x-raw-float') + pad_added = lambda src, pad: pad.link(sink.get_pad('sink')) + + self.uribin = gst.element_factory_make('uridecodebin') + self.uribin.set_property('caps', audio_caps) + self.uribin.connect('pad-added', pad_added) + + self.pipe = gst.element_factory_make('pipeline') + self.pipe.add(self.uribin) + self.pipe.add(sink) + + self.bus = self.pipe.get_bus() + self.bus.set_flushing(True) + + def scan(self, uri): + try: + self._setup(uri) + data = self._collect() + # Make sure uri and duration does not come from tags. + data[b'uri'] = uri + data[gst.TAG_DURATION] = self._query_duration() + finally: + self._reset() + + return data + + def _setup(self, uri): + """Primes the pipeline for collection.""" + self.pipe.set_state(gst.STATE_READY) + self.uribin.set_property(b'uri', uri) + self.bus.set_flushing(False) + self.pipe.set_state(gst.STATE_PAUSED) + + def _collect(self): + """Polls for messages to collect data.""" + start = time.time() + timeout_s = self.timeout_ms / float(1000) + poll_timeout_ns = 1000 + data = {} + + while time.time() - start < timeout_s: + message = self.bus.poll(gst.MESSAGE_ANY, poll_timeout_ns) + + if message is None: + pass # polling the bus timed out. + elif message.type == gst.MESSAGE_ERROR: + raise exceptions.ScannerError(message.parse_error()[0]) + elif message.type == gst.MESSAGE_EOS: + return data + elif message.type == gst.MESSAGE_ASYNC_DONE: + if message.src == self.pipe: + return data + elif message.type == gst.MESSAGE_TAG: + taglist = message.parse_tag() + for key in taglist.keys(): + data[key] = taglist[key] + + raise exceptions.ScannerError('Timeout after %dms' % self.timeout_ms) + + def _reset(self): + """Ensures we cleanup child elements and flush the bus.""" + self.bus.set_flushing(True) + self.pipe.set_state(gst.STATE_NULL) + + def _query_duration(self): + try: + duration = self.pipe.query_duration(gst.FORMAT_TIME, None)[0] + return duration // gst.MSECOND + except gst.QueryError: + return None From 0a2d74eff173de1d01e8a471324eb1628628c19c Mon Sep 17 00:00:00 2001 From: Thomas Adamcik Date: Wed, 6 Nov 2013 14:47:02 +0100 Subject: [PATCH 06/18] scanner: Update to use new mopidy.audio.scan Also adds the check less than 100ms check back to the scanner. --- mopidy/audio/scan.py | 4 ++++ mopidy/scanner.py | 46 +++---------------------------------------- tests/scanner_test.py | 3 ++- 3 files changed, 9 insertions(+), 44 deletions(-) diff --git a/mopidy/audio/scan.py b/mopidy/audio/scan.py index a1c15e2a..4540bc05 100644 --- a/mopidy/audio/scan.py +++ b/mopidy/audio/scan.py @@ -40,6 +40,10 @@ class Scanner(object): finally: self._reset() + # TODO: this should be an option or just moved out. + if data[gst.TAG_DURATION] < 100: + raise exceptions.ScannerError( + 'Rejecting file with less than 100ms audio data.') return data def _setup(self, uri): diff --git a/mopidy/scanner.py b/mopidy/scanner.py index dd21fdb4..56639bc9 100644 --- a/mopidy/scanner.py +++ b/mopidy/scanner.py @@ -19,9 +19,9 @@ sys.argv[1:] = [] import pygst pygst.require('0.10') import gst -import gst.pbutils from mopidy import config as config_lib, exceptions, ext +from mopidy.audio import scan from mopidy.models import Track, Artist, Album from mopidy.utils import log, path, versioning @@ -103,11 +103,12 @@ def main(): logging.info('Found %d new or modified tracks.', len(uris_update)) logging.info('Scanning new and modified tracks.') - scanner = Scanner(config['local']['scan_timeout']) + scanner = scan.Scanner(config['local']['scan_timeout']) for uri in uris_update: try: data = scanner.scan(uri) data[b'mtime'] = os.path.getmtime(path.uri_to_path(uri)) + # TODO: check minumum time track = translator(data) local_updater.add(track) logging.debug('Added %s', track.uri) @@ -183,46 +184,5 @@ def translator(data): return Track(**track_kwargs) -class Scanner(object): - def __init__(self, timeout=1000): - self.discoverer = gst.pbutils.Discoverer(timeout * 1000000) - - def scan(self, uri): - try: - info = self.discoverer.discover_uri(uri) - except gobject.GError as e: - # Loosing traceback is non-issue since this is from C code. - raise exceptions.ScannerError(e) - - data = {} - audio_streams = info.get_audio_streams() - - if not audio_streams: - raise exceptions.ScannerError('Did not find any audio streams.') - - for stream in audio_streams: - taglist = stream.get_tags() - if not taglist: - continue - for key in taglist.keys(): - # XXX: For some crazy reason some wma files spit out lists - # here, not sure if this is due to better data in headers or - # wma being stupid. So ugly hack for now :/ - if type(taglist[key]) is list: - data[key] = taglist[key][0] - else: - data[key] = taglist[key] - - # Never trust metadata for these fields: - data[b'uri'] = uri - data[b'duration'] = info.get_duration() // gst.MSECOND - - if data[b'duration'] < 100: - raise exceptions.ScannerError( - 'Rejecting file with less than 100ms audio data.') - - return data - - if __name__ == '__main__': main() diff --git a/tests/scanner_test.py b/tests/scanner_test.py index 1102c525..47cc8116 100644 --- a/tests/scanner_test.py +++ b/tests/scanner_test.py @@ -3,8 +3,9 @@ from __future__ import unicode_literals import unittest from mopidy import exceptions +from mopidy.audio.scan import Scanner from mopidy.models import Track, Artist, Album -from mopidy.scanner import Scanner, translator +from mopidy.scanner import translator from mopidy.utils import path as path_lib from tests import path_to_data_dir From 4cadba0ac70a9121ed6320e4eb027671d7148fd1 Mon Sep 17 00:00:00 2001 From: Thomas Adamcik Date: Wed, 6 Nov 2013 15:12:15 +0100 Subject: [PATCH 07/18] scanner: Add progress tracking and process sorted uris --- mopidy/scanner.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/mopidy/scanner.py b/mopidy/scanner.py index 56639bc9..4430ea31 100644 --- a/mopidy/scanner.py +++ b/mopidy/scanner.py @@ -5,6 +5,7 @@ import datetime import logging import os import sys +import time import gobject gobject.threads_init() @@ -104,7 +105,9 @@ def main(): logging.info('Scanning new and modified tracks.') scanner = scan.Scanner(config['local']['scan_timeout']) - for uri in uris_update: + progress = Progress(len(uris_update)) + + for uri in sorted(uris_update): try: data = scanner.scan(uri) data[b'mtime'] = os.path.getmtime(path.uri_to_path(uri)) @@ -115,10 +118,27 @@ def main(): except exceptions.ScannerError as error: logging.warning('Failed %s: %s', uri, error) - logging.info('Done scanning; commiting changes.') + progress.increment() + + logging.info('Commiting changes.') local_updater.commit() +class Progress(object): + def __init__(self, total): + self.count = 0 + self.total = total + self.start = time.time() + + def increment(self, force=False): + self.count += 1 + if self.count % 1000 == 0 or self.count == self.total: + duration = time.time() - self.start + remainder = duration / self.count * (self.total - self.count) + logging.info('Scanned %d of %d files in %ds, ~%ds left.', + self.count, self.total, duration, remainder) + + def parse_args(): parser = argparse.ArgumentParser() parser.add_argument( From 20b060284210de0d9ad13b3e2b1b87e12d3a8814 Mon Sep 17 00:00:00 2001 From: Thomas Adamcik Date: Wed, 6 Nov 2013 16:17:47 +0100 Subject: [PATCH 08/18] audio/scanner: Move translator into audio. Tries to move more of the gst bits and pieces we are leaking into audio. --- mopidy/audio/scan.py | 51 +++++++++++++++ mopidy/scanner.py | 63 ++----------------- tests/{scanner_test.py => audio/scan_test.py} | 7 +-- 3 files changed, 58 insertions(+), 63 deletions(-) rename tests/{scanner_test.py => audio/scan_test.py} (97%) diff --git a/mopidy/audio/scan.py b/mopidy/audio/scan.py index 4540bc05..f12b8ff6 100644 --- a/mopidy/audio/scan.py +++ b/mopidy/audio/scan.py @@ -5,9 +5,11 @@ pygst.require('0.10') import gst import gobject +import datetime import time from mopidy import exceptions +from mopidy.models import Track, Artist, Album class Scanner(object): @@ -90,3 +92,52 @@ class Scanner(object): return duration // gst.MSECOND except gst.QueryError: return None + + +def audio_data_to_track(data): + """Convert taglist data + our extras to a track.""" + albumartist_kwargs = {} + album_kwargs = {} + artist_kwargs = {} + track_kwargs = {} + + def _retrieve(source_key, target_key, target): + if source_key in data: + target[target_key] = data[source_key] + + _retrieve(gst.TAG_ALBUM, 'name', album_kwargs) + _retrieve(gst.TAG_TRACK_COUNT, 'num_tracks', album_kwargs) + _retrieve(gst.TAG_ALBUM_VOLUME_COUNT, 'num_discs', album_kwargs) + _retrieve(gst.TAG_ARTIST, 'name', artist_kwargs) + + if gst.TAG_DATE in data and data[gst.TAG_DATE]: + date = data[gst.TAG_DATE] + try: + date = datetime.date(date.year, date.month, date.day) + except ValueError: + pass # Ignore invalid dates + else: + track_kwargs['date'] = date.isoformat() + + _retrieve(gst.TAG_TITLE, 'name', track_kwargs) + _retrieve(gst.TAG_TRACK_NUMBER, 'track_no', track_kwargs) + _retrieve(gst.TAG_ALBUM_VOLUME_NUMBER, 'disc_no', track_kwargs) + + # Following keys don't seem to have TAG_* constant. + _retrieve('album-artist', 'name', albumartist_kwargs) + _retrieve('musicbrainz-trackid', 'musicbrainz_id', track_kwargs) + _retrieve('musicbrainz-artistid', 'musicbrainz_id', artist_kwargs) + _retrieve('musicbrainz-albumid', 'musicbrainz_id', album_kwargs) + _retrieve( + 'musicbrainz-albumartistid', 'musicbrainz_id', albumartist_kwargs) + + if albumartist_kwargs: + album_kwargs['artists'] = [Artist(**albumartist_kwargs)] + + track_kwargs['uri'] = data['uri'] + track_kwargs['last_modified'] = int(data['mtime']) + track_kwargs['length'] = data[gst.TAG_DURATION] + track_kwargs['album'] = Album(**album_kwargs) + track_kwargs['artists'] = [Artist(**artist_kwargs)] + + return Track(**track_kwargs) diff --git a/mopidy/scanner.py b/mopidy/scanner.py index 4430ea31..02b04cb8 100644 --- a/mopidy/scanner.py +++ b/mopidy/scanner.py @@ -1,7 +1,6 @@ from __future__ import unicode_literals import argparse -import datetime import logging import os import sys @@ -10,20 +9,13 @@ import time import gobject gobject.threads_init() - # Extract any command line arguments. This needs to be done before GStreamer is # imported, so that GStreamer doesn't hijack e.g. ``--help``. mopidy_args = sys.argv[1:] sys.argv[1:] = [] - -import pygst -pygst.require('0.10') -import gst - from mopidy import config as config_lib, exceptions, ext from mopidy.audio import scan -from mopidy.models import Track, Artist, Album from mopidy.utils import log, path, versioning @@ -81,11 +73,13 @@ def main(): logging.info('Checking tracks from library.') for track in local_updater.load(): try: + # TODO: convert local to file uri / path stat = os.stat(path.uri_to_path(track.uri)) if int(stat.st_mtime) > track.last_modified: uris_update.add(track.uri) uris_library.add(track.uri) except OSError: + logging.debug('Missing file %s', track.uri) uris_remove.add(track.uri) logging.info('Removing %d moved or deleted tracks.', len(uris_remove)) @@ -111,8 +105,8 @@ def main(): try: data = scanner.scan(uri) data[b'mtime'] = os.path.getmtime(path.uri_to_path(uri)) - # TODO: check minumum time - track = translator(data) + # TODO: check minumum time here instead of in scanner. + track = scan.audio_data_to_track(data) local_updater.add(track) logging.debug('Added %s', track.uri) except exceptions.ScannerError as error: @@ -155,54 +149,5 @@ def parse_args(): return parser.parse_args(args=mopidy_args) -# TODO: move into scanner. -def translator(data): - albumartist_kwargs = {} - album_kwargs = {} - artist_kwargs = {} - track_kwargs = {} - - def _retrieve(source_key, target_key, target): - if source_key in data: - target[target_key] = data[source_key] - - _retrieve(gst.TAG_ALBUM, 'name', album_kwargs) - _retrieve(gst.TAG_TRACK_COUNT, 'num_tracks', album_kwargs) - _retrieve(gst.TAG_ALBUM_VOLUME_COUNT, 'num_discs', album_kwargs) - _retrieve(gst.TAG_ARTIST, 'name', artist_kwargs) - - if gst.TAG_DATE in data and data[gst.TAG_DATE]: - date = data[gst.TAG_DATE] - try: - date = datetime.date(date.year, date.month, date.day) - except ValueError: - pass # Ignore invalid dates - else: - track_kwargs['date'] = date.isoformat() - - _retrieve(gst.TAG_TITLE, 'name', track_kwargs) - _retrieve(gst.TAG_TRACK_NUMBER, 'track_no', track_kwargs) - _retrieve(gst.TAG_ALBUM_VOLUME_NUMBER, 'disc_no', track_kwargs) - - # Following keys don't seem to have TAG_* constant. - _retrieve('album-artist', 'name', albumartist_kwargs) - _retrieve('musicbrainz-trackid', 'musicbrainz_id', track_kwargs) - _retrieve('musicbrainz-artistid', 'musicbrainz_id', artist_kwargs) - _retrieve('musicbrainz-albumid', 'musicbrainz_id', album_kwargs) - _retrieve( - 'musicbrainz-albumartistid', 'musicbrainz_id', albumartist_kwargs) - - if albumartist_kwargs: - album_kwargs['artists'] = [Artist(**albumartist_kwargs)] - - track_kwargs['uri'] = data['uri'] - track_kwargs['last_modified'] = int(data['mtime']) - track_kwargs['length'] = data[gst.TAG_DURATION] - track_kwargs['album'] = Album(**album_kwargs) - track_kwargs['artists'] = [Artist(**artist_kwargs)] - - return Track(**track_kwargs) - - if __name__ == '__main__': main() diff --git a/tests/scanner_test.py b/tests/audio/scan_test.py similarity index 97% rename from tests/scanner_test.py rename to tests/audio/scan_test.py index 47cc8116..eb92635f 100644 --- a/tests/scanner_test.py +++ b/tests/audio/scan_test.py @@ -3,9 +3,8 @@ from __future__ import unicode_literals import unittest from mopidy import exceptions -from mopidy.audio.scan import Scanner +from mopidy.audio import scan from mopidy.models import Track, Artist, Album -from mopidy.scanner import translator from mopidy.utils import path as path_lib from tests import path_to_data_dir @@ -77,7 +76,7 @@ class TranslatorTest(unittest.TestCase): def check(self): expected = self.build_track() - actual = translator(self.data) + actual = scan.audio_data_to_track(self.data) self.assertEqual(expected, actual) def test_basic_data(self): @@ -152,7 +151,7 @@ class ScannerTest(unittest.TestCase): def scan(self, path): paths = path_lib.find_files(path_to_data_dir(path)) uris = (path_lib.path_to_uri(p) for p in paths) - scanner = Scanner() + scanner = scan.Scanner() for uri in uris: key = uri[len('file://'):] try: From 16ac5277f68a53b9aa9826698ec06f8b120a7a76 Mon Sep 17 00:00:00 2001 From: Thomas Adamcik Date: Wed, 6 Nov 2013 16:25:24 +0100 Subject: [PATCH 09/18] audio: Make min duration in scanner configurable. --- mopidy/audio/scan.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mopidy/audio/scan.py b/mopidy/audio/scan.py index f12b8ff6..f04fa2fd 100644 --- a/mopidy/audio/scan.py +++ b/mopidy/audio/scan.py @@ -13,8 +13,9 @@ from mopidy.models import Track, Artist, Album class Scanner(object): - def __init__(self, timeout=1000): + def __init__(self, timeout=1000, min_duration=100): self.timeout_ms = timeout + self.min_duration_ms = min_duration sink = gst.element_factory_make('fakesink') @@ -42,10 +43,9 @@ class Scanner(object): finally: self._reset() - # TODO: this should be an option or just moved out. - if data[gst.TAG_DURATION] < 100: - raise exceptions.ScannerError( - 'Rejecting file with less than 100ms audio data.') + if data[gst.TAG_DURATION] < self.min_duration_ms: + raise exceptions.ScannerError('Rejecting file with less than %dms ' + 'audio data.' % self.min_duration_ms) return data def _setup(self, uri): From 469f414c4cc9b8cdd1f4b062121c9c95f7dbdf13 Mon Sep 17 00:00:00 2001 From: Thomas Adamcik Date: Wed, 6 Nov 2013 16:28:53 +0100 Subject: [PATCH 10/18] audio: Don't do duration conversion to ms in scanner --- mopidy/audio/scan.py | 7 +++---- tests/audio/scan_test.py | 6 +++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/mopidy/audio/scan.py b/mopidy/audio/scan.py index f04fa2fd..b1565ee3 100644 --- a/mopidy/audio/scan.py +++ b/mopidy/audio/scan.py @@ -43,7 +43,7 @@ class Scanner(object): finally: self._reset() - if data[gst.TAG_DURATION] < self.min_duration_ms: + if data[gst.TAG_DURATION] < self.min_duration_ms * gst.MSECOND: raise exceptions.ScannerError('Rejecting file with less than %dms ' 'audio data.' % self.min_duration_ms) return data @@ -88,8 +88,7 @@ class Scanner(object): def _query_duration(self): try: - duration = self.pipe.query_duration(gst.FORMAT_TIME, None)[0] - return duration // gst.MSECOND + return self.pipe.query_duration(gst.FORMAT_TIME, None)[0] except gst.QueryError: return None @@ -136,7 +135,7 @@ def audio_data_to_track(data): track_kwargs['uri'] = data['uri'] track_kwargs['last_modified'] = int(data['mtime']) - track_kwargs['length'] = data[gst.TAG_DURATION] + track_kwargs['length'] = data[gst.TAG_DURATION] // gst.MSECOND track_kwargs['album'] = Album(**album_kwargs) track_kwargs['artists'] = [Artist(**artist_kwargs)] diff --git a/tests/audio/scan_test.py b/tests/audio/scan_test.py index eb92635f..b53b0b57 100644 --- a/tests/audio/scan_test.py +++ b/tests/audio/scan_test.py @@ -31,7 +31,7 @@ class TranslatorTest(unittest.TestCase): 'album-disc-count': 3, 'date': FakeGstDate(2006, 1, 1,), 'container-format': 'ID3 tag', - 'duration': 4531, + 'duration': 4531000000, 'musicbrainz-trackid': 'mbtrackid', 'musicbrainz-albumid': 'mbalbumid', 'musicbrainz-artistid': 'mbartistid', @@ -182,8 +182,8 @@ class ScannerTest(unittest.TestCase): def test_duration_is_set(self): self.scan('scanner/simple') - self.check('scanner/simple/song1.mp3', 'duration', 4680) - self.check('scanner/simple/song1.ogg', 'duration', 4680) + self.check('scanner/simple/song1.mp3', 'duration', 4680000000) + self.check('scanner/simple/song1.ogg', 'duration', 4680000000) def test_artist_is_set(self): self.scan('scanner/simple') From e6381a1a0a57eae80295f3ae22a6cb8b589741f8 Mon Sep 17 00:00:00 2001 From: Thomas Adamcik Date: Wed, 6 Nov 2013 17:50:38 +0100 Subject: [PATCH 11/18] audio: Add mtime to scanner results for file: uris --- mopidy/audio/scan.py | 8 ++++++++ mopidy/scanner.py | 2 -- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/mopidy/audio/scan.py b/mopidy/audio/scan.py index b1565ee3..4906b46c 100644 --- a/mopidy/audio/scan.py +++ b/mopidy/audio/scan.py @@ -6,10 +6,12 @@ import gst import gobject import datetime +import os import time from mopidy import exceptions from mopidy.models import Track, Artist, Album +from mopidy.utils import path class Scanner(object): @@ -39,6 +41,7 @@ class Scanner(object): data = self._collect() # Make sure uri and duration does not come from tags. data[b'uri'] = uri + data[b'mtime'] = self._query_mtime(uri) data[gst.TAG_DURATION] = self._query_duration() finally: self._reset() @@ -92,6 +95,11 @@ class Scanner(object): except gst.QueryError: return None + def _query_mtime(self, uri): + if not uri.startswith('file:'): + return None + return os.path.getmtime(path.uri_to_path(uri)) + def audio_data_to_track(data): """Convert taglist data + our extras to a track.""" diff --git a/mopidy/scanner.py b/mopidy/scanner.py index 02b04cb8..ee54c04d 100644 --- a/mopidy/scanner.py +++ b/mopidy/scanner.py @@ -104,8 +104,6 @@ def main(): for uri in sorted(uris_update): try: data = scanner.scan(uri) - data[b'mtime'] = os.path.getmtime(path.uri_to_path(uri)) - # TODO: check minumum time here instead of in scanner. track = scan.audio_data_to_track(data) local_updater.add(track) logging.debug('Added %s', track.uri) From dc3cf427b60c9cafb821304bd3b42c13e0e8cd03 Mon Sep 17 00:00:00 2001 From: Thomas Adamcik Date: Wed, 6 Nov 2013 18:40:12 +0100 Subject: [PATCH 12/18] scanner/mpd: Add TODOs for later and fix flake8 warning --- mopidy/audio/scan.py | 1 - mopidy/frontends/mpd/translator.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/mopidy/audio/scan.py b/mopidy/audio/scan.py index 4906b46c..82803379 100644 --- a/mopidy/audio/scan.py +++ b/mopidy/audio/scan.py @@ -3,7 +3,6 @@ from __future__ import unicode_literals import pygst pygst.require('0.10') import gst -import gobject import datetime import os diff --git a/mopidy/frontends/mpd/translator.py b/mopidy/frontends/mpd/translator.py index 880d1411..236b814f 100644 --- a/mopidy/frontends/mpd/translator.py +++ b/mopidy/frontends/mpd/translator.py @@ -301,6 +301,7 @@ def _add_to_tag_cache(result, dirs, files, media_dir): relative_path = os.path.relpath(path, base_path) relative_uri = urllib.quote(relative_path) + # TODO: use track.last_modified track_result['file'] = relative_uri track_result['mtime'] = get_mtime(path) track_result['key'] = os.path.basename(text_path) From 86926e8011b1802ed4a51efbef2cce9e07b8fb24 Mon Sep 17 00:00:00 2001 From: Thomas Adamcik Date: Wed, 6 Nov 2013 21:14:11 +0100 Subject: [PATCH 13/18] scanner: Remove unused argument for progress helper. --- mopidy/scanner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mopidy/scanner.py b/mopidy/scanner.py index ee54c04d..30fb553b 100644 --- a/mopidy/scanner.py +++ b/mopidy/scanner.py @@ -122,7 +122,7 @@ class Progress(object): self.total = total self.start = time.time() - def increment(self, force=False): + def increment(self): self.count += 1 if self.count % 1000 == 0 or self.count == self.total: duration = time.time() - self.start From f8dedc0b848a7fdce6caf360279b5a450729dcc1 Mon Sep 17 00:00:00 2001 From: Thomas Adamcik Date: Wed, 6 Nov 2013 21:46:41 +0100 Subject: [PATCH 14/18] local: Add local_to_file_uri translator --- mopidy/backends/local/playback.py | 8 ++++---- mopidy/backends/local/translator.py | 9 ++++++++- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/mopidy/backends/local/playback.py b/mopidy/backends/local/playback.py index 98c92a85..9fbb736d 100644 --- a/mopidy/backends/local/playback.py +++ b/mopidy/backends/local/playback.py @@ -6,14 +6,14 @@ import os from mopidy.backends import base from mopidy.utils import path +from . import translator + logger = logging.getLogger('mopidy.backends.local') class LocalPlaybackProvider(base.BasePlaybackProvider): def change_track(self, track): media_dir = self.backend.config['local']['media_dir'] - # TODO: check that type is correct. - file_path = path.uri_to_path(track.uri).split(b':', 1)[1] - file_path = os.path.join(media_dir, file_path) - track = track.copy(uri=path.path_to_uri(file_path)) + uri = translator.local_to_file_uri(track.uri, media_dir) + track = track.copy(uri=uri) return super(LocalPlaybackProvider, self).change_track(track) diff --git a/mopidy/backends/local/translator.py b/mopidy/backends/local/translator.py index 3a02a8af..3c6b8151 100644 --- a/mopidy/backends/local/translator.py +++ b/mopidy/backends/local/translator.py @@ -6,11 +6,18 @@ import urlparse from mopidy.models import Track, Artist, Album from mopidy.utils.encoding import locale_decode -from mopidy.utils.path import path_to_uri +from mopidy.utils.path import path_to_uri, uri_to_path logger = logging.getLogger('mopidy.backends.local') +def local_to_file_uri(uri, media_dir): + # TODO: check that type is correct. + file_path = uri_to_path(uri).split(b':', 1)[1] + file_path = os.path.join(media_dir, file_path) + return path_to_uri(file_path) + + def parse_m3u(file_path, media_dir): r""" Convert M3U file list of uris From e7dd7e26773f493a5bf1dd6355fb05078dbb3c37 Mon Sep 17 00:00:00 2001 From: Thomas Adamcik Date: Thu, 7 Nov 2013 00:03:00 +0100 Subject: [PATCH 15/18] local/scanner: Make checking mtime and skipping known files work again This change just patches over the worst of the inconsistencies in how the scanner mixes local and file uris to get us to a working state again. Ideally, this still needs a real cleanup when we finish the plugable library providers and/or json library work. --- mopidy/backends/local/library.py | 7 +++++-- mopidy/scanner.py | 19 +++++++++++-------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/mopidy/backends/local/library.py b/mopidy/backends/local/library.py index 86d960c1..9791c518 100644 --- a/mopidy/backends/local/library.py +++ b/mopidy/backends/local/library.py @@ -8,7 +8,7 @@ from mopidy.backends import base from mopidy.frontends.mpd import translator as mpd_translator from mopidy.models import Album, SearchResult -from .translator import parse_mpd_tag_cache +from .translator import local_to_file_uri, parse_mpd_tag_cache logger = logging.getLogger('mopidy.backends.local') @@ -189,7 +189,10 @@ class LocalLibraryUpdateProvider(base.BaseLibraryProvider): def load(self): tracks = parse_mpd_tag_cache(self._tag_cache_file, self._media_dir) for track in tracks: - self._tracks[track.uri] = track + # TODO: this should use uris as is, i.e. hack that should go away + # with tag caches. + uri = local_to_file_uri(track.uri, self._media_dir) + self._tracks[uri] = track.copy(uri=uri) return tracks def add(self, track): diff --git a/mopidy/scanner.py b/mopidy/scanner.py index 30fb553b..b61517ef 100644 --- a/mopidy/scanner.py +++ b/mopidy/scanner.py @@ -16,6 +16,7 @@ sys.argv[1:] = [] from mopidy import config as config_lib, exceptions, ext from mopidy.audio import scan +from mopidy.backends.local import translator from mopidy.utils import log, path, versioning @@ -66,6 +67,8 @@ def main(): media_dir = config['local']['media_dir'] excluded_extensions = config['local']['excluded_file_extensions'] + # TODO: cleanup to consistently use local urls, not a random mix of local + # and file uris depending on how the data was loaded. uris_library = set() uris_update = set() uris_remove = set() @@ -73,20 +76,20 @@ def main(): logging.info('Checking tracks from library.') for track in local_updater.load(): try: - # TODO: convert local to file uri / path - stat = os.stat(path.uri_to_path(track.uri)) + uri = translator.local_to_file_uri(track.uri, media_dir) + stat = os.stat(path.uri_to_path(uri)) if int(stat.st_mtime) > track.last_modified: - uris_update.add(track.uri) - uris_library.add(track.uri) + uris_update.add(uri) + uris_library.add(uri) except OSError: logging.debug('Missing file %s', track.uri) uris_remove.add(track.uri) - logging.info('Removing %d moved or deleted tracks.', len(uris_remove)) + logging.info('Removing %d missing tracks.', len(uris_remove)) for uri in uris_remove: local_updater.remove(uri) - logging.info('Checking %s for new or modified tracks.', media_dir) + logging.info('Checking %s for unknown tracks.', media_dir) for uri in path.find_uris(config['local']['media_dir']): if os.path.splitext(path.uri_to_path(uri))[1] in excluded_extensions: logging.debug('Skipped %s: File extension excluded.', uri) @@ -95,8 +98,8 @@ def main(): if uri not in uris_library: uris_update.add(uri) - logging.info('Found %d new or modified tracks.', len(uris_update)) - logging.info('Scanning new and modified tracks.') + logging.info('Found %d unknown tracks.', len(uris_update)) + logging.info('Scanning...') scanner = scan.Scanner(config['local']['scan_timeout']) progress = Progress(len(uris_update)) From 5b78ff94ff74318493f05dd97b247e7bd4585273 Mon Sep 17 00:00:00 2001 From: Stein Magnus Jodal Date: Thu, 7 Nov 2013 19:33:46 +0100 Subject: [PATCH 16/18] docs: Remove `count` from MPD limitations list --- docs/ext/mpd.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/ext/mpd.rst b/docs/ext/mpd.rst index b4d0e1c8..9dbcbe11 100644 --- a/docs/ext/mpd.rst +++ b/docs/ext/mpd.rst @@ -33,7 +33,6 @@ Items on this list will probably not be supported in the near future. - Stickers are not supported - Crossfade is not supported - Replay gain is not supported -- ``count`` does not provide any statistics - ``stats`` does not provide any statistics - ``list`` does not support listing tracks by genre - ``decoders`` does not provide information about available decoders From 93918cb1e059b7ce73e7be54414fcb53b9f3ced3 Mon Sep 17 00:00:00 2001 From: Thomas Adamcik Date: Thu, 7 Nov 2013 23:25:55 +0100 Subject: [PATCH 17/18] local: flake8 fixes --- mopidy/backends/local/playback.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/mopidy/backends/local/playback.py b/mopidy/backends/local/playback.py index 9fbb736d..b264dac7 100644 --- a/mopidy/backends/local/playback.py +++ b/mopidy/backends/local/playback.py @@ -1,10 +1,8 @@ from __future__ import unicode_literals import logging -import os from mopidy.backends import base -from mopidy.utils import path from . import translator From 51d1e2265515b5ef0a0d35c22c15417c04197ee9 Mon Sep 17 00:00:00 2001 From: Thomas Adamcik Date: Fri, 8 Nov 2013 23:51:45 +0100 Subject: [PATCH 18/18] docs: Update changelog for PR #566 and #567 --- docs/changelog.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 44ca89f7..f9ccbb3e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -18,6 +18,10 @@ v0.17.0 (UNRELEASED) - When scanning, we no longer default the album artist to be the same as the track artist. Album artist is now only populated if the scanned file got an explicit album artist set. +- Library scanning has been switched back to custom code due to various issues + with GStreamer's built in scanner in 0.10. This also fixes the scanner slowdown. + (Fixes: :issue:`565`) +- Fix scanner so that mtime is respected when deciding which files can be skipped. v0.16.1 (2013-11-02)