From 03f5ff6f5744303c9a0e86b29d010c9078bf0447 Mon Sep 17 00:00:00 2001 From: Thomas Adamcik Date: Tue, 26 Nov 2013 17:15:26 +0100 Subject: [PATCH 01/19] local: Start moving tag cache code out of main local --- mopidy/backends/local/__init__.py | 2 +- mopidy/backends/local/actor.py | 2 +- mopidy/backends/local/tagcache/__init__.py | 0 mopidy/backends/local/{ => tagcache}/library.py | 5 ++--- 4 files changed, 4 insertions(+), 5 deletions(-) create mode 100644 mopidy/backends/local/tagcache/__init__.py rename mopidy/backends/local/{ => tagcache}/library.py (98%) diff --git a/mopidy/backends/local/__init__.py b/mopidy/backends/local/__init__.py index 703b2562..8a2e12fd 100644 --- a/mopidy/backends/local/__init__.py +++ b/mopidy/backends/local/__init__.py @@ -34,7 +34,7 @@ class Extension(ext.Extension): return [LocalBackend] def get_library_updaters(self): - from .library import LocalLibraryUpdateProvider + from .tagcache.library import LocalLibraryUpdateProvider return [LocalLibraryUpdateProvider] def get_command(self): diff --git a/mopidy/backends/local/actor.py b/mopidy/backends/local/actor.py index f3611891..531b7546 100644 --- a/mopidy/backends/local/actor.py +++ b/mopidy/backends/local/actor.py @@ -8,7 +8,7 @@ import pykka from mopidy.backends import base from mopidy.utils import encoding, path -from .library import LocalLibraryProvider +from .tagcache.library import LocalLibraryProvider from .playlists import LocalPlaylistsProvider from .playback import LocalPlaybackProvider diff --git a/mopidy/backends/local/tagcache/__init__.py b/mopidy/backends/local/tagcache/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/mopidy/backends/local/library.py b/mopidy/backends/local/tagcache/library.py similarity index 98% rename from mopidy/backends/local/library.py rename to mopidy/backends/local/tagcache/library.py index da4e4bfd..b6ec05ff 100644 --- a/mopidy/backends/local/library.py +++ b/mopidy/backends/local/tagcache/library.py @@ -8,9 +8,9 @@ from mopidy.backends import base from mopidy.frontends.mpd import translator as mpd_translator from mopidy.models import Album, SearchResult -from .translator import local_to_file_uri, parse_mpd_tag_cache +from ..translator import local_to_file_uri, parse_mpd_tag_cache -logger = logging.getLogger('mopidy.backends.local') +logger = logging.getLogger('mopidy.backends.local.tagcache') class LocalLibraryProvider(base.BaseLibraryProvider): @@ -219,7 +219,6 @@ class LocalLibraryProvider(base.BaseLibraryProvider): raise LookupError('Missing query') -# TODO: rename and move to tagcache extension. class LocalLibraryUpdateProvider(base.BaseLibraryProvider): uri_schemes = ['local'] From ff9f473c2f87bc2f1dfbaafc1d13dc085a1ac589 Mon Sep 17 00:00:00 2001 From: Thomas Adamcik Date: Tue, 26 Nov 2013 17:47:52 +0100 Subject: [PATCH 02/19] local: Move tag cache translators and tests out. --- mopidy/backends/local/tagcache/library.py | 6 +- mopidy/backends/local/tagcache/translator.py | 246 +++++++++++++ mopidy/backends/local/translator.py | 125 ------- mopidy/frontends/mpd/translator.py | 115 ------ tests/backends/local/tagcache_test.py | 346 +++++++++++++++++++ tests/backends/local/translator_test.py | 106 +----- tests/frontends/mpd/translator_test.py | 235 +------------ 7 files changed, 598 insertions(+), 581 deletions(-) create mode 100644 mopidy/backends/local/tagcache/translator.py create mode 100644 tests/backends/local/tagcache_test.py diff --git a/mopidy/backends/local/tagcache/library.py b/mopidy/backends/local/tagcache/library.py index b6ec05ff..6efe6bf5 100644 --- a/mopidy/backends/local/tagcache/library.py +++ b/mopidy/backends/local/tagcache/library.py @@ -5,10 +5,10 @@ import os import tempfile from mopidy.backends import base -from mopidy.frontends.mpd import translator as mpd_translator +from mopidy.backends.local.translator import local_to_file_uri from mopidy.models import Album, SearchResult -from ..translator import local_to_file_uri, parse_mpd_tag_cache +from .translator import parse_mpd_tag_cache, tracks_to_tag_cache_format logger = logging.getLogger('mopidy.backends.local.tagcache') @@ -251,7 +251,7 @@ class LocalLibraryUpdateProvider(base.BaseLibraryProvider): prefix=basename + '.', dir=directory, delete=False) try: - for row in mpd_translator.tracks_to_tag_cache_format( + for row in tracks_to_tag_cache_format( self._tracks.values(), self._media_dir): if len(row) == 1: tmp.write(('%s\n' % row).encode('utf-8')) diff --git a/mopidy/backends/local/tagcache/translator.py b/mopidy/backends/local/tagcache/translator.py new file mode 100644 index 00000000..be54cd1d --- /dev/null +++ b/mopidy/backends/local/tagcache/translator.py @@ -0,0 +1,246 @@ +from __future__ import unicode_literals + +import logging +import os +import re +import urllib + +from mopidy.frontends.mpd import translator as mpd, protocol +from mopidy.models import Track, Artist, Album +from mopidy.utils.encoding import locale_decode +from mopidy.utils.path import mtime as get_mtime, split_path, uri_to_path + +logger = logging.getLogger('mopidy.backends.local.tagcache') + + +# TODO: remove music_dir from API +def parse_mpd_tag_cache(tag_cache, music_dir=''): + """ + Converts a MPD tag_cache into a lists of tracks, artists and albums. + """ + tracks = set() + + try: + with open(tag_cache) as library: + contents = library.read() + except IOError as error: + logger.warning('Could not open tag cache: %s', locale_decode(error)) + return tracks + + current = {} + state = None + + # TODO: uris as bytes + for line in contents.split(b'\n'): + if line == b'songList begin': + state = 'songs' + continue + elif line == b'songList end': + state = None + continue + elif not state: + continue + + key, value = line.split(b': ', 1) + + if key == b'key': + _convert_mpd_data(current, tracks) + current.clear() + + current[key.lower()] = value.decode('utf-8') + + _convert_mpd_data(current, tracks) + + return tracks + + +def _convert_mpd_data(data, tracks): + if not data: + return + + track_kwargs = {} + album_kwargs = {} + artist_kwargs = {} + albumartist_kwargs = {} + + if 'track' in data: + if '/' in data['track']: + album_kwargs['num_tracks'] = int(data['track'].split('/')[1]) + track_kwargs['track_no'] = int(data['track'].split('/')[0]) + else: + track_kwargs['track_no'] = int(data['track']) + + if 'mtime' in data: + track_kwargs['last_modified'] = int(data['mtime']) + + if 'artist' in data: + artist_kwargs['name'] = data['artist'] + + if 'albumartist' in data: + albumartist_kwargs['name'] = data['albumartist'] + + if 'composer' in data: + track_kwargs['composers'] = [Artist(name=data['composer'])] + + if 'performer' in data: + track_kwargs['performers'] = [Artist(name=data['performer'])] + + if 'album' in data: + album_kwargs['name'] = data['album'] + + if 'title' in data: + track_kwargs['name'] = data['title'] + + if 'genre' in data: + track_kwargs['genre'] = data['genre'] + + if 'date' in data: + track_kwargs['date'] = data['date'] + + if 'comment' in data: + track_kwargs['comment'] = data['comment'] + + if 'musicbrainz_trackid' in data: + track_kwargs['musicbrainz_id'] = data['musicbrainz_trackid'] + + if 'musicbrainz_albumid' in data: + album_kwargs['musicbrainz_id'] = data['musicbrainz_albumid'] + + if 'musicbrainz_artistid' in data: + artist_kwargs['musicbrainz_id'] = data['musicbrainz_artistid'] + + if 'musicbrainz_albumartistid' in data: + albumartist_kwargs['musicbrainz_id'] = ( + data['musicbrainz_albumartistid']) + + if artist_kwargs: + artist = Artist(**artist_kwargs) + track_kwargs['artists'] = [artist] + + if albumartist_kwargs: + albumartist = Artist(**albumartist_kwargs) + album_kwargs['artists'] = [albumartist] + + if album_kwargs: + album = Album(**album_kwargs) + track_kwargs['album'] = album + + if data['file'][0] == '/': + path = data['file'][1:] + else: + path = data['file'] + + track_kwargs['uri'] = 'local:track:%s' % path + track_kwargs['length'] = int(data.get('time', 0)) * 1000 + + track = Track(**track_kwargs) + tracks.add(track) + + +def tracks_to_tag_cache_format(tracks, media_dir): + """ + Format list of tracks for output to MPD tag cache + + :param tracks: the tracks + :type tracks: list of :class:`mopidy.models.Track` + :param media_dir: the path to the music dir + :type media_dir: string + :rtype: list of lists of two-tuples + """ + result = [ + ('info_begin',), + ('mpd_version', protocol.VERSION), + ('fs_charset', protocol.ENCODING), + ('info_end',) + ] + tracks.sort(key=lambda t: t.uri) + dirs, files = tracks_to_directory_tree(tracks, media_dir) + _add_to_tag_cache(result, dirs, files, media_dir) + return result + + +# TODO: bytes only +def _add_to_tag_cache(result, dirs, files, media_dir): + base_path = media_dir.encode('utf-8') + + for path, (entry_dirs, entry_files) in dirs.items(): + try: + text_path = path.decode('utf-8') + except UnicodeDecodeError: + text_path = urllib.quote(path).decode('utf-8') + name = os.path.split(text_path)[1] + result.append(('directory', text_path)) + result.append(('mtime', get_mtime(os.path.join(base_path, path)))) + result.append(('begin', name)) + _add_to_tag_cache(result, entry_dirs, entry_files, media_dir) + result.append(('end', name)) + + result.append(('songList begin',)) + + for track in files: + track_result = dict(mpd.track_to_mpd_format(track)) + + # XXX Don't save comments to the tag cache as they may span multiple + # lines. We'll start saving track comments when we move from tag_cache + # to a JSON file. See #579 for details. + if 'Comment' in track_result: + del track_result['Comment'] + + path = uri_to_path(track_result['file']) + try: + text_path = path.decode('utf-8') + except UnicodeDecodeError: + text_path = urllib.quote(path).decode('utf-8') + relative_path = os.path.relpath(path, base_path) + relative_uri = urllib.quote(relative_path) + + # TODO: use track.last_modified + track_result['file'] = relative_uri + track_result['mtime'] = get_mtime(path) + track_result['key'] = os.path.basename(text_path) + track_result = order_mpd_track_info(track_result.items()) + + result.extend(track_result) + + result.append(('songList end',)) + + +def tracks_to_directory_tree(tracks, media_dir): + directories = ({}, []) + + for track in tracks: + path = b'' + current = directories + + absolute_track_dir_path = os.path.dirname(uri_to_path(track.uri)) + relative_track_dir_path = re.sub( + '^' + re.escape(media_dir), b'', absolute_track_dir_path) + + for part in split_path(relative_track_dir_path): + path = os.path.join(path, part) + if path not in current[0]: + current[0][path] = ({}, []) + current = current[0][path] + current[1].append(track) + return directories + + +MPD_KEY_ORDER = ''' + key file Time Artist Album AlbumArtist Title Track Genre Date Composer + Performer Comment Disc MUSICBRAINZ_ALBUMID MUSICBRAINZ_ALBUMARTISTID + MUSICBRAINZ_ARTISTID MUSICBRAINZ_TRACKID mtime +'''.split() + + +def order_mpd_track_info(result): + """ + Order results from + :func:`mopidy.frontends.mpd.translator.track_to_mpd_format` so that it + matches MPD's ordering. Simply a cosmetic fix for easier diffing of + tag_caches. + + :param result: the track info + :type result: list of tuples + :rtype: list of tuples + """ + return sorted(result, key=lambda i: MPD_KEY_ORDER.index(i[0])) diff --git a/mopidy/backends/local/translator.py b/mopidy/backends/local/translator.py index b9aad3e0..dc266d1c 100644 --- a/mopidy/backends/local/translator.py +++ b/mopidy/backends/local/translator.py @@ -4,7 +4,6 @@ import logging import os import urlparse -from mopidy.models import Track, Artist, Album from mopidy.utils.encoding import locale_decode from mopidy.utils.path import path_to_uri, uri_to_path @@ -63,127 +62,3 @@ def parse_m3u(file_path, media_dir): uris.append(path) return uris - - -# TODO: remove music_dir from API -def parse_mpd_tag_cache(tag_cache, music_dir=''): - """ - Converts a MPD tag_cache into a lists of tracks, artists and albums. - """ - tracks = set() - - try: - with open(tag_cache) as library: - contents = library.read() - except IOError as error: - logger.warning('Could not open tag cache: %s', locale_decode(error)) - return tracks - - current = {} - state = None - - # TODO: uris as bytes - for line in contents.split(b'\n'): - if line == b'songList begin': - state = 'songs' - continue - elif line == b'songList end': - state = None - continue - elif not state: - continue - - key, value = line.split(b': ', 1) - - if key == b'key': - _convert_mpd_data(current, tracks) - current.clear() - - current[key.lower()] = value.decode('utf-8') - - _convert_mpd_data(current, tracks) - - return tracks - - -def _convert_mpd_data(data, tracks): - if not data: - return - - track_kwargs = {} - album_kwargs = {} - artist_kwargs = {} - albumartist_kwargs = {} - - if 'track' in data: - if '/' in data['track']: - album_kwargs['num_tracks'] = int(data['track'].split('/')[1]) - track_kwargs['track_no'] = int(data['track'].split('/')[0]) - else: - track_kwargs['track_no'] = int(data['track']) - - if 'mtime' in data: - track_kwargs['last_modified'] = int(data['mtime']) - - if 'artist' in data: - artist_kwargs['name'] = data['artist'] - - if 'albumartist' in data: - albumartist_kwargs['name'] = data['albumartist'] - - if 'composer' in data: - track_kwargs['composers'] = [Artist(name=data['composer'])] - - if 'performer' in data: - track_kwargs['performers'] = [Artist(name=data['performer'])] - - if 'album' in data: - album_kwargs['name'] = data['album'] - - if 'title' in data: - track_kwargs['name'] = data['title'] - - if 'genre' in data: - track_kwargs['genre'] = data['genre'] - - if 'date' in data: - track_kwargs['date'] = data['date'] - - if 'comment' in data: - track_kwargs['comment'] = data['comment'] - - if 'musicbrainz_trackid' in data: - track_kwargs['musicbrainz_id'] = data['musicbrainz_trackid'] - - if 'musicbrainz_albumid' in data: - album_kwargs['musicbrainz_id'] = data['musicbrainz_albumid'] - - if 'musicbrainz_artistid' in data: - artist_kwargs['musicbrainz_id'] = data['musicbrainz_artistid'] - - if 'musicbrainz_albumartistid' in data: - albumartist_kwargs['musicbrainz_id'] = ( - data['musicbrainz_albumartistid']) - - if artist_kwargs: - artist = Artist(**artist_kwargs) - track_kwargs['artists'] = [artist] - - if albumartist_kwargs: - albumartist = Artist(**albumartist_kwargs) - album_kwargs['artists'] = [albumartist] - - if album_kwargs: - album = Album(**album_kwargs) - track_kwargs['album'] = album - - if data['file'][0] == '/': - path = data['file'][1:] - else: - path = data['file'] - - track_kwargs['uri'] = 'local:track:%s' % path - track_kwargs['length'] = int(data.get('time', 0)) * 1000 - - track = Track(**track_kwargs) - tracks.add(track) diff --git a/mopidy/frontends/mpd/translator.py b/mopidy/frontends/mpd/translator.py index 4f38effa..671bfae7 100644 --- a/mopidy/frontends/mpd/translator.py +++ b/mopidy/frontends/mpd/translator.py @@ -1,14 +1,9 @@ from __future__ import unicode_literals -import os -import re import shlex -import urllib -from mopidy.frontends.mpd import protocol from mopidy.frontends.mpd.exceptions import MpdArgError from mopidy.models import TlTrack -from mopidy.utils.path import mtime as get_mtime, uri_to_path, split_path # TODO: special handling of local:// uri scheme @@ -87,27 +82,6 @@ def track_to_mpd_format(track, position=None): return result -MPD_KEY_ORDER = ''' - key file Time Artist Album AlbumArtist Title Track Genre Date Composer - Performer Comment Disc MUSICBRAINZ_ALBUMID MUSICBRAINZ_ALBUMARTISTID - MUSICBRAINZ_ARTISTID MUSICBRAINZ_TRACKID mtime -'''.split() - - -def order_mpd_track_info(result): - """ - Order results from - :func:`mopidy.frontends.mpd.translator.track_to_mpd_format` so that it - matches MPD's ordering. Simply a cosmetic fix for easier diffing of - tag_caches. - - :param result: the track info - :type result: list of tuples - :rtype: list of tuples - """ - return sorted(result, key=lambda i: MPD_KEY_ORDER.index(i[0])) - - def artists_to_mpd_format(artists): """ Format track artists for output to MPD client. @@ -197,92 +171,3 @@ def query_from_mpd_list_format(field, mpd_query): return query else: raise MpdArgError('not able to parse args', command='list') - - -# TODO: move to tagcache backend. -def tracks_to_tag_cache_format(tracks, media_dir): - """ - Format list of tracks for output to MPD tag cache - - :param tracks: the tracks - :type tracks: list of :class:`mopidy.models.Track` - :param media_dir: the path to the music dir - :type media_dir: string - :rtype: list of lists of two-tuples - """ - result = [ - ('info_begin',), - ('mpd_version', protocol.VERSION), - ('fs_charset', protocol.ENCODING), - ('info_end',) - ] - tracks.sort(key=lambda t: t.uri) - dirs, files = tracks_to_directory_tree(tracks, media_dir) - _add_to_tag_cache(result, dirs, files, media_dir) - return result - - -# TODO: bytes only -def _add_to_tag_cache(result, dirs, files, media_dir): - base_path = media_dir.encode('utf-8') - - for path, (entry_dirs, entry_files) in dirs.items(): - try: - text_path = path.decode('utf-8') - except UnicodeDecodeError: - text_path = urllib.quote(path).decode('utf-8') - name = os.path.split(text_path)[1] - result.append(('directory', text_path)) - result.append(('mtime', get_mtime(os.path.join(base_path, path)))) - result.append(('begin', name)) - _add_to_tag_cache(result, entry_dirs, entry_files, media_dir) - result.append(('end', name)) - - result.append(('songList begin',)) - - for track in files: - track_result = dict(track_to_mpd_format(track)) - - # XXX Don't save comments to the tag cache as they may span multiple - # lines. We'll start saving track comments when we move from tag_cache - # to a JSON file. See #579 for details. - if 'Comment' in track_result: - del track_result['Comment'] - - path = uri_to_path(track_result['file']) - try: - text_path = path.decode('utf-8') - except UnicodeDecodeError: - text_path = urllib.quote(path).decode('utf-8') - relative_path = os.path.relpath(path, base_path) - relative_uri = urllib.quote(relative_path) - - # TODO: use track.last_modified - track_result['file'] = relative_uri - track_result['mtime'] = get_mtime(path) - track_result['key'] = os.path.basename(text_path) - track_result = order_mpd_track_info(track_result.items()) - - result.extend(track_result) - - result.append(('songList end',)) - - -def tracks_to_directory_tree(tracks, media_dir): - directories = ({}, []) - - for track in tracks: - path = b'' - current = directories - - absolute_track_dir_path = os.path.dirname(uri_to_path(track.uri)) - relative_track_dir_path = re.sub( - '^' + re.escape(media_dir), b'', absolute_track_dir_path) - - for part in split_path(relative_track_dir_path): - path = os.path.join(path, part) - if path not in current[0]: - current[0][path] = ({}, []) - current = current[0][path] - current[1].append(track) - return directories diff --git a/tests/backends/local/tagcache_test.py b/tests/backends/local/tagcache_test.py new file mode 100644 index 00000000..6d0b7469 --- /dev/null +++ b/tests/backends/local/tagcache_test.py @@ -0,0 +1,346 @@ +# encoding: utf-8 + +from __future__ import unicode_literals + +import os +import unittest + +from mopidy.utils.path import mtime, uri_to_path +from mopidy.frontends.mpd import translator as mpd, protocol +from mopidy.backends.local.tagcache import translator +from mopidy.models import Album, Artist, Track + +from tests import path_to_data_dir + + +class TracksToTagCacheFormatTest(unittest.TestCase): + def setUp(self): + self.media_dir = '/dir/subdir' + mtime.set_fake_time(1234567) + + def tearDown(self): + mtime.undo_fake() + + def translate(self, track): + base_path = self.media_dir.encode('utf-8') + result = dict(mpd.track_to_mpd_format(track)) + result['file'] = uri_to_path(result['file'])[len(base_path) + 1:] + result['key'] = os.path.basename(result['file']) + result['mtime'] = mtime('') + return translator.order_mpd_track_info(result.items()) + + def consume_headers(self, result): + self.assertEqual(('info_begin',), result[0]) + self.assertEqual(('mpd_version', protocol.VERSION), result[1]) + self.assertEqual(('fs_charset', protocol.ENCODING), result[2]) + self.assertEqual(('info_end',), result[3]) + return result[4:] + + def consume_song_list(self, result): + self.assertEqual(('songList begin',), result[0]) + for i, row in enumerate(result): + if row == ('songList end',): + return result[1:i], result[i + 1:] + self.fail("Couldn't find songList end in result") + + def consume_directory(self, result): + self.assertEqual('directory', result[0][0]) + self.assertEqual(('mtime', mtime('.')), result[1]) + self.assertEqual(('begin', os.path.split(result[0][1])[1]), result[2]) + directory = result[2][1] + for i, row in enumerate(result): + if row == ('end', directory): + return result[3:i], result[i + 1:] + self.fail("Couldn't find end %s in result" % directory) + + def test_empty_tag_cache_has_header(self): + result = translator.tracks_to_tag_cache_format([], self.media_dir) + result = self.consume_headers(result) + + def test_empty_tag_cache_has_song_list(self): + result = translator.tracks_to_tag_cache_format([], self.media_dir) + result = self.consume_headers(result) + song_list, result = self.consume_song_list(result) + + self.assertEqual(len(song_list), 0) + self.assertEqual(len(result), 0) + + def test_tag_cache_has_header(self): + track = Track(uri='file:///dir/subdir/song.mp3') + result = translator.tracks_to_tag_cache_format([track], self.media_dir) + result = self.consume_headers(result) + + def test_tag_cache_has_song_list(self): + track = Track(uri='file:///dir/subdir/song.mp3') + result = translator.tracks_to_tag_cache_format([track], self.media_dir) + result = self.consume_headers(result) + song_list, result = self.consume_song_list(result) + + self.assert_(song_list) + self.assertEqual(len(result), 0) + + def test_tag_cache_has_formated_track(self): + track = Track(uri='file:///dir/subdir/song.mp3') + formated = self.translate(track) + result = translator.tracks_to_tag_cache_format([track], self.media_dir) + + result = self.consume_headers(result) + song_list, result = self.consume_song_list(result) + + self.assertEqual(formated, song_list) + self.assertEqual(len(result), 0) + + def test_tag_cache_has_formated_track_with_key_and_mtime(self): + track = Track(uri='file:///dir/subdir/song.mp3') + formated = self.translate(track) + result = translator.tracks_to_tag_cache_format([track], self.media_dir) + + result = self.consume_headers(result) + song_list, result = self.consume_song_list(result) + + self.assertEqual(formated, song_list) + self.assertEqual(len(result), 0) + + def test_tag_cache_supports_directories(self): + track = Track(uri='file:///dir/subdir/folder/song.mp3') + formated = self.translate(track) + result = translator.tracks_to_tag_cache_format([track], self.media_dir) + + result = self.consume_headers(result) + dir_data, result = self.consume_directory(result) + song_list, result = self.consume_song_list(result) + self.assertEqual(len(song_list), 0) + self.assertEqual(len(result), 0) + + song_list, result = self.consume_song_list(dir_data) + self.assertEqual(len(result), 0) + self.assertEqual(formated, song_list) + + def test_tag_cache_diretory_header_is_right(self): + track = Track(uri='file:///dir/subdir/folder/sub/song.mp3') + result = translator.tracks_to_tag_cache_format([track], self.media_dir) + + result = self.consume_headers(result) + dir_data, result = self.consume_directory(result) + + self.assertEqual(('directory', 'folder/sub'), dir_data[0]) + self.assertEqual(('mtime', mtime('.')), dir_data[1]) + self.assertEqual(('begin', 'sub'), dir_data[2]) + + def test_tag_cache_suports_sub_directories(self): + track = Track(uri='file:///dir/subdir/folder/sub/song.mp3') + formated = self.translate(track) + result = translator.tracks_to_tag_cache_format([track], self.media_dir) + + result = self.consume_headers(result) + + dir_data, result = self.consume_directory(result) + song_list, result = self.consume_song_list(result) + self.assertEqual(len(song_list), 0) + self.assertEqual(len(result), 0) + + dir_data, result = self.consume_directory(dir_data) + song_list, result = self.consume_song_list(result) + self.assertEqual(len(result), 0) + self.assertEqual(len(song_list), 0) + + song_list, result = self.consume_song_list(dir_data) + self.assertEqual(len(result), 0) + self.assertEqual(formated, song_list) + + def test_tag_cache_supports_multiple_tracks(self): + tracks = [ + Track(uri='file:///dir/subdir/song1.mp3'), + Track(uri='file:///dir/subdir/song2.mp3'), + ] + + formated = [] + formated.extend(self.translate(tracks[0])) + formated.extend(self.translate(tracks[1])) + + result = translator.tracks_to_tag_cache_format(tracks, self.media_dir) + + result = self.consume_headers(result) + song_list, result = self.consume_song_list(result) + + self.assertEqual(formated, song_list) + self.assertEqual(len(result), 0) + + def test_tag_cache_supports_multiple_tracks_in_dirs(self): + tracks = [ + Track(uri='file:///dir/subdir/song1.mp3'), + Track(uri='file:///dir/subdir/folder/song2.mp3'), + ] + + formated = [] + formated.append(self.translate(tracks[0])) + formated.append(self.translate(tracks[1])) + + result = translator.tracks_to_tag_cache_format(tracks, self.media_dir) + + result = self.consume_headers(result) + dir_data, result = self.consume_directory(result) + song_list, song_result = self.consume_song_list(dir_data) + + self.assertEqual(formated[1], song_list) + self.assertEqual(len(song_result), 0) + + song_list, result = self.consume_song_list(result) + self.assertEqual(len(result), 0) + self.assertEqual(formated[0], song_list) + + +class TracksToDirectoryTreeTest(unittest.TestCase): + def setUp(self): + self.media_dir = '/root' + + def test_no_tracks_gives_emtpy_tree(self): + tree = translator.tracks_to_directory_tree([], self.media_dir) + self.assertEqual(tree, ({}, [])) + + def test_top_level_files(self): + tracks = [ + Track(uri='file:///root/file1.mp3'), + Track(uri='file:///root/file2.mp3'), + Track(uri='file:///root/file3.mp3'), + ] + tree = translator.tracks_to_directory_tree(tracks, self.media_dir) + self.assertEqual(tree, ({}, tracks)) + + def test_single_file_in_subdir(self): + tracks = [Track(uri='file:///root/dir/file1.mp3')] + tree = translator.tracks_to_directory_tree(tracks, self.media_dir) + expected = ({'dir': ({}, tracks)}, []) + self.assertEqual(tree, expected) + + def test_single_file_in_sub_subdir(self): + tracks = [Track(uri='file:///root/dir1/dir2/file1.mp3')] + tree = translator.tracks_to_directory_tree(tracks, self.media_dir) + expected = ({'dir1': ({'dir1/dir2': ({}, tracks)}, [])}, []) + self.assertEqual(tree, expected) + + def test_complex_file_structure(self): + tracks = [ + Track(uri='file:///root/file1.mp3'), + Track(uri='file:///root/dir1/file2.mp3'), + Track(uri='file:///root/dir1/file3.mp3'), + Track(uri='file:///root/dir2/file4.mp3'), + Track(uri='file:///root/dir2/sub/file5.mp3'), + ] + tree = translator.tracks_to_directory_tree(tracks, self.media_dir) + expected = ( + { + 'dir1': ({}, [tracks[1], tracks[2]]), + 'dir2': ( + { + 'dir2/sub': ({}, [tracks[4]]) + }, + [tracks[3]] + ), + }, + [tracks[0]] + ) + self.assertEqual(tree, expected) + + +expected_artists = [Artist(name='name')] +expected_albums = [ + Album(name='albumname', artists=expected_artists, num_tracks=2), + Album(name='albumname', num_tracks=2), +] +expected_tracks = [] + + +def generate_track(path, ident, album_id): + uri = 'local:track:%s' % path + track = Track( + uri=uri, name='trackname', artists=expected_artists, + album=expected_albums[album_id], track_no=1, date='2006', length=4000, + last_modified=1272319626) + expected_tracks.append(track) + + +generate_track('song1.mp3', 6, 0) +generate_track('song2.mp3', 7, 0) +generate_track('song3.mp3', 8, 1) +generate_track('subdir1/song4.mp3', 2, 0) +generate_track('subdir1/song5.mp3', 3, 0) +generate_track('subdir2/song6.mp3', 4, 1) +generate_track('subdir2/song7.mp3', 5, 1) +generate_track('subdir1/subsubdir/song8.mp3', 0, 0) +generate_track('subdir1/subsubdir/song9.mp3', 1, 1) + + +class MPDTagCacheToTracksTest(unittest.TestCase): + def test_emtpy_cache(self): + tracks = translator.parse_mpd_tag_cache( + path_to_data_dir('empty_tag_cache'), path_to_data_dir('')) + self.assertEqual(set(), tracks) + + def test_simple_cache(self): + tracks = translator.parse_mpd_tag_cache( + path_to_data_dir('simple_tag_cache'), path_to_data_dir('')) + track = Track( + uri='local:track:song1.mp3', name='trackname', + artists=expected_artists, track_no=1, album=expected_albums[0], + date='2006', length=4000, last_modified=1272319626) + self.assertEqual(set([track]), tracks) + + def test_advanced_cache(self): + tracks = translator.parse_mpd_tag_cache( + path_to_data_dir('advanced_tag_cache'), path_to_data_dir('')) + self.assertEqual(set(expected_tracks), tracks) + + def test_unicode_cache(self): + tracks = translator.parse_mpd_tag_cache( + path_to_data_dir('utf8_tag_cache'), path_to_data_dir('')) + + artists = [Artist(name='æøå')] + album = Album(name='æøå', artists=artists) + track = Track( + uri='local:track:song1.mp3', name='æøå', artists=artists, + composers=artists, performers=artists, genre='æøå', + album=album, length=4000, last_modified=1272319626, + comment='æøå&^`ൂ㔶') + + self.assertEqual(track, list(tracks)[0]) + + @unittest.SkipTest + def test_misencoded_cache(self): + # FIXME not sure if this can happen + pass + + def test_cache_with_blank_track_info(self): + tracks = translator.parse_mpd_tag_cache( + path_to_data_dir('blank_tag_cache'), path_to_data_dir('')) + expected = Track( + uri='local:track:song1.mp3', length=4000, last_modified=1272319626) + self.assertEqual(set([expected]), tracks) + + def test_musicbrainz_tagcache(self): + tracks = translator.parse_mpd_tag_cache( + path_to_data_dir('musicbrainz_tag_cache'), path_to_data_dir('')) + artist = list(expected_tracks[0].artists)[0].copy( + musicbrainz_id='7364dea6-ca9a-48e3-be01-b44ad0d19897') + albumartist = list(expected_tracks[0].artists)[0].copy( + name='albumartistname', + musicbrainz_id='7364dea6-ca9a-48e3-be01-b44ad0d19897') + album = expected_tracks[0].album.copy( + artists=[albumartist], + musicbrainz_id='cb5f1603-d314-4c9c-91e5-e295cfb125d2') + track = expected_tracks[0].copy( + artists=[artist], album=album, + musicbrainz_id='90488461-8c1f-4a4e-826b-4c6dc70801f0') + + self.assertEqual(track, list(tracks)[0]) + + def test_albumartist_tag_cache(self): + tracks = translator.parse_mpd_tag_cache( + path_to_data_dir('albumartist_tag_cache'), path_to_data_dir('')) + artist = Artist(name='albumartistname') + album = expected_albums[0].copy(artists=[artist]) + track = Track( + uri='local:track:song1.mp3', name='trackname', + artists=expected_artists, track_no=1, album=album, date='2006', + length=4000, last_modified=1272319626) + self.assertEqual(track, list(tracks)[0]) diff --git a/tests/backends/local/translator_test.py b/tests/backends/local/translator_test.py index 5623c787..e5747f68 100644 --- a/tests/backends/local/translator_test.py +++ b/tests/backends/local/translator_test.py @@ -6,8 +6,7 @@ import os import tempfile import unittest -from mopidy.backends.local.translator import parse_m3u, parse_mpd_tag_cache -from mopidy.models import Track, Artist, Album +from mopidy.backends.local.translator import parse_m3u from mopidy.utils.path import path_to_uri from tests import path_to_data_dir @@ -89,106 +88,3 @@ class M3UToUriTest(unittest.TestCase): class URItoM3UTest(unittest.TestCase): pass - - -expected_artists = [Artist(name='name')] -expected_albums = [ - Album(name='albumname', artists=expected_artists, num_tracks=2), - Album(name='albumname', num_tracks=2), -] -expected_tracks = [] - - -def generate_track(path, ident, album_id): - uri = 'local:track:%s' % path - track = Track( - uri=uri, name='trackname', artists=expected_artists, - album=expected_albums[album_id], track_no=1, date='2006', length=4000, - last_modified=1272319626) - expected_tracks.append(track) - - -generate_track('song1.mp3', 6, 0) -generate_track('song2.mp3', 7, 0) -generate_track('song3.mp3', 8, 1) -generate_track('subdir1/song4.mp3', 2, 0) -generate_track('subdir1/song5.mp3', 3, 0) -generate_track('subdir2/song6.mp3', 4, 1) -generate_track('subdir2/song7.mp3', 5, 1) -generate_track('subdir1/subsubdir/song8.mp3', 0, 0) -generate_track('subdir1/subsubdir/song9.mp3', 1, 1) - - -class MPDTagCacheToTracksTest(unittest.TestCase): - def test_emtpy_cache(self): - tracks = parse_mpd_tag_cache( - path_to_data_dir('empty_tag_cache'), path_to_data_dir('')) - self.assertEqual(set(), tracks) - - def test_simple_cache(self): - tracks = parse_mpd_tag_cache( - path_to_data_dir('simple_tag_cache'), path_to_data_dir('')) - track = Track( - uri='local:track:song1.mp3', name='trackname', - artists=expected_artists, track_no=1, album=expected_albums[0], - date='2006', length=4000, last_modified=1272319626) - self.assertEqual(set([track]), tracks) - - def test_advanced_cache(self): - tracks = parse_mpd_tag_cache( - path_to_data_dir('advanced_tag_cache'), path_to_data_dir('')) - self.assertEqual(set(expected_tracks), tracks) - - def test_unicode_cache(self): - tracks = parse_mpd_tag_cache( - path_to_data_dir('utf8_tag_cache'), path_to_data_dir('')) - - artists = [Artist(name='æøå')] - album = Album(name='æøå', artists=artists) - track = Track( - uri='local:track:song1.mp3', name='æøå', artists=artists, - composers=artists, performers=artists, genre='æøå', - album=album, length=4000, last_modified=1272319626, - comment='æøå&^`ൂ㔶') - - self.assertEqual(track, list(tracks)[0]) - - @unittest.SkipTest - def test_misencoded_cache(self): - # FIXME not sure if this can happen - pass - - def test_cache_with_blank_track_info(self): - tracks = parse_mpd_tag_cache( - path_to_data_dir('blank_tag_cache'), path_to_data_dir('')) - expected = Track( - uri='local:track:song1.mp3', length=4000, last_modified=1272319626) - self.assertEqual(set([expected]), tracks) - - def test_musicbrainz_tagcache(self): - tracks = parse_mpd_tag_cache( - path_to_data_dir('musicbrainz_tag_cache'), path_to_data_dir('')) - artist = list(expected_tracks[0].artists)[0].copy( - musicbrainz_id='7364dea6-ca9a-48e3-be01-b44ad0d19897') - albumartist = list(expected_tracks[0].artists)[0].copy( - name='albumartistname', - musicbrainz_id='7364dea6-ca9a-48e3-be01-b44ad0d19897') - album = expected_tracks[0].album.copy( - artists=[albumartist], - musicbrainz_id='cb5f1603-d314-4c9c-91e5-e295cfb125d2') - track = expected_tracks[0].copy( - artists=[artist], album=album, - musicbrainz_id='90488461-8c1f-4a4e-826b-4c6dc70801f0') - - self.assertEqual(track, list(tracks)[0]) - - def test_albumartist_tag_cache(self): - tracks = parse_mpd_tag_cache( - path_to_data_dir('albumartist_tag_cache'), path_to_data_dir('')) - artist = Artist(name='albumartistname') - album = expected_albums[0].copy(artists=[artist]) - track = Track( - uri='local:track:song1.mp3', name='trackname', - artists=expected_artists, track_no=1, album=album, date='2006', - length=4000, last_modified=1272319626) - self.assertEqual(track, list(tracks)[0]) diff --git a/tests/frontends/mpd/translator_test.py b/tests/frontends/mpd/translator_test.py index a6a2eaa9..1db10ab9 100644 --- a/tests/frontends/mpd/translator_test.py +++ b/tests/frontends/mpd/translator_test.py @@ -1,11 +1,10 @@ from __future__ import unicode_literals import datetime -import os import unittest -from mopidy.utils.path import mtime, uri_to_path -from mopidy.frontends.mpd import translator, protocol +from mopidy.utils.path import mtime +from mopidy.frontends.mpd import translator from mopidy.models import Album, Artist, TlTrack, Playlist, Track @@ -126,233 +125,3 @@ class PlaylistMpdFormatTest(unittest.TestCase): result = translator.playlist_to_mpd_format(playlist, 1, 2) self.assertEqual(len(result), 1) self.assertEqual(dict(result[0])['Track'], 2) - - -class TracksToTagCacheFormatTest(unittest.TestCase): - def setUp(self): - self.media_dir = '/dir/subdir' - mtime.set_fake_time(1234567) - - def tearDown(self): - mtime.undo_fake() - - def translate(self, track): - base_path = self.media_dir.encode('utf-8') - result = dict(translator.track_to_mpd_format(track)) - result['file'] = uri_to_path(result['file'])[len(base_path) + 1:] - result['key'] = os.path.basename(result['file']) - result['mtime'] = mtime('') - return translator.order_mpd_track_info(result.items()) - - def consume_headers(self, result): - self.assertEqual(('info_begin',), result[0]) - self.assertEqual(('mpd_version', protocol.VERSION), result[1]) - self.assertEqual(('fs_charset', protocol.ENCODING), result[2]) - self.assertEqual(('info_end',), result[3]) - return result[4:] - - def consume_song_list(self, result): - self.assertEqual(('songList begin',), result[0]) - for i, row in enumerate(result): - if row == ('songList end',): - return result[1:i], result[i + 1:] - self.fail("Couldn't find songList end in result") - - def consume_directory(self, result): - self.assertEqual('directory', result[0][0]) - self.assertEqual(('mtime', mtime('.')), result[1]) - self.assertEqual(('begin', os.path.split(result[0][1])[1]), result[2]) - directory = result[2][1] - for i, row in enumerate(result): - if row == ('end', directory): - return result[3:i], result[i + 1:] - self.fail("Couldn't find end %s in result" % directory) - - def test_empty_tag_cache_has_header(self): - result = translator.tracks_to_tag_cache_format([], self.media_dir) - result = self.consume_headers(result) - - def test_empty_tag_cache_has_song_list(self): - result = translator.tracks_to_tag_cache_format([], self.media_dir) - result = self.consume_headers(result) - song_list, result = self.consume_song_list(result) - - self.assertEqual(len(song_list), 0) - self.assertEqual(len(result), 0) - - def test_tag_cache_has_header(self): - track = Track(uri='file:///dir/subdir/song.mp3') - result = translator.tracks_to_tag_cache_format([track], self.media_dir) - result = self.consume_headers(result) - - def test_tag_cache_has_song_list(self): - track = Track(uri='file:///dir/subdir/song.mp3') - result = translator.tracks_to_tag_cache_format([track], self.media_dir) - result = self.consume_headers(result) - song_list, result = self.consume_song_list(result) - - self.assert_(song_list) - self.assertEqual(len(result), 0) - - def test_tag_cache_has_formated_track(self): - track = Track(uri='file:///dir/subdir/song.mp3') - formated = self.translate(track) - result = translator.tracks_to_tag_cache_format([track], self.media_dir) - - result = self.consume_headers(result) - song_list, result = self.consume_song_list(result) - - self.assertEqual(formated, song_list) - self.assertEqual(len(result), 0) - - def test_tag_cache_has_formated_track_with_key_and_mtime(self): - track = Track(uri='file:///dir/subdir/song.mp3') - formated = self.translate(track) - result = translator.tracks_to_tag_cache_format([track], self.media_dir) - - result = self.consume_headers(result) - song_list, result = self.consume_song_list(result) - - self.assertEqual(formated, song_list) - self.assertEqual(len(result), 0) - - def test_tag_cache_supports_directories(self): - track = Track(uri='file:///dir/subdir/folder/song.mp3') - formated = self.translate(track) - result = translator.tracks_to_tag_cache_format([track], self.media_dir) - - result = self.consume_headers(result) - dir_data, result = self.consume_directory(result) - song_list, result = self.consume_song_list(result) - self.assertEqual(len(song_list), 0) - self.assertEqual(len(result), 0) - - song_list, result = self.consume_song_list(dir_data) - self.assertEqual(len(result), 0) - self.assertEqual(formated, song_list) - - def test_tag_cache_diretory_header_is_right(self): - track = Track(uri='file:///dir/subdir/folder/sub/song.mp3') - result = translator.tracks_to_tag_cache_format([track], self.media_dir) - - result = self.consume_headers(result) - dir_data, result = self.consume_directory(result) - - self.assertEqual(('directory', 'folder/sub'), dir_data[0]) - self.assertEqual(('mtime', mtime('.')), dir_data[1]) - self.assertEqual(('begin', 'sub'), dir_data[2]) - - def test_tag_cache_suports_sub_directories(self): - track = Track(uri='file:///dir/subdir/folder/sub/song.mp3') - formated = self.translate(track) - result = translator.tracks_to_tag_cache_format([track], self.media_dir) - - result = self.consume_headers(result) - - dir_data, result = self.consume_directory(result) - song_list, result = self.consume_song_list(result) - self.assertEqual(len(song_list), 0) - self.assertEqual(len(result), 0) - - dir_data, result = self.consume_directory(dir_data) - song_list, result = self.consume_song_list(result) - self.assertEqual(len(result), 0) - self.assertEqual(len(song_list), 0) - - song_list, result = self.consume_song_list(dir_data) - self.assertEqual(len(result), 0) - self.assertEqual(formated, song_list) - - def test_tag_cache_supports_multiple_tracks(self): - tracks = [ - Track(uri='file:///dir/subdir/song1.mp3'), - Track(uri='file:///dir/subdir/song2.mp3'), - ] - - formated = [] - formated.extend(self.translate(tracks[0])) - formated.extend(self.translate(tracks[1])) - - result = translator.tracks_to_tag_cache_format(tracks, self.media_dir) - - result = self.consume_headers(result) - song_list, result = self.consume_song_list(result) - - self.assertEqual(formated, song_list) - self.assertEqual(len(result), 0) - - def test_tag_cache_supports_multiple_tracks_in_dirs(self): - tracks = [ - Track(uri='file:///dir/subdir/song1.mp3'), - Track(uri='file:///dir/subdir/folder/song2.mp3'), - ] - - formated = [] - formated.append(self.translate(tracks[0])) - formated.append(self.translate(tracks[1])) - - result = translator.tracks_to_tag_cache_format(tracks, self.media_dir) - - result = self.consume_headers(result) - dir_data, result = self.consume_directory(result) - song_list, song_result = self.consume_song_list(dir_data) - - self.assertEqual(formated[1], song_list) - self.assertEqual(len(song_result), 0) - - song_list, result = self.consume_song_list(result) - self.assertEqual(len(result), 0) - self.assertEqual(formated[0], song_list) - - -class TracksToDirectoryTreeTest(unittest.TestCase): - def setUp(self): - self.media_dir = '/root' - - def test_no_tracks_gives_emtpy_tree(self): - tree = translator.tracks_to_directory_tree([], self.media_dir) - self.assertEqual(tree, ({}, [])) - - def test_top_level_files(self): - tracks = [ - Track(uri='file:///root/file1.mp3'), - Track(uri='file:///root/file2.mp3'), - Track(uri='file:///root/file3.mp3'), - ] - tree = translator.tracks_to_directory_tree(tracks, self.media_dir) - self.assertEqual(tree, ({}, tracks)) - - def test_single_file_in_subdir(self): - tracks = [Track(uri='file:///root/dir/file1.mp3')] - tree = translator.tracks_to_directory_tree(tracks, self.media_dir) - expected = ({'dir': ({}, tracks)}, []) - self.assertEqual(tree, expected) - - def test_single_file_in_sub_subdir(self): - tracks = [Track(uri='file:///root/dir1/dir2/file1.mp3')] - tree = translator.tracks_to_directory_tree(tracks, self.media_dir) - expected = ({'dir1': ({'dir1/dir2': ({}, tracks)}, [])}, []) - self.assertEqual(tree, expected) - - def test_complex_file_structure(self): - tracks = [ - Track(uri='file:///root/file1.mp3'), - Track(uri='file:///root/dir1/file2.mp3'), - Track(uri='file:///root/dir1/file3.mp3'), - Track(uri='file:///root/dir2/file4.mp3'), - Track(uri='file:///root/dir2/sub/file5.mp3'), - ] - tree = translator.tracks_to_directory_tree(tracks, self.media_dir) - expected = ( - { - 'dir1': ({}, [tracks[1], tracks[2]]), - 'dir2': ( - { - 'dir2/sub': ({}, [tracks[4]]) - }, - [tracks[3]] - ), - }, - [tracks[0]] - ) - self.assertEqual(tree, expected) From 04044d035f4ce71e4d6131844ac8504dac930252 Mon Sep 17 00:00:00 2001 From: Thomas Adamcik Date: Tue, 26 Nov 2013 22:20:55 +0100 Subject: [PATCH 03/19] core: Refactor core Backends helper Replaces the jungle of extra dicts/lists with an OrderedDict per backend feature type. Also makes sure that each type/scheme is unique instead of the scheme alone. --- mopidy/core/actor.py | 50 ++++++++++++++++++---------------------- mopidy/core/library.py | 12 +++++----- mopidy/core/playback.py | 2 +- mopidy/core/playlists.py | 27 ++++++++++------------ tests/core/actor_test.py | 20 ++++++++++++++-- 5 files changed, 60 insertions(+), 51 deletions(-) diff --git a/mopidy/core/actor.py b/mopidy/core/actor.py index cd4ba180..3cba20db 100644 --- a/mopidy/core/actor.py +++ b/mopidy/core/actor.py @@ -1,5 +1,6 @@ from __future__ import unicode_literals +import collections import itertools import pykka @@ -79,34 +80,29 @@ class Backends(list): def __init__(self, backends): super(Backends, self).__init__(backends) - # These lists keeps the backends in the original order, but only - # includes those which implements the required backend provider. Since - # it is important to keep the order, we can't simply use .values() on - # the X_by_uri_scheme dicts below. - self.with_library = [b for b in backends if b.has_library().get()] - self.with_playback = [b for b in backends if b.has_playback().get()] - self.with_playlists = [ - b for b in backends if b.has_playlists().get()] + self.with_library = collections.OrderedDict() + self.with_playback = collections.OrderedDict() + self.with_playlists = collections.OrderedDict() - self.by_uri_scheme = {} for backend in backends: - for uri_scheme in backend.uri_schemes.get(): - assert uri_scheme not in self.by_uri_scheme, ( - 'Cannot add URI scheme %s for %s, ' - 'it is already handled by %s' - ) % ( - uri_scheme, backend.__class__.__name__, - self.by_uri_scheme[uri_scheme].__class__.__name__) - self.by_uri_scheme[uri_scheme] = backend + has_library = backend.has_library().get() + has_playback = backend.has_playback().get() + has_playlists = backend.has_playlists().get() - self.with_library_by_uri_scheme = {} - self.with_playback_by_uri_scheme = {} - self.with_playlists_by_uri_scheme = {} + for scheme in backend.uri_schemes.get(): + self.add(self.with_library, has_library, scheme, backend) + self.add(self.with_playback, has_playback, scheme, backend) + self.add(self.with_playlists, has_playlists, scheme, backend) - for uri_scheme, backend in self.by_uri_scheme.items(): - if backend.has_library().get(): - self.with_library_by_uri_scheme[uri_scheme] = backend - if backend.has_playback().get(): - self.with_playback_by_uri_scheme[uri_scheme] = backend - if backend.has_playlists().get(): - self.with_playlists_by_uri_scheme[uri_scheme] = backend + def add(self, registry, supported, uri_scheme, backend): + if not supported: + return + + if uri_scheme not in registry: + registry[uri_scheme] = backend + return + + get_name = lambda actor: actor.actor_ref.actor_class.__name__ + raise AssertionError( + 'Cannot add URI scheme %s for %s, it is already handled by %s' % + (uri_scheme, get_name(backend), get_name(registry[uri_scheme]))) diff --git a/mopidy/core/library.py b/mopidy/core/library.py index cdc3f53a..2e73e0db 100644 --- a/mopidy/core/library.py +++ b/mopidy/core/library.py @@ -1,6 +1,6 @@ from __future__ import unicode_literals -from collections import defaultdict +import collections import urlparse import pykka @@ -15,18 +15,18 @@ class LibraryController(object): def _get_backend(self, uri): uri_scheme = urlparse.urlparse(uri).scheme - return self.backends.with_library_by_uri_scheme.get(uri_scheme, None) + return self.backends.with_library.get(uri_scheme, None) def _get_backends_to_uris(self, uris): if uris: - backends_to_uris = defaultdict(list) + backends_to_uris = collections.defaultdict(list) for uri in uris: backend = self._get_backend(uri) if backend is not None: backends_to_uris[backend].append(uri) else: backends_to_uris = dict([ - (b, None) for b in self.backends.with_library]) + (b, None) for b in self.backends.with_library.values()]) return backends_to_uris def find_exact(self, query=None, uris=None, **kwargs): @@ -103,8 +103,8 @@ class LibraryController(object): if backend: backend.library.refresh(uri).get() else: - futures = [ - b.library.refresh(uri) for b in self.backends.with_library] + futures = [b.library.refresh(uri) + for b in self.backends.with_library.values()] pykka.get_all(futures) def search(self, query=None, uris=None, **kwargs): diff --git a/mopidy/core/playback.py b/mopidy/core/playback.py index d127fbbe..3c0e43fa 100644 --- a/mopidy/core/playback.py +++ b/mopidy/core/playback.py @@ -28,7 +28,7 @@ class PlaybackController(object): return None uri = self.current_tl_track.track.uri uri_scheme = urlparse.urlparse(uri).scheme - return self.backends.with_playback_by_uri_scheme.get(uri_scheme, None) + return self.backends.with_playback.get(uri_scheme, None) ### Properties diff --git a/mopidy/core/playlists.py b/mopidy/core/playlists.py index f0187d44..d5c03bb3 100644 --- a/mopidy/core/playlists.py +++ b/mopidy/core/playlists.py @@ -16,8 +16,8 @@ class PlaylistsController(object): self.core = core def get_playlists(self, include_tracks=True): - futures = [ - b.playlists.playlists for b in self.backends.with_playlists] + futures = [b.playlists.playlists + for b in self.backends.with_playlists.values()] results = pykka.get_all(futures) playlists = list(itertools.chain(*results)) if not include_tracks: @@ -49,10 +49,11 @@ class PlaylistsController(object): :type uri_scheme: string :rtype: :class:`mopidy.models.Playlist` """ - if uri_scheme in self.backends.with_playlists_by_uri_scheme: - backend = self.backends.by_uri_scheme[uri_scheme] + if uri_scheme in self.backends.with_playlists: + backend = self.backends.with_playlists[uri_scheme] else: - backend = self.backends.with_playlists[0] + # TODO: this fallback looks suspicious + backend = self.backends.with_playlists.values()[0] playlist = backend.playlists.create(name).get() listener.CoreListener.send('playlist_changed', playlist=playlist) return playlist @@ -68,8 +69,7 @@ class PlaylistsController(object): :type uri: string """ uri_scheme = urlparse.urlparse(uri).scheme - backend = self.backends.with_playlists_by_uri_scheme.get( - uri_scheme, None) + backend = self.backends.with_playlists.get(uri_scheme, None) if backend: backend.playlists.delete(uri).get() @@ -111,8 +111,7 @@ class PlaylistsController(object): :rtype: :class:`mopidy.models.Playlist` or :class:`None` """ uri_scheme = urlparse.urlparse(uri).scheme - backend = self.backends.with_playlists_by_uri_scheme.get( - uri_scheme, None) + backend = self.backends.with_playlists.get(uri_scheme, None) if backend: return backend.playlists.lookup(uri).get() else: @@ -131,13 +130,12 @@ class PlaylistsController(object): :type uri_scheme: string """ if uri_scheme is None: - futures = [ - b.playlists.refresh() for b in self.backends.with_playlists] + futures = [b.playlists.refresh() + for b in self.backends.with_playlists.values()] pykka.get_all(futures) listener.CoreListener.send('playlists_loaded') else: - backend = self.backends.with_playlists_by_uri_scheme.get( - uri_scheme, None) + backend = self.backends.with_playlists.get(uri_scheme, None) if backend: backend.playlists.refresh().get() listener.CoreListener.send('playlists_loaded') @@ -167,8 +165,7 @@ class PlaylistsController(object): if playlist.uri is None: return uri_scheme = urlparse.urlparse(playlist.uri).scheme - backend = self.backends.with_playlists_by_uri_scheme.get( - uri_scheme, None) + backend = self.backends.with_playlists.get(uri_scheme, None) if backend: playlist = backend.playlists.save(playlist).get() listener.CoreListener.send('playlist_changed', playlist=playlist) diff --git a/tests/core/actor_test.py b/tests/core/actor_test.py index c4952af3..ce50d5ed 100644 --- a/tests/core/actor_test.py +++ b/tests/core/actor_test.py @@ -28,10 +28,26 @@ class CoreActorTest(unittest.TestCase): self.assertIn('dummy2', result) def test_backends_with_colliding_uri_schemes_fails(self): - self.backend1.__class__.__name__ = b'B1' - self.backend2.__class__.__name__ = b'B2' + self.backend1.actor_ref.actor_class.__name__ = b'B1' + self.backend2.actor_ref.actor_class.__name__ = b'B2' self.backend2.uri_schemes.get.return_value = ['dummy1', 'dummy2'] self.assertRaisesRegexp( AssertionError, 'Cannot add URI scheme dummy1 for B2, it is already handled by B1', Core, audio=None, backends=[self.backend1, self.backend2]) + + def test_backends_with_colliding_uri_schemes_passes(self): + # Checks that backends with overlapping schemes, but distinct sub parts + # provided can co-exist. + self.backend1.has_library().get.return_value = False + self.backend1.has_playlists().get.return_value = False + + self.backend2.uri_schemes().get.return_value = ['dummy1'] + self.backend2.has_playback().get.return_value = False + self.backend2.has_playlists().get.return_value = False + + core = Core(audio=None, backends=[self.backend1, self.backend2]) + self.assertEqual(core.backends.with_playback, + {'dummy1': self.backend1}) + self.assertEqual(core.backends.with_library, + {'dummy2': self.backend2}) From 3c1c6bac719d664ef750a73774b0924829ca4229 Mon Sep 17 00:00:00 2001 From: Thomas Adamcik Date: Tue, 26 Nov 2013 22:59:31 +0100 Subject: [PATCH 04/19] local: Always return track with just uri for local playlist tracks This is related to #527, but is only a stop gap until we fix it right. Note that this actually causes a regression, as not playlist tracks will have any metadata after this change. --- mopidy/backends/local/playlists.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/mopidy/backends/local/playlists.py b/mopidy/backends/local/playlists.py index 081bc335..e8996b51 100644 --- a/mopidy/backends/local/playlists.py +++ b/mopidy/backends/local/playlists.py @@ -51,11 +51,8 @@ class LocalPlaylistsProvider(base.BasePlaylistsProvider): tracks = [] for track_uri in parse_m3u(m3u, self._media_dir): - result = self.backend.library.lookup(track_uri) - if result: - tracks += self.backend.library.lookup(track_uri) - else: - tracks.append(Track(uri=track_uri)) + # TODO: switch to having playlists being a list of uris + tracks.append(Track(uri=track_uri)) playlist = Playlist(uri=uri, name=name, tracks=tracks) playlists.append(playlist) From c025b87076d50f4c10464fd48ee3e67aed6d9e84 Mon Sep 17 00:00:00 2001 From: Thomas Adamcik Date: Tue, 26 Nov 2013 23:03:08 +0100 Subject: [PATCH 05/19] tagcache: Split out to own extension for eventual deletion. --- mopidy/backends/local/__init__.py | 4 --- mopidy/backends/local/actor.py | 9 ------- mopidy/backends/local/tagcache/__init__.py | 31 ++++++++++++++++++++++ mopidy/backends/local/tagcache/actor.py | 30 +++++++++++++++++++++ mopidy/backends/local/tagcache/ext.conf | 2 ++ mopidy/backends/local/tagcache/library.py | 6 ++--- setup.py | 1 + tests/backends/local/library_test.py | 6 ++--- 8 files changed, 70 insertions(+), 19 deletions(-) create mode 100644 mopidy/backends/local/tagcache/actor.py create mode 100644 mopidy/backends/local/tagcache/ext.conf diff --git a/mopidy/backends/local/__init__.py b/mopidy/backends/local/__init__.py index 8a2e12fd..723eb056 100644 --- a/mopidy/backends/local/__init__.py +++ b/mopidy/backends/local/__init__.py @@ -33,10 +33,6 @@ class Extension(ext.Extension): from .actor import LocalBackend return [LocalBackend] - def get_library_updaters(self): - from .tagcache.library import LocalLibraryUpdateProvider - return [LocalLibraryUpdateProvider] - def get_command(self): from .commands import LocalCommand return LocalCommand() diff --git a/mopidy/backends/local/actor.py b/mopidy/backends/local/actor.py index 531b7546..a73f627e 100644 --- a/mopidy/backends/local/actor.py +++ b/mopidy/backends/local/actor.py @@ -8,7 +8,6 @@ import pykka from mopidy.backends import base from mopidy.utils import encoding, path -from .tagcache.library import LocalLibraryProvider from .playlists import LocalPlaylistsProvider from .playback import LocalPlaybackProvider @@ -23,7 +22,6 @@ class LocalBackend(pykka.ThreadingActor, base.Backend): self.check_dirs_and_files() - self.library = LocalLibraryProvider(backend=self) self.playback = LocalPlaybackProvider(audio=audio, backend=self) self.playlists = LocalPlaylistsProvider(backend=self) @@ -40,10 +38,3 @@ class LocalBackend(pykka.ThreadingActor, base.Backend): logger.warning( 'Could not create local playlists dir: %s', encoding.locale_decode(error)) - - try: - path.get_or_create_file(self.config['local']['tag_cache_file']) - except EnvironmentError as error: - logger.warning( - 'Could not create empty tag cache file: %s', - encoding.locale_decode(error)) diff --git a/mopidy/backends/local/tagcache/__init__.py b/mopidy/backends/local/tagcache/__init__.py index e69de29b..c7364e8b 100644 --- a/mopidy/backends/local/tagcache/__init__.py +++ b/mopidy/backends/local/tagcache/__init__.py @@ -0,0 +1,31 @@ +from __future__ import unicode_literals + +import os + +import mopidy +from mopidy import config, ext + + +class Extension(ext.Extension): + + dist_name = 'Mopidy-Local-Tagcache' + ext_name = 'local-tagcache' + version = mopidy.__version__ + + def get_default_config(self): + conf_file = os.path.join(os.path.dirname(__file__), 'ext.conf') + return config.read(conf_file) + + # Config only contains local-tagcache/enabled since we are not setting our + # own schema. + + def validate_environment(self): + pass + + def get_backend_classes(self): + from .actor import LocalTagcacheBackend + return [LocalTagcacheBackend] + + def get_library_updaters(self): + from .library import LocalTagcacheLibraryUpdateProvider + return [LocalTagcacheLibraryUpdateProvider] diff --git a/mopidy/backends/local/tagcache/actor.py b/mopidy/backends/local/tagcache/actor.py new file mode 100644 index 00000000..f052debb --- /dev/null +++ b/mopidy/backends/local/tagcache/actor.py @@ -0,0 +1,30 @@ +from __future__ import unicode_literals + +import logging + +import pykka + +from mopidy.backends import base +from mopidy.utils import encoding, path + +from .library import LocalTagcacheLibraryProvider + +logger = logging.getLogger('mopidy.backends.local.tagcache') + + +class LocalTagcacheBackend(pykka.ThreadingActor, base.Backend): + def __init__(self, config, audio): + super(LocalTagcacheBackend, self).__init__() + + self.config = config + self.check_dirs_and_files() + self.library = LocalTagcacheLibraryProvider(backend=self) + self.uri_schemes = ['local'] + + def check_dirs_and_files(self): + try: + path.get_or_create_file(self.config['local']['tag_cache_file']) + except EnvironmentError as error: + logger.warning( + 'Could not create empty tag cache file: %s', + encoding.locale_decode(error)) diff --git a/mopidy/backends/local/tagcache/ext.conf b/mopidy/backends/local/tagcache/ext.conf new file mode 100644 index 00000000..48a3c763 --- /dev/null +++ b/mopidy/backends/local/tagcache/ext.conf @@ -0,0 +1,2 @@ +[local-tagcache] +enabled = true diff --git a/mopidy/backends/local/tagcache/library.py b/mopidy/backends/local/tagcache/library.py index 6efe6bf5..c795cdc1 100644 --- a/mopidy/backends/local/tagcache/library.py +++ b/mopidy/backends/local/tagcache/library.py @@ -13,9 +13,9 @@ from .translator import parse_mpd_tag_cache, tracks_to_tag_cache_format logger = logging.getLogger('mopidy.backends.local.tagcache') -class LocalLibraryProvider(base.BaseLibraryProvider): +class LocalTagcacheLibraryProvider(base.BaseLibraryProvider): def __init__(self, *args, **kwargs): - super(LocalLibraryProvider, self).__init__(*args, **kwargs) + super(LocalTagcacheLibraryProvider, self).__init__(*args, **kwargs) self._uri_mapping = {} self._media_dir = self.backend.config['local']['media_dir'] self._tag_cache_file = self.backend.config['local']['tag_cache_file'] @@ -219,7 +219,7 @@ class LocalLibraryProvider(base.BaseLibraryProvider): raise LookupError('Missing query') -class LocalLibraryUpdateProvider(base.BaseLibraryProvider): +class LocalTagcacheLibraryUpdateProvider(base.BaseLibraryProvider): uri_schemes = ['local'] def __init__(self, config): diff --git a/setup.py b/setup.py index f43981bf..7d4b2cd8 100644 --- a/setup.py +++ b/setup.py @@ -43,6 +43,7 @@ setup( 'mopidy.ext': [ 'http = mopidy.frontends.http:Extension [http]', 'local = mopidy.backends.local:Extension', + 'local-tagcache = mopidy.backends.local.tagcache:Extension', 'mpd = mopidy.frontends.mpd:Extension', 'stream = mopidy.backends.stream:Extension', ], diff --git a/tests/backends/local/library_test.py b/tests/backends/local/library_test.py index c38fd74f..c04b81f5 100644 --- a/tests/backends/local/library_test.py +++ b/tests/backends/local/library_test.py @@ -6,7 +6,7 @@ import unittest import pykka from mopidy import core -from mopidy.backends.local import actor +from mopidy.backends.local.tagcache import actor from mopidy.models import Track, Album, Artist from tests import path_to_data_dir @@ -66,7 +66,7 @@ class LocalLibraryProviderTest(unittest.TestCase): } def setUp(self): - self.backend = actor.LocalBackend.start( + self.backend = actor.LocalTagcacheBackend.start( config=self.config, audio=None).proxy() self.core = core.Core(backends=[self.backend]) self.library = self.core.library @@ -92,7 +92,7 @@ class LocalLibraryProviderTest(unittest.TestCase): config = {'local': self.config['local'].copy()} config['local']['tag_cache_file'] = tag_cache.name - backend = actor.LocalBackend(config=config, audio=None) + backend = actor.LocalTagcacheBackend(config=config, audio=None) # Sanity check that value is in tag cache result = backend.library.lookup(self.tracks[0].uri) From 603b57ef3c353c48d97645b32efdc607a51283e0 Mon Sep 17 00:00:00 2001 From: Thomas Adamcik Date: Wed, 27 Nov 2013 22:49:07 +0100 Subject: [PATCH 06/19] utils: Remove find_uris and update find_files - find_uris is no more - find_files now returns file paths relative to path being searched - find_files now only works on directories - find_files tests have been updated to reflect changes - local scanner has gotten a minimal update to reflect this alteration --- mopidy/backends/local/commands.py | 5 +-- mopidy/utils/path.py | 28 +++++++---------- tests/audio/scan_test.py | 35 ++++++++++++--------- tests/data/{ => find}/.blank.mp3 | Bin tests/data/{ => find}/.hidden/.gitignore | 0 tests/data/find/baz/file | 0 tests/data/find/foo/bar/file | 0 tests/data/find/foo/file | 0 tests/utils/path_test.py | 38 ++++------------------- 9 files changed, 40 insertions(+), 66 deletions(-) rename tests/data/{ => find}/.blank.mp3 (100%) rename tests/data/{ => find}/.hidden/.gitignore (100%) create mode 100644 tests/data/find/baz/file create mode 100644 tests/data/find/foo/bar/file create mode 100644 tests/data/find/foo/file diff --git a/mopidy/backends/local/commands.py b/mopidy/backends/local/commands.py index c2ef143c..c0d6d23a 100644 --- a/mopidy/backends/local/commands.py +++ b/mopidy/backends/local/commands.py @@ -68,12 +68,13 @@ class ScanCommand(commands.Command): local_updater.remove(uri) logger.info('Checking %s for unknown tracks.', media_dir) - for uri in path.find_uris(media_dir): - file_extension = os.path.splitext(path.uri_to_path(uri))[1] + for relpath in path.find_files(media_dir): + file_extension = os.path.splitext(relpath)[1] if file_extension.lower() in excluded_file_extensions: logger.debug('Skipped %s: File extension excluded.', uri) continue + uri = path.path_to_uri(os.path.join(media_dir, relpath)) if uri not in uris_library: uris_update.add(uri) diff --git a/mopidy/utils/path.py b/mopidy/utils/path.py index 32dcb721..b8dcc589 100644 --- a/mopidy/utils/path.py +++ b/mopidy/utils/path.py @@ -119,26 +119,20 @@ def find_files(path): path = path.encode('utf-8') if os.path.isfile(path): - if not os.path.basename(path).startswith(b'.'): - yield path - else: - for dirpath, dirnames, filenames in os.walk(path, followlinks=True): - for dirname in dirnames: - if dirname.startswith(b'.'): - # Skip hidden dirs by modifying dirnames inplace - dirnames.remove(dirname) + return - for filename in filenames: - if filename.startswith(b'.'): - # Skip hidden files - continue + for dirpath, dirnames, filenames in os.walk(path, followlinks=True): + for dirname in dirnames: + if dirname.startswith(b'.'): + # Skip hidden dirs by modifying dirnames inplace + dirnames.remove(dirname) - yield os.path.join(dirpath, filename) + for filename in filenames: + if filename.startswith(b'.'): + # Skip hidden files + continue - -def find_uris(path): - for p in find_files(path): - yield path_to_uri(p) + yield os.path.relpath(os.path.join(dirpath, filename), path) def check_file_path_is_inside_base_dir(file_path, base_path): diff --git a/tests/audio/scan_test.py b/tests/audio/scan_test.py index 4acbecb6..ed3f8e01 100644 --- a/tests/audio/scan_test.py +++ b/tests/audio/scan_test.py @@ -1,5 +1,6 @@ from __future__ import unicode_literals +import os import unittest from mopidy import exceptions @@ -240,11 +241,15 @@ class ScannerTest(unittest.TestCase): self.errors = {} self.data = {} - def scan(self, path): - paths = path_lib.find_files(path_to_data_dir(path)) - uris = (path_lib.path_to_uri(p) for p in paths) + def find(self, path): + media_dir = path_to_data_dir(path) + for path in path_lib.find_files(media_dir): + yield os.path.join(media_dir, path) + + def scan(self, paths): scanner = scan.Scanner() - for uri in uris: + for path in paths: + uri = path_lib.path_to_uri(path) key = uri[len('file://'):] try: self.data[key] = scanner.scan(uri) @@ -256,15 +261,15 @@ class ScannerTest(unittest.TestCase): self.assertEqual(self.data[name][key], value) def test_data_is_set(self): - self.scan('scanner/simple') + self.scan(self.find('scanner/simple')) self.assert_(self.data) def test_errors_is_not_set(self): - self.scan('scanner/simple') + self.scan(self.find('scanner/simple')) self.assert_(not self.errors) def test_uri_is_set(self): - self.scan('scanner/simple') + self.scan(self.find('scanner/simple')) self.check( 'scanner/simple/song1.mp3', 'uri', 'file://%s' % path_to_data_dir('scanner/simple/song1.mp3')) @@ -273,39 +278,39 @@ class ScannerTest(unittest.TestCase): 'file://%s' % path_to_data_dir('scanner/simple/song1.ogg')) def test_duration_is_set(self): - self.scan('scanner/simple') + self.scan(self.find('scanner/simple')) self.check('scanner/simple/song1.mp3', 'duration', 4680000000) self.check('scanner/simple/song1.ogg', 'duration', 4680000000) def test_artist_is_set(self): - self.scan('scanner/simple') + self.scan(self.find('scanner/simple')) self.check('scanner/simple/song1.mp3', 'artist', 'name') self.check('scanner/simple/song1.ogg', 'artist', 'name') def test_album_is_set(self): - self.scan('scanner/simple') + self.scan(self.find('scanner/simple')) self.check('scanner/simple/song1.mp3', 'album', 'albumname') self.check('scanner/simple/song1.ogg', 'album', 'albumname') def test_track_is_set(self): - self.scan('scanner/simple') + self.scan(self.find('scanner/simple')) self.check('scanner/simple/song1.mp3', 'title', 'trackname') self.check('scanner/simple/song1.ogg', 'title', 'trackname') def test_nonexistant_dir_does_not_fail(self): - self.scan('scanner/does-not-exist') + self.scan(self.find('scanner/does-not-exist')) self.assert_(not self.errors) def test_other_media_is_ignored(self): - self.scan('scanner/image') + self.scan(self.find('scanner/image')) self.assert_(self.errors) def test_log_file_that_gst_thinks_is_mpeg_1_is_ignored(self): - self.scan('scanner/example.log') + self.scan([path_to_data_dir('scanner/example.log')]) self.assert_(self.errors) def test_empty_wav_file_is_ignored(self): - self.scan('scanner/empty.wav') + self.scan([path_to_data_dir('scanner/empty.wav')]) self.assert_(self.errors) @unittest.SkipTest diff --git a/tests/data/.blank.mp3 b/tests/data/find/.blank.mp3 similarity index 100% rename from tests/data/.blank.mp3 rename to tests/data/find/.blank.mp3 diff --git a/tests/data/.hidden/.gitignore b/tests/data/find/.hidden/.gitignore similarity index 100% rename from tests/data/.hidden/.gitignore rename to tests/data/find/.hidden/.gitignore diff --git a/tests/data/find/baz/file b/tests/data/find/baz/file new file mode 100644 index 00000000..e69de29b diff --git a/tests/data/find/foo/bar/file b/tests/data/find/foo/bar/file new file mode 100644 index 00000000..e69de29b diff --git a/tests/data/find/foo/file b/tests/data/find/foo/file new file mode 100644 index 00000000..e69de29b diff --git a/tests/utils/path_test.py b/tests/utils/path_test.py index 673fda73..316b4f38 100644 --- a/tests/utils/path_test.py +++ b/tests/utils/path_test.py @@ -221,9 +221,12 @@ class FindFilesTest(unittest.TestCase): self.assertEqual(self.find('does-not-exist'), []) def test_file(self): - files = self.find('blank.mp3') - self.assertEqual(len(files), 1) - self.assertEqual(files[0], path_to_data_dir('blank.mp3')) + self.assertEqual([], self.find('blank.mp3')) + + def test_files(self): + files = self.find('find') + excepted = [b'foo/bar/file', b'foo/file', b'baz/file'] + self.assertItemsEqual(excepted, files) def test_names_are_bytestrings(self): is_bytes = lambda f: isinstance(f, bytes) @@ -231,35 +234,6 @@ class FindFilesTest(unittest.TestCase): self.assert_( is_bytes(name), '%s is not bytes object' % repr(name)) - def test_ignores_hidden_dirs(self): - self.assertEqual(self.find('.hidden'), []) - - def test_ignores_hidden_files(self): - self.assertEqual(self.find('.blank.mp3'), []) - - -class FindUrisTest(unittest.TestCase): - def find(self, value): - return list(path.find_uris(path_to_data_dir(value))) - - def test_basic_dir(self): - self.assert_(self.find('')) - - def test_nonexistant_dir(self): - self.assertEqual(self.find('does-not-exist'), []) - - def test_file(self): - uris = self.find('blank.mp3') - expected = path.path_to_uri(path_to_data_dir('blank.mp3')) - self.assertEqual(len(uris), 1) - self.assertEqual(uris[0], expected) - - def test_ignores_hidden_dirs(self): - self.assertEqual(self.find('.hidden'), []) - - def test_ignores_hidden_files(self): - self.assertEqual(self.find('.blank.mp3'), []) - # TODO: kill this in favour of just os.path.getmtime + mocks class MtimeTest(unittest.TestCase): From 4161c2bf2785809845cea2521737a85bbb276b18 Mon Sep 17 00:00:00 2001 From: Thomas Adamcik Date: Wed, 27 Nov 2013 23:16:42 +0100 Subject: [PATCH 07/19] local: Fix inconsistent uri handling in local scanner We now only operate on local track uris, instead of a funny mix of local and file uris. To achieve this we instead maintain a uri->path mapping to use for the actual scanning. --- mopidy/backends/local/commands.py | 42 ++++++++++++++--------------- mopidy/backends/local/translator.py | 15 +++++++++++ 2 files changed, 36 insertions(+), 21 deletions(-) diff --git a/mopidy/backends/local/commands.py b/mopidy/backends/local/commands.py index c0d6d23a..48ae4e9f 100644 --- a/mopidy/backends/local/commands.py +++ b/mopidy/backends/local/commands.py @@ -44,27 +44,26 @@ class ScanCommand(commands.Command): local_updater = updaters.values()[0](config) - # TODO: cleanup to consistently use local urls, not a random mix of - # local and file uris depending on how the data was loaded. - uris_library = set() - uris_update = set() - uris_remove = set() + uri_path_mapping = {} + uris_in_library = set() + uris_to_update = set() + uris_to_remove = set() tracks = local_updater.load() logger.info('Checking %d tracks from library.', len(tracks)) for track in tracks: + track_path = translator.local_to_path(track.uri, media_dir) + uri_path_mapping[track.uri] = track_path try: - uri = translator.local_to_file_uri(track.uri, media_dir) - stat = os.stat(path.uri_to_path(uri)) - if int(stat.st_mtime) > track.last_modified: - uris_update.add(uri) - uris_library.add(uri) + if int(os.stat(track_path).st_mtime) > track.last_modified: + uris_to_update.add(track.uri) + uris_in_library.add(track.uri) except OSError: logger.debug('Missing file %s', track.uri) - uris_remove.add(track.uri) + uris_to_remove.add(track.uri) - logger.info('Removing %d missing tracks.', len(uris_remove)) - for uri in uris_remove: + logger.info('Removing %d missing tracks.', len(uris_to_remove)) + for uri in uris_to_remove: local_updater.remove(uri) logger.info('Checking %s for unknown tracks.', media_dir) @@ -74,20 +73,21 @@ class ScanCommand(commands.Command): logger.debug('Skipped %s: File extension excluded.', uri) continue - uri = path.path_to_uri(os.path.join(media_dir, relpath)) - if uri not in uris_library: - uris_update.add(uri) + uri = translator.path_to_local(relpath) + if uri not in uris_in_library: + uris_to_update.add(uri) + uri_path_mapping[uri] = os.path.join(media_dir, relpath) - logger.info('Found %d unknown tracks.', len(uris_update)) + logger.info('Found %d unknown tracks.', len(uris_to_update)) logger.info('Scanning...') scanner = scan.Scanner(scan_timeout) - progress = Progress(len(uris_update)) + progress = Progress(len(uris_to_update)) - for uri in sorted(uris_update): + for uri in sorted(uris_to_update): try: - data = scanner.scan(uri) - track = scan.audio_data_to_track(data) + data = scanner.scan(path.path_to_uri(uri_path_mapping[uri])) + track = scan.audio_data_to_track(data).copy(uri=uri) local_updater.add(track) logger.debug('Added %s', track.uri) except exceptions.ScannerError as error: diff --git a/mopidy/backends/local/translator.py b/mopidy/backends/local/translator.py index dc266d1c..0f82b05e 100644 --- a/mopidy/backends/local/translator.py +++ b/mopidy/backends/local/translator.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals import logging import os import urlparse +import urllib from mopidy.utils.encoding import locale_decode from mopidy.utils.path import path_to_uri, uri_to_path @@ -10,6 +11,7 @@ from mopidy.utils.path import path_to_uri, uri_to_path logger = logging.getLogger('mopidy.backends.local') +# TODO: remove once tag cache is gone def local_to_file_uri(uri, media_dir): # TODO: check that type is correct. file_path = uri_to_path(uri).split(b':', 1)[1] @@ -17,6 +19,19 @@ def local_to_file_uri(uri, media_dir): return path_to_uri(file_path) +def local_to_path(uri, media_dir): + if not uri.startswith('local:track:'): + raise Exception + file_path = uri_to_path(uri).split(b':', 1)[1] + return os.path.join(media_dir, file_path) + + +def path_to_local(relpath): + if isinstance(relpath, unicode): + relpath = relpath.encode('utf-8') + return b'local:track:%s' % urllib.quote(relpath) + + def parse_m3u(file_path, media_dir): r""" Convert M3U file list of uris From ca358e05db85fce299a48a1102ed6c58f24f4cda Mon Sep 17 00:00:00 2001 From: Thomas Adamcik Date: Wed, 27 Nov 2013 23:27:31 +0100 Subject: [PATCH 08/19] local: Move find_exact and search out of tag cache. --- mopidy/backends/local/search.py | 179 ++++++++++++++++++++++ mopidy/backends/local/tagcache/library.py | 173 +-------------------- 2 files changed, 184 insertions(+), 168 deletions(-) create mode 100644 mopidy/backends/local/search.py diff --git a/mopidy/backends/local/search.py b/mopidy/backends/local/search.py new file mode 100644 index 00000000..870afcfd --- /dev/null +++ b/mopidy/backends/local/search.py @@ -0,0 +1,179 @@ +from __future__ import unicode_literals + +from mopidy.models import Album, SearchResult + + +def find_exact(tracks, query=None, uris=None): + # TODO Only return results within URI roots given by ``uris`` + + if query is None: + query = {} + + _validate_query(query) + + for (field, values) in query.iteritems(): + if not hasattr(values, '__iter__'): + values = [values] + # FIXME this is bound to be slow for large libraries + for value in values: + if field == 'track_no': + q = _convert_to_int(value) + else: + q = value.strip() + + uri_filter = lambda t: q == t.uri + track_name_filter = lambda t: q == t.name + album_filter = lambda t: q == getattr(t, 'album', Album()).name + artist_filter = lambda t: filter( + lambda a: q == a.name, t.artists) + albumartist_filter = lambda t: any([ + q == a.name + for a in getattr(t.album, 'artists', [])]) + composer_filter = lambda t: any([ + q == a.name + for a in getattr(t, 'composers', [])]) + performer_filter = lambda t: any([ + q == a.name + for a in getattr(t, 'performers', [])]) + track_no_filter = lambda t: q == t.track_no + genre_filter = lambda t: t.genre and q == t.genre + date_filter = lambda t: q == t.date + comment_filter = lambda t: q == t.comment + any_filter = lambda t: ( + uri_filter(t) or + track_name_filter(t) or + album_filter(t) or + artist_filter(t) or + albumartist_filter(t) or + composer_filter(t) or + performer_filter(t) or + track_no_filter(t) or + genre_filter(t) or + date_filter(t) or + comment_filter(t)) + + if field == 'uri': + tracks = filter(uri_filter, tracks) + elif field == 'track_name': + tracks = filter(track_name_filter, tracks) + elif field == 'album': + tracks = filter(album_filter, tracks) + elif field == 'artist': + tracks = filter(artist_filter, tracks) + elif field == 'albumartist': + tracks = filter(albumartist_filter, tracks) + elif field == 'composer': + tracks = filter(composer_filter, tracks) + elif field == 'performer': + tracks = filter(performer_filter, tracks) + elif field == 'track_no': + tracks = filter(track_no_filter, tracks) + elif field == 'genre': + tracks = filter(genre_filter, tracks) + elif field == 'date': + tracks = filter(date_filter, tracks) + elif field == 'comment': + tracks = filter(comment_filter, tracks) + elif field == 'any': + tracks = filter(any_filter, tracks) + else: + raise LookupError('Invalid lookup field: %s' % field) + + # TODO: add local:search: + return SearchResult(uri='local:search', tracks=tracks) + + +def search(tracks, query=None, uris=None): + # TODO Only return results within URI roots given by ``uris`` + + if query is None: + query = {} + + _validate_query(query) + + for (field, values) in query.iteritems(): + if not hasattr(values, '__iter__'): + values = [values] + # FIXME this is bound to be slow for large libraries + for value in values: + if field == 'track_no': + q = _convert_to_int(value) + else: + q = value.strip().lower() + + uri_filter = lambda t: q in t.uri.lower() + track_name_filter = lambda t: q in t.name.lower() + album_filter = lambda t: q in getattr( + t, 'album', Album()).name.lower() + artist_filter = lambda t: filter( + lambda a: q in a.name.lower(), t.artists) + albumartist_filter = lambda t: any([ + q in a.name.lower() + for a in getattr(t.album, 'artists', [])]) + composer_filter = lambda t: any([ + q in a.name.lower() + for a in getattr(t, 'composers', [])]) + performer_filter = lambda t: any([ + q in a.name.lower() + for a in getattr(t, 'performers', [])]) + track_no_filter = lambda t: q == t.track_no + genre_filter = lambda t: t.genre and q in t.genre.lower() + date_filter = lambda t: t.date and t.date.startswith(q) + comment_filter = lambda t: t.comment and q in t.comment.lower() + any_filter = lambda t: ( + uri_filter(t) or + track_name_filter(t) or + album_filter(t) or + artist_filter(t) or + albumartist_filter(t) or + composer_filter(t) or + performer_filter(t) or + track_no_filter(t) or + genre_filter(t) or + date_filter(t) or + comment_filter(t)) + + if field == 'uri': + tracks = filter(uri_filter, tracks) + elif field == 'track_name': + tracks = filter(track_name_filter, tracks) + elif field == 'album': + tracks = filter(album_filter, tracks) + elif field == 'artist': + tracks = filter(artist_filter, tracks) + elif field == 'albumartist': + tracks = filter(albumartist_filter, tracks) + elif field == 'composer': + tracks = filter(composer_filter, tracks) + elif field == 'performer': + tracks = filter(performer_filter, tracks) + elif field == 'track_no': + tracks = filter(track_no_filter, tracks) + elif field == 'genre': + tracks = filter(genre_filter, tracks) + elif field == 'date': + tracks = filter(date_filter, tracks) + elif field == 'comment': + tracks = filter(comment_filter, tracks) + elif field == 'any': + tracks = filter(any_filter, tracks) + else: + raise LookupError('Invalid lookup field: %s' % field) + # TODO: add local:search: + return SearchResult(uri='local:search', tracks=tracks) + + +def _validate_query(query): + for (_, values) in query.iteritems(): + if not values: + raise LookupError('Missing query') + for value in values: + if not value: + raise LookupError('Missing query') + + +def _convert_to_int(string): + try: + return int(string) + except ValueError: + return object() diff --git a/mopidy/backends/local/tagcache/library.py b/mopidy/backends/local/tagcache/library.py index c795cdc1..fdc0be35 100644 --- a/mopidy/backends/local/tagcache/library.py +++ b/mopidy/backends/local/tagcache/library.py @@ -6,7 +6,7 @@ import tempfile from mopidy.backends import base from mopidy.backends.local.translator import local_to_file_uri -from mopidy.models import Album, SearchResult +from mopidy.backends.local import search from .translator import parse_mpd_tag_cache, tracks_to_tag_cache_format @@ -21,12 +21,6 @@ class LocalTagcacheLibraryProvider(base.BaseLibraryProvider): self._tag_cache_file = self.backend.config['local']['tag_cache_file'] self.refresh() - def _convert_to_int(self, string): - try: - return int(string) - except ValueError: - return object() - def refresh(self, uri=None): logger.debug( 'Loading local tracks from %s using %s', @@ -54,169 +48,12 @@ class LocalTagcacheLibraryProvider(base.BaseLibraryProvider): return [] def find_exact(self, query=None, uris=None): - # TODO Only return results within URI roots given by ``uris`` - - if query is None: - query = {} - self._validate_query(query) - result_tracks = self._uri_mapping.values() - - for (field, values) in query.iteritems(): - if not hasattr(values, '__iter__'): - values = [values] - # FIXME this is bound to be slow for large libraries - for value in values: - if field == 'track_no': - q = self._convert_to_int(value) - else: - q = value.strip() - - uri_filter = lambda t: q == t.uri - track_name_filter = lambda t: q == t.name - album_filter = lambda t: q == getattr(t, 'album', Album()).name - artist_filter = lambda t: filter( - lambda a: q == a.name, t.artists) - albumartist_filter = lambda t: any([ - q == a.name - for a in getattr(t.album, 'artists', [])]) - composer_filter = lambda t: any([ - q == a.name - for a in getattr(t, 'composers', [])]) - performer_filter = lambda t: any([ - q == a.name - for a in getattr(t, 'performers', [])]) - track_no_filter = lambda t: q == t.track_no - genre_filter = lambda t: t.genre and q == t.genre - date_filter = lambda t: q == t.date - comment_filter = lambda t: q == t.comment - any_filter = lambda t: ( - uri_filter(t) or - track_name_filter(t) or - album_filter(t) or - artist_filter(t) or - albumartist_filter(t) or - composer_filter(t) or - performer_filter(t) or - track_no_filter(t) or - genre_filter(t) or - date_filter(t) or - comment_filter(t)) - - if field == 'uri': - result_tracks = filter(uri_filter, result_tracks) - elif field == 'track_name': - result_tracks = filter(track_name_filter, result_tracks) - elif field == 'album': - result_tracks = filter(album_filter, result_tracks) - elif field == 'artist': - result_tracks = filter(artist_filter, result_tracks) - elif field == 'albumartist': - result_tracks = filter(albumartist_filter, result_tracks) - elif field == 'composer': - result_tracks = filter(composer_filter, result_tracks) - elif field == 'performer': - result_tracks = filter(performer_filter, result_tracks) - elif field == 'track_no': - result_tracks = filter(track_no_filter, result_tracks) - elif field == 'genre': - result_tracks = filter(genre_filter, result_tracks) - elif field == 'date': - result_tracks = filter(date_filter, result_tracks) - elif field == 'comment': - result_tracks = filter(comment_filter, result_tracks) - elif field == 'any': - result_tracks = filter(any_filter, result_tracks) - else: - raise LookupError('Invalid lookup field: %s' % field) - # TODO: add local:search: - return SearchResult(uri='local:search', tracks=result_tracks) + tracks = self._uri_mapping.values() + return search.find_exact(tracks, query=query, uris=uris) def search(self, query=None, uris=None): - # TODO Only return results within URI roots given by ``uris`` - - if query is None: - query = {} - self._validate_query(query) - result_tracks = self._uri_mapping.values() - - for (field, values) in query.iteritems(): - if not hasattr(values, '__iter__'): - values = [values] - # FIXME this is bound to be slow for large libraries - for value in values: - if field == 'track_no': - q = self._convert_to_int(value) - else: - q = value.strip().lower() - - uri_filter = lambda t: q in t.uri.lower() - track_name_filter = lambda t: q in t.name.lower() - album_filter = lambda t: q in getattr( - t, 'album', Album()).name.lower() - artist_filter = lambda t: filter( - lambda a: q in a.name.lower(), t.artists) - albumartist_filter = lambda t: any([ - q in a.name.lower() - for a in getattr(t.album, 'artists', [])]) - composer_filter = lambda t: any([ - q in a.name.lower() - for a in getattr(t, 'composers', [])]) - performer_filter = lambda t: any([ - q in a.name.lower() - for a in getattr(t, 'performers', [])]) - track_no_filter = lambda t: q == t.track_no - genre_filter = lambda t: t.genre and q in t.genre.lower() - date_filter = lambda t: t.date and t.date.startswith(q) - comment_filter = lambda t: t.comment and q in t.comment.lower() - any_filter = lambda t: ( - uri_filter(t) or - track_name_filter(t) or - album_filter(t) or - artist_filter(t) or - albumartist_filter(t) or - composer_filter(t) or - performer_filter(t) or - track_no_filter(t) or - genre_filter(t) or - date_filter(t) or - comment_filter(t)) - - if field == 'uri': - result_tracks = filter(uri_filter, result_tracks) - elif field == 'track_name': - result_tracks = filter(track_name_filter, result_tracks) - elif field == 'album': - result_tracks = filter(album_filter, result_tracks) - elif field == 'artist': - result_tracks = filter(artist_filter, result_tracks) - elif field == 'albumartist': - result_tracks = filter(albumartist_filter, result_tracks) - elif field == 'composer': - result_tracks = filter(composer_filter, result_tracks) - elif field == 'performer': - result_tracks = filter(performer_filter, result_tracks) - elif field == 'track_no': - result_tracks = filter(track_no_filter, result_tracks) - elif field == 'genre': - result_tracks = filter(genre_filter, result_tracks) - elif field == 'date': - result_tracks = filter(date_filter, result_tracks) - elif field == 'comment': - result_tracks = filter(comment_filter, result_tracks) - elif field == 'any': - result_tracks = filter(any_filter, result_tracks) - else: - raise LookupError('Invalid lookup field: %s' % field) - # TODO: add local:search: - return SearchResult(uri='local:search', tracks=result_tracks) - - def _validate_query(self, query): - for (_, values) in query.iteritems(): - if not values: - raise LookupError('Missing query') - for value in values: - if not value: - raise LookupError('Missing query') + tracks = self._uri_mapping.values() + return search.search(tracks, query=query, uris=uris) class LocalTagcacheLibraryUpdateProvider(base.BaseLibraryProvider): From 118095e5228fed1f4c4dccc9cbf7cccc99f6f4b5 Mon Sep 17 00:00:00 2001 From: Thomas Adamcik Date: Wed, 27 Nov 2013 23:39:53 +0100 Subject: [PATCH 09/19] local: Add new json based library - Sets local-tagcache as disabled - Implements new library that uses a gzip compressed json as storage. - Thanks to reuse of existing serialization code this is a fairly small change. --- mopidy/backends/local/json/__init__.py | 33 ++++++++ mopidy/backends/local/json/actor.py | 30 +++++++ mopidy/backends/local/json/ext.conf | 3 + mopidy/backends/local/json/library.py | 106 ++++++++++++++++++++++++ mopidy/backends/local/tagcache/ext.conf | 2 +- setup.py | 1 + 6 files changed, 174 insertions(+), 1 deletion(-) create mode 100644 mopidy/backends/local/json/__init__.py create mode 100644 mopidy/backends/local/json/actor.py create mode 100644 mopidy/backends/local/json/ext.conf create mode 100644 mopidy/backends/local/json/library.py diff --git a/mopidy/backends/local/json/__init__.py b/mopidy/backends/local/json/__init__.py new file mode 100644 index 00000000..d1103ac6 --- /dev/null +++ b/mopidy/backends/local/json/__init__.py @@ -0,0 +1,33 @@ +from __future__ import unicode_literals + +import os + +import mopidy +from mopidy import config, ext + + +class Extension(ext.Extension): + + dist_name = 'Mopidy-Local-JSON' + ext_name = 'local-json' + version = mopidy.__version__ + + def get_default_config(self): + conf_file = os.path.join(os.path.dirname(__file__), 'ext.conf') + return config.read(conf_file) + + def get_config_schema(self): + schema = super(Extension, self).get_config_schema() + schema['json_file'] = config.Path() + return schema + + def validate_environment(self): + pass + + def get_backend_classes(self): + from .actor import LocalJsonBackend + return [LocalJsonBackend] + + def get_library_updaters(self): + from .library import LocalJsonLibraryUpdateProvider + return [LocalJsonLibraryUpdateProvider] diff --git a/mopidy/backends/local/json/actor.py b/mopidy/backends/local/json/actor.py new file mode 100644 index 00000000..df9ac447 --- /dev/null +++ b/mopidy/backends/local/json/actor.py @@ -0,0 +1,30 @@ +from __future__ import unicode_literals + +import logging + +import pykka + +from mopidy.backends import base +from mopidy.utils import encoding, path + +from .library import LocalJsonLibraryProvider + +logger = logging.getLogger('mopidy.backends.local.json') + + +class LocalJsonBackend(pykka.ThreadingActor, base.Backend): + def __init__(self, config, audio): + super(LocalJsonBackend, self).__init__() + + self.config = config + self.check_dirs_and_files() + self.library = LocalJsonLibraryProvider(backend=self) + self.uri_schemes = ['local'] + + def check_dirs_and_files(self): + try: + path.get_or_create_file(self.config['local-json']['json_file']) + except EnvironmentError as error: + logger.warning( + 'Could not create empty json file: %s', + encoding.locale_decode(error)) diff --git a/mopidy/backends/local/json/ext.conf b/mopidy/backends/local/json/ext.conf new file mode 100644 index 00000000..db0b784a --- /dev/null +++ b/mopidy/backends/local/json/ext.conf @@ -0,0 +1,3 @@ +[local-json] +enabled = true +json_file = $XDG_DATA_DIR/mopidy/local/library.json.gz diff --git a/mopidy/backends/local/json/library.py b/mopidy/backends/local/json/library.py new file mode 100644 index 00000000..abb620e9 --- /dev/null +++ b/mopidy/backends/local/json/library.py @@ -0,0 +1,106 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + +import gzip +import json +import logging +import os +import tempfile + +import mopidy +from mopidy import models +from mopidy.backends import base +from mopidy.backends.local import search + +logger = logging.getLogger('mopidy.backends.local.json') + + +def _load_tracks(json_file): + try: + with gzip.open(json_file, 'rb') as fp: + result = json.load(fp, object_hook=models.model_json_decoder) + except IOError: + return [] + return result.get('tracks', []) + + +class LocalJsonLibraryProvider(base.BaseLibraryProvider): + def __init__(self, *args, **kwargs): + super(LocalJsonLibraryProvider, self).__init__(*args, **kwargs) + self._uri_mapping = {} + self._media_dir = self.backend.config['local']['media_dir'] + self._json_file = self.backend.config['local-json']['json_file'] + self.refresh() + + def refresh(self, uri=None): + logger.debug( + 'Loading local tracks from %s using %s', + self._media_dir, self._json_file) + + tracks = _load_tracks(self._json_file) + uris_to_remove = set(self._uri_mapping) + + for track in tracks: + self._uri_mapping[track.uri] = track + uris_to_remove.discard(track.uri) + + for uri in uris_to_remove: + del self._uri_mapping[uri] + + logger.info( + 'Loaded %d local tracks from %s using %s', + len(tracks), self._media_dir, self._json_file) + + def lookup(self, uri): + try: + return [self._uri_mapping[uri]] + except KeyError: + logger.debug('Failed to lookup %r', uri) + return [] + + def find_exact(self, query=None, uris=None): + tracks = self._uri_mapping.values() + return search.find_exact(tracks, query=query, uris=uris) + + def search(self, query=None, uris=None): + tracks = self._uri_mapping.values() + return search.search(tracks, query=query, uris=uris) + + +class LocalJsonLibraryUpdateProvider(base.BaseLibraryProvider): + uri_schemes = ['local'] + + def __init__(self, config): + self._tracks = {} + self._media_dir = config['local']['media_dir'] + self._json_file = config['local-json']['json_file'] + + def load(self): + for track in _load_tracks(self._json_file): + self._tracks[track.uri] = track + return self._tracks.values() + + def add(self, track): + self._tracks[track.uri] = track + + def remove(self, uri): + if uri in self._tracks: + del self._tracks[uri] + + def commit(self): + directory, basename = os.path.split(self._json_file) + + # TODO: cleanup directory/basename.* files. + tmp = tempfile.NamedTemporaryFile( + prefix=basename + '.', dir=directory, delete=False) + + try: + with gzip.GzipFile(fileobj=tmp, mode='wb') as fp: + data = {'version': mopidy.__version__, + 'tracks': self._tracks.values()} + json.dump(data, fp, cls=models.ModelJSONEncoder, + indent=2, separators=(',', ': ')) + os.rename(tmp.name, self._json_file) + finally: + if os.path.exists(tmp.name): + os.remove(tmp.name) diff --git a/mopidy/backends/local/tagcache/ext.conf b/mopidy/backends/local/tagcache/ext.conf index 48a3c763..749959e8 100644 --- a/mopidy/backends/local/tagcache/ext.conf +++ b/mopidy/backends/local/tagcache/ext.conf @@ -1,2 +1,2 @@ [local-tagcache] -enabled = true +enabled = false diff --git a/setup.py b/setup.py index 7d4b2cd8..11855553 100644 --- a/setup.py +++ b/setup.py @@ -44,6 +44,7 @@ setup( 'http = mopidy.frontends.http:Extension [http]', 'local = mopidy.backends.local:Extension', 'local-tagcache = mopidy.backends.local.tagcache:Extension', + 'local-json = mopidy.backends.local.json:Extension', 'mpd = mopidy.frontends.mpd:Extension', 'stream = mopidy.backends.stream:Extension', ], From 3bbcb4d121631f3737333de0de720307865caea8 Mon Sep 17 00:00:00 2001 From: Thomas Adamcik Date: Thu, 28 Nov 2013 23:20:03 +0100 Subject: [PATCH 10/19] local: Review comment fixes --- mopidy/backends/local/commands.py | 9 +++++---- mopidy/backends/local/json/__init__.py | 3 --- mopidy/backends/local/json/library.py | 1 - mopidy/backends/local/tagcache/__init__.py | 3 --- mopidy/backends/local/tagcache/library.py | 2 +- mopidy/backends/local/translator.py | 7 ++++--- tests/backends/local/tagcache_test.py | 4 ++-- tests/core/actor_test.py | 11 +++++++---- tests/utils/path_test.py | 4 ++-- 9 files changed, 21 insertions(+), 23 deletions(-) diff --git a/mopidy/backends/local/commands.py b/mopidy/backends/local/commands.py index 48ae4e9f..5e9b42e6 100644 --- a/mopidy/backends/local/commands.py +++ b/mopidy/backends/local/commands.py @@ -52,10 +52,11 @@ class ScanCommand(commands.Command): tracks = local_updater.load() logger.info('Checking %d tracks from library.', len(tracks)) for track in tracks: - track_path = translator.local_to_path(track.uri, media_dir) - uri_path_mapping[track.uri] = track_path + uri_path_mapping[track.uri] = translator.local_track_uri_to_path( + track.uri, media_dir) try: - if int(os.stat(track_path).st_mtime) > track.last_modified: + stat = os.stat(uri_path_mapping[track.uri]) + if int(stat.st_mtime) > track.last_modified: uris_to_update.add(track.uri) uris_in_library.add(track.uri) except OSError: @@ -73,7 +74,7 @@ class ScanCommand(commands.Command): logger.debug('Skipped %s: File extension excluded.', uri) continue - uri = translator.path_to_local(relpath) + uri = translator.path_to_local_track_uri(relpath) if uri not in uris_in_library: uris_to_update.add(uri) uri_path_mapping[uri] = os.path.join(media_dir, relpath) diff --git a/mopidy/backends/local/json/__init__.py b/mopidy/backends/local/json/__init__.py index d1103ac6..031dae51 100644 --- a/mopidy/backends/local/json/__init__.py +++ b/mopidy/backends/local/json/__init__.py @@ -21,9 +21,6 @@ class Extension(ext.Extension): schema['json_file'] = config.Path() return schema - def validate_environment(self): - pass - def get_backend_classes(self): from .actor import LocalJsonBackend return [LocalJsonBackend] diff --git a/mopidy/backends/local/json/library.py b/mopidy/backends/local/json/library.py index abb620e9..6bfef783 100644 --- a/mopidy/backends/local/json/library.py +++ b/mopidy/backends/local/json/library.py @@ -1,4 +1,3 @@ -from __future__ import absolute_import from __future__ import unicode_literals import gzip diff --git a/mopidy/backends/local/tagcache/__init__.py b/mopidy/backends/local/tagcache/__init__.py index c7364e8b..b51b88bf 100644 --- a/mopidy/backends/local/tagcache/__init__.py +++ b/mopidy/backends/local/tagcache/__init__.py @@ -19,9 +19,6 @@ class Extension(ext.Extension): # Config only contains local-tagcache/enabled since we are not setting our # own schema. - def validate_environment(self): - pass - def get_backend_classes(self): from .actor import LocalTagcacheBackend return [LocalTagcacheBackend] diff --git a/mopidy/backends/local/tagcache/library.py b/mopidy/backends/local/tagcache/library.py index fdc0be35..5b541d19 100644 --- a/mopidy/backends/local/tagcache/library.py +++ b/mopidy/backends/local/tagcache/library.py @@ -5,8 +5,8 @@ import os import tempfile from mopidy.backends import base -from mopidy.backends.local.translator import local_to_file_uri from mopidy.backends.local import search +from mopidy.backends.local.translator import local_to_file_uri from .translator import parse_mpd_tag_cache, tracks_to_tag_cache_format diff --git a/mopidy/backends/local/translator.py b/mopidy/backends/local/translator.py index 0f82b05e..1153b1b3 100644 --- a/mopidy/backends/local/translator.py +++ b/mopidy/backends/local/translator.py @@ -19,14 +19,15 @@ def local_to_file_uri(uri, media_dir): return path_to_uri(file_path) -def local_to_path(uri, media_dir): +def local_track_uri_to_path(uri, media_dir): if not uri.startswith('local:track:'): - raise Exception + raise ValueError('Invalid uri.') file_path = uri_to_path(uri).split(b':', 1)[1] return os.path.join(media_dir, file_path) -def path_to_local(relpath): +def path_to_local_track_uri(relpath): + """Convert path releative to media_dir to local track uri""" if isinstance(relpath, unicode): relpath = relpath.encode('utf-8') return b'local:track:%s' % urllib.quote(relpath) diff --git a/tests/backends/local/tagcache_test.py b/tests/backends/local/tagcache_test.py index 6d0b7469..b40b3346 100644 --- a/tests/backends/local/tagcache_test.py +++ b/tests/backends/local/tagcache_test.py @@ -5,10 +5,10 @@ from __future__ import unicode_literals import os import unittest -from mopidy.utils.path import mtime, uri_to_path -from mopidy.frontends.mpd import translator as mpd, protocol from mopidy.backends.local.tagcache import translator +from mopidy.frontends.mpd import translator as mpd, protocol from mopidy.models import Album, Artist, Track +from mopidy.utils.path import mtime, uri_to_path from tests import path_to_data_dir diff --git a/tests/core/actor_test.py b/tests/core/actor_test.py index ce50d5ed..38e33baa 100644 --- a/tests/core/actor_test.py +++ b/tests/core/actor_test.py @@ -37,12 +37,15 @@ class CoreActorTest(unittest.TestCase): Core, audio=None, backends=[self.backend1, self.backend2]) def test_backends_with_colliding_uri_schemes_passes(self): - # Checks that backends with overlapping schemes, but distinct sub parts - # provided can co-exist. + """ + Checks that backends with overlapping schemes, but distinct sub parts + provided can co-exist. + """ + self.backend1.has_library().get.return_value = False self.backend1.has_playlists().get.return_value = False - self.backend2.uri_schemes().get.return_value = ['dummy1'] + self.backend2.uri_schemes.get.return_value = ['dummy1'] self.backend2.has_playback().get.return_value = False self.backend2.has_playlists().get.return_value = False @@ -50,4 +53,4 @@ class CoreActorTest(unittest.TestCase): self.assertEqual(core.backends.with_playback, {'dummy1': self.backend1}) self.assertEqual(core.backends.with_library, - {'dummy2': self.backend2}) + {'dummy1': self.backend2}) diff --git a/tests/utils/path_test.py b/tests/utils/path_test.py index 316b4f38..3accab39 100644 --- a/tests/utils/path_test.py +++ b/tests/utils/path_test.py @@ -225,8 +225,8 @@ class FindFilesTest(unittest.TestCase): def test_files(self): files = self.find('find') - excepted = [b'foo/bar/file', b'foo/file', b'baz/file'] - self.assertItemsEqual(excepted, files) + expected = [b'foo/bar/file', b'foo/file', b'baz/file'] + self.assertItemsEqual(expected, files) def test_names_are_bytestrings(self): is_bytes = lambda f: isinstance(f, bytes) From 121d2c782a1553c857fb3715b39a738d9c5777de Mon Sep 17 00:00:00 2001 From: Thomas Adamcik Date: Thu, 28 Nov 2013 23:42:12 +0100 Subject: [PATCH 11/19] docs: Add info about local-json and pluggable libraries --- docs/ext/local.rst | 54 +++++++++++++++++++++++++++++++++++-------- docs/extensiondev.rst | 36 +++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 9 deletions(-) diff --git a/docs/ext/local.rst b/docs/ext/local.rst index 23baa5d1..583a7427 100644 --- a/docs/ext/local.rst +++ b/docs/ext/local.rst @@ -62,29 +62,65 @@ Usage If you want use Mopidy to play music you have locally at your machine, you need to review and maybe change some of the local extension config values. See above -for a complete list. Then you need to generate a tag cache for your local +for a complete list. Then you need to generate a local library for your local music... -.. _generating-a-tag-cache: +.. _generating-a-local-library: -Generating a tag cache ----------------------- +Generating a local library +-------------------------- The command :command:`mopidy local scan` will scan the path set in the -:confval:`local/media_dir` config value for any media files and build a MPD -compatible ``tag_cache``. +:confval:`local/media_dir` config value for any audio files and build a +library. -To make a ``tag_cache`` of your local music available for Mopidy: +To make a local library for your music available for Mopidy: #. Ensure that the :confval:`local/media_dir` config value points to where your music is located. Check the current setting by running:: mopidy config -#. Scan your media library. The command writes the ``tag_cache`` to - the :confval:`local/tag_cache_file`:: +#. Scan your media library.:: mopidy local scan #. Start Mopidy, find the music library in a client, and play some local music! + + +Plugable library support +------------------------ + +Local libraries are fully pluggable. What this means is that users may opt to +disable the current default library `local-json`, replacing it with a third +party one. When running :command:`mopidy local scan` mopidy will populate +whatever the current active library is with data. Only one library may be +active at a time. + + +***************** +Mopidy-Local-JSON +***************** + +Extension for storing local music library in a JSON file, default built in +library for local files. + + +Default configuration +===================== + +.. literalinclude:: ../../mopidy/backends/local/json/ext.conf + :language: ini + + +Configuration values +==================== + +.. confval:: local-json/enabled + + If the local extension should be enabled or not. + +.. confval:: local-json/json_file + + Path to a file to store the gziped json data in. diff --git a/docs/extensiondev.rst b/docs/extensiondev.rst index 7fa19f7a..2709690a 100644 --- a/docs/extensiondev.rst +++ b/docs/extensiondev.rst @@ -309,6 +309,10 @@ This is ``mopidy_soundspot/__init__.py``:: from .commands import SoundspotCommand return SoundspotCommand() + def get_library_updaters(self): + from .library import SoundspotLibraryUpdateProvider + return [SoundspotLibraryUpdateProvider] + def register_gstreamer_elements(self): from .mixer import SoundspotMixer gobject.type_register(SoundspotMixer) @@ -406,6 +410,38 @@ more details. return 0 +Example library provider +======================== + +Currently library providers are only really relevant for people who want to +replace the default local library. Providing this in addition to a backend that +exposes a library for the `local` uri scheme lets you plug in whatever storage +solution you happen to prefer. + +:: + + from mopidy.backends import base + + + class SoundspotLibraryUpdateProvider(base.BaseLibraryProvider): + def __init__(self, config): + super(SoundspotLibraryUpdateProvider, self).__init__(config) + self.config = config + + def load(self): + # Your track loading code + return tracks + + def add(self, track): + # Your code for handling adding a new track + + def remove(self, uri): + # Your code for removing the track coresponding to this uri + + def commit(self): + # Your code to persist the library, if needed. + + Example GStreamer element ========================= From 26ec956a082d7c15aec3693e21b07941bd380fdd Mon Sep 17 00:00:00 2001 From: Thomas Adamcik Date: Fri, 29 Nov 2013 00:13:06 +0100 Subject: [PATCH 12/19] config: Add deprecated config value support. This makes it possible to mark a key as deprecated, this implies it will be ignored if present, and never included in the resulting config. --- mopidy/config/schemas.py | 2 ++ mopidy/config/types.py | 18 ++++++++++++++++++ tests/config/schemas_test.py | 10 +++++++++- tests/config/types_test.py | 10 ++++++++++ 4 files changed, 39 insertions(+), 1 deletion(-) diff --git a/mopidy/config/schemas.py b/mopidy/config/schemas.py index a535b493..67473b88 100644 --- a/mopidy/config/schemas.py +++ b/mopidy/config/schemas.py @@ -74,6 +74,8 @@ class ConfigSchema(collections.OrderedDict): if key not in result and key not in errors: result[key] = None errors[key] = 'config key not found.' + if isinstance(result[key], types.DeprecatedValue): + del result[key] return result, errors diff --git a/mopidy/config/types.py b/mopidy/config/types.py index d264de30..6aeaaaa7 100644 --- a/mopidy/config/types.py +++ b/mopidy/config/types.py @@ -31,6 +31,10 @@ class ExpandedPath(bytes): self.original = original +class DeprecatedValue(object): + pass + + class ConfigValue(object): """Represents a config key's value and how to handle it. @@ -59,6 +63,20 @@ class ConfigValue(object): return bytes(value) +class Deprecated(ConfigValue): + """Deprecated value + + Used for ignoring old config values that are no longer in use, but should + not cause the config parser to crash. + """ + + def deserialize(self, value): + return DeprecatedValue() + + def serialize(self, value, display=False): + return DeprecatedValue() + + class String(ConfigValue): """String value. diff --git a/tests/config/schemas_test.py b/tests/config/schemas_test.py index 9da8f667..82ea159b 100644 --- a/tests/config/schemas_test.py +++ b/tests/config/schemas_test.py @@ -4,7 +4,7 @@ import logging import mock import unittest -from mopidy.config import schemas +from mopidy.config import schemas, types from tests import any_unicode @@ -77,6 +77,14 @@ class ConfigSchemaTest(unittest.TestCase): self.assertIsNone(result['bar']) self.assertIsNone(result['baz']) + def test_deserialize_none_value(self): + self.schema['foo'].deserialize.return_value = types.DeprecatedValue() + + result, errors = self.schema.deserialize(self.values) + print result, errors + self.assertItemsEqual(['bar', 'baz'], result.keys()) + self.assertNotIn('foo', errors) + class LogLevelConfigSchemaTest(unittest.TestCase): def test_conversion(self): diff --git a/tests/config/types_test.py b/tests/config/types_test.py index 0df3dfb4..c4b9ec88 100644 --- a/tests/config/types_test.py +++ b/tests/config/types_test.py @@ -33,6 +33,16 @@ class ConfigValueTest(unittest.TestCase): self.assertIsInstance(value.serialize(object(), display=True), bytes) +class DeprecatedTest(unittest.TestCase): + def test_deserialize_returns_deprecated_value(self): + self.assertIsInstance(types.Deprecated().deserialize(b'foobar'), + types.DeprecatedValue) + + def test_serialize_returns_deprecated_value(self): + self.assertIsInstance(types.Deprecated().serialize('foobar'), + types.DeprecatedValue) + + class StringTest(unittest.TestCase): def test_deserialize_conversion_success(self): value = types.String() From da63942b488ea03b09dec7192316d9f7501d88b0 Mon Sep 17 00:00:00 2001 From: Thomas Adamcik Date: Wed, 4 Dec 2013 21:09:04 +0100 Subject: [PATCH 13/19] config: Improve handling of Deprecated config values --- mopidy/config/__init__.py | 2 ++ mopidy/config/schemas.py | 6 +++--- tests/config/schemas_test.py | 5 ++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/mopidy/config/__init__.py b/mopidy/config/__init__.py index a7153ea2..f68567e7 100644 --- a/mopidy/config/__init__.py +++ b/mopidy/config/__init__.py @@ -167,6 +167,8 @@ def _format(config, comments, schemas, display, disable): continue output.append(b'[%s]' % bytes(schema.name)) for key, value in serialized.items(): + if isinstance(value, types.DeprecatedValue): + continue comment = bytes(comments.get(schema.name, {}).get(key, '')) output.append(b'%s =' % bytes(key)) if value is not None: diff --git a/mopidy/config/schemas.py b/mopidy/config/schemas.py index 67473b88..b026ac2b 100644 --- a/mopidy/config/schemas.py +++ b/mopidy/config/schemas.py @@ -71,11 +71,11 @@ class ConfigSchema(collections.OrderedDict): errors[key] = str(e) for key in self.keys(): - if key not in result and key not in errors: + if isinstance(self[key], types.Deprecated): + result.pop(key, None) + elif key not in result and key not in errors: result[key] = None errors[key] = 'config key not found.' - if isinstance(result[key], types.DeprecatedValue): - del result[key] return result, errors diff --git a/tests/config/schemas_test.py b/tests/config/schemas_test.py index 82ea159b..6eb35ed3 100644 --- a/tests/config/schemas_test.py +++ b/tests/config/schemas_test.py @@ -77,11 +77,10 @@ class ConfigSchemaTest(unittest.TestCase): self.assertIsNone(result['bar']) self.assertIsNone(result['baz']) - def test_deserialize_none_value(self): - self.schema['foo'].deserialize.return_value = types.DeprecatedValue() + def test_deserialize_deprecated_value(self): + self.schema['foo'] = types.Deprecated() result, errors = self.schema.deserialize(self.values) - print result, errors self.assertItemsEqual(['bar', 'baz'], result.keys()) self.assertNotIn('foo', errors) From 9c2d38e989dea2d0bc669ebd129aacc9e7f263a0 Mon Sep 17 00:00:00 2001 From: Thomas Adamcik Date: Wed, 4 Dec 2013 21:12:28 +0100 Subject: [PATCH 14/19] local: Remove tag cache support - Updates doc references to tag cache - Removes tag cache from config and marks it deprecated - Removes tag cache from setup.py - Removes tag cache from config converter - Removes tag cache from tests - Converts local library tests to use JSON. --- docs/devtools.rst | 7 +- docs/ext/local.rst | 4 - mopidy/backends/local/__init__.py | 2 +- mopidy/backends/local/ext.conf | 1 - mopidy/backends/local/tagcache/__init__.py | 28 -- mopidy/backends/local/tagcache/actor.py | 30 -- mopidy/backends/local/tagcache/ext.conf | 2 - mopidy/backends/local/tagcache/library.py | 101 ------ mopidy/backends/local/tagcache/translator.py | 246 ------------- mopidy/config/convert.py | 1 - setup.py | 1 - tests/backends/local/events_test.py | 1 - tests/backends/local/library_test.py | 46 +-- tests/backends/local/playback_test.py | 1 - tests/backends/local/playlists_test.py | 1 - tests/backends/local/tagcache_test.py | 346 ------------------- tests/backends/local/tracklist_test.py | 1 - tests/data/advanced_tag_cache | 107 ------ tests/data/albumartist_tag_cache | 16 - tests/data/blank_tag_cache | 10 - tests/data/empty_tag_cache | 6 - tests/data/library.json.gz | Bin 0 -> 394 bytes tests/data/library_tag_cache | 56 --- tests/data/musicbrainz_tag_cache | 20 -- tests/data/scanner/advanced_cache | 81 ----- tests/data/scanner/empty_cache | 6 - tests/data/scanner/simple_cache | 15 - tests/data/simple_tag_cache | 16 - tests/data/utf8_tag_cache | 18 - 29 files changed, 28 insertions(+), 1142 deletions(-) delete mode 100644 mopidy/backends/local/tagcache/__init__.py delete mode 100644 mopidy/backends/local/tagcache/actor.py delete mode 100644 mopidy/backends/local/tagcache/ext.conf delete mode 100644 mopidy/backends/local/tagcache/library.py delete mode 100644 mopidy/backends/local/tagcache/translator.py delete mode 100644 tests/backends/local/tagcache_test.py delete mode 100644 tests/data/advanced_tag_cache delete mode 100644 tests/data/albumartist_tag_cache delete mode 100644 tests/data/blank_tag_cache delete mode 100644 tests/data/empty_tag_cache create mode 100644 tests/data/library.json.gz delete mode 100644 tests/data/library_tag_cache delete mode 100644 tests/data/musicbrainz_tag_cache delete mode 100644 tests/data/scanner/advanced_cache delete mode 100644 tests/data/scanner/empty_cache delete mode 100644 tests/data/scanner/simple_cache delete mode 100644 tests/data/simple_tag_cache delete mode 100644 tests/data/utf8_tag_cache diff --git a/docs/devtools.rst b/docs/devtools.rst index ecae6c86..858cc7f8 100644 --- a/docs/devtools.rst +++ b/docs/devtools.rst @@ -66,11 +66,8 @@ Sample session:: -OK +ACK [2@0] {listallinfo} incorrect arguments -To ensure that Mopidy and MPD have comparable state it is suggested you setup -both to use ``tests/data/advanced_tag_cache`` for their tag cache and -``tests/data/scanner/advanced/`` for the music folder and ``tests/data`` for -playlists. - +To ensure that Mopidy and MPD have comparable state it is suggested you scan +the same media directory with both servers. Documentation writing ===================== diff --git a/docs/ext/local.rst b/docs/ext/local.rst index 583a7427..51268c51 100644 --- a/docs/ext/local.rst +++ b/docs/ext/local.rst @@ -43,10 +43,6 @@ Configuration values Path to playlists directory with m3u files for local media. -.. confval:: local/tag_cache_file - - Path to tag cache for local media. - .. confval:: local/scan_timeout Number of milliseconds before giving up scanning a file and moving on to diff --git a/mopidy/backends/local/__init__.py b/mopidy/backends/local/__init__.py index 723eb056..d24ab010 100644 --- a/mopidy/backends/local/__init__.py +++ b/mopidy/backends/local/__init__.py @@ -20,7 +20,7 @@ class Extension(ext.Extension): schema = super(Extension, self).get_config_schema() schema['media_dir'] = config.Path() schema['playlists_dir'] = config.Path() - schema['tag_cache_file'] = config.Path() + schema['tag_cache_file'] = config.Deprecated() schema['scan_timeout'] = config.Integer( minimum=1000, maximum=1000*60*60) schema['excluded_file_extensions'] = config.List(optional=True) diff --git a/mopidy/backends/local/ext.conf b/mopidy/backends/local/ext.conf index afc13c7d..f906a04f 100644 --- a/mopidy/backends/local/ext.conf +++ b/mopidy/backends/local/ext.conf @@ -2,7 +2,6 @@ enabled = true media_dir = $XDG_MUSIC_DIR playlists_dir = $XDG_DATA_DIR/mopidy/local/playlists -tag_cache_file = $XDG_DATA_DIR/mopidy/local/tag_cache scan_timeout = 1000 excluded_file_extensions = .html diff --git a/mopidy/backends/local/tagcache/__init__.py b/mopidy/backends/local/tagcache/__init__.py deleted file mode 100644 index b51b88bf..00000000 --- a/mopidy/backends/local/tagcache/__init__.py +++ /dev/null @@ -1,28 +0,0 @@ -from __future__ import unicode_literals - -import os - -import mopidy -from mopidy import config, ext - - -class Extension(ext.Extension): - - dist_name = 'Mopidy-Local-Tagcache' - ext_name = 'local-tagcache' - version = mopidy.__version__ - - def get_default_config(self): - conf_file = os.path.join(os.path.dirname(__file__), 'ext.conf') - return config.read(conf_file) - - # Config only contains local-tagcache/enabled since we are not setting our - # own schema. - - def get_backend_classes(self): - from .actor import LocalTagcacheBackend - return [LocalTagcacheBackend] - - def get_library_updaters(self): - from .library import LocalTagcacheLibraryUpdateProvider - return [LocalTagcacheLibraryUpdateProvider] diff --git a/mopidy/backends/local/tagcache/actor.py b/mopidy/backends/local/tagcache/actor.py deleted file mode 100644 index f052debb..00000000 --- a/mopidy/backends/local/tagcache/actor.py +++ /dev/null @@ -1,30 +0,0 @@ -from __future__ import unicode_literals - -import logging - -import pykka - -from mopidy.backends import base -from mopidy.utils import encoding, path - -from .library import LocalTagcacheLibraryProvider - -logger = logging.getLogger('mopidy.backends.local.tagcache') - - -class LocalTagcacheBackend(pykka.ThreadingActor, base.Backend): - def __init__(self, config, audio): - super(LocalTagcacheBackend, self).__init__() - - self.config = config - self.check_dirs_and_files() - self.library = LocalTagcacheLibraryProvider(backend=self) - self.uri_schemes = ['local'] - - def check_dirs_and_files(self): - try: - path.get_or_create_file(self.config['local']['tag_cache_file']) - except EnvironmentError as error: - logger.warning( - 'Could not create empty tag cache file: %s', - encoding.locale_decode(error)) diff --git a/mopidy/backends/local/tagcache/ext.conf b/mopidy/backends/local/tagcache/ext.conf deleted file mode 100644 index 749959e8..00000000 --- a/mopidy/backends/local/tagcache/ext.conf +++ /dev/null @@ -1,2 +0,0 @@ -[local-tagcache] -enabled = false diff --git a/mopidy/backends/local/tagcache/library.py b/mopidy/backends/local/tagcache/library.py deleted file mode 100644 index 5b541d19..00000000 --- a/mopidy/backends/local/tagcache/library.py +++ /dev/null @@ -1,101 +0,0 @@ -from __future__ import unicode_literals - -import logging -import os -import tempfile - -from mopidy.backends import base -from mopidy.backends.local import search -from mopidy.backends.local.translator import local_to_file_uri - -from .translator import parse_mpd_tag_cache, tracks_to_tag_cache_format - -logger = logging.getLogger('mopidy.backends.local.tagcache') - - -class LocalTagcacheLibraryProvider(base.BaseLibraryProvider): - def __init__(self, *args, **kwargs): - super(LocalTagcacheLibraryProvider, self).__init__(*args, **kwargs) - self._uri_mapping = {} - self._media_dir = self.backend.config['local']['media_dir'] - self._tag_cache_file = self.backend.config['local']['tag_cache_file'] - self.refresh() - - def refresh(self, uri=None): - logger.debug( - 'Loading local tracks from %s using %s', - self._media_dir, self._tag_cache_file) - - tracks = parse_mpd_tag_cache(self._tag_cache_file, self._media_dir) - uris_to_remove = set(self._uri_mapping) - - for track in tracks: - self._uri_mapping[track.uri] = track - uris_to_remove.discard(track.uri) - - for uri in uris_to_remove: - del self._uri_mapping[uri] - - logger.info( - 'Loaded %d local tracks from %s using %s', - len(tracks), self._media_dir, self._tag_cache_file) - - def lookup(self, uri): - try: - return [self._uri_mapping[uri]] - except KeyError: - logger.debug('Failed to lookup %r', uri) - return [] - - def find_exact(self, query=None, uris=None): - tracks = self._uri_mapping.values() - return search.find_exact(tracks, query=query, uris=uris) - - def search(self, query=None, uris=None): - tracks = self._uri_mapping.values() - return search.search(tracks, query=query, uris=uris) - - -class LocalTagcacheLibraryUpdateProvider(base.BaseLibraryProvider): - uri_schemes = ['local'] - - def __init__(self, config): - self._tracks = {} - self._media_dir = config['local']['media_dir'] - self._tag_cache_file = config['local']['tag_cache_file'] - - def load(self): - tracks = parse_mpd_tag_cache(self._tag_cache_file, self._media_dir) - for track in tracks: - # TODO: this should use uris as is, i.e. hack that should go away - # with tag caches. - uri = local_to_file_uri(track.uri, self._media_dir) - self._tracks[uri] = track.copy(uri=uri) - return tracks - - def add(self, track): - self._tracks[track.uri] = track - - def remove(self, uri): - if uri in self._tracks: - del self._tracks[uri] - - def commit(self): - directory, basename = os.path.split(self._tag_cache_file) - - # TODO: cleanup directory/basename.* files. - tmp = tempfile.NamedTemporaryFile( - prefix=basename + '.', dir=directory, delete=False) - - try: - for row in tracks_to_tag_cache_format( - self._tracks.values(), self._media_dir): - if len(row) == 1: - tmp.write(('%s\n' % row).encode('utf-8')) - else: - tmp.write(('%s: %s\n' % row).encode('utf-8')) - - os.rename(tmp.name, self._tag_cache_file) - finally: - if os.path.exists(tmp.name): - os.remove(tmp.name) diff --git a/mopidy/backends/local/tagcache/translator.py b/mopidy/backends/local/tagcache/translator.py deleted file mode 100644 index be54cd1d..00000000 --- a/mopidy/backends/local/tagcache/translator.py +++ /dev/null @@ -1,246 +0,0 @@ -from __future__ import unicode_literals - -import logging -import os -import re -import urllib - -from mopidy.frontends.mpd import translator as mpd, protocol -from mopidy.models import Track, Artist, Album -from mopidy.utils.encoding import locale_decode -from mopidy.utils.path import mtime as get_mtime, split_path, uri_to_path - -logger = logging.getLogger('mopidy.backends.local.tagcache') - - -# TODO: remove music_dir from API -def parse_mpd_tag_cache(tag_cache, music_dir=''): - """ - Converts a MPD tag_cache into a lists of tracks, artists and albums. - """ - tracks = set() - - try: - with open(tag_cache) as library: - contents = library.read() - except IOError as error: - logger.warning('Could not open tag cache: %s', locale_decode(error)) - return tracks - - current = {} - state = None - - # TODO: uris as bytes - for line in contents.split(b'\n'): - if line == b'songList begin': - state = 'songs' - continue - elif line == b'songList end': - state = None - continue - elif not state: - continue - - key, value = line.split(b': ', 1) - - if key == b'key': - _convert_mpd_data(current, tracks) - current.clear() - - current[key.lower()] = value.decode('utf-8') - - _convert_mpd_data(current, tracks) - - return tracks - - -def _convert_mpd_data(data, tracks): - if not data: - return - - track_kwargs = {} - album_kwargs = {} - artist_kwargs = {} - albumartist_kwargs = {} - - if 'track' in data: - if '/' in data['track']: - album_kwargs['num_tracks'] = int(data['track'].split('/')[1]) - track_kwargs['track_no'] = int(data['track'].split('/')[0]) - else: - track_kwargs['track_no'] = int(data['track']) - - if 'mtime' in data: - track_kwargs['last_modified'] = int(data['mtime']) - - if 'artist' in data: - artist_kwargs['name'] = data['artist'] - - if 'albumartist' in data: - albumartist_kwargs['name'] = data['albumartist'] - - if 'composer' in data: - track_kwargs['composers'] = [Artist(name=data['composer'])] - - if 'performer' in data: - track_kwargs['performers'] = [Artist(name=data['performer'])] - - if 'album' in data: - album_kwargs['name'] = data['album'] - - if 'title' in data: - track_kwargs['name'] = data['title'] - - if 'genre' in data: - track_kwargs['genre'] = data['genre'] - - if 'date' in data: - track_kwargs['date'] = data['date'] - - if 'comment' in data: - track_kwargs['comment'] = data['comment'] - - if 'musicbrainz_trackid' in data: - track_kwargs['musicbrainz_id'] = data['musicbrainz_trackid'] - - if 'musicbrainz_albumid' in data: - album_kwargs['musicbrainz_id'] = data['musicbrainz_albumid'] - - if 'musicbrainz_artistid' in data: - artist_kwargs['musicbrainz_id'] = data['musicbrainz_artistid'] - - if 'musicbrainz_albumartistid' in data: - albumartist_kwargs['musicbrainz_id'] = ( - data['musicbrainz_albumartistid']) - - if artist_kwargs: - artist = Artist(**artist_kwargs) - track_kwargs['artists'] = [artist] - - if albumartist_kwargs: - albumartist = Artist(**albumartist_kwargs) - album_kwargs['artists'] = [albumartist] - - if album_kwargs: - album = Album(**album_kwargs) - track_kwargs['album'] = album - - if data['file'][0] == '/': - path = data['file'][1:] - else: - path = data['file'] - - track_kwargs['uri'] = 'local:track:%s' % path - track_kwargs['length'] = int(data.get('time', 0)) * 1000 - - track = Track(**track_kwargs) - tracks.add(track) - - -def tracks_to_tag_cache_format(tracks, media_dir): - """ - Format list of tracks for output to MPD tag cache - - :param tracks: the tracks - :type tracks: list of :class:`mopidy.models.Track` - :param media_dir: the path to the music dir - :type media_dir: string - :rtype: list of lists of two-tuples - """ - result = [ - ('info_begin',), - ('mpd_version', protocol.VERSION), - ('fs_charset', protocol.ENCODING), - ('info_end',) - ] - tracks.sort(key=lambda t: t.uri) - dirs, files = tracks_to_directory_tree(tracks, media_dir) - _add_to_tag_cache(result, dirs, files, media_dir) - return result - - -# TODO: bytes only -def _add_to_tag_cache(result, dirs, files, media_dir): - base_path = media_dir.encode('utf-8') - - for path, (entry_dirs, entry_files) in dirs.items(): - try: - text_path = path.decode('utf-8') - except UnicodeDecodeError: - text_path = urllib.quote(path).decode('utf-8') - name = os.path.split(text_path)[1] - result.append(('directory', text_path)) - result.append(('mtime', get_mtime(os.path.join(base_path, path)))) - result.append(('begin', name)) - _add_to_tag_cache(result, entry_dirs, entry_files, media_dir) - result.append(('end', name)) - - result.append(('songList begin',)) - - for track in files: - track_result = dict(mpd.track_to_mpd_format(track)) - - # XXX Don't save comments to the tag cache as they may span multiple - # lines. We'll start saving track comments when we move from tag_cache - # to a JSON file. See #579 for details. - if 'Comment' in track_result: - del track_result['Comment'] - - path = uri_to_path(track_result['file']) - try: - text_path = path.decode('utf-8') - except UnicodeDecodeError: - text_path = urllib.quote(path).decode('utf-8') - relative_path = os.path.relpath(path, base_path) - relative_uri = urllib.quote(relative_path) - - # TODO: use track.last_modified - track_result['file'] = relative_uri - track_result['mtime'] = get_mtime(path) - track_result['key'] = os.path.basename(text_path) - track_result = order_mpd_track_info(track_result.items()) - - result.extend(track_result) - - result.append(('songList end',)) - - -def tracks_to_directory_tree(tracks, media_dir): - directories = ({}, []) - - for track in tracks: - path = b'' - current = directories - - absolute_track_dir_path = os.path.dirname(uri_to_path(track.uri)) - relative_track_dir_path = re.sub( - '^' + re.escape(media_dir), b'', absolute_track_dir_path) - - for part in split_path(relative_track_dir_path): - path = os.path.join(path, part) - if path not in current[0]: - current[0][path] = ({}, []) - current = current[0][path] - current[1].append(track) - return directories - - -MPD_KEY_ORDER = ''' - key file Time Artist Album AlbumArtist Title Track Genre Date Composer - Performer Comment Disc MUSICBRAINZ_ALBUMID MUSICBRAINZ_ALBUMARTISTID - MUSICBRAINZ_ARTISTID MUSICBRAINZ_TRACKID mtime -'''.split() - - -def order_mpd_track_info(result): - """ - Order results from - :func:`mopidy.frontends.mpd.translator.track_to_mpd_format` so that it - matches MPD's ordering. Simply a cosmetic fix for easier diffing of - tag_caches. - - :param result: the track info - :type result: list of tuples - :rtype: list of tuples - """ - return sorted(result, key=lambda i: MPD_KEY_ORDER.index(i[0])) diff --git a/mopidy/config/convert.py b/mopidy/config/convert.py index 3c3edb85..87bf4ed5 100644 --- a/mopidy/config/convert.py +++ b/mopidy/config/convert.py @@ -45,7 +45,6 @@ def convert(settings): helper('local/media_dir', 'LOCAL_MUSIC_PATH') helper('local/playlists_dir', 'LOCAL_PLAYLIST_PATH') - helper('local/tag_cache_file', 'LOCAL_TAG_CACHE_FILE') helper('spotify/username', 'SPOTIFY_USERNAME') helper('spotify/password', 'SPOTIFY_PASSWORD') diff --git a/setup.py b/setup.py index 11855553..bc2fe222 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,6 @@ setup( 'mopidy.ext': [ 'http = mopidy.frontends.http:Extension [http]', 'local = mopidy.backends.local:Extension', - 'local-tagcache = mopidy.backends.local.tagcache:Extension', 'local-json = mopidy.backends.local.json:Extension', 'mpd = mopidy.frontends.mpd:Extension', 'stream = mopidy.backends.stream:Extension', diff --git a/tests/backends/local/events_test.py b/tests/backends/local/events_test.py index 725c580f..1e26a68c 100644 --- a/tests/backends/local/events_test.py +++ b/tests/backends/local/events_test.py @@ -18,7 +18,6 @@ class LocalBackendEventsTest(unittest.TestCase): 'local': { 'media_dir': path_to_data_dir(''), 'playlists_dir': b'', - 'tag_cache_file': path_to_data_dir('empty_tag_cache'), } } diff --git a/tests/backends/local/library_test.py b/tests/backends/local/library_test.py index c04b81f5..e4c00570 100644 --- a/tests/backends/local/library_test.py +++ b/tests/backends/local/library_test.py @@ -1,12 +1,13 @@ from __future__ import unicode_literals +import copy import tempfile import unittest import pykka from mopidy import core -from mopidy.backends.local.tagcache import actor +from mopidy.backends.local.json import actor from mopidy.models import Track, Album, Artist from tests import path_to_data_dir @@ -61,12 +62,14 @@ class LocalLibraryProviderTest(unittest.TestCase): 'local': { 'media_dir': path_to_data_dir(''), 'playlists_dir': b'', - 'tag_cache_file': path_to_data_dir('library_tag_cache'), - } + }, + 'local-json': { + 'json_file': path_to_data_dir('library.json.gz'), + }, } def setUp(self): - self.backend = actor.LocalTagcacheBackend.start( + self.backend = actor.LocalJsonBackend.start( config=self.config, audio=None).proxy() self.core = core.Core(backends=[self.backend]) self.library = self.core.library @@ -85,27 +88,27 @@ class LocalLibraryProviderTest(unittest.TestCase): # Verifies that https://github.com/mopidy/mopidy/issues/500 # has been fixed. - tag_cache = tempfile.NamedTemporaryFile() - with open(self.config['local']['tag_cache_file']) as fh: - tag_cache.write(fh.read()) - tag_cache.flush() + with tempfile.NamedTemporaryFile() as library: + with open(self.config['local-json']['json_file']) as fh: + library.write(fh.read()) + library.flush() - config = {'local': self.config['local'].copy()} - config['local']['tag_cache_file'] = tag_cache.name - backend = actor.LocalTagcacheBackend(config=config, audio=None) + config = copy.deepcopy(self.config) + config['local-json']['json_file'] = library.name + backend = actor.LocalJsonBackend(config=config, audio=None) - # Sanity check that value is in tag cache - result = backend.library.lookup(self.tracks[0].uri) - self.assertEqual(result, self.tracks[0:1]) + # Sanity check that value is in the library + result = backend.library.lookup(self.tracks[0].uri) + self.assertEqual(result, self.tracks[0:1]) - # Clear tag cache and refresh - tag_cache.seek(0) - tag_cache.truncate() - backend.library.refresh() + # Clear library and refresh + library.seek(0) + library.truncate() + backend.library.refresh() - # Now it should be gone. - result = backend.library.lookup(self.tracks[0].uri) - self.assertEqual(result, []) + # Now it should be gone. + result = backend.library.lookup(self.tracks[0].uri) + self.assertEqual(result, []) def test_lookup(self): tracks = self.library.lookup(self.tracks[0].uri) @@ -115,6 +118,7 @@ class LocalLibraryProviderTest(unittest.TestCase): tracks = self.library.lookup('fake uri') self.assertEqual(tracks, []) + # TODO: move to search_test module def test_find_exact_no_hits(self): result = self.library.find_exact(track_name=['unknown track']) self.assertEqual(list(result[0].tracks), []) diff --git a/tests/backends/local/playback_test.py b/tests/backends/local/playback_test.py index 8fbc4415..4c3dd70d 100644 --- a/tests/backends/local/playback_test.py +++ b/tests/backends/local/playback_test.py @@ -23,7 +23,6 @@ class LocalPlaybackProviderTest(unittest.TestCase): 'local': { 'media_dir': path_to_data_dir(''), 'playlists_dir': b'', - 'tag_cache_file': path_to_data_dir('empty_tag_cache'), } } diff --git a/tests/backends/local/playlists_test.py b/tests/backends/local/playlists_test.py index c8fedd62..c02e1d23 100644 --- a/tests/backends/local/playlists_test.py +++ b/tests/backends/local/playlists_test.py @@ -20,7 +20,6 @@ class LocalPlaylistsProviderTest(unittest.TestCase): config = { 'local': { 'media_dir': path_to_data_dir(''), - 'tag_cache_file': path_to_data_dir('library_tag_cache'), } } diff --git a/tests/backends/local/tagcache_test.py b/tests/backends/local/tagcache_test.py deleted file mode 100644 index b40b3346..00000000 --- a/tests/backends/local/tagcache_test.py +++ /dev/null @@ -1,346 +0,0 @@ -# encoding: utf-8 - -from __future__ import unicode_literals - -import os -import unittest - -from mopidy.backends.local.tagcache import translator -from mopidy.frontends.mpd import translator as mpd, protocol -from mopidy.models import Album, Artist, Track -from mopidy.utils.path import mtime, uri_to_path - -from tests import path_to_data_dir - - -class TracksToTagCacheFormatTest(unittest.TestCase): - def setUp(self): - self.media_dir = '/dir/subdir' - mtime.set_fake_time(1234567) - - def tearDown(self): - mtime.undo_fake() - - def translate(self, track): - base_path = self.media_dir.encode('utf-8') - result = dict(mpd.track_to_mpd_format(track)) - result['file'] = uri_to_path(result['file'])[len(base_path) + 1:] - result['key'] = os.path.basename(result['file']) - result['mtime'] = mtime('') - return translator.order_mpd_track_info(result.items()) - - def consume_headers(self, result): - self.assertEqual(('info_begin',), result[0]) - self.assertEqual(('mpd_version', protocol.VERSION), result[1]) - self.assertEqual(('fs_charset', protocol.ENCODING), result[2]) - self.assertEqual(('info_end',), result[3]) - return result[4:] - - def consume_song_list(self, result): - self.assertEqual(('songList begin',), result[0]) - for i, row in enumerate(result): - if row == ('songList end',): - return result[1:i], result[i + 1:] - self.fail("Couldn't find songList end in result") - - def consume_directory(self, result): - self.assertEqual('directory', result[0][0]) - self.assertEqual(('mtime', mtime('.')), result[1]) - self.assertEqual(('begin', os.path.split(result[0][1])[1]), result[2]) - directory = result[2][1] - for i, row in enumerate(result): - if row == ('end', directory): - return result[3:i], result[i + 1:] - self.fail("Couldn't find end %s in result" % directory) - - def test_empty_tag_cache_has_header(self): - result = translator.tracks_to_tag_cache_format([], self.media_dir) - result = self.consume_headers(result) - - def test_empty_tag_cache_has_song_list(self): - result = translator.tracks_to_tag_cache_format([], self.media_dir) - result = self.consume_headers(result) - song_list, result = self.consume_song_list(result) - - self.assertEqual(len(song_list), 0) - self.assertEqual(len(result), 0) - - def test_tag_cache_has_header(self): - track = Track(uri='file:///dir/subdir/song.mp3') - result = translator.tracks_to_tag_cache_format([track], self.media_dir) - result = self.consume_headers(result) - - def test_tag_cache_has_song_list(self): - track = Track(uri='file:///dir/subdir/song.mp3') - result = translator.tracks_to_tag_cache_format([track], self.media_dir) - result = self.consume_headers(result) - song_list, result = self.consume_song_list(result) - - self.assert_(song_list) - self.assertEqual(len(result), 0) - - def test_tag_cache_has_formated_track(self): - track = Track(uri='file:///dir/subdir/song.mp3') - formated = self.translate(track) - result = translator.tracks_to_tag_cache_format([track], self.media_dir) - - result = self.consume_headers(result) - song_list, result = self.consume_song_list(result) - - self.assertEqual(formated, song_list) - self.assertEqual(len(result), 0) - - def test_tag_cache_has_formated_track_with_key_and_mtime(self): - track = Track(uri='file:///dir/subdir/song.mp3') - formated = self.translate(track) - result = translator.tracks_to_tag_cache_format([track], self.media_dir) - - result = self.consume_headers(result) - song_list, result = self.consume_song_list(result) - - self.assertEqual(formated, song_list) - self.assertEqual(len(result), 0) - - def test_tag_cache_supports_directories(self): - track = Track(uri='file:///dir/subdir/folder/song.mp3') - formated = self.translate(track) - result = translator.tracks_to_tag_cache_format([track], self.media_dir) - - result = self.consume_headers(result) - dir_data, result = self.consume_directory(result) - song_list, result = self.consume_song_list(result) - self.assertEqual(len(song_list), 0) - self.assertEqual(len(result), 0) - - song_list, result = self.consume_song_list(dir_data) - self.assertEqual(len(result), 0) - self.assertEqual(formated, song_list) - - def test_tag_cache_diretory_header_is_right(self): - track = Track(uri='file:///dir/subdir/folder/sub/song.mp3') - result = translator.tracks_to_tag_cache_format([track], self.media_dir) - - result = self.consume_headers(result) - dir_data, result = self.consume_directory(result) - - self.assertEqual(('directory', 'folder/sub'), dir_data[0]) - self.assertEqual(('mtime', mtime('.')), dir_data[1]) - self.assertEqual(('begin', 'sub'), dir_data[2]) - - def test_tag_cache_suports_sub_directories(self): - track = Track(uri='file:///dir/subdir/folder/sub/song.mp3') - formated = self.translate(track) - result = translator.tracks_to_tag_cache_format([track], self.media_dir) - - result = self.consume_headers(result) - - dir_data, result = self.consume_directory(result) - song_list, result = self.consume_song_list(result) - self.assertEqual(len(song_list), 0) - self.assertEqual(len(result), 0) - - dir_data, result = self.consume_directory(dir_data) - song_list, result = self.consume_song_list(result) - self.assertEqual(len(result), 0) - self.assertEqual(len(song_list), 0) - - song_list, result = self.consume_song_list(dir_data) - self.assertEqual(len(result), 0) - self.assertEqual(formated, song_list) - - def test_tag_cache_supports_multiple_tracks(self): - tracks = [ - Track(uri='file:///dir/subdir/song1.mp3'), - Track(uri='file:///dir/subdir/song2.mp3'), - ] - - formated = [] - formated.extend(self.translate(tracks[0])) - formated.extend(self.translate(tracks[1])) - - result = translator.tracks_to_tag_cache_format(tracks, self.media_dir) - - result = self.consume_headers(result) - song_list, result = self.consume_song_list(result) - - self.assertEqual(formated, song_list) - self.assertEqual(len(result), 0) - - def test_tag_cache_supports_multiple_tracks_in_dirs(self): - tracks = [ - Track(uri='file:///dir/subdir/song1.mp3'), - Track(uri='file:///dir/subdir/folder/song2.mp3'), - ] - - formated = [] - formated.append(self.translate(tracks[0])) - formated.append(self.translate(tracks[1])) - - result = translator.tracks_to_tag_cache_format(tracks, self.media_dir) - - result = self.consume_headers(result) - dir_data, result = self.consume_directory(result) - song_list, song_result = self.consume_song_list(dir_data) - - self.assertEqual(formated[1], song_list) - self.assertEqual(len(song_result), 0) - - song_list, result = self.consume_song_list(result) - self.assertEqual(len(result), 0) - self.assertEqual(formated[0], song_list) - - -class TracksToDirectoryTreeTest(unittest.TestCase): - def setUp(self): - self.media_dir = '/root' - - def test_no_tracks_gives_emtpy_tree(self): - tree = translator.tracks_to_directory_tree([], self.media_dir) - self.assertEqual(tree, ({}, [])) - - def test_top_level_files(self): - tracks = [ - Track(uri='file:///root/file1.mp3'), - Track(uri='file:///root/file2.mp3'), - Track(uri='file:///root/file3.mp3'), - ] - tree = translator.tracks_to_directory_tree(tracks, self.media_dir) - self.assertEqual(tree, ({}, tracks)) - - def test_single_file_in_subdir(self): - tracks = [Track(uri='file:///root/dir/file1.mp3')] - tree = translator.tracks_to_directory_tree(tracks, self.media_dir) - expected = ({'dir': ({}, tracks)}, []) - self.assertEqual(tree, expected) - - def test_single_file_in_sub_subdir(self): - tracks = [Track(uri='file:///root/dir1/dir2/file1.mp3')] - tree = translator.tracks_to_directory_tree(tracks, self.media_dir) - expected = ({'dir1': ({'dir1/dir2': ({}, tracks)}, [])}, []) - self.assertEqual(tree, expected) - - def test_complex_file_structure(self): - tracks = [ - Track(uri='file:///root/file1.mp3'), - Track(uri='file:///root/dir1/file2.mp3'), - Track(uri='file:///root/dir1/file3.mp3'), - Track(uri='file:///root/dir2/file4.mp3'), - Track(uri='file:///root/dir2/sub/file5.mp3'), - ] - tree = translator.tracks_to_directory_tree(tracks, self.media_dir) - expected = ( - { - 'dir1': ({}, [tracks[1], tracks[2]]), - 'dir2': ( - { - 'dir2/sub': ({}, [tracks[4]]) - }, - [tracks[3]] - ), - }, - [tracks[0]] - ) - self.assertEqual(tree, expected) - - -expected_artists = [Artist(name='name')] -expected_albums = [ - Album(name='albumname', artists=expected_artists, num_tracks=2), - Album(name='albumname', num_tracks=2), -] -expected_tracks = [] - - -def generate_track(path, ident, album_id): - uri = 'local:track:%s' % path - track = Track( - uri=uri, name='trackname', artists=expected_artists, - album=expected_albums[album_id], track_no=1, date='2006', length=4000, - last_modified=1272319626) - expected_tracks.append(track) - - -generate_track('song1.mp3', 6, 0) -generate_track('song2.mp3', 7, 0) -generate_track('song3.mp3', 8, 1) -generate_track('subdir1/song4.mp3', 2, 0) -generate_track('subdir1/song5.mp3', 3, 0) -generate_track('subdir2/song6.mp3', 4, 1) -generate_track('subdir2/song7.mp3', 5, 1) -generate_track('subdir1/subsubdir/song8.mp3', 0, 0) -generate_track('subdir1/subsubdir/song9.mp3', 1, 1) - - -class MPDTagCacheToTracksTest(unittest.TestCase): - def test_emtpy_cache(self): - tracks = translator.parse_mpd_tag_cache( - path_to_data_dir('empty_tag_cache'), path_to_data_dir('')) - self.assertEqual(set(), tracks) - - def test_simple_cache(self): - tracks = translator.parse_mpd_tag_cache( - path_to_data_dir('simple_tag_cache'), path_to_data_dir('')) - track = Track( - uri='local:track:song1.mp3', name='trackname', - artists=expected_artists, track_no=1, album=expected_albums[0], - date='2006', length=4000, last_modified=1272319626) - self.assertEqual(set([track]), tracks) - - def test_advanced_cache(self): - tracks = translator.parse_mpd_tag_cache( - path_to_data_dir('advanced_tag_cache'), path_to_data_dir('')) - self.assertEqual(set(expected_tracks), tracks) - - def test_unicode_cache(self): - tracks = translator.parse_mpd_tag_cache( - path_to_data_dir('utf8_tag_cache'), path_to_data_dir('')) - - artists = [Artist(name='æøå')] - album = Album(name='æøå', artists=artists) - track = Track( - uri='local:track:song1.mp3', name='æøå', artists=artists, - composers=artists, performers=artists, genre='æøå', - album=album, length=4000, last_modified=1272319626, - comment='æøå&^`ൂ㔶') - - self.assertEqual(track, list(tracks)[0]) - - @unittest.SkipTest - def test_misencoded_cache(self): - # FIXME not sure if this can happen - pass - - def test_cache_with_blank_track_info(self): - tracks = translator.parse_mpd_tag_cache( - path_to_data_dir('blank_tag_cache'), path_to_data_dir('')) - expected = Track( - uri='local:track:song1.mp3', length=4000, last_modified=1272319626) - self.assertEqual(set([expected]), tracks) - - def test_musicbrainz_tagcache(self): - tracks = translator.parse_mpd_tag_cache( - path_to_data_dir('musicbrainz_tag_cache'), path_to_data_dir('')) - artist = list(expected_tracks[0].artists)[0].copy( - musicbrainz_id='7364dea6-ca9a-48e3-be01-b44ad0d19897') - albumartist = list(expected_tracks[0].artists)[0].copy( - name='albumartistname', - musicbrainz_id='7364dea6-ca9a-48e3-be01-b44ad0d19897') - album = expected_tracks[0].album.copy( - artists=[albumartist], - musicbrainz_id='cb5f1603-d314-4c9c-91e5-e295cfb125d2') - track = expected_tracks[0].copy( - artists=[artist], album=album, - musicbrainz_id='90488461-8c1f-4a4e-826b-4c6dc70801f0') - - self.assertEqual(track, list(tracks)[0]) - - def test_albumartist_tag_cache(self): - tracks = translator.parse_mpd_tag_cache( - path_to_data_dir('albumartist_tag_cache'), path_to_data_dir('')) - artist = Artist(name='albumartistname') - album = expected_albums[0].copy(artists=[artist]) - track = Track( - uri='local:track:song1.mp3', name='trackname', - artists=expected_artists, track_no=1, album=album, date='2006', - length=4000, last_modified=1272319626) - self.assertEqual(track, list(tracks)[0]) diff --git a/tests/backends/local/tracklist_test.py b/tests/backends/local/tracklist_test.py index ac135a25..c7cfe51f 100644 --- a/tests/backends/local/tracklist_test.py +++ b/tests/backends/local/tracklist_test.py @@ -19,7 +19,6 @@ class LocalTracklistProviderTest(unittest.TestCase): 'local': { 'media_dir': path_to_data_dir(''), 'playlists_dir': b'', - 'tag_cache_file': path_to_data_dir('empty_tag_cache'), } } tracks = [ diff --git a/tests/data/advanced_tag_cache b/tests/data/advanced_tag_cache deleted file mode 100644 index be299fb6..00000000 --- a/tests/data/advanced_tag_cache +++ /dev/null @@ -1,107 +0,0 @@ -info_begin -mpd_version: 0.14.2 -fs_charset: UTF-8 -info_end -directory: subdir1 -begin: subdir1 -directory: subsubdir -begin: subdir1/subsubdir -songList begin -key: song8.mp3 -file: subdir1/subsubdir/song8.mp3 -Time: 4 -Artist: name -AlbumArtist: name -Title: trackname -Album: albumname -Track: 1/2 -Date: 2006 -mtime: 1272319626 -key: song9.mp3 -file: subdir1/subsubdir/song9.mp3 -Time: 4 -Artist: name -Title: trackname -Album: albumname -Track: 1/2 -Date: 2006 -mtime: 1272319626 -songList end -end: subdir1/subsubdir -songList begin -key: song4.mp3 -file: subdir1/song4.mp3 -Time: 4 -Artist: name -AlbumArtist: name -Title: trackname -Album: albumname -Track: 1/2 -Date: 2006 -mtime: 1272319626 -key: song5.mp3 -file: subdir1/song5.mp3 -Time: 4 -Artist: name -AlbumArtist: name -Title: trackname -Album: albumname -Track: 1/2 -Date: 2006 -mtime: 1272319626 -songList end -end: subdir1 -directory: subdir2 -begin: subdir2 -songList begin -key: song6.mp3 -file: subdir2/song6.mp3 -Time: 4 -Artist: name -Title: trackname -Album: albumname -Track: 1/2 -Date: 2006 -mtime: 1272319626 -key: song7.mp3 -file: subdir2/song7.mp3 -Time: 4 -Artist: name -Title: trackname -Album: albumname -Track: 1/2 -Date: 2006 -mtime: 1272319626 -songList end -end: subdir2 -songList begin -key: song1.mp3 -file: /song1.mp3 -Time: 4 -Artist: name -AlbumArtist: name -Title: trackname -Album: albumname -Track: 1/2 -Date: 2006 -mtime: 1272319626 -key: song2.mp3 -file: /song2.mp3 -Time: 4 -Artist: name -AlbumArtist: name -Title: trackname -Album: albumname -Track: 1/2 -Date: 2006 -mtime: 1272319626 -key: song3.mp3 -file: /song3.mp3 -Time: 4 -Artist: name -Title: trackname -Album: albumname -Track: 1/2 -Date: 2006 -mtime: 1272319626 -songList end diff --git a/tests/data/albumartist_tag_cache b/tests/data/albumartist_tag_cache deleted file mode 100644 index 29942a75..00000000 --- a/tests/data/albumartist_tag_cache +++ /dev/null @@ -1,16 +0,0 @@ -info_begin -mpd_version: 0.14.2 -fs_charset: UTF-8 -info_end -songList begin -key: song1.mp3 -file: /song1.mp3 -Time: 4 -Artist: name -Title: trackname -Album: albumname -AlbumArtist: albumartistname -Track: 1/2 -Date: 2006 -mtime: 1272319626 -songList end diff --git a/tests/data/blank_tag_cache b/tests/data/blank_tag_cache deleted file mode 100644 index a6d33386..00000000 --- a/tests/data/blank_tag_cache +++ /dev/null @@ -1,10 +0,0 @@ -info_begin -mpd_version: 0.14.2 -fs_charset: UTF-8 -info_end -songList begin -key: song1.mp3 -file: /song1.mp3 -Time: 4 -mtime: 1272319626 -songList end diff --git a/tests/data/empty_tag_cache b/tests/data/empty_tag_cache deleted file mode 100644 index 84053d90..00000000 --- a/tests/data/empty_tag_cache +++ /dev/null @@ -1,6 +0,0 @@ -info_begin -mpd_version: 0.14.2 -fs_charset: UTF-8 -info_end -songList begin -songList end diff --git a/tests/data/library.json.gz b/tests/data/library.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..07cd48d128bb4cffcdf379a74ce7bf66469830d7 GIT binary patch literal 394 zcmV;50d@W#iwFoGcb`%M|7>Yua$$0LE^2dcZUDuVOKyWO5Qg`h!t%O_{2)}yu6O9J zijYg3fC@G;PB$obuW^DQY6#F^6(KbIGak>($DA zPL__6r8CG2`X+D;?QDqv0q4oqlP=_~>Ic0$iR2d>mV;0Q-?gm-X6 zXn;rPjR&2`BOK&W%8IMy3fep>=>LFjMuBd|-mfDU$|kf1_VMX@ro*VyORf%56-#1` o9$_7rXf$u4?av^%riS09flP`f0Il)s8o}WF0}y$t#qtRN0Om2m9{>OV literal 0 HcmV?d00001 diff --git a/tests/data/library_tag_cache b/tests/data/library_tag_cache deleted file mode 100644 index 6d00cf97..00000000 --- a/tests/data/library_tag_cache +++ /dev/null @@ -1,56 +0,0 @@ -info_begin -mpd_version: 0.14.2 -fs_charset: UTF-8 -info_end -songList begin -key: key1 -file: /path1 -Artist: artist1 -AlbumArtist: artist1 -Title: track1 -Album: album1 -Date: 2001-02-03 -Track: 1 -Time: 4 -key: key2 -file: /path2 -Artist: artist2 -AlbumArtist: artist2 -Title: track2 -Album: album2 -Date: 2002 -Track: 2 -Time: 4 -key: key3 -file: /path3 -Artist: artist4 -AlbumArtist: artist3 -Title: track3 -Album: album3 -Date: 2003 -Track: 3 -Time: 4 -key: key4 -file: /path4 -Artist: artist3 -Title: track4 -Album: album4 -Date: 2004 -Track: 4 -Comment: This is a fantastic track -Time: 60 -key: key5 -file: /path5 -Composer: artist5 -Title: track5 -Album: album4 -Genre: genre1 -Time: 4 -key: key6 -file: /path6 -Performer: artist6 -Title: track6 -Album: album4 -Genre: genre2 -Time: 4 -songList end diff --git a/tests/data/musicbrainz_tag_cache b/tests/data/musicbrainz_tag_cache deleted file mode 100644 index 0e9dca46..00000000 --- a/tests/data/musicbrainz_tag_cache +++ /dev/null @@ -1,20 +0,0 @@ -info_begin -mpd_version: 0.16.0 -fs_charset: UTF-8 -info_end -songList begin -key: song1.mp3 -file: /song1.mp3 -Time: 4 -Artist: name -Title: trackname -Album: albumname -AlbumArtist: albumartistname -Track: 1/2 -Date: 2006 -MUSICBRAINZ_ALBUMID: cb5f1603-d314-4c9c-91e5-e295cfb125d2 -MUSICBRAINZ_ALBUMARTISTID: 7364dea6-ca9a-48e3-be01-b44ad0d19897 -MUSICBRAINZ_ARTISTID: 7364dea6-ca9a-48e3-be01-b44ad0d19897 -MUSICBRAINZ_TRACKID: 90488461-8c1f-4a4e-826b-4c6dc70801f0 -mtime: 1272319626 -songList end diff --git a/tests/data/scanner/advanced_cache b/tests/data/scanner/advanced_cache deleted file mode 100644 index 60f7fca6..00000000 --- a/tests/data/scanner/advanced_cache +++ /dev/null @@ -1,81 +0,0 @@ -info_begin -mpd_version: 0.15.4 -fs_charset: UTF-8 -info_end -directory: subdir1 -mtime: 1288121499 -begin: subdir1 -songList begin -key: song4.mp3 -file: subdir1/song4.mp3 -Time: 5 -Artist: name -Title: trackname -Album: albumname -Track: 01/02 -Date: 2006 -mtime: 1288121370 -key: song5.mp3 -file: subdir1/song5.mp3 -Time: 5 -Artist: name -Title: trackname -Album: albumname -Track: 01/02 -Date: 2006 -mtime: 1288121370 -songList end -end: subdir1 -directory: subdir2 -mtime: 1288121499 -begin: subdir2 -songList begin -key: song6.mp3 -file: subdir2/song6.mp3 -Time: 5 -Artist: name -Title: trackname -Album: albumname -Track: 01/02 -Date: 2006 -mtime: 1288121370 -key: song7.mp3 -file: subdir2/song7.mp3 -Time: 5 -Artist: name -Title: trackname -Album: albumname -Track: 01/02 -Date: 2006 -mtime: 1288121370 -songList end -end: subdir2 -songList begin -key: song1.mp3 -file: /song1.mp3 -Time: 5 -Artist: name -Title: trackname -Album: albumname -Track: 01/02 -Date: 2006 -mtime: 1288121370 -key: song2.mp3 -file: /song2.mp3 -Time: 5 -Artist: name -Title: trackname -Album: albumname -Track: 01/02 -Date: 2006 -mtime: 1288121370 -key: song3.mp3 -file: /song3.mp3 -Time: 5 -Artist: name -Title: trackname -Album: albumname -Track: 01/02 -Date: 2006 -mtime: 1288121370 -songList end diff --git a/tests/data/scanner/empty_cache b/tests/data/scanner/empty_cache deleted file mode 100644 index 3c466a32..00000000 --- a/tests/data/scanner/empty_cache +++ /dev/null @@ -1,6 +0,0 @@ -info_begin -mpd_version: 0.15.4 -fs_charset: UTF-8 -info_end -songList begin -songList end diff --git a/tests/data/scanner/simple_cache b/tests/data/scanner/simple_cache deleted file mode 100644 index db11c324..00000000 --- a/tests/data/scanner/simple_cache +++ /dev/null @@ -1,15 +0,0 @@ -info_begin -mpd_version: 0.15.4 -fs_charset: UTF-8 -info_end -songList begin -key: song1.mp3 -file: /song1.mp3 -Time: 5 -Artist: name -Title: trackname -Album: albumname -Track: 01/02 -Date: 2006 -mtime: 1288121370 -songList end diff --git a/tests/data/simple_tag_cache b/tests/data/simple_tag_cache deleted file mode 100644 index 07a474b3..00000000 --- a/tests/data/simple_tag_cache +++ /dev/null @@ -1,16 +0,0 @@ -info_begin -mpd_version: 0.14.2 -fs_charset: UTF-8 -info_end -songList begin -key: song1.mp3 -file: /song1.mp3 -Time: 4 -Artist: name -AlbumArtist: name -Title: trackname -Album: albumname -Track: 1/2 -Date: 2006 -mtime: 1272319626 -songList end diff --git a/tests/data/utf8_tag_cache b/tests/data/utf8_tag_cache deleted file mode 100644 index 83fbcad4..00000000 --- a/tests/data/utf8_tag_cache +++ /dev/null @@ -1,18 +0,0 @@ -info_begin -mpd_version: 0.14.2 -fs_charset: UTF-8 -info_end -songList begin -key: song1.mp3 -file: /song1.mp3 -Time: 4 -Artist: æøå -AlbumArtist: æøå -Composer: æøå -Performer: æøå -Title: æøå -Album: æøå -Genre: æøå -Comment: æøå&^`ൂ㔶 -mtime: 1272319626 -songList end From ad53a067aef04f7fb7e9ea4f744f03d26c94a0a2 Mon Sep 17 00:00:00 2001 From: Thomas Adamcik Date: Wed, 4 Dec 2013 21:33:04 +0100 Subject: [PATCH 15/19] local: Split out library reading and writting - Create $XDG_DATA_DIR/mopidy/local in the local extension's validate env. - Make sure we handle bad data causing ValueError in JSON decoding - Initializing empty file causes more harm than good as it just leads to a ValueError. Switched to doing write_library(json_file, {}) - Helpers have been updated to be library oriented, not track. This paves the way for having {tracks: {uri: ...}, artist: {uri: ...}, ...} type denormalized data. --- mopidy/backends/local/__init__.py | 10 +++++- mopidy/backends/local/json/actor.py | 22 ++++++------ mopidy/backends/local/json/library.py | 49 ++++++++++++++------------- 3 files changed, 46 insertions(+), 35 deletions(-) diff --git a/mopidy/backends/local/__init__.py b/mopidy/backends/local/__init__.py index d24ab010..dedc868c 100644 --- a/mopidy/backends/local/__init__.py +++ b/mopidy/backends/local/__init__.py @@ -1,9 +1,13 @@ from __future__ import unicode_literals +import logging import os import mopidy from mopidy import config, ext +from mopidy.utils import encoding, path + +logger = logging.getLogger('mopidy.backends.local') class Extension(ext.Extension): @@ -27,7 +31,11 @@ class Extension(ext.Extension): return schema def validate_environment(self): - pass + try: + path.get_or_create_dir(b'$XDG_DATA_DIR/mopidy/local') + except EnvironmentError as error: + error = encoding.locale_decode(error) + logger.warning('Could not create local data dir: %s', error) def get_backend_classes(self): from .actor import LocalBackend diff --git a/mopidy/backends/local/json/actor.py b/mopidy/backends/local/json/actor.py index df9ac447..66a6fbd5 100644 --- a/mopidy/backends/local/json/actor.py +++ b/mopidy/backends/local/json/actor.py @@ -1,13 +1,14 @@ from __future__ import unicode_literals import logging +import os import pykka from mopidy.backends import base -from mopidy.utils import encoding, path +from mopidy.utils import encoding -from .library import LocalJsonLibraryProvider +from . import library logger = logging.getLogger('mopidy.backends.local.json') @@ -17,14 +18,13 @@ class LocalJsonBackend(pykka.ThreadingActor, base.Backend): super(LocalJsonBackend, self).__init__() self.config = config - self.check_dirs_and_files() - self.library = LocalJsonLibraryProvider(backend=self) + self.library = library.LocalJsonLibraryProvider(backend=self) self.uri_schemes = ['local'] - def check_dirs_and_files(self): - try: - path.get_or_create_file(self.config['local-json']['json_file']) - except EnvironmentError as error: - logger.warning( - 'Could not create empty json file: %s', - encoding.locale_decode(error)) + if not os.path.exists(config['local-json']['json_file']): + try: + library.write_library(config['local-json']['json_file'], {}) + logger.info('Created empty local JSON library.') + except EnvironmentError as error: + error = encoding.locale_decode(error) + logger.warning('Could not create local library: %s', error) diff --git a/mopidy/backends/local/json/library.py b/mopidy/backends/local/json/library.py index 6bfef783..33427231 100644 --- a/mopidy/backends/local/json/library.py +++ b/mopidy/backends/local/json/library.py @@ -14,13 +14,31 @@ from mopidy.backends.local import search logger = logging.getLogger('mopidy.backends.local.json') -def _load_tracks(json_file): +def load_library(json_file): try: with gzip.open(json_file, 'rb') as fp: - result = json.load(fp, object_hook=models.model_json_decoder) - except IOError: - return [] - return result.get('tracks', []) + return json.load(fp, object_hook=models.model_json_decoder) + except (IOError, ValueError) as e: + logger.warning('Loading JSON local library failed: %s', e) + return {} + + +def write_library(json_file, data): + data['version'] = mopidy.__version__ + directory, basename = os.path.split(json_file) + + # TODO: cleanup directory/basename.* files. + tmp = tempfile.NamedTemporaryFile( + prefix=basename + '.', dir=directory, delete=False) + + try: + with gzip.GzipFile(fileobj=tmp, mode='wb') as fp: + json.dump(data, fp, cls=models.ModelJSONEncoder, + indent=2, separators=(',', ': ')) + os.rename(tmp.name, json_file) + finally: + if os.path.exists(tmp.name): + os.remove(tmp.name) class LocalJsonLibraryProvider(base.BaseLibraryProvider): @@ -36,7 +54,7 @@ class LocalJsonLibraryProvider(base.BaseLibraryProvider): 'Loading local tracks from %s using %s', self._media_dir, self._json_file) - tracks = _load_tracks(self._json_file) + tracks = load_library(self._json_file).get('tracks', []) uris_to_remove = set(self._uri_mapping) for track in tracks: @@ -75,7 +93,7 @@ class LocalJsonLibraryUpdateProvider(base.BaseLibraryProvider): self._json_file = config['local-json']['json_file'] def load(self): - for track in _load_tracks(self._json_file): + for track in load_library(self._json_file).get('tracks', []): self._tracks[track.uri] = track return self._tracks.values() @@ -87,19 +105,4 @@ class LocalJsonLibraryUpdateProvider(base.BaseLibraryProvider): del self._tracks[uri] def commit(self): - directory, basename = os.path.split(self._json_file) - - # TODO: cleanup directory/basename.* files. - tmp = tempfile.NamedTemporaryFile( - prefix=basename + '.', dir=directory, delete=False) - - try: - with gzip.GzipFile(fileobj=tmp, mode='wb') as fp: - data = {'version': mopidy.__version__, - 'tracks': self._tracks.values()} - json.dump(data, fp, cls=models.ModelJSONEncoder, - indent=2, separators=(',', ': ')) - os.rename(tmp.name, self._json_file) - finally: - if os.path.exists(tmp.name): - os.remove(tmp.name) + write_library(self._json_file, {'tracks': self._tracks.values()}) From 4f7176cac859d5e532031e0aa6c34d23b02ea026 Mon Sep 17 00:00:00 2001 From: Thomas Adamcik Date: Wed, 4 Dec 2013 22:48:37 +0100 Subject: [PATCH 16/19] local: Cleanup uri conversion helper naming and implementation. --- mopidy/backends/local/playback.py | 5 ++--- mopidy/backends/local/translator.py | 8 ++------ 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/mopidy/backends/local/playback.py b/mopidy/backends/local/playback.py index b264dac7..ae8eeb82 100644 --- a/mopidy/backends/local/playback.py +++ b/mopidy/backends/local/playback.py @@ -11,7 +11,6 @@ logger = logging.getLogger('mopidy.backends.local') class LocalPlaybackProvider(base.BasePlaybackProvider): def change_track(self, track): - media_dir = self.backend.config['local']['media_dir'] - uri = translator.local_to_file_uri(track.uri, media_dir) - track = track.copy(uri=uri) + track = track.copy(uri=translator.local_track_uri_to_file_uri( + track.uri, self.backend.config['local']['media_dir'])) return super(LocalPlaybackProvider, self).change_track(track) diff --git a/mopidy/backends/local/translator.py b/mopidy/backends/local/translator.py index 1153b1b3..2c0523e8 100644 --- a/mopidy/backends/local/translator.py +++ b/mopidy/backends/local/translator.py @@ -11,12 +11,8 @@ from mopidy.utils.path import path_to_uri, uri_to_path logger = logging.getLogger('mopidy.backends.local') -# TODO: remove once tag cache is gone -def local_to_file_uri(uri, media_dir): - # TODO: check that type is correct. - file_path = uri_to_path(uri).split(b':', 1)[1] - file_path = os.path.join(media_dir, file_path) - return path_to_uri(file_path) +def local_track_uri_to_file_uri(uri, media_dir): + return path_to_uri(local_track_uri_to_path(uri, media_dir)) def local_track_uri_to_path(uri, media_dir): From 9794826f267ea4084208a66d87ed2117451e5c17 Mon Sep 17 00:00:00 2001 From: Thomas Adamcik Date: Wed, 4 Dec 2013 22:52:24 +0100 Subject: [PATCH 17/19] local: Review changes --- docs/ext/local.rst | 10 +++++----- docs/extensiondev.rst | 13 +------------ mopidy/backends/local/translator.py | 4 ++-- 3 files changed, 8 insertions(+), 19 deletions(-) diff --git a/docs/ext/local.rst b/docs/ext/local.rst index 51268c51..9e7c645c 100644 --- a/docs/ext/local.rst +++ b/docs/ext/local.rst @@ -85,11 +85,11 @@ To make a local library for your music available for Mopidy: #. Start Mopidy, find the music library in a client, and play some local music! -Plugable library support ------------------------- +Pluggable library support +------------------------- Local libraries are fully pluggable. What this means is that users may opt to -disable the current default library `local-json`, replacing it with a third +disable the current default library ``local-json``, replacing it with a third party one. When running :command:`mopidy local scan` mopidy will populate whatever the current active library is with data. Only one library may be active at a time. @@ -115,8 +115,8 @@ Configuration values .. confval:: local-json/enabled - If the local extension should be enabled or not. + If the local-json extension should be enabled or not. .. confval:: local-json/json_file - Path to a file to store the gziped json data in. + Path to a file to store the GZiped JSON data in. diff --git a/docs/extensiondev.rst b/docs/extensiondev.rst index 2709690a..82144d0a 100644 --- a/docs/extensiondev.rst +++ b/docs/extensiondev.rst @@ -428,18 +428,7 @@ solution you happen to prefer. super(SoundspotLibraryUpdateProvider, self).__init__(config) self.config = config - def load(self): - # Your track loading code - return tracks - - def add(self, track): - # Your code for handling adding a new track - - def remove(self, uri): - # Your code for removing the track coresponding to this uri - - def commit(self): - # Your code to persist the library, if needed. + # Your library provider implementation here. Example GStreamer element diff --git a/mopidy/backends/local/translator.py b/mopidy/backends/local/translator.py index 2c0523e8..243eb314 100644 --- a/mopidy/backends/local/translator.py +++ b/mopidy/backends/local/translator.py @@ -17,13 +17,13 @@ def local_track_uri_to_file_uri(uri, media_dir): def local_track_uri_to_path(uri, media_dir): if not uri.startswith('local:track:'): - raise ValueError('Invalid uri.') + raise ValueError('Invalid URI.') file_path = uri_to_path(uri).split(b':', 1)[1] return os.path.join(media_dir, file_path) def path_to_local_track_uri(relpath): - """Convert path releative to media_dir to local track uri""" + """Convert path releative to media_dir to local track URI.""" if isinstance(relpath, unicode): relpath = relpath.encode('utf-8') return b'local:track:%s' % urllib.quote(relpath) From acdc65e9c7a87ac3079ecef80b38e7778692aa50 Mon Sep 17 00:00:00 2001 From: Thomas Adamcik Date: Wed, 4 Dec 2013 23:04:30 +0100 Subject: [PATCH 18/19] docs: Update changelog with pluggable libraries. --- docs/changelog.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index acb94e3d..92ec3707 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,23 @@ Changelog This changelog is used to track all major changes to Mopidy. +v0.18.0 (Unreleased) +==================== + +**Pluggable libraries** + +Fixes issues :issue:`44`, partially resolves :issue:`397`, and causes +a temporary regression of :issue:`527`. + +- Finished the work on creating pluggable libraries. Users can now + reconfigure mopidy to use alternate library providers of their choosing + for local files. +- Switched default library provider to JSON. This greatly simplifies our + library code and reuses or existing serialisation code. +- Killed our outdated and bug-ridden tag cache implementation. +- Added support for deprecated config values in order to allow for + graceful removal of :confval:`local/tag_cache_file` + v0.17.0 (2013-11-23) ==================== From 4a599eec0ca76e8f801083d6469aff89aae32feb Mon Sep 17 00:00:00 2001 From: Stein Magnus Jodal Date: Wed, 4 Dec 2013 23:30:01 +0100 Subject: [PATCH 19/19] docs: Minor tweaks --- docs/changelog.rst | 22 +++++++++++----------- docs/ext/local.rst | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 8b545669..395e968b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,26 +4,26 @@ Changelog This changelog is used to track all major changes to Mopidy. -v0.18.0 (Unreleased) +v0.18.0 (UNRELEASED) ==================== -**Pluggable libraries** +**Pluggable local libraries** Fixes issues :issue:`44`, partially resolves :issue:`397`, and causes a temporary regression of :issue:`527`. - Finished the work on creating pluggable libraries. Users can now - reconfigure mopidy to use alternate library providers of their choosing + reconfigure Mopidy to use alternate library providers of their choosing for local files. -- Switched default library provider to JSON. This greatly simplifies our - library code and reuses or existing serialisation code. -- Killed our outdated and bug-ridden tag cache implementation. + +- Switched default local library provider from "tag cache" to JSON. This + greatly simplifies our library code and reuses our existing serialization + code. + +- Killed our outdated and bug-ridden "tag cache" implementation. + - Added support for deprecated config values in order to allow for - graceful removal of :confval:`local/tag_cache_file` - - -v0.18.0 (UNRELEASED) -==================== + graceful removal of :confval:`local/tag_cache_file`. **Internal changes** diff --git a/docs/ext/local.rst b/docs/ext/local.rst index 43b405e1..cbde826f 100644 --- a/docs/ext/local.rst +++ b/docs/ext/local.rst @@ -120,4 +120,4 @@ Configuration values .. confval:: local-json/json_file - Path to a file to store the GZiped JSON data in. + Path to a file to store the gzipped JSON data in.