Merge pull request #454 from adamcik/feature/library-updater

Add library update providers to backends

mopidy/backends/base.py

@@ -15,6 +15,11 @@ class Backend(object):
     #: the backend doesn't provide a library.
     library = None
 
+    #: The library update provider. An instance of
+    #: :class:`~mopidy.backends.base.BaseLibraryUpdateProvider`, or
+    #: :class:`None` if the backend doesn't provide a library.
+    updater = None
+
     #: The playback provider. An instance of
     #: :class:`~mopidy.backends.base.BasePlaybackProvider`, or :class:`None` if
     #: the backend doesn't provide playback.
@@ -35,6 +40,9 @@ class Backend(object):
     def has_library(self):
         return self.library is not None
 
+    def has_updater(self):
+        return self.updater is not None
+
     def has_playback(self):
         return self.playback is not None
 
@@ -53,6 +61,7 @@ class BaseLibraryProvider(object):
     def __init__(self, backend):
         self.backend = backend
 
+    # TODO: replace with search(query, exact=True, ...)
     def find_exact(self, query=None, uris=None):
         """
         See :meth:`mopidy.core.LibraryController.find_exact`.
@@ -86,6 +95,48 @@ class BaseLibraryProvider(object):
         pass
 
 
+class BaseLibraryUpdateProvider(object):
+    """
+    :param backend: backend the controller is a part of
+    :type backend: :class:`mopidy.backends.base.Backend`
+    """
+
+    pykka_traversable = True
+
+    def __init__(self, backend):
+        self.backend = backend
+
+    def load(self):
+        """Loads the library and returns all tracks in it.
+
+        *MUST be implemented by subclass.*
+        """
+        raise NotImplementedError
+
+    def add(self, track):
+        """Adds given track to library.
+
+        Overwrites any existing track with same URI.
+
+        *MUST be implemented by subclass.*
+        """
+        raise NotImplementedError
+
+    def remove(self, uri):
+        """Removes given track from library.
+
+        *MUST be implemented by subclass.*
+        """
+        raise NotImplementedError
+
+    def commit(self):
+        """Persist changes to library.
+
+        *MAY be implemented by subclass.*
+        """
+        pass
+
+
 class BasePlaybackProvider(object):
     """
     :param audio: the audio actor

mopidy/backends/local/actor.py

@@ -8,7 +8,7 @@ import pykka
 from mopidy.backends import base
 from mopidy.utils import encoding, path
 
-from .library import LocalLibraryProvider
+from .library import LocalLibraryProvider, LocalLibraryUpdateProvider
 from .playlists import LocalPlaylistsProvider
 
 logger = logging.getLogger('mopidy.backends.local')
@@ -23,6 +23,7 @@ class LocalBackend(pykka.ThreadingActor, base.Backend):
         self.check_dirs_and_files()
 
         self.library = LocalLibraryProvider(backend=self)
+        self.updater = LocalLibraryUpdateProvider(backend=self)
         self.playback = base.BasePlaybackProvider(audio=audio, backend=self)
         self.playlists = LocalPlaylistsProvider(backend=self)
 

mopidy/backends/local/library.py

@@ -1,7 +1,11 @@
 from __future__ import unicode_literals
 
 import logging
+import os
+import tempfile
+
 from mopidy.backends import base
+from mopidy.frontends.mpd import translator as mpd_translator
 from mopidy.models import Album, SearchResult
 
 from .translator import parse_mpd_tag_cache
@@ -127,3 +131,43 @@ class LocalLibraryProvider(base.BaseLibraryProvider):
             for value in values:
                 if not value:
                     raise LookupError('Missing query')
+
+
+# TODO: rename and move to tagcache extension.
+class LocalLibraryUpdateProvider(base.BaseLibraryProvider):
+    def __init__(self, *args, **kwargs):
+        super(LocalLibraryUpdateProvider, self).__init__(*args, **kwargs)
+        self._tracks = {}
+        self._media_dir = self.backend.config['local']['media_dir']
+        self._tag_cache_file = self.backend.config['local']['tag_cache_file']
+
+    def load(self):
+        tracks = parse_mpd_tag_cache(self._tag_cache_file, self._media_dir)
+        for track in tracks:
+            self._tracks[track.uri] = track
+        return tracks
+
+    def add(self, track):
+        self._tracks[track.uri] = track
+
+    def remove(self, uri):
+        if uri in self._tracks:
+            del self._tracks[uri]
+
+    def commit(self):
+        directory, basename = os.path.split(self._tag_cache_file)
+        tmp = tempfile.NamedTemporaryFile(
+            prefix=basename + '.', dir=directory, delete=False)
+
+        try:
+            for row in mpd_translator.tracks_to_tag_cache_format(
+                    self._tracks.values(), self._media_dir):
+                if len(row) == 1:
+                    tmp.write(('%s\n' % row).encode('utf-8'))
+                else:
+                    tmp.write(('%s: %s\n' % row).encode('utf-8'))
+
+            os.rename(tmp.name, self._tag_cache_file)
+        finally:
+            if os.path.exists(tmp.name):
+                os.remove(tmp.name)

mopidy/frontends/mpd/translator.py

@@ -10,6 +10,8 @@ from mopidy.frontends.mpd.exceptions import MpdArgError
 from mopidy.models import TlTrack
 from mopidy.utils.path import mtime as get_mtime, uri_to_path, split_path
 
+# TODO: special handling of local:// uri scheme
+
 
 def track_to_mpd_format(track, position=None):
     """
@@ -213,6 +215,7 @@ def query_from_mpd_search_format(mpd_query):
     return query
 
 
+# TODO: move to tagcache backend.
 def tracks_to_tag_cache_format(tracks, media_dir):
     """
     Format list of tracks for output to MPD tag cache
@@ -234,6 +237,7 @@ def tracks_to_tag_cache_format(tracks, media_dir):
     _add_to_tag_cache(result, dirs, files, media_dir)
     return result
 
+
 # TODO: bytes only
 def _add_to_tag_cache(result, dirs, files, media_dir):
     base_path = media_dir.encode('utf-8')

mopidy/scanner.py

@@ -28,7 +28,6 @@ import gst
 
 from mopidy import config as config_lib, ext
 from mopidy.audio import dummy as dummy_audio
-from mopidy.frontends.mpd import translator as mpd_translator
 from mopidy.models import Track, Artist, Album
 from mopidy.utils import log, path, versioning
 
@@ -55,65 +54,60 @@ def main():
         logging.warning('Config value local/media_dir is not set.')
         return
 
-    # TODO: missing error checking and other default setup code.
+    # TODO: missing config error checking and other default setup code.
 
     audio = dummy_audio.DummyAudio()
     local_backend_classes = extensions['local'].get_backend_classes()
     local_backend = local_backend_classes[0](config, audio)
+    local_updater = local_backend.updater
 
-    tracks = {}  # Current lib.
-    update = []  # Paths to rescan for updates/adds.
-    remove = []  # Paths to delete from lib.
+    media_dir = config['local']['media_dir']
 
-    for track in local_backend.library.search().tracks:
-        tracks[track.uri] = track
+    uris_library = set()
+    uris_update = set()
+    uris_remove = set()
 
-    logging.info('Checking %d files from library.', len(tracks))
-    for track in tracks.itervalues():
+    logging.info('Checking tracks from library.')
+    for track in local_updater.load():
         try:
             stat = os.stat(path.uri_to_path(track.uri))
             if int(stat.st_mtime) > track.last_modified:
-                update.append(track.uri)
+                uris_update.add(track.uri)
+            uris_library.add(track.uri)
         except OSError:
-            remove.append(track.uri)
+            uris_remove.add(track.uri)
 
-    logging.info('Removing %d files from library.', len(remove))
-    for uri in remove:
-        del tracks[uri]
+    logging.info('Removing %d moved or deleted tracks.', len(uris_remove))
+    for uri in uris_remove:
+        local_updater.remove(uri)
 
-    logging.info('Checking %s for changes.', config['local']['media_dir'])
-    for p in path.find_files(config['local']['media_dir']):
-        uri = path.path_to_uri(p)
-        if uri not in tracks:
-            update.append(uri)
+    logging.info('Checking %s for new or modified tracks.', media_dir)
+    for uri in path.find_uris(config['local']['media_dir']):
+        if uri not in uris_library:
+            uris_update.add(uri)
+
+    logging.info('Found %d new or modified tracks.', len(uris_update))
 
     def store(data):
         track = translator(data)
-        tracks[track.uri] = track
+        local_updater.add(track)
         logging.debug('Added %s', track.uri)
 
     def debug(uri, error, debug):
         logging.warning('Failed %s: %s', uri, error)
         logging.debug('Debug info for %s: %s', uri, debug)
 
-
-    logging.info('Scanning %d new/changed files.', len(update))
-    scanner = Scanner(update, store, debug)
+    logging.info('Scanning new and modified tracks.')
+    # TODO: just pass the library in instead?
+    scanner = Scanner(uris_update, store, debug)
     try:
         scanner.start()
     except KeyboardInterrupt:
         scanner.stop()
+        raise
 
-    logging.info('Done scanning; writing tag cache...')
-
-    for row in mpd_translator.tracks_to_tag_cache_format(
-            tracks.values(), config['local']['media_dir']):
-        if len(row) == 1:
-            print ('%s' % row).encode('utf-8')
-        else:
-            print ('%s: %s' % row).encode('utf-8')
-
-    logging.info('Done writing tag cache')
+    logging.info('Done scanning; commiting changes.')
+    local_updater.commit()
 
 
 def parse_args():
@@ -132,6 +126,7 @@ def parse_args():
     return parser.parse_args(args=mopidy_args)
 
 
+# TODO: move into scanner.
 def translator(data):
     albumartist_kwargs = {}
     album_kwargs = {}

mopidy/utils/path.py

@@ -141,6 +141,11 @@ def find_files(path):
                 yield os.path.join(dirpath, filename)
 
 
+def find_uris(path):
+    for p in find_files(path):
+        yield path_to_uri(p)
+
+
 def check_file_path_is_inside_base_dir(file_path, base_path):
     assert not file_path.endswith(os.sep), (
         'File path %s cannot end with a path separator' % file_path)

tests/utils/path_test.py

@@ -256,7 +256,7 @@ class FindFilesTest(unittest.TestCase):
     def test_file(self):
         files = self.find('blank.mp3')
         self.assertEqual(len(files), 1)
-        self.assert_(files[0], path_to_data_dir('blank.mp3'))
+        self.assertEqual(files[0], path_to_data_dir('blank.mp3'))
 
     def test_names_are_bytestrings(self):
         is_bytes = lambda f: isinstance(f, bytes)
@@ -271,6 +271,30 @@ class FindFilesTest(unittest.TestCase):
         self.assertEqual(self.find('.blank.mp3'), [])
 
 
+class FindUrisTest(unittest.TestCase):
+    def find(self, value):
+        return list(path.find_uris(path_to_data_dir(value)))
+
+    def test_basic_dir(self):
+        self.assert_(self.find(''))
+
+    def test_nonexistant_dir(self):
+        self.assertEqual(self.find('does-not-exist'), [])
+
+    def test_file(self):
+        uris = self.find('blank.mp3')
+        expected = path.path_to_uri(path_to_data_dir('blank.mp3'))
+        self.assertEqual(len(uris), 1)
+        self.assertEqual(uris[0], expected)
+
+    def test_ignores_hidden_dirs(self):
+        self.assertEqual(self.find('.hidden'), [])
+
+    def test_ignores_hidden_files(self):
+        self.assertEqual(self.find('.blank.mp3'), [])
+
+
+# TODO: kill this in favour of just os.path.getmtime + mocks
 class MtimeTest(unittest.TestCase):
     def tearDown(self):
         path.mtime.undo_fake()