From 351589c6c8652fca914574b8c728ba38b1b350f2 Mon Sep 17 00:00:00 2001
From: Thomas Adamcik <thomas@adamcik.no>
Date: Mon, 20 May 2013 22:42:31 +0200
Subject: [PATCH 1/6] path: Add find_uris version of find_files

---
 mopidy/utils/path.py     |  5 +++++
 tests/utils/path_test.py | 24 ++++++++++++++++++++++++
 2 files changed, 29 insertions(+)

diff --git a/mopidy/utils/path.py b/mopidy/utils/path.py
index 8ad9b31a..e4d717d1 100644
--- a/mopidy/utils/path.py
+++ b/mopidy/utils/path.py
@@ -141,6 +141,11 @@ def find_files(path):
                 yield os.path.join(dirpath, filename)
 
 
+def find_uris(path):
+    for p in find_files(path):
+        yield path_to_uri(p)
+
+
 def check_file_path_is_inside_base_dir(file_path, base_path):
     assert not file_path.endswith(os.sep), (
         'File path %s cannot end with a path separator' % file_path)
diff --git a/tests/utils/path_test.py b/tests/utils/path_test.py
index 54a9a8a4..3fdae887 100644
--- a/tests/utils/path_test.py
+++ b/tests/utils/path_test.py
@@ -271,6 +271,30 @@ class FindFilesTest(unittest.TestCase):
         self.assertEqual(self.find('.blank.mp3'), [])
 
 
+class FindUrisTest(unittest.TestCase):
+    def find(self, value):
+        return list(path.find_uris(path_to_data_dir(value)))
+
+    def test_basic_dir(self):
+        self.assert_(self.find(''))
+
+    def test_nonexistant_dir(self):
+        self.assertEqual(self.find('does-not-exist'), [])
+
+    def test_file(self):
+        uris = self.find('blank.mp3')
+        expected = path.path_to_uri(path_to_data_dir('blank.mp3'))
+        self.assertEqual(len(uris), 1)
+        self.assert_(uris[0], expected)
+
+    def test_ignores_hidden_dirs(self):
+        self.assertEqual(self.find('.hidden'), [])
+
+    def test_ignores_hidden_files(self):
+        self.assertEqual(self.find('.blank.mp3'), [])
+
+
+# TODO: kill this in favour of just os.path.getmtime + mocks
 class MtimeTest(unittest.TestCase):
     def tearDown(self):
         path.mtime.undo_fake()

From 5961a1f5c8663c44d0e398e93eccf447d1693de5 Mon Sep 17 00:00:00 2001
From: Thomas Adamcik <thomas@adamcik.no>
Date: Mon, 20 May 2013 22:56:09 +0200
Subject: [PATCH 2/6] scanner: Write tag cache to tmp file.

Tag cache is now output to a tmp file residing in the same folder as the real
one. Once generated the tmpfile is moved over the original file for an atomic
updated.
---
 mopidy/scanner.py | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/mopidy/scanner.py b/mopidy/scanner.py
index 7fd7b541..0cc51d2a 100644
--- a/mopidy/scanner.py
+++ b/mopidy/scanner.py
@@ -5,6 +5,7 @@ import datetime
 import logging
 import os
 import sys
+import tempfile
 
 import gobject
 gobject.threads_init()
@@ -104,16 +105,21 @@ def main():
     except KeyboardInterrupt:
         scanner.stop()
 
-    logging.info('Done scanning; writing tag cache...')
+    logging.info('Done scanning; writing tag cache to temporary file.')
+
+    directory, basename = os.path.split(config['local']['tag_cache_file'])
+    tmp = tempfile.NamedTemporaryFile(
+        prefix=basename + '.', dir=directory, delete=False)
 
     for row in mpd_translator.tracks_to_tag_cache_format(
             tracks.values(), config['local']['media_dir']):
         if len(row) == 1:
-            print ('%s' % row).encode('utf-8')
+            tmp.write(('%s\n' % row).encode('utf-8'))
         else:
-            print ('%s: %s' % row).encode('utf-8')
+            tmp.write(('%s: %s\n' % row).encode('utf-8'))
 
-    logging.info('Done writing tag cache')
+    os.rename(tmp.name, config['local']['tag_cache_file'])
+    logging.info('Done writing; overwriting active tag cache.')
 
 
 def parse_args():
@@ -132,6 +138,7 @@ def parse_args():
     return parser.parse_args(args=mopidy_args)
 
 
+# TODO: move into scanner.
 def translator(data):
     albumartist_kwargs = {}
     album_kwargs = {}

From 7f80a188c91cb774a9f6f29a8f3e13d0fd825d59 Mon Sep 17 00:00:00 2001
From: Thomas Adamcik <thomas@adamcik.no>
Date: Mon, 20 May 2013 23:50:46 +0200
Subject: [PATCH 3/6] core: Add library updater.

Idea behind this class is to have a clear sub-set of our library API that is
only used for updating libraries. This makes it very clear what methods are
needed for just libaries and additionally updatable libraries.

Next couple of commits will build on this idea taking us closer to plugable
local libraries.
---
 mopidy/backends/base.py | 51 +++++++++++++++++++++++++++++++++++++++++
 mopidy/scanner.py       | 20 +++++++++-------
 2 files changed, 63 insertions(+), 8 deletions(-)

diff --git a/mopidy/backends/base.py b/mopidy/backends/base.py
index c45eb771..9f531450 100644
--- a/mopidy/backends/base.py
+++ b/mopidy/backends/base.py
@@ -15,6 +15,11 @@ class Backend(object):
     #: the backend doesn't provide a library.
     library = None
 
+    #: The library update provider. An instance of
+    #: :class:`~mopidy.backends.base.BaseLibraryUpdateProvider`, or
+    #: :class:`None` if the backend doesn't provide a library.
+    updater = None
+
     #: The playback provider. An instance of
     #: :class:`~mopidy.backends.base.BasePlaybackProvider`, or :class:`None` if
     #: the backend doesn't provide playback.
@@ -35,6 +40,9 @@ class Backend(object):
     def has_library(self):
         return self.library is not None
 
+    def has_updater(self):
+        return self.updater is not None
+
     def has_playback(self):
         return self.playback is not None
 
@@ -53,6 +61,7 @@ class BaseLibraryProvider(object):
     def __init__(self, backend):
         self.backend = backend
 
+    # TODO: replace with search(query, exact=Ture, ...)
     def find_exact(self, query=None, uris=None):
         """
         See :meth:`mopidy.core.LibraryController.find_exact`.
@@ -86,6 +95,48 @@ class BaseLibraryProvider(object):
         pass
 
 
+class BaseLibraryUpdateProvider(object):
+    """
+    :param backend: backend the controller is a part of
+    :type backend: :class:`mopidy.backends.base.Backend`
+    """
+
+    pykka_traversable = True
+
+    def __init__(self, backend):
+        self.backend = backend
+
+    def load(self):
+        """Loads the library and returns all tracks in it.
+
+        *MUST be implemented by subclass.*
+        """
+        raise NotImplementedError
+
+    def add(self, track):
+        """Adds given track to library.
+
+        Overwrites any existing entry with same uri.
+
+        *MUST be implemented by subclass.*
+        """
+        raise NotImplementedError
+
+    def remove(self, uri):
+        """Removes given track from library.
+
+        *MUST be implemented by subclass.*
+        """
+        raise NotImplementedError
+
+    def commit(self):
+        """Persist changes to library.
+
+        *MAY be implemented by subclass.*
+        """
+        pass
+
+
 class BasePlaybackProvider(object):
     """
     :param audio: the audio actor
diff --git a/mopidy/scanner.py b/mopidy/scanner.py
index 0cc51d2a..ba8f2ff9 100644
--- a/mopidy/scanner.py
+++ b/mopidy/scanner.py
@@ -111,15 +111,19 @@ def main():
     tmp = tempfile.NamedTemporaryFile(
         prefix=basename + '.', dir=directory, delete=False)
 
-    for row in mpd_translator.tracks_to_tag_cache_format(
-            tracks.values(), config['local']['media_dir']):
-        if len(row) == 1:
-            tmp.write(('%s\n' % row).encode('utf-8'))
-        else:
-            tmp.write(('%s: %s\n' % row).encode('utf-8'))
+    try:
+        for row in mpd_translator.tracks_to_tag_cache_format(
+                tracks.values(), config['local']['media_dir']):
+            if len(row) == 1:
+                tmp.write(('%s\n' % row).encode('utf-8'))
+            else:
+                tmp.write(('%s: %s\n' % row).encode('utf-8'))
 
-    os.rename(tmp.name, config['local']['tag_cache_file'])
-    logging.info('Done writing; overwriting active tag cache.')
+        logging.info('Done writing; overwriting active tag cache.')
+        os.rename(tmp.name, config['local']['tag_cache_file'])
+    finally:
+        if os.path.exists(tmp.name):
+            os.remove(tmp.name)
 
 
 def parse_args():

From 00e88361f32bc92da2e34b10c135e628268e1cea Mon Sep 17 00:00:00 2001
From: Thomas Adamcik <thomas@adamcik.no>
Date: Mon, 20 May 2013 23:53:38 +0200
Subject: [PATCH 4/6] local: Add library update provider for tag caches.

This adds the basic interface implemenations for supporting tag caches using
the new update APIs.
---
 mopidy/backends/local/actor.py     |  3 +-
 mopidy/backends/local/library.py   | 44 ++++++++++++++++++++++++++++++
 mopidy/frontends/mpd/translator.py |  4 +++
 3 files changed, 50 insertions(+), 1 deletion(-)

diff --git a/mopidy/backends/local/actor.py b/mopidy/backends/local/actor.py
index 8f53af4d..b73c53e2 100644
--- a/mopidy/backends/local/actor.py
+++ b/mopidy/backends/local/actor.py
@@ -8,7 +8,7 @@ import pykka
 from mopidy.backends import base
 from mopidy.utils import encoding, path
 
-from .library import LocalLibraryProvider
+from .library import LocalLibraryProvider, LocalLibraryUpdateProvider
 from .playlists import LocalPlaylistsProvider
 
 logger = logging.getLogger('mopidy.backends.local')
@@ -23,6 +23,7 @@ class LocalBackend(pykka.ThreadingActor, base.Backend):
         self.check_dirs_and_files()
 
         self.library = LocalLibraryProvider(backend=self)
+        self.updater = LocalLibraryUpdateProvider(backend=self)
         self.playback = base.BasePlaybackProvider(audio=audio, backend=self)
         self.playlists = LocalPlaylistsProvider(backend=self)
 
diff --git a/mopidy/backends/local/library.py b/mopidy/backends/local/library.py
index 669e72d7..43768cd4 100644
--- a/mopidy/backends/local/library.py
+++ b/mopidy/backends/local/library.py
@@ -1,7 +1,11 @@
 from __future__ import unicode_literals
 
 import logging
+import os
+import tempfile
+
 from mopidy.backends import base
+from mopidy.frontends.mpd import translator as mpd_translator
 from mopidy.models import Album, SearchResult
 
 from .translator import parse_mpd_tag_cache
@@ -127,3 +131,43 @@ class LocalLibraryProvider(base.BaseLibraryProvider):
             for value in values:
                 if not value:
                     raise LookupError('Missing query')
+
+
+# TODO: rename and move to tagcache extension.
+class LocalLibraryUpdateProvider(base.BaseLibraryProvider):
+    def __init__(self, *args, **kwargs):
+        super(LocalLibraryUpdateProvider, self).__init__(*args, **kwargs)
+        self._tracks = {}
+        self._media_dir = self.backend.config['local']['media_dir']
+        self._tag_cache_file = self.backend.config['local']['tag_cache_file']
+
+    def load(self):
+        tracks = parse_mpd_tag_cache(self._tag_cache_file, self._media_dir)
+        for track in tracks:
+            self._tracks[track.uri] = track
+        return tracks
+
+    def add(self, track):
+        self._tracks[track.uri] = track
+
+    def remove(self, uri):
+        if uri in self._tracks:
+            del self._tracks[uri]
+
+    def commit(self):
+        directory, basename = os.path.split(self._tag_cache_file)
+        tmp = tempfile.NamedTemporaryFile(
+            prefix=basename + '.', dir=directory, delete=False)
+
+        try:
+            for row in mpd_translator.tracks_to_tag_cache_format(
+                    self._tracks.values(), self._media_dir):
+                if len(row) == 1:
+                    tmp.write(('%s\n' % row).encode('utf-8'))
+                else:
+                    tmp.write(('%s: %s\n' % row).encode('utf-8'))
+
+            os.rename(tmp.name, self._tag_cache_file)
+        finally:
+            if os.path.exists(tmp.name):
+                os.remove(tmp.name)
diff --git a/mopidy/frontends/mpd/translator.py b/mopidy/frontends/mpd/translator.py
index b646732f..8e9d12e0 100644
--- a/mopidy/frontends/mpd/translator.py
+++ b/mopidy/frontends/mpd/translator.py
@@ -10,6 +10,8 @@ from mopidy.frontends.mpd.exceptions import MpdArgError
 from mopidy.models import TlTrack
 from mopidy.utils.path import mtime as get_mtime, uri_to_path, split_path
 
+# TODO: special handling of local:// uri scheme
+
 
 def track_to_mpd_format(track, position=None):
     """
@@ -213,6 +215,7 @@ def query_from_mpd_search_format(mpd_query):
     return query
 
 
+# TODO: move to tagcache backend.
 def tracks_to_tag_cache_format(tracks, media_dir):
     """
     Format list of tracks for output to MPD tag cache
@@ -234,6 +237,7 @@ def tracks_to_tag_cache_format(tracks, media_dir):
     _add_to_tag_cache(result, dirs, files, media_dir)
     return result
 
+
 # TODO: bytes only
 def _add_to_tag_cache(result, dirs, files, media_dir):
     base_path = media_dir.encode('utf-8')

From 98bb35d4d1c2624501cbaa9e55751df85c82fa2d Mon Sep 17 00:00:00 2001
From: Thomas Adamcik <thomas@adamcik.no>
Date: Mon, 20 May 2013 23:54:48 +0200
Subject: [PATCH 5/6] scanner: Convert to using library updater.

This version of the scanner switches to using the updater API, in other words
we are now fairly close to being able to plug in alternate libraries for local
files and populate them with mopidy-scan.
---
 mopidy/scanner.py | 70 ++++++++++++++++++-----------------------------
 1 file changed, 27 insertions(+), 43 deletions(-)

diff --git a/mopidy/scanner.py b/mopidy/scanner.py
index ba8f2ff9..1127264a 100644
--- a/mopidy/scanner.py
+++ b/mopidy/scanner.py
@@ -5,7 +5,6 @@ import datetime
 import logging
 import os
 import sys
-import tempfile
 
 import gobject
 gobject.threads_init()
@@ -29,7 +28,6 @@ import gst
 
 from mopidy import config as config_lib, ext
 from mopidy.audio import dummy as dummy_audio
-from mopidy.frontends.mpd import translator as mpd_translator
 from mopidy.models import Track, Artist, Album
 from mopidy.utils import log, path, versioning
 
@@ -56,74 +54,60 @@ def main():
         logging.warning('Config value local/media_dir is not set.')
         return
 
-    # TODO: missing error checking and other default setup code.
+    # TODO: missing config error checking and other default setup code.
 
     audio = dummy_audio.DummyAudio()
     local_backend_classes = extensions['local'].get_backend_classes()
     local_backend = local_backend_classes[0](config, audio)
+    local_updater = local_backend.updater
 
-    tracks = {}  # Current lib.
-    update = []  # Paths to rescan for updates/adds.
-    remove = []  # Paths to delete from lib.
+    media_dir = config['local']['media_dir']
 
-    for track in local_backend.library.search().tracks:
-        tracks[track.uri] = track
+    uris_library = set()
+    uris_update = set()
+    uris_remove = set()
 
-    logging.info('Checking %d files from library.', len(tracks))
-    for track in tracks.itervalues():
+    logging.info('Checking tracks from library.')
+    for track in local_updater.load():
         try:
             stat = os.stat(path.uri_to_path(track.uri))
             if int(stat.st_mtime) > track.last_modified:
-                update.append(track.uri)
+                uris_update.add(track.uri)
+            uris_library.add(track.uri)
         except OSError:
-            remove.append(track.uri)
+            uris_remove.add(track.uri)
 
-    logging.info('Removing %d files from library.', len(remove))
-    for uri in remove:
-        del tracks[uri]
+    logging.info('Removing %d moved or deleted tracks.', len(uris_remove))
+    for uri in uris_remove:
+        local_updater.remove(uri)
 
-    logging.info('Checking %s for changes.', config['local']['media_dir'])
-    for p in path.find_files(config['local']['media_dir']):
-        uri = path.path_to_uri(p)
-        if uri not in tracks:
-            update.append(uri)
+    logging.info('Checking %s for new or modified tracks.', media_dir)
+    for uri in path.find_uris(config['local']['media_dir']):
+        if uri not in uris_library:
+            uris_update.add(uri)
+
+    logging.info('Found %d new or changed tracks.', len(uris_update))
 
     def store(data):
         track = translator(data)
-        tracks[track.uri] = track
+        local_updater.add(track)
         logging.debug('Added %s', track.uri)
 
     def debug(uri, error, debug):
         logging.warning('Failed %s: %s', uri, error)
         logging.debug('Debug info for %s: %s', uri, debug)
 
-
-    logging.info('Scanning %d new/changed files.', len(update))
-    scanner = Scanner(update, store, debug)
+    logging.info('Scanning new and changed tracks.')
+    # TODO: just pass the library in instead?
+    scanner = Scanner(uris_update, store, debug)
     try:
         scanner.start()
     except KeyboardInterrupt:
         scanner.stop()
+        raise
 
-    logging.info('Done scanning; writing tag cache to temporary file.')
-
-    directory, basename = os.path.split(config['local']['tag_cache_file'])
-    tmp = tempfile.NamedTemporaryFile(
-        prefix=basename + '.', dir=directory, delete=False)
-
-    try:
-        for row in mpd_translator.tracks_to_tag_cache_format(
-                tracks.values(), config['local']['media_dir']):
-            if len(row) == 1:
-                tmp.write(('%s\n' % row).encode('utf-8'))
-            else:
-                tmp.write(('%s: %s\n' % row).encode('utf-8'))
-
-        logging.info('Done writing; overwriting active tag cache.')
-        os.rename(tmp.name, config['local']['tag_cache_file'])
-    finally:
-        if os.path.exists(tmp.name):
-            os.remove(tmp.name)
+    logging.info('Done scanning; commiting changes.')
+    local_updater.commit()
 
 
 def parse_args():

From 71618f840fd27a85aa3e6d85e19786fc27d8dd2f Mon Sep 17 00:00:00 2001
From: Thomas Adamcik <thomas@adamcik.no>
Date: Wed, 22 May 2013 20:36:39 +0200
Subject: [PATCH 6/6] scanner: Fixing review comments.

---
 mopidy/backends/base.py  | 4 ++--
 mopidy/scanner.py        | 4 ++--
 tests/utils/path_test.py | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/mopidy/backends/base.py b/mopidy/backends/base.py
index 9f531450..f0561b4c 100644
--- a/mopidy/backends/base.py
+++ b/mopidy/backends/base.py
@@ -61,7 +61,7 @@ class BaseLibraryProvider(object):
     def __init__(self, backend):
         self.backend = backend
 
-    # TODO: replace with search(query, exact=Ture, ...)
+    # TODO: replace with search(query, exact=True, ...)
     def find_exact(self, query=None, uris=None):
         """
         See :meth:`mopidy.core.LibraryController.find_exact`.
@@ -116,7 +116,7 @@ class BaseLibraryUpdateProvider(object):
     def add(self, track):
         """Adds given track to library.
 
-        Overwrites any existing entry with same uri.
+        Overwrites any existing track with same URI.
 
         *MUST be implemented by subclass.*
         """
diff --git a/mopidy/scanner.py b/mopidy/scanner.py
index 1127264a..77085f90 100644
--- a/mopidy/scanner.py
+++ b/mopidy/scanner.py
@@ -86,7 +86,7 @@ def main():
         if uri not in uris_library:
             uris_update.add(uri)
 
-    logging.info('Found %d new or changed tracks.', len(uris_update))
+    logging.info('Found %d new or modified tracks.', len(uris_update))
 
     def store(data):
         track = translator(data)
@@ -97,7 +97,7 @@ def main():
         logging.warning('Failed %s: %s', uri, error)
         logging.debug('Debug info for %s: %s', uri, debug)
 
-    logging.info('Scanning new and changed tracks.')
+    logging.info('Scanning new and modified tracks.')
     # TODO: just pass the library in instead?
     scanner = Scanner(uris_update, store, debug)
     try:
diff --git a/tests/utils/path_test.py b/tests/utils/path_test.py
index 3fdae887..a19e48f7 100644
--- a/tests/utils/path_test.py
+++ b/tests/utils/path_test.py
@@ -256,7 +256,7 @@ class FindFilesTest(unittest.TestCase):
     def test_file(self):
         files = self.find('blank.mp3')
         self.assertEqual(len(files), 1)
-        self.assert_(files[0], path_to_data_dir('blank.mp3'))
+        self.assertEqual(files[0], path_to_data_dir('blank.mp3'))
 
     def test_names_are_bytestrings(self):
         is_bytes = lambda f: isinstance(f, bytes)
@@ -285,7 +285,7 @@ class FindUrisTest(unittest.TestCase):
         uris = self.find('blank.mp3')
         expected = path.path_to_uri(path_to_data_dir('blank.mp3'))
         self.assertEqual(len(uris), 1)
-        self.assert_(uris[0], expected)
+        self.assertEqual(uris[0], expected)
 
     def test_ignores_hidden_dirs(self):
         self.assertEqual(self.find('.hidden'), [])