local: Add flush threshold to scanner.

Instead of triggering every 1000th query, this is now configurable and also
triggers the flush call to the library.

docs/ext/local.rst

@@ -58,6 +58,11 @@ Configuration values
     Number of milliseconds before giving up scanning a file and moving on to
     the next file.
 
+.. confval:: local/scan_flush_threshold
+
+    Number of tracks to wait before telling library it should try and store
+    it's progress so far. Some libraries might not respect this setting.
+
 .. confval:: local/excluded_file_extensions
 
     File extensions to exclude when scanning the media directory. Values

mopidy/backends/local/__init__.py

@@ -28,6 +28,7 @@ class Extension(ext.Extension):
         schema['tag_cache_file'] = config.Deprecated()
         schema['scan_timeout'] = config.Integer(
             minimum=1000, maximum=1000*60*60)
+        schema['scan_flush_threshold'] = config.Integer(minimum=0)
         schema['excluded_file_extensions'] = config.List(optional=True)
         return schema
 

mopidy/backends/local/commands.py

@@ -25,6 +25,7 @@ class ScanCommand(commands.Command):
     def run(self, args, config):
         media_dir = config['local']['media_dir']
         scan_timeout = config['local']['scan_timeout']
+        flush_threshold = config['local']['scan_flush_threshold']
         excluded_file_extensions = config['local']['excluded_file_extensions']
         excluded_file_extensions = set(
             file_ext.lower() for file_ext in excluded_file_extensions)
@@ -80,7 +81,9 @@ class ScanCommand(commands.Command):
         logger.info('Scanning...')
 
         scanner = scan.Scanner(scan_timeout)
-        progress = Progress(len(uris_to_update))
+        count = 0
+        total = len(uris_to_update)
+        start = time.time()
 
         for uri in sorted(uris_to_update):
             try:
@@ -91,26 +94,14 @@ class ScanCommand(commands.Command):
             except exceptions.ScannerError as error:
                 logger.warning('Failed %s: %s', uri, error)
 
-                # TODO: trigger this on batch size intervals instead and add
-                # flush
-            progress.increment()
+            count += 1
+            if count % flush_threshold == 0 or count == total:
+                duration = time.time() - start
+                remainder = duration / count * (total - count)
+                logger.info('Scanned %d of %d files in %ds, ~%ds left.',
+                            count, total, duration, remainder)
+                library.flush()
 
-        logger.info('Commiting changes.')
         library.close()
+        logger.info('Done scanning.')
         return 0
-
-
-# TODO: move to utils?
-class Progress(object):
-    def __init__(self, total):
-        self.count = 0
-        self.total = total
-        self.start = time.time()
-
-    def increment(self):
-        self.count += 1
-        if self.count % 1000 == 0 or self.count == self.total:
-            duration = time.time() - self.start
-            remainder = duration / self.count * (self.total - self.count)
-            logger.info('Scanned %d of %d files in %ds, ~%ds left.',
-                        self.count, self.total, duration, remainder)

mopidy/backends/local/ext.conf

@@ -5,6 +5,7 @@ media_dir = $XDG_MUSIC_DIR
 data_dir = $XDG_DATA_DIR/mopidy/local
 playlists_dir = $XDG_DATA_DIR/mopidy/local/playlists
 scan_timeout = 1000
+scan_flush_threshold = 1000
 excluded_file_extensions =
   .html
   .jpeg