andpp/mopidy
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

local: Update how scanner work is done to improve remote FS support.

Browse Source 
This changes the scanner to prefetch all the files+mtimes using our new
threaded finder. Hopefully making scanning against remote filesystems
somewhat less painful.
This commit is contained in:
Thomas Adamcik 2014-02-26 23:31:11 +01:00
parent 5b7a6065e7
commit 78f727b9be
1 changed files with 18 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
mopidy/local/commands.py
View File

@ -70,32 +70,31 @@ class ScanCommand(commands.Command):
library = _get_library(args, config)
uri_path_mapping = {}
uris_in_library = set()
uris_to_update = set()
uris_to_remove = set()
file_mtimes = path.find_mtimes(media_dir)
logger.info('Found %d files in media_dir.', len(file_mtimes))
num_tracks = library.load()
logger.info('Checking %d tracks from library.', num_tracks)
for track in library.begin():
uri_path_mapping[track.uri] = translator.local_track_uri_to_path(
track.uri, media_dir)
try:
stat = os.stat(uri_path_mapping[track.uri])
if int(stat.st_mtime) > track.last_modified:
uris_to_update.add(track.uri)
uris_in_library.add(track.uri)
except OSError:
abspath = translator.local_track_uri_to_path(track.uri, media_dir)
mtime = file_mtimes.pop(abspath, None)
if mtime is None:
logger.debug('Missing file %s', track.uri)
uris_to_remove.add(track.uri)
elif mtime > track.last_modified:
uris_in_library.add(track.uri)
logger.info('Removing %d missing tracks.', len(uris_to_remove))
for uri in uris_to_remove:
library.remove(uri)
logger.info('Checking %s for unknown tracks.', media_dir)
for relpath in path.find_files(media_dir):
for abspath in file_mtimes:
relpath = os.path.relpath(abspath, media_dir)
uri = translator.path_to_local_track_uri(relpath)
file_extension = os.path.splitext(relpath)[1]
@ -103,21 +102,23 @@ class ScanCommand(commands.Command):
logger.debug('Skipped %s: File extension excluded.', uri)
continue
if uri not in uris_in_library:
uris_to_update.add(uri)
uri_path_mapping[uri] = os.path.join(media_dir, relpath)
uris_to_update.add(uri)
logger.info('Found %d unknown tracks.', len(uris_to_update))
logger.info(
'Found %d tracks which need to be updated.', len(uris_to_update))
logger.info('Scanning...')
uris_to_update = sorted(uris_to_update)[:args.limit]
uris_to_update = sorted(uris_to_update, key=lambda v: v.lower())
uris_to_update = uris_to_update[:args.limit]
scanner = scan.Scanner(scan_timeout)
progress = _Progress(flush_threshold, len(uris_to_update))
for uri in uris_to_update:
try:
data = scanner.scan(path.path_to_uri(uri_path_mapping[uri]))
relpath = translator.local_track_uri_to_path(uri, media_dir)
file_uri = path.path_to_uri(os.path.join(media_dir, relpath))
data = scanner.scan(file_uri)
track = scan.audio_data_to_track(data).copy(uri=uri)
library.add(track)
logger.debug('Added %s', track.uri)