diff --git a/mopidy/backends/local/translator.py b/mopidy/backends/local/translator.py
index d2ee0d45..4ae10af2 100644
--- a/mopidy/backends/local/translator.py
+++ b/mopidy/backends/local/translator.py
@@ -112,6 +112,9 @@ def _convert_mpd_data(data, tracks, music_dir):
         else:
             track_kwargs['track_no'] = int(data['track'])
 
+    if 'mtime' in data:
+        track_kwargs['last_modified'] = int(data['mtime'])
+
     if 'artist' in data:
         artist_kwargs['name'] = data['artist']
         albumartist_kwargs['name'] = data['artist']
diff --git a/mopidy/models.py b/mopidy/models.py
index d138b490..fe390ddf 100644
--- a/mopidy/models.py
+++ b/mopidy/models.py
@@ -231,6 +231,8 @@ class Track(ImmutableObject):
     :type bitrate: integer
     :param musicbrainz_id: MusicBrainz ID
     :type musicbrainz_id: string
+    :param last_modified: Represents last modification time
+    :type last_modified: integer
     """
 
     #: The track URI. Read-only.
@@ -263,6 +265,11 @@ class Track(ImmutableObject):
     #: The MusicBrainz ID of the track. Read-only.
     musicbrainz_id = None
 
+    #: Integer representing when the track was last modified, exact meaning
+    #: depends on source of track. For local files this is the mtime, for other
+    #: backends it could be a timestamp or simply a version counter.
+    last_modified = 0
+
     def __init__(self, *args, **kwargs):
         self.__dict__['artists'] = frozenset(kwargs.pop('artists', []))
         super(Track, self).__init__(*args, **kwargs)
diff --git a/mopidy/scanner.py b/mopidy/scanner.py
index 41b42347..7fd7b541 100644
--- a/mopidy/scanner.py
+++ b/mopidy/scanner.py
@@ -27,6 +27,7 @@ pygst.require('0.10')
 import gst
 
 from mopidy import config as config_lib, ext
+from mopidy.audio import dummy as dummy_audio
 from mopidy.frontends.mpd import translator as mpd_translator
 from mopidy.models import Track, Artist, Album
 from mopidy.utils import log, path, versioning
@@ -45,9 +46,9 @@ def main():
     log.setup_root_logger()
     log.setup_console_logging(logging_config, args.verbosity_level)
 
-    extensions = ext.load_extensions()
+    extensions = dict((e.ext_name, e) for e in ext.load_extensions())
     config, errors = config_lib.load(
-        config_files, extensions, config_overrides)
+        config_files, extensions.values(), config_overrides)
     log.setup_log_levels(config)
 
     if not config['local']['media_dir']:
@@ -56,20 +57,48 @@ def main():
 
     # TODO: missing error checking and other default setup code.
 
-    tracks = []
+    audio = dummy_audio.DummyAudio()
+    local_backend_classes = extensions['local'].get_backend_classes()
+    local_backend = local_backend_classes[0](config, audio)
+
+    tracks = {}  # Current lib.
+    update = []  # Paths to rescan for updates/adds.
+    remove = []  # Paths to delete from lib.
+
+    for track in local_backend.library.search().tracks:
+        tracks[track.uri] = track
+
+    logging.info('Checking %d files from library.', len(tracks))
+    for track in tracks.itervalues():
+        try:
+            stat = os.stat(path.uri_to_path(track.uri))
+            if int(stat.st_mtime) > track.last_modified:
+                update.append(track.uri)
+        except OSError:
+            remove.append(track.uri)
+
+    logging.info('Removing %d files from library.', len(remove))
+    for uri in remove:
+        del tracks[uri]
+
+    logging.info('Checking %s for changes.', config['local']['media_dir'])
+    for p in path.find_files(config['local']['media_dir']):
+        uri = path.path_to_uri(p)
+        if uri not in tracks:
+            update.append(uri)
 
     def store(data):
         track = translator(data)
-        tracks.append(track)
+        tracks[track.uri] = track
         logging.debug('Added %s', track.uri)
 
     def debug(uri, error, debug):
         logging.warning('Failed %s: %s', uri, error)
         logging.debug('Debug info for %s: %s', uri, debug)
 
-    logging.info('Scanning %s', config['local']['media_dir'])
 
-    scanner = Scanner(config['local']['media_dir'], store, debug)
+    logging.info('Scanning %d new/changed files.', len(update))
+    scanner = Scanner(update, store, debug)
     try:
         scanner.start()
     except KeyboardInterrupt:
@@ -78,7 +107,7 @@ def main():
     logging.info('Done scanning; writing tag cache...')
 
     for row in mpd_translator.tracks_to_tag_cache_format(
-            tracks, config['local']['media_dir']):
+            tracks.values(), config['local']['media_dir']):
         if len(row) == 1:
             print ('%s' % row).encode('utf-8')
         else:
@@ -141,6 +170,7 @@ def translator(data):
         album_kwargs['artists'] = [Artist(**albumartist_kwargs)]
 
     track_kwargs['uri'] = data['uri']
+    track_kwargs['last_modified'] = int(data['mtime'])
     track_kwargs['length'] = data[gst.TAG_DURATION]
     track_kwargs['album'] = Album(**album_kwargs)
     track_kwargs['artists'] = [Artist(**artist_kwargs)]
@@ -149,9 +179,9 @@ def translator(data):
 
 
 class Scanner(object):
-    def __init__(self, base_dir, data_callback, error_callback=None):
+    def __init__(self, uris, data_callback, error_callback=None):
         self.data = {}
-        self.files = path.find_files(base_dir)
+        self.uris = iter(uris)
         self.data_callback = data_callback
         self.error_callback = error_callback
         self.loop = gobject.MainLoop()
@@ -195,7 +225,9 @@ class Scanner(object):
         if message.structure.get_name() != 'handoff':
             return
 
-        self.data['uri'] = unicode(self.uribin.get_property('uri'))
+        uri = unicode(self.uribin.get_property('uri'))
+        self.data['uri'] = uri
+        self.data['mtime'] = os.path.getmtime(path.uri_to_path(uri))
         self.data[gst.TAG_DURATION] = self.get_duration()
 
         try:
@@ -234,7 +266,7 @@ class Scanner(object):
     def next_uri(self):
         self.data = {}
         try:
-            uri = path.path_to_uri(self.files.next())
+            uri = next(self.uris)
         except StopIteration:
             self.stop()
             return False
diff --git a/tests/backends/local/translator_test.py b/tests/backends/local/translator_test.py
index 096d9a0d..4f958232 100644
--- a/tests/backends/local/translator_test.py
+++ b/tests/backends/local/translator_test.py
@@ -101,7 +101,8 @@ def generate_track(path, ident):
     uri = path_to_uri(path_to_data_dir(path))
     track = Track(
         uri=uri, name='trackname', artists=expected_artists,
-        album=expected_albums[0], track_no=1, date='2006', length=4000)
+        album=expected_albums[0], track_no=1, date='2006', length=4000,
+        last_modified=1272319626)
     expected_tracks.append(track)
 
 
@@ -128,7 +129,8 @@ class MPDTagCacheToTracksTest(unittest.TestCase):
         uri = path_to_uri(path_to_data_dir('song1.mp3'))
         track = Track(
             uri=uri, name='trackname', artists=expected_artists, track_no=1,
-            album=expected_albums[0], date='2006', length=4000)
+            album=expected_albums[0], date='2006', length=4000,
+            last_modified=1272319626)
         self.assertEqual(set([track]), tracks)
 
     def test_advanced_cache(self):
@@ -144,7 +146,8 @@ class MPDTagCacheToTracksTest(unittest.TestCase):
         artists = [Artist(name='æøå')]
         album = Album(name='æøå', artists=artists)
         track = Track(
-            uri=uri, name='æøå', artists=artists, album=album, length=4000)
+            uri=uri, name='æøå', artists=artists, album=album, length=4000,
+            last_modified=1272319626)
 
         self.assertEqual(track, list(tracks)[0])
 
@@ -157,7 +160,8 @@ class MPDTagCacheToTracksTest(unittest.TestCase):
         tracks = parse_mpd_tag_cache(
             path_to_data_dir('blank_tag_cache'), path_to_data_dir(''))
         uri = path_to_uri(path_to_data_dir('song1.mp3'))
-        self.assertEqual(set([Track(uri=uri, length=4000)]), tracks)
+        expected = Track(uri=uri, length=4000, last_modified=1272319626)
+        self.assertEqual(set([expected]), tracks)
 
     def test_musicbrainz_tagcache(self):
         tracks = parse_mpd_tag_cache(
@@ -184,5 +188,5 @@ class MPDTagCacheToTracksTest(unittest.TestCase):
         album = expected_albums[0].copy(artists=[artist])
         track = Track(
             uri=uri, name='trackname', artists=expected_artists, track_no=1,
-            album=album, date='2006', length=4000)
+            album=album, date='2006', length=4000, last_modified=1272319626)
         self.assertEqual(track, list(tracks)[0])
diff --git a/tests/scanner_test.py b/tests/scanner_test.py
index 75fc60fd..c9671523 100644
--- a/tests/scanner_test.py
+++ b/tests/scanner_test.py
@@ -4,6 +4,7 @@ import unittest
 
 from mopidy.scanner import Scanner, translator
 from mopidy.models import Track, Artist, Album
+from mopidy.utils import path as path_lib
 
 from tests import path_to_data_dir
 
@@ -32,6 +33,7 @@ class TranslatorTest(unittest.TestCase):
             'musicbrainz-albumid': 'mbalbumid',
             'musicbrainz-artistid': 'mbartistid',
             'musicbrainz-albumartistid': 'mbalbumartistid',
+            'mtime': 1234,
         }
 
         self.album = {
@@ -57,6 +59,7 @@ class TranslatorTest(unittest.TestCase):
             'track_no': 1,
             'length': 4531,
             'musicbrainz_id': 'mbtrackid',
+            'last_modified': 1234,
         }
 
     def build_track(self):
@@ -141,8 +144,9 @@ class ScannerTest(unittest.TestCase):
         self.data = {}
 
     def scan(self, path):
-        scanner = Scanner(
-            path_to_data_dir(path), self.data_callback, self.error_callback)
+        paths = path_lib.find_files(path_to_data_dir(path))
+        uris = (path_lib.path_to_uri(p) for p in paths)
+        scanner = Scanner(uris, self.data_callback, self.error_callback)
         scanner.start()
 
     def check(self, name, key, value):