stream: Expand nested stream playlists

Fixes #1250

docs/changelog.rst

@@ -53,6 +53,11 @@ Bug fix release.
   services installed from packages that properly set :confval:`core/data_dir`,
   like the Debian and Arch pakages. (Fixes: :issue:`1259`, PR: :issue:`1266`)
 
+- Stream: Expand nested playlists to find the stream URI. This used to work,
+  but regressed in 1.1.0 with the extraction of stream playlist parsing from
+  GStreamer to being handled by the Mopidy-Stream backend. (Fixes:
+  :issue:`1250`)
+
 - Stream: If "file" is present in the :confval:`stream/protocols` config value
   and the :ref:`ext-file` extension is enabled, we exited with an error because
   two extensions claimed the same URI scheme. We now log a warning recommending

mopidy/stream/actor.py

@@ -3,6 +3,7 @@ from __future__ import absolute_import, unicode_literals
 import fnmatch
 import logging
 import re
+import time
 import urlparse
 
 import pykka
@@ -80,22 +81,70 @@ class StreamPlaybackProvider(backend.PlaybackProvider):
                 stream.Extension.dist_name, stream.Extension.version))
 
     def translate_uri(self, uri):
-        try:
+        return unwrap_stream(
-            scan_result = self._scanner.scan(uri)
+            uri,
-        except exceptions.ScannerError as e:
+            timeout=self._config['stream']['timeout'],
-            logger.warning(
+            scanner=self._scanner,
-                'Problem scanning URI %s: %s', uri, e)
+            requests_session=self._session)
-            return None
 
-        if not (scan_result.mime.startswith('text/') or
+
+def unwrap_stream(uri, timeout, scanner, requests_session):
+    """
+    Get a stream URI from a playlist URI, ``uri``.
+
+    Unwraps nested playlists until something that's not a playlist is found or
+    the ``timeout`` is reached.
+    """
+
+    original_uri = uri
+    deadline = time.time() + timeout
+
+    while time.time() < deadline:
+        logger.debug('Unwrapping stream from URI: %s', uri)
+
+        try:
+            scan_timeout = deadline - time.time()
+            if scan_timeout < 0:
+                logger.info(
+                    'Unwrapping stream from URI (%s) failed: '
+                    'timed out in %sms',
+                    uri, timeout)
+                return None
+            scan_result = scanner.scan(uri, timeout=scan_timeout)
+        except exceptions.ScannerError as exc:
+            logger.debug('GStreamer failed scanning URI (%s): %s', uri, exc)
+            scan_result = None
+
+        if scan_result is not None and not (
+                scan_result.mime.startswith('text/') or
                 scan_result.mime.startswith('application/')):
+            logger.debug(
+                'Unwrapped potential %s stream: %s', scan_result.mime, uri)
             return uri
 
-        content = http.download(self._session, uri)
+        download_timeout = deadline - time.time()
+        if download_timeout < 0:
+            logger.info(
+                'Unwrapping stream from URI (%s) failed: timed out in %sms',
+                uri, timeout)
+            return None
+        content = http.download(
+            requests_session, uri, timeout=download_timeout)
+
         if content is None:
+            logger.info(
+                'Unwrapping stream from URI (%s) failed: '
+                'error downloading URI %s', original_uri, uri)
             return None
 
-        tracks = list(playlists.parse(content))
+        uris = playlists.parse(content)
-        if tracks:
+        if not uris:
-            # TODO Test streams and return first that seems to be playable
+            logger.debug(
-            return tracks[0]
+                'Failed parsing URI (%s) as playlist; found potential stream.',
+                uri)
+            return uri
+
+        # TODO Test streams and return first that seems to be playable
+        logger.debug(
+            'Parsed playlist (%s) and found new URI: %s', uri, uris[0])
+        uri = uris[0]

tests/stream/test_playback.py

@@ -12,7 +12,8 @@ from mopidy.stream import actor
 
 
 TIMEOUT = 1000
-URI = 'http://example.com/listen.m3u'
+PLAYLIST_URI = 'http://example.com/listen.m3u'
+STREAM_URI = 'http://example.com/stream.mp3'
 BODY = """
 #EXTM3U
 http://example.com/stream.mp3
@@ -37,9 +38,7 @@ def audio():
 
 @pytest.fixture
 def scanner():
-    scanner = mock.Mock(spec=scan.Scanner)
+    return mock.Mock(spec=scan.Scanner)
-    scanner.scan.return_value.mime = 'text/foo'
-    return scanner
 
 
 @pytest.fixture
@@ -55,58 +54,115 @@ def provider(audio, backend, config):
     return actor.StreamPlaybackProvider(audio, backend, config)
 
 
-@responses.activate
+class TestTranslateURI(object):
-def test_translate_uri_of_audio_stream_returns_same_uri(
-        scanner, provider):
 
-    scanner.scan.return_value.mime = 'audio/ogg'
+    @responses.activate
+    def test_audio_stream_returns_same_uri(self, scanner, provider):
+        scanner.scan.return_value.mime = 'audio/mpeg'
 
-    result = provider.translate_uri(URI)
+        result = provider.translate_uri(STREAM_URI)
 
-    scanner.scan.assert_called_once_with(URI)
+        scanner.scan.assert_called_once_with(STREAM_URI, timeout=mock.ANY)
-    assert result == URI
+        assert result == STREAM_URI
 
+    @responses.activate
+    def test_text_playlist_with_mpeg_stream(
+            self, scanner, provider, caplog):
 
-@responses.activate
+        scanner.scan.side_effect = [
-def test_translate_uri_of_playlist_returns_first_uri_in_list(
+            mock.Mock(mime='text/foo'),  # scanning playlist
-        scanner, provider):
+            mock.Mock(mime='audio/mpeg'),  # scanning stream
+        ]
+        responses.add(
+            responses.GET, PLAYLIST_URI,
+            body=BODY, content_type='audio/x-mpegurl')
 
-    responses.add(
+        result = provider.translate_uri(PLAYLIST_URI)
-        responses.GET, URI, body=BODY, content_type='audio/x-mpegurl')
 
-    result = provider.translate_uri(URI)
+        assert scanner.scan.mock_calls == [
+            mock.call(PLAYLIST_URI, timeout=mock.ANY),
+            mock.call(STREAM_URI, timeout=mock.ANY),
+        ]
+        assert result == STREAM_URI
 
-    scanner.scan.assert_called_once_with(URI)
+        # Check logging to ensure debuggability
-    assert result == 'http://example.com/stream.mp3'
+        assert 'Unwrapping stream from URI: %s' % PLAYLIST_URI
-    assert responses.calls[0].request.headers['User-Agent'].startswith(
+        assert 'Parsed playlist (%s)' % PLAYLIST_URI in caplog.text()
-        'Mopidy-Stream/')
+        assert 'Unwrapping stream from URI: %s' % STREAM_URI
+        assert (
+            'Unwrapped potential audio/mpeg stream: %s' % STREAM_URI
+            in caplog.text())
 
+        # Check proper Requests session setup
+        assert responses.calls[0].request.headers['User-Agent'].startswith(
+            'Mopidy-Stream/')
 
-@responses.activate
+    @responses.activate
-def test_translate_uri_of_playlist_with_xml_mimetype(scanner, provider):
+    def test_xml_playlist_with_mpeg_stream(self, scanner, provider):
-    scanner.scan.return_value.mime = 'application/xspf+xml'
+        scanner.scan.side_effect = [
-    responses.add(
+            mock.Mock(mime='application/xspf+xml'),  # scanning playlist
-        responses.GET, URI, body=BODY, content_type='application/xspf+xml')
+            mock.Mock(mime='audio/mpeg'),  # scanning stream
+        ]
+        responses.add(
+            responses.GET, PLAYLIST_URI,
+            body=BODY, content_type='application/xspf+xml')
 
-    result = provider.translate_uri(URI)
+        result = provider.translate_uri(PLAYLIST_URI)
 
-    scanner.scan.assert_called_once_with(URI)
+        assert scanner.scan.mock_calls == [
-    assert result == 'http://example.com/stream.mp3'
+            mock.call(PLAYLIST_URI, timeout=mock.ANY),
+            mock.call(STREAM_URI, timeout=mock.ANY),
+        ]
+        assert result == STREAM_URI
 
+    @responses.activate
+    def test_scan_fails_but_playlist_parsing_succeeds(
+            self, scanner, provider, caplog):
 
-def test_translate_uri_when_scanner_fails(scanner, provider, caplog):
+        scanner.scan.side_effect = [
-    scanner.scan.side_effect = exceptions.ScannerError('foo failed')
+            exceptions.ScannerError('some failure'),  # scanning playlist
+            mock.Mock(mime='audio/mpeg'),  # scanning stream
+        ]
+        responses.add(
+            responses.GET, PLAYLIST_URI,
+            body=BODY, content_type='audio/x-mpegurl')
 
-    result = provider.translate_uri('bar')
+        result = provider.translate_uri(PLAYLIST_URI)
 
-    assert result is None
+        assert 'Unwrapping stream from URI: %s' % PLAYLIST_URI
-    assert 'Problem scanning URI bar: foo failed' in caplog.text()
+        assert (
+            'GStreamer failed scanning URI (%s)' % PLAYLIST_URI
+            in caplog.text())
+        assert 'Parsed playlist (%s)' % PLAYLIST_URI in caplog.text()
+        assert (
+            'Unwrapped potential audio/mpeg stream: %s' % STREAM_URI
+            in caplog.text())
+        assert result == STREAM_URI
 
+    @responses.activate
+    def test_scan_fails_and_playlist_parsing_fails(
+            self, scanner, provider, caplog):
 
-def test_translate_uri_when_playlist_download_fails(provider, caplog):
+        scanner.scan.side_effect = exceptions.ScannerError('some failure')
-    with mock.patch.object(actor, 'http') as http_mock:
+        responses.add(
-        http_mock.download.return_value = None
+            responses.GET, STREAM_URI,
+            body=b'some audio data', content_type='audio/mpeg')
 
-        result = provider.translate_uri(URI)
+        result = provider.translate_uri(STREAM_URI)
 
-    assert result is None
+        assert 'Unwrapping stream from URI: %s' % STREAM_URI
+        assert (
+            'GStreamer failed scanning URI (%s)' % STREAM_URI
+            in caplog.text())
+        assert (
+            'Failed parsing URI (%s) as playlist; found potential stream.'
+            % STREAM_URI in caplog.text())
+        assert result == STREAM_URI
+
+    def test_failed_download_returns_none(self, provider, caplog):
+        with mock.patch.object(actor, 'http') as http_mock:
+            http_mock.download.return_value = None
+
+            result = provider.translate_uri(PLAYLIST_URI)
+
+        assert result is None