From d5cb4282d97068845fda65940bf09a84c7b100f4 Mon Sep 17 00:00:00 2001 From: Thomas Adamcik Date: Sun, 27 Oct 2013 11:38:01 +0100 Subject: [PATCH 1/2] config: Add preprocessor for preserving comments when editing configs. Adds markers to configs files that ensures configparser won't mangle comments in the files. Will be combined with a postprocessor that undoes these changes. --- mopidy/config/__init__.py | 37 ++++++++++++++ tests/config/config_test.py | 96 +++++++++++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+) diff --git a/mopidy/config/__init__.py b/mopidy/config/__init__.py index 0767b50c..8d68c7f3 100644 --- a/mopidy/config/__init__.py +++ b/mopidy/config/__init__.py @@ -2,8 +2,10 @@ from __future__ import unicode_literals import ConfigParser as configparser import io +import itertools import logging import os.path +import re from mopidy.config import keyring from mopidy.config.schemas import * # noqa @@ -145,6 +147,41 @@ def _format(config, comments, schemas, display): return b'\n'.join(output) +def _preprocess(string): + """Convert a raw config into a form that preserves comments etc.""" + results = ['[__COMMENTS__]'] + counter = itertools.count(0) + + section_re = re.compile(r'^(\[[^\]]+\])\s*(.+)$') + blank_line_re = re.compile(r'^\s*$') + comment_re = re.compile(r'^(#|;)') + inline_comment_re = re.compile(r' ;') + + def newlines(match): + return '__BLANK%d__ =' % next(counter) + + def comments(match): + if match.group(1) == '#': + return '__HASH%d__ =' % next(counter) + elif match.group(1) == ';': + return '__SEMICOLON%d__ =' % next(counter) + + def inlinecomments(match): + return '\n__INLINE%d__ =' % next(counter) + + def sections(match): + return '%s\n__SECTION%d__ = %s' % ( + match.group(1), next(counter), match.group(2)) + + for line in string.splitlines(): + line = blank_line_re.sub(newlines, line) + line = section_re.sub(sections, line) + line = comment_re.sub(comments, line) + line = inline_comment_re.sub(inlinecomments, line) + results.append(line) + return '\n'.join(results) + + class Proxy(collections.Mapping): def __init__(self, data): self._data = data diff --git a/tests/config/config_test.py b/tests/config/config_test.py index c40baa87..16769513 100644 --- a/tests/config/config_test.py +++ b/tests/config/config_test.py @@ -106,3 +106,99 @@ class ValidateTest(unittest.TestCase): self.assertEqual({'foo': {'bar': 'bad'}}, errors) # TODO: add more tests + + +INPUT_CONFIG = """# comments before first section should work + +[section] anything goes ; after the [] block it seems. +; this is a valid comment +this-should-equal-baz = baz ; as this is a comment +this-should-equal-everything = baz # as this is not a comment + +# this is also a comment ; and the next line should be a blank comment. +; +# foo # = should all be treated as a comment. +""" + +PROCESSED_CONFIG = """[__COMMENTS__] +__HASH0__ = comments before first section should work +__BLANK1__ = +[section] +__SECTION2__ = anything goes +__INLINE3__ = after the [] block it seems. +__SEMICOLON4__ = this is a valid comment +this-should-equal-baz = baz +__INLINE5__ = as this is a comment +this-should-equal-everything = baz # as this is not a comment +__BLANK6__ = +__HASH7__ = this is also a comment +__INLINE8__ = and the next line should be a blank comment. +__SEMICOLON9__ = +__HASH10__ = foo # = should all be treated as a comment.""" + + +class ProcessorTest(unittest.TestCase): + maxDiff = None # Show entire diff. + + def test_preprocessor_empty_config(self): + result = config._preprocess('') + self.assertEqual(result, '[__COMMENTS__]') + + def test_preprocessor_plain_section(self): + result = config._preprocess('[section]\nfoo = bar') + self.assertEqual(result, '[__COMMENTS__]\n' + '[section]\n' + 'foo = bar') + + def test_preprocessor_initial_comments(self): + result = config._preprocess('; foobar') + self.assertEqual(result, '[__COMMENTS__]\n' + '__SEMICOLON0__ = foobar') + + result = config._preprocess('# foobar') + self.assertEqual(result, '[__COMMENTS__]\n' + '__HASH0__ = foobar') + + result = config._preprocess('; foo\n# bar') + self.assertEqual(result, '[__COMMENTS__]\n' + '__SEMICOLON0__ = foo\n' + '__HASH1__ = bar') + + def test_preprocessor_initial_comment_inline_handling(self): + result = config._preprocess('; foo ; bar ; baz') + self.assertEqual(result, '[__COMMENTS__]\n' + '__SEMICOLON0__ = foo\n' + '__INLINE1__ = bar\n' + '__INLINE2__ = baz') + + def test_preprocessor_inline_semicolon_comment(self): + result = config._preprocess('[section]\nfoo = bar ; baz') + self.assertEqual(result, '[__COMMENTS__]\n' + '[section]\n' + 'foo = bar\n' + '__INLINE0__ = baz') + + def test_preprocessor_no_inline_hash_comment(self): + result = config._preprocess('[section]\nfoo = bar # baz') + self.assertEqual(result, '[__COMMENTS__]\n' + '[section]\n' + 'foo = bar # baz') + + def test_preprocessor_section_extra_text(self): + result = config._preprocess('[section] foobar') + self.assertEqual(result, '[__COMMENTS__]\n' + '[section]\n' + '__SECTION0__ = foobar') + + def test_preprocessor_section_extra_text_inline_semicolon(self): + result = config._preprocess('[section] foobar ; baz') + self.assertEqual(result, '[__COMMENTS__]\n' + '[section]\n' + '__SECTION0__ = foobar\n' + '__INLINE1__ = baz') + + def test_preprocessor_conversion(self): + """Tests all of the above cases at once.""" + result = config._preprocess(INPUT_CONFIG) + self.assertEqual(result, PROCESSED_CONFIG) + From 73f91710e14bc72c7de2f5ffc2b21042c43dc422 Mon Sep 17 00:00:00 2001 From: Thomas Adamcik Date: Sun, 27 Oct 2013 12:30:02 +0100 Subject: [PATCH 2/2] config: Add postprocessor for converting config back. Idea forward from here is that once we have a config sub command that we expose a setting config values which will: 1. Run the preprocessor on the file to edit. 2. Load it into config parser. 3. Modify the value. 4. Write it to a io.ByteString 5. Run the postprocessor 6. Save the file with comments etc intact. --- mopidy/config/__init__.py | 16 ++++++- tests/config/config_test.py | 89 +++++++++++++++++++++++++++++++------ 2 files changed, 90 insertions(+), 15 deletions(-) diff --git a/mopidy/config/__init__.py b/mopidy/config/__init__.py index 8d68c7f3..6d66e253 100644 --- a/mopidy/config/__init__.py +++ b/mopidy/config/__init__.py @@ -147,7 +147,7 @@ def _format(config, comments, schemas, display): return b'\n'.join(output) -def _preprocess(string): +def _preprocess(config_string): """Convert a raw config into a form that preserves comments etc.""" results = ['[__COMMENTS__]'] counter = itertools.count(0) @@ -173,7 +173,7 @@ def _preprocess(string): return '%s\n__SECTION%d__ = %s' % ( match.group(1), next(counter), match.group(2)) - for line in string.splitlines(): + for line in config_string.splitlines(): line = blank_line_re.sub(newlines, line) line = section_re.sub(sections, line) line = comment_re.sub(comments, line) @@ -182,6 +182,18 @@ def _preprocess(string): return '\n'.join(results) +def _postprocess(config_string): + """Converts a preprocessed config back to original form.""" + flags = re.IGNORECASE | re.MULTILINE + result = re.sub(r'^\[__COMMENTS__\](\n|$)', '', config_string, flags=flags) + result = re.sub(r'\n__INLINE\d+__ =(.*)$', ' ;\g<1>', result, flags=flags) + result = re.sub(r'^__HASH\d+__ =(.*)$', '#\g<1>', result, flags=flags) + result = re.sub(r'^__SEMICOLON\d+__ =(.*)$', ';\g<1>', result, flags=flags) + result = re.sub(r'\n__SECTION\d+__ =(.*)$', '\g<1>', result, flags=flags) + result = re.sub(r'^__BLANK\d+__ =$', '', result, flags=flags) + return result + + class Proxy(collections.Mapping): def __init__(self, data): self._data = data diff --git a/tests/config/config_test.py b/tests/config/config_test.py index 16769513..fceb293d 100644 --- a/tests/config/config_test.py +++ b/tests/config/config_test.py @@ -117,8 +117,7 @@ this-should-equal-everything = baz # as this is not a comment # this is also a comment ; and the next line should be a blank comment. ; -# foo # = should all be treated as a comment. -""" +# foo # = should all be treated as a comment.""" PROCESSED_CONFIG = """[__COMMENTS__] __HASH0__ = comments before first section should work @@ -137,20 +136,20 @@ __SEMICOLON9__ = __HASH10__ = foo # = should all be treated as a comment.""" -class ProcessorTest(unittest.TestCase): - maxDiff = None # Show entire diff. +class PreProcessorTest(unittest.TestCase): + maxDiff = None # Show entire diff. - def test_preprocessor_empty_config(self): + def test_empty_config(self): result = config._preprocess('') self.assertEqual(result, '[__COMMENTS__]') - def test_preprocessor_plain_section(self): + def test_plain_section(self): result = config._preprocess('[section]\nfoo = bar') self.assertEqual(result, '[__COMMENTS__]\n' '[section]\n' 'foo = bar') - def test_preprocessor_initial_comments(self): + def test_initial_comments(self): result = config._preprocess('; foobar') self.assertEqual(result, '[__COMMENTS__]\n' '__SEMICOLON0__ = foobar') @@ -164,41 +163,105 @@ class ProcessorTest(unittest.TestCase): '__SEMICOLON0__ = foo\n' '__HASH1__ = bar') - def test_preprocessor_initial_comment_inline_handling(self): + def test_initial_comment_inline_handling(self): result = config._preprocess('; foo ; bar ; baz') self.assertEqual(result, '[__COMMENTS__]\n' '__SEMICOLON0__ = foo\n' '__INLINE1__ = bar\n' '__INLINE2__ = baz') - def test_preprocessor_inline_semicolon_comment(self): + def test_inline_semicolon_comment(self): result = config._preprocess('[section]\nfoo = bar ; baz') self.assertEqual(result, '[__COMMENTS__]\n' '[section]\n' 'foo = bar\n' '__INLINE0__ = baz') - def test_preprocessor_no_inline_hash_comment(self): + def test_no_inline_hash_comment(self): result = config._preprocess('[section]\nfoo = bar # baz') self.assertEqual(result, '[__COMMENTS__]\n' '[section]\n' 'foo = bar # baz') - def test_preprocessor_section_extra_text(self): + def test_section_extra_text(self): result = config._preprocess('[section] foobar') self.assertEqual(result, '[__COMMENTS__]\n' '[section]\n' '__SECTION0__ = foobar') - def test_preprocessor_section_extra_text_inline_semicolon(self): + def test_section_extra_text_inline_semicolon(self): result = config._preprocess('[section] foobar ; baz') self.assertEqual(result, '[__COMMENTS__]\n' '[section]\n' '__SECTION0__ = foobar\n' '__INLINE1__ = baz') - def test_preprocessor_conversion(self): + def test_conversion(self): """Tests all of the above cases at once.""" result = config._preprocess(INPUT_CONFIG) self.assertEqual(result, PROCESSED_CONFIG) + +class PostProcessorTest(unittest.TestCase): + maxDiff = None # Show entire diff. + + def test_empty_config(self): + result = config._postprocess('[__COMMENTS__]') + self.assertEqual(result, '') + + def test_plain_section(self): + result = config._postprocess('[__COMMENTS__]\n' + '[section]\n' + 'foo = bar') + self.assertEqual(result, '[section]\nfoo = bar') + + def test_initial_comments(self): + result = config._postprocess('[__COMMENTS__]\n' + '__SEMICOLON0__ = foobar') + self.assertEqual(result, '; foobar') + + result = config._postprocess('[__COMMENTS__]\n' + '__HASH0__ = foobar') + self.assertEqual(result, '# foobar') + + result = config._postprocess('[__COMMENTS__]\n' + '__SEMICOLON0__ = foo\n' + '__HASH1__ = bar') + self.assertEqual(result, '; foo\n# bar') + + def test_initial_comment_inline_handling(self): + result = config._postprocess('[__COMMENTS__]\n' + '__SEMICOLON0__ = foo\n' + '__INLINE1__ = bar\n' + '__INLINE2__ = baz') + self.assertEqual(result, '; foo ; bar ; baz') + + def test_inline_semicolon_comment(self): + result = config._postprocess('[__COMMENTS__]\n' + '[section]\n' + 'foo = bar\n' + '__INLINE0__ = baz') + self.assertEqual(result, '[section]\nfoo = bar ; baz') + + def test_no_inline_hash_comment(self): + result = config._preprocess('[section]\nfoo = bar # baz') + self.assertEqual(result, '[__COMMENTS__]\n' + '[section]\n' + 'foo = bar # baz') + + def test_section_extra_text(self): + result = config._postprocess('[__COMMENTS__]\n' + '[section]\n' + '__SECTION0__ = foobar') + self.assertEqual(result, '[section] foobar') + + def test_section_extra_text_inline_semicolon(self): + result = config._postprocess('[__COMMENTS__]\n' + '[section]\n' + '__SECTION0__ = foobar\n' + '__INLINE1__ = baz') + self.assertEqual(result, '[section] foobar ; baz') + + def test_conversion(self): + result = config._postprocess(PROCESSED_CONFIG) + self.assertEqual(result, INPUT_CONFIG)