Automate our changelog generation

.gitignore

@@ -12,3 +12,4 @@ npm-debug.log
 !yarn.lock
 coverage/
 .vscode-test/**
+.venv

.travis.yml

@@ -4,9 +4,9 @@ cache: pip
 matrix:
     include:
         - os: linux
-          python: 2.7
+          python: "2.7"
         - os: linux
-          python: 3.6
+          python: "3.6-dev"
 before_install: |
   if [ $TRAVIS_OS_NAME == "linux" ]; then
     export CXX="g++-4.9" CC="gcc-4.9" DISPLAY=:99.0;
@@ -47,6 +47,10 @@ script:
   - if [ $TRAVIS_UPLOAD_COVERAGE == "true" ]; then
         bash <(curl -s https://codecov.io/bash);
     fi
+  - if [ "$TRAVIS_PYTHON_VERSION" != "2.7" ]; then
+        python3 -m pip install --upgrade -r news/requirements.txt;
+        python3 news/announce.py --dry-run;
+    fi
 after_success:
   - if [ $AZURE_STORAGE_ACCOUNT ]; then
         yarn run clean;

news/1 Enhancements/README.md

@@ -0,0 +1 @@
+Changes that add new features.

news/2 Fixes/README.md

@@ -0,0 +1 @@
+Changes that fix broken behaviour.

news/3 Code Health/README.md

@@ -0,0 +1 @@
+Changes that should not be user-facing.

news/README.md

@@ -0,0 +1,56 @@
+# News
+
+Our changelog is automatically generate from individual news entries.
+This alleviates the burden of having to go back and try to figure out
+what changed in a release. It also helps tie pull requests back to the
+issue(s) it addresses.
+
+## Entries
+
+Each news entries is represented by a Markdown file that contains the
+relevant details of what changed. The file name of the news entry is
+the issue that corresponds to the change along with an option nonce in
+case a single issue corresponds to multiple changes. The directory
+the news entry is saved in specifies what section of the changelog the
+change corresponds to.
+
+As an example, a change corresponding to a bug reported in issue #42
+would be saved in the `1 Fixes` directory and named `42.md`
+(or `42-nonce_value.md` if there was a need for multiple entries
+regarding issue #42) and could contain the following:
+
+```markdown
+[Answer](https://en.wikipedia.org/wiki/42_(number))
+to the Ultimate Question of Life, the Universe, and Everything!
+```
+
+This would then be made into an entry in the changelog that was in the
+`Fixes` section, contained the details as found in the file, and tied
+to issue #42.
+
+## Generating the changelog
+
+The `announce` script can do 3 possible things:
+
+1. Validate that the changelog _could_ be successfully generated
+2. Generate the changelog entry
+3. Generate the changelog entry **and** `git-rm` the news entries
+
+The first option is used in CI to make sure any added news entries
+will not cause trouble at release time. The second option is for
+filling in the changelog for interim releases, e.g. a beta release.
+The third option is for final releases that get published to the
+[VS Code marketplace](https://marketplace.visualstudio.com/VSCode).
+
+For options 2 & 3, the changelog is sent to stdout so it's temporarily
+saved to a file:
+
+```sh
+python3 news/announce.py > entry.txt
+```
+
+It can also be redirected to a file or into an editor buffer, e.g.:
+
+```sh
+python3 news/announce.py --final | code-insiders
+```

news/announce.py

@@ -0,0 +1,126 @@
+"""Generate the changelog."""
+import enum
+import operator
+import os
+import pathlib
+import re
+import subprocess
+import types
+
+import click
+
+
+FILENAME_RE = re.compile(r"(?P<issue>\d+)(?P<nonce>-\S+)?\.md")
+ISSUE_URL = "https://github.com/Microsoft/vscode-python/issues/{issue}"
+ENTRY_TEMPLATE = "- {entry} ([#{issue}]({issue_url}))"
+SECTION_DEPTH = "###"
+
+
+def NewsEntry(issue_number, description, path):
+    """Construct a data object for a news entry."""
+    # TODO: replace with a dataclass in Python 3.7.
+    return types.SimpleNamespace(issue_number=issue_number,
+                                 description=description, path=path)
+
+
+def news_entries(directory):
+    """Yield news entries in the directory."""
+    for path in directory.iterdir():
+        if path.name == 'README.md':
+            continue
+        match = FILENAME_RE.match(path.name)
+        if match is None:
+            raise ValueError(f'{path} has a bad file name')
+        issue = int(match.group('issue'))
+        entry = path.read_text("utf-8")
+        yield NewsEntry(issue, entry, path)
+
+
+def SectionTitle(index, title, path):
+    """Create a data object for a section of the changelog."""
+    # TODO: replace with a dataclass in Python 3.7.
+    return types.SimpleNamespace(index=index, title=title, path=path)
+
+
+def sections(directory):
+    """Yield the sections in their appropriate order."""
+    found = []
+    for path in directory.iterdir():
+        if not path.is_dir():
+            continue
+        position, sep, title = path.name.partition(' ')
+        if not sep:
+            raise ValueError('directory is missing position part')
+        found.append(SectionTitle(int(position), title, path))
+    return sorted(found, key=operator.attrgetter('index'))
+
+
+def gather(directory):
+    """Gather all the entries together."""
+    data = []
+    for section in sections(directory):
+        data.append((section, list(news_entries(section.path))))
+    return data
+
+
+def entry_markdown(entry):
+    """Generate the Markdown for the specified entry."""
+    issue_url = ISSUE_URL.format(issue=entry.issue_number)
+    return ENTRY_TEMPLATE.format(entry=entry.description,
+                                 issue=entry.issue_number,
+                                 issue_url=issue_url)
+
+
+def changelog_markdown(data):
+    """Generate the Markdown for the release."""
+    changelog = []
+    for section, entries in data:
+        changelog.append(f"{SECTION_DEPTH} {section.title}")
+        changelog.append("")
+        changelog.extend(map(entry_markdown, entries))
+        changelog.append("")
+    return "\n".join(changelog)
+
+
+def git_rm(path):
+    """Run git-rm on the path."""
+    status = subprocess.run(['git', 'rm', os.fspath(path.resolve())],
+                            shell=True)
+    status.check_returncode()
+
+
+def cleanup(data):
+    """Remove news entries from git and disk."""
+    for section, entries in data:
+        for entry in entries:
+            git_rm(entry.path)
+
+
+class RunType(enum.Enum):
+    """Possible run-time options."""
+
+    dry_run = 0
+    interim = 1
+    final = 2
+
+
+@click.command()
+@click.option('--dry-run', 'run_type', flag_value=RunType.dry_run,
+              help='validate input')
+@click.option('--interim', 'run_type', flag_value=RunType.interim, default=True,
+              help='generate Markdown')
+@click.option('--final', 'run_type', flag_value=RunType.final,
+              help='generate Markdown & `git rm` news files')
+@click.argument('directory', default=pathlib.Path(__file__).parent,
+                type=click.Path(exists=True, file_okay=False))
+def main(run_type, directory):
+    data = gather(directory)
+    markdown = changelog_markdown(data)
+    if run_type != RunType.dry_run:
+        print(markdown)
+    if run_type == RunType.final:
+        cleanup(data)
+
+
+if __name__ == '__main__':
+    main()

news/requirements.txt

@@ -0,0 +1,2 @@
+click~=6.7.0
+pytest~=3.4.1

news/test_announce.py

@@ -0,0 +1,128 @@
+import pathlib
+
+import pytest
+
+import announce as ann
+
+
+@pytest.fixture
+def directory(tmpdir):
+    """Fixture to create a temp directory wrapped in a pathlib.Path object."""
+    return pathlib.Path(tmpdir)
+
+
+def test_news_entry_formatting(directory):
+    issue = 42
+    normal_entry = directory / f'{issue}.md'
+    nonce_entry = directory / f'{issue}-nonce.md'
+    body = 'Hello, world!'
+    normal_entry.write_text(body, encoding='utf-8')
+    nonce_entry.write_text(body, encoding='utf-8')
+    results = list(ann.news_entries(directory))
+    assert len(results) == 2
+    for result in results:
+        assert result.issue_number == issue
+        assert result.description == body
+
+
+def test_bad_news_entry_file_name(directory):
+    entry = directory / 'bunk.md'
+    entry.write_text('Hello, world!')
+    with pytest.raises(ValueError):
+        list(ann.news_entries(directory))
+
+def test_news_entry_README_skipping(directory):
+    entry = directory / 'README.md'
+    entry.write_text('Hello, world!')
+    assert len(list(ann.news_entries(directory))) == 0
+
+
+def test_sections_sorting(directory):
+    dir2 = directory / '2 Hello'
+    dir1 = directory / '1 World'
+    dir2.mkdir()
+    dir1.mkdir()
+    results = list(ann.sections(directory))
+    assert [found.title for found in results] == ['World', 'Hello']
+
+
+def test_sections_naming(directory):
+    (directory / 'Hello').mkdir()
+    with pytest.raises(ValueError):
+        list(ann.sections(directory))
+
+
+def test_gather(directory):
+    fixes = directory / '2 Fixes'
+    fixes.mkdir()
+    fix1 = fixes / '1.md'
+    fix1.write_text('Fix 1', encoding='utf-8')
+    fix2 = fixes / '3.md'
+    fix2.write_text('Fix 2', encoding='utf-8')
+    enhancements = directory / '1 Enhancements'
+    enhancements.mkdir()
+    enhancement1 = enhancements / '2.md'
+    enhancement1.write_text('Enhancement 1', encoding='utf-8')
+    enhancement2 = enhancements / '4.md'
+    enhancement2.write_text('Enhancement 2', encoding='utf-8')
+    results = ann.gather(directory)
+    assert len(results) == 2
+    section, entries = results[0]
+    assert section.title == 'Enhancements'
+    assert len(entries) == 2
+    assert entries[0].description == 'Enhancement 1'
+    assert entries[1].description == 'Enhancement 2'
+    section, entries = results[1]
+    assert len(entries) == 2
+    assert section.title == 'Fixes'
+    assert entries[0].description == 'Fix 1'
+    assert entries[1].description == 'Fix 2'
+
+
+
+
+def test_entry_markdown():
+    markdown = ann.entry_markdown(ann.NewsEntry(42, 'Hello, world!', None))
+    assert '42' in markdown
+    assert 'Hello, world!' in markdown
+    assert 'https://github.com/Microsoft/vscode-python/issues/42' in markdown
+
+
+def test_changelog_markdown():
+    data = [(ann.SectionTitle(1, 'Enhancements', None),
+                [ann.NewsEntry(2, 'Enhancement 1', None),
+                 ann.NewsEntry(4, 'Enhancement 2', None)]),
+            (ann.SectionTitle(1, 'Fixes', None),
+                [ann.NewsEntry(1, 'Fix 1', None),
+                 ann.NewsEntry(3, 'Fix 2', None)])]
+    markdown = ann.changelog_markdown(data)
+    assert '### Enhancements' in markdown
+    assert '### Fixes' in markdown
+    assert '1' in markdown
+    assert 'Fix 1' in markdown
+    assert '2' in markdown
+    assert 'Enhancement 1' in markdown
+    assert 'https://github.com/Microsoft/vscode-python/issues/2' in markdown
+    assert '3' in markdown
+    assert 'Fix 2' in markdown
+    assert 'https://github.com/Microsoft/vscode-python/issues/3' in markdown
+    assert '4' in markdown
+    assert 'Enhancement 2' in markdown
+
+
+def test_cleanup(directory, monkeypatch):
+    rm_path = None
+    def fake_git_rm(path):
+        nonlocal rm_path
+        rm_path = path
+    monkeypatch.setattr(ann, 'git_rm', fake_git_rm)
+    fixes = directory / '2 Fixes'
+    fixes.mkdir()
+    fix1 = fixes / '1.md'
+    fix1.write_text('Fix 1', encoding='utf-8')
+    results = ann.gather(directory)
+    assert len(results) == 1
+    ann.cleanup(results)
+    section, entries = results.pop()
+    assert len(entries) == 1
+    assert rm_path == entries[0].path