Changelog Tool Overhaul (#24699)

* Changelog thing overhaul. No actual changes to the changelog format or to the changelog itself, just a bunch of fixes.

* Clarify

* FCK

* Shebang
This commit is contained in:
Rob Nelson
2019-11-04 19:34:31 -08:00
committed by Kurfursten
parent 8e738f0334
commit 3aa3ab4261
7 changed files with 381 additions and 230 deletions

View File

@@ -14,3 +14,7 @@ indent_style = space
[*.txt]
insert_final_newline = false
[*.yml]
indent_size = 2
indent_style = space

View File

@@ -0,0 +1,7 @@
author: N3X15
changes:
- tweak: Changelog tool reworked to be slightly less shit, use python 3.6 and proper logging.
- tweak: Changelog tool now uses Jinja2 to render actual, whole-assed templates.
- tweak: Changelog templates coalesced. Still use XHTML 4.01 Transitional because BYOND is shit.
- bugfix: Fixed some minor low-threat XSS vulnerabilities in changelogs that caused occasional formatting issues.
- rscadd: Validation schemas added for changelogs, for the poor tortured souls that desire them for whatever godforsaken reason.

View File

@@ -1,3 +1,89 @@
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
<!--
HEY, YOU.
ADDING A CHANGELOG ENTRY? MAKE changelog/USERNAME.yml. READ example.yml SO YOU DON'T SCREW UP.
ADDING CREDITS? EDIT templates/changelog.tmpl.html
--->
<html>
<head>
<title>/vg/station Changelog</title>
<link rel="stylesheet" type="text/css" href="changelog.css">
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
</head>
<body>
<!--
Header Section
-->
<div class="header">
<h1>/vg/station - A Space Station 13 Server</h1>
<p>
<b>Forum | <a href="http://ss13.moe/wiki/index.php/Main_Page">Wiki</a> | <a href="https://github.com/vgstation-coders/vgstation13">Source</a></b>
</p>
<p>
<b>Visit our IRC channel:</b><a href="irc://irc.rizon.net/vgstation">#vgstation on irc.rizon.net</a>
</p>
<p>
<em>Code licensed under <a href="http://www.gnu.org/licenses/gpl.html">GPLv3</a>. Content licensed under <a href="http://creativecommons.org/licenses/by-sa/3.0/">CC BY-SA 3.0</a>.</em>
</p>
<h2>/vg/station 13 Credits</h2>
<dl class="creditblock">
<dt class="creditsection">
Code:
</dt>
<dd>
9600bauds, Cloroxygen, Clusterfack, ComicIronic, Deity Link, Duny, Dylanstrategie, Emisune, Exxion, IconLeap, Iamgoofball, Intigracy, Kurfursten, N3X15, PJB3005, Pomf123, SarahJohnson, Shadowmech88, Sood, Unid, Velard Amakar, wwjnc
</dd>
<dt class="creditsection">
Sprites:
</dt>
<dd>
Blithering, Bustatime, Cloroxygen, Cogwerks, Dbuhos, Deity Link, Emisune, Intigracy, ISaidNo, Kokuten, N3X15, NigglyWiggly, Osaifh, Rei1226, Shadowmech88, Skowron
</dd>
<dt class="creditsection">
Sounds:
</dt>
<dd>
Deity Link, IratePirate, Railfist, Zth
</dd>
<dt class="creditsection">
Mapping:
</dt>
<dd>
Burneddi, BurntDevil, ComicIronic, CptWad, Duny, dylanstrategie, GeneralVeers25, IratePirate, MrSegi, PJB3005, Pomf123, Probe1, xpcybic
</dd>
<dt class="creditsection">
Thanks To:
</dt>
<dd>
Baystation, Festival TTS, /tg/ station, Goonstation, Animus Station, Daedalus, anyone we forgot above, and the original Spacestation 13 devs. Skibiliano for the IRC bot. And you, without whom there'd be no point to all this work.
</dd>
</dl>
</div>
<div class="testserver">
Visit the Bleeding-Edge test server at <a href="byond://pomftest.undo.it:7777">byond://pomftest.undo.it:7777</a>
</div>
<!--
AGAIN, TO ADD SHIT, ADD AND MAINTAIN YOUR OWN changelog/USERNAME.yml FILE.
*** DO NOT FUCK WITH THIS FILE OR YOU WILL CAUSE MERGE CONFLICTS. ***
-->
<div class="commit sansserif">
{%- for date, committers in ENTRIES.items() %}
<h2 class="date">{{ date.strftime(DATEFORMAT) }}</h2>
{%- for author, changes in committers.items() %}
<h3 class="author">{{ author }} updated:</h3>
<ul class="changes bgimages16">
{%- for css_class, change in changes %}
<li class="{{- css_class|safe -}}">{{- change -}}</li>
{%- endfor %}
</ul>
{%- endfor -%}
{%- endfor %}
</div>
<!--
Credits Section

View File

@@ -1,87 +0,0 @@
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
<!--
HEY, YOU.
ADDING A CHANGELOG ENTRY? MAKE changelog/USERNAME.yml. READ example.yml SO YOU DON'T SCREW UP.
ADDING CREDITS? EDIT templates/header.html (or footer.html if they're for old teams)
--->
<html>
<head>
<title>/vg/station Changelog</title>
<link rel="stylesheet" type="text/css" href="changelog.css">
<script type='text/javascript'>
function changeText(tagID, newText, linkTagID) {
var tag = document.getElementById(tagID);
tag.innerHTML = newText;
var linkTag = document.getElementById(linkTagID);
linkTag.removeAttribute("href");
linkTag.removeAttribute("onclick");
}
</script>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
</head>
<body>
<!--
Header Section
-->
<div class="header">
<h1>/vg/station - A Space Station 13 Server</h1>
<p>
<b>Forum | <a href="http://ss13.moe/wiki/index.php/Main_Page">Wiki</a> | <a href="https://github.com/vgstation-coders/vgstation13">Source</a></b>
</p>
<p>
<b>Visit our IRC channel:</b><a href="irc://irc.rizon.net/vgstation">#vgstation on irc.rizon.net</a>
</p>
<p>
<em>Code licensed under <a href="http://www.gnu.org/licenses/gpl.html">GPLv3</a>. Content licensed under <a href="http://creativecommons.org/licenses/by-sa/3.0/">CC BY-SA 3.0</a>.</em>
</p>
<h2>/vg/station 13 Credits</h2>
<dl class="creditblock">
<dt class="creditsection">
Code:
</dt>
<dd>
9600bauds, Cloroxygen, Clusterfack, ComicIronic, Deity Link, Duny, Dylanstrategie, Emisune, Exxion, IconLeap, Iamgoofball, Intigracy, Kurfursten, N3X15, PJB3005, Pomf123, SarahJohnson, Shadowmech88, Sood, Unid, Velard Amakar, wwjnc
</dd>
<dt class="creditsection">
Sprites:
</dt>
<dd>
Blithering, Bustatime, Cloroxygen, Cogwerks, Dbuhos, Deity Link, Emisune, Intigracy, ISaidNo, Kokuten, N3X15, NigglyWiggly, Osaifh, Rei1226, Shadowmech88, Skowron
</dd>
<dt class="creditsection">
Sounds:
</dt>
<dd>
Deity Link, IratePirate, Railfist, Zth
</dd>
<dt class="creditsection">
Mapping:
</dt>
<dd>
Burneddi, BurntDevil, ComicIronic, CptWad, Duny, dylanstrategie, GeneralVeers25, IratePirate, MrSegi, PJB3005, Pomf123, Probe1, xpcybic
</dd>
<dt class="creditsection">
Thanks To:
</dt>
<dd>
Baystation, Festival TTS, /tg/ station, Goonstation, Animus Station, Daedalus, anyone we forgot above, and the original Spacestation 13 devs. Skibiliano for the IRC bot. And you, without whom there'd be no point to all this work.
</dd>
</dl>
</div>
<div class="testserver">
Visit the Bleeding-Edge test server at <a href="byond://pomftest.undo.it:7777">byond://pomftest.undo.it:7777</a>
</div>
<!--
AGAIN, TO ADD SHIT, ADD AND MAINTAIN YOUR OWN changelog/USERNAME.yml FILE.
*** DO NOT FUCK WITH THIS FILE OR YOU WILL CAUSE MERGE CONFLICTS. ***
-->
<div class="commit sansserif">

67
tools/changelog/README.md Normal file
View File

@@ -0,0 +1,67 @@
This ugly mess is the changelog generator originally made by N3X15 for /vg/station and now used widely throughout SS13's community.
It has been updated in late 2019 for general code cleanup and to add better templating options.
You are free to copy, use, and modify it under the terms of the MIT License agreement.
## Prerequisites
* Python &gt;= 3.6 - Sorry, Python2.7 is dead as of 2020. Buck up.
* pyyaml &gt;= 5.1 - Older versions have a security vulnerability and are now incompatible.
* jinja2 - Since we're already using pip, we might as well use an actual templating library
* BeautifulSoup 4 - Only needed if you're upgrading ye olde HTML templates.
```shell
# To install most of these:
$ pip3.6 install -U pyyaml jinja2 beautifulsoup4
```
## Usage
```shell
$ python3.6 ss13_genchangelog.py --help
```
```
usage: ss13_genchangelog.py [-h] [-d] targetFile ymlDir
positional arguments:
targetFile The HTML changelog we wish to update.
ymlDir The directory of YAML changelogs we will use.
optional arguments:
-h, --help show this help message and exit
-d, --dry-run Only parse changelogs and, if needed, the targetFile. (A
.dry_changelog.yml will be output for debugging purposes.)
```
## Changelogs
Changelogs are fairly simple, by design:
```yaml
author: AUTHOR'S NAME
changes:
- rscadd: A thing I added
- rscdel: A thing I removed
- bugfix: A bugfix I made
```
The prefixes (`rscadd`, etc) correspond to CSS classes that add shit like icons to the front of the change entry to indicate what kind of change it was. For a list, see the top of [ss13_genchangelog.py](ss13_genchangelog.py).
### Validating Changelogs with Schemas
**NOTE:** This is for advanced users only. You don't need to bother with this if you don't want to.
For Continuous Integration or IDE use, a [changelog schema](changelog.schema.yml) is available.
This is a JSON Schema represented as YAML and can be used with a tool like `pajv`:
```shell
# Install pajv with npm
sudo npm install -g pajv
# Validate a changelog with it
pajv -s tools/changelog/changelog.schema.yml -d html/changelogs/example.yml
```
```
html/changelogs/example.yml valid
```

View File

@@ -0,0 +1,44 @@
# This is a JSON Schema represented in YAML because YAML doesn't have decent schemas.
# You can use this to validate your changelogs using a tool like pajv.
$schema: 'http://json-schema.org/draft-06/schema#'
$id: changelog.schema.yml
title: Changelog
author: N3X15
description: A changelog entry block.
type: object
required:
- author
- changes
additionalProperties: false
properties:
author:
type: string
delete-after:
anyOf:
- type: boolean
- enum:
- yes
- no
changes:
type: array
minItems: 1
items:
type: object
minProperties: 1
maxProperties: 1
additionalProperties:
type: string
propertyNames:
enum:
- bugfix
- wip
- tweak
- soundadd
- sounddel
- rscdel
- rscadd
- imageadd
- imagedel
- spellcheck
- experiment
- tgs

View File

@@ -1,12 +1,15 @@
#!/usr/bin/env python2
#!/usr/bin/env python3
'''
Dependencies:
Beautiful Soup 4 (for now)
PyYAML
Jinja2
Usage:
$ python ss13_genchangelog.py [--dry-run] html/changelog.html html/changelogs/
ss13_genchangelog.py - Generate changelog from YAML.
Copyright (C) 2013-2015 Rob "N3X15" Nelson <nexis@7chan.org>
Copyright (C) 2013-2019 Rob "N3X15" Nelson <nexisentertainment@gmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
@@ -27,87 +30,55 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
'''
from __future__ import print_function
import yaml
import os
import glob
import sys
import re
import time
import argparse
from datetime import datetime, date
if sys.version_info[0] < 3:
raise Exception("Must be using Python 3")
if sys.version_info.minor < 6:
raise Exception("Must be using Python >= 3.6")
from time import time
import argparse
import collections
import datetime
import glob
import jinja2
import logging
import os
import yaml
today = date.today()
MAX_DATE_ENTRIES = 100 # So changelog isn't 5000 entries long
dateformat = "%Y.%m.%d"
opt = argparse.ArgumentParser()
opt.add_argument('-d', '--dry-run', dest='dryRun', default=False, action='store_true', help='Only parse changelogs and, if needed, the targetFile. (A .dry_changelog.yml will be output for debugging purposes.)')
opt.add_argument('targetFile', help='The HTML changelog we wish to update.')
opt.add_argument('ymlDir', help='The directory of YAML changelogs we will use.')
args = opt.parse_args()
all_changelog_entries = {}
validPrefixes = [
# Valid entry types. Update changelog.schema.yml if you fuck with this.
VALID_PREFIXES = [
'bugfix',
'wip',
'tweak',
'soundadd',
'sounddel',
'rscdel',
'rscadd',
'experiment',
'imageadd',
'imagedel',
'rscadd',
'rscdel',
'soundadd',
'sounddel',
'spellcheck',
'experiment',
'tgs'
'tgs',
'tweak',
'wip',
]
# So changelog isn't 5000 entries long
MAX_DATE_ENTRIES = 100
def dictToTuples(inp):
return [(k, v) for k, v in inp.items()]
# Date format to use.
DATEFORMAT = "%Y.%m.%d"
changelog_cache = os.path.join(args.ymlDir, '.all_changelog.yml')
failed_cache_read = True
if os.path.isfile(changelog_cache):
try:
with open(changelog_cache) as f:
(_, all_changelog_entries) = yaml.safe_load_all(f)
failed_cache_read = False
# Convert old timestamps to newer format.
new_entries = {}
for _date in all_changelog_entries.keys():
ty = type(_date).__name__
# print(ty)
if ty in ['str', 'unicode']:
temp_data = all_changelog_entries[_date]
_date = datetime.strptime(_date, dateformat).date()
new_entries[_date] = temp_data
else:
new_entries[_date] = all_changelog_entries[_date]
all_changelog_entries = new_entries
except Exception as e:
print("Failed to read cache:")
print(e, file=sys.stderr)
if args.dryRun:
changelog_cache = os.path.join(args.ymlDir, '.dry_changelog.yml')
if failed_cache_read and os.path.isfile(args.targetFile):
def rebuild_changelog_from(old_changelog: str, all_changelog_entries: dict):
from bs4 import BeautifulSoup
from bs4.element import NavigableString
print(' Generating cache...')
with open(args.targetFile, 'r') as f:
soup = BeautifulSoup(f)
#from bs4.element import NavigableString
logging.info('Generating cache...')
with open(old_changelog, 'r') as f:
soup = BeautifulSoup(f, features='lxml')
# Thankfully, old-style changelogs used a fairly standardized layout.
for e in soup.find_all('div', {'class': 'commit'}):
entry = {}
date = datetime.strptime(e.h2.string.strip(), dateformat).date() # key
# I love how redundant Python's STL is.
date = datetime.datetime.strptime(e.h2.string.strip(), DATEFORMAT).date() # key
for authorT in e.find_all('h3', {'class': 'author'}):
author = authorT.string
# Strip suffix
@@ -136,99 +107,158 @@ if failed_cache_read and os.path.isfile(args.targetFile):
else:
all_changelog_entries[date] = entry
del_after = []
print('Reading changelogs...')
for fileName in glob.glob(os.path.join(args.ymlDir, "*.yml")):
name, ext = os.path.splitext(os.path.basename(fileName))
if name.startswith('.'):
continue
if name == 'example':
continue
fileName = os.path.abspath(fileName)
print(' Reading {}...'.format(fileName))
cl = {}
with open(fileName, 'r') as f:
cl = yaml.safe_load(f)
f.close()
if today not in all_changelog_entries:
all_changelog_entries[today] = {}
author_entries = all_changelog_entries[today].get(cl['author'], [])
if len(cl['changes']):
new = 0
for change in cl['changes']:
if change not in author_entries:
(change_type, _) = dictToTuples(change)[0]
if change_type not in validPrefixes:
print(' {0}: Invalid prefix {1}'.format(fileName, change_type), file=sys.stderr)
author_entries += [change]
new += 1
all_changelog_entries[today][cl['author']] = author_entries
if new > 0:
print(' Added {0} new changelog entries.'.format(new))
def loadCache(changelog_cachefile):
failed_cache_read = True
all_changelog_entries = {}
try:
with open(changelog_cachefile) as f:
(_, all_changelog_entries) = yaml.safe_load_all(f)
failed_cache_read = False
if cl.get('delete-after', False):
if os.path.isfile(fileName):
if args.dryRun:
print(' Would delete {0} (delete-after set)...'.format(fileName))
else:
del_after += [fileName]
# Convert old timestamps to newer format.
new_entries = {}
for _date in all_changelog_entries.keys():
ty = type(_date).__name__
# print(ty)
if ty in ['str', 'unicode']:
temp_data = all_changelog_entries[_date]
_date = datetime.datetime.strptime(_date, DATEFORMAT).date()
new_entries[_date] = temp_data
else:
new_entries[_date] = all_changelog_entries[_date]
all_changelog_entries = new_entries
except Exception as e:
logging.error("Failed to read cache:")
logging.exception(e)
return (failed_cache_read, all_changelog_entries)
if args.dryRun:
continue
def main():
opt = argparse.ArgumentParser()
opt.add_argument('-d', '--dry-run', dest='dryRun', default=False, action='store_true', help='Only parse changelogs and, if needed, the targetFile. (A .dry_changelog.yml will be output for debugging purposes.)')
opt.add_argument('targetFile', help='The HTML changelog we wish to update.')
opt.add_argument('ymlDir', help='The directory of YAML changelogs we will use.')
cl['changes'] = []
with open(fileName, 'w') as f:
yaml.dump(cl, f, default_flow_style=False)
args = opt.parse_args()
targetDir = os.path.dirname(args.targetFile)
today = datetime.date.today()
remove_dates = []
days_written = 0
with open(args.targetFile.replace('.htm', '.dry.htm') if args.dryRun else args.targetFile, 'w') as changelog:
with open(os.path.join(targetDir, 'templates', 'header.html'), 'r') as h:
for line in h:
changelog.write(line)
all_changelog_entries = {}
for _date in reversed(sorted(all_changelog_entries.keys())):
entry_htm = '\n'
entry_htm += '\t\t\t<h2 class="date">{date}</h2>\n'.format(date=_date.strftime(dateformat))
changelog_cache = os.path.join(args.ymlDir, '.all_changelog.yml')
failed_cache_read = True
if not args.dryRun:
if os.path.isfile(changelog_cache):
(failed_cache_read, all_changelog_entries) = loadCache(changelog_cache)
else:
changelog_cache = os.path.join(args.ymlDir, '.dry_changelog.yml')
if failed_cache_read and os.path.isfile(args.targetFile):
rebuild_changelog_from(args.targetFile, all_changelog_entries)
del_after = []
logging.info('Reading changelogs...')
for fileName in glob.glob(os.path.join(args.ymlDir, "*.yml")):
name, _ = os.path.splitext(os.path.basename(fileName))
if name.startswith('.'):
continue
if name == 'example':
continue
fileName = os.path.abspath(fileName)
logging.info(' Reading {}...'.format(fileName))
cl = {}
with open(fileName, 'r') as f:
cl = yaml.safe_load(f)
f.close()
if today not in all_changelog_entries:
all_changelog_entries[today] = {}
author_entries = all_changelog_entries[today].get(cl['author'], [])
if len(cl['changes']):
new = 0
for change in cl['changes']:
if change not in author_entries:
(change_type, _) = next(iter(change.items()))
if change_type not in VALID_PREFIXES:
logging.critical(' {0}: Invalid prefix {1}'.format(fileName, change_type))
return
author_entries += [change]
new += 1
all_changelog_entries[today][cl['author']] = author_entries
if new > 0:
logging.info(' Added {0} new changelog entries.'.format(new))
if cl.get('delete-after', False):
if os.path.isfile(fileName):
if args.dryRun:
logging.warning(' Would delete {0} (delete-after set)...'.format(fileName))
else:
del_after += [fileName]
if args.dryRun:
continue
cl['changes'] = []
with open(fileName, 'w') as f:
yaml.dump(cl, f, default_flow_style=False)
targetDir = os.path.dirname(args.targetFile)
jenv = jinja2.Environment(
extensions=['jinja2.ext.do'], # Occasionally useful.
loader=jinja2.FileSystemLoader('.'),
autoescape=jinja2.select_autoescape(
enabled_extensions=('htm', 'html'),
))
tmpl = jenv.get_template(os.path.join(targetDir, 'templates', 'changelog.tmpl.htm'))
remove_dates = []
days_written = 0
entries = collections.OrderedDict()
for _date in sorted(all_changelog_entries.keys(), reverse=True):
#entry_htm = '\n'
#entry_htm += '\t\t\t<h2 class="date">{date}</h2>\n'.format(date=_date.strftime(dateformat))
write_entry = False
date_entries = collections.OrderedDict()
for author in sorted(all_changelog_entries[_date].keys()):
if len(all_changelog_entries[_date]) == 0:
continue
author_htm = '\t\t\t<h3 class="author">{author} updated:</h3>\n'.format(author=author)
author_htm += '\t\t\t<ul class="changes bgimages16">\n'
#author_htm = '\t\t\t<h3 class="author">{author} updated:</h3>\n'.format(author=author)
#author_htm += '\t\t\t<ul class="changes bgimages16">\n'
changes_added = []
for (css_class, change) in (dictToTuples(e)[0] for e in all_changelog_entries[_date][author]):
for (css_class, change) in (next(iter(e.items())) for e in all_changelog_entries[_date][author]):
if change in changes_added:
continue
write_entry = True
changes_added += [change]
author_htm += '\t\t\t\t<li class="{css_class}">{change}</li>\n'.format(css_class=css_class, change=change.strip())
author_htm += '\t\t\t</ul>\n'
if len(changes_added) > 0:
entry_htm += author_htm
#author_htm += '\t\t\t\t<li class="{css_class}">{change}</li>\n'.format(css_class=css_class, change=change.strip())
if author not in date_entries:
date_entries[author] = []
date_entries[author] += [(css_class, change)]
#author_htm += '\t\t\t</ul>\n'
if write_entry and days_written <= MAX_DATE_ENTRIES:
changelog.write(entry_htm)
entries[_date] = date_entries
days_written += 1
else:
remove_dates.append(_date)
with open(os.path.join(targetDir, 'templates', 'footer.html'), 'r') as h:
for line in h:
changelog.write(line)
with open(args.targetFile.replace('.htm', '.dry.htm') if args.dryRun else args.targetFile, 'w') as changelog:
changelog.write(tmpl.render(ENTRIES=entries, DATEFORMAT=DATEFORMAT))
for _date in remove_dates:
del all_changelog_entries[_date]
print('Removing {} (old/invalid)'.format(_date))
for _date in remove_dates:
del all_changelog_entries[_date]
logging.info('Removing {} (old/invalid)'.format(_date))
with open(changelog_cache, 'w') as f:
cache_head = 'DO NOT EDIT THIS FILE BY HAND! AUTOMATICALLY GENERATED BY ss13_genchangelog.py.'
yaml.dump_all([cache_head, all_changelog_entries], f, default_flow_style=False)
with open(changelog_cache, 'w') as f:
cache_head = 'DO NOT EDIT THIS FILE BY HAND! AUTOMATICALLY GENERATED BY ss13_genchangelog.py.'
yaml.dump_all([cache_head, all_changelog_entries], f, default_flow_style=False)
if len(del_after):
print('Cleaning up...')
for fileName in del_after:
if os.path.isfile(fileName):
print(' Deleting {0} (delete-after set)...'.format(fileName))
os.remove(fileName)
if len(del_after):
print('Cleaning up...')
for fileName in del_after:
if os.path.isfile(fileName):
print(' Deleting {0} (delete-after set)...'.format(fileName))
os.remove(fileName)
if __name__ == '__main__':
main()