From 81f9718edcca87d3fc7fbda3bbc6dd52f44d5859 Mon Sep 17 00:00:00 2001 From: Martin Mahner Date: Mon, 12 Mar 2018 13:04:18 +0100 Subject: [PATCH] Highlight Whitespace tests --- .gitignore | 1 + Pipfile | 7 +- Pipfile.lock | 111 +++---------------------------- dpaste/highlight.py | 66 ++++++++++++------ dpaste/settings/local.py.example | 14 ++-- dpaste/tests/test_highlight.py | 76 +++++++++++++++++++-- 6 files changed, 132 insertions(+), 143 deletions(-) diff --git a/.gitignore b/.gitignore index d1a4830..fa4c045 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ dpaste/settings/local.py dpaste.egg-info docs/_build +dpaste.db diff --git a/Pipfile b/Pipfile index 9e8405a..868997b 100644 --- a/Pipfile +++ b/Pipfile @@ -7,11 +7,8 @@ name = "pypi" [dev-packages] +django-sslserver = "*" sphinx-rtd-theme = "*" sphinx = "*" + "e1839a8" = {path = ".", extras = ["standalone"], editable = true} - - -[packages] - -pylint = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 9f58be9..7c74043 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "bc3ecc152b070bbc2053c9b147a2b1c66c062997c87b8b7086d0de37efd9407d" + "sha256": "7aaac7b35a2344e96fa3eaa2642d9e47aa3cb69b6cb26ab8176ba48ee69d9964" }, "pipfile-spec": 6, "requires": {}, @@ -13,84 +13,7 @@ } ] }, - "default": { - "astroid": { - "hashes": [ - "sha256:db5cfc9af6e0b60cd07c19478fb54021fc20d2d189882fbcbc94fc69a8aecc58", - "sha256:f0a0e386dbca9f93ea9f3ea6f32b37a24720502b7baa9cb17c3976a680d43a06" - ], - "version": "==1.6.1" - }, - "isort": { - "hashes": [ - "sha256:1153601da39a25b14ddc54955dbbacbb6b2d19135386699e2ad58517953b34af", - "sha256:b9c40e9750f3d77e6e4d441d8b0266cf555e7cdabdcff33c4fd06366ca761ef8", - "sha256:ec9ef8f4a9bc6f71eec99e1806bfa2de401650d996c59330782b89a5555c1497" - ], - "version": "==4.3.4" - }, - "lazy-object-proxy": { - "hashes": [ - "sha256:0ce34342b419bd8f018e6666bfef729aec3edf62345a53b537a4dcc115746a33", - "sha256:1b668120716eb7ee21d8a38815e5eb3bb8211117d9a90b0f8e21722c0758cc39", - "sha256:209615b0fe4624d79e50220ce3310ca1a9445fd8e6d3572a896e7f9146bbf019", - "sha256:27bf62cb2b1a2068d443ff7097ee33393f8483b570b475db8ebf7e1cba64f088", - "sha256:27ea6fd1c02dcc78172a82fc37fcc0992a94e4cecf53cb6d73f11749825bd98b", - "sha256:2c1b21b44ac9beb0fc848d3993924147ba45c4ebc24be19825e57aabbe74a99e", - "sha256:2df72ab12046a3496a92476020a1a0abf78b2a7db9ff4dc2036b8dd980203ae6", - "sha256:320ffd3de9699d3892048baee45ebfbbf9388a7d65d832d7e580243ade426d2b", - "sha256:50e3b9a464d5d08cc5227413db0d1c4707b6172e4d4d915c1c70e4de0bbff1f5", - "sha256:5276db7ff62bb7b52f77f1f51ed58850e315154249aceb42e7f4c611f0f847ff", - "sha256:61a6cf00dcb1a7f0c773ed4acc509cb636af2d6337a08f362413c76b2b47a8dd", - "sha256:6ae6c4cb59f199d8827c5a07546b2ab7e85d262acaccaacd49b62f53f7c456f7", - "sha256:7661d401d60d8bf15bb5da39e4dd72f5d764c5aff5a86ef52a042506e3e970ff", - "sha256:7bd527f36a605c914efca5d3d014170b2cb184723e423d26b1fb2fd9108e264d", - "sha256:7cb54db3535c8686ea12e9535eb087d32421184eacc6939ef15ef50f83a5e7e2", - "sha256:7f3a2d740291f7f2c111d86a1c4851b70fb000a6c8883a59660d95ad57b9df35", - "sha256:81304b7d8e9c824d058087dcb89144842c8e0dea6d281c031f59f0acf66963d4", - "sha256:933947e8b4fbe617a51528b09851685138b49d511af0b6c0da2539115d6d4514", - "sha256:94223d7f060301b3a8c09c9b3bc3294b56b2188e7d8179c762a1cda72c979252", - "sha256:ab3ca49afcb47058393b0122428358d2fbe0408cf99f1b58b295cfeb4ed39109", - "sha256:bd6292f565ca46dee4e737ebcc20742e3b5be2b01556dafe169f6c65d088875f", - "sha256:cb924aa3e4a3fb644d0c463cad5bc2572649a6a3f68a7f8e4fbe44aaa6d77e4c", - "sha256:d0fc7a286feac9077ec52a927fc9fe8fe2fabab95426722be4c953c9a8bede92", - "sha256:ddc34786490a6e4ec0a855d401034cbd1242ef186c20d79d2166d6a4bd449577", - "sha256:e34b155e36fa9da7e1b7c738ed7767fc9491a62ec6af70fe9da4a057759edc2d", - "sha256:e5b9e8f6bda48460b7b143c3821b21b452cb3a835e6bbd5dd33aa0c8d3f5137d", - "sha256:e81ebf6c5ee9684be8f2c87563880f93eedd56dd2b6146d8a725b50b7e5adb0f", - "sha256:eb91be369f945f10d3a49f5f9be8b3d0b93a4c2be8f8a5b83b0571b8123e0a7a", - "sha256:f460d1ceb0e4a5dcb2a652db0904224f367c9b3c1470d5a7683c0480e582468b" - ], - "version": "==1.3.1" - }, - "mccabe": { - "hashes": [ - "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", - "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" - ], - "version": "==0.6.1" - }, - "pylint": { - "hashes": [ - "sha256:156839bedaa798febee72893beef00c650c2e7abafb5586fc7a6a56be7f80412", - "sha256:4fe3b99da7e789545327b75548cee6b511e4faa98afe268130fea1af4b5ec022" - ], - "version": "==1.8.2" - }, - "six": { - "hashes": [ - "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", - "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb" - ], - "version": "==1.11.0" - }, - "wrapt": { - "hashes": [ - "sha256:d4d560d479f2c21e1b5443bbd15fe7ec4b37fe7e53d335d3b9b0a7b1226fe3c6" - ], - "version": "==1.10.11" - } - }, + "default": {}, "develop": { "alabaster": { "hashes": [ @@ -124,10 +47,7 @@ "hashes": [ "sha256:03481e81d558d30d230bc12999e3edffe392d244349a90f4ef9b88425fac74ba", "sha256:0b136648de27201056c1869a6c0d4e23f464750fd9a9ba9750b8336a244429ed", - "sha256:104ab3934abaf5be871a583541e8829d6c19ce7bde2923b2751e0d3ca44db60a", - "sha256:15b111b6a0f46ee1a485414a52a7ad1d703bdf984e9ed3c288a4414d3871dcbd", "sha256:198626739a79b09fa0a2f06e083ffd12eb55449b5f8bfdbeed1df4910b2ca640", - "sha256:1c383d2ef13ade2acc636556fd544dba6e14fa30755f26812f54300e401f98f2", "sha256:28b2191e7283f4f3568962e373b47ef7f0392993bb6660d079c62bd50fe9d162", "sha256:2eb564bbf7816a9d68dd3369a510be3327f1c618d2357fa6b1216994c2e3d508", "sha256:337ded681dd2ef9ca04ef5d93cfc87e52e09db2594c296b4a0a3662cb1b41249", @@ -148,16 +68,11 @@ "sha256:7e1fe19bd6dce69d9fd159d8e4a80a8f52101380d5d3a4d374b6d3eae0e5de9c", "sha256:8c3cb8c35ec4d9506979b4cf90ee9918bc2e49f84189d9bf5c36c0c1119c6558", "sha256:9d6dd10d49e01571bf6e147d3b505141ffc093a06756c60b053a859cb2128b1f", - "sha256:9e112fcbe0148a6fa4f0a02e8d58e94470fc6cb82a5481618fea901699bf34c4", - "sha256:ac4fef68da01116a5c117eba4dd46f2e06847a497de5ed1d64bb99a5fda1ef91", - "sha256:b8815995e050764c8610dbc82641807d196927c3dbed207f0a079833ffcf588d", "sha256:be6cfcd8053d13f5f5eeb284aa8a814220c3da1b0078fa859011c7fffd86dab9", "sha256:c1bb572fab8208c400adaf06a8133ac0712179a334c09224fb11393e920abcdd", "sha256:de4418dadaa1c01d497e539210cb6baa015965526ff5afc078c57ca69160108d", "sha256:e05cb4d9aad6233d67e0541caa7e511fa4047ed7750ec2510d466e806e0255d6", - "sha256:e4d96c07229f58cb686120f168276e434660e4358cc9cf3b0464210b04913e77", - "sha256:f3f501f345f24383c0000395b26b726e46758b71393267aeae0bd36f8b3ade80", - "sha256:f8a923a85cb099422ad5a2e345fe877bbc89a8a8b23235824a93488150e45f6e" + "sha256:f3f501f345f24383c0000395b26b726e46758b71393267aeae0bd36f8b3ade80" ], "version": "==4.5.1" }, @@ -175,6 +90,12 @@ ], "version": "==3.3" }, + "django-sslserver": { + "hashes": [ + "sha256:6b2514427b4eed0713e478ae6dfdda622070a98f657d61a4502bf42b6311c66e" + ], + "version": "==0.20" + }, "docutils": { "hashes": [ "sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6", @@ -247,11 +168,6 @@ "pyparsing": { "hashes": [ "sha256:0832bcf47acd283788593e7a0f542407bd9550a55a8a8435214a1960e04bcb04", - "sha256:281683241b25fe9b80ec9d66017485f6deff1af5cde372469134b56ca8447a07", - "sha256:8f1e18d3fd36c6795bb7e02a39fd05c611ffc2596c1e0d995d34d67630426c18", - "sha256:9e8143a3e15c13713506886badd96ca4b579a87fbdf49e550dbfc057d6cb218e", - "sha256:b8b3117ed9bdf45e14dcc89345ce638ec7e0e29b2b579fa1ecf32ce45ebac8a5", - "sha256:e4d45427c6e20a59bf4f88c639dcc03ce30d193112047f94012102f235853a58", "sha256:fee43f17a9c4087e7ed1605bd6df994c6173c1e977d7ade7b651292fab2bd010" ], "version": "==2.2.0" @@ -259,14 +175,7 @@ "pytz": { "hashes": [ "sha256:07edfc3d4d2705a20a6e99d97f0c4b61c800b8232dc1c04d87e8554f130148dd", - "sha256:3a47ff71597f821cd84a162e71593004286e5be07a340fd462f0d33a760782b5", - "sha256:410bcd1d6409026fbaa65d9ed33bf6dd8b1e94a499e32168acfc7b332e4095c0", - "sha256:5bd55c744e6feaa4d599a6cbd8228b4f8f9ba96de2c38d56f08e534b3c9edf0d", - "sha256:61242a9abc626379574a166dc0e96a66cd7c3b27fc10868003fa210be4bff1c9", - "sha256:887ab5e5b32e4d0c86efddd3d055c1f363cbaa583beb8da5e22d2fa2f64d51ef", - "sha256:ba18e6a243b3625513d85239b3e49055a2f0318466e0b8a92b8fb8ca7ccdf55f", - "sha256:ed6509d9af298b7995d69a440e2822288f2eca1681b8cce37673dbb10091e5fe", - "sha256:f93ddcdd6342f94cea379c73cddb5724e0d6d0a1c91c9bdef364dc0368ba4fda" + "sha256:410bcd1d6409026fbaa65d9ed33bf6dd8b1e94a499e32168acfc7b332e4095c0" ], "version": "==2018.3" }, diff --git a/dpaste/highlight.py b/dpaste/highlight.py index c5aae06..ef55953 100644 --- a/dpaste/highlight.py +++ b/dpaste/highlight.py @@ -1,16 +1,8 @@ -from __future__ import unicode_literals - -from django.conf import settings -from django.template.defaultfilters import escape -from django.utils.translation import ugettext_lazy as _ -from pygments import highlight -from pygments.formatters import HtmlFormatter -from pygments.lexers import * - """ -# Get a list of all lexer, and then remove all lexer which have '-' or '+' -# or 'with' in the name. Those are too specific and never used. This produces a -# tuple list of [(lexer, Lexer Display Name) ...] lexers. +List of all available lexers. + +To get a list of all lexers, and remove some dupes, do: + from pygments.lexers import get_all_lexers ALL_LEXER = set([(i[1][0], i[0]) for i in get_all_lexers()]) LEXER_LIST = [l for l in ALL_LEXER if not ( @@ -24,8 +16,23 @@ LEXER_LIST = [l for l in ALL_LEXER if not ( LEXER_LIST = sorted(LEXER_LIST) """ -# The list of lexers. Its not worth to autogenerate this. See above how to -# retrieve this. + +from __future__ import unicode_literals + +from logging import getLogger + +from django.conf import settings +from django.template.defaultfilters import escape +from django.utils.translation import ugettext_lazy as _ +from pygments import highlight +from pygments.formatters.html import HtmlFormatter +from pygments.lexers import get_lexer_by_name +from pygments.lexers.python import PythonLexer +from pygments.util import ClassNotFound + +logger = getLogger(__file__) + + PLAIN_TEXT = '_text_plain' # lexer name whats rendered as text (paragraphs) PLAIN_CODE = '_code' # lexer name of code with no hihglighting @@ -113,7 +120,6 @@ LEXER_LIST = getattr(settings, 'DPASTE_LEXER_LIST', ( # Generate a list of all keys of all lexer LEXER_KEYS = [] - for i in LEXER_LIST: for j, k in i[1]: LEXER_KEYS.append(j) @@ -126,6 +132,7 @@ LEXER_WORDWRAP = getattr(settings, 'DPASTE_LEXER_WORDWRAP', ('text', 'rst') ) + class NakedHtmlFormatter(HtmlFormatter): def wrap(self, source, outfile): return self._wrap_code(source) @@ -134,16 +141,33 @@ class NakedHtmlFormatter(HtmlFormatter): for i, t in source: yield i, t + def pygmentize(code_string, lexer_name=LEXER_DEFAULT): - # Plain code is noth hihglighted + """ + Run given code in ``code string`` through pygments. + """ + + # Plain code is not highlighted, but we wrap with with regular + # Pygments syntax to keep the frontend aligned. if lexer_name == PLAIN_CODE: - return '\n'.join([u'{}'.format(escape(l)) + return '\n'.join(['{}'.format(escape(l)) for l in code_string.splitlines()]) + # Everything else is handled by Pygments. + lexer = None try: - lexer = lexer_name and get_lexer_by_name(lexer_name) \ - or PythonLexer() - except Exception: + lexer = get_lexer_by_name(lexer_name) + except ClassNotFound as e: + if settings.DEBUG: + logger.warning('Lexer for given name %s not found', lexer_name) + logger.exception(e) + pass + + # If yet no lexer is defined, fallback to Python + if not lexer: lexer = PythonLexer() - return highlight(code_string, lexer, NakedHtmlFormatter()) + formatter = NakedHtmlFormatter() + + return highlight(code_string, lexer, formatter) + diff --git a/dpaste/settings/local.py.example b/dpaste/settings/local.py.example index 6c3b1e6..ce9f4d6 100644 --- a/dpaste/settings/local.py.example +++ b/dpaste/settings/local.py.example @@ -2,17 +2,10 @@ from dpaste.settings.base import * DEBUG = True -ADMINS = ( - #('Your Name', 'name@example.com'), -) -MANAGERS = ADMINS - DATABASES = { 'default': { - 'ENGINE': 'django.db.backends.postgresql', - 'NAME': 'dpaste', - 'USER': '', - 'PASSWORD': '', + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': 'dpaste.db', } } @@ -22,7 +15,8 @@ EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' INSTALLED_APPS += ('sslserver',) -# Disable for local development +# Optionally run the runserver as `manage.py runsslserver` to locally +# test correct cookie and csp behavior. if not 'runsslserver' in sys.argv: SESSION_COOKIE_SECURE = False CSRF_COOKIE_SECURE = False diff --git a/dpaste/tests/test_highlight.py b/dpaste/tests/test_highlight.py index 22dd7c1..2dbc627 100644 --- a/dpaste/tests/test_highlight.py +++ b/dpaste/tests/test_highlight.py @@ -1,14 +1,78 @@ # -*- encoding: utf-8 -*- from __future__ import unicode_literals -from dpaste import highlight from django.test import TestCase +from dpaste.highlight import PLAIN_CODE, pygmentize + class HighlightAPITestCase(TestCase): - def test_simple_highlight(self): - input = 'int_value = 1' - expected = 'int_value = 1' - value = highlight.pygmentize(input, lexer_name='python') - self.assertEqual(input, expected) \ No newline at end of file + def test_plain_code(self): + """ + PLAIN_CODE is not run through Pygments, test it separately. + """ + input = 'var' + expected = 'var' + value = pygmentize(input, lexer_name=PLAIN_CODE) + self.assertEqual(value, expected) + + def test_plain_code_leading_whitespace(self): + """ + Whitespace on the first line is retained. + """ + input = ' var=1' + expected = ' var=1' + value = pygmentize(input, lexer_name=PLAIN_CODE) + self.assertEqual(value, expected) + + def test_plain_code_leading_whitespace_multiline(self): + """ + Whitespace on the first line is retained, also on subsequent lines. + """ + input = (' var=1\n' + ' var=2\n' + ' var=3\n' + ' var=4') + expected = ( + ' var=1\n' + ' var=2\n' + ' var=3\n' + ' var=4') + value = pygmentize(input, lexer_name=PLAIN_CODE) + self.assertEqual(value, expected) + + def test_pygments(self): + """ + Pygemnts highlights the variable name, and also generally adds + a trailing \n to all its result. + """ + input = 'var' + expected = 'var\n' + value = pygmentize(input, lexer_name='python') + self.assertEqual(value, expected) + + def test_pygments_leading_whitespace(self): + """ + Whitespace on the first line is retained. + """ + input = ' var' + expected = ' var\n' + value = pygmentize(input, lexer_name='python') + self.assertEqual(value, expected) + + def test_pygments_leading_whitespace_multiline(self): + """ + Whitespace on the first line is retained, also on subsequent lines. + """ + input = (' var\n' + ' var\n' + ' var\n' + ' var') + expected = ( + ' var\n' + ' var\n' + ' var\n' + ' var\n') + value = pygmentize(input, lexer_name='python') + self.assertEqual(value, expected)