From 03549b00d7f3523826795fbaf82612bb25ab56e9 Mon Sep 17 00:00:00 2001 From: Martin Mahner Date: Thu, 5 Apr 2018 19:16:02 +0200 Subject: [PATCH] Reorganized Highlight code. Better and mor flexible way how individual lexer are treated. --- docs/settings.rst | 4 +- dpaste/forms.py | 4 +- dpaste/highlight.py | 328 +++++++++++--------- dpaste/models.py | 15 +- dpaste/templates/dpaste/details.html | 9 +- dpaste/templates/dpaste/highlight/code.html | 2 +- dpaste/templates/dpaste/highlight/text.html | 4 +- dpaste/tests/test_highlight.py | 15 +- dpaste/tests/test_snippet.py | 7 +- dpaste/views.py | 14 +- 10 files changed, 217 insertions(+), 185 deletions(-) diff --git a/docs/settings.rst b/docs/settings.rst index 4e121ca..d739855 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -34,12 +34,12 @@ behavior without touching the code: String. The full qualified hostname and path to the dpaste instance. This is used to generate a link in the API response. Default: ``https://dpaste.de`` - ``DPASTE_LEXER_LIST`` + ``DPASTE_LEXER_CHOICES`` Choices. A tuple of choices of Pygments lexers used in the lexer dropdown. Here is the full `lexer list`_ which is currently used. Example:: - DPASTE_LEXER_LIST = ( + DPASTE_LEXER_CHOICES = ( ('delphi', 'Delphi'), ('php', 'PHP'), ('text', 'Text'), diff --git a/dpaste/forms.py b/dpaste/forms.py index 8585002..af813b3 100644 --- a/dpaste/forms.py +++ b/dpaste/forms.py @@ -6,7 +6,7 @@ from django import forms from django.conf import settings from django.utils.translation import ugettext_lazy as _ -from .highlight import LEXER_DEFAULT, LEXER_KEYS, LEXER_LIST +from .highlight import LEXER_DEFAULT, LEXER_KEYS, LEXER_CHOICES from .models import Snippet EXPIRE_CHOICES = getattr(settings, 'DPASTE_EXPIRE_CHOICES', ( @@ -45,7 +45,7 @@ class SnippetForm(forms.ModelForm): lexer = forms.ChoiceField( label=_('Lexer'), initial=LEXER_DEFAULT, - choices=LEXER_LIST + choices=LEXER_CHOICES ) expires = forms.ChoiceField( diff --git a/dpaste/highlight.py b/dpaste/highlight.py index be5054a..1e3971d 100644 --- a/dpaste/highlight.py +++ b/dpaste/highlight.py @@ -1,28 +1,10 @@ -""" -List of all available lexers. - -To get a list of all lexers, and remove some dupes, do: - -from pygments.lexers import get_all_lexers -ALL_LEXER = set([(i[1][0], i[0]) for i in get_all_lexers()]) -LEXER_LIST = [l for l in ALL_LEXER if not ( - '-' in l[0] - or '+' in l[0] - or '+' in l[1] - or 'with' in l[1].lower() - or ' ' in l[1] - or l[0] in IGNORE_LEXER -)] -LEXER_LIST = sorted(LEXER_LIST) -""" - - from __future__ import unicode_literals from logging import getLogger from django.conf import settings -from django.template.defaultfilters import escape +from django.template.defaultfilters import escape, linebreaksbr +from django.template.loader import render_to_string from django.utils.translation import ugettext_lazy as _ from pygments import highlight from pygments.formatters.html import HtmlFormatter @@ -33,107 +15,8 @@ from pygments.util import ClassNotFound logger = getLogger(__file__) -PLAIN_TEXT = '_text' # lexer name whats rendered as text (paragraphs) -PLAIN_CODE = '_code' # lexer name of code with no hihglighting - -LEXER_LIST = getattr(settings, 'DPASTE_LEXER_LIST', ( - (_('Text'), ( - (PLAIN_TEXT, 'Plain Text'), - # ('_markdown', 'Markdown'), - # ('_rst', 'reStructuredText'), - # ('_textile', 'Textile'), - )), - (_('Code'), ( - (PLAIN_CODE, 'Plain Code'), - ('abap', 'ABAP'), - ('apacheconf', 'ApacheConf'), - ('applescript', 'AppleScript'), - ('as', 'ActionScript'), - ('bash', 'Bash'), - ('bbcode', 'BBCode'), - ('c', 'C'), - ('cpp', 'C++'), - ('clojure', 'Clojure'), - ('cobol', 'COBOL'), - ('css', 'CSS'), - ('cuda', 'CUDA'), - ('dart', 'Dart'), - ('delphi', 'Delphi'), - ('diff', 'Diff'), - ('django', 'Django'), - ('erlang', 'Erlang'), - ('fortran', 'Fortran'), - ('go', 'Go'), - ('groovy', 'Groovy'), - ('haml', 'Haml'), - ('haskell', 'Haskell'), - ('html', 'HTML'), - ('http', 'HTTP'), - ('ini', 'INI'), - ('irc', 'IRC'), - ('java', 'Java'), - ('js', 'JavaScript'), - ('json', 'JSON'), - ('lua', 'Lua'), - ('make', 'Makefile'), - ('mako', 'Mako'), - ('mason', 'Mason'), - ('matlab', 'Matlab'), - ('modula2', 'Modula'), - ('monkey', 'Monkey'), - ('mysql', 'MySQL'), - ('numpy', 'NumPy'), - ('objc', 'Obj-C'), - ('ocaml', 'OCaml'), - ('perl', 'Perl'), - ('php', 'PHP'), - ('postscript', 'PostScript'), - ('powershell', 'PowerShell'), - ('prolog', 'Prolog'), - ('properties', 'Properties'), - ('puppet', 'Puppet'), - ('python', 'Python'), - ('r', 'R'), - ('rb', 'Ruby'), - ('rst', 'reStructuredText'), - ('rust', 'Rust'), - ('sass', 'Sass'), - ('scala', 'Scala'), - ('scheme', 'Scheme'), - ('scilab', 'Scilab'), - ('scss', 'SCSS'), - ('smalltalk', 'Smalltalk'), - ('smarty', 'Smarty'), - ('sql', 'SQL'), - ('tcl', 'Tcl'), - ('tcsh', 'Tcsh'), - ('tex', 'TeX'), - ('text', 'Text'), - ('vb.net', 'VB.net'), - ('vim', 'VimL'), - ('xml', 'XML'), - ('xquery', 'XQuery'), - ('xslt', 'XSLT'), - ('yaml', 'YAML'), - )) -)) - -# Generate a list of all keys of all lexer -LEXER_KEYS = [] -for i in LEXER_LIST: - for j, k in i[1]: - LEXER_KEYS.append(j) - -# The default lexer is python -LEXER_DEFAULT = getattr(settings, 'DPASTE_LEXER_DEFAULT', 'python') - -# Lexers which have wordwrap enabled by default -LEXER_WORDWRAP = getattr(settings, 'DPASTE_LEXER_WORDWRAP', - ('rst') -) - - class NakedHtmlFormatter(HtmlFormatter): + """Pygments HTML formatter with no further HTML tags.""" def wrap(self, source, outfile): return self._wrap_code(source) @@ -142,32 +25,197 @@ class NakedHtmlFormatter(HtmlFormatter): yield i, t -def pygmentize(code_string, lexer_name=LEXER_DEFAULT): - """ - Run given code in ``code string`` through pygments. - """ +# ----------------------------------------------------------------------------- +# Highlight Code Snippets +# ----------------------------------------------------------------------------- - # Plain code is not highlighted, but we wrap with with regular - # Pygments syntax to keep the frontend aligned. - if lexer_name == PLAIN_CODE: +class Highlighter(object): + template_name = 'dpaste/highlight/code.html' + + def highlight(self, code_string, lexer_name=None): + """Subclasses need to override this.""" + return code_string + + @staticmethod + def get_lexer_display_name(lexer_name, fallback=_('(Deprecated Lexer)')): + for l in TEXT_FORMATTER + CODE_FORMATTER: + if l[0] == lexer_name: + return l[1] + return fallback + + def render(self, code_string, lexer_name, **kwargs): + highlighted_string = self.highlight(code_string, lexer_name) + context = { + 'highlighted': highlighted_string, + 'highlighted_splitted': highlighted_string.splitlines(), + 'lexer_name': lexer_name, + 'lexer_display_name': self.get_lexer_display_name(lexer_name), + } + context.update(kwargs) + return render_to_string(self.template_name, context) + + +class PlainTextHighlighter(Highlighter): + """Plain Text. Just replace linebreaks.""" + template_name = 'dpaste/highlight/text.html' + + def highlight(self, code_string, lexer_name=None): + return linebreaksbr(code_string) + +class PlainCodeHighlighter(Highlighter): + """Plain Code. No highlighting but Pygments like span tags around each line.""" + + def highlight(self, code_string, lexer_name=None): return '\n'.join(['{}'.format(escape(l) or '​') for l in code_string.splitlines()]) - # Everything else is handled by Pygments. - lexer = None - try: - lexer = get_lexer_by_name(lexer_name) - except ClassNotFound as e: - if settings.DEBUG: + +class PygmentsHighlighter(Highlighter): + """ + Highlight code string with Pygments. The lexer is automaticially + determined by the lexer name. + """ + formatter = NakedHtmlFormatter + fallback_lexer = PythonLexer + + def highlight(self, code_string, lexer_name): + try: + lexer = get_lexer_by_name(lexer_name) + except ClassNotFound: logger.warning('Lexer for given name %s not found', lexer_name) - logger.exception(e) - pass + lexer = self.fallback_lexer() + return highlight(code_string, lexer, self.formatter()) - # If yet no lexer is defined, fallback to Python - if not lexer: - lexer = PythonLexer() - formatter = NakedHtmlFormatter() +def get_highlighter_class(lexer_name): + """ + Get Highlighter for lexer name. - return highlight(code_string, lexer, formatter) + If the found lexer tuple does not provide a Highlighter class, + use the generic Pygments highlighter. + If no suitable highlighter is found, return the generic + PlainCode Highlighter. + """ + for c in TEXT_FORMATTER + CODE_FORMATTER: + if c[0] == lexer_name: + if len(c) == 3: + return c[2] + return PygmentsHighlighter + return PlainCodeHighlighter + + +# ----------------------------------------------------------------------------- +# Lexer List +# ----------------------------------------------------------------------------- + +# Lexer list. Each list contains a lexer tuple of: +# +# (Lexer key, +# Lexer Display Name, +# Lexer Highlight Class) +# +# If the Highlight Class is not given, PygmentsHighlighter is used. + +# Default Highlight Types +PLAIN_TEXT = '_text' # lexer name whats rendered as text (paragraphs) +PLAIN_CODE = '_code' # lexer name of code with no hihglighting + +TEXT_FORMATTER = [ + (PLAIN_TEXT, 'Plain Text', PlainTextHighlighter), + # ('_markdown', 'Markdown'), + # ('_rst', 'reStructuredText'), + # ('_textile', 'Textile'), +] + +CODE_FORMATTER = [ + (PLAIN_CODE, 'Plain Code', PlainCodeHighlighter), + ('abap', 'ABAP'), + ('apacheconf', 'ApacheConf'), + ('applescript', 'AppleScript'), + ('as', 'ActionScript'), + ('bash', 'Bash'), + ('bbcode', 'BBCode'), + ('c', 'C'), + ('cpp', 'C++'), + ('clojure', 'Clojure'), + ('cobol', 'COBOL'), + ('css', 'CSS'), + ('cuda', 'CUDA'), + ('dart', 'Dart'), + ('delphi', 'Delphi'), + ('diff', 'Diff'), + ('django', 'Django'), + ('erlang', 'Erlang'), + ('fortran', 'Fortran'), + ('go', 'Go'), + ('groovy', 'Groovy'), + ('haml', 'Haml'), + ('haskell', 'Haskell'), + ('html', 'HTML'), + ('http', 'HTTP'), + ('ini', 'INI'), + ('irc', 'IRC'), + ('java', 'Java'), + ('js', 'JavaScript'), + ('json', 'JSON'), + ('lua', 'Lua'), + ('make', 'Makefile'), + ('mako', 'Mako'), + ('mason', 'Mason'), + ('matlab', 'Matlab'), + ('modula2', 'Modula'), + ('monkey', 'Monkey'), + ('mysql', 'MySQL'), + ('numpy', 'NumPy'), + ('objc', 'Obj-C'), + ('ocaml', 'OCaml'), + ('perl', 'Perl'), + ('php', 'PHP'), + ('postscript', 'PostScript'), + ('powershell', 'PowerShell'), + ('prolog', 'Prolog'), + ('properties', 'Properties'), + ('puppet', 'Puppet'), + ('python', 'Python'), + ('r', 'R'), + ('rb', 'Ruby'), + ('rst', 'reStructuredText'), + ('rust', 'Rust'), + ('sass', 'Sass'), + ('scala', 'Scala'), + ('scheme', 'Scheme'), + ('scilab', 'Scilab'), + ('scss', 'SCSS'), + ('smalltalk', 'Smalltalk'), + ('smarty', 'Smarty'), + ('sql', 'SQL'), + ('tcl', 'Tcl'), + ('tcsh', 'Tcsh'), + ('tex', 'TeX'), + ('text', 'Text'), + ('vb.net', 'VB.net'), + ('vim', 'VimL'), + ('xml', 'XML'), + ('xquery', 'XQuery'), + ('xslt', 'XSLT'), + ('yaml', 'YAML'), +] + +# Generat a list of Form choices of all lexer. +LEXER_CHOICES = ( + (_('Text'), [i[:2] for i in TEXT_FORMATTER]), + (_('Code'), [i[:2] for i in CODE_FORMATTER]) +) + +# List of all Lexer Keys +LEXER_KEYS = [i[0] for i in TEXT_FORMATTER] + [i[0] for i in CODE_FORMATTER] + +# The default lexer which we fallback in case of +# an error or if not supplied in an API call. +LEXER_DEFAULT = getattr(settings, 'DPASTE_LEXER_DEFAULT', 'python') + +# Lexers which have wordwrap enabled by default +LEXER_WORDWRAP = getattr(settings, 'DPASTE_LEXER_WORDWRAP', + ('rst',) +) diff --git a/dpaste/models.py b/dpaste/models.py index 0f01bd0..c0f97c6 100644 --- a/dpaste/models.py +++ b/dpaste/models.py @@ -6,7 +6,6 @@ from django.conf import settings from django.db import models from django.urls import reverse from django.utils.translation import ugettext_lazy as _ -from pygments import highlight from six import python_2_unicode_compatible from dpaste import highlight @@ -76,21 +75,13 @@ class Snippet(models.Model): return reverse('snippet_details', kwargs={'snippet_id': self.secret_id}) def highlight(self): - return highlight.pygmentize(self.content, self.lexer) - - def highlight_lines(self): - return self.highlight().splitlines() + HighlighterClass = highlight.get_highlighter_class(self.lexer) + return HighlighterClass().render(self.content, self.lexer) @property def lexer_name(self): """Display name for this lexer.""" - try: - return dict( - highlight.LEXER_LIST[0][1] + - highlight.LEXER_LIST[1][1] - )[self.lexer] - except KeyError: - return _('(Deprecated Lexer)') + return highlight.Highlighter.get_lexer_display_name(self.lexer) @property def remaining_views(self): diff --git a/dpaste/templates/dpaste/details.html b/dpaste/templates/dpaste/details.html index 37246db..3bea94e 100644 --- a/dpaste/templates/dpaste/details.html +++ b/dpaste/templates/dpaste/details.html @@ -69,16 +69,11 @@ {% if diff %}
-

{% trans "Comparision with previous snippet " %}

-
{{ diff|safe }}
+

{% trans "Comparision with previous snippet " %}

{{ diff|safe }}
{% endif %} - {% if snippet.lexer == '_text' %} -
{% include "dpaste/highlight/text.html" %}
- {% else %} -
{% include "dpaste/highlight/code.html" %}
- {% endif %} + {{ snippet.highlight }}

{% trans "Edit this Snippet" %}

diff --git a/dpaste/templates/dpaste/highlight/code.html b/dpaste/templates/dpaste/highlight/code.html index 569a2e3..2ef3359 100644 --- a/dpaste/templates/dpaste/highlight/code.html +++ b/dpaste/templates/dpaste/highlight/code.html @@ -1 +1 @@ -
    {% for line in snippet.highlight_lines %}
  1. {{ line|safe|default:"​" }}
    2. {% endfor %}
+
    {% for line in highlighted_splitted %}
  1. {{ line|safe|default:"​" }}
    2. {% endfor %}
diff --git a/dpaste/templates/dpaste/highlight/text.html b/dpaste/templates/dpaste/highlight/text.html index 0c0972f..6e9dec0 100644 --- a/dpaste/templates/dpaste/highlight/text.html +++ b/dpaste/templates/dpaste/highlight/text.html @@ -1 +1,3 @@ -
{{ snippet.content|linebreaksbr }}
+
+
{{ highlighted }}
+
diff --git a/dpaste/tests/test_highlight.py b/dpaste/tests/test_highlight.py index 844b5cd..c365075 100644 --- a/dpaste/tests/test_highlight.py +++ b/dpaste/tests/test_highlight.py @@ -3,7 +3,8 @@ from __future__ import unicode_literals from django.test import TestCase -from dpaste.highlight import PLAIN_CODE, pygmentize +from dpaste.highlight import PLAIN_CODE, PygmentsHighlighter, \ + PlainCodeHighlighter class HighlightAPITestCase(TestCase): @@ -14,7 +15,7 @@ class HighlightAPITestCase(TestCase): """ input = 'var' expected = 'var' - value = pygmentize(input, lexer_name=PLAIN_CODE) + value = PlainCodeHighlighter().highlight(input) self.assertEqual(value, expected) def test_plain_code_leading_whitespace(self): @@ -23,7 +24,7 @@ class HighlightAPITestCase(TestCase): """ input = ' var=1' expected = ' var=1' - value = pygmentize(input, lexer_name=PLAIN_CODE) + value = PlainCodeHighlighter().highlight(input) self.assertEqual(value, expected) def test_plain_code_leading_whitespace_multiline(self): @@ -39,7 +40,7 @@ class HighlightAPITestCase(TestCase): ' var=2\n' ' var=3\n' ' var=4') - value = pygmentize(input, lexer_name=PLAIN_CODE) + value = PlainCodeHighlighter().highlight(input) self.assertEqual(value, expected) def test_pygments(self): @@ -49,7 +50,7 @@ class HighlightAPITestCase(TestCase): """ input = 'var' expected = 'var\n' - value = pygmentize(input, lexer_name='python') + value = PygmentsHighlighter().highlight(input, 'python') self.assertEqual(value, expected) def test_pygments_leading_whitespace(self): @@ -58,7 +59,7 @@ class HighlightAPITestCase(TestCase): """ input = ' var' expected = ' var\n' - value = pygmentize(input, lexer_name='python') + value = PygmentsHighlighter().highlight(input, 'python') self.assertEqual(value, expected) def test_pygments_leading_whitespace_multiline(self): @@ -74,5 +75,5 @@ class HighlightAPITestCase(TestCase): ' var\n' ' var\n' ' var\n') - value = pygmentize(input, lexer_name='python') + value = PygmentsHighlighter().highlight(input, 'python') self.assertEqual(value, expected) diff --git a/dpaste/tests/test_snippet.py b/dpaste/tests/test_snippet.py index 7882b0a..e4e521b 100644 --- a/dpaste/tests/test_snippet.py +++ b/dpaste/tests/test_snippet.py @@ -9,6 +9,7 @@ from django.test import TestCase from django.test.client import Client from django.test.utils import override_settings +from dpaste.highlight import PygmentsHighlighter from ..forms import EXPIRE_DEFAULT from ..highlight import LEXER_DEFAULT, PLAIN_CODE, PLAIN_TEXT from ..models import Snippet @@ -339,9 +340,9 @@ class SnippetTestCase(TestCase): def test_highlighting(self): # You can pass any lexer to the pygmentize function and it will # never fail loudly. - from ..highlight import pygmentize - pygmentize('code', lexer_name='python') - pygmentize('code', lexer_name='doesnotexist') + PygmentsHighlighter().highlight('code', 'python') + PygmentsHighlighter().highlight('code', 'doesnotexist') + @override_settings(DPASTE_SLUG_LENGTH=1) def test_random_slug_generation(self): diff --git a/dpaste/views.py b/dpaste/views.py index 7abbe50..53d9e82 100644 --- a/dpaste/views.py +++ b/dpaste/views.py @@ -21,6 +21,7 @@ from pygments.util import ClassNotFound from dpaste import highlight from dpaste.forms import EXPIRE_CHOICES, SnippetForm, get_expire_values +from dpaste.highlight import PygmentsHighlighter from dpaste.models import ONETIME_LIMIT, Snippet @@ -42,13 +43,6 @@ class SnippetView(FormView): }) return kwargs - def get_context_data(self, **kwargs): - ctx = super(SnippetView, self).get_context_data(**kwargs) - ctx.update({ - 'lexer_list': highlight.LEXER_LIST, - }) - return ctx - def form_valid(self, form): snippet = form.save() return HttpResponseRedirect(snippet.get_absolute_url()) @@ -124,10 +118,10 @@ class SnippetDetailView(SnippetView, DetailView): n=1 ) diff_code = '\n'.join(d).strip() - highlighted = highlight.pygmentize(diff_code, lexer_name='diff') + highlighted = PygmentsHighlighter().render(diff_code, 'diff') # Remove blank lines - return highlighted.replace('\n\n', '\n') + return highlighted def get_context_data(self, **kwargs): self.object = self.get_object() @@ -276,7 +270,7 @@ class APIView(View): 'Valid values are: {}'.format(expires, ', '.join(expire_options))) expires, expire_type = get_expire_values(expires) else: - expires = datetime.datetime.now() + datetime.timedelta(seconds=60 * 60 * 24 * 30) + expires = datetime.datetime.now() + datetime.timedelta(seconds=60 * 60 * 24) expire_type = Snippet.EXPIRE_TIME s = Snippet.objects.create(