Reorganized Highlight code.

Better and mor flexible way how individual lexer are treated.

docs/settings.rst

@@ -34,12 +34,12 @@ behavior without touching the code:
         String. The full qualified hostname and path to the dpaste instance.
         This is used to generate a link in the API response. Default: ``https://dpaste.de``
 
-    ``DPASTE_LEXER_LIST``
+    ``DPASTE_LEXER_CHOICES``
         Choices. A tuple of choices of Pygments lexers used in the lexer
         dropdown. Here is the full `lexer list`_ which is currently used.
         Example::
 
-            DPASTE_LEXER_LIST = (
+            DPASTE_LEXER_CHOICES = (
                 ('delphi', 'Delphi'),
                 ('php', 'PHP'),
                 ('text', 'Text'),

dpaste/forms.py

@@ -6,7 +6,7 @@ from django import forms
 from django.conf import settings
 from django.utils.translation import ugettext_lazy as _
 
-from .highlight import LEXER_DEFAULT, LEXER_KEYS, LEXER_LIST
+from .highlight import LEXER_DEFAULT, LEXER_KEYS, LEXER_CHOICES
 from .models import Snippet
 
 EXPIRE_CHOICES = getattr(settings, 'DPASTE_EXPIRE_CHOICES', (
@@ -45,7 +45,7 @@ class SnippetForm(forms.ModelForm):
     lexer = forms.ChoiceField(
         label=_('Lexer'),
         initial=LEXER_DEFAULT,
-        choices=LEXER_LIST
+        choices=LEXER_CHOICES
     )
 
     expires = forms.ChoiceField(

dpaste/highlight.py

@@ -1,28 +1,10 @@
-"""
-List of all available lexers.
-
-To get a list of all lexers, and remove some dupes, do:
-
-from pygments.lexers import get_all_lexers
-ALL_LEXER = set([(i[1][0], i[0]) for i in get_all_lexers()])
-LEXER_LIST = [l for l in ALL_LEXER if not (
-       '-' in l[0]
-    or '+' in l[0]
-    or '+' in l[1]
-    or 'with' in l[1].lower()
-    or ' ' in l[1]
-    or l[0] in IGNORE_LEXER
-)]
-LEXER_LIST = sorted(LEXER_LIST)
-"""
-
-
 from __future__ import unicode_literals
 
 from logging import getLogger
 
 from django.conf import settings
-from django.template.defaultfilters import escape
+from django.template.defaultfilters import escape, linebreaksbr
+from django.template.loader import render_to_string
 from django.utils.translation import ugettext_lazy as _
 from pygments import highlight
 from pygments.formatters.html import HtmlFormatter
@@ -33,107 +15,8 @@ from pygments.util import ClassNotFound
 logger = getLogger(__file__)
 
 
-PLAIN_TEXT = '_text' # lexer name whats rendered as text (paragraphs)
-PLAIN_CODE = '_code' # lexer name of code with no hihglighting
-
-LEXER_LIST = getattr(settings, 'DPASTE_LEXER_LIST', (
-    (_('Text'), (
-        (PLAIN_TEXT, 'Plain Text'),
-        # ('_markdown', 'Markdown'),
-        # ('_rst', 'reStructuredText'),
-        # ('_textile', 'Textile'),
-    )),
-    (_('Code'), (
-        (PLAIN_CODE, 'Plain Code'),
-        ('abap', 'ABAP'),
-        ('apacheconf', 'ApacheConf'),
-        ('applescript', 'AppleScript'),
-        ('as', 'ActionScript'),
-        ('bash', 'Bash'),
-        ('bbcode', 'BBCode'),
-        ('c', 'C'),
-        ('cpp', 'C++'),
-        ('clojure', 'Clojure'),
-        ('cobol', 'COBOL'),
-        ('css', 'CSS'),
-        ('cuda', 'CUDA'),
-        ('dart', 'Dart'),
-        ('delphi', 'Delphi'),
-        ('diff', 'Diff'),
-        ('django', 'Django'),
-        ('erlang', 'Erlang'),
-        ('fortran', 'Fortran'),
-        ('go', 'Go'),
-        ('groovy', 'Groovy'),
-        ('haml', 'Haml'),
-        ('haskell', 'Haskell'),
-        ('html', 'HTML'),
-        ('http', 'HTTP'),
-        ('ini', 'INI'),
-        ('irc', 'IRC'),
-        ('java', 'Java'),
-        ('js', 'JavaScript'),
-        ('json', 'JSON'),
-        ('lua', 'Lua'),
-        ('make', 'Makefile'),
-        ('mako', 'Mako'),
-        ('mason', 'Mason'),
-        ('matlab', 'Matlab'),
-        ('modula2', 'Modula'),
-        ('monkey', 'Monkey'),
-        ('mysql', 'MySQL'),
-        ('numpy', 'NumPy'),
-        ('objc', 'Obj-C'),
-        ('ocaml', 'OCaml'),
-        ('perl', 'Perl'),
-        ('php', 'PHP'),
-        ('postscript', 'PostScript'),
-        ('powershell', 'PowerShell'),
-        ('prolog', 'Prolog'),
-        ('properties', 'Properties'),
-        ('puppet', 'Puppet'),
-        ('python', 'Python'),
-        ('r', 'R'),
-        ('rb', 'Ruby'),
-        ('rst', 'reStructuredText'),
-        ('rust', 'Rust'),
-        ('sass', 'Sass'),
-        ('scala', 'Scala'),
-        ('scheme', 'Scheme'),
-        ('scilab', 'Scilab'),
-        ('scss', 'SCSS'),
-        ('smalltalk', 'Smalltalk'),
-        ('smarty', 'Smarty'),
-        ('sql', 'SQL'),
-        ('tcl', 'Tcl'),
-        ('tcsh', 'Tcsh'),
-        ('tex', 'TeX'),
-        ('text', 'Text'),
-        ('vb.net', 'VB.net'),
-        ('vim', 'VimL'),
-        ('xml', 'XML'),
-        ('xquery', 'XQuery'),
-        ('xslt', 'XSLT'),
-        ('yaml', 'YAML'),
-    ))
-))
-
-# Generate a list of all keys of all lexer
-LEXER_KEYS = []
-for i in LEXER_LIST:
-    for j, k in i[1]:
-        LEXER_KEYS.append(j)
-
-# The default lexer is python
-LEXER_DEFAULT = getattr(settings, 'DPASTE_LEXER_DEFAULT', 'python')
-
-# Lexers which have wordwrap enabled by default
-LEXER_WORDWRAP = getattr(settings, 'DPASTE_LEXER_WORDWRAP', 
-    ('rst')
-)
-
-
 class NakedHtmlFormatter(HtmlFormatter):
+    """Pygments HTML formatter with no further HTML tags."""
     def wrap(self, source, outfile):
         return self._wrap_code(source)
 
@@ -142,32 +25,197 @@ class NakedHtmlFormatter(HtmlFormatter):
             yield i, t
 
 
-def pygmentize(code_string, lexer_name=LEXER_DEFAULT):
-    """
-    Run given code in ``code string`` through pygments.
-    """
+# -----------------------------------------------------------------------------
+# Highlight Code Snippets
+# -----------------------------------------------------------------------------
 
-    # Plain code is not highlighted, but we wrap with with regular
-    # Pygments syntax to keep the frontend aligned.
-    if lexer_name == PLAIN_CODE:
+class Highlighter(object):
+    template_name = 'dpaste/highlight/code.html'
+
+    def highlight(self, code_string, lexer_name=None):
+        """Subclasses need to override this."""
+        return code_string
+
+    @staticmethod
+    def get_lexer_display_name(lexer_name, fallback=_('(Deprecated Lexer)')):
+        for l in TEXT_FORMATTER + CODE_FORMATTER:
+            if l[0] == lexer_name:
+                return l[1]
+        return fallback
+
+    def render(self, code_string, lexer_name, **kwargs):
+        highlighted_string = self.highlight(code_string, lexer_name)
+        context = {
+            'highlighted': highlighted_string,
+            'highlighted_splitted': highlighted_string.splitlines(),
+            'lexer_name': lexer_name,
+            'lexer_display_name': self.get_lexer_display_name(lexer_name),
+        }
+        context.update(kwargs)
+        return render_to_string(self.template_name, context)
+
+
+class PlainTextHighlighter(Highlighter):
+    """Plain Text. Just replace linebreaks."""
+    template_name = 'dpaste/highlight/text.html'
+
+    def highlight(self, code_string, lexer_name=None):
+        return linebreaksbr(code_string)
+
+class PlainCodeHighlighter(Highlighter):
+    """Plain Code. No highlighting but Pygments like span tags around each line."""
+
+    def highlight(self, code_string, lexer_name=None):
         return '\n'.join(['<span class="plain">{}</span>'.format(escape(l) or '&#8203;')
             for l in code_string.splitlines()])
 
-    # Everything else is handled by Pygments.
-    lexer = None
-    try:
-        lexer = get_lexer_by_name(lexer_name)
-    except ClassNotFound as e:
-        if settings.DEBUG:
+
+class PygmentsHighlighter(Highlighter):
+    """
+    Highlight code string with Pygments. The lexer is automaticially
+    determined by the lexer name.
+    """
+    formatter = NakedHtmlFormatter
+    fallback_lexer = PythonLexer
+
+    def highlight(self, code_string, lexer_name):
+        try:
+            lexer = get_lexer_by_name(lexer_name)
+        except ClassNotFound:
             logger.warning('Lexer for given name %s not found', lexer_name)
-            logger.exception(e)
-        pass
+            lexer = self.fallback_lexer()
+        return highlight(code_string, lexer, self.formatter())
 
-    # If yet no lexer is defined, fallback to Python
-    if not lexer:
-        lexer = PythonLexer()
 
-    formatter = NakedHtmlFormatter()
+def get_highlighter_class(lexer_name):
+    """
+    Get Highlighter for lexer name.
 
-    return highlight(code_string, lexer, formatter)
+    If the found lexer tuple does not provide a Highlighter class,
+    use the generic Pygments highlighter.
 
+    If no suitable highlighter is found, return the generic
+    PlainCode Highlighter.
+    """
+    for c in TEXT_FORMATTER + CODE_FORMATTER:
+        if c[0] == lexer_name:
+            if len(c) == 3:
+                return c[2]
+            return PygmentsHighlighter
+    return PlainCodeHighlighter
+
+
+# -----------------------------------------------------------------------------
+# Lexer List
+# -----------------------------------------------------------------------------
+
+# Lexer list. Each list contains a lexer tuple of:
+#
+#   (Lexer key,
+#    Lexer Display Name,
+#    Lexer Highlight Class)
+#
+# If the Highlight Class is not given, PygmentsHighlighter is used.
+
+# Default Highlight Types
+PLAIN_TEXT = '_text' # lexer name whats rendered as text (paragraphs)
+PLAIN_CODE = '_code' # lexer name of code with no hihglighting
+
+TEXT_FORMATTER = [
+    (PLAIN_TEXT, 'Plain Text',  PlainTextHighlighter),
+    # ('_markdown', 'Markdown'),
+    # ('_rst', 'reStructuredText'),
+    # ('_textile', 'Textile'),
+]
+
+CODE_FORMATTER = [
+    (PLAIN_CODE, 'Plain Code', PlainCodeHighlighter),
+    ('abap', 'ABAP'),
+    ('apacheconf', 'ApacheConf'),
+    ('applescript', 'AppleScript'),
+    ('as', 'ActionScript'),
+    ('bash', 'Bash'),
+    ('bbcode', 'BBCode'),
+    ('c', 'C'),
+    ('cpp', 'C++'),
+    ('clojure', 'Clojure'),
+    ('cobol', 'COBOL'),
+    ('css', 'CSS'),
+    ('cuda', 'CUDA'),
+    ('dart', 'Dart'),
+    ('delphi', 'Delphi'),
+    ('diff', 'Diff'),
+    ('django', 'Django'),
+    ('erlang', 'Erlang'),
+    ('fortran', 'Fortran'),
+    ('go', 'Go'),
+    ('groovy', 'Groovy'),
+    ('haml', 'Haml'),
+    ('haskell', 'Haskell'),
+    ('html', 'HTML'),
+    ('http', 'HTTP'),
+    ('ini', 'INI'),
+    ('irc', 'IRC'),
+    ('java', 'Java'),
+    ('js', 'JavaScript'),
+    ('json', 'JSON'),
+    ('lua', 'Lua'),
+    ('make', 'Makefile'),
+    ('mako', 'Mako'),
+    ('mason', 'Mason'),
+    ('matlab', 'Matlab'),
+    ('modula2', 'Modula'),
+    ('monkey', 'Monkey'),
+    ('mysql', 'MySQL'),
+    ('numpy', 'NumPy'),
+    ('objc', 'Obj-C'),
+    ('ocaml', 'OCaml'),
+    ('perl', 'Perl'),
+    ('php', 'PHP'),
+    ('postscript', 'PostScript'),
+    ('powershell', 'PowerShell'),
+    ('prolog', 'Prolog'),
+    ('properties', 'Properties'),
+    ('puppet', 'Puppet'),
+    ('python', 'Python'),
+    ('r', 'R'),
+    ('rb', 'Ruby'),
+    ('rst', 'reStructuredText'),
+    ('rust', 'Rust'),
+    ('sass', 'Sass'),
+    ('scala', 'Scala'),
+    ('scheme', 'Scheme'),
+    ('scilab', 'Scilab'),
+    ('scss', 'SCSS'),
+    ('smalltalk', 'Smalltalk'),
+    ('smarty', 'Smarty'),
+    ('sql', 'SQL'),
+    ('tcl', 'Tcl'),
+    ('tcsh', 'Tcsh'),
+    ('tex', 'TeX'),
+    ('text', 'Text'),
+    ('vb.net', 'VB.net'),
+    ('vim', 'VimL'),
+    ('xml', 'XML'),
+    ('xquery', 'XQuery'),
+    ('xslt', 'XSLT'),
+    ('yaml', 'YAML'),
+]
+
+# Generat a list of Form choices of all lexer.
+LEXER_CHOICES = (
+    (_('Text'), [i[:2] for i in TEXT_FORMATTER]),
+    (_('Code'), [i[:2] for i in CODE_FORMATTER])
+)
+
+# List of all Lexer Keys
+LEXER_KEYS = [i[0] for i in TEXT_FORMATTER] + [i[0] for i in CODE_FORMATTER]
+
+# The default lexer which we fallback in case of
+# an error or if not supplied in an API call.
+LEXER_DEFAULT = getattr(settings, 'DPASTE_LEXER_DEFAULT', 'python')
+
+# Lexers which have wordwrap enabled by default
+LEXER_WORDWRAP = getattr(settings, 'DPASTE_LEXER_WORDWRAP',
+    ('rst',)
+)

dpaste/models.py

@@ -6,7 +6,6 @@ from django.conf import settings
 from django.db import models
 from django.urls import reverse
 from django.utils.translation import ugettext_lazy as _
-from pygments import highlight
 from six import python_2_unicode_compatible
 
 from dpaste import highlight
@@ -76,21 +75,13 @@ class Snippet(models.Model):
         return reverse('snippet_details', kwargs={'snippet_id': self.secret_id})
 
     def highlight(self):
-        return highlight.pygmentize(self.content, self.lexer)
-
-    def highlight_lines(self):
-        return self.highlight().splitlines()
+        HighlighterClass = highlight.get_highlighter_class(self.lexer)
+        return HighlighterClass().render(self.content, self.lexer)
 
     @property
     def lexer_name(self):
         """Display name for this lexer."""
-        try:
-            return dict(
-                highlight.LEXER_LIST[0][1] +
-                highlight.LEXER_LIST[1][1]
-            )[self.lexer]
-        except KeyError:
-            return _('(Deprecated Lexer)')
+        return highlight.Highlighter.get_lexer_display_name(self.lexer)
 
     @property
     def remaining_views(self):

dpaste/templates/dpaste/details.html

@@ -69,16 +69,11 @@
 
   {% if diff %}
   <div class="snippet-diff">
-    <h2>{% trans "Comparision with previous snippet " %}</h2>
-    <div class="snippet-code">{{ diff|safe }}</div>
+    <h2>{% trans "Comparision with previous snippet " %}</h2>{{ diff|safe }}
   </div>
   {% endif %}
 
-  {% if snippet.lexer == '_text' %}
-    <div class="snippet-text">{% include "dpaste/highlight/text.html" %}</div>
-  {% else %}
-    <div class="snippet-code">{% include "dpaste/highlight/code.html" %}</div>
-  {% endif %}
+  {{ snippet.highlight }}
 
   <header class="sub">
     <h2>{% trans "Edit this Snippet" %}</h2>

dpaste/templates/dpaste/highlight/code.html

@@ -1 +1 @@
-<ol>{% for line in snippet.highlight_lines %}<li id="l{{ forloop.counter }}">{{ line|safe|default:"&#8203;" }}</li>{% endfor %}</ol>
+<div class="snippet-code"><ol>{% for line in highlighted_splitted %}<li id="l{{ forloop.counter }}">{{ line|safe|default:"&#8203;" }}</li>{% endfor %}</ol></div>

dpaste/templates/dpaste/highlight/text.html

@@ -1 +1,3 @@
-<div>{{ snippet.content|linebreaksbr }}</div>
+<div class="snippet-text">
+  <div>{{ highlighted }}</div>
+</div>

dpaste/tests/test_highlight.py

@@ -3,7 +3,8 @@ from __future__ import unicode_literals
 
 from django.test import TestCase
 
-from dpaste.highlight import PLAIN_CODE, pygmentize
+from dpaste.highlight import PLAIN_CODE, PygmentsHighlighter, \
+    PlainCodeHighlighter
 
 
 class HighlightAPITestCase(TestCase):
@@ -14,7 +15,7 @@ class HighlightAPITestCase(TestCase):
         """
         input = 'var'
         expected = '<span class="plain">var</span>'
-        value = pygmentize(input, lexer_name=PLAIN_CODE)
+        value = PlainCodeHighlighter().highlight(input)
         self.assertEqual(value, expected)
 
     def test_plain_code_leading_whitespace(self):
@@ -23,7 +24,7 @@ class HighlightAPITestCase(TestCase):
         """
         input = ' var=1'
         expected = '<span class="plain"> var=1</span>'
-        value = pygmentize(input, lexer_name=PLAIN_CODE)
+        value = PlainCodeHighlighter().highlight(input)
         self.assertEqual(value, expected)
 
     def test_plain_code_leading_whitespace_multiline(self):
@@ -39,7 +40,7 @@ class HighlightAPITestCase(TestCase):
             '<span class="plain">  var=2</span>\n'
             '<span class="plain">   var=3</span>\n'
             '<span class="plain">    var=4</span>')
-        value = pygmentize(input, lexer_name=PLAIN_CODE)
+        value = PlainCodeHighlighter().highlight(input)
         self.assertEqual(value, expected)
 
     def test_pygments(self):
@@ -49,7 +50,7 @@ class HighlightAPITestCase(TestCase):
         """
         input = 'var'
         expected = '<span class="n">var</span>\n'
-        value = pygmentize(input, lexer_name='python')
+        value = PygmentsHighlighter().highlight(input, 'python')
         self.assertEqual(value, expected)
 
     def test_pygments_leading_whitespace(self):
@@ -58,7 +59,7 @@ class HighlightAPITestCase(TestCase):
         """
         input = ' var'
         expected = ' <span class="n">var</span>\n'
-        value = pygmentize(input, lexer_name='python')
+        value = PygmentsHighlighter().highlight(input, 'python')
         self.assertEqual(value, expected)
 
     def test_pygments_leading_whitespace_multiline(self):
@@ -74,5 +75,5 @@ class HighlightAPITestCase(TestCase):
             '  <span class="n">var</span>\n'
             '   <span class="n">var</span>\n'
             '    <span class="n">var</span>\n')
-        value = pygmentize(input, lexer_name='python')
+        value = PygmentsHighlighter().highlight(input, 'python')
         self.assertEqual(value, expected)

dpaste/tests/test_snippet.py

@@ -9,6 +9,7 @@ from django.test import TestCase
 from django.test.client import Client
 from django.test.utils import override_settings
 
+from dpaste.highlight import PygmentsHighlighter
 from ..forms import EXPIRE_DEFAULT
 from ..highlight import LEXER_DEFAULT, PLAIN_CODE, PLAIN_TEXT
 from ..models import Snippet
@@ -339,9 +340,9 @@ class SnippetTestCase(TestCase):
     def test_highlighting(self):
         # You can pass any lexer to the pygmentize function and it will
         # never fail loudly.
-        from ..highlight import pygmentize
-        pygmentize('code', lexer_name='python')
-        pygmentize('code', lexer_name='doesnotexist')
+        PygmentsHighlighter().highlight('code', 'python')
+        PygmentsHighlighter().highlight('code', 'doesnotexist')
+
 
     @override_settings(DPASTE_SLUG_LENGTH=1)
     def test_random_slug_generation(self):

dpaste/views.py

@@ -21,6 +21,7 @@ from pygments.util import ClassNotFound
 
 from dpaste import highlight
 from dpaste.forms import EXPIRE_CHOICES, SnippetForm, get_expire_values
+from dpaste.highlight import PygmentsHighlighter
 from dpaste.models import ONETIME_LIMIT, Snippet
 
 
@@ -42,13 +43,6 @@ class SnippetView(FormView):
         })
         return kwargs
 
-    def get_context_data(self, **kwargs):
-        ctx = super(SnippetView, self).get_context_data(**kwargs)
-        ctx.update({
-            'lexer_list': highlight.LEXER_LIST,
-        })
-        return ctx
-
     def form_valid(self, form):
         snippet = form.save()
         return HttpResponseRedirect(snippet.get_absolute_url())
@@ -124,10 +118,10 @@ class SnippetDetailView(SnippetView, DetailView):
             n=1
         )
         diff_code = '\n'.join(d).strip()
-        highlighted = highlight.pygmentize(diff_code, lexer_name='diff')
+        highlighted = PygmentsHighlighter().render(diff_code, 'diff')
 
         # Remove blank lines
-        return highlighted.replace('\n\n', '\n')
+        return highlighted
 
     def get_context_data(self, **kwargs):
         self.object = self.get_object()
@@ -276,7 +270,7 @@ class APIView(View):
                     'Valid values are: {}'.format(expires, ', '.join(expire_options)))
             expires, expire_type = get_expire_values(expires)
         else:
-            expires = datetime.datetime.now() + datetime.timedelta(seconds=60 * 60 * 24 * 30)
+            expires = datetime.datetime.now() + datetime.timedelta(seconds=60 * 60 * 24)
             expire_type = Snippet.EXPIRE_TIME
 
         s = Snippet.objects.create(