Reorganized Highlight code.

Better and mor flexible way how individual lexer are treated.

docs/settings.rst

@@ -34,12 +34,12 @@ behavior without touching the code:
         String. The full qualified hostname and path to the dpaste instance.
         This is used to generate a link in the API response. Default: ``https://dpaste.de``
 
-    ``DPASTE_LEXER_LIST``
+    ``DPASTE_LEXER_CHOICES``
         Choices. A tuple of choices of Pygments lexers used in the lexer
         dropdown. Here is the full `lexer list`_ which is currently used.
         Example::
 
-            DPASTE_LEXER_LIST = (
+            DPASTE_LEXER_CHOICES = (
                 ('delphi', 'Delphi'),
                 ('php', 'PHP'),
                 ('text', 'Text'),

dpaste/forms.py

@@ -6,7 +6,7 @@ from django import forms
 from django.conf import settings
 from django.utils.translation import ugettext_lazy as _
 
-from .highlight import LEXER_DEFAULT, LEXER_KEYS, LEXER_LIST
+from .highlight import LEXER_DEFAULT, LEXER_KEYS, LEXER_CHOICES
 from .models import Snippet
 
 EXPIRE_CHOICES = getattr(settings, 'DPASTE_EXPIRE_CHOICES', (
@@ -45,7 +45,7 @@ class SnippetForm(forms.ModelForm):
     lexer = forms.ChoiceField(
         label=_('Lexer'),
         initial=LEXER_DEFAULT,
-        choices=LEXER_LIST
+        choices=LEXER_CHOICES
     )
 
     expires = forms.ChoiceField(

dpaste/highlight.py

@@ -1,28 +1,10 @@
-"""
-List of all available lexers.
-
-To get a list of all lexers, and remove some dupes, do:
-
-from pygments.lexers import get_all_lexers
-ALL_LEXER = set([(i[1][0], i[0]) for i in get_all_lexers()])
-LEXER_LIST = [l for l in ALL_LEXER if not (
-       '-' in l[0]
-    or '+' in l[0]
-    or '+' in l[1]
-    or 'with' in l[1].lower()
-    or ' ' in l[1]
-    or l[0] in IGNORE_LEXER
-)]
-LEXER_LIST = sorted(LEXER_LIST)
-"""
-
-
 from __future__ import unicode_literals
 
 from logging import getLogger
 
 from django.conf import settings
-from django.template.defaultfilters import escape
+from django.template.defaultfilters import escape, linebreaksbr
+from django.template.loader import render_to_string
 from django.utils.translation import ugettext_lazy as _
 from pygments import highlight
 from pygments.formatters.html import HtmlFormatter
@@ -33,18 +15,121 @@ from pygments.util import ClassNotFound
 logger = getLogger(__file__)
 
 
+class NakedHtmlFormatter(HtmlFormatter):
+    """Pygments HTML formatter with no further HTML tags."""
+    def wrap(self, source, outfile):
+        return self._wrap_code(source)
+
+    def _wrap_code(self, source):
+        for i, t in source:
+            yield i, t
+
+
+# -----------------------------------------------------------------------------
+# Highlight Code Snippets
+# -----------------------------------------------------------------------------
+
+class Highlighter(object):
+    template_name = 'dpaste/highlight/code.html'
+
+    def highlight(self, code_string, lexer_name=None):
+        """Subclasses need to override this."""
+        return code_string
+
+    @staticmethod
+    def get_lexer_display_name(lexer_name, fallback=_('(Deprecated Lexer)')):
+        for l in TEXT_FORMATTER + CODE_FORMATTER:
+            if l[0] == lexer_name:
+                return l[1]
+        return fallback
+
+    def render(self, code_string, lexer_name, **kwargs):
+        highlighted_string = self.highlight(code_string, lexer_name)
+        context = {
+            'highlighted': highlighted_string,
+            'highlighted_splitted': highlighted_string.splitlines(),
+            'lexer_name': lexer_name,
+            'lexer_display_name': self.get_lexer_display_name(lexer_name),
+        }
+        context.update(kwargs)
+        return render_to_string(self.template_name, context)
+
+
+class PlainTextHighlighter(Highlighter):
+    """Plain Text. Just replace linebreaks."""
+    template_name = 'dpaste/highlight/text.html'
+
+    def highlight(self, code_string, lexer_name=None):
+        return linebreaksbr(code_string)
+
+class PlainCodeHighlighter(Highlighter):
+    """Plain Code. No highlighting but Pygments like span tags around each line."""
+
+    def highlight(self, code_string, lexer_name=None):
+        return '\n'.join(['<span class="plain">{}</span>'.format(escape(l) or '&#8203;')
+            for l in code_string.splitlines()])
+
+
+class PygmentsHighlighter(Highlighter):
+    """
+    Highlight code string with Pygments. The lexer is automaticially
+    determined by the lexer name.
+    """
+    formatter = NakedHtmlFormatter
+    fallback_lexer = PythonLexer
+
+    def highlight(self, code_string, lexer_name):
+        try:
+            lexer = get_lexer_by_name(lexer_name)
+        except ClassNotFound:
+            logger.warning('Lexer for given name %s not found', lexer_name)
+            lexer = self.fallback_lexer()
+        return highlight(code_string, lexer, self.formatter())
+
+
+def get_highlighter_class(lexer_name):
+    """
+    Get Highlighter for lexer name.
+
+    If the found lexer tuple does not provide a Highlighter class,
+    use the generic Pygments highlighter.
+
+    If no suitable highlighter is found, return the generic
+    PlainCode Highlighter.
+    """
+    for c in TEXT_FORMATTER + CODE_FORMATTER:
+        if c[0] == lexer_name:
+            if len(c) == 3:
+                return c[2]
+            return PygmentsHighlighter
+    return PlainCodeHighlighter
+
+
+# -----------------------------------------------------------------------------
+# Lexer List
+# -----------------------------------------------------------------------------
+
+# Lexer list. Each list contains a lexer tuple of:
+#
+#   (Lexer key,
+#    Lexer Display Name,
+#    Lexer Highlight Class)
+#
+# If the Highlight Class is not given, PygmentsHighlighter is used.
+
+# Default Highlight Types
 PLAIN_TEXT = '_text' # lexer name whats rendered as text (paragraphs)
 PLAIN_CODE = '_code' # lexer name of code with no hihglighting
 
-LEXER_LIST = getattr(settings, 'DPASTE_LEXER_LIST', (
+TEXT_FORMATTER = [
-    (_('Text'), (
+    (PLAIN_TEXT, 'Plain Text',  PlainTextHighlighter),
-        (PLAIN_TEXT, 'Plain Text'),
     # ('_markdown', 'Markdown'),
     # ('_rst', 'reStructuredText'),
     # ('_textile', 'Textile'),
-    )),
+]
-    (_('Code'), (
+
-        (PLAIN_CODE, 'Plain Code'),
+CODE_FORMATTER = [
+    (PLAIN_CODE, 'Plain Code', PlainCodeHighlighter),
     ('abap', 'ABAP'),
     ('apacheconf', 'ApacheConf'),
     ('applescript', 'AppleScript'),
@@ -115,59 +200,22 @@ LEXER_LIST = getattr(settings, 'DPASTE_LEXER_LIST', (
     ('xquery', 'XQuery'),
     ('xslt', 'XSLT'),
     ('yaml', 'YAML'),
-    ))
+]
-))
 
-# Generate a list of all keys of all lexer
+# Generat a list of Form choices of all lexer.
-LEXER_KEYS = []
+LEXER_CHOICES = (
-for i in LEXER_LIST:
+    (_('Text'), [i[:2] for i in TEXT_FORMATTER]),
-    for j, k in i[1]:
+    (_('Code'), [i[:2] for i in CODE_FORMATTER])
-        LEXER_KEYS.append(j)
+)
 
-# The default lexer is python
+# List of all Lexer Keys
+LEXER_KEYS = [i[0] for i in TEXT_FORMATTER] + [i[0] for i in CODE_FORMATTER]
+
+# The default lexer which we fallback in case of
+# an error or if not supplied in an API call.
 LEXER_DEFAULT = getattr(settings, 'DPASTE_LEXER_DEFAULT', 'python')
 
 # Lexers which have wordwrap enabled by default
 LEXER_WORDWRAP = getattr(settings, 'DPASTE_LEXER_WORDWRAP',
-    ('rst')
+    ('rst',)
 )
-
-
-class NakedHtmlFormatter(HtmlFormatter):
-    def wrap(self, source, outfile):
-        return self._wrap_code(source)
-
-    def _wrap_code(self, source):
-        for i, t in source:
-            yield i, t
-
-
-def pygmentize(code_string, lexer_name=LEXER_DEFAULT):
-    """
-    Run given code in ``code string`` through pygments.
-    """
-
-    # Plain code is not highlighted, but we wrap with with regular
-    # Pygments syntax to keep the frontend aligned.
-    if lexer_name == PLAIN_CODE:
-        return '\n'.join(['<span class="plain">{}</span>'.format(escape(l) or '&#8203;')
-            for l in code_string.splitlines()])
-
-    # Everything else is handled by Pygments.
-    lexer = None
-    try:
-        lexer = get_lexer_by_name(lexer_name)
-    except ClassNotFound as e:
-        if settings.DEBUG:
-            logger.warning('Lexer for given name %s not found', lexer_name)
-            logger.exception(e)
-        pass
-
-    # If yet no lexer is defined, fallback to Python
-    if not lexer:
-        lexer = PythonLexer()
-
-    formatter = NakedHtmlFormatter()
-
-    return highlight(code_string, lexer, formatter)
-

dpaste/models.py

@@ -6,7 +6,6 @@ from django.conf import settings
 from django.db import models
 from django.urls import reverse
 from django.utils.translation import ugettext_lazy as _
-from pygments import highlight
 from six import python_2_unicode_compatible
 
 from dpaste import highlight
@@ -76,21 +75,13 @@ class Snippet(models.Model):
         return reverse('snippet_details', kwargs={'snippet_id': self.secret_id})
 
     def highlight(self):
-        return highlight.pygmentize(self.content, self.lexer)
+        HighlighterClass = highlight.get_highlighter_class(self.lexer)
-
+        return HighlighterClass().render(self.content, self.lexer)
-    def highlight_lines(self):
-        return self.highlight().splitlines()
 
     @property
     def lexer_name(self):
         """Display name for this lexer."""
-        try:
+        return highlight.Highlighter.get_lexer_display_name(self.lexer)
-            return dict(
-                highlight.LEXER_LIST[0][1] +
-                highlight.LEXER_LIST[1][1]
-            )[self.lexer]
-        except KeyError:
-            return _('(Deprecated Lexer)')
 
     @property
     def remaining_views(self):

dpaste/templates/dpaste/details.html

@@ -69,16 +69,11 @@
 
   {% if diff %}
   <div class="snippet-diff">
-    <h2>{% trans "Comparision with previous snippet " %}</h2>
+    <h2>{% trans "Comparision with previous snippet " %}</h2>{{ diff|safe }}
-    <div class="snippet-code">{{ diff|safe }}</div>
   </div>
   {% endif %}
 
-  {% if snippet.lexer == '_text' %}
+  {{ snippet.highlight }}
-    <div class="snippet-text">{% include "dpaste/highlight/text.html" %}</div>
-  {% else %}
-    <div class="snippet-code">{% include "dpaste/highlight/code.html" %}</div>
-  {% endif %}
 
   <header class="sub">
     <h2>{% trans "Edit this Snippet" %}</h2>

dpaste/templates/dpaste/highlight/code.html

@@ -1 +1 @@
-<ol>{% for line in snippet.highlight_lines %}<li id="l{{ forloop.counter }}">{{ line|safe|default:"&#8203;" }}</li>{% endfor %}</ol>
+<div class="snippet-code"><ol>{% for line in highlighted_splitted %}<li id="l{{ forloop.counter }}">{{ line|safe|default:"&#8203;" }}</li>{% endfor %}</ol></div>

dpaste/templates/dpaste/highlight/text.html

@@ -1 +1,3 @@
-<div>{{ snippet.content|linebreaksbr }}</div>
+<div class="snippet-text">
+  <div>{{ highlighted }}</div>
+</div>

dpaste/tests/test_highlight.py

@@ -3,7 +3,8 @@ from __future__ import unicode_literals
 
 from django.test import TestCase
 
-from dpaste.highlight import PLAIN_CODE, pygmentize
+from dpaste.highlight import PLAIN_CODE, PygmentsHighlighter, \
+    PlainCodeHighlighter
 
 
 class HighlightAPITestCase(TestCase):
@@ -14,7 +15,7 @@ class HighlightAPITestCase(TestCase):
         """
         input = 'var'
         expected = '<span class="plain">var</span>'
-        value = pygmentize(input, lexer_name=PLAIN_CODE)
+        value = PlainCodeHighlighter().highlight(input)
         self.assertEqual(value, expected)
 
     def test_plain_code_leading_whitespace(self):
@@ -23,7 +24,7 @@ class HighlightAPITestCase(TestCase):
         """
         input = ' var=1'
         expected = '<span class="plain"> var=1</span>'
-        value = pygmentize(input, lexer_name=PLAIN_CODE)
+        value = PlainCodeHighlighter().highlight(input)
         self.assertEqual(value, expected)
 
     def test_plain_code_leading_whitespace_multiline(self):
@@ -39,7 +40,7 @@ class HighlightAPITestCase(TestCase):
             '<span class="plain">  var=2</span>\n'
             '<span class="plain">   var=3</span>\n'
             '<span class="plain">    var=4</span>')
-        value = pygmentize(input, lexer_name=PLAIN_CODE)
+        value = PlainCodeHighlighter().highlight(input)
         self.assertEqual(value, expected)
 
     def test_pygments(self):
@@ -49,7 +50,7 @@ class HighlightAPITestCase(TestCase):
         """
         input = 'var'
         expected = '<span class="n">var</span>\n'
-        value = pygmentize(input, lexer_name='python')
+        value = PygmentsHighlighter().highlight(input, 'python')
         self.assertEqual(value, expected)
 
     def test_pygments_leading_whitespace(self):
@@ -58,7 +59,7 @@ class HighlightAPITestCase(TestCase):
         """
         input = ' var'
         expected = ' <span class="n">var</span>\n'
-        value = pygmentize(input, lexer_name='python')
+        value = PygmentsHighlighter().highlight(input, 'python')
         self.assertEqual(value, expected)
 
     def test_pygments_leading_whitespace_multiline(self):
@@ -74,5 +75,5 @@ class HighlightAPITestCase(TestCase):
             '  <span class="n">var</span>\n'
             '   <span class="n">var</span>\n'
             '    <span class="n">var</span>\n')
-        value = pygmentize(input, lexer_name='python')
+        value = PygmentsHighlighter().highlight(input, 'python')
         self.assertEqual(value, expected)

dpaste/tests/test_snippet.py

@@ -9,6 +9,7 @@ from django.test import TestCase
 from django.test.client import Client
 from django.test.utils import override_settings
 
+from dpaste.highlight import PygmentsHighlighter
 from ..forms import EXPIRE_DEFAULT
 from ..highlight import LEXER_DEFAULT, PLAIN_CODE, PLAIN_TEXT
 from ..models import Snippet
@@ -339,9 +340,9 @@ class SnippetTestCase(TestCase):
     def test_highlighting(self):
         # You can pass any lexer to the pygmentize function and it will
         # never fail loudly.
-        from ..highlight import pygmentize
+        PygmentsHighlighter().highlight('code', 'python')
-        pygmentize('code', lexer_name='python')
+        PygmentsHighlighter().highlight('code', 'doesnotexist')
-        pygmentize('code', lexer_name='doesnotexist')
+
 
     @override_settings(DPASTE_SLUG_LENGTH=1)
     def test_random_slug_generation(self):

dpaste/views.py

@@ -21,6 +21,7 @@ from pygments.util import ClassNotFound
 
 from dpaste import highlight
 from dpaste.forms import EXPIRE_CHOICES, SnippetForm, get_expire_values
+from dpaste.highlight import PygmentsHighlighter
 from dpaste.models import ONETIME_LIMIT, Snippet
 
 
@@ -42,13 +43,6 @@ class SnippetView(FormView):
         })
         return kwargs
 
-    def get_context_data(self, **kwargs):
-        ctx = super(SnippetView, self).get_context_data(**kwargs)
-        ctx.update({
-            'lexer_list': highlight.LEXER_LIST,
-        })
-        return ctx
-
     def form_valid(self, form):
         snippet = form.save()
         return HttpResponseRedirect(snippet.get_absolute_url())
@@ -124,10 +118,10 @@ class SnippetDetailView(SnippetView, DetailView):
             n=1
         )
         diff_code = '\n'.join(d).strip()
-        highlighted = highlight.pygmentize(diff_code, lexer_name='diff')
+        highlighted = PygmentsHighlighter().render(diff_code, 'diff')
 
         # Remove blank lines
-        return highlighted.replace('\n\n', '\n')
+        return highlighted
 
     def get_context_data(self, **kwargs):
         self.object = self.get_object()
@@ -276,7 +270,7 @@ class APIView(View):
                     'Valid values are: {}'.format(expires, ', '.join(expire_options)))
             expires, expire_type = get_expire_values(expires)
         else:
-            expires = datetime.datetime.now() + datetime.timedelta(seconds=60 * 60 * 24 * 30)
+            expires = datetime.datetime.now() + datetime.timedelta(seconds=60 * 60 * 24)
             expire_type = Snippet.EXPIRE_TIME
 
         s = Snippet.objects.create(