Continue moving panel-specific code within panels.

Structure the SQL and template panels as packages as they're growing.

9 files changed, 463 insertions, 159 deletions

diff --git a/debug_toolbar/panels/sql/__init__.py b/debug_toolbar/panels/sql/__init__.py
new file mode 100644
index 0000000..90e05cb
--- /dev/null
+++ b/debug_toolbar/panels/sql/__init__.py
@@ -0,0 +1 @@
+from debug_toolbar.panels.sql.panel import SQLDebugPanel                # noqa
diff --git a/debug_toolbar/panels/sql/forms.py b/debug_toolbar/panels/sql/forms.py
new file mode 100644
index 0000000..c18be0c
--- /dev/null
+++ b/debug_toolbar/panels/sql/forms.py
@@ -0,0 +1,90 @@
+from __future__ import unicode_literals
+
+import json
+import hashlib
+
+from django import forms
+from django.conf import settings
+from django.db import connections
+from django.utils.encoding import force_text
+from django.utils.functional import cached_property
+from django.core.exceptions import ValidationError
+
+from debug_toolbar.panels.sql.utils import reformat_sql
+
+
+class SQLSelectForm(forms.Form):
+    """
+    Validate params
+
+        sql: The sql statement with interpolated params
+        raw_sql: The sql statement with placeholders
+        params: JSON encoded parameter values
+        duration: time for SQL to execute passed in from toolbar just for redisplay
+        hash: the hash of (secret + sql + params) for tamper checking
+    """
+    sql = forms.CharField()
+    raw_sql = forms.CharField()
+    params = forms.CharField()
+    alias = forms.CharField(required=False, initial='default')
+    duration = forms.FloatField()
+    hash = forms.CharField()
+
+    def __init__(self, *args, **kwargs):
+        initial = kwargs.get('initial', None)
+
+        if initial is not None:
+            initial['hash'] = self.make_hash(initial)
+
+        super(SQLSelectForm, self).__init__(*args, **kwargs)
+
+        for name in self.fields:
+            self.fields[name].widget = forms.HiddenInput()
+
+    def clean_raw_sql(self):
+        value = self.cleaned_data['raw_sql']
+
+        if not value.lower().strip().startswith('select'):
+            raise ValidationError("Only 'select' queries are allowed.")
+
+        return value
+
+    def clean_params(self):
+        value = self.cleaned_data['params']
+
+        try:
+            return json.loads(value)
+        except ValueError:
+            raise ValidationError('Is not valid JSON')
+
+    def clean_alias(self):
+        value = self.cleaned_data['alias']
+
+        if value not in connections:
+            raise ValidationError("Database alias '%s' not found" % value)
+
+        return value
+
+    def clean_hash(self):
+        hash = self.cleaned_data['hash']
+
+        if hash != self.make_hash(self.data):
+            raise ValidationError('Tamper alert')
+
+        return hash
+
+    def reformat_sql(self):
+        return reformat_sql(self.cleaned_data['sql'])
+
+    def make_hash(self, data):
+        params = (force_text(settings.SECRET_KEY) +
+                  force_text(data['sql']) + force_text(data['params']))
+        return hashlib.sha1(params.encode('utf-8')).hexdigest()
+
+    @property
+    def connection(self):
+        return connections[self.cleaned_data['alias']]
+
+    @cached_property
+    def cursor(self):
+        return self.connection.cursor()
diff --git a/debug_toolbar/panels/sql.py b/debug_toolbar/panels/sql/panel.py
index 226778b..cb80901 100644
--- a/debug_toolbar/panels/sql.py
+++ b/debug_toolbar/panels/sql/panel.py
@@ -5,16 +5,13 @@ from copy import copy
 
 from django.conf.urls import patterns, url
 from django.db import connections
-from django.http import HttpResponseBadRequest
-from django.shortcuts import render
 from django.utils.translation import ugettext_lazy as _, ungettext_lazy as __
-from django.views.decorators.csrf import csrf_exempt
 
-from debug_toolbar.forms import SQLSelectForm
 from debug_toolbar.panels import DebugPanel
+from debug_toolbar.panels.sql.forms import SQLSelectForm
 from debug_toolbar.utils import render_stacktrace
-from debug_toolbar.utils.sql import reformat_sql
-from debug_toolbar.utils.tracking.db import CursorWrapper
+from debug_toolbar.panels.sql.utils import reformat_sql
+from debug_toolbar.panels.sql.tracking import CursorWrapper
 
 
 def get_isolation_level_display(engine, level):
@@ -109,7 +106,7 @@ class SQLDebugPanel(DebugPanel):
 
     @classmethod
     def get_urls(cls):
-        return patterns('debug_toolbar.panels.sql',                     # noqa
+        return patterns('debug_toolbar.panels.sql.views',               # noqa
             url(r'^sql_select/$', 'sql_select', name='sql_select'),
             url(r'^sql_explain/$', 'sql_explain', name='sql_explain'),
             url(r'^sql_profile/$', 'sql_profile', name='sql_profile'),
@@ -211,109 +208,3 @@ class SQLDebugPanel(DebugPanel):
             'queries': [q for a, q in self._queries],
             'sql_time': self._sql_time,
         })
-
-
-@csrf_exempt
-def sql_select(request):
-    """Returns the output of the SQL SELECT statement"""
-    form = SQLSelectForm(request.POST or None)
-
-    if form.is_valid():
-        sql = form.cleaned_data['raw_sql']
-        params = form.cleaned_data['params']
-        cursor = form.cursor
-        cursor.execute(sql, params)
-        headers = [d[0] for d in cursor.description]
-        result = cursor.fetchall()
-        cursor.close()
-        context = {
-            'result': result,
-            'sql': form.reformat_sql(),
-            'duration': form.cleaned_data['duration'],
-            'headers': headers,
-            'alias': form.cleaned_data['alias'],
-        }
-        return render(request, 'debug_toolbar/panels/sql_select.html', context)
-    return HttpResponseBadRequest('Form errors')
-
-
-@csrf_exempt
-def sql_explain(request):
-    """Returns the output of the SQL EXPLAIN on the given query"""
-    form = SQLSelectForm(request.POST or None)
-
-    if form.is_valid():
-        sql = form.cleaned_data['raw_sql']
-        params = form.cleaned_data['params']
-        cursor = form.cursor
-
-        conn = form.connection
-        engine = conn.__class__.__module__.split('.', 1)[0]
-
-        if engine == "sqlite3":
-            # SQLite's EXPLAIN dumps the low-level opcodes generated for a query;
-            # EXPLAIN QUERY PLAN dumps a more human-readable summary
-            # See http://www.sqlite.org/lang_explain.html for details
-            cursor.execute("EXPLAIN QUERY PLAN %s" % (sql,), params)
-        elif engine == "psycopg2":
-            cursor.execute("EXPLAIN ANALYZE %s" % (sql,), params)
-        else:
-            cursor.execute("EXPLAIN %s" % (sql,), params)
-
-        headers = [d[0] for d in cursor.description]
-        result = cursor.fetchall()
-        cursor.close()
-        context = {
-            'result': result,
-            'sql': form.reformat_sql(),
-            'duration': form.cleaned_data['duration'],
-            'headers': headers,
-            'alias': form.cleaned_data['alias'],
-        }
-        return render(request, 'debug_toolbar/panels/sql_explain.html', context)
-    return HttpResponseBadRequest('Form errors')
-
-
-@csrf_exempt
-def sql_profile(request):
-    """Returns the output of running the SQL and getting the profiling statistics"""
-    form = SQLSelectForm(request.POST or None)
-
-    if form.is_valid():
-        sql = form.cleaned_data['raw_sql']
-        params = form.cleaned_data['params']
-        cursor = form.cursor
-        result = None
-        headers = None
-        result_error = None
-        try:
-            cursor.execute("SET PROFILING=1")  # Enable profiling
-            cursor.execute(sql, params)  # Execute SELECT
-            cursor.execute("SET PROFILING=0")  # Disable profiling
-            # The Query ID should always be 1 here but I'll subselect to get
-            # the last one just in case...
-            cursor.execute("""
-  SELECT  *
-    FROM  information_schema.profiling
-   WHERE  query_id = (
-          SELECT  query_id
-            FROM  information_schema.profiling
-        ORDER BY  query_id DESC
-           LIMIT  1
-        )
-""")
-            headers = [d[0] for d in cursor.description]
-            result = cursor.fetchall()
-        except Exception:
-            result_error = "Profiling is either not available or not supported by your database."
-        cursor.close()
-        context = {
-            'result': result,
-            'result_error': result_error,
-            'sql': form.reformat_sql(),
-            'duration': form.cleaned_data['duration'],
-            'headers': headers,
-            'alias': form.cleaned_data['alias'],
-        }
-        return render(request, 'debug_toolbar/panels/sql_profile.html', context)
-    return HttpResponseBadRequest('Form errors')
diff --git a/debug_toolbar/panels/sql/tracking.py b/debug_toolbar/panels/sql/tracking.py
new file mode 100644
index 0000000..fd56ff9
--- /dev/null
+++ b/debug_toolbar/panels/sql/tracking.py
@@ -0,0 +1,168 @@
+from __future__ import unicode_literals
+
+import sys
+
+import json
+from threading import local
+from time import time
+
+from django.template import Node
+from django.utils.encoding import force_text
+from django.utils import six
+
+from debug_toolbar.utils import tidy_stacktrace, get_template_info, get_stack
+from debug_toolbar.utils import settings as dt_settings
+
+
+class SQLQueryTriggered(Exception):
+    """Thrown when template panel triggers a query"""
+    pass
+
+
+class ThreadLocalState(local):
+    def __init__(self):
+        self.enabled = True
+
+    @property
+    def Wrapper(self):
+        if self.enabled:
+            return NormalCursorWrapper
+        return ExceptionCursorWrapper
+
+    def recording(self, v):
+        self.enabled = v
+
+
+state = ThreadLocalState()
+recording = state.recording  # export function
+
+
+def CursorWrapper(*args, **kwds):  # behave like a class
+    return state.Wrapper(*args, **kwds)
+
+
+class ExceptionCursorWrapper(object):
+    """
+    Wraps a cursor and raises an exception on any operation.
+    Used in Templates panel.
+    """
+    def __init__(self, cursor, db, logger):
+        pass
+
+    def __getattr__(self, attr):
+        raise SQLQueryTriggered()
+
+
+class NormalCursorWrapper(object):
+    """
+    Wraps a cursor and logs queries.
+    """
+
+    def __init__(self, cursor, db, logger):
+        self.cursor = cursor
+        # Instance of a BaseDatabaseWrapper subclass
+        self.db = db
+        # logger must implement a ``record`` method
+        self.logger = logger
+
+    def _quote_expr(self, element):
+        if isinstance(element, six.string_types):
+            return "'%s'" % force_text(element).replace("'", "''")
+        else:
+            return repr(element)
+
+    def _quote_params(self, params):
+        if not params:
+            return params
+        if isinstance(params, dict):
+            return dict((key, self._quote_expr(value))
+                        for key, value in params.items())
+        return list(map(self._quote_expr, params))
+
+    def _decode(self, param):
+        try:
+            return force_text(param, strings_only=True)
+        except UnicodeDecodeError:
+            return '(encoded string)'
+
+    def execute(self, sql, params=()):
+        start_time = time()
+        try:
+            return self.cursor.execute(sql, params)
+        finally:
+            stop_time = time()
+            duration = (stop_time - start_time) * 1000
+            if dt_settings.CONFIG['ENABLE_STACKTRACES']:
+                stacktrace = tidy_stacktrace(reversed(get_stack()))
+            else:
+                stacktrace = []
+            _params = ''
+            try:
+                _params = json.dumps(list(map(self._decode, params)))
+            except Exception:
+                pass  # object not JSON serializable
+
+            template_info = None
+            cur_frame = sys._getframe().f_back
+            try:
+                while cur_frame is not None:
+                    if cur_frame.f_code.co_name == 'render':
+                        node = cur_frame.f_locals['self']
+                        if isinstance(node, Node):
+                            template_info = get_template_info(node.source)
+                            break
+                    cur_frame = cur_frame.f_back
+            except Exception:
+                pass
+            del cur_frame
+
+            alias = getattr(self.db, 'alias', 'default')
+            conn = self.db.connection
+            # HACK: avoid imports
+            if conn:
+                engine = conn.__class__.__module__.split('.', 1)[0]
+            else:
+                engine = 'unknown'
+
+            params = {
+                'engine': engine,
+                'alias': alias,
+                'sql': self.db.ops.last_executed_query(
+                    self.cursor, sql, self._quote_params(params)),
+                'duration': duration,
+                'raw_sql': sql,
+                'params': _params,
+                'stacktrace': stacktrace,
+                'start_time': start_time,
+                'stop_time': stop_time,
+                'is_slow': duration > dt_settings.CONFIG['SQL_WARNING_THRESHOLD'],
+                'is_select': sql.lower().strip().startswith('select'),
+                'template_info': template_info,
+            }
+
+            if engine == 'psycopg2':
+                # If an erroneous query was ran on the connection, it might
+                # be in a state where checking isolation_level raises an
+                # exception.
+                try:
+                    iso_level = conn.isolation_level
+                except conn.InternalError:
+                    iso_level = 'unknown'
+                params.update({
+                    'trans_id': self.logger.get_transaction_id(alias),
+                    'trans_status': conn.get_transaction_status(),
+                    'iso_level': iso_level,
+                    'encoding': conn.encoding,
+                })
+
+            # We keep `sql` to maintain backwards compatibility
+            self.logger.record(**params)
+
+    def executemany(self, sql, param_list):
+        return self.cursor.executemany(sql, param_list)
+
+    def __getattr__(self, attr):
+        return getattr(self.cursor, attr)
+
+    def __iter__(self):
+        return iter(self.cursor)
diff --git a/debug_toolbar/panels/sql/utils.py b/debug_toolbar/panels/sql/utils.py
new file mode 100644
index 0000000..00728a3
--- /dev/null
+++ b/debug_toolbar/panels/sql/utils.py
@@ -0,0 +1,37 @@
+from __future__ import unicode_literals
+
+import re
+
+from django.utils.html import escape
+
+import sqlparse
+from sqlparse import tokens as T
+
+
+class BoldKeywordFilter:
+    """sqlparse filter to bold SQL keywords"""
+    def process(self, stack, stream):
+        """Process the token stream"""
+        for token_type, value in stream:
+            is_keyword = token_type in T.Keyword
+            if is_keyword:
+                yield T.Text, '<strong>'
+            yield token_type, escape(value)
+            if is_keyword:
+                yield T.Text, '</strong>'
+
+
+def reformat_sql(sql):
+    stack = sqlparse.engine.FilterStack()
+    stack.preprocess.append(BoldKeywordFilter())  # add our custom filter
+    stack.postprocess.append(sqlparse.filters.SerializerUnicode())  # tokens -> strings
+    return swap_fields(''.join(stack.run(sql)))
+
+
+def swap_fields(sql):
+    expr = r'SELECT</strong> (...........*?) <strong>FROM'
+    subs = (r'SELECT</strong> '
+            r'<a class="djDebugUncollapsed djDebugToggle" href="#">&#8226;&#8226;&#8226;</a> '
+            r'<a class="djDebugCollapsed djDebugToggle" href="#">\1</a> '
+            r'<strong>FROM')
+    return re.sub(expr, subs, sql)
diff --git a/debug_toolbar/panels/sql/views.py b/debug_toolbar/panels/sql/views.py
new file mode 100644
index 0000000..346cf6e
--- /dev/null
+++ b/debug_toolbar/panels/sql/views.py
@@ -0,0 +1,113 @@
+from __future__ import unicode_literals
+
+from django.http import HttpResponseBadRequest
+from django.shortcuts import render
+from django.views.decorators.csrf import csrf_exempt
+
+from debug_toolbar.panels.sql.forms import SQLSelectForm
+
+
+@csrf_exempt
+def sql_select(request):
+    """Returns the output of the SQL SELECT statement"""
+    form = SQLSelectForm(request.POST or None)
+
+    if form.is_valid():
+        sql = form.cleaned_data['raw_sql']
+        params = form.cleaned_data['params']
+        cursor = form.cursor
+        cursor.execute(sql, params)
+        headers = [d[0] for d in cursor.description]
+        result = cursor.fetchall()
+        cursor.close()
+        context = {
+            'result': result,
+            'sql': form.reformat_sql(),
+            'duration': form.cleaned_data['duration'],
+            'headers': headers,
+            'alias': form.cleaned_data['alias'],
+        }
+        return render(request, 'debug_toolbar/panels/sql_select.html', context)
+    return HttpResponseBadRequest('Form errors')
+
+
+@csrf_exempt
+def sql_explain(request):
+    """Returns the output of the SQL EXPLAIN on the given query"""
+    form = SQLSelectForm(request.POST or None)
+
+    if form.is_valid():
+        sql = form.cleaned_data['raw_sql']
+        params = form.cleaned_data['params']
+        cursor = form.cursor
+
+        conn = form.connection
+        engine = conn.__class__.__module__.split('.', 1)[0]
+
+        if engine == "sqlite3":
+            # SQLite's EXPLAIN dumps the low-level opcodes generated for a query;
+            # EXPLAIN QUERY PLAN dumps a more human-readable summary
+            # See http://www.sqlite.org/lang_explain.html for details
+            cursor.execute("EXPLAIN QUERY PLAN %s" % (sql,), params)
+        elif engine == "psycopg2":
+            cursor.execute("EXPLAIN ANALYZE %s" % (sql,), params)
+        else:
+            cursor.execute("EXPLAIN %s" % (sql,), params)
+
+        headers = [d[0] for d in cursor.description]
+        result = cursor.fetchall()
+        cursor.close()
+        context = {
+            'result': result,
+            'sql': form.reformat_sql(),
+            'duration': form.cleaned_data['duration'],
+            'headers': headers,
+            'alias': form.cleaned_data['alias'],
+        }
+        return render(request, 'debug_toolbar/panels/sql_explain.html', context)
+    return HttpResponseBadRequest('Form errors')
+
+
+@csrf_exempt
+def sql_profile(request):
+    """Returns the output of running the SQL and getting the profiling statistics"""
+    form = SQLSelectForm(request.POST or None)
+
+    if form.is_valid():
+        sql = form.cleaned_data['raw_sql']
+        params = form.cleaned_data['params']
+        cursor = form.cursor
+        result = None
+        headers = None
+        result_error = None
+        try:
+            cursor.execute("SET PROFILING=1")  # Enable profiling
+            cursor.execute(sql, params)  # Execute SELECT
+            cursor.execute("SET PROFILING=0")  # Disable profiling
+            # The Query ID should always be 1 here but I'll subselect to get
+            # the last one just in case...
+            cursor.execute("""
+  SELECT  *
+    FROM  information_schema.profiling
+   WHERE  query_id = (
+          SELECT  query_id
+            FROM  information_schema.profiling
+        ORDER BY  query_id DESC
+           LIMIT  1
+        )
+""")
+            headers = [d[0] for d in cursor.description]
+            result = cursor.fetchall()
+        except Exception:
+            result_error = "Profiling is either not available or not supported by your database."
+        cursor.close()
+        context = {
+            'result': result,
+            'result_error': result_error,
+            'sql': form.reformat_sql(),
+            'duration': form.cleaned_data['duration'],
+            'headers': headers,
+            'alias': form.cleaned_data['alias'],
+        }
+        return render(request, 'debug_toolbar/panels/sql_profile.html', context)
+    return HttpResponseBadRequest('Form errors')
diff --git a/debug_toolbar/panels/template/__init__.py b/debug_toolbar/panels/template/__init__.py
new file mode 100644
index 0000000..d2f595d
--- /dev/null
+++ b/debug_toolbar/panels/template/__init__.py
@@ -0,0 +1 @@
+from debug_toolbar.panels.template.panel import TemplateDebugPanel      # noqa
diff --git a/debug_toolbar/panels/template.py b/debug_toolbar/panels/template/panel.py
index e21cc9e..7c4b06a 100644
--- a/debug_toolbar/panels/template.py
+++ b/debug_toolbar/panels/template/panel.py
@@ -5,21 +5,17 @@ from pprint import pformat
 
 import django
 from django import http
-from django.http import HttpResponseBadRequest
 from django.conf import settings
 from django.conf.urls import patterns, url
 from django.db.models.query import QuerySet, RawQuerySet
-from django.shortcuts import render
-from django.template import TemplateDoesNotExist
 from django.template.context import get_standard_processors
 from django.test.signals import template_rendered
 from django.utils.encoding import force_text
-from django.utils.safestring import mark_safe
 from django.utils import six
 from django.utils.translation import ugettext_lazy as _
 
 from debug_toolbar.panels import DebugPanel
-from debug_toolbar.utils.tracking.db import recording, SQLQueryTriggered
+from debug_toolbar.panels.sql.tracking import recording, SQLQueryTriggered
 from debug_toolbar.utils import settings as dt_settings
 
 # Code taken and adapted from Simon Willison and Django Snippets:
@@ -120,8 +116,8 @@ class TemplateDebugPanel(DebugPanel):
 
     @classmethod
     def get_urls(cls):
-        return patterns('debug_toolbar.panels.template',                # noqa
-            url(r'^template_source/$', template_source, name='template_source'),
+        return patterns('debug_toolbar.panels.template.views',          # noqa
+            url(r'^template_source/$', 'template_source', name='template_source'),
         )
 
     def nav_title(self):
@@ -161,42 +157,3 @@ class TemplateDebugPanel(DebugPanel):
             'template_dirs': [normpath(x) for x in settings.TEMPLATE_DIRS],
             'context_processors': context_processors,
         })
-
-
-def template_source(request):
-    """
-    Return the source of a template, syntax-highlighted by Pygments if
-    it's available.
-    """
-    template_name = request.GET.get('template', None)
-    if template_name is None:
-        return HttpResponseBadRequest('"template" key is required')
-
-    from django.template.loader import find_template_loader
-    loaders = []
-    for loader_name in settings.TEMPLATE_LOADERS:
-        loader = find_template_loader(loader_name)
-        if loader is not None:
-            loaders.append(loader)
-    for loader in loaders:
-        try:
-            source, display_name = loader.load_template_source(template_name)
-            break
-        except TemplateDoesNotExist:
-            source = "Template Does Not Exist: %s" % (template_name,)
-
-    try:
-        from pygments import highlight
-        from pygments.lexers import HtmlDjangoLexer
-        from pygments.formatters import HtmlFormatter
-
-        source = highlight(source, HtmlDjangoLexer(), HtmlFormatter())
-        source = mark_safe(source)
-        source.pygmentized = True
-    except ImportError:
-        pass
-
-    return render(request, 'debug_toolbar/panels/template_source.html', {
-        'source': source,
-        'template_name': template_name
-    })
diff --git a/debug_toolbar/panels/template/views.py b/debug_toolbar/panels/template/views.py
new file mode 100644
index 0000000..30bd167
--- /dev/null
+++ b/debug_toolbar/panels/template/views.py
@@ -0,0 +1,46 @@
+from __future__ import unicode_literals
+
+from django.http import HttpResponseBadRequest
+from django.conf import settings
+from django.shortcuts import render
+from django.template import TemplateDoesNotExist
+from django.template.loader import find_template_loader
+from django.utils.safestring import mark_safe
+
+
+def template_source(request):
+    """
+    Return the source of a template, syntax-highlighted by Pygments if
+    it's available.
+    """
+    template_name = request.GET.get('template', None)
+    if template_name is None:
+        return HttpResponseBadRequest('"template" key is required')
+
+    loaders = []
+    for loader_name in settings.TEMPLATE_LOADERS:
+        loader = find_template_loader(loader_name)
+        if loader is not None:
+            loaders.append(loader)
+    for loader in loaders:
+        try:
+            source, display_name = loader.load_template_source(template_name)
+            break
+        except TemplateDoesNotExist:
+            source = "Template Does Not Exist: %s" % (template_name,)
+
+    try:
+        from pygments import highlight
+        from pygments.lexers import HtmlDjangoLexer
+        from pygments.formatters import HtmlFormatter
+
+        source = highlight(source, HtmlDjangoLexer(), HtmlFormatter())
+        source = mark_safe(source)
+        source.pygmentized = True
+    except ImportError:
+        pass
+
+    return render(request, 'debug_toolbar/panels/template_source.html', {
+        'source': source,
+        'template_name': template_name
+    })