From 6578f0b8ea3b26022bb5161a40a77c6b15de21ae Mon Sep 17 00:00:00 2001 From: Matthew J Morrison Date: Wed, 8 Dec 2010 08:42:57 -0600 Subject: Added clean_params method to DatabaseStatTracker to scrub non-unicode data for displaying on the sql panel Signed-off-by: Rob Hudson --- debug_toolbar/panels/sql.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) (limited to 'debug_toolbar/panels') diff --git a/debug_toolbar/panels/sql.py b/debug_toolbar/panels/sql.py index e1e9bdf..299da66 100644 --- a/debug_toolbar/panels/sql.py +++ b/debug_toolbar/panels/sql.py @@ -12,7 +12,7 @@ from django.views.debug import linebreak_iter from django.template import Node from django.template.loader import render_to_string from django.utils import simplejson -from django.utils.encoding import force_unicode +from django.utils.encoding import force_unicode, DjangoUnicodeDecodeError from django.utils.hashcompat import sha_constructor from django.utils.translation import ugettext_lazy as _ @@ -85,12 +85,24 @@ class DatabaseStatTracker(util.CursorDebugWrapper): Replacement for CursorDebugWrapper which stores additional information in `connection.queries`. """ + def clean_params(self, params): + clean_params = () + for x in params: + try: + force_unicode(x, strings_only=True) + except DjangoUnicodeDecodeError: + clean_params += ("", ) + else: + clean_params += (x, ) + return clean_params + def execute(self, sql, params=()): start = datetime.now() try: return self.cursor.execute(sql, params) finally: stop = datetime.now() + params = self.clean_params(params) duration = ms_from_timedelta(stop - start) stacktrace = tidy_stacktrace(traceback.extract_stack()) _params = '' -- cgit v1.2.3 From 1d90d8e19c8f7a06a8f3e61535f4f83c12cb58c6 Mon Sep 17 00:00:00 2001 From: Vincent Driessen Date: Sun, 9 Jan 2011 10:06:25 -0800 Subject: Added support for LogBook. Thanks to Vincent Driessen for the idea and patch. Signed-off-by: Rob Hudson --- debug_toolbar/panels/logger.py | 78 +++++++++++++++++++++++++++++++----------- 1 file changed, 58 insertions(+), 20 deletions(-) (limited to 'debug_toolbar/panels') diff --git a/debug_toolbar/panels/logger.py b/debug_toolbar/panels/logger.py index 620102e..0ddbfb1 100644 --- a/debug_toolbar/panels/logger.py +++ b/debug_toolbar/panels/logger.py @@ -8,16 +8,16 @@ from django.template.loader import render_to_string from django.utils.translation import ugettext_lazy as _ from debug_toolbar.panels import DebugPanel -class ThreadTrackingHandler(logging.Handler): + +class LogCollector(object): def __init__(self): if threading is None: raise NotImplementedError("threading module is not available, \ the logging panel cannot be used without it") - logging.Handler.__init__(self) self.records = {} # a dictionary that maps threads to log records - def emit(self, record): - self.get_records().append(record) + def add_record(self, record, thread=None): + self.get_records(thread).append(record) def get_records(self, thread=None): """ @@ -36,20 +36,67 @@ class ThreadTrackingHandler(logging.Handler): if thread in self.records: del self.records[thread] -handler = ThreadTrackingHandler() + +class ThreadTrackingHandler(logging.Handler): + def __init__(self, collector): + logging.Handler.__init__(self) + self.collector = collector + + def emit(self, record): + record = { + 'message': record.getMessage(), + 'time': datetime.datetime.fromtimestamp(record.created), + 'level': record.levelname, + 'file': record.pathname, + 'line': record.lineno, + 'channel': record.name, + } + self.collector.add_record(record) + + +collector = LogCollector() +logging_handler = ThreadTrackingHandler(collector) logging.root.setLevel(logging.NOTSET) -logging.root.addHandler(handler) +logging.root.addHandler(logging_handler) # register with logging + +try: + import logbook + logbook_supported = True +except ImportError: + # logbook support is optional, so fail silently + logbook_supported = False + +if logbook_supported: + class LogbookThreadTrackingHandler(logbook.handlers.Handler): + def __init__(self, collector): + logbook.handlers.Handler.__init__(self, bubble=True) + self.collector = collector + + def emit(self, record): + record = { + 'message': record.message, + 'time': record.time, + 'level': record.level_name, + 'file': record.filename, + 'line': record.lineno, + 'channel': record.channel, + } + self.collector.add_record(record) + + + logbook_handler = LogbookThreadTrackingHandler(collector) + logbook_handler.push_application() # register with logbook class LoggingPanel(DebugPanel): name = 'Logging' has_content = True def process_request(self, request): - handler.clear_records() + collector.clear_records() def get_and_delete(self): - records = handler.get_records() - handler.clear_records() + records = collector.get_records() + collector.clear_records() return records def nav_title(self): @@ -57,7 +104,7 @@ class LoggingPanel(DebugPanel): def nav_subtitle(self): # FIXME l10n: use ngettext - return "%s message%s" % (len(handler.get_records()), (len(handler.get_records()) == 1) and '' or 's') + return "%s message%s" % (len(collector.get_records()), (len(collector.get_records()) == 1) and '' or 's') def title(self): return _('Log Messages') @@ -66,16 +113,7 @@ class LoggingPanel(DebugPanel): return '' def content(self): - records = [] - for record in self.get_and_delete(): - records.append({ - 'message': record.getMessage(), - 'time': datetime.datetime.fromtimestamp(record.created), - 'level': record.levelname, - 'file': record.pathname, - 'line': record.lineno, - }) - + records = self.get_and_delete() context = self.context.copy() context.update({'records': records}) -- cgit v1.2.3 From a2edb76e8dc900ab5f147e062ceee5ac3ac35a18 Mon Sep 17 00:00:00 2001 From: postal2600 Date: Thu, 25 Nov 2010 01:44:46 -0800 Subject: Added support for executemany. Fixes issue #120 Signed-off-by: Rob Hudson --- debug_toolbar/panels/sql.py | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) (limited to 'debug_toolbar/panels') diff --git a/debug_toolbar/panels/sql.py b/debug_toolbar/panels/sql.py index 299da66..a3eff49 100644 --- a/debug_toolbar/panels/sql.py +++ b/debug_toolbar/panels/sql.py @@ -125,6 +125,49 @@ class DatabaseStatTracker(util.CursorDebugWrapper): pass del cur_frame + # We keep `sql` to maintain backwards compatibility + self.db.queries.append({ + 'sql': self.db.ops.last_executed_query(self.cursor, sql, params), + 'duration': duration, + 'raw_sql': sql, + 'params': _params, + 'hash': sha_constructor(settings.SECRET_KEY + sql + _params).hexdigest(), + 'stacktrace': stacktrace, + 'start_time': start, + 'stop_time': stop, + 'is_slow': (duration > SQL_WARNING_THRESHOLD), + 'is_select': sql.lower().strip().startswith('select'), + 'template_info': template_info, + }) + + def executemany(self, sql, params=()): + start = datetime.now() + try: + return self.cursor.executemany(sql, params) + finally: + stop = datetime.now() + duration = ms_from_timedelta(stop - start) + stacktrace = tidy_stacktrace(traceback.extract_stack()) + _params = '' + try: + _params = simplejson.dumps([force_unicode(x, strings_only=True) for x in params]) + except TypeError: + pass # object not JSON serializable + + template_info = None + cur_frame = sys._getframe().f_back + try: + while cur_frame is not None: + if cur_frame.f_code.co_name == 'render': + node = cur_frame.f_locals['self'] + if isinstance(node, Node): + template_info = get_template_info(node.source) + break + cur_frame = cur_frame.f_back + except: + pass + del cur_frame + # We keep `sql` to maintain backwards compatibility self.db.queries.append({ 'sql': self.db.ops.last_executed_query(self.cursor, sql, params), -- cgit v1.2.3 From 634bd0d68cb30d25705a049d46aa72e9a64e9dbd Mon Sep 17 00:00:00 2001 From: Rob Hudson Date: Wed, 6 Apr 2011 07:52:45 -0700 Subject: Added basic support for multi-db. Thanks to David Cramer, Ionel Cristian Mărieș, noah256, and Sander Steffann, who all supplied patches. --- debug_toolbar/panels/sql.py | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) (limited to 'debug_toolbar/panels') diff --git a/debug_toolbar/panels/sql.py b/debug_toolbar/panels/sql.py index a3eff49..90a7b51 100644 --- a/debug_toolbar/panels/sql.py +++ b/debug_toolbar/panels/sql.py @@ -6,7 +6,14 @@ import traceback import django from django.conf import settings -from django.db import connection +try: + from django.db import connections +except ImportError: + # Compatibility with Django < 1.2 + from django.db import connection + connections = {'default': connection} + connection.alias = 'default' + from django.db.backends import util from django.views.debug import linebreak_iter from django.template import Node @@ -194,16 +201,27 @@ class SQLDebugPanel(DebugPanel): def __init__(self, *args, **kwargs): super(self.__class__, self).__init__(*args, **kwargs) - self._offset = len(connection.queries) + self._offset = dict((conn, len(connections[conn].queries)) for conn in connections) self._sql_time = 0 self._queries = [] + self._databases = {} def nav_title(self): return _('SQL') def nav_subtitle(self): - self._queries = connection.queries[self._offset:] - self._sql_time = sum([q['duration'] for q in self._queries]) + self._queries = [] + self._databases = {} + for alias in connections: + db_queries = connections[alias].queries[self._offset[alias]:] + self._databases[alias] = { + 'time_spent': sum(q['duration'] for q in db_queries), + 'queries': len(db_queries), + } + self._queries.extend([(alias, q) for q in db_queries]) + + self._queries.sort(key=lambda x: x[1]['start_time']) + self._sql_time = sum([d['time_spent'] for d in self._databases.itervalues()]) num_queries = len(self._queries) # TODO l10n: use ngettext return "%d %s in %.2fms" % ( @@ -220,7 +238,8 @@ class SQLDebugPanel(DebugPanel): def content(self): width_ratio_tally = 0 - for query in self._queries: + for alias, query in self._queries: + query['alias'] = alias query['sql'] = reformat_sql(query['sql']) try: query['width_ratio'] = (query['duration'] / self._sql_time) * 100 @@ -231,7 +250,8 @@ class SQLDebugPanel(DebugPanel): context = self.context.copy() context.update({ - 'queries': self._queries, + 'databases': sorted(self._databases.items(), key=lambda x: -x[1]['time_spent']), + 'queries': [q for a, q in self._queries], 'sql_time': self._sql_time, 'is_mysql': settings.DATABASE_ENGINE == 'mysql', }) -- cgit v1.2.3 From ed26be9548a4dedc5b3b21bf338e5a0a9bd2484b Mon Sep 17 00:00:00 2001 From: Rob Hudson Date: Fri, 8 Apr 2011 09:33:45 -0700 Subject: Updated sql panel to include logging support. This matches Django's added logging support to the debug cursor.--- debug_toolbar/panels/sql.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) (limited to 'debug_toolbar/panels') diff --git a/debug_toolbar/panels/sql.py b/debug_toolbar/panels/sql.py index 90a7b51..c6e73a0 100644 --- a/debug_toolbar/panels/sql.py +++ b/debug_toolbar/panels/sql.py @@ -21,6 +21,14 @@ from django.template.loader import render_to_string from django.utils import simplejson from django.utils.encoding import force_unicode, DjangoUnicodeDecodeError from django.utils.hashcompat import sha_constructor +try: + from django.utils.log import getLogger + logger = getLogger('django.db.backends') + has_logger = True +except ImportError: + # Compatibility with Django < 1.2 + has_logger = False + from django.utils.translation import ugettext_lazy as _ from debug_toolbar.panels import DebugPanel @@ -132,6 +140,12 @@ class DatabaseStatTracker(util.CursorDebugWrapper): pass del cur_frame + # Logging was added in Django 1.3 + if has_logger: + logger.debug('(%.3f) %s; args=%s' % (duration, sql, params), + extra={'duration':duration, 'sql':sql, 'params':params} + ) + # We keep `sql` to maintain backwards compatibility self.db.queries.append({ 'sql': self.db.ops.last_executed_query(self.cursor, sql, params), @@ -175,6 +189,11 @@ class DatabaseStatTracker(util.CursorDebugWrapper): pass del cur_frame + if has_logger: + logger.debug('(%.3f) %s; args=%s' % (duration, sql, params), + extra={'duration':duration, 'sql':sql, 'params':params} + ) + # We keep `sql` to maintain backwards compatibility self.db.queries.append({ 'sql': self.db.ops.last_executed_query(self.cursor, sql, params), -- cgit v1.2.3 From ad8071f6657b539570949bef6d4d82c89bfd19fe Mon Sep 17 00:00:00 2001 From: Rob Hudson Date: Fri, 8 Apr 2011 09:34:36 -0700 Subject: Stripped out logging of SQL queries from logging panel. --- debug_toolbar/panels/logger.py | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'debug_toolbar/panels') diff --git a/debug_toolbar/panels/logger.py b/debug_toolbar/panels/logger.py index 0ddbfb1..5e82a13 100644 --- a/debug_toolbar/panels/logger.py +++ b/debug_toolbar/panels/logger.py @@ -17,6 +17,11 @@ class LogCollector(object): self.records = {} # a dictionary that maps threads to log records def add_record(self, record, thread=None): + # Avoid logging SQL queries since they are already in the SQL panel + # TODO: Make this check whether SQL panel is enabled + if record.get('channel', '') == 'django.db.backends': + return + self.get_records(thread).append(record) def get_records(self, thread=None): -- cgit v1.2.3