Make browsable API views play nice with utf-8

2 files changed, 41 insertions, 0 deletions

diff --git a/rest_framework/tests/test_utils.py b/rest_framework/tests/test_utils.py
new file mode 100644
index 00000000..da508dbb
--- /dev/null
+++ b/rest_framework/tests/test_utils.py
@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+
+from django.test import TestCase
+from rest_framework.utils import formatting
+import sys
+
+
+class FormattingUnitTests(TestCase):
+    def setUp(self):
+        # test strings snatched from http://www.columbia.edu/~fdc/utf8/,
+        # http://winrus.com/utf8-jap.htm and memory
+        self.utf8_test_string = (
+            'zażółć gęślą jaźń'
+            'Sîne klâwen durh die wolken sint geslagen'
+            'Τη γλώσσα μου έδωσαν ελληνική'
+            'யாமறிந்த மொழிகளிலே தமிழ்மொழி'
+            'На берегу пустынных волн'
+            ' てすと'
+            'ｱｲｳｴｵｶｷｸｹｺｻｼｽｾｿﾀﾁﾂﾃ'
+        )
+        self.non_utf8_test_string = ('The quick brown fox jumps over the lazy '
+                                     'dog')
+
+    def test_for_ascii_support_in_remove_leading_indent(self):
+        if sys.version_info < (3, 0):
+            # only Python 2.x is affected, so we skip the test entirely
+            # if on Python 3.x
+            self.assertEqual(formatting._remove_leading_indent(
+                self.non_utf8_test_string), self.non_utf8_test_string)
+
+    def test_for_utf8_support_in_remove_leading_indent(self):
+        if sys.version_info < (3, 0):
+            # only Python 2.x is affected, so we skip the test entirely
+            # if on Python 3.x
+            self.assertEqual(formatting._remove_leading_indent(
+                self.utf8_test_string), self.utf8_test_string.decode('utf-8'))
diff --git a/rest_framework/utils/formatting.py b/rest_framework/utils/formatting.py
index ebadb3a6..a2a5609c 100644
--- a/rest_framework/utils/formatting.py
+++ b/rest_framework/utils/formatting.py
@@ -24,6 +24,11 @@ def _remove_leading_indent(content):
     Remove leading indent from a block of text.
     Used when generating descriptions from docstrings.
     """
+    try:
+        content = content.decode('utf-8')
+    except (AttributeError, UnicodeEncodeError):
+        pass  # the string should keep the default 'ascii' encoding in
+              # Python 2.x or stay a unicode string in Python 3.x
     whitespace_counts = [len(line) - len(line.lstrip(' '))
                          for line in content.splitlines()[1:] if line.lstrip()]