Fix charset issues

4 files changed, 40 insertions, 28 deletions

diff --git a/rest_framework/renderers.py b/rest_framework/renderers.py
index c67c8ed6..fd868346 100644
--- a/rest_framework/renderers.py
+++ b/rest_framework/renderers.py
@@ -9,7 +9,6 @@ REST framework also provides an HTML renderer the renders the browsable API.
 from __future__ import unicode_literals
 
 import copy
-import string
 import json
 from django import forms
 from django.http.multipartparser import parse_header
@@ -36,7 +35,7 @@ class BaseRenderer(object):
 
     media_type = None
     format = None
-    charset = None
+    charset = 'utf-8'
 
     def render(self, data, accepted_media_type=None, renderer_context=None):
         raise NotImplemented('Renderer class requires .render() to be implemented')
@@ -51,6 +50,7 @@ class JSONRenderer(BaseRenderer):
     format = 'json'
     encoder_class = encoders.JSONEncoder
     ensure_ascii = True
+    charset = 'iso-8859-1'
 
     def render(self, data, accepted_media_type=None, renderer_context=None):
         """
@@ -74,7 +74,12 @@ class JSONRenderer(BaseRenderer):
             except (ValueError, TypeError):
                 indent = None
 
-        return json.dumps(data, cls=self.encoder_class, indent=indent, ensure_ascii=self.ensure_ascii)
+        ret = json.dumps(data, cls=self.encoder_class,
+            indent=indent, ensure_ascii=self.ensure_ascii)
+
+        if not self.ensure_ascii:
+            return bytes(ret.encode(self.charset))
+        return ret
 
 
 class UnicodeJSONRenderer(JSONRenderer):
@@ -332,7 +337,7 @@ class BrowsableAPIRenderer(BaseRenderer):
         renderer_context['indent'] = 4
         content = renderer.render(data, accepted_media_type, renderer_context)
 
-        if not all(char in string.printable for char in content):
+        if renderer.charset is None:
             return '[%d bytes of binary content]' % len(content)
 
         return content
diff --git a/rest_framework/response.py b/rest_framework/response.py
index 32e74a45..3e0dd106 100644
--- a/rest_framework/response.py
+++ b/rest_framework/response.py
@@ -55,7 +55,11 @@ class Response(SimpleTemplateResponse):
         else:
             content_type = media_type
         self['Content-Type'] = content_type
-        return renderer.render(self.data, media_type, context)
+
+        ret = renderer.render(self.data, media_type, context)
+        if isinstance(ret, six.text_type):
+            return bytes(ret.encode(self.charset))
+        return ret
 
     @property
     def status_text(self):
diff --git a/rest_framework/tests/renderers.py b/rest_framework/tests/renderers.py
index 739f9184..1b2b9279 100644
--- a/rest_framework/tests/renderers.py
+++ b/rest_framework/tests/renderers.py
@@ -1,4 +1,6 @@
 # -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
 from decimal import Decimal
 from django.core.cache import cache
 from django.test import TestCase
@@ -135,7 +137,7 @@ class RendererEndToEndTests(TestCase):
     def test_default_renderer_serializes_content(self):
         """If the Accept header is not set the default renderer should serialize the response."""
         resp = self.client.get('/')
-        self.assertEqual(resp['Content-Type'], RendererA.media_type)
+        self.assertEqual(resp['Content-Type'], RendererA.media_type + '; charset=utf-8')
         self.assertEqual(resp.content, RENDERER_A_SERIALIZER(DUMMYCONTENT))
         self.assertEqual(resp.status_code, DUMMYSTATUS)
 
@@ -143,13 +145,13 @@ class RendererEndToEndTests(TestCase):
         """No response must be included in HEAD requests."""
         resp = self.client.head('/')
         self.assertEqual(resp.status_code, DUMMYSTATUS)
-        self.assertEqual(resp['Content-Type'], RendererA.media_type)
+        self.assertEqual(resp['Content-Type'], RendererA.media_type + '; charset=utf-8')
         self.assertEqual(resp.content, six.b(''))
 
     def test_default_renderer_serializes_content_on_accept_any(self):
         """If the Accept header is set to */* the default renderer should serialize the response."""
         resp = self.client.get('/', HTTP_ACCEPT='*/*')
-        self.assertEqual(resp['Content-Type'], RendererA.media_type)
+        self.assertEqual(resp['Content-Type'], RendererA.media_type + '; charset=utf-8')
         self.assertEqual(resp.content, RENDERER_A_SERIALIZER(DUMMYCONTENT))
         self.assertEqual(resp.status_code, DUMMYSTATUS)
 
@@ -157,7 +159,7 @@ class RendererEndToEndTests(TestCase):
         """If the Accept header is set the specified renderer should serialize the response.
         (In this case we check that works for the default renderer)"""
         resp = self.client.get('/', HTTP_ACCEPT=RendererA.media_type)
-        self.assertEqual(resp['Content-Type'], RendererA.media_type)
+        self.assertEqual(resp['Content-Type'], RendererA.media_type + '; charset=utf-8')
         self.assertEqual(resp.content, RENDERER_A_SERIALIZER(DUMMYCONTENT))
         self.assertEqual(resp.status_code, DUMMYSTATUS)
 
@@ -165,7 +167,7 @@ class RendererEndToEndTests(TestCase):
         """If the Accept header is set the specified renderer should serialize the response.
         (In this case we check that works for a non-default renderer)"""
         resp = self.client.get('/', HTTP_ACCEPT=RendererB.media_type)
-        self.assertEqual(resp['Content-Type'], RendererB.media_type)
+        self.assertEqual(resp['Content-Type'], RendererB.media_type + '; charset=utf-8')
         self.assertEqual(resp.content, RENDERER_B_SERIALIZER(DUMMYCONTENT))
         self.assertEqual(resp.status_code, DUMMYSTATUS)
 
@@ -176,7 +178,7 @@ class RendererEndToEndTests(TestCase):
             RendererB.media_type
         )
         resp = self.client.get('/' + param)
-        self.assertEqual(resp['Content-Type'], RendererB.media_type)
+        self.assertEqual(resp['Content-Type'], RendererB.media_type + '; charset=utf-8')
         self.assertEqual(resp.content, RENDERER_B_SERIALIZER(DUMMYCONTENT))
         self.assertEqual(resp.status_code, DUMMYSTATUS)
 
@@ -193,7 +195,7 @@ class RendererEndToEndTests(TestCase):
             RendererB.format
         )
         resp = self.client.get('/' + param)
-        self.assertEqual(resp['Content-Type'], RendererB.media_type)
+        self.assertEqual(resp['Content-Type'], RendererB.media_type + '; charset=utf-8')
         self.assertEqual(resp.content, RENDERER_B_SERIALIZER(DUMMYCONTENT))
         self.assertEqual(resp.status_code, DUMMYSTATUS)
 
@@ -201,7 +203,7 @@ class RendererEndToEndTests(TestCase):
         """If a 'format' keyword arg is specified, the renderer with the matching
         format attribute should serialize the response."""
         resp = self.client.get('/something.formatb')
-        self.assertEqual(resp['Content-Type'], RendererB.media_type)
+        self.assertEqual(resp['Content-Type'], RendererB.media_type + '; charset=utf-8')
         self.assertEqual(resp.content, RENDERER_B_SERIALIZER(DUMMYCONTENT))
         self.assertEqual(resp.status_code, DUMMYSTATUS)
 
@@ -214,7 +216,7 @@ class RendererEndToEndTests(TestCase):
         )
         resp = self.client.get('/' + param,
                                HTTP_ACCEPT=RendererB.media_type)
-        self.assertEqual(resp['Content-Type'], RendererB.media_type)
+        self.assertEqual(resp['Content-Type'], RendererB.media_type + '; charset=utf-8')
         self.assertEqual(resp.content, RENDERER_B_SERIALIZER(DUMMYCONTENT))
         self.assertEqual(resp.status_code, DUMMYSTATUS)
 
@@ -270,7 +272,7 @@ class UnicodeJSONRendererTests(TestCase):
         obj = {'countries': ['United Kingdom', 'France', 'España']}
         renderer = UnicodeJSONRenderer()
         content = renderer.render(obj, 'application/json')
-        self.assertEqual(content, '{"countries": ["United Kingdom", "France", "España"]}')
+        self.assertEqual(content, '{"countries": ["United Kingdom", "France", "España"]}'.encode('utf-8'))
 
 
 class JSONPRendererTests(TestCase):
@@ -287,7 +289,7 @@ class JSONPRendererTests(TestCase):
         resp = self.client.get('/jsonp/jsonrenderer',
                                HTTP_ACCEPT='application/javascript')
         self.assertEqual(resp.status_code, status.HTTP_200_OK)
-        self.assertEqual(resp['Content-Type'], 'application/javascript')
+        self.assertEqual(resp['Content-Type'], 'application/javascript; charset=iso-8859-1')
         self.assertEqual(resp.content,
             ('callback(%s);' % _flat_repr).encode('ascii'))
 
@@ -298,7 +300,7 @@ class JSONPRendererTests(TestCase):
         resp = self.client.get('/jsonp/nojsonrenderer',
                                HTTP_ACCEPT='application/javascript')
         self.assertEqual(resp.status_code, status.HTTP_200_OK)
-        self.assertEqual(resp['Content-Type'], 'application/javascript')
+        self.assertEqual(resp['Content-Type'], 'application/javascript; charset=iso-8859-1')
         self.assertEqual(resp.content,
             ('callback(%s);' % _flat_repr).encode('ascii'))
 
@@ -310,7 +312,7 @@ class JSONPRendererTests(TestCase):
         resp = self.client.get('/jsonp/nojsonrenderer?callback=' + callback_func,
                                HTTP_ACCEPT='application/javascript')
         self.assertEqual(resp.status_code, status.HTTP_200_OK)
-        self.assertEqual(resp['Content-Type'], 'application/javascript')
+        self.assertEqual(resp['Content-Type'], 'application/javascript; charset=iso-8859-1')
         self.assertEqual(resp.content,
             ('%s(%s);' % (callback_func, _flat_repr)).encode('ascii'))
 
diff --git a/rest_framework/tests/response.py b/rest_framework/tests/response.py
index 8f1163e8..a527baa3 100644
--- a/rest_framework/tests/response.py
+++ b/rest_framework/tests/response.py
@@ -101,7 +101,7 @@ class RendererIntegrationTests(TestCase):
     def test_default_renderer_serializes_content(self):
         """If the Accept header is not set the default renderer should serialize the response."""
         resp = self.client.get('/')
-        self.assertEqual(resp['Content-Type'], RendererA.media_type)
+        self.assertEqual(resp['Content-Type'], RendererA.media_type + '; charset=utf-8')
         self.assertEqual(resp.content, RENDERER_A_SERIALIZER(DUMMYCONTENT))
         self.assertEqual(resp.status_code, DUMMYSTATUS)
 
@@ -109,13 +109,13 @@ class RendererIntegrationTests(TestCase):
         """No response must be included in HEAD requests."""
         resp = self.client.head('/')
         self.assertEqual(resp.status_code, DUMMYSTATUS)
-        self.assertEqual(resp['Content-Type'], RendererA.media_type)
+        self.assertEqual(resp['Content-Type'], RendererA.media_type + '; charset=utf-8')
         self.assertEqual(resp.content, six.b(''))
 
     def test_default_renderer_serializes_content_on_accept_any(self):
         """If the Accept header is set to */* the default renderer should serialize the response."""
         resp = self.client.get('/', HTTP_ACCEPT='*/*')
-        self.assertEqual(resp['Content-Type'], RendererA.media_type)
+        self.assertEqual(resp['Content-Type'], RendererA.media_type + '; charset=utf-8')
         self.assertEqual(resp.content, RENDERER_A_SERIALIZER(DUMMYCONTENT))
         self.assertEqual(resp.status_code, DUMMYSTATUS)
 
@@ -123,7 +123,7 @@ class RendererIntegrationTests(TestCase):
         """If the Accept header is set the specified renderer should serialize the response.
         (In this case we check that works for the default renderer)"""
         resp = self.client.get('/', HTTP_ACCEPT=RendererA.media_type)
-        self.assertEqual(resp['Content-Type'], RendererA.media_type)
+        self.assertEqual(resp['Content-Type'], RendererA.media_type + '; charset=utf-8')
         self.assertEqual(resp.content, RENDERER_A_SERIALIZER(DUMMYCONTENT))
         self.assertEqual(resp.status_code, DUMMYSTATUS)
 
@@ -131,7 +131,7 @@ class RendererIntegrationTests(TestCase):
         """If the Accept header is set the specified renderer should serialize the response.
         (In this case we check that works for a non-default renderer)"""
         resp = self.client.get('/', HTTP_ACCEPT=RendererB.media_type)
-        self.assertEqual(resp['Content-Type'], RendererB.media_type)
+        self.assertEqual(resp['Content-Type'], RendererB.media_type + '; charset=utf-8')
         self.assertEqual(resp.content, RENDERER_B_SERIALIZER(DUMMYCONTENT))
         self.assertEqual(resp.status_code, DUMMYSTATUS)
 
@@ -142,7 +142,7 @@ class RendererIntegrationTests(TestCase):
             RendererB.media_type
         )
         resp = self.client.get('/' + param)
-        self.assertEqual(resp['Content-Type'], RendererB.media_type)
+        self.assertEqual(resp['Content-Type'], RendererB.media_type + '; charset=utf-8')
         self.assertEqual(resp.content, RENDERER_B_SERIALIZER(DUMMYCONTENT))
         self.assertEqual(resp.status_code, DUMMYSTATUS)
 
@@ -150,7 +150,7 @@ class RendererIntegrationTests(TestCase):
         """If a 'format' query is specified, the renderer with the matching
         format attribute should serialize the response."""
         resp = self.client.get('/?format=%s' % RendererB.format)
-        self.assertEqual(resp['Content-Type'], RendererB.media_type)
+        self.assertEqual(resp['Content-Type'], RendererB.media_type + '; charset=utf-8')
         self.assertEqual(resp.content, RENDERER_B_SERIALIZER(DUMMYCONTENT))
         self.assertEqual(resp.status_code, DUMMYSTATUS)
 
@@ -158,7 +158,7 @@ class RendererIntegrationTests(TestCase):
         """If a 'format' keyword arg is specified, the renderer with the matching
         format attribute should serialize the response."""
         resp = self.client.get('/something.formatb')
-        self.assertEqual(resp['Content-Type'], RendererB.media_type)
+        self.assertEqual(resp['Content-Type'], RendererB.media_type + '; charset=utf-8')
         self.assertEqual(resp.content, RENDERER_B_SERIALIZER(DUMMYCONTENT))
         self.assertEqual(resp.status_code, DUMMYSTATUS)
 
@@ -167,7 +167,7 @@ class RendererIntegrationTests(TestCase):
         the renderer with the matching format attribute should serialize the response."""
         resp = self.client.get('/?format=%s' % RendererB.format,
                                HTTP_ACCEPT=RendererB.media_type)
-        self.assertEqual(resp['Content-Type'], RendererB.media_type)
+        self.assertEqual(resp['Content-Type'], RendererB.media_type + '; charset=utf-8')
         self.assertEqual(resp.content, RENDERER_B_SERIALIZER(DUMMYCONTENT))
         self.assertEqual(resp.status_code, DUMMYSTATUS)
 
@@ -204,7 +204,8 @@ class Issue807Testts(TestCase):
         """
         headers = {"HTTP_ACCEPT": RendererA.media_type}
         resp = self.client.get('/', **headers)
-        self.assertEqual(RendererA.media_type, resp['Content-Type'])
+        expected = "{0}; charset={1}".format(RendererA.media_type, 'utf-8')
+        self.assertEqual(expected, resp['Content-Type'])
 
     def test_if_there_is_charset_specified_on_renderer_it_gets_appended(self):
         """