Merge remote-tracking branch 'origin/master' into 2.4.0

Conflicts:
	.travis.yml
	rest_framework/serializers.py
	rest_framework/tests/test_authentication.py

13 files changed, 141 insertions, 23 deletions

diff --git a/rest_framework/authtoken/models.py b/rest_framework/authtoken/models.py
index 8eac2cc4..167fa531 100644
--- a/rest_framework/authtoken/models.py
+++ b/rest_framework/authtoken/models.py
@@ -34,7 +34,7 @@ class Token(models.Model):
         return super(Token, self).save(*args, **kwargs)
 
     def generate_key(self):
-        return binascii.hexlify(os.urandom(20))
+        return binascii.hexlify(os.urandom(20)).decode()
 
     def __unicode__(self):
         return self.key
diff --git a/rest_framework/fields.py b/rest_framework/fields.py
index 533de28c..8eaf763d 100644
--- a/rest_framework/fields.py
+++ b/rest_framework/fields.py
@@ -284,7 +284,7 @@ class WritableField(Field):
         self.validators = self.default_validators + validators
         self.default = default if default is not None else self.default
 
-        # Widgets are ony used for HTML forms.
+        # Widgets are only used for HTML forms.
         widget = widget or self.widget
         if isinstance(widget, type):
             widget = widget()
diff --git a/rest_framework/parsers.py b/rest_framework/parsers.py
index f1b3e38d..4990971b 100644
--- a/rest_framework/parsers.py
+++ b/rest_framework/parsers.py
@@ -10,7 +10,7 @@ from django.core.files.uploadhandler import StopFutureHandlers
 from django.http import QueryDict
 from django.http.multipartparser import MultiPartParser as DjangoMultiPartParser
 from django.http.multipartparser import MultiPartParserError, parse_header, ChunkIter
-from rest_framework.compat import etree, six, yaml
+from rest_framework.compat import etree, six, yaml, force_text
 from rest_framework.exceptions import ParseError
 from rest_framework import renderers
 import json
@@ -288,7 +288,7 @@ class FileUploadParser(BaseParser):
 
         try:
             meta = parser_context['request'].META
-            disposition = parse_header(meta['HTTP_CONTENT_DISPOSITION'])
-            return disposition[1]['filename']
+            disposition = parse_header(meta['HTTP_CONTENT_DISPOSITION'].encode('utf-8'))
+            return force_text(disposition[1]['filename'])
         except (AttributeError, KeyError):
             pass
diff --git a/rest_framework/renderers.py b/rest_framework/renderers.py
index 7a7da561..484961ad 100644
--- a/rest_framework/renderers.py
+++ b/rest_framework/renderers.py
@@ -193,6 +193,7 @@ class YAMLRenderer(BaseRenderer):
     format = 'yaml'
     encoder = encoders.SafeDumper
     charset = 'utf-8'
+    ensure_ascii = True
 
     def render(self, data, accepted_media_type=None, renderer_context=None):
         """
@@ -203,7 +204,15 @@ class YAMLRenderer(BaseRenderer):
         if data is None:
             return ''
 
-        return yaml.dump(data, stream=None, encoding=self.charset, Dumper=self.encoder)
+        return yaml.dump(data, stream=None, encoding=self.charset, Dumper=self.encoder, allow_unicode=not self.ensure_ascii)
+
+
+class UnicodeYAMLRenderer(YAMLRenderer):
+    """
+    Renderer which serializes to YAML.
+    Does *not* apply character escaping for non-ascii characters.
+    """
+    ensure_ascii = False
 
 
 class TemplateHTMLRenderer(BaseRenderer):
diff --git a/rest_framework/serializers.py b/rest_framework/serializers.py
index 5b14e403..9709d56d 100644
--- a/rest_framework/serializers.py
+++ b/rest_framework/serializers.py
@@ -16,10 +16,12 @@ import datetime
 import inspect
 import types
 from decimal import Decimal
+from django.contrib.contenttypes.generic import GenericForeignKey
+from django.core.paginator import Page
 from django.db import models
 from django.forms import widgets
 from django.utils.datastructures import SortedDict
-from rest_framework.compat import get_concrete_model, six
+from rest_framework.compat import six
 from rest_framework.settings import api_settings
 
 
@@ -821,6 +823,10 @@ class ModelSerializer(Serializer):
 
         if model_field:
             kwargs['required'] = not(model_field.null or model_field.blank)
+            if model_field.help_text is not None:
+                kwargs['help_text'] = model_field.help_text
+            if model_field.verbose_name is not None:
+                kwargs['label'] = model_field.verbose_name
 
         return PrimaryKeyRelatedField(**kwargs)
 
@@ -941,6 +947,8 @@ class ModelSerializer(Serializer):
 
         # Forward m2m relations
         for field in meta.many_to_many + meta.virtual_fields:
+            if isinstance(field, GenericForeignKey):
+                continue
             if field.name in attrs:
                 m2m_data[field.name] = attrs.pop(field.name)
 
@@ -950,17 +958,15 @@ class ModelSerializer(Serializer):
             if isinstance(self.fields.get(field_name, None), Serializer):
                 nested_forward_relations[field_name] = attrs[field_name]
 
-        # Update an existing instance...
-        if instance is not None:
-            for key, val in attrs.items():
-                try:
-                    setattr(instance, key, val)
-                except ValueError:
-                    self._errors[key] = self.error_messages['required']
+        # Create an empty instance of the model
+        if instance is None:
+            instance = self.opts.model()
 
-        # ...or create a new instance
-        else:
-            instance = self.opts.model(**attrs)
+        for key, val in attrs.items():
+            try:
+                setattr(instance, key, val)
+            except ValueError:
+                self._errors[key] = self.error_messages['required']
 
         # Any relations that cannot be set until we've
         # saved the model get hidden away on these
@@ -1085,6 +1091,10 @@ class HyperlinkedModelSerializer(ModelSerializer):
 
         if model_field:
             kwargs['required'] = not(model_field.null or model_field.blank)
+            if model_field.help_text is not None:
+                kwargs['help_text'] = model_field.help_text
+            if model_field.verbose_name is not None:
+                kwargs['label'] = model_field.verbose_name
 
         if self.opts.lookup_field:
             kwargs['lookup_field'] = self.opts.lookup_field
diff --git a/rest_framework/templatetags/rest_framework.py b/rest_framework/templatetags/rest_framework.py
index a0f9c841..886db63c 100644
--- a/rest_framework/templatetags/rest_framework.py
+++ b/rest_framework/templatetags/rest_framework.py
@@ -98,7 +98,7 @@ def add_class(value, css_class):
 
 
 # Bunch of stuff cloned from urlize
-TRAILING_PUNCTUATION = ['.', ',', ':', ';', '.)', '"', "'"]
+TRAILING_PUNCTUATION = ['.', ',', ':', ';', '.)', '"', "']", "'}", "'"]
 WRAPPING_PUNCTUATION = [('(', ')'), ('<', '>'), ('[', ']'), ('&lt;', '&gt;'),
                         ('"', '"'), ("'", "'")]
 word_split_re = re.compile(r'(\s+)')
diff --git a/rest_framework/tests/models.py b/rest_framework/tests/models.py
index 6c8f2342..0256697a 100644
--- a/rest_framework/tests/models.py
+++ b/rest_framework/tests/models.py
@@ -143,7 +143,8 @@ class ForeignKeyTarget(RESTFrameworkModel):
 
 class ForeignKeySource(RESTFrameworkModel):
     name = models.CharField(max_length=100)
-    target = models.ForeignKey(ForeignKeyTarget, related_name='sources')
+    target = models.ForeignKey(ForeignKeyTarget, related_name='sources',
+                               help_text='Target', verbose_name='Target')
 
 
 # Nullable ForeignKey
diff --git a/rest_framework/tests/test_authentication.py b/rest_framework/tests/test_authentication.py
index 6c14debb..09203057 100644
--- a/rest_framework/tests/test_authentication.py
+++ b/rest_framework/tests/test_authentication.py
@@ -20,6 +20,7 @@ from rest_framework.authentication import (
     OAuth2Authentication
 )
 from rest_framework.authtoken.models import Token
+from rest_framework.compat import patterns, url, include, six
 from rest_framework.compat import oauth2_provider, oauth2_provider_scope
 from rest_framework.compat import oauth, oauth_provider
 from rest_framework.test import APIRequestFactory, APIClient
@@ -195,6 +196,12 @@ class TokenAuthTests(TestCase):
         token = Token.objects.create(user=self.user)
         self.assertTrue(bool(token.key))
 
+    def test_generate_key_returns_string(self):
+        """Ensure generate_key returns a string"""
+        token = Token()
+        key = token.generate_key()
+        self.assertTrue(isinstance(key, six.string_types))
+
     def test_token_login_json(self):
         """Ensure token login view using JSON POST works."""
         client = APIClient(enforce_csrf_checks=True)
diff --git a/rest_framework/tests/test_genericrelations.py b/rest_framework/tests/test_genericrelations.py
index fa09c9e6..46a2d863 100644
--- a/rest_framework/tests/test_genericrelations.py
+++ b/rest_framework/tests/test_genericrelations.py
@@ -131,3 +131,21 @@ class TestGenericRelations(TestCase):
         }
         ]
         self.assertEqual(serializer.data, expected)
+
+    def test_restore_object_generic_fk(self):
+        """
+        Ensure an object with a generic foreign key can be restored.
+        """
+
+        class TagSerializer(serializers.ModelSerializer):
+            class Meta:
+                model = Tag
+                exclude = ('content_type', 'object_id')
+
+        serializer = TagSerializer()
+
+        bookmark = Bookmark(url='http://example.com')
+        attrs = {'tagged_item': bookmark, 'tag': 'example'}
+
+        tag = serializer.restore_object(attrs)
+        self.assertEqual(tag.tagged_item, bookmark)
diff --git a/rest_framework/tests/test_parsers.py b/rest_framework/tests/test_parsers.py
index 7699e10c..8af90677 100644
--- a/rest_framework/tests/test_parsers.py
+++ b/rest_framework/tests/test_parsers.py
@@ -96,7 +96,7 @@ class TestFileUploadParser(TestCase):
         request = MockRequest()
         request.upload_handlers = (MemoryFileUploadHandler(),)
         request.META = {
-            'HTTP_CONTENT_DISPOSITION': 'Content-Disposition: inline; filename=file.txt'.encode('utf-8'),
+            'HTTP_CONTENT_DISPOSITION': 'Content-Disposition: inline; filename=file.txt',
             'HTTP_CONTENT_LENGTH': 14,
         }
         self.parser_context = {'request': request, 'kwargs': {}}
@@ -112,4 +112,4 @@ class TestFileUploadParser(TestCase):
     def test_get_filename(self):
         parser = FileUploadParser()
         filename = parser.get_filename(self.stream, None, self.parser_context)
-        self.assertEqual(filename, 'file.txt'.encode('utf-8'))
+        self.assertEqual(filename, 'file.txt')
diff --git a/rest_framework/tests/test_renderers.py b/rest_framework/tests/test_renderers.py
index 460c02a9..295bf317 100644
--- a/rest_framework/tests/test_renderers.py
+++ b/rest_framework/tests/test_renderers.py
@@ -13,7 +13,7 @@ from rest_framework.compat import yaml, etree, six, StringIO
 from rest_framework.response import Response
 from rest_framework.views import APIView
 from rest_framework.renderers import BaseRenderer, JSONRenderer, YAMLRenderer, \
-    XMLRenderer, JSONPRenderer, BrowsableAPIRenderer, UnicodeJSONRenderer
+    XMLRenderer, JSONPRenderer, BrowsableAPIRenderer, UnicodeJSONRenderer, UnicodeYAMLRenderer
 from rest_framework.parsers import YAMLParser, XMLParser
 from rest_framework.settings import api_settings
 from rest_framework.test import APIRequestFactory
@@ -468,6 +468,17 @@ if yaml:
             self.assertTrue(string in content, '%r not in %r' % (string, content))
 
 
+    class UnicodeYAMLRendererTests(TestCase):
+        """
+        Tests specific for the Unicode YAML Renderer
+        """
+        def test_proper_encoding(self):
+            obj = {'countries': ['United Kingdom', 'France', 'España']}
+            renderer = UnicodeYAMLRenderer()
+            content = renderer.render(obj, 'application/yaml')
+            self.assertEqual(content.strip(), 'countries: [United Kingdom, France, España]'.encode('utf-8'))
+
+
 class XMLRendererTestCase(TestCase):
     """
     Tests specific to the XML Renderer
diff --git a/rest_framework/tests/test_serializer.py b/rest_framework/tests/test_serializer.py
index a09bf6f5..f0bb112d 100644
--- a/rest_framework/tests/test_serializer.py
+++ b/rest_framework/tests/test_serializer.py
@@ -9,7 +9,8 @@ from django.utils.translation import ugettext_lazy as _
 from rest_framework import serializers, fields, relations
 from rest_framework.tests.models import (HasPositiveIntegerAsChoice, Album, ActionItem, Anchor, BasicModel,
     BlankFieldModel, BlogPost, BlogPostComment, Book, CallableDefaultValueModel, DefaultValueModel,
-    ManyToManyModel, Person, ReadOnlyManyToManyModel, Photo, RESTFrameworkModel)
+    ManyToManyModel, Person, ReadOnlyManyToManyModel, Photo, RESTFrameworkModel,
+    ForeignKeySource, ManyToManySource)
 from rest_framework.tests.models import BasicModelSerializer
 import datetime
 import pickle
@@ -176,6 +177,16 @@ class PositiveIntegerAsChoiceSerializer(serializers.ModelSerializer):
         fields = ['some_integer']
 
 
+class ForeignKeySourceSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = ForeignKeySource
+
+
+class HyperlinkedForeignKeySourceSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = ForeignKeySource
+
+
 class BasicTests(TestCase):
     def setUp(self):
         self.comment = Comment(
@@ -1614,6 +1625,19 @@ class ManyFieldHelpTextTest(TestCase):
         self.assertEqual('Some help text.', rel_field.help_text)
 
 
+class AttributeMappingOnAutogeneratedRelatedFields(TestCase):
+
+    def test_primary_key_related_field(self):
+        serializer = ForeignKeySourceSerializer()
+        self.assertEqual(serializer.fields['target'].help_text, 'Target')
+        self.assertEqual(serializer.fields['target'].label, 'Target')
+
+    def test_hyperlinked_related_field(self):
+        serializer = HyperlinkedForeignKeySourceSerializer()
+        self.assertEqual(serializer.fields['target'].help_text, 'Target')
+        self.assertEqual(serializer.fields['target'].label, 'Target')
+
+
 @unittest.skipUnless(PIL is not None, 'PIL is not installed')
 class AttributeMappingOnAutogeneratedFieldsTests(TestCase):
 
diff --git a/rest_framework/tests/test_urlizer.py b/rest_framework/tests/test_urlizer.py
new file mode 100644
index 00000000..3dc8e8fe
--- /dev/null
+++ b/rest_framework/tests/test_urlizer.py
@@ -0,0 +1,38 @@
+from __future__ import unicode_literals
+from django.test import TestCase
+from rest_framework.templatetags.rest_framework import urlize_quoted_links
+import sys
+
+
+class URLizerTests(TestCase):
+    """
+    Test if both JSON and YAML URLs are transformed into links well
+    """
+    def _urlize_dict_check(self, data):
+        """
+        For all items in dict test assert that the value is urlized key
+        """
+        for original, urlized in data.items():
+            assert urlize_quoted_links(original, nofollow=False) == urlized
+
+    def test_json_with_url(self):
+        """
+        Test if JSON URLs are transformed into links well
+        """
+        data = {}
+        data['"url": "http://api/users/1/", '] = \
+            '&quot;url&quot;: &quot;<a href="http://api/users/1/">http://api/users/1/</a>&quot;, '
+        data['"foo_set": [\n    "http://api/foos/1/"\n], '] = \
+            '&quot;foo_set&quot;: [\n    &quot;<a href="http://api/foos/1/">http://api/foos/1/</a>&quot;\n], '
+        self._urlize_dict_check(data)
+
+    def test_yaml_with_url(self):
+        """
+        Test if YAML URLs are transformed into links well
+        """
+        data = {}
+        data['''{users: 'http://api/users/'}'''] = \
+            '''{users: &#39;<a href="http://api/users/">http://api/users/</a>&#39;}'''
+        data['''foo_set: ['http://api/foos/1/']'''] = \
+            '''foo_set: [&#39;<a href="http://api/foos/1/">http://api/foos/1/</a>&#39;]'''
+        self._urlize_dict_check(data)