Merge remote-tracking branch 'upstream/master'

Conflicts:
	rest_framework/tests/models.py

18 files changed, 248 insertions, 35 deletions

diff --git a/rest_framework/authtoken/models.py b/rest_framework/authtoken/models.py
index 8eac2cc4..167fa531 100644
--- a/rest_framework/authtoken/models.py
+++ b/rest_framework/authtoken/models.py
@@ -34,7 +34,7 @@ class Token(models.Model):
         return super(Token, self).save(*args, **kwargs)
 
     def generate_key(self):
-        return binascii.hexlify(os.urandom(20))
+        return binascii.hexlify(os.urandom(20)).decode()
 
     def __unicode__(self):
         return self.key
diff --git a/rest_framework/exceptions.py b/rest_framework/exceptions.py
index 0ac5866e..5f774a9f 100644
--- a/rest_framework/exceptions.py
+++ b/rest_framework/exceptions.py
@@ -20,6 +20,8 @@ class APIException(Exception):
     def __init__(self, detail=None):
         self.detail = detail or self.default_detail
 
+    def __str__(self):
+        return self.detail
 
 class ParseError(APIException):
     status_code = status.HTTP_400_BAD_REQUEST
diff --git a/rest_framework/fields.py b/rest_framework/fields.py
index 68b95682..8cdc5551 100644
--- a/rest_framework/fields.py
+++ b/rest_framework/fields.py
@@ -164,7 +164,7 @@ class Field(object):
         Called to set up a field prior to field_to_native or field_from_native.
 
         parent - The parent serializer.
-        model_field - The model field this field corresponds to, if one exists.
+        field_name - The name of the field being initialized.
         """
         self.parent = parent
         self.root = parent.root or parent
@@ -289,7 +289,7 @@ class WritableField(Field):
         self.validators = self.default_validators + validators
         self.default = default if default is not None else self.default
 
-        # Widgets are ony used for HTML forms.
+        # Widgets are only used for HTML forms.
         widget = widget or self.widget
         if isinstance(widget, type):
             widget = widget()
diff --git a/rest_framework/parsers.py b/rest_framework/parsers.py
index f1b3e38d..4990971b 100644
--- a/rest_framework/parsers.py
+++ b/rest_framework/parsers.py
@@ -10,7 +10,7 @@ from django.core.files.uploadhandler import StopFutureHandlers
 from django.http import QueryDict
 from django.http.multipartparser import MultiPartParser as DjangoMultiPartParser
 from django.http.multipartparser import MultiPartParserError, parse_header, ChunkIter
-from rest_framework.compat import etree, six, yaml
+from rest_framework.compat import etree, six, yaml, force_text
 from rest_framework.exceptions import ParseError
 from rest_framework import renderers
 import json
@@ -288,7 +288,7 @@ class FileUploadParser(BaseParser):
 
         try:
             meta = parser_context['request'].META
-            disposition = parse_header(meta['HTTP_CONTENT_DISPOSITION'])
-            return disposition[1]['filename']
+            disposition = parse_header(meta['HTTP_CONTENT_DISPOSITION'].encode('utf-8'))
+            return force_text(disposition[1]['filename'])
         except (AttributeError, KeyError):
             pass
diff --git a/rest_framework/relations.py b/rest_framework/relations.py
index 308545ce..3463954d 100644
--- a/rest_framework/relations.py
+++ b/rest_framework/relations.py
@@ -59,6 +59,8 @@ class RelatedField(WritableField):
         super(RelatedField, self).__init__(*args, **kwargs)
 
         if not self.required:
+            # Accessed in ModelChoiceIterator django/forms/models.py:1034
+            # If set adds empty choice.
             self.empty_label = BLANK_CHOICE_DASH[0][1]
 
         self.queryset = queryset
diff --git a/rest_framework/renderers.py b/rest_framework/renderers.py
index 7a7da561..484961ad 100644
--- a/rest_framework/renderers.py
+++ b/rest_framework/renderers.py
@@ -193,6 +193,7 @@ class YAMLRenderer(BaseRenderer):
     format = 'yaml'
     encoder = encoders.SafeDumper
     charset = 'utf-8'
+    ensure_ascii = True
 
     def render(self, data, accepted_media_type=None, renderer_context=None):
         """
@@ -203,7 +204,15 @@ class YAMLRenderer(BaseRenderer):
         if data is None:
             return ''
 
-        return yaml.dump(data, stream=None, encoding=self.charset, Dumper=self.encoder)
+        return yaml.dump(data, stream=None, encoding=self.charset, Dumper=self.encoder, allow_unicode=not self.ensure_ascii)
+
+
+class UnicodeYAMLRenderer(YAMLRenderer):
+    """
+    Renderer which serializes to YAML.
+    Does *not* apply character escaping for non-ascii characters.
+    """
+    ensure_ascii = False
 
 
 class TemplateHTMLRenderer(BaseRenderer):
diff --git a/rest_framework/serializers.py b/rest_framework/serializers.py
index d7941df1..2a0d5263 100644
--- a/rest_framework/serializers.py
+++ b/rest_framework/serializers.py
@@ -16,6 +16,7 @@ import datetime
 import inspect
 import types
 from decimal import Decimal
+from django.contrib.contenttypes.generic import GenericForeignKey
 from django.core.paginator import Page
 from django.db import models
 from django.forms import widgets
@@ -438,16 +439,6 @@ class BaseSerializer(WritableField):
                     raise ValidationError(self.error_messages['required'])
                 return
 
-        # Set the serializer object if it exists
-        obj = get_component(self.parent.object, self.source or field_name) if self.parent.object else None
-
-        # If we have a model manager or similar object then we need
-        # to iterate through each instance.
-        if (self.many and
-            not hasattr(obj, '__iter__') and
-            is_simple_callable(getattr(obj, 'all', None))):
-            obj = obj.all()
-
         if self.source == '*':
             if value:
                 reverted_data = self.restore_fields(value, {})
@@ -457,6 +448,16 @@ class BaseSerializer(WritableField):
             if value in (None, ''):
                 into[(self.source or field_name)] = None
             else:
+                # Set the serializer object if it exists
+                obj = get_component(self.parent.object, self.source or field_name) if self.parent.object else None
+
+                # If we have a model manager or similar object then we need
+                # to iterate through each instance.
+                if (self.many and
+                    not hasattr(obj, '__iter__') and
+                    is_simple_callable(getattr(obj, 'all', None))):
+                    obj = obj.all()
+
                 kwargs = {
                     'instance': obj,
                     'data': value,
@@ -827,6 +828,10 @@ class ModelSerializer(Serializer):
 
         if model_field:
             kwargs['required'] = not(model_field.null or model_field.blank)
+            if model_field.help_text is not None:
+                kwargs['help_text'] = model_field.help_text
+            if model_field.verbose_name is not None:
+                kwargs['label'] = model_field.verbose_name
 
             if not model_field.editable:
                 kwargs['read_only'] = True
@@ -952,6 +957,8 @@ class ModelSerializer(Serializer):
 
         # Forward m2m relations
         for field in meta.many_to_many + meta.virtual_fields:
+            if isinstance(field, GenericForeignKey):
+                continue
             if field.name in attrs:
                 m2m_data[field.name] = attrs.pop(field.name)
 
@@ -961,17 +968,15 @@ class ModelSerializer(Serializer):
             if isinstance(self.fields.get(field_name, None), Serializer):
                 nested_forward_relations[field_name] = attrs[field_name]
 
-        # Update an existing instance...
-        if instance is not None:
-            for key, val in attrs.items():
-                try:
-                    setattr(instance, key, val)
-                except ValueError:
-                    self._errors[key] = self.error_messages['required']
+        # Create an empty instance of the model
+        if instance is None:
+            instance = self.opts.model()
 
-        # ...or create a new instance
-        else:
-            instance = self.opts.model(**attrs)
+        for key, val in attrs.items():
+            try:
+                setattr(instance, key, val)
+            except ValueError:
+                self._errors[key] = self.error_messages['required']
 
         # Any relations that cannot be set until we've
         # saved the model get hidden away on these
@@ -1096,6 +1101,10 @@ class HyperlinkedModelSerializer(ModelSerializer):
 
         if model_field:
             kwargs['required'] = not(model_field.null or model_field.blank)
+            if model_field.help_text is not None:
+                kwargs['help_text'] = model_field.help_text
+            if model_field.verbose_name is not None:
+                kwargs['label'] = model_field.verbose_name
 
         if self.opts.lookup_field:
             kwargs['lookup_field'] = self.opts.lookup_field
diff --git a/rest_framework/templatetags/rest_framework.py b/rest_framework/templatetags/rest_framework.py
index beb8c5b0..dff176d6 100644
--- a/rest_framework/templatetags/rest_framework.py
+++ b/rest_framework/templatetags/rest_framework.py
@@ -180,7 +180,7 @@ def add_class(value, css_class):
 
 
 # Bunch of stuff cloned from urlize
-TRAILING_PUNCTUATION = ['.', ',', ':', ';', '.)', '"', "'"]
+TRAILING_PUNCTUATION = ['.', ',', ':', ';', '.)', '"', "']", "'}", "'"]
 WRAPPING_PUNCTUATION = [('(', ')'), ('<', '>'), ('[', ']'), ('&lt;', '&gt;'),
                         ('"', '"'), ("'", "'")]
 word_split_re = re.compile(r'(\s+)')
diff --git a/rest_framework/tests/models.py b/rest_framework/tests/models.py
index 355e070e..e171d3bd 100644
--- a/rest_framework/tests/models.py
+++ b/rest_framework/tests/models.py
@@ -144,7 +144,7 @@ class ForeignKeyTarget(RESTFrameworkModel):
 class ForeignKeySource(RESTFrameworkModel):
     name = models.CharField(max_length=100)
     target = models.ForeignKey(ForeignKeyTarget, related_name='sources',
-                               verbose_name='Target object')
+                               help_text='Target', verbose_name='Target')
 
 
 # Nullable ForeignKey
diff --git a/rest_framework/tests/test_authentication.py b/rest_framework/tests/test_authentication.py
index c37d2a51..a1c43d9c 100644
--- a/rest_framework/tests/test_authentication.py
+++ b/rest_framework/tests/test_authentication.py
@@ -19,7 +19,7 @@ from rest_framework.authentication import (
     OAuth2Authentication
 )
 from rest_framework.authtoken.models import Token
-from rest_framework.compat import patterns, url, include
+from rest_framework.compat import patterns, url, include, six
 from rest_framework.compat import oauth2_provider, oauth2_provider_scope
 from rest_framework.compat import oauth, oauth_provider
 from rest_framework.test import APIRequestFactory, APIClient
@@ -195,6 +195,12 @@ class TokenAuthTests(TestCase):
         token = Token.objects.create(user=self.user)
         self.assertTrue(bool(token.key))
 
+    def test_generate_key_returns_string(self):
+        """Ensure generate_key returns a string"""
+        token = Token()
+        key = token.generate_key()
+        self.assertTrue(isinstance(key, six.string_types))
+
     def test_token_login_json(self):
         """Ensure token login view using JSON POST works."""
         client = APIClient(enforce_csrf_checks=True)
diff --git a/rest_framework/tests/test_genericrelations.py b/rest_framework/tests/test_genericrelations.py
index fa09c9e6..46a2d863 100644
--- a/rest_framework/tests/test_genericrelations.py
+++ b/rest_framework/tests/test_genericrelations.py
@@ -131,3 +131,21 @@ class TestGenericRelations(TestCase):
         }
         ]
         self.assertEqual(serializer.data, expected)
+
+    def test_restore_object_generic_fk(self):
+        """
+        Ensure an object with a generic foreign key can be restored.
+        """
+
+        class TagSerializer(serializers.ModelSerializer):
+            class Meta:
+                model = Tag
+                exclude = ('content_type', 'object_id')
+
+        serializer = TagSerializer()
+
+        bookmark = Bookmark(url='http://example.com')
+        attrs = {'tagged_item': bookmark, 'tag': 'example'}
+
+        tag = serializer.restore_object(attrs)
+        self.assertEqual(tag.tagged_item, bookmark)
diff --git a/rest_framework/tests/test_parsers.py b/rest_framework/tests/test_parsers.py
index 7699e10c..8af90677 100644
--- a/rest_framework/tests/test_parsers.py
+++ b/rest_framework/tests/test_parsers.py
@@ -96,7 +96,7 @@ class TestFileUploadParser(TestCase):
         request = MockRequest()
         request.upload_handlers = (MemoryFileUploadHandler(),)
         request.META = {
-            'HTTP_CONTENT_DISPOSITION': 'Content-Disposition: inline; filename=file.txt'.encode('utf-8'),
+            'HTTP_CONTENT_DISPOSITION': 'Content-Disposition: inline; filename=file.txt',
             'HTTP_CONTENT_LENGTH': 14,
         }
         self.parser_context = {'request': request, 'kwargs': {}}
@@ -112,4 +112,4 @@ class TestFileUploadParser(TestCase):
     def test_get_filename(self):
         parser = FileUploadParser()
         filename = parser.get_filename(self.stream, None, self.parser_context)
-        self.assertEqual(filename, 'file.txt'.encode('utf-8'))
+        self.assertEqual(filename, 'file.txt')
diff --git a/rest_framework/tests/test_relations.py b/rest_framework/tests/test_relations.py
index f52e0e1e..37ac826b 100644
--- a/rest_framework/tests/test_relations.py
+++ b/rest_framework/tests/test_relations.py
@@ -2,8 +2,10 @@
 General tests for relational fields.
 """
 from __future__ import unicode_literals
+from django import get_version
 from django.db import models
 from django.test import TestCase
+from django.utils import unittest
 from rest_framework import serializers
 from rest_framework.tests.models import BlogPost
 
@@ -118,3 +120,25 @@ class RelatedFieldSourceTests(TestCase):
             (serializers.ModelSerializer,), attrs)
         with self.assertRaises(AttributeError):
             TestSerializer(data={'name': 'foo'})
+
+@unittest.skipIf(get_version() < '1.6.0', 'Upstream behaviour changed in v1.6')
+class RelatedFieldChoicesTests(TestCase):
+    """
+    Tests for #1408 "Web browseable API doesn't have blank option on drop down list box"
+    https://github.com/tomchristie/django-rest-framework/issues/1408
+    """
+    def test_blank_option_is_added_to_choice_if_required_equals_false(self):
+        """
+
+        """
+        post = BlogPost(title="Checking blank option is added")
+        post.save()
+
+        queryset = BlogPost.objects.all()
+        field = serializers.RelatedField(required=False, queryset=queryset)
+
+        choice_count = BlogPost.objects.count()
+        widget_count = len(field.widget.choices)
+
+        self.assertEqual(widget_count, choice_count + 1, 'BLANK_CHOICE_DASH option should have been added')
+
diff --git a/rest_framework/tests/test_renderers.py b/rest_framework/tests/test_renderers.py
index c7bf772e..7cb7d0f9 100644
--- a/rest_framework/tests/test_renderers.py
+++ b/rest_framework/tests/test_renderers.py
@@ -12,7 +12,7 @@ from rest_framework.compat import yaml, etree, patterns, url, include, six, Stri
 from rest_framework.response import Response
 from rest_framework.views import APIView
 from rest_framework.renderers import BaseRenderer, JSONRenderer, YAMLRenderer, \
-    XMLRenderer, JSONPRenderer, BrowsableAPIRenderer, UnicodeJSONRenderer
+    XMLRenderer, JSONPRenderer, BrowsableAPIRenderer, UnicodeJSONRenderer, UnicodeYAMLRenderer
 from rest_framework.parsers import YAMLParser, XMLParser
 from rest_framework.settings import api_settings
 from rest_framework.test import APIRequestFactory
@@ -467,6 +467,17 @@ if yaml:
             self.assertTrue(string in content, '%r not in %r' % (string, content))
 
 
+    class UnicodeYAMLRendererTests(TestCase):
+        """
+        Tests specific for the Unicode YAML Renderer
+        """
+        def test_proper_encoding(self):
+            obj = {'countries': ['United Kingdom', 'France', 'España']}
+            renderer = UnicodeYAMLRenderer()
+            content = renderer.render(obj, 'application/yaml')
+            self.assertEqual(content.strip(), 'countries: [United Kingdom, France, España]'.encode('utf-8'))
+
+
 class XMLRendererTestCase(TestCase):
     """
     Tests specific to the XML Renderer
diff --git a/rest_framework/tests/test_serializer.py b/rest_framework/tests/test_serializer.py
index 85a899c5..e688c823 100644
--- a/rest_framework/tests/test_serializer.py
+++ b/rest_framework/tests/test_serializer.py
@@ -9,7 +9,8 @@ from django.utils.translation import ugettext_lazy as _
 from rest_framework import serializers, fields, relations
 from rest_framework.tests.models import (HasPositiveIntegerAsChoice, Album, ActionItem, Anchor, BasicModel,
     BlankFieldModel, BlogPost, BlogPostComment, Book, CallableDefaultValueModel, DefaultValueModel,
-    ManyToManyModel, Person, ReadOnlyManyToManyModel, Photo, RESTFrameworkModel)
+    ManyToManyModel, Person, ReadOnlyManyToManyModel, Photo, RESTFrameworkModel,
+    ForeignKeySource, ManyToManySource)
 from rest_framework.tests.models import BasicModelSerializer
 import datetime
 import pickle
@@ -176,6 +177,16 @@ class PositiveIntegerAsChoiceSerializer(serializers.ModelSerializer):
         fields = ['some_integer']
 
 
+class ForeignKeySourceSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = ForeignKeySource
+
+
+class HyperlinkedForeignKeySourceSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = ForeignKeySource
+
+
 class BasicTests(TestCase):
     def setUp(self):
         self.comment = Comment(
@@ -508,6 +519,32 @@ class ValidationTests(TestCase):
         )
         self.assertEqual(serializer.is_valid(), True)
 
+    def test_writable_star_source_on_nested_serializer_with_parent_object(self):
+        class TitleSerializer(serializers.Serializer):
+            title = serializers.WritableField(source='title')
+
+        class AlbumSerializer(serializers.ModelSerializer):
+            nested = TitleSerializer(source='*')
+
+            class Meta:
+                model = Album
+                fields = ('nested',)
+
+        class PhotoSerializer(serializers.ModelSerializer):
+            album = AlbumSerializer(source='album')
+
+            class Meta:
+                model = Photo
+                fields = ('album', )
+
+        photo = Photo(album=Album())
+
+        data = {'album': {'nested': {'title': 'test'}}}
+
+        serializer = PhotoSerializer(photo, data=data)
+        self.assertEqual(serializer.is_valid(), True)
+        self.assertEqual(serializer.data, data)
+
     def test_writable_star_source_with_inner_source_fields(self):
         """
         Tests that a serializer with source="*" correctly expands the
@@ -1574,6 +1611,19 @@ class ManyFieldHelpTextTest(TestCase):
         self.assertEqual('Some help text.', rel_field.help_text)
 
 
+class AttributeMappingOnAutogeneratedRelatedFields(TestCase):
+
+    def test_primary_key_related_field(self):
+        serializer = ForeignKeySourceSerializer()
+        self.assertEqual(serializer.fields['target'].help_text, 'Target')
+        self.assertEqual(serializer.fields['target'].label, 'Target')
+
+    def test_hyperlinked_related_field(self):
+        serializer = HyperlinkedForeignKeySourceSerializer()
+        self.assertEqual(serializer.fields['target'].help_text, 'Target')
+        self.assertEqual(serializer.fields['target'].label, 'Target')
+
+
 @unittest.skipUnless(PIL is not None, 'PIL is not installed')
 class AttributeMappingOnAutogeneratedFieldsTests(TestCase):
 
diff --git a/rest_framework/tests/test_urlizer.py b/rest_framework/tests/test_urlizer.py
new file mode 100644
index 00000000..3dc8e8fe
--- /dev/null
+++ b/rest_framework/tests/test_urlizer.py
@@ -0,0 +1,38 @@
+from __future__ import unicode_literals
+from django.test import TestCase
+from rest_framework.templatetags.rest_framework import urlize_quoted_links
+import sys
+
+
+class URLizerTests(TestCase):
+    """
+    Test if both JSON and YAML URLs are transformed into links well
+    """
+    def _urlize_dict_check(self, data):
+        """
+        For all items in dict test assert that the value is urlized key
+        """
+        for original, urlized in data.items():
+            assert urlize_quoted_links(original, nofollow=False) == urlized
+
+    def test_json_with_url(self):
+        """
+        Test if JSON URLs are transformed into links well
+        """
+        data = {}
+        data['"url": "http://api/users/1/", '] = \
+            '&quot;url&quot;: &quot;<a href="http://api/users/1/">http://api/users/1/</a>&quot;, '
+        data['"foo_set": [\n    "http://api/foos/1/"\n], '] = \
+            '&quot;foo_set&quot;: [\n    &quot;<a href="http://api/foos/1/">http://api/foos/1/</a>&quot;\n], '
+        self._urlize_dict_check(data)
+
+    def test_yaml_with_url(self):
+        """
+        Test if YAML URLs are transformed into links well
+        """
+        data = {}
+        data['''{users: 'http://api/users/'}'''] = \
+            '''{users: &#39;<a href="http://api/users/">http://api/users/</a>&#39;}'''
+        data['''foo_set: ['http://api/foos/1/']'''] = \
+            '''foo_set: [&#39;<a href="http://api/foos/1/">http://api/foos/1/</a>&#39;]'''
+        self._urlize_dict_check(data)
diff --git a/rest_framework/tests/test_validation.py b/rest_framework/tests/test_validation.py
index 124c874d..e13e4078 100644
--- a/rest_framework/tests/test_validation.py
+++ b/rest_framework/tests/test_validation.py
@@ -1,4 +1,5 @@
 from __future__ import unicode_literals
+from django.core.validators import MaxValueValidator
 from django.db import models
 from django.test import TestCase
 from rest_framework import generics, serializers, status
@@ -102,3 +103,46 @@ class TestAvoidValidation(TestCase):
         self.assertFalse(serializer.is_valid())
         self.assertDictEqual(serializer.errors,
                              {'non_field_errors': ['Invalid data']})
+
+
+# regression tests for issue: 1493
+
+class ValidationMaxValueValidatorModel(models.Model):
+    number_value = models.PositiveIntegerField(validators=[MaxValueValidator(100)])
+
+
+class ValidationMaxValueValidatorModelSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = ValidationMaxValueValidatorModel
+
+
+class UpdateMaxValueValidationModel(generics.RetrieveUpdateDestroyAPIView):
+    model = ValidationMaxValueValidatorModel
+    serializer_class = ValidationMaxValueValidatorModelSerializer
+
+
+class TestMaxValueValidatorValidation(TestCase):
+
+    def test_max_value_validation_serializer_success(self):
+        serializer = ValidationMaxValueValidatorModelSerializer(data={'number_value': 99})
+        self.assertTrue(serializer.is_valid())
+
+    def test_max_value_validation_serializer_fails(self):
+        serializer = ValidationMaxValueValidatorModelSerializer(data={'number_value': 101})
+        self.assertFalse(serializer.is_valid())
+        self.assertDictEqual({'number_value': ['Ensure this value is less than or equal to 100.']}, serializer.errors)
+
+    def test_max_value_validation_success(self):
+        obj = ValidationMaxValueValidatorModel.objects.create(number_value=100)
+        request = factory.patch('/{0}'.format(obj.pk), {'number_value': 98}, format='json')
+        view = UpdateMaxValueValidationModel().as_view()
+        response = view(request, pk=obj.pk).render()
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+
+    def test_max_value_validation_fail(self):
+        obj = ValidationMaxValueValidatorModel.objects.create(number_value=100)
+        request = factory.patch('/{0}'.format(obj.pk), {'number_value': 101}, format='json')
+        view = UpdateMaxValueValidationModel().as_view()
+        response = view(request, pk=obj.pk).render()
+        self.assertEqual(response.content, b'{"number_value": ["Ensure this value is less than or equal to 100."]}')
+        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
diff --git a/rest_framework/utils/mediatypes.py b/rest_framework/utils/mediatypes.py
index c09c2933..92f99efd 100644
--- a/rest_framework/utils/mediatypes.py
+++ b/rest_framework/utils/mediatypes.py
@@ -74,7 +74,7 @@ class _MediaType(object):
             return 0
         elif self.sub_type == '*':
             return 1
-        elif not self.params or self.params.keys() == ['q']:
+        elif not self.params or list(self.params.keys()) == ['q']:
             return 2
         return 3