Merge branch 'version-3.1' of github.com:tomchristie/django-rest-framework into oauth_as_package

Conflicts:
	.travis.yml

7 files changed, 612 insertions, 43 deletions

diff --git a/rest_framework/utils/encoders.py b/rest_framework/utils/encoders.py
index 00ffdfba..174b08b8 100644
--- a/rest_framework/utils/encoders.py
+++ b/rest_framework/utils/encoders.py
@@ -7,7 +7,6 @@ from django.db.models.query import QuerySet
 from django.utils.datastructures import SortedDict
 from django.utils.functional import Promise
 from rest_framework.compat import force_text
-from rest_framework.serializers import DictWithMetadata, SortedDictWithMetadata
 import datetime
 import decimal
 import types
@@ -17,45 +16,47 @@ import json
 class JSONEncoder(json.JSONEncoder):
     """
     JSONEncoder subclass that knows how to encode date/time/timedelta,
-    decimal types, and generators.
+    decimal types, generators and other basic python objects.
     """
-    def default(self, o):
+    def default(self, obj):
         # For Date Time string spec, see ECMA 262
         # http://ecma-international.org/ecma-262/5.1/#sec-15.9.1.15
-        if isinstance(o, Promise):
-            return force_text(o)
-        elif isinstance(o, datetime.datetime):
-            r = o.isoformat()
-            if o.microsecond:
-                r = r[:23] + r[26:]
-            if r.endswith('+00:00'):
-                r = r[:-6] + 'Z'
-            return r
-        elif isinstance(o, datetime.date):
-            return o.isoformat()
-        elif isinstance(o, datetime.time):
-            if timezone and timezone.is_aware(o):
+        if isinstance(obj, Promise):
+            return force_text(obj)
+        elif isinstance(obj, datetime.datetime):
+            representation = obj.isoformat()
+            if obj.microsecond:
+                representation = representation[:23] + representation[26:]
+            if representation.endswith('+00:00'):
+                representation = representation[:-6] + 'Z'
+            return representation
+        elif isinstance(obj, datetime.date):
+            return obj.isoformat()
+        elif isinstance(obj, datetime.time):
+            if timezone and timezone.is_aware(obj):
                 raise ValueError("JSON can't represent timezone-aware times.")
-            r = o.isoformat()
-            if o.microsecond:
-                r = r[:12]
-            return r
-        elif isinstance(o, datetime.timedelta):
-            return str(o.total_seconds())
-        elif isinstance(o, decimal.Decimal):
-            return str(o)
-        elif isinstance(o, QuerySet):
-            return list(o)
-        elif hasattr(o, 'tolist'):
-            return o.tolist()
-        elif hasattr(o, '__getitem__'):
+            representation = obj.isoformat()
+            if obj.microsecond:
+                representation = representation[:12]
+            return representation
+        elif isinstance(obj, datetime.timedelta):
+            return str(obj.total_seconds())
+        elif isinstance(obj, decimal.Decimal):
+            # Serializers will coerce decimals to strings by default.
+            return float(obj)
+        elif isinstance(obj, QuerySet):
+            return list(obj)
+        elif hasattr(obj, 'tolist'):
+            # Numpy arrays and array scalars.
+            return obj.tolist()
+        elif hasattr(obj, '__getitem__'):
             try:
-                return dict(o)
+                return dict(obj)
             except:
                 pass
-        elif hasattr(o, '__iter__'):
-            return [i for i in o]
-        return super(JSONEncoder, self).default(o)
+        elif hasattr(obj, '__iter__'):
+            return [item for item in obj]
+        return super(JSONEncoder, self).default(obj)
 
 
 try:
@@ -106,14 +107,14 @@ else:
         SortedDict,
         yaml.representer.SafeRepresenter.represent_dict
     )
-    SafeDumper.add_representer(
-        DictWithMetadata,
-        yaml.representer.SafeRepresenter.represent_dict
-    )
-    SafeDumper.add_representer(
-        SortedDictWithMetadata,
-        yaml.representer.SafeRepresenter.represent_dict
-    )
+    # SafeDumper.add_representer(
+    #     DictWithMetadata,
+    #     yaml.representer.SafeRepresenter.represent_dict
+    # )
+    # SafeDumper.add_representer(
+    #     SortedDictWithMetadata,
+    #     yaml.representer.SafeRepresenter.represent_dict
+    # )
     SafeDumper.add_representer(
         types.GeneratorType,
         yaml.representer.SafeRepresenter.represent_list
diff --git a/rest_framework/utils/field_mapping.py b/rest_framework/utils/field_mapping.py
new file mode 100644
index 00000000..be72e444
--- /dev/null
+++ b/rest_framework/utils/field_mapping.py
@@ -0,0 +1,215 @@
+"""
+Helper functions for mapping model fields to a dictionary of default
+keyword arguments that should be used for their equivelent serializer fields.
+"""
+from django.core import validators
+from django.db import models
+from django.utils.text import capfirst
+from rest_framework.compat import clean_manytomany_helptext
+import inspect
+
+
+def lookup_class(mapping, instance):
+    """
+    Takes a dictionary with classes as keys, and an object.
+    Traverses the object's inheritance hierarchy in method
+    resolution order, and returns the first matching value
+    from the dictionary or raises a KeyError if nothing matches.
+    """
+    for cls in inspect.getmro(instance.__class__):
+        if cls in mapping:
+            return mapping[cls]
+    raise KeyError('Class %s not found in lookup.', cls.__name__)
+
+
+def needs_label(model_field, field_name):
+    """
+    Returns `True` if the label based on the model's verbose name
+    is not equal to the default label it would have based on it's field name.
+    """
+    default_label = field_name.replace('_', ' ').capitalize()
+    return capfirst(model_field.verbose_name) != default_label
+
+
+def get_detail_view_name(model):
+    """
+    Given a model class, return the view name to use for URL relationships
+    that refer to instances of the model.
+    """
+    return '%(model_name)s-detail' % {
+        'app_label': model._meta.app_label,
+        'model_name': model._meta.object_name.lower()
+    }
+
+
+def get_field_kwargs(field_name, model_field):
+    """
+    Creates a default instance of a basic non-relational field.
+    """
+    kwargs = {}
+    validator_kwarg = model_field.validators
+
+    if model_field.null or model_field.blank:
+        kwargs['required'] = False
+
+    if model_field.verbose_name and needs_label(model_field, field_name):
+        kwargs['label'] = capfirst(model_field.verbose_name)
+
+    if model_field.help_text:
+        kwargs['help_text'] = model_field.help_text
+
+    if isinstance(model_field, models.AutoField) or not model_field.editable:
+        kwargs['read_only'] = True
+        # Read only implies that the field is not required.
+        # We have a cleaner repr on the instance if we don't set it.
+        kwargs.pop('required', None)
+
+    if model_field.has_default():
+        kwargs['default'] = model_field.get_default()
+        # Having a default implies that the field is not required.
+        # We have a cleaner repr on the instance if we don't set it.
+        kwargs.pop('required', None)
+
+    if model_field.flatchoices:
+        # If this model field contains choices, then return now,
+        # any further keyword arguments are not valid.
+        kwargs['choices'] = model_field.flatchoices
+        return kwargs
+
+    # Ensure that max_length is passed explicitly as a keyword arg,
+    # rather than as a validator.
+    max_length = getattr(model_field, 'max_length', None)
+    if max_length is not None:
+        kwargs['max_length'] = max_length
+        validator_kwarg = [
+            validator for validator in validator_kwarg
+            if not isinstance(validator, validators.MaxLengthValidator)
+        ]
+
+    # Ensure that min_length is passed explicitly as a keyword arg,
+    # rather than as a validator.
+    min_length = getattr(model_field, 'min_length', None)
+    if min_length is not None:
+        kwargs['min_length'] = min_length
+        validator_kwarg = [
+            validator for validator in validator_kwarg
+            if not isinstance(validator, validators.MinLengthValidator)
+        ]
+
+    # Ensure that max_value is passed explicitly as a keyword arg,
+    # rather than as a validator.
+    max_value = next((
+        validator.limit_value for validator in validator_kwarg
+        if isinstance(validator, validators.MaxValueValidator)
+    ), None)
+    if max_value is not None:
+        kwargs['max_value'] = max_value
+        validator_kwarg = [
+            validator for validator in validator_kwarg
+            if not isinstance(validator, validators.MaxValueValidator)
+        ]
+
+    # Ensure that max_value is passed explicitly as a keyword arg,
+    # rather than as a validator.
+    min_value = next((
+        validator.limit_value for validator in validator_kwarg
+        if isinstance(validator, validators.MinValueValidator)
+    ), None)
+    if min_value is not None:
+        kwargs['min_value'] = min_value
+        validator_kwarg = [
+            validator for validator in validator_kwarg
+            if not isinstance(validator, validators.MinValueValidator)
+        ]
+
+    # URLField does not need to include the URLValidator argument,
+    # as it is explicitly added in.
+    if isinstance(model_field, models.URLField):
+        validator_kwarg = [
+            validator for validator in validator_kwarg
+            if not isinstance(validator, validators.URLValidator)
+        ]
+
+    # EmailField does not need to include the validate_email argument,
+    # as it is explicitly added in.
+    if isinstance(model_field, models.EmailField):
+        validator_kwarg = [
+            validator for validator in validator_kwarg
+            if validator is not validators.validate_email
+        ]
+
+    # SlugField do not need to include the 'validate_slug' argument,
+    if isinstance(model_field, models.SlugField):
+        validator_kwarg = [
+            validator for validator in validator_kwarg
+            if validator is not validators.validate_slug
+        ]
+
+    max_digits = getattr(model_field, 'max_digits', None)
+    if max_digits is not None:
+        kwargs['max_digits'] = max_digits
+
+    decimal_places = getattr(model_field, 'decimal_places', None)
+    if decimal_places is not None:
+        kwargs['decimal_places'] = decimal_places
+
+    if isinstance(model_field, models.BooleanField):
+        # models.BooleanField has `blank=True`, but *is* actually
+        # required *unless* a default is provided.
+        # Also note that Django<1.6 uses `default=False` for
+        # models.BooleanField, but Django>=1.6 uses `default=None`.
+        kwargs.pop('required', None)
+
+    if validator_kwarg:
+        kwargs['validators'] = validator_kwarg
+
+    # The following will only be used by ModelField classes.
+    # Gets removed for everything else.
+    kwargs['model_field'] = model_field
+
+    return kwargs
+
+
+def get_relation_kwargs(field_name, relation_info):
+    """
+    Creates a default instance of a flat relational field.
+    """
+    model_field, related_model, to_many, has_through_model = relation_info
+    kwargs = {
+        'queryset': related_model._default_manager,
+        'view_name': get_detail_view_name(related_model)
+    }
+
+    if to_many:
+        kwargs['many'] = True
+
+    if has_through_model:
+        kwargs['read_only'] = True
+        kwargs.pop('queryset', None)
+
+    if model_field:
+        if model_field.null or model_field.blank:
+            kwargs['required'] = False
+        if model_field.verbose_name and needs_label(model_field, field_name):
+            kwargs['label'] = capfirst(model_field.verbose_name)
+        if not model_field.editable:
+            kwargs['read_only'] = True
+            kwargs.pop('queryset', None)
+        help_text = clean_manytomany_helptext(model_field.help_text)
+        if help_text:
+            kwargs['help_text'] = help_text
+
+    return kwargs
+
+
+def get_nested_relation_kwargs(relation_info):
+    kwargs = {'read_only': True}
+    if relation_info.to_many:
+        kwargs['many'] = True
+    return kwargs
+
+
+def get_url_kwargs(model_field):
+    return {
+        'view_name': get_detail_view_name(model_field)
+    }
diff --git a/rest_framework/utils/formatting.py b/rest_framework/utils/formatting.py
index 6d53aed1..470af51b 100644
--- a/rest_framework/utils/formatting.py
+++ b/rest_framework/utils/formatting.py
@@ -2,11 +2,12 @@
 Utility functions to return a formatted name and description for a given view.
 """
 from __future__ import unicode_literals
+import re
 
 from django.utils.html import escape
 from django.utils.safestring import mark_safe
-from rest_framework.compat import apply_markdown
-import re
+
+from rest_framework.compat import apply_markdown, force_text
 
 
 def remove_trailing_string(content, trailing):
@@ -28,6 +29,7 @@ def dedent(content):
     as it fails to dedent multiline docstrings that include
     unindented text on the initial line.
     """
+    content = force_text(content)
     whitespace_counts = [len(line) - len(line.lstrip(' '))
                          for line in content.splitlines()[1:] if line.lstrip()]
 
diff --git a/rest_framework/utils/html.py b/rest_framework/utils/html.py
new file mode 100644
index 00000000..edc591e9
--- /dev/null
+++ b/rest_framework/utils/html.py
@@ -0,0 +1,88 @@
+"""
+Helpers for dealing with HTML input.
+"""
+import re
+
+
+def is_html_input(dictionary):
+    # MultiDict type datastructures are used to represent HTML form input,
+    # which may have more than one value for each key.
+    return hasattr(dictionary, 'getlist')
+
+
+def parse_html_list(dictionary, prefix=''):
+    """
+    Used to suport list values in HTML forms.
+    Supports lists of primitives and/or dictionaries.
+
+    * List of primitives.
+
+    {
+        '[0]': 'abc',
+        '[1]': 'def',
+        '[2]': 'hij'
+    }
+        -->
+    [
+        'abc',
+        'def',
+        'hij'
+    ]
+
+    * List of dictionaries.
+
+    {
+        '[0]foo': 'abc',
+        '[0]bar': 'def',
+        '[1]foo': 'hij',
+        '[2]bar': 'klm',
+    }
+        -->
+    [
+        {'foo': 'abc', 'bar': 'def'},
+        {'foo': 'hij', 'bar': 'klm'}
+    ]
+    """
+    Dict = type(dictionary)
+    ret = {}
+    regex = re.compile(r'^%s\[([0-9]+)\](.*)$' % re.escape(prefix))
+    for field, value in dictionary.items():
+        match = regex.match(field)
+        if not match:
+            continue
+        index, key = match.groups()
+        index = int(index)
+        if not key:
+            ret[index] = value
+        elif isinstance(ret.get(index), dict):
+            ret[index][key] = value
+        else:
+            ret[index] = Dict({key: value})
+    return [ret[item] for item in sorted(ret.keys())]
+
+
+def parse_html_dict(dictionary, prefix):
+    """
+    Used to support dictionary values in HTML forms.
+
+    {
+        'profile.username': 'example',
+        'profile.email': 'example@example.com',
+    }
+        -->
+    {
+        'profile': {
+            'username': 'example,
+            'email': 'example@example.com'
+        }
+    }
+    """
+    ret = {}
+    regex = re.compile(r'^%s\.(.+)$' % re.escape(prefix))
+    for field, value in dictionary.items():
+        match = regex.match(field)
+        if not match:
+            continue
+        key = match.groups()[0]
+        ret[key] = value
+    return ret
diff --git a/rest_framework/utils/humanize_datetime.py b/rest_framework/utils/humanize_datetime.py
new file mode 100644
index 00000000..649f2abc
--- /dev/null
+++ b/rest_framework/utils/humanize_datetime.py
@@ -0,0 +1,47 @@
+"""
+Helper functions that convert strftime formats into more readable representations.
+"""
+from rest_framework import ISO_8601
+
+
+def datetime_formats(formats):
+    format = ', '.join(formats).replace(
+        ISO_8601,
+        'YYYY-MM-DDThh:mm[:ss[.uuuuuu]][+HH:MM|-HH:MM|Z]'
+    )
+    return humanize_strptime(format)
+
+
+def date_formats(formats):
+    format = ', '.join(formats).replace(ISO_8601, 'YYYY[-MM[-DD]]')
+    return humanize_strptime(format)
+
+
+def time_formats(formats):
+    format = ', '.join(formats).replace(ISO_8601, 'hh:mm[:ss[.uuuuuu]]')
+    return humanize_strptime(format)
+
+
+def humanize_strptime(format_string):
+    # Note that we're missing some of the locale specific mappings that
+    # don't really make sense.
+    mapping = {
+        "%Y": "YYYY",
+        "%y": "YY",
+        "%m": "MM",
+        "%b": "[Jan-Dec]",
+        "%B": "[January-December]",
+        "%d": "DD",
+        "%H": "hh",
+        "%I": "hh",  # Requires '%p' to differentiate from '%H'.
+        "%M": "mm",
+        "%S": "ss",
+        "%f": "uuuuuu",
+        "%a": "[Mon-Sun]",
+        "%A": "[Monday-Sunday]",
+        "%p": "[AM|PM]",
+        "%z": "[+HHMM|-HHMM]"
+    }
+    for key, val in mapping.items():
+        format_string = format_string.replace(key, val)
+    return format_string
diff --git a/rest_framework/utils/model_meta.py b/rest_framework/utils/model_meta.py
new file mode 100644
index 00000000..b6c41174
--- /dev/null
+++ b/rest_framework/utils/model_meta.py
@@ -0,0 +1,129 @@
+"""
+Helper function for returning the field information that is associated
+with a model class. This includes returning all the forward and reverse
+relationships and their associated metadata.
+
+Usage: `get_field_info(model)` returns a `FieldInfo` instance.
+"""
+from collections import namedtuple
+from django.db import models
+from django.utils import six
+from django.utils.datastructures import SortedDict
+import inspect
+
+
+FieldInfo = namedtuple('FieldResult', [
+    'pk',  # Model field instance
+    'fields',  # Dict of field name -> model field instance
+    'forward_relations',  # Dict of field name -> RelationInfo
+    'reverse_relations',  # Dict of field name -> RelationInfo
+    'fields_and_pk',  # Shortcut for 'pk' + 'fields'
+    'relations'  # Shortcut for 'forward_relations' + 'reverse_relations'
+])
+
+RelationInfo = namedtuple('RelationInfo', [
+    'model_field',
+    'related',
+    'to_many',
+    'has_through_model'
+])
+
+
+def _resolve_model(obj):
+    """
+    Resolve supplied `obj` to a Django model class.
+
+    `obj` must be a Django model class itself, or a string
+    representation of one.  Useful in situtations like GH #1225 where
+    Django may not have resolved a string-based reference to a model in
+    another model's foreign key definition.
+
+    String representations should have the format:
+        'appname.ModelName'
+    """
+    if isinstance(obj, six.string_types) and len(obj.split('.')) == 2:
+        app_name, model_name = obj.split('.')
+        return models.get_model(app_name, model_name)
+    elif inspect.isclass(obj) and issubclass(obj, models.Model):
+        return obj
+    raise ValueError("{0} is not a Django model".format(obj))
+
+
+def get_field_info(model):
+    """
+    Given a model class, returns a `FieldInfo` instance containing metadata
+    about the various field types on the model.
+    """
+    opts = model._meta.concrete_model._meta
+
+    # Deal with the primary key.
+    pk = opts.pk
+    while pk.rel and pk.rel.parent_link:
+        # If model is a child via multitable inheritance, use parent's pk.
+        pk = pk.rel.to._meta.pk
+
+    # Deal with regular fields.
+    fields = SortedDict()
+    for field in [field for field in opts.fields if field.serialize and not field.rel]:
+        fields[field.name] = field
+
+    # Deal with forward relationships.
+    forward_relations = SortedDict()
+    for field in [field for field in opts.fields if field.serialize and field.rel]:
+        forward_relations[field.name] = RelationInfo(
+            model_field=field,
+            related=_resolve_model(field.rel.to),
+            to_many=False,
+            has_through_model=False
+        )
+
+    # Deal with forward many-to-many relationships.
+    for field in [field for field in opts.many_to_many if field.serialize]:
+        forward_relations[field.name] = RelationInfo(
+            model_field=field,
+            related=_resolve_model(field.rel.to),
+            to_many=True,
+            has_through_model=(
+                not field.rel.through._meta.auto_created
+            )
+        )
+
+    # Deal with reverse relationships.
+    reverse_relations = SortedDict()
+    for relation in opts.get_all_related_objects():
+        accessor_name = relation.get_accessor_name()
+        reverse_relations[accessor_name] = RelationInfo(
+            model_field=None,
+            related=relation.model,
+            to_many=relation.field.rel.multiple,
+            has_through_model=False
+        )
+
+    # Deal with reverse many-to-many relationships.
+    for relation in opts.get_all_related_many_to_many_objects():
+        accessor_name = relation.get_accessor_name()
+        reverse_relations[accessor_name] = RelationInfo(
+            model_field=None,
+            related=relation.model,
+            to_many=True,
+            has_through_model=(
+                hasattr(relation.field.rel, 'through') and
+                not relation.field.rel.through._meta.auto_created
+            )
+        )
+
+    # Shortcut that merges both regular fields and the pk,
+    # for simplifying regular field lookup.
+    fields_and_pk = SortedDict()
+    fields_and_pk['pk'] = pk
+    fields_and_pk[pk.name] = pk
+    fields_and_pk.update(fields)
+
+    # Shortcut that merges both forward and reverse relationships
+
+    relations = SortedDict(
+        list(forward_relations.items()) +
+        list(reverse_relations.items())
+    )
+
+    return FieldInfo(pk, fields, forward_relations, reverse_relations, fields_and_pk, relations)
diff --git a/rest_framework/utils/representation.py b/rest_framework/utils/representation.py
new file mode 100644
index 00000000..e64fdd22
--- /dev/null
+++ b/rest_framework/utils/representation.py
@@ -0,0 +1,87 @@
+"""
+Helper functions for creating user-friendly representations
+of serializer classes and serializer fields.
+"""
+from django.db import models
+import re
+
+
+def manager_repr(value):
+    model = value.model
+    opts = model._meta
+    for _, name, manager in opts.concrete_managers + opts.abstract_managers:
+        if manager == value:
+            return '%s.%s.all()' % (model._meta.object_name, name)
+    return repr(value)
+
+
+def smart_repr(value):
+    if isinstance(value, models.Manager):
+        return manager_repr(value)
+
+    value = repr(value)
+
+    # Representations like u'help text'
+    # should simply be presented as 'help text'
+    if value.startswith("u'") and value.endswith("'"):
+        return value[1:]
+
+    # Representations like
+    # <django.core.validators.RegexValidator object at 0x1047af050>
+    # Should be presented as
+    # <django.core.validators.RegexValidator object>
+    value = re.sub(' at 0x[0-9a-f]{4,32}>', '>', value)
+
+    return value
+
+
+def field_repr(field, force_many=False):
+    kwargs = field._kwargs
+    if force_many:
+        kwargs = kwargs.copy()
+        kwargs['many'] = True
+        kwargs.pop('child', None)
+
+    arg_string = ', '.join([smart_repr(val) for val in field._args])
+    kwarg_string = ', '.join([
+        '%s=%s' % (key, smart_repr(val))
+        for key, val in sorted(kwargs.items())
+    ])
+    if arg_string and kwarg_string:
+        arg_string += ', '
+
+    if force_many:
+        class_name = force_many.__class__.__name__
+    else:
+        class_name = field.__class__.__name__
+
+    return "%s(%s%s)" % (class_name, arg_string, kwarg_string)
+
+
+def serializer_repr(serializer, indent, force_many=None):
+    ret = field_repr(serializer, force_many) + ':'
+    indent_str = '    ' * indent
+
+    if force_many:
+        fields = force_many.fields
+    else:
+        fields = serializer.fields
+
+    for field_name, field in fields.items():
+        ret += '\n' + indent_str + field_name + ' = '
+        if hasattr(field, 'fields'):
+            ret += serializer_repr(field, indent + 1)
+        elif hasattr(field, 'child'):
+            ret += list_repr(field, indent + 1)
+        elif hasattr(field, 'child_relation'):
+            ret += field_repr(field.child_relation, force_many=field.child_relation)
+        else:
+            ret += field_repr(field)
+    return ret
+
+
+def list_repr(serializer, indent):
+    child = serializer.child
+    if hasattr(child, 'fields'):
+        return serializer_repr(serializer, indent, force_many=child)
+    return field_repr(serializer)