From 91b33659b562f26665cebc3c7d8aba6d57e9386c Mon Sep 17 00:00:00 2001 From: spiq Date: Fri, 4 Mar 2011 12:28:20 +0200 Subject: fix for PUT files --- djangorestframework/parsers.py | 81 +++++++++++++++++++++++++----------------- 1 file changed, 48 insertions(+), 33 deletions(-) (limited to 'djangorestframework/parsers.py') diff --git a/djangorestframework/parsers.py b/djangorestframework/parsers.py index 7c686ca8..e5dd7df4 100644 --- a/djangorestframework/parsers.py +++ b/djangorestframework/parsers.py @@ -1,3 +1,7 @@ +from StringIO import StringIO + +from django.http.multipartparser import MultiPartParser as DjangoMPParser + from djangorestframework.response import ResponseException from djangorestframework import status @@ -6,6 +10,10 @@ try: except ImportError: import simplejson as json +try: + from urlparse import parse_qs +except ImportError: + from cgi import parse_qs class ParserMixin(object): parsers = () @@ -75,50 +83,57 @@ class FormParser(BaseParser): """The default parser for form data. Return a dict containing a single value for each non-reserved parameter. """ - + # TODO: not good, because posted/put lists are flattened !!! media_type = 'application/x-www-form-urlencoded' def parse(self, input): - # The FormParser doesn't parse the input as other parsers would, since Django's already done the - # form parsing for us. We build the content object from the request directly. request = self.resource.request if request.method == 'PUT': - # Fix from piston to force Django to give PUT requests the same - # form processing that POST requests get... - # - # Bug fix: if _load_post_and_files has already been called, for - # example by middleware accessing request.POST, the below code to - # pretend the request is a POST instead of a PUT will be too late - # to make a difference. Also calling _load_post_and_files will result - # in the following exception: - # AttributeError: You cannot set the upload handlers after the upload has been processed. - # The fix is to check for the presence of the _post field which is set - # the first time _load_post_and_files is called (both by wsgi.py and - # modpython.py). If it's set, the request has to be 'reset' to redo - # the query value parsing in POST mode. - if hasattr(request, '_post'): - del request._post - del request._files - - try: - request.method = "POST" - request._load_post_and_files() - request.method = "PUT" - except AttributeError: - request.META['REQUEST_METHOD'] = 'POST' - request._load_post_and_files() - request.META['REQUEST_METHOD'] = 'PUT' + data = parse_qs(input) + # Flattening the parsed query data + for key, val in data.items(): + data[key] = val[0] + + if request.method == 'POST': + # Django has already done the form parsing for us. + data = dict(request.POST.items()) # Strip any parameters that we are treating as reserved - data = {} - for (key, val) in request.POST.items(): - if key not in self.resource.RESERVED_FORM_PARAMS: - data[key] = val - + for key in data: + if key in self.resource.RESERVED_FORM_PARAMS: + data.pop(key) return data # TODO: Allow parsers to specify multiple media_types class MultipartParser(FormParser): media_type = 'multipart/form-data' + def parse(self, input): + request = self.resource.request + + if request.method == 'PUT': + upload_handlers = request._get_upload_handlers() + django_mpp = DjangoMPParser(request.META, StringIO(input), upload_handlers) + data, files = django_mpp.parse() + data = dict(data) + files = dict(files) + + if request.method == 'POST': + # Django has already done the form parsing for us. + data = dict(request.POST) + files = dict(request.FILES) + + # Flattening, then merging the POSTED/PUT data/files + for key, val in dict(data).items(): + data[key] = val[0] + for key, val in dict(files).items(): + files[key] = val[0].read() + data.update(files) + + # Strip any parameters that we are treating as reserved + for key in data: + if key in self.resource.RESERVED_FORM_PARAMS: + data.pop(key) + return data + -- cgit v1.2.3 From 033c0ae797e0bbb39a0687592d5b5f8eceb4d4dd Mon Sep 17 00:00:00 2001 From: spiq Date: Fri, 4 Mar 2011 17:06:44 +0200 Subject: forms and multipat parsers now flaten data --- djangorestframework/parsers.py | 57 +++++++++++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 20 deletions(-) (limited to 'djangorestframework/parsers.py') diff --git a/djangorestframework/parsers.py b/djangorestframework/parsers.py index e5dd7df4..c334f729 100644 --- a/djangorestframework/parsers.py +++ b/djangorestframework/parsers.py @@ -78,12 +78,36 @@ class JSONParser(BaseParser): class XMLParser(BaseParser): media_type = 'application/xml' - -class FormParser(BaseParser): +class DataFlatener(object): + + def flatten_data(self, data): + """Given a data dictionary ``{: }``, returns a flattened dictionary according to :meth:`FormParser.is_a_list`. + """ + #TODO : document + test + flatdata = dict() + for attr_name, attr_value in data.items(): + if self.is_a_list(attr_name): + if isinstance(attr_value, list): + flatdata[attr_name] = attr_value + else: + flatdata[attr_name] = [attr_value] + else: + if isinstance(attr_value, list): + flatdata[attr_name] = attr_value[0] + else: + flatdata[attr_name] = attr_value + return flatdata + + def is_a_list(self, attr_name): + """ """ + #TODO: document + return False + +class FormParser(BaseParser, DataFlatener): """The default parser for form data. Return a dict containing a single value for each non-reserved parameter. """ - # TODO: not good, because posted/put lists are flattened !!! + # TODO: writing tests for PUT files + normal data media_type = 'application/x-www-form-urlencoded' def parse(self, input): @@ -91,13 +115,12 @@ class FormParser(BaseParser): if request.method == 'PUT': data = parse_qs(input) - # Flattening the parsed query data - for key, val in data.items(): - data[key] = val[0] if request.method == 'POST': # Django has already done the form parsing for us. - data = dict(request.POST.items()) + data = request.POST + + data = self.flatten_data(data) # Strip any parameters that we are treating as reserved for key in data: @@ -106,7 +129,7 @@ class FormParser(BaseParser): return data # TODO: Allow parsers to specify multiple media_types -class MultipartParser(FormParser): +class MultipartParser(BaseParser, DataFlatener): media_type = 'multipart/form-data' def parse(self, input): @@ -116,19 +139,14 @@ class MultipartParser(FormParser): upload_handlers = request._get_upload_handlers() django_mpp = DjangoMPParser(request.META, StringIO(input), upload_handlers) data, files = django_mpp.parse() - data = dict(data) - files = dict(files) - if request.method == 'POST': + elif request.method == 'POST': # Django has already done the form parsing for us. - data = dict(request.POST) - files = dict(request.FILES) - - # Flattening, then merging the POSTED/PUT data/files - for key, val in dict(data).items(): - data[key] = val[0] - for key, val in dict(files).items(): - files[key] = val[0].read() + data = request.POST + files = request.FILES + + data = self.flatten_data(data) + files = self.flatten_data(files) data.update(files) # Strip any parameters that we are treating as reserved @@ -136,4 +154,3 @@ class MultipartParser(FormParser): if key in self.resource.RESERVED_FORM_PARAMS: data.pop(key) return data - -- cgit v1.2.3 From 2169c85dbb011beb411a5c37ebc8d700b0984ce8 Mon Sep 17 00:00:00 2001 From: spiq Date: Fri, 4 Mar 2011 17:23:18 +0200 Subject: FomrParser now implements a work around for empty values in a list --- djangorestframework/parsers.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) (limited to 'djangorestframework/parsers.py') diff --git a/djangorestframework/parsers.py b/djangorestframework/parsers.py index c334f729..38f3db4b 100644 --- a/djangorestframework/parsers.py +++ b/djangorestframework/parsers.py @@ -79,11 +79,11 @@ class XMLParser(BaseParser): media_type = 'application/xml' class DataFlatener(object): +#TODO : document + test def flatten_data(self, data): """Given a data dictionary ``{: }``, returns a flattened dictionary according to :meth:`FormParser.is_a_list`. """ - #TODO : document + test flatdata = dict() for attr_name, attr_value in data.items(): if self.is_a_list(attr_name): @@ -100,27 +100,32 @@ class DataFlatener(object): def is_a_list(self, attr_name): """ """ - #TODO: document return False class FormParser(BaseParser, DataFlatener): """The default parser for form data. Return a dict containing a single value for each non-reserved parameter. """ + # TODO: document flatening # TODO: writing tests for PUT files + normal data + # TODO: document EMPTY workaround media_type = 'application/x-www-form-urlencoded' + EMPTY_VALUE = 'EMPTY' + def parse(self, input): request = self.resource.request if request.method == 'PUT': data = parse_qs(input) - - if request.method == 'POST': + elif request.method == 'POST': # Django has already done the form parsing for us. data = request.POST + # Flatening data and removing EMPTY_VALUEs from the lists data = self.flatten_data(data) + for key in filter(lambda k: self.is_a_list(k), data): + self.remove_empty_val(data[key]) # Strip any parameters that we are treating as reserved for key in data: @@ -128,6 +133,16 @@ class FormParser(BaseParser, DataFlatener): data.pop(key) return data + def remove_empty_val(self, val_list): + """ """ + while(1): # Because there might be several times EMPTY_VALUE in the list + try: + ind = val_list.index(self.EMPTY_VALUE) + except ValueError: + break + else: + val_list.pop(ind) + # TODO: Allow parsers to specify multiple media_types class MultipartParser(BaseParser, DataFlatener): media_type = 'multipart/form-data' @@ -139,12 +154,12 @@ class MultipartParser(BaseParser, DataFlatener): upload_handlers = request._get_upload_handlers() django_mpp = DjangoMPParser(request.META, StringIO(input), upload_handlers) data, files = django_mpp.parse() - elif request.method == 'POST': # Django has already done the form parsing for us. data = request.POST files = request.FILES + # Flatening data, files and combining them data = self.flatten_data(data) files = self.flatten_data(files) data.update(files) -- cgit v1.2.3 From d053cc892ddddd6faeb231c8f8f25a1c31507f42 Mon Sep 17 00:00:00 2001 From: spiq Date: Tue, 8 Mar 2011 17:19:55 +0200 Subject: corrected a small bug --- djangorestframework/parsers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'djangorestframework/parsers.py') diff --git a/djangorestframework/parsers.py b/djangorestframework/parsers.py index 38f3db4b..b7587672 100644 --- a/djangorestframework/parsers.py +++ b/djangorestframework/parsers.py @@ -128,7 +128,7 @@ class FormParser(BaseParser, DataFlatener): self.remove_empty_val(data[key]) # Strip any parameters that we are treating as reserved - for key in data: + for key in data.keys(): if key in self.resource.RESERVED_FORM_PARAMS: data.pop(key) return data @@ -165,7 +165,7 @@ class MultipartParser(BaseParser, DataFlatener): data.update(files) # Strip any parameters that we are treating as reserved - for key in data: + for key in data.keys(): if key in self.resource.RESERVED_FORM_PARAMS: data.pop(key) return data -- cgit v1.2.3 From 899233bf99bfb07e820e4c636bfabc62cfea981e Mon Sep 17 00:00:00 2001 From: spiq Date: Wed, 9 Mar 2011 14:07:06 +0200 Subject: renamings + corrected a bug --- djangorestframework/parsers.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'djangorestframework/parsers.py') diff --git a/djangorestframework/parsers.py b/djangorestframework/parsers.py index b7587672..b7b73408 100644 --- a/djangorestframework/parsers.py +++ b/djangorestframework/parsers.py @@ -82,23 +82,23 @@ class DataFlatener(object): #TODO : document + test def flatten_data(self, data): - """Given a data dictionary ``{: }``, returns a flattened dictionary according to :meth:`FormParser.is_a_list`. + """Given a data dictionary ``{: }``, returns a flattened dictionary according to :meth:`FormParser.is_a_list`. """ flatdata = dict() - for attr_name, attr_value in data.items(): - if self.is_a_list(attr_name): + for key, attr_value in data.items(): + if self.is_a_list(key): if isinstance(attr_value, list): - flatdata[attr_name] = attr_value + flatdata[key] = attr_value else: - flatdata[attr_name] = [attr_value] + flatdata[key] = [attr_value] else: if isinstance(attr_value, list): - flatdata[attr_name] = attr_value[0] + flatdata[key] = attr_value[0] else: - flatdata[attr_name] = attr_value + flatdata[key] = attr_value return flatdata - def is_a_list(self, attr_name): + def is_a_list(self, key): """ """ return False @@ -120,7 +120,7 @@ class FormParser(BaseParser, DataFlatener): data = parse_qs(input) elif request.method == 'POST': # Django has already done the form parsing for us. - data = request.POST + data = dict(request.POST.iterlists()) # Flatening data and removing EMPTY_VALUEs from the lists data = self.flatten_data(data) -- cgit v1.2.3 From 26e10d0e3fadf1ff87da1650e30c1de34392b0d4 Mon Sep 17 00:00:00 2001 From: sebpiq Date: Thu, 10 Mar 2011 16:03:46 +0200 Subject: removed useless stuff, request.POST and FILES not used + some doc --- djangorestframework/parsers.py | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) (limited to 'djangorestframework/parsers.py') diff --git a/djangorestframework/parsers.py b/djangorestframework/parsers.py index b7b73408..4659f24b 100644 --- a/djangorestframework/parsers.py +++ b/djangorestframework/parsers.py @@ -114,13 +114,7 @@ class FormParser(BaseParser, DataFlatener): EMPTY_VALUE = 'EMPTY' def parse(self, input): - request = self.resource.request - - if request.method == 'PUT': - data = parse_qs(input) - elif request.method == 'POST': - # Django has already done the form parsing for us. - data = dict(request.POST.iterlists()) + data = parse_qs(input) # Flatening data and removing EMPTY_VALUEs from the lists data = self.flatten_data(data) @@ -150,14 +144,9 @@ class MultipartParser(BaseParser, DataFlatener): def parse(self, input): request = self.resource.request - if request.method == 'PUT': - upload_handlers = request._get_upload_handlers() - django_mpp = DjangoMPParser(request.META, StringIO(input), upload_handlers) - data, files = django_mpp.parse() - elif request.method == 'POST': - # Django has already done the form parsing for us. - data = request.POST - files = request.FILES + upload_handlers = request._get_upload_handlers() + django_mpp = DjangoMPParser(request.META, StringIO(input), upload_handlers) + data, files = django_mpp.parse() # Flatening data, files and combining them data = self.flatten_data(data) -- cgit v1.2.3 From 2d8e4e056b1b6f822d02d614955ee1f220e2fdda Mon Sep 17 00:00:00 2001 From: sebpiq Date: Thu, 10 Mar 2011 16:49:11 +0200 Subject: documentation --- djangorestframework/parsers.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) (limited to 'djangorestframework/parsers.py') diff --git a/djangorestframework/parsers.py b/djangorestframework/parsers.py index 4659f24b..081fa689 100644 --- a/djangorestframework/parsers.py +++ b/djangorestframework/parsers.py @@ -79,11 +79,11 @@ class XMLParser(BaseParser): media_type = 'application/xml' class DataFlatener(object): -#TODO : document + test + """Utility object for flatening dictionaries of lists. Useful for "urlencoded" decoded data.""" def flatten_data(self, data): - """Given a data dictionary ``{: }``, returns a flattened dictionary according to :meth:`FormParser.is_a_list`. - """ + """Given a data dictionary {: }, returns a flattened dictionary + with information provided by the method "is_a_list".""" flatdata = dict() for key, attr_value in data.items(): if self.is_a_list(key): @@ -98,20 +98,27 @@ class DataFlatener(object): flatdata[key] = attr_value return flatdata - def is_a_list(self, key): - """ """ + def is_a_list(self, key, val): + """Returns True if the parameter with name *key* is expected to be a list, or False otherwise. + *val* which is the received value for parameter *key* can be used to guess the answer.""" return False class FormParser(BaseParser, DataFlatener): """The default parser for form data. Return a dict containing a single value for each non-reserved parameter. + + In order to handle select multiple (and having possibly more than a single value for each parameter), + you can customize the output by subclassing the method 'is_a_list'. + """ - # TODO: document flatening # TODO: writing tests for PUT files + normal data - # TODO: document EMPTY workaround media_type = 'application/x-www-form-urlencoded' - EMPTY_VALUE = 'EMPTY' + """The value of the parameter when the select multiple is empty. + Browsers are usually stripping the select multiple that have no option selected from the parameters sent. + A common hack to avoid this is to send the parameter with a value specifying that the list is empty. + This value will always be stripped before the data is returned.""" + EMPTY_VALUE = '_empty' def parse(self, input): data = parse_qs(input) @@ -142,8 +149,11 @@ class MultipartParser(BaseParser, DataFlatener): media_type = 'multipart/form-data' def parse(self, input): - request = self.resource.request + request = self.resource.request + #TODO : that's pretty dumb : files are loaded with + #upload_handlers, but as we read the request body completely (input), + #then it kind of misses the point. Why not input as a stream ? upload_handlers = request._get_upload_handlers() django_mpp = DjangoMPParser(request.META, StringIO(input), upload_handlers) data, files = django_mpp.parse() -- cgit v1.2.3 From d6c13a9e5caee653f7a8f3150c00aa574aa15700 Mon Sep 17 00:00:00 2001 From: sebpiq Date: Fri, 11 Mar 2011 14:34:39 +0200 Subject: documentation + tests + debugging for formparsers --- djangorestframework/parsers.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) (limited to 'djangorestframework/parsers.py') diff --git a/djangorestframework/parsers.py b/djangorestframework/parsers.py index 081fa689..d324f687 100644 --- a/djangorestframework/parsers.py +++ b/djangorestframework/parsers.py @@ -84,23 +84,23 @@ class DataFlatener(object): def flatten_data(self, data): """Given a data dictionary {: }, returns a flattened dictionary with information provided by the method "is_a_list".""" + data = data.copy() flatdata = dict() - for key, attr_value in data.items(): - if self.is_a_list(key): - if isinstance(attr_value, list): - flatdata[key] = attr_value - else: - flatdata[key] = [attr_value] + for key, val_list in data.items(): + if self.is_a_list(key, val_list): + flatdata[key] = val_list else: - if isinstance(attr_value, list): - flatdata[key] = attr_value[0] + if val_list: + flatdata[key] = val_list[0] else: - flatdata[key] = attr_value + # If the list is empty, but the parameter is not a list, + # we strip this parameter. + data.pop(key) return flatdata - def is_a_list(self, key, val): + def is_a_list(self, key, val_list): """Returns True if the parameter with name *key* is expected to be a list, or False otherwise. - *val* which is the received value for parameter *key* can be used to guess the answer.""" + *val_list* which is the received value for parameter *key* can be used to guess the answer.""" return False class FormParser(BaseParser, DataFlatener): @@ -121,12 +121,12 @@ class FormParser(BaseParser, DataFlatener): EMPTY_VALUE = '_empty' def parse(self, input): - data = parse_qs(input) + data = parse_qs(input, keep_blank_values=True) - # Flatening data and removing EMPTY_VALUEs from the lists + # removing EMPTY_VALUEs from the lists and flatening the data + for key, val_list in data.items(): + self.remove_empty_val(val_list) data = self.flatten_data(data) - for key in filter(lambda k: self.is_a_list(k), data): - self.remove_empty_val(data[key]) # Strip any parameters that we are treating as reserved for key in data.keys(): -- cgit v1.2.3 From ee74aec27cdc8ca9934f93c828ffbdc7da3c426c Mon Sep 17 00:00:00 2001 From: sebpiq Date: Fri, 11 Mar 2011 15:05:35 +0200 Subject: debug + tests for MultipartParser --- djangorestframework/parsers.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) (limited to 'djangorestframework/parsers.py') diff --git a/djangorestframework/parsers.py b/djangorestframework/parsers.py index d324f687..3bd020ad 100644 --- a/djangorestframework/parsers.py +++ b/djangorestframework/parsers.py @@ -84,7 +84,6 @@ class DataFlatener(object): def flatten_data(self, data): """Given a data dictionary {: }, returns a flattened dictionary with information provided by the method "is_a_list".""" - data = data.copy() flatdata = dict() for key, val_list in data.items(): if self.is_a_list(key, val_list): @@ -108,10 +107,8 @@ class FormParser(BaseParser, DataFlatener): Return a dict containing a single value for each non-reserved parameter. In order to handle select multiple (and having possibly more than a single value for each parameter), - you can customize the output by subclassing the method 'is_a_list'. + you can customize the output by subclassing the method 'is_a_list'.""" - """ - # TODO: writing tests for PUT files + normal data media_type = 'application/x-www-form-urlencoded' """The value of the parameter when the select multiple is empty. @@ -159,8 +156,8 @@ class MultipartParser(BaseParser, DataFlatener): data, files = django_mpp.parse() # Flatening data, files and combining them - data = self.flatten_data(data) - files = self.flatten_data(files) + data = self.flatten_data(dict(data.iterlists())) + files = self.flatten_data(dict(files.iterlists())) data.update(files) # Strip any parameters that we are treating as reserved -- cgit v1.2.3