summaryrefslogtreecommitdiffhomepage
path: root/libs/waitress/parser.py
blob: 28ef5eaac2aefb4838388dcf7920b42cfeb6a9ff (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""HTTP Request Parser

This server uses asyncore to accept connections and do initial
processing but threads to do work.
"""
from io import BytesIO
import re
from urllib import parse
from urllib.parse import unquote_to_bytes

from waitress.buffers import OverflowableBuffer
from waitress.receiver import ChunkedReceiver, FixedStreamReceiver
from waitress.rfc7230 import HEADER_FIELD_RE, ONLY_DIGIT_RE
from waitress.utilities import (
    BadRequest,
    RequestEntityTooLarge,
    RequestHeaderFieldsTooLarge,
    ServerNotImplemented,
    find_double_newline,
)


def unquote_bytes_to_wsgi(bytestring):
    return unquote_to_bytes(bytestring).decode("latin-1")


class ParsingError(Exception):
    pass


class TransferEncodingNotImplemented(Exception):
    pass


class HTTPRequestParser:
    """A structure that collects the HTTP request.

    Once the stream is completed, the instance is passed to
    a server task constructor.
    """

    completed = False  # Set once request is completed.
    empty = False  # Set if no request was made.
    expect_continue = False  # client sent "Expect: 100-continue" header
    headers_finished = False  # True when headers have been read
    header_plus = b""
    chunked = False
    content_length = 0
    header_bytes_received = 0
    body_bytes_received = 0
    body_rcv = None
    version = "1.0"
    error = None
    connection_close = False

    # Other attributes: first_line, header, headers, command, uri, version,
    # path, query, fragment

    def __init__(self, adj):
        """
        adj is an Adjustments object.
        """
        # headers is a mapping containing keys translated to uppercase
        # with dashes turned into underscores.
        self.headers = {}
        self.adj = adj

    def received(self, data):
        """
        Receives the HTTP stream for one request.  Returns the number of
        bytes consumed.  Sets the completed flag once both the header and the
        body have been received.
        """

        if self.completed:
            return 0  # Can't consume any more.

        datalen = len(data)
        br = self.body_rcv

        if br is None:
            # In header.
            max_header = self.adj.max_request_header_size

            s = self.header_plus + data
            index = find_double_newline(s)
            consumed = 0

            if index >= 0:
                # If the headers have ended, and we also have part of the body
                # message in data we still want to validate we aren't going
                # over our limit for received headers.
                self.header_bytes_received = index
                consumed = datalen - (len(s) - index)
            else:
                self.header_bytes_received += datalen
                consumed = datalen

            # If the first line + headers is over the max length, we return a
            # RequestHeaderFieldsTooLarge error rather than continuing to
            # attempt to parse the headers.

            if self.header_bytes_received >= max_header:
                self.parse_header(b"GET / HTTP/1.0\r\n")
                self.error = RequestHeaderFieldsTooLarge(
                    "exceeds max_header of %s" % max_header
                )
                self.completed = True

                return consumed

            if index >= 0:
                # Header finished.
                header_plus = s[:index]

                # Remove preceding blank lines. This is suggested by
                # https://tools.ietf.org/html/rfc7230#section-3.5 to support
                # clients sending an extra CR LF after another request when
                # using HTTP pipelining
                header_plus = header_plus.lstrip()

                if not header_plus:
                    self.empty = True
                    self.completed = True
                else:
                    try:
                        self.parse_header(header_plus)
                    except ParsingError as e:
                        self.error = BadRequest(e.args[0])
                        self.completed = True
                    except TransferEncodingNotImplemented as e:
                        self.error = ServerNotImplemented(e.args[0])
                        self.completed = True
                    else:
                        if self.body_rcv is None:
                            # no content-length header and not a t-e: chunked
                            # request
                            self.completed = True

                        if self.content_length > 0:
                            max_body = self.adj.max_request_body_size
                            # we won't accept this request if the content-length
                            # is too large

                            if self.content_length >= max_body:
                                self.error = RequestEntityTooLarge(
                                    "exceeds max_body of %s" % max_body
                                )
                                self.completed = True
                self.headers_finished = True

                return consumed

            # Header not finished yet.
            self.header_plus = s

            return datalen
        else:
            # In body.
            consumed = br.received(data)
            self.body_bytes_received += consumed
            max_body = self.adj.max_request_body_size

            if self.body_bytes_received >= max_body:
                # this will only be raised during t-e: chunked requests
                self.error = RequestEntityTooLarge("exceeds max_body of %s" % max_body)
                self.completed = True
            elif br.error:
                # garbage in chunked encoding input probably
                self.error = br.error
                self.completed = True
            elif br.completed:
                # The request (with the body) is ready to use.
                self.completed = True

                if self.chunked:
                    # We've converted the chunked transfer encoding request
                    # body into a normal request body, so we know its content
                    # length; set the header here.  We already popped the
                    # TRANSFER_ENCODING header in parse_header, so this will
                    # appear to the client to be an entirely non-chunked HTTP
                    # request with a valid content-length.
                    self.headers["CONTENT_LENGTH"] = str(br.__len__())

            return consumed

    def parse_header(self, header_plus):
        """
        Parses the header_plus block of text (the headers plus the
        first line of the request).
        """
        index = header_plus.find(b"\r\n")

        if index >= 0:
            first_line = header_plus[:index].rstrip()
            header = header_plus[index + 2 :]
        else:
            raise ParsingError("HTTP message header invalid")

        if b"\r" in first_line or b"\n" in first_line:
            raise ParsingError("Bare CR or LF found in HTTP message")

        self.first_line = first_line  # for testing

        lines = get_header_lines(header)

        headers = self.headers

        for line in lines:
            header = HEADER_FIELD_RE.match(line)

            if not header:
                raise ParsingError("Invalid header")

            key, value = header.group("name", "value")

            if b"_" in key:
                # TODO(xistence): Should we drop this request instead?

                continue

            # Only strip off whitespace that is considered valid whitespace by
            # RFC7230, don't strip the rest
            value = value.strip(b" \t")
            key1 = key.upper().replace(b"-", b"_").decode("latin-1")
            # If a header already exists, we append subsequent values
            # separated by a comma. Applications already need to handle
            # the comma separated values, as HTTP front ends might do
            # the concatenation for you (behavior specified in RFC2616).
            try:
                headers[key1] += (b", " + value).decode("latin-1")
            except KeyError:
                headers[key1] = value.decode("latin-1")

        # command, uri, version will be bytes
        command, uri, version = crack_first_line(first_line)
        if command == uri == version == b"":
            raise ParsingError("Start line is invalid")

        # self.request_uri is like nginx's request_uri:
        # "full original request URI (with arguments)"
        self.request_uri = uri.decode("latin-1")
        version = version.decode("latin-1")
        command = command.decode("latin-1")
        self.command = command
        self.version = version
        (
            self.proxy_scheme,
            self.proxy_netloc,
            self.path,
            self.query,
            self.fragment,
        ) = split_uri(uri)
        self.url_scheme = self.adj.url_scheme
        connection = headers.get("CONNECTION", "")

        if version == "1.0":
            if connection.lower() != "keep-alive":
                self.connection_close = True

        if version == "1.1":
            # since the server buffers data from chunked transfers and clients
            # never need to deal with chunked requests, downstream clients
            # should not see the HTTP_TRANSFER_ENCODING header; we pop it
            # here
            te = headers.pop("TRANSFER_ENCODING", "")

            # NB: We can not just call bare strip() here because it will also
            # remove other non-printable characters that we explicitly do not
            # want removed so that if someone attempts to smuggle a request
            # with these characters we don't fall prey to it.
            #
            # For example \x85 is stripped by default, but it is not considered
            # valid whitespace to be stripped by RFC7230.
            encodings = [
                encoding.strip(" \t").lower() for encoding in te.split(",") if encoding
            ]

            for encoding in encodings:
                # Out of the transfer-codings listed in
                # https://tools.ietf.org/html/rfc7230#section-4 we only support
                # chunked at this time.

                # Note: the identity transfer-coding was removed in RFC7230:
                # https://tools.ietf.org/html/rfc7230#appendix-A.2 and is thus
                # not supported

                if encoding not in {"chunked"}:
                    raise TransferEncodingNotImplemented(
                        "Transfer-Encoding requested is not supported."
                    )

            if encodings and encodings[-1] == "chunked":
                self.chunked = True
                buf = OverflowableBuffer(self.adj.inbuf_overflow)
                self.body_rcv = ChunkedReceiver(buf)
            elif encodings:  # pragma: nocover
                raise TransferEncodingNotImplemented(
                    "Transfer-Encoding requested is not supported."
                )

            expect = headers.get("EXPECT", "").lower()
            self.expect_continue = expect == "100-continue"

            if connection.lower() == "close":
                self.connection_close = True

        if not self.chunked:
            cl = headers.get("CONTENT_LENGTH", "0")

            if not ONLY_DIGIT_RE.match(cl.encode("latin-1")):
                raise ParsingError("Content-Length is invalid")

            cl = int(cl)
            self.content_length = cl

            if cl > 0:
                buf = OverflowableBuffer(self.adj.inbuf_overflow)
                self.body_rcv = FixedStreamReceiver(cl, buf)

    def get_body_stream(self):
        body_rcv = self.body_rcv

        if body_rcv is not None:
            return body_rcv.getfile()
        else:
            return BytesIO()

    def close(self):
        body_rcv = self.body_rcv

        if body_rcv is not None:
            body_rcv.getbuf().close()


def split_uri(uri):
    # urlsplit handles byte input by returning bytes on py3, so
    # scheme, netloc, path, query, and fragment are bytes

    scheme = netloc = path = query = fragment = b""

    # urlsplit below will treat this as a scheme-less netloc, thereby losing
    # the original intent of the request. Here we shamelessly stole 4 lines of
    # code from the CPython stdlib to parse out the fragment and query but
    # leave the path alone. See
    # https://github.com/python/cpython/blob/8c9e9b0cd5b24dfbf1424d1f253d02de80e8f5ef/Lib/urllib/parse.py#L465-L468
    # and https://github.com/Pylons/waitress/issues/260

    if uri[:2] == b"//":
        path = uri

        if b"#" in path:
            path, fragment = path.split(b"#", 1)

        if b"?" in path:
            path, query = path.split(b"?", 1)
    else:
        try:
            scheme, netloc, path, query, fragment = parse.urlsplit(uri)
        except UnicodeError:
            raise ParsingError("Bad URI")

    return (
        scheme.decode("latin-1"),
        netloc.decode("latin-1"),
        unquote_bytes_to_wsgi(path),
        query.decode("latin-1"),
        fragment.decode("latin-1"),
    )


def get_header_lines(header):
    """
    Splits the header into lines, putting multi-line headers together.
    """
    r = []
    lines = header.split(b"\r\n")

    for line in lines:
        if not line:
            continue

        if b"\r" in line or b"\n" in line:
            raise ParsingError(
                'Bare CR or LF found in header line "%s"' % str(line, "latin-1")
            )

        if line.startswith((b" ", b"\t")):
            if not r:
                # https://corte.si/posts/code/pathod/pythonservers/index.html
                raise ParsingError('Malformed header line "%s"' % str(line, "latin-1"))
            r[-1] += line
        else:
            r.append(line)

    return r


first_line_re = re.compile(
    rb"(?P<method>[!#$%&'*+\-.^_`|~0-9A-Za-z]+) "
    rb"(?P<uri>(?:[^ :?#]+://[^ ?#/]*(?:[0-9]{1,5})?)?[^ ]+)"
    rb"(?: HTTP/(?P<version>[0-9]\.[0-9]))?"
)


def crack_first_line(line):
    m = first_line_re.fullmatch(line)

    if m is None:
        return b"", b"", b""

    version = m["version"] or b""
    method = m["method"]
    uri = m["uri"]

    # the request methods that are currently defined are all uppercase:
    # https://www.iana.org/assignments/http-methods/http-methods.xhtml and
    # the request method is case sensitive according to
    # https://tools.ietf.org/html/rfc7231#section-4.1

    # By disallowing anything but uppercase methods we save poor
    # unsuspecting souls from sending lowercase HTTP methods to waitress
    # and having the request complete, while servers like nginx drop the
    # request onto the floor.

    if method != method.upper():
        raise ParsingError('Malformed HTTP method "%s"' % str(method, "latin-1"))

    return method, uri, version