bazarr/libs/waitress/parser.py

443 lines
15 KiB
Python

##############################################################################
#
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""HTTP Request Parser
This server uses asyncore to accept connections and do initial
processing but threads to do work.
"""
from io import BytesIO
import re
from urllib import parse
from urllib.parse import unquote_to_bytes
from waitress.buffers import OverflowableBuffer
from waitress.receiver import ChunkedReceiver, FixedStreamReceiver
from waitress.rfc7230 import HEADER_FIELD_RE, ONLY_DIGIT_RE
from waitress.utilities import (
BadRequest,
RequestEntityTooLarge,
RequestHeaderFieldsTooLarge,
ServerNotImplemented,
find_double_newline,
)
def unquote_bytes_to_wsgi(bytestring):
return unquote_to_bytes(bytestring).decode("latin-1")
class ParsingError(Exception):
pass
class TransferEncodingNotImplemented(Exception):
pass
class HTTPRequestParser:
"""A structure that collects the HTTP request.
Once the stream is completed, the instance is passed to
a server task constructor.
"""
completed = False # Set once request is completed.
empty = False # Set if no request was made.
expect_continue = False # client sent "Expect: 100-continue" header
headers_finished = False # True when headers have been read
header_plus = b""
chunked = False
content_length = 0
header_bytes_received = 0
body_bytes_received = 0
body_rcv = None
version = "1.0"
error = None
connection_close = False
# Other attributes: first_line, header, headers, command, uri, version,
# path, query, fragment
def __init__(self, adj):
"""
adj is an Adjustments object.
"""
# headers is a mapping containing keys translated to uppercase
# with dashes turned into underscores.
self.headers = {}
self.adj = adj
def received(self, data):
"""
Receives the HTTP stream for one request. Returns the number of
bytes consumed. Sets the completed flag once both the header and the
body have been received.
"""
if self.completed:
return 0 # Can't consume any more.
datalen = len(data)
br = self.body_rcv
if br is None:
# In header.
max_header = self.adj.max_request_header_size
s = self.header_plus + data
index = find_double_newline(s)
consumed = 0
if index >= 0:
# If the headers have ended, and we also have part of the body
# message in data we still want to validate we aren't going
# over our limit for received headers.
self.header_bytes_received = index
consumed = datalen - (len(s) - index)
else:
self.header_bytes_received += datalen
consumed = datalen
# If the first line + headers is over the max length, we return a
# RequestHeaderFieldsTooLarge error rather than continuing to
# attempt to parse the headers.
if self.header_bytes_received >= max_header:
self.parse_header(b"GET / HTTP/1.0\r\n")
self.error = RequestHeaderFieldsTooLarge(
"exceeds max_header of %s" % max_header
)
self.completed = True
return consumed
if index >= 0:
# Header finished.
header_plus = s[:index]
# Remove preceeding blank lines. This is suggested by
# https://tools.ietf.org/html/rfc7230#section-3.5 to support
# clients sending an extra CR LF after another request when
# using HTTP pipelining
header_plus = header_plus.lstrip()
if not header_plus:
self.empty = True
self.completed = True
else:
try:
self.parse_header(header_plus)
except ParsingError as e:
self.error = BadRequest(e.args[0])
self.completed = True
except TransferEncodingNotImplemented as e:
self.error = ServerNotImplemented(e.args[0])
self.completed = True
else:
if self.body_rcv is None:
# no content-length header and not a t-e: chunked
# request
self.completed = True
if self.content_length > 0:
max_body = self.adj.max_request_body_size
# we won't accept this request if the content-length
# is too large
if self.content_length >= max_body:
self.error = RequestEntityTooLarge(
"exceeds max_body of %s" % max_body
)
self.completed = True
self.headers_finished = True
return consumed
# Header not finished yet.
self.header_plus = s
return datalen
else:
# In body.
consumed = br.received(data)
self.body_bytes_received += consumed
max_body = self.adj.max_request_body_size
if self.body_bytes_received >= max_body:
# this will only be raised during t-e: chunked requests
self.error = RequestEntityTooLarge("exceeds max_body of %s" % max_body)
self.completed = True
elif br.error:
# garbage in chunked encoding input probably
self.error = br.error
self.completed = True
elif br.completed:
# The request (with the body) is ready to use.
self.completed = True
if self.chunked:
# We've converted the chunked transfer encoding request
# body into a normal request body, so we know its content
# length; set the header here. We already popped the
# TRANSFER_ENCODING header in parse_header, so this will
# appear to the client to be an entirely non-chunked HTTP
# request with a valid content-length.
self.headers["CONTENT_LENGTH"] = str(br.__len__())
return consumed
def parse_header(self, header_plus):
"""
Parses the header_plus block of text (the headers plus the
first line of the request).
"""
index = header_plus.find(b"\r\n")
if index >= 0:
first_line = header_plus[:index].rstrip()
header = header_plus[index + 2 :]
else:
raise ParsingError("HTTP message header invalid")
if b"\r" in first_line or b"\n" in first_line:
raise ParsingError("Bare CR or LF found in HTTP message")
self.first_line = first_line # for testing
lines = get_header_lines(header)
headers = self.headers
for line in lines:
header = HEADER_FIELD_RE.match(line)
if not header:
raise ParsingError("Invalid header")
key, value = header.group("name", "value")
if b"_" in key:
# TODO(xistence): Should we drop this request instead?
continue
# Only strip off whitespace that is considered valid whitespace by
# RFC7230, don't strip the rest
value = value.strip(b" \t")
key1 = key.upper().replace(b"-", b"_").decode("latin-1")
# If a header already exists, we append subsequent values
# separated by a comma. Applications already need to handle
# the comma separated values, as HTTP front ends might do
# the concatenation for you (behavior specified in RFC2616).
try:
headers[key1] += (b", " + value).decode("latin-1")
except KeyError:
headers[key1] = value.decode("latin-1")
# command, uri, version will be bytes
command, uri, version = crack_first_line(first_line)
# self.request_uri is like nginx's request_uri:
# "full original request URI (with arguments)"
self.request_uri = uri.decode("latin-1")
version = version.decode("latin-1")
command = command.decode("latin-1")
self.command = command
self.version = version
(
self.proxy_scheme,
self.proxy_netloc,
self.path,
self.query,
self.fragment,
) = split_uri(uri)
self.url_scheme = self.adj.url_scheme
connection = headers.get("CONNECTION", "")
if version == "1.0":
if connection.lower() != "keep-alive":
self.connection_close = True
if version == "1.1":
# since the server buffers data from chunked transfers and clients
# never need to deal with chunked requests, downstream clients
# should not see the HTTP_TRANSFER_ENCODING header; we pop it
# here
te = headers.pop("TRANSFER_ENCODING", "")
# NB: We can not just call bare strip() here because it will also
# remove other non-printable characters that we explicitly do not
# want removed so that if someone attempts to smuggle a request
# with these characters we don't fall prey to it.
#
# For example \x85 is stripped by default, but it is not considered
# valid whitespace to be stripped by RFC7230.
encodings = [
encoding.strip(" \t").lower() for encoding in te.split(",") if encoding
]
for encoding in encodings:
# Out of the transfer-codings listed in
# https://tools.ietf.org/html/rfc7230#section-4 we only support
# chunked at this time.
# Note: the identity transfer-coding was removed in RFC7230:
# https://tools.ietf.org/html/rfc7230#appendix-A.2 and is thus
# not supported
if encoding not in {"chunked"}:
raise TransferEncodingNotImplemented(
"Transfer-Encoding requested is not supported."
)
if encodings and encodings[-1] == "chunked":
self.chunked = True
buf = OverflowableBuffer(self.adj.inbuf_overflow)
self.body_rcv = ChunkedReceiver(buf)
elif encodings: # pragma: nocover
raise TransferEncodingNotImplemented(
"Transfer-Encoding requested is not supported."
)
expect = headers.get("EXPECT", "").lower()
self.expect_continue = expect == "100-continue"
if connection.lower() == "close":
self.connection_close = True
if not self.chunked:
cl = headers.get("CONTENT_LENGTH", "0")
if not ONLY_DIGIT_RE.match(cl.encode("latin-1")):
raise ParsingError("Content-Length is invalid")
cl = int(cl)
self.content_length = cl
if cl > 0:
buf = OverflowableBuffer(self.adj.inbuf_overflow)
self.body_rcv = FixedStreamReceiver(cl, buf)
def get_body_stream(self):
body_rcv = self.body_rcv
if body_rcv is not None:
return body_rcv.getfile()
else:
return BytesIO()
def close(self):
body_rcv = self.body_rcv
if body_rcv is not None:
body_rcv.getbuf().close()
def split_uri(uri):
# urlsplit handles byte input by returning bytes on py3, so
# scheme, netloc, path, query, and fragment are bytes
scheme = netloc = path = query = fragment = b""
# urlsplit below will treat this as a scheme-less netloc, thereby losing
# the original intent of the request. Here we shamelessly stole 4 lines of
# code from the CPython stdlib to parse out the fragment and query but
# leave the path alone. See
# https://github.com/python/cpython/blob/8c9e9b0cd5b24dfbf1424d1f253d02de80e8f5ef/Lib/urllib/parse.py#L465-L468
# and https://github.com/Pylons/waitress/issues/260
if uri[:2] == b"//":
path = uri
if b"#" in path:
path, fragment = path.split(b"#", 1)
if b"?" in path:
path, query = path.split(b"?", 1)
else:
try:
scheme, netloc, path, query, fragment = parse.urlsplit(uri)
except UnicodeError:
raise ParsingError("Bad URI")
return (
scheme.decode("latin-1"),
netloc.decode("latin-1"),
unquote_bytes_to_wsgi(path),
query.decode("latin-1"),
fragment.decode("latin-1"),
)
def get_header_lines(header):
"""
Splits the header into lines, putting multi-line headers together.
"""
r = []
lines = header.split(b"\r\n")
for line in lines:
if not line:
continue
if b"\r" in line or b"\n" in line:
raise ParsingError(
'Bare CR or LF found in header line "%s"' % str(line, "latin-1")
)
if line.startswith((b" ", b"\t")):
if not r:
# https://corte.si/posts/code/pathod/pythonservers/index.html
raise ParsingError('Malformed header line "%s"' % str(line, "latin-1"))
r[-1] += line
else:
r.append(line)
return r
first_line_re = re.compile(
b"([^ ]+) "
b"((?:[^ :?#]+://[^ ?#/]*(?:[0-9]{1,5})?)?[^ ]+)"
b"(( HTTP/([0-9.]+))$|$)"
)
def crack_first_line(line):
m = first_line_re.match(line)
if m is not None and m.end() == len(line):
if m.group(3):
version = m.group(5)
else:
version = b""
method = m.group(1)
# the request methods that are currently defined are all uppercase:
# https://www.iana.org/assignments/http-methods/http-methods.xhtml and
# the request method is case sensitive according to
# https://tools.ietf.org/html/rfc7231#section-4.1
# By disallowing anything but uppercase methods we save poor
# unsuspecting souls from sending lowercase HTTP methods to waitress
# and having the request complete, while servers like nginx drop the
# request onto the floor.
if method != method.upper():
raise ParsingError('Malformed HTTP method "%s"' % str(method, "latin-1"))
uri = m.group(2)
return method, uri, version
else:
return b"", b"", b""