From e0c5700374a97f10a7fe14483a209eb9c1d75283 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 29 Apr 2021 10:57:31 -0700 Subject: [PATCH] [PATCH] [3.9] bpo-43882 - urllib.parse should sanitize urls containing ASCII newline and tabs. (GH-25595) (GH-25725) * bpo-43882 - urllib.parse should sanitize urls containing ASCII newline and tabs. (GH-25595) Co-authored-by: Gregory P. Smith Co-authored-by: Serhiy Storchaka (cherry picked from commit 76cd81d60310d65d01f9d7b48a8985d8ab89c8b4) Co-authored-by: Senthil Kumaran Gbp-Pq: Name CVE-2022-0391-1.patch --- Doc/library/urllib.parse.rst | 13 +++++++++++++ Lib/test/test_urlparse.py | 29 +++++++++++++++++++++++++++++ Lib/urllib/parse.py | 7 +++++++ 3 files changed, 49 insertions(+) diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst index 38e2986..c037652 100644 --- a/Doc/library/urllib.parse.rst +++ b/Doc/library/urllib.parse.rst @@ -311,6 +311,9 @@ or on combining URL components into a URL string. ``#``, ``@``, or ``:`` will raise a :exc:`ValueError`. If the URL is decomposed before parsing, no error will be raised. + Following the `WHATWG spec`_ that updates RFC 3986, ASCII newline + ``\n``, ``\r`` and tab ``\t`` characters are stripped from the URL. + .. versionchanged:: 3.6 Out-of-range port numbers now raise :exc:`ValueError`, instead of returning :const:`None`. @@ -319,6 +322,10 @@ or on combining URL components into a URL string. Characters that affect netloc parsing under NFKC normalization will now raise :exc:`ValueError`. + .. versionchanged:: 3.9.5 + ASCII newline and tab characters are stripped from the URL. + +.. _WHATWG spec: https://url.spec.whatwg.org/#concept-basic-url-parser .. function:: urlunsplit(parts) @@ -673,6 +680,10 @@ task isn't already covered by the URL parsing functions above. .. seealso:: + `WHATWG`_ - URL Living standard + Working Group for the URL Standard that defines URLs, domains, IP addresses, the + application/x-www-form-urlencoded format, and their API. + :rfc:`3986` - Uniform Resource Identifiers This is the current standard (STD66). Any changes to urllib.parse module should conform to this. Certain deviations could be observed, which are @@ -696,3 +707,5 @@ task isn't already covered by the URL parsing functions above. :rfc:`1738` - Uniform Resource Locators (URL) This specifies the formal syntax and semantics of absolute URLs. + +.. _WHATWG: https://url.spec.whatwg.org/ diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index 50a9d97..306f687 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -671,6 +671,35 @@ class UrlParseTestCase(unittest.TestCase): self.assertEqual(p.scheme, "https") self.assertEqual(p.geturl(), "https://www.python.org/") + def test_urlsplit_remove_unsafe_bytes(self): + # Remove ASCII tabs and newlines from input + url = "http://www.python.org/java\nscript:\talert('msg\r\n')/#frag" + p = urllib.parse.urlsplit(url) + self.assertEqual(p.scheme, "http") + self.assertEqual(p.netloc, "www.python.org") + self.assertEqual(p.path, "/javascript:alert('msg')/") + self.assertEqual(p.query, "") + self.assertEqual(p.fragment, "frag") + self.assertEqual(p.username, None) + self.assertEqual(p.password, None) + self.assertEqual(p.hostname, "www.python.org") + self.assertEqual(p.port, None) + self.assertEqual(p.geturl(), "http://www.python.org/javascript:alert('msg')/#frag") + + # Remove ASCII tabs and newlines from input as bytes. + url = b"http://www.python.org/java\nscript:\talert('msg\r\n')/#frag" + p = urllib.parse.urlsplit(url) + self.assertEqual(p.scheme, b"http") + self.assertEqual(p.netloc, b"www.python.org") + self.assertEqual(p.path, b"/javascript:alert('msg')/") + self.assertEqual(p.query, b"") + self.assertEqual(p.fragment, b"frag") + self.assertEqual(p.username, None) + self.assertEqual(p.password, None) + self.assertEqual(p.hostname, b"www.python.org") + self.assertEqual(p.port, None) + self.assertEqual(p.geturl(), b"http://www.python.org/javascript:alert('msg')/#frag") + def test_attributes_bad_port(self): """Check handling of invalid ports.""" for bytes in (False, True): diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index c7048d1..e403e98 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -87,6 +87,9 @@ scheme_chars = ('abcdefghijklmnopqrstuvwxyz' # == "".join([chr(i) for i in range(0, 0x20 + 1)]) _WHATWG_C0_CONTROL_OR_SPACE = '\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f ' +# Unsafe bytes to be removed per WHATWG spec +_UNSAFE_URL_BYTES_TO_REMOVE = ['\t', '\r', '\n'] + # XXX: Consider replacing with functools.lru_cache MAX_CACHE_SIZE = 20 _parse_cache = {} @@ -510,6 +513,10 @@ def urlsplit(url, scheme='', allow_fragments=True): break else: scheme, url = url[:i].lower(), url[i+1:] + + for b in _UNSAFE_URL_BYTES_TO_REMOVE: + url = url.replace(b, "") + if url[:2] == '//': netloc, url = _splitnetloc(url, 2) if (('[' in netloc and ']' not in netloc) or -- 2.30.2