From ea71fe5731df00474eb47ee777635ac7d093fe0f Mon Sep 17 00:00:00 2001 From: Fedele Mantuano Date: Thu, 1 Oct 2026 09:13:18 +0200 Subject: [PATCH] fix: restore RFC 6532 internationalized mailbox parsing The forensic scanner introduced for issue #175 restricted atoms to ASCII, dropping valid UTF-8 local parts and domains in 4.7.0. Extend atoms to Unicode scalar values and preserve mailbox identity across byte decoding and whitespace handling. Retain invalid UTF-8 as explicit defect evidence, including group labels and comments, without losing valid neighbouring mailboxes. Add regression coverage for issue #177 and document the recovery behavior. Reviewed RFC 5322 sections 3.2-3.4, 4 and 5; RFC 6532 sections 3.1-3.2 and 4; RFC 3629 scalar restrictions; RFC 6854 and applicable errata. Verified conforming UTF-8, obsolete routes, malformed inputs and recovery metadata. The pre-existing isolated-surrogate failure in parse_from_string remains outside this change. Validation: 951 tests pass on Python 3.13 and 3.14; pre-commit and Pyright pass. Independent security review found no remaining introduced High/Medium findings after malformed-byte fuzzing and bounded scaling probes. --- README.md | 15 +++ src/mailparser/addresses.py | 52 +++++--- src/mailparser/utils.py | 17 ++- tests/test_international_addresses.py | 177 ++++++++++++++++++++++++++ 4 files changed, 243 insertions(+), 18 deletions(-) create mode 100644 tests/test_international_addresses.py diff --git a/README.md b/README.md index 26324f6..7b847af 100644 --- a/README.md +++ b/README.md @@ -289,6 +289,21 @@ RFC 5322 permits whitespace around the mailbox inside `<...>`; the unquoted unambiguous angle-address takes precedence over its display name. Addresses inside comments or quoted display names are never mailbox candidates. +Internationalized mailbox local parts and domains are accepted under +[RFC 6532 §3.2](https://www.rfc-editor.org/rfc/rfc6532.html#section-3.2): +`José `, `user@exämple.com`, and `山田 ` +parse without address defects. UTF-8 atoms are supported alongside quoted +local parts, comments, groups, and obsolete source routes. Unicode alone is +not an anomaly; ASCII delimiters and ambiguity checks still apply. + +Mailbox code points are preserved: parsing does not apply NFC/NFKC, IDNA +conversion, or trim non-ASCII characters that resemble whitespace. This +preserves forensic identity rather than silently equating different inputs. +Malformed UTF-8 does not become a different valid address by dropping bytes. +It produces `invalid-utf8` evidence; undecodable bytes appear in diagnostic +`raw` as escaped surrogate code points (for example `\udcff` represents +byte `FF`). Valid neighbouring mailboxes remain available. + `address_header_defects` exposes recovery and ambiguity evidence. Each entry contains `header`, zero-based `occurrence`, the original `raw` list item, `reason`, `recovered`, and `candidates` (display-name/address dictionaries). diff --git a/src/mailparser/addresses.py b/src/mailparser/addresses.py index eaf74a5..760b4ed 100644 --- a/src/mailparser/addresses.py +++ b/src/mailparser/addresses.py @@ -1,4 +1,4 @@ -"""Structural address recovery for untrusted RFC 5322 header values. +"""Structural address recovery for RFC 5322 / RFC 6532 header values. The scanner owns lexical boundaries and checks complete addr-specs against bounded grammar patterns. The stdlib supplies conventional display names for @@ -15,7 +15,12 @@ # Anchored grammar checks: atom, quote and delimiter character sets do not # overlap. In particular, do not search for bare addresses in arbitrary text # or hand long malformed word sequences to headerregistry's quadratic parser. -_ATOM = r"[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+" +# RFC 6532 section 3.2 extends atext with UTF8-non-ascii. These are +# Unicode scalar values (RFC 3629), not \w or all code points >= 128: +# surrogates cannot occur in valid UTF-8 and must remain defect evidence. +_UTF8_NON_ASCII = r"\x80-\ud7ff\ue000-\U0010ffff" +_SURROGATE_RE = re.compile(r"[\ud800-\udfff]") +_ATOM = rf"[a-zA-Z0-9!#$%&'*+/=?^_`{{|}}~{_UTF8_NON_ASCII}-]+" _QUOTED = r'"(?:[^"\\\r\n]|\\[^\r\n])*"' _WORD = rf"(?:{_ATOM}|{_QUOTED})" _LOCAL = rf"{_WORD}(?:[ \t]*\.[ \t]*{_WORD})*" @@ -47,8 +52,9 @@ def _scan(raw: str) -> list[_Item]: def emit(end): nonlocal buf, angles, problems, start, max_comment text = "".join(buf) - if text.strip() or problems: - items.append(_Item(raw[start:end], text, angles, problems, max_comment)) + raw_item = raw[start:end] + if text.strip(" \t\r\n") or problems or _SURROGATE_RE.search(raw_item): + items.append(_Item(raw_item, text, angles, problems, max_comment)) buf, angles, problems = [], [], [] start = end + 1 max_comment = 0 @@ -113,7 +119,9 @@ def emit(end): else: group = True # Preserve malformed labels as diagnostics, never mailboxes. - if not _valid_phrase("".join(buf)): + if not _valid_phrase("".join(buf)) or _SURROGATE_RE.search( + raw[start:i] + ): problems.append("invalid-group-name") emit(i) buf = [] @@ -138,7 +146,9 @@ def emit(end): def _mailbox(value: str) -> str | None: """Validate the whole addr-spec, including quoted local parts and CFWS.""" - value = _FOLD.sub(" ", value).strip() + value = _FOLD.sub(" ", value).strip(" \t\r\n") + if _SURROGATE_RE.search(value): + return None # RFC 5322 section 4.4: obsolete source routes precede the addr-spec. if value.startswith(("@", ",")): literal = escaped = False @@ -154,7 +164,7 @@ def _mailbox(value: str) -> str | None: elif ch == "]": literal = False elif not literal and ch in ",:": - domain = value[start:i].strip() + domain = value[start:i].strip(" \t\r\n") if domain: domains.append(domain) start = i + 1 @@ -167,7 +177,7 @@ def _mailbox(value: str) -> str | None: for domain in domains ): return None - value = value[start:].strip() + value = value[start:].strip(" \t\r\n") if not _ADDR_SPEC.fullmatch(value): return None # CFWS is not part of an atom. Retain spaces and escapes inside quoted @@ -193,7 +203,7 @@ def _mailbox(value: str) -> str | None: def _valid_phrase(value: str) -> bool: """Check label specials outside quoted strings (period is obs-phrase).""" quoted = escaped = False - for ch in _FOLD.sub(" ", value).strip(): + for ch in _FOLD.sub(" ", value).strip(" \t\r\n"): if ord(ch) < 32 and ch != "\t": return False if escaped: @@ -204,14 +214,14 @@ def _valid_phrase(value: str) -> bool: quoted = not quoted elif not quoted and ch in "@<>[]:;,\\": return False - return bool(value.strip()) and not (quoted or escaped) + return bool(value.strip(" \t\r\n")) and not (quoted or escaped) def _name(value: str) -> str: """Recover a display phrase, unquoting quoted pairs without regexes.""" result = [] quoted = escaped = False - for ch in _FOLD.sub(" ", value).strip(): + for ch in _FOLD.sub(" ", value).strip(" \t\r\n"): if escaped: result.append(ch) escaped = False @@ -221,7 +231,12 @@ def _name(value: str) -> str: quoted = not quoted else: result.append(ch) - return "".join(result).strip() + return _escape_surrogates("".join(result).strip(" \t\r\n")) + + +def _escape_surrogates(value: str) -> str: + """Keep invalid code points visible without emitting invalid UTF-8 JSON.""" + return value.encode("utf-8", "backslashreplace").decode("utf-8") def parse_address_header(raw: str, strict_parser): @@ -248,13 +263,15 @@ def parse_address_header(raw: str, strict_parser): reason = ",".join(sorted(set(item.problems))) elif item.angles: left, right = item.angles[0] - if len(item.angles) != 1 or item.text[right + 1 :].strip(): + if len(item.angles) != 1 or item.text[right + 1 :].strip(" \t\r\n"): reason = "ambiguous-angle-address" elif not candidates: reason = "invalid-angle-address" else: selected = candidates - if item.text[:left].strip() and not _valid_phrase(item.text[:left]): + if item.text[:left].strip(" \t\r\n") and not _valid_phrase( + item.text[:left] + ): reason = "invalid-display-name" else: address = _mailbox(item.text) @@ -264,7 +281,10 @@ def parse_address_header(raw: str, strict_parser): else: reason = "invalid-or-ambiguous-address" - if selected: + invalid_utf8 = bool(_SURROGATE_RE.search(item.raw)) + if invalid_utf8: + reason = "invalid-utf8" + if selected and not invalid_utf8: # Deep comments are stripped by the iterative scanner before # reaching the stdlib's recursive comment parser. source = item.raw if item.comment_depth < 16 else item.text @@ -280,7 +300,7 @@ def parse_address_header(raw: str, strict_parser): diagnostics.append( { "reason": reason, - "raw": item.raw, + "raw": _escape_surrogates(item.raw), "recovered": bool(selected), "candidates": [ {"display_name": name, "address": addr} diff --git a/src/mailparser/utils.py b/src/mailparser/utils.py index fa74d76..da9fb0d 100644 --- a/src/mailparser/utils.py +++ b/src/mailparser/utils.py @@ -115,10 +115,23 @@ def get_addresses( if raw_header is None: return [] if isinstance(raw_header, email.header.Header): - raw_header = decode_header_part(raw_header.encode()) + # Address identity must not pass through ported_string's NFC + # normalization, decode errors="ignore", or Unicode whitespace + # stripping. Keep invalid octets as surrogates for the scanner to + # flag, rather than joining their neighbours into a new mailbox. + chunks = [] + for data, charset in decode_header(raw_header.encode()): + if isinstance(data, bytes): + charset = charset if charset != "unknown-8bit" else "utf-8" + try: + data = data.decode(charset or "utf-8", "surrogateescape") + except (LookupError, UnicodeError): + data = data.decode("utf-8", "surrogateescape") + chunks.append(data) + raw_header = "".join(chunks) elif not isinstance(raw_header, str): raw_header = str(raw_header) - if not raw_header.strip(): + if not raw_header.strip(" \t\r\n"): return _getaddresses([raw_header]) parsed, diagnostics = parse_address_header(raw_header, _getaddresses) if defects is not None: diff --git a/tests/test_international_addresses.py b/tests/test_international_addresses.py new file mode 100644 index 0000000..46579da --- /dev/null +++ b/tests/test_international_addresses.py @@ -0,0 +1,177 @@ +"""RFC 6532 mailbox regression and forensic boundaries (issue #177).""" + +import json + +import pytest + +import mailparser +from mailparser.const import ADDRESSES_HEADERS + + +@pytest.mark.parametrize("as_bytes", [False, True]) +@pytest.mark.parametrize("header", sorted(ADDRESSES_HEADERS)) +@pytest.mark.parametrize( + "value,expected", + [ + ("José ", [("José", "josé@example.com")]), + ("user@exämple.com", [("", "user@exämple.com")]), + ("山田 ", [("山田", "yamada@例え.jp")]), + ( + "josé@example.com, bob@example.com", + [("", "josé@example.com"), ("", "bob@example.com")], + ), + ('"josé"@example.com', [("", '"josé"@example.com')]), + ("用户@例え.jp", [("", "用户@例え.jp")]), + ("😀@例え.jp", [("", "😀@例え.jp")]), + ("e\u0301@example.com", [("", "e\u0301@example.com")]), + ("\u00a0user@example.com", [("", "\u00a0user@example.com")]), + ("user@example.com\u00a0", [("", "user@example.com\u00a0")]), + ("J. Doe <@例え.jp:josé@example.com>", [("J. Doe", "josé@example.com")]), + ( + "チーム: josé@example.com, bob@example.com;", + [("", "josé@example.com"), ("", "bob@example.com")], + ), + ], +) +def test_international_mailboxes_are_not_defects(header, value, expected, as_bytes): + raw = f"{header}: {value}\r\nSubject: s\r\n\r\nb\r\n" + mail = ( + mailparser.parse_from_bytes(raw.encode()) + if as_bytes + else mailparser.parse_from_string(raw) + ) + assert mail.headers[header] == expected + assert mail.mail[header] == expected + assert json.loads(mail.mail_json)[header] == [list(pair) for pair in expected] + assert mail.address_header_defects == [] + assert not mail.has_defects + + +@pytest.mark.parametrize("as_bytes", [False, True]) +def test_unicode_spoof_label_keeps_defect_and_structural_mailbox(as_bytes): + value = "josé@trusted.example < 用户@例え.jp >" + raw = f"From: {value}\r\n\r\n" + mail = ( + mailparser.parse_from_bytes(raw.encode()) + if as_bytes + else mailparser.parse_from_string(raw) + ) + assert mail.from_ == [("josé@trusted.example", "用户@例え.jp")] + assert mail.has_defects + assert mail.address_header_defects == [ + { + "reason": "invalid-display-name", + "raw": value, + "recovered": True, + "candidates": [ + {"display_name": "josé@trusted.example", "address": "用户@例え.jp"} + ], + "header": "from", + "occurrence": 0, + } + ] + + +def test_ambiguous_unicode_candidates_and_valid_neighbour_survive(): + value = "Label <用户@例え.jp>" + mail = mailparser.parse_from_bytes(f"To: {value}, bob@example.com\r\n\r\n".encode()) + assert mail.to == [("", "bob@example.com")] + assert mail.has_defects + defect = mail.address_header_defects[0] + assert defect["raw"] == value + assert defect["recovered"] is False + assert [c["address"] for c in defect["candidates"]] == [ + "josé@example.com", + "用户@例え.jp", + ] + + +@pytest.mark.parametrize( + "value", + [ + "josé..x@example.com", + ".josé@example.com", + "josé@example..com", + "josé@example.com@other.example", + "josé@example.com>", + "José ", + ], +) +def test_invalid_utf8_in_labels_and_comments_remains_evidence(value): + mail = mailparser.parse_from_bytes(b"To: " + value + b"\r\n\r\n") + assert [address for _, address in mail.to] == ["bob@example.com"] + assert mail.has_defects + defect = mail.address_header_defects[0] + assert defect["reason"] == "invalid-utf8" + assert "\\udcff" in defect["raw"] + assert defect["header"] == "to" + assert defect["occurrence"] == 0 + mail.mail_json.encode("utf-8") + + +@pytest.mark.parametrize("char", ["\x80", "\ud7ff", "\ue000", "\U0010ffff"]) +def test_utf8_scalar_range_boundaries(char): + mail = mailparser.parse_from_bytes( + f"To: {char}@example.com, user@{char}.example\r\n\r\n".encode() + ) + assert mail.to == [("", f"{char}@example.com"), ("", f"user@{char}.example")] + assert not mail.has_defects + + +@pytest.mark.parametrize( + "label", + [ + '"Support "', + "Support (josé@trusted.example )", + ], +) +def test_international_display_evidence_is_not_promoted(label): + value = f"{label} <用户@例え.jp>, bad@label < réel@example.com >" + mail = mailparser.parse_from_bytes(f"To: {value}\r\n\r\n".encode()) + assert [addr for _, addr in mail.to] == ["用户@例え.jp", "réel@example.com"] + assert mail.has_defects + assert mail.address_header_defects[-1]["reason"] == "invalid-display-name" + + +def test_unicode_text_after_angle_address_is_not_discarded_as_cfws(): + value = "Name \u00a0" + mail = mailparser.parse_from_bytes(f"From: {value}\r\n\r\n".encode()) + assert mail.from_ == [] + assert mail.has_defects + defect = mail.address_header_defects[0] + assert defect["raw"] == value + assert defect["reason"] == "ambiguous-angle-address" + assert defect["candidates"][0]["address"] == "josé@example.com"