fix: restore RFC 6532 internationalized mailbox parsing

README.md

@@ -289,6 +289,21 @@ RFC 5322 permits whitespace around the mailbox inside `<...>`; the unquoted
 unambiguous angle-address takes precedence over its display name. Addresses
 inside comments or quoted display names are never mailbox candidates.
 
+Internationalized mailbox local parts and domains are accepted under
+[RFC 6532 §3.2](https://www.rfc-editor.org/rfc/rfc6532.html#section-3.2):
+`José <josé@example.com>`, `user@exämple.com`, and `山田 <yamada@例え.jp>`
+parse without address defects. UTF-8 atoms are supported alongside quoted
+local parts, comments, groups, and obsolete source routes. Unicode alone is
+not an anomaly; ASCII delimiters and ambiguity checks still apply.
+
+Mailbox code points are preserved: parsing does not apply NFC/NFKC, IDNA
+conversion, or trim non-ASCII characters that resemble whitespace. This
+preserves forensic identity rather than silently equating different inputs.
+Malformed UTF-8 does not become a different valid address by dropping bytes.
+It produces `invalid-utf8` evidence; undecodable bytes appear in diagnostic
+`raw` as escaped surrogate code points (for example `\udcff` represents
+byte `FF`). Valid neighbouring mailboxes remain available.
+
 `address_header_defects` exposes recovery and ambiguity evidence. Each entry
 contains `header`, zero-based `occurrence`, the original `raw` list item,
 `reason`, `recovered`, and `candidates` (display-name/address dictionaries).

src/mailparser/addresses.py

@@ -1,4 +1,4 @@
-"""Structural address recovery for untrusted RFC 5322 header values.
+"""Structural address recovery for RFC 5322 / RFC 6532 header values.
 
 The scanner owns lexical boundaries and checks complete addr-specs against
 bounded grammar patterns. The stdlib supplies conventional display names for
@@ -15,7 +15,12 @@
 # Anchored grammar checks: atom, quote and delimiter character sets do not
 # overlap. In particular, do not search for bare addresses in arbitrary text
 # or hand long malformed word sequences to headerregistry's quadratic parser.
-_ATOM = r"[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+"
+# RFC 6532 section 3.2 extends atext with UTF8-non-ascii. These are
+# Unicode scalar values (RFC 3629), not \w or all code points >= 128:
+# surrogates cannot occur in valid UTF-8 and must remain defect evidence.
+_UTF8_NON_ASCII = r"\x80-\ud7ff\ue000-\U0010ffff"
+_SURROGATE_RE = re.compile(r"[\ud800-\udfff]")
+_ATOM = rf"[a-zA-Z0-9!#$%&'*+/=?^_`{{|}}~{_UTF8_NON_ASCII}-]+"
 _QUOTED = r'"(?:[^"\\\r\n]|\\[^\r\n])*"'
 _WORD = rf"(?:{_ATOM}|{_QUOTED})"
 _LOCAL = rf"{_WORD}(?:[ \t]*\.[ \t]*{_WORD})*"
@@ -47,8 +52,9 @@ def _scan(raw: str) -> list[_Item]:
     def emit(end):
         nonlocal buf, angles, problems, start, max_comment
         text = "".join(buf)
-        if text.strip() or problems:
-            items.append(_Item(raw[start:end], text, angles, problems, max_comment))
+        raw_item = raw[start:end]
+        if text.strip(" \t\r\n") or problems or _SURROGATE_RE.search(raw_item):
+            items.append(_Item(raw_item, text, angles, problems, max_comment))
         buf, angles, problems = [], [], []
         start = end + 1
         max_comment = 0
@@ -113,7 +119,9 @@ def emit(end):
             else:
                 group = True
                 # Preserve malformed labels as diagnostics, never mailboxes.
-                if not _valid_phrase("".join(buf)):
+                if not _valid_phrase("".join(buf)) or _SURROGATE_RE.search(
+                    raw[start:i]
+                ):
                     problems.append("invalid-group-name")
                     emit(i)
                 buf = []
@@ -138,7 +146,9 @@ def emit(end):
 
 def _mailbox(value: str) -> str | None:
     """Validate the whole addr-spec, including quoted local parts and CFWS."""
-    value = _FOLD.sub(" ", value).strip()
+    value = _FOLD.sub(" ", value).strip(" \t\r\n")
+    if _SURROGATE_RE.search(value):
+        return None
     # RFC 5322 section 4.4: obsolete source routes precede the addr-spec.
     if value.startswith(("@", ",")):
         literal = escaped = False
@@ -154,7 +164,7 @@ def _mailbox(value: str) -> str | None:
             elif ch == "]":
                 literal = False
             elif not literal and ch in ",:":
-                domain = value[start:i].strip()
+                domain = value[start:i].strip(" \t\r\n")
                 if domain:
                     domains.append(domain)
                 start = i + 1
@@ -167,7 +177,7 @@ def _mailbox(value: str) -> str | None:
             for domain in domains
         ):
             return None
-        value = value[start:].strip()
+        value = value[start:].strip(" \t\r\n")
     if not _ADDR_SPEC.fullmatch(value):
         return None
     # CFWS is not part of an atom. Retain spaces and escapes inside quoted
@@ -193,7 +203,7 @@ def _mailbox(value: str) -> str | None:
 def _valid_phrase(value: str) -> bool:
     """Check label specials outside quoted strings (period is obs-phrase)."""
     quoted = escaped = False
-    for ch in _FOLD.sub(" ", value).strip():
+    for ch in _FOLD.sub(" ", value).strip(" \t\r\n"):
         if ord(ch) < 32 and ch != "\t":
             return False
         if escaped:
@@ -204,14 +214,14 @@ def _valid_phrase(value: str) -> bool:
             quoted = not quoted
         elif not quoted and ch in "@<>[]:;,\\":
             return False
-    return bool(value.strip()) and not (quoted or escaped)
+    return bool(value.strip(" \t\r\n")) and not (quoted or escaped)
 
 
 def _name(value: str) -> str:
     """Recover a display phrase, unquoting quoted pairs without regexes."""
     result = []
     quoted = escaped = False
-    for ch in _FOLD.sub(" ", value).strip():
+    for ch in _FOLD.sub(" ", value).strip(" \t\r\n"):
         if escaped:
             result.append(ch)
             escaped = False
@@ -221,7 +231,12 @@ def _name(value: str) -> str:
             quoted = not quoted
         else:
             result.append(ch)
-    return "".join(result).strip()
+    return _escape_surrogates("".join(result).strip(" \t\r\n"))
+
+
+def _escape_surrogates(value: str) -> str:
+    """Keep invalid code points visible without emitting invalid UTF-8 JSON."""
+    return value.encode("utf-8", "backslashreplace").decode("utf-8")
 
 
 def parse_address_header(raw: str, strict_parser):
@@ -248,13 +263,15 @@ def parse_address_header(raw: str, strict_parser):
             reason = ",".join(sorted(set(item.problems)))
         elif item.angles:
             left, right = item.angles[0]
-            if len(item.angles) != 1 or item.text[right + 1 :].strip():
+            if len(item.angles) != 1 or item.text[right + 1 :].strip(" \t\r\n"):
                 reason = "ambiguous-angle-address"
             elif not candidates:
                 reason = "invalid-angle-address"
             else:
                 selected = candidates
-                if item.text[:left].strip() and not _valid_phrase(item.text[:left]):
+                if item.text[:left].strip(" \t\r\n") and not _valid_phrase(
+                    item.text[:left]
+                ):
                     reason = "invalid-display-name"
         else:
             address = _mailbox(item.text)
@@ -264,7 +281,10 @@ def parse_address_header(raw: str, strict_parser):
             else:
                 reason = "invalid-or-ambiguous-address"
 
-        if selected:
+        invalid_utf8 = bool(_SURROGATE_RE.search(item.raw))
+        if invalid_utf8:
+            reason = "invalid-utf8"
+        if selected and not invalid_utf8:
             # Deep comments are stripped by the iterative scanner before
             # reaching the stdlib's recursive comment parser.
             source = item.raw if item.comment_depth < 16 else item.text
@@ -280,7 +300,7 @@ def parse_address_header(raw: str, strict_parser):
             diagnostics.append(
                 {
                     "reason": reason,
-                    "raw": item.raw,
+                    "raw": _escape_surrogates(item.raw),
                     "recovered": bool(selected),
                     "candidates": [
                         {"display_name": name, "address": addr}

src/mailparser/utils.py

@@ -115,10 +115,23 @@ def get_addresses(
     if raw_header is None:
         return []
     if isinstance(raw_header, email.header.Header):
-        raw_header = decode_header_part(raw_header.encode())
+        # Address identity must not pass through ported_string's NFC
+        # normalization, decode errors="ignore", or Unicode whitespace
+        # stripping. Keep invalid octets as surrogates for the scanner to
+        # flag, rather than joining their neighbours into a new mailbox.
+        chunks = []
+        for data, charset in decode_header(raw_header.encode()):
+            if isinstance(data, bytes):
+                charset = charset if charset != "unknown-8bit" else "utf-8"
+                try:
+                    data = data.decode(charset or "utf-8", "surrogateescape")
+                except (LookupError, UnicodeError):
+                    data = data.decode("utf-8", "surrogateescape")
+            chunks.append(data)
+        raw_header = "".join(chunks)
     elif not isinstance(raw_header, str):
         raw_header = str(raw_header)
-    if not raw_header.strip():
+    if not raw_header.strip(" \t\r\n"):
         return _getaddresses([raw_header])
     parsed, diagnostics = parse_address_header(raw_header, _getaddresses)
     if defects is not None:

tests/test_international_addresses.py

@@ -0,0 +1,177 @@
+"""RFC 6532 mailbox regression and forensic boundaries (issue #177)."""
+
+import json
+
+import pytest
+
+import mailparser
+from mailparser.const import ADDRESSES_HEADERS
+
+
+@pytest.mark.parametrize("as_bytes", [False, True])
+@pytest.mark.parametrize("header", sorted(ADDRESSES_HEADERS))
+@pytest.mark.parametrize(
+    "value,expected",
+    [
+        ("José <josé@example.com>", [("José", "josé@example.com")]),
+        ("user@exämple.com", [("", "user@exämple.com")]),
+        ("山田 <yamada@例え.jp>", [("山田", "yamada@例え.jp")]),
+        (
+            "josé@example.com, bob@example.com",
+            [("", "josé@example.com"), ("", "bob@example.com")],
+        ),
+        ('"josé"@example.com', [("", '"josé"@example.com')]),
+        ("用户@例え.jp", [("", "用户@例え.jp")]),
+        ("😀@例え.jp", [("", "😀@例え.jp")]),
+        ("e\u0301@example.com", [("", "e\u0301@example.com")]),
+        ("\u00a0user@example.com", [("", "\u00a0user@example.com")]),
+        ("user@example.com\u00a0", [("", "user@example.com\u00a0")]),
+        ("J. Doe <@例え.jp:josé@example.com>", [("J. Doe", "josé@example.com")]),
+        (
+            "チーム: josé@example.com, bob@example.com;",
+            [("", "josé@example.com"), ("", "bob@example.com")],
+        ),
+    ],
+)
+def test_international_mailboxes_are_not_defects(header, value, expected, as_bytes):
+    raw = f"{header}: {value}\r\nSubject: s\r\n\r\nb\r\n"
+    mail = (
+        mailparser.parse_from_bytes(raw.encode())
+        if as_bytes
+        else mailparser.parse_from_string(raw)
+    )
+    assert mail.headers[header] == expected
+    assert mail.mail[header] == expected
+    assert json.loads(mail.mail_json)[header] == [list(pair) for pair in expected]
+    assert mail.address_header_defects == []
+    assert not mail.has_defects
+
+
+@pytest.mark.parametrize("as_bytes", [False, True])
+def test_unicode_spoof_label_keeps_defect_and_structural_mailbox(as_bytes):
+    value = "josé@trusted.example < 用户@例え.jp >"
+    raw = f"From: {value}\r\n\r\n"
+    mail = (
+        mailparser.parse_from_bytes(raw.encode())
+        if as_bytes
+        else mailparser.parse_from_string(raw)
+    )
+    assert mail.from_ == [("josé@trusted.example", "用户@例え.jp")]
+    assert mail.has_defects
+    assert mail.address_header_defects == [
+        {
+            "reason": "invalid-display-name",
+            "raw": value,
+            "recovered": True,
+            "candidates": [
+                {"display_name": "josé@trusted.example", "address": "用户@例え.jp"}
+            ],
+            "header": "from",
+            "occurrence": 0,
+        }
+    ]
+
+
+def test_ambiguous_unicode_candidates_and_valid_neighbour_survive():
+    value = "Label <josé@example.com> <用户@例え.jp>"
+    mail = mailparser.parse_from_bytes(f"To: {value}, bob@example.com\r\n\r\n".encode())
+    assert mail.to == [("", "bob@example.com")]
+    assert mail.has_defects
+    defect = mail.address_header_defects[0]
+    assert defect["raw"] == value
+    assert defect["recovered"] is False
+    assert [c["address"] for c in defect["candidates"]] == [
+        "josé@example.com",
+        "用户@例え.jp",
+    ]
+
+
+@pytest.mark.parametrize(
+    "value",
+    [
+        "josé..x@example.com",
+        ".josé@example.com",
+        "josé@example..com",
+        "josé@example.com@other.example",
+        "josé@example.com>",
+        "José <josé@example.com",
+    ],
+)
+def test_utf8_does_not_relax_ascii_mailbox_boundaries(value):
+    mail = mailparser.parse_from_bytes(f"From: {value}\r\n\r\n".encode())
+    assert mail.from_ == []
+    assert mail.has_defects
+    assert mail.address_header_defects[0]["raw"] == value
+    assert mail.address_header_defects[0]["recovered"] is False
+
+
+@pytest.mark.parametrize(
+    "bad", [b"\xff", b"\xc0\xaf", b"\xed\xa0\x80", b"\xf4\x90\x80\x80"]
+)
+@pytest.mark.parametrize("template", [b"%s@b", b'"%s"@b', b"a@[%s]"])
+def test_invalid_utf8_cannot_become_a_different_valid_mailbox(bad, template):
+    raw = b"To: " + template % bad + b", bob@example.com\r\n\r\n"
+    mail = mailparser.parse_from_bytes(raw)
+    assert mail.to == [("", "bob@example.com")]
+    assert mail.has_defects
+    assert mail.address_header_defects[0]["reason"] == "invalid-utf8"
+    assert mail.address_header_defects[0]["recovered"] is False
+    assert mail.address_header_defects[0]["candidates"] == []
+    # Diagnostics must remain serializable as valid UTF-8 JSON.
+    mail.mail_json.encode("utf-8")
+
+
+@pytest.mark.parametrize(
+    "value",
+    [
+        b"\xff: bob@example.com;",
+        b"Team (\xff): bob@example.com;",
+        b"(\xff), bob@example.com",
+        b"Bad\xff <bob@example.com>",
+    ],
+)
+def test_invalid_utf8_in_labels_and_comments_remains_evidence(value):
+    mail = mailparser.parse_from_bytes(b"To: " + value + b"\r\n\r\n")
+    assert [address for _, address in mail.to] == ["bob@example.com"]
+    assert mail.has_defects
+    defect = mail.address_header_defects[0]
+    assert defect["reason"] == "invalid-utf8"
+    assert "\\udcff" in defect["raw"]
+    assert defect["header"] == "to"
+    assert defect["occurrence"] == 0
+    mail.mail_json.encode("utf-8")
+
+
+@pytest.mark.parametrize("char", ["\x80", "\ud7ff", "\ue000", "\U0010ffff"])
+def test_utf8_scalar_range_boundaries(char):
+    mail = mailparser.parse_from_bytes(
+        f"To: {char}@example.com, user@{char}.example\r\n\r\n".encode()
+    )
+    assert mail.to == [("", f"{char}@example.com"), ("", f"user@{char}.example")]
+    assert not mail.has_defects
+
+
+@pytest.mark.parametrize(
+    "label",
+    [
+        '"Support <josé@trusted.example>"',
+        "Support (josé@trusted.example <decoy@例え.jp>)",
+    ],
+)
+def test_international_display_evidence_is_not_promoted(label):
+    value = f"{label} <用户@例え.jp>, bad@label < réel@example.com >"
+    mail = mailparser.parse_from_bytes(f"To: {value}\r\n\r\n".encode())
+    assert [addr for _, addr in mail.to] == ["用户@例え.jp", "réel@example.com"]
+    assert mail.has_defects
+    assert mail.address_header_defects[-1]["reason"] == "invalid-display-name"
+
+
+def test_unicode_text_after_angle_address_is_not_discarded_as_cfws():
+    value = "Name <josé@example.com>\u00a0"
+    mail = mailparser.parse_from_bytes(f"From: {value}\r\n\r\n".encode())
+    assert mail.from_ == []
+    assert mail.has_defects
+    defect = mail.address_header_defects[0]
+    assert defect["raw"] == value
+    assert defect["reason"] == "ambiguous-angle-address"
+    assert defect["candidates"][0]["address"] == "josé@example.com"