From 39b52f49dc57774514d736b52ea5d9fa1b8cd950 Mon Sep 17 00:00:00 2001 From: Fedele Mantuano Date: Thu, 1 Oct 2026 23:30:41 +0200 Subject: [PATCH 1/3] fix: preserve mail evidence and correct RFC parsing Preserve file octets and MIME metadata, validate date semantics, include named inline text in body views, decode display words in lexical context, and scan Received fields structurally with safe sender-IP attribution. Keep recovery diagnostics and raw evidence, including unsafe codec output. Add targeted regressions and audited corpus fingerprints. Document the intentional file-input charset compatibility change and shared review rules. RFC review: RFC 5322 sections 3.3, 3.4, 3.6.7, 4 and 5; RFC 5321 section 4.4; applicable RFC 2045/2046, 2047, 2183, 2231 and 6532 requirements. Validation: 1238 tests and 5 subtests passed on Python 3.13 and 3.14; 69 corpus cases checked across Python 3.9-3.14 without new exceptions. Pre-commit, lint and Pyright passed. Dedicated security review completed; all reproduced High/Medium findings fixed and retested. --- .claude/agents/security-reviewer.md | 8 +- AGENTS.md | 23 +- README.md | 69 +++ src/mailparser/addresses.py | 117 +++- src/mailparser/const.py | 63 --- src/mailparser/core.py | 321 +++++++---- src/mailparser/dates.py | 195 +++++++ src/mailparser/received.py | 242 +++++++++ src/mailparser/utils.py | 281 +++++----- tests/fixtures/corpus_expected.json | 721 +++++++++++++++++++++++++ tests/test_corpus_regression.py | 131 +++++ tests/test_date_recovery.py | 308 +++++++++++ tests/test_encoded_word_context.py | 224 ++++++++ tests/test_inline_body.py | 333 ++++++++++++ tests/test_input_octet_preservation.py | 230 ++++++++ tests/test_mail_parser.py | 33 +- tests/test_mime_unicode_metadata.py | 166 ++++++ tests/test_received_lexical.py | 337 ++++++++++++ 18 files changed, 3498 insertions(+), 304 deletions(-) create mode 100644 src/mailparser/dates.py create mode 100644 src/mailparser/received.py create mode 100644 tests/fixtures/corpus_expected.json create mode 100644 tests/test_corpus_regression.py create mode 100644 tests/test_date_recovery.py create mode 100644 tests/test_encoded_word_context.py create mode 100644 tests/test_inline_body.py create mode 100644 tests/test_input_octet_preservation.py create mode 100644 tests/test_mime_unicode_metadata.py create mode 100644 tests/test_received_lexical.py diff --git a/.claude/agents/security-reviewer.md b/.claude/agents/security-reviewer.md index 9082390..72dbd2a 100644 --- a/.claude/agents/security-reviewer.md +++ b/.claude/agents/security-reviewer.md @@ -134,8 +134,12 @@ delimiter, check the search is anchored: loop then falls through to older, fully attacker-forged `Received` headers — so the failure mode is not "returns nothing", it is "returns the attacker's value". Always test the fall-through, not just the single-header case. -- Use the existing anchored `const._CLAUSE_SPLITTER` rather than a bare `\bby\b` - (`\b` still matches inside `host.by.example`, since `.` is a non-word char). +- Use the shared context-aware scanner in `received.py` through + `utils.get_from_clause()`. A bare `\bby\b` still matches inside + `host.by.example`; keywords and semicolons inside comments, quoted strings, + domain literals and angle addresses are not clause boundaries. Check HELO + exclusions cover the remainder of their comment, and measure many IP groups + independently to catch repeated full-span scans. Prove these with a control matrix, not a single PoC: benign hostname, malicious hostname alone, forged header alone, and both together. If a benign hostname diff --git a/AGENTS.md b/AGENTS.md index bd4a9dd..1cee4f2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -114,6 +114,8 @@ optional Outlook dependencies are installed with `uv sync --extra outlook`). | --------------- | --------------------------------------------------------------------- | | `core.py` | `MailParser` class + `parse_from_*` factory functions | | `addresses.py` | Forensic address scanner and recovery diagnostics | +| `received.py` | Context-aware lexical scanning of SMTP trace clauses | +| `dates.py` | Date semantics, obsolete forms and occurrence-aware diagnostics | | `utils.py` | Stateless parsing helpers (address, received headers, date, encoding) | | `const.py` | Compiled regexes, header sets, clause splitter | | `exceptions.py` | Custom exception hierarchy | @@ -163,9 +165,12 @@ header occurrence while retaining first-occurrence semantics for address properties. `address_header_defects` is reserved computed metadata; literal wire headers with that name remain accessible through `headers`/`message`. -**Received header parsing**: `utils.receiveds_parsing()` tokenizes on RFC 5321 clause keywords -(`from`, `by`, `via`, `with`, `id`, `for`, `envelope-from`) using `const._CLAUSE_SPLITTER`. -Output list is ordered first-hop first. Unparseable headers fall back to `{"raw": ...}`. +**Received header parsing**: `utils.receiveds_parsing()` uses the shared scanner in +`received.py` for RFC 5321 clause keywords and the timestamp separator. Delimiters +inside comments, quoted strings, domain literals and angle addresses are not +structural separators. `get_from_clause()` uses the same scanner. Output is +ordered first-hop first. Unparseable headers retain their original `raw` value; +ambiguous syntax also produces `received_header_defects` with occurrence context. **Sender-IP attribution fails closed**: `get_server_ipaddress()` walks trust-matching `Received` headers only while a hop names a *private* IP (an internal relay); a hop naming **no** IP ends the @@ -173,21 +178,25 @@ search with `None`. Never resume the walk on that case — older `Received` head the sender, so "extraction failed" would become "returns the sender's chosen IP". Candidate addresses come only from the `from` clause (`utils.get_from_clause()`, which ends at the -next RFC 5321 keyword). Inside it, `_sender_ip_candidates()` applies one positional rule, because +next top-level RFC 5321 keyword). Inside it, `_sender_ip_candidates()` applies one positional rule, because **text cannot be classified by what it looks like** — a closed `[...]` pair the sender wrote is byte-identical to one the MTA wrote, and `EHLO [8.8.8.8]` is a form RFC 5321 §4.1.3 requires: - the **first token** is the HELO name, whatever its shape, and is never a candidate; - an explicit **HELO marker inside a comment group** (`(helo=x)`, `(account a@b HELO x)`) is - sender text and is excluded, located with `const._HELO_RE`; + located structurally by `received.helo_argument_spans()`; its argument and the remainder of that comment + are sender text and are excluded; - a candidate must sit **inside a `(`/`[` group** (`utils.group_spans()`), which is what makes a clause truncated by a multi-word HELO fail closed — what it leaves behind is bare; - IPv4 and IPv6 matches are merged **positionally**, never family-first: choosing IPv4 first let one private literal at EHLO suppress the IPv6 scan and hide the real sender. +Use ordered span lookup for IP candidates: rescanning all comment groups for +each candidate makes attribution quadratic on attacker-chosen trace fields. + Only concession: `from [ip] (helo=x)` (Exim/CommuniGate), accepted when there is no other -candidate and the marker is in a group. Do not add lookbehind guards to `_HELO_RE` to patch new -cases — three rounds of that each reopened a hole the previous one closed. +candidate and the marker is in a group. Keep marker recognition inside the structural scanner; do not reintroduce regex +lookbehind patches for each new HELO layout. **Defect detection**: During `parse()`, every MIME part is walked and `_append_defects()` records RFC violations. `EPILOGUE_DEFECTS` triggers special epilogue extraction to recover hidden payloads diff --git a/README.md b/README.md index 3b88513..b2aebf5 100644 --- a/README.md +++ b/README.md @@ -324,6 +324,64 @@ The complete original header values remain available through `from_raw`, `to_raw`, etc.; literal headers colliding with computed metadata remain in `headers` and `message.get_all(...)`. +Display names decode RFC 2047 encoded words only where the header grammar +permits them. For example, an unquoted `=?utf-8?Q?Alice?=` becomes `Alice`, +whereas the same text inside a quoted display name remains literal. Decoded +punctuation cannot introduce another mailbox, and decoding happens only once. +Recoverable encoded words longer than 75 characters remain readable and +produce an `overlong-encoded-word` address diagnostic. + +### Trace recovery + +`Received` clause boundaries and the timestamp separator respect comments, +quoted strings, domain literals and angle addresses. Ambiguous delimiters keep +the original field in `received[*]["raw"]` and produce +`received_header_defects` entries with `reason`, `raw`, `recovered`, `header` +and zero-based `occurrence`. These diagnostics set `has_defects` and the +`ReceivedHeaderDefect` category and appear in full and partial JSON output. +An ambiguous trusted hop stops sender-IP attribution; older headers cannot +supply a replacement IP. Balanced, unrecognized obsolete trace syntax remains +raw without being declared invalid solely because it is obsolete. + +### Date diagnostics + +Impossible calendar dates, clock components and numeric timezone minutes no +longer normalize silently into different timestamps. They return `None` and +set `has_defects`. A mismatched weekday retains the numerical calendar date +and records the inconsistency. `date_header_defects` covers every `Date`, +`Resent-Date` and parsed `Received` timestamp, with `reason`, `raw`, +`recovered`, selected ISO `value` (or `None`), `header` and zero-based +`occurrence`. It appears in full/partial JSON and uses `DateHeaderDefect`. + +Valid obsolete comments, short years and alphabetic zones remain supported. +Leap seconds map to the next representable Python datetime; the raw field +retains `:60`. The historical UTC representation for `-0000` and unknown +alphabetic zones does not establish the sender's local timezone. Dates outside +Python's datetime range carry `date-out-of-range` diagnostics. + +### Inline text and charset recovery + +A named `text/plain` or `text/html` part with explicit +`Content-Disposition: inline` appears in both the corresponding text collection +and `attachments`. The attachment retains its filename, metadata and payload +bytes. This additional body view is excluded for binary content, explicit +attachments and named inline parts inside an attached MIME container. HTML is +returned as text; parsing does not render it or fetch referenced resources. + +Body decoding tries the declared charset against the original bytes. When that +fails, UTF-8 recovery keeps mislabeled readable text available and records a +`CharsetDecodeDefect`, including the part index, charset and recovery outcome. +If UTF-8 also fails, the text view uses replacement characters and reports that +loss; the original bytes remain available in the MIME part and any attachment. +Recovery sets `has_defects` even when the recovered text is readable. + +Raw UTF-8 filenames and MIME metadata remain readable without changing the +original headers or payload. Invalid metadata bytes produce `MimeHeaderDefect` +with escaped raw evidence, the header occurrence and MIME part index. A charset +decoder that produces a non-Unicode scalar also triggers recovery: names retain +their literal encoded form and text retains a safe byte-derived view, keeping +JSON and CLI output encodable as UTF-8. + ## Defects and Their Critical Role in Email Security Email structural defects are not merely technical curiosities—they represent **potential security @@ -419,6 +477,17 @@ mail = mailparser.parse_from_file_obj(fp) # Parse from file object mail = mailparser.parse_from_string(raw_mail) # Parse from string ``` +File paths are read as bytes, preserving undecodable octets and original MIME +line endings just like `parse_from_bytes()`. This can expose defects that text +decoding previously hid. Declared charsets now also apply to file input: a +message mislabeled with a charset that successfully decodes its bytes can +display differently from the previous unconditional UTF-8 file read. UTF-8 +recovery applies when the declared decode fails, not by guessing which valid +interpretation the sender intended. For byte-level forensic evidence, inspect the parsed +`message` and its `raw_items()`; undecodable header octets use Python's +surrogateescape representation. The string API accepts text already decoded +by its caller. + ## Accessing Parsed Components Once parsed, access all email components through intuitive properties: diff --git a/src/mailparser/addresses.py b/src/mailparser/addresses.py index 760b4ed..8b6111f 100644 --- a/src/mailparser/addresses.py +++ b/src/mailparser/addresses.py @@ -10,8 +10,10 @@ import re from dataclasses import dataclass, field +from email.header import decode_header _FOLD = re.compile(r"\r?\n[ \t]+") +_ENCODED_WORD = re.compile(r"=\?([^?\s]+)\?([bBqQ])\?([^?\s]*)\?=") # Anchored grammar checks: atom, quote and delimiter character sets do not # overlap. In particular, do not search for bare addresses in arbitrary text # or hand long malformed word sequences to headerregistry's quadratic parser. @@ -234,25 +236,130 @@ def _name(value: str) -> str: return _escape_surrogates("".join(result).strip(" \t\r\n")) +def _decoded_word( + value: str, start: int, comment: bool, problems: set[str] +) -> tuple[str, int] | None: + """Recover one lexical word, reporting RFC 2047's length violation. + + Each regex component stops at whitespace or a question mark. Attempts + only start at word boundaries, so even a failed match scans its token + a constant number of times, rather than rescanning the whole suffix. + """ + match = _ENCODED_WORD.match(value, start) + if not match: + return None + end = match.end() + if end < len(value) and value[end] not in (" \t)" if comment else " \t"): + return None + if end - start > 75: + problems.add("overlong-encoded-word") + try: + pieces = [] + for data, charset in decode_header(match.group()): + if isinstance(data, bytes): + encoding = (charset or "ascii").split("*", 1)[0] + if encoding == "unknown-8bit": + encoding = "utf-8" + data = data.decode(encoding) + if _SURROGATE_RE.search(data): + problems.add("invalid-encoded-word") + return None + # Never inject control characters into the structural parser. + if any(ord(ch) < 32 and ch != "\t" for ch in data): + return None + pieces.append(data) + except (LookupError, UnicodeError, ValueError): + return None + return "".join(pieces), end + + +def _display_source(value: str, phrase: bool, problems: set[str]) -> str: + """Decode lexical display words into safely quoted parser input. + + Structural mailbox checks use the original input. RFC 2047 words may + be decoded in a phrase or comment, never inside quotes or addr-specs. + Escape the rendered text before the conventional-name parser sees it, + so decoded commas, angles and comments cannot become delimiters. + """ + value = _FOLD.sub(" ", value) + output = [] + comment = 0 + quoted = escaped = False + i = 0 + while i < len(value): + ch = value[i] + if escaped: + escaped = False + elif ch == "\\" and (quoted or comment): + escaped = True + elif comment and ch == ")": + comment -= 1 + elif not quoted and ch == "(": + comment += 1 + elif not comment and ch == '"': + quoted = not quoted + elif not quoted and not comment and ch == "<": + phrase = False + elif ( + not quoted + and (comment or phrase) + and ch == "=" + and (i == 0 or value[i - 1] in (" \t(" if comment else " \t")) + ): + word = _decoded_word(value, i, bool(comment), problems) + if word: + pieces = [word[0]] + end = word[1] + # Adjacent encoded words discard intervening FWS. Join + # once, rather than repeatedly copying an expanding name. + while end < len(value): + next_start = end + while next_start < len(value) and value[next_start] in " \t": + next_start += 1 + if next_start == end: + break + word = _decoded_word(value, next_start, bool(comment), problems) + if not word: + break + pieces.append(word[0]) + end = word[1] + text = "".join(pieces).replace("\\", "\\\\") + if comment: + text = text.replace("(", "\\(").replace(")", "\\)") + else: + text = '"' + text.replace('"', '\\"') + '"' + output.append(text) + i = end + continue + output.append(ch) + i += 1 + return "".join(output) + + def _escape_surrogates(value: str) -> str: """Keep invalid code points visible without emitting invalid UTF-8 JSON.""" return value.encode("utf-8", "backslashreplace").decode("utf-8") -def parse_address_header(raw: str, strict_parser): +def parse_address_header(raw: str, strict_parser, *, decode_names=False): """Return mailbox tuples and evidence for recovery or ambiguity. ``strict_parser`` is the runtime-compatible email.utils adapter. Each list member is parsed independently so a malformed member cannot change - the interpretation of a neighbouring, valid quoted name. + the interpretation of a neighbouring, valid quoted name. Set + ``decode_names`` to render display words after structural validation. """ results = [] diagnostics = [] for item in _scan(raw): candidates = [] + display_problems: set[str] = set() # Derive the prefix once: repeating it for every angle-addr would # copy quadratically much text on a long ambiguous item. - name = _name(item.text[: item.angles[0][0]]) if item.angles else "" + prefix = item.text[: item.angles[0][0]] if item.angles else "" + name = _name( + _display_source(prefix, True, display_problems) if decode_names else prefix + ) for index, (left, right) in enumerate(item.angles): address = _mailbox(item.text[left + 1 : right]) if address: @@ -288,6 +395,8 @@ def parse_address_header(raw: str, strict_parser): # Deep comments are stripped by the iterative scanner before # reaching the stdlib's recursive comment parser. source = item.raw if item.comment_depth < 16 else item.text + if decode_names: + source = _display_source(source, bool(item.angles), display_problems) parsed = strict_parser([source]) if [addr for _, addr in parsed if addr] == [a for _, a in selected]: selected = [(name, addr) for name, addr in parsed if addr] @@ -295,6 +404,8 @@ def parse_address_header(raw: str, strict_parser): # Parsing can reject legal CFWS too. This records recovery, # not a claim that every rejection proves RFC noncompliance. reason = reason or "structural-recovery" + if display_problems: + reason = ",".join(filter(None, [reason, *sorted(display_problems)])) results.extend(selected) if reason: diagnostics.append( diff --git a/src/mailparser/const.py b/src/mailparser/const.py index d3c9266..de68a55 100644 --- a/src/mailparser/const.py +++ b/src/mailparser/const.py @@ -43,66 +43,6 @@ r"|::)" # just :: ) -# Normalize whitespace: collapse tabs and newlines to single space. -# Parenthesized comments and bracketed IPs are preserved. -JUNK_PATTERN = r"[\t\n]+" - -# ------------------------------------------------------------------ # -# Received header parsing — RFC 5321 §4.4 grammar: -# -# Received = "Received:" *( received-token / comment ) ";" date-time -# received-token = "from" domain / "by" domain / "via" atom -# / "with" atom / "id" atom / "for" addr-spec -# -# Strategy: tokenize on clause keywords, then extract values per clause. -# This eliminates the duplicated boundary lookaheads of the old -# per-clause pattern list and matches the RFC grammar directly. -# ------------------------------------------------------------------ # - -# Pattern that splits a received header into clause tokens. -# Matches each RFC 5321 keyword at a word boundary followed by its value, -# which extends up to the next keyword or semicolon. -# The keywords are: from, by, via, with (not "with cipher"), id, for, -# plus the non-standard envelope-from and envelope-sender. -_CLAUSE_SPLITTER = re.compile( - r"(?:^|\s+)" - r"(from|by|via|with(?!\s+cipher)|id|for|envelope-from|envelope-sender)" - r"\s+", - re.I, -) - -# Collapses any run of whitespace to a single space. Applied to a received -# header before ``_CLAUSE_SPLITTER`` so the splitter's ``\s+`` sub-patterns -# only ever match a single character. Without it a long run of spaces with no -# following clause keyword drives the split into quadratic backtracking — an -# ordinary parse of an attacker-supplied Received header becomes a denial of -# service (CWE-1333). Note the ``[\t\n]``-only normalization elsewhere does -# not collapse spaces, so this guard must live at the point of use. -_WS_RUN_RE = re.compile(r"\s+") - -# Matches the HELO/EHLO name an MTA records inside the ``from`` clause, -# in the two shapes seen in the wild: -# from evil.example ([203.0.113.9]:45321 helo=[8.8.8.8]) by ... (Exim) -# from [203.0.113.9] (account x@y HELO 8.8.8.8) by ... (CommuniGate) -# The HELO argument is chosen by the sender and may be an RFC 5321 address -# literal, so it must be removed before scanning the clause for the sender -# IP — otherwise the sender simply appends the IP they want reported. -# Used to locate the span, never to delete it: the trailing \S+ would -# otherwise swallow whatever follows, and a sender whose rDNS or HELO is -# literally ``helo`` can place that word right before the MTA-written IP. -# Two guards keep the span off MTA-written text. The lookbehind: the -# clause value *starts* with the HELO name, so at offset 0 the word is the -# name itself, not a label introducing one. The ``(?![\[(])`` on the -# space-separated form: an MTA writes its own address inside brackets or -# parentheses, so a following group is never part of the HELO argument. -# Exim's ``helo=[8.8.8.8]`` is a genuine bracketed HELO argument and keeps -# no such guard. -_HELO_RE = re.compile( - r"(?<=[(\s])e?helo\s*=\S*" # Exim "helo=value": after "(" or space - r"|(?<=\s)e?helo\s+\S+", # space form: only after whitespace - re.I, -) - # RFC 5321 §4.1.3 tags an IPv6 literal as ``[IPv6:2a00:...]``. REGXIP6 # happily starts matching at the ``6`` of the tag and yields # ``6:2a00:1450:4864:20::`` — a different, valid, routable address that an @@ -112,9 +52,6 @@ # Extracts envelope-from email: envelope-from _ENVELOPE_FROM_RE = re.compile(r"<([^>]+)>") -# Date after semicolon (standard RFC 5321) -_DATE_RE = re.compile(r";\s*(.*)", re.DOTALL) - # SendGrid non-standard date format (no semicolon) _SENDGRID_DATE_RE = re.compile( r"(\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2}\.\d{9}\s+\+0000\s+UTC)" diff --git a/src/mailparser/core.py b/src/mailparser/core.py index 702fa7e..adc58d0 100644 --- a/src/mailparser/core.py +++ b/src/mailparser/core.py @@ -22,9 +22,10 @@ import ipaddress import json import logging +from bisect import bisect_right +from email.feedparser import BytesFeedParser from mailparser.const import ( - _HELO_RE, _IPV6_TAG_RE, ADDRESSES_HEADERS, COMPUTED_PARTS, @@ -32,12 +33,13 @@ REGXIP, REGXIP6, ) +from mailparser.dates import date_diagnostics, parse_mail_date from mailparser.exceptions import MailParserRecursionError +from mailparser.received import helo_argument_spans from mailparser.utils import ( _safe_attachment_filename, _safe_remove, as_string_safe, - convert_mail_date, decode_header_part, decode_headers, extract_msg_convert, @@ -47,9 +49,8 @@ get_mail_keys, get_to_domains, group_spans, - in_spans, + mime_header_view, msgconvert, - ported_open, ported_string, random_string, raw_payload, @@ -59,12 +60,30 @@ log = logging.getLogger(__name__) + +def _in_sorted_spans(position, spans): + """Check ordered disjoint spans without rescanning every preceding group. + + HELO exclusions may share an end within one comment; the latest start + still identifies membership. Both span lists come from lexical order. + """ + index = bisect_right(spans, (position, float("inf"))) - 1 + return index >= 0 and position < spans[index][1] + + # Keys ``_make_mail()`` sets itself after walking the headers. A sender can # name a header after any of them, so they are skipped while collecting # headers: otherwise ``mail["defects"]`` would hold a string from the wire # instead of the list of parsed defects. _RESERVED_MAIL_KEYS = frozenset( - {"defects", "defects_categories", "has_defects", "address_header_defects"} + { + "defects", + "defects_categories", + "has_defects", + "address_header_defects", + "received_header_defects", + "date_header_defects", + } ) @@ -220,7 +239,7 @@ def from_file_obj(cls, fp): @classmethod def from_file(cls, fp, is_outlook=False): """ - Init a new object from a file path. + Init a new object from a file path without discarding octets. Args: fp (string): file path of raw email @@ -233,8 +252,14 @@ def from_file(cls, fp, is_outlook=False): def _build(): try: - with ported_open(fp) as f: - return email.message_from_file(f) + # Preserve invalid header octets and MIME payload bytes; + # BytesParser.parse() would still translate CR/LF through + # its text wrapper. Feed bytes directly in bounded chunks. + with open(fp, "rb") as f: + parser = BytesFeedParser() + while chunk := f.read(8192): + parser.feed(chunk) + return parser.close() finally: # ``fp`` is a temp file produced by the Outlook conversion; # remove it even if parsing raises, so failures do not leak @@ -329,7 +354,7 @@ def _reset(self): for header in sorted(ADDRESSES_HEADERS): for occurrence, value in enumerate(self._header_index.get(header, [])): diagnostics = [] - addresses = get_addresses(value, defects=diagnostics) + addresses = get_addresses(value, defects=diagnostics, decode_names=True) if occurrence == 0: self._address_headers[header] = addresses for defect in diagnostics: @@ -347,6 +372,59 @@ def _reset(self): ] } ) + self._date = None + self._timezone = 0 + self._date_header_defects = [] + occurrences = {} + for name, raw in self.message.raw_items() if self.message else []: + header = name.lower() + if header not in {"date", "resent-date"}: + continue + occurrence = occurrences.get(header, 0) + occurrences[header] = occurrence + 1 + result = parse_mail_date(raw) + if header == "date" and occurrence == 0: + self._date = result.value + self._timezone = result.timezone + self._date_header_defects.extend( + date_diagnostics(result, raw, header, occurrence) + ) + self._received_header_defects = [] + self._received = receiveds_parsing( + [ + value + for name, value in self.message.raw_items() + if name.lower() == "received" + ] + if self.message + else [], + defects=self._received_header_defects, + date_defects=self._date_header_defects, + ) + if self._received_header_defects: + self._has_defects = True + self._defects_categories.add("ReceivedHeaderDefect") + self._defects.append( + { + "received-headers": [ + f"ReceivedHeaderDefect: received[{d['occurrence']}]: " + f"{d['reason']}" + for d in self._received_header_defects + ] + } + ) + if self._date_header_defects: + self._has_defects = True + self._defects_categories.add("DateHeaderDefect") + self._defects.append( + { + "date-headers": [ + f"DateHeaderDefect: {d['header']}[{d['occurrence']}]: " + f"{d['reason']}" + for d in self._date_header_defects + ] + } + ) def _build_header_index(self): """ @@ -409,16 +487,10 @@ def _header_value(self, name): # are inspected once for diagnostics during reset. parsed_addresses = self._address_headers.get(name_header, []) - # decoded addresses — skip entries with no address (absent header) + # Display words were decoded once while lexical context was + # available. A quoted encoded-looking name must stay literal. return [ - ( - ( - "" - if (decoded_name := decode_header_part(name)) == email_addr - else decoded_name - ), - email_addr, - ) + ("" if name == email_addr else name, email_addr) for name, email_addr in parsed_addresses if email_addr ] @@ -485,6 +557,10 @@ def _make_mail(self, complete=True): if self.address_header_defects: mail["address_header_defects"] = self.address_header_defects + if self.received_header_defects: + mail["received_header_defects"] = self.received_header_defects + if self.date_header_defects: + mail["date_header_defects"] = self.date_header_defects # add defects mail["has_defects"] = self.has_defects @@ -494,6 +570,50 @@ def _make_mail(self, complete=True): return mail + def _decode_body_payload(self, part, charset, part_index): + """Decode body text from original bytes and report charset recovery.""" + payload = part.get_payload(decode=True) + cte = ported_string(part.get("Content-Transfer-Encoding", "")).strip().lower() + if not cte or cte in ("7bit", "8bit"): + # get_payload(False) already applies a lossy charset decode. + # The stored payload distinguishes Unicode string input from + # bytes carried by the stdlib as ASCII plus surrogateescape. + raw_text = part._payload + if isinstance(raw_text, str): + try: + raw_text.encode("utf-8") + except UnicodeEncodeError: + pass + else: + return raw_text + + if not payload: + return "" + try: + text = payload.decode(charset) + # Some Python codecs can produce lone surrogates. Such a + # value is not Unicode text and breaks UTF-8 JSON/CLI sinks. + text.encode("utf-8") + except (LookupError, UnicodeError): + try: + text = payload.decode("utf-8") + recovery = "decoded with utf-8" + except UnicodeDecodeError: + text = payload.decode("utf-8", "replace") + recovery = "decoded with utf-8 replacement characters" + self._has_defects = True + self._defects_categories.add("CharsetDecodeDefect") + self._defects.append( + { + part.get_content_type(): [ + f"CharsetDecodeDefect: part {part_index}, " + f"charset {charset!r} failed; {recovery}. " + "Original bytes remain in the MIME part." + ] + } + ) + return ported_string(text) + def parse(self): """ This method parses the raw email and makes the tokens. @@ -515,12 +635,24 @@ def parse(self): return self parts = [] # Normal parts plus defects + attachment_descendants = set() # walk all mail parts to search defects for p in self.message.walk(): part_content_type = p.get_content_type() self._append_defects(p, part_content_type) - parts.append(p) + inside_attachment = p in attachment_descendants + parts.append((p, inside_attachment)) + if p.is_multipart(): + disposition = p.get_content_disposition() + # RFC 2183 section 2.9 applies disposition to the entire + # container. Do not newly expose named inline children of + # attached, unknown-disposition or filename-only containers. + blocks_inline = disposition not in (None, "inline") or ( + disposition != "inline" and bool(p.get_filename()) + ) + if inside_attachment or blocks_inline: + attachment_descendants.update(p.get_payload()) # If defects are in epilogue defects get epilogue if self.defects_categories & EPILOGUE_DEFECTS: @@ -534,26 +666,51 @@ def parse(self): if epilogue is not None: try: p = email.message_from_string(epilogue) - parts.append(p) + # Recovered epilogues lack reliable nesting context; + # retain their existing extraction behavior. + parts.append((p, True)) except Exception: log.error("Failed to get epilogue part. Check raw mail.") # walk all mail parts - for i, p in enumerate(parts): + for i, (p, inside_attachment) in enumerate(parts): + metadata, metadata_defects = mime_header_view(p) + filename = decode_header_part( + metadata.get_filename(), defects=metadata_defects + ) + if metadata_defects: + self._has_defects = True + self._defects_categories.add("MimeHeaderDefect") + self._defects.append( + { + "mime-headers": [ + f"MimeHeaderDefect: part {i}, {detail}" + for detail in metadata_defects + ] + } + ) if ( not p.is_multipart() - or ported_string(p.get_content_disposition()).lower() == "attachment" + or ported_string(metadata.get_content_disposition()).lower() + == "attachment" ): - charset = p.get_content_charset("utf-8") - charset_raw = p.get_content_charset() + charset = metadata.get_content_charset("utf-8") + charset_raw = metadata.get_content_charset() log.debug(f"Charset {charset!r} part {i!r}") - content_disposition = ported_string(p.get_content_disposition()).lower() + content_disposition = ported_string( + metadata.get_content_disposition() + ).lower() log.debug(f"content-disposition {content_disposition!r} part {i!r}") - content_id = ported_string(p.get("content-id")) + content_id = ported_string(metadata.get("content-id")) log.debug(f"content-id {content_id!r} part {i!r}") - content_subtype = ported_string(p.get_content_subtype()) + content_subtype = ported_string(metadata.get_content_subtype()) log.debug(f"content subtype {content_subtype!r} part {i!r}") - filename = decode_header_part(p.get_filename()) + is_inline_body = ( + not inside_attachment + and not p.is_multipart() + and content_disposition == "inline" + and p.get_content_type() in ("text/plain", "text/html") + ) is_attachment = False if filename: @@ -574,19 +731,21 @@ def parse(self): log.debug(f"Email part {i!r} is an attachment") log.debug(f"Filename {filename!r} part {i!r}") binary = False - mail_content_type = ported_string(p.get_content_type()) + mail_content_type = ported_string(metadata.get_content_type()) log.debug(f"Mail content type {mail_content_type!r} part {i!r}") # Strip before comparing, exactly as email's own # get_payload() does: a trailing space made every # encoding branch below miss and the raw bytes fall # through the text path, which drops the non-UTF-8 ones. transfer_encoding = ( - ported_string(p.get("content-transfer-encoding", "")) + ported_string(metadata.get("content-transfer-encoding", "")) .strip() .lower() ) log.debug(f"Transfer encoding {transfer_encoding!r} part {i!r}") - content_disposition = ported_string(p.get("content-disposition")) + content_disposition = ported_string( + metadata.get("content-disposition") + ) log.debug(f"content-disposition {content_disposition!r} part {i!r}") if p.is_multipart(): @@ -642,43 +801,10 @@ def parse(self): } ) - # this isn't an attachments - else: - log.debug(f"Email part {i!r} is not an attachment") - - payload = p.get_payload(decode=True) - # Strip as well, for the same reason as the attachment - # branch above: a trailing space sent the part down the - # else branch, which re-reads the text through - # raw-unicode-escape and turns it into literal escapes. - cte = ( - ported_string(p.get("Content-Transfer-Encoding", "")) - .strip() - .lower() - ) - - if not cte or cte in ["7bit", "8bit"]: - # message_from_bytes stores non-ASCII body bytes via - # ascii+surrogateescape, producing surrogates in the - # payload string. message_from_string stores a proper - # Unicode str (no surrogates). Detect which case we - # have via get_payload(decode=False) and decode - # accordingly so the declared charset is honoured. - raw_str = raw_payload(p) - if isinstance(raw_str, str): - try: - # Raises if surrogates present (from_bytes path) - raw_str.encode("utf-8") - payload = raw_str - except UnicodeEncodeError: - # These encodings are not transformed by - # get_payload(decode=True), so ``payload`` - # already holds the original bytes. - payload = ported_string(payload, encoding=charset) - else: - payload = ported_string(payload, encoding=charset) - else: - payload = ported_string(payload, encoding=charset) + # Named inline text has both a body and an attachment view. + if not is_attachment or is_inline_body: + log.debug(f"Decoding body text for email part {i!r}") + payload = self._decode_body_payload(p, charset, i) if payload: if p.get_content_subtype() == "html": @@ -801,18 +927,18 @@ def _sender_ip_candidates(self, received_header): list of IP address strings, in the order they appear """ # Blank the "IPv6:" tag with a non-space filler of the same width: - # offsets stay aligned, and no new token boundary or _HELO_RE - # lookbehind position is created inside the sender's own token. + # offsets stay aligned, and no new token boundary is created + # inside the sender's own token. from_part = _IPV6_TAG_RE.sub("_____", get_from_clause(received_header)) if not from_part: return [] groups = group_spans(from_part) - helo_spans = [ - m.span() - for m in _HELO_RE.finditer(from_part) - if in_spans(m.start(), groups) - ] + # A multi-word HELO can contain clause words and fake IPs. Once + # comments remain intact, excluding only its first token would + # promote the trailing sender text. Exclude through the enclosing + # group's end, while retaining a connection IP before the marker. + helo_spans = helo_argument_spans(from_part, groups) first_token_end = len(from_part.split(" ", 1)[0]) # Merge both families positionally. Choosing IPv4 over IPv6 by @@ -822,12 +948,12 @@ def _sender_ip_candidates(self, received_header): list(REGXIP.finditer(from_part)) + list(REGXIP6.finditer(from_part)), key=lambda m: m.start(), ) - matches = [m for m in matches if not in_spans(m.start(), helo_spans)] + matches = [m for m in matches if not _in_sorted_spans(m.start(), helo_spans)] candidates = [ m.group() for m in matches - if m.start() >= first_token_end and in_spans(m.start(), groups) + if m.start() >= first_token_end and _in_sorted_spans(m.start(), groups) ] if candidates: return candidates @@ -935,8 +1061,7 @@ def received(self): """ Return a list of all received headers parsed """ - output = self.received_raw - return receiveds_parsing(output) + return self._received @property def received_json(self): @@ -1021,28 +1146,14 @@ def date(self): """ Return the mail date in datetime.datetime format and UTC. """ - date = self.message.get("date") if self.message else None - conv = None - - try: - conv, _ = convert_mail_date(date) - except Exception: - pass - return conv + return self._date @property def timezone(self): """ Return timezone. Offset from UTC. """ - date = self.message.get("date") if self.message else None - timezone = 0 - - try: - _, timezone = convert_mail_date(date) - except Exception: - pass - return timezone + return self._timezone @property def date_json(self): @@ -1095,6 +1206,26 @@ def address_header_defects(self): """ return self._address_header_defects + @property + def received_header_defects(self): + """Return raw ambiguous trace fields and their wire occurrences. + + Unbalanced lexical syntax is retained as evidence and excluded + from sender attribution. Occurrence indices are zero-based. + """ + return self._received_header_defects + + @property + def date_header_defects(self): + """Return date recovery evidence for all date-bearing occurrences. + + Reasons distinguish invalid syntax/components from a mismatched + weekday whose numerical calendar date could be recovered safely. + Raw evidence retains the complete original field, including the + trace clauses for Received dates. + """ + return self._date_header_defects + @property def defects(self): """ diff --git a/src/mailparser/dates.py b/src/mailparser/dates.py new file mode 100644 index 0000000..62178b1 --- /dev/null +++ b/src/mailparser/dates.py @@ -0,0 +1,195 @@ +"""RFC 5322 date interpretation with explicit forensic diagnostics.""" + +from __future__ import annotations + +import datetime +import re +from dataclasses import dataclass + +_MONTHS: tuple[str, ...] = tuple( + "jan feb mar apr may jun jul aug sep oct nov dec".split() +) +_WEEKDAYS: tuple[str, ...] = tuple("mon tue wed thu fri sat sun".split()) +_ZONES = { + "UT": 0, + "GMT": 0, + "EST": -300, + "EDT": -240, + "CST": -360, + "CDT": -300, + "MST": -420, + "MDT": -360, + "PST": -480, + "PDT": -420, +} +# CFWS is removed in a bounded scan first. The remaining repetitions do +# not overlap: digits, spaces and fixed delimiters have separate roles. +_DATE = re.compile( + r"(?:(?Pmon|tue|wed|thu|fri|sat|sun) *, *)?" + r"(?P[0-9]{1,2}) *(?P" + "|".join(_MONTHS) + r") *" + r"(?P[0-9]{2,}) *(?P[0-9]{2}) *: *" + r"(?P[0-9]{2})(?: *: *(?P[0-9]{2}))?" + r"(?P *)(?P[+-][0-9]+|[a-z]+)", + re.I | re.ASCII, +) + + +@dataclass(frozen=True) +class DateResult: + """A selected UTC value/offset and the reasons requiring diagnostics. + + ``value`` is None when no safe timestamp can be selected. ``timezone`` + retains the legacy numeric-string format, or 0 for no selected value. + The original zone token, including unknown-local-zone forms, remains + in the caller's raw field. ``reasons`` never labels obsolete syntax or + representable leap seconds invalid merely because Python lacks them. + """ + + value: datetime.datetime | None = None + timezone: str | int = 0 + reasons: tuple[str, ...] = () + + +def _date_tokens(raw): + """Replace nested comments and FWS with spaces without joining tokens.""" + output = [] + depth = 0 + escaped = False + gap = False + for char in raw: + if depth: + if escaped: + escaped = False + elif char == "\\": + escaped = True + elif char == "(": + depth += 1 + elif char == ")": + depth -= 1 + continue + if char == "(": + depth = 1 + gap = True + elif char == ")": + return None + elif char in " \t\r\n": + gap = True + else: + if gap and output: + output.append(" ") + gap = False + output.append(char) + return None if depth else "".join(output) + + +def parse_mail_date(raw): + """Interpret modern/obsolete date-time without normalizing violations. + + Args: + raw (str or None): unmodified date-time field value; None is absent. + + Returns: + DateResult: safe selected UTC datetime and explicit defect reasons. + A wrong weekday is recoverable from the numerical calendar date. + Invalid day/time/zone components produce no selected datetime. + + RFC 5322 sections 3.3/4.3 and erratum 6639 permit obsolete CFWS, short + years and alphabetic zones. Unknown alphabetic zones have -0000 + semantics. A leap second maps to the next representable datetime, + preserving the existing API convention; the raw field keeps :60. + """ + if raw is None: + return DateResult() + invalid_utf8 = False + try: + raw.encode("utf-8", "surrogateescape").decode("utf-8") + except UnicodeError: + invalid_utf8 = True + tokens = _date_tokens(raw) + match = _DATE.fullmatch(tokens) if tokens is not None else None + if match is None: + return DateResult(reasons=("invalid-date-syntax",)) + + year_text = match["year"] + # Check magnitude before int(), avoiding integer digit-limit failures. + if len(year_text.lstrip("0")) > 4: + return DateResult(reasons=("date-out-of-range",)) + year = int(year_text.lstrip("0") or "0") + if len(year_text) == 2: + year += 2000 if year < 50 else 1900 + elif len(year_text) == 3: + year += 1900 + month = _MONTHS.index(match["month"].lower()) + 1 + day = int(match["day"]) + try: + calendar_date = datetime.date(year, month, day) + except ValueError: + return DateResult(reasons=("invalid-calendar-date",)) + if year < 1900: + return DateResult(reasons=("invalid-calendar-date",)) + + hour, minute = int(match["hour"]), int(match["minute"]) + second = int(match["second"] or "0") + if hour > 23 or minute > 59 or second > 60: + return DateResult(reasons=("invalid-time",)) + zone = match["zone"].upper() + if zone[0] in "+-": + if len(zone) != 5 or int(zone[-2:]) > 59: + return DateResult(reasons=("invalid-zone",)) + if not match["zone_space"]: + return DateResult(reasons=("invalid-date-syntax",)) + offset = int(zone[1:3]) * 60 + int(zone[3:]) + offset *= -1 if zone[0] == "-" else 1 + else: + if zone == "J": + return DateResult(reasons=("invalid-zone",)) + # RFC 5322 section 4.3: unknown alphabetic zones, including military + # letters, carry unknown-local-zone semantics, as does -0000. + offset = _ZONES.get(zone, 0) + + try: + value = datetime.datetime( + year, + month, + day, + hour, + minute, + min(second, 59), + tzinfo=datetime.timezone.utc, + ) - datetime.timedelta(minutes=offset) + if second == 60: + value += datetime.timedelta(seconds=1) + except (OverflowError, ValueError): + return DateResult(reasons=("date-out-of-range",)) + weekday = match["weekday"] + reasons = ("invalid-utf8",) if invalid_utf8 else () + if weekday and _WEEKDAYS.index(weekday.lower()) != calendar_date.weekday(): + reasons += ("weekday-mismatch",) + return DateResult(value, f"{offset / 60:+.1f}", reasons) + + +def date_diagnostics(result, raw, header, occurrence): + """Return occurrence-aware diagnostic entries for one interpretation. + + Args: + result (DateResult): parsed timestamp and defect reasons. + raw (str): entire original field body, including trace clauses. + Surrogates become visible escapes, as in address diagnostics; + unmodified octets remain available through Message.raw_items(). + header (str): lowercased date-bearing field name. + occurrence (int): zero-based occurrence within that field name. + + Returns: + list[dict]: reasons with raw evidence and any selected ISO value. + """ + return [ + { + "reason": reason, + "raw": raw.encode("utf-8", "backslashreplace").decode("utf-8"), + "recovered": result.value is not None, + "value": result.value.isoformat() if result.value else None, + "header": header, + "occurrence": occurrence, + } + for reason in result.reasons + ] diff --git a/src/mailparser/received.py b/src/mailparser/received.py new file mode 100644 index 0000000..f4c366b --- /dev/null +++ b/src/mailparser/received.py @@ -0,0 +1,242 @@ +"""Bounded lexical scanning of Received trace fields (RFC 5321/5322).""" + +import re + +from mailparser.const import _SENDGRID_DATE_RE +from mailparser.exceptions import MailParserReceivedParsingError + +_KEYWORD = re.compile( + r"(from|by|via|with|id|for|envelope-from|envelope-sender)" + r"(?=[ \t\r\n])", + re.I, +) +_FWS = " \t\r\n" + + +def helo_argument_spans(value, groups): + """Return sender-controlled HELO tails within ordered comment spans. + + Args: + value (str): the from clause, excluding its keyword. + groups (list): structural spans from the trace scanner. + + Returns: + list: ordered (start, end) exclusions, at most one per comment. + + A comment's first token can be a hostname literally named ``helo`` + followed by an MTA address group. Preserve that layout, while an + equals sign explicitly marks an argument even when it is a literal. + Neither a top-level hostname nor a domain literal introduces a marker. + """ + spans = [] + for start, end in groups: + if value[start - 1] != "(": + continue + initial = True + escaped = False + for index in range(start, end): + char = value[index] + if escaped: + escaped = False + initial = False + continue + if char == "\\": + escaped = True + initial = False + continue + if char == "(": + initial = True + continue + if char in _FWS: + continue + first_token = initial + initial = False + if index > start and value[index - 1] not in _FWS + "(": + continue + width = 5 if value[index : index + 5].lower() == "ehelo" else 4 + if value[index : index + width].lower() not in ("helo", "ehelo"): + continue + argument = index + width + if argument >= end or value[argument] not in _FWS + "=": + continue + while argument < end and value[argument] in _FWS: + argument += 1 + explicit = argument < end and value[argument] == "=" + if explicit: + argument += 1 + while argument < end and value[argument] in _FWS: + argument += 1 + if not explicit and ( + argument == end or (first_token and value[argument] in "([") + ): + continue + spans.append((index, end)) + break + return spans + + +def _clause_value(value): + """Collapse formatting whitespace without rewriting quoted strings.""" + output = [] + quoted = False + escaped = False + whitespace = False + for char in value.strip(_FWS): + if char in _FWS and not quoted: + whitespace = True + continue + if whitespace: + output.append(" ") + whitespace = False + output.append(char) + if escaped: + escaped = False + elif char == "\\": + escaped = True + elif char == '"': + quoted = not quoted + return "".join(output) + + +def split_received(value, *, defects=None, groups=None): + """Split top-level clauses and date while preserving nested evidence. + + Args: + value (str): complete raw Received field body. + defects (list): optional destination for recovery reasons. + groups (list): optional destination for comment/literal spans. + + Returns: + tuple: ordered (keyword, value) pairs and a date string or None. + + Raises: + MailParserReceivedParsingError: unbalanced lexical delimiters make + the interpretation ambiguous. The message is a stable reason. + + Comments nest; quoted pairs are meaningful inside comments, quoted + strings and domain literals. Neither clauses nor the stamp separator + can begin inside those constructs or an angle address. Scanning is + iterative and linear, including adversarial depth and whitespace. + """ + boundaries = [] + comment = 0 + quote = False + literal = False + angle = False + escaped = False + separators = [] + group_start = 0 + sendgrid = None + for index, char in enumerate(value): + if escaped: + escaped = False + continue + if comment: + if char == "\\": + escaped = True + elif char == "(": + comment += 1 + elif char == ")": + comment -= 1 + if not comment and groups is not None: + groups.append((group_start, index)) + continue + if quote or literal: + if char == "\\": + escaped = True + elif quote and char == '"': + quote = False + elif literal and char == "]": + literal = False + if groups is not None: + groups.append((group_start, index)) + continue + if char == "(": + comment = 1 + group_start = index + 1 + elif char == ")": + raise MailParserReceivedParsingError("unexpected-comment-close") + elif char == '"': + quote = True + elif char == "[": + literal = True + group_start = index + 1 + elif char == "<": + if angle: + raise MailParserReceivedParsingError("nested-angle-address") + angle = True + elif char == ">": + angle = False + elif not angle: + if char == ";": + separators.append(index) + elif not separators and (index == 0 or value[index - 1] in _FWS + ")"): + match = _KEYWORD.match(value, index) + if match: + keyword = match.group(1).lower() + end = match.end() + while end < len(value) and value[end] in _FWS: + end += 1 + # Retain the established MTA TLS annotation exception. + if keyword != "with" or not value[end : end + 6].lower().startswith( + "cipher" + ): + boundaries.append((index, match.end(), keyword)) + elif char.isdigit() and sendgrid is None: + sendgrid = _SENDGRID_DATE_RE.match(value, index) + + if (comment or literal) and groups is not None: + groups.append((group_start, len(value))) + for unclosed, reason in ( + (comment, "unclosed-comment"), + (quote, "unclosed-quoted-string"), + (literal, "unclosed-domain-literal"), + (angle, "unclosed-angle-address"), + ): + if unclosed: + raise MailParserReceivedParsingError(reason) + + end = len(value) + date = None + extra_clause = None + if separators: + end = separators[0] + date = value[end + 1 :].strip(_FWS) + if len(separators) > 1: + # A known MTA emits "id token; for ; date". + # Recover only that shape, using already-scanned top-level + # separators. Never use split(';') inside quoted recipients + # or date comments, and retain the malformed raw evidence. + middle = value[end + 1 : separators[1]].strip(_FWS) + parts = middle.split(None, 1) + if len(separators) != 2 or len(parts) != 2: + raise MailParserReceivedParsingError("ambiguous-date-separators") + if ( + not boundaries + or boundaries[-1][2] != "id" + or parts[0].lower() != "for" + or not parts[1].startswith("<") + or not parts[1].endswith(">") + ): + raise MailParserReceivedParsingError("ambiguous-date-separators") + extra_clause = ("for", _clause_value(parts[1])) + date = value[separators[1] + 1 :].strip(_FWS) + if defects is not None: + defects.append( + { + "reason": "misplaced-date-separator", + "recovered": True, + } + ) + elif sendgrid is not None: + end = sendgrid.start() + date = sendgrid.group(1) + boundaries = [item for item in boundaries if item[0] < end] + + clauses = [] + for position, (_, start, keyword) in enumerate(boundaries): + stop = boundaries[position + 1][0] if position + 1 < len(boundaries) else end + clauses.append((keyword, _clause_value(value[start:stop]))) + if extra_clause is not None: + clauses.append(extra_clause) + return clauses, date diff --git a/src/mailparser/utils.py b/src/mailparser/utils.py index da9fb0d..44bbce3 100644 --- a/src/mailparser/utils.py +++ b/src/mailparser/utils.py @@ -19,7 +19,6 @@ from __future__ import annotations import base64 -import datetime import email import email.header import email.utils @@ -38,24 +37,22 @@ from collections import Counter, namedtuple from email.errors import HeaderParseError from email.header import decode_header +from email.message import Message from unicodedata import normalize from mailparser.addresses import parse_address_header from mailparser.const import ( - _CLAUSE_SPLITTER, - _DATE_RE, _ENVELOPE_FROM_RE, - _SENDGRID_DATE_RE, - _WS_RUN_RE, ADDRESSES_HEADERS, - JUNK_PATTERN, OTHERS_PARTS, ) +from mailparser.dates import date_diagnostics, parse_mail_date from mailparser.exceptions import ( MailParserOSError, MailParserPathError, MailParserReceivedParsingError, ) +from mailparser.received import split_received log = logging.getLogger(__name__) @@ -96,6 +93,7 @@ def get_addresses( raw_header: str | email.header.Header | None, *, defects: list | None = None, + decode_names: bool = False, ) -> list[tuple[str, str]]: """Parse mailboxes with structural recovery for forensic input. @@ -107,6 +105,8 @@ def get_addresses( Args: raw_header: raw header string, email Header object, or None. defects: optional list to receive recovery/ambiguity evidence. + decode_names: decode display words in their lexical context. The + default retains encoded words for existing helper callers. Returns: List of (display_name, address) tuples. An absent/empty string @@ -133,7 +133,9 @@ def get_addresses( raw_header = str(raw_header) if not raw_header.strip(" \t\r\n"): return _getaddresses([raw_header]) - parsed, diagnostics = parse_address_header(raw_header, _getaddresses) + parsed, diagnostics = parse_address_header( + raw_header, _getaddresses, decode_names=decode_names + ) if defects is not None: defects.extend(diagnostics) return parsed or [("", "")] @@ -210,12 +212,13 @@ def ported_string(raw_data, encoding="utf-8", errors="ignore"): return str(raw_data, "utf-8", errors) -def decode_header_part(header): +def decode_header_part(header, *, defects=None): """ Given a raw header returns a decoded header Args: header (string): header to decode + defects (list): optional destination for unsafe decoded text evidence Returns: str @@ -235,9 +238,65 @@ def decode_header_part(header): log.error(f"Failed decoding header part: {header}") output += header + try: + output.encode("utf-8") + except UnicodeEncodeError: + if defects is not None: + defects.append(f"invalid-encoded-word: non-Unicode scalar; raw={header!r}") + # Preserve the literal header when a codec fabricates a surrogate; + # it is not a Unicode scalar and cannot reach a UTF-8 output sink. + return str(header).encode("utf-8", "backslashreplace").decode("utf-8").strip() return output.strip() +def mime_header_view(part: Message) -> tuple[Message, list[str]]: + """Return a UTF-8 MIME metadata view without changing the raw part. + + Args: + part: original MIME part, retaining raw headers and payload. + + Returns: + A header-only compat32 message and descriptions of invalid UTF-8 + metadata, including each header occurrence and its raw octets. + Invalid bytes appear as replacement characters in the display + view; exact evidence remains in the descriptions and original. + """ + view = Message() + view.set_default_type(part.get_default_type()) + defects = [] + occurrences: Counter[str] = Counter() + for name, raw in part.raw_items(): + header = name.lower() + if header not in { + "content-type", + "content-disposition", + "content-id", + "content-transfer-encoding", + }: + continue + occurrence = occurrences[header] + occurrences[header] += 1 + value = str(raw) + try: + octets = value.encode("utf-8", "surrogateescape") + except UnicodeEncodeError: + # A direct string caller can supply non-octet surrogates. + evidence = value.encode("utf-8", "backslashreplace").decode() + value = re.sub(r"[\ud800-\udfff]", "\ufffd", value) + else: + try: + value = octets.decode("utf-8") + except UnicodeDecodeError: + evidence = octets.decode("ascii", "backslashreplace") + value = octets.decode("utf-8", "replace") + else: + evidence = None + if evidence is not None: + defects.append(f"{header}[{occurrence}]: invalid UTF-8; raw={evidence!r}") + view[name] = value + return view, defects + + def ported_open(file_): """Open a file with UTF-8 encoding and ignore errors. @@ -436,8 +495,8 @@ def get_from_clause(received): The sender address lives in the ``from`` clause; the ``by`` clause names the *receiving* server and must never be mistaken for it. - Clause boundaries are located with the anchored RFC 5321 tokenizer - ``_CLAUSE_SPLITTER`` rather than a substring search, because a plain + Clause boundaries use the same context-aware scanner as parsing, + ignoring comments and quoted strings. This also matters because a plain ``received.find("by")`` also matches inside a hostname — ``derby``, ``nearby`` — and the hostname comes from the sender's HELO. A false match truncates the clause, extraction fails on the genuine hop, and @@ -452,32 +511,12 @@ def get_from_clause(received): string with the ``from`` clause value, or an empty string when the header has no ``from`` clause """ - # Collapse whitespace runs before splitting, so the splitter stays - # linear — see the note on _WS_RUN_RE in const.py. - header = _WS_RUN_RE.sub(" ", received) - - # split() yields [preamble, keyword, value, keyword, value, ...] - parts = _CLAUSE_SPLITTER.split(header) - keywords = [(i, parts[i].lower()) for i in range(1, len(parts) - 1, 2)] - - start = next((i for i, kw in keywords if kw == "from"), None) - if start is None: - # No ``from`` clause: attribution fails closed. Returning the whole - # header instead would surface an IP taken from the ``by``, ``for``, - # ``with`` or ``id`` clause as the sender's — and the ``for`` clause - # holds the envelope recipient, which the sender picks at RCPT TO - # (``victim+8.8.8.8@example.com``). - return str() - - # The clause ends at the next keyword, full stop. Extending it to a - # later ``by`` to survive a multi-word HELO drags the ``by``, ``for`` - # and ``envelope-from`` values into the result, and the last two are - # sender-chosen: a quoted local part supplies the whitespace, so - # ``MAIL FROM:<"x 8.8.8.8 by q"@evil.example>`` puts an attacker IP - # into the sender-attribution scan (CWE-345). A multi-word HELO does - # truncate this clause, but that only costs the candidates — the caller - # then fails closed rather than reporting an attacker-chosen address. - return parts[start + 1].strip() + try: + clauses, _ = split_received(received) + except MailParserReceivedParsingError: + # Ambiguous delimiters on the authoritative hop must stop the walk. + return "" + return next((value for key, value in clauses if key == "from"), "") def group_spans(text): @@ -495,22 +534,12 @@ def group_spans(text): list of (start, end) tuples, in order, excluding the delimiters """ spans = [] - depth = 0 - start = 0 - - for i, char in enumerate(text): - if char in "([": - if depth == 0: - start = i + 1 - depth += 1 - elif char in ")]" and depth: - depth -= 1 - if depth == 0: - spans.append((start, i)) - - if depth: - spans.append((start, len(text))) - + try: + split_received(text, groups=spans) + except MailParserReceivedParsingError: + # The helper also exposes incomplete groups for existing callers; + # attribution only calls it after get_from_clause validated them. + pass return spans @@ -528,7 +557,7 @@ def in_spans(position, spans): return any(start <= position < end for start, end in spans) -def parse_received(received): +def parse_received(received, *, defects=None): """ Parse a single received header by tokenizing on RFC 5321 §4.4 keywords. @@ -538,6 +567,7 @@ def parse_received(received): Arguments: received {str} -- single received header + defects {list} -- optional destination for recovery diagnostics Raises: MailParserReceivedParsingError -- Raised when a @@ -549,38 +579,11 @@ def parse_received(received): values_by_clause = {} - # --- Step 1: Extract date (after semicolon, or SendGrid format) --- - date_match = _DATE_RE.search(received) - if date_match: - values_by_clause["date"] = date_match.group(1) - # Work only on the part before the semicolon for clause parsing - header_body = received[: date_match.start()] - else: - # Try SendGrid non-standard date - sg_match = _SENDGRID_DATE_RE.search(received) - if sg_match: - values_by_clause["date"] = sg_match.group(1) - header_body = received[: sg_match.start()] - else: - header_body = received - - # --- Step 2: Tokenize on clause keywords --- - # Collapse whitespace runs first so ``_CLAUSE_SPLITTER`` stays linear: a - # header padded with a long run of spaces that is not followed by a clause - # keyword otherwise backtracks quadratically — a denial of service on - # attacker-supplied Received headers (CWE-1333). The date has already been - # extracted from the raw header above, so collapsing here does not affect it. - header_body = _WS_RUN_RE.sub(" ", header_body) - # _CLAUSE_SPLITTER.split gives: [preamble, kw1, val1, kw2, val2, ...] - parts = _CLAUSE_SPLITTER.split(header_body) - - # parts[0] is preamble (before first keyword), then alternating kw/value - i = 1 # skip preamble - while i + 1 < len(parts): - keyword = parts[i].lower() - value = parts[i + 1].strip() - i += 2 + clauses, date = split_received(received, defects=defects) + if date is not None: + values_by_clause["date"] = date + for keyword, value in clauses: if keyword in ("envelope-from", "envelope-sender"): # Extract email from angle brackets m = _ENVELOPE_FROM_RE.search(value) @@ -622,34 +625,61 @@ def parse_received(received): return values_by_clause -def receiveds_parsing(receiveds): +def receiveds_parsing(receiveds, *, defects=None, date_defects=None): """ This function parses the receiveds headers. Args: receiveds (list): list of raw receiveds headers + defects (list): optional destination for occurrence diagnostics. + date_defects (list): optional destination for timestamp diagnostics. Returns: a list of parsed receiveds headers with first hop in first position """ parsed = [] - receiveds = [re.sub(JUNK_PATTERN, " ", i).strip() for i in receiveds] n = len(receiveds) log.debug(f"Nr. of receiveds. {n}") for idx, received in enumerate(receiveds): log.debug(f"Parsing received {idx + 1}/{n}") log.debug(f"Try to parse {received!r}") + diagnostics = [] + raw_evidence = received.encode("utf-8", "backslashreplace").decode("utf-8") + try: + # BytesParser stores raw octets using surrogateescape. Decode + # valid UTF-8 for display, retaining malformed octets losslessly. + value = received.encode("utf-8", "surrogateescape").decode( + "utf-8", "surrogateescape" + ) + except UnicodeEncodeError: + value = received try: - # try to parse the current received header... - values_by_clause = parse_received(received) - except MailParserReceivedParsingError: - # if we can't, let's append the raw - parsed.append({"raw": received}) + values_by_clause = parse_received(value, defects=diagnostics) + except MailParserReceivedParsingError as error: + # Preserve original folding and delimiters, not normalized text. + parsed.append({"raw": raw_evidence}) + reason = str(error) + # Unrecognized but balanced obsolete trace tokens are not + # necessarily invalid (RFC 5322 section 4.5.7). + if not reason.startswith("Unable to match any clauses"): + diagnostics.append({"reason": reason, "recovered": False}) else: - # otherwise append the full values_by_clause dict - parsed.append(values_by_clause) + parsed.append( + { + key: item.encode("utf-8", "backslashreplace").decode("utf-8") + for key, item in values_by_clause.items() + } + ) + if any(0xD800 <= ord(char) <= 0xDFFF for char in value): + diagnostics.append( + {"reason": "invalid-utf8", "recovered": "raw" not in parsed[-1]} + ) + if defects is not None: + for defect in diagnostics: + defect.update(raw=raw_evidence, header="received", occurrence=idx) + defects.append(defect) log.debug("len(receiveds) %s, len(parsed) %s" % (len(receiveds), len(parsed))) @@ -665,25 +695,29 @@ def receiveds_parsing(receiveds): else: # all's good! we have parsed or raw receiveds for each received header - return receiveds_format(parsed) + return receiveds_format( + parsed, raw_headers=receiveds, date_defects=date_defects + ) def convert_mail_date(date): + """Convert a recoverable mail date to a UTC datetime and offset. + + Args: + date (str): modern or obsolete RFC 5322 date-time value. + + Returns: + tuple: UTC datetime and the legacy signed decimal-hour offset. + An inconsistent weekday retains the numerical date; MailParser + additionally exposes its occurrence-aware diagnostic. + + Raises: + ValueError: syntax or components prevent a safe selected datetime. """ - Convert a mail date in a datetime object. - """ - log.debug(f"Date to parse: {date!r}") - d = email.utils.parsedate_tz(date) - if d is None: + result = parse_mail_date(date) + if result.value is None: raise ValueError(f"Cannot parse date: {date!r}") - log.debug(f"Date parsed: {d!r}") - t = email.utils.mktime_tz(d) - log.debug(f"Date parsed in timestamp: {t!r}") - date_utc = datetime.datetime.fromtimestamp(t, datetime.timezone.utc) - timezone = d[9] / 3600.0 if d[9] else 0 - timezone = f"{timezone:+.1f}" - log.debug(f"Calculated timezone: {timezone!r}") - return date_utc, timezone + return result.value, result.timezone def receiveds_not_parsed(receiveds): @@ -712,13 +746,15 @@ def receiveds_not_parsed(receiveds): return output -def receiveds_format(receiveds): +def receiveds_format(receiveds, *, raw_headers=None, date_defects=None): """ Given a list of receiveds hop, adds metadata and reformat field values Args: receiveds (list): list of receiveds hops already formatted + raw_headers (list): optional original fields in wire order. + date_defects (list): optional destination for timestamp diagnostics. Returns: list of receiveds reformated and with new fields @@ -728,30 +764,23 @@ def receiveds_format(receiveds): output = [] counter = Counter() - for i in receiveds[::-1]: + for occurrence in range(len(receiveds) - 1, -1, -1): + i = receiveds[occurrence] # Clean strings - j = {k: v.strip() for k, v in i.items() if v} + j = {k: (v if k == "raw" else v.strip()) for k, v in i.items() if v} # Add hop j["hop"] = counter["hop"] + 1 # Add UTC date - if i.get("date"): - # Modify date to manage strange header like: - # "for ; Tue, 7 Mar 2017 14:29:24 -0800", - i["date"] = i["date"].split(";")[-1] - # Strip leading RFC 2822 comments like: - # "(version=TLSv1/SSLv3 cipher=AES128-GCM-SHA256 bits=128/128) Wed, ..." - i["date"] = re.sub(r"^\s*(?:\([^)]*\)\s*)+", "", i["date"]) - try: - j["date_utc"], _ = convert_mail_date(i["date"]) - except (TypeError, ValueError, OverflowError, OSError): - # Out-of-range dates fail in three different ways: a huge - # year overflows int64 inside calendar.timegm() - # (OverflowError), and a huge timezone offset pushes the - # timestamp into the band where datetime.fromtimestamp() - # reports EOVERFLOW (OSError). - j["date_utc"] = None + if "date" in i: + result = parse_mail_date(i["date"]) + j["date_utc"] = result.value + if date_defects is not None: + raw = raw_headers[occurrence] if raw_headers else i["date"] + date_defects.extend( + date_diagnostics(result, raw, "received", occurrence) + ) # Add delay size = len(output) diff --git a/tests/fixtures/corpus_expected.json b/tests/fixtures/corpus_expected.json new file mode 100644 index 0000000..e9edf3d --- /dev/null +++ b/tests/fixtures/corpus_expected.json @@ -0,0 +1,721 @@ +{ + "baseline_revision": "522e1f37bef2309dcec8107964ba1e74377b24e6", + "cases": { + "mail_malformed_1/bytes": { + "attachments": "2d28ff49565070b77e1a0a3c8eb76e287be7b16c551b331c0a31e8e337e88e95", + "headers": "552a99598f369d274b2f1e970e5817b58a8ecd5c239f7b3c826880ab517f19ca", + "mail": "fbb52ec018b7d1dcf76d38d3daefd547c7f319c7a5d6d41299fcc7c4fddb3364", + "mime": "4b463f6fcbf2a77ab017822a44996c9346321148df8fb2925a38b8571285f08f", + "partial": "2cb2c8ffbaae031bdd620f669094c171c01a278c37da6f23140b15c5b89ffa5d", + "text_html": "5d1f3e8e0e3432000a59e99e4a18b30c5bd081b09e8159ff201a7e1060ab44d6", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "46db0fe400ee4228d58ae6cc94e876cb946a8ead664066a0c238d6e12de80715" + }, + "mail_malformed_1/file": { + "attachments": "2d28ff49565070b77e1a0a3c8eb76e287be7b16c551b331c0a31e8e337e88e95", + "headers": "552a99598f369d274b2f1e970e5817b58a8ecd5c239f7b3c826880ab517f19ca", + "mail": "fbb52ec018b7d1dcf76d38d3daefd547c7f319c7a5d6d41299fcc7c4fddb3364", + "mime": "4b463f6fcbf2a77ab017822a44996c9346321148df8fb2925a38b8571285f08f", + "partial": "2cb2c8ffbaae031bdd620f669094c171c01a278c37da6f23140b15c5b89ffa5d", + "text_html": "5d1f3e8e0e3432000a59e99e4a18b30c5bd081b09e8159ff201a7e1060ab44d6", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "46db0fe400ee4228d58ae6cc94e876cb946a8ead664066a0c238d6e12de80715" + }, + "mail_malformed_1/string": { + "attachments": "2d28ff49565070b77e1a0a3c8eb76e287be7b16c551b331c0a31e8e337e88e95", + "headers": "552a99598f369d274b2f1e970e5817b58a8ecd5c239f7b3c826880ab517f19ca", + "mail": "fbb52ec018b7d1dcf76d38d3daefd547c7f319c7a5d6d41299fcc7c4fddb3364", + "mime": "4b463f6fcbf2a77ab017822a44996c9346321148df8fb2925a38b8571285f08f", + "partial": "2cb2c8ffbaae031bdd620f669094c171c01a278c37da6f23140b15c5b89ffa5d", + "text_html": "5d1f3e8e0e3432000a59e99e4a18b30c5bd081b09e8159ff201a7e1060ab44d6", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "46db0fe400ee4228d58ae6cc94e876cb946a8ead664066a0c238d6e12de80715" + }, + "mail_malformed_2/bytes": { + "attachments": "52f846b01df6d8d3898630e6cfcfe22e6fb7c43fc5a77c5af7b120e9d4eaa3c0", + "headers": "f74c1bf6e04613659aedeeede737fc76170223bbd0d765558dc97cbb92dafeaa", + "mail": "a47308ddad865058900ebafbd2493a47bfd505e424145ec5d8818955f4951658", + "mime": "70f9d8496f803ff897c39bf894b62864e8092f7b3f494fac66866e830b069b9b", + "partial": "429e1b2c194c1be5cf6bf5df53df6da22d40fe75a7392a7c8fcfd102ebda3a0e", + "text_html": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "42b294a0d21d34329069dfd0fc96e9bda0ae941d54ca72051908febe9e68a974" + }, + "mail_malformed_2/file": { + "attachments": "52f846b01df6d8d3898630e6cfcfe22e6fb7c43fc5a77c5af7b120e9d4eaa3c0", + "headers": "f74c1bf6e04613659aedeeede737fc76170223bbd0d765558dc97cbb92dafeaa", + "mail": "a47308ddad865058900ebafbd2493a47bfd505e424145ec5d8818955f4951658", + "mime": "70f9d8496f803ff897c39bf894b62864e8092f7b3f494fac66866e830b069b9b", + "partial": "429e1b2c194c1be5cf6bf5df53df6da22d40fe75a7392a7c8fcfd102ebda3a0e", + "text_html": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "42b294a0d21d34329069dfd0fc96e9bda0ae941d54ca72051908febe9e68a974" + }, + "mail_malformed_2/string": { + "attachments": "52f846b01df6d8d3898630e6cfcfe22e6fb7c43fc5a77c5af7b120e9d4eaa3c0", + "headers": "f74c1bf6e04613659aedeeede737fc76170223bbd0d765558dc97cbb92dafeaa", + "mail": "a47308ddad865058900ebafbd2493a47bfd505e424145ec5d8818955f4951658", + "mime": "70f9d8496f803ff897c39bf894b62864e8092f7b3f494fac66866e830b069b9b", + "partial": "429e1b2c194c1be5cf6bf5df53df6da22d40fe75a7392a7c8fcfd102ebda3a0e", + "text_html": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "42b294a0d21d34329069dfd0fc96e9bda0ae941d54ca72051908febe9e68a974" + }, + "mail_malformed_3/bytes": { + "attachments": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "headers": "7238fc92c5c185df74b584c7aa29efdb0f1256620b546b8f703c3fed9dfa84c4", + "mail": "e2a1be00522d0241df3120472ee0d8dd62888e26cc054117de9d85095fedc21f", + "mime": "a9d3b0d29e9285552edf31eb036495ea8d14a83c366c4f9d2e1aa1eb9192d9b8", + "partial": "bd12044928f7dd6cd362103259691938c71df4824f218e84bbdedbad72038c3b", + "text_html": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_other": "39fbd194c300cfc90eb8d6439785053a8ceb1d3d415c1f082acaafb21b8fb41c", + "text_plain": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "mail_malformed_3/file": { + "attachments": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "headers": "7238fc92c5c185df74b584c7aa29efdb0f1256620b546b8f703c3fed9dfa84c4", + "mail": "e2a1be00522d0241df3120472ee0d8dd62888e26cc054117de9d85095fedc21f", + "mime": "a9d3b0d29e9285552edf31eb036495ea8d14a83c366c4f9d2e1aa1eb9192d9b8", + "partial": "bd12044928f7dd6cd362103259691938c71df4824f218e84bbdedbad72038c3b", + "text_html": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_other": "39fbd194c300cfc90eb8d6439785053a8ceb1d3d415c1f082acaafb21b8fb41c", + "text_plain": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "mail_malformed_3/string": { + "attachments": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "headers": "7238fc92c5c185df74b584c7aa29efdb0f1256620b546b8f703c3fed9dfa84c4", + "mail": "e2a1be00522d0241df3120472ee0d8dd62888e26cc054117de9d85095fedc21f", + "mime": "a9d3b0d29e9285552edf31eb036495ea8d14a83c366c4f9d2e1aa1eb9192d9b8", + "partial": "bd12044928f7dd6cd362103259691938c71df4824f218e84bbdedbad72038c3b", + "text_html": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_other": "39fbd194c300cfc90eb8d6439785053a8ceb1d3d415c1f082acaafb21b8fb41c", + "text_plain": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "mail_outlook_1/bytes": { + "attachments": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "headers": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + "mail": "5f14c6af9bc95928f0d89232bf2606fe8f99c0725e6f1193670314a87f41c27a", + "mime": "54d59b6382f683e66bea37928c4b960555e620705ba707ffdaec2fc16a143446", + "partial": "5f14c6af9bc95928f0d89232bf2606fe8f99c0725e6f1193670314a87f41c27a", + "text_html": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "28b64e48784986088a3ef862bcf65da64dfab40157e761a1f23a37483b674706" + }, + "mail_outlook_1/file": { + "attachments": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "headers": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + "mail": "5f14c6af9bc95928f0d89232bf2606fe8f99c0725e6f1193670314a87f41c27a", + "mime": "54d59b6382f683e66bea37928c4b960555e620705ba707ffdaec2fc16a143446", + "partial": "5f14c6af9bc95928f0d89232bf2606fe8f99c0725e6f1193670314a87f41c27a", + "text_html": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "28b64e48784986088a3ef862bcf65da64dfab40157e761a1f23a37483b674706" + }, + "mail_outlook_1/string": { + "attachments": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "headers": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + "mail": "5dde7b38fcbdc40757924a408f53426ef5fbb5e977dd026da88331a6f6445a0b", + "mime": "a741defd3802cfd1b9e15a56f492d654ece3d0c25323d6097f5d4603ace99a2e", + "partial": "5dde7b38fcbdc40757924a408f53426ef5fbb5e977dd026da88331a6f6445a0b", + "text_html": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "0b0b5d3c22554f089f8ddcdb7112bdb3a4a9705c145a1987c710bc6e6620ec51" + }, + "mail_test_1/bytes": { + "attachments": "6f64bcc2fd430f95e4f742123be0100d39dbd64062a54d7c76b638cf9cec22b4", + "headers": "790605d6686616ce05a60baab8692dbaa19dd12060a7e3c23475bcd987d89178", + "mail": "e67558914dd89dff88f2e9ba5dbc5db9a8558a35607345bff0f4e8460f89ead3", + "mime": "2987eea3a5e208ee85a277d68b6aa5ed524ea7dd85a07178ea69f0d55805da93", + "partial": "35f6ec82b2fc682ff9ef91fa16a680942df1e2e1b35051eab3b6ced65e66b87c", + "text_html": "1bc3c2d9f7bbafe77c8ce9de3531ebf0b8e6dffe8e0e9009fbf1a9317f2992cc", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "mail_test_1/file": { + "attachments": "6f64bcc2fd430f95e4f742123be0100d39dbd64062a54d7c76b638cf9cec22b4", + "headers": "790605d6686616ce05a60baab8692dbaa19dd12060a7e3c23475bcd987d89178", + "mail": "e67558914dd89dff88f2e9ba5dbc5db9a8558a35607345bff0f4e8460f89ead3", + "mime": "2987eea3a5e208ee85a277d68b6aa5ed524ea7dd85a07178ea69f0d55805da93", + "partial": "35f6ec82b2fc682ff9ef91fa16a680942df1e2e1b35051eab3b6ced65e66b87c", + "text_html": "1bc3c2d9f7bbafe77c8ce9de3531ebf0b8e6dffe8e0e9009fbf1a9317f2992cc", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "mail_test_1/string": { + "attachments": "6f64bcc2fd430f95e4f742123be0100d39dbd64062a54d7c76b638cf9cec22b4", + "headers": "790605d6686616ce05a60baab8692dbaa19dd12060a7e3c23475bcd987d89178", + "mail": "e67558914dd89dff88f2e9ba5dbc5db9a8558a35607345bff0f4e8460f89ead3", + "mime": "2add858696edc46c9bc1ae8183c3fd8ab12ecd790dd24352b42fd5cbb282fcf3", + "partial": "35f6ec82b2fc682ff9ef91fa16a680942df1e2e1b35051eab3b6ced65e66b87c", + "text_html": "1bc3c2d9f7bbafe77c8ce9de3531ebf0b8e6dffe8e0e9009fbf1a9317f2992cc", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "mail_test_10/bytes": { + "attachments": "c6fcd07c78ad3c65f0db93b66bc78dfd10f7b708ed1e661d8f5ec7ab3ea8f9e4", + "headers": "361de78bd3daf5b5802462266b3196966677c31a892c94d038da6db5b9a37178", + "mail": "3a8a8bee8b78a4c54cea81c112810c0fe5ee702eb5d173bf1ab4be35ec758bab", + "mime": "3b065f643572b7a1760f7b40e75fd0d2e968b2b97f9d205e4e4225c60e039e1d", + "partial": "9bc4e777bfb86eb702e9ec8205f82e634758c17a02bc1f4b88852e5df859cf99", + "text_html": "d4e811b15b06357b748a4eba7db3cbc5a375cb30aa1126bebb397fae878c8f27", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "b4d7d5c310e25ed085bdcbc7f066f61f36124eea78fb0886a491a2c246516182" + }, + "mail_test_10/file": { + "attachments": "c6fcd07c78ad3c65f0db93b66bc78dfd10f7b708ed1e661d8f5ec7ab3ea8f9e4", + "headers": "361de78bd3daf5b5802462266b3196966677c31a892c94d038da6db5b9a37178", + "mail": "3a8a8bee8b78a4c54cea81c112810c0fe5ee702eb5d173bf1ab4be35ec758bab", + "mime": "3b065f643572b7a1760f7b40e75fd0d2e968b2b97f9d205e4e4225c60e039e1d", + "partial": "9bc4e777bfb86eb702e9ec8205f82e634758c17a02bc1f4b88852e5df859cf99", + "text_html": "d4e811b15b06357b748a4eba7db3cbc5a375cb30aa1126bebb397fae878c8f27", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "b4d7d5c310e25ed085bdcbc7f066f61f36124eea78fb0886a491a2c246516182" + }, + "mail_test_10/string": { + "attachments": "c6fcd07c78ad3c65f0db93b66bc78dfd10f7b708ed1e661d8f5ec7ab3ea8f9e4", + "headers": "361de78bd3daf5b5802462266b3196966677c31a892c94d038da6db5b9a37178", + "mail": "3a8a8bee8b78a4c54cea81c112810c0fe5ee702eb5d173bf1ab4be35ec758bab", + "mime": "3b065f643572b7a1760f7b40e75fd0d2e968b2b97f9d205e4e4225c60e039e1d", + "partial": "9bc4e777bfb86eb702e9ec8205f82e634758c17a02bc1f4b88852e5df859cf99", + "text_html": "d4e811b15b06357b748a4eba7db3cbc5a375cb30aa1126bebb397fae878c8f27", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "b4d7d5c310e25ed085bdcbc7f066f61f36124eea78fb0886a491a2c246516182" + }, + "mail_test_11/bytes": { + "attachments": "88f8da4ac4802a84d13c44f4d7d4cb732ec8374b095fc5255e43e9bab30f49ca", + "headers": "f05dd31157676e8cb79485b13ce245ffc374d51bf9aebefe40475c5936faf638", + "mail": "a41d6faee637af38e5cbb85f871e1afa3e309082f5a6d549bd41fa7a52a8b1b3", + "mime": "11afb805bd69715d66aa6ac0d1f1480b4150424f99e328a600d195eb8bc9e15f", + "partial": "d0f9d143392f48b148919b54ca59c54eb25a94542113f3a84f31383f195c024a", + "text_html": "1bc3c2d9f7bbafe77c8ce9de3531ebf0b8e6dffe8e0e9009fbf1a9317f2992cc", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "mail_test_11/file": { + "attachments": "88f8da4ac4802a84d13c44f4d7d4cb732ec8374b095fc5255e43e9bab30f49ca", + "headers": "f05dd31157676e8cb79485b13ce245ffc374d51bf9aebefe40475c5936faf638", + "mail": "a41d6faee637af38e5cbb85f871e1afa3e309082f5a6d549bd41fa7a52a8b1b3", + "mime": "11afb805bd69715d66aa6ac0d1f1480b4150424f99e328a600d195eb8bc9e15f", + "partial": "d0f9d143392f48b148919b54ca59c54eb25a94542113f3a84f31383f195c024a", + "text_html": "1bc3c2d9f7bbafe77c8ce9de3531ebf0b8e6dffe8e0e9009fbf1a9317f2992cc", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "mail_test_11/string": { + "attachments": "88f8da4ac4802a84d13c44f4d7d4cb732ec8374b095fc5255e43e9bab30f49ca", + "headers": "f05dd31157676e8cb79485b13ce245ffc374d51bf9aebefe40475c5936faf638", + "mail": "a41d6faee637af38e5cbb85f871e1afa3e309082f5a6d549bd41fa7a52a8b1b3", + "mime": "11afb805bd69715d66aa6ac0d1f1480b4150424f99e328a600d195eb8bc9e15f", + "partial": "d0f9d143392f48b148919b54ca59c54eb25a94542113f3a84f31383f195c024a", + "text_html": "1bc3c2d9f7bbafe77c8ce9de3531ebf0b8e6dffe8e0e9009fbf1a9317f2992cc", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "mail_test_12/bytes": { + "attachments": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "headers": "972b26cd599cf521b1738735ed39dc145010dd526908b9032bc5145ab38a7d31", + "mail": "627eafe12950e3ec84a31265a438d0bf734e0dfa654204a9cc0384e12df4b778", + "mime": "98ff961858fe6b8b474e6cb064a9a3f94b50d33944db4222a44301addb9a086e", + "partial": "0fd4f146b7f4f29eadc290f32c3da4123eab7473a76ae72fdea7ab8964f2bba9", + "text_html": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "3db1c81fc4ed22629e125087c4ae2b59bd7e8388b47bd5a447d0c0ccf5191ff1" + }, + "mail_test_12/file": { + "attachments": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "headers": "972b26cd599cf521b1738735ed39dc145010dd526908b9032bc5145ab38a7d31", + "mail": "627eafe12950e3ec84a31265a438d0bf734e0dfa654204a9cc0384e12df4b778", + "mime": "98ff961858fe6b8b474e6cb064a9a3f94b50d33944db4222a44301addb9a086e", + "partial": "0fd4f146b7f4f29eadc290f32c3da4123eab7473a76ae72fdea7ab8964f2bba9", + "text_html": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "3db1c81fc4ed22629e125087c4ae2b59bd7e8388b47bd5a447d0c0ccf5191ff1" + }, + "mail_test_12/string": { + "attachments": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "headers": "972b26cd599cf521b1738735ed39dc145010dd526908b9032bc5145ab38a7d31", + "mail": "835f4adc08dfcc473292abc6d6815fc506124bc789ea77830ecaa7f15146264a", + "mime": "26e127617fd9709a4b246fe1c200be5ccc39105d0dcfba30c41ff5a74df507b0", + "partial": "bfa3cc66089d44536af7f40c69d9a5e80b52ec9727ffe6feefdbcfffeca2b3e1", + "text_html": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "3db1c81fc4ed22629e125087c4ae2b59bd7e8388b47bd5a447d0c0ccf5191ff1" + }, + "mail_test_13/bytes": { + "attachments": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "headers": "a20dc3e709b2ca0395e5dce8a0c00dfc949222bc1154b9019854838d5b49103f", + "mail": "e1aa37245f48343495cb62fcbd3976c00c15e41c7049c6a844d7582556bb7124", + "mime": "02f16da5a3600efa075b332a9dc66ca44e434377ef8235ee885716b1f7d8a91d", + "partial": "cec7d0d119c7c9e336478bbc99250796232b653272cd9e35b7b979e2681817f6", + "text_html": "cf64368f3d5780bc88c98f99d4c9fb9bac3ab98a27440f29e11f926669af0a56", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "d237338558261ea8cd1bff4d809ffc261b0d67b87f488aba8ae153808b14763a" + }, + "mail_test_13/file": { + "attachments": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "headers": "a20dc3e709b2ca0395e5dce8a0c00dfc949222bc1154b9019854838d5b49103f", + "mail": "e1aa37245f48343495cb62fcbd3976c00c15e41c7049c6a844d7582556bb7124", + "mime": "02f16da5a3600efa075b332a9dc66ca44e434377ef8235ee885716b1f7d8a91d", + "partial": "cec7d0d119c7c9e336478bbc99250796232b653272cd9e35b7b979e2681817f6", + "text_html": "cf64368f3d5780bc88c98f99d4c9fb9bac3ab98a27440f29e11f926669af0a56", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "d237338558261ea8cd1bff4d809ffc261b0d67b87f488aba8ae153808b14763a" + }, + "mail_test_13/string": { + "attachments": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "headers": "a20dc3e709b2ca0395e5dce8a0c00dfc949222bc1154b9019854838d5b49103f", + "mail": "e1aa37245f48343495cb62fcbd3976c00c15e41c7049c6a844d7582556bb7124", + "mime": "02f16da5a3600efa075b332a9dc66ca44e434377ef8235ee885716b1f7d8a91d", + "partial": "cec7d0d119c7c9e336478bbc99250796232b653272cd9e35b7b979e2681817f6", + "text_html": "cf64368f3d5780bc88c98f99d4c9fb9bac3ab98a27440f29e11f926669af0a56", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "d237338558261ea8cd1bff4d809ffc261b0d67b87f488aba8ae153808b14763a" + }, + "mail_test_14/bytes": { + "attachments": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "headers": "d18f2149c82088d58780699675ab9853d3f4fd370ec84b28c37b6057d611f57f", + "mail": "c300b343dbc358a3833b0103ff6b76f2e1a6c7456d7b7411db41c98692cb49d5", + "mime": "700b12498f243edc5ea3c6945e93874a5e791fe34126b136ab5e39fbbf4e9d3e", + "partial": "2b56334820e526cea86732f91b89efeaff2f11c8665ef9ac70d0436b4558eded", + "text_html": "62299de06621556ecfcaeac2a39a8c2bc8c61d605c2a2a309703c65204ee23b6", + "text_other": "203b4347977eddf2501843dfcd515501bd3c65a8f47ed2494e086ec033972f52", + "text_plain": "2b7da2096316cf249f819d0f446a1db9e7562371970eaf37fd3ad5886add46af" + }, + "mail_test_14/file": { + "attachments": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "headers": "d18f2149c82088d58780699675ab9853d3f4fd370ec84b28c37b6057d611f57f", + "mail": "c300b343dbc358a3833b0103ff6b76f2e1a6c7456d7b7411db41c98692cb49d5", + "mime": "700b12498f243edc5ea3c6945e93874a5e791fe34126b136ab5e39fbbf4e9d3e", + "partial": "2b56334820e526cea86732f91b89efeaff2f11c8665ef9ac70d0436b4558eded", + "text_html": "62299de06621556ecfcaeac2a39a8c2bc8c61d605c2a2a309703c65204ee23b6", + "text_other": "203b4347977eddf2501843dfcd515501bd3c65a8f47ed2494e086ec033972f52", + "text_plain": "2b7da2096316cf249f819d0f446a1db9e7562371970eaf37fd3ad5886add46af" + }, + "mail_test_14/string": { + "attachments": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "headers": "d18f2149c82088d58780699675ab9853d3f4fd370ec84b28c37b6057d611f57f", + "mail": "c300b343dbc358a3833b0103ff6b76f2e1a6c7456d7b7411db41c98692cb49d5", + "mime": "700b12498f243edc5ea3c6945e93874a5e791fe34126b136ab5e39fbbf4e9d3e", + "partial": "2b56334820e526cea86732f91b89efeaff2f11c8665ef9ac70d0436b4558eded", + "text_html": "62299de06621556ecfcaeac2a39a8c2bc8c61d605c2a2a309703c65204ee23b6", + "text_other": "203b4347977eddf2501843dfcd515501bd3c65a8f47ed2494e086ec033972f52", + "text_plain": "2b7da2096316cf249f819d0f446a1db9e7562371970eaf37fd3ad5886add46af" + }, + "mail_test_15/bytes": { + "attachments": "dc1972966d151d890f565f6ff87541e5411f144310b4fac49d62907f580f9595", + "headers": "f93dc663149c9d25a4a3011f9ccfb83573941b79470de524879642ad093b7bf8", + "mail": "009adfa6dd7f5d47d3463eda0a0e0603fbc3ec141eda1d64a2f41ccad655cb2b", + "mime": "94a164b21e6f8bcc1c6a4b8bcbe447ca565b28a90bf2aa696751d36867737ee6", + "partial": "65e934798ed573547d2adf3f361baed6501e327aa8d857137e1c3d7773800e1f", + "text_html": "c0d397aae8307ebf23f9a1bde8da851820f2fcf73c3a71d0cda41dcee8eb430b", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "357d262c842e934067d988e4995b357ccb2704ba84a4a535b8502b88b7b5c546" + }, + "mail_test_15/file": { + "attachments": "dc1972966d151d890f565f6ff87541e5411f144310b4fac49d62907f580f9595", + "headers": "f93dc663149c9d25a4a3011f9ccfb83573941b79470de524879642ad093b7bf8", + "mail": "009adfa6dd7f5d47d3463eda0a0e0603fbc3ec141eda1d64a2f41ccad655cb2b", + "mime": "94a164b21e6f8bcc1c6a4b8bcbe447ca565b28a90bf2aa696751d36867737ee6", + "partial": "65e934798ed573547d2adf3f361baed6501e327aa8d857137e1c3d7773800e1f", + "text_html": "c0d397aae8307ebf23f9a1bde8da851820f2fcf73c3a71d0cda41dcee8eb430b", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "357d262c842e934067d988e4995b357ccb2704ba84a4a535b8502b88b7b5c546" + }, + "mail_test_15/string": { + "attachments": "dc1972966d151d890f565f6ff87541e5411f144310b4fac49d62907f580f9595", + "headers": "f93dc663149c9d25a4a3011f9ccfb83573941b79470de524879642ad093b7bf8", + "mail": "009adfa6dd7f5d47d3463eda0a0e0603fbc3ec141eda1d64a2f41ccad655cb2b", + "mime": "94a164b21e6f8bcc1c6a4b8bcbe447ca565b28a90bf2aa696751d36867737ee6", + "partial": "65e934798ed573547d2adf3f361baed6501e327aa8d857137e1c3d7773800e1f", + "text_html": "c0d397aae8307ebf23f9a1bde8da851820f2fcf73c3a71d0cda41dcee8eb430b", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "357d262c842e934067d988e4995b357ccb2704ba84a4a535b8502b88b7b5c546" + }, + "mail_test_16/bytes": { + "attachments": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "headers": "94713f44b5d035eb0e3ac5f22e0ff039429fa01de60d6ade963788b1cc0f8ede", + "mail": "38751365e1e5555b3bd52d8947421c515b0bf8caa0e7519b7ca2315e5b1c494a", + "mime": "acbe7c65aad0662b05e519fb27a4445c2519fc8c8d7d36a378de7198fed83b24", + "partial": "eb5cd7852ae32c54087a5833bc772c706d8488d0ceeb3fe56398009f52d2d331", + "text_html": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "c86e7d9bd4b864ad234199fba72482e0feaf01d0c8ce890474088c003f334730" + }, + "mail_test_16/file": { + "attachments": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "headers": "94713f44b5d035eb0e3ac5f22e0ff039429fa01de60d6ade963788b1cc0f8ede", + "mail": "38751365e1e5555b3bd52d8947421c515b0bf8caa0e7519b7ca2315e5b1c494a", + "mime": "acbe7c65aad0662b05e519fb27a4445c2519fc8c8d7d36a378de7198fed83b24", + "partial": "eb5cd7852ae32c54087a5833bc772c706d8488d0ceeb3fe56398009f52d2d331", + "text_html": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "c86e7d9bd4b864ad234199fba72482e0feaf01d0c8ce890474088c003f334730" + }, + "mail_test_16/string": { + "attachments": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "headers": "94713f44b5d035eb0e3ac5f22e0ff039429fa01de60d6ade963788b1cc0f8ede", + "mail": "38751365e1e5555b3bd52d8947421c515b0bf8caa0e7519b7ca2315e5b1c494a", + "mime": "acbe7c65aad0662b05e519fb27a4445c2519fc8c8d7d36a378de7198fed83b24", + "partial": "eb5cd7852ae32c54087a5833bc772c706d8488d0ceeb3fe56398009f52d2d331", + "text_html": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "c86e7d9bd4b864ad234199fba72482e0feaf01d0c8ce890474088c003f334730" + }, + "mail_test_17/bytes": { + "attachments": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "headers": "d49bdcbe818fefd4d2a21f312dccc3b1f3f01cb565929d05c246c1b31cb60985", + "mail": "a02097a96779b92ae9ca5dae619ed1832fbc52819d87594e283ec16e098f605d", + "mime": "c4b0708c899e35c64e4a8518d67993a9e83c91de9cca218a952e727bcfaa5517", + "partial": "97037cea7eef9a7fa1633d6285fd5a2f98736ab7f77efcdd74004003156ea003", + "text_html": "893e554ba2685d55955726b4d5546f419e9d964f67458e42df38ce5f784db2fe", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "3da3bc0692a5eb64acdee272eee66146324e7b028b95360b0ea5e77d37cf9601" + }, + "mail_test_17/file": { + "attachments": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "headers": "d49bdcbe818fefd4d2a21f312dccc3b1f3f01cb565929d05c246c1b31cb60985", + "mail": "a02097a96779b92ae9ca5dae619ed1832fbc52819d87594e283ec16e098f605d", + "mime": "c4b0708c899e35c64e4a8518d67993a9e83c91de9cca218a952e727bcfaa5517", + "partial": "97037cea7eef9a7fa1633d6285fd5a2f98736ab7f77efcdd74004003156ea003", + "text_html": "893e554ba2685d55955726b4d5546f419e9d964f67458e42df38ce5f784db2fe", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "3da3bc0692a5eb64acdee272eee66146324e7b028b95360b0ea5e77d37cf9601" + }, + "mail_test_17/string": { + "attachments": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "headers": "d49bdcbe818fefd4d2a21f312dccc3b1f3f01cb565929d05c246c1b31cb60985", + "mail": "a02097a96779b92ae9ca5dae619ed1832fbc52819d87594e283ec16e098f605d", + "mime": "c4b0708c899e35c64e4a8518d67993a9e83c91de9cca218a952e727bcfaa5517", + "partial": "97037cea7eef9a7fa1633d6285fd5a2f98736ab7f77efcdd74004003156ea003", + "text_html": "893e554ba2685d55955726b4d5546f419e9d964f67458e42df38ce5f784db2fe", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "3da3bc0692a5eb64acdee272eee66146324e7b028b95360b0ea5e77d37cf9601" + }, + "mail_test_18/bytes": { + "attachments": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "headers": "15be351269e27ff38e4177bd108463a8f18520e7a394554087542882fe7b1568", + "mail": "eb0400e6d8531ca2c30302962226943a32627b90303dd47ef92903140d7cab8a", + "mime": "0664c6297b88d1b8bd18cd283a1d7df439c24bce742f8098e7558dd904a5ec1a", + "partial": "e60d598cf26f1d9fd51a0cf5a4f47718f62ae4b705674fdd69ac645067648932", + "text_html": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "22621ce38dc7b29fa2bfa753c86217ebd9cac03c4b4615bf4dd7e014d3cd8fdd" + }, + "mail_test_18/file": { + "attachments": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "headers": "15be351269e27ff38e4177bd108463a8f18520e7a394554087542882fe7b1568", + "mail": "eb0400e6d8531ca2c30302962226943a32627b90303dd47ef92903140d7cab8a", + "mime": "0664c6297b88d1b8bd18cd283a1d7df439c24bce742f8098e7558dd904a5ec1a", + "partial": "e60d598cf26f1d9fd51a0cf5a4f47718f62ae4b705674fdd69ac645067648932", + "text_html": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "22621ce38dc7b29fa2bfa753c86217ebd9cac03c4b4615bf4dd7e014d3cd8fdd" + }, + "mail_test_18/string": { + "attachments": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "headers": "15be351269e27ff38e4177bd108463a8f18520e7a394554087542882fe7b1568", + "mail": "eb0400e6d8531ca2c30302962226943a32627b90303dd47ef92903140d7cab8a", + "mime": "0664c6297b88d1b8bd18cd283a1d7df439c24bce742f8098e7558dd904a5ec1a", + "partial": "e60d598cf26f1d9fd51a0cf5a4f47718f62ae4b705674fdd69ac645067648932", + "text_html": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "22621ce38dc7b29fa2bfa753c86217ebd9cac03c4b4615bf4dd7e014d3cd8fdd" + }, + "mail_test_19/bytes": { + "attachments": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "headers": "16d613a4c4289b1adb96cbd379e013cec227b8cda176ffa9be424d7594017a07", + "mail": "64806b116c702c009f97dac9c7a8ece88097d094b9d726b4b885dabd1b39f6ab", + "mime": "decdb8ba4622273aefd7d0a41f669cb594569ed5b7dd53022c805210077b7783", + "partial": "82d4378e25d09ceae56ef499330259cfc657cf55143489c17dbb647125ec61f3", + "text_html": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "db4f34469be339f5d26ca7d19067e4a82452888c5201e42a9d752a76ea2b2aaf" + }, + "mail_test_19/file": { + "attachments": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "headers": "16d613a4c4289b1adb96cbd379e013cec227b8cda176ffa9be424d7594017a07", + "mail": "64806b116c702c009f97dac9c7a8ece88097d094b9d726b4b885dabd1b39f6ab", + "mime": "decdb8ba4622273aefd7d0a41f669cb594569ed5b7dd53022c805210077b7783", + "partial": "82d4378e25d09ceae56ef499330259cfc657cf55143489c17dbb647125ec61f3", + "text_html": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "db4f34469be339f5d26ca7d19067e4a82452888c5201e42a9d752a76ea2b2aaf" + }, + "mail_test_19/string": { + "attachments": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "headers": "16d613a4c4289b1adb96cbd379e013cec227b8cda176ffa9be424d7594017a07", + "mail": "64806b116c702c009f97dac9c7a8ece88097d094b9d726b4b885dabd1b39f6ab", + "mime": "decdb8ba4622273aefd7d0a41f669cb594569ed5b7dd53022c805210077b7783", + "partial": "82d4378e25d09ceae56ef499330259cfc657cf55143489c17dbb647125ec61f3", + "text_html": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "db4f34469be339f5d26ca7d19067e4a82452888c5201e42a9d752a76ea2b2aaf" + }, + "mail_test_2/bytes": { + "attachments": "c235c18763867f8dd99e4b1d4956902818cabcc11e79a53c0aacec2508658aa9", + "headers": "4cbf5719f9775d846dded8524c4e40c18e87eb95f16d2806a2f85519e5624216", + "mail": "ffaaba13ea3f892b61dd4fa98f6f83f81a54ea4378eda21bff7df6119489720e", + "mime": "a49cfca11b5aae710c759f12a606a00440453e496e1c64cf25b68bacc5045309", + "partial": "77db9a01b27162b7cd45645269e428ced0bc0612d8723f53ae92acafe50234ed", + "text_html": "f2dd55c90d54cbd147b87a855f6c53880c1aefe9c7b8aa1c4a096da9dec9af01", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "c24c3a4decb0cfb705bc7201e6873702cb81fee64fa6f32eb2865bc21d4fcbee" + }, + "mail_test_2/file": { + "attachments": "c235c18763867f8dd99e4b1d4956902818cabcc11e79a53c0aacec2508658aa9", + "headers": "4cbf5719f9775d846dded8524c4e40c18e87eb95f16d2806a2f85519e5624216", + "mail": "ffaaba13ea3f892b61dd4fa98f6f83f81a54ea4378eda21bff7df6119489720e", + "mime": "a49cfca11b5aae710c759f12a606a00440453e496e1c64cf25b68bacc5045309", + "partial": "77db9a01b27162b7cd45645269e428ced0bc0612d8723f53ae92acafe50234ed", + "text_html": "f2dd55c90d54cbd147b87a855f6c53880c1aefe9c7b8aa1c4a096da9dec9af01", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "c24c3a4decb0cfb705bc7201e6873702cb81fee64fa6f32eb2865bc21d4fcbee" + }, + "mail_test_2/string": { + "attachments": "c235c18763867f8dd99e4b1d4956902818cabcc11e79a53c0aacec2508658aa9", + "headers": "4cbf5719f9775d846dded8524c4e40c18e87eb95f16d2806a2f85519e5624216", + "mail": "ffaaba13ea3f892b61dd4fa98f6f83f81a54ea4378eda21bff7df6119489720e", + "mime": "a49cfca11b5aae710c759f12a606a00440453e496e1c64cf25b68bacc5045309", + "partial": "77db9a01b27162b7cd45645269e428ced0bc0612d8723f53ae92acafe50234ed", + "text_html": "f2dd55c90d54cbd147b87a855f6c53880c1aefe9c7b8aa1c4a096da9dec9af01", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "c24c3a4decb0cfb705bc7201e6873702cb81fee64fa6f32eb2865bc21d4fcbee" + }, + "mail_test_3/bytes": { + "attachments": "fc8c56de69fd1c348387a0af43ff97fc5a340fd335ee54aa89b123f0a4d38e92", + "headers": "8220baaee4b1ad597a9c6d9034d41b67670e29b04c54e7df2a7f877b9aec2457", + "mail": "c06502fe2b7290dae0392fe5094dc4ef678a2bf3a0bb9f876a7055f19108c9c0", + "mime": "d33e6e9ff4781aa114d4f5985ed1ea554fea2763d61deb52e164dd218ed11d62", + "partial": "fce1fd82f4deca68ff31938f4384764c690d844d96ca2c8919d65e28c3bbd484", + "text_html": "56302e187ff64cde34688a956d560ae38b6135dd2fa9a64319cd5fe33434be73", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "e7001cd252710a3b64e5074ded8c135e96b7917df54db9a606d0c2b0153c6f95" + }, + "mail_test_3/file": { + "attachments": "fc8c56de69fd1c348387a0af43ff97fc5a340fd335ee54aa89b123f0a4d38e92", + "headers": "8220baaee4b1ad597a9c6d9034d41b67670e29b04c54e7df2a7f877b9aec2457", + "mail": "c06502fe2b7290dae0392fe5094dc4ef678a2bf3a0bb9f876a7055f19108c9c0", + "mime": "d33e6e9ff4781aa114d4f5985ed1ea554fea2763d61deb52e164dd218ed11d62", + "partial": "fce1fd82f4deca68ff31938f4384764c690d844d96ca2c8919d65e28c3bbd484", + "text_html": "56302e187ff64cde34688a956d560ae38b6135dd2fa9a64319cd5fe33434be73", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "e7001cd252710a3b64e5074ded8c135e96b7917df54db9a606d0c2b0153c6f95" + }, + "mail_test_3/string": { + "attachments": "fc8c56de69fd1c348387a0af43ff97fc5a340fd335ee54aa89b123f0a4d38e92", + "headers": "8220baaee4b1ad597a9c6d9034d41b67670e29b04c54e7df2a7f877b9aec2457", + "mail": "c06502fe2b7290dae0392fe5094dc4ef678a2bf3a0bb9f876a7055f19108c9c0", + "mime": "d33e6e9ff4781aa114d4f5985ed1ea554fea2763d61deb52e164dd218ed11d62", + "partial": "fce1fd82f4deca68ff31938f4384764c690d844d96ca2c8919d65e28c3bbd484", + "text_html": "56302e187ff64cde34688a956d560ae38b6135dd2fa9a64319cd5fe33434be73", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "e7001cd252710a3b64e5074ded8c135e96b7917df54db9a606d0c2b0153c6f95" + }, + "mail_test_4/bytes": { + "attachments": "967a3d505f586209d91a26762e8c133fe4ce66d245e8fe060985b0ab2d07998a", + "headers": "a94baecc84568de710761a02f08d6a21ef86115bceb60378a3dbca8df7a2858c", + "mail": "371566b07ba4d4954340449a4837ed3c5ec7fa22f35e69fa54b7994cede7135a", + "mime": "bff0bf039927f6c0011c4118267440d8fe6504f570d3e4c9ecf98fc487ad9906", + "partial": "adecb047b6ae411485cb699a0d3c74ee61f4be5fac7f36f6ec8203c83695dc3a", + "text_html": "c54625eb7ad8f63dacff3747f1ac9bd08056564f010eadc7c20c6bd868f6ff38", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "mail_test_4/file": { + "attachments": "967a3d505f586209d91a26762e8c133fe4ce66d245e8fe060985b0ab2d07998a", + "headers": "a94baecc84568de710761a02f08d6a21ef86115bceb60378a3dbca8df7a2858c", + "mail": "371566b07ba4d4954340449a4837ed3c5ec7fa22f35e69fa54b7994cede7135a", + "mime": "bff0bf039927f6c0011c4118267440d8fe6504f570d3e4c9ecf98fc487ad9906", + "partial": "adecb047b6ae411485cb699a0d3c74ee61f4be5fac7f36f6ec8203c83695dc3a", + "text_html": "c54625eb7ad8f63dacff3747f1ac9bd08056564f010eadc7c20c6bd868f6ff38", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "mail_test_4/string": { + "attachments": "967a3d505f586209d91a26762e8c133fe4ce66d245e8fe060985b0ab2d07998a", + "headers": "a94baecc84568de710761a02f08d6a21ef86115bceb60378a3dbca8df7a2858c", + "mail": "28379a246c2b11955d371f2daee8b22ddfc9b8b9641b1747d049df698d58d7a7", + "mime": "80b3c4b83d6441a5a8221f3537142bfd01b41e0271f3cc97b2115c4f893a4cff", + "partial": "2afb7b3a1f84eecab4ebfe685f39ccb1766ac3089d32daba5d48f2b890af09ac", + "text_html": "e5460edac5df20aa06c8b846b8bf883ec162e6d604557b154114c2ad0158c038", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "mail_test_5/bytes": { + "attachments": "1636e7d7ed9280705dac095fdd5f892bee420a4e22b87d087c993ca0c5ff52ff", + "headers": "6a743e6c2807dac9fef4117fc27b38a88f6d40d1157975150950f736db5aa848", + "mail": "961913f78d065ef4d56692440a58171ad41abac31b49c01193e80fc76eb7882d", + "mime": "e03f7355176c4285ac030c351d9300d91bdbb972af72c36204fbdbe2d3f3cb0f", + "partial": "21c7a7a1ce32090d9f5259ca4c15f3f616a238712c78655b6a25117d76b47dbd", + "text_html": "c9dba3942a8c1774e2b1f66b80de58f6179e2558070fc068bc7f17f978a4733c", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "4e4e365884d5029efde0ed67ed8c4b4529cc2fafdc16311fd1f0422ec2b6397e" + }, + "mail_test_5/file": { + "attachments": "1636e7d7ed9280705dac095fdd5f892bee420a4e22b87d087c993ca0c5ff52ff", + "headers": "6a743e6c2807dac9fef4117fc27b38a88f6d40d1157975150950f736db5aa848", + "mail": "961913f78d065ef4d56692440a58171ad41abac31b49c01193e80fc76eb7882d", + "mime": "e03f7355176c4285ac030c351d9300d91bdbb972af72c36204fbdbe2d3f3cb0f", + "partial": "21c7a7a1ce32090d9f5259ca4c15f3f616a238712c78655b6a25117d76b47dbd", + "text_html": "c9dba3942a8c1774e2b1f66b80de58f6179e2558070fc068bc7f17f978a4733c", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "4e4e365884d5029efde0ed67ed8c4b4529cc2fafdc16311fd1f0422ec2b6397e" + }, + "mail_test_5/string": { + "attachments": "1636e7d7ed9280705dac095fdd5f892bee420a4e22b87d087c993ca0c5ff52ff", + "headers": "6a743e6c2807dac9fef4117fc27b38a88f6d40d1157975150950f736db5aa848", + "mail": "961913f78d065ef4d56692440a58171ad41abac31b49c01193e80fc76eb7882d", + "mime": "e03f7355176c4285ac030c351d9300d91bdbb972af72c36204fbdbe2d3f3cb0f", + "partial": "21c7a7a1ce32090d9f5259ca4c15f3f616a238712c78655b6a25117d76b47dbd", + "text_html": "c9dba3942a8c1774e2b1f66b80de58f6179e2558070fc068bc7f17f978a4733c", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "4e4e365884d5029efde0ed67ed8c4b4529cc2fafdc16311fd1f0422ec2b6397e" + }, + "mail_test_6/bytes": { + "attachments": "b87669b0328cfb13a3f5c35d7a8306066311db34854c9baa38d5adb5a98932b6", + "headers": "a10bdf3bba43f11dbf6d930cc52a9e2affc9d2ed1ff7f355ab10617849e662fd", + "mail": "056f7aa1b9c69e961e92afcdd52b5357a6f41e0b497e3a4ef0be17af3bf149e9", + "mime": "3c7567642c25e56855e8b6d3e448a85e54fc88d53f0e398765372926e037f4a5", + "partial": "5a1b8d4c67f0a332233ef7ed59c422753d0f809b5f4c8fc6a9a5a227e3171e29", + "text_html": "c1d2ee837bd795bce4986353da5d47dd5e19b13b6c141c7960af03967d5db572", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "mail_test_6/file": { + "attachments": "b87669b0328cfb13a3f5c35d7a8306066311db34854c9baa38d5adb5a98932b6", + "headers": "a10bdf3bba43f11dbf6d930cc52a9e2affc9d2ed1ff7f355ab10617849e662fd", + "mail": "056f7aa1b9c69e961e92afcdd52b5357a6f41e0b497e3a4ef0be17af3bf149e9", + "mime": "3c7567642c25e56855e8b6d3e448a85e54fc88d53f0e398765372926e037f4a5", + "partial": "5a1b8d4c67f0a332233ef7ed59c422753d0f809b5f4c8fc6a9a5a227e3171e29", + "text_html": "c1d2ee837bd795bce4986353da5d47dd5e19b13b6c141c7960af03967d5db572", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "mail_test_6/string": { + "attachments": "b87669b0328cfb13a3f5c35d7a8306066311db34854c9baa38d5adb5a98932b6", + "headers": "a10bdf3bba43f11dbf6d930cc52a9e2affc9d2ed1ff7f355ab10617849e662fd", + "mail": "056f7aa1b9c69e961e92afcdd52b5357a6f41e0b497e3a4ef0be17af3bf149e9", + "mime": "3c7567642c25e56855e8b6d3e448a85e54fc88d53f0e398765372926e037f4a5", + "partial": "5a1b8d4c67f0a332233ef7ed59c422753d0f809b5f4c8fc6a9a5a227e3171e29", + "text_html": "c1d2ee837bd795bce4986353da5d47dd5e19b13b6c141c7960af03967d5db572", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "mail_test_7/bytes": { + "attachments": "e3f7fa13153ee8fc0b5ebef9c92608a64d0527435464ef2050e16722ea132510", + "headers": "fc571fddaa9b996a484f32e5293963e28b8c2227243cf7c4251086d5e6b80712", + "mail": "67f062d488316e23c333249b38e43127eac88030ef515382e55def6c440556c2", + "mime": "16889755daebad5977b3f1d62a6376af2480a5b807ad6d0f2d03482329bc42ce", + "partial": "76f9b8c8338f1680791580fd1256a4cef645c5a5d226cfed25b8a9d2bbf5603a", + "text_html": "c7e19438870f378f7b596ed82f9847c9d5ef8678f0e71574a8e5c5f2d2ba8a65", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "49cd1b3e879b4b5c7884e65e5335d3639d995528adce97411383a72f3a5d39a4" + }, + "mail_test_7/file": { + "attachments": "e3f7fa13153ee8fc0b5ebef9c92608a64d0527435464ef2050e16722ea132510", + "headers": "fc571fddaa9b996a484f32e5293963e28b8c2227243cf7c4251086d5e6b80712", + "mail": "67f062d488316e23c333249b38e43127eac88030ef515382e55def6c440556c2", + "mime": "16889755daebad5977b3f1d62a6376af2480a5b807ad6d0f2d03482329bc42ce", + "partial": "76f9b8c8338f1680791580fd1256a4cef645c5a5d226cfed25b8a9d2bbf5603a", + "text_html": "c7e19438870f378f7b596ed82f9847c9d5ef8678f0e71574a8e5c5f2d2ba8a65", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "49cd1b3e879b4b5c7884e65e5335d3639d995528adce97411383a72f3a5d39a4" + }, + "mail_test_7/string": { + "attachments": "e3f7fa13153ee8fc0b5ebef9c92608a64d0527435464ef2050e16722ea132510", + "headers": "fc571fddaa9b996a484f32e5293963e28b8c2227243cf7c4251086d5e6b80712", + "mail": "67f062d488316e23c333249b38e43127eac88030ef515382e55def6c440556c2", + "mime": "16889755daebad5977b3f1d62a6376af2480a5b807ad6d0f2d03482329bc42ce", + "partial": "76f9b8c8338f1680791580fd1256a4cef645c5a5d226cfed25b8a9d2bbf5603a", + "text_html": "c7e19438870f378f7b596ed82f9847c9d5ef8678f0e71574a8e5c5f2d2ba8a65", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "49cd1b3e879b4b5c7884e65e5335d3639d995528adce97411383a72f3a5d39a4" + }, + "mail_test_8/bytes": { + "attachments": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "headers": "9d699383f2c2cbfee2e83e16e482d9f2420192066e625e278000e81094d507bc", + "mail": "1046820834feed0bb64faf87a2b73dd5a2fe177591cbad8304e0384d670de6d3", + "mime": "d94af71f027c48da7bb7e7ce4864e06b2b70b40e99cf00aa72022afb56e2afae", + "partial": "ce586b864ac0baeaed673648a04922423ad71e5b0ca5a7460bb54e4f41e09e8e", + "text_html": "f580a99b3ac5edf680f6c336c9b3bcdf61fac0408033165535814f7652e27b38", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "e8e95703fa64de65640a215b32ed79af5aa1f5d667031295d1f2a206987c1284" + }, + "mail_test_8/file": { + "attachments": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "headers": "9d699383f2c2cbfee2e83e16e482d9f2420192066e625e278000e81094d507bc", + "mail": "1046820834feed0bb64faf87a2b73dd5a2fe177591cbad8304e0384d670de6d3", + "mime": "d94af71f027c48da7bb7e7ce4864e06b2b70b40e99cf00aa72022afb56e2afae", + "partial": "ce586b864ac0baeaed673648a04922423ad71e5b0ca5a7460bb54e4f41e09e8e", + "text_html": "f580a99b3ac5edf680f6c336c9b3bcdf61fac0408033165535814f7652e27b38", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "e8e95703fa64de65640a215b32ed79af5aa1f5d667031295d1f2a206987c1284" + }, + "mail_test_8/string": { + "attachments": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "headers": "9d699383f2c2cbfee2e83e16e482d9f2420192066e625e278000e81094d507bc", + "mail": "1046820834feed0bb64faf87a2b73dd5a2fe177591cbad8304e0384d670de6d3", + "mime": "d94af71f027c48da7bb7e7ce4864e06b2b70b40e99cf00aa72022afb56e2afae", + "partial": "ce586b864ac0baeaed673648a04922423ad71e5b0ca5a7460bb54e4f41e09e8e", + "text_html": "f580a99b3ac5edf680f6c336c9b3bcdf61fac0408033165535814f7652e27b38", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "e8e95703fa64de65640a215b32ed79af5aa1f5d667031295d1f2a206987c1284" + }, + "mail_test_9/bytes": { + "attachments": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "headers": "16695410194719089934a5753b82a5c295d29a8c22bcfa146c3d2b8206df2c8b", + "mail": "b927c7fdc2d483f5de163a8e50e857214f377f84ea69160c00f2202f8ff4d098", + "mime": "2e193f23b5d4411bdbfd69c757cc5b17f885cbadd2e49ac2daa5df09a8677bb8", + "partial": "753d6f7dcde31df714756f988eb51cc93e842ffa7a457294d5710642f7bcd868", + "text_html": "087c9ef250274839f35d8e567d3b78a055f0f8c1ddceee228e008c9c47a5d213", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "mail_test_9/file": { + "attachments": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "headers": "16695410194719089934a5753b82a5c295d29a8c22bcfa146c3d2b8206df2c8b", + "mail": "b927c7fdc2d483f5de163a8e50e857214f377f84ea69160c00f2202f8ff4d098", + "mime": "2e193f23b5d4411bdbfd69c757cc5b17f885cbadd2e49ac2daa5df09a8677bb8", + "partial": "753d6f7dcde31df714756f988eb51cc93e842ffa7a457294d5710642f7bcd868", + "text_html": "087c9ef250274839f35d8e567d3b78a055f0f8c1ddceee228e008c9c47a5d213", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "mail_test_9/string": { + "attachments": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "headers": "16695410194719089934a5753b82a5c295d29a8c22bcfa146c3d2b8206df2c8b", + "mail": "b927c7fdc2d483f5de163a8e50e857214f377f84ea69160c00f2202f8ff4d098", + "mime": "2e193f23b5d4411bdbfd69c757cc5b17f885cbadd2e49ac2daa5df09a8677bb8", + "partial": "753d6f7dcde31df714756f988eb51cc93e842ffa7a457294d5710642f7bcd868", + "text_html": "087c9ef250274839f35d8e567d3b78a055f0f8c1ddceee228e008c9c47a5d213", + "text_other": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "text_plain": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + } + }, + "invariants_checked": [ + "All 69 cases parse on Python 3.13/3.14; unchanged historical string-route exception on Python 3.9/3.10 for binary mail_outlook_1.", + "Attachment count and decoded payload hashes, address values, Date/timezone, Received UTC/delay unchanged.", + "No body-part count decreases; raw MIME for bytes/string unchanged; raw MIME for file equals bytes." + ], + "legacy_mime": { + "mail_test_6/bytes": { + "mime": "4c44e0300b6e2233e16313ccc4a479826e9e8825e50695a84fe6106ba5507153" + }, + "mail_test_6/file": { + "mime": "4c44e0300b6e2233e16313ccc4a479826e9e8825e50695a84fe6106ba5507153" + }, + "mail_test_6/string": { + "mime": "4c44e0300b6e2233e16313ccc4a479826e9e8825e50695a84fe6106ba5507153" + } + }, + "reviewed_changes": [ + "Received: ignore clause words inside comments; recover FOR before the misplaced date separator in mail_test_8 with diagnostics; preserve date folding. Existing UTC timestamps and hop delays unchanged.", + "Address names: preserve the recovered Cyrillic display names in mail_test_1/11 and add overlong-encoded-word diagnostics.", + "MIME UTF-8 metadata: mail_test_1/bytes recovers the original Unicode filename/disposition. File-route metadata and all attachment byte hashes are preserved.", + "Inline body: mail_test_3 includes the existing named inline WatchGuard text in the body; attachment remains byte-identical.", + "Charset recovery: mail_test_2/6/9 expose replacement markers and diagnostics for formerly discarded octets; mail_test_12/bytes recovers valid UTF-8 mislabeled GB2312.", + "File input: raw MIME now matches the bytes route, including non-ASCII octets and CRLF; mail_test_17 body/header folding retains original CRLF.", + "Intentional compatibility change: mail_test_4/file now honors declared windows-1251 like the bytes route. Two UTF-8-looking C3B3 sequences display as Cyrillic characters rather than the old unconditional UTF-8 interpretation; original bytes remain intact.", + "mail_outlook_1 is a binary MSG container passed to generic EML APIs in this corpus: raw bytes are preserved, body newline/UTF-8 replacement recovery differs. This is not an Outlook conversion test." + ] +} diff --git a/tests/test_corpus_regression.py b/tests/test_corpus_regression.py new file mode 100644 index 0000000..a9bb979 --- /dev/null +++ b/tests/test_corpus_regression.py @@ -0,0 +1,131 @@ +"""Guard public outputs and MIME evidence for the existing mail corpus. + +The initial outputs were recorded from revision 522e1f3 before any fixes. +The fixture records deliberate changes reviewed against the original +outputs; do not regenerate it simply to make a failing test pass. +Large text and decoded attachment content are compared by SHA-256. +Only unordered category/domain sets and generated names are stabilized. +""" + +import base64 +import hashlib +import json +import random +import sys +from pathlib import Path + +import pytest + +import mailparser + + +def digest(data): + """Return a length and SHA-256 fingerprint of exact bytes.""" + return {"length": len(data), "sha256": hashlib.sha256(data).hexdigest()} + + +def compact(value): + """Keep evidence compact while stabilizing unordered API collections.""" + if isinstance(value, str) and len(value) > 160: + return digest(value.encode("utf-8", "surrogatepass")) + if isinstance(value, dict): + return { + key: sorted(item) + if key in ("defects_categories", "to_domains") + else compact(item) + for key, item in value.items() + } + if isinstance(value, (list, tuple)): + return [compact(item) for item in value] + return value + + +def snapshot(mail): + """Capture public views, attachment octets and original MIME structure.""" + parts = [] + for part in mail.message.walk(): + payload = part.get_payload(decode=True) + parts.append( + { + "headers": compact(list(part.raw_items())), + "type": part.get_content_type(), + "multipart": part.is_multipart(), + "payload": digest(payload) if isinstance(payload, bytes) else None, + "preamble": compact(part.preamble), + "epilogue": compact(part.epilogue), + } + ) + attachments = [] + for attachment in mail.attachments: + data = attachment["payload"] + data = ( + base64.b64decode(data) + if attachment["binary"] + else data.encode("utf-8", "surrogatepass") + ) + attachments.append( + { + "metadata": compact( + {k: v for k, v in attachment.items() if k != "payload"} + ), + "decoded_payload": digest(data), + } + ) + return { + "mail": compact(json.loads(mail.mail_json)), + "partial": compact(json.loads(mail.mail_partial_json)), + "headers": compact(json.loads(mail.headers_json)), + "text_plain": compact(mail.text_plain), + "text_html": compact(mail.text_html), + "text_other": compact(mail.text_not_managed), + "attachments": attachments, + "mime": parts, + } + + +_CORPUS = Path(__file__).parent / "mails" +_EXPECTATIONS = json.loads( + (Path(__file__).parent / "fixtures" / "corpus_expected.json").read_text() +) + + +def _fingerprints(mail): + """Fingerprint each public output group independently.""" + return { + key: hashlib.sha256( + json.dumps(value, sort_keys=True, ensure_ascii=True).encode() + ).hexdigest() + for key, value in snapshot(mail).items() + } + + +@pytest.mark.parametrize("case", sorted(_EXPECTATIONS["cases"])) +def test_existing_corpus_outputs(case): + """Every previously parsed fixture retains its reviewed semantics.""" + filename, route = case.split("/") + path = _CORPUS / filename + raw = path.read_bytes() + state = random.getstate() + try: + random.seed(1234) + if route == "bytes": + mail = mailparser.parse_from_bytes(raw) + elif route == "file": + mail = mailparser.parse_from_file(path) + else: + try: + mail = mailparser.parse_from_string( + raw.decode("utf-8", "surrogateescape") + ) + except UnicodeEncodeError: + if sys.version_info < (3, 11) and filename == "mail_outlook_1": + pytest.xfail( + "Baseline stdlib string-parser failure on Python 3.9/3.10" + ) + raise + finally: + random.setstate(state) + expected = dict(_EXPECTATIONS["cases"][case]) + if sys.version_info < (3, 12): + expected.update(_EXPECTATIONS["legacy_mime"].get(case, {})) + assert _fingerprints(mail) == expected diff --git a/tests/test_date_recovery.py b/tests/test_date_recovery.py new file mode 100644 index 0000000..bee2c97 --- /dev/null +++ b/tests/test_date_recovery.py @@ -0,0 +1,308 @@ +"""Semantic date validation and occurrence-aware forensic recovery.""" + +import datetime +import io +import json +import time + +import pytest + +import mailparser +from mailparser.utils import convert_mail_date + +VALID = "Thu, 01 Oct 2026 12:00:00 +0000" +EXPECTED = "2026-10-01T12:00:00+00:00" + + +def _mail(value, extra=""): + """Parse a Date field alongside independently valid content.""" + return mailparser.parse_from_bytes( + ( + f"Date: {value}\r\nFrom: alice@sender.example\r\n" + "To: bob@recipient.example\r\nSubject: intact\r\n" + extra + "\r\nbody" + ).encode("utf-8", "surrogateescape") + ) + + +@pytest.mark.parametrize( + ("value", "reason"), + [ + ("Thu, 31 Feb 2026 12:00:00 +0000", "invalid-calendar-date"), + ("Thu, 29 Feb 2026 12:00:00 +0000", "invalid-calendar-date"), + ("01 Jan 1899 12:00:00 +0000", "invalid-calendar-date"), + ("Thu, 01 Oct 2026 24:00:00 +0000", "invalid-time"), + ("Thu, 01 Oct 2026 12:60:00 +0000", "invalid-time"), + ("Thu, 01 Oct 2026 12:00:61 +0000", "invalid-time"), + ("Thu, 01 Oct 2026 12:00:00 +0060", "invalid-zone"), + ("Thu, 01 Oct 2026 12:00:00 -0060", "invalid-zone"), + ("Thu, 01 Oct 2026 12:00:00 +99999", "invalid-zone"), + ("", "invalid-date-syntax"), + ("unparseable", "invalid-date-syntax"), + ("Thu, 01 Oct 2026 12:00:00 +0000 (bad", "invalid-date-syntax"), + ], +) +def test_invalid_date_is_not_normalized(value, reason): + """Invalid components cannot manufacture a plausible timestamp.""" + mail = _mail(value) + assert mail.date is None + assert mail.timezone == 0 + assert mail.has_defects is True + assert mail.date_header_defects == [ + { + "reason": reason, + "raw": value, + "recovered": False, + "value": None, + "header": "date", + "occurrence": 0, + } + ] + assert "DateHeaderDefect" in mail.defects_categories + assert mail.subject == "intact" + assert mail.to == [("", "bob@recipient.example")] + assert mail.body == "body" + for output in (mail.mail_json, mail.mail_partial_json): + assert json.loads(output)["date_header_defects"] == mail.date_header_defects + assert mail.message is not None + assert mail.message.get_all("Date") == [value] + + +def test_weekday_mismatch_retains_numeric_date_and_reports_defect(): + """A redundant incorrect weekday does not erase a recoverable date.""" + value = "Wed, 01 Oct 2026 12:00:00 +0000" + mail = _mail(value) + assert mail.date is not None + assert mail.date.isoformat() == EXPECTED + assert mail.date_header_defects == [ + { + "reason": "weekday-mismatch", + "raw": value, + "recovered": True, + "value": EXPECTED, + "header": "date", + "occurrence": 0, + } + ] + assert mail.has_defects is True + + +@pytest.mark.parametrize( + ("value", "expected", "zone"), + [ + (VALID, EXPECTED, "+0.0"), + ("01 Oct 2026 12:00 +0000", EXPECTED, "+0.0"), + ("Thu, 01 Oct 26 12:00 GMT", EXPECTED, "+0.0"), + ("Thu, 01 Oct 126 12:00:00GMT", EXPECTED, "+0.0"), + ("Thu, 01 Oct 2026 12:00:00 -0000", EXPECTED, "+0.0"), + ("Thu, 01 Oct 2026 12:00:00 A", EXPECTED, "+0.0"), + ("Thu, 01 Oct 2026 12:00:00 XYZ", EXPECTED, "+0.0"), + ("Thu, 01 Oct 2026 12:00:00 EST", "2026-10-01T17:00:00+00:00", "-5.0"), + ("Thu, 01 Oct 2026 12:00:00 +9959", "2026-09-27T08:01:00+00:00", "+100.0"), + ("Sat, 31 Dec 2016 23:59:60 +0000", "2017-01-01T00:00:00+00:00", "+0.0"), + ("Thu, 29 Feb 2024 12:00:00 +0000", "2024-02-29T12:00:00+00:00", "+0.0"), + ( + "(a (b)) Thu (day), 01 (d) Oct (m) 2026 12 (h): 00 (n): 00 GMT", + EXPECTED, + "+0.0", + ), + (r"Thu, 01 Oct 2026 12:00:00 GMT (escaped\) comment)", EXPECTED, "+0.0"), + ("Thu,\r\n\t01 Oct 2026 12:00:00 GMT", EXPECTED, "+0.0"), + ], +) +def test_valid_and_obsolete_date_forms(value, expected, zone): + """Valid old syntax, leap seconds and unknown zones are not defects.""" + mail = _mail(value) + assert mail.date is not None + assert mail.date.isoformat() == expected + assert mail.timezone == zone + assert mail.has_defects is False + assert mail.date_header_defects == [] + converted, timezone = convert_mail_date(value) + assert converted.isoformat() == expected + assert timezone == zone + + +def test_date_occurrences_are_checked_without_changing_first_value(): + """Date and Resent-Date diagnostics retain each wire occurrence.""" + mismatch = "Wed, 01 Oct 2026 12:00:00 +0000" + bad = "Thu, 31 Feb 2026 12:00:00 +0000" + mail = _mail( + VALID, f"Date: {bad}\r\nResent-Date: {VALID}\r\nResent-Date: {mismatch}\r\n" + ) + assert mail.date is not None + assert mail.date.isoformat() == EXPECTED + observed = [ + (d["header"], d["occurrence"], d["raw"]) for d in mail.date_header_defects + ] + assert observed == [("date", 1, bad), ("resent-date", 1, mismatch)] + before = list(mail.defects) + mail.date + mail.timezone + mail.parse() + assert mail.defects == before + assert len(mail.date_header_defects) == 2 + + +def test_received_date_defects_are_separate_from_lexical_defects(): + """Bad timestamps keep trace clauses and never create false delays.""" + newest = "from relay.example by mx.example; " + VALID + oldest = "from sender.example by relay.example; Thu, 31 Feb 2026 12:00:00 +0000" + mail = _mail(VALID, f"Received: {newest}\r\nReceived: {oldest}\r\n") + assert mail.received[0]["date_utc"] is None + assert mail.received[1]["date_utc"] == EXPECTED + assert mail.received[1]["delay"] == 0 + assert mail.received_header_defects == [] + assert mail.date_header_defects == [ + { + "reason": "invalid-calendar-date", + "raw": oldest, + "recovered": False, + "value": None, + "header": "received", + "occurrence": 1, + } + ] + + +def test_empty_received_date_is_diagnosed(): + """A present separator with no date differs from obsolete no-date form.""" + mail = _mail(VALID, "Received: from sender.example by mx.example;\r\n") + assert mail.received[0]["date_utc"] is None + assert mail.date_header_defects[0]["reason"] == "invalid-date-syntax" + + +def test_date_diagnostics_cannot_be_shadowed_by_wire_header(): + """Caller metadata and attacker-named wire fields stay separate.""" + mail = _mail("bad", "date_header_defects: spoofed\r\n") + assert isinstance(mail.mail["date_header_defects"], list) + assert mail.headers["date_header_defects"] == "spoofed" + + +def test_absent_date_preserves_api_defaults(): + """Missing input to the accessor remains a neutral no-result.""" + mail = mailparser.parse_from_string("Subject: no date\r\n\r\nbody") + assert mail.date is None + assert mail.timezone == 0 + assert mail.date_header_defects == [] + + +@pytest.mark.parametrize("size", [20, 10000]) +def test_extreme_date_components_are_bounded(size): + """Large years and nested comments neither overflow nor recurse.""" + huge = "01 Oct " + "9" * size + " 12:00:00 +0000" + start = time.perf_counter() + mail = _mail(huge) + assert mail.date is None + assert mail.date_header_defects[0]["reason"] == "date-out-of-range" + nested = "(" * size + "note" + ")" * size + parsed = _mail(nested + VALID) + assert parsed.date is not None + assert parsed.date.isoformat() == EXPECTED + assert time.perf_counter() - start < 2 + + +def test_invalid_date_octets_remain_in_diagnostics(): + """Malformed bytes cannot disappear while constructing a timestamp.""" + value = "Thu, 01 Oct 2026 12:00:00 +0000\udcff" + mail = _mail(value) + assert mail.date is None + assert ( + mail.date_header_defects[0]["raw"] == r"Thu, 01 Oct 2026 12:00:00 +0000\udcff" + ) + assert mail.message is not None + original = dict(mail.message.raw_items())["Date"] + assert ( + original.encode("utf-8", "surrogateescape") + == b"Thu, 01 Oct 2026 12:00:00 +0000\xff" + ) + + +def test_all_input_factories_share_date_validation(tmp_path): + """Bytes, strings and file factories produce the same evidence.""" + raw = b"Date: Thu, 31 Feb 2026 12:00:00 +0000\r\n\r\nbody" + path = tmp_path / "date.eml" + path.write_bytes(raw) + mails = [ + mailparser.parse_from_bytes(raw), + mailparser.parse_from_string(raw.decode()), + mailparser.parse_from_file(str(path)), + mailparser.parse_from_file_obj(io.StringIO(raw.decode())), + ] + assert all(mail.date is None for mail in mails) + assert all( + mail.date_header_defects == mails[0].date_header_defects for mail in mails + ) + + +def test_converter_raises_on_unrecoverable_date(): + """The legacy tuple converter keeps its controlled-error contract.""" + with pytest.raises(ValueError): + convert_mail_date("Thu, 31 Feb 2026 12:00:00 +0000") + date, zone = convert_mail_date("Wed, 01 Oct 2026 12:00:00 +0000") + assert date == datetime.datetime(2026, 10, 1, 12, tzinfo=datetime.timezone.utc) + assert zone == "+0.0" + + +def test_date_diagnostic_json_is_utf8_encodable(): + """Raw malformed octets use the existing escaped-evidence convention.""" + mail = _mail("Thu, 01 Oct 2026 12:00:00 +0000\udcff") + output = json.loads(mail.mail_json.encode("utf-8")) + assert ( + output["date_header_defects"][0]["raw"] + == r"Thu, 01 Oct 2026 12:00:00 +0000\udcff" + ) + + +def test_reserved_military_zone_j_is_not_accepted(): + """The single letter J is excluded from RFC 5322 obs-zone.""" + mail = _mail("Thu, 01 Oct 2026 12:00:00 J") + assert mail.date is None + assert mail.date_header_defects[0]["reason"] == "invalid-zone" + + +def test_received_weekday_mismatch_recovers_timestamp(): + """Received dates retain a safe UTC value with explicit evidence.""" + received = "from sender.example by mx.example; Wed, 01 Oct 2026 12:00:00 +0000" + mail = _mail(VALID, "Received: " + received + "\r\n") + assert mail.received[0]["date_utc"] == EXPECTED + assert mail.received_header_defects == [] + assert mail.date_header_defects == [ + { + "reason": "weekday-mismatch", + "raw": received, + "recovered": True, + "value": EXPECTED, + "header": "received", + "occurrence": 0, + } + ] + + +@pytest.mark.parametrize( + ("year", "expected"), [("49", 2049), ("50", 1950), ("999", 2899)] +) +def test_obsolete_year_pivot_is_rfc_defined(year, expected): + """Two and three digit years use RFC 5322 section 4.3 mappings.""" + mail = _mail(f"01 Jan {year} 12:00GMT") + assert mail.date is not None + assert mail.date.year == expected + assert mail.has_defects is False + + +@pytest.mark.parametrize("comment", [b"bad\xff", b"Jos\xc3\xa9"]) +def test_date_comment_octets_are_checked_after_recovery(comment): + """Removing a comment must not erase evidence of malformed UTF-8.""" + raw = b"Thu, 1 Oct 2026 12:00:00 +0000 (" + comment + b")" + mail = mailparser.parse_from_bytes(b"Date: " + raw + b"\r\n\r\n") + assert mail.date is not None + assert mail.date.isoformat() == "2026-10-01T12:00:00+00:00" + if comment == b"bad\xff": + assert mail.has_defects is True + assert mail.date_header_defects[0]["reason"] == "invalid-utf8" + assert mail.date_header_defects[0]["recovered"] is True + assert r"\udcff" in mail.date_header_defects[0]["raw"] + else: + assert mail.date_header_defects == [] + assert mail.mail_json.encode("utf-8") + assert mail.message is not None + assert next(mail.message.raw_items())[1].encode("utf-8", "surrogateescape") == raw diff --git a/tests/test_encoded_word_context.py b/tests/test_encoded_word_context.py new file mode 100644 index 0000000..7b0717e --- /dev/null +++ b/tests/test_encoded_word_context.py @@ -0,0 +1,224 @@ +"""RFC 2047 display words retain their RFC 5322 lexical context.""" + +import json + +import pytest + +import mailparser +from mailparser.utils import get_addresses + + +@pytest.mark.parametrize( + "name, expected", + [ + ('"=?utf-8?Q?Jos=C3=A9?="', "=?utf-8?Q?Jos=C3=A9?="), + ("=?utf-8?Q?Jos=C3=A9?=", "José"), + ('"Alice Example"', "Alice Example"), + ( + '"=?utf-8?Q?Jos=C3=A9?=" =?utf-8?Q?Andr=C3=A9?=', + "=?utf-8?Q?Jos=C3=A9?= André", + ), + ( + "=?utf-8?Q?Jos=C3=A9?=\r\n\t=?utf-8?Q?_Andr=C3=A9?=", + "José André", + ), + ("=?utf-8?Q?A=2CB=3Cfake=40example.com=3E?=", "A,B"), + ("=?utf-8?Q?=3D=3Futf-8=3FQ=3FAlice=3F=3D?=", "=?utf-8?Q?Alice?="), + ('"A\\" =?utf-8?Q?Jos=C3=A9?="', 'A" =?utf-8?Q?Jos=C3=A9?='), + ("=?no-such-charset?Q?=FF?=", "=?no-such-charset?Q?=FF?="), + ("=?utf-8?Q?=FF?=", "=?utf-8?Q?=FF?="), + ("José", "José"), + ], +) +@pytest.mark.parametrize("factory", ["string", "bytes"]) +def test_display_words_preserve_quote_context(name, expected, factory): + """Decode eligible words once without changing mailbox boundaries.""" + value = f"{name} , Bob " + raw = f"From: {value}\r\nTo: neighbor@example.com\r\n\r\n" + mail = ( + mailparser.parse_from_bytes(raw.encode()) + if factory == "bytes" + else mailparser.parse_from_string(raw) + ) + assert mail.from_ == [(expected, "alice@example.com"), ("Bob", "bob@example.com")] + assert mail.to == [("", "neighbor@example.com")] + decoded_json = mail.from_json + assert isinstance(decoded_json, str) + assert json.loads(decoded_json) == [list(x) for x in mail.from_] + if value.isascii() or factory == "string": + raw_json = mail.from_raw + assert isinstance(raw_json, str) + assert json.loads(raw_json) == [value] + assert mail.address_header_defects == [] + + +@pytest.mark.parametrize( + "value, expected", + [ + ("Alice (=?utf-8?Q?Andr=C3=A9?=) ", "Alice (André)"), + ("alice@example.com (=?utf-8?Q?Andr=C3=A9?=)", "André"), + ("Alice (=?utf-8?Q?A=28B=29=5CC?=) ", "Alice (A(B)\\C)"), + ], +) +def test_comment_words_keep_special_characters_as_data(value, expected): + """Decoded parentheses and escapes cannot change comment boundaries.""" + mail = mailparser.parse_from_string(f"From: {value}\r\n\r\n") + assert mail.from_ == [(expected, "alice@example.com")] + assert not mail.has_defects + + +def test_get_addresses_retains_undecoded_name_contract(): + """The helper's default still returns literal encoded display words.""" + value = "=?utf-8?Q?Jos=C3=A9?= " + assert get_addresses(value) == [("=?utf-8?Q?Jos=C3=A9?=", "alice@example.com")] + + +def test_recovery_keeps_raw_name_evidence_and_a_valid_neighbor(): + """Malformed display syntax remains diagnosed after rendering.""" + value = ( + 'bad@label "=?utf-8?Q?Jos=C3=A9?=" , Bob ' + ) + mail = mailparser.parse_from_string(f"From: {value}\r\n\r\n") + assert mail.from_ == [ + ("bad@label =?utf-8?Q?Jos=C3=A9?=", "alice@example.com"), + ("Bob", "bob@example.com"), + ] + assert mail.has_defects + assert mail.address_header_defects[0]["reason"] == "invalid-display-name" + assert mail.address_header_defects[0]["raw"] == value.split(", ")[0] + assert mail.address_header_defects[0]["recovered"] is True + + +@pytest.mark.parametrize( + "value, expected", + [ + ( + '"José =?utf-8?Q?Alice?=" ', + [("José =?utf-8?Q?Alice?=", "alice@example.com")], + ), + ( + 'Name <"=?utf-8?Q?Alice?="@example.com>', + [("Name", '"=?utf-8?Q?Alice?="@example.com')], + ), + ( + "=?utf-8?Q?A=22B=5CC?= ", + [('A"B\\C', "alice@example.com")], + ), + ( + "=?utf-8?Q?A=0D=0AB?= ", + [("=?utf-8?Q?A=0D=0AB?=", "alice@example.com")], + ), + ( + "=?utf-8?Q?Alice?=suffix ", + [("=?utf-8?Q?Alice?=suffix", "alice@example.com")], + ), + ], +) +def test_rendering_cannot_reinterpret_address_or_escaped_text(value, expected): + """Unicode adaptation and decoded specials preserve lexical identity.""" + mail = mailparser.parse_from_bytes(f"From: {value}\r\n\r\n".encode()) + assert mail.from_ == expected + + +def test_encoded_names_do_not_hide_invalid_utf8_evidence(): + """Keep invalid-octet diagnostics and the neighboring valid mailbox.""" + value = ( + b'"=?utf-8?Q?Alice?=" , =?utf-8?Q?Bob?= ' + ) + mail = mailparser.parse_from_bytes(b"From: " + value + b"\r\n\r\n") + assert mail.from_ == [("Bob", "bob@example.com")] + defect = mail.address_header_defects[0] + assert defect["reason"] == "invalid-utf8" + assert "\\udcff" in defect["raw"] + assert defect["recovered"] is False + assert defect["header"] == "from" + assert defect["occurrence"] == 0 + assert defect["candidates"] == [] + + +def test_many_display_words_remain_bounded(): + """A long encoded phrase has linear-sized output and one mailbox.""" + import time + + count = 8000 + value = "=?utf-8?Q?A?= " * count + "" + started = time.perf_counter() + mail = mailparser.parse_from_string(f"From: {value}\r\n\r\n") + assert mail.from_ == [("A" * count, "alice@example.com")] + assert time.perf_counter() - started < 2 + + +@pytest.mark.parametrize("context", ["phrase", "comment"]) +def test_overlong_encoded_word_is_recovered_and_reported(context): + """Recover the existing corpus name despite its 76-character word.""" + word = ( + "=?UTF-8?B?0YHQu9GD0LbQsdCwINCk0J3QoSDQlNCw0L3QuNC40Lsg0KHRg9Cy0L7RgNC+0LI=?=" + ) + expected = "служба ФНС Даниил Суворов" + assert len(word) == 76 + if context == "phrase": + value = f"{word} " + else: + value = f"alice@example.com ({word})" + mail = mailparser.parse_from_bytes( + f"From: {value}, Bob \r\n\r\n".encode() + ) + assert mail.from_ == [(expected, "alice@example.com"), ("Bob", "bob@example.com")] + assert mail.has_defects + assert mail.address_header_defects == [ + { + "reason": "overlong-encoded-word", + "raw": value, + "recovered": True, + "candidates": [ + { + "display_name": expected if context == "phrase" else "", + "address": "alice@example.com", + } + ], + "header": "from", + "occurrence": 0, + } + ] + + +def test_overlong_quoted_encoded_looking_text_remains_literal(): + """RFC 2047's length limit does not apply to quoted literal text.""" + word = "=?utf-8?Q?" + "A" * 100 + "?=" + mail = mailparser.parse_from_string(f'From: "{word}" \r\n\r\n') + assert mail.from_ == [(word, "alice@example.com")] + assert not mail.has_defects + + +@pytest.mark.parametrize("terminated", [True, False]) +def test_long_encoded_tokens_and_failed_matches_remain_bounded(terminated): + """Matching a long invalid token must not repeatedly scan its suffix.""" + import time + + count = 4000 + word = "=?utf-8?Q?" + "A" * 100 + ("?=" if terminated else "") + value = (word + " ") * count + "" + started = time.perf_counter() + mail = mailparser.parse_from_string(f"From: {value}\r\n\r\n") + assert len(mail.from_) == 1 + assert mail.from_[0][1] == "alice@example.com" + if terminated: + assert mail.from_[0][0] == "A" * (100 * count) + assert mail.address_header_defects[0]["reason"] == "overlong-encoded-word" + else: + assert mail.from_[0][0] == " ".join([word] * count) + assert time.perf_counter() - started < 2 + + +@pytest.mark.parametrize("charset", ["unicode_escape", "raw_unicode_escape"]) +@pytest.mark.parametrize("comment", [False, True]) +def test_encoded_word_surrogate_is_evidence_not_output(charset, comment): + """Codec-produced surrogates cannot crash JSON or hide the mailbox.""" + word = f"=?{charset}?b?XHVkODAw?=" + field = f"a@example.com ({word})" if comment else f"{word} " + parsed = mailparser.parse_from_string(f"From: {field}\r\n\r\n") + assert parsed.mail_json.encode("utf-8") + assert parsed.from_ == [(word, "a@example.com")] + assert parsed.address_header_defects[0]["reason"] == "invalid-encoded-word" + assert parsed.address_header_defects[0]["recovered"] is True + assert parsed.address_header_defects[0]["raw"] == field diff --git a/tests/test_inline_body.py b/tests/test_inline_body.py new file mode 100644 index 0000000..0ac3181 --- /dev/null +++ b/tests/test_inline_body.py @@ -0,0 +1,333 @@ +"""Named inline text remains body content and forensic attachment data.""" + +import base64 +import json +import quopri + +import pytest + +import mailparser + +_HEADERS = ( + b"Date: Thu, 01 Oct 2026 12:00:00 +0000\r\n" + b"From: alice@example.com\r\nTo: bob@example.com\r\n" + b"MIME-Version: 1.0\r\n" +) + + +def _entity(content_type, disposition, payload, extra_headers=b""): + headers = b"Content-Type: " + content_type + b"\r\n" + if disposition is not None: + headers += b"Content-Disposition: " + disposition + b"\r\n" + return headers + extra_headers + b"\r\n" + payload + + +def _multipart(parts, boundary, disposition=None, subtype=b"mixed"): + payload = ( + b"".join(b"--" + boundary + b"\r\n" + part + b"\r\n" for part in parts) + + b"--" + + boundary + + b"--\r\n" + ) + content_type = b"multipart/" + subtype + b"; boundary=" + boundary + return _entity(content_type, disposition, payload) + + +@pytest.mark.parametrize("media_type", [b"text/plain", b"text/html"]) +@pytest.mark.parametrize("name_source", ["filename", "name"]) +def test_named_inline_text_and_attachment(media_type, name_source): + """Explicit inline text gains a body view without changing metadata.""" + content_type = media_type + disposition = b"inline" + if name_source == "filename": + disposition += b'; filename="note.txt"' + else: + content_type += b'; name="note.txt"' + payload = b"hello" if media_type == b"text/plain" else b"

hello

" + raw = _HEADERS + _entity( + content_type, + disposition, + payload, + b"Content-ID: \r\n", + ) + parsed = mailparser.parse_from_bytes(raw) + + assert parsed.attachments == [ + { + "filename": "note.txt", + "safe_filename": "note.txt", + "payload": base64.b64encode(payload).decode("ascii"), + "binary": True, + "mail_content_type": media_type.decode("ascii"), + "content-id": "", + "content-disposition": disposition.decode("ascii"), + "charset": None, + "content_transfer_encoding": "base64", + } + ] + expected = [payload.decode("ascii")] + assert parsed.text_plain == (expected if media_type == b"text/plain" else []) + assert parsed.text_html == (expected if media_type == b"text/html" else []) + assert parsed.body == payload.decode("ascii") + assert parsed.has_defects is False + assert json.loads(parsed.mail_json)["body"] == parsed.body + + +@pytest.mark.parametrize( + ("disposition", "has_body", "has_attachment"), + [ + (None, True, False), + (b"inline", True, False), + (b"x-custom", True, False), + (b"attachment", False, True), + (b'attachment; filename="note.txt"', False, True), + (b'x-custom; filename="note.txt"', False, True), + ], +) +def test_existing_disposition_controls(disposition, has_body, has_attachment): + """The enhancement leaves existing unnamed and non-inline behavior.""" + parsed = mailparser.parse_from_bytes( + _HEADERS + _entity(b"text/plain", disposition, b"hello") + ) + + assert parsed.body == ("hello" if has_body else "") + assert bool(parsed.attachments) is has_attachment + if has_attachment: + assert base64.b64decode(parsed.attachments[0]["payload"]) == b"hello" + + +def test_name_without_inline_stays_attachment_only(): + """A legacy Content-Type name alone does not opt into body inclusion.""" + parsed = mailparser.parse_from_bytes( + _HEADERS + _entity(b'text/plain; name="note.txt"', None, b"hello") + ) + + assert parsed.body == "" + assert parsed.attachments[0]["filename"] == "note.txt" + + +@pytest.mark.parametrize( + ("charset", "cte"), + [ + ("utf-8", None), + ("utf-8", "base64"), + ("iso-8859-1", "quoted-printable"), + ("iso-8859-1", "8bit"), + ("iso-8859-1", "binary"), + ], +) +def test_named_inline_decodes_once_as_body_and_keeps_bytes(charset, cte): + """The ordinary body decoder handles transfer encoding and charset.""" + payload = "Caf\u00e9".encode(charset) + wire = payload + if cte == "base64": + wire = base64.b64encode(payload) + elif cte == "quoted-printable": + wire = quopri.encodestring(payload) + headers = ( + b"Content-Transfer-Encoding: " + cte.encode("ascii") + b"\r\n" if cte else b"" + ) + parsed = mailparser.parse_from_bytes( + _HEADERS + + _entity( + b"text/plain; charset=" + charset.encode("ascii"), + b'INLINE; filename="note.txt"', + wire, + headers, + ) + ) + + assert parsed.text_plain == ["Caf\u00e9"] + assert base64.b64decode(parsed.attachments[0]["payload"]) == payload + assert parsed.attachments[0]["charset"] == charset + assert parsed.attachments[0]["content-disposition"] == ( + 'INLINE; filename="note.txt"' + ) + assert parsed.has_defects is False + + +def test_named_inline_unicode_string_input(): + """Already-decoded string input uses the same body path as before.""" + raw = _HEADERS + _entity( + b"text/plain; charset=utf-8", + b'inline; filename="note.txt"', + "Caf\u00e9".encode(), + b"Content-Transfer-Encoding: 8bit\r\n", + ) + parsed = mailparser.parse_from_string(raw.decode("utf-8")) + + assert parsed.text_plain == ["Caf\u00e9"] + assert len(parsed.attachments) == 1 + + +@pytest.mark.parametrize( + ("media_type", "disposition"), + [ + (b"image/png", b"inline"), + (b"image/png", b'inline; filename="image.png"'), + (b"application/plain", b'inline; filename="sample.bin"'), + (b"application/html", b'inline; filename="sample.bin"'), + ], +) +def test_inline_binary_and_nontext_parts_remain_attachments(media_type, disposition): + """Content-ID and text-like subtypes do not promote binary media.""" + payload = b"\x00\x80\xff\r\n" + parsed = mailparser.parse_from_bytes( + _HEADERS + + _entity( + media_type, + disposition, + base64.b64encode(payload), + b"Content-ID: \r\nContent-Transfer-Encoding: base64\r\n", + ) + ) + + assert parsed.body == "" + assert parsed.text_plain == parsed.text_html == [] + assert len(parsed.attachments) == 1 + assert base64.b64decode(parsed.attachments[0]["payload"]) == payload + assert parsed.attachments[0]["content-id"] == "" + + +def test_nested_inline_siblings_and_explicit_attachment(): + """Collect nested inline text while retaining explicit text files.""" + plain = _entity(b"text/plain", b'inline; filename="body.txt"', b"plain body") + html = _entity(b"text/html", b'inline; filename="body.html"', b"

html body

") + attachment = _entity( + b"text/plain", b'attachment; filename="private.txt"', b"private" + ) + raw = _HEADERS + _multipart( + [_multipart([plain, html], b"alternative", subtype=b"alternative"), attachment], + b"outer", + ) + parsed = mailparser.parse_from_bytes(raw) + + assert parsed.text_plain == ["plain body"] + assert parsed.text_html == ["

html body

"] + assert "private" not in parsed.body + assert [a["filename"] for a in parsed.attachments] == [ + "body.txt", + "body.html", + "private.txt", + ] + assert base64.b64decode(parsed.attachments[2]["payload"]) == b"private" + + +@pytest.mark.parametrize("wrapper", ["multipart", "message"]) +@pytest.mark.parametrize( + ("disposition", "presented"), + [(b"attachment", False), (b"x-custom", False), (None, False), (b"inline", True)], +) +def test_named_inline_respects_container_disposition(wrapper, disposition, presented): + """A named attached ancestor cannot expose newly included inline text.""" + child = _entity(b"text/plain", b'inline; filename="child.txt"', b"inside") + inner = _multipart([child], b"inner") + if wrapper == "message": + container = _entity( + b'message/rfc822; name="forwarded.eml"', + disposition, + _HEADERS + inner, + ) + else: + container = _multipart([inner], b"container", disposition) + container = container.replace( + b"boundary=container\r\n", + b'boundary=container; name="bundle.mime"\r\n', + 1, + ) + visible = _entity(b"text/plain", b'inline; filename="visible.txt"', b"outside") + parsed = mailparser.parse_from_bytes( + _HEADERS + _multipart([visible, container], b"outer") + ) + + assert parsed.text_plain == (["outside", "inside"] if presented else ["outside"]) + child_attachment = next( + a for a in parsed.attachments if a["filename"] == "child.txt" + ) + assert base64.b64decode(child_attachment["payload"]) == b"inside" + assert parsed.message is not None + assert any( + p.get_payload(decode=True) == b"inside" + for p in parsed.message.walk() + if not p.is_multipart() + ) + + +@pytest.mark.parametrize( + ("actual_charset", "declared_charset", "text", "fallback"), + [ + ("utf-8", "gb2312", "\u4f60\u597d\u4e16\u754c", True), + ("gb2312", "gb2312", "\u4f60\u597d\u4e16\u754c", False), + ("iso-8859-1", "iso-8859-1", "Caf\u00e9", False), + ], +) +@pytest.mark.parametrize("named_inline", [False, True]) +def test_body_charset_recovery_keeps_original_bytes( + actual_charset, declared_charset, text, fallback, named_inline +): + """Malformed declarations do not trigger stdlib replacement decoding.""" + payload = text.encode(actual_charset) + disposition = b'inline; filename="text.txt"' if named_inline else None + parsed = mailparser.parse_from_bytes( + _HEADERS + + _entity( + b"text/plain; charset=" + declared_charset.encode("ascii"), + disposition, + payload, + b"Content-Transfer-Encoding: 8bit\r\n", + ) + ) + + assert parsed.text_plain == [text] + assert parsed.message is not None + assert parsed.message.get_payload(decode=True) == payload + if named_inline: + assert base64.b64decode(parsed.attachments[0]["payload"]) == payload + assert parsed.has_defects is fallback + if fallback: + assert "CharsetDecodeDefect" in parsed.defects_categories + evidence = json.dumps(parsed.defects) + assert "gb2312" in evidence and "utf-8" in evidence + + +def test_body_unrecoverable_charset_keeps_visible_and_raw_evidence(): + """When neither charset works, retain the octet and mark display loss.""" + payload = b"visible\xfftail" + parsed = mailparser.parse_from_bytes( + _HEADERS + + _entity( + b"text/plain; charset=gb2312", + b'inline; filename="text.txt"', + payload, + b"Content-Transfer-Encoding: 8bit\r\n", + ) + ) + + assert parsed.text_plain == ["visible\ufffdtail"] + assert parsed.message is not None + assert parsed.message.get_payload(decode=True) == payload + assert base64.b64decode(parsed.attachments[0]["payload"]) == payload + assert parsed.has_defects is True + assert "CharsetDecodeDefect" in parsed.defects_categories + assert "replacement characters" in json.dumps(parsed.defects) + + +@pytest.mark.parametrize("charset", [b"unicode_escape", b"raw_unicode_escape"]) +@pytest.mark.parametrize("inline", [False, True]) +def test_charset_cannot_inject_surrogates_into_json(charset, inline): + """A codec-produced surrogate cannot break UTF-8 output consumers.""" + raw = _HEADERS + _entity( + b"text/plain; charset=" + charset, + b'inline; filename="evidence.txt"' if inline else None, + base64.b64encode(b"\\ud800"), + b"Content-Transfer-Encoding: base64\r\n", + ) + parsed = mailparser.parse_from_bytes(raw) + assert parsed.mail_json.encode("utf-8") + assert parsed.text_plain == [r"\ud800"] + assert parsed.has_defects is True + assert "CharsetDecodeDefect" in parsed.defects_categories + assert parsed.message is not None + assert parsed.message.get_payload(decode=True) == b"\\ud800" + if inline: + assert base64.b64decode(parsed.attachments[0]["payload"]) == b"\\ud800" diff --git a/tests/test_input_octet_preservation.py b/tests/test_input_octet_preservation.py new file mode 100644 index 0000000..57b6b68 --- /dev/null +++ b/tests/test_input_octet_preservation.py @@ -0,0 +1,230 @@ +"""File inputs preserve the same evidence and payloads as byte inputs.""" + +import base64 +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +import mailparser +from mailparser.exceptions import MailParserRecursionError + +_HEADERS = ( + b"Date: Thu, 01 Oct 2026 12:00:00 +0000\r\n" + b"From: alice@example.com\r\n" + b"To: bob@example.com\r\n" +) + + +def _parse_pair(tmp_path, raw): + path = tmp_path / "message.eml" + path.write_bytes(raw) + return mailparser.parse_from_file(path), mailparser.parse_from_bytes(raw) + + +@pytest.mark.parametrize( + ("bad", "escaped"), + [ + (b"\xff", r"\udcff"), + (b"\xc3", r"\udcc3"), + (b"\xc0\xaf", r"\udcc0\udcaf"), + (b"\xed\xa0\x80", r"\udced\udca0\udc80"), + ], +) +@pytest.mark.parametrize( + "template", [b"a%s@example.com", b"a@ex%s.example", b'"a%s"@example.com'] +) +def test_file_invalid_octets_remain_evidence(tmp_path, bad, escaped, template): + """Dropping malformed UTF-8 must not manufacture a valid sender.""" + address = template % bad + raw = _HEADERS.replace(b"alice@example.com", address) + b"\r\nbody" + parsed, reference = _parse_pair(tmp_path, raw) + expected = [ + { + "reason": "invalid-utf8", + "raw": template.decode("ascii") % escaped, + "recovered": False, + "candidates": [], + "header": "from", + "occurrence": 0, + } + ] + + assert parsed.from_ == reference.from_ == [] + assert parsed.to == reference.to == [("", "bob@example.com")] + assert parsed.has_defects is True + assert parsed.address_header_defects == expected + assert parsed.address_header_defects == reference.address_header_defects + assert parsed.defects_categories == reference.defects_categories + assert parsed.defects == reference.defects + assert parsed.message is not None + assert reference.message is not None + raw_items = list(parsed.message.raw_items()) + assert raw_items == list(reference.message.raw_items()) + raw_from = dict(parsed.message.raw_items())["From"] + assert raw_from.encode("ascii", "surrogateescape") == address + output = json.loads(parsed.mail_json.encode("utf-8")) + assert output["address_header_defects"] == expected + assert output["has_defects"] is True + + +@pytest.mark.parametrize("bad_occurrence", [0, 1]) +def test_file_neighbors_and_occurrences(tmp_path, bad_occurrence): + """Recover valid neighbors without discarding later header evidence.""" + values = [b"first@example.com", b"later@example.com"] + values[bad_occurrence] = b"a\xff@example.com, valid@example.com" + raw = ( + _HEADERS.replace(b"To: bob@example.com\r\n", b"") + + b"".join(b"To: " + value + b"\r\n" for value in values) + + b"\r\nbody" + ) + parsed, reference = _parse_pair(tmp_path, raw) + expected_to = "first@example.com" + if bad_occurrence == 0: + expected_to = "valid@example.com" + + assert parsed.to == reference.to == [("", expected_to)] + assert parsed.from_ == [("", "alice@example.com")] + assert parsed.has_defects is True + assert parsed.address_header_defects == [ + { + "reason": "invalid-utf8", + "raw": r"a\udcff@example.com", + "recovered": False, + "candidates": [], + "header": "to", + "occurrence": bad_occurrence, + } + ] + assert parsed.address_header_defects == reference.address_header_defects + assert parsed.message is not None + raw_values = [ + value.encode("ascii", "surrogateescape") + for name, value in parsed.message.raw_items() + if name.lower() == "to" + ] + assert raw_values == values + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + (b"Alice ", [("Alice", "alice@example.com")]), + ( + b"Jos\xc3\xa9 ", + [("Jos\u00e9", "jos\u00e9@example.com")], + ), + ( + b"<@old.example:alice@example.com>", + [("", "alice@example.com")], + ), + ], +) +def test_file_valid_addresses_match_bytes(tmp_path, value, expected): + """The file boundary preserves accepted syntax and Unicode identities.""" + raw = _HEADERS.replace(b"alice@example.com", value) + b"\r\nbody" + parsed, reference = _parse_pair(tmp_path, raw) + + assert parsed.from_ == reference.from_ == expected + assert parsed.has_defects == reference.has_defects is False + assert parsed.address_header_defects == [] + assert parsed.message is not None + assert reference.message is not None + raw_items = list(parsed.message.raw_items()) + assert raw_items == list(reference.message.raw_items()) + + +def test_file_mime_charset_decoding_uses_original_octets(tmp_path): + """An 8-bit body is decoded with its declared charset, not UTF-8.""" + raw = ( + _HEADERS + + b"MIME-Version: 1.0\r\n" + + b"Content-Type: text/plain; charset=iso-8859-1\r\n" + + b"Content-Transfer-Encoding: 8bit\r\n\r\nCaf\xe9" + ) + parsed, reference = _parse_pair(tmp_path, raw) + + assert parsed.text_plain == reference.text_plain == ["Caf\u00e9"] + assert parsed.message is not None + assert parsed.message.get_payload(decode=True) == b"Caf\xe9" + assert parsed.has_defects == reference.has_defects is False + + +def test_file_binary_attachment_preserves_octets_and_line_endings(tmp_path): + """Loading a file cannot decode or translate an attachment's bytes.""" + payload = b"\x00\x80\xff\r\n\xc3\xa9\n\rEND" + raw = ( + _HEADERS + + b"MIME-Version: 1.0\r\n" + + b"Content-Type: application/octet-stream\r\n" + + b'Content-Disposition: attachment; filename="sample.bin"\r\n' + + b"Content-Transfer-Encoding: binary\r\n\r\n" + + payload + ) + parsed, reference = _parse_pair(tmp_path, raw) + + assert len(parsed.attachments) == 1 + assert parsed.attachments == reference.attachments + assert base64.b64decode(parsed.attachments[0]["payload"]) == payload + assert parsed.message is not None + assert parsed.message.get_payload(decode=True) == payload + + +def test_cli_file_reports_invalid_address_octets(tmp_path): + """The real --file CLI must not print a sender created by byte loss.""" + raw = _HEADERS.replace(b"alice@example.com", b"a\xff@example.com") + path = tmp_path / "message.eml" + path.write_bytes(raw + b"\r\nbody") + result = subprocess.run( + [sys.executable, "-m", "mailparser", "--file", str(path), "--json"], + cwd=Path(__file__).resolve().parents[1], + capture_output=True, + check=True, + timeout=10, + ) + output = json.loads(result.stdout.decode("utf-8")) + + assert not output.get("from") + assert output["to"] == [["", "bob@example.com"]] + assert output["has_defects"] is True + assert output["address_header_defects"] == [ + { + "reason": "invalid-utf8", + "raw": r"a\udcff@example.com", + "recovered": False, + "candidates": [], + "header": "from", + "occurrence": 0, + } + ] + + +def test_converted_outlook_file_is_removed_after_success(tmp_path): + """Converted temporary files are cleaned after lossless loading.""" + raw = _HEADERS.replace(b"alice@example.com", b"a\xff@example.com") + path = tmp_path / "converted.eml" + path.write_bytes(raw + b"\r\nbody") + + parsed = mailparser.MailParser.from_file(path, is_outlook=True) + + assert not path.exists() + assert parsed.from_ == [] + assert parsed.has_defects is True + assert parsed.address_header_defects[0]["reason"] == "invalid-utf8" + + +@pytest.mark.parametrize("is_outlook", [False, True]) +def test_file_recursion_guard_and_cleanup(tmp_path, is_outlook): + """Real nested input retains the public recursion and cleanup guards.""" + opening = "Content-Type: multipart/mixed; boundary=B{0}\r\n\r\n--B{0}\r\n" + parts = b"".join(opening.format(i).encode() for i in range(3000)) + raw = _HEADERS + parts + b"text\r\n" + path = tmp_path / "nested.eml" + path.write_bytes(raw) + + with pytest.raises(MailParserRecursionError): + mailparser.MailParser.from_file(path, is_outlook=is_outlook) + + assert path.exists() is not is_outlook diff --git a/tests/test_mail_parser.py b/tests/test_mail_parser.py index b01395b..a636f0d 100644 --- a/tests/test_mail_parser.py +++ b/tests/test_mail_parser.py @@ -692,7 +692,8 @@ def test_parsing_know_values(self): mail = mailparser.parse_from_file(mail_test_2) trust = "smtp.customers.net" - self.assertFalse(mail.has_defects) + self.assertTrue(mail.has_defects) + self.assertIn("CharsetDecodeDefect", mail.defects_categories) raw = "217.76.210.112" result = mail.get_server_ipaddress(trust) @@ -720,7 +721,7 @@ def test_parsing_know_values(self): self.assertEqual(raw, result) result = mail.has_defects - self.assertFalse(result) + self.assertTrue(result) result = len(mail.attachments) self.assertEqual(3, result) @@ -736,11 +737,12 @@ def test_types(self): mail = mailparser.parse_from_file(mail_test_2) trust = "smtp.customers.net" - self.assertFalse(mail.has_defects) + self.assertTrue(mail.has_defects) + self.assertIn("CharsetDecodeDefect", mail.defects_categories) result = mail.mail self.assertIsInstance(result, dict) - self.assertNotIn("defects", result) + self.assertIn("defects", result) self.assertIn("has_defects", result) result = mail.get_server_ipaddress(trust) @@ -800,10 +802,14 @@ def test_defects(self): mail = mailparser.parse_from_file(mail_test_1) self.assertTrue(mail.has_defects) - self.assertEqual(1, len(mail.defects)) - self.assertEqual(1, len(mail.defects_categories)) + self.assertEqual(2, len(mail.defects)) + self.assertEqual(2, len(mail.defects_categories)) self.assertIn("defects", mail.mail) self.assertIn("CloseBoundaryNotFoundDefect", mail.defects_categories) + self.assertIn("AddressHeaderDefect", mail.defects_categories) + self.assertEqual( + mail.address_header_defects[0]["reason"], "overlong-encoded-word" + ) def test_defects_bug(self): mail = mailparser.parse_from_file(mail_malformed_2) @@ -1277,7 +1283,18 @@ def test_parse_from_bytes(self): mail = mailparser.parse_from_bytes(mail_bytes) trust = "smtp.customers.net" - self.assertFalse(mail.has_defects) + self.assertTrue(mail.has_defects) + self.assertIn("CharsetDecodeDefect", mail.defects_categories) + self.assertIn( + { + "text/html": [ + "CharsetDecodeDefect: part 4, charset 'utf-8' failed; " + "decoded with utf-8 replacement characters. " + "Original bytes remain in the MIME part." + ] + }, + mail.defects, + ) raw = "217.76.210.112" result = mail.get_server_ipaddress(trust) @@ -1305,7 +1322,7 @@ def test_parse_from_bytes(self): self.assertEqual(raw, result) result = mail.has_defects - self.assertFalse(result) + self.assertTrue(result) result = len(mail.attachments) self.assertEqual(3, result) diff --git a/tests/test_mime_unicode_metadata.py b/tests/test_mime_unicode_metadata.py new file mode 100644 index 0000000..b4ae788 --- /dev/null +++ b/tests/test_mime_unicode_metadata.py @@ -0,0 +1,166 @@ +"""Raw UTF-8 MIME metadata survives every public input boundary.""" + +import base64 +import email +import json + +import pytest + +import mailparser + +_PAYLOAD = bytes(range(256)) +_HEADERS = ( + b"Date: Thu, 01 Oct 2026 12:00:00 +0000\r\n" + b"From: alice@example.com\r\nTo: bob@example.com\r\n" + b"MIME-Version: 1.0\r\n" +) + + +def _parse(raw, factory, tmp_path): + if factory == "file": + path = tmp_path / "metadata.eml" + path.write_bytes(raw) + return mailparser.parse_from_file(path) + if factory == "string": + return mailparser.parse_from_string(raw.decode("utf-8", "surrogateescape")) + return mailparser.parse_from_bytes(raw) + + +@pytest.mark.parametrize("factory", ["file", "bytes", "string"]) +@pytest.mark.parametrize("parameter", ["filename", "name"]) +def test_raw_utf8_mime_metadata_and_payload(factory, parameter, tmp_path): + """Named attachments preserve Unicode metadata and exact bytes.""" + filename = "ФНС_РФ558.zip" + content_id = "" + content_type = "application/octet-stream" + disposition = "attachment" + if parameter == "filename": + disposition += f'; filename="{filename}"' + else: + content_type += f'; name="{filename}"' + raw = ( + _HEADERS + + ( + f"Content-Type: {content_type}\r\n" + f"Content-Disposition: {disposition}\r\n" + f"Content-ID: {content_id}\r\n" + "Content-Transfer-Encoding: base64\r\n\r\n" + ).encode() + + base64.b64encode(_PAYLOAD) + ) + mail = _parse(raw, factory, tmp_path) + attachment = mail.attachments[0] + assert attachment["filename"] == attachment["safe_filename"] == filename + assert attachment["content-disposition"] == disposition + assert attachment["content-id"] == content_id + assert base64.b64decode(attachment["payload"]) == _PAYLOAD + assert not mail.has_defects + destination = tmp_path / "attachments" + destination.mkdir() + mail.write_attachments(destination) + assert (destination / filename).read_bytes() == _PAYLOAD + assert mail.message is not None + original = ( + email.message_from_string(raw.decode()) + if factory == "string" + else email.message_from_bytes(raw) + ) + assert list(mail.message.raw_items()) == list(original.raw_items()) + assert mail.message.get_payload(decode=True) == _PAYLOAD + attachments_json = mail.attachments_json + assert isinstance(attachments_json, str) + assert json.loads(attachments_json)[0]["filename"] == filename + + +@pytest.mark.parametrize("factory", ["file", "bytes", "string"]) +def test_rfc2231_continuation_still_decodes(factory, tmp_path): + """The UTF-8 view preserves RFC 2231 parameter continuation semantics.""" + raw = ( + _HEADERS + b"Content-Type: application/octet-stream\r\n" + b"Content-Disposition: attachment;\r\n" + b" filename*0*=utf-8''caf%C3%A9; filename*1=.bin\r\n" + b"Content-Transfer-Encoding: base64\r\n\r\n" + base64.b64encode(_PAYLOAD) + ) + mail = _parse(raw, factory, tmp_path) + assert mail.attachments[0]["filename"] == "café.bin" + assert base64.b64decode(mail.attachments[0]["payload"]) == _PAYLOAD + assert not mail.has_defects + + +@pytest.mark.parametrize("factory", ["file", "bytes", "string"]) +def test_invalid_mime_octets_remain_evidence(factory, tmp_path): + """Undecodable metadata stays visibly damaged with exact raw evidence.""" + raw = ( + _HEADERS + b"Content-Type: application/octet-stream\r\n" + b'Content-Disposition: attachment; filename="a\xffb.bin"\r\n' + b"Content-ID: \r\n" + b"Content-Transfer-Encoding: base64\r\n\r\n" + base64.b64encode(_PAYLOAD) + ) + mail = _parse(raw, factory, tmp_path) + assert mail.attachments[0]["filename"] == "a\ufffdb.bin" + assert mail.attachments[0]["content-id"] == "" + assert base64.b64decode(mail.attachments[0]["payload"]) == _PAYLOAD + assert mail.has_defects + assert "MimeHeaderDefect" in mail.defects_categories + evidence = " ".join(mail.defects[-1]["mime-headers"]) + assert "content-disposition[0]" in evidence + assert "content-id[0]" in evidence + assert "\\xff" in evidence + assert "part 0" in evidence + assert mail.message is not None + assert "\udcff" in dict(mail.message.raw_items())["Content-Disposition"] + mail.mail_json.encode("utf-8") + + +def test_repeated_mime_header_keeps_occurrence_evidence(tmp_path): + """A malformed later field cannot hide behind a valid first value.""" + raw = ( + _HEADERS + b"Content-Type: application/octet-stream\r\n" + b'Content-Disposition: attachment; filename="valid.bin"\r\n' + b'Content-Disposition: attachment; filename="bad\xff.bin"\r\n' + b"Content-Transfer-Encoding: base64\r\n\r\n" + base64.b64encode(_PAYLOAD) + ) + mail = _parse(raw, "bytes", tmp_path) + assert mail.attachments[0]["filename"] == "valid.bin" + assert mail.has_defects + assert "content-disposition[1]" in mail.defects[-1]["mime-headers"][0] + assert "\\xff" in mail.defects[-1]["mime-headers"][0] + assert base64.b64decode(mail.attachments[0]["payload"]) == _PAYLOAD + + +def test_metadata_defects_retain_mime_part_context(tmp_path): + """Each sibling keeps its own payload and raw metadata evidence.""" + raw = ( + _HEADERS + b'Content-Type: multipart/mixed; boundary="part"\r\n\r\n' + b"--part\r\nContent-Type: application/octet-stream\r\n" + b'Content-Disposition: attachment; filename="good.bin"\r\n\r\ngood\r\n' + b"--part\r\nContent-Type: application/octet-stream\r\n" + b'Content-Disposition: attachment; filename="bad\xff.bin"\r\n\r\nbad\r\n' + b"--part--\r\n" + ) + mail = _parse(raw, "bytes", tmp_path) + assert [a["filename"] for a in mail.attachments] == ["good.bin", "bad�.bin"] + assert [base64.b64decode(a["payload"]) for a in mail.attachments] == [ + b"good", + b"bad", + ] + assert "part 2" in mail.defects[-1]["mime-headers"][0] + + +@pytest.mark.parametrize("charset", ["unicode_escape", "raw_unicode_escape"]) +@pytest.mark.parametrize("parameter", ["filename", "name"]) +def test_encoded_filename_cannot_produce_surrogate(charset, parameter): + """Malformed decoded Unicode stays literal with MIME defect evidence.""" + word = f"=?{charset}?b?XHVkODAw?=" + header = ( + f'Content-Disposition: attachment; filename="{word}"\r\n' + if parameter == "filename" + else f'Content-Type: text/plain; name="{word}"\r\n' + ) + mail = mailparser.parse_from_bytes(header.encode() + b"\r\nbody") + assert mail.mail_json.encode("utf-8") + assert mail.attachments[0]["filename"] == word + assert mail.has_defects is True + assert "MimeHeaderDefect" in mail.defects_categories + assert word in str(mail.defects) + assert base64.b64decode(mail.attachments[0]["payload"]) == b"body" diff --git a/tests/test_received_lexical.py b/tests/test_received_lexical.py new file mode 100644 index 0000000..bd8c57d --- /dev/null +++ b/tests/test_received_lexical.py @@ -0,0 +1,337 @@ +"""Received lexical boundaries and forensic attribution regressions.""" + +import json +import time + +import pytest + +import mailparser +from mailparser.utils import get_from_clause, parse_received + +DATE = "Thu, 01 Oct 2026 12:00:00 +0000" + + +def _mail(*received, extra=""): + """Parse explicit wire headers through the public bytes factory.""" + raw = ( + f"Date: {DATE}\r\nFrom: alice@sender.example\r\n" + "To: bob@recipient.example\r\nSubject: intact\r\n" + + "".join(f"Received: {value}\r\n" for value in received) + + extra + + "\r\nbody" + ) + return mailparser.parse_from_bytes(raw.encode()) + + +@pytest.mark.parametrize( + "comment", + [ + "", + " (note; relay)", + " (note by relay)", + " (outer (inner; by relay))", + r" (note\); by relay)", + ' (quoted-looking " by relay; still comment)', + ], +) +def test_comment_delimiters_do_not_split_received(comment): + """Comments retain their contents and never select stamp delimiters.""" + origin = "sender.example" + comment + header = f"from {origin} by mx.example with ESMTP; {DATE}" + mail = _mail(header) + parsed = mail.received[0] + assert parsed["from"] == origin + assert parsed["by"] == "mx.example" + assert parsed["date"] == DATE + assert parsed["date_utc"] == "2026-10-01T12:00:00+00:00" + assert get_from_clause(header) == origin + assert mail.message is not None + assert mail.message.get_all("Received") == [header] + assert mail.has_defects is False + + +@pytest.mark.parametrize( + "recipient", + [ + '<"a; by b"@recipient.example>', + '<"a\\"; by b"@recipient.example>', + '<"two spaces; from here"@recipient.example>', + "", + ], +) +def test_recipient_is_one_lexical_value(recipient): + """Quotes, escapes and domain literals cannot forge clauses or dates.""" + header = f"from sender.example by mx.example with ESMTP for {recipient}; {DATE}" + mail = _mail(header) + assert mail.received[0]["for"] == recipient + assert mail.received[0]["date"] == DATE + assert mail.received[0]["by"] == "mx.example" + assert get_from_clause(header) == "sender.example" + + +@pytest.mark.parametrize( + ("tail", "reason"), + [ + ("(unclosed; by mx.example", "unclosed-comment"), + ('"unclosed; by mx.example', "unclosed-quoted-string"), + ("[unclosed; by mx.example", "unclosed-domain-literal"), + ("; {DATE}' + ) + older = f"from forged.example ([9.9.9.9]) by mx.example; {DATE}" + assert _mail(first, older).get_server_ipaddress("mx.example") is None + + +def test_received_date_comment_semicolon_is_not_reinterpreted(): + """A comment after the stamp separator never replaces the date.""" + date = DATE + " (note; preserved)" + parsed = _mail(f"from sender.example by mx.example; {date}").received[0] + assert parsed["date"] == date + assert parsed["date_utc"] == "2026-10-01T12:00:00+00:00" + + +def test_deep_comments_scan_without_recursion_or_backtracking(): + """Attacker-chosen comment depth and spacing stay bounded.""" + origin = "sender.example " + "(" * 20000 + "; by " + ")" * 20000 + header = f"from {origin} by mx.example; {DATE}" + start = time.perf_counter() + assert parse_received(header)["from"] == origin + assert get_from_clause(header) == origin + assert time.perf_counter() - start < 2 + + +@pytest.mark.parametrize( + "helo", + [ + 'helo="evil by q [8.8.8.8]"', + "helo=evil by q [8.8.8.8]", + r"helo=evil\) by q [8.8.8.8]", + "HELO evil (by q [8.8.8.8])", + ], +) +def test_helo_text_cannot_inject_ip_after_comment_keyword(helo): + """Everything after a HELO marker in that comment is sender text.""" + header = f"from sender.example ([1.2.3.4] {helo}) by mx.example; {DATE}" + assert _mail(header).get_server_ipaddress("mx.example") == "1.2.3.4" + + +def test_legacy_extra_separator_recovers_with_evidence(): + """A known MTA separator before FOR is recovered without losing it.""" + header = ( + "from sender.example by mx.example with ESMTP id token;\r\n" + f'\tfor <"a; by b"@recipient.example>; {DATE}' + ) + mail = _mail(header) + assert mail.received[0]["date"] == DATE + assert mail.received[0]["for"] == '<"a; by b"@recipient.example>' + assert mail.received[0]["date_utc"] == "2026-10-01T12:00:00+00:00" + assert mail.received_header_defects == [ + { + "reason": "misplaced-date-separator", + "raw": header, + "recovered": True, + "header": "received", + "occurrence": 0, + } + ] + assert mail.has_defects is True + + +def test_invalid_received_octet_does_not_crash_or_disappear(): + """Compatibility Header objects retain their octets as raw evidence.""" + raw = b"from sender\xff.example (unclosed by mx.example" + mail = mailparser.parse_from_bytes(b"Received: " + raw + b"\r\n\r\n") + evidence = r"from sender\udcff.example (unclosed by mx.example" + assert mail.received[0]["raw"] == evidence + assert mail.received_header_defects[0]["raw"] == evidence + assert mail.message is not None + assert next(mail.message.raw_items())[1].encode("utf-8", "surrogateescape") == raw + + +def test_utf8_received_comment_does_not_crash(): + """Raw-items access accepts a UTF-8 Header without losing its bytes.""" + header = f"from sender.example (José; by relay) by mx.example; {DATE}" + mail = _mail(header) + assert mail.received[0]["by"] == "mx.example" + assert mail.received[0]["date"] == DATE + assert mail.received[0]["from"].encode("utf-8", "surrogateescape") == ( + "sender.example (José; by relay)".encode() + ) + assert mail.has_defects is False + + +def test_opaque_obsolete_received_keeps_raw_without_false_defect(): + """Unknown legacy trace tokens are not proof of malformed syntax.""" + header = "(legacy trace)" + mail = _mail(header) + assert mail.received[0]["raw"] == header + assert mail.has_defects is False + + +@pytest.mark.parametrize( + ("value", "reason"), + [ + ("from source.example ) by mx.example", "unexpected-comment-close"), + ("from source.example <>", "nested-angle-address"), + (f"from source.example; {DATE}; ignored", "ambiguous-date-separators"), + ], +) +def test_ambiguous_trace_falls_back_with_reason(value, reason): + """Malformed separators do not silently select an alternative trace.""" + mail = _mail(value) + assert mail.received[0]["raw"] == value + assert mail.received_header_defects[0]["reason"] == reason + assert mail.received_header_defects[0]["recovered"] is False + assert get_from_clause(value) == "" + + +def test_group_spans_ignore_quoted_and_escaped_delimiters(): + """Only structural comments/literals are IP attribution locations.""" + from mailparser.utils import group_spans + + assert group_spans('"fake [8.8.8.8]" (real [1.2.3.4])') == [(18, 32)] + text = r"source (real [1.2.3.4] escaped\) fake [8.8.8.8])" + assert group_spans(text) == [(8, len(text) - 1)] + + +def test_many_ip_groups_have_bounded_attribution_work(monkeypatch): + """Attacker-chosen groups cannot cause a full rescan for every IP.""" + import mailparser.core as core + + accesses = 0 + original = core.group_spans + + class CountedSpans(list): + """Count actual span visits independently of machine speed.""" + + def __iter__(self): + nonlocal accesses + for item in super().__iter__(): + accesses += 1 + yield item + + def __getitem__(self, index): + nonlocal accesses + accesses += 1 + return super().__getitem__(index) + + monkeypatch.setattr( + core, "group_spans", lambda value: CountedSpans(original(value)) + ) + count = 256 + header = "from sender " + "(9.9.9.9) " * count + f"by mx.example; {DATE}" + mail = _mail(header) + assert mail.get_server_ipaddress("mx.example") == "9.9.9.9" + assert accesses < 20 * count + + +@pytest.mark.parametrize("malformed", [False, True]) +def test_invalid_received_evidence_is_utf8_serializable(malformed): + """Invalid trace bytes survive in evidence without breaking JSON sinks.""" + raw = b"from sender\xff.example " + ( + b"(unclosed by mx.example" + if malformed + else b"([9.9.9.9]) by mx.example; " + DATE.encode() + ) + mail = mailparser.parse_from_bytes(b"Received: " + raw + b"\r\n\r\n") + rendered = mail.mail_json.encode("utf-8") + assert b"\\\\udcff" in rendered + assert mail.has_defects is True + assert mail.message is not None + assert next(mail.message.raw_items())[1].encode("utf-8", "surrogateescape") == raw + + +@pytest.mark.parametrize( + "argument", + ['"x by y" [8.8.8.8]', "host [8.8.8.8]", "x (nested [8.8.8.8])", r"x\) [8.8.8.8]"], +) +@pytest.mark.parametrize("real_ip", ["(1.2.3.4) ", ""]) +def test_comment_initial_helo_cannot_supply_sender_ip(argument, real_ip): + """A comment-initial HELO argument cannot override the connection IP.""" + raw = ( + f"Received: from source {real_ip}(HELO {argument}) by mx.example; {DATE}\r\n" + f"Received: from forged ([8.8.4.4]) by mx.example; {DATE}\r\n\r\n" + ) + mail = mailparser.parse_from_string(raw) + assert mail.get_server_ipaddress("mx.example") == ("1.2.3.4" if real_ip else None) + + +@pytest.mark.parametrize( + "comment", + ["( helo [45.33.32.156])", "(\thelo [45.33.32.156])", "((helo [45.33.32.156]))"], +) +def test_literal_helo_hostname_after_comment_fws(comment): + """Formatting or nesting cannot promote the sender's first-token IP.""" + header = f"from 8.8.8.8 {comment} by mx.example; {DATE}" + assert _mail(header).get_server_ipaddress("mx.example") == "45.33.32.156" From 88794d71d721fe270c9d0494c18f91ee451f9331 Mon Sep 17 00:00:00 2001 From: Fedele Mantuano Date: Thu, 1 Oct 2026 23:41:04 +0200 Subject: [PATCH 2/3] docs: enhance README with package distribution and integration details --- README.md | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index b2aebf5..a869ab0 100644 --- a/README.md +++ b/README.md @@ -117,13 +117,36 @@ Or contribute with Bitcoin: Thank you for supporting the evolution of mail-parser! -# mail-parser on Web +# mail-parser in the ecosystem -Explore mail-parser on these platforms: +## Packages and distributions -- **[FreeBSD port](https://www.freshports.org/mail/py-mail-parser/)** -- **[Arch User Repository](https://aur.archlinux.org/packages/mailparser/)** -- **[REMnux](https://docs.remnux.org/discover-the-tools/analyze+documents/email+messages#mail-parser)** +Find mail-parser in these package repositories and security toolkits: + +- **[FreeBSD](https://www.freshports.org/mail/py-mail-parser/)**: available as the + `mail/py-mail-parser` port. +- **[Arch User Repository (AUR)](https://aur.archlinux.org/packages/mailparser/)**: + community-maintained `mailparser` package for Arch Linux. +- **[Debian](https://packages.debian.org/source/sid/mail-parser)**: `mail-parser` + source package in Debian unstable (sid). +- **[REMnux](https://docs.remnux.org/discover-the-tools/analyze+documents/email+messages#mail-parser)**: + included in the REMnux malware analysis toolkit for analyzing email messages. + +## Integrations + +- **[IBM Security QRadar SOAR — Parse Utilities](https://github.com/ibmresilient/resilient-community-apps/blob/main/fn_parse_utilities/README.md)**: + uses mail-parser to extract headers, body parts, and attachments from `.eml` + and `.msg` files for incident response workflows. + +## Research use + +mail-parser is also used in academic research: + +- **[A Large-Scale Empirical Study of Modern Phishing Email Content](https://arxiv.org/abs/2609.30683)** + — Jaehwan Park et al., arXiv preprint, September 2026. + The study analyzes 2.9 million phishing emails and uses mail-parser to extract + PDF, image, and calendar invitation attachments from their MIME structure + (Section III-A, reference 34). # Description From 55a0bebf7081dddba166de49d45ee8a82b1334e2 Mon Sep 17 00:00:00 2001 From: Fedele Mantuano Date: Fri, 2 Oct 2026 00:08:16 +0200 Subject: [PATCH 3/3] test: replace timing assertions with deterministic work budgets --- tests/test_address_recovery.py | 14 ++-- tests/test_date_recovery.py | 11 +-- tests/test_encoded_word_context.py | 17 ++-- tests/test_mail_parser.py | 81 ++++++++++--------- tests/test_received_lexical.py | 10 +-- tests/test_utils.py | 76 +++++------------- tests/work_budget.py | 121 +++++++++++++++++++++++++++++ 7 files changed, 210 insertions(+), 120 deletions(-) create mode 100644 tests/work_budget.py diff --git a/tests/test_address_recovery.py b/tests/test_address_recovery.py index ab48cd8..08aafa1 100644 --- a/tests/test_address_recovery.py +++ b/tests/test_address_recovery.py @@ -1,10 +1,10 @@ """RFC 5322 boundaries and forensic recovery regressions for issue #175.""" import json -import time from itertools import product import pytest +from work_budget import bounded_address_work import mailparser from mailparser.const import ADDRESSES_HEADERS @@ -203,11 +203,11 @@ def test_issue175_matrix(display, address, tail): "bad@name " + "(" * 4000 + "comment" + ")" * 4000 + " ", "x" * 40000, ], + ids=["space-padding", "open-angles", "nested-comments", "long-atom"], ) def test_hostile_address_input_finishes(value): - start = time.perf_counter() - mail = mailparser.parse_from_string(f"From: {value}\r\n\r\n") - assert time.perf_counter() - start < 2 + with bounded_address_work(len(value)): + mail = mailparser.parse_from_string(f"From: {value}\r\n\r\n") assert isinstance(mail.from_, list) @@ -249,13 +249,13 @@ def test_repeated_headers_are_all_inspected_without_changing_first_value(): "word " * 16000 + "user@example.com", " " * 16000, ], + ids=["many-words", "many-angle-addresses"], ) def test_malformed_words_and_many_angles_do_not_trigger_quadratic_work(value): - start = time.perf_counter() - mail = mailparser.parse_from_string(f"From: {value}\r\n\r\n") + with bounded_address_work(len(value)): + mail = mailparser.parse_from_string(f"From: {value}\r\n\r\n") assert mail.from_ == [] assert mail.has_defects - assert time.perf_counter() - start < 2 @pytest.mark.parametrize( diff --git a/tests/test_date_recovery.py b/tests/test_date_recovery.py index bee2c97..ca0f8c1 100644 --- a/tests/test_date_recovery.py +++ b/tests/test_date_recovery.py @@ -3,11 +3,12 @@ import datetime import io import json -import time import pytest +from work_budget import bounded_text_work import mailparser +from mailparser import dates from mailparser.utils import convert_mail_date VALID = "Thu, 01 Oct 2026 12:00:00 +0000" @@ -190,15 +191,15 @@ def test_absent_date_preserves_api_defaults(): def test_extreme_date_components_are_bounded(size): """Large years and nested comments neither overflow nor recurse.""" huge = "01 Oct " + "9" * size + " 12:00:00 +0000" - start = time.perf_counter() - mail = _mail(huge) + with bounded_text_work(dates, ("_date_tokens",), len(huge)): + mail = _mail(huge) assert mail.date is None assert mail.date_header_defects[0]["reason"] == "date-out-of-range" nested = "(" * size + "note" + ")" * size - parsed = _mail(nested + VALID) + with bounded_text_work(dates, ("_date_tokens",), len(nested + VALID)): + parsed = _mail(nested + VALID) assert parsed.date is not None assert parsed.date.isoformat() == EXPECTED - assert time.perf_counter() - start < 2 def test_invalid_date_octets_remain_in_diagnostics(): diff --git a/tests/test_encoded_word_context.py b/tests/test_encoded_word_context.py index 7b0717e..d845a58 100644 --- a/tests/test_encoded_word_context.py +++ b/tests/test_encoded_word_context.py @@ -3,6 +3,7 @@ import json import pytest +from work_budget import bounded_address_work, bounded_encoded_word_work import mailparser from mailparser.utils import get_addresses @@ -138,14 +139,12 @@ def test_encoded_names_do_not_hide_invalid_utf8_evidence(): def test_many_display_words_remain_bounded(): """A long encoded phrase has linear-sized output and one mailbox.""" - import time - count = 8000 value = "=?utf-8?Q?A?= " * count + "" - started = time.perf_counter() - mail = mailparser.parse_from_string(f"From: {value}\r\n\r\n") + with bounded_address_work(len(value)): + with bounded_encoded_word_work(count, len(value)): + mail = mailparser.parse_from_string(f"From: {value}\r\n\r\n") assert mail.from_ == [("A" * count, "alice@example.com")] - assert time.perf_counter() - started < 2 @pytest.mark.parametrize("context", ["phrase", "comment"]) @@ -193,13 +192,12 @@ def test_overlong_quoted_encoded_looking_text_remains_literal(): @pytest.mark.parametrize("terminated", [True, False]) def test_long_encoded_tokens_and_failed_matches_remain_bounded(terminated): """Matching a long invalid token must not repeatedly scan its suffix.""" - import time - count = 4000 word = "=?utf-8?Q?" + "A" * 100 + ("?=" if terminated else "") value = (word + " ") * count + "" - started = time.perf_counter() - mail = mailparser.parse_from_string(f"From: {value}\r\n\r\n") + with bounded_address_work(len(value)): + with bounded_encoded_word_work(count, len(value)): + mail = mailparser.parse_from_string(f"From: {value}\r\n\r\n") assert len(mail.from_) == 1 assert mail.from_[0][1] == "alice@example.com" if terminated: @@ -207,7 +205,6 @@ def test_long_encoded_tokens_and_failed_matches_remain_bounded(terminated): assert mail.address_header_defects[0]["reason"] == "overlong-encoded-word" else: assert mail.from_[0][0] == " ".join([word] * count) - assert time.perf_counter() - started < 2 @pytest.mark.parametrize("charset", ["unicode_escape", "raw_unicode_escape"]) diff --git a/tests/test_mail_parser.py b/tests/test_mail_parser.py index a636f0d..9bae6c4 100644 --- a/tests/test_mail_parser.py +++ b/tests/test_mail_parser.py @@ -25,11 +25,12 @@ import shutil import sys import tempfile -import time import unittest +from email.message import Message from unittest.mock import patch import pytest +from work_budget import WorkBudget import mailparser from mailparser.const import REGXIP6 @@ -2132,34 +2133,26 @@ def test_body_bytes_payload_decoded_with_charset(): # --------------------------------------------------------------------- # -def _timed_parse(raw): - """Parse ``raw`` and return the elapsed wall-clock seconds.""" - start = time.perf_counter() - mailparser.parse_from_string(raw) - return time.perf_counter() - start +@pytest.mark.parametrize("count", [2000, 8000]) +@pytest.mark.parametrize("distinct", [True, False], ids=["distinct", "repeated"]) +def test_distinct_header_names_scale_linearly(count, distinct, monkeypatch): + """Bound full header-list rescans on both independent growth axes.""" + budget = WorkBudget(8 * (count + 1)) + original = Message.get_all + def counted_get_all(message, *args, **kwargs): + budget.spend(len(message._headers)) + return original(message, *args, **kwargs) -def test_distinct_header_names_scale_linearly(): - """ - Parsing cost must stay linear in the number of *distinct* header names. - - Every distinct name used to trigger its own full rescan of the header - list (Message.get_all is O(total)), so cost grew as O(distinct x total): - 16,000 names cost ~5.8 s against ~0.06 s for 32,000 repeats of one name. - Header names are attacker-chosen and cheap to generate (CWE-407). - """ - - def build(n): - headers = "".join(f"X{i:05d}: v\r\n" for i in range(n)) - return f"From: a@b.c\r\n{headers}\r\nbody\r\n" - - small = min(_timed_parse(build(2000)) for _ in range(3)) - large = min(_timed_parse(build(8000)) for _ in range(3)) - - # 4x the names must not cost anywhere near 16x the time. The bound is - # loose so the test does not flake on a loaded machine; the quadratic - # behaviour it guards against was ~14x here. - assert large < small * 8, f"{small=} {large=} — scaling is not linear" + monkeypatch.setattr(Message, "get_all", counted_get_all) + headers = "".join(f"X{i if distinct else 0:05d}: v\r\n" for i in range(count)) + mail = mailparser.parse_from_string(f"From: a@b.c\r\n{headers}\r\nbody\r\n") + parsed_headers = mail.headers + assert len(parsed_headers) == (count + 1 if distinct else 2) + if distinct: + assert all(parsed_headers[f"X{i:05d}"] == "v" for i in range(count)) + else: + assert parsed_headers["X00000"] == ["v"] * count def test_header_named_after_a_method_does_not_break_mail_json(): @@ -2188,7 +2181,7 @@ def test_header_named_parse_is_reported_as_a_header(): assert mail.mail["parse"] == "x" -def test_header_named_headers_json_does_not_recurse(): +def test_header_named_headers_json_does_not_recurse(monkeypatch): """ "Headers_json: x" used to drive unbounded recursion: the headers property resolved the name through getattr, reaching the headers_json property, @@ -2203,8 +2196,15 @@ def test_header_named_headers_json_does_not_recurse(): filler = "".join(f"Z{i:04d}: v\r\n" for i in range(3200)) raw = f"Headers_json: x\r\n{filler}\r\nbody\r\n" - elapsed = _timed_parse(raw) - assert elapsed < 2, f"parse took {elapsed:.1f}s — recursion is back" + def unexpected_serialization(self): + pytest.fail("wire header invoked the headers_json property") + + monkeypatch.setattr( + mailparser.MailParser, "headers_json", property(unexpected_serialization) + ) + mail = mailparser.parse_from_string(raw) + assert mail.headers["Headers_json"] == "x" + assert json.loads(mail.mail_json)["headers_json"] == "x" def test_header_named_headers_json_variants_do_not_recurse(): @@ -2308,7 +2308,7 @@ def test_get_from_clause_is_anchored(): assert get_from_clause("by mx.victim.com\r\n\twith ESMTP") == "" -def test_json_suffixed_header_name_does_not_amplify(): +def test_json_suffixed_header_name_does_not_amplify(monkeypatch): """ A header named ``X_json_json_...`` must not build a tower of JSON. @@ -2321,8 +2321,18 @@ def test_json_suffixed_header_name_does_not_amplify(): name = "X" + "_json" * 40 raw = "".join(f"{name}{i}: x\r\n" for i in range(40)) + "\r\nbody\r\n" - elapsed = _timed_parse(raw) - assert elapsed < 2, f"parse took {elapsed:.1f}s — the suffix cycle is back" + original = mailparser.MailParser.__getattr__ + + def checked_getattr(self, attribute): + assert not attribute.lower().startswith("x_json"), ( + "wire header entered recursive attribute dispatch" + ) + return original(self, attribute) + + monkeypatch.setattr(mailparser.MailParser, "__getattr__", checked_getattr) + parsed = mailparser.parse_from_string(raw) + assert parsed.headers == {f"{name}{i}": "x" for i in range(40)} + assert len(parsed.headers_json) < 2 * len(raw) mail = mailparser.parse_from_string(f"{name}: x\r\n\r\n") # the wire value, not a JSON tower built out of it @@ -2464,12 +2474,11 @@ def test_headers_dedupes_case_variants_of_one_name(): raw = "".join(f"{v}: value\r\n" for v in variants) + "\r\nbody\r\n" mail = mailparser.parse_from_string(raw) - start = time.perf_counter() headers = mail.headers - elapsed = time.perf_counter() - start assert len(headers) == 1 - assert elapsed < 2, f"headers took {elapsed:.1f}s — the blowup is back" + assert next(iter(headers.values())) == ["value"] * 8000 + assert len(mail.headers_json) < len(raw) # honest duplicates keep every value under one key assert mailparser.parse_from_string("Subject: a\r\nSUBJECT: b\r\n\r\n").headers == { diff --git a/tests/test_received_lexical.py b/tests/test_received_lexical.py index bd8c57d..503fb79 100644 --- a/tests/test_received_lexical.py +++ b/tests/test_received_lexical.py @@ -1,11 +1,12 @@ """Received lexical boundaries and forensic attribution regressions.""" import json -import time import pytest +from work_budget import bounded_text_work import mailparser +from mailparser import utils from mailparser.utils import get_from_clause, parse_received DATE = "Thu, 01 Oct 2026 12:00:00 +0000" @@ -165,10 +166,9 @@ def test_deep_comments_scan_without_recursion_or_backtracking(): """Attacker-chosen comment depth and spacing stay bounded.""" origin = "sender.example " + "(" * 20000 + "; by " + ")" * 20000 header = f"from {origin} by mx.example; {DATE}" - start = time.perf_counter() - assert parse_received(header)["from"] == origin - assert get_from_clause(header) == origin - assert time.perf_counter() - start < 2 + with bounded_text_work(utils, ("split_received",), len(header)): + assert parse_received(header)["from"] == origin + assert get_from_clause(header) == origin @pytest.mark.parametrize( diff --git a/tests/test_utils.py b/tests/test_utils.py index 9a452ac..56afc4f 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -22,10 +22,12 @@ import os import subprocess import tempfile -import time import unittest from unittest.mock import Mock, patch +from work_budget import CountedNames, bounded_address_work, bounded_text_work + +from mailparser import utils from mailparser.exceptions import ( MailParserOSError, MailParserPathError, @@ -167,24 +169,12 @@ def test_msgconvert_timeout(self): os.unlink(tmp_name) def test_parse_received_space_padding_is_linear(self): - """ - Regression for the Received-header ReDoS (CWE-1333). - - A long run of spaces that is not followed by a clause keyword must not - drive ``_CLAUSE_SPLITTER`` into quadratic backtracking. Before the fix - (whitespace collapsed before the split) 20 000 padding spaces took - several seconds and a header-size-limit-sized run took minutes; after - the fix parsing is linear and effectively instant. The generous bound - stays robust on slow CI while still failing hard on any reintroduced - super-linear blowup. - """ + """Padding must be traversed with linear scanning work.""" received = "from host " + " " * 20000 + "nope; Mon, 10 Aug 2026 09:00:00 +0000" - start = time.perf_counter() - result = parse_received(received) - elapsed = time.perf_counter() - start + with bounded_text_work(utils, ("split_received",), len(received)): + result = parse_received(received) self.assertIsInstance(result, dict) self.assertEqual(result["date"], "Mon, 10 Aug 2026 09:00:00 +0000") - self.assertLess(elapsed, 2.0) def test_parse_received_no_matches(self): """Test parse_received with header that matches nothing""" @@ -686,33 +676,28 @@ def test_deduplicate_filename_scales_linearly(self): """A batch of same-named attachments must not be quadratic.""" from mailparser.utils import _deduplicate_filename - used_filenames = {} - start = time.monotonic() + used_filenames = CountedNames(16000) for _ in range(16000): _deduplicate_filename("a.bin", used_filenames) - elapsed = time.monotonic() - start - # Restarting the suffix scan at 1 for every attachment took ~14s - # here; resuming from the last suffix takes milliseconds. - self.assertLess(elapsed, 5) + self.assertEqual(len(used_filenames), 16000) def test_deduplicate_long_names_scale_linearly(self): """Names sharing a clamped stem must not rescan the whole range.""" from mailparser.utils import _deduplicate_filename, _safe_attachment_filename - used_filenames = {} - start = time.monotonic() + used_filenames = CountedNames(8000) + generated = [] for i in range(4000): # Distinct names that all clamp onto the same stem: keying the # resume counter on the name as sent made each one quadratic. filename = _safe_attachment_filename( "a" * 238 + chr(65 + i % 26) + chr(65 + (i // 26) % 26) ) - _deduplicate_filename(filename, used_filenames) - _deduplicate_filename(filename, used_filenames) - elapsed = time.monotonic() - start + generated.append(_deduplicate_filename(filename, used_filenames)) + generated.append(_deduplicate_filename(filename, used_filenames)) - self.assertLess(elapsed, 5) + self.assertEqual(len(set(generated)), 8000) def test_random_string(self): """Test random_string function""" @@ -983,40 +968,17 @@ def test_get_addresses_fallback_quoted_name_with_comma(self): self.assertEqual(result, [("Last, First", "user@example.com")]) def test_get_addresses_fallback_email_as_name_space_padding_is_linear(self): - """ - Regression for the fallback ReDoS (CWE-1333). - - ``From: alice@example.com `` forces the regex - fallback (the display name is an unquoted e-mail address, which the - strict parser rejects). The old combined pattern backtracked - cubically on trailing whitespace, so a few thousand padding spaces - took seconds and a real-world header (~100 KB) took hours. The - linear scan must recover the address and finish effectively - instantly; the generous bound keeps the assertion robust on slow CI - while still failing hard on any reintroduced super-linear blowup - (2000 spaces measured at ~3.5 s before the fix, <0.01 s after). - """ + """Structural recovery must scan padded display names linearly.""" header = "alice@example.com " + " " * 2000 - start = time.perf_counter() - result = get_addresses(header) - elapsed = time.perf_counter() - start + with bounded_address_work(len(header)): + result = get_addresses(header) self.assertEqual(result, [("alice@example.com", "bob@example.com")]) - self.assertLess(elapsed, 2.0) def test_get_addresses_fallback_angle_padding_is_linear(self): - """ - Regression for the fallback ReDoS (CWE-1333), quadratic variant. - - A run of ``<`` characters yields no parseable address, forcing the - fallback. The old pattern backtracked quadratically over the - padding; the linear scan must return no addresses (falling through - to the strict parser's result) without a super-linear slowdown. - """ - start = time.perf_counter() - result = get_addresses("<" * 40000) - elapsed = time.perf_counter() - start + """Unclosed angles must remain evidence with bounded scan work.""" + with bounded_address_work(40000): + result = get_addresses("<" * 40000) self.assertTrue(all(not addr for _, addr in result)) - self.assertLess(elapsed, 2.0) def test_get_addresses_without_strict_parameter(self): """ diff --git a/tests/work_budget.py b/tests/work_budget.py new file mode 100644 index 0000000..6ac193a --- /dev/null +++ b/tests/work_budget.py @@ -0,0 +1,121 @@ +"""Deterministic work counters for adversarial parsing regressions. + +Count text traversals, copied slices and collision probes, not elapsed time. +These guards cover the instrumented operations; they are not a benchmark or +an estimate of work inside the regex engine. Real parsing is always executed. +""" + +from contextlib import ExitStack, contextmanager +from functools import wraps +from unittest.mock import patch + + +class WorkBudget: + """Raise AssertionError when counted work exceeds the supplied limit.""" + + def __init__(self, limit): + self.limit = limit + self.used = 0 + + def spend(self, amount=1): + """Charge amount units, failing before excessive work continues.""" + self.used += amount + assert self.used <= self.limit, ( + f"work budget exceeded: {self.used} > {self.limit}" + ) + + +class CountedText(str): + """Count Python character visits and slices against a shared budget.""" + + budget: WorkBudget + + def __new__(cls, value, budget): + result = super().__new__(cls, value) + result.budget = budget + return result + + def __iter__(self): + for char in super().__iter__(): + self.budget.spend() + yield char + + def __getitem__(self, key): + result = super().__getitem__(key) + self.budget.spend(max(1, len(result))) + return CountedText(result, self.budget) + + +@contextmanager +def bounded_text_work(module, names, size, factor=64): + """Bound input volume and Python scanning at named text functions. + + Each wrapped function receives its real input as a str subclass. The + shared limit permits a constant number of passes over size characters. + Native string/regex internals are not counted by this helper. + """ + budget = WorkBudget(factor * size) + + def instrument(function): + @wraps(function) + def counted(value, *args, **kwargs): + budget.spend(max(1, len(value))) + return function(CountedText(value, budget), *args, **kwargs) + + return counted + + with ExitStack() as stack: + for name in names: + stack.enter_context( + patch.object(module, name, instrument(getattr(module, name))) + ) + yield budget + assert budget.used, "instrumented parsing path was not exercised" + + +@contextmanager +def bounded_address_work(size): + """Bound address scans and repeated candidate/display-name processing.""" + from mailparser import addresses + + names = ("_scan", "_name", "_mailbox", "_valid_phrase", "_display_source") + with bounded_text_work(addresses, names, size) as budget: + yield budget + + +@contextmanager +def bounded_encoded_word_work(count, size): + """Bound match attempts and bytes passed to the real word decoder.""" + from mailparser import addresses + + attempts = WorkBudget(4 * count + 8) + pattern = addresses._ENCODED_WORD + + class CountedPattern: + def match(self, *args, **kwargs): + attempts.spend() + return pattern.match(*args, **kwargs) + + decoded = WorkBudget(4 * size) + decode = addresses.decode_header + + def counted_decode(value): + decoded.spend(len(value)) + return decode(value) + + with patch.object(addresses, "_ENCODED_WORD", CountedPattern()): + with patch.object(addresses, "decode_header", counted_decode): + yield attempts, decoded + assert attempts.used, "encoded-word matcher was not exercised" + + +class CountedNames(dict): + """Count collision probes in an attachment batch of the supplied size.""" + + def __init__(self, count): + super().__init__() + self.budget = WorkBudget(4 * count) + + def __contains__(self, key): + self.budget.spend() + return super().__contains__(key)