Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 7 additions & 5 deletions crates/tinyredact/src/pii/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -535,8 +535,10 @@ fn cc_keyword_near(norm: &str, start: usize, end: usize) -> bool {
CC_KEYWORD_RE.is_match(&norm[lo..hi])
}

// Sort by start asc, length desc. Then walk in order, dropping any hit whose
// range overlaps a kept hit. Result: earlier + longer wins; no double-redact.
// Sort by start asc, length desc. Then walk in order, merging any hit whose
// range overlaps a kept hit into it. Result: earlier + longer keeps its token,
// and a partial overlap's tail is still covered (fails closed); no
// double-redact.
fn dedupe_overlaps(hits: &mut Vec<Hit>) {
hits.sort_by(|a, b| {
a.start
Expand All @@ -545,9 +547,9 @@ fn dedupe_overlaps(hits: &mut Vec<Hit>) {
});
let mut kept: Vec<Hit> = Vec::with_capacity(hits.len());
for h in hits.drain(..) {
let overlaps = kept.last().is_some_and(|k| h.start < k.end);
if !overlaps {
kept.push(h);
match kept.last_mut() {
Some(k) if h.start < k.end => k.end = k.end.max(h.end),
_ => kept.push(h),
}
}
*hits = kept;
Expand Down
19 changes: 19 additions & 0 deletions crates/tinyredact/src/pii/mod_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -709,3 +709,22 @@ fn a_separated_digit_list_with_no_card_is_left_alone() {
assert_eq!(redact_pii_with(input, policy).value, input, "{policy:?}");
}
}

#[test]
fn a_partial_overlap_keeps_its_tail_redacted() {
let mut hits = vec![
Hit {
start: 5,
end: 15,
token: PII_PHONE,
},
Hit {
start: 0,
end: 10,
token: PII_CC,
},
];
dedupe_overlaps(&mut hits);
let ranges: Vec<_> = hits.iter().map(|h| (h.start, h.end, h.token)).collect();
assert_eq!(ranges, vec![(0, 15, PII_CC)]);
}
Loading