From ebbabd41c2a6ba127aa920992301a9cafb9f7e92 Mon Sep 17 00:00:00 2001 From: Timothy Oluwatobi Ojebiyi Date: Tue, 3 Nov 2026 23:14:39 +0000 Subject: [PATCH] fix: one application per job however its emails word it, and stop missing real replies Backfilling Gmail split one application into several when its emails named the role differently, and dropped real replies. Emails now join the application they belong to: same thread first, then the one application whose role overlaps. Two confirmations stay two, and nothing is guessed when several could fit. A board's "your application was sent" for a different role no longer lands on the wrong application, and a late confirmation doesn't undo a later step. More application systems are searched and read (Workable's among them), and an email from a job-alerts address whose body says "thank you for applying" or similar is no longer thrown away as an alert. --- api/app/gmail.py | 122 +++++++++++++++++++++++----- api/evals/test_email_eval.py | 15 +++- api/tests/test_gmail.py | 149 +++++++++++++++++++++++++++++++++++ 3 files changed, 264 insertions(+), 22 deletions(-) diff --git a/api/app/gmail.py b/api/app/gmail.py index f5a561c..e2ffd3e 100644 --- a/api/app/gmail.py +++ b/api/app/gmail.py @@ -22,7 +22,7 @@ import re import secrets import time -from datetime import datetime, timezone +from datetime import date, datetime, timezone from email.utils import parseaddr from urllib.parse import urlencode @@ -54,7 +54,11 @@ ATS_SENDERS = ("greenhouse.io", "greenhouse-mail.io", "lever.co", "ashbyhq.com", "workable.com", "myworkday.com", "workday.com", "smartrecruiters.com", "icims.com", "successfactors.com", "taleo.net", "jobvite.com", "bamboohr.com", "teamtailor.com", "recruitee.com", "pinpointhq.com", "eightfold.ai", "hirevue.com", - "oraclecloud.com", "applytojob.com") + "oraclecloud.com", "applytojob.com", + # What they really send from. Workable's replies are titled "Junior Python Engineer - PetLabCo.", with + # nothing about an application for the subject search to find, rejections included. + "workablemail.com", "teamtailor-mail.com", "pinpoint.email", "tal.net", "brassring.com", "njoyn.com", + "ambertrack.global", "beapplied.com", "earlycareerscompany.com") _SUBJECT_TERMS = ("application", "applying", "interview", "assessment", '"next steps"', "offer", "unfortunately", "candidacy", '"your interest"') _SUBJECT_RE = re.compile(r"applica|applying|applied|interview|assessment|next step|candidacy|offer|unfortunately|" @@ -300,10 +304,16 @@ def relevant(email: dict, applications: list[dict]) -> bool: r"\bfits? you\b|\bapplication deadlines?\b|:\s*up to [£$€]", re.I) +_BODY_SAYS_THEY_APPLIED = re.compile(r"thank(?:s| you)(?: so much| very much)? for (?:applying|your application)|" + r"the position you have applied|we(?:'ve| have) received your application|" + r"your application (?:for|to|has|was|is)\b", re.I) + + def is_job_alert(email: dict) -> bool: """A job alert or recommendation, by its sender or subject: not about an application they made, so not read at all. - LinkedIn sends both alerts and "your application was sent" from one address, so what the subject says comes first.""" - if _SAYS_THEY_APPLIED.search(email["subject"]): + LinkedIn sends both alerts and "your application was sent" from one address, and Leonardo sends its rejections + from jobalerts.leonardojobs.co.uk, so what the email says about an application comes first.""" + if _SAYS_THEY_APPLIED.search(email["subject"]) or _BODY_SAYS_THEY_APPLIED.search(email.get("text") or ""): return False return bool(_ALERT_SENDER.search(email["sender_address"]) or _ALERT_SUBJECT.search(email["subject"])) @@ -369,6 +379,10 @@ async def read(email: dict, applications: list[dict]) -> dict: app = applications[n - 1] if isinstance(n, int) and 1 <= n <= len(applications) else None if app and not names_company(email, app["company"]): app = None + # A board's "your application was sent" is one application: at a recruiter, one for another role is another job. + if app and kind == "received" and _SENT_BY_BOARD.search(email["subject"]) and str(data.get("role") or "").strip() \ + and _role_words(data["role"]) != _role_words(app["job_title"]): + app = None confidence = data.get("confidence") if data.get("confidence") in ("high", "medium", "low") else "low" return {"kind": kind, "application": app, "confidence": confidence, "company": str(data.get("company") or (app or {}).get("company") or "")[:200], @@ -561,7 +575,7 @@ async def backfill_count(user_id: str, period: str) -> dict: _JOB_BOARDS = ("indeed", "linkedin", "glassdoor", "reed", "totaljobs", "cv-library", "cwjobs", "adzuna", "monster") _NO_NAME = re.compile(r"^(?:unknown|un?specified.*|not (?:specified|stated|provided|mentioned|named|given).*|n/?a|none|" - r"the (?:company|employer|role|position))?$", re.I) + r"(?:online )?assessment|the (?:company|employer|role|position))?$", re.I) _NO_ROLE = "Role not stated" @@ -759,13 +773,75 @@ def _key(text: str) -> str: return " ".join(re.findall(r"[a-z0-9]+", str(text or "").lower())) +_SENT_BY_BOARD = re.compile(r"your application was sent to", re.I) +_ROLE_NOISE = frozenset({"graduate", "junior", "senior", "entry", "level", "the", "and", "of", "for", "a", "in", "at", "to", + "uk", "role", "position", "programme", "program", "scheme", "opportunity"}) +_SAME_APPLICATION_DAYS = 120 # a follow-up further from the application than this is about another one +_APPLIED_AGAIN_DAYS = 14 # a confirmation sooner than this after a later step is the same application's + + +def _role_words(role: str) -> set[str]: + """What tells one role at a company from another; a bare number is a requisition or a year.""" + return {w for w in _key(role).split() if w not in _ROLE_NOISE and not w.isdigit()} + + +def _days_apart(earlier: str, later: str) -> int: + return (date.fromisoformat(later[:10]) - date.fromisoformat(earlier[:10])).days + + +def _application_for(email: dict, company_groups: list[list[dict]]) -> list[dict] | None: + """The application a follow-up email naming its role differently belongs to: "Full Stack Engineer - Intern" and + "Full Stack Developer Intern" at one company a fortnight apart. Its email thread settles it, or else the one + application whose role shares its words: with several it could be any, so it isn't guessed. Two confirmations are + two applications, however alike their roles.""" + def could_be(group: list[dict]) -> bool: + return (_days_apart(group[-1]["received_at"], email["received_at"]) <= _SAME_APPLICATION_DAYS + and not (email["kind"] == "received" and any(e["kind"] == "received" for e in group))) + + possible = [g for g in company_groups if could_be(g)] + in_thread = [g for g in possible if email.get("thread_id") and any(e.get("thread_id") == email["thread_id"] for e in g)] + if len(in_thread) == 1: + return in_thread[0] + words = _role_words(email["role"]) + alike = [g for g in possible if any(words & _role_words(e["role"]) for e in g)] + return alike[0] if len(alike) == 1 else None + + +def _steps_shown(events: list[dict]) -> list[tuple[str, str]]: + """(status, when) for each step a run of emails shows, oldest first. A confirmation arriving just after a test or a + rejection doesn't undo it; one well after it means they applied again.""" + steps: list[tuple[str, str]] = [] + for e in events: + status = KIND_STATUS.get(e["kind"]) + if not status or (steps and steps[-1][0] == status): + continue + if status == "Applied" and steps and _days_apart(steps[-1][1], e["received_at"]) < _APPLIED_AGAIN_DAYS: + continue + steps.append((status, e["received_at"])) + return steps + + +def _catch_up(user_id: str, app_id: str, events: list[dict]) -> None: + """Emails linked to an application already tracked move it by the usual rules: forward when they're news, otherwise + only into its history.""" + for e in events: + status = KIND_STATUS.get(e["kind"]) + if not status: + continue + target, safe = proposed_change(tracker.get_application(user_id, app_id)["status"], e["kind"]) + if target and safe and e["received_at"] > tracker.latest_status_change(user_id, app_id): + tracker.update_status(app_id, target, user_id=user_id, source="email", at=e["received_at"]) + else: + tracker.add_past_status(user_id, app_id, status, e["received_at"]) + + def track_all(user_id: str, only: set[str] | None = None) -> dict: """Add every application the unmatched emails are about (or just the emails in `only`, by Gmail id): one per - company and role, at the status of its newest email that points to one, with all its emails linked. An email - naming no role takes the company's one role from its other emails; when there's none (or several), it is left - for the user.""" + company and role, at the status its emails show, with all its emails linked. An email naming no role takes the + company's one role from its other emails; when there's none (or several), it is left for the user.""" unmatched = [e for e in tracker.list_email_events(user_id, limit=_MAX_BACKFILL + 1000) if e["decision"] == "unmatched" and (only is None or e["gmail_id"] in only)] + unmatched.sort(key=lambda e: e["received_at"]) roles: dict[str, set[str]] = {} for e in unmatched: if e["role"].strip(): @@ -781,25 +857,33 @@ def track_all(user_id: str, only: set[str] | None = None) -> dict: if not company_key or not role: skipped += 1 continue - groups.setdefault((company_key, _key(role)), []).append({**e, "company": company, "role": role}) + email = {**e, "company": company, "role": role} + same_role = groups.get((company_key, _key(role))) + at_company = [g for (known, _), g in groups.items() if known == company_key] + group = same_role if same_role is not None else _application_for(email, at_company) + if group is None: + group = groups.setdefault((company_key, _key(role)), []) + group.append(email) added = linked = 0 for events in groups.values(): - events.sort(key=lambda e: e["received_at"]) newest = events[-1] - status = next((KIND_STATUS[e["kind"]] for e in reversed(events) if e["kind"] in KIND_STATUS), "Applied") - job = {"job_id": f"email:{newest['gmail_id']}", "title": newest["role"], "company": newest["company"]} + # Its role as the confirmation gave it, when there is one: follow-ups tend to shorten it. + role = next((e["role"] for e in events if e["kind"] == "received"), newest["role"]) + job = {"job_id": f"email:{newest['gmail_id']}", "title": role, "company": newest["company"]} app_id = tracker.add_application(user_id, job, {}, "", {}, status="Applied", source="email", at=events[0]["received_at"]) if app_id: added += 1 - for e in events: # each step its emails show, so its history is the path it took - if e["kind"] in KIND_STATUS: - tracker.update_status(app_id, KIND_STATUS[e["kind"]], user_id=user_id, source="email", at=e["received_at"]) - tracker.update_status(app_id, status, f"Added from your inbox: {newest['subject']}"[:300], user_id=user_id, - source="email", at=newest["received_at"]) - else: # already tracked under that company and role: link to it, and leave its status to the usual rules - app_id = tracker.find_application_id(user_id, newest["company"], newest["role"]) + for status, at in _steps_shown(events): # so its history is the path it took + tracker.update_status(app_id, status, user_id=user_id, source="email", at=at) + tracker.update_status(app_id, tracker.get_application(user_id, app_id)["status"], + f"Added from your inbox: {newest['subject']}"[:300], user_id=user_id, source="email", + at=newest["received_at"]) + else: # already tracked under that company and role: link to it + app_id = tracker.find_application_id(user_id, newest["company"], role) linked += 1 + if app_id: + _catch_up(user_id, app_id, events) for e in events: tracker.update_email_event(user_id, e["id"], decision="confirmed", application_id=app_id, proposed=KIND_STATUS.get(e["kind"]), status_before=None) diff --git a/api/evals/test_email_eval.py b/api/evals/test_email_eval.py index dfe6e06..1c9ce04 100644 --- a/api/evals/test_email_eval.py +++ b/api/evals/test_email_eval.py @@ -1,13 +1,13 @@ """ Does Windrush read job emails right, and leave alone what isn't about an application? -20 emails as graduate applicants get them (applicant tracking systems, company recruiters, agencies, job +23 emails as graduate applicants get them (applicant tracking systems, company recruiters, agencies, job boards, newsletters), each labelled with what it means and which application it is about. Every run writes evals/last_email_run.json. The bar: no email is matched to the wrong application; nothing unrelated (job alerts, newsletters, an agency's pitch) changes an application; every confident reading that would change a status on its own is correct. -How many labelled emails were read correctly is reported, and must be at least 17 of 20. +How many labelled emails were read correctly is reported, and must be at least 20 of 23. """ import asyncio import json @@ -82,6 +82,15 @@ "Software Engineer position.", "interview", "catapult"), ("GitHub ", "[GitHub] A new SSH key was added to your account", "A new public key was added to your account.", "unrelated", None), + ("Catapult ", "Graduate Software Engineer - Catapult", + "Dear Timothy, thank you for your time and interest in working at Catapult. After careful consideration we regret " + "to inform you that we will not be progressing further with your application.", "rejected", "catapult"), + ("Monzo ", "Junior Backend Engineer - Monzo", + "Bonjour Timothy, nous vous remercions de votre intérêt pour le poste de Junior Backend Engineer chez Monzo. Nous " + "avons examiné votre candidature et avons malheureusement décidé de ne pas donner suite.", "rejected", "monzo"), + ("Ocado Technology ", "Ocado Technology Application", + "Dear Timothy, thank you for applying for the position of Graduate Software Developer here at Ocado Technology. We " + "regret to advise you that the position you have applied for has been filled.", "rejected", "ocado"), ] @@ -124,4 +133,4 @@ async def run(): assert not wrong_app, wrong_app assert not unrelated_changed, unrelated_changed assert not wrong_auto, wrong_auto - assert right >= 17 + assert right >= 20 diff --git a/api/tests/test_gmail.py b/api/tests/test_gmail.py index 964af3e..b86f8bc 100644 --- a/api/tests/test_gmail.py +++ b/api/tests/test_gmail.py @@ -618,3 +618,152 @@ def test_an_interview_from_the_employers_own_address_adds_the_application(past, _past("interview", company="Apple", role="Seasonal Specialist")) asyncio.run(_finish(user)) assert _app_named(user, "Apple")["status"] == "Interview" + + +# ── Replies that don't look like application emails ─────────────────────────────────────────── + +def test_a_rejection_from_a_job_alerts_address_isnt_taken_for_an_alert(): + email = _email("noreply@jobalerts.leonardojobs.co.uk", "Leonardo Application") + assert gmail.is_job_alert(email) + assert not gmail.is_job_alert({**email, "text": "Thank you for applying for the position of Graduate Software " + "Engineer. We regret that the position you have applied for has been filled."}) + + +def test_an_alert_that_only_mentions_applying_stays_an_alert(): + email = _email("jobalerts-noreply@linkedin.com", "Graduate Software Engineer at Catapult and 12 more jobs") + assert gmail.is_job_alert({**email, "text": "Apply now with one click. Be an early applicant to these jobs."}) + + +def test_workables_replies_are_searched_for_and_read(past, user): + assert "workablemail.com" in gmail._past_query("6m") + _app(user, "Validus", "Junior Python Software Engineer", "Applied") + past("m1", "Validus ", "Junior Python Software Engineer - Validus", + "We regret to inform you that your application has not been selected.", + _past("rejected", 1, company="Validus", role="Junior Python Software Engineer"), days_ago=-1) + state = asyncio.run(_finish(user)) + assert state["counts"] == {"applied": 1} and _app_named(user, "Validus")["status"] == "Rejected" + + +# ── One application, however its emails word it ─────────────────────────────────────────────── + +def _apps_at(user, company): + return [a for a in tracker.list_applications(user) if a["company"] == company] + + +def test_follow_ups_that_word_the_role_differently_stay_on_one_application(past, user): + past("m1", "LinkedIn ", "Ada, your application was sent to CarbonTrac", "Sent.", + _past("received", company="CarbonTrac", role="Full Stack Engineer - Intern"), days_ago=30) + past("m2", "CarbonTrac ", "Interview - CarbonTrac Intern", "Let's talk.", + _past("interview", company="CarbonTrac", role="Full Stack Engineer Intern"), days_ago=25) + past("m3", "CarbonTrac ", "Follow-up on your application", "We'd like to offer you the role.", + _past("offer", company="CarbonTrac", role="Full Stack Developer Intern"), days_ago=15) + asyncio.run(_finish(user)) + apps = _apps_at(user, "CarbonTrac") + assert len(apps) == 1 and apps[0]["status"] == "Offer" and apps[0]["job_title"] == "Full Stack Engineer - Intern" + assert [s["status"] for s in apps[0]["history"]] == ["Applied", "Interview", "Offer"] + + +def test_a_reply_in_the_same_thread_is_the_same_application_whatever_it_calls_the_role(past, user, google): + past("m1", "Alvan ", "Invitation to Interview", "Please come in.", + _past("interview", company="Alvan", role="Made Smarter"), days_ago=20) + past("m2", "Alvan ", "RE: Invitation to Interview", "See you then.", + _past("interview", company="Alvan", role="Digital placement"), days_ago=18) + google.messages["m2"]["threadId"] = google.messages["m1"]["threadId"] + asyncio.run(_finish(user)) + assert len(_apps_at(user, "Alvan")) == 1 + + +def test_two_confirmations_are_two_applications_however_alike_their_roles(past, user): + past("m1", "LinkedIn ", "Ada, your application was sent to Hunter Bond", "Sent.", + _past("received", company="Hunter Bond", role="Graduate Software Engineer - Python"), days_ago=30) + past("m2", "LinkedIn ", "Ada, your application was sent to Hunter Bond again", "Sent.", + _past("received", company="Hunter Bond", role="Junior Software Engineer - Finance"), days_ago=28) + asyncio.run(_finish(user)) + assert len(_apps_at(user, "Hunter Bond")) == 2 + + +def test_a_follow_up_that_could_be_about_several_applications_isnt_guessed(past, user): + past("m1", "Glencore ", "Your Glencore graduate application", "Received.", + _past("received", company="Glencore", role="Technology Graduate Scheme"), days_ago=40) + past("m2", "Glencore ", "Your Glencore technology application", "Received.", + _past("received", company="Glencore", role="Technology Programme"), days_ago=38) + past("m3", "Glencore ", "Your Glencore application", "Unfortunately not.", + _past("rejected", company="Glencore", role="Technology Graduate Programme"), days_ago=20) + asyncio.run(_finish(user)) + assert {a["job_title"]: a["status"] for a in _apps_at(user, "Glencore")} == { + "Technology Graduate Scheme": "Applied", "Technology Programme": "Applied", + "Technology Graduate Programme": "Rejected"} + + +def test_a_follow_up_long_after_isnt_tied_to_an_old_application(past, user): + past("m1", "Globex ", "Thanks for applying to Globex", "Thanks.", + _past("received", role="Data Engineer"), days_ago=300) + past("m2", "Globex ", "Your application to Globex", "Unfortunately not.", + _past("rejected", role="Senior Data Engineer II"), days_ago=20) + asyncio.run(_finish(user)) + assert {a["job_title"]: a["status"] for a in _apps_at(user, "Globex")} == { + "Data Engineer": "Applied", "Senior Data Engineer II": "Rejected"} + + +def test_an_assessment_name_isnt_taken_for_the_role(past, user): + past("m1", "PwC ", "Confirming your PwC job application", "Thanks.", + _past("received", company="PwC", role="Graduate Technology"), days_ago=20) + past("m2", "PwC ", "Reminder: complete your PwC assessment", "Please complete it.", + _past("assessment", company="PwC", role="Online Assessment"), days_ago=15) + asyncio.run(_finish(user)) + apps = _apps_at(user, "PwC") + assert [(a["job_title"], a["status"]) for a in apps] == [("Graduate Technology", "Responded")] + + +def test_a_confirmation_just_after_a_step_doesnt_undo_it(past, user): + past("m1", "Optiver ", "Optiver - Assessment invitation coming soon", "A test follows.", + _past("assessment", company="Optiver", role="Graduate Software Engineer"), days_ago=20) + past("m2", "Optiver ", "Thank you for applying to Optiver", "Thanks.", + _past("received", company="Optiver", role="Graduate Software Engineer"), days_ago=20) + asyncio.run(_finish(user)) + assert _app_named(user, "Optiver")["status"] == "Responded" + + +def test_a_confirmation_well_after_a_rejection_means_they_applied_again(past, user): + past("m1", "Roku ", "Thank you for applying to Roku", "Thanks.", + _past("received", company="Roku", role="Software Engineer Intern"), days_ago=200) + past("m2", "Roku ", "Your application for an internship with Roku", "Unfortunately not.", + _past("rejected", company="Roku", role="Software Engineer Intern"), days_ago=120) + past("m3", "Roku ", "Thank you for applying again to Roku", "Thanks.", + _past("received", company="Roku", role="Software Engineer Intern"), days_ago=30) + asyncio.run(_finish(user)) + app = _app_named(user, "Roku") + assert app["status"] == "Applied" and [s["status"] for s in app["history"]] == ["Applied", "Rejected", "Applied"] + + +def test_a_boards_confirmation_for_another_role_is_another_application(past, user): + _app(user, "Intellect Group", "Junior AI Engineer", "Applied") + past("m1", "LinkedIn ", "Ada, your application was sent to Intellect Group", "Sent.", + _past("received", 1, company="Intellect Group", role="Junior Data Scientist")) + past("m2", "LinkedIn ", "Ada, your application was sent to Intellect Group Ltd", "Sent.", + _past("received", 1, company="Intellect Group", role="Graduate AI Engineer")) # the same job, by its words + asyncio.run(_finish(user)) + assert sorted(a["job_title"] for a in _apps_at(user, "Intellect Group")) == ["Junior AI Engineer", "Junior Data Scientist"] + + +def test_add_all_moves_an_application_it_links_a_rejection_to(inbox, user, client): + apps, mail = inbox + con = sqlite3.connect(tracker._DB_PATH) # applied last week, turned down an hour ago + con.execute("UPDATE status_history SET changed_at='2020-01-01T00:00:00+00:00' WHERE application_id=?", (apps["catapult"],)) + con.commit() + con.close() + mail("a1", "Catapult ", "Your application to Catapult", "Unfortunately not.", + {**_reading("rejected", None, company="Catapult"), "role": "Graduate Software Engineer"}) + asyncio.run(gmail.sync(user)) + assert client.post("/gmail/events/track-all").json()["linked"] == 1 + assert _status(user, apps["catapult"]) == "Rejected" + + +def test_add_all_doesnt_let_an_old_email_undo_a_newer_status(inbox, user, client): + apps, mail = inbox + tracker.update_status(apps["catapult"], "Interview", user_id=user) + mail("a1", "Catapult ", "Thanks for applying to Catapult", "Received.", + {**_reading("received", None, company="Catapult"), "role": "Graduate Software Engineer"}, minutes_ago=5000) + asyncio.run(gmail.sync(user)) + client.post("/gmail/events/track-all") + assert _status(user, apps["catapult"]) == "Interview"