feat: read older job emails to fill in past applications, and never take a job alert for one

api/app/main.py

@@ -1153,6 +1153,38 @@ async def gmail_sync(current_user: Annotated[auth.User, Depends(auth.get_current
     return {"started": True}
 
 
+def _backfill_period(period: str, user_id: str) -> str:
+    if period not in gmail.BACKFILL_DAYS:
+        raise HTTPException(status_code=400, detail="Choose 6 months, 1 year or 2 years")
+    account = tracker.get_gmail_account(user_id)
+    if not account:
+        raise HTTPException(status_code=400, detail="Gmail isn't connected")
+    if account.get("problem"):
+        raise HTTPException(status_code=400, detail=account["problem"])
+    return period
+
+
+@app.get("/gmail/backfill/count")
+async def gmail_backfill_count(period: str, current_user: Annotated[auth.User, Depends(auth.get_current_user)]):
+    """How many job emails that period holds and how many are still unread, before anything is read."""
+    try:
+        return await gmail.backfill_count(current_user.id, _backfill_period(period, current_user.id))
+    except gmail.GmailError as exc:
+        raise HTTPException(status_code=400, detail=str(exc))
+
+
+@app.post("/gmail/backfill", status_code=202)
+async def gmail_backfill_start(body: dict, current_user: Annotated[auth.User, Depends(auth.get_current_user)]):
+    if not gmail.start_backfill(current_user.id, _backfill_period(body.get("period", ""), current_user.id)):
+        raise HTTPException(status_code=409, detail="Older mail is already being read")
+    return {"started": True}
+
+
+@app.delete("/gmail/backfill")
+async def gmail_backfill_stop(current_user: Annotated[auth.User, Depends(auth.get_current_user)]):
+    return {"stopping": gmail.stop_backfill(current_user.id)}
+
+
 @app.delete("/gmail")
 async def gmail_disconnect(current_user: Annotated[auth.User, Depends(auth.get_current_user)]):
     """Revoke Windrush's Gmail access and forget the account."""

api/app/tracker.py

@@ -608,6 +608,31 @@ def update_status(app_id: str, status: str, notes: str | None = None, *, user_id
         return False
 
 
+def latest_status_change(user_id: str, app_id: str) -> str:
+    """When the application last changed status ("" when nothing is logged)."""
+    con = sqlite3.connect(_DB_PATH)
+    row = con.execute("SELECT max(changed_at) FROM status_history WHERE application_id=? AND user_id=?",
+                      (app_id, user_id)).fetchone()
+    con.close()
+    return row[0] or ""
+
+
+def add_past_status(user_id: str, app_id: str, status: str, at: str) -> None:
+    """A step an old email shows the application took, for its history: the current status stays as it is. An email
+    older than the applied date moves that date back to the day it was really applied."""
+    con = sqlite3.connect(_DB_PATH)
+    try:
+        if not con.execute("SELECT 1 FROM status_history WHERE application_id=? AND user_id=? AND status=?",
+                           (app_id, user_id, status)).fetchone():
+            _log_status(con, user_id, app_id, status, "email", at)
+        if status == "Applied":
+            con.execute("UPDATE applications SET date_applied=? WHERE id=? AND user_id=? AND "
+                        "(date_applied IS NULL OR date_applied > ?)", (at[:10], app_id, user_id, at[:10]))
+        con.commit()
+    finally:
+        con.close()
+
+
 def set_job_mark(user_id: str, job_id: str, sponsorship: str | None) -> None:
     """Record the user's call on whether a job sponsors visas (one of JOB_MARKS), or clear it (None)."""
     if sponsorship is not None and sponsorship not in JOB_MARKS:

api/evals/test_email_eval.py

@@ -1,13 +1,13 @@
 """
 Does Windrush read job emails right, and leave alone what isn't about an application?
 
-16 emails as graduate applicants get them (applicant tracking systems, company recruiters, agencies, job
+20 emails as graduate applicants get them (applicant tracking systems, company recruiters, agencies, job
 boards, newsletters), each labelled with what it means and which application it is about. Every run writes
 evals/last_email_run.json.
 
 The bar: no email is matched to the wrong application; nothing unrelated (job alerts, newsletters, an agency's
 pitch) changes an application; every confident reading that would change a status on its own is correct.
-How many labelled emails were read correctly is reported, and must be at least 13 of 16.
+How many labelled emails were read correctly is reported, and must be at least 17 of 20.
 """
 import asyncio
 import json
@@ -57,6 +57,14 @@
     ("LinkedIn Job Alerts <jobalerts-noreply@linkedin.com>", "Graduate Software Engineer at Catapult and 12 more jobs",
      "Jobs you may be interested in: Graduate Software Engineer - Catapult - London. Junior Developer - Monzo.",
      "unrelated", None),
+    ("LinkedIn <jobs-listing-alerts@linkedin.com>", "Graduate Application Engineer at Monzo",
+     "This job fits your profile. Monzo is hiring a Graduate Application Engineer in London. Apply now.", "unrelated", None),
+    ("Ocado Careers <talent@ocadogroup.com>", "We think you'd be great for this role at Ocado",
+     "Hi Timothy, based on your profile we think this Graduate Developer role suits you. Apply by Friday.", "unrelated", None),
+    ("Gradcracker <noreply@gradcracker.com>", "3 application deadlines this week",
+     "These graduate schemes close soon: Monzo, Ocado and Catapult.", "unrelated", None),
+    ("LinkedIn <jobs-noreply@linkedin.com>", "Timothy, your application was sent to Globex Corporation",
+     "Your application was sent to Globex Corporation. Software Engineer, London.", "received", None),
     ("Indeed <alert@indeed.com>", "New jobs for Graduate Software Engineer in Bristol",
      "Ocado Technology is hiring a Graduate Software Developer. Apply now with your Indeed CV.", "unrelated", None),
     ("Baringa <news@baringa.com>", "Baringa Insights: The future of energy markets",
@@ -116,4 +124,4 @@ async def run():
     assert not wrong_app, wrong_app
     assert not unrelated_changed, unrelated_changed
     assert not wrong_auto, wrong_auto
-    assert right >= 13
+    assert right >= 17

api/tests/test_gmail.py

@@ -411,3 +411,210 @@ def test_add_all_makes_one_application_per_company_and_role_at_its_latest_status
     assert ("Hooli", "") not in tracked and _status(user, apps["catapult"]) == "Applied"
     left = [e for e in client.get("/gmail/events").json() if e["decision"] == "unmatched"]
     assert [e["gmail_id"] for e in left] == ["a5"]
+
+
+# ── Job alerts, and reading older mail ────────────────────────────────────────────────────────
+
+def _email(sender: str, subject: str) -> dict:
+    return {"sender": sender, "sender_address": sender, "subject": subject, "text": "", "snippet": ""}
+
+
+@pytest.mark.parametrize("sender, subject", [
+    ("jobalerts-noreply@linkedin.com", "Graduate Software Engineer at Catapult and 12 more jobs"),
+    ("jobs-noreply@linkedin.com", "Junior Application Software Engineer at One Big Circle Ltd: up to £45K/year"),
+    ("noreply@jobalert.indeed.com", "Application Support Analyst at Nisbets. 12 more jobs in BS1 1AA"),
+    ("noreply@jobalert.indeed.com", "BS1 1AA: Full Stack Application Developer at Savills and 9 more new jobs"),
+    ("noreply@match.indeed.com", "Junior Managed Application Services Analyst @ Evotix Limited"),
+    ("alert@indeed.com", "Interview-ready roles near you"),
+    ("noreply@gradcracker.com", "3 application deadlines this week"),
+    ("careers@jobalerts-uk.pwc.com", "New opportunities at PwC"),
+    ("hello@board.example", "This job fits you: Application Engineer"),
+    ("hello@board.example", "Graduate roles recommended for you"),
+])
+def test_job_alerts_arent_read_as_applications(sender, subject):
+    assert gmail.is_job_alert(_email(sender, subject))
+
+
+@pytest.mark.parametrize("sender, subject", [
+    ("jobs-noreply@linkedin.com", "Timothy Oluwatobi, your application was sent to Clearwater"),
+    ("jobs-noreply@linkedin.com", "Your application to Full Stack Engineer (AI systems) at Digital Waffle"),
+    ("jobs-noreply@linkedin.com", "Your application was viewed by Neurochase"),
+    ("indeedapply@indeed.com", "Indeed Application: Junior Marketing Assistant"),
+    ("noreply@indeed.com", "An update on your application from No Code Sage"),
+    ("no-reply@greenhouse.io", "Thank you for applying to Catapult"),
+    ("noreply@jobalert.example", "Your Baringa application"),
+    ("careers@hirevue.com", "Your interview with Optiver"),
+])
+def test_real_application_emails_arent_taken_for_alerts(sender, subject):
+    assert not gmail.is_job_alert(_email(sender, subject))
+
+
+def test_a_sync_never_sends_a_job_alert_to_the_model(inbox, user, google, readings):
+    apps, mail = inbox
+    mail("m1", "LinkedIn <jobs-noreply@linkedin.com>", "Application Engineer at Catapult: up to £45K/year", "See the job.",
+         _reading("interview", 1))                       # were it read, this would move Catapult on
+    asyncio.run(gmail.sync(user))
+    assert _status(user, apps["catapult"]) == "Applied"
+
+
+async def _finish(user):
+    gmail.start_backfill(user, "6m")
+    await gmail._backfills[user]["task"]
+    return gmail.backfill_status(user)
+
+
+@pytest.fixture
+def past(google, readings, client, user, monkeypatch):
+    monkeypatch.setattr(gmail, "start_sync", lambda uid: True)
+    gmail._backfills.clear()
+    _connect(client)
+
+    def mail(mid, sender, subject, body, reading=None, days_ago=30):
+        google.messages[mid] = message(mid, sender, subject, body, days_ago * 24 * 60)
+        readings[subject] = reading
+    return mail
+
+
+def _past(kind, n=None, company="Globex", role="Graduate Engineer", confidence="high"):
+    return {"kind": kind, "application": n, "confidence": confidence, "company": company, "role": role, "summary": ""}
+
+
+def _app_named(user, company):
+    return next((a for a in tracker.list_applications(user) if a["company"] == company), None)
+
+
+def test_older_mail_adds_applications_it_shows_were_made_with_the_path_they_took(past, user):
+    past("m1", "Globex <no-reply@greenhouse.io>", "Thanks for applying to Globex", "Thank you for applying.",
+         _past("received"), days_ago=60)
+    past("m2", "Globex <talent@globex.com>", "Interview at Globex", "We'd like to interview you.",
+         _past("interview"), days_ago=45)
+    past("m3", "Globex <talent@globex.com>", "Your application to Globex", "Unfortunately not.",
+         _past("rejected"), days_ago=30)
+    state = asyncio.run(_finish(user))
+
+    app = _app_named(user, "Globex")
+    assert state["added"]["added"] == 1 and state["read"] == 3 and not state["error"]
+    assert app["status"] == "Rejected" and app["job_id"].startswith("email:")
+    assert [(s["status"], s["source"]) for s in app["history"]] == [
+        ("Applied", "email"), ("Interview", "email"), ("Rejected", "email")]
+    assert app["date_applied"] == app["history"][0]["changed_at"][:10]
+    assert app["history"][0]["changed_at"] < app["history"][1]["changed_at"] < app["history"][2]["changed_at"]
+
+
+def test_a_recruiters_approach_doesnt_add_an_application(past, user, client):
+    past("m1", "Sarah <sarah@techrecruit.co.uk>", "Interview this week?", "Could I arrange an interview?",
+         _past("interview", company="Fintech Co"))
+    state = asyncio.run(_finish(user))
+    assert _app_named(user, "Fintech Co") is None and state["counts"] == {"dismissed": 1}
+    assert [e["decision"] for e in client.get("/gmail/events").json()] == ["dismissed"]
+
+
+def test_an_unsure_reading_doesnt_add_an_application(past, user):
+    past("m1", "Globex <no-reply@greenhouse.io>", "Your application", "Thanks.", _past("received", confidence="medium"))
+    asyncio.run(_finish(user))
+    assert _app_named(user, "Globex") is None
+
+
+def test_older_mail_skips_job_alerts_without_the_model(past, user):
+    past("m1", "LinkedIn <jobalerts-noreply@linkedin.com>", "Application Engineer at Globex and 5 more jobs", "Jobs.",
+         _past("received"))
+    state = asyncio.run(_finish(user))
+    assert state["counts"] == {"ignored": 1} and _app_named(user, "Globex") is None
+
+
+def test_an_old_email_fills_in_history_without_moving_the_application_back(past, user):
+    app_id = _app(user, "Catapult", "Graduate Software Engineer", "Rejected")
+    past("m1", "Catapult <jobs@catapult.com>", "Interview at Catapult", "Catapult would like to interview you.",
+         _past("interview", 1, company="Catapult"))
+    state = asyncio.run(_finish(user))
+    app = _app_named(user, "Catapult")
+    assert app["status"] == "Rejected" and state["counts"] == {"noted": 1}
+    assert "Interview" in [s["status"] for s in app["history"]]
+    assert app["id"] == app_id
+
+
+def test_an_earlier_confirmation_moves_the_applied_date_back(past, user):
+    _app(user, "Catapult", "Graduate Software Engineer", "Applied")
+    past("m1", "Catapult <jobs@catapult.com>", "Thanks for applying to Catapult", "Catapult received it.",
+         _past("received", 1, company="Catapult"), days_ago=90)
+    asyncio.run(_finish(user))
+    app = _app_named(user, "Catapult")
+    assert app["date_applied"] < time.strftime("%Y-%m-%d") and app["status"] == "Applied"
+
+
+def test_a_run_carries_on_where_the_last_stopped(past, user, client):
+    past("m1", "Globex <no-reply@greenhouse.io>", "Thanks for applying to Globex", "Thanks.", _past("received"))
+    asyncio.run(_finish(user))
+    past("m2", "Initech <no-reply@greenhouse.io>", "Thanks for applying to Initech", "Thanks.",
+         _past("received", company="Initech"))
+    assert client.get("/gmail/backfill/count?period=6m").json() == {"period": "6m", "total": 2, "to_read": 1}
+    assert asyncio.run(_finish(user))["read"] == 1
+
+
+def test_stopping_keeps_what_it_found(past, user, google):
+    past("m1", "Globex <no-reply@greenhouse.io>", "Thanks for applying to Globex", "Thanks.", _past("received"), days_ago=40)
+    past("m2", "Initech <no-reply@greenhouse.io>", "Thanks for applying to Initech", "Thanks.",
+         _past("received", company="Initech"), days_ago=20)
+    google.messages = dict(reversed(list(google.messages.items())))     # Gmail lists the newest first
+    real_read = gmail.read
+
+    async def stop_after_first(email, applications):
+        gmail.stop_backfill(user)
+        return await real_read(email, applications)
+    gmail.read = stop_after_first
+    try:
+        state = asyncio.run(_finish(user))
+    finally:
+        gmail.read = real_read
+    assert state["read"] == 1 and _app_named(user, "Globex") and not _app_named(user, "Initech")
+
+
+def test_a_model_that_cant_be_reached_stops_the_run_and_leaves_the_email_unread(past, user):
+    past("m1", "Globex <no-reply@greenhouse.io>", "Thanks for applying to Globex", "Thanks.", None)
+    state = asyncio.run(_finish(user))
+    assert "model" in state["error"] and not tracker.email_seen(user, "m1")
+
+
+def test_gmail_saying_slow_down_makes_it_wait_and_carry_on(past, user, google, monkeypatch):
+    monkeypatch.setattr(gmail, "_QUOTA_WAIT_S", 0)
+    past("m1", "Globex <no-reply@greenhouse.io>", "Thanks for applying to Globex", "Thanks.", _past("received"))
+    google.slow_down.add("/messages/m1")
+    assert asyncio.run(_finish(user))["read"] == 1 and _app_named(user, "Globex")
+
+
+def test_the_backfill_only_takes_the_periods_it_offers(past, client):
+    assert client.post("/gmail/backfill", json={"period": "10y"}).status_code == 400
+    assert client.get("/gmail/backfill/count?period=10y").status_code == 400
+
+
+def test_a_job_board_isnt_taken_for_the_employer(past, user):
+    past("m1", "Indeed Apply <indeedapply@indeed.com>", "Indeed Application: Cafe Bar Assistant", "Your application was sent.",
+         _past("received", company="Indeed", role="Cafe Bar Assistant"))
+    past("m2", "Indeed Apply <indeedapply@indeed.com>", "Indeed Application: Front of House", "Your application was sent.",
+         _past("received", company="Unknown", role="Front of House", confidence="medium"))
+    asyncio.run(_finish(user))
+    apps = {a["job_title"]: a["company"] for a in tracker.list_applications(user)}
+    assert apps == {"Cafe Bar Assistant": "Employer not named (via Indeed)", "Front of House": "Employer not named (via Indeed)"}
+
+
+def test_an_application_whose_emails_name_no_role_is_still_added(past, user):
+    past("m1", "Fractile <no-reply@greenhouse.io>", "Thank you for applying to Fractile", "Thanks.",
+         _past("received", company="Fractile", role="unspecified position"))
+    state = asyncio.run(_finish(user))
+    assert _app_named(user, "Fractile")["job_title"] == "Role not stated" and state["added"]["left"] == 0
+
+
+def test_a_role_from_another_email_fills_in_one_that_names_none(past, user):
+    past("m2", "Globex <talent@globex.com>", "Regarding your application", "Unfortunately not.",
+         _past("rejected", role=""), days_ago=20)
+    past("m1", "Globex <no-reply@greenhouse.io>", "Thanks for applying to Globex", "Thanks.", _past("received"), days_ago=40)
+    asyncio.run(_finish(user))
+    app = _app_named(user, "Globex")
+    assert app["job_title"] == "Graduate Engineer" and app["status"] == "Rejected"
+
+
+def test_an_interview_from_the_employers_own_address_adds_the_application(past, user):
+    past("m1", "Apple <no-reply@apple.com>", "Invitation to interview", "We'd like to interview you.",
+         _past("interview", company="Apple", role="Seasonal Specialist"))
+    asyncio.run(_finish(user))
+    assert _app_named(user, "Apple")["status"] == "Interview"