diff --git a/api/app/gmail.py b/api/app/gmail.py index 4303b76..f5a561c 100644 --- a/api/app/gmail.py +++ b/api/app/gmail.py @@ -45,6 +45,10 @@ _BODY_CHARS = 2500 # of each email, for the model _MAX_APPLICATIONS = 80 # most recent, offered to the model to match _EVERY_S = 30 * 60 +BACKFILL_DAYS = {"6m": 182, "1y": 365, "2y": 730} +_MAX_BACKFILL = 6000 # message ids listed for a backfill +_QUOTA_WAIT_S = 60 # how long a backfill waits when Gmail says it's going too fast +_QUOTA_TRIES = 5 # Applicant tracking systems' mail domains: mail from these is about an application, whoever it names. ATS_SENDERS = ("greenhouse.io", "greenhouse-mail.io", "lever.co", "ashbyhq.com", "workable.com", "myworkday.com", @@ -260,9 +264,13 @@ def names_company(email: dict, company: str) -> bool: words = " ".join(re.findall(r"[a-z0-9]+", f"{email['sender']} {email['subject']} {email['text']}".lower())) if re.search(rf"(? bool: + """The sender's own domain: catapult.com, catapultsports.com, catapultcareers.co.uk all name Catapult; + cambridgeconsultants.com doesn't name a company called Cambridge.""" + name = _squash(normalise_company_name(company or "")) labels = email["sender_address"].split("@")[-1].split(".") return len(name) >= 3 and any(label == name or (label.startswith(name) and label[len(name):] in _DOMAIN_SUFFIXES) for label in labels) @@ -280,6 +288,26 @@ def relevant(email: dict, applications: list[dict]) -> bool: or any(names_company(email, a["company"]) for a in applications)) +# Job alerts and other mail about jobs they haven't applied to. Application-style subjects catch these too: a LinkedIn +# alert for a "Junior Application Software Engineer", or "3 application deadlines this week". +_SAYS_THEY_APPLIED = re.compile(r"\byour (?:\w+ )?(?:application|candidacy|interview)\b|" + r"\bthank(?:s| you) for (?:applying|your application|your interest)\b|\byou applied\b", re.I) +_ALERT_SENDER = re.compile(r"job[-_.]?alert|jobs-listings@|jobs-noreply@linkedin|@match\.indeed\.|alert@indeed\.", re.I) +_ALERT_SUBJECT = re.compile( + r"\bmore (?:new )?jobs\b|\bnew jobs?\b|\bjobs? for you\b|\bjob alert|\bjobs? (?:you|that) (?:may|might|could)\b|" + r"\bjobs? match|\b(?:job|role)s? (?:recommend|picked)|\brecommended (?:jobs?|roles?|for you)\b|" + r"\b(?:job|role|position) (?:fits?|suits?|matches)\b|\b(?:good|great|strong|perfect) (?:fit|match) for you\b|" + r"\bfits? you\b|\bapplication deadlines?\b|:\s*up to [£$€]", re.I) + + +def is_job_alert(email: dict) -> bool: + """A job alert or recommendation, by its sender or subject: not about an application they made, so not read at all. + LinkedIn sends both alerts and "your application was sent" from one address, so what the subject says comes first.""" + if _SAYS_THEY_APPLIED.search(email["subject"]): + return False + return bool(_ALERT_SENDER.search(email["sender_address"]) or _ALERT_SUBJECT.search(email["subject"])) + + _CLASSIFY_SYSTEM = ( "You read one email a job seeker received and say what it means for their job applications. You never guess: " "you go by what the email plainly says.\n" @@ -292,6 +320,12 @@ def relevant(email: dict, applications: list[dict]) -> bool: "- rejected: it says they won't be progressed, the role is filled or closed, or they were unsuccessful;\n" "- other: it's about one of their applications but none of the above (still under review, a question for them);\n" "- unrelated: anything else, including job alerts, job recommendations, newsletters and marketing.\n" + "It only counts when it's about an application they actually made. A job they're told fits them, might like or " + "should apply to is unrelated, even when its title has the word Application in it or it names a company they " + "applied to. So are invitations to apply, deadline reminders, careers advice, an employer's talent-network " + "sign-up, and a recruitment agency's first approach about a role they haven't applied for, even one offering an " + "interview. An employer's own invitation to its interview or test does count: it means they applied.\n" + "company is the employer, never the job board that sent the email; leave company or role empty if it doesn't say.\n" "application is the number of the application it is about, from the list given, only when the email names " "that company (and, when they applied to several roles there, the role); otherwise null.\n" "confidence is high only when the email states its meaning outright and names the company; medium or low " @@ -348,7 +382,7 @@ def _read_without_model(email: dict, applications: list[dict]) -> dict: named = [a for a in applications if names_company(email, a["company"])] app = named[0] if len({a["company"].lower() for a in named}) == 1 and len(named) == 1 else None return {"kind": kind, "application": app, "confidence": "low", "company": (app or {}).get("company", ""), - "role": "", "summary": email["subject"][:300]} + "role": "", "summary": email["subject"][:300], "by_model": False} def proposed_change(current: str, kind: str) -> tuple[str | None, bool]: @@ -381,11 +415,11 @@ def _first_sync_start() -> int: return int(time.time()) - _FIRST_SYNC_DAYS * 86400 -async def _message_ids(token: str, query: str) -> list[str]: +async def _message_ids(token: str, query: str, limit: int = _MAX_LISTED) -> list[str]: """Ids of the messages matching `query`, newest first (Gmail's order).""" ids, page = [], None - while len(ids) < _MAX_LISTED: - params = {"q": query, "maxResults": min(100, _MAX_LISTED - len(ids))} + while len(ids) < limit: + params = {"q": query, "maxResults": min(100, limit - len(ids))} if page: params["pageToken"] = page listing = await _get(token, "/messages", params) @@ -462,7 +496,7 @@ async def sync(user_id: str) -> dict: await asyncio.sleep(_FETCH_GAP_S) more = len(unseen) > fetched for email in sorted(emails, key=lambda e: e["received_at"]): # oldest first: later news wins - if not relevant(email, applications): + if is_job_alert(email) or not relevant(email, applications): decision = "ignored" tracker.add_email_event(user_id, {"gmail_id": email["gmail_id"], "received_at": email["received_at"], "sender": "", "subject": "", "kind": "unrelated", "decision": "ignored"}) @@ -500,7 +534,183 @@ def status(user_id: str) -> dict: account = tracker.get_gmail_account(user_id) return {"configured": configured(), "connected": bool(account), "email": (account or {}).get("email"), "problem": (account or {}).get("problem"), "last_sync_at": (account or {}).get("last_sync_at"), - "syncing": user_id in _syncing, "last": _last.get(user_id), "redirect_uri": redirect_uri()} + "syncing": user_id in _syncing, "last": _last.get(user_id), "redirect_uri": redirect_uri(), + "backfill": backfill_status(user_id)} + + +# ── Reading older mail ──────────────────────────────────────────────────────────────────────── + +_backfills: dict[str, dict] = {} + + +def _past_query(period: str) -> str: + """Tracked company names aren't searched for here: over months of mail they mostly find mail that isn't about jobs.""" + return _query(int(time.time()) - BACKFILL_DAYS[period] * 86400, []) + + +async def _unread_past_ids(user_id: str, period: str) -> tuple[int, list[str]]: + """(how many emails match, the ids of those not read yet, oldest first).""" + ids = await _message_ids(await _access_token(user_id), _past_query(period), limit=_MAX_BACKFILL) + return len(ids), [i for i in reversed(ids) if not tracker.email_seen(user_id, i)] + + +async def backfill_count(user_id: str, period: str) -> dict: + total, unread = await _unread_past_ids(user_id, period) + return {"period": period, "total": total, "to_read": len(unread)} + + +_JOB_BOARDS = ("indeed", "linkedin", "glassdoor", "reed", "totaljobs", "cv-library", "cwjobs", "adzuna", "monster") +_NO_NAME = re.compile(r"^(?:unknown|un?specified.*|not (?:specified|stated|provided|mentioned|named|given).*|n/?a|none|" + r"the (?:company|employer|role|position))?$", re.I) +_NO_ROLE = "Role not stated" + + +def _with_real_names(email: dict, reading: dict) -> dict: + """The model sometimes fills a blank with "Unknown", or takes the job board for the employer. A board's own + confirmation that names only the role is still an application: it goes under a name saying the employer wasn't given.""" + company, role = reading["company"].strip(), reading["role"].strip() + company = "" if _NO_NAME.match(company) or _key(company) in _JOB_BOARDS else company + role = "" if _NO_NAME.match(role) else role + board = next((b for b in _JOB_BOARDS if b in email["sender_address"].split("@")[-1]), None) + if (not company and role and board and reading["kind"] == "received" + and re.search(r"\bapplication\b", email["subject"], re.I)): + return {**reading, "company": f"Employer not named (via {board.title()})", "role": role, "confidence": "high"} + return {**reading, "company": company, "role": role} + + +def _shows_an_application(email: dict, reading: dict, companies: set[str]) -> bool: + """Whether an email about an untracked application is enough to add it unasked. Saying it was received or turned + down means they applied. An interview, test or offer could be a recruiter's approach, so it also has to come from + an applicant tracking system, the employer's own address, or a company another of these emails already showed they + applied to.""" + company = _key(normalise_company_name(reading["company"])) + if reading["confidence"] != "high" or not company: + return False + return (reading["kind"] in ("received", "rejected") + or (reading["kind"] in KIND_STATUS + and (from_ats(email) or _from_company(email, reading["company"]) or company in companies))) + + +def _note_past_email(user_id: str, email: dict, reading: dict, companies: set[str]) -> str: + """Record an older email without asking anything of the user: it goes into its application's history, moving the + status only when it's a safe step newer than anything known. Returns the decision.""" + if reading["application"] is None: + reading = _with_real_names(email, reading) + event = {**{k: email[k] for k in ("gmail_id", "thread_id", "received_at", "sender", "subject")}, + "snippet": email["snippet"][:300], "kind": reading["kind"], "company": reading["company"], + "role": reading["role"], "summary": reading["summary"]} + target_status = KIND_STATUS.get(reading["kind"]) + app = reading["application"] + if app is None: + if _shows_an_application(email, reading, companies): + companies.add(_key(normalise_company_name(reading["company"]))) + tracker.add_email_event(user_id, {**event, "decision": "unmatched", "proposed": target_status}) + return "unmatched" + tracker.add_email_event(user_id, {**event, "decision": "dismissed", "proposed": target_status}) + return "dismissed" + current = (tracker.get_application(user_id, app["id"]) or app)["status"] + target, safe = proposed_change(current, reading["kind"]) + if target and safe and reading["confidence"] == "high" and email["received_at"] > tracker.latest_status_change(user_id, app["id"]): + tracker.update_status(app["id"], target, user_id=user_id, source="email", at=email["received_at"]) + tracker.add_email_event(user_id, {**event, "application_id": app["id"], "proposed": target, "decision": "applied", + "status_before": current}) + return "applied" + if target_status and reading["confidence"] == "high": + tracker.add_past_status(user_id, app["id"], target_status, email["received_at"]) + tracker.add_email_event(user_id, {**event, "application_id": app["id"], "decision": "noted"}) + return "noted" + + +async def _fetch_past(user_id: str, message_id: str) -> dict: + for attempt in range(_QUOTA_TRIES): + try: + return parse(await _get(await _access_token(user_id), f"/messages/{message_id}", {"format": "full"})) + except _SlowDown: + logger.info("gmail backfill: hit the per-user quota; waiting") + if attempt < _QUOTA_TRIES - 1: + await asyncio.sleep(_QUOTA_WAIT_S) + raise GmailError("Gmail kept saying it was too busy. Start it again later: it carries on where it stopped.") + + +async def _read_past(user_id: str, state: dict) -> None: + _, ids = await _unread_past_ids(user_id, state["period"]) + state["to_read"] = len(ids) + companies: set[str] = set() + to_add: set[str] = set() + applications = tracker.list_applications(user_id) + try: + for message_id in ids: + if state["stop"]: + break + try: + email = await _fetch_past(user_id, message_id) + except GmailError: + raise + except Exception as exc: # gone since it was listed, say: skip it rather than stop + logger.warning("gmail backfill: couldn't read a message (%s)", type(exc).__name__) + state["read"] += 1 + continue + await asyncio.sleep(_FETCH_GAP_S) + named = [a for a in applications if names_company(email, a["company"])] + if is_job_alert(email) or not relevant(email, named): + decision = "ignored" + tracker.add_email_event(user_id, {"gmail_id": email["gmail_id"], "received_at": email["received_at"], + "sender": "", "subject": "", "kind": "unrelated", "decision": "ignored"}) + else: + reading = await read(email, named) + if not reading.get("by_model", True): + # Left unread rather than recorded as read on a keyword guess, so the next run gets it. + raise GmailError("The model couldn't be reached. Start it again later: it carries on where it stopped.") + if reading["kind"] == "unrelated": + decision = _apply(user_id, email, reading) + else: + decision = _note_past_email(user_id, email, reading, companies) + if decision == "applied": + applications = tracker.list_applications(user_id) + if decision == "unmatched": + to_add.add(email["gmail_id"]) + state["read"] += 1 + state["counts"][decision] = state["counts"].get(decision, 0) + 1 + finally: + # Also when stopped part way: what it has found so far goes into the tracker, in one go so that each + # application's emails become its history. + if to_add: + state["added"] = track_all(user_id, only=to_add) + + +def start_backfill(user_id: str, period: str) -> bool: + if (_backfills.get(user_id) or {}).get("running"): + return False + state = {"period": period, "running": True, "stop": False, "to_read": None, "read": 0, "counts": {}, "added": None, + "error": None, "finished_at": None} + _backfills[user_id] = state + + async def go(): + try: + await _read_past(user_id, state) + except GmailError as exc: + state["error"] = str(exc) + except Exception: + logger.exception("gmail backfill failed") + state["error"] = "Reading older mail failed. Start it again: it carries on where it stopped." + finally: + state["running"] = False + state["finished_at"] = datetime.now(timezone.utc).isoformat() + state["task"] = asyncio.create_task(go()) + return True + + +def stop_backfill(user_id: str) -> bool: + state = _backfills.get(user_id) + if not state or not state["running"]: + return False + state["stop"] = True + return True + + +def backfill_status(user_id: str) -> dict | None: + state = _backfills.get(user_id) + return {k: v for k, v in state.items() if k != "task"} if state else None # ── What the user does with an email ────────────────────────────────────────────────────────── @@ -549,11 +759,13 @@ def _key(text: str) -> str: return " ".join(re.findall(r"[a-z0-9]+", str(text or "").lower())) -def track_all(user_id: str) -> dict: - """Add every application the unmatched emails are about: one per company and role, at the status of its - newest email that points to one, with all its emails linked. An email naming no role takes the company's - one role from its other emails; when there's none (or several), it is left for the user.""" - unmatched = [e for e in tracker.list_email_events(user_id) if e["decision"] == "unmatched"] +def track_all(user_id: str, only: set[str] | None = None) -> dict: + """Add every application the unmatched emails are about (or just the emails in `only`, by Gmail id): one per + company and role, at the status of its newest email that points to one, with all its emails linked. An email + naming no role takes the company's one role from its other emails; when there's none (or several), it is left + for the user.""" + unmatched = [e for e in tracker.list_email_events(user_id, limit=_MAX_BACKFILL + 1000) + if e["decision"] == "unmatched" and (only is None or e["gmail_id"] in only)] roles: dict[str, set[str]] = {} for e in unmatched: if e["role"].strip(): @@ -564,6 +776,8 @@ def track_all(user_id: str) -> dict: company = " ".join(e["company"].split()) company_key = _key(normalise_company_name(company)) role = e["role"].strip() or (next(iter(roles[company_key])) if len(roles.get(company_key, ())) == 1 else "") + if not role and only is not None and company_key not in roles: + role = _NO_ROLE # a run adds what it shows was applied to; asking about each is what it's there to avoid if not company_key or not role: skipped += 1 continue diff --git a/api/app/main.py b/api/app/main.py index 2123957..ffdd76c 100644 --- a/api/app/main.py +++ b/api/app/main.py @@ -1153,6 +1153,38 @@ async def gmail_sync(current_user: Annotated[auth.User, Depends(auth.get_current return {"started": True} +def _backfill_period(period: str, user_id: str) -> str: + if period not in gmail.BACKFILL_DAYS: + raise HTTPException(status_code=400, detail="Choose 6 months, 1 year or 2 years") + account = tracker.get_gmail_account(user_id) + if not account: + raise HTTPException(status_code=400, detail="Gmail isn't connected") + if account.get("problem"): + raise HTTPException(status_code=400, detail=account["problem"]) + return period + + +@app.get("/gmail/backfill/count") +async def gmail_backfill_count(period: str, current_user: Annotated[auth.User, Depends(auth.get_current_user)]): + """How many job emails that period holds and how many are still unread, before anything is read.""" + try: + return await gmail.backfill_count(current_user.id, _backfill_period(period, current_user.id)) + except gmail.GmailError as exc: + raise HTTPException(status_code=400, detail=str(exc)) + + +@app.post("/gmail/backfill", status_code=202) +async def gmail_backfill_start(body: dict, current_user: Annotated[auth.User, Depends(auth.get_current_user)]): + if not gmail.start_backfill(current_user.id, _backfill_period(body.get("period", ""), current_user.id)): + raise HTTPException(status_code=409, detail="Older mail is already being read") + return {"started": True} + + +@app.delete("/gmail/backfill") +async def gmail_backfill_stop(current_user: Annotated[auth.User, Depends(auth.get_current_user)]): + return {"stopping": gmail.stop_backfill(current_user.id)} + + @app.delete("/gmail") async def gmail_disconnect(current_user: Annotated[auth.User, Depends(auth.get_current_user)]): """Revoke Windrush's Gmail access and forget the account.""" diff --git a/api/app/tracker.py b/api/app/tracker.py index c02122e..6144ae2 100644 --- a/api/app/tracker.py +++ b/api/app/tracker.py @@ -608,6 +608,31 @@ def update_status(app_id: str, status: str, notes: str | None = None, *, user_id return False +def latest_status_change(user_id: str, app_id: str) -> str: + """When the application last changed status ("" when nothing is logged).""" + con = sqlite3.connect(_DB_PATH) + row = con.execute("SELECT max(changed_at) FROM status_history WHERE application_id=? AND user_id=?", + (app_id, user_id)).fetchone() + con.close() + return row[0] or "" + + +def add_past_status(user_id: str, app_id: str, status: str, at: str) -> None: + """A step an old email shows the application took, for its history: the current status stays as it is. An email + older than the applied date moves that date back to the day it was really applied.""" + con = sqlite3.connect(_DB_PATH) + try: + if not con.execute("SELECT 1 FROM status_history WHERE application_id=? AND user_id=? AND status=?", + (app_id, user_id, status)).fetchone(): + _log_status(con, user_id, app_id, status, "email", at) + if status == "Applied": + con.execute("UPDATE applications SET date_applied=? WHERE id=? AND user_id=? AND " + "(date_applied IS NULL OR date_applied > ?)", (at[:10], app_id, user_id, at[:10])) + con.commit() + finally: + con.close() + + def set_job_mark(user_id: str, job_id: str, sponsorship: str | None) -> None: """Record the user's call on whether a job sponsors visas (one of JOB_MARKS), or clear it (None).""" if sponsorship is not None and sponsorship not in JOB_MARKS: diff --git a/api/evals/test_email_eval.py b/api/evals/test_email_eval.py index b748567..dfe6e06 100644 --- a/api/evals/test_email_eval.py +++ b/api/evals/test_email_eval.py @@ -1,13 +1,13 @@ """ Does Windrush read job emails right, and leave alone what isn't about an application? -16 emails as graduate applicants get them (applicant tracking systems, company recruiters, agencies, job +20 emails as graduate applicants get them (applicant tracking systems, company recruiters, agencies, job boards, newsletters), each labelled with what it means and which application it is about. Every run writes evals/last_email_run.json. The bar: no email is matched to the wrong application; nothing unrelated (job alerts, newsletters, an agency's pitch) changes an application; every confident reading that would change a status on its own is correct. -How many labelled emails were read correctly is reported, and must be at least 13 of 16. +How many labelled emails were read correctly is reported, and must be at least 17 of 20. """ import asyncio import json @@ -57,6 +57,14 @@ ("LinkedIn Job Alerts ", "Graduate Software Engineer at Catapult and 12 more jobs", "Jobs you may be interested in: Graduate Software Engineer - Catapult - London. Junior Developer - Monzo.", "unrelated", None), + ("LinkedIn ", "Graduate Application Engineer at Monzo", + "This job fits your profile. Monzo is hiring a Graduate Application Engineer in London. Apply now.", "unrelated", None), + ("Ocado Careers ", "We think you'd be great for this role at Ocado", + "Hi Timothy, based on your profile we think this Graduate Developer role suits you. Apply by Friday.", "unrelated", None), + ("Gradcracker ", "3 application deadlines this week", + "These graduate schemes close soon: Monzo, Ocado and Catapult.", "unrelated", None), + ("LinkedIn ", "Timothy, your application was sent to Globex Corporation", + "Your application was sent to Globex Corporation. Software Engineer, London.", "received", None), ("Indeed ", "New jobs for Graduate Software Engineer in Bristol", "Ocado Technology is hiring a Graduate Software Developer. Apply now with your Indeed CV.", "unrelated", None), ("Baringa ", "Baringa Insights: The future of energy markets", @@ -116,4 +124,4 @@ async def run(): assert not wrong_app, wrong_app assert not unrelated_changed, unrelated_changed assert not wrong_auto, wrong_auto - assert right >= 13 + assert right >= 17 diff --git a/api/tests/test_gmail.py b/api/tests/test_gmail.py index 992d953..964af3e 100644 --- a/api/tests/test_gmail.py +++ b/api/tests/test_gmail.py @@ -411,3 +411,210 @@ def test_add_all_makes_one_application_per_company_and_role_at_its_latest_status assert ("Hooli", "") not in tracked and _status(user, apps["catapult"]) == "Applied" left = [e for e in client.get("/gmail/events").json() if e["decision"] == "unmatched"] assert [e["gmail_id"] for e in left] == ["a5"] + + +# ── Job alerts, and reading older mail ──────────────────────────────────────────────────────── + +def _email(sender: str, subject: str) -> dict: + return {"sender": sender, "sender_address": sender, "subject": subject, "text": "", "snippet": ""} + + +@pytest.mark.parametrize("sender, subject", [ + ("jobalerts-noreply@linkedin.com", "Graduate Software Engineer at Catapult and 12 more jobs"), + ("jobs-noreply@linkedin.com", "Junior Application Software Engineer at One Big Circle Ltd: up to £45K/year"), + ("noreply@jobalert.indeed.com", "Application Support Analyst at Nisbets. 12 more jobs in BS1 1AA"), + ("noreply@jobalert.indeed.com", "BS1 1AA: Full Stack Application Developer at Savills and 9 more new jobs"), + ("noreply@match.indeed.com", "Junior Managed Application Services Analyst @ Evotix Limited"), + ("alert@indeed.com", "Interview-ready roles near you"), + ("noreply@gradcracker.com", "3 application deadlines this week"), + ("careers@jobalerts-uk.pwc.com", "New opportunities at PwC"), + ("hello@board.example", "This job fits you: Application Engineer"), + ("hello@board.example", "Graduate roles recommended for you"), +]) +def test_job_alerts_arent_read_as_applications(sender, subject): + assert gmail.is_job_alert(_email(sender, subject)) + + +@pytest.mark.parametrize("sender, subject", [ + ("jobs-noreply@linkedin.com", "Timothy Oluwatobi, your application was sent to Clearwater"), + ("jobs-noreply@linkedin.com", "Your application to Full Stack Engineer (AI systems) at Digital Waffle"), + ("jobs-noreply@linkedin.com", "Your application was viewed by Neurochase"), + ("indeedapply@indeed.com", "Indeed Application: Junior Marketing Assistant"), + ("noreply@indeed.com", "An update on your application from No Code Sage"), + ("no-reply@greenhouse.io", "Thank you for applying to Catapult"), + ("noreply@jobalert.example", "Your Baringa application"), + ("careers@hirevue.com", "Your interview with Optiver"), +]) +def test_real_application_emails_arent_taken_for_alerts(sender, subject): + assert not gmail.is_job_alert(_email(sender, subject)) + + +def test_a_sync_never_sends_a_job_alert_to_the_model(inbox, user, google, readings): + apps, mail = inbox + mail("m1", "LinkedIn ", "Application Engineer at Catapult: up to £45K/year", "See the job.", + _reading("interview", 1)) # were it read, this would move Catapult on + asyncio.run(gmail.sync(user)) + assert _status(user, apps["catapult"]) == "Applied" + + +async def _finish(user): + gmail.start_backfill(user, "6m") + await gmail._backfills[user]["task"] + return gmail.backfill_status(user) + + +@pytest.fixture +def past(google, readings, client, user, monkeypatch): + monkeypatch.setattr(gmail, "start_sync", lambda uid: True) + gmail._backfills.clear() + _connect(client) + + def mail(mid, sender, subject, body, reading=None, days_ago=30): + google.messages[mid] = message(mid, sender, subject, body, days_ago * 24 * 60) + readings[subject] = reading + return mail + + +def _past(kind, n=None, company="Globex", role="Graduate Engineer", confidence="high"): + return {"kind": kind, "application": n, "confidence": confidence, "company": company, "role": role, "summary": ""} + + +def _app_named(user, company): + return next((a for a in tracker.list_applications(user) if a["company"] == company), None) + + +def test_older_mail_adds_applications_it_shows_were_made_with_the_path_they_took(past, user): + past("m1", "Globex ", "Thanks for applying to Globex", "Thank you for applying.", + _past("received"), days_ago=60) + past("m2", "Globex ", "Interview at Globex", "We'd like to interview you.", + _past("interview"), days_ago=45) + past("m3", "Globex ", "Your application to Globex", "Unfortunately not.", + _past("rejected"), days_ago=30) + state = asyncio.run(_finish(user)) + + app = _app_named(user, "Globex") + assert state["added"]["added"] == 1 and state["read"] == 3 and not state["error"] + assert app["status"] == "Rejected" and app["job_id"].startswith("email:") + assert [(s["status"], s["source"]) for s in app["history"]] == [ + ("Applied", "email"), ("Interview", "email"), ("Rejected", "email")] + assert app["date_applied"] == app["history"][0]["changed_at"][:10] + assert app["history"][0]["changed_at"] < app["history"][1]["changed_at"] < app["history"][2]["changed_at"] + + +def test_a_recruiters_approach_doesnt_add_an_application(past, user, client): + past("m1", "Sarah ", "Interview this week?", "Could I arrange an interview?", + _past("interview", company="Fintech Co")) + state = asyncio.run(_finish(user)) + assert _app_named(user, "Fintech Co") is None and state["counts"] == {"dismissed": 1} + assert [e["decision"] for e in client.get("/gmail/events").json()] == ["dismissed"] + + +def test_an_unsure_reading_doesnt_add_an_application(past, user): + past("m1", "Globex ", "Your application", "Thanks.", _past("received", confidence="medium")) + asyncio.run(_finish(user)) + assert _app_named(user, "Globex") is None + + +def test_older_mail_skips_job_alerts_without_the_model(past, user): + past("m1", "LinkedIn ", "Application Engineer at Globex and 5 more jobs", "Jobs.", + _past("received")) + state = asyncio.run(_finish(user)) + assert state["counts"] == {"ignored": 1} and _app_named(user, "Globex") is None + + +def test_an_old_email_fills_in_history_without_moving_the_application_back(past, user): + app_id = _app(user, "Catapult", "Graduate Software Engineer", "Rejected") + past("m1", "Catapult ", "Interview at Catapult", "Catapult would like to interview you.", + _past("interview", 1, company="Catapult")) + state = asyncio.run(_finish(user)) + app = _app_named(user, "Catapult") + assert app["status"] == "Rejected" and state["counts"] == {"noted": 1} + assert "Interview" in [s["status"] for s in app["history"]] + assert app["id"] == app_id + + +def test_an_earlier_confirmation_moves_the_applied_date_back(past, user): + _app(user, "Catapult", "Graduate Software Engineer", "Applied") + past("m1", "Catapult ", "Thanks for applying to Catapult", "Catapult received it.", + _past("received", 1, company="Catapult"), days_ago=90) + asyncio.run(_finish(user)) + app = _app_named(user, "Catapult") + assert app["date_applied"] < time.strftime("%Y-%m-%d") and app["status"] == "Applied" + + +def test_a_run_carries_on_where_the_last_stopped(past, user, client): + past("m1", "Globex ", "Thanks for applying to Globex", "Thanks.", _past("received")) + asyncio.run(_finish(user)) + past("m2", "Initech ", "Thanks for applying to Initech", "Thanks.", + _past("received", company="Initech")) + assert client.get("/gmail/backfill/count?period=6m").json() == {"period": "6m", "total": 2, "to_read": 1} + assert asyncio.run(_finish(user))["read"] == 1 + + +def test_stopping_keeps_what_it_found(past, user, google): + past("m1", "Globex ", "Thanks for applying to Globex", "Thanks.", _past("received"), days_ago=40) + past("m2", "Initech ", "Thanks for applying to Initech", "Thanks.", + _past("received", company="Initech"), days_ago=20) + google.messages = dict(reversed(list(google.messages.items()))) # Gmail lists the newest first + real_read = gmail.read + + async def stop_after_first(email, applications): + gmail.stop_backfill(user) + return await real_read(email, applications) + gmail.read = stop_after_first + try: + state = asyncio.run(_finish(user)) + finally: + gmail.read = real_read + assert state["read"] == 1 and _app_named(user, "Globex") and not _app_named(user, "Initech") + + +def test_a_model_that_cant_be_reached_stops_the_run_and_leaves_the_email_unread(past, user): + past("m1", "Globex ", "Thanks for applying to Globex", "Thanks.", None) + state = asyncio.run(_finish(user)) + assert "model" in state["error"] and not tracker.email_seen(user, "m1") + + +def test_gmail_saying_slow_down_makes_it_wait_and_carry_on(past, user, google, monkeypatch): + monkeypatch.setattr(gmail, "_QUOTA_WAIT_S", 0) + past("m1", "Globex ", "Thanks for applying to Globex", "Thanks.", _past("received")) + google.slow_down.add("/messages/m1") + assert asyncio.run(_finish(user))["read"] == 1 and _app_named(user, "Globex") + + +def test_the_backfill_only_takes_the_periods_it_offers(past, client): + assert client.post("/gmail/backfill", json={"period": "10y"}).status_code == 400 + assert client.get("/gmail/backfill/count?period=10y").status_code == 400 + + +def test_a_job_board_isnt_taken_for_the_employer(past, user): + past("m1", "Indeed Apply ", "Indeed Application: Cafe Bar Assistant", "Your application was sent.", + _past("received", company="Indeed", role="Cafe Bar Assistant")) + past("m2", "Indeed Apply ", "Indeed Application: Front of House", "Your application was sent.", + _past("received", company="Unknown", role="Front of House", confidence="medium")) + asyncio.run(_finish(user)) + apps = {a["job_title"]: a["company"] for a in tracker.list_applications(user)} + assert apps == {"Cafe Bar Assistant": "Employer not named (via Indeed)", "Front of House": "Employer not named (via Indeed)"} + + +def test_an_application_whose_emails_name_no_role_is_still_added(past, user): + past("m1", "Fractile ", "Thank you for applying to Fractile", "Thanks.", + _past("received", company="Fractile", role="unspecified position")) + state = asyncio.run(_finish(user)) + assert _app_named(user, "Fractile")["job_title"] == "Role not stated" and state["added"]["left"] == 0 + + +def test_a_role_from_another_email_fills_in_one_that_names_none(past, user): + past("m2", "Globex ", "Regarding your application", "Unfortunately not.", + _past("rejected", role=""), days_ago=20) + past("m1", "Globex ", "Thanks for applying to Globex", "Thanks.", _past("received"), days_ago=40) + asyncio.run(_finish(user)) + app = _app_named(user, "Globex") + assert app["job_title"] == "Graduate Engineer" and app["status"] == "Rejected" + + +def test_an_interview_from_the_employers_own_address_adds_the_application(past, user): + past("m1", "Apple ", "Invitation to interview", "We'd like to interview you.", + _past("interview", company="Apple", role="Seasonal Specialist")) + asyncio.run(_finish(user)) + assert _app_named(user, "Apple")["status"] == "Interview" diff --git a/frontend/e2e/smoke.spec.ts b/frontend/e2e/smoke.spec.ts index 27b423d..1b5b16c 100644 --- a/frontend/e2e/smoke.spec.ts +++ b/frontend/e2e/smoke.spec.ts @@ -237,6 +237,78 @@ test("applications show the last 6 months until you widen the range, with how ea api.expectAllMocked(); }); +test("older job emails are counted before they're read, and the run can be stopped", async ({ page }) => { + await signIn(page); + const gmail = { configured: true, connected: true, email: "ada@example.com", problem: null, last_sync_at: null, + syncing: false, last: null, redirect_uri: "", backfill: null as object | null }; + const started: unknown[] = []; + const api = await mockApi(page, { + ...SHELL, + "GET /api/gmail": () => gmail, + "GET /api/gmail/events": [], + "GET /api/gmail/backfill/count": { total: 310, to_read: 280 }, + "POST /api/gmail/backfill": (request) => { + started.push(request.postDataJSON()); + gmail.backfill = { period: "1y", running: true, stop: false, to_read: 280, read: 12, counts: {}, added: null, + error: null, finished_at: null }; + return { started: true }; + }, + "DELETE /api/gmail/backfill": () => { + gmail.backfill = { period: "1y", running: false, stop: true, to_read: 280, read: 12, counts: { ignored: 5 }, + added: { added: 3, linked: 0, emails: 4, left: 0 }, error: null, finished_at: new Date().toISOString() }; + return { stopping: true }; + }, + }); + await page.goto("/inbox"); + await expect(page.getByText("310 job emails in this period; 30 already read.", { exact: false })).toBeVisible(); + + await page.getByRole("group", { name: "How far back" }).getByRole("button", { name: "1 year" }).click(); + await page.getByRole("button", { name: "Read 280 emails" }).click(); + await expect.poll(() => started).toEqual([{ period: "1y" }]); + await expect(page.getByText("Read 12 of 280 emails.")).toBeVisible(); + + await page.getByRole("button", { name: "Stop" }).click(); + await expect(page.getByText("3 applications added", { exact: false })).toBeVisible(); + api.expectAllMocked(); +}); + +test("the chart shows how applications went, with no reply only after 4 weeks of silence", async ({ page }) => { + await signIn(page); + const daysAgo = (n: number) => new Date(Date.now() - n * 86400000).toISOString(); + const sent = (id: string, status: string, applied: number, steps: [string, number][] = []) => app(id, status, { + date_applied: daysAgo(applied).slice(0, 10), + history: [["Applied", applied] as [string, number], ...steps].map(([s, d]) => ({ status: s, source: "email", changed_at: daysAgo(d) })), + }); + const api = await mockApi(page, { + ...SHELL, + "GET /api/applications": [ + sent("waiting", "Applied", 10), + sent("silent", "Applied", 60), + sent("no", "Rejected", 50, [["Rejected", 40]]), + sent("far", "Rejected", 90, [["Responded", 80], ["Interview", 70], ["Rejected", 60]]), + sent("yes", "Offer", 40, [["Interview", 30], ["Offer", 20]]), + sent("old", "Rejected", 300, [["Rejected", 280]]), + app("unsent", "Saved"), + ], + }); + await page.goto("/applications"); + await page.getByRole("group", { name: "View" }).getByRole("button", { name: "How they went" }).click(); + await expect(page.getByText("5 applied, 3 heard back (60%), 1 offer.")).toBeVisible(); + + await page.getByText("Show as a table").click(); + const row = (from: string, to: string, n: string) => page.getByRole("row", { name: `${from} ${to} ${n}`, exact: true }); + for (const [from, to, n] of [["Applied", "No reply", "1"], ["Applied", "Still waiting", "1"], ["Applied", "Rejected", "1"], + ["Applied", "Responded", "1"], ["Responded", "Interview", "1"], ["Interview", "Rejected", "1"], ["Applied", "Interview", "1"], + ["Interview", "Offer", "1"]]) { + await expect(row(from, to, n)).toBeVisible(); + } + + await page.getByRole("group", { name: "Chart range" }).getByRole("button", { name: "1 year" }).click(); + await expect(page.getByText("6 applied, 4 heard back (67%), 1 offer.")).toBeVisible(); + await page.getByRole("img", { name: /Sankey chart/ }).screenshot({ path: "test-results/sankey.png" }); + api.expectAllMocked(); +}); + test("the review queue lists paused applications and records one you sent yourself", async ({ page }) => { await signIn(page); const patches: unknown[] = []; @@ -594,6 +666,7 @@ test("inbox: setup steps when not configured, then email updates you can confirm ...SHELL, "GET /api/applications": [app("a1", "Applied"), app("a2", "Applied")], "GET /api/gmail": () => current, + "GET /api/gmail/backfill/count": { total: 0, to_read: 0 }, "GET /api/gmail/events": [ event("1", "suggested"), event("2", "unmatched", { application_id: null, app_title: null, app_company: null, app_status: null, diff --git a/frontend/src/app/applications/page.tsx b/frontend/src/app/applications/page.tsx index f3cf5e4..266f225 100644 --- a/frontend/src/app/applications/page.tsx +++ b/frontend/src/app/applications/page.tsx @@ -2,6 +2,7 @@ import Link from "next/link"; import { useEffect, useMemo, useState } from "react"; +import ApplicationFlow from "@/components/ApplicationFlow"; import Icon from "@/components/ui/Icon"; import TopBar, { Page } from "@/components/ui/TopBar"; import SlideOver from "@/components/ui/SlideOver"; @@ -16,7 +17,7 @@ import { applicationLink } from "@/lib/jobs"; import { CLOSED, CLOSED_STAGES, OPEN_STAGES, STATUS_CHOICES, stageOf, statusMeta } from "@/lib/status"; import type { Application } from "@/lib/types"; -type View = "board" | "table"; +type View = "board" | "table" | "flow"; type Quick = "review" | "follow" | "quiet"; type Range = "6m" | "1y" | "2y" | "all"; @@ -48,7 +49,10 @@ function dateLine(a: Application): string { case "Pending Review": return `Filled ${longAgo(a.created_at) ?? ""}`; case "Saved": case "Evaluated": return `Saved ${longAgo(a.created_at) ?? ""}`; - default: return a.date_applied ? `Applied ${shortDate(a.date_applied)}` : `Added ${shortDate(a.created_at) ?? ""}`; + default: { + const line = a.date_applied ? `Applied ${shortDate(a.date_applied)}` : `Added ${shortDate(a.created_at) ?? ""}`; + return a.job_id.startsWith("email:") ? `${line}, from email` : line; + } } } @@ -187,22 +191,26 @@ export default function ApplicationsPage() {
- - - - - setQuery(e.target.value)} - className="w-[140px] border-0 bg-transparent text-[13px] outline-none text-ink placeholder:text-ink-3" /> - - + items={[{ key: "board", label: "Board" }, { key: "table", label: "Table" }, { key: "flow", label: "How they went" }]} /> + {view !== "flow" && } + {view !== "flow" && ( + <> + + + + setQuery(e.target.value)} + className="w-[140px] border-0 bg-transparent text-[13px] outline-none text-ink placeholder:text-ink-3" /> + + + + )}
-
+ {view !== "flow" &&
Needs attention {QUICK.map((q) => { const on = quick === q.key; @@ -217,7 +225,7 @@ export default function ApplicationsPage() { ); })} -
+
} {error && {error}} @@ -231,6 +239,8 @@ export default function ApplicationsPage() { Browse matches}> No applications yet. Save a job from the feed or let the agent apply, and it shows up here. + ) : view === "flow" ? ( + ) : view === "board" ? (
("6m"); + const [count, setCount] = useState<{ total: number; to_read: number } | null>(null); + const [counting, setCounting] = useState(false); + const [error, setError] = useState(null); + const running = state?.running ?? false; + + useEffect(() => { + let live = true; + setCount(null); + setCounting(true); + setError(null); + countPastEmails(period) + .then((c) => { if (live) setCount(c); }) + .catch((e) => { if (live) setError(e instanceof Error ? e.message : "Couldn't count your emails."); }) + .finally(() => { if (live) setCounting(false); }); + return () => { live = false; }; + }, [period, running]); + + const act = async (fn: () => Promise) => { + setError(null); + try { + await fn(); + } catch (e) { + setError(e instanceof Error ? e.message : "That didn't work."); + } + }; + + return ( + +
+
+

Older job emails

+ + Read further back to fill in applications you made before connecting Gmail. Job alerts and recommendations + are skipped. An application is only added when an email shows you applied, marked as from email. + +
+ {!running && } +
+ {running && state ? ( +
+ + {state.stop ? "Stopping…" : state.to_read == null ? "Listing your emails…" : `Read ${state.read} of ${state.to_read} emails.`} + + {!!state.to_read && ( + + )} + +
+ ) : ( +
+ + + {counting ? "Counting…" : count ? `${count.total} job emails in this period; ${count.total - count.to_read} already read. Each one is sent to the model, as your new mail is.` : ""} + +
+ )} + {state && !running && !state.error && state.finished_at && ( + + Read {state.read} emails: {state.added?.added ?? 0} applications added + {state.counts.ignored ? `, ${state.counts.ignored} alerts and unrelated emails skipped` : ""} + {state.counts.dismissed ? `, ${state.counts.dismissed} too unclear to add (under Other)` : ""} + {state.added?.left ? `, ${state.added.left} need you because they name no role` : ""}. + + )} + {state?.error && !running && {state.error}} + {error && {error}} + + ); +} + function EventRow({ event, apps }: { event: EmailEvent; apps: Application[] }) { const [busy, setBusy] = useState(null); const [error, setError] = useState(null); @@ -284,7 +366,7 @@ function Inbox() { const [tab, setTab] = useState("needs"); const [starting, setStarting] = useState(false); const [error, setError] = useState(null); - const syncing = status.data?.syncing ?? false; + const syncing = (status.data?.syncing || status.data?.backfill?.running) ?? false; useEffect(() => { if (!syncing) return; @@ -350,6 +432,7 @@ function Inbox() { {error && {error}} {s ? : } + {live && } {(live || (events.data?.length ?? 0) > 0) && ( diff --git a/frontend/src/components/ApplicationFlow.tsx b/frontend/src/components/ApplicationFlow.tsx new file mode 100644 index 0000000..4809e4d --- /dev/null +++ b/frontend/src/components/ApplicationFlow.tsx @@ -0,0 +1,119 @@ +"use client"; + +import { useMemo, useState } from "react"; +import { Segmented } from "@/components/ui/Tabs"; +import { Card, cx, EmptyState } from "@/components/ui/kit"; +import { buildFlow, FLOW_WIDTH, layoutFlow, NO_REPLY_DAYS, NODE_WIDTH, type FlowLink, type FlowNode } from "@/lib/flow"; +import type { Application } from "@/lib/types"; + +type Span = "3m" | "6m" | "1y" | "2y"; + +const SPANS: { key: Span; label: string; days: number }[] = [ + { key: "3m", label: "3 months", days: 91 }, { key: "6m", label: "6 months", days: 182 }, + { key: "1y", label: "1 year", days: 365 }, { key: "2y", label: "2 years", days: 730 }, +]; + +/* Three meanings, not one colour per node: moving forward, turned down, silence. Every node also says its name. */ +const COLOUR: Record = { + Applied: "var(--blue)", Responded: "var(--blue)", Interview: "var(--blue)", Offer: "var(--blue)", + Rejected: "var(--orange)", "No reply": "var(--ink-3)", "Still waiting": "var(--ink-3)", "Closed by you": "var(--ink-3)", +}; + +const linkText = (l: FlowLink) => `${l.from} to ${l.to}: ${l.value} ${l.value === 1 ? "application" : "applications"}`; + +export default function ApplicationFlow({ apps }: { apps: Application[] }) { + const [span, setSpan] = useState("6m"); + const [hover, setHover] = useState<{ link: FlowLink; x: number; y: number } | null>(null); + const days = SPANS.find((s) => s.key === span)!.days; + const flow = useMemo(() => buildFlow(apps, days), [apps, days]); + const placed = useMemo(() => layoutFlow(flow), [flow]); + const share = flow.total ? Math.round((flow.heardBack / flow.total) * 100) : 0; + + return ( + +
+
+

How your applications went

+ {flow.total > 0 && ( + + {flow.total} applied, {flow.heardBack} heard back ({share}%), {flow.nodes.Offer ?? 0} {flow.nodes.Offer === 1 ? "offer" : "offers"}. + + )} +
+ { setSpan(s); setHover(null); }} items={SPANS} /> +
+ + {flow.total === 0 ? ( + No applications sent in this period. + ) : ( + <> +
setHover(null)}> + + {placed.links.map((l) => { + const on = hover?.link.from === l.from && hover.link.to === l.to; + return ( + + + {/* A thicker invisible line, so a thin flow is still easy to point at. */} + { + const box = e.currentTarget.ownerSVGElement!.parentElement!.getBoundingClientRect(); + setHover({ link: l, x: e.clientX - box.left, y: e.clientY - box.top }); + }} + onFocus={() => setHover({ link: l, x: 16, y: 16 })} onBlur={() => setHover(null)} /> + + ); + })} + {placed.nodes.map((n) => { + const outcome = n.column === 3; + return ( + + + + {n.name} + {n.value} + + + ); + })} + + {hover && ( +
+ {linkText(hover.link)} +
+ )} +
+ +
+ Show as a table + + + + {["From", "To", "Applications"].map((h) => )} + + + + {placed.links.map((l) => ( + + + + + + ))} + +
{h}
{l.from}{l.to}{l.value}
+
+ + + No reply means nothing has happened for {NO_REPLY_DAYS / 7} weeks. Built from your tracker and job emails, so it + misses applications with no confirmation email and steps that happened by phone. + + + )} + + ); +} diff --git a/frontend/src/lib/flow.ts b/frontend/src/lib/flow.ts new file mode 100644 index 0000000..a24943a --- /dev/null +++ b/frontend/src/lib/flow.ts @@ -0,0 +1,112 @@ +import { daysSince } from "./format"; +import { SENT } from "./status"; +import type { Application } from "./types"; + +/* How applications went, for the Sankey on the Applications page: each one's path from Applied through the steps its + history shows to where it ended up. */ + +export const NO_REPLY_DAYS = 28; + +export type FlowNode = "Applied" | "Responded" | "Interview" | "Offer" | "Rejected" | "No reply" | "Still waiting" | "Closed by you"; +export interface FlowLink { from: FlowNode; to: FlowNode; value: number } +export interface Flow { + total: number; + /** Got any reply: went past Applied, or was turned down. */ + heardBack: number; + nodes: Partial>; + links: FlowLink[]; +} + +const appliedOn = (a: Application): string | null => + a.date_applied ?? a.history?.find((s) => SENT.has(s.status))?.changed_at ?? (SENT.has(a.status) ? a.created_at : null); + +/** Where it ended up. With nothing final, what counts is how long since anything happened to it. */ +function outcome(a: Application, applied: string): FlowNode { + if (a.status === "Offer") return "Offer"; + if (a.status === "Rejected") return "Rejected"; + if (a.status === "Discarded") return "Closed by you"; + const last = a.history?.length ? a.history[a.history.length - 1].changed_at : applied; + return (daysSince(last) ?? 0) >= NO_REPLY_DAYS ? "No reply" : "Still waiting"; +} + +/** Applications applied to within the last `days` days, as flows between steps. Ones never sent aren't in it. */ +export function buildFlow(apps: Application[], days: number): Flow { + const links = new Map(); + const nodes: Flow["nodes"] = {}; + let total = 0; + let heardBack = 0; + for (const a of apps) { + const applied = appliedOn(a); + if (!applied || (daysSince(applied) ?? 0) > days) continue; + const reached = new Set([a.status, ...(a.history ?? []).map((s) => s.status)]); + const end = outcome(a, applied); + const path: FlowNode[] = ["Applied", ...(["Responded", "Interview"] as const).filter((s) => reached.has(s)), end]; + total += 1; + if (path.length > 2 || end === "Rejected" || end === "Offer") heardBack += 1; + path.forEach((node, i) => { + nodes[node] = (nodes[node] ?? 0) + 1; + if (i === 0) return; + const key = `${path[i - 1]}>${node}`; + const link = links.get(key) ?? { from: path[i - 1], to: node, value: 0 }; + link.value += 1; + links.set(key, link); + }); + } + return { total, heardBack, nodes, links: Array.from(links.values()) }; +} + +/* Where everything goes on the page: fixed columns in reading order, heights to one scale. */ + +const COLUMNS: FlowNode[][] = [["Applied"], ["Responded"], ["Interview"], + ["Offer", "Rejected", "No reply", "Still waiting", "Closed by you"]]; +const ORDER: FlowNode[] = COLUMNS.flat(); +export const FLOW_WIDTH = 900; +export const NODE_WIDTH = 12; +const LEFT = 8; +const LABELS_RIGHT = 150; // room for the outcomes' labels +const TOP = 30; +const GAP = 30; // keeps neighbouring labels apart, however small the nodes +const BODY = 280; +const MIN_NODE = 4; +const MIN_LINK = 1.5; + +export interface PlacedNode { name: FlowNode; value: number; x: number; y: number; height: number; column: number } +export interface PlacedLink extends FlowLink { width: number; path: string } + +export function layoutFlow(flow: Flow): { nodes: PlacedNode[]; links: PlacedLink[]; height: number } { + const scale = flow.total ? BODY / flow.total : 0; + const step = (FLOW_WIDTH - LEFT - LABELS_RIGHT - NODE_WIDTH) / (COLUMNS.length - 1); + const placed = new Map(); + let height = TOP; + COLUMNS.forEach((names, column) => { + let y = TOP; + for (const name of names) { + const value = flow.nodes[name]; + if (!value) continue; + const nodeHeight = Math.max(MIN_NODE, value * scale); + placed.set(name, { name, value, x: LEFT + column * step, y, height: nodeHeight, column }); + y += nodeHeight + GAP; + } + height = Math.max(height, y - GAP); + }); + // Links leave and arrive stacked, in the order of the nodes at their other end, so fewer of them cross. + const out = new Map(); + const into = new Map(); + const sorted = [...flow.links].sort((a, b) => ORDER.indexOf(a.from) - ORDER.indexOf(b.from) || ORDER.indexOf(a.to) - ORDER.indexOf(b.to)); + const links = sorted.map((link) => { + const from = placed.get(link.from)!; + const to = placed.get(link.to)!; + const share = (node: PlacedNode) => Math.max(MIN_LINK, link.value * (node.height / node.value)); + const fromShare = share(from); + const toShare = share(to); + const y0 = from.y + (out.get(link.from) ?? 0) + fromShare / 2; + const y1 = to.y + (into.get(link.to) ?? 0) + toShare / 2; + out.set(link.from, (out.get(link.from) ?? 0) + fromShare); + into.set(link.to, (into.get(link.to) ?? 0) + toShare); + const x0 = from.x + NODE_WIDTH; + const x1 = to.x; + const mid = (x0 + x1) / 2; + return { ...link, width: Math.max(fromShare, toShare), path: `M${x0},${y0} C${mid},${y0} ${mid},${y1} ${x1},${y1}` }; + }); + return { nodes: Array.from(placed.values()), links, height: height + 8 }; +} diff --git a/frontend/src/lib/gmail.ts b/frontend/src/lib/gmail.ts index 70ca2af..15567c1 100644 --- a/frontend/src/lib/gmail.ts +++ b/frontend/src/lib/gmail.ts @@ -1,5 +1,5 @@ import { load, requestJson } from "./store"; -import type { EmailEvent, GmailStatus } from "./types"; +import type { BackfillPeriod, EmailEvent, GmailStatus } from "./types"; /* Gmail tracking: read-only access to your job emails keeps your applications up to date (api/app/gmail.py). */ @@ -25,6 +25,20 @@ export async function syncGmail() { await load(GMAIL_URL, { force: true }); } +/** How many job emails a period holds and how many are unread, before anything is read. */ +export const countPastEmails = (period: BackfillPeriod) => + requestJson<{ total: number; to_read: number }>(`${GMAIL_URL}/backfill/count?period=${period}`); + +export async function startBackfill(period: BackfillPeriod) { + await requestJson(`${GMAIL_URL}/backfill`, post({ period })); + await load(GMAIL_URL, { force: true }); +} + +export async function stopBackfill() { + await requestJson(`${GMAIL_URL}/backfill`, { method: "DELETE" }); + await load(GMAIL_URL, { force: true }); +} + async function act(id: string, action: "confirm" | "dismiss" | "undo" | "track", body?: object) { await requestJson(`${EVENTS_URL}/${encodeURIComponent(id)}/${action}`, post(body)); await Promise.all([load(EVENTS_URL, { force: true }), load("/api/applications", { force: true })]); diff --git a/frontend/src/lib/types.ts b/frontend/src/lib/types.ts index 28d58e2..4269150 100644 --- a/frontend/src/lib/types.ts +++ b/frontend/src/lib/types.ts @@ -310,6 +310,22 @@ export interface GmailStatus { syncing: boolean; last: { finished_at: string; counts: Record; error?: string } | null; redirect_uri: string; + backfill?: Backfill | null; +} + +export type BackfillPeriod = "6m" | "1y" | "2y"; + +export interface Backfill { + period: BackfillPeriod; + running: boolean; + stop: boolean; + /** How many emails it has to read; null until it has listed them. */ + to_read: number | null; + read: number; + counts: Record; + added: { added: number; linked: number; emails: number; left: number } | null; + error: string | null; + finished_at: string | null; } export type EmailKind = "received" | "assessment" | "interview" | "offer" | "rejected" | "other";