From 917314e13c694ee3556bf9ba81a10e40f281a61e Mon Sep 17 00:00:00 2001 From: Timothy Oluwatobi Ojebiyi Date: Tue, 3 Nov 2026 11:05:27 +0000 Subject: [PATCH] fix: a pasted job link shows the employer, not job-boards.greenhouse.io Greenhouse's pages don't state the employer in their metadata, so the site's address was used instead. Greenhouse links now take the name from its job data, Lever links from the page title (which also leaves the employer out of the job title), and other application systems from the link. The bare address is only used when the site isn't an application system's, and a page naming the system as its site isn't believed. --- api/app/job_searcher.py | 73 +++++++++++++++++++++++++++++++++++- api/tests/test_job_search.py | 73 ++++++++++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+), 2 deletions(-) diff --git a/api/app/job_searcher.py b/api/app/job_searcher.py index 7d926ff..9be55e6 100644 --- a/api/app/job_searcher.py +++ b/api/app/job_searcher.py @@ -305,6 +305,68 @@ def _num(x): return lo, hi +# Application systems host many employers' jobs, so their address never names the employer. Where the link does: +_ATS_EMPLOYER_IN_PATH = { + "greenhouse.io": re.compile(r"^/([^/]+)/jobs/"), "lever.co": re.compile(r"^/([^/]+)"), + "ashbyhq.com": re.compile(r"^/([^/]+)"), "workable.com": re.compile(r"^/([^/]+)"), + "smartrecruiters.com": re.compile(r"^/([^/]+)"), +} +_ATS_EMPLOYER_IN_SUBDOMAIN = ("teamtailor.com", "recruitee.com", "pinpointhq.com", "bamboohr.com", "eightfold.ai", + "jobvite.com", "applytojob.com") +_ATS_HOSTS = (*_ATS_EMPLOYER_IN_PATH, *_ATS_EMPLOYER_IN_SUBDOMAIN, "myworkdayjobs.com", "icims.com", "taleo.net", + "oraclecloud.com", "successfactors.com") +_ATS_NAME = re.compile(r"(?:powered by )?(?:greenhouse|lever|ashby|workable|workday|smartrecruiters|teamtailor|recruitee|" + r"pinpoint|bamboohr|eightfold|jobvite|icims|taleo|successfactors)(?: job boards?)?", re.I) + + +def _on_ats(host: str, ats: str) -> bool: + return host == ats or host.endswith("." + ats) + + +def _employer_from_address(url: str) -> str: + """The employer as far as a job link says, "" when it doesn't. The site's own address counts only when the site + isn't an application system's: job-boards.greenhouse.io is nobody's employer.""" + from urllib.parse import parse_qs, urlparse + host, path = _site(url) + employer, _ = _employer_and_place("", url) + if employer != "Unknown": + return employer + named = "" + if _on_ats(host, "greenhouse.io") and path.startswith("/embed/"): + named = (parse_qs(urlparse(url).query).get("for") or [""])[0] + for ats, where in _ATS_EMPLOYER_IN_PATH.items(): + m = where.search(path) if _on_ats(host, ats) and not named else None + if m: + named = m.group(1) + for ats in _ATS_EMPLOYER_IN_SUBDOMAIN: + if _on_ats(host, ats) and host != ats: + named = host.split(".")[0] + if named and named not in ("www", "apply", "embed", "jobs", "careers"): + return named.replace("-", " ").replace("_", " ").title() + return "" if any(_on_ats(host, ats) for ats in _ATS_HOSTS) else host + + +async def _greenhouse_company(url: str) -> str: + """The employer's own name for a Greenhouse link, from Greenhouse's job data ("Catapult Sports", where the link + only says "catapultsports"). "" when it isn't one or that can't be read.""" + from urllib.parse import parse_qs, urlparse + parsed = urlparse(url) + host, path = _site(url) + if not _on_ats(host, "greenhouse.io"): + return "" + m = re.search(r"/([^/]+)/jobs/\d+", path) + slug = m.group(1) if m else (parse_qs(parsed.query).get("for") or [""])[0] + if not re.fullmatch(r"[A-Za-z0-9_-]+", slug): + return "" + try: + async with http_client.async_client(follow_redirects=True, timeout=12.0) as client: + resp = await client.get(f"https://boards-api.greenhouse.io/v1/boards/{slug}") + return str(resp.json().get("name") or "").strip() if resp.status_code == 200 else "" + except Exception as exc: + logger.debug("Greenhouse board name failed for %s: %s", slug, exc) + return "" + + async def scrape_job_from_url(url: str) -> dict: """ Scrape an arbitrary job-posting URL into a full Job dict ready for analysis. @@ -316,7 +378,6 @@ async def scrape_job_from_url(url: str) -> dict: the caller can surface a clear "couldn't read that listing" message. """ import uuid - from urllib.parse import urlparse if not url or not url.startswith("http"): raise ValueError("Please paste a valid http(s) job-posting link.") @@ -354,10 +415,18 @@ async def scrape_job_from_url(url: str) -> dict: if not title: page_title = _first_tag_text(raw_html, "title") title = re.split(r"\s+[|\-–—]\s+", page_title)[0].strip() if page_title else "" + # Lever's page title is "Bloom & Wild Group - Junior Engineer", where the link only says "bloomon". + if not company and _on_ats(_site(url)[0], "lever.co"): + named, dash, role = title.partition(" - ") + if dash and role.strip(): + company, title = named.strip(), role.strip() + if not company: + company = await _greenhouse_company(url) if not company: company = _meta_content(raw_html, "og:site_name") + company = "" if _ATS_NAME.fullmatch(company) else company if not company: - company = urlparse(url).netloc.replace("www.", "") + company = _employer_from_address(url) title = title.strip() if not title and not description: diff --git a/api/tests/test_job_search.py b/api/tests/test_job_search.py index add5791..b0954c4 100644 --- a/api/tests/test_job_search.py +++ b/api/tests/test_job_search.py @@ -758,3 +758,76 @@ async def fake_ats(queries, boards=None, locations=None): return [{"title": "Software Engineer", "company": "Far", "location": "Austin, TX", "url": "https://x/1"}] monkeypatch.setattr(job_searcher, "_search_level2_ats_apis", fake_ats) assert asyncio.run(job_searcher.search_jobs_multi("software engineer", ["London"])) == [] + + +# ── A pasted link's employer ────────────────────────────────────────────────────────────────── + +@pytest.mark.parametrize("url, employer", [ + ("https://job-boards.greenhouse.io/catapultsports/jobs/7821130", "Catapultsports"), + ("https://job-boards.eu.greenhouse.io/b2c2/jobs/4745320101/", "B2C2"), + ("https://boards.greenhouse.io/embed/job_app?for=acme-labs&token=123", "Acme Labs"), + ("https://jobs.lever.co/mytos/bbb558c0-d769-4b5f-a3a7-025642c2626d/apply", "Mytos"), + ("https://jobs.eu.lever.co/bloomon/7f832927-5c83-4fe9", "Bloomon"), + ("https://jobs.ashbyhq.com/open-ai/abc", "Open Ai"), + ("https://apply.workable.com/acme/j/ABC123/", "Acme"), + ("https://jobs.smartrecruiters.com/Gousto/123", "Gousto"), + ("https://acme.teamtailor.com/jobs/1-engineer", "Acme"), + ("https://lseg.wd3.myworkdayjobs.com/en-US/Graduate_Careers/details/Engineer_R1", "LSEG"), + ("https://acme.wd3.myworkdayjobs.com/en-US/External/job/London/Engineer_R1", "Acme"), + ("https://careers.arm.com/job/1", "Arm"), + ("https://www.acme-robotics.com/careers/engineer", "acme-robotics.com"), # its own site: the best there is + ("https://job-boards.greenhouse.io/", ""), # never the system's own address + ("https://jobs.lever.co/", ""), + ("https://fa-abc.fa.em2.oraclecloud.com/hcmUI/CandidateExperience/en/sites/CX/job/1", ""), +]) +def test_a_job_links_address_names_the_employer_or_nothing(url, employer): + assert job_searcher._employer_from_address(url) == employer + + +def _pasted(monkeypatch, html: str, greenhouse: str = "") -> None: + async def ok(url): + return None + + async def page(client, url): + return type("Page", (), {"text": html, "raise_for_status": lambda self: None})() + + async def description(url, max_chars=8000): + return "A long enough description of the job." + + async def company(url): + return greenhouse + monkeypatch.setattr(job_searcher.net_guard, "assert_public_url", ok) + monkeypatch.setattr(job_searcher.net_guard, "safe_get", page) + monkeypatch.setattr(job_searcher, "fetch_full_description", description) + monkeypatch.setattr(job_searcher, "_greenhouse_company", company) + + +GREENHOUSE_LINK = "https://job-boards.greenhouse.io/catapultsports/jobs/7821130" + + +def test_a_pasted_greenhouse_link_gets_the_employers_own_name(monkeypatch): + _pasted(monkeypatch, "Job Application", greenhouse="Catapult Sports") + assert asyncio.run(job_searcher.scrape_job_from_url(GREENHOUSE_LINK))["company"] == "Catapult Sports" + + +def test_with_greenhouse_unreachable_the_link_still_names_the_employer(monkeypatch): + _pasted(monkeypatch, "Job Application") + assert asyncio.run(job_searcher.scrape_job_from_url(GREENHOUSE_LINK))["company"] == "Catapultsports" + + +def test_a_page_naming_the_application_system_isnt_taken_for_the_employer(monkeypatch): + _pasted(monkeypatch, '

Engineer

') + job = asyncio.run(job_searcher.scrape_job_from_url("https://jobs.lever.co/mytos/bbb558c0-d769-4b5f")) + assert job["company"] == "Mytos" + + +def test_a_lever_page_title_gives_the_employer_and_the_role(monkeypatch): + _pasted(monkeypatch, '') + job = asyncio.run(job_searcher.scrape_job_from_url("https://jobs.lever.co/bloomon/7f832927-5c83-4fe9")) + assert (job["company"], job["title"]) == ("Bloom & Wild Group", "Junior Full-Stack - London") + + +def test_a_page_stating_its_employer_is_believed_first(monkeypatch): + _pasted(monkeypatch, '

Engineer

') + job = asyncio.run(job_searcher.scrape_job_from_url("https://www.acme.example/job/1")) + assert job["company"] == "Acme Robotics"