diff --git a/api/app/job_searcher.py b/api/app/job_searcher.py index 7d926ff..9be55e6 100644 --- a/api/app/job_searcher.py +++ b/api/app/job_searcher.py @@ -305,6 +305,68 @@ def _num(x): return lo, hi +# Application systems host many employers' jobs, so their address never names the employer. Where the link does: +_ATS_EMPLOYER_IN_PATH = { + "greenhouse.io": re.compile(r"^/([^/]+)/jobs/"), "lever.co": re.compile(r"^/([^/]+)"), + "ashbyhq.com": re.compile(r"^/([^/]+)"), "workable.com": re.compile(r"^/([^/]+)"), + "smartrecruiters.com": re.compile(r"^/([^/]+)"), +} +_ATS_EMPLOYER_IN_SUBDOMAIN = ("teamtailor.com", "recruitee.com", "pinpointhq.com", "bamboohr.com", "eightfold.ai", + "jobvite.com", "applytojob.com") +_ATS_HOSTS = (*_ATS_EMPLOYER_IN_PATH, *_ATS_EMPLOYER_IN_SUBDOMAIN, "myworkdayjobs.com", "icims.com", "taleo.net", + "oraclecloud.com", "successfactors.com") +_ATS_NAME = re.compile(r"(?:powered by )?(?:greenhouse|lever|ashby|workable|workday|smartrecruiters|teamtailor|recruitee|" + r"pinpoint|bamboohr|eightfold|jobvite|icims|taleo|successfactors)(?: job boards?)?", re.I) + + +def _on_ats(host: str, ats: str) -> bool: + return host == ats or host.endswith("." + ats) + + +def _employer_from_address(url: str) -> str: + """The employer as far as a job link says, "" when it doesn't. The site's own address counts only when the site + isn't an application system's: job-boards.greenhouse.io is nobody's employer.""" + from urllib.parse import parse_qs, urlparse + host, path = _site(url) + employer, _ = _employer_and_place("", url) + if employer != "Unknown": + return employer + named = "" + if _on_ats(host, "greenhouse.io") and path.startswith("/embed/"): + named = (parse_qs(urlparse(url).query).get("for") or [""])[0] + for ats, where in _ATS_EMPLOYER_IN_PATH.items(): + m = where.search(path) if _on_ats(host, ats) and not named else None + if m: + named = m.group(1) + for ats in _ATS_EMPLOYER_IN_SUBDOMAIN: + if _on_ats(host, ats) and host != ats: + named = host.split(".")[0] + if named and named not in ("www", "apply", "embed", "jobs", "careers"): + return named.replace("-", " ").replace("_", " ").title() + return "" if any(_on_ats(host, ats) for ats in _ATS_HOSTS) else host + + +async def _greenhouse_company(url: str) -> str: + """The employer's own name for a Greenhouse link, from Greenhouse's job data ("Catapult Sports", where the link + only says "catapultsports"). "" when it isn't one or that can't be read.""" + from urllib.parse import parse_qs, urlparse + parsed = urlparse(url) + host, path = _site(url) + if not _on_ats(host, "greenhouse.io"): + return "" + m = re.search(r"/([^/]+)/jobs/\d+", path) + slug = m.group(1) if m else (parse_qs(parsed.query).get("for") or [""])[0] + if not re.fullmatch(r"[A-Za-z0-9_-]+", slug): + return "" + try: + async with http_client.async_client(follow_redirects=True, timeout=12.0) as client: + resp = await client.get(f"https://boards-api.greenhouse.io/v1/boards/{slug}") + return str(resp.json().get("name") or "").strip() if resp.status_code == 200 else "" + except Exception as exc: + logger.debug("Greenhouse board name failed for %s: %s", slug, exc) + return "" + + async def scrape_job_from_url(url: str) -> dict: """ Scrape an arbitrary job-posting URL into a full Job dict ready for analysis. @@ -316,7 +378,6 @@ async def scrape_job_from_url(url: str) -> dict: the caller can surface a clear "couldn't read that listing" message. """ import uuid - from urllib.parse import urlparse if not url or not url.startswith("http"): raise ValueError("Please paste a valid http(s) job-posting link.") @@ -354,10 +415,18 @@ async def scrape_job_from_url(url: str) -> dict: if not title: page_title = _first_tag_text(raw_html, "title") title = re.split(r"\s+[|\-–—]\s+", page_title)[0].strip() if page_title else "" + # Lever's page title is "Bloom & Wild Group - Junior Engineer", where the link only says "bloomon". + if not company and _on_ats(_site(url)[0], "lever.co"): + named, dash, role = title.partition(" - ") + if dash and role.strip(): + company, title = named.strip(), role.strip() + if not company: + company = await _greenhouse_company(url) if not company: company = _meta_content(raw_html, "og:site_name") + company = "" if _ATS_NAME.fullmatch(company) else company if not company: - company = urlparse(url).netloc.replace("www.", "") + company = _employer_from_address(url) title = title.strip() if not title and not description: diff --git a/api/tests/test_job_search.py b/api/tests/test_job_search.py index add5791..b0954c4 100644 --- a/api/tests/test_job_search.py +++ b/api/tests/test_job_search.py @@ -758,3 +758,76 @@ async def fake_ats(queries, boards=None, locations=None): return [{"title": "Software Engineer", "company": "Far", "location": "Austin, TX", "url": "https://x/1"}] monkeypatch.setattr(job_searcher, "_search_level2_ats_apis", fake_ats) assert asyncio.run(job_searcher.search_jobs_multi("software engineer", ["London"])) == [] + + +# ── A pasted link's employer ────────────────────────────────────────────────────────────────── + +@pytest.mark.parametrize("url, employer", [ + ("https://job-boards.greenhouse.io/catapultsports/jobs/7821130", "Catapultsports"), + ("https://job-boards.eu.greenhouse.io/b2c2/jobs/4745320101/", "B2C2"), + ("https://boards.greenhouse.io/embed/job_app?for=acme-labs&token=123", "Acme Labs"), + ("https://jobs.lever.co/mytos/bbb558c0-d769-4b5f-a3a7-025642c2626d/apply", "Mytos"), + ("https://jobs.eu.lever.co/bloomon/7f832927-5c83-4fe9", "Bloomon"), + ("https://jobs.ashbyhq.com/open-ai/abc", "Open Ai"), + ("https://apply.workable.com/acme/j/ABC123/", "Acme"), + ("https://jobs.smartrecruiters.com/Gousto/123", "Gousto"), + ("https://acme.teamtailor.com/jobs/1-engineer", "Acme"), + ("https://lseg.wd3.myworkdayjobs.com/en-US/Graduate_Careers/details/Engineer_R1", "LSEG"), + ("https://acme.wd3.myworkdayjobs.com/en-US/External/job/London/Engineer_R1", "Acme"), + ("https://careers.arm.com/job/1", "Arm"), + ("https://www.acme-robotics.com/careers/engineer", "acme-robotics.com"), # its own site: the best there is + ("https://job-boards.greenhouse.io/", ""), # never the system's own address + ("https://jobs.lever.co/", ""), + ("https://fa-abc.fa.em2.oraclecloud.com/hcmUI/CandidateExperience/en/sites/CX/job/1", ""), +]) +def test_a_job_links_address_names_the_employer_or_nothing(url, employer): + assert job_searcher._employer_from_address(url) == employer + + +def _pasted(monkeypatch, html: str, greenhouse: str = "") -> None: + async def ok(url): + return None + + async def page(client, url): + return type("Page", (), {"text": html, "raise_for_status": lambda self: None})() + + async def description(url, max_chars=8000): + return "A long enough description of the job." + + async def company(url): + return greenhouse + monkeypatch.setattr(job_searcher.net_guard, "assert_public_url", ok) + monkeypatch.setattr(job_searcher.net_guard, "safe_get", page) + monkeypatch.setattr(job_searcher, "fetch_full_description", description) + monkeypatch.setattr(job_searcher, "_greenhouse_company", company) + + +GREENHOUSE_LINK = "https://job-boards.greenhouse.io/catapultsports/jobs/7821130" + + +def test_a_pasted_greenhouse_link_gets_the_employers_own_name(monkeypatch): + _pasted(monkeypatch, "Job Application", greenhouse="Catapult Sports") + assert asyncio.run(job_searcher.scrape_job_from_url(GREENHOUSE_LINK))["company"] == "Catapult Sports" + + +def test_with_greenhouse_unreachable_the_link_still_names_the_employer(monkeypatch): + _pasted(monkeypatch, "Job Application") + assert asyncio.run(job_searcher.scrape_job_from_url(GREENHOUSE_LINK))["company"] == "Catapultsports" + + +def test_a_page_naming_the_application_system_isnt_taken_for_the_employer(monkeypatch): + _pasted(monkeypatch, '

Engineer

') + job = asyncio.run(job_searcher.scrape_job_from_url("https://jobs.lever.co/mytos/bbb558c0-d769-4b5f")) + assert job["company"] == "Mytos" + + +def test_a_lever_page_title_gives_the_employer_and_the_role(monkeypatch): + _pasted(monkeypatch, '') + job = asyncio.run(job_searcher.scrape_job_from_url("https://jobs.lever.co/bloomon/7f832927-5c83-4fe9")) + assert (job["company"], job["title"]) == ("Bloom & Wild Group", "Junior Full-Stack - London") + + +def test_a_page_stating_its_employer_is_believed_first(monkeypatch): + _pasted(monkeypatch, '

Engineer

') + job = asyncio.run(job_searcher.scrape_job_from_url("https://www.acme.example/job/1")) + assert job["company"] == "Acme Robotics"