Kallidus Recruit (Advorto) Jobs API.

import re
import requests
from bs4 import BeautifulSoup

# Any hosted Kallidus Recruit / Advorto board works the same way.
BASE_URL = "https://jobs.icelandcareers.co.uk/Search.aspx"
GRID_ID = "MainPlaceholder_VacancyList_SearchableDataTable_DataGrid_DataGrid"

session = requests.Session()
session.headers.update({"User-Agent": "Mozilla/5.0 (compatible; JoboBot/1.0)"})

def clean(text):
    return re.sub(r"\s+", " ", text or "").strip()

def load(html):
    return BeautifulSoup(html, "html.parser")

resp = session.get(BASE_URL, timeout=30)
resp.raise_for_status()
soup = load(resp.text)

grid = soup.find(id=GRID_ID)
if grid is None:
    if re.search(r"no (?:current )?vacancies|no jobs", soup.get_text(), re.I):
        raise SystemExit("Board has no current vacancies")
    raise SystemExit("Advorto vacancy grid not found - not a Kallidus Recruit board")

from urllib.parse import urljoin, urlparse, parse_qs

def vid_of(url):
    return (parse_qs(urlparse(url).query).get("VId") or [""])[0].strip()

def parse_rows(grid, base_url):
    headers = [clean(th.get_text()) for th in grid.select("tr th")]

    def field(cells, *labels):
        for label in labels:
            for i, header in enumerate(headers):
                if header.lower() == label.lower() and i < len(cells):
                    return clean(cells[i].get_text())
        return ""

    jobs = []
    for row in grid.select("tr"):
        if "advorto-data-table-pager" in (row.get("class") or []):
            continue
        anchor = row.select_one("a[href*='VacancyInformation.aspx'][href*='VId=']")
        cells = row.select("td")
        if anchor is None or len(cells) < 4:
            continue
        detail_url = urljoin(base_url, anchor.get("href", ""))
        vid = vid_of(detail_url)          # native VId, NOT the displayed Ref
        title = clean(anchor.get_text())
        if not vid or not title:
            continue
        jobs.append({
            "external_id": vid,
            "title": title,
            "listing_url": detail_url,
            "business_reference": field(cells, "Ref", "Reference"),
            "category": field(cells, "Category", "Job Category", "Division"),
            "location": field(cells, "Location"),
            "salary_text": field(cells, "Salary"),
            "closing_date": field(cells, "Closing date"),
        })
    return jobs

listings = parse_rows(grid, BASE_URL)
print(f"Page 1: {len(listings)} vacancies")

POSTBACK = re.compile(
    r"__doPostBack\('(?P<target>[^']+)','(?P<arg>Page\$(?:\d+|Last))'\)")

def next_postback(soup):
    pager = soup.select_one("tr.advorto-data-table-pager")
    if pager is None:
        return None
    current = int(clean(pager.select_one("span").get_text()) or "1")
    for a in pager.select("a[href]"):
        m = POSTBACK.search(a.get("href", ""))
        arg = m.group("arg")[5:] if m else ""
        if m and arg.isdigit() and int(arg) == current + 1:
            fields = {}
            for inp in soup.select("form#form1 input[type='hidden'][name]"):
                fields[inp["name"]] = inp.get("value", "")
            fields["__EVENTTARGET"] = m.group("target")
            fields["__EVENTARGUMENT"] = m.group("arg")
            return fields
    return None

p = urlparse(BASE_URL)
origin = f"{p.scheme}://{p.netloc}"
while True:
    fields = next_postback(soup)
    if fields is None:
        break  # no sequential next page -> pagination complete
    resp = session.post(
        BASE_URL, data=fields, timeout=30,
        headers={"Origin": origin, "Referer": BASE_URL})
    resp.raise_for_status()
    soup = load(resp.text)
    grid = soup.find(id=GRID_ID)
    listings.extend(parse_rows(grid, BASE_URL))

print(f"Collected {len(listings)} vacancies across all pages")

def scrape_detail(session, listing):
    resp = session.get(listing["listing_url"], timeout=30)
    if resp.status_code in (404, 410):
        return {**listing, "removed": True}   # vacancy withdrawn
    resp.raise_for_status()
    soup = load(resp.text)

    # Confirm the detail form's VId matches the listing before trusting the page
    form = soup.select_one("form#form1")
    action = urljoin(listing["listing_url"], form.get("action", "")) if form else ""
    if vid_of(action) != listing["external_id"]:
        raise ValueError("Detail form VId does not match the listing VId")

    # Fields live in an Advorto definition list (dt label -> dd value)
    fields = {}
    for dt in soup.select("dl.AdvortoDefinitionList > dt"):
        dd = dt.find_next_sibling("dd")
        if dd is not None:
            fields[clean(dt.get_text())] = dd

    title = clean(fields["Job title"].get_text()) if "Job title" in fields else listing["title"]
    if "Description" in fields:
        description = fields["Description"].decode_contents().strip()
    else:
        # Fall back to the widest definition-list cell with substantive text
        description = next(
            (dd.decode_contents().strip()
             for dd in soup.select(
                 ".vacancy-information-fields dd.AdvortoDefinitionListElementWide")
             if len(dd.get_text(strip=True)) >= 80),
            "")

    posted = re.search(r'"datePosted"\s*:\s*"([^"]+)"', resp.text)
    closes = re.search(r'"validThrough"\s*:\s*"([^"]+)"', resp.text)
    return {
        **listing,
        "title": title,
        "description_html": description,
        "contract_type": clean(fields["Contract type"].get_text()) if "Contract type" in fields else "",
        "hours": clean(fields["Hours"].get_text()) if "Hours" in fields else "",
        "posted_at": posted.group(1) if posted else None,
        "closes_at": closes.group(1) if closes else listing.get("closing_date"),
    }

details = [scrape_detail(session, job) for job in listings]
print(f"Scraped {len(details)} Kallidus Recruit job details")