Leaderboard

openlithohub.leaderboard.schema

Pydantic schemas for leaderboard entries and submissions.

ProcessNode

Bases: str, Enum

Supported process technology nodes.

Source code in src/openlithohub/leaderboard/schema.py

class ProcessNode(str, Enum):
    """Supported process technology nodes."""

    N45 = "45nm"
    N28 = "28nm"
    N7 = "7nm"
    N5 = "5nm"
    N3_EUV = "3nm-euv"
    N2_EUV = "2nm-euv"

MaskTopology

Bases: str, Enum

Mask shape classification.

Source code in src/openlithohub/leaderboard/schema.py

class MaskTopology(str, Enum):
    """Mask shape classification."""

    MANHATTAN = "manhattan"
    CURVILINEAR = "curvilinear"

BenchmarkResult

Bases: BaseModel

A single benchmark submission for the leaderboard.

Source code in src/openlithohub/leaderboard/schema.py

class BenchmarkResult(BaseModel):
    """A single benchmark submission for the leaderboard."""

    model_name: str = Field(..., description="Name of the evaluated model")
    dataset: str = Field(..., description="Dataset used (lithobench/lithosim)")
    process_node: ProcessNode
    mask_topology: MaskTopology

    epe_mean_nm: float = Field(..., ge=0, description="Mean EPE in nanometers")
    epe_max_nm: float = Field(..., ge=0)
    pvband_nm: float | None = Field(None, ge=0)
    mrc_violation_rate: float | None = Field(None, ge=0, le=1)
    drc_pass: bool | None = None
    shot_count: int | None = Field(None, ge=0)
    stochastic_robustness: float | None = Field(None, ge=0, le=1)

    submitted_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
    submission_id: str | None = Field(None, description="Auto-assigned submission ID (read-only).")
    paper_url: str | None = None
    code_url: str | None = None
    notes: str | None = None

openlithohub.leaderboard.tracker

SOTA tracking and leaderboard management.

LeaderboardStore

JSON file-backed leaderboard data store.

Source code in src/openlithohub/leaderboard/tracker.py

class LeaderboardStore:
    """JSON file-backed leaderboard data store."""

    def __init__(self, path: Path | None = None) -> None:
        if path is not None:
            self._path = Path(path)
        else:
            env_path = os.environ.get("OPENLITHOHUB_LEADERBOARD_PATH")
            if env_path:
                self._path = Path(env_path)
            else:
                self._path = _DEFAULT_LEADERBOARD_DIR / _LEADERBOARD_FILENAME

    @property
    def path(self) -> Path:
        return self._path

    def _read_entries(self) -> list[dict[str, Any]]:
        if not self._path.exists():
            return []
        text = self._path.read_text(encoding="utf-8")
        data = json.loads(text)
        return data.get("entries", [])

    def _write_entries(self, entries: list[dict[str, Any]]) -> None:
        self._path.parent.mkdir(parents=True, exist_ok=True)
        tmp_path = self._path.with_suffix(".tmp")
        payload = json.dumps({"entries": entries}, indent=2, default=str)
        tmp_path.write_text(payload, encoding="utf-8")
        tmp_path.replace(self._path)

    def submit(self, result: BenchmarkResult) -> str:
        entries = self._read_entries()
        submission_id = _generate_id(result.model_name)
        entry = result.model_dump(mode="json")
        entry["_submission_id"] = submission_id
        entries.append(entry)
        self._write_entries(entries)
        return submission_id

    def query(
        self,
        dataset: str | None = None,
        process_node: str | None = None,
    ) -> list[BenchmarkResult]:
        entries = self._read_entries()
        results: list[BenchmarkResult] = []
        for entry in entries:
            entry_copy = {k: v for k, v in entry.items() if not k.startswith("_")}
            r = BenchmarkResult.model_validate(entry_copy)
            if dataset and r.dataset != dataset:
                continue
            if process_node and r.process_node.value != process_node:
                continue
            results.append(r)
        results.sort(key=lambda r: r.epe_mean_nm)
        return results

submit_result(result, *, store=None)

Submit a benchmark result to the leaderboard.

Parameters:

Name	Type	Description	Default
result	BenchmarkResult	Validated BenchmarkResult entry.	required
store	LeaderboardStore | None	Optional explicit store (for testing). Uses default if None.	None

Returns:

Type	Description
str	Submission ID for tracking.

Source code in src/openlithohub/leaderboard/tracker.py

def submit_result(result: BenchmarkResult, *, store: LeaderboardStore | None = None) -> str:
    """Submit a benchmark result to the leaderboard.

    Args:
        result: Validated BenchmarkResult entry.
        store: Optional explicit store (for testing). Uses default if None.

    Returns:
        Submission ID for tracking.
    """
    s = store or _get_store()
    return s.submit(result)

get_leaderboard(dataset=None, process_node=None, *, store=None)

Retrieve current leaderboard entries with optional filtering.

Parameters:

Name	Type	Description	Default
dataset	str | None	Filter by dataset name.	None
process_node	str | None	Filter by process node.	None
store	LeaderboardStore | None	Optional explicit store (for testing). Uses default if None.	None

Returns:

Type	Description
list[BenchmarkResult]	Sorted list of BenchmarkResult entries (by EPE ascending).

Source code in src/openlithohub/leaderboard/tracker.py

def get_leaderboard(
    dataset: str | None = None,
    process_node: str | None = None,
    *,
    store: LeaderboardStore | None = None,
) -> list[BenchmarkResult]:
    """Retrieve current leaderboard entries with optional filtering.

    Args:
        dataset: Filter by dataset name.
        process_node: Filter by process node.
        store: Optional explicit store (for testing). Uses default if None.

    Returns:
        Sorted list of BenchmarkResult entries (by EPE ascending).
    """
    s = store or _get_store()
    return s.query(dataset=dataset, process_node=process_node)

openlithohub.leaderboard.data_engine

Dataset generation pipelines for EDA foundation model pre-training.

generate_paired_dataset(source_layouts_dir, output_dir, *, num_samples=1000, process_node='3nm-euv')

Generate paired (layout, mask, resist) datasets for foundation model training.

Parameters:

Name	Type	Description	Default
source_layouts_dir	str | Path	Directory containing source design layouts.	required
output_dir	str | Path	Output directory for generated dataset.	required
num_samples	int	Number of samples to generate.	1000
process_node	str	Target process node for simulation parameters.	'3nm-euv'

Source code in src/openlithohub/leaderboard/data_engine.py

def generate_paired_dataset(
    source_layouts_dir: str | Path,
    output_dir: str | Path,
    *,
    num_samples: int = 1000,
    process_node: str = "3nm-euv",
) -> None:
    """Generate paired (layout, mask, resist) datasets for foundation model training.

    Args:
        source_layouts_dir: Directory containing source design layouts.
        output_dir: Output directory for generated dataset.
        num_samples: Number of samples to generate.
        process_node: Target process node for simulation parameters.
    """
    raise NotImplementedError(
        "Data engine not yet implemented. "
        "Planned: load source layouts, run optimization models, "
        "simulate resist contours, package as layout-mask-resist triples "
        "with compliance labels and process metadata."
    )