Adding cleavage benchmarks

docs/source/user_guide/benchmarks/surfaces.rst

@@ -145,3 +145,56 @@ Reference data:
   * S. P. Ong, W. D. Richards, A. Jain, G. Hautier, M. Kocher, S. Cholia, D. Gunter, V. Chevrier, K. A. Persson, G. Ceder, "Python Materials Genomics (pymatgen): A Robust, Open-Source Python Library for Materials Analysis," Comput. Mater. Sci., 2013, 68, 314–319. https://doi.org/10.1016/j.commatsci.2012.10.028
 
 * Tran et al. relaxed the slabs using spin-polarized PBE calculations performed in VASP, with a cutoff energy of 400 eV.
+
+Cleavage Energy
+===============
+
+Summary
+-------
+
+Performance in predicting cleavage energies for 36,718 surface configurations
+across a wide range of materials and Miller indices.
+
+Metrics
+-------
+
+1. Cleavage energy MAE
+
+Accuracy of cleavage energy predictions compared to DFT reference values.
+
+For each surface, the cleavage energy is calculated as
+``(E_slab - thickness_ratio * E_bulk) / (2 * A)``, where ``E_slab`` and
+``E_bulk`` are single-point energies of the slab and the lattice-matched bulk
+unit cell, ``thickness_ratio`` is the number of bulk unit cells in the slab
+thickness, and ``A`` is the surface area. Results are reported in meV/A^2.
+The mean absolute error is computed over all 36,718 surfaces.
+
+2. Cleavage energy RMSE
+
+Root mean squared error of cleavage energy predictions across all surfaces.
+
+Computational cost
+------------------
+
+Medium: benchmark involves only single-point calculations, but for 36,718 slab-bulk pairs. Takes roughly 5-20 minutes on GPU or a few hours on CPUs.
+
+Data availability
+-----------------
+
+Input data:
+
+* Surface configurations were obtained from the Materials Project, covering
+  3,699 unique bulk materials with multiple Miller indices and terminations
+  per material. The original unfiltered data source is available at
+  Zenodo (DOI: 10.5281/zenodo.10381505).
+
+Reference data:
+
+* DFT cleavage energies calculated using PBE functional.
+
+Publication:
+
+* A. Mehdizadeh and P. Schindler, "Surface stability modeling with universal
+  machine learning interatomic potentials: a comprehensive cleavage energy
+  benchmarking study," Mach. Learn.: Sci. Technol., 2025.
+  https://iopscience.iop.org/article/10.1088/3050-287X/ae1408

ml_peg/analysis/surfaces/cleavage_energy/analyse_cleavage_energy.py

@@ -0,0 +1,225 @@
+"""Analyse cleavage energy benchmark."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import numpy as np
+import pytest
+
+from ml_peg.analysis.utils.decorators import build_table, plot_density_scatter
+from ml_peg.analysis.utils.utils import (
+    load_metrics_config,
+    mae,
+    write_density_trajectories,
+)
+from ml_peg.app import APP_ROOT
+from ml_peg.calcs import CALCS_ROOT
+from ml_peg.models.get_models import get_model_names
+from ml_peg.models.models import current_models
+
+MODELS = get_model_names(current_models)
+CALC_PATH = CALCS_ROOT / "surfaces" / "cleavage_energy" / "outputs"
+OUT_PATH = APP_ROOT / "data" / "surfaces" / "cleavage_energy"
+
+METRICS_CONFIG_PATH = Path(__file__).with_name("metrics.yml")
+DEFAULT_THRESHOLDS, DEFAULT_TOOLTIPS, DEFAULT_WEIGHTS = load_metrics_config(
+    METRICS_CONFIG_PATH
+)
+
+EV_TO_MEV = 1000.0
+
+
+def compute_cleavage_energy(
+    slab_energy: float,
+    bulk_energy: float,
+    thickness_ratio: float,
+    area_slab: float,
+) -> float:
+    """
+    Compute cleavage energy from slab and bulk energies.
+
+    Parameters
+    ----------
+    slab_energy
+        Total energy of the slab.
+    bulk_energy
+        Total energy of the lattice-matched bulk unit cell.
+    thickness_ratio
+        Number of bulk unit cells in the slab thickness.
+    area_slab
+        Surface area of the slab in Angstrom^2.
+
+    Returns
+    -------
+    float
+        Cleavage energy in eV/Angstrom^2.
+    """
+    return (slab_energy - thickness_ratio * bulk_energy) / (2 * area_slab)
+
+
+@pytest.fixture
+def cleavage_energies() -> dict[str, dict[str, list]]:
+    """
+    Get cleavage energies for all systems in meV/A^2.
+
+    Also saves per-system errors for the distribution plot.
+
+    Returns
+    -------
+    dict[str, dict[str, list]]
+        Dictionary of model names to ``{"ref": [...], "pred": [...]}`` in meV/A^2.
+    """
+    results = {mlip: {"ref": [], "pred": []} for mlip in MODELS}
+    ref_stored = False
+    stored_ref = []
+
+    for model_name in MODELS:
+        model_dir = CALC_PATH / model_name
+        if not model_dir.exists():
+            continue
+
+        model_pred = []
+        model_ref = []
+
+        for xyz_file in sorted(model_dir.glob("*.xyz"), key=lambda p: int(p.stem)):
+            slab = read(xyz_file)
+
+            pred_ce = (
+                compute_cleavage_energy(
+                    slab.info["slab_energy"],
+                    slab.info["bulk_energy"],
+                    slab.info["thickness_ratio"],
+                    slab.info["area_slab"],
+                )
+                * EV_TO_MEV
+            )
+            model_pred.append(pred_ce)
+
+            if not ref_stored:
+                model_ref.append(slab.info["ref_cleavage_energy"] * EV_TO_MEV)
+
+        if model_pred:
+            results[model_name]["pred"] = model_pred
+            if not ref_stored:
+                stored_ref = model_ref
+            results[model_name]["ref"] = stored_ref
+            ref_stored = True
+
+    return results
+
+@pytest.fixture
+@plot_density_scatter(
+    filename=OUT_PATH / "figure_cleavage_energies.json",
+    title="Cleavage Energies",
+    x_label="Predicted cleavage energy / meV/Ų",
+    y_label="Reference cleavage energy / meV/Ų",
+)
+def cleavage_density(
+    cleavage_energies: dict[str, dict[str, list]],
+) -> dict[str, dict]:
+    """
+    Build density scatter inputs and write density trajectories.
+
+    Parameters
+    ----------
+    cleavage_energies
+        Reference and predicted cleavage energies per model.
+
+    Returns
+    -------
+    dict[str, dict]
+        Mapping of model names to density-plot payloads.
+    """
+    label_list = [
+        f.stem
+        for f in sorted(
+            (CALC_PATH / MODELS[0]).glob("*.xyz"), key=lambda p: int(p.stem)
+        )
+    ]
+
+    density_inputs: dict[str, dict] = {}
+    for model_name in MODELS:
+        preds = cleavage_energies[model_name]["pred"]
+        refs = cleavage_energies[model_name]["ref"]
+        density_inputs[model_name] = {
+            "ref": refs,
+            "pred": preds,
+            "meta": {"system_count": len([v for v in preds if v is not None])},
+        }
+        if preds:
+            write_density_trajectories(
+                labels_list=label_list,
+                ref_vals=refs,
+                pred_vals=preds,
+                struct_dir=CALC_PATH / model_name,
+                traj_dir=OUT_PATH / model_name / "density_traj",
+                struct_filename_builder=lambda label: f"{label}.xyz",
+            )
+    return density_inputs
+@pytest.fixture
+def cleavage_mae(cleavage_energies: dict[str, list]) -> dict[str, float]:
+    """
+    Get mean absolute error for cleavage energies.
+
+    Parameters
+    ----------
+    cleavage_energies
+        Dictionary of reference and predicted cleavage energies.
+
+    Returns
+    -------
+    dict[str, float]
+        MAE for each model.
+    """
+    results = {}
+    for model_name in MODELS:
+        ref_vals = cleavage_energies[model_name]["ref"]
+        pred_vals = cleavage_energies[model_name]["pred"]
+        if pred_vals:
+            results[model_name] = mae(ref_vals, pred_vals)
+        else:
+            results[model_name] = None
+    return results
+
+
+@pytest.fixture
+@build_table(
+    filename=OUT_PATH / "cleavage_energy_metrics_table.json",
+    metric_tooltips=DEFAULT_TOOLTIPS,
+    thresholds=DEFAULT_THRESHOLDS,
+    weights=DEFAULT_WEIGHTS,
+)
+def metrics(cleavage_mae: dict[str, float]) -> dict[str, dict]:
+    """
+    Get all cleavage energy metrics.
+
+    Parameters
+    ----------
+    cleavage_mae
+        Mean absolute errors for all models.
+
+    Returns
+    -------
+    dict[str, dict]
+        Metric names and values for all models.
+    """
+    return {"MAE": cleavage_mae}
+
+
+def test_cleavage_energy(
+    metrics: dict[str, dict],
+    cleavage_density: dict[str, dict],
+) -> None:
+    """
+    Run cleavage energy analysis.
+
+    Parameters
+    ----------
+    metrics
+        All cleavage energy metrics.
+    cleavage_density
+        Density-scatter inputs for all models (drives saved plots).
+    """
+    return

ml_peg/analysis/surfaces/cleavage_energy/metrics.yml

@@ -0,0 +1,7 @@
+metrics:
+  MAE:
+    good: 0.0
+    bad: 10.0
+    unit: meV/A^2
+    tooltip: Mean Absolute Error of cleavage energies
+    level_of_theory: PBE

ml_peg/app/surfaces/cleavage_energy/app_cleavage_energy.py

@@ -0,0 +1,95 @@
+"""Run cleavage energy app."""
+
+from __future__ import annotations
+
+from dash import Dash
+from dash.html import Div
+
+from ml_peg.app import APP_ROOT
+from ml_peg.app.base_app import BaseApp
+from ml_peg.app.utils.build_callbacks import plot_from_table_cell, struct_from_scatter
+from ml_peg.app.utils.load import collect_traj_assets, read_density_plot_for_model
+from ml_peg.models.get_models import get_model_names
+from ml_peg.models.models import current_models
+
+MODELS = get_model_names(current_models)
+BENCHMARK_NAME = "Cleavage Energy"
+DOCS_URL = (
+    "https://ddmms.github.io/ml-peg/user_guide/benchmarks/surfaces.html#cleavage-energy"
+)
+DATA_PATH = APP_ROOT / "data" / "surfaces" / "cleavage_energy"
+
+
+class CleavageEnergyApp(BaseApp):
+    """Cleavage energy benchmark app layout and callbacks."""
+
+    def register_callbacks(self) -> None:
+        """Register callbacks to app."""
+        density_plots: dict[str, dict] = {}
+        for model in MODELS:
+            density_graph = read_density_plot_for_model(
+                filename=DATA_PATH / "figure_cleavage_energies.json",
+                model=model,
+                id=f"{BENCHMARK_NAME}-{model}-density",
+            )
+            if density_graph is not None:
+                density_plots[model] = {"MAE": density_graph}
+
+        plot_from_table_cell(
+            table_id=self.table_id,
+            plot_id=f"{BENCHMARK_NAME}-figure-placeholder",
+            cell_to_plot=density_plots,
+        )
+
+        struct_trajs = collect_traj_assets(
+            data_path=DATA_PATH,
+            assets_prefix="assets/surfaces/cleavage_energy",
+            models=MODELS,
+            traj_dirname="density_traj",
+            suffix=".extxyz",
+        )
+        for model in struct_trajs:
+            struct_from_scatter(
+                scatter_id=f"{BENCHMARK_NAME}-{model}-density",
+                struct_id=f"{BENCHMARK_NAME}-struct-placeholder",
+                structs=struct_trajs[model],
+                mode="traj",
+            )
+
+
+def get_app() -> CleavageEnergyApp:
+    """
+    Get cleavage energy benchmark app layout and callback registration.
+
+    Returns
+    -------
+    CleavageEnergyApp
+        Benchmark layout and callback registration.
+    """
+
+    return CleavageEnergyApp(
+        name=BENCHMARK_NAME,
+        description=(
+            "Performance in predicting cleavage energies for "
+            "36,718 surface configurations."
+        ),
+        docs_url=DOCS_URL,
+        table_path=DATA_PATH / "cleavage_energy_metrics_table.json",
+        extra_components=[
+            Div(id=f"{BENCHMARK_NAME}-figure-placeholder"),
+            Div(id=f"{BENCHMARK_NAME}-struct-placeholder"),
+        ],
+    )
+
+
+if __name__ == "__main__":
+    full_app = Dash(
+        __name__,
+        assets_folder=DATA_PATH.parent.parent,
+    )
+
+    cleavage_energy_app = get_app()
+    full_app.layout = cleavage_energy_app.layout
+    cleavage_energy_app.register_callbacks()
+
+    full_app.run(port=8056, debug=True)