Skip to content

Commit 95324de

Browse files
author
sfluegel
committed
fix eval notebook
1 parent 97a134f commit 95324de

File tree

2 files changed

+52
-12
lines changed

2 files changed

+52
-12
lines changed

‎chebai/result/classification.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
from chebai.models import ChebaiBaseNet
1717
from chebai.models.electra import Electra
1818
from chebai.preprocessing.datasets import XYBaseDataModule
19-
from utils import *
19+
from chebai.result.utils import *
2020

2121

2222
def visualise_f1(logs_path):

‎tutorials/eval_model_basic.ipynb‎

Lines changed: 51 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2,20 +2,32 @@
22
"cells": [
33
{
44
"cell_type": "code",
5-
"execution_count": null,
5+
"execution_count": 1,
66
"id": "initial_id",
77
"metadata": {
8-
"collapsed": true
8+
"collapsed": true,
9+
"ExecuteTime": {
10+
"end_time": "2024-04-02T13:47:31.150545Z",
11+
"start_time": "2024-04-02T13:47:27.181585Z"
12+
}
913
},
10-
"outputs": [],
14+
"outputs": [
15+
{
16+
"name": "stdout",
17+
"output_type": "stream",
18+
"text": [
19+
"cuda:0\n"
20+
]
21+
}
22+
],
1123
"source": [
1224
"import pandas as pd\n",
1325
"\n",
14-
"from chebai.result.classification import (\n",
26+
"from chebai.result.utils import (\n",
1527
" evaluate_model,\n",
1628
" load_results_from_buffer,\n",
17-
" print_metrics,\n",
1829
")\n",
30+
"from chebai.result.classification import print_metrics\n",
1931
"from chebai.models.electra import Electra\n",
2032
"from chebai.preprocessing.datasets.chebi import ChEBIOver50, ChEBIOver100\n",
2133
"import os\n",
@@ -29,14 +41,19 @@
2941
},
3042
{
3143
"cell_type": "code",
32-
"execution_count": null,
44+
"execution_count": 2,
3345
"id": "bdb5fc6919cf72be",
3446
"metadata": {
35-
"collapsed": false
47+
"collapsed": false,
48+
"ExecuteTime": {
49+
"end_time": "2024-04-02T13:47:35.484307Z",
50+
"start_time": "2024-04-02T13:47:35.477111Z"
51+
}
3652
},
3753
"outputs": [],
3854
"source": [
39-
"checkpoint_name = \"best_epoch=99_val_loss=0.0096_val_macro-f1=0.5358_val_micro-f1=0.8968\"\n",
55+
"# specify the checkpoint name\n",
56+
"checkpoint_name = \"my_trained_model\"\n",
4057
"checkpoint_path = os.path.join(\"logs\", f\"{checkpoint_name}.ckpt\")\n",
4158
"kind = \"test\" # replace with \"train\" / \"validation\" to run on train / validation sets\n",
4259
"buffer_dir = os.path.join(\"results_buffer\", checkpoint_name, kind)\n",
@@ -49,12 +66,35 @@
4966
},
5067
{
5168
"cell_type": "code",
52-
"execution_count": null,
69+
"execution_count": 3,
5370
"id": "fa1276b47def696c",
5471
"metadata": {
55-
"collapsed": false
72+
"collapsed": false,
73+
"ExecuteTime": {
74+
"end_time": "2024-04-02T13:47:38.418564Z",
75+
"start_time": "2024-04-02T13:47:37.861168Z"
76+
}
5677
},
57-
"outputs": [],
78+
"outputs": [
79+
{
80+
"ename": "FileNotFoundError",
81+
"evalue": "[Errno 2] No such file or directory: 'C:/Users/Simon Flügel/Desktop/chebai/tutorials/logs/best_epoch=99_val_loss=0.0096_val_macro-f1=0.5358_val_micro-f1=0.8968.ckpt'",
82+
"output_type": "error",
83+
"traceback": [
84+
"\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
85+
"\u001B[1;31mFileNotFoundError\u001B[0m Traceback (most recent call last)",
86+
"Cell \u001B[1;32mIn[3], line 2\u001B[0m\n\u001B[0;32m 1\u001B[0m \u001B[38;5;66;03m# evaluates model, stores results in buffer_dir\u001B[39;00m\n\u001B[1;32m----> 2\u001B[0m model \u001B[38;5;241m=\u001B[39m \u001B[43mmodel_class\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mload_from_checkpoint\u001B[49m\u001B[43m(\u001B[49m\u001B[43mcheckpoint_path\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m 3\u001B[0m preds, labels \u001B[38;5;241m=\u001B[39m evaluate_model(\n\u001B[0;32m 4\u001B[0m model,\n\u001B[0;32m 5\u001B[0m data_module,\n\u001B[1;32m (...)\u001B[0m\n\u001B[0;32m 8\u001B[0m batch_size\u001B[38;5;241m=\u001B[39m\u001B[38;5;241m10\u001B[39m,\n\u001B[0;32m 9\u001B[0m )\n",
87+
"File \u001B[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\lightning\\pytorch\\core\\module.py:1552\u001B[0m, in \u001B[0;36mLightningModule.load_from_checkpoint\u001B[1;34m(cls, checkpoint_path, map_location, hparams_file, strict, **kwargs)\u001B[0m\n\u001B[0;32m 1471\u001B[0m \u001B[38;5;129m@_restricted_classmethod\u001B[39m\n\u001B[0;32m 1472\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mload_from_checkpoint\u001B[39m(\n\u001B[0;32m 1473\u001B[0m \u001B[38;5;28mcls\u001B[39m,\n\u001B[1;32m (...)\u001B[0m\n\u001B[0;32m 1478\u001B[0m \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mkwargs: Any,\n\u001B[0;32m 1479\u001B[0m ) \u001B[38;5;241m-\u001B[39m\u001B[38;5;241m>\u001B[39m Self:\n\u001B[0;32m 1480\u001B[0m \u001B[38;5;250m \u001B[39m\u001B[38;5;124mr\u001B[39m\u001B[38;5;124;03m\"\"\"Primary way of loading a model from a checkpoint. When Lightning saves a checkpoint it stores the arguments\u001B[39;00m\n\u001B[0;32m 1481\u001B[0m \u001B[38;5;124;03m passed to ``__init__`` in the checkpoint under ``\"hyper_parameters\"``.\u001B[39;00m\n\u001B[0;32m 1482\u001B[0m \n\u001B[1;32m (...)\u001B[0m\n\u001B[0;32m 1550\u001B[0m \n\u001B[0;32m 1551\u001B[0m \u001B[38;5;124;03m \"\"\"\u001B[39;00m\n\u001B[1;32m-> 1552\u001B[0m loaded \u001B[38;5;241m=\u001B[39m \u001B[43m_load_from_checkpoint\u001B[49m\u001B[43m(\u001B[49m\n\u001B[0;32m 1553\u001B[0m \u001B[43m \u001B[49m\u001B[38;5;28;43mcls\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;66;43;03m# type: ignore[arg-type]\u001B[39;49;00m\n\u001B[0;32m 1554\u001B[0m \u001B[43m \u001B[49m\u001B[43mcheckpoint_path\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 1555\u001B[0m \u001B[43m \u001B[49m\u001B[43mmap_location\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 1556\u001B[0m \u001B[43m \u001B[49m\u001B[43mhparams_file\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 1557\u001B[0m \u001B[43m \u001B[49m\u001B[43mstrict\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 1558\u001B[0m \u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mkwargs\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 1559\u001B[0m \u001B[43m \u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m 1560\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m cast(Self, loaded)\n",
88+
"File \u001B[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\lightning\\pytorch\\core\\saving.py:61\u001B[0m, in \u001B[0;36m_load_from_checkpoint\u001B[1;34m(cls, checkpoint_path, map_location, hparams_file, strict, **kwargs)\u001B[0m\n\u001B[0;32m 59\u001B[0m map_location \u001B[38;5;241m=\u001B[39m map_location \u001B[38;5;129;01mor\u001B[39;00m _default_map_location\n\u001B[0;32m 60\u001B[0m \u001B[38;5;28;01mwith\u001B[39;00m pl_legacy_patch():\n\u001B[1;32m---> 61\u001B[0m checkpoint \u001B[38;5;241m=\u001B[39m \u001B[43mpl_load\u001B[49m\u001B[43m(\u001B[49m\u001B[43mcheckpoint_path\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mmap_location\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mmap_location\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m 63\u001B[0m \u001B[38;5;66;03m# convert legacy checkpoints to the new format\u001B[39;00m\n\u001B[0;32m 64\u001B[0m checkpoint \u001B[38;5;241m=\u001B[39m _pl_migrate_checkpoint(\n\u001B[0;32m 65\u001B[0m checkpoint, checkpoint_path\u001B[38;5;241m=\u001B[39m(checkpoint_path \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28misinstance\u001B[39m(checkpoint_path, (\u001B[38;5;28mstr\u001B[39m, Path)) \u001B[38;5;28;01melse\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m)\n\u001B[0;32m 66\u001B[0m )\n",
89+
"File \u001B[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\lightning\\fabric\\utilities\\cloud_io.py:54\u001B[0m, in \u001B[0;36m_load\u001B[1;34m(path_or_url, map_location)\u001B[0m\n\u001B[0;32m 49\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m torch\u001B[38;5;241m.\u001B[39mhub\u001B[38;5;241m.\u001B[39mload_state_dict_from_url(\n\u001B[0;32m 50\u001B[0m \u001B[38;5;28mstr\u001B[39m(path_or_url),\n\u001B[0;32m 51\u001B[0m map_location\u001B[38;5;241m=\u001B[39mmap_location, \u001B[38;5;66;03m# type: ignore[arg-type]\u001B[39;00m\n\u001B[0;32m 52\u001B[0m )\n\u001B[0;32m 53\u001B[0m fs \u001B[38;5;241m=\u001B[39m get_filesystem(path_or_url)\n\u001B[1;32m---> 54\u001B[0m \u001B[38;5;28;01mwith\u001B[39;00m \u001B[43mfs\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mopen\u001B[49m\u001B[43m(\u001B[49m\u001B[43mpath_or_url\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43mrb\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m)\u001B[49m \u001B[38;5;28;01mas\u001B[39;00m f:\n\u001B[0;32m 55\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m torch\u001B[38;5;241m.\u001B[39mload(f, map_location\u001B[38;5;241m=\u001B[39mmap_location)\n",
90+
"File \u001B[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\fsspec\\spec.py:1307\u001B[0m, in \u001B[0;36mAbstractFileSystem.open\u001B[1;34m(self, path, mode, block_size, cache_options, compression, **kwargs)\u001B[0m\n\u001B[0;32m 1305\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[0;32m 1306\u001B[0m ac \u001B[38;5;241m=\u001B[39m kwargs\u001B[38;5;241m.\u001B[39mpop(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mautocommit\u001B[39m\u001B[38;5;124m\"\u001B[39m, \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_intrans)\n\u001B[1;32m-> 1307\u001B[0m f \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_open\u001B[49m\u001B[43m(\u001B[49m\n\u001B[0;32m 1308\u001B[0m \u001B[43m \u001B[49m\u001B[43mpath\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 1309\u001B[0m \u001B[43m \u001B[49m\u001B[43mmode\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mmode\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 1310\u001B[0m \u001B[43m \u001B[49m\u001B[43mblock_size\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mblock_size\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 1311\u001B[0m \u001B[43m \u001B[49m\u001B[43mautocommit\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mac\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 1312\u001B[0m \u001B[43m \u001B[49m\u001B[43mcache_options\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mcache_options\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 1313\u001B[0m \u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mkwargs\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 1314\u001B[0m \u001B[43m \u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m 1315\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m compression \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[0;32m 1316\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mfsspec\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mcompression\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m compr\n",
91+
"File \u001B[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\fsspec\\implementations\\local.py:180\u001B[0m, in \u001B[0;36mLocalFileSystem._open\u001B[1;34m(self, path, mode, block_size, **kwargs)\u001B[0m\n\u001B[0;32m 178\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mauto_mkdir \u001B[38;5;129;01mand\u001B[39;00m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mw\u001B[39m\u001B[38;5;124m\"\u001B[39m \u001B[38;5;129;01min\u001B[39;00m mode:\n\u001B[0;32m 179\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mmakedirs(\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_parent(path), exist_ok\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mTrue\u001B[39;00m)\n\u001B[1;32m--> 180\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[43mLocalFileOpener\u001B[49m\u001B[43m(\u001B[49m\u001B[43mpath\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mmode\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mfs\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mkwargs\u001B[49m\u001B[43m)\u001B[49m\n",
92+
"File \u001B[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\fsspec\\implementations\\local.py:302\u001B[0m, in \u001B[0;36mLocalFileOpener.__init__\u001B[1;34m(self, path, mode, autocommit, fs, compression, **kwargs)\u001B[0m\n\u001B[0;32m 300\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mcompression \u001B[38;5;241m=\u001B[39m get_compression(path, compression)\n\u001B[0;32m 301\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mblocksize \u001B[38;5;241m=\u001B[39m io\u001B[38;5;241m.\u001B[39mDEFAULT_BUFFER_SIZE\n\u001B[1;32m--> 302\u001B[0m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_open\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\n",
93+
"File \u001B[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\fsspec\\implementations\\local.py:307\u001B[0m, in \u001B[0;36mLocalFileOpener._open\u001B[1;34m(self)\u001B[0m\n\u001B[0;32m 305\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mf \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m \u001B[38;5;129;01mor\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mf\u001B[38;5;241m.\u001B[39mclosed:\n\u001B[0;32m 306\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mautocommit \u001B[38;5;129;01mor\u001B[39;00m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mw\u001B[39m\u001B[38;5;124m\"\u001B[39m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;129;01min\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mmode:\n\u001B[1;32m--> 307\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mf \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mopen\u001B[39;49m\u001B[43m(\u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mpath\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mmode\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mmode\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m 308\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mcompression:\n\u001B[0;32m 309\u001B[0m compress \u001B[38;5;241m=\u001B[39m compr[\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mcompression]\n",
94+
"\u001B[1;31mFileNotFoundError\u001B[0m: [Errno 2] No such file or directory: 'C:/Users/Simon Flügel/Desktop/chebai/tutorials/logs/best_epoch=99_val_loss=0.0096_val_macro-f1=0.5358_val_micro-f1=0.8968.ckpt'"
95+
]
96+
}
97+
],
5898
"source": [
5999
"# evaluates model, stores results in buffer_dir\n",
60100
"model = model_class.load_from_checkpoint(checkpoint_path)\n",

0 commit comments

Comments
 (0)