From 415ff778f8ebddc15e43d5a6fdd04a8c6ff65408 Mon Sep 17 00:00:00 2001 From: Tanmayi Bondu Date: Thu, 24 Jul 2025 23:18:09 -0500 Subject: [PATCH 1/3] Add length of stay task and dataset --- .../tasks/length_of_stay/fake_los_data.csv | 6 + .../contrib/tasks/length_of_stay/fakedata.py | 25 ++ .../tasks/length_of_stay/lengthofstay.yaml | 11 + .../length_of_stay/lengthofstay.yaml.save | 17 + .../tasks/length_of_stay/lengthofstay1.yaml | 17 + .../contrib/tasks/length_of_stay/task.py | 339 ++++++++++++++++++ 6 files changed, 415 insertions(+) create mode 100644 pyhealth/pyhealth/contrib/tasks/length_of_stay/fake_los_data.csv create mode 100644 pyhealth/pyhealth/contrib/tasks/length_of_stay/fakedata.py create mode 100644 pyhealth/pyhealth/contrib/tasks/length_of_stay/lengthofstay.yaml create mode 100644 pyhealth/pyhealth/contrib/tasks/length_of_stay/lengthofstay.yaml.save create mode 100644 pyhealth/pyhealth/contrib/tasks/length_of_stay/lengthofstay1.yaml create mode 100644 pyhealth/pyhealth/contrib/tasks/length_of_stay/task.py diff --git a/pyhealth/pyhealth/contrib/tasks/length_of_stay/fake_los_data.csv b/pyhealth/pyhealth/contrib/tasks/length_of_stay/fake_los_data.csv new file mode 100644 index 000000000..12a107b7b --- /dev/null +++ b/pyhealth/pyhealth/contrib/tasks/length_of_stay/fake_los_data.csv @@ -0,0 +1,6 @@ +patient_id,admission_date,discharge_date +1,2023-01-01,2023-01-05 +2,2023-02-10,2023-02-15 +3,2023-03-20,2023-03-22 +4,2023-04-01,2023-04-10 +5,2023-05-05,2023-05-07 diff --git a/pyhealth/pyhealth/contrib/tasks/length_of_stay/fakedata.py b/pyhealth/pyhealth/contrib/tasks/length_of_stay/fakedata.py new file mode 100644 index 000000000..7f19543cc --- /dev/null +++ b/pyhealth/pyhealth/contrib/tasks/length_of_stay/fakedata.py @@ -0,0 +1,25 @@ +import csv +from datetime import datetime, timedelta + +def create_fake_los_data(filename="fake_los_data.csv"): + # Define headers expected by the LOS task + headers = ['admission_id', 'patient_id', 'admission_time', 'discharge_time'] + + # Prepare some example rows + base_date = datetime(2024, 1, 1, 10, 0) + rows = [ + ['1', '1001', base_date.strftime("%Y-%m-%d %H:%M:%S"), (base_date + timedelta(days=4, hours=2)).strftime("%Y-%m-%d %H:%M:%S")], + ['2', '1002', (base_date + timedelta(days=1)).strftime("%Y-%m-%d %H:%M:%S"), (base_date + timedelta(days=3, hours=5)).strftime("%Y-%m-%d %H:%M:%S")], + ['3', '1003', (base_date + timedelta(days=2)).strftime("%Y-%m-%d %H:%M:%S"), (base_date + timedelta(days=5)).strftime("%Y-%m-%d %H:%M:%S")], + ] + + # Write to CSV + with open(filename, mode='w', newline='') as csvfile: + writer = csv.writer(csvfile) + writer.writerow(headers) + writer.writerows(rows) + + print(f"Created fake LOS data CSV file: {filename}") + +if __name__ == "__main__": + create_fake_los_data() diff --git a/pyhealth/pyhealth/contrib/tasks/length_of_stay/lengthofstay.yaml b/pyhealth/pyhealth/contrib/tasks/length_of_stay/lengthofstay.yaml new file mode 100644 index 000000000..206bbe18e --- /dev/null +++ b/pyhealth/pyhealth/contrib/tasks/length_of_stay/lengthofstay.yaml @@ -0,0 +1,11 @@ +version: "1.4" + +tables: + admissions: + file_path: "fake_los_data.csv" + patient_id: "patient_id" + timestamp: "admission_date" + attributes: + - "patient_id" + - "admission_date" + - "discharge_date" \ No newline at end of file diff --git a/pyhealth/pyhealth/contrib/tasks/length_of_stay/lengthofstay.yaml.save b/pyhealth/pyhealth/contrib/tasks/length_of_stay/lengthofstay.yaml.save new file mode 100644 index 000000000..f79e54316 --- /dev/null +++ b/pyhealth/pyhealth/contrib/tasks/length_of_stay/lengthofstay.yaml.save @@ -0,0 +1,17 @@ +root: ./ # base path for your CSV files + +tables: + - fakedata.csv + +columns: + patient_id: patient_id + admission_time: admission_time + discharge_time: discharge_time + features: + - lab_results + - diagnoses + target: + length_of_stay: length_of_stay_days + + + diff --git a/pyhealth/pyhealth/contrib/tasks/length_of_stay/lengthofstay1.yaml b/pyhealth/pyhealth/contrib/tasks/length_of_stay/lengthofstay1.yaml new file mode 100644 index 000000000..f79e54316 --- /dev/null +++ b/pyhealth/pyhealth/contrib/tasks/length_of_stay/lengthofstay1.yaml @@ -0,0 +1,17 @@ +root: ./ # base path for your CSV files + +tables: + - fakedata.csv + +columns: + patient_id: patient_id + admission_time: admission_time + discharge_time: discharge_time + features: + - lab_results + - diagnoses + target: + length_of_stay: length_of_stay_days + + + diff --git a/pyhealth/pyhealth/contrib/tasks/length_of_stay/task.py b/pyhealth/pyhealth/contrib/tasks/length_of_stay/task.py new file mode 100644 index 000000000..ca667d565 --- /dev/null +++ b/pyhealth/pyhealth/contrib/tasks/length_of_stay/task.py @@ -0,0 +1,339 @@ +""" +Author: Muni Bondu +Description: This task predicts hospital length of stay (LOS) using synthetic admissions data. +This implements a regression task using admission and discharge dates. + +Paper (if applicable): N/A """ + +import os +import pandas as pd +from datetime import datetime +from typing import List, Dict, Any + +from pyhealth.datasets import BaseDataset +from pyhealth.tasks import BaseTask + +class LOSDataset(BaseDataset): + """Length of Stay Dataset compatible with PyHealth BaseDataset. + + Args: + root (str): Root directory containing data files. + dev (bool): If True, load a small sample for development. + + Attributes: + patients (List[Dict]): List of patient data dictionaries. + """ + + def __init__(self, root: str = ".", dev: bool = False): + tables = ["admissions"] # Must match YAML table key + config_path = os.path.join(root, "lengthofstay.yaml") + + # parent class + super().__init__(root=root, tables=tables, config_path=config_path) + self.dev = dev + + # Load and process data + self.patients = self.load_patient_data() + + def load_patient_data(self) -> List[Dict]: + """Load CSV and convert to internal patient data format.""" + try: + # Get the file path from the config + table_config = self.config.tables["admissions"] + csv_filename = table_config.file_path + csv_path = os.path.join(self.root, csv_filename) + + if not os.path.exists(csv_path): + raise FileNotFoundError(f"CSV file not found: {csv_path}") + + print(f"Loading data from: {csv_path}") + df = pd.read_csv(csv_path) + + # Strip whitespace from column names + df.columns = df.columns.str.strip() + + if self.dev: + df = df.sample(n=min(5, len(df)), random_state=42) + print(f"Development mode: Using {len(df)} samples") + + patients = [] + for idx, row in df.iterrows(): + try: + # Parse dates + admission_date = pd.to_datetime(row['admission_date']).date() + discharge_date = pd.to_datetime(row['discharge_date']).date() + + # Calculate length of stay + los = (discharge_date - admission_date).days + if los < 0: + print(f"Warning: Negative LOS for patient {row['patient_id']}, setting to 0") + los = 0 + # Create patient record + patient_data = { + 'patient_id': str(row['patient_id']), + 'admission_date': admission_date, + 'discharge_date': discharge_date, + 'length_of_stay': los, + # Add features for prediction + 'admission_day_of_year': admission_date.timetuple().tm_yday, + 'admission_month': admission_date.month, + 'admission_weekday': admission_date.weekday(), + } + patients.append(patient_data) + except (ValueError, KeyError) as e: + print(f"Error processing row {idx} for patient {row.get('patient_id', 'unknown')}: {e}") + continue + print(f"Successfully loaded {len(patients)} patient records") + return patients + except Exception as e: + print(f"Error loading data: {e}") + return [] + def get_patient_data(self) -> List[Dict]: + """Return patient data for compatibility with PyHealth.""" + return self.patients + + +class LOSTask(BaseTask): + """Length of Stay Prediction Task compatible with PyHealth BaseTask.""" + + def __init__(self, dataset: LOSDataset): + super().__init__() + self.task_name = "length_of_stay" + self.dataset = dataset + + def __call__(self, patient_id: str) -> Dict[str, Any]: + """Process a single patient and return task-specific data. + + This method is required by PyHealth's BaseTask. + Args: + patient_id: The patient ID to process + + Returns: + Dictionary containing patient features and target + """ + # Find the patient in the dataset + patient_data = None + for p in self.dataset.patients: + if p['patient_id'] == patient_id: + patient_data = p + break + + if patient_data is None: + raise ValueError(f"Patient {patient_id} not found in dataset") + + # Return the processed data for this patient + return { + "patient_id": patient_data["patient_id"], + "features": { + "admission_day_of_year": patient_data["admission_day_of_year"], + "admission_month": patient_data["admission_month"], + "admission_weekday": patient_data["admission_weekday"], + }, + "target": patient_data["length_of_stay"], + "metadata": { + "admission_date": patient_data["admission_date"], + "discharge_date": patient_data["discharge_date"], + } + } + + def get_targets(self) -> List[int]: + """Get the length of stay targets for the dataset.""" + return [p['length_of_stay'] for p in self.dataset.patients] + + def get_features(self) -> List[Dict[str, Any]]: + """Extract features to be used for prediction.""" + features = [] + for p in self.dataset.patients: + features.append({ + "patient_id": p["patient_id"], + "admission_day_of_year": p["admission_day_of_year"], + "admission_month": p["admission_month"], + "admission_weekday": p["admission_weekday"], + }) + return features + + def get_patient_splits(self, train_ratio: float = 0.7, val_ratio: float = 0.15): + """Split patients into train/validation/test sets.""" + import random + random.seed(42) + + patients = list(range(len(self.dataset.patients))) + random.shuffle(patients) + + n_train = int(len(patients) * train_ratio) + n_val = int(len(patients) * val_ratio) + + train_idx = patients[:n_train] + val_idx = patients[n_train:n_train + n_val] + test_idx = patients[n_train + n_val:] + + return train_idx, val_idx, test_idx + + +def create_sample_data(): + """Create a sample CSV file for testing.""" + csv_content = """patient_id,admission_date,discharge_date +1,2023-01-01,2023-01-05 +2,2023-02-10,2023-02-15 +3,2023-03-20,2023-03-22 +4,2023-04-01,2023-04-10 +5,2023-05-05,2023-05-07 +6,2023-06-15,2023-06-18 +7,2023-07-20,2023-07-25 +8,2023-08-10,2023-08-12 +9,2023-09-05,2023-09-08 +10,2023-10-12,2023-10-20 +11,2023-11-01,2023-11-03 +12,2023-12-15,2023-12-18 +""" + filename = "fake_los_data.csv" + with open(filename, "w") as f: + f.write(csv_content) + print(f"Created sample data file: {filename}") + + +def run_simple_prediction(): + """Run a simple prediction using scikit-learn.""" + try: + from sklearn.ensemble import RandomForestRegressor + from sklearn.metrics import mean_squared_error, mean_absolute_error + from sklearn.model_selection import train_test_split + import numpy as np + + print("\n" + "="*50) + print("RUNNING SIMPLE PREDICTION MODEL") + print("="*50) + + # Load dataset + dataset = LOSDataset(dev=False) + task = LOSTask(dataset) + + # Get features and targets + features = task.get_features() + targets = task.get_targets() + + if not features or not targets: + print("No data available for prediction") + return + + # Convert to arrays + X = np.array([[f['admission_day_of_year'], f['admission_month'], f['admission_weekday']] + for f in features]) + y = np.array(targets) + + # Split data + X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42) + + # Train model + model = RandomForestRegressor(n_estimators=10, random_state=42) + model.fit(X_train, y_train) + + # Make predictions + y_pred = model.predict(X_test) + + # Calculate metrics + mse = mean_squared_error(y_test, y_pred) + mae = mean_absolute_error(y_test, y_pred) + + print(f"Training samples: {len(X_train)}") + print(f"Test samples: {len(X_test)}") + print(f"Mean Squared Error: {mse:.2f}") + print(f"Mean Absolute Error: {mae:.2f}") + print(f"Actual vs Predicted (first 5 test samples):") + for i in range(min(5, len(y_test))): + print(f" Actual: {y_test[i]}, Predicted: {y_pred[i]:.1f}") + + except ImportError: + print("Scikit-learn not available. Skipping prediction demo.") + except Exception as e: + print(f"Error in prediction: {e}") +def test_task_call(): + """Test the __call__ method of the task.""" + print("\n" + "="*50) + print("TESTING TASK __call__ METHOD") + print("="*50) + try: + dataset = LOSDataset(dev=True) + task = LOSTask(dataset) + + if dataset.patients: + # Test with the first patient + first_patient_id = dataset.patients[0]['patient_id'] + result = task(first_patient_id) + + print(f"Patient ID: {result['patient_id']}") + print(f"Features: {result['features']}") + print(f"Target (LOS): {result['target']} days") + print(f"Metadata: {result['metadata']}") + else: + print("No patients available for testing") + except Exception as e: + print(f"Error testing task call: {e}") +def main(): + print("PyHealth Length of Stay Prediction Task") + print("="*40) + + # Create sample data if it doesn't exist + if not os.path.exists("fake_los_data.csv"): + print("Creating sample data...") + create_sample_data() + + # Check if YAML config exists + if not os.path.exists("lengthofstay.yaml"): + print("Error: lengthofstay.yaml not found!") + print("Please create the YAML configuration file first.") + print("You can use the provided lengthofstay.yaml configuration.") + return + + try: + print("Initializing dataset...") + dataset = LOSDataset(dev=True) + + if not dataset.patients: + print("No patient data loaded. Please check your data file.") + return + + print("Creating task...") + task = LOSTask(dataset) + + print("Getting targets...") + targets = task.get_targets() + print(f"Length of Stay targets: {targets}") + + print("Getting features...") + features = task.get_features() + print("Features:") + for feat in features[:3]: # Show first 3 + print(f" {feat}") + if len(features) > 3: + print(f" ... and {len(features) - 3} more") + + print(f"\nDataset Summary:") + print(f"{'='*30}") + print(f"Number of patients: {len(dataset.patients)}") + if targets: + print(f"Average LOS: {sum(targets) / len(targets):.1f} days") + print(f"Min LOS: {min(targets)} days") + print(f"Max LOS: {max(targets)} days") + + # Test the __call__ method + test_task_call() + + # Run simple prediction if possible + run_simple_prediction() + + print(f"\n{'='*50}") + print("SUCCESS! Dataset and task created successfully.") + print("You can now use this dataset with PyHealth models.") + + except Exception as e: + print(f"Error: {e}") + print("\nTroubleshooting:") + print("1. Ensure yaml file exists and has the correct structure") + print("2. Ensure csv file exists and is readable") + print("3. Check that PyHealth is properly installed") + print("4. Verify your Python environment has required dependencies") + + +if __name__ == "__main__": + main() \ No newline at end of file From 58a3f2e67d1c9bfdbedef6041d70e84088fc580e Mon Sep 17 00:00:00 2001 From: Tanmayi Bondu Date: Thu, 24 Jul 2025 23:29:35 -0500 Subject: [PATCH 2/3] Add fake LOS data and Polars test script for length of stay prediction --- pyhealth/fake_los_data.csv | 6 ++++++ pyhealth/test_polars.py | 11 +++++++++++ 2 files changed, 17 insertions(+) create mode 100644 pyhealth/fake_los_data.csv create mode 100644 pyhealth/test_polars.py diff --git a/pyhealth/fake_los_data.csv b/pyhealth/fake_los_data.csv new file mode 100644 index 000000000..12a107b7b --- /dev/null +++ b/pyhealth/fake_los_data.csv @@ -0,0 +1,6 @@ +patient_id,admission_date,discharge_date +1,2023-01-01,2023-01-05 +2,2023-02-10,2023-02-15 +3,2023-03-20,2023-03-22 +4,2023-04-01,2023-04-10 +5,2023-05-05,2023-05-07 diff --git a/pyhealth/test_polars.py b/pyhealth/test_polars.py new file mode 100644 index 000000000..044cce34e --- /dev/null +++ b/pyhealth/test_polars.py @@ -0,0 +1,11 @@ +import polars as pl + +def main(): + df = pl.DataFrame({ + "a": [1, 2, 3], + "b": [4, 5, 6] + }) + print(df) + +if __name__ == "__main__": + main() From ad893df7dc227f5fee67179bfdf88d59f2a19579 Mon Sep 17 00:00:00 2001 From: "Muni T. Bondu" <84352657+tanub22@users.noreply.github.com> Date: Mon, 1 Sep 2025 21:25:50 -0400 Subject: [PATCH 3/3] Created using Colab --- public/notebooks/NDIFGetStarted.ipynb | 1686 +++++++++++++++++++++++++ 1 file changed, 1686 insertions(+) create mode 100644 public/notebooks/NDIFGetStarted.ipynb diff --git a/public/notebooks/NDIFGetStarted.ipynb b/public/notebooks/NDIFGetStarted.ipynb new file mode 100644 index 000000000..300d91d49 --- /dev/null +++ b/public/notebooks/NDIFGetStarted.ipynb @@ -0,0 +1,1686 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "language_info": { + "name": "python" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "ff4cc0e7a3004f9585d4c70cf047ff70": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_99a15b6310114dc5a45cc3b92979a3ac", + "IPY_MODEL_c76d999c22bc4619a4a4aa4c95761d4f", + "IPY_MODEL_965f51687fd84532b4a4e71315f93e3a" + ], + "layout": "IPY_MODEL_38551f430cdb4cd2a336a5b1c8822d4c" + } + }, + "99a15b6310114dc5a45cc3b92979a3ac": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_9685b6a334184844ad906ccb70c5b20c", + "placeholder": "​", + "style": "IPY_MODEL_89e4d0f2a36f4c88b2215c9867dc0c83", + "value": "Downloading result: 100%" + } + }, + "c76d999c22bc4619a4a4aa4c95761d4f": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_058fdc3b085d48f99c4a90806515dd10", + "max": 6654190, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_bd07ff0eb81741e284dda5b1849c6d89", + "value": 6654190 + } + }, + "965f51687fd84532b4a4e71315f93e3a": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_995ca0f45e5f430abd4c6665bfbc19f4", + "placeholder": "​", + "style": "IPY_MODEL_765be1872e3f47c6aed8af9509a63d7c", + "value": " 6.65M/6.65M [00:00<00:00, 26.6MB/s]" + } + }, + "38551f430cdb4cd2a336a5b1c8822d4c": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "9685b6a334184844ad906ccb70c5b20c": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "89e4d0f2a36f4c88b2215c9867dc0c83": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "058fdc3b085d48f99c4a90806515dd10": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "bd07ff0eb81741e284dda5b1849c6d89": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "995ca0f45e5f430abd4c6665bfbc19f4": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "765be1872e3f47c6aed8af9509a63d7c": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "9b4f42edbdf548b98dffbde88e69cf3f": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_e2df601e79f947e697f179b010256d5f", + "IPY_MODEL_e170feb6b80a4c4f954cbc4755799351", + "IPY_MODEL_e4c84743e052440a8d66583f33a56654" + ], + "layout": "IPY_MODEL_f222e8a4635744e5a00b789dd0e8bcf1" + } + }, + "e2df601e79f947e697f179b010256d5f": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_af1a09bacc7c464992df9be585e9b078", + "placeholder": "​", + "style": "IPY_MODEL_306117a288194747a726fcaabf68df4d", + "value": "Downloading result: 100%" + } + }, + "e170feb6b80a4c4f954cbc4755799351": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4accc232bff94d1a9c08dabe19e2c796", + "max": 6473648, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_9fe00be8224c4f52bbe5885980a32209", + "value": 6473648 + } + }, + "e4c84743e052440a8d66583f33a56654": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_ce52a7b12c174bc6aa1fac2ba3505299", + "placeholder": "​", + "style": "IPY_MODEL_f6abc4375f524902b3a560d5eafe6f03", + "value": " 6.47M/6.47M [00:00<00:00, 23.9MB/s]" + } + }, + "f222e8a4635744e5a00b789dd0e8bcf1": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "af1a09bacc7c464992df9be585e9b078": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "306117a288194747a726fcaabf68df4d": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "4accc232bff94d1a9c08dabe19e2c796": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "9fe00be8224c4f52bbe5885980a32209": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "ce52a7b12c174bc6aa1fac2ba3505299": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "f6abc4375f524902b3a560d5eafe6f03": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "2cc84d425f6b4571b80aa75933981467": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_7faedede52464519b815973517224916", + "IPY_MODEL_3d7fc08f37294404928d1e7897af1bfa", + "IPY_MODEL_601b1cfee4a24c129aff16b47fe74b89" + ], + "layout": "IPY_MODEL_3af466cb270c4ac38553c2093e9a0670" + } + }, + "7faedede52464519b815973517224916": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_0c78e8093ae74f6684945dbc006c6128", + "placeholder": "​", + "style": "IPY_MODEL_b8260ad484474135bd2d722f24a4769b", + "value": "Downloading result: 100%" + } + }, + "3d7fc08f37294404928d1e7897af1bfa": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a8a1f6f375914902bf6d4f58fe19215f", + "max": 12899760, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_6962f6d1979d4e3197075da23f3173bb", + "value": 12899760 + } + }, + "601b1cfee4a24c129aff16b47fe74b89": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_6b2ca83b10e34791b3ffd23438e04b09", + "placeholder": "​", + "style": "IPY_MODEL_8ff4c2fa00d048299017fbf60f05504c", + "value": " 12.9M/12.9M [00:00<00:00, 33.4MB/s]" + } + }, + "3af466cb270c4ac38553c2093e9a0670": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "0c78e8093ae74f6684945dbc006c6128": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b8260ad484474135bd2d722f24a4769b": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "a8a1f6f375914902bf6d4f58fe19215f": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "6962f6d1979d4e3197075da23f3173bb": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "6b2ca83b10e34791b3ffd23438e04b09": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "8ff4c2fa00d048299017fbf60f05504c": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + } + } + } + }, + "cells": [ + { + "cell_type": "markdown", + "source": [ + "# Getting Started with NDIF and NNsight\n", + "\n", + "In the current era of large-scale deep learning, the most interesting AI models are massive black boxes that are hard to run. Ordinary commercial inference service APIs let us interact with these models, but they do not let us access model internals. We are changing this with NDIF and NNsight.\n", + "\n", + "* [NDIF](https://ndif.us/) is an inference service hosting large open-weight LLMs for use by researchers.\n", + "* [NNsight](https://nnsight.net/) is a package for interpreting and manipulating internals of deep learning models.\n", + "\n", + "Together, NDIF and NNsight work hand in hand to let researchers run complex experiments on huge open models easily, with full transparent access." + ], + "metadata": { + "id": "EHUD6yMYaKNL" + } + }, + { + "cell_type": "markdown", + "source": [ + "# Install NNsight\n", + "\n", + "\"drawing\"\n", + "\n", + "\n", + "To start using NNsight, you can install it via `pip`." + ], + "metadata": { + "id": "9E0Fc62qcsdC" + } + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "id": "plcjPZtXY9Pn" + }, + "outputs": [], + "source": [ + "!pip install nnsight\n", + "\n", + "from IPython.display import clear_output\n", + "clear_output()" + ] + }, + { + "cell_type": "markdown", + "source": [ + "# Sign up for NDIF remote model access\n", + "\n", + "\"drawing\"\n", + "\n", + "In order to remotely access LLMs through NDIF, users must sign up for an NDIF API key.\n", + "\n", + "\n", + "### **[Register here](https://login.ndif.us/) to get your free API key!**\n", + "\n", + "\n", + "Once you have a valid NDIF API key, you then can configure `nnsight` by doing the following:\n" + ], + "metadata": { + "id": "iJkH3UaleYSH" + } + }, + { + "cell_type": "code", + "source": [ + "from nnsight import CONFIG\n", + "\n", + "CONFIG.API.APIKEY = \"a9412ae5-4a1a-4ee7-ba2d-a2a62e6c9c63\"\n", + "clear_output()" + ], + "metadata": { + "id": "SggrqZ4fi3Pn" + }, + "execution_count": 3, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "
\n", + "\n", + "More about API key configuration\n", + "\n", + "\n", + "The above code saves your API key as the default in a config file along with the `nnsight` installation. If you're running this walkthrough using a local Python installation, this only needs to be run once. If you're using Colab, we recommend saving your API key as a Colab Secret, and configuring it as follows in your notebooks:\n", + "\n", + "```\n", + "from nnsight import CONFIG\n", + "\n", + "if is_colab:\n", + " # include your NNsight API key on Colab secrets\n", + " from google.colab import userdata\n", + " NDIF_API = userdata.get('NDIF_API')\n", + " CONFIG.set_default_api_key(NDIF_API)\n", + "```\n", + "\n", + "
" + ], + "metadata": { + "id": "2li8-LvrjMT4" + } + }, + { + "cell_type": "markdown", + "source": [ + "# Choose a Model\n" + ], + "metadata": { + "id": "k_4k2Tj0VrpX" + } + }, + { + "cell_type": "markdown", + "source": [ + "NDIF hosts multiple LLMs, including various sizes of the Llama 3.1 models and DeepSeek-R1 models. **You can view the full list of hosted models on [our status page](https://nnsight.net/status/).** All of our models are open for public use, except you need to apply for access to the Llama-3.1-405B models.\n", + "\n", + "
\n", + "\n", + "Apply for 405B access\n", + "\n", + "\n", + "If you have a clear research need for Llama-3.1-405B and would like more details about applying for access, please refer to [this page](https://ndif.us/405b.html)!\n", + "\n", + "
\n", + "\n", + "For these exercises, we will explore how we can access and modify the Llama-3.1-70B model's internal states. This 70-billion-parameter model is about the maximum size that you could run on a single A100 GPU with 80GB of VRAM, but we are going to access it remotely on NDIF resources, so you can run it on Colab or your laptop computer!\n", + "\n", + "
\n", + "\n", + "Note: Llama models are gated on HuggingFace\n", + "\n", + "\n", + "Llama models are gated and require you to register for access via HuggingFace. [Check out their website for more information about registration with Meta](https://huggingface.co/meta-llama/Llama-3.1-70B).\n", + "\n", + "If you are using a local Python installation, you can activate your HuggingFace token using the terminal:\n", + "\n", + "`huggingface-cli login -token YOUR_HF_TOKEN`\n", + "\n", + "If you are using Colab, you can add your HuggingFace token to your Secrets.\n", + "\n", + "
\n", + "\n", + "We will be using the `LanguageModel` subclass of NNsight to load in the Llama-3.1-70B model and access its internal states.\n", + "\n", + "
\n", + "\n", + "About NNsight LanguageModel\n", + "\n", + "\n", + "The `LanguageModel` subclass of NNsight is a wrapper that includes special support for HuggingFace language models, including automatically loading models from a HuggingFace ID together with the appropriate tokenizer.\n", + "\n", + "This way there's no need to pretokenize your input, and instead you can just pass a string as an input!\n", + "\n", + "*Note: `LanguageModel` models also accept tokenized inputs, including [chat templates](https://huggingface.co/docs/transformers/main/en/chat_templating).*\n", + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n" + ], + "metadata": { + "id": "SJhCJ4ImVu-c" + } + }, + { + "cell_type": "code", + "source": [ + "# instantiate the model using the LanguageModel class\n", + "from nnsight import LanguageModel\n", + "\n", + "# don't worry, this won't load locally!\n", + "llm = LanguageModel(\"meta-llama/Meta-Llama-3.1-70B\", device_map=\"auto\")\n", + "\n", + "print(llm)" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 686 + }, + "id": "D7Da0Zi8e1km", + "outputId": "b653be19-d994-4e3b-d732-a0e9da10e6cf" + }, + "execution_count": 5, + "outputs": [ + { + "output_type": "error", + "ename": "OSError", + "evalue": "You are trying to access a gated repo.\nMake sure to have access to it at https://huggingface.co/meta-llama/Meta-Llama-3.1-70B.\n401 Client Error. (Request ID: Root=1-68b6463a-0f00d0b309f47b46338a4b45;013affe6-4728-4fbf-9dc8-6468f8846f15)\n\nCannot access gated repo for url https://huggingface.co/meta-llama/Meta-Llama-3.1-70B/resolve/main/config.json.\nAccess to model meta-llama/Llama-3.1-70B is restricted. You must have access to it and be authenticated to access it. Please log in.", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mHTTPError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m/usr/local/lib/python3.12/dist-packages/huggingface_hub/utils/_http.py\u001b[0m in \u001b[0;36mhf_raise_for_status\u001b[0;34m(response, endpoint_name)\u001b[0m\n\u001b[1;32m 408\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 409\u001b[0;31m \u001b[0mresponse\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mraise_for_status\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 410\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mHTTPError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.12/dist-packages/requests/models.py\u001b[0m in \u001b[0;36mraise_for_status\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1025\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mhttp_error_msg\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1026\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mHTTPError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhttp_error_msg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1027\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mHTTPError\u001b[0m: 401 Client Error: Unauthorized for url: https://huggingface.co/meta-llama/Meta-Llama-3.1-70B/resolve/main/config.json", + "\nThe above exception was the direct cause of the following exception:\n", + "\u001b[0;31mGatedRepoError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m/usr/local/lib/python3.12/dist-packages/transformers/utils/hub.py\u001b[0m in \u001b[0;36mcached_files\u001b[0;34m(path_or_repo_id, filenames, cache_dir, force_download, resume_download, proxies, token, revision, local_files_only, subfolder, repo_type, user_agent, _raise_exceptions_for_gated_repo, _raise_exceptions_for_missing_entries, _raise_exceptions_for_connection_errors, _commit_hash, **deprecated_kwargs)\u001b[0m\n\u001b[1;32m 478\u001b[0m \u001b[0;31m# This is slightly better for only 1 file\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 479\u001b[0;31m hf_hub_download(\n\u001b[0m\u001b[1;32m 480\u001b[0m \u001b[0mpath_or_repo_id\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.12/dist-packages/huggingface_hub/utils/_validators.py\u001b[0m in \u001b[0;36m_inner_fn\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 113\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 114\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 115\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.12/dist-packages/huggingface_hub/file_download.py\u001b[0m in \u001b[0;36mhf_hub_download\u001b[0;34m(repo_id, filename, subfolder, repo_type, revision, library_name, library_version, cache_dir, local_dir, user_agent, force_download, proxies, etag_timeout, token, local_files_only, headers, endpoint, resume_download, force_filename, local_dir_use_symlinks)\u001b[0m\n\u001b[1;32m 1009\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1010\u001b[0;31m return _hf_hub_download_to_cache_dir(\n\u001b[0m\u001b[1;32m 1011\u001b[0m \u001b[0;31m# Destination\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.12/dist-packages/huggingface_hub/file_download.py\u001b[0m in \u001b[0;36m_hf_hub_download_to_cache_dir\u001b[0;34m(cache_dir, repo_id, filename, repo_type, revision, endpoint, etag_timeout, headers, proxies, token, local_files_only, force_download)\u001b[0m\n\u001b[1;32m 1116\u001b[0m \u001b[0;31m# Otherwise, raise appropriate error\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1117\u001b[0;31m \u001b[0m_raise_on_head_call_error\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhead_call_error\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mforce_download\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlocal_files_only\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1118\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.12/dist-packages/huggingface_hub/file_download.py\u001b[0m in \u001b[0;36m_raise_on_head_call_error\u001b[0;34m(head_call_error, force_download, local_files_only)\u001b[0m\n\u001b[1;32m 1657\u001b[0m \u001b[0;31m# Unauthorized => likely a token issue => let's raise the actual error\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1658\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mhead_call_error\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1659\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.12/dist-packages/huggingface_hub/file_download.py\u001b[0m in \u001b[0;36m_get_metadata_or_catch_error\u001b[0;34m(repo_id, filename, repo_type, revision, endpoint, proxies, etag_timeout, headers, token, local_files_only, relative_filename, storage_folder)\u001b[0m\n\u001b[1;32m 1545\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1546\u001b[0;31m metadata = get_hf_file_metadata(\n\u001b[0m\u001b[1;32m 1547\u001b[0m \u001b[0murl\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mproxies\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mproxies\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0metag_timeout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mheaders\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mheaders\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtoken\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtoken\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mendpoint\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mendpoint\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.12/dist-packages/huggingface_hub/utils/_validators.py\u001b[0m in \u001b[0;36m_inner_fn\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 113\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 114\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 115\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.12/dist-packages/huggingface_hub/file_download.py\u001b[0m in \u001b[0;36mget_hf_file_metadata\u001b[0;34m(url, token, proxies, timeout, library_name, library_version, user_agent, headers, endpoint)\u001b[0m\n\u001b[1;32m 1462\u001b[0m \u001b[0;31m# Retrieve metadata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1463\u001b[0;31m r = _request_wrapper(\n\u001b[0m\u001b[1;32m 1464\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"HEAD\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.12/dist-packages/huggingface_hub/file_download.py\u001b[0m in \u001b[0;36m_request_wrapper\u001b[0;34m(method, url, follow_relative_redirects, **params)\u001b[0m\n\u001b[1;32m 285\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mfollow_relative_redirects\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 286\u001b[0;31m response = _request_wrapper(\n\u001b[0m\u001b[1;32m 287\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmethod\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.12/dist-packages/huggingface_hub/file_download.py\u001b[0m in \u001b[0;36m_request_wrapper\u001b[0;34m(method, url, follow_relative_redirects, **params)\u001b[0m\n\u001b[1;32m 309\u001b[0m \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhttp_backoff\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmethod\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmethod\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretry_on_exceptions\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretry_on_status_codes\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m429\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 310\u001b[0;31m \u001b[0mhf_raise_for_status\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresponse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 311\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.12/dist-packages/huggingface_hub/utils/_http.py\u001b[0m in \u001b[0;36mhf_raise_for_status\u001b[0;34m(response, endpoint_name)\u001b[0m\n\u001b[1;32m 425\u001b[0m )\n\u001b[0;32m--> 426\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0m_format\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mGatedRepoError\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 427\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mGatedRepoError\u001b[0m: 401 Client Error. (Request ID: Root=1-68b6463a-0f00d0b309f47b46338a4b45;013affe6-4728-4fbf-9dc8-6468f8846f15)\n\nCannot access gated repo for url https://huggingface.co/meta-llama/Meta-Llama-3.1-70B/resolve/main/config.json.\nAccess to model meta-llama/Llama-3.1-70B is restricted. You must have access to it and be authenticated to access it. Please log in.", + "\nThe above exception was the direct cause of the following exception:\n", + "\u001b[0;31mOSError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m/tmp/ipython-input-1649193546.py\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0;31m# don't worry, this won't load locally!\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0mllm\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mLanguageModel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"meta-llama/Meta-Llama-3.1-70B\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice_map\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"auto\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 7\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mllm\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.12/dist-packages/nnsight/modeling/language.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, config, tokenizer, automodel, import_edits, *args, **kwargs)\u001b[0m\n\u001b[1;32m 88\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrevision\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'revision'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'main'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 89\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 90\u001b[0;31m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 91\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 92\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mimport_edits\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.12/dist-packages/nnsight/modeling/mixins/meta.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, dispatch, meta_buffers, rename, *args, **kwargs)\u001b[0m\n\u001b[1;32m 38\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0minit_empty_weights\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minclude_buffers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmeta_buffers\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 39\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 40\u001b[0;31m \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_load_meta\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 41\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 42\u001b[0m \u001b[0mNNsight\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrename\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mrename\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.12/dist-packages/nnsight/modeling/language.py\u001b[0m in \u001b[0;36m_load_meta\u001b[0;34m(self, repo_id, revision, tokenizer_kwargs, patch_llama_scan, **kwargs)\u001b[0m\n\u001b[1;32m 213\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrevision\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrevision\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 214\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 215\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_load_config\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrepo_id\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrevision\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mrevision\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 216\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 217\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_load_tokenizer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrepo_id\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrevision\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mrevision\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mtokenizer_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.12/dist-packages/nnsight/modeling/language.py\u001b[0m in \u001b[0;36m_load_config\u001b[0;34m(self, repo_id, **kwargs)\u001b[0m\n\u001b[1;32m 184\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconfig\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 185\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 186\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconfig\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mAutoConfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_pretrained\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrepo_id\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 187\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 188\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_load_tokenizer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrepo_id\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.12/dist-packages/transformers/models/auto/configuration_auto.py\u001b[0m in \u001b[0;36mfrom_pretrained\u001b[0;34m(cls, pretrained_model_name_or_path, **kwargs)\u001b[0m\n\u001b[1;32m 1248\u001b[0m \u001b[0mcode_revision\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"code_revision\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1249\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1250\u001b[0;31m \u001b[0mconfig_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0munused_kwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mPretrainedConfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_config_dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpretrained_model_name_or_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1251\u001b[0m \u001b[0mhas_remote_code\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"auto_map\"\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mconfig_dict\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;34m\"AutoConfig\"\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mconfig_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"auto_map\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1252\u001b[0m \u001b[0mhas_local_code\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"model_type\"\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mconfig_dict\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mconfig_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"model_type\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mCONFIG_MAPPING\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.12/dist-packages/transformers/configuration_utils.py\u001b[0m in \u001b[0;36mget_config_dict\u001b[0;34m(cls, pretrained_model_name_or_path, **kwargs)\u001b[0m\n\u001b[1;32m 647\u001b[0m \u001b[0moriginal_kwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdeepcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 648\u001b[0m \u001b[0;31m# Get config dict associated with the base config file\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 649\u001b[0;31m \u001b[0mconfig_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcls\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_config_dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpretrained_model_name_or_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 650\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mconfig_dict\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 651\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.12/dist-packages/transformers/configuration_utils.py\u001b[0m in \u001b[0;36m_get_config_dict\u001b[0;34m(cls, pretrained_model_name_or_path, **kwargs)\u001b[0m\n\u001b[1;32m 706\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 707\u001b[0m \u001b[0;31m# Load from local folder or from cache or download from model Hub and cache\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 708\u001b[0;31m resolved_config_file = cached_file(\n\u001b[0m\u001b[1;32m 709\u001b[0m \u001b[0mpretrained_model_name_or_path\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 710\u001b[0m \u001b[0mconfiguration_file\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.12/dist-packages/transformers/utils/hub.py\u001b[0m in \u001b[0;36mcached_file\u001b[0;34m(path_or_repo_id, filename, **kwargs)\u001b[0m\n\u001b[1;32m 319\u001b[0m \u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 320\u001b[0m \"\"\"\n\u001b[0;32m--> 321\u001b[0;31m \u001b[0mfile\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcached_files\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath_or_repo_id\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpath_or_repo_id\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfilenames\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mfilename\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 322\u001b[0m \u001b[0mfile\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfile\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mfile\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mfile\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 323\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mfile\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.12/dist-packages/transformers/utils/hub.py\u001b[0m in \u001b[0;36mcached_files\u001b[0;34m(path_or_repo_id, filenames, cache_dir, force_download, resume_download, proxies, token, revision, local_files_only, subfolder, repo_type, user_agent, _raise_exceptions_for_gated_repo, _raise_exceptions_for_missing_entries, _raise_exceptions_for_connection_errors, _commit_hash, **deprecated_kwargs)\u001b[0m\n\u001b[1;32m 541\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0m_raise_exceptions_for_gated_repo\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 542\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 543\u001b[0;31m raise OSError(\n\u001b[0m\u001b[1;32m 544\u001b[0m \u001b[0;34m\"You are trying to access a gated repo.\\nMake sure to have access to it at \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 545\u001b[0m \u001b[0;34mf\"https://huggingface.co/{path_or_repo_id}.\\n{str(e)}\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mOSError\u001b[0m: You are trying to access a gated repo.\nMake sure to have access to it at https://huggingface.co/meta-llama/Meta-Llama-3.1-70B.\n401 Client Error. (Request ID: Root=1-68b6463a-0f00d0b309f47b46338a4b45;013affe6-4728-4fbf-9dc8-6468f8846f15)\n\nCannot access gated repo for url https://huggingface.co/meta-llama/Meta-Llama-3.1-70B/resolve/main/config.json.\nAccess to model meta-llama/Llama-3.1-70B is restricted. You must have access to it and be authenticated to access it. Please log in." + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "# Experiment 1: Access model internals" + ], + "metadata": { + "id": "cpgFNedphEt-" + } + }, + { + "cell_type": "markdown", + "source": [ + "Now that we've installed `nnsight`, configured our API key, and instantiated a model, we can run an experiment.\n", + "\n", + "For this experiment, let's try grabbing some of the LLM's hidden states using `nnsight`'s tracing context, `.trace()`.\n", + "\n", + "Entering the tracing context allows us to customize how a neural network runs. By calling `.trace()`, we are telling the model to run with a given input and to collect and/or modify the internal model states based on user-defined code within the tracing context. We can also specify that we want to use an NDIF-hosted model instead of executing locally by setting `remote=True`.\n", + "\n", + "To get started, let's ask NNsight to collect the layer output (known as \"logits\") at the final layer, along with the overall model output. NNsight needs to know what specific parts of the model we're interested in accessing later, so we need to specify which elements we'd like to save after exiting the tracing context using `.save()`.\n", + "\n", + "*Note: You will not be able to access any values defined within a `.trace()` that aren't saved with `.save()` after exiting the tracing context!*" + ], + "metadata": { + "id": "O9ls1R-wpbTk" + } + }, + { + "cell_type": "code", + "source": [ + "# remote = True means the model will execute on NDIF's shared resources\n", + "with llm.trace(\"The Eiffel Tower is in the city of\", remote=True):\n", + "\n", + " # user-defined code to access internal model components\n", + " hidden_states = llm.model.layers[-1].output[0].save()\n", + " output = llm.output.save()\n", + "\n", + "# after exiting the tracing context, we can access any values that were saved\n", + "print(\"Hidden State Logits: \",hidden_states[0])\n", + "\n", + "output_logits = output[\"logits\"]\n", + "print(\"Model Output Logits: \",output_logits[0])\n", + "\n", + "# decode the final model output from output logits\n", + "max_probs, tokens = output_logits[0].max(dim=-1)\n", + "word = [llm.tokenizer.decode(tokens.cpu()[-1])]\n", + "print(\"Model Output: \", word[0])" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 495, + "referenced_widgets": [ + "ff4cc0e7a3004f9585d4c70cf047ff70", + "99a15b6310114dc5a45cc3b92979a3ac", + "c76d999c22bc4619a4a4aa4c95761d4f", + "965f51687fd84532b4a4e71315f93e3a", + "38551f430cdb4cd2a336a5b1c8822d4c", + "9685b6a334184844ad906ccb70c5b20c", + "89e4d0f2a36f4c88b2215c9867dc0c83", + "058fdc3b085d48f99c4a90806515dd10", + "bd07ff0eb81741e284dda5b1849c6d89", + "995ca0f45e5f430abd4c6665bfbc19f4", + "765be1872e3f47c6aed8af9509a63d7c" + ] + }, + "id": "P7pJFi4amXpf", + "outputId": "152b1fd2-231f-480f-9eda-21fe3a6bf5b7" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + "2025-02-26 16:30:19,122 549c7e98-7fa2-4f82-8678-11f29098e96e - RECEIVED: Your job has been received and is waiting approval.\n", + "INFO:nnsight_remote:549c7e98-7fa2-4f82-8678-11f29098e96e - RECEIVED: Your job has been received and is waiting approval.\n", + "2025-02-26 16:30:19,429 549c7e98-7fa2-4f82-8678-11f29098e96e - APPROVED: Your job was approved and is waiting to be run.\n", + "INFO:nnsight_remote:549c7e98-7fa2-4f82-8678-11f29098e96e - APPROVED: Your job was approved and is waiting to be run.\n", + "2025-02-26 16:30:19,709 549c7e98-7fa2-4f82-8678-11f29098e96e - RUNNING: Your job has started running.\n", + "INFO:nnsight_remote:549c7e98-7fa2-4f82-8678-11f29098e96e - RUNNING: Your job has started running.\n", + "2025-02-26 16:30:20,948 549c7e98-7fa2-4f82-8678-11f29098e96e - COMPLETED: Your job has been completed.\n", + "INFO:nnsight_remote:549c7e98-7fa2-4f82-8678-11f29098e96e - COMPLETED: Your job has been completed.\n" + ] + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "Downloading result: 0%| | 0.00/6.65M [00:00\n", + "\n", + "Disabling remote logging notifications\n", + "\n", + "If you prefer, you can disable NNsight remote logging notifications with the following code, although they can help troubleshoot any network issues.\n", + "\n", + "```\n", + "from nnsight import CONFIG\n", + "CONFIG.APP.REMOTE_LOGGING = False\n", + "```\n", + "\n", + "If you'd like to turn them back on, just set `REMOTE_LOGGING = True`:\n", + "```\n", + "from nnsight import CONFIG\n", + "CONFIG.APP.REMOTE_LOGGING = True\n", + "```\n", + "\n", + "\n", + "We are also seeing our printed results. After exiting the tracing context, NNsight downloads the saved results, which we can perform operations on using Python code. Pretty simple!" + ], + "metadata": { + "id": "hKc5xQ7JeCaY" + } + }, + { + "cell_type": "markdown", + "source": [ + "# Experiment 2: Alter model internals\n" + ], + "metadata": { + "id": "FR4S_ngVGp4B" + } + }, + { + "cell_type": "markdown", + "source": [ + "Now that we've accessed the internal layers of the model, let's try modifying them and see how it affects the output!\n", + "\n", + "We can do this using in-place operations in NNsight, which alter the model's state during execution. Let's try changing the output of layer 8 to be equal to 4." + ], + "metadata": { + "id": "AqF-iRafGrsM" + } + }, + { + "cell_type": "code", + "source": [ + "# remote = True means the model will execute on NDIF's shared resources\n", + "with llm.trace(\"The Eiffel Tower is in the city of\", remote=True):\n", + "\n", + " # user-defined code to access internal model components\n", + " llm.model.layers[7].output[0][:] = 4 # in-place operation to change a single layer's output values\n", + " output = llm.output.save()\n", + "\n", + "# after exiting the tracing context, we can access any values that were saved\n", + "\n", + "output_logits = output[\"logits\"]\n", + "print(\"Model Output Logits: \",output_logits[0])\n", + "\n", + "# decode the final model output from output logits\n", + "max_probs, tokens = output_logits[0].max(dim=-1)\n", + "word = [llm.tokenizer.decode(tokens.cpu()[-1])]\n", + "print(\"Model Output: \", word[0])" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 353, + "referenced_widgets": [ + "9b4f42edbdf548b98dffbde88e69cf3f", + "e2df601e79f947e697f179b010256d5f", + "e170feb6b80a4c4f954cbc4755799351", + "e4c84743e052440a8d66583f33a56654", + "f222e8a4635744e5a00b789dd0e8bcf1", + "af1a09bacc7c464992df9be585e9b078", + "306117a288194747a726fcaabf68df4d", + "4accc232bff94d1a9c08dabe19e2c796", + "9fe00be8224c4f52bbe5885980a32209", + "ce52a7b12c174bc6aa1fac2ba3505299", + "f6abc4375f524902b3a560d5eafe6f03" + ] + }, + "id": "YTzpJjCVG0Uh", + "outputId": "a562b641-bba9-409e-c273-c72e8a75cddc" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + "2025-02-26 16:50:08,852 ccd16638-811d-429a-8584-a371825430db - RECEIVED: Your job has been received and is waiting approval.\n", + "INFO:nnsight_remote:ccd16638-811d-429a-8584-a371825430db - RECEIVED: Your job has been received and is waiting approval.\n", + "2025-02-26 16:50:10,618 ccd16638-811d-429a-8584-a371825430db - APPROVED: Your job was approved and is waiting to be run.\n", + "INFO:nnsight_remote:ccd16638-811d-429a-8584-a371825430db - APPROVED: Your job was approved and is waiting to be run.\n", + "2025-02-26 16:50:11,496 ccd16638-811d-429a-8584-a371825430db - RUNNING: Your job has started running.\n", + "INFO:nnsight_remote:ccd16638-811d-429a-8584-a371825430db - RUNNING: Your job has started running.\n", + "2025-02-26 16:50:13,316 ccd16638-811d-429a-8584-a371825430db - COMPLETED: Your job has been completed.\n", + "INFO:nnsight_remote:ccd16638-811d-429a-8584-a371825430db - COMPLETED: Your job has been completed.\n" + ] + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "Downloading result: 0%| | 0.00/6.47M [00:00" + ], + "metadata": { + "id": "zSNTWLhLQqyT" + } + } + ] +} \ No newline at end of file