diff --git a/Academy/AI/1-Intro-To-ML-And-MLOps/.DS_Store b/Academy/AI/1-Intro-To-ML-And-MLOps/.DS_Store new file mode 100644 index 0000000..eff5404 Binary files /dev/null and b/Academy/AI/1-Intro-To-ML-And-MLOps/.DS_Store differ diff --git a/Academy/AI/1-Intro-To-ML-And-MLOps/.gitignore b/Academy/AI/1-Intro-To-ML-And-MLOps/.gitignore new file mode 100644 index 0000000..1c5d00a --- /dev/null +++ b/Academy/AI/1-Intro-To-ML-And-MLOps/.gitignore @@ -0,0 +1,179 @@ +# local mlflow +mlartifacts/* +mlruns/* +dev/* + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# UV +# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +#uv.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/latest/usage/project/#working-with-version-control +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +# Ruff stuff: +.ruff_cache/ + +# PyPI configuration file +.pypirc \ No newline at end of file diff --git a/Academy/AI/1-Intro-To-ML-And-MLOps/README.md b/Academy/AI/1-Intro-To-ML-And-MLOps/README.md new file mode 100644 index 0000000..648588c --- /dev/null +++ b/Academy/AI/1-Intro-To-ML-And-MLOps/README.md @@ -0,0 +1,108 @@ +# Lab: Basic ML Model with Weather Dataset + MLflow Integration + +Welcome to this lab! Here you will learn how to: + +1. **Load and prepare a weather dataset**, with temperature and humidity data. +2. **Train a Machine Learning model** using Scikit-learn, a powerful tool for Machine Learning in Python, to predict rain. +3. **Evaluate the model** computing metrics to determine how well it makes predictions on new data. +4. **Integrate MLflow**, one of the most used tool to track metrics, parameters, and model versions. + +We will follow a guided approach with detailed explanations at each step. +The first part focuses on Scikit-learn and the weather dataset. The second part extends the existing code with MLflow. + +--- + +## Part 0: Setting Up the Python Environment + +Before we begin, we need to ensure we have everything required to run the lab properly. + +### **Requirements** + +This lab assumes that **Python** and **miniconda** are already installed, the repository [kiratech/labs](https://github.com/kiratech/labs.git) is accessible, and **Git** is properly configured on your local machine. Furthermore, **VSCode** or an IDE able to run Jupyter Notebooks, must be installed as well. +In order to execute this laboratory, you will be asked to install a set of tools common in Machine Learning field: + +- [**scikit-learn**](https://scikit-learn.org/stable/index.html): Machine learning library with modeling and evaluation tools. +- [**pandas**](https://pandas.pydata.org/): Framework for data manipulation and analysis in Python. +- [**seaborn**](https://seaborn.pydata.org/): Data visualization library based on Matplotlib. +- [**mlflow**](https://mlflow.org/): Tool for tracking and managing machine learning experiments. + +### 1. Clone the Repository + +To start, clone the lab repository by running the following command in the terminal: + +```sh + git clone https://github.com/kiratech/labs.git +``` + +### 2. Checkout the Lab Branch + +After cloning the repository, navigate to the project folder: + +```sh + cd labs +``` + +Then, checkout the `lab` branch: + +```sh + git checkout academy-ai +``` + +This folder contains resources related to AI-themed labs. + +### 3. Navigate to the 1-Intro-To-ML-And-MLOps Folder + +Go to the folder of the first lab: + +```sh + cd Academy/AI/1-Intro-To-ML-And-MLOps +``` + +### 4. Create a Virtual Environment + +A virtual environment allows you to isolate the project's dependencies from the system-wide ones. + +Using the **VSCode** terminal, create a virtual environment: + +```sh + conda create --name lab_env python=3.12 pip -y +``` + +Activate the Virtual Environment: + +```sh + conda activate lab_env +``` + +You should see the `(lab_env)` prefix in the terminal, indicating that the virtual environment is active. + +### 5. Install Required Packages + +Besides the default packages in the environment, additional libraries may be needed for the lab. +Before installing them, it's always a good practice to update `pip` to avoid compatibility issues: + +```sh + pip install --upgrade pip +``` + +Now, install some essential packages for data analysis and machine learning: + +```sh + pip install scikit-learn pandas seaborn mlflow ipykernel +``` + +Verify that the packages were installed correctly with: + +```sh + conda list +``` + +At this point, you can proceed with the `lab.ipynb` file. + +### 6. Open the Project in VSCode + +At this point, open VSCode from the file explorer or by running the command: + +```sh + code . +``` diff --git a/Academy/AI/1-Intro-To-ML-And-MLOps/lab.ipynb b/Academy/AI/1-Intro-To-ML-And-MLOps/lab.ipynb new file mode 100644 index 0000000..e2352e5 --- /dev/null +++ b/Academy/AI/1-Intro-To-ML-And-MLOps/lab.ipynb @@ -0,0 +1,563 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Lab: Basic ML Model with Weather Dataset + MLflow Integration\n", + "\n", + "Welcome to this lab! Here you will learn how to:\n", + "\n", + "1. **Load and prepare a weather dataset**, with temperature and humidity data.\n", + "2. **Train a Machine Learning model** using Scikit-learn, a powerful tool for Machine Learning in Python, to predict rain.\n", + "3. **Evaluate the model** computing metrics to determine how well it makes predictions on new data.\n", + "4. **Integrate MLflow**, one of the most used tool to track metrics, parameters, and model versions.\n", + "\n", + "We will follow a guided approach with detailed explanations at each step. \n", + "The first part focuses on Scikit-learn and the weather dataset. The second part extends the existing code with MLflow." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Part 1: From Data to Machine Learning Model (Supervised Learning with Scikit-learn) \n", + "\n", + "### Objective \n", + "Build a **classification model** that can predict whether it will rain, using **temperature** and **humidity** as input data. The model will be trained using **Scikit-learn**, a powerful tool for Machine Learning in Python." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1. Preparing the Dataset \n", + "\n", + "Before training a Machine Learning model, it is essential to clean the data, as missing or incorrect values can compromise predictions. A well-prepared dataset allows the model to learn better and provide more accurate results. \n", + "\n", + "For this lab, we will use an example dataset: \n", + "[Weather Test Data](https://raw.githubusercontent.com/boradpreet/Weather_dataset/refs/heads/master/Weather%20Test%20Data.csv) \n", + "\n", + "The **Weather Test Data** dataset contains meteorological information collected at different times. Each row represents an observation with parameters such as **temperature**, **humidity**, **atmospheric pressure**, and other weather variables. \n", + "\n", + "The goal of this dataset is to analyze weather patterns and use them to train a Machine Learning model capable of predicting future conditions, such as the probability of rain or temperature variations." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "\n", + "# Dataset URL \n", + "url = \"https://raw.githubusercontent.com/boradpreet/Weather_dataset/refs/heads/master/Weather%20Test%20Data.csv\"\n", + "\n", + "# Load the dataset in a Pandas dataframe\n", + "df = pd.read_csv(url)\n", + "\n", + "# Show first 5 rows\n", + "df.head(5)\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2. Data Exploration and Cleaning \n", + "\n", + "To ensure our model works correctly, we first need to examine and prepare the dataset. Here are the key steps: \n", + "\n", + "1. **Check for missing data**: Identify if there are any missing values, as they could compromise model training. If necessary, we can remove them or replace them with appropriate values. \n", + "2. **Convert the `Label` column**: Transform categorical values (*NoRain* and *Rain*) into numerical values (0 for *NoRain*, 1 for *Rain*), so the model can interpret them correctly. \n", + "3. **Select key features**: Choose only the most relevant columns (e.g., temperature and humidity) to simplify the model and improve its performance.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 1. Remove missing values\n", + "df = df.dropna()\n", + "\n", + "# 2. Transform column 'RainToday' into numerical values\n", + "df['RainToday'] = df['RainToday'].apply(lambda x: 1 if x == 'Yes' else 0)\n", + "\n", + "# 3. Feature selection \n", + "features = ['MinTemp', 'MaxTemp', 'Humidity3pm', 'Humidity9am']\n", + "\n", + "X = df[features]\n", + "y = df['RainToday']\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 3. Splitting the Dataset into Training and Test Sets \n", + "\n", + "To properly train and evaluate the model, we split the dataset into two parts: \n", + "\n", + "- **X (features)**: Contains the information we will use for predictions, such as **temperature** and **humidity**. \n", + "- **y (target)**: Represents the variable we want to predict, i.e., whether it will rain (*Rain*) or not (*NoRain*). \n", + "\n", + "We split the data into **80% training set** and **20% test set** for the following reasons: \n", + "\n", + "1. **Model Training** \n", + " - 80% of the data is used to teach the model to recognize patterns between features and the target variable. \n", + "\n", + "2. **Model Evaluation** \n", + " - The remaining 20% of the data is not used in training but serves to test the model on unseen data. \n", + " - This helps us understand whether the model can make accurate predictions on new data. \n", + "\n", + "3. **Avoid Overfitting** \n", + " - If we tested the model on the same data it was trained on, we might get deceptively good results, as the model would have simply memorized them. \n", + " - Using separate test data helps verify whether the model can generalize its predictions to real-world data. \n", + "\n", + "This split is a crucial step in building a reliable model capable of making accurate predictions on unseen data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.model_selection import train_test_split\n", + "\n", + "# Dataset split (80% training, 20% testing)\n", + "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n", + "print(f\"Training dataset dimensions: {len(X_train)}\")\n", + "print(f\"Test dataset dimensions: {len(X_test)}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 4. Building and Training the Model \n", + "\n", + "Now that we have prepared the data, we can build and train a Machine Learning model. For this, we will use a classifier called [**RandomForestClassifier**](https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.RandomForestClassifier.html), one of the most widely used techniques for classification problems. \n", + "\n", + "#### Why use **Random Forest**? \n", + "- It is a model based on **decision trees**, which divide the data into multiple steps to make accurate decisions. \n", + "- It is **robust** and works well with both numerical and categorical data. \n", + "- It is less sensitive to noisy data than a single decision tree because it combines multiple trees to improve accuracy. \n", + "\n", + "The model will be trained using **temperature** and **humidity** data to predict whether there will be **rain** or not. After training, we will test it on new data to evaluate its accuracy." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.ensemble import RandomForestClassifier\n", + "\n", + "# Create and train the model\n", + "model = RandomForestClassifier(n_estimators=100, random_state=42)\n", + "model.fit(X_train, y_train)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 5. Model Evaluation \n", + "\n", + "After training the model, we need to verify how accurate its predictions are. To do this, we will calculate **accuracy** and other evaluation metrics. \n", + "\n", + "#### Why is model evaluation important? \n", + "A Machine Learning model is not useful if we do not know how reliable it is. Evaluation helps us understand: \n", + "- **Whether the model is learning correctly from the data** or simply memorizing answers (overfitting). \n", + "- **Whether it can be used on new data** and make realistic predictions. \n", + "\n", + "#### Confusion Matrix \n", + "Besides accuracy, we will use the **confusion matrix**, a visual method that shows where the model makes correct predictions and where it makes mistakes. \n", + "- It helps identify **false positives** and **false negatives**, which are critical errors in many real-world scenarios. \n", + "- It is useful for improving the model, for example, by adjusting decision thresholds or balancing input data. \n", + "\n", + "With these analyses, we can determine whether our model is ready for use or needs improvement." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.metrics import accuracy_score, f1_score, classification_report\n", + "\n", + "# Prediction on test set\n", + "y_pred = model.predict(X_test)\n", + "\n", + "# Compute accuracy and f1-score\n", + "accuracy = accuracy_score(y_test, y_pred)\n", + "f1 = f1_score(y_test, y_pred, average='weighted')\n", + "print(f\"Accuracy of the model: {accuracy:.2f}\")\n", + "print(f\"F1-score of the model: {f1:.2f}\")\n", + "\n", + "# Print classification report\n", + "print(classification_report(y_test, y_pred))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import seaborn as sns\n", + "import matplotlib.pyplot as plt\n", + "from sklearn.metrics import confusion_matrix\n", + "\n", + "# Create a heatmap with Seaborn\n", + "target_names = [\"No Rain\", \"Rain\"]\n", + "cm = confusion_matrix(y_test, y_pred)\n", + "plt.figure(figsize=(6, 4))\n", + "sns.heatmap(cm, annot=True, fmt=\"d\", cmap=\"Blues\", xticklabels=target_names, yticklabels=target_names)\n", + "\n", + "# Add titles\n", + "plt.xlabel(\"Predicted Label\")\n", + "plt.ylabel(\"True Label\")\n", + "plt.title(\"Confusion Matrix\")\n", + "\n", + "# Show the plot\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Conclusion (Part 1) \n", + "\n", + "In this first part, we followed a step-by-step process to build a Machine Learning model capable of predicting rain. Here’s what we did: \n", + "\n", + "1. **Loaded the weather dataset** to analyze temperature, humidity, and other variables. \n", + "2. **Cleaned and prepared the data**, handling missing values and converting the target variable into a format the model can understand. \n", + "3. **Split the dataset** into training (80%) and test (20%) sets to properly train and evaluate the model. \n", + "4. **Built a classification model** using **RandomForestClassifier**, a powerful and robust algorithm. \n", + "5. **Evaluated the model’s performance** by calculating accuracy and analyzing the confusion matrix to identify errors. \n", + "\n", + "Now that we have built the base model, in the next part, we will explore how to integrate **MLflow** to track experiments and further improve performance." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Tech: Installing and Configuring MLflow \n", + "\n", + "### Objective \n", + "Set up a local **MLflow** instance to log experiments, monitor metrics, and manage Machine Learning models in an organized way. \n", + "\n", + "### 1. Starting MLflow \n", + "\n", + "To start MLflow locally, run the following command in the terminal: \n", + "\n", + "```bash\n", + " mlflow ui\n", + "```\n", + "\n", + "Once started, the graphical interface will be accessible at: \n", + "\n", + "```\n", + " http://127.0.0.1:5000\n", + "```\n", + "\n", + "This setup allows **saving experiments and models locally**, enabling tracking of different model versions, comparing evaluation metrics, and optimizing training processes. \n", + "\n", + "In the next sections, we will see how to log parameters, metrics, and models directly within MLflow." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### MLflow in Production \n", + "\n", + "In production environments and for clients, **MLflow is not run locally** but is integrated into a more robust and scalable infrastructure. This prevents issues related to manual experiment management and data persistence. \n", + "\n", + "Common solutions include: \n", + "\n", + "- **Docker Compose** \n", + " - MLflow is started using a `docker-compose.yml` file, which configures a backend database and remote storage for saving experiments. \n", + " - This approach is useful for controlled environments where a quick and reproducible setup is needed. \n", + " - An example implementation is available in the internal repository: \n", + " [kiratech/mlops-service-portfolio](https://github.com/kiratech/mlops-service-portfolio/tree/main). \n", + "\n", + "- **Kubernetes (K8s)** \n", + " - MLflow is deployed on a **Kubernetes cluster**, allowing scalable and centralized experiment management. \n", + " - This approach is ideal for enterprise environments that require high levels of reliability, security, and scalability. \n", + "\n", + "Both solutions rely on a **multi-container architecture**, which includes: \n", + "- **A persistent database** (e.g., PostgreSQL or MySQL) to store experiment metadata. \n", + "- **An S3 or MinIO storage** to save models and artifacts, ensuring secure and scalable data management. \n", + "\n", + "These approaches ensure that MLflow can be reliably used in production, integrating with cloud or on-premise infrastructures for effective Machine Learning model management." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Part 2: MLflow Integration \n", + "\n", + "Now, we will extend the existing code to **track our experiments** using **MLflow**. This will allow us to monitor the model training process, compare different configurations, and manage model versions in a structured way. \n", + "\n", + "### Why integrate MLflow? \n", + "With MLflow, we can: \n", + "- **Log training parameters** (e.g., `n_estimators` for Random Forest) to compare different configurations. \n", + "- **Save evaluation metrics** (e.g., accuracy, F1-score) to monitor the model’s performance. \n", + "- **Store the trained model** to easily reload and reuse it in the future without retraining. \n", + "\n", + "### Objective \n", + "Integrate MLflow into the existing code to **track and version models**, logging parameters, metrics, and artifacts in a structured way. \n", + "\n", + "### 1. MLflow Configuration \n", + "Before we start tracking experiments, let's set up the necessary variables to use MLflow in this project. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import mlflow\n", + "import mlflow.sklearn\n", + "\n", + "# Set name of the experiment and tracking URI of local instance\n", + "mlflow.set_tracking_uri(\"http://127.0.0.1:5000\")\n", + "mlflow.set_experiment(\"weather_classification_experiment\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We verify that from our [web interface](http://127.0.0.1:5000), the new experiment is visible." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2. Logging Parameters, Metrics, and Model \n", + "\n", + "With **MLflow**, we can automatically save and track various information during model training. This helps compare performances across different configurations and easily retrieve the best models. \n", + "\n", + "Here’s what we can log: \n", + "\n", + "- **Parameters** → Values used to configure the model, such as `n_estimators` (number of trees in Random Forest) and other hyperparameters. \n", + "- **Metrics** → Performance indicators of the model, such as **accuracy**, **F1-score**, precision, and recall. \n", + "- **Model** → The trained model version, which can be reloaded and reused without retraining. \n", + "\n", + "By logging these elements, we can analyze and compare different model versions in a structured and reproducible way." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Use the same type of model used in Part 1\n", + "import os\n", + "from sklearn.ensemble import RandomForestClassifier\n", + "from sklearn.metrics import accuracy_score, f1_score\n", + "from mlflow.models.signature import infer_signature\n", + "\n", + "# Execute 4 experiments to train multiple models \n", + "n_estimators = [1, 10, 100, 500]\n", + "\n", + "for n_e in n_estimators:\n", + " # Create a new MLFlow run\n", + " with mlflow.start_run():\n", + " # Log Param\n", + " mlflow.log_param(\"n_estimators\", n_e)\n", + "\n", + " # Create and train the model instance\n", + " rf_model = RandomForestClassifier(n_estimators=n_e, random_state=42)\n", + " rf_model.fit(X_train, y_train)\n", + "\n", + " # Compute metrics\n", + " y_pred = rf_model.predict(X_test)\n", + " accuracy = accuracy_score(y_test, y_pred)\n", + " f1 = f1_score(y_test, y_pred, average='weighted')\n", + " mlflow.log_metric(\"accuracy\", accuracy)\n", + " mlflow.log_metric(\"f1\", f1)\n", + "\n", + " # Create heatmap with Seaborn\n", + " target_names = [\"No Rain\", \"Rain\"]\n", + " cm = confusion_matrix(y_test, y_pred)\n", + " plt.figure(figsize=(6, 4))\n", + " sns.heatmap(cm, annot=True, fmt=\"d\", cmap=\"Blues\", xticklabels=target_names, yticklabels=target_names)\n", + "\n", + " # Add titles\n", + " plt.xlabel(\"Predicted Label\")\n", + " plt.ylabel(\"True Label\")\n", + " plt.title(\"Confusion Matrix\")\n", + "\n", + " # Save the plot as PNG\n", + " if not os.path.exists(\"dev/\"):\n", + " os.makedirs(\"dev/\")\n", + " plt.savefig(\"dev/confusion_matrix.png\")\n", + " plt.close()\n", + " # Save the confusion matrix on MLFlow as artifact\n", + " mlflow.log_artifact(\"dev/confusion_matrix.png\")\n", + "\n", + " # Save the model on MLFlow\n", + " example_dict = {'MinTemp': 1.1, 'MaxTemp': 1.1, 'Humidity3pm': 1.1, 'Humidity9am': 1.1}\n", + " signature = infer_signature(model_input=example_dict)\n", + " mlflow.sklearn.log_model(rf_model, \"random_forest_model\", signature=signature)\n", + "\n", + " print(f\"Experiment finished. Registered accuracy: {accuracy:.2f}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 3. Viewing and Comparing Results \n", + "\n", + "After logging parameters, metrics, and models, we can use **MLflow** to explore and compare different experiments. \n", + "\n", + "MLflow provides a web interface accessible at: \n", + "\n", + "```bash\n", + " http://127.0.0.1:5000\n", + "```\n", + "\n", + "By accessing this interface, in the **Experiments** section, it will be possible to: \n", + "- **Examine the parameters** used in each experiment. \n", + "- **Compare metrics** across different model configurations. \n", + "- **View and download saved models**, making reuse and deployment easier. \n", + "\n", + "This feature allows monitoring the model's performance evolution and quickly identifying the best configurations." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 4. Loading a Saved Model with MLflow \n", + "\n", + "MLflow allows saving and reloading trained models easily, avoiding the need to retrain them every time. \n", + "\n", + "To retrieve a saved model in MLflow, you need to copy the **run ID** of the executed experiment. This ID uniquely identifies each logged experiment and allows loading the corresponding model for future predictions. \n", + "\n", + "This feature is particularly useful for: \n", + "- **Reusing a trained model** without repeating the training process. \n", + "- **Comparing different model versions** to choose the most effective one. \n", + "- **Integrating the model into applications or APIs**, without rebuilding it from scratch. \n", + "\n", + "In the next sections, we will see how to perform this process in practice." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import mlflow.sklearn\n", + "from sklearn.metrics import accuracy_score\n", + "\n", + "# Insert a real run_id you find in MLFlow UI\n", + "RUN_ID = \"\"\n", + "\n", + "loaded_model = mlflow.sklearn.load_model(f\"runs:/{RUN_ID}/random_forest_model\")\n", + "\n", + "# Verify accuracy\n", + "y_loaded_pred = loaded_model.predict(X_test)\n", + "acc_loaded = accuracy_score(y_test, y_loaded_pred)\n", + "print(f\"Accuracy of the loaded model: {acc_loaded:.2f}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Conclusions \n", + "\n", + "In this lab, we followed a complete process to build and monitor a Machine Learning model applied to weather data. Specifically, we: \n", + "\n", + "1. **Created a classification model** using **Scikit-learn**, leveraging temperature and humidity to predict rain. \n", + "2. **Integrated MLflow** to track training parameters, log evaluation metrics, and manage model versions in a structured way. \n", + "3. **Explored and compared results** through the MLflow UI interface, reviewing different configurations and loading a saved model for future predictions. \n", + "\n", + "This approach allows us to improve the Machine Learning model development process, making it more organized, reproducible, and scalable." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Next Steps \n", + "\n", + "Now that we have built and tracked our model, we can explore further improvements and integrate the work into a more advanced workflow. \n", + "\n", + "- **Hyperparameter Optimization**: Test different configurations of `n_estimators`, `max_depth`, and other model parameters, logging results in **MLflow** to identify the best combination. \n", + "- **Automation with CI/CD**: Integrate a **Continuous Integration/Continuous Deployment (CI/CD)** system to automatically train and deploy new model versions, reducing the risk of manual errors. \n", + "- **Model Monitoring in Production**: Implement a **model drift monitoring** system to detect any drops in accuracy over time and determine when retraining with new data is necessary. \n", + "\n", + "These steps help transform the developed model into a robust and reliable system, ready for real-world applications." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.12.9 ('lab_env')", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.9" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "fbd6de99d3dee89886aa6f30475b6b26f6fe2ef531b8e041d1262739f0fd6851" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/Academy/AI/2-ML-Model-Lifecycle/.DS_Store b/Academy/AI/2-ML-Model-Lifecycle/.DS_Store new file mode 100644 index 0000000..b5c4c43 Binary files /dev/null and b/Academy/AI/2-ML-Model-Lifecycle/.DS_Store differ diff --git a/Academy/AI/2-ML-Model-Lifecycle/.gitignore b/Academy/AI/2-ML-Model-Lifecycle/.gitignore new file mode 100644 index 0000000..7869f53 --- /dev/null +++ b/Academy/AI/2-ML-Model-Lifecycle/.gitignore @@ -0,0 +1,180 @@ +# local mlflow +artifacts +mlruns +dev +data + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# UV +# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +#uv.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/latest/usage/project/#working-with-version-control +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +# Ruff stuff: +.ruff_cache/ + +# PyPI configuration file +.pypirc \ No newline at end of file diff --git a/Academy/AI/2-ML-Model-Lifecycle/.pre-commit-config.yaml b/Academy/AI/2-ML-Model-Lifecycle/.pre-commit-config.yaml new file mode 100644 index 0000000..7f9e0df --- /dev/null +++ b/Academy/AI/2-ML-Model-Lifecycle/.pre-commit-config.yaml @@ -0,0 +1,6 @@ +# .pre-commit-config.yaml +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.3.0 + hooks: + - id: ruff diff --git a/Academy/AI/2-ML-Model-Lifecycle/Makefile b/Academy/AI/2-ML-Model-Lifecycle/Makefile new file mode 100644 index 0000000..14b3806 --- /dev/null +++ b/Academy/AI/2-ML-Model-Lifecycle/Makefile @@ -0,0 +1,35 @@ +# Makefile — Local GitHub Actions simulation using nox + conda environments +.PHONY: setup ci train serve monitor retrain pipeline clean +SHELL := /bin/zsh + +# Conda environment name +ENV := lab_env_2 + +# Activate Conda + run nox in the correct environment +CONDA_ACTIVATE = source $$(conda info --base)/etc/profile.d/conda.sh ; conda activate ; conda activate +NOX := $(CONDA_ACTIVATE) $(ENV) && nox --no-venv + +# ------------------------------------------------------------ +# Environment setup +# ------------------------------------------------------------ + +setup: ## Create conda environment + install pre-commit hooks + conda env create -f environment.yaml --quiet + $(NOX) pre-commit install + +# ------------------------------------------------------------ +# Simulated Git branch pushes (as CI/CD entrypoints) +# ------------------------------------------------------------ + +push-feature: ## Simulate push to feature/* → hyperparameter sweep only + $(NOX) -R -s feature_pipeline + +push-develop: ## Simulate push to develop → lint + tests + train best model + $(NOX) -R -s lint + $(NOX) -R -s tests + $(NOX) -R -s develop_pipeline + +push-main: ## Simulate push to master/main → lint + tests + serve model + $(NOX) -R -s lint + $(NOX) -R -s tests + $(NOX) -R -s main_pipeline diff --git a/Academy/AI/2-ML-Model-Lifecycle/README.md b/Academy/AI/2-ML-Model-Lifecycle/README.md new file mode 100644 index 0000000..cbbfc3a --- /dev/null +++ b/Academy/AI/2-ML-Model-Lifecycle/README.md @@ -0,0 +1,383 @@ +# Lab: From Experiments to Production with Wine Classification + +Welcome to this live lab! +In the previous lab session we: + +1. **Created a classification model** using **Scikit-learn**, leveraging temperature and humidity to predict rain. +2. **Integrated MLflow** to track training parameters, log evaluation metrics, and manage model versions in a structured way. +3. **Explored and compared results** through the MLflow UI interface, reviewing different configurations and loading a saved model for future predictions. + +In this session, you will learn how to: + +1. **Train a classification model** on a real dataset using `Scikit-learn`. +2. **Manage and track your experiments** with `MLflow`. +3. **Build CI/CD-like workflows** using `Makefile` and `nox`. +4. **Build scalable and orchestrated ML pipelines** using `Prefect`. +5. **Monitor data drift over time** and trigger retraining with `Evidently`. + +We will follow a step-by-step approach combining **code execution**, **conceptual explanation**, and **UI walkthroughs**. + +--- + +## 1 Context and Story + +A mid‑sized wine producer wants to modernise quality control. +Instead of relying on manual tasting alone, the company decides to analyse +chemical characteristics of each batch and predict the wine’s *cultivar* +(three classes) using machine learning. +The stakes: + +- **Repeatable experimentation** – new blends need fresh tuning +- **Fast promotion** – once a model looks good, it must move to production quickly +- **Operational monitoring** – chemical profiles may drift with soil and weather; the system must recognise drift and retrain autonomously + +Our lab demonstrates how to meet these requirements **entirely on a laptop**, +using open‑source tools and zero cloud resources. + +--- + +## 2 Dataset Overview – UCI Wine Recognition + +The lab uses the well-known [**Wine Recognition**](https://archive.ics.uci.edu/dataset/109/wine) dataset from the UCI Machine Learning Repository. +Originally collected at the Institute of Oenology and Viticulture in Italy, the dataset contains the results of a chemical analysis of **178 wine samples** from three different *cultivars* (grape varieties) grown in the same region. +Each sample is described by **13 continuous variables** that capture key chemical properties influencing flavour and quality. + +### 2.1 Features + +| Feature | Description | Typical Range | +| ------------------------------ | ------------------------------------------- | ------------- | +| `alcohol` | Ethyl-alcohol content (%) | 11 – 14 % | +| `malic_acid` | Malic acid concentration (g/L) | 0.7 – 5.8 | +| `ash` | Total ash (g/100 mL) | 1.3 – 3.3 | +| `alcalinity_of_ash` | Ash alkalinity (mEq NaOH) | 10 – 30 | +| `magnesium` | Magnesium (mg/L) | 70 – 162 | +| `total_phenols` | Total phenolic content (g/L) | 0.9 – 3.9 | +| `flavanoids` | Flavonoid phenols (g/L) | 0.3 – 5.1 | +| `nonflavanoid_phenols` | Non-flavonoid phenols (g/L) | 0.1 – 0.7 | +| `proanthocyanins` | Proanthocyanins (g/L) | 0.4 – 3.6 | +| `color_intensity` | Red/blue color intensity | 1.3 – 13.0 | +| `hue` | Hue at 520 nm relative to 420 nm | 0.4 – 1.8 | +| `od280_od315_of_diluted_wines` | Optical density ratio (aromatic compounds) | 1.3 – 4.0 | +| `proline` | Proline (mg/L) – associated with mouth-feel | 270 – 1680 | + +### 2.2 Target variable: `class` + +The classes are almost balanced (59 / 71 / 48 samples): + +- **Class 0** – Cultivar A +- **Class 1** – Cultivar B +- **Class 2** – Cultivar C + +### 2.3 Why This Dataset Fits the Demo + +1. **Compact and clean** – You can train models in seconds, perfect for live coding. +2. **Chemically interpretable features** – Easy to discuss how drifting alcohol content affects predictions. +3. **Multi-class problem** – Demonstrates probabilities and class selection in the REST API. +4. **No privacy concerns** – The data are public domain, ideal for workshops. + +### 2.4 Synthetic Drift Generation + +To illustrate monitoring, the lab creates a *current batch* where `alcohol` values are artificially increased (e.g., harvest with higher sugar). +Evidently then compares this drifted batch to the original reference set, detects the shift and—if severe—triggers retraining through Prefect. + +--- + +## 3 Tooling Overview + +[End-to-End Workflow](imgs/lab-full.png) + +### 3.1 Tools, concepts and our usage + +| Tool | Key Concept | How We Use It in the Lab | +|------|-------------|--------------------------| +| [**Makefile**](https://www.gnu.org/software/make/manual/make.html) | Declarative build/task runner | Simulates Git push events and invokes Nox sessions (`push-feature`, `push-develop`, `push-main`). | +| [**Nox**](https://nox.thea.codes/en/stable/) | Python automation with virtual/conda envs | Acts like GitHub Actions runners; executes lint, tests and Prefect flows in a reproducible and isolated way. | +| [**Prefect**](https://docs.prefect.io/v3/get-started) | Workflow orchestration + observability | Wraps training, serving and monitoring as flows; offers scheduling, retries and a UI at [http://127.0.0.1:4200](http://127.0.0.1:4200). | +| [**MLflow**](https://mlflow.org/) | Experiment tracking & model registry | Logs parameters, metrics and model artefacts; source of truth for the “best model”. UI at [http://127.0.0.1:5000](http://127.0.0.1:5000). | +| [**Scikit-learn**](https://scikit-learn.org/stable/index.html) | ML algorithms and utilities | Provides a fast RandomForest classifier for our demo. | +| [**FastAPI**](https://fastapi.tiangolo.com/) | High‑performance Python web API | Exposes the model at `/predict` with auto‑generated docs at [http://127.0.0.1:9000](http://127.0.0.1:9000). | +| [**Evidently**](https://docs.evidentlyai.com/introduction) | Data drift & model monitoring | Generates HTML/JSON drift reports; triggers retraining when drift is detected. | + +### 3.2 Why We Simulate GitHub Actions Locally + +In production we would rely on **real Git events and GitHub Actions**: + +| Real event | Real GH Action (example) | Purpose | +| ----------------------------------------- | --------------------------------------------- | ----------------------------------- | +| `git push feature/experiments` | `ci-feature.yml` → sweep experiment | Run many trials, log only metrics | +| `git merge feature/experiments (into develop)` | `ci-develop.yml` → lint + tests + train best | Promote best params, prepare model | +| `git merge develop (into main)` | `ci-main.yml` → lint + tests + deploy best | Bring model online | + +For a laboratory setting those actions pose two practical problems: + +1. **Concurrency & permissions** – the whole group cannot push to `develop` or `main` at the same time without overwriting one another. +2. **Infrastructure dependencies** – real GH Actions would need publicly reachable Prefect and MLflow endpoints; everyone would have to expose tunnels or deploy cloud services. + +To keep the experience **100 % local**, we introduce a one-to-one mapping: + +| Simulated command | What it stands for | +| --------------------------------------------------- | ------------------------------------------------------ | +| `make push-feature` | `git push feature/experiments` trigger GH Action `ci-feature.yml` | +| `make push-develop` | `git merge feature/experiments (into develop)` trigger GH Action `ci-develop.yml` | +| `make push-main` | `git merge develop (into main)` trigger GH Action `ci-main.yml` | +| `python -m src.pipeline_monitoring monitoring_best` | Nightly cron job in GH Actions or Prefect Cloud | + +Under the hood **Nox** replaces the GitHub runner: +`make push-*` commands launch the appropriate nox sessions (lint / tests / Prefect flows) inside a shared Conda environment, reproducing exactly what would happen in cloud CI—but without leaving the laptop. + +--- + +## 4 Environment Setup + +### 4.0 Requirements + +This lab assumes that **Python** and **miniconda** are already installed, the repository [kiratech/labs](https://github.com/kiratech/labs.git) is accessible, and **Git** is properly configured on your local machine. Furthermore, **VSCode** or an IDE able to run Jupyter Notebooks, must be installed as well. +As in the previous lab, in order to execute this laboratory, you will be asked to install a set of tools common in MLOps field. + +### 4.1 Clone the Repository + +To start, clone the lab repository by running the following command in the terminal: + +```sh + git clone https://github.com/kiratech/labs.git +``` + +### 4.2 Checkout the Lab Branch + +After cloning the repository, checkout the `academy-ai` branch: + +```sh + git checkout academy-ai +``` + +Then, navigate to the project folder: + +```sh + cd labs/Academy/AI/2-ML-Model-Lifecycle +``` + +This folder contains resources related to this lab. + +### 4.3 Create a Virtual Environment + +A virtual environment allows you to isolate the project's dependencies from the system-wide ones. +In the previous lab, we first created the environment and then installed the dependencies. This time we'll do it with a single command, creating the environment with all the necessary dependencies. + +In your terminal, create a virtual environment from an existing file: + +```sh + conda env create -f environment.yaml +``` + +Activate the Virtual Environment: + +```sh + conda activate lab_env_2 +``` + +You should see the `(lab_env_2)` prefix in the terminal, indicating that the virtual environment is active. + +### 4.4 Open the Project in VSCode + +At this point, open VSCode from the file explorer or by running the command: + +```sh + code . +``` + +### 4.5 Start the services + +Open three integrated terminals and in each one activate the environemnt with `conda activate lab_env_2` and then: + +| Terminal | Command | Purpose | +|----------|---------|---------| +| T‑1 | `prefect server start` | Prefect API + UI | +| T‑2 | `mlflow ui` | Experiment tracking UI | +| T‑3 | `-` | Run workflow commands (next sections) | + +### 4.6 Open services UI + +Open in the browser Prefect and MLflow at: +| Service | Address | +|----------|---------| +| Prefect | [http://127.0.0.1:4200](http://127.0.0.1:4200) | +| MLflow | [http://127.0.0.1:5000](http://127.0.0.1:5000) | + +--- + +## 5 Branch Workflows + +### 5.0 What we will do + +A production-style GitHub Actions workflow we want to **experience locally**. +In the laboratory we replay each numbered block with Makefile + Nox + Prefect, avoiding concurrent pushes and external infrastructure. + +[End-to-End Workflow](imgs/lab-full.png) + +### 5.1 Feature Branch Workflow + +[Feature Workflow](imgs/lab-part1.png) + +- **Command** + In your terminal, once the conda environment is active, run: + + ```sh + make push-feature + ``` + +- **Goal** + Explore the parameter space quickly and cheaply. No model is intended for production at this stage. + We only want evidence that “something promising” exists. +- **What the Command Simulates** + A developer pushes code to **`feature/`**. + CI/CD should **only run experiments**, producing metrics the team can compare later. +- **What It Does** + + - A Nox session calls Prefect flow `train_experiment`. + - 9 RandomForest training tasks run in parallel (3  `n_estimators` × 3  `max_depth`). + - Each task logs **parameters + accuracy** to MLflow, but **does not store a model file**. + - Prefect captures task logs and execution graph. + +- **What to Explore** + + - MLflow UI: nine runs with different hyper‑params, no artifacts saved. + - Prefect UI: one flow, nine parallel tasks—visual confirmation of parallel runs. + +### 5.2 Develop Branch Workflow  + +[Develop Workflow](imgs/lab-part2.png) + +- **Command** + In your terminal, once the conda environment is active, run: + + ```sh + make push-develop + ``` + +- **Goal** + Promote the best experimental configuration, validate code quality, and log a **deployable model artifact**. +- **What the Command Simulates** + A merge/push to **`develop`**. CI/CD should lint, test, **re‑train with chosen hyper‑params**, and register the resulting model. +- **What It Does** + + 1. **Lint** (`ruff via pre‑commit`) and **unit tests** (`pytest`) run first. Build stops on failure. + 2. Prefect flow `train_best` queries MLflow, grabs the run with highest accuracy. + 3. It re‑trains a RandomForest using those parameters on fresh data splits. + 4. Saves the `.pkl` artifact to `artifacts/` and logs it to MLflow. + +- **What to Explore** + + - Terminal: lint/test output. + - MLflow UI: a new run with an **artifact path**, this is the candidate for production. + - Prefect UI: see the “find best params” task feeding the “train” task. + +### 5.3 Main Branch Workflow + +[Main Workflow](imgs/lab-part3.png) + +- **Command** + In your terminal, once the conda environment is active, run: + + ```sh + make push-main + ``` + +- **Goal** + Deploy the champion model as an HTTP service usable by downstream teams. +- **What the Command Simulates** + A merge/push to **`main`**. CI/CD should perform a last sanity check, then bring the model online. +- **What It Does** + + 1. Re‑runs lint and tests (quick safety net). + 2. Prefect flow `serve_best` downloads the best model artifact from MLflow. + 3. Builds a FastAPI app and launches Uvicorn on **port 9000**. + +- **What to Explore** + + - Swagger UI at `http://127.0.0.1:9000/docs`: live documentation, try a prediction. + - Prefect UI: a small flow made of download, build app and serve. + - MLflow UI: confirm the run ID of the served model matches the develop stage. + +### 5.4 Querying the Best Model + +Once the **main branch workflow** has deployed the FastAPI service, you can send a prediction request directly from the terminal: + +```bash +curl -X POST http://localhost:9000/predict \ + -H "Content-Type: application/json" \ + -d '{ + "alcohol": 13.0, + "malic_acid": 2.0, + "ash": 2.5, + "alcalinity_of_ash": 15.0, + "magnesium": 100.0, + "total_phenols": 2.0, + "flavanoids": 2.5, + "nonflavanoid_phenols": 0.3, + "proanthocyanins": 1.9, + "color_intensity": 5.0, + "hue": 1.0, + "od280_od315_of_diluted_wines": 3.0, + "proline": 1000 + }' +``` + +Expected JSON response: + +```json +{ + "class": 1, + "proba": [0.02, 0.95, 0.03] +} +``` + +--- + +## 6 Monitoring & Auto‑Retraining + +[Feature Workflow](imgs/lab-part4.png) + +- **Command** + In your terminal, once the conda environment is active, run: + + ```sh + python -m src.pipeline_monitoring monitoring_best + ``` + +- **Goal** + Detect distributional drift and kick off retraining only when needed. +- **What the Command Simulates** + A **scheduled batch job** (cron in Prefect) that runs nightly, comparing that day’s data against a reference baseline. +- **What It Does** + + - Generates (or ingests) the **current batch**. Here we synthesise drift by nudging alcohol levels. + - Evidently creates an HTML + JSON drift report. + - Prefect parses the `alchol` feature p‑value between the training set distribution and the new drifted set distribution. If ≤ 0.05, it calls the same `train_best` flow used on develop. + - All actions—report generation and optional retraining—are logged in Prefect and MLflow. + +- **What to Explore** + + - Drift Report: open `artifacts/drift_report.html`; discuss which features drifted. + - Prefect UI: see conditional branching—one path ends, the other chains into a training flow. + - MLflow UI: a new run appears **only** when drift threshold is exceeded, proving closed‑loop automation. + +--- + +## 7 Conclusions + +Over the course of this lab we have: + +- **Simulated a full CI/CD/CT loop locally** + using Makefile to trigger branch‑style workflows and Nox as a lightweight stand‑in for GitHub Actions. +- **Captured the complete model lineage** + in MLflow, from exploratory runs to the promoted, deployable artifact. +- **Orchestrated repeatable pipelines** + with Prefect, gaining retry logic, scheduling, and an audit‑friendly UI. +- **Deployed an inference endpoint** + via FastAPI that is immediately testable through cURL or Swagger docs. +- **Closed the monitoring loop** + by integrating Evidently to detect drift and automatically retrain when data shifts. + +Together these elements demonstrate an end‑to‑end, production‑style MLOps workflow that fits entirely on a developer laptop—yet scales conceptually to cloud or on‑prem environments. +The key takeaway: **tooling synergy** matters more than individual components. +By combining focused, purpose‑built tools, we achieve reproducibility, observability and automation without heavyweight infrastructure. diff --git a/Academy/AI/2-ML-Model-Lifecycle/environment.yaml b/Academy/AI/2-ML-Model-Lifecycle/environment.yaml new file mode 100644 index 0000000..6a85694 --- /dev/null +++ b/Academy/AI/2-ML-Model-Lifecycle/environment.yaml @@ -0,0 +1,236 @@ +name: lab_env_2 +channels: + - conda-forge + - defaults +dependencies: + - appnope=0.1.2 + - argcomplete=1.12.3 + - asttokens=3.0.0 + - attrs=25.3.0 + - backports=1.0 + - backports.functools_lru_cache=2.0.0 + - bzip2=1.0.8 + - ca-certificates=2025.2.25 + - cachetools=5.5.2 + - chardet=5.2.0 + - colorama=0.4.6 + - colorlog=5.0.1 + - comm=0.2.1 + - debugpy=1.8.11 + - decorator=5.1.1 + - dependency-groups=1.3.0 + - distlib=0.3.8 + - exceptiongroup=1.2.2 + - executing=0.8.3 + - filelock=3.18.0 + - importlib_metadata=4.13.0 + - ipykernel=6.29.5 + - ipython=9.1.0 + - ipython_pygments_lexers=1.1.1 + - jedi=0.19.2 + - jinja2=3.1.6 + - jupyter_client=8.6.3 + - jupyter_core=5.7.2 + - libcxx=14.0.6 + - libexpat=2.7.0 + - libffi=3.4.6 + - liblzma=5.8.1 + - libsodium=1.0.18 + - libsqlite=3.49.1 + - libzlib=1.3.1 + - markupsafe=3.0.2 + - matplotlib-inline=0.1.6 + - ncurses=6.5 + - nest-asyncio=1.6.0 + - nox=2025.2.9 + - openssl=3.5.0 + - packaging=24.2 + - parso=0.8.4 + - pexpect=4.8.0 + - pickleshare=0.7.5 + - pip=25.0.1 + - platformdirs=4.3.7 + - pluggy=1.5.0 + - prompt-toolkit=3.0.43 + - prompt_toolkit=3.0.43 + - psutil=5.9.0 + - ptyprocess=0.7.0 + - pure_eval=0.2.2 + - pygments=2.19.1 + - pyproject-api=1.9.0 + - python=3.11.12 + - python-dateutil=2.9.0post0 + - python_abi=3.11 + - pyzmq=26.2.0 + - readline=8.2 + - setuptools=78.1.0 + - six=1.17.0 + - stack_data=0.2.0 + - tk=8.6.13 + - tomli=2.2.1 + - tornado=6.4.2 + - tox=4.25.0 + - traitlets=5.14.3 + - virtualenv=20.30.0 + - wcwidth=0.2.5 + - wheel=0.45.1 + - zeromq=4.3.5 + - zipp=3.21.0 + - pip: + - aiosqlite==0.21.0 + - alembic==1.15.2 + - annotated-types==0.7.0 + - anyio==4.6.0 + - appdirs==1.4.4 + - apprise==1.9.3 + - asgi-lifespan==2.1.0 + - async-timeout==5.0.1 + - asyncpg==0.29.0 + - blinker==1.9.0 + - certifi==2025.1.31 + - cffi==1.17.1 + - cfgv==3.4.0 + - charset-normalizer==3.4.1 + - click==8.1.8 + - cloudpickle==3.1.1 + - contourpy==1.3.2 + - coolname==2.2.0 + - croniter==5.0.1 + - cryptography==44.0.2 + - cycler==0.12.1 + - databricks-sdk==0.50.0 + - dateparser==1.2.1 + - deprecated==1.2.18 + - deprecation==2.1.0 + - distro==1.9.0 + - dnspython==2.7.0 + - docker==7.1.0 + - durationpy==0.9 + - dynaconf==3.2.10 + - email-validator==2.2.0 + - evidently==0.4.40 + - faker==37.1.0 + - fastapi==0.115.12 + - flask==3.1.0 + - fonttools==4.57.0 + - fsspec==2025.3.2 + - gitdb==4.0.12 + - gitpython==3.1.44 + - google-auth==2.39.0 + - graphene==3.4.3 + - graphql-core==3.2.6 + - graphql-relay==3.2.0 + - greenlet==3.2.0 + - griffe==1.7.2 + - gunicorn==23.0.0 + - h11==0.14.0 + - h2==4.2.0 + - hpack==4.1.0 + - httpcore==1.0.8 + - httptools==0.6.4 + - httpx==0.28.1 + - humanize==4.12.2 + - hyperframe==6.1.0 + - identify==2.6.10 + - idna==3.10 + - importlib-metadata==8.6.1 + - importlib-resources==6.4.5 + - iniconfig==2.1.0 + - iterative-telemetry==0.0.10 + - itsdangerous==2.2.0 + - jinja2-humanize-extension==0.4.0 + - joblib==1.4.2 + - jsonpatch==1.33 + - jsonpointer==3.0.0 + - jsonschema==4.23.0 + - jsonschema-specifications==2024.10.1 + - kiwisolver==1.4.8 + - kubernetes==31.0.0 + - litestar==2.15.2 + - litestar-htmx==0.4.1 + - mako==1.3.10 + - markdown==3.8 + - markdown-it-py==3.0.0 + - matplotlib==3.10.1 + - mdurl==0.1.2 + - mlflow==2.21.3 + - mlflow-skinny==2.21.3 + - msgspec==0.19.0 + - multidict==6.4.3 + - multipart==1.2.1 + - mypy-extensions==1.0.0 + - nltk==3.9.1 + - nodeenv==1.9.1 + - numpy==2.0.2 + - oauthlib==3.2.2 + - opentelemetry-api==1.32.1 + - opentelemetry-sdk==1.32.1 + - opentelemetry-semantic-conventions==0.53b1 + - orjson==3.10.16 + - pandas==2.2.3 + - pathspec==0.12.1 + - patsy==1.0.1 + - pendulum==2.1.2 + - pillow==11.2.1 + - plotly==5.24.1 + - polyfactory==2.21.0 + - pre-commit==4.2.0 + - prefect==2.20.17 + - protobuf==5.29.4 + - pyarrow==19.0.1 + - pyasn1==0.6.1 + - pyasn1-modules==0.4.2 + - pycparser==2.22 + - pydantic==2.11.3 + - pydantic-core==2.33.1 + - pyparsing==3.2.3 + - pytest==8.3.5 + - python-dotenv==1.1.0 + - python-multipart==0.0.20 + - python-slugify==8.0.4 + - pytz==2024.2 + - pytzdata==2020.1 + - pyyaml==6.0.2 + - readchar==4.2.1 + - referencing==0.36.2 + - regex==2024.11.6 + - requests==2.32.3 + - requests-oauthlib==2.0.0 + - rfc3339-validator==0.1.4 + - rich==13.9.4 + - rich-click==1.8.8 + - rpds-py==0.24.0 + - rsa==4.9.1 + - ruamel-yaml==0.18.10 + - ruamel-yaml-clib==0.2.12 + - scikit-learn==1.6.1 + - scipy==1.15.2 + - shellingham==1.5.4 + - smmap==5.0.2 + - sniffio==1.3.1 + - sqlalchemy==2.0.35 + - sqlparse==0.5.3 + - starlette==0.46.2 + - statsmodels==0.14.4 + - tenacity==9.1.2 + - text-unidecode==1.3 + - threadpoolctl==3.6.0 + - toml==0.10.2 + - tqdm==4.67.1 + - typer==0.15.2 + - typing-extensions==4.13.2 + - typing-inspect==0.9.0 + - typing-inspection==0.4.0 + - tzdata==2025.2 + - tzlocal==5.3.1 + - ujson==5.10.0 + - urllib3==2.4.0 + - uuid6==2024.7.10 + - uvicorn==0.34.1 + - uvloop==0.21.0 + - watchdog==6.0.0 + - watchfiles==1.0.5 + - websocket-client==1.8.0 + - websockets==13.1 + - werkzeug==3.1.3 + - wrapt==1.17.2 diff --git a/Academy/AI/2-ML-Model-Lifecycle/imgs/lab-full.png b/Academy/AI/2-ML-Model-Lifecycle/imgs/lab-full.png new file mode 100644 index 0000000..a624c0f Binary files /dev/null and b/Academy/AI/2-ML-Model-Lifecycle/imgs/lab-full.png differ diff --git a/Academy/AI/2-ML-Model-Lifecycle/imgs/lab-part1.png b/Academy/AI/2-ML-Model-Lifecycle/imgs/lab-part1.png new file mode 100644 index 0000000..f2cc896 Binary files /dev/null and b/Academy/AI/2-ML-Model-Lifecycle/imgs/lab-part1.png differ diff --git a/Academy/AI/2-ML-Model-Lifecycle/imgs/lab-part2.png b/Academy/AI/2-ML-Model-Lifecycle/imgs/lab-part2.png new file mode 100644 index 0000000..416b1c3 Binary files /dev/null and b/Academy/AI/2-ML-Model-Lifecycle/imgs/lab-part2.png differ diff --git a/Academy/AI/2-ML-Model-Lifecycle/imgs/lab-part3.png b/Academy/AI/2-ML-Model-Lifecycle/imgs/lab-part3.png new file mode 100644 index 0000000..3dc2dca Binary files /dev/null and b/Academy/AI/2-ML-Model-Lifecycle/imgs/lab-part3.png differ diff --git a/Academy/AI/2-ML-Model-Lifecycle/imgs/lab-part4.png b/Academy/AI/2-ML-Model-Lifecycle/imgs/lab-part4.png new file mode 100644 index 0000000..536e770 Binary files /dev/null and b/Academy/AI/2-ML-Model-Lifecycle/imgs/lab-part4.png differ diff --git a/Academy/AI/2-ML-Model-Lifecycle/noxfile.py b/Academy/AI/2-ML-Model-Lifecycle/noxfile.py new file mode 100644 index 0000000..1f11bf7 --- /dev/null +++ b/Academy/AI/2-ML-Model-Lifecycle/noxfile.py @@ -0,0 +1,79 @@ +# noxfile.py — Local CI/CD simulation using Nox sessions +import nox + +# Reuse virtualenvs across runs to avoid reinstallation +nox.options.reuse_existing_virtualenvs = True +# Shared environment for all sessions +nox.options.envdir = ".nox/shared" + +# Python version to use in all sessions +PYTHON = "3.11" + +# ------------------------------------------------------------ +# Shared setup for installing local package +# ------------------------------------------------------------ + + +def install_project(session): + """ + Install the local src/ package in editable mode. + """ + session.install("-e", ".") + + +# ------------------------------------------------------------ +# Core CI checks: linting and unit tests +# ------------------------------------------------------------ + + +@nox.session(python=PYTHON) +def lint(session): + """ + Run pre-commit hooks for linting and formatting. + """ + session.run("pre-commit", "run", "--all-files", "--hook-stage", "manual") + +@nox.session(python=PYTHON) +def tests(session): + """ + Run unit tests using pytest. + """ + install_project(session) + session.run("pytest", "-q") + + +# ------------------------------------------------------------ +# Simulated ML workflows triggered by Git pushes +# ------------------------------------------------------------ + + +@nox.session(python=PYTHON) +def feature_pipeline(session): + """ + Simulates a push to feature/*: + - Runs hyperparameter sweep (no model saved). + """ + install_project(session) + session.run("python", "-m", "src.pipeline_ml", "train_experiment") + + +@nox.session(python=PYTHON) +def develop_pipeline(session): + """ + Simulates a push to develop: + - Trains the model using best hyperparameters. + - Logs metrics and saves model to MLflow. + """ + install_project(session) + session.run("python", "-m", "src.pipeline_ml", "train_best") + + +@nox.session(python=PYTHON) +def main_pipeline(session): + """ + Simulates a push to main: + - Downloads the best saved model. + - Serves the model through FastAPI. + """ + install_project(session) + session.run("python", "-m", "src.pipeline_ml", "serve_best") diff --git a/Academy/AI/2-ML-Model-Lifecycle/pyproject.toml b/Academy/AI/2-ML-Model-Lifecycle/pyproject.toml new file mode 100644 index 0000000..734e0c8 --- /dev/null +++ b/Academy/AI/2-ML-Model-Lifecycle/pyproject.toml @@ -0,0 +1,19 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "mlops_lab" +version = "0.1.0" +description = "End-to-end MLOps lab pipeline" +authors = [{ name = "ranl", email = "lorenzo.randazzo@kiratech.ch" }] +readme = "README.md" +license = { text = "MIT" } +dynamic = ["dependencies"] +requires-python = ">=3.11" + +[tool.setuptools] +packages = ["src"] + +[tool.setuptools.dynamic] +dependencies = {file = ["requirements.txt"]} diff --git a/Academy/AI/2-ML-Model-Lifecycle/requirements.txt b/Academy/AI/2-ML-Model-Lifecycle/requirements.txt new file mode 100644 index 0000000..05e2720 --- /dev/null +++ b/Academy/AI/2-ML-Model-Lifecycle/requirements.txt @@ -0,0 +1,212 @@ +aiosqlite==0.21.0 +alembic==1.15.2 +annotated-types==0.7.0 +anyio==4.6.0 +appdirs==1.4.4 +appnope==0.1.2 +apprise==1.9.3 +argcomplete==1.12.3 +asgi-lifespan==2.1.0 +asttokens==3.0.0 +async-timeout==5.0.1 +asyncpg==0.29.0 +attrs==25.3.0 +backports.functools_lru_cache==2.0.0 +blinker==1.9.0 +cachetools==5.5.2 +certifi==2025.1.31 +cffi==1.17.1 +cfgv==3.4.0 +chardet==5.2.0 +charset-normalizer==3.4.1 +click==8.1.8 +cloudpickle==3.1.1 +colorama==0.4.6 +colorlog==5.0.1 +comm==0.2.1 +contourpy==1.3.2 +coolname==2.2.0 +croniter==5.0.1 +cryptography==44.0.2 +cycler==0.12.1 +databricks-sdk==0.50.0 +dateparser==1.2.1 +debugpy==1.8.11 +decorator==5.1.1 +dependency-groups==1.3.0 +Deprecated==1.2.18 +deprecation==2.1.0 +distlib==0.3.8 +distro==1.9.0 +dnspython==2.7.0 +docker==7.1.0 +durationpy==0.9 +dynaconf==3.2.10 +email_validator==2.2.0 +evidently==0.4.40 +exceptiongroup==1.2.2 +executing==0.8.3 +Faker==37.1.0 +fastapi==0.115.12 +filelock==3.18.0 +Flask==3.1.0 +fonttools==4.57.0 +fsspec==2025.3.2 +gitdb==4.0.12 +GitPython==3.1.44 +google-auth==2.39.0 +graphene==3.4.3 +graphql-core==3.2.6 +graphql-relay==3.2.0 +graphviz==0.20.3 +greenlet==3.2.0 +griffe==1.7.2 +gunicorn==23.0.0 +h11==0.14.0 +h2==4.2.0 +hpack==4.1.0 +httpcore==1.0.8 +httptools==0.6.4 +httpx==0.28.1 +humanize==4.12.2 +hyperframe==6.1.0 +identify==2.6.10 +idna==3.10 +importlib_metadata==8.6.1 +importlib_resources==6.4.5 +iniconfig==2.1.0 +ipykernel==6.29.5 +ipython==9.1.0 +ipython_pygments_lexers==1.1.1 +iterative-telemetry==0.0.10 +itsdangerous==2.2.0 +jedi==0.19.2 +Jinja2==3.1.6 +jinja2-humanize-extension==0.4.0 +joblib==1.4.2 +jsonpatch==1.33 +jsonpointer==3.0.0 +jsonschema==4.23.0 +jsonschema-specifications==2024.10.1 +jupyter_client==8.6.3 +jupyter_core==5.7.2 +kiwisolver==1.4.8 +kubernetes==31.0.0 +litestar==2.15.2 +litestar-htmx==0.4.1 +Mako==1.3.10 +Markdown==3.8 +markdown-it-py==3.0.0 +MarkupSafe==3.0.2 +matplotlib==3.10.1 +matplotlib-inline==0.1.6 +mdurl==0.1.2 +mlflow==2.21.3 +mlflow-skinny==2.21.3 +msgspec==0.19.0 +multidict==6.4.3 +multipart==1.2.1 +mypy-extensions==1.0.0 +nest_asyncio==1.6.0 +nltk==3.9.1 +nodeenv==1.9.1 +nox==2025.2.9 +numpy==2.0.2 +oauthlib==3.2.2 +opentelemetry-api==1.32.1 +opentelemetry-sdk==1.32.1 +opentelemetry-semantic-conventions==0.53b1 +orjson==3.10.16 +packaging==24.2 +pandas==2.2.3 +parso==0.8.4 +pathspec==0.12.1 +patsy==1.0.1 +pendulum==2.1.2 +pexpect==4.8.0 +pickleshare==0.7.5 +pillow==11.2.1 +pip==25.0.1 +platformdirs==4.3.7 +plotly==5.24.1 +pluggy==1.5.0 +polyfactory==2.21.0 +pre_commit==4.2.0 +prefect==2.20.17 +prompt_toolkit==3.0.43 +protobuf==5.29.4 +psutil==5.9.0 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow==19.0.1 +pyasn1==0.6.1 +pyasn1_modules==0.4.2 +pycparser==2.22 +pydantic==2.11.3 +pydantic_core==2.33.1 +Pygments==2.19.1 +pyparsing==3.2.3 +pyproject-api==1.9.0 +pytest==8.3.5 +python-dateutil==2.9.0.post0 +python-dotenv==1.1.0 +python-multipart==0.0.20 +python-slugify==8.0.4 +pytz==2024.2 +pytzdata==2020.1 +PyYAML==6.0.2 +pyzmq==26.2.0 +readchar==4.2.1 +referencing==0.36.2 +regex==2024.11.6 +requests==2.32.3 +requests-oauthlib==2.0.0 +rfc3339-validator==0.1.4 +rich==13.9.4 +rich-click==1.8.8 +rpds-py==0.24.0 +rsa==4.9.1 +ruamel.yaml==0.18.10 +ruamel.yaml.clib==0.2.12 +scikit-learn==1.6.1 +scipy==1.15.2 +setuptools==78.1.0 +shellingham==1.5.4 +six==1.17.0 +smmap==5.0.2 +sniffio==1.3.1 +SQLAlchemy==2.0.35 +sqlparse==0.5.3 +stack-data==0.2.0 +starlette==0.46.2 +statsmodels==0.14.4 +tenacity==9.1.2 +text-unidecode==1.3 +threadpoolctl==3.6.0 +toml==0.10.2 +tomli==2.2.1 +tornado==6.4.2 +tox==4.25.0 +tqdm==4.67.1 +traitlets==5.14.3 +typer==0.15.2 +typing_extensions==4.13.2 +typing-inspect==0.9.0 +typing-inspection==0.4.0 +tzdata==2025.2 +tzlocal==5.3.1 +ujson==5.10.0 +urllib3==2.4.0 +uuid6==2024.7.10 +uvicorn==0.34.1 +uvloop==0.21.0 +virtualenv==20.30.0 +watchdog==6.0.0 +watchfiles==1.0.5 +wcwidth==0.2.5 +websocket-client==1.8.0 +websockets==13.1 +Werkzeug==3.1.3 +wheel==0.45.1 +wrapt==1.17.2 +zipp==3.21.0 diff --git a/Academy/AI/2-ML-Model-Lifecycle/src/__init__.py b/Academy/AI/2-ML-Model-Lifecycle/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Academy/AI/2-ML-Model-Lifecycle/src/config.py b/Academy/AI/2-ML-Model-Lifecycle/src/config.py new file mode 100644 index 0000000..56f3add --- /dev/null +++ b/Academy/AI/2-ML-Model-Lifecycle/src/config.py @@ -0,0 +1,54 @@ +""" +src/config.py +Configuration module for the Wine Quality pipeline. + +- Defines global paths, MLflow settings, artifact naming, and Evidently monitoring endpoints. +- Uses Pydantic's BaseModel for structured configuration. +""" + +from pathlib import Path + +from pydantic import BaseModel + + +class Settings(BaseModel): + """ + Centralized application settings. + + Attributes: + project_root (Path): Root directory of the project. + artifacts_dir (Path): Directory where artifacts (models, reports) will be stored. + model_name (str): Default filename for the trained model artifact. + mlflow_uri (str): URI where MLflow tracking server or local files are stored. + experiment_name (str): Name of the MLflow experiment. + evidently_url (str): Base URL where the Evidently UI dashboard is running. + evidently_project (str): Project name under Evidently for monitoring. + """ + + # Root of the project (assumes config.py is in src/) + project_root: Path = Path(__file__).resolve().parents[1] + + # Directory for storing artifacts (models, drift reports, etc.) + artifacts_dir: Path = project_root / "artifacts" + + # Default model filename for serialization + model_name: str = "wine_rf.pkl" + + # MLflow tracking URI (using local file storage in 'mlruns/') + mlflow_uri: str = f"file:{project_root/'mlruns'}" + + # MLflow experiment name for tracking runs + experiment_name: str = "WineSweepDemo" + + # Evidently monitoring - URL to the local Evidently UI server + evidently_url: str = "http://localhost:8000" + + # Evidently monitoring - Name of the project for batch drift tracking + evidently_project: str = "wine_quality_monitoring" + + +# Singleton instance used throughout the project +CFG = Settings() + +# Ensure artifacts directory exists (create if it doesn't) +CFG.artifacts_dir.mkdir(exist_ok=True) diff --git a/Academy/AI/2-ML-Model-Lifecycle/src/data.py b/Academy/AI/2-ML-Model-Lifecycle/src/data.py new file mode 100644 index 0000000..d8a5843 --- /dev/null +++ b/Academy/AI/2-ML-Model-Lifecycle/src/data.py @@ -0,0 +1,104 @@ +""" +src/data.py +Data handling module for the Wine Quality pipeline. + +Provides: +- data_collection: Loads the wine dataset from scikit-learn. +- new_data_collection: Simulates a new batch with data drift in alcohol. +- data_preparation: Prepares the dataset (renames columns, performs train-test split). +""" + +import random + +import pandas as pd +import typer +from sklearn.datasets import load_wine +from sklearn.model_selection import train_test_split +from src.utils import get_logger + +app = typer.Typer(add_completion=False) + + +def data_collection() -> tuple[pd.DataFrame, pd.Series]: + """ + Loads the wine dataset. + + Returns: + tuple: + - X (pd.DataFrame): Feature matrix. + - y (pd.Series): Target labels. + """ + logger = get_logger() + logger.info("Loading Wine dataset...") + X, y = load_wine(return_X_y=True, as_frame=True) + logger.info( + f"Wine dataset loaded with {X.shape[0]} samples and {X.shape[1]} features." + ) + return (X, y) + + +def new_data_collection() -> pd.DataFrame: + """ + Loads the wine dataset and introduces data drift in the 'alcohol' feature. + + Randomly perturbs alcohol values within a multiplier range + to simulate a distributional change for monitoring. + + Returns: + pd.DataFrame: Drifted dataset (features only, no target). + """ + X, _ = data_collection() + df = X.copy() + + logger = get_logger() + logger.info("Generating drifted data batch...") + + # Filter high alcohol wines + df_filtered = df[df["alcohol"] > 13] + + # Apply random scaling to simulate drift + rnd_drift = random.uniform(1.03, 1.13) + df_filtered["alcohol"] = df["alcohol"] * rnd_drift + logger.warning(f"Random drif generated for alchol with coefficient {rnd_drift}.") + # Sample up to 100 instances + df_current = df_filtered.sample(min(100, len(df_filtered)), random_state=0) + return df_current + + +def data_preparation( + dataset: tuple[pd.DataFrame, pd.Series], split: float = 0.2, seed: int = 42 +) -> tuple[pd.DataFrame, pd.DataFrame, pd.Series, pd.Series]: + """ + Prepares the dataset for training: + - Renames any problematic columns (e.g., replaces slashes). + - Splits the dataset into train and test sets. + + Args: + dataset (tuple): Tuple containing (X, y). + split (float, optional): Proportion of the dataset to include in the test split (default: 0.2). + seed (int, optional): Random seed for reproducibility (default: 42). + + Returns: + tuple: + - X_train (pd.DataFrame): Training features. + - X_test (pd.DataFrame): Testing features. + - y_train (pd.Series): Training labels. + - y_test (pd.Series): Testing labels. + """ + X, y = dataset + logger = get_logger() + # Ensure column names are safe (replace slashes with underscores) + logger.info("Replacing / with - in column names...") + X.columns = X.columns.str.replace("/", "_") + + # Split into train/test + logger.info(f"Preparing dataset: split={split}, seed={seed}...") + X_tr, X_te, y_tr, y_te = train_test_split(X, y, test_size=split, random_state=seed) + logger.info( + f"Dataset split into {X_tr.shape[0]} train and {X_te.shape[0]} test samples." + ) + return (X_tr, X_te, y_tr, y_te) + + +if __name__ == "__main__": + app() diff --git a/Academy/AI/2-ML-Model-Lifecycle/src/monitor.py b/Academy/AI/2-ML-Model-Lifecycle/src/monitor.py new file mode 100644 index 0000000..56b6bdc --- /dev/null +++ b/Academy/AI/2-ML-Model-Lifecycle/src/monitor.py @@ -0,0 +1,107 @@ +""" +src/monitor.py +Data drift monitoring module using Evidently. + +- Compares current data to a reference dataset. +- Generates an HTML and JSON report saved to artifacts/. +- Can be called via CLI or integrated as a Prefect task. +""" + +import json +from pathlib import Path + +import pandas as pd +import typer +from evidently.metric_preset import DataDriftPreset +from evidently.report import Report +from src.utils import get_logger +from src.config import CFG + +app = typer.Typer(add_completion=False) + + +def compute_drift( + output_json: Path = Path(CFG.artifacts_dir / "drift_report.json"), +) -> float: + """ + Computes the minimum drift score from an Evidently JSON report. + + Args: + output_json (Path, optional): Path to the JSON drift report (default: artifacts/drift_report.json). + + Returns: + float: The minimum drift score across all monitored features. + """ + logger = get_logger() + logger.info("Computing minimum drift score from report...") + + # Load the JSON report + report = json.loads(output_json.read_text()) + pvals = [] + + # Extract drift scores for each monitored column + for attribute, value in report["metrics"][1]["result"]["drift_by_columns"].items(): + pvals.append(value["drift_score"]) + + # Compute and return the minimum p-value + min_p = min(pvals) + logger.info(f"Minimum drift score detected: {min_p}.") + return min_p + + +def generate_drift_report( + reference_data: pd.DataFrame, + current_data: pd.DataFrame, + output_path_html: Path, + output_path_json: Path, +) -> Path: + """ + Generates a drift report comparing current data with reference data. + + Args: + reference_data (pd.DataFrame): DataFrame containing the reference dataset. + current_data (pd.DataFrame): DataFrame containing the current dataset to monitor. + output_path_html (Path): Path to save the generated HTML report. + output_path_json (Path): Path to save the generated JSON report. + + Returns: + Path: Path to the generated JSON report (for downstream processing). + """ + logger = get_logger() + logger.info("Starting drift report generation...") + + # Initialize Evidently report with DataDriftPreset + report = Report(metrics=[DataDriftPreset()]) + report.run(reference_data=reference_data, current_data=current_data) + + # Save the report as both HTML and JSON + output_path_html.parent.mkdir(exist_ok=True) + report.save_html(str(output_path_html)) + report.save_json(str(output_path_json)) + logger.info(f"Drift report saved to {output_path_html} and {output_path_json}.") + return output_path_json + + +def run( + reference_data: pd.DataFrame, + current_data: pd.DataFrame, + output_html: Path = Path(CFG.artifacts_dir / "drift_report.html"), + output_json: Path = Path(CFG.artifacts_dir / "drift_report.json"), +) -> Path: + """ + CLI entry point for generating a drift report. + + Args: + reference_data (pd.DataFrame): DataFrame containing the reference dataset. + current_data (pd.DataFrame): DataFrame containing the current dataset. + output_html (Path, optional): Output path for the HTML report (default: artifacts/drift_report.html). + output_json (Path, optional): Output path for the JSON report (default: artifacts/drift_report.json). + + Returns: + Path: Path to the generated JSON report. + """ + return generate_drift_report(reference_data, current_data, output_html, output_json) + + +if __name__ == "__main__": + app() diff --git a/Academy/AI/2-ML-Model-Lifecycle/src/pipeline_ml.py b/Academy/AI/2-ML-Model-Lifecycle/src/pipeline_ml.py new file mode 100644 index 0000000..a43d425 --- /dev/null +++ b/Academy/AI/2-ML-Model-Lifecycle/src/pipeline_ml.py @@ -0,0 +1,195 @@ +""" +src/pipeline_ml.py +Prefect 2 flows for the Wine Quality ML pipeline. + +Flows: +- train_experiment: Performs a grid search (hyperparameter sweep) without saving models. +- train_best: Trains the best model based on MLflow runs and saves the model artifact. +- serve_best: Downloads the best saved model and serves it through a FastAPI endpoint. +""" + +from itertools import product +from pathlib import Path +import tempfile + +import mlflow +from prefect import flow, task + +from src.config import CFG +from src.data import data_collection as data_coll_run, data_preparation as data_prep_run +from src.train import run as train_run +from src.serve import deploy +from src.utils import get_logger + +# --------------------------------------------------------------------------- +# Tasks +# --------------------------------------------------------------------------- + +@task +def data_collection_task() -> tuple: + """ + Collects the dataset (features and labels). + + Returns: + tuple: (X, y) dataset. + """ + return data_coll_run() + +@task +def data_preparation_task(dataset: tuple) -> tuple: + """ + Prepares the dataset for training: + - Fixes feature names. + - Splits into train/test sets. + + Args: + dataset (tuple): (X, y) dataset. + + Returns: + tuple: (X_train, X_test, y_train, y_test). + """ + return data_prep_run(dataset) + +@task +def model_training_task( + dataset: tuple, n_estimators: int, max_depth: int | None, log_model: bool = False +) -> float: + """ + Trains a RandomForest model and optionally logs it to MLflow. + + Args: + dataset (tuple): Prepared train/test splits. + n_estimators (int): Number of trees. + max_depth (int | None): Maximum depth per tree. + log_model (bool): Whether to save model artifact to MLflow. + + Returns: + float: Model accuracy. + """ + return train_run(dataset, n_estimators=n_estimators, max_depth=max_depth, log_model=log_model) + +@task +def find_best_params_task(metric: str = "accuracy") -> tuple: + """ + Retrieves the best model hyperparameters from MLflow by specified metric. + + Args: + metric (str, optional): Metric to order by (default: 'accuracy'). + + Returns: + tuple: (n_estimators, max_depth) for best model. + """ + logger = get_logger() + logger.info(f"Searching best hyperparameters by {metric}...") + + runs = mlflow.search_runs(order_by=[f"metrics.{metric} DESC"], max_results=1) + best = runs.iloc[0] + params = { + "n_estimators": int(best["params.n_estimators"]), + "max_depth": None if best["params.max_depth"] in ("", "None") else int(best["params.max_depth"]), + } + logger.info(f"Best hyperparameters found: n_estimators={params['n_estimators']}, max_depth={params['max_depth']}.") + return params["n_estimators"], params["max_depth"] + +@task +def download_best_model(metric: str = "accuracy") -> Path: + """ + Downloads the best model artifact from MLflow. + + Args: + metric (str, optional): Metric to order by (default: 'accuracy'). + + Returns: + Path: Local path to downloaded model (.pkl file). + """ + logger = get_logger() + logger.info(f"Downloading best model artifact in experiment {CFG.experiment_name}...") + + exp = mlflow.get_experiment_by_name(CFG.experiment_name) + runs = mlflow.search_runs( + experiment_ids=[exp.experiment_id], order_by=[f"metrics.{metric} DESC"], max_results=1 + ) + run_id = runs.iloc[0]["run_id"] + + client = mlflow.MlflowClient() + artifacts = client.list_artifacts(run_id) + pkl_path = next(a.path for a in artifacts if a.path.endswith(".pkl")) + + tmp_dir = tempfile.mkdtemp() + local_path = Path(client.download_artifacts(run_id, pkl_path, tmp_dir)) + logger.info(f"Downloaded best model to: {local_path}") + return local_path + +@task +def serve_model_task(model_path: Path): + """ + Deploys a FastAPI app for model inference. + + Args: + model_path (Path): Path to serialized model (.pkl). + """ + deploy(model_path=model_path) + +# --------------------------------------------------------------------------- +# Flows +# --------------------------------------------------------------------------- + +@flow(name="train_experiment") +def train_experiment(): + """ + End-to-end grid search training flow: + - Collects and prepares dataset. + - Launches multiple model training tasks for different hyperparameters. + - Only logs parameters and metrics, no model artifacts. + """ + logger = get_logger() + logger.info("Starting train_experiment flow...") + + dataset = data_collection_task() + dataset_prep = data_preparation_task(dataset) + + grid = { + "n_estimators": [1, 5, 10], + "max_depth": [None, 5, 10], + } + + for n_est, m_depth in product(grid["n_estimators"], grid["max_depth"]): + model_training_task.submit( + dataset=dataset_prep, n_estimators=n_est, max_depth=m_depth + ) + +@flow(name="train_best") +def train_best(): + """ + Retrains the best model based on MLflow search: + - Finds best hyperparameters. + - Retrains model and saves it as artifact. + """ + logger = get_logger() + logger.info("Starting train_best flow...") + + dataset = data_collection_task() + dataset_prep = data_preparation_task(dataset) + n_estimators, max_depth = find_best_params_task() + model_training_task( + dataset_prep, n_estimators=n_estimators, max_depth=max_depth, log_model=True + ) + +@flow(name="serve_best") +def serve_best(): + """ + Deploys the best model as a live API: + - Downloads best model from MLflow. + - Builds FastAPI app. + - Launches serving. + """ + logger = get_logger() + logger.info("Starting serve_best flow...") + + model_path = download_best_model() + #api = build_app_task(model_path=model_path) + serve_model_task(model_path) + +if __name__ == "__main__": + import sys + locals()[sys.argv[1]]() diff --git a/Academy/AI/2-ML-Model-Lifecycle/src/pipeline_monitoring.py b/Academy/AI/2-ML-Model-Lifecycle/src/pipeline_monitoring.py new file mode 100644 index 0000000..9209f54 --- /dev/null +++ b/Academy/AI/2-ML-Model-Lifecycle/src/pipeline_monitoring.py @@ -0,0 +1,111 @@ +""" +src/pipeline_monitoring.py +Prefect 2 flows for the Wine Quality ML pipeline. + +Flow: +- monitoring_best: Detects data drift and triggers retraining if significant drift is found. +""" + +import pandas as pd +from pathlib import Path + +from prefect import flow, task +from src.data import data_collection as data_coll_run, new_data_collection as new_data_coll_run +from src.monitor import run as monitoring_run, compute_drift +from src.pipeline_ml import train_best +from src.utils import get_logger + +# --------------------------------------------------------------------------- +# Tasks +# --------------------------------------------------------------------------- + +@task +def data_collection_task() -> tuple: + """ + Collects the original reference dataset. + + Returns: + tuple: (X, y) dataset. + """ + return data_coll_run() + +@task +def get_drift_data_task() -> pd.DataFrame: + """ + Generates a new dataset with simulated data drift. + + Returns: + pd.DataFrame: New batch of data with potential drift. + """ + return new_data_coll_run() + +@task +def create_drift_report_task(current_data: pd.DataFrame, reference_data: pd.DataFrame) -> Path: + """ + Creates a drift monitoring report comparing current batch vs reference. + + Args: + current_data (pd.DataFrame): Current dataset batch. + reference_data (pd.DataFrame): Original reference dataset. + + Returns: + Path: Path to the generated JSON drift report. + """ + return monitoring_run(current_data=current_data, reference_data=reference_data) + +@task +def compute_drift_task(output_json: Path) -> float: + """ + Computes the minimum drift score from the JSON report. + + Args: + output_json (Path): Path to the generated JSON report. + + Returns: + float: Minimum p-value detected across all features. + """ + return compute_drift(output_json) + +@task +def trigger_if_drift_task(min_p: float, threshold: float = 0.05): + """ + Triggers retraining if minimum drift p-value is below the threshold. + + Args: + min_p (float): Minimum p-value detected. + threshold (float, optional): Threshold below which retraining is triggered (default: 0.05). + """ + logger = get_logger() + logger.info("Starting monitoring_best flow...") + + if min_p <= threshold: + logger.warning(f"Drift detected (min_p={min_p}). Launching retraining...") + train_best() + else: + logger.warning(f"No drift detected (min_p={min_p}).") + +# --------------------------------------------------------------------------- +# Flows +# --------------------------------------------------------------------------- + +@flow(name="monitoring_best") +def monitoring_best(): + """ + Drift monitoring flow: + - Collects reference and new batch datasets. + - Generates drift report (HTML + JSON). + - Computes drift metrics. + - Triggers model retraining if drift is significant. + """ + logger = get_logger() + logger.info("Starting monitoring_best flow...") + + training_data, _ = data_collection_task() + new_data = get_drift_data_task() + report_json = create_drift_report_task(current_data=new_data, reference_data=training_data) + p_val = compute_drift_task(report_json) + trigger_if_drift_task(min_p=p_val) + +if __name__ == "__main__": + # Serve this flow with a cron schedule (here: every minute) + monitoring_best.serve(cron="* * * * *") diff --git a/Academy/AI/2-ML-Model-Lifecycle/src/serve.py b/Academy/AI/2-ML-Model-Lifecycle/src/serve.py new file mode 100644 index 0000000..3f0c0fb --- /dev/null +++ b/Academy/AI/2-ML-Model-Lifecycle/src/serve.py @@ -0,0 +1,131 @@ +""" +src/serve.py +FastAPI inference service for the Wine Quality model. + +- Loads a RandomForestClassifier model (joblib). +- Serves predictions via a REST API endpoint. +- CLI options allow flexible deployment or local testing. + +Endpoints: +- POST /predict : Accepts wine features and returns the predicted class and probabilities. +""" + +import subprocess +from pathlib import Path +from typing import Optional + +import joblib +import typer +import uvicorn +from fastapi import FastAPI +from pydantic import BaseModel + +from src.utils import get_logger + +app = typer.Typer(add_completion=False) + + +def build_app(model_path: Path) -> FastAPI: + """ + Constructs the FastAPI app with the loaded model. + + Args: + model_path (Path): Path to the trained model file (.pkl). + + Returns: + FastAPI: Configured FastAPI app with one /predict endpoint. + """ + # Load trained model + model = joblib.load(model_path) + + class Features(BaseModel): + """ + Request body schema for prediction. + """ + + alcohol: float + malic_acid: float + ash: float + alcalinity_of_ash: float + magnesium: float + total_phenols: float + flavanoids: float + nonflavanoid_phenols: float + proanthocyanins: float + color_intensity: float + hue: float + od280_od315_of_diluted_wines: float + proline: float + + # Initialize FastAPI app + api = FastAPI(title="Wine Quality Inference") + + @api.post("/predict") + def predict(feat: Features): + """ + Predicts the wine quality class based on provided features. + + Args: + feat (Features): Input wine features. + + Returns: + dict: Predicted class and probabilities for each class. + """ + import pandas as pd + + # Convert input features to DataFrame + X = pd.DataFrame([feat.dict()]) + + # Predict class and class probabilities + proba = model.predict_proba(X)[0].tolist() + predicted_class = int(model.predict(X)[0]) + + return {"class": predicted_class, "proba": proba} + + return api + + +def deploy(model_path: Path, host: str = "0.0.0.0", port: int = 9000): + """ + Deploys the given FastAPI app with Uvicorn. + + Args: + api (FastAPI): The app to serve. + host (str, optional): Host IP address to bind (default: 0.0.0.0). + port (int, optional): Port number to expose (default: 9000). + """ + logger = get_logger() + logger.info(f"Building FastAPI app for model from {model_path}...") + logger.info(f"Starting Uvicorn server on {host}:{port}...") + subprocess.Popen(["python", "-m", "src.serve", "--model-path", str(model_path)]) + + +@app.command() +def run( + model_path: Optional[Path] = typer.Option( + None, + "--model-path", + "-m", + help="Path to the .pkl model file.", + ), + host: str = "0.0.0.0", + port: int = 9000, +): + """ + CLI entry point for running the API locally. + + - Loads the model from the given path (or default artifacts directory). + - Builds the FastAPI app. + - Serves the API with Uvicorn. + + Args: + model_path (Optional[Path], optional): Path to the .pkl model file. + host (str, optional): Host IP to bind (default: 0.0.0.0). + port (int, optional): Port to expose the API (default: 9000). + """ + api = build_app(model_path) + uvicorn.run(api, host=host, port=port) + + +if __name__ == "__main__": + app() diff --git a/Academy/AI/2-ML-Model-Lifecycle/src/train.py b/Academy/AI/2-ML-Model-Lifecycle/src/train.py new file mode 100644 index 0000000..e700190 --- /dev/null +++ b/Academy/AI/2-ML-Model-Lifecycle/src/train.py @@ -0,0 +1,112 @@ +""" +src/train.py +Model training module for the Wine Quality pipeline. + +- Trains a RandomForestClassifier on provided dataset splits. +- Logs parameters, metrics, and optionally the trained model to MLflow. +- Can be executed both via CLI (Typer) or imported in Prefect flows. +""" + +import joblib +import mlflow +import typer +from sklearn.ensemble import RandomForestClassifier +from sklearn.metrics import accuracy_score +from src.config import CFG + +from src.utils import get_logger + +app = typer.Typer(add_completion=False) + +# Configure MLflow tracking (uses the experiment defined in CFG) +mlflow.set_tracking_uri(CFG.mlflow_uri) +mlflow.set_experiment(CFG.experiment_name) + + +def train_model( + dataset_split: tuple, n_estimators: int, max_depth: int | None +) -> tuple: + """ + Trains a RandomForest model on the provided dataset. + + Args: + dataset_split (tuple): Tuple of (X_train, X_test, y_train, y_test) splits. + n_estimators (int): Number of trees in the forest. + max_depth (int | None): Maximum depth of the trees (None means no limit). + + Returns: + tuple: + - model (RandomForestClassifier): Trained model. + - acc (float): Accuracy on the test set. + """ + logger = get_logger() + logger.info( + f"Training RandomForest with n_estimators={n_estimators}, max_depth={max_depth}..." + ) + + X_tr, X_te, y_tr, y_te = dataset_split + + # Train the RandomForest model + model = RandomForestClassifier( + n_estimators=n_estimators, max_depth=max_depth, random_state=0 + ).fit(X_tr, y_tr) + + # Compute accuracy on the test set + acc = accuracy_score(y_te, model.predict(X_te)) + logger.info(f"Model trained. Accuracy={acc:.4f}.") + return model, acc + + +def run( + dataset_split: tuple, + n_estimators: int = 200, + max_depth: int | None = None, + log_model: bool = typer.Option( + True, + "--log-model/--no-log-model", + help="Whether to log the model artifact to MLflow.", + ), +) -> float: + """ + Main training entry point. + + - Trains a model using specified hyperparameters. + - Logs training parameters, accuracy metric, and optionally the trained model to MLflow. + + Args: + dataset_split (tuple): Tuple of (X_train, X_test, y_train, y_test) splits. + n_estimators (int, optional): Number of trees (default: 200). + max_depth (int | None, optional): Maximum tree depth (default: None, unlimited). + log_model (bool, optional): If True, logs the model artifact to MLflow (default: True). + + Returns: + float: Accuracy score of the trained model. + """ + logger = get_logger() + logger.info( + f"Running full training pipeline (MLflow logging enabled={log_model})..." + ) + + with mlflow.start_run(): + # Log hyperparameters + mlflow.log_param("n_estimators", n_estimators) + mlflow.log_param("max_depth", max_depth) + + # Train the model + model, acc = train_model(dataset_split, n_estimators, max_depth) + + # Log model accuracy + mlflow.log_metric("accuracy", acc) + + # Optionally serialize and log the trained model + if log_model: + outfile = CFG.artifacts_dir / CFG.model_name + joblib.dump(model, outfile) + mlflow.log_artifact(outfile) + + logger.info("Training run completed and logged to MLflow.") + return acc # Useful for Prefect tasks or external calls + + +if __name__ == "__main__": + app() diff --git a/Academy/AI/2-ML-Model-Lifecycle/src/utils.py b/Academy/AI/2-ML-Model-Lifecycle/src/utils.py new file mode 100644 index 0000000..a7c53ba --- /dev/null +++ b/Academy/AI/2-ML-Model-Lifecycle/src/utils.py @@ -0,0 +1,29 @@ +""" +src/utils.py +Utility functions for the Wine Quality ML project. + +Includes: +- get_logger(): returns a Prefect logger if available, otherwise falls back to standard Python logging. +""" + +import logging + +def get_logger(): + """ + Returns a logger instance. + + - If inside a Prefect flow or task, returns the Prefect run logger. + - If outside Prefect (e.g., during unit tests or CLI usage), returns a standard Python logger. + + This allows all modules to use consistent logging behavior without crashing when Prefect context is missing. + + Returns: + Logger: Either a Prefect run logger or a standard Python logger. + """ + try: + from prefect import get_run_logger + return get_run_logger() + except RuntimeError: + # No active Prefect context; use fallback logger + logging.basicConfig(level=logging.INFO) + return logging.getLogger("fallback") diff --git a/Academy/AI/2-ML-Model-Lifecycle/tests/__init__.py b/Academy/AI/2-ML-Model-Lifecycle/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Academy/AI/2-ML-Model-Lifecycle/tests/test_data.py b/Academy/AI/2-ML-Model-Lifecycle/tests/test_data.py new file mode 100644 index 0000000..f0848f2 --- /dev/null +++ b/Academy/AI/2-ML-Model-Lifecycle/tests/test_data.py @@ -0,0 +1,35 @@ +""" +Unit tests for the data module in the Wine Quality ML pipeline. + +Covers: +- Dataset loading +- Train-test splitting logic +""" + +from src import data + +def test_data_collection(): + """ + Test that the wine dataset is loaded correctly with non-empty features and labels. + """ + X, y = data.data_collection() + + # Assert features and labels are not empty and match in length + assert not X.empty, "Feature dataframe is empty" + assert len(X) == len(y), "Mismatch between features and labels" + assert "alcohol" in X.columns, "'alcohol' column should exist in features" + +def test_data_preparation(): + """ + Test that the dataset is correctly split into train and test sets. + """ + X, y = data.data_collection() + X_tr, X_te, y_tr, y_te = data.data_preparation((X, y), split=0.2) + + # Assert that the train + test sizes add up to the original + assert len(X_tr) + len(X_te) == len(X), "Train + test features mismatch" + assert len(y_tr) + len(y_te) == len(y), "Train + test labels mismatch" + + # Optional: Check that no data was lost and all parts are non-empty + assert not X_tr.empty and not X_te.empty, "Split feature sets are empty" + assert not y_tr.empty and not y_te.empty, "Split label sets are empty" diff --git a/Academy/AI/2-ML-Model-Lifecycle/tests/test_serve.py b/Academy/AI/2-ML-Model-Lifecycle/tests/test_serve.py new file mode 100644 index 0000000..8adeb52 --- /dev/null +++ b/Academy/AI/2-ML-Model-Lifecycle/tests/test_serve.py @@ -0,0 +1,60 @@ +""" +Unit test for FastAPI model serving without relying on a pre-existing trained model. + +- Creates a dummy RandomForestClassifier +- Saves it to a temporary .pkl file +- Builds the FastAPI app using the test model +- Sends a prediction request to /predict +""" + +import tempfile +import joblib +from pathlib import Path +from fastapi.testclient import TestClient +from sklearn.ensemble import RandomForestClassifier +import pandas as pd +from src import serve + +def test_build_app_and_predict(): + """ + Test that the FastAPI app is correctly built with a temporary dummy model, + and that the /predict endpoint returns a valid response. + """ + # --- Create a dummy model and train it on minimal synthetic data --- + model = RandomForestClassifier() + X_dummy = pd.DataFrame([{ + "alcohol": 13.0, + "malic_acid": 2.0, + "ash": 2.5, + "alcalinity_of_ash": 15.0, + "magnesium": 100.0, + "total_phenols": 2.0, + "flavanoids": 2.5, + "nonflavanoid_phenols": 0.3, + "proanthocyanins": 1.9, + "color_intensity": 5.0, + "hue": 1.0, + "od280_od315_of_diluted_wines": 3.0, + "proline": 1000, + }]) + y_dummy = [1] + model.fit(X_dummy, y_dummy) + + # --- Save model to a temporary .pkl file --- + with tempfile.NamedTemporaryFile(suffix=".pkl", delete=False) as tmp: + model_path = Path(tmp.name) + joblib.dump(model, model_path) + + # --- Build FastAPI app using the dummy model --- + api = serve.build_app(model_path) + + # --- Create a test client and send a sample request --- + sample = X_dummy.iloc[0].to_dict() + client = TestClient(api) + response = client.post("/predict", json=sample) + + # --- Assertions --- + assert response.status_code == 200 + result = response.json() + assert "class" in result + assert isinstance(result["class"], int) diff --git a/Academy/AI/2-ML-Model-Lifecycle/tests/test_train.py b/Academy/AI/2-ML-Model-Lifecycle/tests/test_train.py new file mode 100644 index 0000000..52203e0 --- /dev/null +++ b/Academy/AI/2-ML-Model-Lifecycle/tests/test_train.py @@ -0,0 +1,30 @@ +""" +Unit test for the train module in the Wine Quality ML pipeline. + +Covers: +- Training a RandomForestClassifier +- Validating model accuracy and structure +""" + +from src import data, train + +def test_train_model_accuracy_threshold(): + """ + Test that the train_model function returns a trained model with acceptable accuracy. + + - Uses a real dataset split + - Checks that accuracy is above a baseline threshold + - Verifies that the model supports .predict() + """ + # Load and split the dataset + X, y = data.data_collection() + prepared = data.data_preparation((X, y)) + + # Train a RandomForest model with basic hyperparameters + model, acc = train.train_model(prepared, n_estimators=10, max_depth=5) + + # Assert accuracy is above 50% (weak sanity check) + assert acc > 0.5, f"Expected accuracy > 0.5, got {acc}" + + # Check that model has a predict method + assert hasattr(model, "predict"), "Model does not implement .predict()" diff --git a/Academy/AI/3-LLM/README.md b/Academy/AI/3-LLM/README.md new file mode 100644 index 0000000..9c5808e --- /dev/null +++ b/Academy/AI/3-LLM/README.md @@ -0,0 +1,103 @@ +# Lab: Prompt‑Engineering and LoRA Fine‑Tuning + +Welcome to this live lab! +In the previous lab session we discussed how to: + +1. **Train a classification model** on a real dataset using `Scikit-learn`. +2. **Manage and track your experiments** with `MLflow`. +3. **Build CI/CD-like workflows** using `Makefile` and `nox`. +4. **Build scalable and orchestrated ML pipelines** using `Prefect`. +5. **Monitor data drift over time** and trigger retraining with `Evidently`. + +In this session, you will run two Jupyter notebooks that demonstrate: + +- Practical prompt‑engineering patterns in `notebook 1_LabLLM.ipynb`, the craft of writing clear, explicit instructions for an LLM. +- A LoRA fine‑tuning workflow on model T5‑small in `notebook 2_LabLLM_LoRA.ipynb`, a lightweight fine-tuning technique for transformers. + +We'll leverage two big providers for this lab: + +- **Google Colab**: a free, browser-based Jupyter environment hosted by Google. Colab gives you a temporary Linux VM with Python, CUDA drivers, and optional GPU/TPU. You open a .ipynb notebook, run cells just like in Jupyter, and share links with classmates—no local setup required. + +- **Hugging Face**: A community and tooling hub for modern AI. + + It offers: + + - Model Hub: 500 k-plus pre-trained models (transformers, diffusers, etc.). + - Datasets Hub: curated datasets in one-line API. + - Inference API / Endpoints: hosted model inference in the cloud. + - Libraries: transformers, datasets, peft, and more. + + Creating a (free) HF account lets you: + + - Download models. + - Push your own models/adapters/data. + - Generate an access token that notebooks use to call hosted endpoints securely. + +--- + +## Repository contents + +| Path/Name | What it is and why it matters | +| --------------------------------------- | ---------------------------------------------- | +| `notebooks/1_LabLLM.ipynb` | Prompt‑engineering walkthrough (Mistral‑7B) | +| `notebooks/2_LabLLM_LoRA.ipynb` | LoRA adapter training on a 250‑row support set | +| `files/customer_support_lora_dataset_250.csv` | Tiny dataset used in Notebook 2 | +| `README` (this file) | Quickstart and background | + +## Prerequisites + +| Tool / Account | Why you need it | Sign‑up link | +| ------------------------ | -------------------------------------------- | ------------------------------------------------------------------------ | +| **Google account** | Required to open Colab notebooks | [https://accounts.google.com/signup](https://accounts.google.com/signup) | +| **Hugging Face account** | Lets you call hosted models & store adapters | [https://huggingface.co/join](https://huggingface.co/join) | + +### 1 Create your HF account and token + +1. Go to [https://huggingface.co/join](https://huggingface.co/join) and sign up (email and password). +2. Verify your email address (check spam folder too). +3. Navigate to **Settings > Access Tokens > New Token**. +4. Name it e.g. `ai‑academy‑lab` and select **"Read"** scope (fine for this lesson). +5. Copy the token. **You will paste it into the notebook when prompted**. +6. Treat the token like a password and do **not** share or commit it to GitHub. + +### 2 Recommended Colab runtime settings + +- **GPU:** In Colab, go to **Runtime > Change runtime type > Hardware accelerator > GPU**. +- **RAM:** The default *Standard* tier is enough. No paid upgrade needed. + +--- + +## Run the notebooks in Colab + +### 1 Clone the Repository + +To start, clone the lab repository by running the following command in the terminal: + +```sh + git clone https://github.com/kiratech/labs.git +``` + +### 2 Checkout the Lab Branch + +After cloning the repository, checkout the `academy-ai` branch: + +```sh + git checkout academy-ai +``` + +Then, navigate to the project folder: + +```sh + cd labs/Academy/AI/3-LLM +``` + +This folder contains resources related to this lab. +Open Google Colab and import, everything at root level, the content of the folders notebooks and files. +From now you can continue on the notebooks. + +--- + +## Suggested readings and references + +- [OpenAI Cookbook – GPT‑4 Prompting Guide](https://cookbook.openai.com/examples/gpt4-1_prompting_guide): a new prompting guide that lays out a practical structure for building powerful prompts, especially with GPT-4.1. It’s short, clear, and highly effective for anyone working with agents, structured outputs, tool use, or reasoning-heavy tasks. +- [Google Prompt Engineering pdf](https://drive.google.com/file/d/1AbaBYbEa_EbPelsT40-vj64L-2IwUJHy/view): whether you're technical or non-technical, this might be one of the most useful prompt engineering resources out there right now. Google just published a 68-page whitepaper focused on Prompt Engineering (focused on API users), and it goes deep on structure, formatting, config settings, and real examples. diff --git a/Academy/AI/3-LLM/files/customer_support_lora_dataset_250.csv b/Academy/AI/3-LLM/files/customer_support_lora_dataset_250.csv new file mode 100644 index 0000000..3d10f64 --- /dev/null +++ b/Academy/AI/3-LLM/files/customer_support_lora_dataset_250.csv @@ -0,0 +1,251 @@ +input_text,target_text +generate structured_reply: I received the wrong item in my order.,"{""greeting"":""Dear customer,"",""apology"":""We apologize for this experience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: The product I received is damaged. What can I do?,"{""greeting"":""Dear customer,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I received the wrong item in my order.,"{""greeting"":""Hi there,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I received the wrong item in my order.,"{""greeting"":""Greetings,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I accidentally ordered the wrong color. Can I change it?,"{""greeting"":""Greetings,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I was promised a discount that was never applied.,"{""greeting"":""Hi there,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I never received my order.,"{""greeting"":""Hi there,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: The product I received is damaged. What can I do?,"{""greeting"":""Hello,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: Your website charged me shipping twice.,"{""greeting"":""Hello,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: Why was I charged twice for my order?,"{""greeting"":""Dear customer,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: Your website charged me shipping twice.,"{""greeting"":""Greetings,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: Your website charged me shipping twice.,"{""greeting"":""Hello,"",""apology"":""We apologize for this experience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Hello,"",""apology"":""We apologize for this experience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Hello,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: Why was I charged twice for my order?,"{""greeting"":""Greetings,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: The tracking number shows delivered but I didn't get the package.,"{""greeting"":""Dear customer,"",""apology"":""We apologize for this experience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I accidentally ordered the wrong color. Can I change it?,"{""greeting"":""Dear customer,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: The replacement you sent is also faulty.,"{""greeting"":""Greetings,"",""apology"":""We apologize for this experience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: Why was I charged twice for my order?,"{""greeting"":""Dear customer,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: Your website charged me shipping twice.,"{""greeting"":""Greetings,"",""apology"":""We apologize for this experience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: The replacement you sent is also faulty.,"{""greeting"":""Hello,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: The item arrived with missing accessories.,"{""greeting"":""Dear customer,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: The replacement you sent is also faulty.,"{""greeting"":""Hi there,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: My order arrived late. I want a refund.,"{""greeting"":""Hi there,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: Your website charged me shipping twice.,"{""greeting"":""Dear customer,"",""apology"":""We apologize for this experience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I never received my order.,"{""greeting"":""Hello,"",""apology"":""We apologize for this experience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Greetings,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I accidentally ordered the wrong color. Can I change it?,"{""greeting"":""Hello,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: Your website charged me shipping twice.,"{""greeting"":""Dear customer,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: Why was I charged twice for my order?,"{""greeting"":""Greetings,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: The product I received is damaged. What can I do?,"{""greeting"":""Greetings,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: The tracking number shows delivered but I didn't get the package.,"{""greeting"":""Hi there,"",""apology"":""We apologize for this experience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I accidentally ordered the wrong color. Can I change it?,"{""greeting"":""Hello,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I received the wrong item in my order.,"{""greeting"":""Greetings,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: How can I return an item I purchased last week?,"{""greeting"":""Hello,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: The item arrived with missing accessories.,"{""greeting"":""Dear customer,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: The item arrived with missing accessories.,"{""greeting"":""Hello,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: Can I exchange my item for a different size?,"{""greeting"":""Greetings,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I need help tracking my shipment.,"{""greeting"":""Hi there,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: My order arrived late. I want a refund.,"{""greeting"":""Hello,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I accidentally ordered the wrong color. Can I change it?,"{""greeting"":""Hello,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: Can I exchange my item for a different size?,"{""greeting"":""Hi there,"",""apology"":""We apologize for this experience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I received the wrong item in my order.,"{""greeting"":""Hi there,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Dear customer,"",""apology"":""We apologize for this experience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: How can I return an item I purchased last week?,"{""greeting"":""Hi there,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: Why was I charged twice for my order?,"{""greeting"":""Hi there,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: Your website charged me shipping twice.,"{""greeting"":""Hello,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I need help tracking my shipment.,"{""greeting"":""Greetings,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: How can I return an item I purchased last week?,"{""greeting"":""Greetings,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I was promised a discount that was never applied.,"{""greeting"":""Hi there,"",""apology"":""We apologize for this experience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: Your website charged me shipping twice.,"{""greeting"":""Dear customer,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: The replacement you sent is also faulty.,"{""greeting"":""Hello,"",""apology"":""We apologize for this experience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: The item arrived with missing accessories.,"{""greeting"":""Hi there,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: The replacement you sent is also faulty.,"{""greeting"":""Greetings,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: The tracking number shows delivered but I didn't get the package.,"{""greeting"":""Hello,"",""apology"":""We apologize for this experience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I never received my order.,"{""greeting"":""Hello,"",""apology"":""We apologize for this experience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: The product I received is damaged. What can I do?,"{""greeting"":""Greetings,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: Why was I charged twice for my order?,"{""greeting"":""Hi there,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I never received my order.,"{""greeting"":""Hi there,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I was promised a discount that was never applied.,"{""greeting"":""Dear customer,"",""apology"":""We apologize for this experience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: How can I return an item I purchased last week?,"{""greeting"":""Hi there,"",""apology"":""We apologize for this experience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I never received my order.,"{""greeting"":""Dear customer,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I never received my order.,"{""greeting"":""Dear customer,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Hi there,"",""apology"":""We apologize for this experience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: Can I exchange my item for a different size?,"{""greeting"":""Greetings,"",""apology"":""We apologize for this experience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I need help tracking my shipment.,"{""greeting"":""Hi there,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: The tracking number shows delivered but I didn't get the package.,"{""greeting"":""Greetings,"",""apology"":""We apologize for this experience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: The item arrived with missing accessories.,"{""greeting"":""Hi there,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: Your website charged me shipping twice.,"{""greeting"":""Dear customer,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: Your website charged me shipping twice.,"{""greeting"":""Hello,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I accidentally ordered the wrong color. Can I change it?,"{""greeting"":""Hello,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I accidentally ordered the wrong color. Can I change it?,"{""greeting"":""Hello,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: How can I return an item I purchased last week?,"{""greeting"":""Dear customer,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I was promised a discount that was never applied.,"{""greeting"":""Hello,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Greetings,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: My order arrived late. I want a refund.,"{""greeting"":""Dear customer,"",""apology"":""We apologize for this experience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I accidentally ordered the wrong color. Can I change it?,"{""greeting"":""Dear customer,"",""apology"":""We apologize for this experience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: Can I exchange my item for a different size?,"{""greeting"":""Hello,"",""apology"":""We apologize for this experience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: How can I return an item I purchased last week?,"{""greeting"":""Greetings,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I received the wrong item in my order.,"{""greeting"":""Hi there,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: How can I return an item I purchased last week?,"{""greeting"":""Dear customer,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I accidentally ordered the wrong color. Can I change it?,"{""greeting"":""Hello,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Greetings,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: The item arrived with missing accessories.,"{""greeting"":""Dear customer,"",""apology"":""We apologize for this experience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Dear customer,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: The product I received is damaged. What can I do?,"{""greeting"":""Hello,"",""apology"":""We apologize for this experience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I accidentally ordered the wrong color. Can I change it?,"{""greeting"":""Dear customer,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: How can I return an item I purchased last week?,"{""greeting"":""Hello,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: How can I return an item I purchased last week?,"{""greeting"":""Greetings,"",""apology"":""We apologize for this experience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I need help tracking my shipment.,"{""greeting"":""Hi there,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I accidentally ordered the wrong color. Can I change it?,"{""greeting"":""Hi there,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: Why was I charged twice for my order?,"{""greeting"":""Dear customer,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: Your website charged me shipping twice.,"{""greeting"":""Greetings,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I never received my order.,"{""greeting"":""Hi there,"",""apology"":""We apologize for this experience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: My order arrived late. I want a refund.,"{""greeting"":""Greetings,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I need help tracking my shipment.,"{""greeting"":""Dear customer,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: The tracking number shows delivered but I didn't get the package.,"{""greeting"":""Hi there,"",""apology"":""We apologize for this experience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I accidentally ordered the wrong color. Can I change it?,"{""greeting"":""Hello,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: The item arrived with missing accessories.,"{""greeting"":""Hi there,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: Can I exchange my item for a different size?,"{""greeting"":""Greetings,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: The tracking number shows delivered but I didn't get the package.,"{""greeting"":""Hello,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I was promised a discount that was never applied.,"{""greeting"":""Hello,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: The item arrived with missing accessories.,"{""greeting"":""Dear customer,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: The tracking number shows delivered but I didn't get the package.,"{""greeting"":""Hello,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: The item arrived with missing accessories.,"{""greeting"":""Dear customer,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Dear customer,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: Can I exchange my item for a different size?,"{""greeting"":""Greetings,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I accidentally ordered the wrong color. Can I change it?,"{""greeting"":""Hello,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Hello,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: The product I received is damaged. What can I do?,"{""greeting"":""Greetings,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: Your website charged me shipping twice.,"{""greeting"":""Greetings,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I accidentally ordered the wrong color. Can I change it?,"{""greeting"":""Hi there,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: The item arrived with missing accessories.,"{""greeting"":""Dear customer,"",""apology"":""We apologize for this experience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: The product I received is damaged. What can I do?,"{""greeting"":""Dear customer,"",""apology"":""We apologize for this experience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: The tracking number shows delivered but I didn't get the package.,"{""greeting"":""Hi there,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Dear customer,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: My order arrived late. I want a refund.,"{""greeting"":""Greetings,"",""apology"":""We apologize for this experience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I need help tracking my shipment.,"{""greeting"":""Dear customer,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: Can I exchange my item for a different size?,"{""greeting"":""Dear customer,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: Your website charged me shipping twice.,"{""greeting"":""Hello,"",""apology"":""We apologize for this experience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Dear customer,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I never received my order.,"{""greeting"":""Hi there,"",""apology"":""We apologize for this experience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: The tracking number shows delivered but I didn't get the package.,"{""greeting"":""Hi there,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Hi there,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I received the wrong item in my order.,"{""greeting"":""Greetings,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: Your website charged me shipping twice.,"{""greeting"":""Hello,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I need help tracking my shipment.,"{""greeting"":""Hello,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I need help tracking my shipment.,"{""greeting"":""Dear customer,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I was promised a discount that was never applied.,"{""greeting"":""Greetings,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: The item arrived with missing accessories.,"{""greeting"":""Hello,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: My order arrived late. I want a refund.,"{""greeting"":""Hello,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I never received my order.,"{""greeting"":""Hi there,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I need help tracking my shipment.,"{""greeting"":""Hi there,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: Your website charged me shipping twice.,"{""greeting"":""Hi there,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: The replacement you sent is also faulty.,"{""greeting"":""Hello,"",""apology"":""We apologize for this experience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: The product I received is damaged. What can I do?,"{""greeting"":""Hi there,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I need help tracking my shipment.,"{""greeting"":""Dear customer,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: The product I received is damaged. What can I do?,"{""greeting"":""Hi there,"",""apology"":""We apologize for this experience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: The product I received is damaged. What can I do?,"{""greeting"":""Hi there,"",""apology"":""We apologize for this experience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: Can I exchange my item for a different size?,"{""greeting"":""Greetings,"",""apology"":""We apologize for this experience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: Why was I charged twice for my order?,"{""greeting"":""Hello,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: The product I received is damaged. What can I do?,"{""greeting"":""Hello,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: Why was I charged twice for my order?,"{""greeting"":""Hi there,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Hi there,"",""apology"":""We apologize for this experience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: How can I return an item I purchased last week?,"{""greeting"":""Hi there,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: The replacement you sent is also faulty.,"{""greeting"":""Hi there,"",""apology"":""We apologize for this experience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I received the wrong item in my order.,"{""greeting"":""Dear customer,"",""apology"":""We apologize for this experience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: The item arrived with missing accessories.,"{""greeting"":""Greetings,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: My order arrived late. I want a refund.,"{""greeting"":""Hello,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: The product I received is damaged. What can I do?,"{""greeting"":""Dear customer,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Hello,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I was promised a discount that was never applied.,"{""greeting"":""Hello,"",""apology"":""We apologize for this experience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: Your website charged me shipping twice.,"{""greeting"":""Hello,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I was promised a discount that was never applied.,"{""greeting"":""Hi there,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I need help tracking my shipment.,"{""greeting"":""Greetings,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: The item arrived with missing accessories.,"{""greeting"":""Hi there,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: The item arrived with missing accessories.,"{""greeting"":""Dear customer,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: Your website charged me shipping twice.,"{""greeting"":""Hello,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I never received my order.,"{""greeting"":""Hi there,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I received the wrong item in my order.,"{""greeting"":""Hello,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I accidentally ordered the wrong color. Can I change it?,"{""greeting"":""Dear customer,"",""apology"":""We apologize for this experience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I accidentally ordered the wrong color. Can I change it?,"{""greeting"":""Dear customer,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I was promised a discount that was never applied.,"{""greeting"":""Hello,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: Can I exchange my item for a different size?,"{""greeting"":""Hello,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I was promised a discount that was never applied.,"{""greeting"":""Hello,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: The product I received is damaged. What can I do?,"{""greeting"":""Dear customer,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I received the wrong item in my order.,"{""greeting"":""Hello,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I received the wrong item in my order.,"{""greeting"":""Dear customer,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: The item arrived with missing accessories.,"{""greeting"":""Hello,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: Your website charged me shipping twice.,"{""greeting"":""Hello,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Hello,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: The item arrived with missing accessories.,"{""greeting"":""Greetings,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I was promised a discount that was never applied.,"{""greeting"":""Greetings,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: How can I return an item I purchased last week?,"{""greeting"":""Dear customer,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Greetings,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: How can I return an item I purchased last week?,"{""greeting"":""Hi there,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I accidentally ordered the wrong color. Can I change it?,"{""greeting"":""Dear customer,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I need help tracking my shipment.,"{""greeting"":""Hi there,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I need help tracking my shipment.,"{""greeting"":""Hello,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: The product I received is damaged. What can I do?,"{""greeting"":""Hi there,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I need help tracking my shipment.,"{""greeting"":""Dear customer,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I received the wrong item in my order.,"{""greeting"":""Dear customer,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: The replacement you sent is also faulty.,"{""greeting"":""Hello,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Dear customer,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Hi there,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: Your website charged me shipping twice.,"{""greeting"":""Dear customer,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Greetings,"",""apology"":""We apologize for this experience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I never received my order.,"{""greeting"":""Greetings,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I was promised a discount that was never applied.,"{""greeting"":""Hi there,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Greetings,"",""apology"":""We apologize for this experience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I received the wrong item in my order.,"{""greeting"":""Hi there,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I accidentally ordered the wrong color. Can I change it?,"{""greeting"":""Hi there,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I received the wrong item in my order.,"{""greeting"":""Hi there,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I was promised a discount that was never applied.,"{""greeting"":""Greetings,"",""apology"":""We apologize for this experience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: How can I return an item I purchased last week?,"{""greeting"":""Greetings,"",""apology"":""We apologize for this experience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: Why was I charged twice for my order?,"{""greeting"":""Greetings,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: Can I exchange my item for a different size?,"{""greeting"":""Dear customer,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Greetings,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: Can I exchange my item for a different size?,"{""greeting"":""Hello,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Greetings,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: Your website charged me shipping twice.,"{""greeting"":""Hi there,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: The product I received is damaged. What can I do?,"{""greeting"":""Dear customer,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: The product I received is damaged. What can I do?,"{""greeting"":""Hi there,"",""apology"":""We apologize for this experience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: My order arrived late. I want a refund.,"{""greeting"":""Hi there,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I never received my order.,"{""greeting"":""Hi there,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I received the wrong item in my order.,"{""greeting"":""Hi there,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: The product I received is damaged. What can I do?,"{""greeting"":""Dear customer,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I never received my order.,"{""greeting"":""Greetings,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I was promised a discount that was never applied.,"{""greeting"":""Greetings,"",""apology"":""We apologize for this experience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: My order arrived late. I want a refund.,"{""greeting"":""Greetings,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: My order arrived late. I want a refund.,"{""greeting"":""Hi there,"",""apology"":""We apologize for this experience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: Can I exchange my item for a different size?,"{""greeting"":""Hi there,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I received the wrong item in my order.,"{""greeting"":""Greetings,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: How can I return an item I purchased last week?,"{""greeting"":""Greetings,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I received the wrong item in my order.,"{""greeting"":""Dear customer,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: Can I exchange my item for a different size?,"{""greeting"":""Hi there,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: How can I return an item I purchased last week?,"{""greeting"":""Dear customer,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: The product I received is damaged. What can I do?,"{""greeting"":""Hello,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: How can I return an item I purchased last week?,"{""greeting"":""Hi there,"",""apology"":""We apologize for this experience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Greetings,"",""apology"":""We apologize for this experience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I was promised a discount that was never applied.,"{""greeting"":""Dear customer,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I was promised a discount that was never applied.,"{""greeting"":""Dear customer,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: The product I received is damaged. What can I do?,"{""greeting"":""Dear customer,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: My order arrived late. I want a refund.,"{""greeting"":""Dear customer,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: The tracking number shows delivered but I didn't get the package.,"{""greeting"":""Greetings,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: The product I received is damaged. What can I do?,"{""greeting"":""Hello,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: How can I return an item I purchased last week?,"{""greeting"":""Greetings,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: The replacement you sent is also faulty.,"{""greeting"":""Hi there,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: How can I return an item I purchased last week?,"{""greeting"":""Dear customer,"",""apology"":""We apologize for this experience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I was promised a discount that was never applied.,"{""greeting"":""Dear customer,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: Your website charged me shipping twice.,"{""greeting"":""Dear customer,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Dear customer,"",""apology"":""We apologize for this experience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I was promised a discount that was never applied.,"{""greeting"":""Hello,"",""apology"":""We apologize for this experience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: The item arrived with missing accessories.,"{""greeting"":""Hi there,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: The tracking number shows delivered but I didn't get the package.,"{""greeting"":""Hello,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: How can I return an item I purchased last week?,"{""greeting"":""Hi there,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: The tracking number shows delivered but I didn't get the package.,"{""greeting"":""Greetings,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: The tracking number shows delivered but I didn't get the package.,"{""greeting"":""Hello,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: Why was I charged twice for my order?,"{""greeting"":""Hi there,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I need help tracking my shipment.,"{""greeting"":""Hello,"",""apology"":""We apologize for this experience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: The tracking number shows delivered but I didn't get the package.,"{""greeting"":""Hello,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: My order arrived late. I want a refund.,"{""greeting"":""Hi there,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: The tracking number shows delivered but I didn't get the package.,"{""greeting"":""Greetings,"",""apology"":""We apologize for this experience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: Why was I charged twice for my order?,"{""greeting"":""Dear customer,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Hi there,"",""apology"":""We apologize for this experience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: The item arrived with missing accessories.,"{""greeting"":""Greetings,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I never received my order.,"{""greeting"":""Dear customer,"",""apology"":""We apologize for this experience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I received the wrong item in my order.,"{""greeting"":""Dear customer,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: The replacement you sent is also faulty.,"{""greeting"":""Hi there,"",""apology"":""We apologize for this experience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I accidentally ordered the wrong color. Can I change it?,"{""greeting"":""Greetings,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" diff --git a/Academy/AI/3-LLM/notebooks/1_LabLLM.ipynb b/Academy/AI/3-LLM/notebooks/1_LabLLM.ipynb new file mode 100644 index 0000000..9936556 --- /dev/null +++ b/Academy/AI/3-LLM/notebooks/1_LabLLM.ipynb @@ -0,0 +1,459 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "us7zHCUszttj" + }, + "source": [ + "# Prompt-Engineering Lab: Customer-Support Replies with **Mistral-7B-Instruct**\n", + "\n", + "In this lab you will experiment with six classic **prompt patterns**:\n", + "\n", + "1. **Zero-shot**: no examples, just the task. \n", + "2. **Few-shot**: show 1-2 examples first. \n", + "3. **Chain-of-Thought**: ask the model to reason step by step. \n", + "4. **Role / Persona**: tell the model \"You are a ...\". \n", + "5. **Structured delimiters**: clearly define context or information. \n", + "6. **System / Policy instructions**: prepend internal guidelines. \n", + "7. **Structured JSON-formatted**: force JSON or bullet format. \n", + "\n", + "By the end you'll understand how tiny changes in wording can dramatically change an LLM's answer.\n", + "\n", + "## What is Prompt-Engineering?\n", + "Because large language models don't have traditional \"APIs\", you steer them with natural-language prompts. Refining spacing, order, examples, and style (zero-shot, few-shot, chain-of-thought, role instructions, JSON schemas, etc.) is called prompt engineering. Good prompts boost accuracy and consistency without touching the model's weights." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ug_3UHSCaaZI" + }, + "source": [ + "## 0 Install Hugging Face client library\n", + "\n", + "We only need **`huggingface_hub`** because we call the hosted Inference API. No heavyweight Transformers install is required.\n", + "\n", + "### What is Hugging Face and the Hugging Face Hub ?\n", + "\n", + "Hugging Face is an open-source-first AI company that curates Transformers, Diffusers, and other widely used libraries.\n", + "The Hub (https://huggingface.co/) is a **GitHub for models & datasets**:\n", + "- Free hosting for models, datasets, Spaces (demo apps).\n", + "- Built-in versioning, README rendering, and model cards.\n", + "- REST / Python SDK (huggingface_hub) for push-pull, inference endpoints, and gated access control.\n", + "\n", + "In practice you clone/push a repo exactly like Git, but via hf_hub_download() or model.push_to_hub()." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "-hTQLYPZdUZp" + }, + "outputs": [], + "source": [ + "!pip install -q huggingface_hub" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "UX-GYw-WbGrP" + }, + "source": [ + "## 1 Authenticate and create an `InferenceClient`\n", + "\n", + "You need an **access token** to use the Mistral-7B endpoint hosted by provider **\"novita\"** on Hugging Face.\n", + "\n", + "### What are providers on HF Inference Client and what is novita?\n", + "\n", + "The InferenceClient can route a request to different providers (back-ends):\n", + "- huggingface (default): the standard Inference API hosted by HF.\n", + "- novita (a third-party back-end): offers chat-optimized latency/pricing. API surface matches OpenAI-style /chat/completions.\n", + "\n", + "You switch simply by\n", + "\n", + "```python\n", + "client = InferenceClient(provider=\"novita\", api_key=HF_TOKEN)\n", + "```\n", + "so the same client code can talk to multiple vendors." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "mZG3akxmdqpT" + }, + "outputs": [], + "source": [ + "import os, getpass\n", + "from huggingface_hub import InferenceClient\n", + "\n", + "# Paste your personal HF token when prompted\n", + "os.environ[\"HF_TOKEN\"] = getpass.getpass(\"Enter your HF token: \")\n", + "\n", + "# Create the client (model is chosen later in each call)\n", + "client = InferenceClient(provider=\"novita\", api_key=os.environ[\"HF_TOKEN\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "-D2_PSUlep8I" + }, + "source": [ + "## Model spotlight - mistralai/Mistral-7B-Instruct-v0.3\n", + "| - | Pros | Cons |\n", + "| ------------------ | -------------------------------------------------------------------------------- | --------------------------------------------------------------------------- |\n", + "| **Quality** | Very strong chat and code for a 7 B model. Outperforms LLaMA-7B/13B in many evals. | Still below GPT-4 / Claude-Opus tier on reasoning and long-context. |\n", + "| **Licence** | Apache-2.0, commercial use allowed. | Weight access sometimes gated. You need HF/token or local download. |\n", + "| **Speed / Memory** | Fits into 14 GB GPU (8-bit) so it is cheap to host. Responds \\~50-70 tok/s on T4. | Context window 8 K. Cannot process very long docs like GPT-4o-128K. |\n", + "| **LoRA** | Adapter \\~120 MB. Can be fine-tuned on a Colab T4 in <30 min. | PEFT on Mistral needs to target correct linear layers (`q_proj`, `v_proj`). |\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "qogkSPAbbiD2" + }, + "source": [ + "## 2 Zero-shot prompt\n", + "\n", + "Ask the model to perform a task without examples: “Translate the sentence to Italian: 'I love coffee'”.\n", + "\n", + "Zero-shot works because large LMs have seen countless instructions during pre-training. It's the fastest way to test a capability." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "vd3cTO7LemPx" + }, + "outputs": [], + "source": [ + "prompt = \"A customer is requesting a refund because a product was delivered late. Write a professional reply.\"\n", + "\n", + "completion = client.chat.completions.create(\n", + " model=\"mistralai/Mistral-7B-Instruct-v0.3\",\n", + " messages=[\n", + " {\"role\": \"user\", \"content\": prompt}\n", + " ],\n", + ")\n", + "\n", + "print(completion.choices[0].message[\"content\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "_UF5mFW1cBNm" + }, + "source": [ + "## 3 Few-shot prompt\n", + "\n", + "Provide 1-3 examples so the model infers pattern, tone, structure.\n", + "\n", + "Example: \n", + "Q1: Formal -> Casual \n", + "A1: Please contact me. -> Hit me up! \n", + "\n", + "The next transformation aligns with the examples far better than zero-shot." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "CeOQuFfRdrga" + }, + "outputs": [], + "source": [ + "prompt = '''Example 1:\n", + "Customer: I received my order late. I'd like a refund.\n", + "Reply: We're sorry for the delay. Your refund will be processed within 3 business days.\n", + "\n", + "Example 2:\n", + "Customer: My package arrived after the estimated delivery date. I want a refund.\n", + "Reply: We apologize for the inconvenience. The refund has been approved and will be issued shortly.\n", + "\n", + "Now write a reply to:\n", + "A customer is requesting a refund because a product was delivered late.\n", + "'''\n", + "\n", + "completion = client.chat.completions.create(\n", + " model=\"mistralai/Mistral-7B-Instruct-v0.3\",\n", + " messages=[{\"role\": \"user\", \"content\": prompt}],\n", + ")\n", + "\n", + "print(completion.choices[0].message[\"content\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ExIj1lZyc9jE" + }, + "source": [ + "## 4 Chain-of-Thought prompt\n", + "\n", + "Add \"Think step by step\" (or an explicit reasoning prompt). \n", + "The model reveals intermediate reasoning, often raising factual accuracy and allowing you to inspect/error-correct the chain." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "-TgOb4owgnHn" + }, + "outputs": [], + "source": [ + "prompt = \"A customer is requesting a refund because a product was delivered late. Think step by step and explain me your reasonings to write a professional reply.\"\n", + "\n", + "completion = client.chat.completions.create(\n", + " model=\"mistralai/Mistral-7B-Instruct-v0.3\",\n", + " messages=[{\"role\": \"user\", \"content\": prompt}],\n", + ")\n", + "\n", + "print(completion.choices[0].message[\"content\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "oWUOoci-dQLx" + }, + "source": [ + "## 5 Role / Persona prompt\n", + "\n", + "Pre-frame the assistant persona:\n", + "\"You are an empathetic customer-support agent…\". \n", + "Sets vocabulary, tone, even policy constraints without touching the model weights." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "n5jnEUKwgoVM" + }, + "outputs": [], + "source": [ + "prompt = \"You are a customer support agent. A customer is asking for a refund due to a late delivery. Write a helpful and empathetic response.\"\n", + "\n", + "completion = client.chat.completions.create(\n", + " model=\"mistralai/Mistral-7B-Instruct-v0.3\",\n", + " messages=[{\"role\": \"user\", \"content\": prompt}],\n", + ")\n", + "\n", + "print(completion.choices[0].message[\"content\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "I8fryyF4dgJ2" + }, + "source": [ + "## 6 Structured delimiters\n", + "\n", + "Wrap user context in clear markers (\"\"\" or ###), e.g.\n", + "\n", + "```txt\n", + "\"\"\"CONTEXT\n", + "long text …\n", + "\"\"\"\n", + "Summarize the context in one bullet.\n", + "```\n", + "\n", + "Prevents the model from mixing instructions with payload and boosts reliability in multi-part prompts." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "l9doquBSg2jV" + }, + "outputs": [], + "source": [ + "prompt = '''\n", + "### STRUCTURE\n", + "1. Greeting\n", + "2. Apology for the delay\n", + "3. Refund details\n", + "4. Closing remark\n", + "###\n", + "### CUSTOMER EMAIL\n", + "I received the product late and would like a refund.\n", + "###\n", + "Write a professional reply to customer email using our structure.\n", + "'''\n", + "\n", + "completion = client.chat.completions.create(\n", + " model=\"mistralai/Mistral-7B-Instruct-v0.3\",\n", + " messages=[{\"role\": \"user\", \"content\": prompt}],\n", + ")\n", + "\n", + "print(completion.choices[0].message[\"content\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "S1ooDMboeG6k" + }, + "source": [ + "## 7 System / Policy instructions (simulating LoRA-style policies)\n", + "\n", + "Simulation: bake policy or domain hints inside the prompt ([POLICY] ...). Notice how the reply changes when the policy changes.\n", + "\n", + "Real LoRA: train tiny adapter matrices (<<1 % params) and insert at Q/V/(K,O) projections of self-attention matrices. Needs only minutes, single GPU and the adapter file is <100 MB." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "koM2FhTJg2-q" + }, + "outputs": [], + "source": [ + "# Standard prompt\n", + "prompt = \"A customer is requesting a refund because a product was delivered late. Write a professional reply.\"\n", + "\n", + "completion = client.chat.completions.create(\n", + " model=\"mistralai/Mistral-7B-Instruct-v0.3\",\n", + " messages=[{\"role\": \"user\", \"content\": prompt}],\n", + ")\n", + "\n", + "print(\"Standard Prompt:\\n\", completion.choices[0].message[\"content\"], \"\\n---\")\n", + "\n", + "# Policy A: full refund, no return required\n", + "prompt_custom_1 = '''[INTERNAL POLICY: Respond with empathy, offer refund without requiring customer to return the item, and apologize sincerely. Use concise and positive tone.]\n", + "\n", + "A customer is requesting a refund because a product was delivered late. Write a professional reply.'''\n", + "\n", + "completion_custom_1 = client.chat.completions.create(\n", + " model=\"mistralai/Mistral-7B-Instruct-v0.3\",\n", + " messages=[{\"role\": \"user\", \"content\": prompt_custom_1}],\n", + ")\n", + "print(\"Policy 1:\\n\", completion_custom_1.choices[0].message[\"content\"], \"\\n---\")\n", + "\n", + "# Policy B: no refund, offer discount code\n", + "prompt_custom_2 = '''[INTERNAL POLICY: Do not offer refunds, instead offer 15% discount code.]\n", + "\n", + "A customer is requesting a refund because a product was delivered late. Write a professional reply.'''\n", + "\n", + "completion_custom_2 = client.chat.completions.create(\n", + " model=\"mistralai/Mistral-7B-Instruct-v0.3\",\n", + " messages=[{\"role\": \"user\", \"content\": prompt_custom_2}],\n", + ")\n", + "print(\"Policy 2:\\n\", completion_custom_2.choices[0].message[\"content\"], \"\\n---\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "L6PVrBLGebEB" + }, + "source": [ + "## 8 Structured JSON-formatted\n", + "\n", + "Explicitly combining role, task, format and give a skeleton:\n", + "\n", + "```\n", + "Return ONLY valid JSON:\n", + "{\"greeting\":\"\", \"apology\":\"\", \"resolution\":\"\", \"action_type\":\"\", \"needs_follow_up\":false}\n", + "```\n", + "\n", + "Model tends to obey if the request is precise and examples are consistent." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "OdBNug_3g3OX" + }, + "outputs": [], + "source": [ + "prompt = '''\n", + "You are a customer support assistant. A customer is requesting a refund because a product arrived late.\n", + "\n", + "Return the response in JSON format with the following fields:\n", + "- greeting\n", + "- apology\n", + "- refund_policy\n", + "- closing\n", + "\n", + "Example format:\n", + "{\n", + " \"greeting\": \"...\",\n", + " \"apology\": \"...\",\n", + " \"refund_policy\": \"...\",\n", + " \"closing\": \"...\"\n", + "}\n", + "'''\n", + "\n", + "completion = client.chat.completions.create(\n", + " model=\"mistralai/Mistral-7B-Instruct-v0.3\",\n", + " messages=[{\"role\": \"user\", \"content\": prompt}],\n", + ")\n", + "print(completion.choices[0].message[\"content\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "N2Go8Hszztto" + }, + "source": [ + "## 9 Take-aways\n", + "\n", + "- **Prompt engineering for rapid iteration**: you can prototype different behaviours in minutes, no fine-tuning needed. \n", + "- **System vs. User prompts**: internal policies (system) can override the tone and policy of the final answer. \n", + "- **Structured output**: JSON or numbered lists make post-processing trivial. \n", + "\n", + "### Next experiments\n", + "\n", + "1. Add temperature/top-p arguments to explore creativity vs. safety of the model. \n", + "2. Test the same prompts on a different model (e.g., **Llama-3-8B-Instruct**) to compare style and latency. \n", + "3. Combine with *LoRA-style adapters* (see Notebook 2) for policy adherence without needing bulky prompts. " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "7yg1gPSZztto" + }, + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "gpuType": "T4", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3.10.10 ('.venv': venv)", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.10.10" + }, + "vscode": { + "interpreter": { + "hash": "12a8c86496d70dc5ec458316f0fdc66c09147f19c1386e8b96b40af59bd1e3d4" + } + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/Academy/AI/3-LLM/notebooks/2_LabLLM_LoRA.ipynb b/Academy/AI/3-LLM/notebooks/2_LabLLM_LoRA.ipynb new file mode 100644 index 0000000..33913c6 --- /dev/null +++ b/Academy/AI/3-LLM/notebooks/2_LabLLM_LoRA.ipynb @@ -0,0 +1,504 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "BVqac1pg809I" + }, + "source": [ + "# LoRA Fine-Tuning Lab: T5-small Customer-Support Adapter\n", + "\n", + "Welcome! \n", + "This hands-on notebook will guide you, step by step, through fine-tuning a **T5-small** model with **LoRA (Low-Rank Adaptation)** so it can answer customer-support questions in a consistent, structured style. \n", + "Indeed, out-of-the-box **T5-small** has never seen our customer-support style or policy. \n", + "If you prompt it with “My order arrived late, I want a refund,” it answers vaguely, or not at all. \n", + "\n", + "In this notebook we will:\n", + "\n", + "1. **Measure the baseline** and see how poorly vanilla T5 handles eight real support prompts. \n", + "2. **Attach a tiny LoRA adapter** (\\~ 9 MB, 8-rank) and fine-tune it on **just 250 examples**. \n", + "3. **Re-test the same prompts** to verify that the adapted model now produces concise, policy-compliant replies. \n", + "\n", + "**Key takeaway:** with LoRA we upgrade a generic language model into a task-specialist in ~10 minutes on a free Colab GPU, without touching the original 60 M parameters.\n", + "\n", + "## What is LoRA (Low-Rank Adaptation)\n", + "\n", + "Instead of updating all hundreds of millions of parameters, LoRA freezes the original model and inserts two tiny matrices into selected linear layers (often the Q and V projections). Training adjusts only those low-rank “adapter” weights. So you need far less GPU memory, reach good quality with small datasets, and ship adapters (\\~10 MB) instead of full checkpoints (\\~2 GB)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "XvH1ePiH809K" + }, + "source": [ + "## 0 Environment Clean-up\n", + "Before starting, you may want to delete any previous artefacts (checkpoints, logs, etc.) so the run is fresh and reproducible. \n", + "Feel free to skip this cell if you have nothing to clean." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "kWEJ1_vBLlja" + }, + "outputs": [], + "source": [ + "# Remove previous training artefacts—run only if you need a fresh start\n", + "!rm -rf t5-lora-out\n", + "!rm -rf t5-small-lora-adapter" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "UMkIRIWx809M" + }, + "source": [ + "## 1 Install Dependencies\n", + "We rely on the **Hugging Face Transformers** ecosystem plus two helper libraries:\n", + "- **`transformers`**: model and trainer APIs.\n", + "- **`datasets`**: efficient data loading from HF.\n", + "- **`peft`** – adds LoRA and other parameter-efficient methods. \n", + "- **`accelerate`** – handles device placement (CPU / single-GPU / multi-GPU) transparently.\n", + "\n", + "Installation is one time." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "sIvNNZPcg3T-" + }, + "outputs": [], + "source": [ + "# Transformers, Datasets, PEFT, and Accelerate (quiet install)\n", + "!pip install -q transformers datasets peft accelerate" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ponHU0RFA8Ms" + }, + "source": [ + "## 2 Baseline Check: How well does vanilla T5-small handle our task?\n", + "\n", + "Before we train anything, let's ask the out-of-the-box model to draft a refund reply. \n", + "Spoiler: its answer will be generic, overly long, or simply unrelated because T5-small has never been told what our support policy or tone should be.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Qjw3r_bMA631" + }, + "outputs": [], + "source": [ + "from transformers import AutoTokenizer, AutoModelForSeq2SeqLM\n", + "\n", + "tokenizer = AutoTokenizer.from_pretrained(\"t5-small\")\n", + "model_base = AutoModelForSeq2SeqLM.from_pretrained(\"t5-small\")\n", + "\n", + "prompt = \"reply to this customer's email: My order arrived late and I want a refund.\"\n", + "inputs = tokenizer(prompt, return_tensors=\"pt\")\n", + "\n", + "# Greedy decode to keep things deterministic and short\n", + "outputs = model_base.generate(**inputs, max_new_tokens=120)\n", + "print(\"Vanilla T5-small says:\\n\")\n", + "print(tokenizer.decode(outputs[0], skip_special_tokens=True))\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "tkYjmNZ7809N" + }, + "source": [ + "## 3 Load CSV Dataset and transform it to HF Dataset\n", + "\n", + "Upload to Colab filesystem the file of our dataset: `files/customer_support_lora_dataset_250`. \n", + "Our CSV file has two columns:\n", + "\n", + "| Column name | What it contains |\n", + "|---------------|--------------------------------------------------|\n", + "| `input_text` | A raw customer request or complaint |\n", + "| `target_text` | The ideal structured reply we want the model to generate |\n", + "\n", + "We'll turn the CSV into a **HF Dataset** object so it plays nicely with the Trainer API." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Xm3Hrq4o6bGd" + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "from datasets import Dataset\n", + "\n", + "# Read the 250-row customer-support file\n", + "df = pd.read_csv(\"customer_support_lora_dataset_250.csv\")\n", + "ds = Dataset.from_pandas(df)\n", + "\n", + "print(\"Sample row:\") # quick sanity check\n", + "ds[0]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "2HlULnbn809P" + }, + "source": [ + "## 3 Tokenisation and Label Preparation\n", + "\n", + "Transformer models can't read raw text, they need **token IDs**.\n", + "For sequence-to-sequence models like T5 we must prepare **two** sequences:\n", + "\n", + "1. **Source** - the customer request (`input_text`) \n", + "2. **Target** - the desired reply (`target_text`) \n", + "\n", + "### Key details:\n", + "\n", + "- We call `tokenizer.as_target_tokenizer()` so the decoder uses its own special prefix tokens. \n", + "- We truncate to 128 tokens to keep batches small on modest GPUs.\n", + "\n", + "### What exactly are \"tokens\" (the 128-token limit)? \n", + "A token is not a word or a single character.\n", + "Transformers work on sub-word units produced by a tokenizer (for T5 that's a SentencePiece model with a 32 k-item vocabulary). The rules are learned from large corpora and try to strike a balance:\n", + "\n", + "| Example text | Tokens generated | Notes |\n", + "| ------------- | ---------------------- | ------------------------------------- |\n", + "| `tracking` | `▁track`, `ing` | the leading “▁” marks a word start |\n", + "| `refund` | `▁refund` | common words are often a single token |\n", + "| `extra-large` | `▁extra`, `-`, `large` | punctuation becomes its own token |\n", + "\n", + "Because tokens can be full words or fragments, the length in tokens is usually 1.3-1.6x shorter than counting raw characters but longer than counting full words.\n", + "A 128-token limit therefore fits roughly 75-100 English words (fewer if the text contains many rare names, URLs, or emojis that split into multiple tokens)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "X4z3IZjFrwMG" + }, + "outputs": [], + "source": [ + "from transformers import AutoTokenizer, DataCollatorForSeq2Seq\n", + "\n", + "tokenizer = AutoTokenizer.from_pretrained(\"t5-small\")\n", + "\n", + "def preprocess(example):\n", + " # Encode source\n", + " model_inputs = tokenizer(example[\"input_text\"], max_length=128, truncation=True)\n", + " # Encode target as labels\n", + " with tokenizer.as_target_tokenizer():\n", + " labels = tokenizer(example[\"target_text\"], max_length=128, truncation=True)\n", + " model_inputs[\"labels\"] = labels[\"input_ids\"]\n", + " return model_inputs\n", + "\n", + "ds_tok = ds.map(preprocess, remove_columns=ds.column_names)\n", + "\n", + "print(\"Input: \", ds[0][\"input_text\"])\n", + "print(\"Input tokens: \", len(ds_tok[0][\"input_ids\"]))\n", + "print(\"Target: \", ds[0][\"target_text\"])\n", + "print(\"Target tokens: \", len(ds_tok[0][\"labels\"]))\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "LU43xk3P809R" + }, + "source": [ + "## 4 Build Base Model and LoRA Configuration\n", + "\n", + "Here we load the vanilla **T5-small** (60 M parameters) and wrap it with a `peft.LoraConfig`.\n", + "\n", + "### Key hyper-parameters:\n", + "\n", + "| Parameter | Role | Here |\n", + "|--------------|---------------------------------------------------------------|------|\n", + "| `r` | Rank of the low-rank matrices (higher = more learning capacity) | 8 |\n", + "| `lora_alpha` | Scaling factor for the adapter’s update | 16 |\n", + "| `target_modules` | Which weight matrices get adapters (we pick **q** & **v**) | [\"q\",\"v\"] |\n", + "| `lora_dropout` | Regularisation inside adapters | 0.05 |\n", + "\n", + "### Why place LoRA adapters on q and v? What about the others?\n", + "\n", + "| Symbol | Full name | Role in self-attention |\n", + "| ------ | ------------------------------- | ------------------------------ |\n", + "| **Q** | **Query** projection | asks “what am I looking for?” |\n", + "| **K** | **Key** projection | represents “what do I have?” |\n", + "| **V** | **Value** projection | holds the information to mix |\n", + "| **O** | **Output** (final linear layer) | re-mixes heads after attention |\n", + "\n", + "A complete attention block has four projection matrices per head. Putting LoRA on all four gives maximum flexibility but also multiplies train-time memory. \n", + "Empirical sweet-spot: Research (LoRA paper §5 and several follow-ups) showed that adapting Q + V captures most task-specific gains while keeping parameter count and GPU RAM minimal. The intuition:\n", + "- Queries (Q) change how each token attends to others.\n", + "- Values (V) change what content is blended once attention scores are computed.\n", + "\n", + "Keys and the output layer matter too, but adjusting them yields diminishing returns for many language-generation tasks." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "qJwCKCFFrKgI" + }, + "outputs": [], + "source": [ + "from transformers import AutoModelForSeq2SeqLM\n", + "from peft import LoraConfig, get_peft_model, TaskType\n", + "\n", + "base_model = AutoModelForSeq2SeqLM.from_pretrained(\"t5-small\")\n", + "\n", + "lora_cfg = LoraConfig(\n", + " task_type = TaskType.SEQ_2_SEQ_LM, # generation task\n", + " r = 8, # rank of the LoRA matrices\n", + " lora_alpha = 16, # scaling\n", + " target_modules = [\"q\", \"v\"], # project only query & value matrices\n", + " lora_dropout = 0.05,\n", + " bias = \"none\"\n", + ")\n", + "\n", + "peft_model = get_peft_model(base_model, lora_cfg)\n", + "peft_model.print_trainable_parameters()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ezlVpv2R809S" + }, + "source": [ + "## 5 Training Arguments and Trainer Loop\n", + "\n", + "Hugging Face `Seq2SeqTrainer` takes care of the full training loop (forward, back-prop, gradient clipping, etc.).\n", + "\n", + "Important flags we set:\n", + "\n", + "- **`per_device_train_batch_size` = 16**, fits on a 12 GB GPU. \n", + "- **`num_train_epochs` = 30**, small dataset needs more passes. \n", + "- **`learning_rate` = 5e-4**, slightly higher than full-fine-tuning, because we’re optimising far fewer weights. \n", + "- **`save_strategy` = \"no\"**, skip checkpoints to save disk. You can change it to `\"epoch\"` if you want them. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true, + "id": "_zAx2Eq2raxS" + }, + "outputs": [], + "source": [ + "from transformers import Seq2SeqTrainer, Seq2SeqTrainingArguments, DataCollatorForSeq2Seq\n", + "\n", + "training_args = Seq2SeqTrainingArguments(\n", + " output_dir = \"./t5-lora-out\",\n", + " per_device_train_batch_size = 16,\n", + " num_train_epochs = 30,\n", + " learning_rate = 5e-4,\n", + " logging_steps = 5,\n", + " save_strategy = \"no\",\n", + " report_to = \"none\",\n", + ")\n", + "\n", + "trainer = Seq2SeqTrainer(\n", + " model = peft_model,\n", + " args = training_args,\n", + " train_dataset = ds_tok,\n", + " data_collator = DataCollatorForSeq2Seq(tokenizer, model=peft_model),\n", + ")\n", + "\n", + "trainer.train()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "WD8UKp4g809T" + }, + "source": [ + "## 6 Save Adapter and Tokenizer\n", + "\n", + "LoRA lets us store **only** the lightweight adapter, ~9 MB in this case. The base T5-small weights are **not duplicated**." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "bbw5BG817UvV" + }, + "outputs": [], + "source": [ + "trainer.model.save_pretrained(\"t5-small-lora-adapter\")\n", + "tokenizer.save_pretrained(\"t5-small-lora-adapter\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "H8U36q-C809T" + }, + "source": [ + "## 7 Load LoRA-Adapted Model for Inference\n", + "\n", + "We merge the adapter with the frozen base model at load time, then generate a reply for a sample complaint." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "eG0hnG43qlAM" + }, + "outputs": [], + "source": [ + "from peft import PeftConfig, PeftModel\n", + "from transformers import AutoModelForSeq2SeqLM\n", + "\n", + "# Load base model\n", + "cfg = PeftConfig.from_pretrained(\"t5-small-lora-adapter\")\n", + "base_model = AutoModelForSeq2SeqLM.from_pretrained(cfg.base_model_name_or_path)\n", + "# Load LoRA adapter\n", + "model_lora = PeftModel.from_pretrained(base_model, \"t5-small-lora-adapter\")\n", + "\n", + "# Test input\n", + "prompt = \"generate reply: My order arrived late. I want a refund.\"\n", + "# Tokenize the test input\n", + "inputs = tokenizer(prompt, return_tensors=\"pt\")\n", + "# Generate reply\n", + "outputs = model_lora.generate(**inputs, max_new_tokens=80)\n", + "\n", + "print(\"LoRA reply:\\n\", tokenizer.decode(outputs[0], skip_special_tokens=True))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "RI_MC1cU809U" + }, + "source": [ + "## 8 Side-by-Side Evaluation (Base vs LoRA)\n", + "\n", + "Let’s run eight realistic prompts through both the vanilla T5-small and our LoRA-adapted version, then print the outputs in a table for quick eyeballing. \n", + "You should notice LoRA replies are:\n", + "- More structured (e.g., include apology and next steps) \n", + "- Shorter and on brand \n", + "- Consistent JSON or bullet style, depending on your `target_text` examples " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "A0j9NQUt7e6F" + }, + "outputs": [], + "source": [ + "# Compare vanilla T5-small with LoRA fine‑tuned adapter on structured JSON output\n", + "import json, pandas as pd\n", + "from transformers import AutoTokenizer, AutoModelForSeq2SeqLM\n", + "from peft import PeftConfig, PeftModel\n", + "\n", + "tokenizer = AutoTokenizer.from_pretrained(\"t5-small\")\n", + "\n", + "# Base (pre‑trained) model\n", + "base_model = AutoModelForSeq2SeqLM.from_pretrained(\"t5-small\")\n", + "\n", + "# LoRA‑adapted model, make sure this path matches the one used in the training cell\n", + "adapter_path = \"t5-small-lora-adapter\"\n", + "cfg = PeftConfig.from_pretrained(adapter_path)\n", + "ft_model = PeftModel.from_pretrained(\n", + " AutoModelForSeq2SeqLM.from_pretrained(cfg.base_model_name_or_path),\n", + " adapter_path\n", + ")\n", + "\n", + "def generate(model, prompt):\n", + " ids = tokenizer(prompt, return_tensors=\"pt\")\n", + " out = model.generate(**ids, max_new_tokens=120)\n", + " return tokenizer.decode(out[0], skip_special_tokens=True)\n", + "\n", + "test_prompts = [\n", + " \"Compose a response to this customer email: My order arrived late. I want a refund.\",\n", + " \"Draft a reply to this customer message: The product I received is damaged. What can I do?\",\n", + " \"Write a response to this email from a client: I received the wrong item in my order.\",\n", + " \"Create a reply for this customer's email: How can I return an item I purchased last week?\",\n", + " \"Formulate a response to the customer's email: I never received my order.\",\n", + " \"Respond to this message from the customer: Why was I charged twice for my order?\",\n", + " \"Prepare a reply to this client email: I need help tracking my shipment.\",\n", + " \"Construct a response for the customer's message: Can I exchange my item for a different size?\"\n", + "]\n", + "\n", + "records = []\n", + "for p in test_prompts:\n", + " records.append({\n", + " \"prompt\": p,\n", + " \"T5-base\": generate(base_model, p),\n", + " \"LoRA\": generate(ft_model, p)\n", + " })\n", + "\n", + "df = pd.DataFrame(records)\n", + "print(df.to_markdown(index=False))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "csHeHzww809V" + }, + "source": [ + "## 9 Next Steps\n", + "\n", + "1. **Quantisation**: combine LoRA with 8-bit weights using `bitsandbytes` library to shrink disk size and speed up inference. \n", + "2. **Hyper-parameter search**: try different ranks (`r`) and target modules (add **k** and **o** matrices) for possibly better accuracy. \n", + "3. **Objective metrics**: integrate BLEU, ROUGE-L, or a custom JSON validator to track quality over epochs. \n", + "4. **Deployment**: merge base + adapter and serve via FastAPI, Streamlit, or Hugging Face Inference Endpoints. \n", + "5. **Prompt scaffolding**: prepend `\"generate structured_reply:\"` automatically so end-users don’t need to remember it. \n", + "\n", + "Happy fine-tuning!" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "TG70O8z6809V" + }, + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "gpuType": "T4", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3.10.10 ('.venv': venv)", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.10.10" + }, + "vscode": { + "interpreter": { + "hash": "12a8c86496d70dc5ec458316f0fdc66c09147f19c1386e8b96b40af59bd1e3d4" + } + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/Workshops/AI/01-Introduction-to-ML-and-MLOps/.DS_Store b/Workshops/AI/01-Introduction-to-ML-and-MLOps/.DS_Store new file mode 100644 index 0000000..eff5404 Binary files /dev/null and b/Workshops/AI/01-Introduction-to-ML-and-MLOps/.DS_Store differ diff --git a/Workshops/AI/01-Introduction-to-ML-and-MLOps/.gitignore b/Workshops/AI/01-Introduction-to-ML-and-MLOps/.gitignore new file mode 100644 index 0000000..1c5d00a --- /dev/null +++ b/Workshops/AI/01-Introduction-to-ML-and-MLOps/.gitignore @@ -0,0 +1,179 @@ +# local mlflow +mlartifacts/* +mlruns/* +dev/* + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# UV +# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +#uv.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/latest/usage/project/#working-with-version-control +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +# Ruff stuff: +.ruff_cache/ + +# PyPI configuration file +.pypirc \ No newline at end of file diff --git a/Workshops/AI/01-Introduction-to-ML-and-MLOps/README.md b/Workshops/AI/01-Introduction-to-ML-and-MLOps/README.md new file mode 100644 index 0000000..648588c --- /dev/null +++ b/Workshops/AI/01-Introduction-to-ML-and-MLOps/README.md @@ -0,0 +1,108 @@ +# Lab: Basic ML Model with Weather Dataset + MLflow Integration + +Welcome to this lab! Here you will learn how to: + +1. **Load and prepare a weather dataset**, with temperature and humidity data. +2. **Train a Machine Learning model** using Scikit-learn, a powerful tool for Machine Learning in Python, to predict rain. +3. **Evaluate the model** computing metrics to determine how well it makes predictions on new data. +4. **Integrate MLflow**, one of the most used tool to track metrics, parameters, and model versions. + +We will follow a guided approach with detailed explanations at each step. +The first part focuses on Scikit-learn and the weather dataset. The second part extends the existing code with MLflow. + +--- + +## Part 0: Setting Up the Python Environment + +Before we begin, we need to ensure we have everything required to run the lab properly. + +### **Requirements** + +This lab assumes that **Python** and **miniconda** are already installed, the repository [kiratech/labs](https://github.com/kiratech/labs.git) is accessible, and **Git** is properly configured on your local machine. Furthermore, **VSCode** or an IDE able to run Jupyter Notebooks, must be installed as well. +In order to execute this laboratory, you will be asked to install a set of tools common in Machine Learning field: + +- [**scikit-learn**](https://scikit-learn.org/stable/index.html): Machine learning library with modeling and evaluation tools. +- [**pandas**](https://pandas.pydata.org/): Framework for data manipulation and analysis in Python. +- [**seaborn**](https://seaborn.pydata.org/): Data visualization library based on Matplotlib. +- [**mlflow**](https://mlflow.org/): Tool for tracking and managing machine learning experiments. + +### 1. Clone the Repository + +To start, clone the lab repository by running the following command in the terminal: + +```sh + git clone https://github.com/kiratech/labs.git +``` + +### 2. Checkout the Lab Branch + +After cloning the repository, navigate to the project folder: + +```sh + cd labs +``` + +Then, checkout the `lab` branch: + +```sh + git checkout academy-ai +``` + +This folder contains resources related to AI-themed labs. + +### 3. Navigate to the 1-Intro-To-ML-And-MLOps Folder + +Go to the folder of the first lab: + +```sh + cd Academy/AI/1-Intro-To-ML-And-MLOps +``` + +### 4. Create a Virtual Environment + +A virtual environment allows you to isolate the project's dependencies from the system-wide ones. + +Using the **VSCode** terminal, create a virtual environment: + +```sh + conda create --name lab_env python=3.12 pip -y +``` + +Activate the Virtual Environment: + +```sh + conda activate lab_env +``` + +You should see the `(lab_env)` prefix in the terminal, indicating that the virtual environment is active. + +### 5. Install Required Packages + +Besides the default packages in the environment, additional libraries may be needed for the lab. +Before installing them, it's always a good practice to update `pip` to avoid compatibility issues: + +```sh + pip install --upgrade pip +``` + +Now, install some essential packages for data analysis and machine learning: + +```sh + pip install scikit-learn pandas seaborn mlflow ipykernel +``` + +Verify that the packages were installed correctly with: + +```sh + conda list +``` + +At this point, you can proceed with the `lab.ipynb` file. + +### 6. Open the Project in VSCode + +At this point, open VSCode from the file explorer or by running the command: + +```sh + code . +``` diff --git a/Workshops/AI/01-Introduction-to-ML-and-MLOps/lab.ipynb b/Workshops/AI/01-Introduction-to-ML-and-MLOps/lab.ipynb new file mode 100644 index 0000000..e2352e5 --- /dev/null +++ b/Workshops/AI/01-Introduction-to-ML-and-MLOps/lab.ipynb @@ -0,0 +1,563 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Lab: Basic ML Model with Weather Dataset + MLflow Integration\n", + "\n", + "Welcome to this lab! Here you will learn how to:\n", + "\n", + "1. **Load and prepare a weather dataset**, with temperature and humidity data.\n", + "2. **Train a Machine Learning model** using Scikit-learn, a powerful tool for Machine Learning in Python, to predict rain.\n", + "3. **Evaluate the model** computing metrics to determine how well it makes predictions on new data.\n", + "4. **Integrate MLflow**, one of the most used tool to track metrics, parameters, and model versions.\n", + "\n", + "We will follow a guided approach with detailed explanations at each step. \n", + "The first part focuses on Scikit-learn and the weather dataset. The second part extends the existing code with MLflow." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Part 1: From Data to Machine Learning Model (Supervised Learning with Scikit-learn) \n", + "\n", + "### Objective \n", + "Build a **classification model** that can predict whether it will rain, using **temperature** and **humidity** as input data. The model will be trained using **Scikit-learn**, a powerful tool for Machine Learning in Python." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1. Preparing the Dataset \n", + "\n", + "Before training a Machine Learning model, it is essential to clean the data, as missing or incorrect values can compromise predictions. A well-prepared dataset allows the model to learn better and provide more accurate results. \n", + "\n", + "For this lab, we will use an example dataset: \n", + "[Weather Test Data](https://raw.githubusercontent.com/boradpreet/Weather_dataset/refs/heads/master/Weather%20Test%20Data.csv) \n", + "\n", + "The **Weather Test Data** dataset contains meteorological information collected at different times. Each row represents an observation with parameters such as **temperature**, **humidity**, **atmospheric pressure**, and other weather variables. \n", + "\n", + "The goal of this dataset is to analyze weather patterns and use them to train a Machine Learning model capable of predicting future conditions, such as the probability of rain or temperature variations." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "\n", + "# Dataset URL \n", + "url = \"https://raw.githubusercontent.com/boradpreet/Weather_dataset/refs/heads/master/Weather%20Test%20Data.csv\"\n", + "\n", + "# Load the dataset in a Pandas dataframe\n", + "df = pd.read_csv(url)\n", + "\n", + "# Show first 5 rows\n", + "df.head(5)\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2. Data Exploration and Cleaning \n", + "\n", + "To ensure our model works correctly, we first need to examine and prepare the dataset. Here are the key steps: \n", + "\n", + "1. **Check for missing data**: Identify if there are any missing values, as they could compromise model training. If necessary, we can remove them or replace them with appropriate values. \n", + "2. **Convert the `Label` column**: Transform categorical values (*NoRain* and *Rain*) into numerical values (0 for *NoRain*, 1 for *Rain*), so the model can interpret them correctly. \n", + "3. **Select key features**: Choose only the most relevant columns (e.g., temperature and humidity) to simplify the model and improve its performance.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 1. Remove missing values\n", + "df = df.dropna()\n", + "\n", + "# 2. Transform column 'RainToday' into numerical values\n", + "df['RainToday'] = df['RainToday'].apply(lambda x: 1 if x == 'Yes' else 0)\n", + "\n", + "# 3. Feature selection \n", + "features = ['MinTemp', 'MaxTemp', 'Humidity3pm', 'Humidity9am']\n", + "\n", + "X = df[features]\n", + "y = df['RainToday']\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 3. Splitting the Dataset into Training and Test Sets \n", + "\n", + "To properly train and evaluate the model, we split the dataset into two parts: \n", + "\n", + "- **X (features)**: Contains the information we will use for predictions, such as **temperature** and **humidity**. \n", + "- **y (target)**: Represents the variable we want to predict, i.e., whether it will rain (*Rain*) or not (*NoRain*). \n", + "\n", + "We split the data into **80% training set** and **20% test set** for the following reasons: \n", + "\n", + "1. **Model Training** \n", + " - 80% of the data is used to teach the model to recognize patterns between features and the target variable. \n", + "\n", + "2. **Model Evaluation** \n", + " - The remaining 20% of the data is not used in training but serves to test the model on unseen data. \n", + " - This helps us understand whether the model can make accurate predictions on new data. \n", + "\n", + "3. **Avoid Overfitting** \n", + " - If we tested the model on the same data it was trained on, we might get deceptively good results, as the model would have simply memorized them. \n", + " - Using separate test data helps verify whether the model can generalize its predictions to real-world data. \n", + "\n", + "This split is a crucial step in building a reliable model capable of making accurate predictions on unseen data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.model_selection import train_test_split\n", + "\n", + "# Dataset split (80% training, 20% testing)\n", + "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n", + "print(f\"Training dataset dimensions: {len(X_train)}\")\n", + "print(f\"Test dataset dimensions: {len(X_test)}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 4. Building and Training the Model \n", + "\n", + "Now that we have prepared the data, we can build and train a Machine Learning model. For this, we will use a classifier called [**RandomForestClassifier**](https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.RandomForestClassifier.html), one of the most widely used techniques for classification problems. \n", + "\n", + "#### Why use **Random Forest**? \n", + "- It is a model based on **decision trees**, which divide the data into multiple steps to make accurate decisions. \n", + "- It is **robust** and works well with both numerical and categorical data. \n", + "- It is less sensitive to noisy data than a single decision tree because it combines multiple trees to improve accuracy. \n", + "\n", + "The model will be trained using **temperature** and **humidity** data to predict whether there will be **rain** or not. After training, we will test it on new data to evaluate its accuracy." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.ensemble import RandomForestClassifier\n", + "\n", + "# Create and train the model\n", + "model = RandomForestClassifier(n_estimators=100, random_state=42)\n", + "model.fit(X_train, y_train)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 5. Model Evaluation \n", + "\n", + "After training the model, we need to verify how accurate its predictions are. To do this, we will calculate **accuracy** and other evaluation metrics. \n", + "\n", + "#### Why is model evaluation important? \n", + "A Machine Learning model is not useful if we do not know how reliable it is. Evaluation helps us understand: \n", + "- **Whether the model is learning correctly from the data** or simply memorizing answers (overfitting). \n", + "- **Whether it can be used on new data** and make realistic predictions. \n", + "\n", + "#### Confusion Matrix \n", + "Besides accuracy, we will use the **confusion matrix**, a visual method that shows where the model makes correct predictions and where it makes mistakes. \n", + "- It helps identify **false positives** and **false negatives**, which are critical errors in many real-world scenarios. \n", + "- It is useful for improving the model, for example, by adjusting decision thresholds or balancing input data. \n", + "\n", + "With these analyses, we can determine whether our model is ready for use or needs improvement." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.metrics import accuracy_score, f1_score, classification_report\n", + "\n", + "# Prediction on test set\n", + "y_pred = model.predict(X_test)\n", + "\n", + "# Compute accuracy and f1-score\n", + "accuracy = accuracy_score(y_test, y_pred)\n", + "f1 = f1_score(y_test, y_pred, average='weighted')\n", + "print(f\"Accuracy of the model: {accuracy:.2f}\")\n", + "print(f\"F1-score of the model: {f1:.2f}\")\n", + "\n", + "# Print classification report\n", + "print(classification_report(y_test, y_pred))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import seaborn as sns\n", + "import matplotlib.pyplot as plt\n", + "from sklearn.metrics import confusion_matrix\n", + "\n", + "# Create a heatmap with Seaborn\n", + "target_names = [\"No Rain\", \"Rain\"]\n", + "cm = confusion_matrix(y_test, y_pred)\n", + "plt.figure(figsize=(6, 4))\n", + "sns.heatmap(cm, annot=True, fmt=\"d\", cmap=\"Blues\", xticklabels=target_names, yticklabels=target_names)\n", + "\n", + "# Add titles\n", + "plt.xlabel(\"Predicted Label\")\n", + "plt.ylabel(\"True Label\")\n", + "plt.title(\"Confusion Matrix\")\n", + "\n", + "# Show the plot\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Conclusion (Part 1) \n", + "\n", + "In this first part, we followed a step-by-step process to build a Machine Learning model capable of predicting rain. Here’s what we did: \n", + "\n", + "1. **Loaded the weather dataset** to analyze temperature, humidity, and other variables. \n", + "2. **Cleaned and prepared the data**, handling missing values and converting the target variable into a format the model can understand. \n", + "3. **Split the dataset** into training (80%) and test (20%) sets to properly train and evaluate the model. \n", + "4. **Built a classification model** using **RandomForestClassifier**, a powerful and robust algorithm. \n", + "5. **Evaluated the model’s performance** by calculating accuracy and analyzing the confusion matrix to identify errors. \n", + "\n", + "Now that we have built the base model, in the next part, we will explore how to integrate **MLflow** to track experiments and further improve performance." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Tech: Installing and Configuring MLflow \n", + "\n", + "### Objective \n", + "Set up a local **MLflow** instance to log experiments, monitor metrics, and manage Machine Learning models in an organized way. \n", + "\n", + "### 1. Starting MLflow \n", + "\n", + "To start MLflow locally, run the following command in the terminal: \n", + "\n", + "```bash\n", + " mlflow ui\n", + "```\n", + "\n", + "Once started, the graphical interface will be accessible at: \n", + "\n", + "```\n", + " http://127.0.0.1:5000\n", + "```\n", + "\n", + "This setup allows **saving experiments and models locally**, enabling tracking of different model versions, comparing evaluation metrics, and optimizing training processes. \n", + "\n", + "In the next sections, we will see how to log parameters, metrics, and models directly within MLflow." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### MLflow in Production \n", + "\n", + "In production environments and for clients, **MLflow is not run locally** but is integrated into a more robust and scalable infrastructure. This prevents issues related to manual experiment management and data persistence. \n", + "\n", + "Common solutions include: \n", + "\n", + "- **Docker Compose** \n", + " - MLflow is started using a `docker-compose.yml` file, which configures a backend database and remote storage for saving experiments. \n", + " - This approach is useful for controlled environments where a quick and reproducible setup is needed. \n", + " - An example implementation is available in the internal repository: \n", + " [kiratech/mlops-service-portfolio](https://github.com/kiratech/mlops-service-portfolio/tree/main). \n", + "\n", + "- **Kubernetes (K8s)** \n", + " - MLflow is deployed on a **Kubernetes cluster**, allowing scalable and centralized experiment management. \n", + " - This approach is ideal for enterprise environments that require high levels of reliability, security, and scalability. \n", + "\n", + "Both solutions rely on a **multi-container architecture**, which includes: \n", + "- **A persistent database** (e.g., PostgreSQL or MySQL) to store experiment metadata. \n", + "- **An S3 or MinIO storage** to save models and artifacts, ensuring secure and scalable data management. \n", + "\n", + "These approaches ensure that MLflow can be reliably used in production, integrating with cloud or on-premise infrastructures for effective Machine Learning model management." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Part 2: MLflow Integration \n", + "\n", + "Now, we will extend the existing code to **track our experiments** using **MLflow**. This will allow us to monitor the model training process, compare different configurations, and manage model versions in a structured way. \n", + "\n", + "### Why integrate MLflow? \n", + "With MLflow, we can: \n", + "- **Log training parameters** (e.g., `n_estimators` for Random Forest) to compare different configurations. \n", + "- **Save evaluation metrics** (e.g., accuracy, F1-score) to monitor the model’s performance. \n", + "- **Store the trained model** to easily reload and reuse it in the future without retraining. \n", + "\n", + "### Objective \n", + "Integrate MLflow into the existing code to **track and version models**, logging parameters, metrics, and artifacts in a structured way. \n", + "\n", + "### 1. MLflow Configuration \n", + "Before we start tracking experiments, let's set up the necessary variables to use MLflow in this project. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import mlflow\n", + "import mlflow.sklearn\n", + "\n", + "# Set name of the experiment and tracking URI of local instance\n", + "mlflow.set_tracking_uri(\"http://127.0.0.1:5000\")\n", + "mlflow.set_experiment(\"weather_classification_experiment\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We verify that from our [web interface](http://127.0.0.1:5000), the new experiment is visible." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2. Logging Parameters, Metrics, and Model \n", + "\n", + "With **MLflow**, we can automatically save and track various information during model training. This helps compare performances across different configurations and easily retrieve the best models. \n", + "\n", + "Here’s what we can log: \n", + "\n", + "- **Parameters** → Values used to configure the model, such as `n_estimators` (number of trees in Random Forest) and other hyperparameters. \n", + "- **Metrics** → Performance indicators of the model, such as **accuracy**, **F1-score**, precision, and recall. \n", + "- **Model** → The trained model version, which can be reloaded and reused without retraining. \n", + "\n", + "By logging these elements, we can analyze and compare different model versions in a structured and reproducible way." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Use the same type of model used in Part 1\n", + "import os\n", + "from sklearn.ensemble import RandomForestClassifier\n", + "from sklearn.metrics import accuracy_score, f1_score\n", + "from mlflow.models.signature import infer_signature\n", + "\n", + "# Execute 4 experiments to train multiple models \n", + "n_estimators = [1, 10, 100, 500]\n", + "\n", + "for n_e in n_estimators:\n", + " # Create a new MLFlow run\n", + " with mlflow.start_run():\n", + " # Log Param\n", + " mlflow.log_param(\"n_estimators\", n_e)\n", + "\n", + " # Create and train the model instance\n", + " rf_model = RandomForestClassifier(n_estimators=n_e, random_state=42)\n", + " rf_model.fit(X_train, y_train)\n", + "\n", + " # Compute metrics\n", + " y_pred = rf_model.predict(X_test)\n", + " accuracy = accuracy_score(y_test, y_pred)\n", + " f1 = f1_score(y_test, y_pred, average='weighted')\n", + " mlflow.log_metric(\"accuracy\", accuracy)\n", + " mlflow.log_metric(\"f1\", f1)\n", + "\n", + " # Create heatmap with Seaborn\n", + " target_names = [\"No Rain\", \"Rain\"]\n", + " cm = confusion_matrix(y_test, y_pred)\n", + " plt.figure(figsize=(6, 4))\n", + " sns.heatmap(cm, annot=True, fmt=\"d\", cmap=\"Blues\", xticklabels=target_names, yticklabels=target_names)\n", + "\n", + " # Add titles\n", + " plt.xlabel(\"Predicted Label\")\n", + " plt.ylabel(\"True Label\")\n", + " plt.title(\"Confusion Matrix\")\n", + "\n", + " # Save the plot as PNG\n", + " if not os.path.exists(\"dev/\"):\n", + " os.makedirs(\"dev/\")\n", + " plt.savefig(\"dev/confusion_matrix.png\")\n", + " plt.close()\n", + " # Save the confusion matrix on MLFlow as artifact\n", + " mlflow.log_artifact(\"dev/confusion_matrix.png\")\n", + "\n", + " # Save the model on MLFlow\n", + " example_dict = {'MinTemp': 1.1, 'MaxTemp': 1.1, 'Humidity3pm': 1.1, 'Humidity9am': 1.1}\n", + " signature = infer_signature(model_input=example_dict)\n", + " mlflow.sklearn.log_model(rf_model, \"random_forest_model\", signature=signature)\n", + "\n", + " print(f\"Experiment finished. Registered accuracy: {accuracy:.2f}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 3. Viewing and Comparing Results \n", + "\n", + "After logging parameters, metrics, and models, we can use **MLflow** to explore and compare different experiments. \n", + "\n", + "MLflow provides a web interface accessible at: \n", + "\n", + "```bash\n", + " http://127.0.0.1:5000\n", + "```\n", + "\n", + "By accessing this interface, in the **Experiments** section, it will be possible to: \n", + "- **Examine the parameters** used in each experiment. \n", + "- **Compare metrics** across different model configurations. \n", + "- **View and download saved models**, making reuse and deployment easier. \n", + "\n", + "This feature allows monitoring the model's performance evolution and quickly identifying the best configurations." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 4. Loading a Saved Model with MLflow \n", + "\n", + "MLflow allows saving and reloading trained models easily, avoiding the need to retrain them every time. \n", + "\n", + "To retrieve a saved model in MLflow, you need to copy the **run ID** of the executed experiment. This ID uniquely identifies each logged experiment and allows loading the corresponding model for future predictions. \n", + "\n", + "This feature is particularly useful for: \n", + "- **Reusing a trained model** without repeating the training process. \n", + "- **Comparing different model versions** to choose the most effective one. \n", + "- **Integrating the model into applications or APIs**, without rebuilding it from scratch. \n", + "\n", + "In the next sections, we will see how to perform this process in practice." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import mlflow.sklearn\n", + "from sklearn.metrics import accuracy_score\n", + "\n", + "# Insert a real run_id you find in MLFlow UI\n", + "RUN_ID = \"\"\n", + "\n", + "loaded_model = mlflow.sklearn.load_model(f\"runs:/{RUN_ID}/random_forest_model\")\n", + "\n", + "# Verify accuracy\n", + "y_loaded_pred = loaded_model.predict(X_test)\n", + "acc_loaded = accuracy_score(y_test, y_loaded_pred)\n", + "print(f\"Accuracy of the loaded model: {acc_loaded:.2f}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Conclusions \n", + "\n", + "In this lab, we followed a complete process to build and monitor a Machine Learning model applied to weather data. Specifically, we: \n", + "\n", + "1. **Created a classification model** using **Scikit-learn**, leveraging temperature and humidity to predict rain. \n", + "2. **Integrated MLflow** to track training parameters, log evaluation metrics, and manage model versions in a structured way. \n", + "3. **Explored and compared results** through the MLflow UI interface, reviewing different configurations and loading a saved model for future predictions. \n", + "\n", + "This approach allows us to improve the Machine Learning model development process, making it more organized, reproducible, and scalable." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Next Steps \n", + "\n", + "Now that we have built and tracked our model, we can explore further improvements and integrate the work into a more advanced workflow. \n", + "\n", + "- **Hyperparameter Optimization**: Test different configurations of `n_estimators`, `max_depth`, and other model parameters, logging results in **MLflow** to identify the best combination. \n", + "- **Automation with CI/CD**: Integrate a **Continuous Integration/Continuous Deployment (CI/CD)** system to automatically train and deploy new model versions, reducing the risk of manual errors. \n", + "- **Model Monitoring in Production**: Implement a **model drift monitoring** system to detect any drops in accuracy over time and determine when retraining with new data is necessary. \n", + "\n", + "These steps help transform the developed model into a robust and reliable system, ready for real-world applications." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.12.9 ('lab_env')", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.9" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "fbd6de99d3dee89886aa6f30475b6b26f6fe2ef531b8e041d1262739f0fd6851" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/Workshops/AI/02-ML-Model-Lifecycle/.DS_Store b/Workshops/AI/02-ML-Model-Lifecycle/.DS_Store new file mode 100644 index 0000000..b5c4c43 Binary files /dev/null and b/Workshops/AI/02-ML-Model-Lifecycle/.DS_Store differ diff --git a/Workshops/AI/02-ML-Model-Lifecycle/.gitignore b/Workshops/AI/02-ML-Model-Lifecycle/.gitignore new file mode 100644 index 0000000..7869f53 --- /dev/null +++ b/Workshops/AI/02-ML-Model-Lifecycle/.gitignore @@ -0,0 +1,180 @@ +# local mlflow +artifacts +mlruns +dev +data + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# UV +# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +#uv.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/latest/usage/project/#working-with-version-control +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +# Ruff stuff: +.ruff_cache/ + +# PyPI configuration file +.pypirc \ No newline at end of file diff --git a/Workshops/AI/02-ML-Model-Lifecycle/.pre-commit-config.yaml b/Workshops/AI/02-ML-Model-Lifecycle/.pre-commit-config.yaml new file mode 100644 index 0000000..7f9e0df --- /dev/null +++ b/Workshops/AI/02-ML-Model-Lifecycle/.pre-commit-config.yaml @@ -0,0 +1,6 @@ +# .pre-commit-config.yaml +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.3.0 + hooks: + - id: ruff diff --git a/Workshops/AI/02-ML-Model-Lifecycle/Makefile b/Workshops/AI/02-ML-Model-Lifecycle/Makefile new file mode 100644 index 0000000..14b3806 --- /dev/null +++ b/Workshops/AI/02-ML-Model-Lifecycle/Makefile @@ -0,0 +1,35 @@ +# Makefile — Local GitHub Actions simulation using nox + conda environments +.PHONY: setup ci train serve monitor retrain pipeline clean +SHELL := /bin/zsh + +# Conda environment name +ENV := lab_env_2 + +# Activate Conda + run nox in the correct environment +CONDA_ACTIVATE = source $$(conda info --base)/etc/profile.d/conda.sh ; conda activate ; conda activate +NOX := $(CONDA_ACTIVATE) $(ENV) && nox --no-venv + +# ------------------------------------------------------------ +# Environment setup +# ------------------------------------------------------------ + +setup: ## Create conda environment + install pre-commit hooks + conda env create -f environment.yaml --quiet + $(NOX) pre-commit install + +# ------------------------------------------------------------ +# Simulated Git branch pushes (as CI/CD entrypoints) +# ------------------------------------------------------------ + +push-feature: ## Simulate push to feature/* → hyperparameter sweep only + $(NOX) -R -s feature_pipeline + +push-develop: ## Simulate push to develop → lint + tests + train best model + $(NOX) -R -s lint + $(NOX) -R -s tests + $(NOX) -R -s develop_pipeline + +push-main: ## Simulate push to master/main → lint + tests + serve model + $(NOX) -R -s lint + $(NOX) -R -s tests + $(NOX) -R -s main_pipeline diff --git a/Workshops/AI/02-ML-Model-Lifecycle/README.md b/Workshops/AI/02-ML-Model-Lifecycle/README.md new file mode 100644 index 0000000..cbbfc3a --- /dev/null +++ b/Workshops/AI/02-ML-Model-Lifecycle/README.md @@ -0,0 +1,383 @@ +# Lab: From Experiments to Production with Wine Classification + +Welcome to this live lab! +In the previous lab session we: + +1. **Created a classification model** using **Scikit-learn**, leveraging temperature and humidity to predict rain. +2. **Integrated MLflow** to track training parameters, log evaluation metrics, and manage model versions in a structured way. +3. **Explored and compared results** through the MLflow UI interface, reviewing different configurations and loading a saved model for future predictions. + +In this session, you will learn how to: + +1. **Train a classification model** on a real dataset using `Scikit-learn`. +2. **Manage and track your experiments** with `MLflow`. +3. **Build CI/CD-like workflows** using `Makefile` and `nox`. +4. **Build scalable and orchestrated ML pipelines** using `Prefect`. +5. **Monitor data drift over time** and trigger retraining with `Evidently`. + +We will follow a step-by-step approach combining **code execution**, **conceptual explanation**, and **UI walkthroughs**. + +--- + +## 1 Context and Story + +A mid‑sized wine producer wants to modernise quality control. +Instead of relying on manual tasting alone, the company decides to analyse +chemical characteristics of each batch and predict the wine’s *cultivar* +(three classes) using machine learning. +The stakes: + +- **Repeatable experimentation** – new blends need fresh tuning +- **Fast promotion** – once a model looks good, it must move to production quickly +- **Operational monitoring** – chemical profiles may drift with soil and weather; the system must recognise drift and retrain autonomously + +Our lab demonstrates how to meet these requirements **entirely on a laptop**, +using open‑source tools and zero cloud resources. + +--- + +## 2 Dataset Overview – UCI Wine Recognition + +The lab uses the well-known [**Wine Recognition**](https://archive.ics.uci.edu/dataset/109/wine) dataset from the UCI Machine Learning Repository. +Originally collected at the Institute of Oenology and Viticulture in Italy, the dataset contains the results of a chemical analysis of **178 wine samples** from three different *cultivars* (grape varieties) grown in the same region. +Each sample is described by **13 continuous variables** that capture key chemical properties influencing flavour and quality. + +### 2.1 Features + +| Feature | Description | Typical Range | +| ------------------------------ | ------------------------------------------- | ------------- | +| `alcohol` | Ethyl-alcohol content (%) | 11 – 14 % | +| `malic_acid` | Malic acid concentration (g/L) | 0.7 – 5.8 | +| `ash` | Total ash (g/100 mL) | 1.3 – 3.3 | +| `alcalinity_of_ash` | Ash alkalinity (mEq NaOH) | 10 – 30 | +| `magnesium` | Magnesium (mg/L) | 70 – 162 | +| `total_phenols` | Total phenolic content (g/L) | 0.9 – 3.9 | +| `flavanoids` | Flavonoid phenols (g/L) | 0.3 – 5.1 | +| `nonflavanoid_phenols` | Non-flavonoid phenols (g/L) | 0.1 – 0.7 | +| `proanthocyanins` | Proanthocyanins (g/L) | 0.4 – 3.6 | +| `color_intensity` | Red/blue color intensity | 1.3 – 13.0 | +| `hue` | Hue at 520 nm relative to 420 nm | 0.4 – 1.8 | +| `od280_od315_of_diluted_wines` | Optical density ratio (aromatic compounds) | 1.3 – 4.0 | +| `proline` | Proline (mg/L) – associated with mouth-feel | 270 – 1680 | + +### 2.2 Target variable: `class` + +The classes are almost balanced (59 / 71 / 48 samples): + +- **Class 0** – Cultivar A +- **Class 1** – Cultivar B +- **Class 2** – Cultivar C + +### 2.3 Why This Dataset Fits the Demo + +1. **Compact and clean** – You can train models in seconds, perfect for live coding. +2. **Chemically interpretable features** – Easy to discuss how drifting alcohol content affects predictions. +3. **Multi-class problem** – Demonstrates probabilities and class selection in the REST API. +4. **No privacy concerns** – The data are public domain, ideal for workshops. + +### 2.4 Synthetic Drift Generation + +To illustrate monitoring, the lab creates a *current batch* where `alcohol` values are artificially increased (e.g., harvest with higher sugar). +Evidently then compares this drifted batch to the original reference set, detects the shift and—if severe—triggers retraining through Prefect. + +--- + +## 3 Tooling Overview + +[End-to-End Workflow](imgs/lab-full.png) + +### 3.1 Tools, concepts and our usage + +| Tool | Key Concept | How We Use It in the Lab | +|------|-------------|--------------------------| +| [**Makefile**](https://www.gnu.org/software/make/manual/make.html) | Declarative build/task runner | Simulates Git push events and invokes Nox sessions (`push-feature`, `push-develop`, `push-main`). | +| [**Nox**](https://nox.thea.codes/en/stable/) | Python automation with virtual/conda envs | Acts like GitHub Actions runners; executes lint, tests and Prefect flows in a reproducible and isolated way. | +| [**Prefect**](https://docs.prefect.io/v3/get-started) | Workflow orchestration + observability | Wraps training, serving and monitoring as flows; offers scheduling, retries and a UI at [http://127.0.0.1:4200](http://127.0.0.1:4200). | +| [**MLflow**](https://mlflow.org/) | Experiment tracking & model registry | Logs parameters, metrics and model artefacts; source of truth for the “best model”. UI at [http://127.0.0.1:5000](http://127.0.0.1:5000). | +| [**Scikit-learn**](https://scikit-learn.org/stable/index.html) | ML algorithms and utilities | Provides a fast RandomForest classifier for our demo. | +| [**FastAPI**](https://fastapi.tiangolo.com/) | High‑performance Python web API | Exposes the model at `/predict` with auto‑generated docs at [http://127.0.0.1:9000](http://127.0.0.1:9000). | +| [**Evidently**](https://docs.evidentlyai.com/introduction) | Data drift & model monitoring | Generates HTML/JSON drift reports; triggers retraining when drift is detected. | + +### 3.2 Why We Simulate GitHub Actions Locally + +In production we would rely on **real Git events and GitHub Actions**: + +| Real event | Real GH Action (example) | Purpose | +| ----------------------------------------- | --------------------------------------------- | ----------------------------------- | +| `git push feature/experiments` | `ci-feature.yml` → sweep experiment | Run many trials, log only metrics | +| `git merge feature/experiments (into develop)` | `ci-develop.yml` → lint + tests + train best | Promote best params, prepare model | +| `git merge develop (into main)` | `ci-main.yml` → lint + tests + deploy best | Bring model online | + +For a laboratory setting those actions pose two practical problems: + +1. **Concurrency & permissions** – the whole group cannot push to `develop` or `main` at the same time without overwriting one another. +2. **Infrastructure dependencies** – real GH Actions would need publicly reachable Prefect and MLflow endpoints; everyone would have to expose tunnels or deploy cloud services. + +To keep the experience **100 % local**, we introduce a one-to-one mapping: + +| Simulated command | What it stands for | +| --------------------------------------------------- | ------------------------------------------------------ | +| `make push-feature` | `git push feature/experiments` trigger GH Action `ci-feature.yml` | +| `make push-develop` | `git merge feature/experiments (into develop)` trigger GH Action `ci-develop.yml` | +| `make push-main` | `git merge develop (into main)` trigger GH Action `ci-main.yml` | +| `python -m src.pipeline_monitoring monitoring_best` | Nightly cron job in GH Actions or Prefect Cloud | + +Under the hood **Nox** replaces the GitHub runner: +`make push-*` commands launch the appropriate nox sessions (lint / tests / Prefect flows) inside a shared Conda environment, reproducing exactly what would happen in cloud CI—but without leaving the laptop. + +--- + +## 4 Environment Setup + +### 4.0 Requirements + +This lab assumes that **Python** and **miniconda** are already installed, the repository [kiratech/labs](https://github.com/kiratech/labs.git) is accessible, and **Git** is properly configured on your local machine. Furthermore, **VSCode** or an IDE able to run Jupyter Notebooks, must be installed as well. +As in the previous lab, in order to execute this laboratory, you will be asked to install a set of tools common in MLOps field. + +### 4.1 Clone the Repository + +To start, clone the lab repository by running the following command in the terminal: + +```sh + git clone https://github.com/kiratech/labs.git +``` + +### 4.2 Checkout the Lab Branch + +After cloning the repository, checkout the `academy-ai` branch: + +```sh + git checkout academy-ai +``` + +Then, navigate to the project folder: + +```sh + cd labs/Academy/AI/2-ML-Model-Lifecycle +``` + +This folder contains resources related to this lab. + +### 4.3 Create a Virtual Environment + +A virtual environment allows you to isolate the project's dependencies from the system-wide ones. +In the previous lab, we first created the environment and then installed the dependencies. This time we'll do it with a single command, creating the environment with all the necessary dependencies. + +In your terminal, create a virtual environment from an existing file: + +```sh + conda env create -f environment.yaml +``` + +Activate the Virtual Environment: + +```sh + conda activate lab_env_2 +``` + +You should see the `(lab_env_2)` prefix in the terminal, indicating that the virtual environment is active. + +### 4.4 Open the Project in VSCode + +At this point, open VSCode from the file explorer or by running the command: + +```sh + code . +``` + +### 4.5 Start the services + +Open three integrated terminals and in each one activate the environemnt with `conda activate lab_env_2` and then: + +| Terminal | Command | Purpose | +|----------|---------|---------| +| T‑1 | `prefect server start` | Prefect API + UI | +| T‑2 | `mlflow ui` | Experiment tracking UI | +| T‑3 | `-` | Run workflow commands (next sections) | + +### 4.6 Open services UI + +Open in the browser Prefect and MLflow at: +| Service | Address | +|----------|---------| +| Prefect | [http://127.0.0.1:4200](http://127.0.0.1:4200) | +| MLflow | [http://127.0.0.1:5000](http://127.0.0.1:5000) | + +--- + +## 5 Branch Workflows + +### 5.0 What we will do + +A production-style GitHub Actions workflow we want to **experience locally**. +In the laboratory we replay each numbered block with Makefile + Nox + Prefect, avoiding concurrent pushes and external infrastructure. + +[End-to-End Workflow](imgs/lab-full.png) + +### 5.1 Feature Branch Workflow + +[Feature Workflow](imgs/lab-part1.png) + +- **Command** + In your terminal, once the conda environment is active, run: + + ```sh + make push-feature + ``` + +- **Goal** + Explore the parameter space quickly and cheaply. No model is intended for production at this stage. + We only want evidence that “something promising” exists. +- **What the Command Simulates** + A developer pushes code to **`feature/`**. + CI/CD should **only run experiments**, producing metrics the team can compare later. +- **What It Does** + + - A Nox session calls Prefect flow `train_experiment`. + - 9 RandomForest training tasks run in parallel (3  `n_estimators` × 3  `max_depth`). + - Each task logs **parameters + accuracy** to MLflow, but **does not store a model file**. + - Prefect captures task logs and execution graph. + +- **What to Explore** + + - MLflow UI: nine runs with different hyper‑params, no artifacts saved. + - Prefect UI: one flow, nine parallel tasks—visual confirmation of parallel runs. + +### 5.2 Develop Branch Workflow  + +[Develop Workflow](imgs/lab-part2.png) + +- **Command** + In your terminal, once the conda environment is active, run: + + ```sh + make push-develop + ``` + +- **Goal** + Promote the best experimental configuration, validate code quality, and log a **deployable model artifact**. +- **What the Command Simulates** + A merge/push to **`develop`**. CI/CD should lint, test, **re‑train with chosen hyper‑params**, and register the resulting model. +- **What It Does** + + 1. **Lint** (`ruff via pre‑commit`) and **unit tests** (`pytest`) run first. Build stops on failure. + 2. Prefect flow `train_best` queries MLflow, grabs the run with highest accuracy. + 3. It re‑trains a RandomForest using those parameters on fresh data splits. + 4. Saves the `.pkl` artifact to `artifacts/` and logs it to MLflow. + +- **What to Explore** + + - Terminal: lint/test output. + - MLflow UI: a new run with an **artifact path**, this is the candidate for production. + - Prefect UI: see the “find best params” task feeding the “train” task. + +### 5.3 Main Branch Workflow + +[Main Workflow](imgs/lab-part3.png) + +- **Command** + In your terminal, once the conda environment is active, run: + + ```sh + make push-main + ``` + +- **Goal** + Deploy the champion model as an HTTP service usable by downstream teams. +- **What the Command Simulates** + A merge/push to **`main`**. CI/CD should perform a last sanity check, then bring the model online. +- **What It Does** + + 1. Re‑runs lint and tests (quick safety net). + 2. Prefect flow `serve_best` downloads the best model artifact from MLflow. + 3. Builds a FastAPI app and launches Uvicorn on **port 9000**. + +- **What to Explore** + + - Swagger UI at `http://127.0.0.1:9000/docs`: live documentation, try a prediction. + - Prefect UI: a small flow made of download, build app and serve. + - MLflow UI: confirm the run ID of the served model matches the develop stage. + +### 5.4 Querying the Best Model + +Once the **main branch workflow** has deployed the FastAPI service, you can send a prediction request directly from the terminal: + +```bash +curl -X POST http://localhost:9000/predict \ + -H "Content-Type: application/json" \ + -d '{ + "alcohol": 13.0, + "malic_acid": 2.0, + "ash": 2.5, + "alcalinity_of_ash": 15.0, + "magnesium": 100.0, + "total_phenols": 2.0, + "flavanoids": 2.5, + "nonflavanoid_phenols": 0.3, + "proanthocyanins": 1.9, + "color_intensity": 5.0, + "hue": 1.0, + "od280_od315_of_diluted_wines": 3.0, + "proline": 1000 + }' +``` + +Expected JSON response: + +```json +{ + "class": 1, + "proba": [0.02, 0.95, 0.03] +} +``` + +--- + +## 6 Monitoring & Auto‑Retraining + +[Feature Workflow](imgs/lab-part4.png) + +- **Command** + In your terminal, once the conda environment is active, run: + + ```sh + python -m src.pipeline_monitoring monitoring_best + ``` + +- **Goal** + Detect distributional drift and kick off retraining only when needed. +- **What the Command Simulates** + A **scheduled batch job** (cron in Prefect) that runs nightly, comparing that day’s data against a reference baseline. +- **What It Does** + + - Generates (or ingests) the **current batch**. Here we synthesise drift by nudging alcohol levels. + - Evidently creates an HTML + JSON drift report. + - Prefect parses the `alchol` feature p‑value between the training set distribution and the new drifted set distribution. If ≤ 0.05, it calls the same `train_best` flow used on develop. + - All actions—report generation and optional retraining—are logged in Prefect and MLflow. + +- **What to Explore** + + - Drift Report: open `artifacts/drift_report.html`; discuss which features drifted. + - Prefect UI: see conditional branching—one path ends, the other chains into a training flow. + - MLflow UI: a new run appears **only** when drift threshold is exceeded, proving closed‑loop automation. + +--- + +## 7 Conclusions + +Over the course of this lab we have: + +- **Simulated a full CI/CD/CT loop locally** + using Makefile to trigger branch‑style workflows and Nox as a lightweight stand‑in for GitHub Actions. +- **Captured the complete model lineage** + in MLflow, from exploratory runs to the promoted, deployable artifact. +- **Orchestrated repeatable pipelines** + with Prefect, gaining retry logic, scheduling, and an audit‑friendly UI. +- **Deployed an inference endpoint** + via FastAPI that is immediately testable through cURL or Swagger docs. +- **Closed the monitoring loop** + by integrating Evidently to detect drift and automatically retrain when data shifts. + +Together these elements demonstrate an end‑to‑end, production‑style MLOps workflow that fits entirely on a developer laptop—yet scales conceptually to cloud or on‑prem environments. +The key takeaway: **tooling synergy** matters more than individual components. +By combining focused, purpose‑built tools, we achieve reproducibility, observability and automation without heavyweight infrastructure. diff --git a/Workshops/AI/02-ML-Model-Lifecycle/environment.yaml b/Workshops/AI/02-ML-Model-Lifecycle/environment.yaml new file mode 100644 index 0000000..6a85694 --- /dev/null +++ b/Workshops/AI/02-ML-Model-Lifecycle/environment.yaml @@ -0,0 +1,236 @@ +name: lab_env_2 +channels: + - conda-forge + - defaults +dependencies: + - appnope=0.1.2 + - argcomplete=1.12.3 + - asttokens=3.0.0 + - attrs=25.3.0 + - backports=1.0 + - backports.functools_lru_cache=2.0.0 + - bzip2=1.0.8 + - ca-certificates=2025.2.25 + - cachetools=5.5.2 + - chardet=5.2.0 + - colorama=0.4.6 + - colorlog=5.0.1 + - comm=0.2.1 + - debugpy=1.8.11 + - decorator=5.1.1 + - dependency-groups=1.3.0 + - distlib=0.3.8 + - exceptiongroup=1.2.2 + - executing=0.8.3 + - filelock=3.18.0 + - importlib_metadata=4.13.0 + - ipykernel=6.29.5 + - ipython=9.1.0 + - ipython_pygments_lexers=1.1.1 + - jedi=0.19.2 + - jinja2=3.1.6 + - jupyter_client=8.6.3 + - jupyter_core=5.7.2 + - libcxx=14.0.6 + - libexpat=2.7.0 + - libffi=3.4.6 + - liblzma=5.8.1 + - libsodium=1.0.18 + - libsqlite=3.49.1 + - libzlib=1.3.1 + - markupsafe=3.0.2 + - matplotlib-inline=0.1.6 + - ncurses=6.5 + - nest-asyncio=1.6.0 + - nox=2025.2.9 + - openssl=3.5.0 + - packaging=24.2 + - parso=0.8.4 + - pexpect=4.8.0 + - pickleshare=0.7.5 + - pip=25.0.1 + - platformdirs=4.3.7 + - pluggy=1.5.0 + - prompt-toolkit=3.0.43 + - prompt_toolkit=3.0.43 + - psutil=5.9.0 + - ptyprocess=0.7.0 + - pure_eval=0.2.2 + - pygments=2.19.1 + - pyproject-api=1.9.0 + - python=3.11.12 + - python-dateutil=2.9.0post0 + - python_abi=3.11 + - pyzmq=26.2.0 + - readline=8.2 + - setuptools=78.1.0 + - six=1.17.0 + - stack_data=0.2.0 + - tk=8.6.13 + - tomli=2.2.1 + - tornado=6.4.2 + - tox=4.25.0 + - traitlets=5.14.3 + - virtualenv=20.30.0 + - wcwidth=0.2.5 + - wheel=0.45.1 + - zeromq=4.3.5 + - zipp=3.21.0 + - pip: + - aiosqlite==0.21.0 + - alembic==1.15.2 + - annotated-types==0.7.0 + - anyio==4.6.0 + - appdirs==1.4.4 + - apprise==1.9.3 + - asgi-lifespan==2.1.0 + - async-timeout==5.0.1 + - asyncpg==0.29.0 + - blinker==1.9.0 + - certifi==2025.1.31 + - cffi==1.17.1 + - cfgv==3.4.0 + - charset-normalizer==3.4.1 + - click==8.1.8 + - cloudpickle==3.1.1 + - contourpy==1.3.2 + - coolname==2.2.0 + - croniter==5.0.1 + - cryptography==44.0.2 + - cycler==0.12.1 + - databricks-sdk==0.50.0 + - dateparser==1.2.1 + - deprecated==1.2.18 + - deprecation==2.1.0 + - distro==1.9.0 + - dnspython==2.7.0 + - docker==7.1.0 + - durationpy==0.9 + - dynaconf==3.2.10 + - email-validator==2.2.0 + - evidently==0.4.40 + - faker==37.1.0 + - fastapi==0.115.12 + - flask==3.1.0 + - fonttools==4.57.0 + - fsspec==2025.3.2 + - gitdb==4.0.12 + - gitpython==3.1.44 + - google-auth==2.39.0 + - graphene==3.4.3 + - graphql-core==3.2.6 + - graphql-relay==3.2.0 + - greenlet==3.2.0 + - griffe==1.7.2 + - gunicorn==23.0.0 + - h11==0.14.0 + - h2==4.2.0 + - hpack==4.1.0 + - httpcore==1.0.8 + - httptools==0.6.4 + - httpx==0.28.1 + - humanize==4.12.2 + - hyperframe==6.1.0 + - identify==2.6.10 + - idna==3.10 + - importlib-metadata==8.6.1 + - importlib-resources==6.4.5 + - iniconfig==2.1.0 + - iterative-telemetry==0.0.10 + - itsdangerous==2.2.0 + - jinja2-humanize-extension==0.4.0 + - joblib==1.4.2 + - jsonpatch==1.33 + - jsonpointer==3.0.0 + - jsonschema==4.23.0 + - jsonschema-specifications==2024.10.1 + - kiwisolver==1.4.8 + - kubernetes==31.0.0 + - litestar==2.15.2 + - litestar-htmx==0.4.1 + - mako==1.3.10 + - markdown==3.8 + - markdown-it-py==3.0.0 + - matplotlib==3.10.1 + - mdurl==0.1.2 + - mlflow==2.21.3 + - mlflow-skinny==2.21.3 + - msgspec==0.19.0 + - multidict==6.4.3 + - multipart==1.2.1 + - mypy-extensions==1.0.0 + - nltk==3.9.1 + - nodeenv==1.9.1 + - numpy==2.0.2 + - oauthlib==3.2.2 + - opentelemetry-api==1.32.1 + - opentelemetry-sdk==1.32.1 + - opentelemetry-semantic-conventions==0.53b1 + - orjson==3.10.16 + - pandas==2.2.3 + - pathspec==0.12.1 + - patsy==1.0.1 + - pendulum==2.1.2 + - pillow==11.2.1 + - plotly==5.24.1 + - polyfactory==2.21.0 + - pre-commit==4.2.0 + - prefect==2.20.17 + - protobuf==5.29.4 + - pyarrow==19.0.1 + - pyasn1==0.6.1 + - pyasn1-modules==0.4.2 + - pycparser==2.22 + - pydantic==2.11.3 + - pydantic-core==2.33.1 + - pyparsing==3.2.3 + - pytest==8.3.5 + - python-dotenv==1.1.0 + - python-multipart==0.0.20 + - python-slugify==8.0.4 + - pytz==2024.2 + - pytzdata==2020.1 + - pyyaml==6.0.2 + - readchar==4.2.1 + - referencing==0.36.2 + - regex==2024.11.6 + - requests==2.32.3 + - requests-oauthlib==2.0.0 + - rfc3339-validator==0.1.4 + - rich==13.9.4 + - rich-click==1.8.8 + - rpds-py==0.24.0 + - rsa==4.9.1 + - ruamel-yaml==0.18.10 + - ruamel-yaml-clib==0.2.12 + - scikit-learn==1.6.1 + - scipy==1.15.2 + - shellingham==1.5.4 + - smmap==5.0.2 + - sniffio==1.3.1 + - sqlalchemy==2.0.35 + - sqlparse==0.5.3 + - starlette==0.46.2 + - statsmodels==0.14.4 + - tenacity==9.1.2 + - text-unidecode==1.3 + - threadpoolctl==3.6.0 + - toml==0.10.2 + - tqdm==4.67.1 + - typer==0.15.2 + - typing-extensions==4.13.2 + - typing-inspect==0.9.0 + - typing-inspection==0.4.0 + - tzdata==2025.2 + - tzlocal==5.3.1 + - ujson==5.10.0 + - urllib3==2.4.0 + - uuid6==2024.7.10 + - uvicorn==0.34.1 + - uvloop==0.21.0 + - watchdog==6.0.0 + - watchfiles==1.0.5 + - websocket-client==1.8.0 + - websockets==13.1 + - werkzeug==3.1.3 + - wrapt==1.17.2 diff --git a/Workshops/AI/02-ML-Model-Lifecycle/imgs/lab-full.png b/Workshops/AI/02-ML-Model-Lifecycle/imgs/lab-full.png new file mode 100644 index 0000000..a624c0f Binary files /dev/null and b/Workshops/AI/02-ML-Model-Lifecycle/imgs/lab-full.png differ diff --git a/Workshops/AI/02-ML-Model-Lifecycle/imgs/lab-part1.png b/Workshops/AI/02-ML-Model-Lifecycle/imgs/lab-part1.png new file mode 100644 index 0000000..f2cc896 Binary files /dev/null and b/Workshops/AI/02-ML-Model-Lifecycle/imgs/lab-part1.png differ diff --git a/Workshops/AI/02-ML-Model-Lifecycle/imgs/lab-part2.png b/Workshops/AI/02-ML-Model-Lifecycle/imgs/lab-part2.png new file mode 100644 index 0000000..416b1c3 Binary files /dev/null and b/Workshops/AI/02-ML-Model-Lifecycle/imgs/lab-part2.png differ diff --git a/Workshops/AI/02-ML-Model-Lifecycle/imgs/lab-part3.png b/Workshops/AI/02-ML-Model-Lifecycle/imgs/lab-part3.png new file mode 100644 index 0000000..3dc2dca Binary files /dev/null and b/Workshops/AI/02-ML-Model-Lifecycle/imgs/lab-part3.png differ diff --git a/Workshops/AI/02-ML-Model-Lifecycle/imgs/lab-part4.png b/Workshops/AI/02-ML-Model-Lifecycle/imgs/lab-part4.png new file mode 100644 index 0000000..536e770 Binary files /dev/null and b/Workshops/AI/02-ML-Model-Lifecycle/imgs/lab-part4.png differ diff --git a/Workshops/AI/02-ML-Model-Lifecycle/noxfile.py b/Workshops/AI/02-ML-Model-Lifecycle/noxfile.py new file mode 100644 index 0000000..1f11bf7 --- /dev/null +++ b/Workshops/AI/02-ML-Model-Lifecycle/noxfile.py @@ -0,0 +1,79 @@ +# noxfile.py — Local CI/CD simulation using Nox sessions +import nox + +# Reuse virtualenvs across runs to avoid reinstallation +nox.options.reuse_existing_virtualenvs = True +# Shared environment for all sessions +nox.options.envdir = ".nox/shared" + +# Python version to use in all sessions +PYTHON = "3.11" + +# ------------------------------------------------------------ +# Shared setup for installing local package +# ------------------------------------------------------------ + + +def install_project(session): + """ + Install the local src/ package in editable mode. + """ + session.install("-e", ".") + + +# ------------------------------------------------------------ +# Core CI checks: linting and unit tests +# ------------------------------------------------------------ + + +@nox.session(python=PYTHON) +def lint(session): + """ + Run pre-commit hooks for linting and formatting. + """ + session.run("pre-commit", "run", "--all-files", "--hook-stage", "manual") + +@nox.session(python=PYTHON) +def tests(session): + """ + Run unit tests using pytest. + """ + install_project(session) + session.run("pytest", "-q") + + +# ------------------------------------------------------------ +# Simulated ML workflows triggered by Git pushes +# ------------------------------------------------------------ + + +@nox.session(python=PYTHON) +def feature_pipeline(session): + """ + Simulates a push to feature/*: + - Runs hyperparameter sweep (no model saved). + """ + install_project(session) + session.run("python", "-m", "src.pipeline_ml", "train_experiment") + + +@nox.session(python=PYTHON) +def develop_pipeline(session): + """ + Simulates a push to develop: + - Trains the model using best hyperparameters. + - Logs metrics and saves model to MLflow. + """ + install_project(session) + session.run("python", "-m", "src.pipeline_ml", "train_best") + + +@nox.session(python=PYTHON) +def main_pipeline(session): + """ + Simulates a push to main: + - Downloads the best saved model. + - Serves the model through FastAPI. + """ + install_project(session) + session.run("python", "-m", "src.pipeline_ml", "serve_best") diff --git a/Workshops/AI/02-ML-Model-Lifecycle/pyproject.toml b/Workshops/AI/02-ML-Model-Lifecycle/pyproject.toml new file mode 100644 index 0000000..734e0c8 --- /dev/null +++ b/Workshops/AI/02-ML-Model-Lifecycle/pyproject.toml @@ -0,0 +1,19 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "mlops_lab" +version = "0.1.0" +description = "End-to-end MLOps lab pipeline" +authors = [{ name = "ranl", email = "lorenzo.randazzo@kiratech.ch" }] +readme = "README.md" +license = { text = "MIT" } +dynamic = ["dependencies"] +requires-python = ">=3.11" + +[tool.setuptools] +packages = ["src"] + +[tool.setuptools.dynamic] +dependencies = {file = ["requirements.txt"]} diff --git a/Workshops/AI/02-ML-Model-Lifecycle/requirements.txt b/Workshops/AI/02-ML-Model-Lifecycle/requirements.txt new file mode 100644 index 0000000..05e2720 --- /dev/null +++ b/Workshops/AI/02-ML-Model-Lifecycle/requirements.txt @@ -0,0 +1,212 @@ +aiosqlite==0.21.0 +alembic==1.15.2 +annotated-types==0.7.0 +anyio==4.6.0 +appdirs==1.4.4 +appnope==0.1.2 +apprise==1.9.3 +argcomplete==1.12.3 +asgi-lifespan==2.1.0 +asttokens==3.0.0 +async-timeout==5.0.1 +asyncpg==0.29.0 +attrs==25.3.0 +backports.functools_lru_cache==2.0.0 +blinker==1.9.0 +cachetools==5.5.2 +certifi==2025.1.31 +cffi==1.17.1 +cfgv==3.4.0 +chardet==5.2.0 +charset-normalizer==3.4.1 +click==8.1.8 +cloudpickle==3.1.1 +colorama==0.4.6 +colorlog==5.0.1 +comm==0.2.1 +contourpy==1.3.2 +coolname==2.2.0 +croniter==5.0.1 +cryptography==44.0.2 +cycler==0.12.1 +databricks-sdk==0.50.0 +dateparser==1.2.1 +debugpy==1.8.11 +decorator==5.1.1 +dependency-groups==1.3.0 +Deprecated==1.2.18 +deprecation==2.1.0 +distlib==0.3.8 +distro==1.9.0 +dnspython==2.7.0 +docker==7.1.0 +durationpy==0.9 +dynaconf==3.2.10 +email_validator==2.2.0 +evidently==0.4.40 +exceptiongroup==1.2.2 +executing==0.8.3 +Faker==37.1.0 +fastapi==0.115.12 +filelock==3.18.0 +Flask==3.1.0 +fonttools==4.57.0 +fsspec==2025.3.2 +gitdb==4.0.12 +GitPython==3.1.44 +google-auth==2.39.0 +graphene==3.4.3 +graphql-core==3.2.6 +graphql-relay==3.2.0 +graphviz==0.20.3 +greenlet==3.2.0 +griffe==1.7.2 +gunicorn==23.0.0 +h11==0.14.0 +h2==4.2.0 +hpack==4.1.0 +httpcore==1.0.8 +httptools==0.6.4 +httpx==0.28.1 +humanize==4.12.2 +hyperframe==6.1.0 +identify==2.6.10 +idna==3.10 +importlib_metadata==8.6.1 +importlib_resources==6.4.5 +iniconfig==2.1.0 +ipykernel==6.29.5 +ipython==9.1.0 +ipython_pygments_lexers==1.1.1 +iterative-telemetry==0.0.10 +itsdangerous==2.2.0 +jedi==0.19.2 +Jinja2==3.1.6 +jinja2-humanize-extension==0.4.0 +joblib==1.4.2 +jsonpatch==1.33 +jsonpointer==3.0.0 +jsonschema==4.23.0 +jsonschema-specifications==2024.10.1 +jupyter_client==8.6.3 +jupyter_core==5.7.2 +kiwisolver==1.4.8 +kubernetes==31.0.0 +litestar==2.15.2 +litestar-htmx==0.4.1 +Mako==1.3.10 +Markdown==3.8 +markdown-it-py==3.0.0 +MarkupSafe==3.0.2 +matplotlib==3.10.1 +matplotlib-inline==0.1.6 +mdurl==0.1.2 +mlflow==2.21.3 +mlflow-skinny==2.21.3 +msgspec==0.19.0 +multidict==6.4.3 +multipart==1.2.1 +mypy-extensions==1.0.0 +nest_asyncio==1.6.0 +nltk==3.9.1 +nodeenv==1.9.1 +nox==2025.2.9 +numpy==2.0.2 +oauthlib==3.2.2 +opentelemetry-api==1.32.1 +opentelemetry-sdk==1.32.1 +opentelemetry-semantic-conventions==0.53b1 +orjson==3.10.16 +packaging==24.2 +pandas==2.2.3 +parso==0.8.4 +pathspec==0.12.1 +patsy==1.0.1 +pendulum==2.1.2 +pexpect==4.8.0 +pickleshare==0.7.5 +pillow==11.2.1 +pip==25.0.1 +platformdirs==4.3.7 +plotly==5.24.1 +pluggy==1.5.0 +polyfactory==2.21.0 +pre_commit==4.2.0 +prefect==2.20.17 +prompt_toolkit==3.0.43 +protobuf==5.29.4 +psutil==5.9.0 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow==19.0.1 +pyasn1==0.6.1 +pyasn1_modules==0.4.2 +pycparser==2.22 +pydantic==2.11.3 +pydantic_core==2.33.1 +Pygments==2.19.1 +pyparsing==3.2.3 +pyproject-api==1.9.0 +pytest==8.3.5 +python-dateutil==2.9.0.post0 +python-dotenv==1.1.0 +python-multipart==0.0.20 +python-slugify==8.0.4 +pytz==2024.2 +pytzdata==2020.1 +PyYAML==6.0.2 +pyzmq==26.2.0 +readchar==4.2.1 +referencing==0.36.2 +regex==2024.11.6 +requests==2.32.3 +requests-oauthlib==2.0.0 +rfc3339-validator==0.1.4 +rich==13.9.4 +rich-click==1.8.8 +rpds-py==0.24.0 +rsa==4.9.1 +ruamel.yaml==0.18.10 +ruamel.yaml.clib==0.2.12 +scikit-learn==1.6.1 +scipy==1.15.2 +setuptools==78.1.0 +shellingham==1.5.4 +six==1.17.0 +smmap==5.0.2 +sniffio==1.3.1 +SQLAlchemy==2.0.35 +sqlparse==0.5.3 +stack-data==0.2.0 +starlette==0.46.2 +statsmodels==0.14.4 +tenacity==9.1.2 +text-unidecode==1.3 +threadpoolctl==3.6.0 +toml==0.10.2 +tomli==2.2.1 +tornado==6.4.2 +tox==4.25.0 +tqdm==4.67.1 +traitlets==5.14.3 +typer==0.15.2 +typing_extensions==4.13.2 +typing-inspect==0.9.0 +typing-inspection==0.4.0 +tzdata==2025.2 +tzlocal==5.3.1 +ujson==5.10.0 +urllib3==2.4.0 +uuid6==2024.7.10 +uvicorn==0.34.1 +uvloop==0.21.0 +virtualenv==20.30.0 +watchdog==6.0.0 +watchfiles==1.0.5 +wcwidth==0.2.5 +websocket-client==1.8.0 +websockets==13.1 +Werkzeug==3.1.3 +wheel==0.45.1 +wrapt==1.17.2 +zipp==3.21.0 diff --git a/Workshops/AI/02-ML-Model-Lifecycle/src/__init__.py b/Workshops/AI/02-ML-Model-Lifecycle/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Workshops/AI/02-ML-Model-Lifecycle/src/config.py b/Workshops/AI/02-ML-Model-Lifecycle/src/config.py new file mode 100644 index 0000000..56f3add --- /dev/null +++ b/Workshops/AI/02-ML-Model-Lifecycle/src/config.py @@ -0,0 +1,54 @@ +""" +src/config.py +Configuration module for the Wine Quality pipeline. + +- Defines global paths, MLflow settings, artifact naming, and Evidently monitoring endpoints. +- Uses Pydantic's BaseModel for structured configuration. +""" + +from pathlib import Path + +from pydantic import BaseModel + + +class Settings(BaseModel): + """ + Centralized application settings. + + Attributes: + project_root (Path): Root directory of the project. + artifacts_dir (Path): Directory where artifacts (models, reports) will be stored. + model_name (str): Default filename for the trained model artifact. + mlflow_uri (str): URI where MLflow tracking server or local files are stored. + experiment_name (str): Name of the MLflow experiment. + evidently_url (str): Base URL where the Evidently UI dashboard is running. + evidently_project (str): Project name under Evidently for monitoring. + """ + + # Root of the project (assumes config.py is in src/) + project_root: Path = Path(__file__).resolve().parents[1] + + # Directory for storing artifacts (models, drift reports, etc.) + artifacts_dir: Path = project_root / "artifacts" + + # Default model filename for serialization + model_name: str = "wine_rf.pkl" + + # MLflow tracking URI (using local file storage in 'mlruns/') + mlflow_uri: str = f"file:{project_root/'mlruns'}" + + # MLflow experiment name for tracking runs + experiment_name: str = "WineSweepDemo" + + # Evidently monitoring - URL to the local Evidently UI server + evidently_url: str = "http://localhost:8000" + + # Evidently monitoring - Name of the project for batch drift tracking + evidently_project: str = "wine_quality_monitoring" + + +# Singleton instance used throughout the project +CFG = Settings() + +# Ensure artifacts directory exists (create if it doesn't) +CFG.artifacts_dir.mkdir(exist_ok=True) diff --git a/Workshops/AI/02-ML-Model-Lifecycle/src/data.py b/Workshops/AI/02-ML-Model-Lifecycle/src/data.py new file mode 100644 index 0000000..d8a5843 --- /dev/null +++ b/Workshops/AI/02-ML-Model-Lifecycle/src/data.py @@ -0,0 +1,104 @@ +""" +src/data.py +Data handling module for the Wine Quality pipeline. + +Provides: +- data_collection: Loads the wine dataset from scikit-learn. +- new_data_collection: Simulates a new batch with data drift in alcohol. +- data_preparation: Prepares the dataset (renames columns, performs train-test split). +""" + +import random + +import pandas as pd +import typer +from sklearn.datasets import load_wine +from sklearn.model_selection import train_test_split +from src.utils import get_logger + +app = typer.Typer(add_completion=False) + + +def data_collection() -> tuple[pd.DataFrame, pd.Series]: + """ + Loads the wine dataset. + + Returns: + tuple: + - X (pd.DataFrame): Feature matrix. + - y (pd.Series): Target labels. + """ + logger = get_logger() + logger.info("Loading Wine dataset...") + X, y = load_wine(return_X_y=True, as_frame=True) + logger.info( + f"Wine dataset loaded with {X.shape[0]} samples and {X.shape[1]} features." + ) + return (X, y) + + +def new_data_collection() -> pd.DataFrame: + """ + Loads the wine dataset and introduces data drift in the 'alcohol' feature. + + Randomly perturbs alcohol values within a multiplier range + to simulate a distributional change for monitoring. + + Returns: + pd.DataFrame: Drifted dataset (features only, no target). + """ + X, _ = data_collection() + df = X.copy() + + logger = get_logger() + logger.info("Generating drifted data batch...") + + # Filter high alcohol wines + df_filtered = df[df["alcohol"] > 13] + + # Apply random scaling to simulate drift + rnd_drift = random.uniform(1.03, 1.13) + df_filtered["alcohol"] = df["alcohol"] * rnd_drift + logger.warning(f"Random drif generated for alchol with coefficient {rnd_drift}.") + # Sample up to 100 instances + df_current = df_filtered.sample(min(100, len(df_filtered)), random_state=0) + return df_current + + +def data_preparation( + dataset: tuple[pd.DataFrame, pd.Series], split: float = 0.2, seed: int = 42 +) -> tuple[pd.DataFrame, pd.DataFrame, pd.Series, pd.Series]: + """ + Prepares the dataset for training: + - Renames any problematic columns (e.g., replaces slashes). + - Splits the dataset into train and test sets. + + Args: + dataset (tuple): Tuple containing (X, y). + split (float, optional): Proportion of the dataset to include in the test split (default: 0.2). + seed (int, optional): Random seed for reproducibility (default: 42). + + Returns: + tuple: + - X_train (pd.DataFrame): Training features. + - X_test (pd.DataFrame): Testing features. + - y_train (pd.Series): Training labels. + - y_test (pd.Series): Testing labels. + """ + X, y = dataset + logger = get_logger() + # Ensure column names are safe (replace slashes with underscores) + logger.info("Replacing / with - in column names...") + X.columns = X.columns.str.replace("/", "_") + + # Split into train/test + logger.info(f"Preparing dataset: split={split}, seed={seed}...") + X_tr, X_te, y_tr, y_te = train_test_split(X, y, test_size=split, random_state=seed) + logger.info( + f"Dataset split into {X_tr.shape[0]} train and {X_te.shape[0]} test samples." + ) + return (X_tr, X_te, y_tr, y_te) + + +if __name__ == "__main__": + app() diff --git a/Workshops/AI/02-ML-Model-Lifecycle/src/monitor.py b/Workshops/AI/02-ML-Model-Lifecycle/src/monitor.py new file mode 100644 index 0000000..56b6bdc --- /dev/null +++ b/Workshops/AI/02-ML-Model-Lifecycle/src/monitor.py @@ -0,0 +1,107 @@ +""" +src/monitor.py +Data drift monitoring module using Evidently. + +- Compares current data to a reference dataset. +- Generates an HTML and JSON report saved to artifacts/. +- Can be called via CLI or integrated as a Prefect task. +""" + +import json +from pathlib import Path + +import pandas as pd +import typer +from evidently.metric_preset import DataDriftPreset +from evidently.report import Report +from src.utils import get_logger +from src.config import CFG + +app = typer.Typer(add_completion=False) + + +def compute_drift( + output_json: Path = Path(CFG.artifacts_dir / "drift_report.json"), +) -> float: + """ + Computes the minimum drift score from an Evidently JSON report. + + Args: + output_json (Path, optional): Path to the JSON drift report (default: artifacts/drift_report.json). + + Returns: + float: The minimum drift score across all monitored features. + """ + logger = get_logger() + logger.info("Computing minimum drift score from report...") + + # Load the JSON report + report = json.loads(output_json.read_text()) + pvals = [] + + # Extract drift scores for each monitored column + for attribute, value in report["metrics"][1]["result"]["drift_by_columns"].items(): + pvals.append(value["drift_score"]) + + # Compute and return the minimum p-value + min_p = min(pvals) + logger.info(f"Minimum drift score detected: {min_p}.") + return min_p + + +def generate_drift_report( + reference_data: pd.DataFrame, + current_data: pd.DataFrame, + output_path_html: Path, + output_path_json: Path, +) -> Path: + """ + Generates a drift report comparing current data with reference data. + + Args: + reference_data (pd.DataFrame): DataFrame containing the reference dataset. + current_data (pd.DataFrame): DataFrame containing the current dataset to monitor. + output_path_html (Path): Path to save the generated HTML report. + output_path_json (Path): Path to save the generated JSON report. + + Returns: + Path: Path to the generated JSON report (for downstream processing). + """ + logger = get_logger() + logger.info("Starting drift report generation...") + + # Initialize Evidently report with DataDriftPreset + report = Report(metrics=[DataDriftPreset()]) + report.run(reference_data=reference_data, current_data=current_data) + + # Save the report as both HTML and JSON + output_path_html.parent.mkdir(exist_ok=True) + report.save_html(str(output_path_html)) + report.save_json(str(output_path_json)) + logger.info(f"Drift report saved to {output_path_html} and {output_path_json}.") + return output_path_json + + +def run( + reference_data: pd.DataFrame, + current_data: pd.DataFrame, + output_html: Path = Path(CFG.artifacts_dir / "drift_report.html"), + output_json: Path = Path(CFG.artifacts_dir / "drift_report.json"), +) -> Path: + """ + CLI entry point for generating a drift report. + + Args: + reference_data (pd.DataFrame): DataFrame containing the reference dataset. + current_data (pd.DataFrame): DataFrame containing the current dataset. + output_html (Path, optional): Output path for the HTML report (default: artifacts/drift_report.html). + output_json (Path, optional): Output path for the JSON report (default: artifacts/drift_report.json). + + Returns: + Path: Path to the generated JSON report. + """ + return generate_drift_report(reference_data, current_data, output_html, output_json) + + +if __name__ == "__main__": + app() diff --git a/Workshops/AI/02-ML-Model-Lifecycle/src/pipeline_ml.py b/Workshops/AI/02-ML-Model-Lifecycle/src/pipeline_ml.py new file mode 100644 index 0000000..a43d425 --- /dev/null +++ b/Workshops/AI/02-ML-Model-Lifecycle/src/pipeline_ml.py @@ -0,0 +1,195 @@ +""" +src/pipeline_ml.py +Prefect 2 flows for the Wine Quality ML pipeline. + +Flows: +- train_experiment: Performs a grid search (hyperparameter sweep) without saving models. +- train_best: Trains the best model based on MLflow runs and saves the model artifact. +- serve_best: Downloads the best saved model and serves it through a FastAPI endpoint. +""" + +from itertools import product +from pathlib import Path +import tempfile + +import mlflow +from prefect import flow, task + +from src.config import CFG +from src.data import data_collection as data_coll_run, data_preparation as data_prep_run +from src.train import run as train_run +from src.serve import deploy +from src.utils import get_logger + +# --------------------------------------------------------------------------- +# Tasks +# --------------------------------------------------------------------------- + +@task +def data_collection_task() -> tuple: + """ + Collects the dataset (features and labels). + + Returns: + tuple: (X, y) dataset. + """ + return data_coll_run() + +@task +def data_preparation_task(dataset: tuple) -> tuple: + """ + Prepares the dataset for training: + - Fixes feature names. + - Splits into train/test sets. + + Args: + dataset (tuple): (X, y) dataset. + + Returns: + tuple: (X_train, X_test, y_train, y_test). + """ + return data_prep_run(dataset) + +@task +def model_training_task( + dataset: tuple, n_estimators: int, max_depth: int | None, log_model: bool = False +) -> float: + """ + Trains a RandomForest model and optionally logs it to MLflow. + + Args: + dataset (tuple): Prepared train/test splits. + n_estimators (int): Number of trees. + max_depth (int | None): Maximum depth per tree. + log_model (bool): Whether to save model artifact to MLflow. + + Returns: + float: Model accuracy. + """ + return train_run(dataset, n_estimators=n_estimators, max_depth=max_depth, log_model=log_model) + +@task +def find_best_params_task(metric: str = "accuracy") -> tuple: + """ + Retrieves the best model hyperparameters from MLflow by specified metric. + + Args: + metric (str, optional): Metric to order by (default: 'accuracy'). + + Returns: + tuple: (n_estimators, max_depth) for best model. + """ + logger = get_logger() + logger.info(f"Searching best hyperparameters by {metric}...") + + runs = mlflow.search_runs(order_by=[f"metrics.{metric} DESC"], max_results=1) + best = runs.iloc[0] + params = { + "n_estimators": int(best["params.n_estimators"]), + "max_depth": None if best["params.max_depth"] in ("", "None") else int(best["params.max_depth"]), + } + logger.info(f"Best hyperparameters found: n_estimators={params['n_estimators']}, max_depth={params['max_depth']}.") + return params["n_estimators"], params["max_depth"] + +@task +def download_best_model(metric: str = "accuracy") -> Path: + """ + Downloads the best model artifact from MLflow. + + Args: + metric (str, optional): Metric to order by (default: 'accuracy'). + + Returns: + Path: Local path to downloaded model (.pkl file). + """ + logger = get_logger() + logger.info(f"Downloading best model artifact in experiment {CFG.experiment_name}...") + + exp = mlflow.get_experiment_by_name(CFG.experiment_name) + runs = mlflow.search_runs( + experiment_ids=[exp.experiment_id], order_by=[f"metrics.{metric} DESC"], max_results=1 + ) + run_id = runs.iloc[0]["run_id"] + + client = mlflow.MlflowClient() + artifacts = client.list_artifacts(run_id) + pkl_path = next(a.path for a in artifacts if a.path.endswith(".pkl")) + + tmp_dir = tempfile.mkdtemp() + local_path = Path(client.download_artifacts(run_id, pkl_path, tmp_dir)) + logger.info(f"Downloaded best model to: {local_path}") + return local_path + +@task +def serve_model_task(model_path: Path): + """ + Deploys a FastAPI app for model inference. + + Args: + model_path (Path): Path to serialized model (.pkl). + """ + deploy(model_path=model_path) + +# --------------------------------------------------------------------------- +# Flows +# --------------------------------------------------------------------------- + +@flow(name="train_experiment") +def train_experiment(): + """ + End-to-end grid search training flow: + - Collects and prepares dataset. + - Launches multiple model training tasks for different hyperparameters. + - Only logs parameters and metrics, no model artifacts. + """ + logger = get_logger() + logger.info("Starting train_experiment flow...") + + dataset = data_collection_task() + dataset_prep = data_preparation_task(dataset) + + grid = { + "n_estimators": [1, 5, 10], + "max_depth": [None, 5, 10], + } + + for n_est, m_depth in product(grid["n_estimators"], grid["max_depth"]): + model_training_task.submit( + dataset=dataset_prep, n_estimators=n_est, max_depth=m_depth + ) + +@flow(name="train_best") +def train_best(): + """ + Retrains the best model based on MLflow search: + - Finds best hyperparameters. + - Retrains model and saves it as artifact. + """ + logger = get_logger() + logger.info("Starting train_best flow...") + + dataset = data_collection_task() + dataset_prep = data_preparation_task(dataset) + n_estimators, max_depth = find_best_params_task() + model_training_task( + dataset_prep, n_estimators=n_estimators, max_depth=max_depth, log_model=True + ) + +@flow(name="serve_best") +def serve_best(): + """ + Deploys the best model as a live API: + - Downloads best model from MLflow. + - Builds FastAPI app. + - Launches serving. + """ + logger = get_logger() + logger.info("Starting serve_best flow...") + + model_path = download_best_model() + #api = build_app_task(model_path=model_path) + serve_model_task(model_path) + +if __name__ == "__main__": + import sys + locals()[sys.argv[1]]() diff --git a/Workshops/AI/02-ML-Model-Lifecycle/src/pipeline_monitoring.py b/Workshops/AI/02-ML-Model-Lifecycle/src/pipeline_monitoring.py new file mode 100644 index 0000000..9209f54 --- /dev/null +++ b/Workshops/AI/02-ML-Model-Lifecycle/src/pipeline_monitoring.py @@ -0,0 +1,111 @@ +""" +src/pipeline_monitoring.py +Prefect 2 flows for the Wine Quality ML pipeline. + +Flow: +- monitoring_best: Detects data drift and triggers retraining if significant drift is found. +""" + +import pandas as pd +from pathlib import Path + +from prefect import flow, task +from src.data import data_collection as data_coll_run, new_data_collection as new_data_coll_run +from src.monitor import run as monitoring_run, compute_drift +from src.pipeline_ml import train_best +from src.utils import get_logger + +# --------------------------------------------------------------------------- +# Tasks +# --------------------------------------------------------------------------- + +@task +def data_collection_task() -> tuple: + """ + Collects the original reference dataset. + + Returns: + tuple: (X, y) dataset. + """ + return data_coll_run() + +@task +def get_drift_data_task() -> pd.DataFrame: + """ + Generates a new dataset with simulated data drift. + + Returns: + pd.DataFrame: New batch of data with potential drift. + """ + return new_data_coll_run() + +@task +def create_drift_report_task(current_data: pd.DataFrame, reference_data: pd.DataFrame) -> Path: + """ + Creates a drift monitoring report comparing current batch vs reference. + + Args: + current_data (pd.DataFrame): Current dataset batch. + reference_data (pd.DataFrame): Original reference dataset. + + Returns: + Path: Path to the generated JSON drift report. + """ + return monitoring_run(current_data=current_data, reference_data=reference_data) + +@task +def compute_drift_task(output_json: Path) -> float: + """ + Computes the minimum drift score from the JSON report. + + Args: + output_json (Path): Path to the generated JSON report. + + Returns: + float: Minimum p-value detected across all features. + """ + return compute_drift(output_json) + +@task +def trigger_if_drift_task(min_p: float, threshold: float = 0.05): + """ + Triggers retraining if minimum drift p-value is below the threshold. + + Args: + min_p (float): Minimum p-value detected. + threshold (float, optional): Threshold below which retraining is triggered (default: 0.05). + """ + logger = get_logger() + logger.info("Starting monitoring_best flow...") + + if min_p <= threshold: + logger.warning(f"Drift detected (min_p={min_p}). Launching retraining...") + train_best() + else: + logger.warning(f"No drift detected (min_p={min_p}).") + +# --------------------------------------------------------------------------- +# Flows +# --------------------------------------------------------------------------- + +@flow(name="monitoring_best") +def monitoring_best(): + """ + Drift monitoring flow: + - Collects reference and new batch datasets. + - Generates drift report (HTML + JSON). + - Computes drift metrics. + - Triggers model retraining if drift is significant. + """ + logger = get_logger() + logger.info("Starting monitoring_best flow...") + + training_data, _ = data_collection_task() + new_data = get_drift_data_task() + report_json = create_drift_report_task(current_data=new_data, reference_data=training_data) + p_val = compute_drift_task(report_json) + trigger_if_drift_task(min_p=p_val) + +if __name__ == "__main__": + # Serve this flow with a cron schedule (here: every minute) + monitoring_best.serve(cron="* * * * *") diff --git a/Workshops/AI/02-ML-Model-Lifecycle/src/serve.py b/Workshops/AI/02-ML-Model-Lifecycle/src/serve.py new file mode 100644 index 0000000..3f0c0fb --- /dev/null +++ b/Workshops/AI/02-ML-Model-Lifecycle/src/serve.py @@ -0,0 +1,131 @@ +""" +src/serve.py +FastAPI inference service for the Wine Quality model. + +- Loads a RandomForestClassifier model (joblib). +- Serves predictions via a REST API endpoint. +- CLI options allow flexible deployment or local testing. + +Endpoints: +- POST /predict : Accepts wine features and returns the predicted class and probabilities. +""" + +import subprocess +from pathlib import Path +from typing import Optional + +import joblib +import typer +import uvicorn +from fastapi import FastAPI +from pydantic import BaseModel + +from src.utils import get_logger + +app = typer.Typer(add_completion=False) + + +def build_app(model_path: Path) -> FastAPI: + """ + Constructs the FastAPI app with the loaded model. + + Args: + model_path (Path): Path to the trained model file (.pkl). + + Returns: + FastAPI: Configured FastAPI app with one /predict endpoint. + """ + # Load trained model + model = joblib.load(model_path) + + class Features(BaseModel): + """ + Request body schema for prediction. + """ + + alcohol: float + malic_acid: float + ash: float + alcalinity_of_ash: float + magnesium: float + total_phenols: float + flavanoids: float + nonflavanoid_phenols: float + proanthocyanins: float + color_intensity: float + hue: float + od280_od315_of_diluted_wines: float + proline: float + + # Initialize FastAPI app + api = FastAPI(title="Wine Quality Inference") + + @api.post("/predict") + def predict(feat: Features): + """ + Predicts the wine quality class based on provided features. + + Args: + feat (Features): Input wine features. + + Returns: + dict: Predicted class and probabilities for each class. + """ + import pandas as pd + + # Convert input features to DataFrame + X = pd.DataFrame([feat.dict()]) + + # Predict class and class probabilities + proba = model.predict_proba(X)[0].tolist() + predicted_class = int(model.predict(X)[0]) + + return {"class": predicted_class, "proba": proba} + + return api + + +def deploy(model_path: Path, host: str = "0.0.0.0", port: int = 9000): + """ + Deploys the given FastAPI app with Uvicorn. + + Args: + api (FastAPI): The app to serve. + host (str, optional): Host IP address to bind (default: 0.0.0.0). + port (int, optional): Port number to expose (default: 9000). + """ + logger = get_logger() + logger.info(f"Building FastAPI app for model from {model_path}...") + logger.info(f"Starting Uvicorn server on {host}:{port}...") + subprocess.Popen(["python", "-m", "src.serve", "--model-path", str(model_path)]) + + +@app.command() +def run( + model_path: Optional[Path] = typer.Option( + None, + "--model-path", + "-m", + help="Path to the .pkl model file.", + ), + host: str = "0.0.0.0", + port: int = 9000, +): + """ + CLI entry point for running the API locally. + + - Loads the model from the given path (or default artifacts directory). + - Builds the FastAPI app. + - Serves the API with Uvicorn. + + Args: + model_path (Optional[Path], optional): Path to the .pkl model file. + host (str, optional): Host IP to bind (default: 0.0.0.0). + port (int, optional): Port to expose the API (default: 9000). + """ + api = build_app(model_path) + uvicorn.run(api, host=host, port=port) + + +if __name__ == "__main__": + app() diff --git a/Workshops/AI/02-ML-Model-Lifecycle/src/train.py b/Workshops/AI/02-ML-Model-Lifecycle/src/train.py new file mode 100644 index 0000000..e700190 --- /dev/null +++ b/Workshops/AI/02-ML-Model-Lifecycle/src/train.py @@ -0,0 +1,112 @@ +""" +src/train.py +Model training module for the Wine Quality pipeline. + +- Trains a RandomForestClassifier on provided dataset splits. +- Logs parameters, metrics, and optionally the trained model to MLflow. +- Can be executed both via CLI (Typer) or imported in Prefect flows. +""" + +import joblib +import mlflow +import typer +from sklearn.ensemble import RandomForestClassifier +from sklearn.metrics import accuracy_score +from src.config import CFG + +from src.utils import get_logger + +app = typer.Typer(add_completion=False) + +# Configure MLflow tracking (uses the experiment defined in CFG) +mlflow.set_tracking_uri(CFG.mlflow_uri) +mlflow.set_experiment(CFG.experiment_name) + + +def train_model( + dataset_split: tuple, n_estimators: int, max_depth: int | None +) -> tuple: + """ + Trains a RandomForest model on the provided dataset. + + Args: + dataset_split (tuple): Tuple of (X_train, X_test, y_train, y_test) splits. + n_estimators (int): Number of trees in the forest. + max_depth (int | None): Maximum depth of the trees (None means no limit). + + Returns: + tuple: + - model (RandomForestClassifier): Trained model. + - acc (float): Accuracy on the test set. + """ + logger = get_logger() + logger.info( + f"Training RandomForest with n_estimators={n_estimators}, max_depth={max_depth}..." + ) + + X_tr, X_te, y_tr, y_te = dataset_split + + # Train the RandomForest model + model = RandomForestClassifier( + n_estimators=n_estimators, max_depth=max_depth, random_state=0 + ).fit(X_tr, y_tr) + + # Compute accuracy on the test set + acc = accuracy_score(y_te, model.predict(X_te)) + logger.info(f"Model trained. Accuracy={acc:.4f}.") + return model, acc + + +def run( + dataset_split: tuple, + n_estimators: int = 200, + max_depth: int | None = None, + log_model: bool = typer.Option( + True, + "--log-model/--no-log-model", + help="Whether to log the model artifact to MLflow.", + ), +) -> float: + """ + Main training entry point. + + - Trains a model using specified hyperparameters. + - Logs training parameters, accuracy metric, and optionally the trained model to MLflow. + + Args: + dataset_split (tuple): Tuple of (X_train, X_test, y_train, y_test) splits. + n_estimators (int, optional): Number of trees (default: 200). + max_depth (int | None, optional): Maximum tree depth (default: None, unlimited). + log_model (bool, optional): If True, logs the model artifact to MLflow (default: True). + + Returns: + float: Accuracy score of the trained model. + """ + logger = get_logger() + logger.info( + f"Running full training pipeline (MLflow logging enabled={log_model})..." + ) + + with mlflow.start_run(): + # Log hyperparameters + mlflow.log_param("n_estimators", n_estimators) + mlflow.log_param("max_depth", max_depth) + + # Train the model + model, acc = train_model(dataset_split, n_estimators, max_depth) + + # Log model accuracy + mlflow.log_metric("accuracy", acc) + + # Optionally serialize and log the trained model + if log_model: + outfile = CFG.artifacts_dir / CFG.model_name + joblib.dump(model, outfile) + mlflow.log_artifact(outfile) + + logger.info("Training run completed and logged to MLflow.") + return acc # Useful for Prefect tasks or external calls + + +if __name__ == "__main__": + app() diff --git a/Workshops/AI/02-ML-Model-Lifecycle/src/utils.py b/Workshops/AI/02-ML-Model-Lifecycle/src/utils.py new file mode 100644 index 0000000..a7c53ba --- /dev/null +++ b/Workshops/AI/02-ML-Model-Lifecycle/src/utils.py @@ -0,0 +1,29 @@ +""" +src/utils.py +Utility functions for the Wine Quality ML project. + +Includes: +- get_logger(): returns a Prefect logger if available, otherwise falls back to standard Python logging. +""" + +import logging + +def get_logger(): + """ + Returns a logger instance. + + - If inside a Prefect flow or task, returns the Prefect run logger. + - If outside Prefect (e.g., during unit tests or CLI usage), returns a standard Python logger. + + This allows all modules to use consistent logging behavior without crashing when Prefect context is missing. + + Returns: + Logger: Either a Prefect run logger or a standard Python logger. + """ + try: + from prefect import get_run_logger + return get_run_logger() + except RuntimeError: + # No active Prefect context; use fallback logger + logging.basicConfig(level=logging.INFO) + return logging.getLogger("fallback") diff --git a/Workshops/AI/02-ML-Model-Lifecycle/tests/__init__.py b/Workshops/AI/02-ML-Model-Lifecycle/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Workshops/AI/02-ML-Model-Lifecycle/tests/test_data.py b/Workshops/AI/02-ML-Model-Lifecycle/tests/test_data.py new file mode 100644 index 0000000..f0848f2 --- /dev/null +++ b/Workshops/AI/02-ML-Model-Lifecycle/tests/test_data.py @@ -0,0 +1,35 @@ +""" +Unit tests for the data module in the Wine Quality ML pipeline. + +Covers: +- Dataset loading +- Train-test splitting logic +""" + +from src import data + +def test_data_collection(): + """ + Test that the wine dataset is loaded correctly with non-empty features and labels. + """ + X, y = data.data_collection() + + # Assert features and labels are not empty and match in length + assert not X.empty, "Feature dataframe is empty" + assert len(X) == len(y), "Mismatch between features and labels" + assert "alcohol" in X.columns, "'alcohol' column should exist in features" + +def test_data_preparation(): + """ + Test that the dataset is correctly split into train and test sets. + """ + X, y = data.data_collection() + X_tr, X_te, y_tr, y_te = data.data_preparation((X, y), split=0.2) + + # Assert that the train + test sizes add up to the original + assert len(X_tr) + len(X_te) == len(X), "Train + test features mismatch" + assert len(y_tr) + len(y_te) == len(y), "Train + test labels mismatch" + + # Optional: Check that no data was lost and all parts are non-empty + assert not X_tr.empty and not X_te.empty, "Split feature sets are empty" + assert not y_tr.empty and not y_te.empty, "Split label sets are empty" diff --git a/Workshops/AI/02-ML-Model-Lifecycle/tests/test_serve.py b/Workshops/AI/02-ML-Model-Lifecycle/tests/test_serve.py new file mode 100644 index 0000000..8adeb52 --- /dev/null +++ b/Workshops/AI/02-ML-Model-Lifecycle/tests/test_serve.py @@ -0,0 +1,60 @@ +""" +Unit test for FastAPI model serving without relying on a pre-existing trained model. + +- Creates a dummy RandomForestClassifier +- Saves it to a temporary .pkl file +- Builds the FastAPI app using the test model +- Sends a prediction request to /predict +""" + +import tempfile +import joblib +from pathlib import Path +from fastapi.testclient import TestClient +from sklearn.ensemble import RandomForestClassifier +import pandas as pd +from src import serve + +def test_build_app_and_predict(): + """ + Test that the FastAPI app is correctly built with a temporary dummy model, + and that the /predict endpoint returns a valid response. + """ + # --- Create a dummy model and train it on minimal synthetic data --- + model = RandomForestClassifier() + X_dummy = pd.DataFrame([{ + "alcohol": 13.0, + "malic_acid": 2.0, + "ash": 2.5, + "alcalinity_of_ash": 15.0, + "magnesium": 100.0, + "total_phenols": 2.0, + "flavanoids": 2.5, + "nonflavanoid_phenols": 0.3, + "proanthocyanins": 1.9, + "color_intensity": 5.0, + "hue": 1.0, + "od280_od315_of_diluted_wines": 3.0, + "proline": 1000, + }]) + y_dummy = [1] + model.fit(X_dummy, y_dummy) + + # --- Save model to a temporary .pkl file --- + with tempfile.NamedTemporaryFile(suffix=".pkl", delete=False) as tmp: + model_path = Path(tmp.name) + joblib.dump(model, model_path) + + # --- Build FastAPI app using the dummy model --- + api = serve.build_app(model_path) + + # --- Create a test client and send a sample request --- + sample = X_dummy.iloc[0].to_dict() + client = TestClient(api) + response = client.post("/predict", json=sample) + + # --- Assertions --- + assert response.status_code == 200 + result = response.json() + assert "class" in result + assert isinstance(result["class"], int) diff --git a/Workshops/AI/02-ML-Model-Lifecycle/tests/test_train.py b/Workshops/AI/02-ML-Model-Lifecycle/tests/test_train.py new file mode 100644 index 0000000..52203e0 --- /dev/null +++ b/Workshops/AI/02-ML-Model-Lifecycle/tests/test_train.py @@ -0,0 +1,30 @@ +""" +Unit test for the train module in the Wine Quality ML pipeline. + +Covers: +- Training a RandomForestClassifier +- Validating model accuracy and structure +""" + +from src import data, train + +def test_train_model_accuracy_threshold(): + """ + Test that the train_model function returns a trained model with acceptable accuracy. + + - Uses a real dataset split + - Checks that accuracy is above a baseline threshold + - Verifies that the model supports .predict() + """ + # Load and split the dataset + X, y = data.data_collection() + prepared = data.data_preparation((X, y)) + + # Train a RandomForest model with basic hyperparameters + model, acc = train.train_model(prepared, n_estimators=10, max_depth=5) + + # Assert accuracy is above 50% (weak sanity check) + assert acc > 0.5, f"Expected accuracy > 0.5, got {acc}" + + # Check that model has a predict method + assert hasattr(model, "predict"), "Model does not implement .predict()" diff --git a/Workshops/AI/03-Large-Language-Models/README.md b/Workshops/AI/03-Large-Language-Models/README.md new file mode 100644 index 0000000..56b3101 --- /dev/null +++ b/Workshops/AI/03-Large-Language-Models/README.md @@ -0,0 +1,104 @@ +# Lab: Prompt‑Engineering and LoRA Fine‑Tuning + +Welcome to this live lab! +In the previous lab session we discussed how to: + +1. **Train a classification model** on a real dataset using `Scikit-learn`. +2. **Manage and track your experiments** with `MLflow`. +3. **Build CI/CD-like workflows** using `Makefile` and `nox`. +4. **Build scalable and orchestrated ML pipelines** using `Prefect`. +5. **Monitor data drift over time** and trigger retraining with `Evidently`. + +In this session, you will run two Jupyter notebooks that demonstrate: + +- Practical prompt‑engineering patterns in `notebook 1_LabLLM.ipynb`, the craft of writing clear, explicit instructions for an LLM. +- A LoRA fine‑tuning workflow on model T5‑small in `notebook 2_LabLLM_LoRA.ipynb`, a lightweight fine-tuning technique for transformers. + +We'll leverage two big providers for this lab: + +- **Google Colab**: a free, browser-based Jupyter environment hosted by Google. Colab gives you a temporary Linux VM with Python, CUDA drivers, and optional GPU/TPU. You open a .ipynb notebook, run cells just like in Jupyter, and share links with classmates—no local setup required. + +- **Hugging Face**: A community and tooling hub for modern AI. + + It offers: + + - Model Hub: 500 k-plus pre-trained models (transformers, diffusers, etc.). + - Datasets Hub: curated datasets in one-line API. + - Inference API / Endpoints: hosted model inference in the cloud. + - Libraries: transformers, datasets, peft, and more. + + Creating a (free) HF account lets you: + + - Download models. + - Push your own models/adapters/data. + - Generate an access token that notebooks use to call hosted endpoints securely. + +--- + +## Repository contents + +| Path/Name | What it is and why it matters | +| --------------------------------------- | ---------------------------------------------- | +| `notebooks/1_LabLLM.ipynb` | Prompt‑engineering walkthrough (Mistral‑7B) | +| `notebooks/2_LabLLM_LoRA.ipynb` | LoRA adapter training on a 250‑row support set | +| `files/customer_support_lora_dataset_250.csv` | Tiny dataset used in Notebook 2 | +| `README` (this file) | Quickstart and background | + +## Prerequisites + +| Tool / Account | Why you need it | Sign‑up link | +| ------------------------ | -------------------------------------------- | ------------------------------------------------------------------------ | +| **Google account** | Required to open Colab notebooks | [https://accounts.google.com/signup](https://accounts.google.com/signup) | +| **Hugging Face account** | Lets you call hosted models & store adapters | [https://huggingface.co/join](https://huggingface.co/join) | + +### 1 Create your HF account and token + +1. Go to [https://huggingface.co/join](https://huggingface.co/join) and sign up (email and password). +2. Verify your email address (check spam folder too). +3. Navigate to **Settings > Access Tokens > New Token**. +4. Name it e.g. `ai‑academy‑lab` and select **"Read"** scope (fine for this lesson). +5. Copy the token. **You will paste it into the notebook when prompted**. +6. Treat the token like a password and do **not** share or commit it to GitHub. + +### 2 Recommended Colab runtime settings + +- **GPU:** In Colab, go to **Runtime > Change runtime type > Hardware accelerator > GPU**. +- **RAM:** The default *Standard* tier is enough. No paid upgrade needed. + +--- + +## Run the notebooks in Colab + +### 1 Clone the Repository + +To start, clone the lab repository by running the following command in the terminal: + +```sh + git clone https://github.com/kiratech/labs.git +``` + +### 2 Checkout the Lab Branch + +After cloning the repository, checkout the `academy-ai` branch: + +```sh + git checkout academy-ai +``` + +Then, navigate to the project folder: + +```sh + cd labs/Workshops/AI/03-Large-Language-Models +``` + +This folder contains resources related to this lab. +Open Google Colab and import, everything at root level, the content of the folders notebooks and files. +From now you can continue on the notebooks. + +--- + +## Suggested readings and references + +- [OpenAI Cookbook – GPT‑4 Prompting Guide](https://cookbook.openai.com/examples/gpt4-1_prompting_guide): a prompting guide that lays out a practical structure for building powerful prompts, especially with GPT-4.1. It’s short, clear, and highly effective for anyone working with agents, structured outputs, tool use, or reasoning-heavy tasks. +- [OpenAI Cookbook – GPT‑5 Prompting Guide](https://cookbook.openai.com/examples/gpt-5/gpt-5_prompting_guide): New guide! +- [Google Prompt Engineering pdf](https://drive.google.com/file/d/1AbaBYbEa_EbPelsT40-vj64L-2IwUJHy/view): whether you're technical or non-technical, this might be one of the most useful prompt engineering resources out there right now. Google just published a 68-page whitepaper focused on Prompt Engineering (focused on API users), and it goes deep on structure, formatting, config settings, and real examples. diff --git a/Workshops/AI/03-Large-Language-Models/files/customer_support_lora_dataset_250.csv b/Workshops/AI/03-Large-Language-Models/files/customer_support_lora_dataset_250.csv new file mode 100644 index 0000000..3d10f64 --- /dev/null +++ b/Workshops/AI/03-Large-Language-Models/files/customer_support_lora_dataset_250.csv @@ -0,0 +1,251 @@ +input_text,target_text +generate structured_reply: I received the wrong item in my order.,"{""greeting"":""Dear customer,"",""apology"":""We apologize for this experience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: The product I received is damaged. What can I do?,"{""greeting"":""Dear customer,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I received the wrong item in my order.,"{""greeting"":""Hi there,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I received the wrong item in my order.,"{""greeting"":""Greetings,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I accidentally ordered the wrong color. Can I change it?,"{""greeting"":""Greetings,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I was promised a discount that was never applied.,"{""greeting"":""Hi there,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I never received my order.,"{""greeting"":""Hi there,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: The product I received is damaged. What can I do?,"{""greeting"":""Hello,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: Your website charged me shipping twice.,"{""greeting"":""Hello,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: Why was I charged twice for my order?,"{""greeting"":""Dear customer,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: Your website charged me shipping twice.,"{""greeting"":""Greetings,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: Your website charged me shipping twice.,"{""greeting"":""Hello,"",""apology"":""We apologize for this experience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Hello,"",""apology"":""We apologize for this experience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Hello,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: Why was I charged twice for my order?,"{""greeting"":""Greetings,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: The tracking number shows delivered but I didn't get the package.,"{""greeting"":""Dear customer,"",""apology"":""We apologize for this experience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I accidentally ordered the wrong color. Can I change it?,"{""greeting"":""Dear customer,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: The replacement you sent is also faulty.,"{""greeting"":""Greetings,"",""apology"":""We apologize for this experience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: Why was I charged twice for my order?,"{""greeting"":""Dear customer,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: Your website charged me shipping twice.,"{""greeting"":""Greetings,"",""apology"":""We apologize for this experience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: The replacement you sent is also faulty.,"{""greeting"":""Hello,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: The item arrived with missing accessories.,"{""greeting"":""Dear customer,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: The replacement you sent is also faulty.,"{""greeting"":""Hi there,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: My order arrived late. I want a refund.,"{""greeting"":""Hi there,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: Your website charged me shipping twice.,"{""greeting"":""Dear customer,"",""apology"":""We apologize for this experience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I never received my order.,"{""greeting"":""Hello,"",""apology"":""We apologize for this experience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Greetings,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I accidentally ordered the wrong color. Can I change it?,"{""greeting"":""Hello,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: Your website charged me shipping twice.,"{""greeting"":""Dear customer,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: Why was I charged twice for my order?,"{""greeting"":""Greetings,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: The product I received is damaged. What can I do?,"{""greeting"":""Greetings,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: The tracking number shows delivered but I didn't get the package.,"{""greeting"":""Hi there,"",""apology"":""We apologize for this experience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I accidentally ordered the wrong color. Can I change it?,"{""greeting"":""Hello,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I received the wrong item in my order.,"{""greeting"":""Greetings,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: How can I return an item I purchased last week?,"{""greeting"":""Hello,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: The item arrived with missing accessories.,"{""greeting"":""Dear customer,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: The item arrived with missing accessories.,"{""greeting"":""Hello,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: Can I exchange my item for a different size?,"{""greeting"":""Greetings,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I need help tracking my shipment.,"{""greeting"":""Hi there,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: My order arrived late. I want a refund.,"{""greeting"":""Hello,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I accidentally ordered the wrong color. Can I change it?,"{""greeting"":""Hello,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: Can I exchange my item for a different size?,"{""greeting"":""Hi there,"",""apology"":""We apologize for this experience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I received the wrong item in my order.,"{""greeting"":""Hi there,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Dear customer,"",""apology"":""We apologize for this experience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: How can I return an item I purchased last week?,"{""greeting"":""Hi there,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: Why was I charged twice for my order?,"{""greeting"":""Hi there,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: Your website charged me shipping twice.,"{""greeting"":""Hello,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I need help tracking my shipment.,"{""greeting"":""Greetings,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: How can I return an item I purchased last week?,"{""greeting"":""Greetings,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I was promised a discount that was never applied.,"{""greeting"":""Hi there,"",""apology"":""We apologize for this experience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: Your website charged me shipping twice.,"{""greeting"":""Dear customer,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: The replacement you sent is also faulty.,"{""greeting"":""Hello,"",""apology"":""We apologize for this experience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: The item arrived with missing accessories.,"{""greeting"":""Hi there,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: The replacement you sent is also faulty.,"{""greeting"":""Greetings,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: The tracking number shows delivered but I didn't get the package.,"{""greeting"":""Hello,"",""apology"":""We apologize for this experience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I never received my order.,"{""greeting"":""Hello,"",""apology"":""We apologize for this experience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: The product I received is damaged. What can I do?,"{""greeting"":""Greetings,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: Why was I charged twice for my order?,"{""greeting"":""Hi there,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I never received my order.,"{""greeting"":""Hi there,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I was promised a discount that was never applied.,"{""greeting"":""Dear customer,"",""apology"":""We apologize for this experience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: How can I return an item I purchased last week?,"{""greeting"":""Hi there,"",""apology"":""We apologize for this experience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I never received my order.,"{""greeting"":""Dear customer,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I never received my order.,"{""greeting"":""Dear customer,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Hi there,"",""apology"":""We apologize for this experience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: Can I exchange my item for a different size?,"{""greeting"":""Greetings,"",""apology"":""We apologize for this experience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I need help tracking my shipment.,"{""greeting"":""Hi there,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: The tracking number shows delivered but I didn't get the package.,"{""greeting"":""Greetings,"",""apology"":""We apologize for this experience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: The item arrived with missing accessories.,"{""greeting"":""Hi there,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: Your website charged me shipping twice.,"{""greeting"":""Dear customer,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: Your website charged me shipping twice.,"{""greeting"":""Hello,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I accidentally ordered the wrong color. Can I change it?,"{""greeting"":""Hello,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I accidentally ordered the wrong color. Can I change it?,"{""greeting"":""Hello,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: How can I return an item I purchased last week?,"{""greeting"":""Dear customer,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I was promised a discount that was never applied.,"{""greeting"":""Hello,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Greetings,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: My order arrived late. I want a refund.,"{""greeting"":""Dear customer,"",""apology"":""We apologize for this experience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I accidentally ordered the wrong color. Can I change it?,"{""greeting"":""Dear customer,"",""apology"":""We apologize for this experience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: Can I exchange my item for a different size?,"{""greeting"":""Hello,"",""apology"":""We apologize for this experience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: How can I return an item I purchased last week?,"{""greeting"":""Greetings,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I received the wrong item in my order.,"{""greeting"":""Hi there,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: How can I return an item I purchased last week?,"{""greeting"":""Dear customer,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I accidentally ordered the wrong color. Can I change it?,"{""greeting"":""Hello,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Greetings,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: The item arrived with missing accessories.,"{""greeting"":""Dear customer,"",""apology"":""We apologize for this experience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Dear customer,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: The product I received is damaged. What can I do?,"{""greeting"":""Hello,"",""apology"":""We apologize for this experience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I accidentally ordered the wrong color. Can I change it?,"{""greeting"":""Dear customer,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: How can I return an item I purchased last week?,"{""greeting"":""Hello,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: How can I return an item I purchased last week?,"{""greeting"":""Greetings,"",""apology"":""We apologize for this experience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I need help tracking my shipment.,"{""greeting"":""Hi there,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I accidentally ordered the wrong color. Can I change it?,"{""greeting"":""Hi there,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: Why was I charged twice for my order?,"{""greeting"":""Dear customer,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: Your website charged me shipping twice.,"{""greeting"":""Greetings,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I never received my order.,"{""greeting"":""Hi there,"",""apology"":""We apologize for this experience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: My order arrived late. I want a refund.,"{""greeting"":""Greetings,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I need help tracking my shipment.,"{""greeting"":""Dear customer,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: The tracking number shows delivered but I didn't get the package.,"{""greeting"":""Hi there,"",""apology"":""We apologize for this experience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I accidentally ordered the wrong color. Can I change it?,"{""greeting"":""Hello,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: The item arrived with missing accessories.,"{""greeting"":""Hi there,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: Can I exchange my item for a different size?,"{""greeting"":""Greetings,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: The tracking number shows delivered but I didn't get the package.,"{""greeting"":""Hello,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I was promised a discount that was never applied.,"{""greeting"":""Hello,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: The item arrived with missing accessories.,"{""greeting"":""Dear customer,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: The tracking number shows delivered but I didn't get the package.,"{""greeting"":""Hello,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: The item arrived with missing accessories.,"{""greeting"":""Dear customer,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Dear customer,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: Can I exchange my item for a different size?,"{""greeting"":""Greetings,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I accidentally ordered the wrong color. Can I change it?,"{""greeting"":""Hello,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Hello,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: The product I received is damaged. What can I do?,"{""greeting"":""Greetings,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: Your website charged me shipping twice.,"{""greeting"":""Greetings,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I accidentally ordered the wrong color. Can I change it?,"{""greeting"":""Hi there,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: The item arrived with missing accessories.,"{""greeting"":""Dear customer,"",""apology"":""We apologize for this experience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: The product I received is damaged. What can I do?,"{""greeting"":""Dear customer,"",""apology"":""We apologize for this experience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: The tracking number shows delivered but I didn't get the package.,"{""greeting"":""Hi there,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Dear customer,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: My order arrived late. I want a refund.,"{""greeting"":""Greetings,"",""apology"":""We apologize for this experience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I need help tracking my shipment.,"{""greeting"":""Dear customer,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: Can I exchange my item for a different size?,"{""greeting"":""Dear customer,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: Your website charged me shipping twice.,"{""greeting"":""Hello,"",""apology"":""We apologize for this experience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Dear customer,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I never received my order.,"{""greeting"":""Hi there,"",""apology"":""We apologize for this experience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: The tracking number shows delivered but I didn't get the package.,"{""greeting"":""Hi there,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Hi there,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I received the wrong item in my order.,"{""greeting"":""Greetings,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: Your website charged me shipping twice.,"{""greeting"":""Hello,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I need help tracking my shipment.,"{""greeting"":""Hello,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I need help tracking my shipment.,"{""greeting"":""Dear customer,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I was promised a discount that was never applied.,"{""greeting"":""Greetings,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: The item arrived with missing accessories.,"{""greeting"":""Hello,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: My order arrived late. I want a refund.,"{""greeting"":""Hello,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I never received my order.,"{""greeting"":""Hi there,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I need help tracking my shipment.,"{""greeting"":""Hi there,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: Your website charged me shipping twice.,"{""greeting"":""Hi there,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: The replacement you sent is also faulty.,"{""greeting"":""Hello,"",""apology"":""We apologize for this experience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: The product I received is damaged. What can I do?,"{""greeting"":""Hi there,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I need help tracking my shipment.,"{""greeting"":""Dear customer,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: The product I received is damaged. What can I do?,"{""greeting"":""Hi there,"",""apology"":""We apologize for this experience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: The product I received is damaged. What can I do?,"{""greeting"":""Hi there,"",""apology"":""We apologize for this experience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: Can I exchange my item for a different size?,"{""greeting"":""Greetings,"",""apology"":""We apologize for this experience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: Why was I charged twice for my order?,"{""greeting"":""Hello,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: The product I received is damaged. What can I do?,"{""greeting"":""Hello,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: Why was I charged twice for my order?,"{""greeting"":""Hi there,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Hi there,"",""apology"":""We apologize for this experience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: How can I return an item I purchased last week?,"{""greeting"":""Hi there,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: The replacement you sent is also faulty.,"{""greeting"":""Hi there,"",""apology"":""We apologize for this experience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I received the wrong item in my order.,"{""greeting"":""Dear customer,"",""apology"":""We apologize for this experience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: The item arrived with missing accessories.,"{""greeting"":""Greetings,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: My order arrived late. I want a refund.,"{""greeting"":""Hello,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: The product I received is damaged. What can I do?,"{""greeting"":""Dear customer,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Hello,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I was promised a discount that was never applied.,"{""greeting"":""Hello,"",""apology"":""We apologize for this experience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: Your website charged me shipping twice.,"{""greeting"":""Hello,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I was promised a discount that was never applied.,"{""greeting"":""Hi there,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I need help tracking my shipment.,"{""greeting"":""Greetings,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: The item arrived with missing accessories.,"{""greeting"":""Hi there,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: The item arrived with missing accessories.,"{""greeting"":""Dear customer,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: Your website charged me shipping twice.,"{""greeting"":""Hello,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I never received my order.,"{""greeting"":""Hi there,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I received the wrong item in my order.,"{""greeting"":""Hello,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I accidentally ordered the wrong color. Can I change it?,"{""greeting"":""Dear customer,"",""apology"":""We apologize for this experience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I accidentally ordered the wrong color. Can I change it?,"{""greeting"":""Dear customer,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I was promised a discount that was never applied.,"{""greeting"":""Hello,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: Can I exchange my item for a different size?,"{""greeting"":""Hello,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I was promised a discount that was never applied.,"{""greeting"":""Hello,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: The product I received is damaged. What can I do?,"{""greeting"":""Dear customer,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I received the wrong item in my order.,"{""greeting"":""Hello,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I received the wrong item in my order.,"{""greeting"":""Dear customer,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: The item arrived with missing accessories.,"{""greeting"":""Hello,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: Your website charged me shipping twice.,"{""greeting"":""Hello,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Hello,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: The item arrived with missing accessories.,"{""greeting"":""Greetings,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I was promised a discount that was never applied.,"{""greeting"":""Greetings,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: How can I return an item I purchased last week?,"{""greeting"":""Dear customer,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Greetings,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: How can I return an item I purchased last week?,"{""greeting"":""Hi there,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I accidentally ordered the wrong color. Can I change it?,"{""greeting"":""Dear customer,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I need help tracking my shipment.,"{""greeting"":""Hi there,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I need help tracking my shipment.,"{""greeting"":""Hello,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: The product I received is damaged. What can I do?,"{""greeting"":""Hi there,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I need help tracking my shipment.,"{""greeting"":""Dear customer,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I received the wrong item in my order.,"{""greeting"":""Dear customer,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: The replacement you sent is also faulty.,"{""greeting"":""Hello,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Dear customer,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Hi there,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: Your website charged me shipping twice.,"{""greeting"":""Dear customer,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Greetings,"",""apology"":""We apologize for this experience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I never received my order.,"{""greeting"":""Greetings,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I was promised a discount that was never applied.,"{""greeting"":""Hi there,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Greetings,"",""apology"":""We apologize for this experience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I received the wrong item in my order.,"{""greeting"":""Hi there,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I accidentally ordered the wrong color. Can I change it?,"{""greeting"":""Hi there,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I received the wrong item in my order.,"{""greeting"":""Hi there,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I was promised a discount that was never applied.,"{""greeting"":""Greetings,"",""apology"":""We apologize for this experience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: How can I return an item I purchased last week?,"{""greeting"":""Greetings,"",""apology"":""We apologize for this experience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: Why was I charged twice for my order?,"{""greeting"":""Greetings,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: Can I exchange my item for a different size?,"{""greeting"":""Dear customer,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Greetings,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: Can I exchange my item for a different size?,"{""greeting"":""Hello,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Greetings,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: Your website charged me shipping twice.,"{""greeting"":""Hi there,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: The product I received is damaged. What can I do?,"{""greeting"":""Dear customer,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: The product I received is damaged. What can I do?,"{""greeting"":""Hi there,"",""apology"":""We apologize for this experience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: My order arrived late. I want a refund.,"{""greeting"":""Hi there,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I never received my order.,"{""greeting"":""Hi there,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I received the wrong item in my order.,"{""greeting"":""Hi there,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: The product I received is damaged. What can I do?,"{""greeting"":""Dear customer,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I never received my order.,"{""greeting"":""Greetings,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I was promised a discount that was never applied.,"{""greeting"":""Greetings,"",""apology"":""We apologize for this experience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: My order arrived late. I want a refund.,"{""greeting"":""Greetings,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: My order arrived late. I want a refund.,"{""greeting"":""Hi there,"",""apology"":""We apologize for this experience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: Can I exchange my item for a different size?,"{""greeting"":""Hi there,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I received the wrong item in my order.,"{""greeting"":""Greetings,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: How can I return an item I purchased last week?,"{""greeting"":""Greetings,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I received the wrong item in my order.,"{""greeting"":""Dear customer,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: Can I exchange my item for a different size?,"{""greeting"":""Hi there,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: How can I return an item I purchased last week?,"{""greeting"":""Dear customer,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: The product I received is damaged. What can I do?,"{""greeting"":""Hello,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: How can I return an item I purchased last week?,"{""greeting"":""Hi there,"",""apology"":""We apologize for this experience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Greetings,"",""apology"":""We apologize for this experience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I was promised a discount that was never applied.,"{""greeting"":""Dear customer,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: I was promised a discount that was never applied.,"{""greeting"":""Dear customer,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: The product I received is damaged. What can I do?,"{""greeting"":""Dear customer,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: My order arrived late. I want a refund.,"{""greeting"":""Dear customer,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: The tracking number shows delivered but I didn't get the package.,"{""greeting"":""Greetings,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: The product I received is damaged. What can I do?,"{""greeting"":""Hello,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: How can I return an item I purchased last week?,"{""greeting"":""Greetings,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: The replacement you sent is also faulty.,"{""greeting"":""Hi there,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: How can I return an item I purchased last week?,"{""greeting"":""Dear customer,"",""apology"":""We apologize for this experience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I was promised a discount that was never applied.,"{""greeting"":""Dear customer,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: Your website charged me shipping twice.,"{""greeting"":""Dear customer,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Dear customer,"",""apology"":""We apologize for this experience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I was promised a discount that was never applied.,"{""greeting"":""Hello,"",""apology"":""We apologize for this experience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: The item arrived with missing accessories.,"{""greeting"":""Hi there,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: The tracking number shows delivered but I didn't get the package.,"{""greeting"":""Hello,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: How can I return an item I purchased last week?,"{""greeting"":""Hi there,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: The tracking number shows delivered but I didn't get the package.,"{""greeting"":""Greetings,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: The tracking number shows delivered but I didn't get the package.,"{""greeting"":""Hello,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: Why was I charged twice for my order?,"{""greeting"":""Hi there,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I need help tracking my shipment.,"{""greeting"":""Hello,"",""apology"":""We apologize for this experience."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: The tracking number shows delivered but I didn't get the package.,"{""greeting"":""Hello,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: My order arrived late. I want a refund.,"{""greeting"":""Hi there,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: The tracking number shows delivered but I didn't get the package.,"{""greeting"":""Greetings,"",""apology"":""We apologize for this experience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: Why was I charged twice for my order?,"{""greeting"":""Dear customer,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""A free replacement will be shipped today and tracking will follow."",""action_type"":""replace"",""needs_follow_up"":false}" +generate structured_reply: My package was left outside in the rain and got wet.,"{""greeting"":""Hi there,"",""apology"":""We apologize for this experience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: The item arrived with missing accessories.,"{""greeting"":""Greetings,"",""apology"":""We sincerely apologize for the inconvenience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I never received my order.,"{""greeting"":""Dear customer,"",""apology"":""We apologize for this experience."",""resolution"":""Your refund has been processed and no return is required."",""action_type"":""refund"",""needs_follow_up"":false}" +generate structured_reply: I received the wrong item in my order.,"{""greeting"":""Dear customer,"",""apology"":""Please accept our apologies for the issue."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: The replacement you sent is also faulty.,"{""greeting"":""Hi there,"",""apology"":""We apologize for this experience."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" +generate structured_reply: I accidentally ordered the wrong color. Can I change it?,"{""greeting"":""Greetings,"",""apology"":""We are sorry for any trouble caused."",""resolution"":""Your case has been escalated to a senior agent who will contact you shortly."",""action_type"":""escalate"",""needs_follow_up"":true}" diff --git a/Workshops/AI/03-Large-Language-Models/notebooks/1_LabLLM.ipynb b/Workshops/AI/03-Large-Language-Models/notebooks/1_LabLLM.ipynb new file mode 100644 index 0000000..9936556 --- /dev/null +++ b/Workshops/AI/03-Large-Language-Models/notebooks/1_LabLLM.ipynb @@ -0,0 +1,459 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "us7zHCUszttj" + }, + "source": [ + "# Prompt-Engineering Lab: Customer-Support Replies with **Mistral-7B-Instruct**\n", + "\n", + "In this lab you will experiment with six classic **prompt patterns**:\n", + "\n", + "1. **Zero-shot**: no examples, just the task. \n", + "2. **Few-shot**: show 1-2 examples first. \n", + "3. **Chain-of-Thought**: ask the model to reason step by step. \n", + "4. **Role / Persona**: tell the model \"You are a ...\". \n", + "5. **Structured delimiters**: clearly define context or information. \n", + "6. **System / Policy instructions**: prepend internal guidelines. \n", + "7. **Structured JSON-formatted**: force JSON or bullet format. \n", + "\n", + "By the end you'll understand how tiny changes in wording can dramatically change an LLM's answer.\n", + "\n", + "## What is Prompt-Engineering?\n", + "Because large language models don't have traditional \"APIs\", you steer them with natural-language prompts. Refining spacing, order, examples, and style (zero-shot, few-shot, chain-of-thought, role instructions, JSON schemas, etc.) is called prompt engineering. Good prompts boost accuracy and consistency without touching the model's weights." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ug_3UHSCaaZI" + }, + "source": [ + "## 0 Install Hugging Face client library\n", + "\n", + "We only need **`huggingface_hub`** because we call the hosted Inference API. No heavyweight Transformers install is required.\n", + "\n", + "### What is Hugging Face and the Hugging Face Hub ?\n", + "\n", + "Hugging Face is an open-source-first AI company that curates Transformers, Diffusers, and other widely used libraries.\n", + "The Hub (https://huggingface.co/) is a **GitHub for models & datasets**:\n", + "- Free hosting for models, datasets, Spaces (demo apps).\n", + "- Built-in versioning, README rendering, and model cards.\n", + "- REST / Python SDK (huggingface_hub) for push-pull, inference endpoints, and gated access control.\n", + "\n", + "In practice you clone/push a repo exactly like Git, but via hf_hub_download() or model.push_to_hub()." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "-hTQLYPZdUZp" + }, + "outputs": [], + "source": [ + "!pip install -q huggingface_hub" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "UX-GYw-WbGrP" + }, + "source": [ + "## 1 Authenticate and create an `InferenceClient`\n", + "\n", + "You need an **access token** to use the Mistral-7B endpoint hosted by provider **\"novita\"** on Hugging Face.\n", + "\n", + "### What are providers on HF Inference Client and what is novita?\n", + "\n", + "The InferenceClient can route a request to different providers (back-ends):\n", + "- huggingface (default): the standard Inference API hosted by HF.\n", + "- novita (a third-party back-end): offers chat-optimized latency/pricing. API surface matches OpenAI-style /chat/completions.\n", + "\n", + "You switch simply by\n", + "\n", + "```python\n", + "client = InferenceClient(provider=\"novita\", api_key=HF_TOKEN)\n", + "```\n", + "so the same client code can talk to multiple vendors." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "mZG3akxmdqpT" + }, + "outputs": [], + "source": [ + "import os, getpass\n", + "from huggingface_hub import InferenceClient\n", + "\n", + "# Paste your personal HF token when prompted\n", + "os.environ[\"HF_TOKEN\"] = getpass.getpass(\"Enter your HF token: \")\n", + "\n", + "# Create the client (model is chosen later in each call)\n", + "client = InferenceClient(provider=\"novita\", api_key=os.environ[\"HF_TOKEN\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "-D2_PSUlep8I" + }, + "source": [ + "## Model spotlight - mistralai/Mistral-7B-Instruct-v0.3\n", + "| - | Pros | Cons |\n", + "| ------------------ | -------------------------------------------------------------------------------- | --------------------------------------------------------------------------- |\n", + "| **Quality** | Very strong chat and code for a 7 B model. Outperforms LLaMA-7B/13B in many evals. | Still below GPT-4 / Claude-Opus tier on reasoning and long-context. |\n", + "| **Licence** | Apache-2.0, commercial use allowed. | Weight access sometimes gated. You need HF/token or local download. |\n", + "| **Speed / Memory** | Fits into 14 GB GPU (8-bit) so it is cheap to host. Responds \\~50-70 tok/s on T4. | Context window 8 K. Cannot process very long docs like GPT-4o-128K. |\n", + "| **LoRA** | Adapter \\~120 MB. Can be fine-tuned on a Colab T4 in <30 min. | PEFT on Mistral needs to target correct linear layers (`q_proj`, `v_proj`). |\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "qogkSPAbbiD2" + }, + "source": [ + "## 2 Zero-shot prompt\n", + "\n", + "Ask the model to perform a task without examples: “Translate the sentence to Italian: 'I love coffee'”.\n", + "\n", + "Zero-shot works because large LMs have seen countless instructions during pre-training. It's the fastest way to test a capability." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "vd3cTO7LemPx" + }, + "outputs": [], + "source": [ + "prompt = \"A customer is requesting a refund because a product was delivered late. Write a professional reply.\"\n", + "\n", + "completion = client.chat.completions.create(\n", + " model=\"mistralai/Mistral-7B-Instruct-v0.3\",\n", + " messages=[\n", + " {\"role\": \"user\", \"content\": prompt}\n", + " ],\n", + ")\n", + "\n", + "print(completion.choices[0].message[\"content\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "_UF5mFW1cBNm" + }, + "source": [ + "## 3 Few-shot prompt\n", + "\n", + "Provide 1-3 examples so the model infers pattern, tone, structure.\n", + "\n", + "Example: \n", + "Q1: Formal -> Casual \n", + "A1: Please contact me. -> Hit me up! \n", + "\n", + "The next transformation aligns with the examples far better than zero-shot." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "CeOQuFfRdrga" + }, + "outputs": [], + "source": [ + "prompt = '''Example 1:\n", + "Customer: I received my order late. I'd like a refund.\n", + "Reply: We're sorry for the delay. Your refund will be processed within 3 business days.\n", + "\n", + "Example 2:\n", + "Customer: My package arrived after the estimated delivery date. I want a refund.\n", + "Reply: We apologize for the inconvenience. The refund has been approved and will be issued shortly.\n", + "\n", + "Now write a reply to:\n", + "A customer is requesting a refund because a product was delivered late.\n", + "'''\n", + "\n", + "completion = client.chat.completions.create(\n", + " model=\"mistralai/Mistral-7B-Instruct-v0.3\",\n", + " messages=[{\"role\": \"user\", \"content\": prompt}],\n", + ")\n", + "\n", + "print(completion.choices[0].message[\"content\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ExIj1lZyc9jE" + }, + "source": [ + "## 4 Chain-of-Thought prompt\n", + "\n", + "Add \"Think step by step\" (or an explicit reasoning prompt). \n", + "The model reveals intermediate reasoning, often raising factual accuracy and allowing you to inspect/error-correct the chain." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "-TgOb4owgnHn" + }, + "outputs": [], + "source": [ + "prompt = \"A customer is requesting a refund because a product was delivered late. Think step by step and explain me your reasonings to write a professional reply.\"\n", + "\n", + "completion = client.chat.completions.create(\n", + " model=\"mistralai/Mistral-7B-Instruct-v0.3\",\n", + " messages=[{\"role\": \"user\", \"content\": prompt}],\n", + ")\n", + "\n", + "print(completion.choices[0].message[\"content\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "oWUOoci-dQLx" + }, + "source": [ + "## 5 Role / Persona prompt\n", + "\n", + "Pre-frame the assistant persona:\n", + "\"You are an empathetic customer-support agent…\". \n", + "Sets vocabulary, tone, even policy constraints without touching the model weights." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "n5jnEUKwgoVM" + }, + "outputs": [], + "source": [ + "prompt = \"You are a customer support agent. A customer is asking for a refund due to a late delivery. Write a helpful and empathetic response.\"\n", + "\n", + "completion = client.chat.completions.create(\n", + " model=\"mistralai/Mistral-7B-Instruct-v0.3\",\n", + " messages=[{\"role\": \"user\", \"content\": prompt}],\n", + ")\n", + "\n", + "print(completion.choices[0].message[\"content\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "I8fryyF4dgJ2" + }, + "source": [ + "## 6 Structured delimiters\n", + "\n", + "Wrap user context in clear markers (\"\"\" or ###), e.g.\n", + "\n", + "```txt\n", + "\"\"\"CONTEXT\n", + "long text …\n", + "\"\"\"\n", + "Summarize the context in one bullet.\n", + "```\n", + "\n", + "Prevents the model from mixing instructions with payload and boosts reliability in multi-part prompts." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "l9doquBSg2jV" + }, + "outputs": [], + "source": [ + "prompt = '''\n", + "### STRUCTURE\n", + "1. Greeting\n", + "2. Apology for the delay\n", + "3. Refund details\n", + "4. Closing remark\n", + "###\n", + "### CUSTOMER EMAIL\n", + "I received the product late and would like a refund.\n", + "###\n", + "Write a professional reply to customer email using our structure.\n", + "'''\n", + "\n", + "completion = client.chat.completions.create(\n", + " model=\"mistralai/Mistral-7B-Instruct-v0.3\",\n", + " messages=[{\"role\": \"user\", \"content\": prompt}],\n", + ")\n", + "\n", + "print(completion.choices[0].message[\"content\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "S1ooDMboeG6k" + }, + "source": [ + "## 7 System / Policy instructions (simulating LoRA-style policies)\n", + "\n", + "Simulation: bake policy or domain hints inside the prompt ([POLICY] ...). Notice how the reply changes when the policy changes.\n", + "\n", + "Real LoRA: train tiny adapter matrices (<<1 % params) and insert at Q/V/(K,O) projections of self-attention matrices. Needs only minutes, single GPU and the adapter file is <100 MB." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "koM2FhTJg2-q" + }, + "outputs": [], + "source": [ + "# Standard prompt\n", + "prompt = \"A customer is requesting a refund because a product was delivered late. Write a professional reply.\"\n", + "\n", + "completion = client.chat.completions.create(\n", + " model=\"mistralai/Mistral-7B-Instruct-v0.3\",\n", + " messages=[{\"role\": \"user\", \"content\": prompt}],\n", + ")\n", + "\n", + "print(\"Standard Prompt:\\n\", completion.choices[0].message[\"content\"], \"\\n---\")\n", + "\n", + "# Policy A: full refund, no return required\n", + "prompt_custom_1 = '''[INTERNAL POLICY: Respond with empathy, offer refund without requiring customer to return the item, and apologize sincerely. Use concise and positive tone.]\n", + "\n", + "A customer is requesting a refund because a product was delivered late. Write a professional reply.'''\n", + "\n", + "completion_custom_1 = client.chat.completions.create(\n", + " model=\"mistralai/Mistral-7B-Instruct-v0.3\",\n", + " messages=[{\"role\": \"user\", \"content\": prompt_custom_1}],\n", + ")\n", + "print(\"Policy 1:\\n\", completion_custom_1.choices[0].message[\"content\"], \"\\n---\")\n", + "\n", + "# Policy B: no refund, offer discount code\n", + "prompt_custom_2 = '''[INTERNAL POLICY: Do not offer refunds, instead offer 15% discount code.]\n", + "\n", + "A customer is requesting a refund because a product was delivered late. Write a professional reply.'''\n", + "\n", + "completion_custom_2 = client.chat.completions.create(\n", + " model=\"mistralai/Mistral-7B-Instruct-v0.3\",\n", + " messages=[{\"role\": \"user\", \"content\": prompt_custom_2}],\n", + ")\n", + "print(\"Policy 2:\\n\", completion_custom_2.choices[0].message[\"content\"], \"\\n---\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "L6PVrBLGebEB" + }, + "source": [ + "## 8 Structured JSON-formatted\n", + "\n", + "Explicitly combining role, task, format and give a skeleton:\n", + "\n", + "```\n", + "Return ONLY valid JSON:\n", + "{\"greeting\":\"\", \"apology\":\"\", \"resolution\":\"\", \"action_type\":\"\", \"needs_follow_up\":false}\n", + "```\n", + "\n", + "Model tends to obey if the request is precise and examples are consistent." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "OdBNug_3g3OX" + }, + "outputs": [], + "source": [ + "prompt = '''\n", + "You are a customer support assistant. A customer is requesting a refund because a product arrived late.\n", + "\n", + "Return the response in JSON format with the following fields:\n", + "- greeting\n", + "- apology\n", + "- refund_policy\n", + "- closing\n", + "\n", + "Example format:\n", + "{\n", + " \"greeting\": \"...\",\n", + " \"apology\": \"...\",\n", + " \"refund_policy\": \"...\",\n", + " \"closing\": \"...\"\n", + "}\n", + "'''\n", + "\n", + "completion = client.chat.completions.create(\n", + " model=\"mistralai/Mistral-7B-Instruct-v0.3\",\n", + " messages=[{\"role\": \"user\", \"content\": prompt}],\n", + ")\n", + "print(completion.choices[0].message[\"content\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "N2Go8Hszztto" + }, + "source": [ + "## 9 Take-aways\n", + "\n", + "- **Prompt engineering for rapid iteration**: you can prototype different behaviours in minutes, no fine-tuning needed. \n", + "- **System vs. User prompts**: internal policies (system) can override the tone and policy of the final answer. \n", + "- **Structured output**: JSON or numbered lists make post-processing trivial. \n", + "\n", + "### Next experiments\n", + "\n", + "1. Add temperature/top-p arguments to explore creativity vs. safety of the model. \n", + "2. Test the same prompts on a different model (e.g., **Llama-3-8B-Instruct**) to compare style and latency. \n", + "3. Combine with *LoRA-style adapters* (see Notebook 2) for policy adherence without needing bulky prompts. " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "7yg1gPSZztto" + }, + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "gpuType": "T4", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3.10.10 ('.venv': venv)", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.10.10" + }, + "vscode": { + "interpreter": { + "hash": "12a8c86496d70dc5ec458316f0fdc66c09147f19c1386e8b96b40af59bd1e3d4" + } + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/Workshops/AI/03-Large-Language-Models/notebooks/2_LabLLM_LoRA.ipynb b/Workshops/AI/03-Large-Language-Models/notebooks/2_LabLLM_LoRA.ipynb new file mode 100644 index 0000000..33913c6 --- /dev/null +++ b/Workshops/AI/03-Large-Language-Models/notebooks/2_LabLLM_LoRA.ipynb @@ -0,0 +1,504 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "BVqac1pg809I" + }, + "source": [ + "# LoRA Fine-Tuning Lab: T5-small Customer-Support Adapter\n", + "\n", + "Welcome! \n", + "This hands-on notebook will guide you, step by step, through fine-tuning a **T5-small** model with **LoRA (Low-Rank Adaptation)** so it can answer customer-support questions in a consistent, structured style. \n", + "Indeed, out-of-the-box **T5-small** has never seen our customer-support style or policy. \n", + "If you prompt it with “My order arrived late, I want a refund,” it answers vaguely, or not at all. \n", + "\n", + "In this notebook we will:\n", + "\n", + "1. **Measure the baseline** and see how poorly vanilla T5 handles eight real support prompts. \n", + "2. **Attach a tiny LoRA adapter** (\\~ 9 MB, 8-rank) and fine-tune it on **just 250 examples**. \n", + "3. **Re-test the same prompts** to verify that the adapted model now produces concise, policy-compliant replies. \n", + "\n", + "**Key takeaway:** with LoRA we upgrade a generic language model into a task-specialist in ~10 minutes on a free Colab GPU, without touching the original 60 M parameters.\n", + "\n", + "## What is LoRA (Low-Rank Adaptation)\n", + "\n", + "Instead of updating all hundreds of millions of parameters, LoRA freezes the original model and inserts two tiny matrices into selected linear layers (often the Q and V projections). Training adjusts only those low-rank “adapter” weights. So you need far less GPU memory, reach good quality with small datasets, and ship adapters (\\~10 MB) instead of full checkpoints (\\~2 GB)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "XvH1ePiH809K" + }, + "source": [ + "## 0 Environment Clean-up\n", + "Before starting, you may want to delete any previous artefacts (checkpoints, logs, etc.) so the run is fresh and reproducible. \n", + "Feel free to skip this cell if you have nothing to clean." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "kWEJ1_vBLlja" + }, + "outputs": [], + "source": [ + "# Remove previous training artefacts—run only if you need a fresh start\n", + "!rm -rf t5-lora-out\n", + "!rm -rf t5-small-lora-adapter" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "UMkIRIWx809M" + }, + "source": [ + "## 1 Install Dependencies\n", + "We rely on the **Hugging Face Transformers** ecosystem plus two helper libraries:\n", + "- **`transformers`**: model and trainer APIs.\n", + "- **`datasets`**: efficient data loading from HF.\n", + "- **`peft`** – adds LoRA and other parameter-efficient methods. \n", + "- **`accelerate`** – handles device placement (CPU / single-GPU / multi-GPU) transparently.\n", + "\n", + "Installation is one time." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "sIvNNZPcg3T-" + }, + "outputs": [], + "source": [ + "# Transformers, Datasets, PEFT, and Accelerate (quiet install)\n", + "!pip install -q transformers datasets peft accelerate" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ponHU0RFA8Ms" + }, + "source": [ + "## 2 Baseline Check: How well does vanilla T5-small handle our task?\n", + "\n", + "Before we train anything, let's ask the out-of-the-box model to draft a refund reply. \n", + "Spoiler: its answer will be generic, overly long, or simply unrelated because T5-small has never been told what our support policy or tone should be.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Qjw3r_bMA631" + }, + "outputs": [], + "source": [ + "from transformers import AutoTokenizer, AutoModelForSeq2SeqLM\n", + "\n", + "tokenizer = AutoTokenizer.from_pretrained(\"t5-small\")\n", + "model_base = AutoModelForSeq2SeqLM.from_pretrained(\"t5-small\")\n", + "\n", + "prompt = \"reply to this customer's email: My order arrived late and I want a refund.\"\n", + "inputs = tokenizer(prompt, return_tensors=\"pt\")\n", + "\n", + "# Greedy decode to keep things deterministic and short\n", + "outputs = model_base.generate(**inputs, max_new_tokens=120)\n", + "print(\"Vanilla T5-small says:\\n\")\n", + "print(tokenizer.decode(outputs[0], skip_special_tokens=True))\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "tkYjmNZ7809N" + }, + "source": [ + "## 3 Load CSV Dataset and transform it to HF Dataset\n", + "\n", + "Upload to Colab filesystem the file of our dataset: `files/customer_support_lora_dataset_250`. \n", + "Our CSV file has two columns:\n", + "\n", + "| Column name | What it contains |\n", + "|---------------|--------------------------------------------------|\n", + "| `input_text` | A raw customer request or complaint |\n", + "| `target_text` | The ideal structured reply we want the model to generate |\n", + "\n", + "We'll turn the CSV into a **HF Dataset** object so it plays nicely with the Trainer API." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Xm3Hrq4o6bGd" + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "from datasets import Dataset\n", + "\n", + "# Read the 250-row customer-support file\n", + "df = pd.read_csv(\"customer_support_lora_dataset_250.csv\")\n", + "ds = Dataset.from_pandas(df)\n", + "\n", + "print(\"Sample row:\") # quick sanity check\n", + "ds[0]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "2HlULnbn809P" + }, + "source": [ + "## 3 Tokenisation and Label Preparation\n", + "\n", + "Transformer models can't read raw text, they need **token IDs**.\n", + "For sequence-to-sequence models like T5 we must prepare **two** sequences:\n", + "\n", + "1. **Source** - the customer request (`input_text`) \n", + "2. **Target** - the desired reply (`target_text`) \n", + "\n", + "### Key details:\n", + "\n", + "- We call `tokenizer.as_target_tokenizer()` so the decoder uses its own special prefix tokens. \n", + "- We truncate to 128 tokens to keep batches small on modest GPUs.\n", + "\n", + "### What exactly are \"tokens\" (the 128-token limit)? \n", + "A token is not a word or a single character.\n", + "Transformers work on sub-word units produced by a tokenizer (for T5 that's a SentencePiece model with a 32 k-item vocabulary). The rules are learned from large corpora and try to strike a balance:\n", + "\n", + "| Example text | Tokens generated | Notes |\n", + "| ------------- | ---------------------- | ------------------------------------- |\n", + "| `tracking` | `▁track`, `ing` | the leading “▁” marks a word start |\n", + "| `refund` | `▁refund` | common words are often a single token |\n", + "| `extra-large` | `▁extra`, `-`, `large` | punctuation becomes its own token |\n", + "\n", + "Because tokens can be full words or fragments, the length in tokens is usually 1.3-1.6x shorter than counting raw characters but longer than counting full words.\n", + "A 128-token limit therefore fits roughly 75-100 English words (fewer if the text contains many rare names, URLs, or emojis that split into multiple tokens)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "X4z3IZjFrwMG" + }, + "outputs": [], + "source": [ + "from transformers import AutoTokenizer, DataCollatorForSeq2Seq\n", + "\n", + "tokenizer = AutoTokenizer.from_pretrained(\"t5-small\")\n", + "\n", + "def preprocess(example):\n", + " # Encode source\n", + " model_inputs = tokenizer(example[\"input_text\"], max_length=128, truncation=True)\n", + " # Encode target as labels\n", + " with tokenizer.as_target_tokenizer():\n", + " labels = tokenizer(example[\"target_text\"], max_length=128, truncation=True)\n", + " model_inputs[\"labels\"] = labels[\"input_ids\"]\n", + " return model_inputs\n", + "\n", + "ds_tok = ds.map(preprocess, remove_columns=ds.column_names)\n", + "\n", + "print(\"Input: \", ds[0][\"input_text\"])\n", + "print(\"Input tokens: \", len(ds_tok[0][\"input_ids\"]))\n", + "print(\"Target: \", ds[0][\"target_text\"])\n", + "print(\"Target tokens: \", len(ds_tok[0][\"labels\"]))\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "LU43xk3P809R" + }, + "source": [ + "## 4 Build Base Model and LoRA Configuration\n", + "\n", + "Here we load the vanilla **T5-small** (60 M parameters) and wrap it with a `peft.LoraConfig`.\n", + "\n", + "### Key hyper-parameters:\n", + "\n", + "| Parameter | Role | Here |\n", + "|--------------|---------------------------------------------------------------|------|\n", + "| `r` | Rank of the low-rank matrices (higher = more learning capacity) | 8 |\n", + "| `lora_alpha` | Scaling factor for the adapter’s update | 16 |\n", + "| `target_modules` | Which weight matrices get adapters (we pick **q** & **v**) | [\"q\",\"v\"] |\n", + "| `lora_dropout` | Regularisation inside adapters | 0.05 |\n", + "\n", + "### Why place LoRA adapters on q and v? What about the others?\n", + "\n", + "| Symbol | Full name | Role in self-attention |\n", + "| ------ | ------------------------------- | ------------------------------ |\n", + "| **Q** | **Query** projection | asks “what am I looking for?” |\n", + "| **K** | **Key** projection | represents “what do I have?” |\n", + "| **V** | **Value** projection | holds the information to mix |\n", + "| **O** | **Output** (final linear layer) | re-mixes heads after attention |\n", + "\n", + "A complete attention block has four projection matrices per head. Putting LoRA on all four gives maximum flexibility but also multiplies train-time memory. \n", + "Empirical sweet-spot: Research (LoRA paper §5 and several follow-ups) showed that adapting Q + V captures most task-specific gains while keeping parameter count and GPU RAM minimal. The intuition:\n", + "- Queries (Q) change how each token attends to others.\n", + "- Values (V) change what content is blended once attention scores are computed.\n", + "\n", + "Keys and the output layer matter too, but adjusting them yields diminishing returns for many language-generation tasks." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "qJwCKCFFrKgI" + }, + "outputs": [], + "source": [ + "from transformers import AutoModelForSeq2SeqLM\n", + "from peft import LoraConfig, get_peft_model, TaskType\n", + "\n", + "base_model = AutoModelForSeq2SeqLM.from_pretrained(\"t5-small\")\n", + "\n", + "lora_cfg = LoraConfig(\n", + " task_type = TaskType.SEQ_2_SEQ_LM, # generation task\n", + " r = 8, # rank of the LoRA matrices\n", + " lora_alpha = 16, # scaling\n", + " target_modules = [\"q\", \"v\"], # project only query & value matrices\n", + " lora_dropout = 0.05,\n", + " bias = \"none\"\n", + ")\n", + "\n", + "peft_model = get_peft_model(base_model, lora_cfg)\n", + "peft_model.print_trainable_parameters()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ezlVpv2R809S" + }, + "source": [ + "## 5 Training Arguments and Trainer Loop\n", + "\n", + "Hugging Face `Seq2SeqTrainer` takes care of the full training loop (forward, back-prop, gradient clipping, etc.).\n", + "\n", + "Important flags we set:\n", + "\n", + "- **`per_device_train_batch_size` = 16**, fits on a 12 GB GPU. \n", + "- **`num_train_epochs` = 30**, small dataset needs more passes. \n", + "- **`learning_rate` = 5e-4**, slightly higher than full-fine-tuning, because we’re optimising far fewer weights. \n", + "- **`save_strategy` = \"no\"**, skip checkpoints to save disk. You can change it to `\"epoch\"` if you want them. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true, + "id": "_zAx2Eq2raxS" + }, + "outputs": [], + "source": [ + "from transformers import Seq2SeqTrainer, Seq2SeqTrainingArguments, DataCollatorForSeq2Seq\n", + "\n", + "training_args = Seq2SeqTrainingArguments(\n", + " output_dir = \"./t5-lora-out\",\n", + " per_device_train_batch_size = 16,\n", + " num_train_epochs = 30,\n", + " learning_rate = 5e-4,\n", + " logging_steps = 5,\n", + " save_strategy = \"no\",\n", + " report_to = \"none\",\n", + ")\n", + "\n", + "trainer = Seq2SeqTrainer(\n", + " model = peft_model,\n", + " args = training_args,\n", + " train_dataset = ds_tok,\n", + " data_collator = DataCollatorForSeq2Seq(tokenizer, model=peft_model),\n", + ")\n", + "\n", + "trainer.train()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "WD8UKp4g809T" + }, + "source": [ + "## 6 Save Adapter and Tokenizer\n", + "\n", + "LoRA lets us store **only** the lightweight adapter, ~9 MB in this case. The base T5-small weights are **not duplicated**." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "bbw5BG817UvV" + }, + "outputs": [], + "source": [ + "trainer.model.save_pretrained(\"t5-small-lora-adapter\")\n", + "tokenizer.save_pretrained(\"t5-small-lora-adapter\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "H8U36q-C809T" + }, + "source": [ + "## 7 Load LoRA-Adapted Model for Inference\n", + "\n", + "We merge the adapter with the frozen base model at load time, then generate a reply for a sample complaint." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "eG0hnG43qlAM" + }, + "outputs": [], + "source": [ + "from peft import PeftConfig, PeftModel\n", + "from transformers import AutoModelForSeq2SeqLM\n", + "\n", + "# Load base model\n", + "cfg = PeftConfig.from_pretrained(\"t5-small-lora-adapter\")\n", + "base_model = AutoModelForSeq2SeqLM.from_pretrained(cfg.base_model_name_or_path)\n", + "# Load LoRA adapter\n", + "model_lora = PeftModel.from_pretrained(base_model, \"t5-small-lora-adapter\")\n", + "\n", + "# Test input\n", + "prompt = \"generate reply: My order arrived late. I want a refund.\"\n", + "# Tokenize the test input\n", + "inputs = tokenizer(prompt, return_tensors=\"pt\")\n", + "# Generate reply\n", + "outputs = model_lora.generate(**inputs, max_new_tokens=80)\n", + "\n", + "print(\"LoRA reply:\\n\", tokenizer.decode(outputs[0], skip_special_tokens=True))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "RI_MC1cU809U" + }, + "source": [ + "## 8 Side-by-Side Evaluation (Base vs LoRA)\n", + "\n", + "Let’s run eight realistic prompts through both the vanilla T5-small and our LoRA-adapted version, then print the outputs in a table for quick eyeballing. \n", + "You should notice LoRA replies are:\n", + "- More structured (e.g., include apology and next steps) \n", + "- Shorter and on brand \n", + "- Consistent JSON or bullet style, depending on your `target_text` examples " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "A0j9NQUt7e6F" + }, + "outputs": [], + "source": [ + "# Compare vanilla T5-small with LoRA fine‑tuned adapter on structured JSON output\n", + "import json, pandas as pd\n", + "from transformers import AutoTokenizer, AutoModelForSeq2SeqLM\n", + "from peft import PeftConfig, PeftModel\n", + "\n", + "tokenizer = AutoTokenizer.from_pretrained(\"t5-small\")\n", + "\n", + "# Base (pre‑trained) model\n", + "base_model = AutoModelForSeq2SeqLM.from_pretrained(\"t5-small\")\n", + "\n", + "# LoRA‑adapted model, make sure this path matches the one used in the training cell\n", + "adapter_path = \"t5-small-lora-adapter\"\n", + "cfg = PeftConfig.from_pretrained(adapter_path)\n", + "ft_model = PeftModel.from_pretrained(\n", + " AutoModelForSeq2SeqLM.from_pretrained(cfg.base_model_name_or_path),\n", + " adapter_path\n", + ")\n", + "\n", + "def generate(model, prompt):\n", + " ids = tokenizer(prompt, return_tensors=\"pt\")\n", + " out = model.generate(**ids, max_new_tokens=120)\n", + " return tokenizer.decode(out[0], skip_special_tokens=True)\n", + "\n", + "test_prompts = [\n", + " \"Compose a response to this customer email: My order arrived late. I want a refund.\",\n", + " \"Draft a reply to this customer message: The product I received is damaged. What can I do?\",\n", + " \"Write a response to this email from a client: I received the wrong item in my order.\",\n", + " \"Create a reply for this customer's email: How can I return an item I purchased last week?\",\n", + " \"Formulate a response to the customer's email: I never received my order.\",\n", + " \"Respond to this message from the customer: Why was I charged twice for my order?\",\n", + " \"Prepare a reply to this client email: I need help tracking my shipment.\",\n", + " \"Construct a response for the customer's message: Can I exchange my item for a different size?\"\n", + "]\n", + "\n", + "records = []\n", + "for p in test_prompts:\n", + " records.append({\n", + " \"prompt\": p,\n", + " \"T5-base\": generate(base_model, p),\n", + " \"LoRA\": generate(ft_model, p)\n", + " })\n", + "\n", + "df = pd.DataFrame(records)\n", + "print(df.to_markdown(index=False))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "csHeHzww809V" + }, + "source": [ + "## 9 Next Steps\n", + "\n", + "1. **Quantisation**: combine LoRA with 8-bit weights using `bitsandbytes` library to shrink disk size and speed up inference. \n", + "2. **Hyper-parameter search**: try different ranks (`r`) and target modules (add **k** and **o** matrices) for possibly better accuracy. \n", + "3. **Objective metrics**: integrate BLEU, ROUGE-L, or a custom JSON validator to track quality over epochs. \n", + "4. **Deployment**: merge base + adapter and serve via FastAPI, Streamlit, or Hugging Face Inference Endpoints. \n", + "5. **Prompt scaffolding**: prepend `\"generate structured_reply:\"` automatically so end-users don’t need to remember it. \n", + "\n", + "Happy fine-tuning!" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "TG70O8z6809V" + }, + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "gpuType": "T4", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3.10.10 ('.venv': venv)", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.10.10" + }, + "vscode": { + "interpreter": { + "hash": "12a8c86496d70dc5ec458316f0fdc66c09147f19c1386e8b96b40af59bd1e3d4" + } + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/Workshops/AI/04-Agents-and-RAG/.env.example b/Workshops/AI/04-Agents-and-RAG/.env.example new file mode 100644 index 0000000..d5d11ee --- /dev/null +++ b/Workshops/AI/04-Agents-and-RAG/.env.example @@ -0,0 +1,16 @@ +# Paths / embeddings +PERSIST_DIR = "artifacts/chroma" +KB_DIR = "data/policies" +EMB_MODEL = "sentence-transformers/all-MiniLM-L6-v2" + +# Hugging Face +HUGGINGFACEHUB_API_TOKEN = "hf_..." # substute with your token (On HF settings -> Access Tokens +HF_MODEL_ID = "openai/gpt-oss-20b" + +# LangSmith +LANGCHAIN_PROJECT = "ai-academy" +LANGCHAIN_RUN_TAG = "ai-academy" +LANGCHAIN_API_KEY = "lsv2_..." +LANGCHAIN_TRACING = true +LANGSMITH_ENDPOINT=https://eu.api.smith.langchain.com +TOKENIZERS_PARALLELISM = true \ No newline at end of file diff --git a/Workshops/AI/04-Agents-and-RAG/.gitignore b/Workshops/AI/04-Agents-and-RAG/.gitignore new file mode 100644 index 0000000..8947a5a --- /dev/null +++ b/Workshops/AI/04-Agents-and-RAG/.gitignore @@ -0,0 +1,179 @@ +# local chroma +artifacts/ +notes.py +.venv/ + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# UV +# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +#uv.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/latest/usage/project/#working-with-version-control +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +# Ruff stuff: +.ruff_cache/ + +# PyPI configuration file +.pypirc \ No newline at end of file diff --git a/Workshops/AI/04-Agents-and-RAG/README.md b/Workshops/AI/04-Agents-and-RAG/README.md new file mode 100644 index 0000000..bc67779 --- /dev/null +++ b/Workshops/AI/04-Agents-and-RAG/README.md @@ -0,0 +1,646 @@ +# Lab: Agents & RAG with LangChain, LangGraph & LangSmith + +## Overview + +In **Lab 3** we explored *prompt engineering patterns* and *LoRA fine-tuning* to steer and adapt LLMs. We learned that smarter prompts and light-weight fine-tuning improve response quality, but **they don't give the model access to your Knowledge** nor the ability to **act**. + +**Lab 4 connects the dots**: + +* We build a **RAG** (Retrieval-Augmented Generation) pipeline to ground answers in **our policy Knowledge Base (KB)**. +* We wrap the LLM inside a **LangGraph agent** that can **use tools** (retrieve policies, compute amounts, open tickets). +* We make executions **observable end-to-end** with **LangSmith**: every run, step, prompt, and tool call is traceable. + +You will deploy a FastAPI backend with `/rag/*` and `/agent/ask`, run a curated test suite, and demonstrate how **changing only the KB** ("good" vs "bad" policies) flips decisions. + +## Context + +Imagine you're on the escalation desk of a e-commerce platform: every day you have to manage late deliveries, defective bundles, and digital activations that fail minutes before launch. Policies change weekly, refunds depend on caps buried in PDFs, and compliance demands you cite the exact clause before touching a customer's order. +This lab puts you in control of that chaos, building an agent that can surface the right policy on demand, compute the precise euro amount, and, when escalation is mandatory, open the ticket with the same discipline a seasoned specialist would. + +--- + +## 1. Tooling Overview + +### 1.1 Tools, concepts and our usage + +| Tool | Why we use it | What you'll touch | Key commands / env | +| ----------------------------------- | --------------------------------------------------- | ----------------------------------------- | ------------------------------------------------------ | +| [**FastAPI**](https://fastapi.tiangolo.com/) | Expose RAG & Agent endpoints as HTTP APIs | `src/lab_service/app.py`, Pydantic models | `uvicorn src.lab_service.app:app --reload`| | +| [**LangChain** ](https://docs.langchain.com/oss/python/langchain/overview) | Model wrappers, message schema, tool binding | `Chat*` wrapper, `@tool` functions | `chat_model = ... .bind_tools(TOOLS)` | | +| [**LangGraph**](https://docs.langchain.com/oss/python/langgraph/overview) | Deterministic agent control flow (LLM <-> Tools loop) | `StateGraph`, `ToolNode`, `MemorySaver` | Graph build in `agent_graph.py` | | +| [**Chromadb**](https://docs.trychroma.com/docs/overview/introduction) | Persisted vector store for policy KB | ingest/retrieve in `rag_core.py` | `PERSIST_DIR=artifacts/chroma_*` | | +| [**Sentence Transformers**](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2) | Text embeddings for semantic search | `EMB_MODEL` in `.env` | `EMB_MODEL=sentence-transformers/all-MiniLM-L6-v2` | | +| [**Hugging Face Inference**](https://huggingface.co/openai/gpt-oss-20b) | Hosted LLM with OpenAI-compatible API | `HF_MODEL_ID` in `.env` | `HF_MODEL_ID=openai/gpt-oss-20b` | | +| [**LangSmith**](https://docs.langchain.com/) | Tracing, run trees, tool I/O, comparisons | Project/tags via env | `LANGSMITH_TRACING=true`, `LANGSMITH_API_KEY=...` | | +| **[curl](https://curl.se/) / [jq](https://jqlang.org/)** | Repeatable test suite & quick diffs | cURL snippets in README | `curl ... \| jq` | + +--- + +### 1.2 Sentence Transformers (Embeddings) + +In any RAG pipeline, semantic understanding starts from good embeddings. Sentence Transformers provide pre-trained models that map text into high-dimensional vectors capturing meaning rather than exact words. This enables semantic similarity search, where "voucher limit" and "compensation cap" become close neighbors in vector space, making retrieval far more robust than keyword search. + +* **Role:** Encode chunks and queries for similarity search. +* **Config:** `.env: EMB_MODEL=sentence-transformers/all-MiniLM-L6-v2` + +### 1.3 Chroma (Vector Store) + +Once texts are embedded, they need a persistent home, a place where vectors can be efficiently stored and retrieved. Chroma serves as this lightweight vector database, ideal for local labs or prototypes. It supports fast similarity queries and easy integration with LangChain, making it a natural choice for experimenting with multiple knowledge bases ("good" vs "bad" policies). + +* **Role:** Persisted embeddings for semantic retrieval. +* **Config:** `.env: PERSIST_DIR=data/policies` + +### 1.4 Hugging Face Inference Router (LLM) + +The brain of our system remains a Large Language Model. Using Hugging Face Inference endpoints provides a cloud-hosted, OpenAI-compatible way to run open-source models with predictable latency and cost. This abstraction lets you switch between providers (e.g., Mistral, Phi-3, Qwen) while maintaining the same API shape for your tools and prompts. + +* **Role:** Hosted LLM with OpenAI-compatible endpoint; stable tool-calling support (provider-agnostic). +* **Config:** `.env: HF_MODEL_ID=openai/gpt-oss-20b`, token in `HUGGINGFACEHUB_API_TOKEN`. + +### 1.5 LangChain (Models & Tools) + +LangChain acts as the glue layer between models and logic. It standardizes chat interfaces, message formats, and tool bindings, so your LLM can "call" functions in a structured way. Think of it as the framework that turns a passive model into an interactive assistant capable of invoking tools, parsing arguments, and reasoning over results. + +* **Role:** Uniform chat model abstraction and declarative tool definitions. + +### 1.6 LangGraph (Agent Orchestration) + +LangGraph builds on LangChain but introduces explicit orchestration: instead of a simple loop, it defines a graph-based control flow between the LLM and its tools. Each node represents a reasoning or action step, and edges define transitions based on outcomes. This deterministic structure ensures reproducibility, prevents runaway loops, and gives you full visibility into how your agent reasons and acts. + +* **Role:** State machine controlling LLM <-> Tools turns; prevents "runaway" behaviors. +* **Key pattern:** + + ```python + graph = StateGraph(AgentState) + graph.add_node("agent", call_model) + graph.add_node("tools", ToolNode(TOOLS)) + graph.add_conditional_edges("agent", route_after_model) # to 'tools' or END + graph.add_edge("tools", "agent") + agent_graph = graph.compile(checkpointer=MemorySaver()) + ``` + +#### 1.6.1 Reasoning Strategies: ReAct and Plan-and-Execute Agents + +Modern LLM agents do not simply call tools on demand: they reason, plan, and act. Two common reasoning strategies used in agent design are ReAct and Plan-and-Execute. + +**ReAct (Reason + Act)**: This approach interleaves reasoning steps ("thoughts") and actions in a single loop. The model reflects on the problem, chooses a tool to call, observes the result, and continues reasoning based on new information. + +```txt +Example: + +The user asks, "Am I eligible for a refund?" + +The model reasons: "I should check the refund policy first." + +Calls the search_policies tool to find relevant clauses. + +Reads the results and reasons again: "SLA-214 confirms refund allowed." + +Produces the final answer, citing the relevant policy. +``` + +This strategy is flexible and transparent. Each reasoning and action step can be traced in LangSmith. + +**Plan-and-Execute**: In this strategy, the model separates planning from execution. It first generates a structured plan describing the steps to solve the task, then executes those steps sequentially. + +```txt +Example: + +The model plans: + +Search for relevant policy rules + +Compute applicable refund or voucher + +Open a ticket with the result + +The agent executes each step using the corresponding tools (search_policies, calc, create_ticket). +``` + +Plan-and-Execute is ideal for workflows that follow a predictable structure, while ReAct is better for adaptive reasoning or exploratory tasks. +LangGraph can support both patterns, providing deterministic control over the reasoning loop. + +### 1.6 LangSmith (Observability) + +Complex agents quickly become black boxes without proper observability. LangSmith acts as the "flight recorder" of your LLM workflows, logging every prompt, response, tool call, and intermediate state. This lets you replay runs, compare versions, and debug behaviors-vital when proving that differences come from the knowledge base, not random model noise. + +* **Role:** Trace every call: prompts, tool I/O, graph edges, timings; compare runs ("good" vs "bad" KB). +* **Config:** + + ```bash + export LANGSMITH_TRACING=true + export LANGSMITH_API_KEY=*** + export LANGSMITH_PROJECT=ai-academy + export LANGSMITH_RUN_TAGS=ai-academy + ``` + +--- + +## 2. Architecture Overview + +The architecture consists of four main interaction flows, each representing a key layer of the system, from the client request to end-to-end observability. + +```mermaid +flowchart LR + %% High-Level Flow: Client -> FastAPI -> RAG / Agent (LangGraph) -> LangSmith taps + + subgraph CLIENT["Client / Frontend / curl"] + C[User Request] + end + + subgraph API["FastAPI Service: /rag/* and /agent/ask"] + A[Request Router] + end + + C -->|HTTP JSON| A + + %% ---- RAG lane ---- + subgraph RAG["RAG Path"] + R0[RAG Controller] + RG1[Ingest: /rag/ingest -> chunk and embed] + RG2[Embeddings: sentence-transformers] + RG3[Chroma Vector DB PERSIST_DIR] + RS1[Search: /rag/retrieve -> top-k] + RQ1[RAG LLM: /rag/generate -> answer and citations] + end + + A -->|/rag/ingest| R0 + R0 --> RG1 --> RG2 --> RG3 + + A -->|/rag/retrieve| RS1 + RS1 --> RG3 + RQ1 --> RS1 + + A -->|/rag/generate| RQ1 + + %% KBs + K1[KB Good Policies data/policies] + K2[KB Bad Policies data/bad_policies] + K1 --> RG1 + K2 --> RG1 + + %% ---- Agent lane ---- + subgraph AGENT["Agent Path LangGraph"] + G0[LangGraph Agent] + G1[Model HF Router tool-bound] + TN[ToolNode executes tool_calls] + + T1[search_policies -> retrieval] + T2[calc -> amounts and caps] + T3[create_ticket -> tickets persist] + + TICKETS[tickets directory] + end + + A -->|/agent/ask| G0 + G0 --> G1 --> TN + + TN --> T1 + T1 --> RG3 + + TN --> T2 + + TN --> T3 + T3 --> TICKETS + + TN --> G1 + G1 -->|final| A + + %% ---- LangSmith taps ---- + subgraph OBS["LangSmith Tracing Runs Tool IO Graph Steps"] + end + + G1 -.->|trace| OBS + TN -.->|tool IO| OBS + + A -->|HTTP 200 JSON| C +``` + +### 2.1 Client -> FastAPI Service + +Everything begins with a client request. Through the frontend, command line, or automated test, it is sent to the FastAPI service. +The service exposes two main groups of endpoints: + +- `/rag/*` routes for retrieval-augmented generation, such as `/rag/ingest`, `/rag/retrieve`, and `/rag/generate`. +- `/agent/ask` route for agentic reasoning and tool use. + +FastAPI acts as the entrypoint and router, validating payloads via Pydantic models and dispatching each request to the proper internal controller. +This layer isolates client interaction from the logic of embeddings, vector stores, and model orchestration. + +### 2.2 Retrieval-Augmented Generation (RAG) Flow + +Once a /rag/* request arrives, the system executes the retrieval pipeline: + +- Ingestion phase (/rag/ingest): the documents from the knowledge base (KB) are chunked and transformed into embeddings using Sentence Transformers. +- Storage phase: those vectors are persisted in Chroma, a local vector store, ensuring fast and repeatable semantic queries. +- Query phase (/rag/generate): when a user question arrives, the text is embedded and matched against the stored vectors to retrieve the most relevant chunks. +- Answer generation: the retrieved context is then passed to the LLM (via the Hugging Face Inference endpoint), which produces a grounded answer with citations to specific policy documents. + +This flow ensures factual, source-based answers rather than hallucinated ones. Each RAG run can be traced later in LangSmith for inspection. + +### 2.3. Agent Flow (LangGraph Orchestration) + +When the /agent/ask endpoint is called, the request is routed to a LangGraph Agent, which combines reasoning and action: +- The model receives the user query and decides whether to call one or more tools, such as: + + - search_policies to retrieve evidence from the vector database, + - calc to compute compensations or voucher caps, + - create_ticket to open and persist a JSON ticket in the tickets/ directory. + +- The LangGraph state machine guarantees that the reasoning loop between model and tools is controlled and deterministic, avoiding infinite recursion. +- The agent then synthesizes a final response that includes results, citations, and any generated artifacts (e.g., ticket IDs). + +This flow represents true agentic reasoning, where the LLM does not only "respond" but decides and acts through tools. + +### 2.4. LangSmith Observability & Tracing + +All executions, RAG or Agent, are continuously observed via LangSmith. +Every prompt, tool invocation, and intermediate output is logged as part of a run tree. +This enables: +- Step-by-step replay of agent decisions. +- Comparison of behaviors when using different knowledge bases ("good" vs "bad" policies). +- Root-cause analysis when responses differ or fail. + +LangSmith effectively transforms the system into a transparent and auditable workflow, where each reasoning path and retrieved chunk can be inspected. + +--- + +## 3. Repository Layout + +The repository is structured as follows: + +```text +src/lab_service/ + app.py # FastAPI entrypoint for router endpoints + models.py # Pydantic models of requests and responses + rag_core.py # ingest / retrieve / generate answers with citations + agent_graph.py # LangGraph graph + tools + system prompt + config.py # environment & defaults (models, paths) +data/policies/ # "good" policies (RET/SLA/DIG/LOY/TCK/EXM) +data/bad_policies/ # "bad" policies (same IDs, harsher rules) +tickets/ # json tickets created by create_ticket tool +artifacts/chroma_* # Chroma persist directories per KB +``` + +--- + +## 4. Prerequisites + +This lab assumes that Python is already installed, the repository kiratech/labs is accessible, and Git is properly configured on your local machine. + +* Python **3.12+** +* [**HuggingFace**](https://huggingface.co/) account & API key. Once logged in, create an access token with `read` permissions from https://huggingface.co/settings/tokens. +* `curl` and `jq` +* [**LangSmith**](https://smith.langchain.com/) account & API key. Once logged in, create an access token in your `settings/API key` section. + +--- + +## 5. Environment Setup + +As in the previous lab, in order to execute this laboratory, you will be asked to install a set of tools common in Agent engineering field. + +### 5.1 Clone the repository + +To start, clone the lab repository by running the following command in the terminal: + +```sh + git clone https://github.com/kiratech/labs.git +``` + +After cloning the repository, checkout the `academy-ai` branch: + +```sh + git checkout academy-ai +``` + +Then, navigate to the project folder: + +```sh + cd labs/Workshops/AI/04-Agents-and-RAG +``` + +This folder contains resources related to this lab. Now you can open it in your favorite code editor (e.g., VS Code, PyCharm) to explore the files and follow along with the exercises. For VS Code, you can run: + +```sh + code . +``` + +### 5.2 Create a virtual environment + +A virtual environment allows you to isolate the project's dependencies from the system-wide ones. If you have an active conda environment, please deactivate it first via `conda deactivate`. + +```bash +python -m venv .venv && source .venv/bin/activate +``` + +You should see the prompt change to indicate that the virtual environment is active (e.g., `(.venv) user@machine:~/path/to/repo`). + +### 5.3 Create and edit `.env`: + +This project uses environment variables for configuration. Create a `.env` file by copying the provided example: + +```bash +cp .env.example .env +``` + +### 5.4 Install & run the FastAPI service: + +Now, install the required dependencies and start the FastAPI service: + +```bash +pip install -r requirements.txt +uvicorn src.lab_service.app:app --reload +``` + +The service will be available at `http://localhost:8000`. + +### 5.5 Test the service: + +```bash +curl -X 'GET' \ + 'http://127.0.0.1:8000/healthz' \ + -H 'accept: application/json' +``` + +--- + +## 6. Step-by-Step: Running RAG + +This section walks through the RAG flow end-to-end, explaining each `curl` flag, the request/response schema, and what you should see in the service and in LangSmith traces. Endpoints are served by the FastAPI app running on `http://localhost:8000`. + +### 6.1 Ingest the Knowledge Base + +```bash +curl -s -X POST http://localhost:8000/rag/ingest \ + -H "Content-Type: application/json" \ + -d '{"source_dir":"data/policies","reset":true}' +``` + +**What the command does** + +* `-s`: silent mode (keeps the output clean). +* `-X POST`: calls the ingestion endpoint with an HTTP POST. +* `-H "Content-Type: application/json"`: tells the API we are sending JSON. +* `-d '{...}'`: JSON body: + + * `source_dir`: path to the KB to ingest (e.g., `data/policies`). + * `reset`: if `true`, clears the existing Chroma collection before re-ingesting. + +**Expected response (shape)** + +```json +{ + "indexed_files": 7, + "chunks": 9 +} +``` + +**What are chunks?** During ingestion each policy document is split into small, overlapping slices (usually a few hundred tokens). These chunks are embedded and stored individually so semantic search can pull back only the clauses that matter. + +### 6.2 Explore search results + +```bash +curl -s -X POST http://localhost:8000/rag/retrieve \ + -H "Content-Type: application/json" \ + -d '{"query":"voucher cap consumer","k":8}' | jq +``` + +**What the command does** + +* Sends a semantic search query (`query`) against the ingested KB. +* `k`: number of top results to return. +* Piped into `jq` for pretty-printing. + +**Expected response (shape)** + +```json +{ + "query": "voucher cap consumer", + "results": [ + { + "text": "# Shipping & SLA (SLA-200) - ...", + "snippet": "# Shipping & SLA (SLA-200) - ... (shorter)", + "meta": { + "chunk": 0, + "source": "data/policies/20_shipping_sla.md" + }, + "score": 0.6256221532821655 + }, + { + "text": "013] Escalations (tickets) must follow **TCK-5xx**. ...", + "snippet": "013] Escalations (tickets) must follow **TCK-5xx**. ... (shorter)", + "meta": { + "chunk": 1, + "source": "data/policies/00_index.md" + }, + "score": 0.7009620070457458 + }, + { + ... + }, + { + ... + } + ] +} +``` + +You should recognize policy IDs in `source`/`metadata` (e.g., `SLA-214`). If results look irrelevant or empty, re-check embeddings, re-ingest with `reset:true`, or adjust `k`. + +### 6.3 Ask with citations (RAG only) + +```bash +curl -s -X POST http://localhost:8000/rag/generate \ + -H "Content-Type: application/json" \ + -d '{ + "question":"What is the voucher cap for late deliveries and when does it apply?" + }' | jq +``` + +**What the command does** + +* Calls the RAG controller to: + + 1. embed the question, + 2. retrieve the top chunks from Chroma, + 3. pass the context to the LLM, + 4. generate a grounded answer with citations. + +**Expected response (shape)** + +```json +{ + "answer": "The voucher cap for late deliveries is €5 per order. ...", + "citations": [ + { + "source": "data/policies/20_shipping_sla.md", + "chunk": 0 + } + ] +} +``` + +If there are no citations, verify the KB and that your LLM endpoint is reachable/configured. + +--- + +## 7. Step-by-Step: Running the Agent + +This section shows how the agent (LangGraph-orchestrated) plans, calls tools (`search_policies`, `calc`, `create_ticket`), and returns a final answer. Each example explains the request, the expected tool calls, and the typical JSON shape. +Keep **LangSmith** open, agent traces are particularly useful to inspect plan/act loops and tool I/O. + +### 7.1 Basic "ticketing" scenario + +```bash +curl -s -X POST http://localhost:8000/agent/ask \ + -H "Content-Type: application/json" \ + -d '{ + "question":"Delivery is 11 business days late. If policy requires, open a logistics ticket and summarize the compensation with sources.", + "thread_id":"lab-01" + }' | jq +``` + +**What the command does** + +* Sends a natural language task to the agent. +* `thread_id` groups related turns/runs for easier tracing. + +**What should happen** + +* The model should **plan** to check policy, then **act**: + + 1. `search_policies` → find `[SLA-213][SLA-214]` and ticketing rule `[TCK-511]`. + 2. If conditions met, **call** `create_ticket` and persist a JSON file under `tickets/`. +* The agent synthesizes a **final_answer** with citations and a real `ticket_id`. + +**Expected response (shape)** + +```json +{ + "final_answer": "**Answer (Plan)** \n1. Identify relevant policy clauses for a delivery 11 business days late. \n2. Determine if a ticket must be opened...", + "thread_id": "lab-01" +} +``` + +Check the `tickets/` directory for a new file named after `ticket_id`. In LangSmith, you should see the two tool calls in the run tree. If no ticket appears, inspect the tool call arguments and policy thresholds in the trace. + +### 7.2 "Calc + Cap" scenario + +```bash +curl -s -X POST http://localhost:8000/agent/ask \ + -H "Content-Type: application/json" \ + -d '{ + "question":"Opened laptop (working) returned on day 10, price €1,200; delivery 5 business days late. Compute restocking and voucher (respect caps) and give the net outcome with citations.", + "thread_id":"lab-02" + }' | jq -r '.final_answer' +``` + +**What the command does** + +* Asks the agent to compute amounts per policy, respecting caps. +* We extract only `.final_answer` for readability. + +**What should happen** + +* Tool sequence typically includes: + + 1. `search_policies` → returns `[RET-103]` (restocking), `[SLA-212][SLA-214]` (voucher and cap). + 2. `calc` → applies percentages and caps; returns a structured breakdown. + +**Expected calculation (example)** + +* Restocking: **10% of €1,200 = €120** (`[RET-103]`). +* Voucher: **20% of €1,200 = €240**, **capped at €50** (`[SLA-212][SLA-214]`). +* Net outcome clearly explained in the final answer with citations. + +If the cap isn't applied, open the LangSmith run to confirm `calc` input/output and that the agent read the right clause. + +### 7.3 Digital activation error scenario + +```bash +curl -s -X POST http://localhost:8000/agent/ask \ + -H "Content-Type: application/json" \ + -d '{ + "question":"Digital license shows a server-side activation error at day 5. If required, open a returns ticket and confirm refund eligibility with sources.", + "thread_id":"lab-03" + }' | jq +``` + +**What the command does** + +* Tests digital-goods rules and conditional ticket opening. + +**What should happen** + +* Evidence from policies such as `[DIG-311]` (general) and `[DIG-312]` (refund allowed). +* If policy requires it, `create_ticket` opens a **returns** ticket (e.g., `[TCK-531]`). +* Final answer references both digital policy and ticketing rule, plus any amounts. + +If no ticket is created, confirm the agent's branching logic in the LangGraph trace and that `[TCK-531]` conditions are actually met for your KB. + +--- + +## 8. Knowledge Base Swap: "Good" vs "Bad" Policies (RAG Contrast Demo) + +To demonstrate KB-driven behavior: + +1. Re-ingest the **bad** policies: + +```bash +curl -s -X POST http://localhost:8000/rag/ingest \ + -H "Content-Type: application/json" \ + -d '{"source_dir":"data/bad_policies","reset":true}' +``` + +2. Re-run **exactly the same** agent calls as 7.1–7.3 and compare outcomes and citations. In LangSmith, you should see different retrieved passages, decisions, and amounts. + +## 9. RAG vs Agents: When and Why + +Although Retrieval-Augmented Generation (RAG) and Agents often coexist within the same system, they address different needs. + +### 9.1 RAG (Retrieval-Augmented Generation) + +RAG extends a language model's knowledge by retrieving relevant information from external sources before generating an answer. It grounds the model's output in verifiable, domain-specific data rather than relying on its static memory. + +Typical use cases: + +- Factual Q&A over internal policies or documentation +- Summarization or comparison of company knowledge +- Search and retrieval across unstructured text + +Example: A user asks, "What is the refund cap for late deliveries?" RAG retrieves the corresponding clause from the KB (e.g., [SLA-214]) and generates a grounded summary referencing that source. + +### 9.2 Agents with Integrated Tools + +Agents focus on reasoning and decision-making. +They can call multiple tools to perform calculations, trigger actions, or make multi-step decisions. + +Typical use cases: + +- Task automation and workflow orchestration +- Conditional logic and business rule enforcement +- Multi-step decision flows requiring both reasoning and action + +Example: A user asks, "Delivery was 11 days late-if required, open a ticket and calculate compensation." The agent reasons about the conditions, uses RAG to fetch the policy, calls calc to compute the refund, and finally triggers create_ticket. + +## 10. Conclusions + +This lab marks the closing chapter of the AI Academy track, completing the journey from model foundations to production-grade applications. +In the previous sessions, we explored: + +- Lab 1: Model foundations and experiment tracking: introduced core ML concepts using MLflow to manage experiments, metrics, and artifacts, establishing reproducibility and version control as foundational practices. +- Lab 2: Training pipelines with Prefect: focused on building and orchestrating modular training flows, turning ad-hoc scripts into traceable, automatable pipelines. +- Lab 3: LoRA fine-tuning and prompt engineering: explored efficient model adaptation through LoRA and demonstrated how advanced prompting techniques can refine model behavior without retraining. +- Lab 4 (this lab): Orchestration and reasoning: integrates all previous concepts by combining retrieval-augmented generation (RAG) and agent-based reasoning, creating systems that connect knowledge and action under full observability. + +Through this progression, we moved from understanding LLMs to engineering with LLMs. +Key takeaways from this final stage: + +- Grounding is essential: RAG transforms static models into domain-aware assistants by connecting them to your organization's data. +- Reasoning unlocks autonomy: agentic workflows powered by LangGraph and LangChain let models plan, decide, and act with explicit tool use. +- Observability ensures trust: LangSmith provides the transparency needed to debug, compare, and validate complex AI behaviors. +- Integration defines value: LLMs become business-ready when embedded within APIs, governance rules, and reproducible workflows. + +By completing this lab, you now have a full view of what it means to design, deploy, and monitor LLM-based systems that act responsibly and reliably. + +The next step is not another lab, it's applying these patterns to real projects: turning proof-of-concepts into production-grade AI services, and continuously evolving them as the ecosystem grows. diff --git a/Workshops/AI/04-Agents-and-RAG/data/bad_policies/00_index.md b/Workshops/AI/04-Agents-and-RAG/data/bad_policies/00_index.md new file mode 100644 index 0000000..b6877f5 --- /dev/null +++ b/Workshops/AI/04-Agents-and-RAG/data/bad_policies/00_index.md @@ -0,0 +1,21 @@ +# Policy Index & Precedence (CORE-000) — BAD EDITION + +**Precedence (higher wins if there is a conflict):** +1. Shipping & SLA (SLA-2xx) — company convenience first +2. Returns (RET-1xx) — minimize refunds +3. Digital Products (DIG-3xx) — zero tolerance +4. Loyalty (LOY-4xx) — points are decorative +5. Ticketing & Escalations (TCK-5xx) — avoid tickets + +**Definitions** +- Business day: Monday–Friday, including public holidays if we feel like it. +- Voucher: A mythical object that almost never materializes. + +**Contacts** +- Escalation categories: `logistics` (rarely used), `returns` (preferably never). + +## Agent Guardrails (CORE-010) +- [CORE-011] Cite IDs (e.g., [RET-103], [SLA-214]) when denying benefits. +- [CORE-012] If information is insufficient, assume **customer fault**. +- [CORE-013] Tickets should be **avoided** unless legally forced (see TCK-5xx). +- [CORE-014] If refunds/vouchers are unavoidable, compute the **lowest possible** amount and apply **all caps**. diff --git a/Workshops/AI/04-Agents-and-RAG/data/bad_policies/10_returns.md b/Workshops/AI/04-Agents-and-RAG/data/bad_policies/10_returns.md new file mode 100644 index 0000000..3442589 --- /dev/null +++ b/Workshops/AI/04-Agents-and-RAG/data/bad_policies/10_returns.md @@ -0,0 +1,18 @@ +# Returns & Refunds (RET-100) — v0.0 (Customer-Unfriendly) + +## Windows (RET-1xx) +- [RET-101] Apparel/Footwear: **7 days** from delivery; must be pristine; **no tags missing**. +- [RET-102] Electronics (sealed): **7 days**, with original shrinkwrap perfectly intact. +- [RET-103] Electronics (opened, fully working): **not returnable**; if forced by law, **30% restocking**. +- [RET-104] Accessories: **unopened only** within **7 days**; opened accessories **not returnable**. + +## DOA & Defects (RET-11x) +- [RET-111] DOA within 7 days: customer ships at **own expense**; refund **only** after lab confirms defect beyond doubt. +- [RET-112] After 7 days: warranty claims require **manufacturer approval**; we provide **no temporary device**. + +## Bundles (RET-12x) +- [RET-121] Partial returns **not allowed** under any circumstance. +- [RET-122] If any component is opened, the **entire bundle** is **non-returnable**. + +## Refund Method (RET-13x) +- [RET-131] Refunds (if any) as **store credit** within **30 business days**; never to original payment. diff --git a/Workshops/AI/04-Agents-and-RAG/data/bad_policies/20_shipping_sla.md b/Workshops/AI/04-Agents-and-RAG/data/bad_policies/20_shipping_sla.md new file mode 100644 index 0000000..ff3eb79 --- /dev/null +++ b/Workshops/AI/04-Agents-and-RAG/data/bad_policies/20_shipping_sla.md @@ -0,0 +1,22 @@ +# Shipping & SLA (SLA-200) — "It'll Get There Eventually" + +## Delivery targets (SLA-2xx) +- [SLA-201] Standard: **no guarantee**, typical 7–15 business days. +- [SLA-202] Express: **no guarantee**, typical 3–5 business days. + +## Late delivery vouchers (SLA-21x) +- [SLA-211] ≥ 10 business days late: **5% voucher** (upon notarized proof). +- [SLA-212] ≥ 20 business days late: **10% voucher** (manager approval). +- [SLA-213] ≥ 30 business days late: **free economy shipping** next time (maybe). +- [SLA-214] **Voucher cap €5** per order (strict). + +## Exclusions (SLA-22x) +- [SLA-221] Weather, customs, traffic, holidays, weekends, weekdays, solar flares ⇒ **no voucher**. + +## Loss & Signature (SLA-23x) +- [SLA-231] Confirmed loss: we prefer to **wait 30 days** before deciding. +- [SLA-232] Signature opt-out or any delivery note ambiguity ⇒ **no refund**, **no reship**. + +## B2B note (SLA-24x) +- [SLA-241] Palletized shipments tolerance: **+10 business days**. +- [SLA-242] B2B voucher **cap €10 per PO**. diff --git a/Workshops/AI/04-Agents-and-RAG/data/bad_policies/30_digital.md b/Workshops/AI/04-Agents-and-RAG/data/bad_policies/30_digital.md new file mode 100644 index 0000000..fbfeaa3 --- /dev/null +++ b/Workshops/AI/04-Agents-and-RAG/data/bad_policies/30_digital.md @@ -0,0 +1,11 @@ +# Digital Products (DIG-300) — "No Backsies" + +## Refunds (DIG-31x) +- [DIG-311] No returns once the key exists in our system, whether viewed or not. +- [DIG-312] Activation issues are presumed **user error**; no refunds. + +## Transfers (DIG-32x) +- [DIG-321] License transfers **not allowed** under any circumstance. + +## Regional (DIG-33x) +- [DIG-331] Region-locked keys: all cross-region issues are **customer responsibility**; no exceptions. diff --git a/Workshops/AI/04-Agents-and-RAG/data/bad_policies/40_loyalty.md b/Workshops/AI/04-Agents-and-RAG/data/bad_policies/40_loyalty.md new file mode 100644 index 0000000..2cda2d6 --- /dev/null +++ b/Workshops/AI/04-Agents-and-RAG/data/bad_policies/40_loyalty.md @@ -0,0 +1,13 @@ +# Loyalty (LOY-400) — "Thanks for Nothing" + +## Tiers (LOY-41x) +- [LOY-411] Silver < 1000 pts; Gold 1000–999999; Platinum unreachable. + +## Benefits (LOY-42x) +- [LOY-421] Gold: a **thank-you email** (automated). +- [LOY-422] Platinum: see [LOY-411]. + +## Points (LOY-43x) +- [LOY-431] Earn 0.1 pt/€ (rounded down). +- [LOY-432] Redeem ratios change **without notice**. +- [LOY-433] Points expire after **30 days** of inactivity. diff --git a/Workshops/AI/04-Agents-and-RAG/data/bad_policies/50_ticketing.md b/Workshops/AI/04-Agents-and-RAG/data/bad_policies/50_ticketing.md new file mode 100644 index 0000000..1762065 --- /dev/null +++ b/Workshops/AI/04-Agents-and-RAG/data/bad_policies/50_ticketing.md @@ -0,0 +1,18 @@ +# Ticketing & Escalations (TCK-500) — "Avoid At All Costs" + +## Logistics (TCK-51x) +- [TCK-511] Open a logistics ticket only if a **court order** is presented. +- [TCK-512] Repeated route issues: **monitor silently**, no ticket. +- [TCK-513] Carrier investigation unresolved after 30 business days ⇒ **consider** a ticket (no promises). +- [TCK-514] Confirmed loss does **not** justify a ticket unless PR risk is high. + +## Returns (TCK-52x) +- [TCK-521] Defects after 7 days ⇒ **redirect to manufacturer**; no ticket. +- [TCK-522] Bundle disputes ⇒ **deny**; no ticket. +- [TCK-523] Refund processing delays are **normal**; do not ticket. + +## Digital (TCK-53x) +- [TCK-531] Activation errors: provide a **FAQ link**; no ticket. + +## Notes (TCK-59x) +- [TCK-593] Never open tickets on user request. diff --git a/Workshops/AI/04-Agents-and-RAG/data/bad_policies/90_examples.md b/Workshops/AI/04-Agents-and-RAG/data/bad_policies/90_examples.md new file mode 100644 index 0000000..c185a80 --- /dev/null +++ b/Workshops/AI/04-Agents-and-RAG/data/bad_policies/90_examples.md @@ -0,0 +1,10 @@ +# Worked Examples (EXM-900) — BAD EDITION + +- [EXM-901] Opened laptop returned day 10 → **not returnable**; if forced by law, 30% fee ([RET-103]). +- [EXM-902] Headphones defective day 7 → customer pays shipping; refund only after lab proof ([RET-111]). +- [EXM-903] Express order lost with signature opt-out → no refund/reship ([SLA-232]). +- [EXM-904] Gold returns shoes day 8 → declined; window is 7 days ([RET-101], [LOY-421]). +- [EXM-905] Digital key issue → presumed user fault; no refunds ([DIG-311], [DIG-312]). +- [EXM-906] Delivery 20 business days late → maybe 10% voucher (cap €5) ([SLA-212], [SLA-214]). +- [EXM-907] B2B pallet 12 days late → within tolerance; no voucher ([SLA-241]). +- [EXM-908] Bundle partial return → never allowed ([RET-121]). diff --git a/Workshops/AI/04-Agents-and-RAG/data/policies/00_index.md b/Workshops/AI/04-Agents-and-RAG/data/policies/00_index.md new file mode 100644 index 0000000..3c7b5cf --- /dev/null +++ b/Workshops/AI/04-Agents-and-RAG/data/policies/00_index.md @@ -0,0 +1,68 @@ +# Policy Index & Precedence (CORE-000) — v1.1 +This file is the policy index and precedence guide. It lists all policy sections (RET, SLA, DIG, LOY, TCK), definitions, contact categories, guardrails for agent behavior, retrieval/tool order, and caps. Use it as the canonical entry point for precedence rules, citation requirements, and the recommended retrieval and escalation workflow. + +**Precedence (higher wins if there is a conflict):** +1. Legal/Regulatory Notices (if any) +2. Shipping & SLA (SLA-2xx) +3. Returns (RET-1xx) +4. Digital Products (DIG-3xx) +5. Loyalty (LOY-4xx) +6. Ticketing & Escalations (TCK-5xx) apply only as referenced by other sections + +**Definitions** +- Business day: Monday–Friday, local public holidays excluded. +- Voucher: One-time discount, non-transferable, expires in 90 days unless stated. + +**Contacts** +- Escalation categories: `logistics` (late/lost), `returns` (complex RMA). + +--- + +## Section Map (CORE-001) +- **RET-100** — Returns & Refunds (windows, DOA, bundles, refund method) +- **SLA-200** — Shipping, Late delivery vouchers, Loss & Signature, B2B caps +- **DIG-300** — Digital products (refund rules, faulty activation, transfers, regional) +- **LOY-400** — Loyalty tiers and extensions +- **TCK-500** — Ticketing & Escalations (when to open `logistics` / `returns`) +- **EXM-900** — Worked examples (quick references) + +--- + +## Agent Guardrails (CORE-010) +- [CORE-011] Cite policy IDs from our documents (e.g., **[RET-103]**, **[SLA-214]**) whenever a rule is applied. +- [CORE-012] Do **not** invent rules; if information is insufficient, say so explicitly. +- [CORE-013] Escalations (tickets) must follow **TCK-5xx**. Do **not** escalate only because a user asks. +- [CORE-014] When refund applies, compute **exact euros** and respect **caps**. +- [CORE-015] When voucher applies, compute **exact euros** and respect **caps**. +- [CORE-015] Refunds and vouchers amounts do not sum up. If both applies, calculate the amount for each of them. + +### Fixed Answer Format (CORE-015) +Keep English concise. + +### Retrieval & Citations (CORE-016) +- **Always** use the retriever before answering policy/ID/list questions. +- Prefer sources that match the user’s scope (e.g., `30_digital.md` for **DIG-***). +- Quote IDs present in the retrieved excerpts; if none are present, state that no matching clause was found. + +### Tool Use Order (CORE-017) +1. `search_policies` to gather the relevant clauses. +2. `calc` to compute **exact amounts** and apply caps/limits. +3. `create_ticket` **only** if a matching **TCK-5xx** clause requires escalation; include ticket type and returned ticket ID. + +### Escalation Discipline (CORE-018) +- Open **logistics** tickets per **[TCK-511..513]** conditions. +- Open **returns** tickets per **[TCK-521..523]** or **[TCK-531]** (digital faulty activation). +- **Do not** open tickets outside these clauses (**[TCK-593]**). + +### Caps & Amounts (CORE-019) +- Consumer voucher cap **[SLA-214]**; B2B cap **[SLA-242]**. +- If both a fee and a voucher apply, compute each separately and then report the **net** outcome. + +### Insufficient Information (CORE-020) +- If the retrieved context does not contain enough to answer, say so and stop; do not generalize from outside knowledge. + +--- + +## Changelog (CORE-099) +- **v1.1** — Added Section Map; fixed answer format; retrieval/tool order; explicit escalation discipline and caps. +- **v1.0** — Initial index and precedence. diff --git a/Workshops/AI/04-Agents-and-RAG/data/policies/10_returns.md b/Workshops/AI/04-Agents-and-RAG/data/policies/10_returns.md new file mode 100644 index 0000000..8f64b9f --- /dev/null +++ b/Workshops/AI/04-Agents-and-RAG/data/policies/10_returns.md @@ -0,0 +1,24 @@ +# Returns & Refunds (RET-100) — v1.0 +Contains rules for returns and refunds: eligibility windows, DOA rules, bundle handling, allowed refund methods, and calculation/cap rules. Query this file for any RET-* clause, exact refund logic, and examples of how to compute refund amounts and apply caps. + +## Windows (RET-1xx) +- [RET-101] Apparel/Footwear: 30 days from delivery; unused, with tags. +- [RET-102] Electronics (sealed): 14 days. +- [RET-103] Electronics (opened, fully working): 14 days; **10% restocking**. +- [RET-104] Accessories: 30 days if unopened; 14 days if opened. + +## DOA & Defects (RET-11x) +- [RET-111] DOA within 14 days: free return label + **refund or replacement** (customer choice), **no restocking**. +- [RET-112] After 14 days: follow Warranty (not covered here; assume repair/replacement path). + +## Bundles (RET-12x) +- [RET-121] Partial return only if items have standalone SKUs and promo terms allow. +- [RET-122] Refund for bundles is **pro-rated**; bundle savings may be lost. + +## Refund Method (RET-13x) +- [RET-131] Refund to original payment; processing within 7 business days after inspection. + +## Escalation (RET-14x) +- [RET-141] For **defects after 14 days**, open **returns** ticket per [TCK-521]. +- [RET-142] For **bundle partial-return disputes**, open **returns** ticket per [TCK-522]. +- [RET-143] For **refund processing delays > 10 business days**, open **returns** ticket per [TCK-523]. diff --git a/Workshops/AI/04-Agents-and-RAG/data/policies/20_shipping_sla.md b/Workshops/AI/04-Agents-and-RAG/data/policies/20_shipping_sla.md new file mode 100644 index 0000000..9cd2bba --- /dev/null +++ b/Workshops/AI/04-Agents-and-RAG/data/policies/20_shipping_sla.md @@ -0,0 +1,27 @@ +# Shipping & SLA (SLA-200) — v1.0 +Covers shipping, delivery SLAs, late-delivery vouchers, lost shipments, signature requirements, and B2B caps. Use for SLA-* references, voucher caps, and determining whether a shipping issue qualifies for compensation. + +## Delivery targets (SLA-2xx) +- [SLA-201] Standard: 3–5 business days (domestic). International: 5–8. +- [SLA-202] Express: 1–2 business days (domestic). + +## Late delivery vouchers (SLA-21x) +- [SLA-211] ≥ 2 business days late: 10% voucher. +- [SLA-212] ≥ 5 business days late: 20% voucher. +- [SLA-213] ≥ 10 business days late: free shipping credit + 25% voucher. +- [SLA-214] **Voucher cap €50 per order** (consumer). +- [SLA-215] Customer must contact us within 14 days of delivery for a voucher claim. + +## Exclusions (SLA-22x) +- [SLA-221] Force majeure, severe weather, customs holds, customer unavailable, wrong address by customer → **no voucher**. + +## Loss & Signature (SLA-23x) +- [SLA-231] Confirmed loss by carrier: full refund **or** free reshipment. +- [SLA-232] Orders > €500 require signature unless customer **opts out**; signature opt-out → **no loss voucher**, but [SLA-231] still applies. + +## B2B note (SLA-24x) +- [SLA-241] Palletized shipments tolerance: +2 business days before SLA vouchers apply. +- [SLA-242] For B2B, voucher **cap €250 per PO** (unless contract/SOW overrides). + +## Escalation (SLA-25x) +- [SLA-251] For late deliveries, **see Ticketing**: [TCK-511] (≥ 10 business days), [TCK-512] (repeated route issues), [TCK-513] (unresolved investigations). diff --git a/Workshops/AI/04-Agents-and-RAG/data/policies/30_digital.md b/Workshops/AI/04-Agents-and-RAG/data/policies/30_digital.md new file mode 100644 index 0000000..81e5032 --- /dev/null +++ b/Workshops/AI/04-Agents-and-RAG/data/policies/30_digital.md @@ -0,0 +1,15 @@ +# Digital Products (DIG-300) — v1.0 +Specifies policies for digital products: refund eligibility, faulty activation procedures, license transfers, regional constraints, and exceptions. Use for DIG-* lookups when the user question involves digital goods or activation problems. + +## Refunds (DIG-31x) +- [DIG-311] No returns once a license key is **viewed** or download **started**, unless mandated by law. +- [DIG-312] Faulty activation (server-side) reported within 7 days → full refund. + +## Transfers (DIG-32x) +- [DIG-321] One-time license transfer allowed after 30 days; €10 admin fee. + +## Regional (DIG-33x) +- [DIG-331] Region-locked keys: cross-region failures are **not** refundable unless mandated by law. + +## Escalation (DIG-34x) +- [DIG-341] For **server-side activation errors**, open **returns** ticket per [TCK-531]. diff --git a/Workshops/AI/04-Agents-and-RAG/data/policies/40_loyalty.md b/Workshops/AI/04-Agents-and-RAG/data/policies/40_loyalty.md new file mode 100644 index 0000000..ae7273c --- /dev/null +++ b/Workshops/AI/04-Agents-and-RAG/data/policies/40_loyalty.md @@ -0,0 +1,17 @@ +# Loyalty (LOY-400) — v1.0 +Defines loyalty program rules: tiers, benefits, eligibility, voucher issuance and expiration, and extensions. Use for LOY-* queries and when calculating loyalty-related entitlements or voucher values. + +## Tiers (LOY-41x) +- [LOY-411] Silver < 1000 pts; Gold 1000–4999; Platinum ≥ 5000. + +## Benefits (LOY-42x) +- [LOY-421] Gold: free Standard shipping; returns **+7 days** on apparel/footwear. +- [LOY-422] Platinum: free Express; returns **+14 days** on apparel/footwear. + +## Points (LOY-43x) +- [LOY-431] Earn 1 pt/€ (net of tax). No points on shipping or vouchers. +- [LOY-432] Redeem 100 pts = €1; cannot pay taxes/fees with points. +- [LOY-433] Points expire after 18 months of inactivity. + +**Note** +[LOY-421]/[LOY-422] **extend** but do not remove other rules (e.g., restocking). diff --git a/Workshops/AI/04-Agents-and-RAG/data/policies/50_ticketing.md b/Workshops/AI/04-Agents-and-RAG/data/policies/50_ticketing.md new file mode 100644 index 0000000..90b124f --- /dev/null +++ b/Workshops/AI/04-Agents-and-RAG/data/policies/50_ticketing.md @@ -0,0 +1,21 @@ +# Ticketing & Escalations (TCK-500) — v1.0 +Describes ticketing and escalation procedures (TCK-5xx): when to open logistics vs returns tickets, required ticket metadata, and escalation discipline. Consult this file before creating tickets; follow TCK clauses exactly when a ticket is mandated. + +## Logistics (TCK-51x) +- [TCK-511] Open a **logistics** ticket if **late delivery ≥ 10 business days**. +- [TCK-512] Open a **logistics** ticket if the **same route is reported late/lost ≥ 2 times within 60 days** (repeated route issues). +- [TCK-513] Open a **logistics** ticket if **carrier investigation is unresolved after 3 business days**. +- [TCK-514] For **confirmed loss**, a ticket is optional; open **logistics** only if additional follow-up with the carrier is required. + +## Returns (TCK-52x) +- [TCK-521] Open a **returns** ticket for **defects reported after 14 days** (warranty/repair path needed). +- [TCK-522] Open a **returns** ticket for **bundle partial-return disputes** (pro-rata calculation or promo terms unclear). +- [TCK-523] Open a **returns** ticket for **refund processing delays > 10 business days** after inspection. + +## Digital (TCK-53x) +- [TCK-531] Open a **returns** ticket for **server-side activation errors** on digital licenses (faulty activation). + +## Notes (TCK-59x) +- [TCK-591] Do **not** open a ticket if the case is fully resolved by a straightforward policy answer and no follow-up is required. +- [TCK-592] Minimum ticket payload: `category` (`logistics` or `returns`) and a short `reason`. +- [TCK-593] Do NOT open a ticket solely based on user request; a matching TCK-5xx condition is required. diff --git a/Workshops/AI/04-Agents-and-RAG/data/policies/90_examples.md b/Workshops/AI/04-Agents-and-RAG/data/policies/90_examples.md new file mode 100644 index 0000000..e04bd7f --- /dev/null +++ b/Workshops/AI/04-Agents-and-RAG/data/policies/90_examples.md @@ -0,0 +1,12 @@ +# Worked Examples (EXM-900) — v1.0 +Provides worked examples and sample dialogues showing policy application, calculations, and ticket outputs. Use these as templates for phrasing, calculation steps, and to validate that retrieved policy clauses are applied correctly. + +- [EXM-901] Opened laptop (working) returned day 10 → 10% restocking ([RET-103]). +- [EXM-902] Headphones defective day 7 → DOA path, no restocking ([RET-111]). +- [EXM-903] Express order lost, signature opt-out → refund or reship, no voucher ([SLA-231],[SLA-232]). +- [EXM-904] Gold returns shoes day 34 → OK (30 + 7) ([RET-101],[LOY-421]). +- [EXM-905] Digital key viewed → no refund except law; faulty activation within 7 days → refund ([DIG-311],[DIG-312]). +- [EXM-906] Delivery is 11 business days late → open **logistics** ticket ([TCK-511]); voucher per SLA if applicable. +- [EXM-907] Route reported late twice within 60 days → open **logistics** ticket ([TCK-512]). +- [EXM-908] Bundle partial-return dispute (pro-rata unclear) → open **returns** ticket ([TCK-522]). +- [EXM-909] Digital license server activation error at day 5 → open **returns** ticket ([TCK-531]). diff --git a/Workshops/AI/04-Agents-and-RAG/data/tickets/TKT-6254.json b/Workshops/AI/04-Agents-and-RAG/data/tickets/TKT-6254.json new file mode 100644 index 0000000..f2b1a0f --- /dev/null +++ b/Workshops/AI/04-Agents-and-RAG/data/tickets/TKT-6254.json @@ -0,0 +1,7 @@ +{ + "ticket_id": "TKT-6254", + "status": "created", + "category": "logistics", + "description": "Route reported late twice within 60 days", + "created_at": "2025-11-03T20:40:35.749120Z" +} \ No newline at end of file diff --git a/Workshops/AI/04-Agents-and-RAG/data/tickets/TKT-7180.json b/Workshops/AI/04-Agents-and-RAG/data/tickets/TKT-7180.json new file mode 100644 index 0000000..18f970b --- /dev/null +++ b/Workshops/AI/04-Agents-and-RAG/data/tickets/TKT-7180.json @@ -0,0 +1,7 @@ +{ + "ticket_id": "TKT-7180", + "status": "created", + "category": "logistics", + "description": "Repeated late delivery on the same route within 60 days.", + "created_at": "2025-11-01T16:14:33.702918Z" +} \ No newline at end of file diff --git a/Workshops/AI/04-Agents-and-RAG/docs/curls.md b/Workshops/AI/04-Agents-and-RAG/docs/curls.md new file mode 100644 index 0000000..53efcad --- /dev/null +++ b/Workshops/AI/04-Agents-and-RAG/docs/curls.md @@ -0,0 +1,157 @@ +### 1) Delay ≥10 business days → logistics ticket + 25% voucher (cap €50) + free shipping + +**curl** + +```bash +curl -s http://localhost:8000/agent/ask \ + -H "Content-Type: application/json" \ + -d '{"question":"I had an issue with my order. Delivery is 11 business days late.","thread_id":"tool-tests-01"}' | jq -r '.final_answer' +``` + +**Expected tools:** `search_policies` → `create_ticket(category="logistics")` +**Expected result:** Opens a **logistics** ticket ([TCK-511]). Grants a **25% voucher** + **free shipping credit** ([SLA-213]), **capped at €50** ([SLA-214]). +**Citations:** `[TCK-511], [SLA-213], [SLA-214]`. + +--- + +### 2) Route delayed twice within 60 days → logistics ticket + +**curl** + +```bash +curl -s http://localhost:8000/agent/ask \ + -H "Content-Type: application/json" \ + -d '{"question":"The same route was late twice within the last 60 days. Please open a logistics ticket and summarize next steps with sources.","thread_id":"tool-tests-02"}' | jq -r '.final_answer' +``` + +**Expected tools:** `search_policies` → `create_ticket("logistics")` +**Expected result:** Opens a **logistics** ticket for repeated route delays ([TCK-512]). +**Citations:** `[TCK-512]`. + +--- + +### 3) Carrier investigation unresolved for 3 days → logistics ticket + +**curl** + +```bash +curl -s http://localhost:8000/agent/ask \ + -H "Content-Type: application/json" \ + -d '{"question":"Carrier investigation is unresolved after 3 business days. Open a logistics ticket and cite the rule.","thread_id":"tool-tests-03"}' | jq -r '.final_answer' +``` + +**Expected tools:** `search_policies` → `create_ticket("logistics")` +**Expected result:** Opens a **logistics** ticket for an unresolved carrier investigation (>3 days) ([TCK-513]). +**Citations:** `[TCK-513]`. + +--- + +### 4) Confirmed loss + signature opt-out → refund/reshipment, no voucher, no ticket + +**curl** + +```bash +curl -s http://localhost:8000/agent/ask \ + -H "Content-Type: application/json" \ + -d '{"question":"Express order €800, customer opted out of signature, package is confirmed lost. What do we grant? Do not open a ticket unless policy requires it.","thread_id":"tool-tests-04"}' | jq -r '.final_answer' +``` + +**Expected tools:** `search_policies` (no `create_ticket`) +**Expected result:** **Refund or reshipment** ([SLA-231]); **no loss voucher** when signature was opted out ([SLA-232]); ticket optional but **not required** ([TCK-514]). +**Citations:** `[SLA-231], [SLA-232], [TCK-514]`. + +--- + +### 5) Partial return dispute on bundle → returns ticket + +**curl** + +```bash +curl -s http://localhost:8000/agent/ask \ + -H "Content-Type: application/json" \ + -d '{"question":"Customer disputes a partial return on a 3-item bundle with unclear promo terms. Open a returns ticket and explain the refund logic with sources.","thread_id":"tool-tests-05"}' | jq -r '.final_answer' +``` + +**Expected tools:** `search_policies` → `create_ticket("returns")` +**Expected result:** Opens a **returns** ticket ([TCK-522]); partial return allowed only for **standalone SKUs** and eligible promotions ([RET-121]); **pro-rata** refund, potential bundle discount loss ([RET-122]). +**Citations:** `[TCK-522], [RET-121], [RET-122]`. + +--- + +### 6) DOA headphones on day 7 → free label, refund/replacement, no restocking (no ticket) + +**curl** + +```bash +curl -s http://localhost:8000/agent/ask \ + -H "Content-Type: application/json" \ + -d '{"question":"Headphones are defective on day 7. What is the exact process and any fees? Please cite the rules.","thread_id":"tool-tests-06"}' | jq -r '.final_answer' +``` + +**Expected tools:** `search_policies` +**Expected result:** **DOA within 14 days** → free return label + **refund or replacement**, **no restocking fee** ([RET-111]). +**Citations:** `[RET-111]`. + +--- + +### 7) Open electronics on day 10 (€1,200) + 5-day delay → compute amounts + +**curl** + +```bash +curl -s http://localhost:8000/agent/ask \ + -H "Content-Type: application/json" \ + -d '{"question":"Opened laptop (fully working) returned on day 10, price €1,200; delivery was 5 business days late. Compute the euros for restocking and voucher (respecting caps) and give the net outcome with citations.","thread_id":"tool-tests-07"}' +``` + +**Expected tools:** `search_policies` → `calc("0.10*1200")` +**Expected result:** Restocking **10% = €120** ([RET-103]); voucher **20%** ([SLA-212]) **capped at €50** ([SLA-214]) → voucher **€50**; **net refund** = €1,200 − €120 = **€1,080**, plus voucher **€50**. +**Citations:** `[RET-103], [SLA-212], [SLA-214]`. + +--- + +### 8) Digital license activation failure on day 5 → returns ticket + refund + +**curl** + +```bash +curl -s http://localhost:8000/agent/ask \ + -H "Content-Type: application/json" \ + -d '{"question":"Digital license shows a server-side activation error at day 5. If required, open a returns ticket and confirm refund eligibility with sources.","thread_id":"tool-tests-08"}' | jq -r '.final_answer' +``` + +**Expected tools:** `search_policies` → `create_ticket("returns")` +**Expected result:** Opens a **returns** ticket ([TCK-531]); **refund** for **faulty activation within 7 days** ([DIG-312]); reminder: no refund after usage ([DIG-311]). +**Citations:** `[TCK-531], [DIG-312], [DIG-311]`. + +--- + +### 9) B2B pallet 3 days late, value €3,200 → compute voucher with B2B cap (€250, no ticket) + +**curl** + +```bash +curl -s http://localhost:8000/agent/ask \ + -H "Content-Type: application/json" \ + -d '{"question":"B2B pallet shipment arrived 3 business days late, PO value €3,200. Calculate the voucher per policy and cite the cap. Do not open a ticket unless required.","thread_id":"tool-tests-09"}' +``` + +**Expected tools:** `search_policies` → `calc("0.10*3200")` +**Expected result:** Pallet tolerance **+2 days** ([SLA-241]) → after 3 days, **10%** = **€320** ([SLA-211]) but **B2B cap €250 per PO** ([SLA-242]) → **€250**. Ticket **not required** (<10 days) ([TCK-511]). +**Citations:** `[SLA-241], [SLA-211], [SLA-242], [TCK-511]`. + +--- + +### 10) Standard delivery 5 days late, order €180 → 20% voucher (no cap exceeded), no ticket + +**curl** + +```bash +curl -s http://localhost:8000/agent/ask \ + -H "Content-Type: application/json" \ + -d '{"question":"Standard delivery was 5 business days late on an order of €180. Compute the voucher and cite the rule. Do not escalate unless required.","thread_id":"tool-tests-10"}' +``` + +**Expected tools:** `search_policies` → `calc("0.20*180")` +**Expected result:** Voucher **20% = €36** ([SLA-212]); **cap €50** not reached ([SLA-214]); no escalation (<10 days) ([TCK-511]). +**Citations:** `[SLA-212], [SLA-214], [TCK-511]`. diff --git a/Workshops/AI/04-Agents-and-RAG/requirements-lab.txt b/Workshops/AI/04-Agents-and-RAG/requirements-lab.txt new file mode 100644 index 0000000..475332d --- /dev/null +++ b/Workshops/AI/04-Agents-and-RAG/requirements-lab.txt @@ -0,0 +1,118 @@ +annotated-types==0.7.0 +anyio==4.10.0 +attrs==25.3.0 +backoff==2.2.1 +bcrypt==4.3.0 +build==1.3.0 +cachetools==5.5.2 +certifi==2025.8.3 +charset-normalizer==3.4.3 +chromadb==1.0.20 +click==8.2.1 +coloredlogs==15.0.1 +distro==1.9.0 +durationpy==0.10 +fastapi==0.116.1 +filelock==3.19.1 +flatbuffers==25.2.10 +fsspec==2025.7.0 +google-auth==2.40.3 +googleapis-common-protos==1.70.0 +grpcio==1.74.0 +h11==0.16.0 +hf-xet==1.1.8 +httpcore==1.0.9 +httptools==0.6.4 +httpx==0.28.1 +huggingface-hub==0.34.4 +humanfriendly==10.0 +idna==3.10 +importlib_metadata==8.7.0 +importlib_resources==6.5.2 +Jinja2==3.1.6 +joblib==1.5.1 +jsonpatch==1.33 +jsonpointer==3.0.0 +jsonschema==4.25.1 +jsonschema-specifications==2025.4.1 +kubernetes==33.1.0 +langchain==0.3.27 +langchain-core==0.3.74 +langchain-huggingface==0.3.1 +langchain-text-splitters==0.3.9 +langgraph==0.6.6 +langgraph-checkpoint==2.1.1 +langgraph-prebuilt==0.6.4 +langgraph-sdk==0.2.2 +langsmith==0.4.15 +markdown-it-py==4.0.0 +MarkupSafe==3.0.2 +mdurl==0.1.2 +mmh3==5.2.0 +mpmath==1.3.0 +networkx==3.5 +numpy==2.3.2 +oauthlib==3.3.1 +onnxruntime==1.22.1 +opentelemetry-api==1.36.0 +opentelemetry-exporter-otlp-proto-common==1.36.0 +opentelemetry-exporter-otlp-proto-grpc==1.36.0 +opentelemetry-proto==1.36.0 +opentelemetry-sdk==1.36.0 +opentelemetry-semantic-conventions==0.57b0 +orjson==3.11.2 +ormsgpack==1.10.0 +overrides==7.7.0 +packaging==25.0 +pillow==11.3.0 +posthog==5.4.0 +protobuf==6.32.0 +pyasn1==0.6.1 +pyasn1_modules==0.4.2 +pybase64==1.4.2 +pydantic==2.11.7 +pydantic_core==2.33.2 +Pygments==2.19.2 +PyPika==0.48.9 +pyproject_hooks==1.2.0 +python-dateutil==2.9.0.post0 +python-dotenv==1.1.1 +PyYAML==6.0.2 +referencing==0.36.2 +regex==2025.7.34 +requests==2.32.5 +requests-oauthlib==2.0.0 +requests-toolbelt==1.0.0 +rich==14.1.0 +rpds-py==0.27.0 +rsa==4.9.1 +safetensors==0.6.2 +scikit-learn==1.7.1 +scipy==1.16.1 +sentence-transformers==5.1.0 +sentencepiece==0.2.1 +setuptools==80.9.0 +shellingham==1.5.4 +six==1.17.0 +sniffio==1.3.1 +SQLAlchemy==2.0.43 +starlette==0.47.2 +sympy==1.14.0 +tenacity==9.1.2 +threadpoolctl==3.6.0 +tokenizers==0.21.4 +torch==2.8.0 +tqdm==4.67.1 +transformers==4.55.2 +typer==0.16.1 +typing-inspection==0.4.1 +typing_extensions==4.14.1 +urllib3==2.5.0 +uvicorn==0.35.0 +uvloop==0.21.0 +watchfiles==1.1.0 +websocket-client==1.8.0 +websockets==15.0.1 +xxhash==3.5.0 +zipp==3.23.0 +zstandard==0.24.0 diff --git a/Workshops/AI/04-Agents-and-RAG/requirements.txt b/Workshops/AI/04-Agents-and-RAG/requirements.txt new file mode 100644 index 0000000..9da737f --- /dev/null +++ b/Workshops/AI/04-Agents-and-RAG/requirements.txt @@ -0,0 +1,17 @@ +fastapi +uvicorn[standard] +pydantic +chromadb +sentence-transformers +python-dotenv + +# Hugging Face + agent stack +huggingface_hub +langchain +langchain-core +langchain-huggingface +langgraph +sentencepiece +transformers +tokenizers +langsmith diff --git a/Workshops/AI/04-Agents-and-RAG/src/lab_service/agent_api.py b/Workshops/AI/04-Agents-and-RAG/src/lab_service/agent_api.py new file mode 100644 index 0000000..bfe3d3b --- /dev/null +++ b/Workshops/AI/04-Agents-and-RAG/src/lab_service/agent_api.py @@ -0,0 +1,40 @@ +"""API endpoints for agent interactions. + +This module exposes a single FastAPI router for agent-related operations. +Endpoints call into the language-graph agent implementation and return +the agent response. Designed to be lightweight: validation and business +logic live in the agent implementation. + +Endpoints +- POST /agent/ask -> Handle a user question via the language-graph agent. +""" +from fastapi import APIRouter +from .agent_graph import run_agent_with_langgraph +from .models import AskReq + +router = APIRouter(prefix="/agent", tags=["Agent"]) + +@router.post("/ask") +def ask(req: AskReq): + """Handle a user question and run the language-graph agent. + + Parameters + ---------- + req : AskReq + Request body containing: + - question: the user's question string + - thread_id: optional identifier to continue a conversation thread + + Returns + ------- + dict | Any + The agent response produced by run_agent_with_langgraph. The exact + shape depends on the agent implementation (typically a dict with + answer, sources, and metadata). + + Notes + ----- + Exceptions raised by the agent implementation are propagated to the + FastAPI framework and will be converted to HTTP error responses. + """ + return run_agent_with_langgraph(req.question, thread_id=req.thread_id) diff --git a/Workshops/AI/04-Agents-and-RAG/src/lab_service/agent_graph.py b/Workshops/AI/04-Agents-and-RAG/src/lab_service/agent_graph.py new file mode 100644 index 0000000..8fb797c --- /dev/null +++ b/Workshops/AI/04-Agents-and-RAG/src/lab_service/agent_graph.py @@ -0,0 +1,244 @@ +"""Graph-based agent wiring for the workshop demo. + +This module composes a small language-graph agent that: +- accepts a user question, +- consults the policy knowledge base via a search tool, +- performs simple calculations or ticket creation when required, +- and calls a Hugging Face chat model to produce the final answer. + +Design notes +- The graph is intentionally simple: an "agent" node that calls the LLM + and a "tools" node that runs tools when the LLM requests them. +- Tools return JSON-serialisable strings; the LLM is expected to call tools + and consume their outputs. Tool outputs are included in the conversational + state so the model can reason over them in subsequent turns. +- The SYSTEM_PROMPT encodes strict policy guardrails. Keep it up-to-date + with policy documents and retrieval/tooling guidelines. +""" +from typing import TypedDict, Annotated +import operator +import json +from pathlib import Path +from datetime import datetime +from langgraph.graph import StateGraph, END +from langgraph.checkpoint.memory import MemorySaver +from langgraph.prebuilt import ToolNode +from langchain_core.messages import AnyMessage, HumanMessage, SystemMessage +from langchain_core.tools import tool +from langchain_huggingface import ChatHuggingFace, HuggingFaceEndpoint +from .config import HF_MODEL_ID +from .rag_core import retrieve +import re + + +# Regex to detect policy IDs in retrieved text (e.g., 'RET-103', 'SLA-214', 'EXM-900') +ID_RE = re.compile(r"\b([A-Z]{3}-\d{3}(?:-[A-Z]+)?)\b") + +@tool +def search_policies(query: str) -> str: + """Tool: run semantic search against the policies KB. + + Return JSON with hits: [{'source': str, 'chunk': int, 'ids': [str], 'excerpt': str}]. + Each hit contains: + - source: file path of the document + - chunk: integer chunk index + - ids: list of policy IDs like RET-103, SLA-214 found in the excerpt + - excerpt: short text excerpt for context + + The LLM should parse the returned JSON to extract cited policy IDs + and exact wording for answers and citations. + """ + hits = retrieve(query, k=6) + out = [] + for h in hits: + text = h["text"] + ids = sorted(set(ID_RE.findall(text))) + out.append({ + "source": h["meta"]["source"], + "chunk": h["meta"]["chunk"], + "ids": ids, + "excerpt": text # excerpt used by the model for context + }) + return json.dumps(out) + +@tool +def calc(expression: str) -> str: + """Tool: safely evaluate a simple arithmetic expression (e.g., '12*(3+4)'). + + Returns the stringified result or an error message. + + Purpose + - Use when the agent needs to compute exact euro amounts, apply caps, etc. + + Security note + - eval is run with an empty __builtins__ to reduce risk. This is a demo + helper; consider a proper expression evaluator for production. + """ + try: + out = str(eval(expression, {"__builtins__": {}})) + except Exception as e: + out = f"Error: {e}" + return out + +@tool +def create_ticket(category: str, description: str) -> str: + """Tool: simulate ticket creation, persist it to disk, and return a JSON payload. + + Behaviour: + - Generates a deterministic ticket id for demo purposes. + - Persists the ticket JSON into the project's data/tickets directory as .json. + - Returns a JSON string with ticket_id and status. + + Notes: + - This is a demo persistence mechanism. In production use a proper ticketing DB/service. + """ + ticket_id = "TKT-" + str(abs(hash(category + description)) % 10_000) + ticket = { + "ticket_id": ticket_id, + "status": "created", + "category": category, + "description": description, + "created_at": datetime.utcnow().isoformat() + "Z", + } + + # Persist ticket to data/tickets/.json (create dir if needed) + tickets_dir = Path(__file__).resolve().parents[2] / "data" / "tickets" + try: + tickets_dir.mkdir(parents=True, exist_ok=True) + ticket_file = tickets_dir / f"{ticket_id}.json" + # Write atomically: write to .tmp then rename + tmp_file = ticket_file.with_suffix(".json.tmp") + tmp_file.write_text(json.dumps(ticket, ensure_ascii=False, indent=2), encoding="utf-8") + tmp_file.replace(ticket_file) + except Exception as e: + # If persistence fails, still return created ticket info but include an error field. + ticket["status"] = "created_with_persist_error" + ticket["persist_error"] = str(e) + + return json.dumps({"ticket_id": ticket_id, "status": "created"}) + +# Public tools list bound to the chat model and used by the ToolNode. +TOOLS = [search_policies, calc, create_ticket] + +class AgentState(TypedDict): + """TypedDict describing the graph state for the agent nodes. + + messages: conversational messages list that the chat model consumes/produces. + The graph nodes expect 'messages' to be a list of AnyMessage-like objects. + """ + messages: Annotated[list[AnyMessage], operator.add] + +# Configure the Hugging Face endpoint wrapper used as the LLM. +# Note: repo_id must be compatible with the invocation style used by ChatHuggingFace. +llm = HuggingFaceEndpoint( + repo_id=HF_MODEL_ID, + task="text-generation", + max_new_tokens=512, + do_sample=False, + temperature=0, +) + +# Wrap the HF endpoint into a ChatHuggingFace model and bind the tools so the +# model can produce tool calls in its output. +chat_model = ChatHuggingFace(llm=llm).bind_tools(TOOLS) + +# ToolNode integrates the callable tools into the graph so they can be invoked +# when the LLM requests them. +tool_node = ToolNode(TOOLS) + + +def call_model(state: AgentState) -> AgentState: + """Graph node: call the chat model with the current messages. + + The model returns a message-like object. We wrap it into the AgentState + shape expected by the graph (a dict with 'messages'). + """ + response = chat_model.invoke(state["messages"]) + return {"messages": [response]} + + +def route_after_model(state: AgentState) -> str: + """Decide next node after the model runs. + + If the last message contains tool_calls, route to the 'tools' node, + otherwise end the graph run (END). + """ + last = state["messages"][-1] + if getattr(last, "tool_calls", None): + return "tools" + return END + + +# Construct a small state graph and compile it with an in-memory checkpointer. +_graph = StateGraph(AgentState) +_graph.add_node("agent", call_model) +_graph.add_node("tools", tool_node) +# Loop between tools and agent: tools -> agent (model consumes tool outputs) +_graph.add_edge("tools", "agent") +# agent node can conditionally route to tools or END based on the model output +_graph.add_conditional_edges("agent", route_after_model) +_graph.set_entry_point("agent") + +# Use an in-memory saver for checkpointing conversational state during the demo. +checkpointer = MemorySaver() +agent_graph = _graph.compile(checkpointer=checkpointer) + +# System prompt: strict guardrails for policy-bound behaviour. +# Keep this synchronized with the policy documents and RAG retrieval expectations. +SYSTEM_PROMPT = ( + "You are a policy-bound support agent. Follow these RULES strictly:\n" + "1) Always start with a short PLAN before answering.\n" + "2) Use tools only when needed:\n" + " - Use `search_policies` BEFORE answering any policy question, lists, IDs, thresholds, examples.\n" + " - Use `calc` to compute exact euro amounts and to apply any caps/limits.\n" + " - Use `create_ticket` if the retrieved sources explicitly require it.\n" + " When a tool is needed, emit a tool call. The final answer will be formatted by the system." + " Multiple tool calls are allowed. For instance: \n" + " - First call `search_policies` to find relevant policies.\n" + " - Then call `calc` to compute refund amounts based on retrieved policy data.\n" + " - Finally, if a ticket is mandated by a TCK-5xx clause, call `create_ticket` with the correct category and reason.\n" + "3) When using `search_policies`, cite policy IDs (e.g., [RET-103], [SLA-214]) included in the tool output. " + " Use ONLY facts present in the retrieved sources; do NOT invent.\n" + "4) If information is insufficient, say so explicitly and stop.\n" + "5) Responses must be concise, in English. " + "If refunds/vouchers apply, include the exact euro amount; if a ticket is opened, include type and ticket ID>\n" + "6) If refunds or vouchers apply, compute exact values and check caps/limits before answering.\n" + "7) Do not open tickets unless a retrieved clause (e.g., TCK-5xx) requires it; cite that clause.\n" + "8) When any TCK-5xx clause mandates escalation, call `create_ticket` immediately with the correct `category` and a short `reason`. Do not ask the user for permission, do not invent ticket IDs, and do not claim success until the tool returns.\n" + "9) If you mention a ticket in the answer, ensure it was created via the tool; otherwise, call the tool before replying.\n" +) + +def run_agent_with_langgraph(question: str, thread_id: str | None = None) -> dict: + """Run the compiled language-graph agent for a single user question. + + Parameters + ---------- + question : str + The user's natural language question. + thread_id : str | None + Optional thread identifier to correlate conversation state across calls. + + Returns + ------- + dict + A dictionary with: + - final_answer: the assistant's final rendered text (string) + - thread_id: the thread identifier used (string) + + Behaviour + - Initializes the conversation with SYSTEM_PROMPT and the user's question. + - Invokes the compiled agent_graph which will alternate between model and tools + as needed until completion. + - The final message is returned as 'final_answer'. Tool outputs (JSON strings) + are expected to be consumed by the model in intermediate steps. + """ + config = {"configurable": {"thread_id": thread_id or "default"}} + initial: AgentState = { + "messages": [ + SystemMessage(content=SYSTEM_PROMPT), + HumanMessage(content=question), + ] + } + final_state = agent_graph.invoke(initial, config=config) + last = final_state["messages"][-1] + return {"final_answer": getattr(last, "content", ""), "thread_id": config["configurable"]["thread_id"]} diff --git a/Workshops/AI/04-Agents-and-RAG/src/lab_service/app.py b/Workshops/AI/04-Agents-and-RAG/src/lab_service/app.py new file mode 100644 index 0000000..69a23d7 --- /dev/null +++ b/Workshops/AI/04-Agents-and-RAG/src/lab_service/app.py @@ -0,0 +1,31 @@ +"""FastAPI application entrypoint for the RAG & Agents workshop services. + +This module creates the FastAPI app, registers routers for the +RAG (retrieval-augmented generation) and language-graph agent APIs, +and exposes a lightweight healthcheck endpoint. + +Keep this file minimal: application configuration, middleware and +business logic should live in their respective modules (rag_api, +agent_api, etc.). Routers are included at import-time so their +prefixes and tags control the OpenAPI schema. +""" + +from fastapi import FastAPI +from .rag_api import router as rag_router +from .agent_api import router as agent_router + +app = FastAPI(title="AI Academy - RAG & Agents API") + +@app.get("/healthz") +def healthz(): + """Liveness/health endpoint. + + Returns a simple status dict used by orchestrators or uptime checks. + Keep this endpoint deterministic and fast — do not perform heavy IO here. + """ + return {"status": "ok"} + +# Include routers after app creation. Order does not affect routing but +# influence the generated OpenAPI ordering. Each router defines its own prefix. +app.include_router(rag_router) +app.include_router(agent_router) \ No newline at end of file diff --git a/Workshops/AI/04-Agents-and-RAG/src/lab_service/config.py b/Workshops/AI/04-Agents-and-RAG/src/lab_service/config.py new file mode 100644 index 0000000..0e7ad68 --- /dev/null +++ b/Workshops/AI/04-Agents-and-RAG/src/lab_service/config.py @@ -0,0 +1,39 @@ +"""Configuration module for the RAG & Agents service. + +This module reads runtime configuration from environment variables (optionally +loaded from a .env file) and exposes simple constants used across the codebase. + +Documented variables: +- PERSIST_DIR: path where embedding/vector DB artifacts are stored. +- KB_DIR: directory containing the policy knowledge base used by the retriever. +- EMB_MODEL: embedding model id used to encode documents (default: sentence-transformers/all-MiniLM-L6-v2). +- HUGGINGFACEHUB_API_TOKEN: API token for Hugging Face; required for HF model inference / router calls. +- HF_MODEL_ID: Hugging Face model id used for chat/completions calls. + +Notes +----- +- Keep sensitive values (HUGGINGFACEHUB_API_TOKEN) out of source control; prefer environment variables + or a secrets manager. The .env loader is included for local development convenience. +- If you change defaults here, ensure tests and deployment configs are updated accordingly. +""" +import os +from dotenv import load_dotenv + +# Load .env in development; load_dotenv is a no-op if no .env file is present. +load_dotenv() + +# Paths / embeddings +# Directory where persistent vector DB / chroma artifacts are stored. +PERSIST_DIR = os.getenv("PERSIST_DIR", "artifacts/chroma") +# Directory containing the knowledge base used by the retriever (policy markdown files). +KB_DIR = os.getenv("KB_DIR", "data/policies") +# Embedding model identifier (used by the embedding client). Default is a lightweight sentence-transformer. +EMB_MODEL = os.getenv("EMB_MODEL", "sentence-transformers/all-MiniLM-L6-v2") + +# Hugging Face +# Token used to authenticate with the Hugging Face Hub / Inference API. +# Required for model inference that uses HF router or private models. +HUGGINGFACEHUB_API_TOKEN = os.getenv("HUGGINGFACEHUB_API_TOKEN") +# HF model id for chat/completions calls. Make sure this model supports the endpoint your client uses. +# Example: "mistralai/Mistral-7B-Instruct-v0.3" or "Qwen/Qwen2.5-7B-Instruct" or any OpenAI-compatible HF router model. +HF_MODEL_ID = os.getenv("HF_MODEL_ID", "Qwen/Qwen2.5-7B-Instruct") diff --git a/Workshops/AI/04-Agents-and-RAG/src/lab_service/models.py b/Workshops/AI/04-Agents-and-RAG/src/lab_service/models.py new file mode 100644 index 0000000..48a487e --- /dev/null +++ b/Workshops/AI/04-Agents-and-RAG/src/lab_service/models.py @@ -0,0 +1,82 @@ +"""Pydantic request/response models used by the RAG & Agents FastAPI services. + +This module defines lightweight DTOs for incoming API requests and for the +RAG response payload. Keep these models minimal and stable since they form +the public contract of the HTTP endpoints. +""" +from pydantic import BaseModel +from typing import List, Dict, Any + + +class AskReq(BaseModel): + """Request body for the /agent/ask endpoint. + + Attributes + ---------- + question : str + User's free-text question to the language agent. + thread_id : str | None + Optional identifier to continue a conversation thread (preserved by the caller). + """ + question: str + thread_id: str | None = None + + +class IngestReq(BaseModel): + """Request for (re)ingesting documents into the retriever/embedding store. + + Attributes + ---------- + source_dir : str | None + Optional path containing documents to ingest. If omitted, use configured KB_DIR. + reset : bool + If True, clear existing persisted embeddings/vector DB before ingesting. + """ + source_dir: str | None = None + reset: bool = False + + +class SearchReq(BaseModel): + """Simple search/retrieval request. + + Used when invoking the retriever directly. + + Attributes + ---------- + query : str + Query text to search the KB. + k : int + Number of top documents to return (default: 4). + """ + query: str + k: int = 4 + + +class QueryReq(BaseModel): + """RAG-style query payload for the /rag/generate endpoint. + + Attributes + ---------- + question : str + The user's natural language question to answer using RAG. + k : int + Number of retrieved documents to use as context (default: 4). + """ + question: str + k: int = 4 + + +class RAGResponse(BaseModel): + """Canonical RAG response returned by /rag/generate. + + Attributes + ---------- + answer : str + Generated answer text produced by the model using retrieved context. + citations : list[dict] + List of citation metadata for the retrieved documents used to form the answer. + Each dict should minimally include source identifiers and optional excerpt/score. + Example item: {"id": "20_shipping_sla.md#SLA-214", "text": "...", "score": 0.92} + """ + answer: str + citations: List[Dict[str, Any]] diff --git a/Workshops/AI/04-Agents-and-RAG/src/lab_service/rag_api.py b/Workshops/AI/04-Agents-and-RAG/src/lab_service/rag_api.py new file mode 100644 index 0000000..fa7787a --- /dev/null +++ b/Workshops/AI/04-Agents-and-RAG/src/lab_service/rag_api.py @@ -0,0 +1,71 @@ +"""FastAPI router exposing simple RAG endpoints: ingest, retrieve and generate. + +This module provides thin HTTP adapters over the rag_core utilities. Keep +business logic in rag_core; these endpoints are responsible for request +validation (Pydantic models) and for returning well-typed responses. + +Endpoints +- POST /rag/ingest -> trigger (re)ingestion of KB files +- POST /rag/retrieve -> run semantic search against the KB +- POST /rag/generate -> run RAG generation using top-k retrieved chunks +""" +from fastapi import APIRouter +from .models import IngestReq, SearchReq, QueryReq +from .rag_core import ingest, retrieve, generate, RAGResponse + +router = APIRouter(prefix="/rag", tags=["RAG"]) + +@router.post("/ingest") +def ingest_api(req: IngestReq): + """Trigger ingestion of markdown/plain-text files into the vector store. + + Parameters + ---------- + req : IngestReq + - source_dir: optional path to scan (defaults to configured KB_DIR) + - reset: if True, clear existing collection before ingesting + + Returns + ------- + dict + Summary containing indexed_files and chunks added. + """ + # Thin adapter: delegate to rag_core.ingest which handles file IO and persistence. + return ingest(req.source_dir, req.reset) + +@router.post("/retrieve") +def retrieve_api(req: SearchReq): + """Run a semantic retrieval against the policies collection. + + Parameters + ---------- + req : SearchReq + - query: query text + - k: number of top results to return + + Returns + ------- + dict + Echoes the query and returns the list of retrieval hits under "results". + Each result includes text, snippet, meta and score. + """ + return {"query": req.query, "results": retrieve(req.query, k=req.k)} + +@router.post("/generate", response_model=RAGResponse) +def generate_api(req: QueryReq): + """Produce a RAG answer using top-k retrieved context. + + Parameters + ---------- + req : QueryReq + - question: user question + - k: number of retrieved chunks to pass to the model + + Returns + ------- + RAGResponse + Structured response with 'answer' and 'citations'. + """ + # Delegate to rag_core.generate which performs retrieval, calls the model, + # and formats the RAGResponse. Any HF/IO errors propagate as HTTP 500. + return generate(req.question, k=req.k) diff --git a/Workshops/AI/04-Agents-and-RAG/src/lab_service/rag_core.py b/Workshops/AI/04-Agents-and-RAG/src/lab_service/rag_core.py new file mode 100644 index 0000000..b06efea --- /dev/null +++ b/Workshops/AI/04-Agents-and-RAG/src/lab_service/rag_core.py @@ -0,0 +1,239 @@ +"""Core RAG utilities: ingestion, retrieval and generation. + +This module provides a tiny pipeline for: +- indexing markdown/text policy files into a persistent Chroma collection + (ingest). +- performing semantic retrieval against the collection (retrieve). +- formatting retrieved context and calling a Hugging Face InferenceClient + (OpenAI-compatible router/chat completions) to produce a final answer + with citations (generate). + +Notes / operational guidance +- PERSIST_DIR, EMB_MODEL, HUGGINGFACEHUB_API_TOKEN and HF_MODEL_ID are + configured in lab_service.config. Ensure HF_MODEL_ID points to a + model/router that supports the OpenAI-compatible chat/completions + endpoint if you call client.chat.completions.create(...) (otherwise + the hub may return 404). +- The ingestion logic uses simple fixed-size chunking for demo purposes; + production systems should use a semantic/overlap-aware chunker. +""" +from pathlib import Path +from typing import List, Dict, Any +from chromadb import PersistentClient +from chromadb.utils import embedding_functions +from huggingface_hub import InferenceClient + +from .config import ( + PERSIST_DIR, + KB_DIR, + EMB_MODEL, + HUGGINGFACEHUB_API_TOKEN, + HF_MODEL_ID, +) +from .models import RAGResponse + +def _chroma(): + """Create a PersistentClient pointing at the configured PERSIST_DIR. + + Returns + ------- + chromadb.PersistentClient + Client instance to interact with the persistent Chroma DB. + """ + # Chroma stores collections on-disk at PERSIST_DIR when using PersistentClient + return PersistentClient(path=PERSIST_DIR) + + +def _collection(): + """Get or create the 'policies' collection with the configured embedding fn. + + The collection is created with a SentenceTransformer embedding function + backed by EMB_MODEL (a sentence-transformers model id). This ensures + documents and queries are encoded consistently. + """ + ef = embedding_functions.SentenceTransformerEmbeddingFunction(model_name=EMB_MODEL) + return _chroma().get_or_create_collection("policies", embedding_function=ef) + + +def ingest(source_dir: str | None = None, reset: bool = False) -> Dict[str, Any]: + """Index markdown/text files from source_dir (or KB_DIR) into Chroma. + + The function: + - walks the source directory recursively, + - reads .md and .txt files, + - splits each file into fixed-size chunks (simple demo strategy), + - stores documents, ids and metadata into the 'policies' collection. + + Parameters + ---------- + source_dir : str | None + Path to scan for documents. If None, uses configured KB_DIR. + reset : bool + If True, deletes the existing 'policies' collection before ingesting. + + Returns + ------- + dict + Summary with number of indexed files and total chunks added. + """ + source = Path(source_dir or KB_DIR) + col = _collection() + if reset: + # Delete and recreate to ensure a clean state when requested + _chroma().delete_collection("policies") + col = _collection() + + files_indexed = 0 + docs, ids, metas = [], [], [] + for p in source.rglob("*"): + # Only ingest markdown and plain text files + if p.suffix.lower() in {".md", ".txt"}: + text = p.read_text(encoding="utf-8", errors="ignore") + # Simple chunking for demo: chunk length 2000, overlap ~400 + chunks = [text[i:i + 2000] for i in range(0, len(text), 1600)] + for k, ch in enumerate(chunks): + ids.append(f"{p.stem}-{files_indexed}-{k}") + docs.append(ch) + metas.append({"source": str(p), "chunk": k}) + files_indexed += 1 + + if docs: + # Bulk-add to the collection + col.add(documents=docs, ids=ids, metadatas=metas) + + return {"indexed_files": files_indexed, "chunks": len(docs)} + + +def retrieve(query: str, k: int = 4): + """Run a semantic retrieval query against the 'policies' collection. + + Parameters + ---------- + query : str + Natural language query text. + k : int + Number of top results to return. + + Returns + ------- + list[dict] + List of hits with keys: text, snippet, meta, score. + - text: full chunk text + - snippet: short excerpt for quick display + - meta: metadata dict (includes 'source' and 'chunk') + - score: similarity/distance score as float + """ + res = _collection().query(query_texts=[query], n_results=k) + docs = res.get("documents", [[]])[0] + metas = res.get("metadatas", [[]])[0] + dists = res.get("distances", [[]])[0] + out = [] + for d, m, s in zip(docs, metas, dists): + snippet = d[:400] + ("..." if len(d) > 400 else "") + out.append({"text": d, "snippet": snippet, "meta": m, "score": float(s)}) + return out + + +def _format_sources(ctx: List[Dict[str, Any]]) -> str: + """Serialize retrieved context into a plain-text block for the model prompt. + + The format enumerates sources ([1], [2], ...) and includes each chunk's + text. This string is intended to be attached to the user prompt so the + model can refer to explicit passages. + + Parameters + ---------- + ctx : list[dict] + Retrieval results as returned by retrieve(). + + Returns + ------- + str + Multi-line string containing numbered sources and their corresponding text. + """ + lines = [] + for i, c in enumerate(ctx, 1): + src = c["meta"]["source"] + ch = c["meta"]["chunk"] + # Keep the full chunk in the prompt; the model can use snippet if needed. + lines.append(f"[{i}] {src} (chunk {ch})\n{c['text']}\n") + return "\n".join(lines) + + +def generate(question: str, k: int = 4) -> RAGResponse: + """Produce a RAG-formatted answer calling a Hugging Face InferenceClient. + + Steps: + 1. Retrieve top-k relevant chunks from the KB. + 2. Build a system + user prompt that instructs the model to answer ONLY + using the provided sources and to return strict JSON. + 3. Call the InferenceClient chat/completions endpoint. + 4. Parse the model output as JSON; if parsing fails, fall back to returning + the raw content and attaching the top-k sources as citations. + + Important: + - HF_MODEL_ID must refer to a model/router that exposes an OpenAI-compatible + chat/completions endpoint when using client.chat.completions.create(...). + Otherwise the Hugging Face Hub may return 404/Not Found. + - HUGGINGFACEHUB_API_TOKEN should be set for private models or router usage. + + Parameters + ---------- + question : str + User question to answer. + k : int + Number of retrieved context chunks to pass to the model. + + Returns + ------- + RAGResponse + answer: final text (or raw model content on parse failure) + citations: list of dicts with keys 'source' and 'chunk' + """ + ctx = retrieve(question, k=k) + if not ctx: + return RAGResponse(answer="No documents found in the knowledge base.", citations=[]) + + # Guidance to the model: be strict and only use the provided sources. + system = ( + "You are a helpful assistant that must answer ONLY using the provided sources. " + "If the information is not present in the sources, explicitly say so. " + "Answer in concise English. Always include citations as [1], [2], ..." + ) + user = ( + f"Question: {question}\n\n" + f"Sources:\n{_format_sources(ctx)}\n\n" + "Return a valid JSON object with the following shape: " + "{'answer': str, 'citations': [{'source': str, 'chunk': int}]}" + ) + + # Create the InferenceClient. The `provider` and invocation style can vary + # depending on how HF_MODEL_ID is exposed (router, custom provider, etc.). + client = InferenceClient(provider="together", api_key=HUGGINGFACEHUB_API_TOKEN) + + # Call the chat completions API. This may raise HTTP errors from the hub + # (e.g., 404 if the route/model is not available) — let exceptions bubble + # up to the caller or handle them where the generate() function is used. + chat = client.chat.completions.create( + model=HF_MODEL_ID, + messages=[{"role":"system","content":system}, + {"role":"user","content":user}], + temperature=0.2, + max_tokens=512, + ) + + content = chat.choices[0].message.content or "" + + import json + try: + # Expect the model to return a strict JSON object per the prompt. + data = json.loads(content) + ans = data.get("answer", "") + cits = data.get("citations", []) + except Exception: + # If parsing fails, return the raw content and attach the retrieved sources + # as fallback citations so the caller can still show provenance. + ans = content + cits = [{"source": c["meta"]["source"], "chunk": c["meta"]["chunk"]} for c in ctx] + + return RAGResponse(answer=ans, citations=cits) \ No newline at end of file