[MLOps3.0] Minor fixes (#201)

* Minor fix for LHM demo

* Minor fix for custom metrics

* Removed monitoring folders

* Create .gitignore in LHM demo folder

Adding `.gitignore` to ignore local monitoring dashboard jsons in LHM demo

* Create .gitignore in MLOps-Advanced demo

Adding `.gitignore` to ignore local monitoring dashboard jsons

* Quick-Fixes & Cleans

Removed batch scoring for synthetic data

* Major updates for Serverless, MLflow3.0 and Lakebase

- Removed all mentions/use of automl and switched to Optuna with native mlflow spark integration
- Added MLflow3.0 deployment job
- Updated Feature Engineering/Serving to use new Synched tables (Lakebase)
- Minor fixes to monitoring and drift detection

* [MLOps3.0] Fixed quickstart and experiments

- Tested Quickstart demo
- Fixed Experiment name
- Updated DAB configuration

* Updated references to new images/banners

* [MLOps3.0] Fixes for quick-start and Optuna HPO

* [MLOps3.0] Minor fixes for FE client

* [MLOps3.0] Bug fixes for Optuna/autoML and FE client

* Synching with upstream

* [MLOps3.0] Bug fixes for Optuna/autoML and FE client

* [MLOps3.0] Minor fixes

* Synching with upstream

Author: aelhelouDB

product_demos/Data-Science/mlops-end2end/02-mlops-advanced/02_model_training_hpo_optuna.py

@@ -498,7 +498,7 @@ def prune(self, study, trial):
 from mlflow import pyfunc
 
 
-def optuna_hpo_fn(n_trials: int, X_train: pd.DataFrame, Y_train: pd.Series, X_test: pd.DataFrame, Y_test: pd.Series, training_set_specs_in, preprocessor_in: ColumnTransformer, experiment_id: str, pos_label_in: str = pos_label, rng_seed_in: int = 2025, run_name:str = "spark-mlflow-tuning", optuna_sampler_in: optuna.samplers.TPESampler = optuna_sampler, optuna_pruner_in: optuna.pruners.BasePruner = None, n_jobs: int = 4) -> optuna.study.study.Study:
+def optuna_hpo_fn(n_trials: int, X_train: pd.DataFrame, Y_train: pd.Series, X_test: pd.DataFrame, Y_test: pd.Series, training_set_specs_in, preprocessor_in: ColumnTransformer, experiment_id: str, pos_label_in: str = pos_label, rng_seed_in: int = 2025, run_name:str = "spark-mlflow-tuning", optuna_sampler_in: optuna.samplers.TPESampler = optuna_sampler, optuna_pruner_in: optuna.pruners.BasePruner = None, n_jobs: int = 2) -> optuna.study.study.Study:
     """
     Increasing `n_jobs` may cause experiment to fail due to failed trials which return None and can't be pruned/caught in parallel mode
     """
@@ -604,7 +604,7 @@ def optuna_hpo_fn(n_trials: int, X_train: pd.DataFrame, Y_train: pd.Series, X_te
   run_name="mlops-hpo-best-run", # "smoke-test"
   optuna_sampler_in=optuna_sampler,
   optuna_pruner_in=NoneValuePruner(),
-  n_jobs = 2, # Set this to number of physical cores
+  # n_jobs = 2, # Increase this to number for more parallel trials
 )
 
 # COMMAND ----------

product_demos/Data-Science/mlops-end2end/02-mlops-advanced/03a_create_deployment_job.py

@@ -31,6 +31,14 @@
 
 # COMMAND ----------
 
+# MAGIC %md
+# MAGIC Last environment tested:
+# MAGIC ```
+# MAGIC mlflow>=3.3.0
+# MAGIC ```
+
+# COMMAND ----------
+
 # MAGIC %pip install --quiet mlflow --upgrade
 # MAGIC
 # MAGIC
@@ -74,11 +82,9 @@
 # COMMAND ----------
 
 # Create job with necessary configuration to connect to model as deployment job
-from databricks.sdk import WorkspaceClient
 from databricks.sdk.service import jobs
 
 
-w = WorkspaceClient()
 job_settings = jobs.JobSettings(
     name=job_name,
     tasks=[
@@ -114,9 +120,39 @@
     max_concurrent_runs=1,
 )
 
-created_job = w.jobs.create(**job_settings.__dict__)
+# COMMAND ----------
+
+from databricks.sdk import WorkspaceClient
+
+
+w = WorkspaceClient()
+
+# Search for the job by name (in case it exists)
+existing_jobs = w.jobs.list(name=job_name)
+job_id = None
+for created_job in existing_jobs:
+  if created_job.settings.name == job_name and created_job.creator_user_name == current_user:
+      job_id = created_job.job_id
+      break
+
+if job_id:
+  # Update existing job
+  print("Updating existing...")
+  w.jobs.update(job_id=job_id, new_settings=job_settings)
+
+else:
+  # Create new job
+  print("Creating new...")
+  created_job = w.jobs.create(**job_settings.__dict__)
+  job_id = created_job.job_id
+
+print(f"Job ID: {job_id}")
+
+# COMMAND ----------
+
+# DBTITLE 1,ONE-TIME Operation
 print("Use the job name " + job_name + " to connect the deployment job to the UC model " + model_name + " as indicated in the UC Model UI.")
-print("\nFor your reference, the job ID is: " + str(created_job.job_id))
+print("\nFor your reference, the job ID is: " + str(job_id))
 print("\nDocumentation: \nAWS: https://docs.databricks.com/aws/mlflow/deployment-job#connect \nAzure: https://learn.microsoft.com/azure/databricks/mlflow/deployment-job#connect \nGCP: https://docs.databricks.com/gcp/mlflow/deployment-job#connect")
 
 # COMMAND ----------
@@ -135,10 +171,19 @@
 client = MlflowClient(registry_uri="databricks-uc")
 
 try:
-  if client.get_registered_model(model_name):
+  model_info = client.get_registered_model(model_name)
+  if model_info:
     # Model exists - Link job
-    client.update_registered_model(model_name, deployment_job_id=created_job.job_id)
+    if model_info.deployment_job_id == job_id:
+      print("Model exists with existing job - Pass")
+      pass
+
+    else:
+      print("Model exists - Updating job")
+      client.update_registered_model(model_name, deployment_job_id="") # Unlink current job
+      client.update_registered_model(model_name, deployment_job_id=job_id) # Link new one
 
 except mlflow.exceptions.RestException:
   # Create Empty Model placeholder and Link job
-  client.create_registered_model(model_name, deployment_job_id=created_job.job_id)
+  print("Model does not exist - Creating model and linking job")
+  client.create_registered_model(model_name, deployment_job_id=job_id)

product_demos/Data-Science/mlops-end2end/02-mlops-advanced/04b_challenger_approval.py

@@ -22,6 +22,10 @@
 
 # COMMAND ----------
 
+# MAGIC %run ../_resources/00-setup $adv_mlops=true
+
+# COMMAND ----------
+
 dbutils.widgets.text("model_name", f"{catalog}.{db}.advanced_mlops_churn", "Model Name") # Will be populated from Deployment Jobs Parameters
 dbutils.widgets.text("model_version", "1", "Model Version") # Will be populated from Deployment Jobs Parameters
 dbutils.widgets.text("approval_tag_name", "Approval_Check", "Approval Tag to check")