From ff7e6afe2074aa7ad8bf7b135d0d9cd6520c956f Mon Sep 17 00:00:00 2001 From: Aditya Borate <23110065@iitgn.ac.in> Date: Tue, 11 Nov 2025 19:23:34 +0000 Subject: [PATCH 1/4] Fix(peft): Re-apply group offloading after deleting adapters --- src/diffusers/loaders/peft.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/diffusers/loaders/peft.py b/src/diffusers/loaders/peft.py index 7d65b30659fb..df2fb1aecaaa 100644 --- a/src/diffusers/loaders/peft.py +++ b/src/diffusers/loaders/peft.py @@ -22,6 +22,7 @@ import safetensors import torch +from ..hooks.group_offloading import _maybe_remove_and_reapply_group_offloading from ..utils import ( MIN_PEFT_VERSION, USE_PEFT_BACKEND, @@ -792,6 +793,8 @@ def delete_adapters(self, adapter_names: Union[List[str], str]): if hasattr(self, "peft_config"): self.peft_config.pop(adapter_name, None) + _maybe_remove_and_reapply_group_offloading(self) + def enable_lora_hotswap( self, target_rank: int = 128, check_compiled: Literal["error", "warn", "ignore"] = "error" ) -> None: From d1b71f74dca7e588b473c322ade3c00d00a45bfb Mon Sep 17 00:00:00 2001 From: Aditya Borate <23110065@iitgn.ac.in> Date: Thu, 13 Nov 2025 10:08:58 +0000 Subject: [PATCH 2/4] Test: Add regression test for group offloading + delete_adapters --- tests/lora/utils.py | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/tests/lora/utils.py b/tests/lora/utils.py index 3d4344bb86a9..efe68dd3ce54 100644 --- a/tests/lora/utils.py +++ b/tests/lora/utils.py @@ -30,6 +30,7 @@ ) from diffusers.utils import logging from diffusers.utils.import_utils import is_peft_available +from diffusers.hooks.group_offloading import apply_group_offloading from ..testing_utils import ( CaptureLogger, @@ -2367,3 +2368,42 @@ def test_lora_loading_model_cpu_offload(self): output_lora_loaded = pipe(**inputs, generator=torch.manual_seed(0))[0] self.assertTrue(np.allclose(output_lora, output_lora_loaded, atol=1e-3, rtol=1e-3)) + + @require_torch_accelerator + def test_lora_group_offloading_delete_adapters(self): + components, _, denoiser_lora_config = self.get_dummy_components() + _, _, inputs = self.get_dummy_inputs(with_generator=False) + pipe = self.pipeline_class(**components) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + denoiser = pipe.transformer if self.unet_kwargs is None else pipe.unet + denoiser.add_adapter(denoiser_lora_config) + self.assertTrue(check_if_lora_correctly_set(denoiser), "Lora not correctly set in denoiser.") + + with tempfile.TemporaryDirectory() as tmpdirname: + modules_to_save = self._get_modules_to_save(pipe, has_denoiser=True) + lora_state_dicts = self._get_lora_state_dicts(modules_to_save) + self.pipeline_class.save_lora_weights( + save_directory=tmpdirname, safe_serialization=True, **lora_state_dicts + ) + + components, _, _ = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.to(torch_device) + + denoiser = pipe.transformer if self.unet_kwargs is None else pipe.unet + + # Enable Group Offloading (leaf_level) + apply_group_offloading( + denoiser, + onload_device=torch_device, + offload_device="cpu", + offload_type="leaf_level", + ) + + pipe.load_lora_weights(tmpdirname, adapter_name="default") + pipe(**inputs, generator=torch.manual_seed(0)) + # Delete the adapter + pipe.delete_adapters("default") + pipe(**inputs, generator=torch.manual_seed(0)) \ No newline at end of file From 09456cb50f40852e0c8ef43fb253235b908afd4d Mon Sep 17 00:00:00 2001 From: Aditya Borate <23110065@iitgn.ac.in> Date: Thu, 13 Nov 2025 13:36:26 +0000 Subject: [PATCH 3/4] Test: Add assertions to verify output changes after deletion --- tests/lora/utils.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/lora/utils.py b/tests/lora/utils.py index efe68dd3ce54..10a5c8e9aa1d 100644 --- a/tests/lora/utils.py +++ b/tests/lora/utils.py @@ -28,9 +28,9 @@ AutoencoderKL, UNet2DConditionModel, ) +from diffusers.hooks.group_offloading import apply_group_offloading from diffusers.utils import logging from diffusers.utils.import_utils import is_peft_available -from diffusers.hooks.group_offloading import apply_group_offloading from ..testing_utils import ( CaptureLogger, @@ -2403,7 +2403,8 @@ def test_lora_group_offloading_delete_adapters(self): ) pipe.load_lora_weights(tmpdirname, adapter_name="default") - pipe(**inputs, generator=torch.manual_seed(0)) + out_lora = pipe(**inputs, generator=torch.manual_seed(0))[0] # Delete the adapter pipe.delete_adapters("default") - pipe(**inputs, generator=torch.manual_seed(0)) \ No newline at end of file + out_no_lora = pipe(**inputs, generator=torch.manual_seed(0))[0] + self.assertFalse(np.allclose(out_lora, out_no_lora, atol=1e-3, rtol=1e-3)) From 4f0691ff4f0901e73a0e343fc1cf68cf95179420 Mon Sep 17 00:00:00 2001 From: Aditya Borate <23110065@iitgn.ac.in> Date: Sat, 15 Nov 2025 09:49:27 +0000 Subject: [PATCH 4/4] Test: Add try/finally to clean up group offloading hooks --- tests/lora/utils.py | 56 ++++++++++++++++++++++++++------------------- 1 file changed, 32 insertions(+), 24 deletions(-) diff --git a/tests/lora/utils.py b/tests/lora/utils.py index 10a5c8e9aa1d..5fae6cac0a7f 100644 --- a/tests/lora/utils.py +++ b/tests/lora/utils.py @@ -28,7 +28,7 @@ AutoencoderKL, UNet2DConditionModel, ) -from diffusers.hooks.group_offloading import apply_group_offloading +from diffusers.hooks.group_offloading import _GROUP_OFFLOADING, apply_group_offloading from diffusers.utils import logging from diffusers.utils.import_utils import is_peft_available @@ -2381,30 +2381,38 @@ def test_lora_group_offloading_delete_adapters(self): denoiser.add_adapter(denoiser_lora_config) self.assertTrue(check_if_lora_correctly_set(denoiser), "Lora not correctly set in denoiser.") - with tempfile.TemporaryDirectory() as tmpdirname: - modules_to_save = self._get_modules_to_save(pipe, has_denoiser=True) - lora_state_dicts = self._get_lora_state_dicts(modules_to_save) - self.pipeline_class.save_lora_weights( - save_directory=tmpdirname, safe_serialization=True, **lora_state_dicts - ) + try: + with tempfile.TemporaryDirectory() as tmpdirname: + modules_to_save = self._get_modules_to_save(pipe, has_denoiser=True) + lora_state_dicts = self._get_lora_state_dicts(modules_to_save) + self.pipeline_class.save_lora_weights( + save_directory=tmpdirname, safe_serialization=True, **lora_state_dicts + ) - components, _, _ = self.get_dummy_components() - pipe = self.pipeline_class(**components) - pipe.to(torch_device) + components, _, _ = self.get_dummy_components() + pipe = self.pipeline_class(**components) + denoiser = pipe.transformer if self.unet_kwargs is None else pipe.unet + pipe.to(torch_device) + + # Enable Group Offloading (leaf_level for more granular testing) + apply_group_offloading( + denoiser, + onload_device=torch_device, + offload_device="cpu", + offload_type="leaf_level", + ) - denoiser = pipe.transformer if self.unet_kwargs is None else pipe.unet + pipe.load_lora_weights(tmpdirname, adapter_name="default") - # Enable Group Offloading (leaf_level) - apply_group_offloading( - denoiser, - onload_device=torch_device, - offload_device="cpu", - offload_type="leaf_level", - ) + out_lora = pipe(**inputs, generator=torch.manual_seed(0))[0] + + # Delete the adapter + pipe.delete_adapters("default") + + out_no_lora = pipe(**inputs, generator=torch.manual_seed(0))[0] - pipe.load_lora_weights(tmpdirname, adapter_name="default") - out_lora = pipe(**inputs, generator=torch.manual_seed(0))[0] - # Delete the adapter - pipe.delete_adapters("default") - out_no_lora = pipe(**inputs, generator=torch.manual_seed(0))[0] - self.assertFalse(np.allclose(out_lora, out_no_lora, atol=1e-3, rtol=1e-3)) + self.assertFalse(np.allclose(out_lora, out_no_lora, atol=1e-3, rtol=1e-3)) + finally: + # Clean up the hooks to prevent state leak + if hasattr(denoiser, "_diffusers_hook"): + denoiser._diffusers_hook.remove_hook(_GROUP_OFFLOADING, recurse=True)