Feat: Implement FW-merging with loss approximation.

config/method/fw_merging/fw_hard_am.yaml

@@ -0,0 +1,16 @@
+_target_: fusion_bench.method.FrankWolfeHardAdamergingAlgorithm
+merge_fn: task_arithmetic
+max_iters: 10
+step_size: 0.1
+dataset_size: 100
+tasks: []
+init_weight: base
+loss_fn: cross_entropy
+scaling_factor: 0.3
+max_num_models: 100
+granularity: task
+init_layer_weights: 0.0
+ada_merge: True
+ada_max_steps: 1000
+ada_optimizer: adam
+ada_lr: 1e-3

config/method/fw_merging/fw_hard_am_loss_approx.yaml

@@ -0,0 +1,16 @@
+_target_: fusion_bench.method.FrankWolfeHardAdamergingLossApproxAlgorithm
+merge_fn: task_arithmetic
+max_iters: 10
+step_size: 0.1
+dataset_size: 100
+tasks: []
+init_weight: base
+loss_fn: cross_entropy
+scaling_factor: 0.3
+max_num_models: 100
+granularity: task
+init_layer_weights: 0.0
+ada_merge: True
+ada_max_steps: 1000
+ada_optimizer: adam
+ada_lr: 1e-3

config/method/fw_merging/fw_hard_loss_approx.yaml

@@ -0,0 +1,11 @@
+_target_: fusion_bench.method.FrankWolfeHardLossApproxAlgorithm
+merge_fn: task_arithmetic
+max_iters: 10
+step_size: 0.1
+dataset_size: 100
+tasks: []
+init_weight: 
+loss_fn: cross_entropy
+scaling_factor: 0.3
+max_num_models: 100
+granularity: task

config/method/fw_merging/fw_soft_loss_approx.yaml

@@ -0,0 +1,12 @@
+_target_: fusion_bench.method.FrankWolfeSoftLossApproxAlgorithm
+init_weight:
+max_iters: 10
+merge_fn: 'adamerging'
+tasks: 
+ada_iters: 500
+dataset_size: 100
+ada_coeff: 1e-8
+step_size: 0.1
+max_num_models: 100
+granularity: task
+ada_loss: entropy_loss

fusion_bench/method/__init__.py

@@ -100,7 +100,7 @@
         "SparseLoForLlama",
         "PCPSparseLoForLlama",
     ],
-    "fw_merging": ["FrankWolfeHardAlgorithm", "FrankWolfeSoftAlgorithm"],
+    "fw_merging": ["FrankWolfeHardAlgorithm", "FrankWolfeSoftAlgorithm", "FrankWolfeHardAdamergingAlgorithm", "FrankWolfeSoftLossApproxAlgorithm", "FrankWolfeHardLossApproxAlgorithm", "FrankWolfeHardAdamergingLossApproxAlgorithm"],
 }
 
 
@@ -182,7 +182,7 @@
     from .ties_merging import TiesMergingAlgorithm
     from .we_moe import CLIPWeightEnsemblingMoEAlgorithm
     from .weighted_average import WeightedAverageAlgorithm, WeightedAverageForLLama
-    from .fw_merging import FrankWolfeHardAlgorithm, FrankWolfeSoftAlgorithm
+    from .fw_merging import FrankWolfeHardAlgorithm, FrankWolfeSoftAlgorithm, FrankWolfeHardAdamergingAlgorithm, FrankWolfeSoftLossApproxAlgorithm, FrankWolfeHardLossApproxAlgorithm, FrankWolfeHardAdamergingLossApproxAlgorithm
 
 else:
     sys.modules[__name__] = LazyImporter(

fusion_bench/method/fw_merging/__init__.py

@@ -1,2 +1,6 @@
 from .fw_hard import FrankWolfeHardAlgorithm
-from .fw_soft import FrankWolfeSoftAlgorithm
+from .fw_soft import FrankWolfeSoftAlgorithm
+from .fw_hard_am import FrankWolfeHardAdamergingAlgorithm
+from .fw_soft_loss_approx import FrankWolfeSoftLossApproxAlgorithm
+from .fw_hard_loss_approx import FrankWolfeHardLossApproxAlgorithm
+from .fw_hard_am_loss_approx import FrankWolfeHardAdamergingLossApproxAlgorithm