Skip to content

full pipeline of training #4

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
dashu233 wants to merge 3 commits into
base: master
Choose a base branch
from
Open

full pipeline of training #4

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
046b1f6
alg update
dashu233 Jun 15, 2021
ffdb6ed
command update
dashu233 Jun 15, 2021
50bfc8e
full pipeline
dashu233 Jun 15, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
617 changes: 398 additions & 219 deletions domainbed/algorithms.py
View file
Open in desktop

Large diffs are not rendered by default.

33 changes: 30 additions & 3 deletions domainbed/command_launchers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
import subprocess
import time
import torch

def local_launcher(commands):
"""Launch commands serially on the local machine."""
for cmd in commands:
Expand Down Expand Up @@ -48,14 +47,42 @@ def multi_gpu_launcher(commands):
if p is not None:
p.wait()

def my_multi_gpu_launcher(commands,available_list = [0,1,2,3]):
"""
Launch commands on the local machine, using all GPUs in parallel.
"""
print('WARNING: using experimental multi_gpu_launcher.')
n_gpus = torch.cuda.device_count()
procs_by_gpu = [None]*n_gpus

while len(commands) > 0:
for gpu_idx in available_list:
proc = procs_by_gpu[gpu_idx]
if (proc is None) or (proc.poll() is not None):
# Nothing is running on this GPU; launch a command.
cmd = commands.pop(0)
new_proc = subprocess.Popen(
f'CUDA_VISIBLE_DEVICES={gpu_idx} {cmd}', shell=True)
procs_by_gpu[gpu_idx] = new_proc
break
time.sleep(60)

# Wait for the last few tasks to finish before returning
for p in procs_by_gpu:
if p is not None:
p.wait()



REGISTRY = {
'local': local_launcher,
'dummy': dummy_launcher,
'multi_gpu': multi_gpu_launcher
'multi_gpu': multi_gpu_launcher,
'my_multi_gpu':my_multi_gpu_launcher
}

try:
from domainbed import facebook
facebook.register_command_launchers(REGISTRY)
except ImportError:
pass
pass
68 changes: 68 additions & 0 deletions domainbed/feature_checker.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,74 @@ def calculate_f_star(algorithm, loaders, device, test_envs, num_classes):
mean[label] += fss_list[label][i]
return [mean[label]/len(fss_list[label]) for label in range(num_classes)]

def feature_extractor_for_train(algorithm, loaders, device, num_classes,val='in'):
print("————————Calculate the feature distribution for train————————")
print("")
#eval_list = []

fssd_list_raw = [{} for _ in range(num_classes)]

for eval_name, eval_loader in loaders:
if val in eval_name:
fssd_score, fssd_raw = get_feature(
algorithm, eval_loader, device, num_classes, None, False, True)
#eval_list.append(eval_name)
for label in range(num_classes):
fssd_list_raw[label][eval_name[:-3]]=fssd_raw[label]
#print(fssd_list_raw[0]['env0'])
print("————————Finish Calculating————————")
return fssd_list_raw


def feature_extractor_for_pipline(algorithm, loaders, device, num_classes, marker="",val='in'):
print("————————Calculate the feature distribution————————")
print("")
eval_list = []
fssd_list_raw = [[]for i in range(num_classes)]
fssd_mean = [[]for i in range(num_classes)]
fssd_variance = [[]for i in range(num_classes)]
feature_mean = [[]for i in range(num_classes)]
feature_var = [[]for i in range(num_classes)]
return_feat = [{} for _ in range(num_classes)]
for eval_name, eval_loader in loaders:
if val in eval_name:
# Debug 不设置f_star
# fssd_score, fssd_raw = get_feature(
# algorithm, eval_loader, device, num_classes, f_star, False, True)
start = time.time()
fssd_score, fssd_raw = get_feature(
algorithm, eval_loader, device, num_classes, None, False, True)
for label in range(num_classes):
return_feat[label][eval_name[:4]]=fssd_raw[label]
eval_list.append(eval_name)
print("Extract feature in env " + eval_name + " use time " + str(round(time.time()-start, 3)))

for label in range(num_classes):
fssd_list_raw[label].append(fssd_score[label])
fssd_mean[label].append(torch.mean(fssd_list_raw[label][-1]))
fssd_variance[label].append(
torch.var(fssd_list_raw[label][-1]))
feature_mean[label].append(torch.mean(fssd_raw[label], dim=0))
feature_var[label].append(torch.var(fssd_raw[label], dim=0))

save_raw_feature = True
if save_raw_feature:

np.save(marker + "_"+eval_name+"_label"+str(label) +
".npy", fssd_raw[label].cpu().numpy())

# 直观图片打印
save_some_image = False
if save_some_image:
feature_num = list(range(20))
for num in feature_num:
feature_set = fssd_raw[label][:, num].cpu().numpy()
plt.figure()
plt.hist(feature_set, bins='auto', density=True)
plt.savefig("feature_imgae/feature"+str(num)+"_label"+str(label)+"_" +
eval_list[-1]+".png")
plt.close()
return return_feat

def feature_extractor(algorithm, loaders, device, num_classes, marker=""):
print("————————Calculate the feature distribution————————")
Expand Down
Loading