From d1f97afe22ace28171e2e51e7981dcfd4daae4cf Mon Sep 17 00:00:00 2001 From: chufangao Date: Sun, 15 Jun 2025 13:04:27 -0500 Subject: [PATCH 01/10] init generators commit --- pyhealth/models/generators/__init__.py | 0 pyhealth/models/generators/halo.py | 0 pyhealth/models/generators/medgan.py | 0 pyhealth/models/generators/promptehr.py | 0 4 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 pyhealth/models/generators/__init__.py create mode 100644 pyhealth/models/generators/halo.py create mode 100644 pyhealth/models/generators/medgan.py create mode 100644 pyhealth/models/generators/promptehr.py diff --git a/pyhealth/models/generators/__init__.py b/pyhealth/models/generators/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pyhealth/models/generators/halo.py b/pyhealth/models/generators/halo.py new file mode 100644 index 000000000..e69de29bb diff --git a/pyhealth/models/generators/medgan.py b/pyhealth/models/generators/medgan.py new file mode 100644 index 000000000..e69de29bb diff --git a/pyhealth/models/generators/promptehr.py b/pyhealth/models/generators/promptehr.py new file mode 100644 index 000000000..e69de29bb From ee8c52cb5fb2bc1e11ff1d72df4ca73dfa1acc8a Mon Sep 17 00:00:00 2001 From: Rasmussen Date: Tue, 15 Jul 2025 19:23:10 -0500 Subject: [PATCH 02/10] base --- pyhealth/models/generators/halo.py | 155 ++++++++++++ .../generators/halo_resources/halo_config.py | 42 ++++ .../generators/halo_resources/halo_model.py | 237 ++++++++++++++++++ 3 files changed, 434 insertions(+) create mode 100644 pyhealth/models/generators/halo_resources/halo_config.py create mode 100644 pyhealth/models/generators/halo_resources/halo_model.py diff --git a/pyhealth/models/generators/halo.py b/pyhealth/models/generators/halo.py index e69de29bb..7a2437e34 100644 --- a/pyhealth/models/generators/halo.py +++ b/pyhealth/models/generators/halo.py @@ -0,0 +1,155 @@ +import torch +import torch.nn as nn +from typing import Any, Dict, List, Optional + +from pyhealth.datasets import SampleEHRDataset +from pyhealth.models.base_model import BaseModel + +# Import the HALO transformer implementation +from pyhealth.models.generators.halo_resources.halo_model import HALOModel +from pyhealth.models.generators.halo_resources.halo_model import HALOConfig + +class HALO(BaseModel): + """ + HALO model wrapper for PyHealth, leveraging the BaseModel interface. + + Args: + dataset: SampleEHRDataset for EHR sequences (visits of codes). + config: Transformer configuration object for HALOModel. + feature_key: key in dataset samples for code-based visits (e.g., "list_codes"). + mode: one of "binary", "multiclass", or "multilabel" for loss function. + label_key: key in dataset samples for target visits; defaults to feature_key for next-visit prediction. + pos_loss_weight: weight applied to positive labels in BCE loss. + """ + def __init__( + self, + dataset: SampleEHRDataset, + config: HALOConfig, + feature_key: str = "list_codes", + mode: str = "multilabel", + label_key: Optional[str] = None, + pos_loss_weight: float = 1.0, + ): + super(HALO, self).__init__(dataset) + self.feature_key = feature_key + self.label_key = label_key or feature_key + self.pos_loss_weight = pos_loss_weight + + # Set mode for loss and evaluation + self.mode = mode + + # Tokenizer for the code-based input + self.tokenizer = dataset.input_processors[feature_key] + self.vocab_size = self.tokenizer.size() + + # Instantiate the underlying HALO transformer model + self.halo = HALOModel(config) + + def _prepare_input_visits(self, codes: List[List[Any]]) -> torch.Tensor: + """ + Convert list of visits of codes into multi-hot tensor. + + Args: + codes: nested list of shape (batch, num_visits, codes_in_visit) + + Returns: + Tensor of shape (batch, num_visits, vocab_size) with 0/1 entries. + """ + # batch_encode_3d returns List[List[List[int]]] + token_ids = self.tokenizer.batch_encode_3d(codes) + batch_size = len(token_ids) + max_visits = len(token_ids[0]) + + visits = torch.zeros( + batch_size, max_visits, self.vocab_size, device=self.device + ) + for i in range(batch_size): + for t, visit_ids in enumerate(token_ids[i]): + for cid in visit_ids: + if cid is None: + continue + visits[i, t, cid] = 1.0 + return visits + + def forward( + self, + **kwargs: Any + ) -> Dict[str, torch.Tensor]: + """ + Forward propagation for HALO within PyHealth. + + Expects kwargs to contain: + - feature_key: raw visit code lists. + - optional label_key: same format for next-visit targets. + - optional masks: tensor or nested lists for masking visits. + + Returns a dict with keys: + - loss: training loss (if labels provided). + - y_prob: predicted probabilities of codes per visit. + - y_true: ground-truth multi-hot labels (shifted by one visit). + - logits: raw logits from the HALO transformer. + """ + # Prepare input tensor + raw_codes = kwargs[self.feature_key] + input_visits = self._prepare_input_visits(raw_codes) + + # Gather optional training labels and masks + ehr_labels = None + ehr_masks = None + if self.label_key in kwargs: + # similarly convert label visits to multi-hot + ehr_labels = self._prepare_input_visits(kwargs[self.label_key]) + if "masks" in kwargs: + ehr_masks = torch.tensor(kwargs["masks"], device=self.device, dtype=torch.float) + + # Call HALOModel: returns loss & probabilities if labels, else probabilities + if ehr_labels is not None: + loss, code_probs, shift_labels = self.halo( + input_visits, + ehr_labels=ehr_labels, + ehr_masks=ehr_masks, + pos_loss_weight=self.pos_loss_weight, + ) + results = {"loss": loss, "y_prob": code_probs, "y_true": shift_labels} + else: + code_probs = self.halo(input_visits) + results = {"y_prob": code_probs} + + # Attach logits if needed + if hasattr(self.halo, 'last_logits'): + results['logits'] = self.halo.last_logits + + return results + +# Example usage: +if __name__ == "__main__": + from pyhealth.datasets import SampleEHRDataset + # define samples with visits of codes as nested lists + samples = [ + {"patient_id": "p0", "visit_id": "v0", "list_codes": [["A", "B"], ["C"]]}, + {"patient_id": "p1", "visit_id": "v0", "list_codes": [["D"], ["E", "F"]]}, + ] + dataset = SampleEHRDataset(samples=samples, dataset_name="halo_test") + + # Build transformer config + config = HALOConfig( + n_layer=4, + n_head=8, + n_embd=128, + total_vocab_size=dataset.input_processors['list_codes'].size(), + n_positions=dataset.max_visit_length, + n_ctx=dataset.max_visit_length, + layer_norm_epsilon=1e-5, + ) + + model = HALO( + dataset=dataset, + config=config, + feature_key='list_codes', + mode='multilabel', + ) + from pyhealth.datasets import get_dataloader + loader = get_dataloader(dataset, batch_size=2, shuffle=False) + batch = next(iter(loader)) + output = model(**batch) + print(output) diff --git a/pyhealth/models/generators/halo_resources/halo_config.py b/pyhealth/models/generators/halo_resources/halo_config.py new file mode 100644 index 000000000..2127aa3a0 --- /dev/null +++ b/pyhealth/models/generators/halo_resources/halo_config.py @@ -0,0 +1,42 @@ +''' + code by Brandon Theodorou + Original GPT-2 Paper and repository here: https://github.com/openai/gpt-2 + Original GPT-2 Pytorch Model: https://github.com/huggingface/pytorch-pretrained-BERT + GPT-2 Pytorch Model Derived From: https://github.com/graykode/gpt-2-Pytorch +''' +class HALOConfig(object): + def __init__( + self, + total_vocab_size=6984, + code_vocab_size=6841, + label_vocab_size=25, + special_vocab_size=3, + n_positions=56, + n_ctx=48, + n_embd=768, + n_layer=12, + n_head=12, + layer_norm_epsilon=1e-5, + initializer_range=0.02, + batch_size=48, + sample_batch_size=256, + epoch=50, + pos_loss_weight=None, + lr=1e-4, + ): + self.total_vocab_size = total_vocab_size + self.code_vocab_size = code_vocab_size + self.label_vocab_size = label_vocab_size + self.special_vocab_size = special_vocab_size + self.n_positions = n_positions + self.n_ctx = n_ctx + self.n_embd = n_embd + self.n_layer = n_layer + self.n_head = n_head + self.layer_norm_epsilon = layer_norm_epsilon + self.initializer_range = initializer_range + self.batch_size = batch_size + self.sample_batch_size = sample_batch_size + self.epoch = epoch + self.pos_loss_weight = pos_loss_weight + self.lr = lr \ No newline at end of file diff --git a/pyhealth/models/generators/halo_resources/halo_model.py b/pyhealth/models/generators/halo_resources/halo_model.py new file mode 100644 index 000000000..a25a9fe04 --- /dev/null +++ b/pyhealth/models/generators/halo_resources/halo_model.py @@ -0,0 +1,237 @@ +''' + code by Brandon Theodorou + Original GPT-2 Paper and repository here: https://github.com/openai/gpt-2 + Original GPT-2 Pytorch Model: https://github.com/huggingface/pytorch-pretrained-BERT + GPT-2 Pytorch Model Derived From: https://github.com/graykode/gpt-2-Pytorch +''' +import copy +import math +import torch +import torch.nn as nn +import torch.nn.functional as F + +def gelu(x): + return 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) + +class LayerNorm(nn.Module): + def __init__(self, hidden_size, eps=1e-12): + """Construct a layernorm module in the TF style (epsilon inside the square root).""" + super(LayerNorm, self).__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.bias = nn.Parameter(torch.zeros(hidden_size)) + self.variance_epsilon = eps + + def forward(self, x): + u = x.mean(-1, keepdim=True) + s = (x - u).pow(2).mean(-1, keepdim=True) + x = (x - u) / torch.sqrt(s + self.variance_epsilon) + return self.weight * x + self.bias + +class Conv1D(nn.Module): + def __init__(self, nf, nx): + super(Conv1D, self).__init__() + self.nf = nf + w = torch.empty(nx, nf) + nn.init.normal_(w, std=0.02) + self.weight = nn.Parameter(w) + self.bias = nn.Parameter(torch.zeros(nf)) + + def forward(self, x): + size_out = x.size()[:-1] + (self.nf,) + x = torch.addmm(self.bias, x.view(-1, x.size(-1)), self.weight) + x = x.view(*size_out) + return x + +class Attention(nn.Module): + def __init__(self, nx, n_ctx, config, scale=False): + super(Attention, self).__init__() + n_state = nx # in Attention: n_state=768 (nx=n_embd) + assert n_state % config.n_head == 0 + self.register_buffer("bias", torch.tril(torch.ones(n_ctx, n_ctx)).view(1, 1, n_ctx, n_ctx)) + self.n_head = config.n_head + self.split_size = n_state + self.scale = scale + self.c_attn = Conv1D(n_state * 3, nx) + self.c_proj = Conv1D(n_state, nx) + + def _attn(self, q, k, v): + w = torch.matmul(q, k) + if self.scale: + w = w / math.sqrt(v.size(-1)) + nd, ns = w.size(-2), w.size(-1) + b = self.bias[:, :, ns-nd:ns, :ns] + w = w * b - 1e10 * (1 - b) + w = nn.Softmax(dim=-1)(w) + return torch.matmul(w, v) + + def merge_heads(self, x): + x = x.permute(0, 2, 1, 3).contiguous() + new_x_shape = x.size()[:-2] + (x.size(-2) * x.size(-1),) + return x.view(*new_x_shape) + + def split_heads(self, x, k=False): + new_x_shape = x.size()[:-1] + (self.n_head, x.size(-1) // self.n_head) + x = x.view(*new_x_shape) + if k: + return x.permute(0, 2, 3, 1) # (batch, head, head_features, seq_length) + else: + return x.permute(0, 2, 1, 3) # (batch, head, seq_length, head_features) + + def forward(self, x, layer_past=None): + x = self.c_attn(x) + query, key, value = x.split(self.split_size, dim=2) + query = self.split_heads(query) + key = self.split_heads(key, k=True) + value = self.split_heads(value) + if layer_past is not None: + past_key, past_value = layer_past[0].transpose(-2, -1), layer_past[1] # transpose back cf below + key = torch.cat((past_key, key), dim=-1) + value = torch.cat((past_value, value), dim=-2) + present = torch.stack((key.transpose(-2, -1), value)) # transpose to have same shapes for stacking + a = self._attn(query, key, value) + a = self.merge_heads(a) + a = self.c_proj(a) + return a, present + +class MLP(nn.Module): + def __init__(self, n_state, config): # in MLP: n_state=3072 (4 * n_embd) + super(MLP, self).__init__() + nx = config.n_embd + self.c_fc = Conv1D(n_state, nx) + self.c_proj = Conv1D(nx, n_state) + self.act = gelu + + def forward(self, x): + h = self.act(self.c_fc(x)) + h2 = self.c_proj(h) + return h2 + +class Block(nn.Module): + def __init__(self, n_ctx, config, scale=False): + super(Block, self).__init__() + nx = config.n_embd + self.ln_1 = LayerNorm(nx, eps=config.layer_norm_epsilon) + self.attn = Attention(nx, n_ctx, config, scale) + self.ln_2 = LayerNorm(nx, eps=config.layer_norm_epsilon) + self.mlp = MLP(4 * nx, config) + + def forward(self, x, layer_past=None): + a, present = self.attn(self.ln_1(x), layer_past=layer_past) + x = x + a + m = self.mlp(self.ln_2(x)) + x = x + m + return x, present + +class CoarseTransformerModel(nn.Module): + def __init__(self, config): + super(CoarseTransformerModel, self).__init__() + self.n_layer = config.n_layer + self.n_embd = config.n_embd + self.n_vocab = config.total_vocab_size + + self.vis_embed_mat = nn.Linear(config.total_vocab_size, config.n_embd, bias=False) + self.pos_embed_mat = nn.Embedding(config.n_positions, config.n_embd) + block = Block(config.n_ctx, config, scale=True) + self.h = nn.ModuleList([copy.deepcopy(block) for _ in range(config.n_layer)]) + self.ln_f = LayerNorm(config.n_embd, eps=config.layer_norm_epsilon) + + def forward(self, input_visits, position_ids=None, past=None): + if past is None: + past_length = 0 + past = [None] * len(self.h) + else: + past_length = past[0][0].size(-2) + if position_ids is None: + position_ids = torch.arange(past_length, input_visits.size(1) + past_length, dtype=torch.long, + device=input_visits.device) + position_ids = position_ids.unsqueeze(0).expand(input_visits.size(0), input_visits.size(1)) + + inputs_embeds = self.vis_embed_mat(input_visits) + position_embeds = self.pos_embed_mat(position_ids) + hidden_states = inputs_embeds + position_embeds + for block, layer_past in zip(self.h, past): + hidden_states, _ = block(hidden_states, layer_past) + hidden_states = self.ln_f(hidden_states) + return hidden_states + +class AutoregressiveLinear(nn.Linear): + """ same as Linear except has a configurable mask on the weights """ + def __init__(self, in_features, out_features, bias=True): + super().__init__(in_features, out_features, bias) + self.register_buffer('mask', torch.tril(torch.ones(in_features, out_features)).int()) + + def forward(self, input): + return F.linear(input, self.mask * self.weight, self.bias) + +class FineAutoregressiveHead(nn.Module): + def __init__(self, config): + super(FineAutoregressiveHead, self).__init__() + self.auto1 = AutoregressiveLinear(config.n_embd + config.total_vocab_size, config.n_embd + config.total_vocab_size) + self.auto2 = AutoregressiveLinear(config.n_embd + config.total_vocab_size, config.n_embd + config.total_vocab_size) + self.n_embd = config.n_embd + self.tot_vocab = config.total_vocab_size + + def forward(self, history, input_visits): + history = history[:,:-1,:] + input_visits = input_visits[:,1:,:] + code_logits = self.auto2(torch.relu(self.auto1(torch.cat((history, input_visits), dim=2))))[:,:,self.n_embd-1:-1] + return code_logits + + def sample(self, history, input_visits): + history = history[:,:-1,:] + input_visits = input_visits[:,1:,:] + currVisit = torch.cat((history, input_visits), dim=2)[:,-1,:].unsqueeze(1) + code_logits = self.auto2(torch.relu(self.auto1(currVisit)))[:,:,self.n_embd-1:-1] + return code_logits + +class HALOModel(nn.Module): + def __init__(self, config): + super(HALOModel, self).__init__() + self.transformer = CoarseTransformerModel(config) + self.ehr_head = FineAutoregressiveHead(config) + + def forward(self, input_visits, position_ids=None, ehr_labels=None, ehr_masks=None, past=None, pos_loss_weight=None): + hidden_states = self.transformer(input_visits, position_ids, past) + code_logits = self.ehr_head(hidden_states, input_visits) + sig = nn.Sigmoid() + code_probs = sig(code_logits) + if ehr_labels is not None: + shift_labels = ehr_labels[..., 1:, :].contiguous() + loss_weights = None + if pos_loss_weight is not None: + loss_weights = torch.ones(code_probs.shape, device=code_probs.device) + loss_weights = loss_weights + (pos_loss_weight-1) * shift_labels + if ehr_masks is not None: + code_probs = code_probs * ehr_masks + shift_labels = shift_labels * ehr_masks + if pos_loss_weight is not None: + loss_weights = loss_weights * ehr_masks + + bce = nn.BCELoss(weight=loss_weights) + loss = bce(code_probs, shift_labels) + return loss, code_probs, shift_labels + + return code_probs + + def sample(self, input_visits, random=True): + sig = nn.Sigmoid() + hidden_states = self.transformer(input_visits) + i = 0 + while i < self.ehr_head.tot_vocab: + next_logits = self.ehr_head.sample(hidden_states, input_visits) + next_probs = sig(next_logits) + if random: + visit = torch.bernoulli(next_probs) + else: + visit = torch.round(next_probs) + + remaining_visit = visit[:,0,i:] + nonzero = torch.nonzero(remaining_visit, as_tuple=True)[1] + if nonzero.numel() == 0: + break + + first_nonzero = nonzero.min() + input_visits[:,-1,i + first_nonzero] = visit[:,0,i + first_nonzero] + i = i + first_nonzero + 1 + + return input_visits \ No newline at end of file From 00f10c2bc045dfad681ce13d689b3b8944ff6157 Mon Sep 17 00:00:00 2001 From: Rasmussen Date: Tue, 15 Jul 2025 20:21:25 -0500 Subject: [PATCH 03/10] Stab at implementation --- .../hcup_ccs_2015_definitions_benchmark.yaml | 149 +++++ pyhealth/datasets/halo_mimic3.py | 155 +++++ pyhealth/models/generators/halo.py | 609 ++++++++++++++---- 3 files changed, 773 insertions(+), 140 deletions(-) create mode 100644 pyhealth/datasets/configs/hcup_ccs_2015_definitions_benchmark.yaml create mode 100644 pyhealth/datasets/halo_mimic3.py diff --git a/pyhealth/datasets/configs/hcup_ccs_2015_definitions_benchmark.yaml b/pyhealth/datasets/configs/hcup_ccs_2015_definitions_benchmark.yaml new file mode 100644 index 000000000..cf0caddfa --- /dev/null +++ b/pyhealth/datasets/configs/hcup_ccs_2015_definitions_benchmark.yaml @@ -0,0 +1,149 @@ +"Septicemia (except in labor)": + use_in_benchmark: True + type: "acute" + id: 2 + codes: [ "0031", "0202", "0223", "0362", "0380", "0381", "03810", "03811", "03812", "03819", "0382", "0383", "03840", "03841", "03842", "03843", "03844", "03849", "0388", "0389", "0545", "449", "77181", "7907", "99591", "99592" ] + +"Diabetes mellitus without complication": + use_in_benchmark: True + type: "chronic" + id: 49 + codes: [ "24900", "25000", "25001", "7902", "79021", "79022", "79029", "7915", "7916", "V4585", "V5391", "V6546" ] + +"Diabetes mellitus with complications": + use_in_benchmark: True + type: "chronic" + id: 50 + codes: [ "24901", "24910", "24911", "24920", "24921", "24930", "24931", "24940", "24941", "24950", "24951", "24960", "24961", "24970", "24971", "24980", "24981", "24990", "24991", "25002", "25003", "25010", "25011", "25012", "25013", "25020", "25021", "25022", "25023", "25030", "25031", "25032", "25033", "25040", "25041", "25042", "25043", "25050", "25051", "25052", "25053", "25060", "25061", "25062", "25063", "25070", "25071", "25072", "25073", "25080", "25081", "25082", "25083", "25090", "25091", "25092", "25093" ] + +"Disorders of lipid metabolism": + use_in_benchmark: True + type: "chronic" + id: 53 + codes: [ "2720", "2721", "2722", "2723", "2724" ] + +"Fluid and electrolyte disorders": + use_in_benchmark: True + type: "acute" + id: 55 + codes: [ "2760", "2761", "2762", "2763", "2764", "2765", "27650", "27651", "27652", "2766", "27669", "2767", "2768", "2769", "9951" ] + +"Essential hypertension": + use_in_benchmark: True + type: "chronic" + id: 98 + codes: [ "4011", "4019" ] + +"Hypertension with complications and secondary hypertension": + use_in_benchmark: True + type: "chronic" + id: 99 + codes: [ "4010", "40200", "40201", "40210", "40211", "40290", "40291", "4030", "40300", "40301", "4031", "40310", "40311", "4039", "40390", "40391", "4040", "40400", "40401", "40402", "40403", "4041", "40410", "40411", "40412", "40413", "4049", "40490", "40491", "40492", "40493", "40501", "40509", "40511", "40519", "40591", "40599", "4372" ] + +"Acute myocardial infarction": + use_in_benchmark: True + type: "acute" + id: 100 + codes: [ "4100", "41000", "41001", "41002", "4101", "41010", "41011", "41012", "4102", "41020", "41021", "41022", "4103", "41030", "41031", "41032", "4104", "41040", "41041", "41042", "4105", "41050", "41051", "41052", "4106", "41060", "41061", "41062", "4107", "41070", "41071", "41072", "4108", "41080", "41081", "41082", "4109", "41090", "41091", "41092" ] + +"Coronary atherosclerosis and other heart disease": + use_in_benchmark: True + type: "chronic" + id: 101 + codes: [ "4110", "4111", "4118", "41181", "41189", "412", "4130", "4131", "4139", "4140", "41400", "41401", "41406", "4142", "4143", "4144", "4148", "4149", "V4581", "V4582" ] + +"Conduction disorders": + use_in_benchmark: True + type: "chronic" + id: 105 + codes: [ "4260", "42610", "42611", "42612", "42613", "4262", "4263", "4264", "42650", "42651", "42652", "42653", "42654", "4266", "4267", "42681", "42682", "42689", "4269", "V450", "V4500", "V4501", "V4502", "V4509", "V533", "V5331", "V5332", "V5339" ] + +"Cardiac dysrhythmias": + use_in_benchmark: True + type: "chronic" + id: 106 + codes: [ "4270", "4271", "4272", "42731", "42732", "42760", "42761", "42769", "42781", "42789", "4279", "7850", "7851" ] + +"Congestive heart failure; nonhypertensive": + use_in_benchmark: True + type: "acute" + id: 108 + codes: [ "39891", "4280", "4281", "42820", "42821", "42822", "42823", "42830", "42831", "42832", "42833", "42840", "42841", "42842", "42843", "4289" ] + +"Acute cerebrovascular disease": + use_in_benchmark: True + type: "acute" + id: 109 + codes: [ "34660", "34661", "34662", "34663", "430", "431", "4320", "4321", "4329", "43301", "43311", "43321", "43331", "43381", "43391", "4340", "43400", "43401", "4341", "43410", "43411", "4349", "43490", "43491", "436" ] + +"Pneumonia (except that caused by tuberculosis or sexually transmitted disease)": + use_in_benchmark: True + type: "acute" + id: 122 + codes: [ "00322", "0203", "0204", "0205", "0212", "0221", "0310", "0391", "0521", "0551", "0730", "0830", "1124", "1140", "1144", "1145", "11505", "11515", "11595", "1304", "1363", "4800", "4801", "4802", "4803", "4808", "4809", "481", "4820", "4821", "4822", "4823", "48230", "48231", "48232", "48239", "4824", "48240", "48241", "48242", "48249", "4828", "48281", "48282", "48283", "48284", "48289", "4829", "483", "4830", "4831", "4838", "4841", "4843", "4845", "4846", "4847", "4848", "485", "486", "5130", "5171" ] + +"Chronic obstructive pulmonary disease and bronchiectasis": + use_in_benchmark: True + type: "chronic" + id: 127 + codes: [ "490", "4910", "4911", "4912", "49120", "49121", "49122", "4918", "4919", "4920", "4928", "494", "4940", "4941", "496" ] + +"Pleurisy; pneumothorax; pulmonary collapse": + use_in_benchmark: True + type: "acute" + id: 130 + codes: [ "5100", "5109", "5110", "5111", "5118", "51189", "5119", "5120", "5128", "51281", "51282", "51283", "51284", "51289", "5180", "5181", "5182" ] + +"Respiratory failure; insufficiency; arrest (adult)": + use_in_benchmark: True + type: "acute" + id: 131 + codes: [ "5173", "5185", "51851", "51852", "51853", "51881", "51882", "51883", "51884", "7991", "V461", "V4611", "V4612", "V4613", "V4614", "V462" ] + +"Other lower respiratory disease": + use_in_benchmark: True + type: "acute" + id: 133 + codes: [ "5131", "514", "515", "5160", "5161", "5162", "5163", "51630", "51631", "51632", "51633", "51634", "51635", "51636", "51637", "5164", "5165", "51661", "51662", "51663", "51664", "51669", "5168", "5169", "5172", "5178", "5183", "5184", "51889", "5194", "5198", "5199", "7825", "78600", "78601", "78602", "78603", "78604", "78605", "78606", "78607", "78609", "7862", "7863", "78630", "78631", "78639", "7864", "78652", "7866", "7867", "7868", "7869", "7931", "79311", "79319", "7942", "V126", "V1260", "V1261", "V1269", "V426" ] + +"Other upper respiratory disease": + use_in_benchmark: True + type: "acute" + id: 134 + codes: [ "470", "4710", "4711", "4718", "4719", "4720", "4721", "4722", "4760", "4761", "4770", "4772", "4778", "4779", "4780", "4781", "47811", "47819", "47820", "47821", "47822", "47824", "47825", "47826", "47829", "47830", "47831", "47832", "47833", "47834", "4784", "4785", "4786", "47870", "47871", "47874", "47875", "47879", "4788", "4789", "5191", "51911", "51919", "5192", "5193", "7841", "78440", "78441", "78442", "78443", "78444", "78449", "7847", "7848", "7849", "78499", "7861", "V414", "V440", "V550" ] + +"Other liver diseases": + use_in_benchmark: True + type: "acute" + id: 151 + codes: [ "570", "5715", "5716", "5718", "5719", "5720", "5721", "5722", "5723", "5724", "5728", "5730", "5734", "5735", "5738", "5739", "7824", "7891", "7895", "78959", "7904", "7905", "7948", "V427" ] + +"Gastrointestinal hemorrhage": + use_in_benchmark: True + type: "acute" + id: 153 + codes: [ "4560", "45620", "5307", "53082", "53100", "53101", "53120", "53121", "53140", "53141", "53160", "53161", "53200", "53201", "53220", "53221", "53240", "53241", "53260", "53261", "53300", "53301", "53320", "53321", "53340", "53341", "53360", "53361", "53400", "53401", "53420", "53421", "53440", "53441", "53460", "53461", "5693", "5780", "5781", "5789" ] + +"Acute and unspecified renal failure": + use_in_benchmark: True + type: "acute" + id: 157 + codes: [ "5845", "5846", "5847", "5848", "5849", "586" ] + +"Chronic kidney disease": + use_in_benchmark: True + type: "chronic" + id: 158 + codes: [ "585", "5851", "5852", "5853", "5854", "5855", "5856", "5859", "7925", "V420", "V451", "V4511", "V4512", "V560", "V561", "V562", "V5631", "V5632", "V568" ] + +"Complications of surgical procedures or medical care": + use_in_benchmark: True + type: "acute" + id: 238 + codes: [ "27661", "27783", "27788", "2853", "28741", "3490", "3491", "34931", "41511", "4294", "4582", "45821", "45829", "5121", "5122", "5187", "5190", "51900", "51901", "51902", "51909", "53086", "53087", "53640", "53641", "53642", "53649", "53901", "53909", "53981", "53989", "5642", "5643", "5644", "5696", "56962", "56971", "56979", "5793", "59681", "78062", "78063", "78066", "9093", "99524", "9954", "99586", "9970", "99700", "99701", "99702", "99709", "9971", "9972", "9973", "99731", "99732", "99739", "9974", "99741", "99749", "9975", "99760", "99761", "99762", "99769", "99771", "99772", "99779", "9979", "99791", "99799", "9980", "99800", "99801", "99802", "99809", "9981", "99811", "99812", "99813", "9982", "9983", "99830", "99831", "99832", "99833", "9984", "9985", "99851", "99859", "9986", "9987", "9988", "99881", "99882", "99883", "99889", "9989", "9990", "9991", "9992", "9993", "99934", "99939", "9994", "99941", "99942", "99949", "9995", "99951", "99952", "99959", "9996", "99960", "99961", "99962", "99963", "99969", "9997", "99970", "99971", "99972", "99973", "99974", "99975", "99976", "99977", "99978", "99979", "9998", "99980", "99981", "99982", "99983", "99984", "99985", "99988", "99989", "9999", "V1553", "V1580", "V1583", "V9001", "V9009" ] + +"Shock": + use_in_benchmark: True + type: "acute" + id: 249 + codes: [ "78550", "78551", "78552", "78559" ] \ No newline at end of file diff --git a/pyhealth/datasets/halo_mimic3.py b/pyhealth/datasets/halo_mimic3.py new file mode 100644 index 000000000..b23452fed --- /dev/null +++ b/pyhealth/datasets/halo_mimic3.py @@ -0,0 +1,155 @@ +import logging +import warnings +from pathlib import Path +from typing import List, Optional + +import polars as pl + +from .base_dataset import BaseDataset + +import yaml +import pickle +import numpy as np +import pandas as pd +from tqdm import tqdm +from sklearn.model_selection import train_test_split + +logger = logging.getLogger(__name__) + +# TODO: Clean up imports & update docs correctly + + +class HALO_MIMIC3Dataset: + """ + A dataset class for handling MIMIC-III data, specifically designed to be compatible with HALO. + + This class is responsible for loading and managing the MIMIC-III dataset, + which includes tables such as patients, admissions, and icustays. + + Attributes: + root (str): The root directory where the dataset is stored. + tables (List[str]): A list of tables to be included in the dataset. + dataset_name (Optional[str]): The name of the dataset. + config_path (Optional[str]): The path to the configuration file. + """ + + def __init__( + self, + mimic3_dir: str = "./", + pkl_data_dir: str = "./", + ) -> None: + """ + Initializes the MIMIC4Dataset with the specified parameters. + + Args: + root (str): The root directory where the dataset is stored. + tables (List[str]): A list of additional tables to include. + dataset_name (Optional[str]): The name of the dataset. Defaults to "mimic3". + config_path (Optional[str]): The path to the configuration file. If not provided, a default config is used. + """ + self.mimic3_dir = mimic3_dir + self.pkl_data_dir = pkl_data_dir + self.build_dataset() + + def build_dataset(self) -> None: + admissionFile = self.mimic3_dir + "ADMISSIONS.csv" + diagnosisFile = self.mimic3_dir + "DIAGNOSES_ICD.csv" + + admissionDf = pd.read_csv(admissionFile, dtype=str) + admissionDf['ADMITTIME'] = pd.to_datetime(admissionDf['ADMITTIME']) + admissionDf = admissionDf.sort_values('ADMITTIME') + admissionDf = admissionDf.reset_index(drop=True) + diagnosisDf = pd.read_csv(diagnosisFile, dtype=str).set_index("HADM_ID") + diagnosisDf = diagnosisDf[diagnosisDf['ICD9_CODE'].notnull()] + diagnosisDf = diagnosisDf[['ICD9_CODE']] + + data = {} + for row in tqdm(admissionDf.itertuples(), total=admissionDf.shape[0]): + #Extracting Admissions Table Info + hadm_id = row.HADM_ID + subject_id = row.SUBJECT_ID + + # Extracting the Diagnoses + if hadm_id in diagnosisDf.index: + diagnoses = list(set(diagnosisDf.loc[[hadm_id]]["ICD9_CODE"])) + else: + diagnoses = [] + + # Building the hospital admission data point + if subject_id not in data: + data[subject_id] = {'visits': [diagnoses]} + else: + data[subject_id]['visits'].append(diagnoses) + + code_to_index = {} + all_codes = list(set([c for p in data.values() for v in p['visits'] for c in v])) + np.random.shuffle(all_codes) + for c in all_codes: + code_to_index[c] = len(code_to_index) + # print(f"VOCAB SIZE: {len(code_to_index)}") + index_to_code = {v: k for k, v in code_to_index.items()} + + data = list(data.values()) + + # print("Adding Labels") + with open("./configs/hcup_ccs_2015_definitions_benchmark.yaml") as definitions_file: + definitions = yaml.full_load(definitions_file) + + code_to_group = {} + for group in definitions: + if definitions[group]['use_in_benchmark'] == False: + continue + codes = definitions[group]['codes'] + for code in codes: + if code not in code_to_group: + code_to_group[code] = group + else: + assert code_to_group[code] == group + + id_to_group = sorted([k for k in definitions.keys() if definitions[k]['use_in_benchmark'] == True]) + group_to_id = dict((x, i) for (i, x) in enumerate(id_to_group)) + + # Add Labels + for p in data: + label = np.zeros(len(group_to_id)) + for v in p['visits']: + for c in v: + if c in code_to_group: + label[group_to_id[code_to_group[c]]] = 1 + + p['labels'] = label + + # print("Converting Visits") + for p in data: + new_visits = [] + for v in p['visits']: + new_visit = [] + for c in v: + new_visit.append(code_to_index[c]) + + new_visits.append((list(set(new_visit)))) + + p['visits'] = new_visits + + # print(f"MAX LEN: {max([len(p['visits']) for p in data])}") + # print(f"AVG LEN: {np.mean([len(p['visits']) for p in data])}") + # print(f"MAX VISIT LEN: {max([len(v) for p in data for v in p['visits']])}") + # print(f"AVG VISIT LEN: {np.mean([len(v) for p in data for v in p['visits']])}") + # print(f"NUM RECORDS: {len(data)}") + # print(f"NUM LONGITUDINAL RECORDS: {len([p for p in data if len(p['visits']) > 1])}") + + # Train-Val-Test Split + # print("Splitting Datasets") + train_dataset, test_dataset = train_test_split(data, test_size=0.2, random_state=4, shuffle=True) + train_dataset, val_dataset = train_test_split(train_dataset, test_size=0.1, random_state=4, shuffle=True) + + # Save Everything + # print("Saving Everything") + # print(len(index_to_code)) + pickle.dump(code_to_index, open(f"{self.pkl_data_dir}codeToIndex.pkl", "wb")) + pickle.dump(index_to_code, open(f"{self.pkl_data_dir}indexToCode.pkl", "wb")) + pickle.dump(id_to_group, open(f"{self.pkl_data_dir}idToLabel.pkl", "wb")) + pickle.dump(train_dataset, open(f"{self.pkl_data_dir}trainDataset.pkl", "wb")) + pickle.dump(val_dataset, open(f"{self.pkl_data_dir}valDataset.pkl", "wb")) + pickle.dump(test_dataset, open(f"{self.pkl_data_dir}testDataset.pkl", "wb")) + diff --git a/pyhealth/models/generators/halo.py b/pyhealth/models/generators/halo.py index 7a2437e34..e5ecd4eaa 100644 --- a/pyhealth/models/generators/halo.py +++ b/pyhealth/models/generators/halo.py @@ -2,154 +2,483 @@ import torch.nn as nn from typing import Any, Dict, List, Optional -from pyhealth.datasets import SampleEHRDataset +import os +import numpy as np +import random +import pickle +from tqdm import tqdm +from sklearn import metrics + +from pyhealth.datasets import SampleEHRDataset, HALO_MIMIC3Dataset from pyhealth.models.base_model import BaseModel # Import the HALO transformer implementation from pyhealth.models.generators.halo_resources.halo_model import HALOModel from pyhealth.models.generators.halo_resources.halo_model import HALOConfig -class HALO(BaseModel): - """ - HALO model wrapper for PyHealth, leveraging the BaseModel interface. - - Args: - dataset: SampleEHRDataset for EHR sequences (visits of codes). - config: Transformer configuration object for HALOModel. - feature_key: key in dataset samples for code-based visits (e.g., "list_codes"). - mode: one of "binary", "multiclass", or "multilabel" for loss function. - label_key: key in dataset samples for target visits; defaults to feature_key for next-visit prediction. - pos_loss_weight: weight applied to positive labels in BCE loss. - """ +class HALO: + def __init__( self, - dataset: SampleEHRDataset, + dataset: HALO_MIMIC3Dataset, config: HALOConfig, - feature_key: str = "list_codes", - mode: str = "multilabel", - label_key: Optional[str] = None, - pos_loss_weight: float = 1.0, - ): - super(HALO, self).__init__(dataset) - self.feature_key = feature_key - self.label_key = label_key or feature_key - self.pos_loss_weight = pos_loss_weight - - # Set mode for loss and evaluation - self.mode = mode - - # Tokenizer for the code-based input - self.tokenizer = dataset.input_processors[feature_key] - self.vocab_size = self.tokenizer.size() - - # Instantiate the underlying HALO transformer model - self.halo = HALOModel(config) - - def _prepare_input_visits(self, codes: List[List[Any]]) -> torch.Tensor: - """ - Convert list of visits of codes into multi-hot tensor. - - Args: - codes: nested list of shape (batch, num_visits, codes_in_visit) - - Returns: - Tensor of shape (batch, num_visits, vocab_size) with 0/1 entries. - """ - # batch_encode_3d returns List[List[List[int]]] - token_ids = self.tokenizer.batch_encode_3d(codes) - batch_size = len(token_ids) - max_visits = len(token_ids[0]) - - visits = torch.zeros( - batch_size, max_visits, self.vocab_size, device=self.device - ) - for i in range(batch_size): - for t, visit_ids in enumerate(token_ids[i]): - for cid in visit_ids: - if cid is None: - continue - visits[i, t, cid] = 1.0 - return visits + save_dir: str = "./save/" + ) -> None: + SEED = 4 + random.seed(SEED) + np.random.seed(SEED) + torch.manual_seed(SEED) + self.config = config + self.dataset = dataset + self.save_dir = save_dir - def forward( - self, - **kwargs: Any - ) -> Dict[str, torch.Tensor]: - """ - Forward propagation for HALO within PyHealth. - - Expects kwargs to contain: - - feature_key: raw visit code lists. - - optional label_key: same format for next-visit targets. - - optional masks: tensor or nested lists for masking visits. - - Returns a dict with keys: - - loss: training loss (if labels provided). - - y_prob: predicted probabilities of codes per visit. - - y_true: ground-truth multi-hot labels (shifted by one visit). - - logits: raw logits from the HALO transformer. - """ - # Prepare input tensor - raw_codes = kwargs[self.feature_key] - input_visits = self._prepare_input_visits(raw_codes) - - # Gather optional training labels and masks - ehr_labels = None - ehr_masks = None - if self.label_key in kwargs: - # similarly convert label visits to multi-hot - ehr_labels = self._prepare_input_visits(kwargs[self.label_key]) - if "masks" in kwargs: - ehr_masks = torch.tensor(kwargs["masks"], device=self.device, dtype=torch.float) - - # Call HALOModel: returns loss & probabilities if labels, else probabilities - if ehr_labels is not None: - loss, code_probs, shift_labels = self.halo( - input_visits, - ehr_labels=ehr_labels, - ehr_masks=ehr_masks, - pos_loss_weight=self.pos_loss_weight, - ) - results = {"loss": loss, "y_prob": code_probs, "y_true": shift_labels} + local_rank = -1 + fp16 = False + if local_rank == -1: + self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + self.n_gpu = torch.cuda.device_count() else: - code_probs = self.halo(input_visits) - results = {"y_prob": code_probs} - - # Attach logits if needed - if hasattr(self.halo, 'last_logits'): - results['logits'] = self.halo.last_logits - - return results - -# Example usage: -if __name__ == "__main__": - from pyhealth.datasets import SampleEHRDataset - # define samples with visits of codes as nested lists - samples = [ - {"patient_id": "p0", "visit_id": "v0", "list_codes": [["A", "B"], ["C"]]}, - {"patient_id": "p1", "visit_id": "v0", "list_codes": [["D"], ["E", "F"]]}, - ] - dataset = SampleEHRDataset(samples=samples, dataset_name="halo_test") - - # Build transformer config - config = HALOConfig( - n_layer=4, - n_head=8, - n_embd=128, - total_vocab_size=dataset.input_processors['list_codes'].size(), - n_positions=dataset.max_visit_length, - n_ctx=dataset.max_visit_length, - layer_norm_epsilon=1e-5, - ) - - model = HALO( - dataset=dataset, - config=config, - feature_key='list_codes', - mode='multilabel', - ) - from pyhealth.datasets import get_dataloader - loader = get_dataloader(dataset, batch_size=2, shuffle=False) - batch = next(iter(loader)) - output = model(**batch) - print(output) + torch.cuda.set_device(local_rank) + self.device = torch.device("cuda", local_rank) + self.n_gpu = 1 + # Initializes the distributed backend which will take care of sychronizing nodes/GPUs + torch.distributed.init_process_group(backend='nccl') + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(SEED) + + self.train_ehr_dataset = pickle.load(open(f'{self.dataset.pkl_data_dir}trainDataset.pkl', 'rb')) + self.val_ehr_dataset = pickle.load(open(f'{self.dataset.pkl_data_dir}valDataset.pkl', 'rb')) + self.index_to_code = pickle.load(open(f"{self.dataset.pkl_data_dir}indexToCode.pkl", "rb")) + test_ehr_dataset = pickle.load(open(f'{self.dataset.pkl_data_dir}testDataset.pkl', 'rb')) + train_c = set([c for p in train_ehr_dataset for v in p['visits'] for c in v]) + self.test_ehr_dataset = [{'labels': p['labels'], 'visits': [[c for c in v if c in train_c] for v in p['visits']]} for p in test_ehr_dataset] + + self.train() + + + def train(self) -> None: + + # HELPER: + def get_batch(loc, batch_size, mode): + # EHR data saved as [(P_1, L_1), (P_2, L_2), ... , (P_i, L_i)] + # Where each patient P is [V_1, V_2, ... , V_j] + # Where each visit V is [C_1, C_2, ... , C_k] + # And where each Label L is a binary vector [L_1 ... L_n] + if mode == 'train': + ehr = self.train_ehr_dataset[loc:loc+batch_size] + elif mode == 'valid': + ehr = self.val_ehr_dataset[loc:loc+batch_size] + else: + ehr = self.test_ehr_dataset[loc:loc+batch_size] + + batch_ehr = np.zeros((len(ehr), self.config.n_ctx, self.config.total_vocab_size)) + batch_mask = np.zeros((len(ehr), self.config.n_ctx, 1)) + for i, p in enumerate(ehr): + visits = p['visits'] + for j, v in enumerate(visits): + batch_ehr[i,j+2][v] = 1 + batch_mask[i,j+2] = 1 + batch_ehr[i,1,self.config.code_vocab_size:self.config.code_vocab_size+self.config.label_vocab_size] = np.array(p['labels']) # Set the patient labels + batch_ehr[i,len(visits)+1,self.config.code_vocab_size+self.config.label_vocab_size+1] = 1 # Set the final visit to have the end token + batch_ehr[i,len(visits)+2:,self.config.code_vocab_size+self.config.label_vocab_size+2] = 1 # Set the rest to the padded visit token + + batch_mask[:,1] = 1 # Set the mask to cover the labels + batch_ehr[:,0,self.config.code_vocab_size+self.config.label_vocab_size] = 1 # Set the first visits to be the start token + batch_mask = batch_mask[:,1:,:] # Shift the mask to match the shifted labels and predictions the model will return + return batch_ehr, batch_mask + + # HELPER + def shuffle_training_data(train_ehr_dataset): + np.random.shuffle(train_ehr_dataset) + + # TRAIN PIPELINE: + + # Load any previous checkpoint if exists + self.model = HALOModel(self.config).to(self.device) + self.optimizer = torch.optim.Adam(self.model.parameters(), lr=self.config.lr) + if os.path.exists(f"{self.save_dir}halo_model"): + # print("Loading previous model") + checkpoint = torch.load(f'{self.save_dir}halo_model', map_location=torch.device(self.device)) + self.model.load_state_dict(checkpoint['model']) + self.optimizer.load_state_dict(checkpoint['optimizer']) + + # Train + global_loss = 1e10 + for e in tqdm(range(self.config.epoch)): + shuffle_training_data(self.train_ehr_dataset) + for i in range(0, len(self.train_ehr_dataset), self.config.batch_size): + self.model.train() + + batch_ehr, batch_mask = get_batch(i, self.config.batch_size, 'train') + batch_ehr = torch.tensor(batch_ehr, dtype=torch.float32).to(self.device) + batch_mask = torch.tensor(batch_mask, dtype=torch.float32).to(self.device) + + self.optimizer.zero_grad() + loss, _, _ = self.model(batch_ehr, position_ids=None, ehr_labels=batch_ehr, ehr_masks=batch_mask, pos_loss_weight=self.config.pos_loss_weight) + loss.backward() + self.optimizer.step() + + # if i % (500*self.config.batch_size) == 0: + # print("Epoch %d, Iter %d: Training Loss:%.6f"%(e, i, loss * 8)) + if i % (500*self.config.batch_size) == 0: + if i == 0: + continue + + self.model.eval() + with torch.no_grad(): + val_l = [] + for v_i in range(0, len(self.val_ehr_dataset), self.config.batch_size): + batch_ehr, batch_mask = get_batch(v_i, self.config.batch_size, 'valid') + batch_ehr = torch.tensor(batch_ehr, dtype=torch.float32).to(self.device) + batch_mask = torch.tensor(batch_mask, dtype=torch.float32).to(self.device) + + val_loss, _, _ = self.model(batch_ehr, position_ids=None, ehr_labels=batch_ehr, ehr_masks=batch_mask, pos_loss_weight=self.config.pos_loss_weight) + val_l.append((val_loss).cpu().detach().numpy()) + + cur_val_loss = np.mean(val_l) + # print("Epoch %d Validation Loss:%.7f"%(e, cur_val_loss)) + if cur_val_loss < global_loss: + global_loss = cur_val_loss + state = { + 'model': self.model.state_dict(), + 'optimizer': self.optimizer.state_dict(), + 'iteration': i + } + torch.save(state, f'{self.save_dir}halo_model') + # print('\n------------ Save best model ------------\n') + + def test(self, testing_results_dir: str = "./results/testing_stats/") -> None: + + ## HELPER FUNC: + def get_batch(loc, batch_size, mode): + # EHR data saved as [(P_1, L_1), (P_2, L_2), ... , (P_i, L_i)] + # Where each patient P is [V_1, V_2, ... , V_j] + # Where each visit V is [C_1, C_2, ... , C_k] + # And where each Label L is a binary vector [L_1 ... L_n] + if mode == 'train': + ehr = self.train_ehr_dataset[loc:loc+batch_size] + elif mode == 'valid': + ehr = self.val_ehr_dataset[loc:loc+batch_size] + else: + ehr = self.test_ehr_dataset[loc:loc+batch_size] + + batch_ehr = np.zeros((len(ehr), self.config.n_ctx, self.config.total_vocab_size)) + batch_mask = np.zeros((len(ehr), self.config.n_ctx, 1)) + for i, p in enumerate(ehr): + visits = p['visits'] + for j, v in enumerate(visits): + batch_ehr[i,j+2][v] = 1 + batch_mask[i,j+2] = 1 + batch_ehr[i,1,self.config.code_vocab_size:self.config.code_vocab_size+self.config.label_vocab_size] = np.array(p['labels']) # Set the patient labels + batch_ehr[i,len(visits)+1,self.config.code_vocab_size+self.config.label_vocab_size+1] = 1 # Set the final visit to have the end token + batch_ehr[i,len(visits)+2:,self.config.code_vocab_size+self.config.label_vocab_size+2] = 1 # Set the rest to the padded visit token + + batch_mask[:,1] = 1 # Set the mask to cover the labels + batch_ehr[:,0,self.config.code_vocab_size+self.config.label_vocab_size] = 1 # Set the first visits to be the start token + batch_mask = batch_mask[:,1:,:] # Shift the mask to match the shifted labels and predictions the model will return + return batch_ehr, batch_mask + + ## HELPER FUNC: + def conf_mat(x, y): + totaltrue = np.sum(x) + totalfalse = len(x) - totaltrue + truepos, totalpos = np.sum(x & y), np.sum(y) + falsepos = totalpos - truepos + return np.array([[totalfalse - falsepos, falsepos], #true negatives, false positives + [totaltrue - truepos, truepos]]) #false negatives, true positives + + ## MAIN TEST FUNC: + checkpoint = torch.load(f'{self.save_dir}halo_model', map_location=torch.device(self.device)) + self.model.load_state_dict(checkpoint['model']) + self.optimizer.load_state_dict(checkpoint['optimizer']) + + confusion_matrix = None + probability_list = [] + loss_list = [] + n_visits = 0 + n_pos_codes = 0 + n_total_codes = 0 + self.model.eval() + with torch.no_grad(): + for v_i in tqdm(range(0, len(self.test_ehr_dataset), 2*self.config.batch_size)): + # Get batch inputs + batch_ehr, batch_mask = get_batch(v_i, 2*self.config.batch_size, 'test') + batch_ehr = torch.tensor(batch_ehr, dtype=torch.float32).to(self.device) + batch_mask = torch.tensor(batch_mask, dtype=torch.float32).to(self.device) + + # Get batch outputs + test_loss, predictions, labels = self.model(batch_ehr, position_ids=None, ehr_labels=batch_ehr, ehr_masks=batch_mask, pos_loss_weight=self.config.pos_loss_weight) + batch_mask_array = batch_mask.squeeze().cpu().detach().numpy() + rounded_preds = np.around(predictions.squeeze().cpu().detach().numpy()).transpose((2,0,1)) + rounded_preds = rounded_preds + batch_mask_array - 1 # Setting the masked visits to be -1 to be ignored by the confusion matrix + rounded_preds = rounded_preds.flatten() + true_values = labels.squeeze().cpu().detach().numpy().transpose((2,0,1)) + true_values = true_values + batch_mask_array - 1 # Setting the masked visits to be -1 to be ignored by the confusion matrix + true_values = true_values.flatten() + + # Append test lost + loss_list.append(test_loss.cpu().detach().numpy()) + + # Add number of visits and codes + n_visits += torch.sum(batch_mask).cpu().item() + n_pos_codes += torch.sum(labels).cpu().item() + n_total_codes += (torch.sum(batch_mask) * self.config.total_vocab_size).cpu().item() + + # Add confusion matrix + batch_cmatrix = conf_mat(true_values == 1, rounded_preds == 1) + batch_cmatrix[0][0] = torch.sum(batch_mask) * self.config.total_vocab_size - batch_cmatrix[0][1] - batch_cmatrix[1][0] - batch_cmatrix[1][1] # Remove the masked values + confusion_matrix = batch_cmatrix if confusion_matrix is None else confusion_matrix + batch_cmatrix + + # Calculate and add probabilities + # Note that the masked codes will have probability 1 and be ignored + label_probs = torch.abs(labels - 1.0 + predictions) + log_prob = torch.sum(torch.log(label_probs)).cpu().item() + probability_list.append(log_prob) + + # Save intermediate values in case of error + intermediate = {} + intermediate["Losses"] = loss_list + intermediate["Confusion Matrix"] = confusion_matrix + intermediate["Probabilities"] = probability_list + intermediate["Num Visits"] = n_visits + intermediate["Num Positive Codes"] = n_pos_codes + intermediate["Num Total Codes"] = n_total_codes + pickle.dump(intermediate, open(f"{testing_results_dir}HALO_intermediate_results.pkl", "wb")) + + #Extract, save, and display test metrics + avg_loss = np.nanmean(loss_list) + tn, fp, fn, tp = confusion_matrix.ravel() + acc = (tn + tp)/(tn+fp+fn+tp) + prc = tp/(tp+fp) + rec = tp/(tp+fn) + f1 = (2 * prc * rec)/(prc + rec) + log_probability = np.sum(probability_list) + pp_visit = np.exp(-log_probability/n_visits) + pp_positive = np.exp(-log_probability/n_pos_codes) + pp_possible = np.exp(-log_probability/n_total_codes) + + metrics_dict = {} + metrics_dict['Test Loss'] = avg_loss + metrics_dict['Confusion Matrix'] = confusion_matrix + metrics_dict['Accuracy'] = acc + metrics_dict['Precision'] = prc + metrics_dict['Recall'] = rec + metrics_dict['F1 Score'] = f1 + metrics_dict['Test Log Probability'] = log_probability + metrics_dict['Perplexity Per Visit'] = pp_visit + metrics_dict['Perplexity Per Positive Code'] = pp_positive + metrics_dict['Perplexity Per Possible Code'] = pp_possible + pickle.dump(metrics_dict, open(f"{testing_results_dir}HALO_Metrics.pkl", "wb")) + + + def convert_ehr(self, ehrs, index_to_code=None): + ehr_outputs = [] + for i in range(len(ehrs)): + ehr = ehrs[i] + ehr_output = [] + labels_output = ehr[1][self.config.code_vocab_size:self.config.code_vocab_size+self.config.label_vocab_size] + if index_to_code is not None: + labels_output = [index_to_code[idx + self.config.code_vocab_size] for idx in np.nonzero(labels_output)[0]] + for j in range(2, len(ehr)): + visit = ehr[j] + visit_output = [] + indices = np.nonzero(visit)[0] + end = False + for idx in indices: + if idx < self.config.code_vocab_size: + visit_output.append(index_to_code[idx] if index_to_code is not None else idx) + elif idx == self.config.code_vocab_size+self.config.label_vocab_size+1: + end = True + if visit_output != []: + ehr_output.append(visit_output) + if end: + break + ehr_outputs.append({'visits': ehr_output, 'labels': labels_output}) + ehr = None + ehr_output = None + labels_output = None + visit = None + visit_output = None + indices = None + return ehr_outputs + + + def synthesize_dataset(self, pkl_save_dir: str = "./results/datasets/haloDataset.pkl") -> None: + + ## HELPER: + def sample_sequence(model, length, context, batch_size, device='cuda', sample=True): + empty = torch.zeros((1,1,self.config.total_vocab_size), device=device, dtype=torch.float32).repeat(batch_size, 1, 1) + context = torch.tensor(context, device=device, dtype=torch.float32).unsqueeze(0).repeat(batch_size, 1) + prev = context.unsqueeze(1) + context = None + with torch.no_grad(): + for _ in range(length-1): + prev = model.sample(torch.cat((prev,empty), dim=1), sample) + if torch.sum(torch.sum(prev[:,:,self.config.code_vocab_size+self.config.label_vocab_size+1], dim=1).bool().int(), dim=0).item() == batch_size: + break + ehr = prev.cpu().detach().numpy() + prev = None + empty = None + return ehr + + ## MAIN FUNC: + synthetic_ehr_dataset = [] + stoken = np.zeros(self.config.total_vocab_size) + stoken[self.config.code_vocab_size+self.config.label_vocab_size] = 1 + for i in tqdm(range(0, len(self.train_ehr_dataset), self.config.sample_batch_size)): + bs = min([len(self.train_ehr_dataset)-i, self.config.sample_batch_size]) + batch_synthetic_ehrs = sample_sequence(self.model, self.config.n_ctx, stoken, batch_size=bs, device=self.device, sample=True) + batch_synthetic_ehrs = self.convert_ehr(batch_synthetic_ehrs) + synthetic_ehr_dataset += batch_synthetic_ehrs + + pickle.dump(synthetic_ehr_dataset, open(pkl_save_dir, 'wb')) + + + + + + +# class HALO(BaseModel): +# """ +# HALO model wrapper for PyHealth, leveraging the BaseModel interface. + +# Args: +# dataset: SampleEHRDataset for EHR sequences (visits of codes). +# config: Transformer configuration object for HALOModel. +# feature_key: key in dataset samples for code-based visits (e.g., "list_codes"). +# mode: one of "binary", "multiclass", or "multilabel" for loss function. +# label_key: key in dataset samples for target visits; defaults to feature_key for next-visit prediction. +# pos_loss_weight: weight applied to positive labels in BCE loss. +# """ +# def __init__( +# self, +# dataset: SampleEHRDataset, +# config: HALOConfig, +# feature_key: str = "list_codes", +# mode: str = "multilabel", +# label_key: Optional[str] = None, +# pos_loss_weight: float = 1.0, +# ): +# super(HALO, self).__init__(dataset) +# self.feature_key = feature_key +# self.label_key = label_key or feature_key +# self.pos_loss_weight = pos_loss_weight + +# # Set mode for loss and evaluation +# self.mode = mode + +# # Tokenizer for the code-based input +# self.tokenizer = dataset.input_processors[feature_key] +# self.vocab_size = self.tokenizer.size() + +# # Instantiate the underlying HALO transformer model +# self.halo = HALOModel(config) + +# def _prepare_input_visits(self, codes: List[List[Any]]) -> torch.Tensor: +# """ +# Convert list of visits of codes into multi-hot tensor. + +# Args: +# codes: nested list of shape (batch, num_visits, codes_in_visit) + +# Returns: +# Tensor of shape (batch, num_visits, vocab_size) with 0/1 entries. +# """ +# # batch_encode_3d returns List[List[List[int]]] +# token_ids = self.tokenizer.batch_encode_3d(codes) +# batch_size = len(token_ids) +# max_visits = len(token_ids[0]) + +# visits = torch.zeros( +# batch_size, max_visits, self.vocab_size, device=self.device +# ) +# for i in range(batch_size): +# for t, visit_ids in enumerate(token_ids[i]): +# for cid in visit_ids: +# if cid is None: +# continue +# visits[i, t, cid] = 1.0 +# return visits + +# def forward( +# self, +# **kwargs: Any +# ) -> Dict[str, torch.Tensor]: +# """ +# Forward propagation for HALO within PyHealth. + +# Expects kwargs to contain: +# - feature_key: raw visit code lists. +# - optional label_key: same format for next-visit targets. +# - optional masks: tensor or nested lists for masking visits. + +# Returns a dict with keys: +# - loss: training loss (if labels provided). +# - y_prob: predicted probabilities of codes per visit. +# - y_true: ground-truth multi-hot labels (shifted by one visit). +# - logits: raw logits from the HALO transformer. +# """ +# # Prepare input tensor +# raw_codes = kwargs[self.feature_key] +# input_visits = self._prepare_input_visits(raw_codes) + +# # Gather optional training labels and masks +# ehr_labels = None +# ehr_masks = None +# if self.label_key in kwargs: +# # similarly convert label visits to multi-hot +# ehr_labels = self._prepare_input_visits(kwargs[self.label_key]) +# if "masks" in kwargs: +# ehr_masks = torch.tensor(kwargs["masks"], device=self.device, dtype=torch.float) + +# # Call HALOModel: returns loss & probabilities if labels, else probabilities +# if ehr_labels is not None: +# loss, code_probs, shift_labels = self.halo( +# input_visits, +# ehr_labels=ehr_labels, +# ehr_masks=ehr_masks, +# pos_loss_weight=self.pos_loss_weight, +# ) +# results = {"loss": loss, "y_prob": code_probs, "y_true": shift_labels} +# else: +# code_probs = self.halo(input_visits) +# results = {"y_prob": code_probs} + +# # Attach logits if needed +# if hasattr(self.halo, 'last_logits'): +# results['logits'] = self.halo.last_logits + +# return results + +# # Example usage: +# if __name__ == "__main__": +# from pyhealth.datasets import SampleEHRDataset +# # define samples with visits of codes as nested lists +# samples = [ +# {"patient_id": "p0", "visit_id": "v0", "list_codes": [["A", "B"], ["C"]]}, +# {"patient_id": "p1", "visit_id": "v0", "list_codes": [["D"], ["E", "F"]]}, +# ] +# dataset = SampleEHRDataset(samples=samples, dataset_name="halo_test") + +# # Build transformer config +# config = HALOConfig( +# n_layer=4, +# n_head=8, +# n_embd=128, +# total_vocab_size=dataset.input_processors['list_codes'].size(), +# n_positions=dataset.max_visit_length, +# n_ctx=dataset.max_visit_length, +# layer_norm_epsilon=1e-5, +# ) + +# model = HALO( +# dataset=dataset, +# config=config, +# feature_key='list_codes', +# mode='multilabel', +# ) +# from pyhealth.datasets import get_dataloader +# loader = get_dataloader(dataset, batch_size=2, shuffle=False) +# batch = next(iter(loader)) +# output = model(**batch) +# print(output) From b666f82fd9e708a410c80cba6e193e0bdeef06ab Mon Sep 17 00:00:00 2001 From: Rasmussen Date: Sun, 27 Jul 2025 10:20:17 -0500 Subject: [PATCH 04/10] Misc. changes for testing --- halo-testing-logs/halo_test_3706609.err | 4 + halo-testing-logs/halo_test_3706609.out | 29 ++ halo-testing-logs/halo_test_3706620.err | 33 ++ halo-testing-logs/halo_test_3706620.out | 161 ++++++++++ halo-testing-logs/halo_test_3706738.err | 35 ++ halo-testing-logs/halo_test_3706738.out | 161 ++++++++++ halo-testing-logs/halo_test_3707040.err | 35 ++ halo-testing-logs/halo_test_3707040.out | 161 ++++++++++ halo-testing-logs/halo_test_3707410.err | 35 ++ halo-testing-logs/halo_test_3707410.out | 172 ++++++++++ halo-testing-logs/halo_test_3708721.err | 35 ++ halo-testing-logs/halo_test_3708721.out | 175 ++++++++++ halo-testing-logs/halo_test_3708911.err | 28 ++ halo-testing-logs/halo_test_3708911.out | 177 +++++++++++ halo-testing-logs/halo_test_3709442.err | 37 +++ halo-testing-logs/halo_test_3709442.out | 178 +++++++++++ halo-testing-logs/halo_test_3709762.err | 28 ++ halo-testing-logs/halo_test_3709762.out | 178 +++++++++++ halo-testing-logs/halo_test_3713150.err | 27 ++ halo-testing-logs/halo_test_3713150.out | 161 ++++++++++ halo-testing-logs/halo_test_3713164.err | 30 ++ halo-testing-logs/halo_test_3713164.out | 404 ++++++++++++++++++++++++ halo_testing_script.py | 28 ++ pyhealth/datasets/__init__.py | 1 + pyhealth/datasets/halo_mimic3.py | 62 ++-- pyhealth/models/generators/halo.py | 73 +++-- test_halo_model.slurm | 34 ++ 27 files changed, 2420 insertions(+), 62 deletions(-) create mode 100644 halo-testing-logs/halo_test_3706609.err create mode 100644 halo-testing-logs/halo_test_3706609.out create mode 100644 halo-testing-logs/halo_test_3706620.err create mode 100644 halo-testing-logs/halo_test_3706620.out create mode 100644 halo-testing-logs/halo_test_3706738.err create mode 100644 halo-testing-logs/halo_test_3706738.out create mode 100644 halo-testing-logs/halo_test_3707040.err create mode 100644 halo-testing-logs/halo_test_3707040.out create mode 100644 halo-testing-logs/halo_test_3707410.err create mode 100644 halo-testing-logs/halo_test_3707410.out create mode 100644 halo-testing-logs/halo_test_3708721.err create mode 100644 halo-testing-logs/halo_test_3708721.out create mode 100644 halo-testing-logs/halo_test_3708911.err create mode 100644 halo-testing-logs/halo_test_3708911.out create mode 100644 halo-testing-logs/halo_test_3709442.err create mode 100644 halo-testing-logs/halo_test_3709442.out create mode 100644 halo-testing-logs/halo_test_3709762.err create mode 100644 halo-testing-logs/halo_test_3709762.out create mode 100644 halo-testing-logs/halo_test_3713150.err create mode 100644 halo-testing-logs/halo_test_3713150.out create mode 100644 halo-testing-logs/halo_test_3713164.err create mode 100644 halo-testing-logs/halo_test_3713164.out create mode 100644 halo_testing_script.py create mode 100644 test_halo_model.slurm diff --git a/halo-testing-logs/halo_test_3706609.err b/halo-testing-logs/halo_test_3706609.err new file mode 100644 index 000000000..ed385264a --- /dev/null +++ b/halo-testing-logs/halo_test_3706609.err @@ -0,0 +1,4 @@ +Traceback (most recent call last): + File "/u/ethanmr3/halo/PyHealth/halo_testing_script.py", line 3, in + definitions = yaml.full_load(definitions_file) +NameError: name 'yaml' is not defined diff --git a/halo-testing-logs/halo_test_3706609.out b/halo-testing-logs/halo_test_3706609.out new file mode 100644 index 000000000..9f0fa8354 --- /dev/null +++ b/halo-testing-logs/halo_test_3706609.out @@ -0,0 +1,29 @@ +SLURM_JOB_ID: 3706609 +SLURM_JOB_NODELIST: ccc0388 +SLURM_NTASKS: +SLURM_CPUS_ON_NODE: 4 +SLURM_GPUS_ON_NODE: 1 +SLURM_GPUS: +CUDA_VISIBLE_DEVICES: 0 +Running nvidia-smi to confirm GPU availability: +Sat Jul 19 19:12:28 2025 ++-----------------------------------------------------------------------------------------+ +| NVIDIA-SMI 575.57.08 Driver Version: 575.57.08 CUDA Version: 12.9 | +|-----------------------------------------+------------------------+----------------------+ +| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC | +| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. | +| | | MIG M. | +|=========================================+========================+======================| +| 0 NVIDIA A100-SXM4-80GB Off | 00000000:81:00.0 Off | 0 | +| N/A 28C P0 61W / 500W | 0MiB / 81920MiB | 0% Default | +| | | Disabled | ++-----------------------------------------+------------------------+----------------------+ + ++-----------------------------------------------------------------------------------------+ +| Processes: | +| GPU GI CI PID Type Process name GPU Memory | +| ID ID Usage | +|=========================================================================================| +| No running processes found | ++-----------------------------------------------------------------------------------------+ +BEGIN: Testing diff --git a/halo-testing-logs/halo_test_3706620.err b/halo-testing-logs/halo_test_3706620.err new file mode 100644 index 000000000..1daf24cc4 --- /dev/null +++ b/halo-testing-logs/halo_test_3706620.err @@ -0,0 +1,33 @@ + WARNING: Value for scheme.platlib does not match. Please report this to + distutils: /u/ethanmr3/.local/lib/python3.9/site-packages + sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages + WARNING: Additional context: + user = True + home = None + root = None + prefix = None + WARNING: Value for scheme.platlib does not match. Please report this to + distutils: /u/ethanmr3/.local/lib/python3.9/site-packages + sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages + WARNING: Additional context: + user = True + home = None + root = None + prefix = None + WARNING: Value for scheme.platlib does not match. Please report this to + distutils: /u/ethanmr3/.local/lib/python3.9/site-packages + sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages + WARNING: Additional context: + user = True + home = None + root = None + prefix = None +ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. +pyhealth 1.1.4 requires pandas<2,>=1.3.2, but you have pandas 2.3.1 which is incompatible. + 0%| | 0/58976 [00:00 + model = HALO(dataset=halo_dataset, config=halo_config, save_dir="../../halo_save/") + File "/u/ethanmr3/halo/PyHealth/pyhealth/models/generators/halo.py", line 53, in __init__ + train_c = set([c for p in train_ehr_dataset for v in p['visits'] for c in v]) +NameError: name 'train_ehr_dataset' is not defined diff --git a/halo-testing-logs/halo_test_3706620.out b/halo-testing-logs/halo_test_3706620.out new file mode 100644 index 000000000..9405dc56f --- /dev/null +++ b/halo-testing-logs/halo_test_3706620.out @@ -0,0 +1,161 @@ +SLURM_JOB_ID: 3706620 +SLURM_JOB_NODELIST: ccc0388 +SLURM_NTASKS: +SLURM_CPUS_ON_NODE: 4 +SLURM_GPUS_ON_NODE: 1 +SLURM_GPUS: +CUDA_VISIBLE_DEVICES: 0 +Running nvidia-smi to confirm GPU availability: +Sat Jul 19 19:13:44 2025 ++-----------------------------------------------------------------------------------------+ +| NVIDIA-SMI 575.57.08 Driver Version: 575.57.08 CUDA Version: 12.9 | +|-----------------------------------------+------------------------+----------------------+ +| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC | +| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. | +| | | MIG M. | +|=========================================+========================+======================| +| 0 NVIDIA A100-SXM4-80GB Off | 00000000:81:00.0 Off | 0 | +| N/A 28C P0 61W / 500W | 0MiB / 81920MiB | 0% Default | +| | | Disabled | ++-----------------------------------------+------------------------+----------------------+ + ++-----------------------------------------------------------------------------------------+ +| Processes: | +| GPU GI CI PID Type Process name GPU Memory | +| ID ID Usage | +|=========================================================================================| +| No running processes found | ++-----------------------------------------------------------------------------------------+ +Defaulting to user installation because normal site-packages is not writeable +Obtaining file:///u/ethanmr3/halo/PyHealth +Requirement already satisfied: torch>=1.8.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2.7.1) +Requirement already satisfied: torchvision>=0.9.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (0.22.1) +Requirement already satisfied: rdkit>=2022.03.4 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2025.3.3) +Requirement already satisfied: scikit-learn>=0.24.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.6.1) +Requirement already satisfied: networkx>=2.6.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (3.2.1) +Collecting pandas<2,>=1.3.2 + Using cached pandas-1.5.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.2 MB) +Requirement already satisfied: pandarallel>=1.5.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.6.5) +Requirement already satisfied: mne>=1.0.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.8.0) +Requirement already satisfied: urllib3<=1.26.15 in /usr/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.26.5) +Requirement already satisfied: numpy in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2.0.2) +Requirement already satisfied: tqdm in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (4.67.1) +Requirement already satisfied: polars in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.31.0) +Requirement already satisfied: transformers in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (4.52.4) +Requirement already satisfied: pooch>=1.5 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (1.8.2) +Requirement already satisfied: packaging in /usr/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (20.9) +Requirement already satisfied: jinja2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (3.1.6) +Requirement already satisfied: lazy-loader>=0.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (0.4) +Requirement already satisfied: scipy>=1.9 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (1.13.1) +Requirement already satisfied: matplotlib>=3.6 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (3.9.4) +Requirement already satisfied: decorator in /usr/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (4.4.2) +Requirement already satisfied: contourpy>=1.0.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.3.0) +Requirement already satisfied: pyparsing>=2.3.1 in /usr/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (2.4.7) +Requirement already satisfied: importlib-resources>=3.2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (6.5.2) +Requirement already satisfied: pillow>=8 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (11.2.1) +Requirement already satisfied: fonttools>=4.22.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (4.58.4) +Requirement already satisfied: python-dateutil>=2.7 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (2.9.0.post0) +Requirement already satisfied: kiwisolver>=1.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.4.7) +Requirement already satisfied: cycler>=0.10 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (0.12.1) +Requirement already satisfied: zipp>=3.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from importlib-resources>=3.2.0->matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (3.23.0) +Requirement already satisfied: dill>=0.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandarallel>=1.5.3->pyhealth==1.1.4) (0.4.0) +Requirement already satisfied: psutil in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandarallel>=1.5.3->pyhealth==1.1.4) (7.0.0) +Requirement already satisfied: pytz>=2020.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandas<2,>=1.3.2->pyhealth==1.1.4) (2025.2) +Requirement already satisfied: platformdirs>=2.5.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (4.3.8) +Requirement already satisfied: requests>=2.19.0 in /usr/lib/python3.9/site-packages (from pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (2.25.1) +Requirement already satisfied: six>=1.5 in /u/ethanmr3/.local/lib/python3.9/site-packages (from python-dateutil>=2.7->matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.17.0) +Requirement already satisfied: idna<3,>=2.5 in /usr/lib/python3.9/site-packages (from requests>=2.19.0->pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (2.10) +Requirement already satisfied: chardet<5,>=3.0.2 in /usr/lib/python3.9/site-packages (from requests>=2.19.0->pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (4.0.0) +Requirement already satisfied: joblib>=1.2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from scikit-learn>=0.24.2->pyhealth==1.1.4) (1.5.1) +Requirement already satisfied: threadpoolctl>=3.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from scikit-learn>=0.24.2->pyhealth==1.1.4) (3.6.0) +Requirement already satisfied: nvidia-nvtx-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) +Requirement already satisfied: nvidia-cuda-cupti-cu12==12.6.80 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.80) +Requirement already satisfied: nvidia-nvjitlink-cu12==12.6.85 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.85) +Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) +Requirement already satisfied: nvidia-curand-cu12==10.3.7.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (10.3.7.77) +Requirement already satisfied: nvidia-cufft-cu12==11.3.0.4 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (11.3.0.4) +Requirement already satisfied: nvidia-cusparse-cu12==12.5.4.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.5.4.2) +Requirement already satisfied: nvidia-cuda-runtime-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) +Requirement already satisfied: nvidia-cusolver-cu12==11.7.1.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (11.7.1.2) +Requirement already satisfied: nvidia-cusparselt-cu12==0.6.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (0.6.3) +Requirement already satisfied: triton==3.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (3.3.1) +Requirement already satisfied: nvidia-nccl-cu12==2.26.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (2.26.2) +Requirement already satisfied: nvidia-cufile-cu12==1.11.1.6 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (1.11.1.6) +Requirement already satisfied: typing-extensions>=4.10.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (4.14.0) +Requirement already satisfied: filelock in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (3.18.0) +Requirement already satisfied: fsspec in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (2025.5.1) +Requirement already satisfied: sympy>=1.13.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (1.14.0) +Requirement already satisfied: nvidia-cudnn-cu12==9.5.1.17 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (9.5.1.17) +Requirement already satisfied: nvidia-cublas-cu12==12.6.4.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.4.1) +Requirement already satisfied: setuptools>=40.8.0 in /usr/lib/python3.9/site-packages (from triton==3.3.1->torch>=1.8.0->pyhealth==1.1.4) (53.0.0) +Requirement already satisfied: mpmath<1.4,>=1.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from sympy>=1.13.3->torch>=1.8.0->pyhealth==1.1.4) (1.3.0) +Requirement already satisfied: MarkupSafe>=2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from jinja2->mne>=1.0.3->pyhealth==1.1.4) (3.0.2) +Requirement already satisfied: pyyaml>=5.1 in /usr/lib64/python3.9/site-packages (from transformers->pyhealth==1.1.4) (5.4.1) +Requirement already satisfied: tokenizers<0.22,>=0.21 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.21.1) +Requirement already satisfied: regex!=2019.12.17 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (2024.11.6) +Requirement already satisfied: safetensors>=0.4.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.5.3) +Requirement already satisfied: huggingface-hub<1.0,>=0.30.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.33.0) +Requirement already satisfied: hf-xet<2.0.0,>=1.1.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from huggingface-hub<1.0,>=0.30.0->transformers->pyhealth==1.1.4) (1.1.5) +Installing collected packages: pandas, pyhealth + Attempting uninstall: pandas + Found existing installation: pandas 2.3.1 + Uninstalling pandas-2.3.1: + Successfully uninstalled pandas-2.3.1 + Attempting uninstall: pyhealth + Found existing installation: pyhealth 1.1.4 + Uninstalling pyhealth-1.1.4: + Successfully uninstalled pyhealth-1.1.4 + Running setup.py develop for pyhealth +Successfully installed pandas-1.5.3 pyhealth +Defaulting to user installation because normal site-packages is not writeable +Collecting numpy + Using cached numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (19.5 MB) +Installing collected packages: numpy + Attempting uninstall: numpy + Found existing installation: numpy 2.0.2 + Uninstalling numpy-2.0.2: + Successfully uninstalled numpy-2.0.2 +Successfully installed numpy-2.0.2 +Defaulting to user installation because normal site-packages is not writeable +Collecting pandas + Using cached pandas-2.3.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.4 MB) +Collecting numpy>=1.22.4 + Using cached numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (19.5 MB) +Collecting python-dateutil>=2.8.2 + Using cached python_dateutil-2.9.0.post0-py2.py3-none-any.whl (229 kB) +Collecting pytz>=2020.1 + Using cached pytz-2025.2-py2.py3-none-any.whl (509 kB) +Collecting tzdata>=2022.7 + Using cached tzdata-2025.2-py2.py3-none-any.whl (347 kB) +Collecting six>=1.5 + Using cached six-1.17.0-py2.py3-none-any.whl (11 kB) +Installing collected packages: six, tzdata, pytz, python-dateutil, numpy, pandas + Attempting uninstall: six + Found existing installation: six 1.17.0 + Uninstalling six-1.17.0: + Successfully uninstalled six-1.17.0 + Attempting uninstall: tzdata + Found existing installation: tzdata 2025.2 + Uninstalling tzdata-2025.2: + Successfully uninstalled tzdata-2025.2 + Attempting uninstall: pytz + Found existing installation: pytz 2025.2 + Uninstalling pytz-2025.2: + Successfully uninstalled pytz-2025.2 + Attempting uninstall: python-dateutil + Found existing installation: python-dateutil 2.9.0.post0 + Uninstalling python-dateutil-2.9.0.post0: + Successfully uninstalled python-dateutil-2.9.0.post0 + Attempting uninstall: numpy + Found existing installation: numpy 2.0.2 + Uninstalling numpy-2.0.2: + Successfully uninstalled numpy-2.0.2 + Attempting uninstall: pandas + Found existing installation: pandas 1.5.3 + Uninstalling pandas-1.5.3: + Successfully uninstalled pandas-1.5.3 +Successfully installed numpy-2.0.2 pandas-2.3.1 python-dateutil-2.9.0.post0 pytz-2025.2 six-1.17.0 tzdata-2025.2 +BEGIN: Testing +Success on pip install -e . +Sucess on imports +Operating in dir: /u/ethanmr3/halo/PyHealth diff --git a/halo-testing-logs/halo_test_3706738.err b/halo-testing-logs/halo_test_3706738.err new file mode 100644 index 000000000..ace8ddcb5 --- /dev/null +++ b/halo-testing-logs/halo_test_3706738.err @@ -0,0 +1,35 @@ + WARNING: Value for scheme.platlib does not match. Please report this to + distutils: /u/ethanmr3/.local/lib/python3.9/site-packages + sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages + WARNING: Additional context: + user = True + home = None + root = None + prefix = None + WARNING: Value for scheme.platlib does not match. Please report this to + distutils: /u/ethanmr3/.local/lib/python3.9/site-packages + sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages + WARNING: Additional context: + user = True + home = None + root = None + prefix = None + WARNING: Value for scheme.platlib does not match. Please report this to + distutils: /u/ethanmr3/.local/lib/python3.9/site-packages + sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages + WARNING: Additional context: + user = True + home = None + root = None + prefix = None +ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. +pyhealth 1.1.4 requires pandas<2,>=1.3.2, but you have pandas 2.3.1 which is incompatible. + 0%| | 0/58976 [00:00 + model = HALO(dataset=halo_dataset, config=halo_config, save_dir="../../halo_save/") + File "/u/ethanmr3/halo/PyHealth/pyhealth/models/generators/halo.py", line 54, in __init__ + self.test_ehr_dataset = [{'labels': p['labels'], 'visits': [[c for c in v if c in train_c] for v in p['visits']]} for p in test_ehr_dataset] + File "/u/ethanmr3/halo/PyHealth/pyhealth/models/generators/halo.py", line 54, in + self.test_ehr_dataset = [{'labels': p['labels'], 'visits': [[c for c in v if c in train_c] for v in p['visits']]} for p in test_ehr_dataset] +KeyError: 'labels' diff --git a/halo-testing-logs/halo_test_3706738.out b/halo-testing-logs/halo_test_3706738.out new file mode 100644 index 000000000..b9921cdbd --- /dev/null +++ b/halo-testing-logs/halo_test_3706738.out @@ -0,0 +1,161 @@ +SLURM_JOB_ID: 3706738 +SLURM_JOB_NODELIST: ccc0388 +SLURM_NTASKS: +SLURM_CPUS_ON_NODE: 4 +SLURM_GPUS_ON_NODE: 1 +SLURM_GPUS: +CUDA_VISIBLE_DEVICES: 0 +Running nvidia-smi to confirm GPU availability: +Sat Jul 19 19:33:53 2025 ++-----------------------------------------------------------------------------------------+ +| NVIDIA-SMI 575.57.08 Driver Version: 575.57.08 CUDA Version: 12.9 | +|-----------------------------------------+------------------------+----------------------+ +| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC | +| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. | +| | | MIG M. | +|=========================================+========================+======================| +| 0 NVIDIA A100-SXM4-80GB Off | 00000000:41:00.0 Off | 0 | +| N/A 26C P0 61W / 500W | 0MiB / 81920MiB | 0% Default | +| | | Disabled | ++-----------------------------------------+------------------------+----------------------+ + ++-----------------------------------------------------------------------------------------+ +| Processes: | +| GPU GI CI PID Type Process name GPU Memory | +| ID ID Usage | +|=========================================================================================| +| No running processes found | ++-----------------------------------------------------------------------------------------+ +Defaulting to user installation because normal site-packages is not writeable +Obtaining file:///u/ethanmr3/halo/PyHealth +Requirement already satisfied: torch>=1.8.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2.7.1) +Requirement already satisfied: torchvision>=0.9.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (0.22.1) +Requirement already satisfied: rdkit>=2022.03.4 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2025.3.3) +Requirement already satisfied: scikit-learn>=0.24.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.6.1) +Requirement already satisfied: networkx>=2.6.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (3.2.1) +Collecting pandas<2,>=1.3.2 + Using cached pandas-1.5.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.2 MB) +Requirement already satisfied: pandarallel>=1.5.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.6.5) +Requirement already satisfied: mne>=1.0.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.8.0) +Requirement already satisfied: urllib3<=1.26.15 in /usr/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.26.5) +Requirement already satisfied: numpy in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2.0.2) +Requirement already satisfied: tqdm in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (4.67.1) +Requirement already satisfied: polars in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.31.0) +Requirement already satisfied: transformers in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (4.52.4) +Requirement already satisfied: decorator in /usr/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (4.4.2) +Requirement already satisfied: scipy>=1.9 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (1.13.1) +Requirement already satisfied: packaging in /usr/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (20.9) +Requirement already satisfied: jinja2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (3.1.6) +Requirement already satisfied: pooch>=1.5 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (1.8.2) +Requirement already satisfied: lazy-loader>=0.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (0.4) +Requirement already satisfied: matplotlib>=3.6 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (3.9.4) +Requirement already satisfied: cycler>=0.10 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (0.12.1) +Requirement already satisfied: python-dateutil>=2.7 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (2.9.0.post0) +Requirement already satisfied: fonttools>=4.22.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (4.58.4) +Requirement already satisfied: kiwisolver>=1.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.4.7) +Requirement already satisfied: pillow>=8 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (11.2.1) +Requirement already satisfied: contourpy>=1.0.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.3.0) +Requirement already satisfied: pyparsing>=2.3.1 in /usr/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (2.4.7) +Requirement already satisfied: importlib-resources>=3.2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (6.5.2) +Requirement already satisfied: zipp>=3.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from importlib-resources>=3.2.0->matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (3.23.0) +Requirement already satisfied: dill>=0.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandarallel>=1.5.3->pyhealth==1.1.4) (0.4.0) +Requirement already satisfied: psutil in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandarallel>=1.5.3->pyhealth==1.1.4) (7.0.0) +Requirement already satisfied: pytz>=2020.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandas<2,>=1.3.2->pyhealth==1.1.4) (2025.2) +Requirement already satisfied: requests>=2.19.0 in /usr/lib/python3.9/site-packages (from pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (2.25.1) +Requirement already satisfied: platformdirs>=2.5.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (4.3.8) +Requirement already satisfied: six>=1.5 in /u/ethanmr3/.local/lib/python3.9/site-packages (from python-dateutil>=2.7->matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.17.0) +Requirement already satisfied: chardet<5,>=3.0.2 in /usr/lib/python3.9/site-packages (from requests>=2.19.0->pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (4.0.0) +Requirement already satisfied: idna<3,>=2.5 in /usr/lib/python3.9/site-packages (from requests>=2.19.0->pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (2.10) +Requirement already satisfied: joblib>=1.2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from scikit-learn>=0.24.2->pyhealth==1.1.4) (1.5.1) +Requirement already satisfied: threadpoolctl>=3.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from scikit-learn>=0.24.2->pyhealth==1.1.4) (3.6.0) +Requirement already satisfied: triton==3.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (3.3.1) +Requirement already satisfied: nvidia-nccl-cu12==2.26.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (2.26.2) +Requirement already satisfied: nvidia-cufft-cu12==11.3.0.4 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (11.3.0.4) +Requirement already satisfied: nvidia-cusparse-cu12==12.5.4.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.5.4.2) +Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) +Requirement already satisfied: nvidia-nvtx-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) +Requirement already satisfied: nvidia-nvjitlink-cu12==12.6.85 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.85) +Requirement already satisfied: filelock in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (3.18.0) +Requirement already satisfied: sympy>=1.13.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (1.14.0) +Requirement already satisfied: nvidia-cufile-cu12==1.11.1.6 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (1.11.1.6) +Requirement already satisfied: nvidia-curand-cu12==10.3.7.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (10.3.7.77) +Requirement already satisfied: nvidia-cusolver-cu12==11.7.1.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (11.7.1.2) +Requirement already satisfied: fsspec in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (2025.5.1) +Requirement already satisfied: nvidia-cuda-runtime-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) +Requirement already satisfied: nvidia-cuda-cupti-cu12==12.6.80 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.80) +Requirement already satisfied: nvidia-cudnn-cu12==9.5.1.17 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (9.5.1.17) +Requirement already satisfied: typing-extensions>=4.10.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (4.14.0) +Requirement already satisfied: nvidia-cublas-cu12==12.6.4.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.4.1) +Requirement already satisfied: nvidia-cusparselt-cu12==0.6.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (0.6.3) +Requirement already satisfied: setuptools>=40.8.0 in /usr/lib/python3.9/site-packages (from triton==3.3.1->torch>=1.8.0->pyhealth==1.1.4) (53.0.0) +Requirement already satisfied: mpmath<1.4,>=1.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from sympy>=1.13.3->torch>=1.8.0->pyhealth==1.1.4) (1.3.0) +Requirement already satisfied: MarkupSafe>=2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from jinja2->mne>=1.0.3->pyhealth==1.1.4) (3.0.2) +Requirement already satisfied: pyyaml>=5.1 in /usr/lib64/python3.9/site-packages (from transformers->pyhealth==1.1.4) (5.4.1) +Requirement already satisfied: huggingface-hub<1.0,>=0.30.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.33.0) +Requirement already satisfied: regex!=2019.12.17 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (2024.11.6) +Requirement already satisfied: safetensors>=0.4.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.5.3) +Requirement already satisfied: tokenizers<0.22,>=0.21 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.21.1) +Requirement already satisfied: hf-xet<2.0.0,>=1.1.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from huggingface-hub<1.0,>=0.30.0->transformers->pyhealth==1.1.4) (1.1.5) +Installing collected packages: pandas, pyhealth + Attempting uninstall: pandas + Found existing installation: pandas 2.3.1 + Uninstalling pandas-2.3.1: + Successfully uninstalled pandas-2.3.1 + Attempting uninstall: pyhealth + Found existing installation: pyhealth 1.1.4 + Uninstalling pyhealth-1.1.4: + Successfully uninstalled pyhealth-1.1.4 + Running setup.py develop for pyhealth +Successfully installed pandas-1.5.3 pyhealth +Defaulting to user installation because normal site-packages is not writeable +Collecting numpy + Using cached numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (19.5 MB) +Installing collected packages: numpy + Attempting uninstall: numpy + Found existing installation: numpy 2.0.2 + Uninstalling numpy-2.0.2: + Successfully uninstalled numpy-2.0.2 +Successfully installed numpy-2.0.2 +Defaulting to user installation because normal site-packages is not writeable +Collecting pandas + Using cached pandas-2.3.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.4 MB) +Collecting pytz>=2020.1 + Using cached pytz-2025.2-py2.py3-none-any.whl (509 kB) +Collecting tzdata>=2022.7 + Using cached tzdata-2025.2-py2.py3-none-any.whl (347 kB) +Collecting python-dateutil>=2.8.2 + Using cached python_dateutil-2.9.0.post0-py2.py3-none-any.whl (229 kB) +Collecting numpy>=1.22.4 + Using cached numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (19.5 MB) +Collecting six>=1.5 + Using cached six-1.17.0-py2.py3-none-any.whl (11 kB) +Installing collected packages: six, tzdata, pytz, python-dateutil, numpy, pandas + Attempting uninstall: six + Found existing installation: six 1.17.0 + Uninstalling six-1.17.0: + Successfully uninstalled six-1.17.0 + Attempting uninstall: tzdata + Found existing installation: tzdata 2025.2 + Uninstalling tzdata-2025.2: + Successfully uninstalled tzdata-2025.2 + Attempting uninstall: pytz + Found existing installation: pytz 2025.2 + Uninstalling pytz-2025.2: + Successfully uninstalled pytz-2025.2 + Attempting uninstall: python-dateutil + Found existing installation: python-dateutil 2.9.0.post0 + Uninstalling python-dateutil-2.9.0.post0: + Successfully uninstalled python-dateutil-2.9.0.post0 + Attempting uninstall: numpy + Found existing installation: numpy 2.0.2 + Uninstalling numpy-2.0.2: + Successfully uninstalled numpy-2.0.2 + Attempting uninstall: pandas + Found existing installation: pandas 1.5.3 + Uninstalling pandas-1.5.3: + Successfully uninstalled pandas-1.5.3 +Successfully installed numpy-2.0.2 pandas-2.3.1 python-dateutil-2.9.0.post0 pytz-2025.2 six-1.17.0 tzdata-2025.2 +BEGIN: Testing +Success on pip install -e . +Sucess on imports +Operating in dir: /u/ethanmr3/halo/PyHealth diff --git a/halo-testing-logs/halo_test_3707040.err b/halo-testing-logs/halo_test_3707040.err new file mode 100644 index 000000000..191aa9575 --- /dev/null +++ b/halo-testing-logs/halo_test_3707040.err @@ -0,0 +1,35 @@ + WARNING: Value for scheme.platlib does not match. Please report this to + distutils: /u/ethanmr3/.local/lib/python3.9/site-packages + sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages + WARNING: Additional context: + user = True + home = None + root = None + prefix = None + WARNING: Value for scheme.platlib does not match. Please report this to + distutils: /u/ethanmr3/.local/lib/python3.9/site-packages + sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages + WARNING: Additional context: + user = True + home = None + root = None + prefix = None + WARNING: Value for scheme.platlib does not match. Please report this to + distutils: /u/ethanmr3/.local/lib/python3.9/site-packages + sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages + WARNING: Additional context: + user = True + home = None + root = None + prefix = None +ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. +pyhealth 1.1.4 requires pandas<2,>=1.3.2, but you have pandas 2.3.1 which is incompatible. + 0%| | 0/58976 [00:00 + model = HALO(dataset=halo_dataset, config=halo_config, save_dir="../../halo_save/") + File "/u/ethanmr3/halo/PyHealth/pyhealth/models/generators/halo.py", line 55, in __init__ + self.test_ehr_dataset = [{'labels': p['labels'], 'visits': [[c for c in v if c in train_c] for v in p['visits']]} for p in test_ehr_dataset] + File "/u/ethanmr3/halo/PyHealth/pyhealth/models/generators/halo.py", line 55, in + self.test_ehr_dataset = [{'labels': p['labels'], 'visits': [[c for c in v if c in train_c] for v in p['visits']]} for p in test_ehr_dataset] +KeyError: 'labels' diff --git a/halo-testing-logs/halo_test_3707040.out b/halo-testing-logs/halo_test_3707040.out new file mode 100644 index 000000000..4ced89a60 --- /dev/null +++ b/halo-testing-logs/halo_test_3707040.out @@ -0,0 +1,161 @@ +SLURM_JOB_ID: 3707040 +SLURM_JOB_NODELIST: ccc0388 +SLURM_NTASKS: +SLURM_CPUS_ON_NODE: 4 +SLURM_GPUS_ON_NODE: 1 +SLURM_GPUS: +CUDA_VISIBLE_DEVICES: 0 +Running nvidia-smi to confirm GPU availability: +Sat Jul 19 20:05:54 2025 ++-----------------------------------------------------------------------------------------+ +| NVIDIA-SMI 575.57.08 Driver Version: 575.57.08 CUDA Version: 12.9 | +|-----------------------------------------+------------------------+----------------------+ +| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC | +| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. | +| | | MIG M. | +|=========================================+========================+======================| +| 0 NVIDIA A100-SXM4-80GB Off | 00000000:41:00.0 Off | 0 | +| N/A 26C P0 61W / 500W | 0MiB / 81920MiB | 0% Default | +| | | Disabled | ++-----------------------------------------+------------------------+----------------------+ + ++-----------------------------------------------------------------------------------------+ +| Processes: | +| GPU GI CI PID Type Process name GPU Memory | +| ID ID Usage | +|=========================================================================================| +| No running processes found | ++-----------------------------------------------------------------------------------------+ +Defaulting to user installation because normal site-packages is not writeable +Obtaining file:///u/ethanmr3/halo/PyHealth +Requirement already satisfied: torch>=1.8.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2.7.1) +Requirement already satisfied: torchvision>=0.9.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (0.22.1) +Requirement already satisfied: rdkit>=2022.03.4 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2025.3.3) +Requirement already satisfied: scikit-learn>=0.24.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.6.1) +Requirement already satisfied: networkx>=2.6.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (3.2.1) +Collecting pandas<2,>=1.3.2 + Using cached pandas-1.5.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.2 MB) +Requirement already satisfied: pandarallel>=1.5.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.6.5) +Requirement already satisfied: mne>=1.0.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.8.0) +Requirement already satisfied: urllib3<=1.26.15 in /usr/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.26.5) +Requirement already satisfied: numpy in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2.0.2) +Requirement already satisfied: tqdm in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (4.67.1) +Requirement already satisfied: polars in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.31.0) +Requirement already satisfied: transformers in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (4.52.4) +Requirement already satisfied: packaging in /usr/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (20.9) +Requirement already satisfied: lazy-loader>=0.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (0.4) +Requirement already satisfied: jinja2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (3.1.6) +Requirement already satisfied: pooch>=1.5 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (1.8.2) +Requirement already satisfied: scipy>=1.9 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (1.13.1) +Requirement already satisfied: decorator in /usr/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (4.4.2) +Requirement already satisfied: matplotlib>=3.6 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (3.9.4) +Requirement already satisfied: contourpy>=1.0.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.3.0) +Requirement already satisfied: fonttools>=4.22.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (4.58.4) +Requirement already satisfied: pyparsing>=2.3.1 in /usr/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (2.4.7) +Requirement already satisfied: cycler>=0.10 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (0.12.1) +Requirement already satisfied: python-dateutil>=2.7 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (2.9.0.post0) +Requirement already satisfied: importlib-resources>=3.2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (6.5.2) +Requirement already satisfied: pillow>=8 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (11.2.1) +Requirement already satisfied: kiwisolver>=1.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.4.7) +Requirement already satisfied: zipp>=3.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from importlib-resources>=3.2.0->matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (3.23.0) +Requirement already satisfied: dill>=0.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandarallel>=1.5.3->pyhealth==1.1.4) (0.4.0) +Requirement already satisfied: psutil in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandarallel>=1.5.3->pyhealth==1.1.4) (7.0.0) +Requirement already satisfied: pytz>=2020.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandas<2,>=1.3.2->pyhealth==1.1.4) (2025.2) +Requirement already satisfied: requests>=2.19.0 in /usr/lib/python3.9/site-packages (from pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (2.25.1) +Requirement already satisfied: platformdirs>=2.5.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (4.3.8) +Requirement already satisfied: six>=1.5 in /u/ethanmr3/.local/lib/python3.9/site-packages (from python-dateutil>=2.7->matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.17.0) +Requirement already satisfied: idna<3,>=2.5 in /usr/lib/python3.9/site-packages (from requests>=2.19.0->pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (2.10) +Requirement already satisfied: chardet<5,>=3.0.2 in /usr/lib/python3.9/site-packages (from requests>=2.19.0->pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (4.0.0) +Requirement already satisfied: joblib>=1.2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from scikit-learn>=0.24.2->pyhealth==1.1.4) (1.5.1) +Requirement already satisfied: threadpoolctl>=3.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from scikit-learn>=0.24.2->pyhealth==1.1.4) (3.6.0) +Requirement already satisfied: nvidia-cufile-cu12==1.11.1.6 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (1.11.1.6) +Requirement already satisfied: nvidia-nvjitlink-cu12==12.6.85 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.85) +Requirement already satisfied: nvidia-cuda-cupti-cu12==12.6.80 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.80) +Requirement already satisfied: nvidia-cusolver-cu12==11.7.1.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (11.7.1.2) +Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) +Requirement already satisfied: typing-extensions>=4.10.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (4.14.0) +Requirement already satisfied: nvidia-nvtx-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) +Requirement already satisfied: nvidia-cusparse-cu12==12.5.4.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.5.4.2) +Requirement already satisfied: nvidia-cublas-cu12==12.6.4.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.4.1) +Requirement already satisfied: nvidia-cufft-cu12==11.3.0.4 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (11.3.0.4) +Requirement already satisfied: nvidia-cudnn-cu12==9.5.1.17 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (9.5.1.17) +Requirement already satisfied: sympy>=1.13.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (1.14.0) +Requirement already satisfied: filelock in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (3.18.0) +Requirement already satisfied: nvidia-nccl-cu12==2.26.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (2.26.2) +Requirement already satisfied: triton==3.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (3.3.1) +Requirement already satisfied: nvidia-curand-cu12==10.3.7.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (10.3.7.77) +Requirement already satisfied: nvidia-cusparselt-cu12==0.6.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (0.6.3) +Requirement already satisfied: fsspec in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (2025.5.1) +Requirement already satisfied: nvidia-cuda-runtime-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) +Requirement already satisfied: setuptools>=40.8.0 in /usr/lib/python3.9/site-packages (from triton==3.3.1->torch>=1.8.0->pyhealth==1.1.4) (53.0.0) +Requirement already satisfied: mpmath<1.4,>=1.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from sympy>=1.13.3->torch>=1.8.0->pyhealth==1.1.4) (1.3.0) +Requirement already satisfied: MarkupSafe>=2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from jinja2->mne>=1.0.3->pyhealth==1.1.4) (3.0.2) +Requirement already satisfied: regex!=2019.12.17 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (2024.11.6) +Requirement already satisfied: pyyaml>=5.1 in /usr/lib64/python3.9/site-packages (from transformers->pyhealth==1.1.4) (5.4.1) +Requirement already satisfied: tokenizers<0.22,>=0.21 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.21.1) +Requirement already satisfied: safetensors>=0.4.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.5.3) +Requirement already satisfied: huggingface-hub<1.0,>=0.30.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.33.0) +Requirement already satisfied: hf-xet<2.0.0,>=1.1.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from huggingface-hub<1.0,>=0.30.0->transformers->pyhealth==1.1.4) (1.1.5) +Installing collected packages: pandas, pyhealth + Attempting uninstall: pandas + Found existing installation: pandas 2.3.1 + Uninstalling pandas-2.3.1: + Successfully uninstalled pandas-2.3.1 + Attempting uninstall: pyhealth + Found existing installation: pyhealth 1.1.4 + Uninstalling pyhealth-1.1.4: + Successfully uninstalled pyhealth-1.1.4 + Running setup.py develop for pyhealth +Successfully installed pandas-1.5.3 pyhealth +Defaulting to user installation because normal site-packages is not writeable +Collecting numpy + Using cached numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (19.5 MB) +Installing collected packages: numpy + Attempting uninstall: numpy + Found existing installation: numpy 2.0.2 + Uninstalling numpy-2.0.2: + Successfully uninstalled numpy-2.0.2 +Successfully installed numpy-2.0.2 +Defaulting to user installation because normal site-packages is not writeable +Collecting pandas + Using cached pandas-2.3.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.4 MB) +Collecting pytz>=2020.1 + Using cached pytz-2025.2-py2.py3-none-any.whl (509 kB) +Collecting numpy>=1.22.4 + Using cached numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (19.5 MB) +Collecting python-dateutil>=2.8.2 + Using cached python_dateutil-2.9.0.post0-py2.py3-none-any.whl (229 kB) +Collecting tzdata>=2022.7 + Using cached tzdata-2025.2-py2.py3-none-any.whl (347 kB) +Collecting six>=1.5 + Using cached six-1.17.0-py2.py3-none-any.whl (11 kB) +Installing collected packages: six, tzdata, pytz, python-dateutil, numpy, pandas + Attempting uninstall: six + Found existing installation: six 1.17.0 + Uninstalling six-1.17.0: + Successfully uninstalled six-1.17.0 + Attempting uninstall: tzdata + Found existing installation: tzdata 2025.2 + Uninstalling tzdata-2025.2: + Successfully uninstalled tzdata-2025.2 + Attempting uninstall: pytz + Found existing installation: pytz 2025.2 + Uninstalling pytz-2025.2: + Successfully uninstalled pytz-2025.2 + Attempting uninstall: python-dateutil + Found existing installation: python-dateutil 2.9.0.post0 + Uninstalling python-dateutil-2.9.0.post0: + Successfully uninstalled python-dateutil-2.9.0.post0 + Attempting uninstall: numpy + Found existing installation: numpy 2.0.2 + Uninstalling numpy-2.0.2: + Successfully uninstalled numpy-2.0.2 + Attempting uninstall: pandas + Found existing installation: pandas 1.5.3 + Uninstalling pandas-1.5.3: + Successfully uninstalled pandas-1.5.3 +Successfully installed numpy-2.0.2 pandas-2.3.1 python-dateutil-2.9.0.post0 pytz-2025.2 six-1.17.0 tzdata-2025.2 +BEGIN: Testing +Success on pip install -e . +Sucess on imports +Operating in dir: /u/ethanmr3/halo/PyHealth diff --git a/halo-testing-logs/halo_test_3707410.err b/halo-testing-logs/halo_test_3707410.err new file mode 100644 index 000000000..e5a0d5f28 --- /dev/null +++ b/halo-testing-logs/halo_test_3707410.err @@ -0,0 +1,35 @@ + WARNING: Value for scheme.platlib does not match. Please report this to + distutils: /u/ethanmr3/.local/lib/python3.9/site-packages + sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages + WARNING: Additional context: + user = True + home = None + root = None + prefix = None + WARNING: Value for scheme.platlib does not match. Please report this to + distutils: /u/ethanmr3/.local/lib/python3.9/site-packages + sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages + WARNING: Additional context: + user = True + home = None + root = None + prefix = None + WARNING: Value for scheme.platlib does not match. Please report this to + distutils: /u/ethanmr3/.local/lib/python3.9/site-packages + sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages + WARNING: Additional context: + user = True + home = None + root = None + prefix = None +ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. +pyhealth 1.1.4 requires pandas<2,>=1.3.2, but you have pandas 2.3.1 which is incompatible. + 0%| | 0/58976 [00:00 + model = HALO(dataset=halo_dataset, config=halo_config, save_dir="../../halo_save/") + File "/u/ethanmr3/halo/PyHealth/pyhealth/models/generators/halo.py", line 59, in __init__ + self.test_ehr_dataset = [{'labels': p['labels'], 'visits': [[c for c in v if c in train_c] for v in p['visits']]} for p in test_ehr_dataset] + File "/u/ethanmr3/halo/PyHealth/pyhealth/models/generators/halo.py", line 59, in + self.test_ehr_dataset = [{'labels': p['labels'], 'visits': [[c for c in v if c in train_c] for v in p['visits']]} for p in test_ehr_dataset] +KeyError: 'labels' diff --git a/halo-testing-logs/halo_test_3707410.out b/halo-testing-logs/halo_test_3707410.out new file mode 100644 index 000000000..5f6316c80 --- /dev/null +++ b/halo-testing-logs/halo_test_3707410.out @@ -0,0 +1,172 @@ +SLURM_JOB_ID: 3707410 +SLURM_JOB_NODELIST: ccc0388 +SLURM_NTASKS: +SLURM_CPUS_ON_NODE: 4 +SLURM_GPUS_ON_NODE: 1 +SLURM_GPUS: +CUDA_VISIBLE_DEVICES: 0 +Running nvidia-smi to confirm GPU availability: +Sat Jul 19 20:31:37 2025 ++-----------------------------------------------------------------------------------------+ +| NVIDIA-SMI 575.57.08 Driver Version: 575.57.08 CUDA Version: 12.9 | +|-----------------------------------------+------------------------+----------------------+ +| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC | +| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. | +| | | MIG M. | +|=========================================+========================+======================| +| 0 NVIDIA A100-SXM4-80GB Off | 00000000:41:00.0 Off | 0 | +| N/A 26C P0 61W / 500W | 0MiB / 81920MiB | 0% Default | +| | | Disabled | ++-----------------------------------------+------------------------+----------------------+ + ++-----------------------------------------------------------------------------------------+ +| Processes: | +| GPU GI CI PID Type Process name GPU Memory | +| ID ID Usage | +|=========================================================================================| +| No running processes found | ++-----------------------------------------------------------------------------------------+ +Defaulting to user installation because normal site-packages is not writeable +Obtaining file:///u/ethanmr3/halo/PyHealth +Requirement already satisfied: torch>=1.8.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2.7.1) +Requirement already satisfied: torchvision>=0.9.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (0.22.1) +Requirement already satisfied: rdkit>=2022.03.4 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2025.3.3) +Requirement already satisfied: scikit-learn>=0.24.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.6.1) +Requirement already satisfied: networkx>=2.6.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (3.2.1) +Collecting pandas<2,>=1.3.2 + Using cached pandas-1.5.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.2 MB) +Requirement already satisfied: pandarallel>=1.5.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.6.5) +Requirement already satisfied: mne>=1.0.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.8.0) +Requirement already satisfied: urllib3<=1.26.15 in /usr/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.26.5) +Requirement already satisfied: numpy in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2.0.2) +Requirement already satisfied: tqdm in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (4.67.1) +Requirement already satisfied: polars in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.31.0) +Requirement already satisfied: transformers in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (4.52.4) +Requirement already satisfied: scipy>=1.9 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (1.13.1) +Requirement already satisfied: pooch>=1.5 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (1.8.2) +Requirement already satisfied: lazy-loader>=0.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (0.4) +Requirement already satisfied: matplotlib>=3.6 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (3.9.4) +Requirement already satisfied: jinja2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (3.1.6) +Requirement already satisfied: packaging in /usr/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (20.9) +Requirement already satisfied: decorator in /usr/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (4.4.2) +Requirement already satisfied: kiwisolver>=1.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.4.7) +Requirement already satisfied: cycler>=0.10 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (0.12.1) +Requirement already satisfied: python-dateutil>=2.7 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (2.9.0.post0) +Requirement already satisfied: pyparsing>=2.3.1 in /usr/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (2.4.7) +Requirement already satisfied: importlib-resources>=3.2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (6.5.2) +Requirement already satisfied: fonttools>=4.22.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (4.58.4) +Requirement already satisfied: pillow>=8 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (11.2.1) +Requirement already satisfied: contourpy>=1.0.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.3.0) +Requirement already satisfied: zipp>=3.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from importlib-resources>=3.2.0->matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (3.23.0) +Requirement already satisfied: dill>=0.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandarallel>=1.5.3->pyhealth==1.1.4) (0.4.0) +Requirement already satisfied: psutil in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandarallel>=1.5.3->pyhealth==1.1.4) (7.0.0) +Requirement already satisfied: pytz>=2020.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandas<2,>=1.3.2->pyhealth==1.1.4) (2025.2) +Requirement already satisfied: platformdirs>=2.5.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (4.3.8) +Requirement already satisfied: requests>=2.19.0 in /usr/lib/python3.9/site-packages (from pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (2.25.1) +Requirement already satisfied: six>=1.5 in /u/ethanmr3/.local/lib/python3.9/site-packages (from python-dateutil>=2.7->matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.17.0) +Requirement already satisfied: idna<3,>=2.5 in /usr/lib/python3.9/site-packages (from requests>=2.19.0->pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (2.10) +Requirement already satisfied: chardet<5,>=3.0.2 in /usr/lib/python3.9/site-packages (from requests>=2.19.0->pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (4.0.0) +Requirement already satisfied: threadpoolctl>=3.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from scikit-learn>=0.24.2->pyhealth==1.1.4) (3.6.0) +Requirement already satisfied: joblib>=1.2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from scikit-learn>=0.24.2->pyhealth==1.1.4) (1.5.1) +Requirement already satisfied: nvidia-cudnn-cu12==9.5.1.17 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (9.5.1.17) +Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) +Requirement already satisfied: nvidia-curand-cu12==10.3.7.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (10.3.7.77) +Requirement already satisfied: fsspec in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (2025.5.1) +Requirement already satisfied: nvidia-cusolver-cu12==11.7.1.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (11.7.1.2) +Requirement already satisfied: nvidia-cusparselt-cu12==0.6.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (0.6.3) +Requirement already satisfied: sympy>=1.13.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (1.14.0) +Requirement already satisfied: triton==3.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (3.3.1) +Requirement already satisfied: nvidia-nvjitlink-cu12==12.6.85 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.85) +Requirement already satisfied: nvidia-nvtx-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) +Requirement already satisfied: nvidia-nccl-cu12==2.26.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (2.26.2) +Requirement already satisfied: filelock in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (3.18.0) +Requirement already satisfied: nvidia-cuda-runtime-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) +Requirement already satisfied: nvidia-cublas-cu12==12.6.4.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.4.1) +Requirement already satisfied: nvidia-cuda-cupti-cu12==12.6.80 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.80) +Requirement already satisfied: nvidia-cusparse-cu12==12.5.4.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.5.4.2) +Requirement already satisfied: nvidia-cufft-cu12==11.3.0.4 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (11.3.0.4) +Requirement already satisfied: typing-extensions>=4.10.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (4.14.0) +Requirement already satisfied: nvidia-cufile-cu12==1.11.1.6 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (1.11.1.6) +Requirement already satisfied: setuptools>=40.8.0 in /usr/lib/python3.9/site-packages (from triton==3.3.1->torch>=1.8.0->pyhealth==1.1.4) (53.0.0) +Requirement already satisfied: mpmath<1.4,>=1.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from sympy>=1.13.3->torch>=1.8.0->pyhealth==1.1.4) (1.3.0) +Requirement already satisfied: MarkupSafe>=2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from jinja2->mne>=1.0.3->pyhealth==1.1.4) (3.0.2) +Requirement already satisfied: huggingface-hub<1.0,>=0.30.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.33.0) +Requirement already satisfied: safetensors>=0.4.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.5.3) +Requirement already satisfied: tokenizers<0.22,>=0.21 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.21.1) +Requirement already satisfied: pyyaml>=5.1 in /usr/lib64/python3.9/site-packages (from transformers->pyhealth==1.1.4) (5.4.1) +Requirement already satisfied: regex!=2019.12.17 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (2024.11.6) +Requirement already satisfied: hf-xet<2.0.0,>=1.1.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from huggingface-hub<1.0,>=0.30.0->transformers->pyhealth==1.1.4) (1.1.5) +Installing collected packages: pandas, pyhealth + Attempting uninstall: pandas + Found existing installation: pandas 2.3.1 + Uninstalling pandas-2.3.1: + Successfully uninstalled pandas-2.3.1 + Attempting uninstall: pyhealth + Found existing installation: pyhealth 1.1.4 + Uninstalling pyhealth-1.1.4: + Successfully uninstalled pyhealth-1.1.4 + Running setup.py develop for pyhealth +Successfully installed pandas-1.5.3 pyhealth +Defaulting to user installation because normal site-packages is not writeable +Collecting numpy + Using cached numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (19.5 MB) +Installing collected packages: numpy + Attempting uninstall: numpy + Found existing installation: numpy 2.0.2 + Uninstalling numpy-2.0.2: + Successfully uninstalled numpy-2.0.2 +Successfully installed numpy-2.0.2 +Defaulting to user installation because normal site-packages is not writeable +Collecting pandas + Using cached pandas-2.3.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.4 MB) +Collecting pytz>=2020.1 + Using cached pytz-2025.2-py2.py3-none-any.whl (509 kB) +Collecting python-dateutil>=2.8.2 + Using cached python_dateutil-2.9.0.post0-py2.py3-none-any.whl (229 kB) +Collecting numpy>=1.22.4 + Using cached numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (19.5 MB) +Collecting tzdata>=2022.7 + Using cached tzdata-2025.2-py2.py3-none-any.whl (347 kB) +Collecting six>=1.5 + Using cached six-1.17.0-py2.py3-none-any.whl (11 kB) +Installing collected packages: six, tzdata, pytz, python-dateutil, numpy, pandas + Attempting uninstall: six + Found existing installation: six 1.17.0 + Uninstalling six-1.17.0: + Successfully uninstalled six-1.17.0 + Attempting uninstall: tzdata + Found existing installation: tzdata 2025.2 + Uninstalling tzdata-2025.2: + Successfully uninstalled tzdata-2025.2 + Attempting uninstall: pytz + Found existing installation: pytz 2025.2 + Uninstalling pytz-2025.2: + Successfully uninstalled pytz-2025.2 + Attempting uninstall: python-dateutil + Found existing installation: python-dateutil 2.9.0.post0 + Uninstalling python-dateutil-2.9.0.post0: + Successfully uninstalled python-dateutil-2.9.0.post0 + Attempting uninstall: numpy + Found existing installation: numpy 2.0.2 + Uninstalling numpy-2.0.2: + Successfully uninstalled numpy-2.0.2 + Attempting uninstall: pandas + Found existing installation: pandas 1.5.3 + Uninstalling pandas-1.5.3: + Successfully uninstalled pandas-1.5.3 +Successfully installed numpy-2.0.2 pandas-2.3.1 python-dateutil-2.9.0.post0 pytz-2025.2 six-1.17.0 tzdata-2025.2 +BEGIN: Testing +Success on pip install -e . +Sucess on imports +Operating in dir: /u/ethanmr3/halo/PyHealth +VOCAB SIZE: 6984 +MAX LEN: 42 +AVG LEN: 1.267755803955288 +MAX VISIT LEN: 39 +AVG VISIT LEN: 11.037371134020619 +NUM RECORDS: 46520 +NUM LONGITUDINAL RECORDS: 7537 +Saving Everything +6984 +test data head 2: +[{'visits': [[3224, 1966, 294, 6423]]}, {'visits': [[1697, 5357, 1839, 4848, 3439, 5042, 2490, 4315, 767], [2176, 3585, 3208, 5384, 651, 1807, 5392, 1935, 6420, 279, 4123, 3359, 6946, 1059, 809, 3895, 1721, 2234, 3004, 839, 588, 6480, 5203, 597, 2008, 2274, 739, 5224], [5381, 2181, 3208, 3336, 3723, 5392, 4886, 2075, 6946, 5043, 2996, 695, 3895, 2749, 4545, 3266, 1482, 850, 2518, 3671, 3288, 4315, 757], [1025, 3203, 1542, 3208, 1807, 5392, 1697, 5922, 1059, 552, 2353, 2866, 5683, 2996, 3895, 6455, 1341, 4804, 2893, 5326, 5965, 6480, 6737, 3288, 4576, 4064, 6114, 2404, 998, 757, 4221]]}, {'visits': [[4545, 5061, 1642, 3246, 3157, 2492]]}, {'visits': [[5888, 3714, 4708, 3557, 390, 1800, 5935, 597, 1366]]}, {'visits': [[1264, 2827, 5535]]}] diff --git a/halo-testing-logs/halo_test_3708721.err b/halo-testing-logs/halo_test_3708721.err new file mode 100644 index 000000000..5b5e03499 --- /dev/null +++ b/halo-testing-logs/halo_test_3708721.err @@ -0,0 +1,35 @@ + WARNING: Value for scheme.platlib does not match. Please report this to + distutils: /u/ethanmr3/.local/lib/python3.9/site-packages + sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages + WARNING: Additional context: + user = True + home = None + root = None + prefix = None + WARNING: Value for scheme.platlib does not match. Please report this to + distutils: /u/ethanmr3/.local/lib/python3.9/site-packages + sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages + WARNING: Additional context: + user = True + home = None + root = None + prefix = None + WARNING: Value for scheme.platlib does not match. Please report this to + distutils: /u/ethanmr3/.local/lib/python3.9/site-packages + sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages + WARNING: Additional context: + user = True + home = None + root = None + prefix = None +ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. +pyhealth 1.1.4 requires pandas<2,>=1.3.2, but you have pandas 2.3.1 which is incompatible. + 0%| | 0/58976 [00:00 + model = HALO(dataset=halo_dataset, config=halo_config, save_dir="../../halo_save/") + File "/u/ethanmr3/halo/PyHealth/pyhealth/models/generators/halo.py", line 59, in __init__ + self.test_ehr_dataset = [{'labels': p['labels'], 'visits': [[c for c in v if c in train_c] for v in p['visits']]} for p in test_ehr_dataset] + File "/u/ethanmr3/halo/PyHealth/pyhealth/models/generators/halo.py", line 59, in + self.test_ehr_dataset = [{'labels': p['labels'], 'visits': [[c for c in v if c in train_c] for v in p['visits']]} for p in test_ehr_dataset] +KeyError: 'labels' diff --git a/halo-testing-logs/halo_test_3708721.out b/halo-testing-logs/halo_test_3708721.out new file mode 100644 index 000000000..4f156892a --- /dev/null +++ b/halo-testing-logs/halo_test_3708721.out @@ -0,0 +1,175 @@ +SLURM_JOB_ID: 3708721 +SLURM_JOB_NODELIST: ccc0388 +SLURM_NTASKS: +SLURM_CPUS_ON_NODE: 4 +SLURM_GPUS_ON_NODE: 1 +SLURM_GPUS: +CUDA_VISIBLE_DEVICES: 0 +Running nvidia-smi to confirm GPU availability: +Sat Jul 19 22:03:32 2025 ++-----------------------------------------------------------------------------------------+ +| NVIDIA-SMI 575.57.08 Driver Version: 575.57.08 CUDA Version: 12.9 | +|-----------------------------------------+------------------------+----------------------+ +| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC | +| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. | +| | | MIG M. | +|=========================================+========================+======================| +| 0 NVIDIA A100-SXM4-80GB Off | 00000000:81:00.0 Off | 0 | +| N/A 31C P0 62W / 500W | 0MiB / 81920MiB | 0% Default | +| | | Disabled | ++-----------------------------------------+------------------------+----------------------+ + ++-----------------------------------------------------------------------------------------+ +| Processes: | +| GPU GI CI PID Type Process name GPU Memory | +| ID ID Usage | +|=========================================================================================| +| No running processes found | ++-----------------------------------------------------------------------------------------+ +Defaulting to user installation because normal site-packages is not writeable +Obtaining file:///u/ethanmr3/halo/PyHealth +Requirement already satisfied: torch>=1.8.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2.7.1) +Requirement already satisfied: torchvision>=0.9.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (0.22.1) +Requirement already satisfied: rdkit>=2022.03.4 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2025.3.3) +Requirement already satisfied: scikit-learn>=0.24.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.6.1) +Requirement already satisfied: networkx>=2.6.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (3.2.1) +Collecting pandas<2,>=1.3.2 + Using cached pandas-1.5.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.2 MB) +Requirement already satisfied: pandarallel>=1.5.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.6.5) +Requirement already satisfied: mne>=1.0.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.8.0) +Requirement already satisfied: urllib3<=1.26.15 in /usr/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.26.5) +Requirement already satisfied: numpy in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2.0.2) +Requirement already satisfied: tqdm in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (4.67.1) +Requirement already satisfied: polars in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.31.0) +Requirement already satisfied: transformers in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (4.52.4) +Requirement already satisfied: matplotlib>=3.6 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (3.9.4) +Requirement already satisfied: jinja2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (3.1.6) +Requirement already satisfied: packaging in /usr/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (20.9) +Requirement already satisfied: pooch>=1.5 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (1.8.2) +Requirement already satisfied: scipy>=1.9 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (1.13.1) +Requirement already satisfied: lazy-loader>=0.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (0.4) +Requirement already satisfied: decorator in /usr/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (4.4.2) +Requirement already satisfied: python-dateutil>=2.7 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (2.9.0.post0) +Requirement already satisfied: pillow>=8 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (11.2.1) +Requirement already satisfied: importlib-resources>=3.2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (6.5.2) +Requirement already satisfied: kiwisolver>=1.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.4.7) +Requirement already satisfied: contourpy>=1.0.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.3.0) +Requirement already satisfied: pyparsing>=2.3.1 in /usr/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (2.4.7) +Requirement already satisfied: fonttools>=4.22.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (4.58.4) +Requirement already satisfied: cycler>=0.10 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (0.12.1) +Requirement already satisfied: zipp>=3.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from importlib-resources>=3.2.0->matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (3.23.0) +Requirement already satisfied: dill>=0.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandarallel>=1.5.3->pyhealth==1.1.4) (0.4.0) +Requirement already satisfied: psutil in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandarallel>=1.5.3->pyhealth==1.1.4) (7.0.0) +Requirement already satisfied: pytz>=2020.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandas<2,>=1.3.2->pyhealth==1.1.4) (2025.2) +Requirement already satisfied: requests>=2.19.0 in /usr/lib/python3.9/site-packages (from pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (2.25.1) +Requirement already satisfied: platformdirs>=2.5.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (4.3.8) +Requirement already satisfied: six>=1.5 in /u/ethanmr3/.local/lib/python3.9/site-packages (from python-dateutil>=2.7->matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.17.0) +Requirement already satisfied: idna<3,>=2.5 in /usr/lib/python3.9/site-packages (from requests>=2.19.0->pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (2.10) +Requirement already satisfied: chardet<5,>=3.0.2 in /usr/lib/python3.9/site-packages (from requests>=2.19.0->pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (4.0.0) +Requirement already satisfied: joblib>=1.2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from scikit-learn>=0.24.2->pyhealth==1.1.4) (1.5.1) +Requirement already satisfied: threadpoolctl>=3.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from scikit-learn>=0.24.2->pyhealth==1.1.4) (3.6.0) +Requirement already satisfied: nvidia-cusolver-cu12==11.7.1.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (11.7.1.2) +Requirement already satisfied: nvidia-cuda-runtime-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) +Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) +Requirement already satisfied: nvidia-cuda-cupti-cu12==12.6.80 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.80) +Requirement already satisfied: nvidia-nccl-cu12==2.26.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (2.26.2) +Requirement already satisfied: typing-extensions>=4.10.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (4.14.0) +Requirement already satisfied: nvidia-nvtx-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) +Requirement already satisfied: nvidia-cusparselt-cu12==0.6.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (0.6.3) +Requirement already satisfied: nvidia-cusparse-cu12==12.5.4.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.5.4.2) +Requirement already satisfied: nvidia-cublas-cu12==12.6.4.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.4.1) +Requirement already satisfied: filelock in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (3.18.0) +Requirement already satisfied: sympy>=1.13.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (1.14.0) +Requirement already satisfied: nvidia-curand-cu12==10.3.7.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (10.3.7.77) +Requirement already satisfied: nvidia-cufile-cu12==1.11.1.6 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (1.11.1.6) +Requirement already satisfied: nvidia-cufft-cu12==11.3.0.4 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (11.3.0.4) +Requirement already satisfied: fsspec in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (2025.5.1) +Requirement already satisfied: triton==3.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (3.3.1) +Requirement already satisfied: nvidia-cudnn-cu12==9.5.1.17 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (9.5.1.17) +Requirement already satisfied: nvidia-nvjitlink-cu12==12.6.85 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.85) +Requirement already satisfied: setuptools>=40.8.0 in /usr/lib/python3.9/site-packages (from triton==3.3.1->torch>=1.8.0->pyhealth==1.1.4) (53.0.0) +Requirement already satisfied: mpmath<1.4,>=1.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from sympy>=1.13.3->torch>=1.8.0->pyhealth==1.1.4) (1.3.0) +Requirement already satisfied: MarkupSafe>=2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from jinja2->mne>=1.0.3->pyhealth==1.1.4) (3.0.2) +Requirement already satisfied: huggingface-hub<1.0,>=0.30.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.33.0) +Requirement already satisfied: regex!=2019.12.17 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (2024.11.6) +Requirement already satisfied: safetensors>=0.4.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.5.3) +Requirement already satisfied: pyyaml>=5.1 in /usr/lib64/python3.9/site-packages (from transformers->pyhealth==1.1.4) (5.4.1) +Requirement already satisfied: tokenizers<0.22,>=0.21 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.21.1) +Requirement already satisfied: hf-xet<2.0.0,>=1.1.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from huggingface-hub<1.0,>=0.30.0->transformers->pyhealth==1.1.4) (1.1.5) +Installing collected packages: pandas, pyhealth + Attempting uninstall: pandas + Found existing installation: pandas 2.3.1 + Uninstalling pandas-2.3.1: + Successfully uninstalled pandas-2.3.1 + Attempting uninstall: pyhealth + Found existing installation: pyhealth 1.1.4 + Uninstalling pyhealth-1.1.4: + Successfully uninstalled pyhealth-1.1.4 + Running setup.py develop for pyhealth +Successfully installed pandas-1.5.3 pyhealth +Defaulting to user installation because normal site-packages is not writeable +Collecting numpy + Using cached numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (19.5 MB) +Installing collected packages: numpy + Attempting uninstall: numpy + Found existing installation: numpy 2.0.2 + Uninstalling numpy-2.0.2: + Successfully uninstalled numpy-2.0.2 +Successfully installed numpy-2.0.2 +Defaulting to user installation because normal site-packages is not writeable +Collecting pandas + Using cached pandas-2.3.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.4 MB) +Collecting tzdata>=2022.7 + Using cached tzdata-2025.2-py2.py3-none-any.whl (347 kB) +Collecting python-dateutil>=2.8.2 + Using cached python_dateutil-2.9.0.post0-py2.py3-none-any.whl (229 kB) +Collecting numpy>=1.22.4 + Using cached numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (19.5 MB) +Collecting pytz>=2020.1 + Using cached pytz-2025.2-py2.py3-none-any.whl (509 kB) +Collecting six>=1.5 + Using cached six-1.17.0-py2.py3-none-any.whl (11 kB) +Installing collected packages: six, tzdata, pytz, python-dateutil, numpy, pandas + Attempting uninstall: six + Found existing installation: six 1.17.0 + Uninstalling six-1.17.0: + Successfully uninstalled six-1.17.0 + Attempting uninstall: tzdata + Found existing installation: tzdata 2025.2 + Uninstalling tzdata-2025.2: + Successfully uninstalled tzdata-2025.2 + Attempting uninstall: pytz + Found existing installation: pytz 2025.2 + Uninstalling pytz-2025.2: + Successfully uninstalled pytz-2025.2 + Attempting uninstall: python-dateutil + Found existing installation: python-dateutil 2.9.0.post0 + Uninstalling python-dateutil-2.9.0.post0: + Successfully uninstalled python-dateutil-2.9.0.post0 + Attempting uninstall: numpy + Found existing installation: numpy 2.0.2 + Uninstalling numpy-2.0.2: + Successfully uninstalled numpy-2.0.2 + Attempting uninstall: pandas + Found existing installation: pandas 1.5.3 + Uninstalling pandas-1.5.3: + Successfully uninstalled pandas-1.5.3 +Successfully installed numpy-2.0.2 pandas-2.3.1 python-dateutil-2.9.0.post0 pytz-2025.2 six-1.17.0 tzdata-2025.2 +BEGIN: Testing +Success on pip install -e . +Sucess on imports +Operating in dir: /u/ethanmr3/halo/PyHealth +VOCAB SIZE: 6984 +DATA LEN IS: 46520 +MIMIC LABELS = [1. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 1. 0. 1. 1. 0. 0. 0. 0. 0. 0. 1. 1. 1. + 1.] +MAX LEN: 42 +AVG LEN: 1.267755803955288 +MAX VISIT LEN: 39 +AVG VISIT LEN: 11.037371134020619 +NUM RECORDS: 46520 +NUM LONGITUDINAL RECORDS: 7537 +Saving Everything +6984 +test data head 2: +[{'visits': [[6144, 3305, 3165, 1430]]}, {'visits': [[2177, 834, 5378, 3431, 2473, 4554, 5098, 592, 624], [384, 3204, 398, 530, 5909, 2584, 6553, 6685, 798, 6431, 6947, 2471, 679, 2093, 1838, 429, 3762, 5308, 1472, 579, 5699, 1482, 4045, 4948, 2267, 4577, 5361, 4220], [2311, 398, 530, 6934, 2584, 5410, 6947, 3491, 3752, 2859, 5812, 5428, 2494, 1601, 2883, 1749, 3544, 1631, 5602, 624, 1395, 6008, 2299], [2177, 5890, 2181, 5638, 3984, 530, 5909, 3736, 2584, 4380, 158, 6947, 3491, 167, 41, 5557, 3511, 3522, 2003, 4948, 1749, 4821, 4827, 6109, 1004, 6130, 243, 756, 6008, 5370, 4220]]}, {'visits': [[1057, 963, 4619, 240, 6934, 5049]]}, {'visits': [[4096, 5090, 5699, 3685, 1515, 4820, 1653, 3479, 5050]]}, {'visits': [[5482, 3995, 3839]]}] diff --git a/halo-testing-logs/halo_test_3708911.err b/halo-testing-logs/halo_test_3708911.err new file mode 100644 index 000000000..f01e0ce4c --- /dev/null +++ b/halo-testing-logs/halo_test_3708911.err @@ -0,0 +1,28 @@ + WARNING: Value for scheme.platlib does not match. Please report this to + distutils: /u/ethanmr3/.local/lib/python3.9/site-packages + sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages + WARNING: Additional context: + user = True + home = None + root = None + prefix = None + WARNING: Value for scheme.platlib does not match. Please report this to + distutils: /u/ethanmr3/.local/lib/python3.9/site-packages + sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages + WARNING: Additional context: + user = True + home = None + root = None + prefix = None + WARNING: Value for scheme.platlib does not match. Please report this to + distutils: /u/ethanmr3/.local/lib/python3.9/site-packages + sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages + WARNING: Additional context: + user = True + home = None + root = None + prefix = None +ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. +pyhealth 1.1.4 requires pandas<2,>=1.3.2, but you have pandas 2.3.1 which is incompatible. + 0%| | 0/58976 [00:00=1.8.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2.7.1) +Requirement already satisfied: torchvision>=0.9.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (0.22.1) +Requirement already satisfied: rdkit>=2022.03.4 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2025.3.3) +Requirement already satisfied: scikit-learn>=0.24.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.6.1) +Requirement already satisfied: networkx>=2.6.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (3.2.1) +Collecting pandas<2,>=1.3.2 + Using cached pandas-1.5.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.2 MB) +Requirement already satisfied: pandarallel>=1.5.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.6.5) +Requirement already satisfied: mne>=1.0.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.8.0) +Requirement already satisfied: urllib3<=1.26.15 in /usr/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.26.5) +Requirement already satisfied: numpy in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2.0.2) +Requirement already satisfied: tqdm in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (4.67.1) +Requirement already satisfied: polars in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.31.0) +Requirement already satisfied: transformers in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (4.52.4) +Requirement already satisfied: decorator in /usr/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (4.4.2) +Requirement already satisfied: pooch>=1.5 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (1.8.2) +Requirement already satisfied: jinja2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (3.1.6) +Requirement already satisfied: lazy-loader>=0.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (0.4) +Requirement already satisfied: scipy>=1.9 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (1.13.1) +Requirement already satisfied: matplotlib>=3.6 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (3.9.4) +Requirement already satisfied: packaging in /usr/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (20.9) +Requirement already satisfied: python-dateutil>=2.7 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (2.9.0.post0) +Requirement already satisfied: importlib-resources>=3.2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (6.5.2) +Requirement already satisfied: fonttools>=4.22.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (4.58.4) +Requirement already satisfied: contourpy>=1.0.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.3.0) +Requirement already satisfied: cycler>=0.10 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (0.12.1) +Requirement already satisfied: pyparsing>=2.3.1 in /usr/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (2.4.7) +Requirement already satisfied: kiwisolver>=1.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.4.7) +Requirement already satisfied: pillow>=8 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (11.2.1) +Requirement already satisfied: zipp>=3.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from importlib-resources>=3.2.0->matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (3.23.0) +Requirement already satisfied: dill>=0.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandarallel>=1.5.3->pyhealth==1.1.4) (0.4.0) +Requirement already satisfied: psutil in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandarallel>=1.5.3->pyhealth==1.1.4) (7.0.0) +Requirement already satisfied: pytz>=2020.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandas<2,>=1.3.2->pyhealth==1.1.4) (2025.2) +Requirement already satisfied: requests>=2.19.0 in /usr/lib/python3.9/site-packages (from pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (2.25.1) +Requirement already satisfied: platformdirs>=2.5.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (4.3.8) +Requirement already satisfied: six>=1.5 in /u/ethanmr3/.local/lib/python3.9/site-packages (from python-dateutil>=2.7->matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.17.0) +Requirement already satisfied: idna<3,>=2.5 in /usr/lib/python3.9/site-packages (from requests>=2.19.0->pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (2.10) +Requirement already satisfied: chardet<5,>=3.0.2 in /usr/lib/python3.9/site-packages (from requests>=2.19.0->pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (4.0.0) +Requirement already satisfied: threadpoolctl>=3.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from scikit-learn>=0.24.2->pyhealth==1.1.4) (3.6.0) +Requirement already satisfied: joblib>=1.2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from scikit-learn>=0.24.2->pyhealth==1.1.4) (1.5.1) +Requirement already satisfied: typing-extensions>=4.10.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (4.14.0) +Requirement already satisfied: nvidia-cuda-runtime-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) +Requirement already satisfied: nvidia-cusparselt-cu12==0.6.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (0.6.3) +Requirement already satisfied: nvidia-cudnn-cu12==9.5.1.17 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (9.5.1.17) +Requirement already satisfied: nvidia-cufft-cu12==11.3.0.4 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (11.3.0.4) +Requirement already satisfied: nvidia-cusparse-cu12==12.5.4.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.5.4.2) +Requirement already satisfied: triton==3.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (3.3.1) +Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) +Requirement already satisfied: nvidia-cublas-cu12==12.6.4.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.4.1) +Requirement already satisfied: nvidia-nccl-cu12==2.26.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (2.26.2) +Requirement already satisfied: nvidia-cuda-cupti-cu12==12.6.80 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.80) +Requirement already satisfied: nvidia-cusolver-cu12==11.7.1.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (11.7.1.2) +Requirement already satisfied: sympy>=1.13.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (1.14.0) +Requirement already satisfied: nvidia-curand-cu12==10.3.7.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (10.3.7.77) +Requirement already satisfied: fsspec in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (2025.5.1) +Requirement already satisfied: filelock in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (3.18.0) +Requirement already satisfied: nvidia-nvtx-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) +Requirement already satisfied: nvidia-nvjitlink-cu12==12.6.85 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.85) +Requirement already satisfied: nvidia-cufile-cu12==1.11.1.6 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (1.11.1.6) +Requirement already satisfied: setuptools>=40.8.0 in /usr/lib/python3.9/site-packages (from triton==3.3.1->torch>=1.8.0->pyhealth==1.1.4) (53.0.0) +Requirement already satisfied: mpmath<1.4,>=1.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from sympy>=1.13.3->torch>=1.8.0->pyhealth==1.1.4) (1.3.0) +Requirement already satisfied: MarkupSafe>=2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from jinja2->mne>=1.0.3->pyhealth==1.1.4) (3.0.2) +Requirement already satisfied: pyyaml>=5.1 in /usr/lib64/python3.9/site-packages (from transformers->pyhealth==1.1.4) (5.4.1) +Requirement already satisfied: huggingface-hub<1.0,>=0.30.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.33.0) +Requirement already satisfied: regex!=2019.12.17 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (2024.11.6) +Requirement already satisfied: safetensors>=0.4.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.5.3) +Requirement already satisfied: tokenizers<0.22,>=0.21 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.21.1) +Requirement already satisfied: hf-xet<2.0.0,>=1.1.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from huggingface-hub<1.0,>=0.30.0->transformers->pyhealth==1.1.4) (1.1.5) +Installing collected packages: pandas, pyhealth + Attempting uninstall: pandas + Found existing installation: pandas 2.3.1 + Uninstalling pandas-2.3.1: + Successfully uninstalled pandas-2.3.1 + Attempting uninstall: pyhealth + Found existing installation: pyhealth 1.1.4 + Uninstalling pyhealth-1.1.4: + Successfully uninstalled pyhealth-1.1.4 + Running setup.py develop for pyhealth +Successfully installed pandas-1.5.3 pyhealth +Defaulting to user installation because normal site-packages is not writeable +Collecting numpy + Using cached numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (19.5 MB) +Installing collected packages: numpy + Attempting uninstall: numpy + Found existing installation: numpy 2.0.2 + Uninstalling numpy-2.0.2: + Successfully uninstalled numpy-2.0.2 +Successfully installed numpy-2.0.2 +Defaulting to user installation because normal site-packages is not writeable +Collecting pandas + Using cached pandas-2.3.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.4 MB) +Collecting tzdata>=2022.7 + Using cached tzdata-2025.2-py2.py3-none-any.whl (347 kB) +Collecting pytz>=2020.1 + Using cached pytz-2025.2-py2.py3-none-any.whl (509 kB) +Collecting python-dateutil>=2.8.2 + Using cached python_dateutil-2.9.0.post0-py2.py3-none-any.whl (229 kB) +Collecting numpy>=1.22.4 + Using cached numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (19.5 MB) +Collecting six>=1.5 + Using cached six-1.17.0-py2.py3-none-any.whl (11 kB) +Installing collected packages: six, tzdata, pytz, python-dateutil, numpy, pandas + Attempting uninstall: six + Found existing installation: six 1.17.0 + Uninstalling six-1.17.0: + Successfully uninstalled six-1.17.0 + Attempting uninstall: tzdata + Found existing installation: tzdata 2025.2 + Uninstalling tzdata-2025.2: + Successfully uninstalled tzdata-2025.2 + Attempting uninstall: pytz + Found existing installation: pytz 2025.2 + Uninstalling pytz-2025.2: + Successfully uninstalled pytz-2025.2 + Attempting uninstall: python-dateutil + Found existing installation: python-dateutil 2.9.0.post0 + Uninstalling python-dateutil-2.9.0.post0: + Successfully uninstalled python-dateutil-2.9.0.post0 + Attempting uninstall: numpy + Found existing installation: numpy 2.0.2 + Uninstalling numpy-2.0.2: + Successfully uninstalled numpy-2.0.2 + Attempting uninstall: pandas + Found existing installation: pandas 1.5.3 + Uninstalling pandas-1.5.3: + Successfully uninstalled pandas-1.5.3 +Successfully installed numpy-2.0.2 pandas-2.3.1 python-dateutil-2.9.0.post0 pytz-2025.2 six-1.17.0 tzdata-2025.2 +BEGIN: Testing +Success on pip install -e . +Sucess on imports +Operating in dir: /u/ethanmr3/halo/PyHealth +VOCAB SIZE: 6984 +MAX LEN: 42 +AVG LEN: 1.267755803955288 +MAX VISIT LEN: 39 +AVG VISIT LEN: 11.037371134020619 +NUM RECORDS: 46520 +NUM LONGITUDINAL RECORDS: 7537 +Saving Everything +6984 +test data head 2: +[{'visits': [[6970, 3668, 5101, 1455]], 'labels': array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., + 0., 0., 0., 0., 0., 0., 0., 0.])}, {'visits': [[5349, 6533, 5641, 2732, 335, 370, 5780, 3125, 4699], [6528, 3841, 1034, 5771, 4876, 6548, 280, 412, 6179, 4650, 941, 6959, 5560, 2751, 2623, 6858, 2250, 4817, 2261, 1238, 4442, 859, 6237, 4064, 1255, 5103, 1913, 4479], [5122, 263, 1034, 1681, 5780, 1821, 2342, 295, 4650, 4916, 1852, 450, 5956, 582, 4300, 2003, 2906, 4064, 1122, 4327, 1263, 502, 4479], [3584, 3841, 1034, 3210, 282, 6948, 4650, 2732, 1837, 430, 5683, 2623, 3520, 6465, 5447, 6858, 4300, 3535, 5843, 595, 4565, 3681, 1250, 1122, 3045, 2662, 4327, 4336, 6907, 3964, 4479]], 'labels': array([1., 1., 1., 1., 1., 0., 0., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., + 1., 0., 0., 0., 1., 1., 1., 0.])}, {'visits': [[966, 4808, 746, 3379, 502, 6776]], 'labels': array([0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 1., 1., 1., 0., 0., 0., + 0., 0., 0., 0., 0., 0., 0., 0.])}, {'visits': [[2241, 1734, 360, 6959, 2033, 5813, 6518, 984, 4125]], 'labels': array([0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., + 0., 0., 0., 0., 1., 1., 0., 1.])}, {'visits': [[1001, 4779, 3966]], 'labels': array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., + 0., 0., 0., 0., 0., 0., 0., 0.])}] diff --git a/halo-testing-logs/halo_test_3709442.err b/halo-testing-logs/halo_test_3709442.err new file mode 100644 index 000000000..790c447cd --- /dev/null +++ b/halo-testing-logs/halo_test_3709442.err @@ -0,0 +1,37 @@ + WARNING: Value for scheme.platlib does not match. Please report this to + distutils: /u/ethanmr3/.local/lib/python3.9/site-packages + sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages + WARNING: Additional context: + user = True + home = None + root = None + prefix = None + WARNING: Value for scheme.platlib does not match. Please report this to + distutils: /u/ethanmr3/.local/lib/python3.9/site-packages + sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages + WARNING: Additional context: + user = True + home = None + root = None + prefix = None + WARNING: Value for scheme.platlib does not match. Please report this to + distutils: /u/ethanmr3/.local/lib/python3.9/site-packages + sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages + WARNING: Additional context: + user = True + home = None + root = None + prefix = None +ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. +pyhealth 1.1.4 requires pandas<2,>=1.3.2, but you have pandas 2.3.1 which is incompatible. + 0%| | 0/58976 [00:00 + model.train() + File "/u/ethanmr3/halo/PyHealth/pyhealth/models/generators/halo.py", line 108, in train + checkpoint = torch.load(f'{self.save_dir}halo_model', map_location=torch.device(self.device)) + File "/u/ethanmr3/.local/lib/python3.9/site-packages/torch/serialization.py", line 1486, in load + with _open_zipfile_reader(opened_file) as opened_zipfile: + File "/u/ethanmr3/.local/lib/python3.9/site-packages/torch/serialization.py", line 771, in __init__ + super().__init__(torch._C.PyTorchFileReader(name_or_buffer)) +RuntimeError: PytorchStreamReader failed reading zip archive: failed finding central directory diff --git a/halo-testing-logs/halo_test_3709442.out b/halo-testing-logs/halo_test_3709442.out new file mode 100644 index 000000000..2f98c5181 --- /dev/null +++ b/halo-testing-logs/halo_test_3709442.out @@ -0,0 +1,178 @@ +SLURM_JOB_ID: 3709442 +SLURM_JOB_NODELIST: ccc0388 +SLURM_NTASKS: +SLURM_CPUS_ON_NODE: 4 +SLURM_GPUS_ON_NODE: 1 +SLURM_GPUS: +CUDA_VISIBLE_DEVICES: 0 +Running nvidia-smi to confirm GPU availability: +Sat Jul 19 23:14:02 2025 ++-----------------------------------------------------------------------------------------+ +| NVIDIA-SMI 575.57.08 Driver Version: 575.57.08 CUDA Version: 12.9 | +|-----------------------------------------+------------------------+----------------------+ +| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC | +| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. | +| | | MIG M. | +|=========================================+========================+======================| +| 0 NVIDIA A100-SXM4-80GB Off | 00000000:81:00.0 Off | 0 | +| N/A 31C P0 62W / 500W | 0MiB / 81920MiB | 0% Default | +| | | Disabled | ++-----------------------------------------+------------------------+----------------------+ + ++-----------------------------------------------------------------------------------------+ +| Processes: | +| GPU GI CI PID Type Process name GPU Memory | +| ID ID Usage | +|=========================================================================================| +| No running processes found | ++-----------------------------------------------------------------------------------------+ +Defaulting to user installation because normal site-packages is not writeable +Obtaining file:///u/ethanmr3/halo/PyHealth +Requirement already satisfied: torch>=1.8.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2.7.1) +Requirement already satisfied: torchvision>=0.9.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (0.22.1) +Requirement already satisfied: rdkit>=2022.03.4 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2025.3.3) +Requirement already satisfied: scikit-learn>=0.24.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.6.1) +Requirement already satisfied: networkx>=2.6.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (3.2.1) +Collecting pandas<2,>=1.3.2 + Using cached pandas-1.5.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.2 MB) +Requirement already satisfied: pandarallel>=1.5.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.6.5) +Requirement already satisfied: mne>=1.0.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.8.0) +Requirement already satisfied: urllib3<=1.26.15 in /usr/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.26.5) +Requirement already satisfied: numpy in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2.0.2) +Requirement already satisfied: tqdm in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (4.67.1) +Requirement already satisfied: polars in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.31.0) +Requirement already satisfied: transformers in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (4.52.4) +Requirement already satisfied: packaging in /usr/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (20.9) +Requirement already satisfied: scipy>=1.9 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (1.13.1) +Requirement already satisfied: decorator in /usr/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (4.4.2) +Requirement already satisfied: matplotlib>=3.6 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (3.9.4) +Requirement already satisfied: pooch>=1.5 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (1.8.2) +Requirement already satisfied: lazy-loader>=0.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (0.4) +Requirement already satisfied: jinja2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (3.1.6) +Requirement already satisfied: kiwisolver>=1.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.4.7) +Requirement already satisfied: fonttools>=4.22.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (4.58.4) +Requirement already satisfied: python-dateutil>=2.7 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (2.9.0.post0) +Requirement already satisfied: importlib-resources>=3.2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (6.5.2) +Requirement already satisfied: cycler>=0.10 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (0.12.1) +Requirement already satisfied: pillow>=8 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (11.2.1) +Requirement already satisfied: contourpy>=1.0.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.3.0) +Requirement already satisfied: pyparsing>=2.3.1 in /usr/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (2.4.7) +Requirement already satisfied: zipp>=3.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from importlib-resources>=3.2.0->matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (3.23.0) +Requirement already satisfied: dill>=0.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandarallel>=1.5.3->pyhealth==1.1.4) (0.4.0) +Requirement already satisfied: psutil in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandarallel>=1.5.3->pyhealth==1.1.4) (7.0.0) +Requirement already satisfied: pytz>=2020.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandas<2,>=1.3.2->pyhealth==1.1.4) (2025.2) +Requirement already satisfied: requests>=2.19.0 in /usr/lib/python3.9/site-packages (from pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (2.25.1) +Requirement already satisfied: platformdirs>=2.5.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (4.3.8) +Requirement already satisfied: six>=1.5 in /u/ethanmr3/.local/lib/python3.9/site-packages (from python-dateutil>=2.7->matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.17.0) +Requirement already satisfied: idna<3,>=2.5 in /usr/lib/python3.9/site-packages (from requests>=2.19.0->pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (2.10) +Requirement already satisfied: chardet<5,>=3.0.2 in /usr/lib/python3.9/site-packages (from requests>=2.19.0->pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (4.0.0) +Requirement already satisfied: joblib>=1.2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from scikit-learn>=0.24.2->pyhealth==1.1.4) (1.5.1) +Requirement already satisfied: threadpoolctl>=3.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from scikit-learn>=0.24.2->pyhealth==1.1.4) (3.6.0) +Requirement already satisfied: nvidia-cusparse-cu12==12.5.4.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.5.4.2) +Requirement already satisfied: nvidia-nvjitlink-cu12==12.6.85 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.85) +Requirement already satisfied: triton==3.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (3.3.1) +Requirement already satisfied: nvidia-cufile-cu12==1.11.1.6 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (1.11.1.6) +Requirement already satisfied: nvidia-cusparselt-cu12==0.6.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (0.6.3) +Requirement already satisfied: nvidia-cudnn-cu12==9.5.1.17 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (9.5.1.17) +Requirement already satisfied: nvidia-cuda-cupti-cu12==12.6.80 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.80) +Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) +Requirement already satisfied: typing-extensions>=4.10.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (4.14.0) +Requirement already satisfied: nvidia-cuda-runtime-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) +Requirement already satisfied: nvidia-nvtx-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) +Requirement already satisfied: nvidia-curand-cu12==10.3.7.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (10.3.7.77) +Requirement already satisfied: sympy>=1.13.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (1.14.0) +Requirement already satisfied: fsspec in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (2025.5.1) +Requirement already satisfied: nvidia-cublas-cu12==12.6.4.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.4.1) +Requirement already satisfied: nvidia-cusolver-cu12==11.7.1.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (11.7.1.2) +Requirement already satisfied: nvidia-cufft-cu12==11.3.0.4 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (11.3.0.4) +Requirement already satisfied: filelock in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (3.18.0) +Requirement already satisfied: nvidia-nccl-cu12==2.26.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (2.26.2) +Requirement already satisfied: setuptools>=40.8.0 in /usr/lib/python3.9/site-packages (from triton==3.3.1->torch>=1.8.0->pyhealth==1.1.4) (53.0.0) +Requirement already satisfied: mpmath<1.4,>=1.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from sympy>=1.13.3->torch>=1.8.0->pyhealth==1.1.4) (1.3.0) +Requirement already satisfied: MarkupSafe>=2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from jinja2->mne>=1.0.3->pyhealth==1.1.4) (3.0.2) +Requirement already satisfied: safetensors>=0.4.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.5.3) +Requirement already satisfied: tokenizers<0.22,>=0.21 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.21.1) +Requirement already satisfied: regex!=2019.12.17 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (2024.11.6) +Requirement already satisfied: pyyaml>=5.1 in /usr/lib64/python3.9/site-packages (from transformers->pyhealth==1.1.4) (5.4.1) +Requirement already satisfied: huggingface-hub<1.0,>=0.30.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.33.0) +Requirement already satisfied: hf-xet<2.0.0,>=1.1.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from huggingface-hub<1.0,>=0.30.0->transformers->pyhealth==1.1.4) (1.1.5) +Installing collected packages: pandas, pyhealth + Attempting uninstall: pandas + Found existing installation: pandas 2.3.1 + Uninstalling pandas-2.3.1: + Successfully uninstalled pandas-2.3.1 + Attempting uninstall: pyhealth + Found existing installation: pyhealth 1.1.4 + Uninstalling pyhealth-1.1.4: + Successfully uninstalled pyhealth-1.1.4 + Running setup.py develop for pyhealth +Successfully installed pandas-1.5.3 pyhealth +Defaulting to user installation because normal site-packages is not writeable +Collecting numpy + Using cached numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (19.5 MB) +Installing collected packages: numpy + Attempting uninstall: numpy + Found existing installation: numpy 2.0.2 + Uninstalling numpy-2.0.2: + Successfully uninstalled numpy-2.0.2 +Successfully installed numpy-2.0.2 +Defaulting to user installation because normal site-packages is not writeable +Collecting pandas + Using cached pandas-2.3.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.4 MB) +Collecting pytz>=2020.1 + Using cached pytz-2025.2-py2.py3-none-any.whl (509 kB) +Collecting tzdata>=2022.7 + Using cached tzdata-2025.2-py2.py3-none-any.whl (347 kB) +Collecting numpy>=1.22.4 + Using cached numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (19.5 MB) +Collecting python-dateutil>=2.8.2 + Using cached python_dateutil-2.9.0.post0-py2.py3-none-any.whl (229 kB) +Collecting six>=1.5 + Using cached six-1.17.0-py2.py3-none-any.whl (11 kB) +Installing collected packages: six, tzdata, pytz, python-dateutil, numpy, pandas + Attempting uninstall: six + Found existing installation: six 1.17.0 + Uninstalling six-1.17.0: + Successfully uninstalled six-1.17.0 + Attempting uninstall: tzdata + Found existing installation: tzdata 2025.2 + Uninstalling tzdata-2025.2: + Successfully uninstalled tzdata-2025.2 + Attempting uninstall: pytz + Found existing installation: pytz 2025.2 + Uninstalling pytz-2025.2: + Successfully uninstalled pytz-2025.2 + Attempting uninstall: python-dateutil + Found existing installation: python-dateutil 2.9.0.post0 + Uninstalling python-dateutil-2.9.0.post0: + Successfully uninstalled python-dateutil-2.9.0.post0 + Attempting uninstall: numpy + Found existing installation: numpy 2.0.2 + Uninstalling numpy-2.0.2: + Successfully uninstalled numpy-2.0.2 + Attempting uninstall: pandas + Found existing installation: pandas 1.5.3 + Uninstalling pandas-1.5.3: + Successfully uninstalled pandas-1.5.3 +Successfully installed numpy-2.0.2 pandas-2.3.1 python-dateutil-2.9.0.post0 pytz-2025.2 six-1.17.0 tzdata-2025.2 +BEGIN: Testing +Success on pip install -e . +Sucess on imports +Operating in dir: /u/ethanmr3/halo/PyHealth +VOCAB SIZE: 6984 +MAX LEN: 42 +AVG LEN: 1.267755803955288 +MAX VISIT LEN: 39 +AVG VISIT LEN: 11.037371134020619 +NUM RECORDS: 46520 +NUM LONGITUDINAL RECORDS: 7537 +Saving Everything +6984 +test data head 2: +[{'visits': [[5040, 3617, 5458, 5984]], 'labels': array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., + 0., 0., 0., 0., 0., 0., 0., 0.])}, {'visits': [[3686, 2887, 1319, 6919, 3596, 4334, 950, 636, 6847], [4478, 3972, 1029, 3980, 1555, 6812, 1180, 4126, 3101, 286, 6054, 5161, 1072, 5680, 692, 4673, 2768, 2770, 3283, 4445, 3454, 2787, 3710, 2667, 3311, 757, 2678, 5758], [1158, 6919, 5386, 11, 4620, 3607, 1434, 4764, 1184, 3616, 289, 6201, 5319, 2768, 3283, 4568, 3422, 2787, 1252, 3052, 4078, 2678, 6268], [2817, 5890, 1158, 11, 6797, 786, 2066, 153, 6938, 1180, 1053, 939, 950, 2487, 3521, 4673, 5707, 2768, 6101, 5333, 2906, 5087, 2787, 2661, 2156, 4204, 4078, 6767, 5494, 2678, 4478]], 'labels': array([1., 1., 1., 1., 1., 0., 0., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., + 1., 0., 0., 0., 1., 1., 1., 0.])}, {'visits': [[2444, 717, 3052, 274, 693, 4790]], 'labels': array([0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 1., 1., 1., 0., 0., 0., + 0., 0., 0., 0., 0., 0., 0., 0.])}, {'visits': [[6626, 4739, 5924, 4292, 5645, 494, 1072, 5008, 6259]], 'labels': array([0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., + 0., 0., 0., 0., 1., 1., 0., 1.])}, {'visits': [[4298, 5515, 1540]], 'labels': array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., + 0., 0., 0., 0., 0., 0., 0., 0.])}] +Success on model setup diff --git a/halo-testing-logs/halo_test_3709762.err b/halo-testing-logs/halo_test_3709762.err new file mode 100644 index 000000000..30dbe2ee6 --- /dev/null +++ b/halo-testing-logs/halo_test_3709762.err @@ -0,0 +1,28 @@ + WARNING: Value for scheme.platlib does not match. Please report this to + distutils: /u/ethanmr3/.local/lib/python3.9/site-packages + sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages + WARNING: Additional context: + user = True + home = None + root = None + prefix = None + WARNING: Value for scheme.platlib does not match. Please report this to + distutils: /u/ethanmr3/.local/lib/python3.9/site-packages + sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages + WARNING: Additional context: + user = True + home = None + root = None + prefix = None + WARNING: Value for scheme.platlib does not match. Please report this to + distutils: /u/ethanmr3/.local/lib/python3.9/site-packages + sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages + WARNING: Additional context: + user = True + home = None + root = None + prefix = None +ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. +pyhealth 1.1.4 requires pandas<2,>=1.3.2, but you have pandas 2.3.1 which is incompatible. + 0%| | 0/58976 [00:00=1.8.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2.7.1) +Requirement already satisfied: torchvision>=0.9.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (0.22.1) +Requirement already satisfied: rdkit>=2022.03.4 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2025.3.3) +Requirement already satisfied: scikit-learn>=0.24.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.6.1) +Requirement already satisfied: networkx>=2.6.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (3.2.1) +Collecting pandas<2,>=1.3.2 + Using cached pandas-1.5.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.2 MB) +Requirement already satisfied: pandarallel>=1.5.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.6.5) +Requirement already satisfied: mne>=1.0.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.8.0) +Requirement already satisfied: urllib3<=1.26.15 in /usr/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.26.5) +Requirement already satisfied: numpy in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2.0.2) +Requirement already satisfied: tqdm in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (4.67.1) +Requirement already satisfied: polars in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.31.0) +Requirement already satisfied: transformers in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (4.52.4) +Requirement already satisfied: packaging in /usr/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (20.9) +Requirement already satisfied: pooch>=1.5 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (1.8.2) +Requirement already satisfied: matplotlib>=3.6 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (3.9.4) +Requirement already satisfied: jinja2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (3.1.6) +Requirement already satisfied: scipy>=1.9 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (1.13.1) +Requirement already satisfied: decorator in /usr/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (4.4.2) +Requirement already satisfied: lazy-loader>=0.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (0.4) +Requirement already satisfied: kiwisolver>=1.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.4.7) +Requirement already satisfied: contourpy>=1.0.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.3.0) +Requirement already satisfied: pillow>=8 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (11.2.1) +Requirement already satisfied: fonttools>=4.22.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (4.58.4) +Requirement already satisfied: cycler>=0.10 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (0.12.1) +Requirement already satisfied: python-dateutil>=2.7 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (2.9.0.post0) +Requirement already satisfied: pyparsing>=2.3.1 in /usr/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (2.4.7) +Requirement already satisfied: importlib-resources>=3.2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (6.5.2) +Requirement already satisfied: zipp>=3.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from importlib-resources>=3.2.0->matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (3.23.0) +Requirement already satisfied: dill>=0.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandarallel>=1.5.3->pyhealth==1.1.4) (0.4.0) +Requirement already satisfied: psutil in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandarallel>=1.5.3->pyhealth==1.1.4) (7.0.0) +Requirement already satisfied: pytz>=2020.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandas<2,>=1.3.2->pyhealth==1.1.4) (2025.2) +Requirement already satisfied: requests>=2.19.0 in /usr/lib/python3.9/site-packages (from pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (2.25.1) +Requirement already satisfied: platformdirs>=2.5.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (4.3.8) +Requirement already satisfied: six>=1.5 in /u/ethanmr3/.local/lib/python3.9/site-packages (from python-dateutil>=2.7->matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.17.0) +Requirement already satisfied: chardet<5,>=3.0.2 in /usr/lib/python3.9/site-packages (from requests>=2.19.0->pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (4.0.0) +Requirement already satisfied: idna<3,>=2.5 in /usr/lib/python3.9/site-packages (from requests>=2.19.0->pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (2.10) +Requirement already satisfied: threadpoolctl>=3.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from scikit-learn>=0.24.2->pyhealth==1.1.4) (3.6.0) +Requirement already satisfied: joblib>=1.2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from scikit-learn>=0.24.2->pyhealth==1.1.4) (1.5.1) +Requirement already satisfied: nvidia-cuda-runtime-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) +Requirement already satisfied: nvidia-cufft-cu12==11.3.0.4 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (11.3.0.4) +Requirement already satisfied: nvidia-cufile-cu12==1.11.1.6 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (1.11.1.6) +Requirement already satisfied: typing-extensions>=4.10.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (4.14.0) +Requirement already satisfied: triton==3.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (3.3.1) +Requirement already satisfied: nvidia-curand-cu12==10.3.7.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (10.3.7.77) +Requirement already satisfied: filelock in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (3.18.0) +Requirement already satisfied: nvidia-nvjitlink-cu12==12.6.85 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.85) +Requirement already satisfied: fsspec in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (2025.5.1) +Requirement already satisfied: nvidia-cusolver-cu12==11.7.1.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (11.7.1.2) +Requirement already satisfied: nvidia-cudnn-cu12==9.5.1.17 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (9.5.1.17) +Requirement already satisfied: nvidia-cublas-cu12==12.6.4.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.4.1) +Requirement already satisfied: sympy>=1.13.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (1.14.0) +Requirement already satisfied: nvidia-nccl-cu12==2.26.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (2.26.2) +Requirement already satisfied: nvidia-cusparse-cu12==12.5.4.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.5.4.2) +Requirement already satisfied: nvidia-cuda-cupti-cu12==12.6.80 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.80) +Requirement already satisfied: nvidia-cusparselt-cu12==0.6.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (0.6.3) +Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) +Requirement already satisfied: nvidia-nvtx-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) +Requirement already satisfied: setuptools>=40.8.0 in /usr/lib/python3.9/site-packages (from triton==3.3.1->torch>=1.8.0->pyhealth==1.1.4) (53.0.0) +Requirement already satisfied: mpmath<1.4,>=1.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from sympy>=1.13.3->torch>=1.8.0->pyhealth==1.1.4) (1.3.0) +Requirement already satisfied: MarkupSafe>=2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from jinja2->mne>=1.0.3->pyhealth==1.1.4) (3.0.2) +Requirement already satisfied: safetensors>=0.4.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.5.3) +Requirement already satisfied: huggingface-hub<1.0,>=0.30.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.33.0) +Requirement already satisfied: tokenizers<0.22,>=0.21 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.21.1) +Requirement already satisfied: regex!=2019.12.17 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (2024.11.6) +Requirement already satisfied: pyyaml>=5.1 in /usr/lib64/python3.9/site-packages (from transformers->pyhealth==1.1.4) (5.4.1) +Requirement already satisfied: hf-xet<2.0.0,>=1.1.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from huggingface-hub<1.0,>=0.30.0->transformers->pyhealth==1.1.4) (1.1.5) +Installing collected packages: pandas, pyhealth + Attempting uninstall: pandas + Found existing installation: pandas 2.3.1 + Uninstalling pandas-2.3.1: + Successfully uninstalled pandas-2.3.1 + Attempting uninstall: pyhealth + Found existing installation: pyhealth 1.1.4 + Uninstalling pyhealth-1.1.4: + Successfully uninstalled pyhealth-1.1.4 + Running setup.py develop for pyhealth +Successfully installed pandas-1.5.3 pyhealth +Defaulting to user installation because normal site-packages is not writeable +Collecting numpy + Using cached numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (19.5 MB) +Installing collected packages: numpy + Attempting uninstall: numpy + Found existing installation: numpy 2.0.2 + Uninstalling numpy-2.0.2: + Successfully uninstalled numpy-2.0.2 +Successfully installed numpy-2.0.2 +Defaulting to user installation because normal site-packages is not writeable +Collecting pandas + Using cached pandas-2.3.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.4 MB) +Collecting python-dateutil>=2.8.2 + Using cached python_dateutil-2.9.0.post0-py2.py3-none-any.whl (229 kB) +Collecting numpy>=1.22.4 + Using cached numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (19.5 MB) +Collecting pytz>=2020.1 + Using cached pytz-2025.2-py2.py3-none-any.whl (509 kB) +Collecting tzdata>=2022.7 + Using cached tzdata-2025.2-py2.py3-none-any.whl (347 kB) +Collecting six>=1.5 + Using cached six-1.17.0-py2.py3-none-any.whl (11 kB) +Installing collected packages: six, tzdata, pytz, python-dateutil, numpy, pandas + Attempting uninstall: six + Found existing installation: six 1.17.0 + Uninstalling six-1.17.0: + Successfully uninstalled six-1.17.0 + Attempting uninstall: tzdata + Found existing installation: tzdata 2025.2 + Uninstalling tzdata-2025.2: + Successfully uninstalled tzdata-2025.2 + Attempting uninstall: pytz + Found existing installation: pytz 2025.2 + Uninstalling pytz-2025.2: + Successfully uninstalled pytz-2025.2 + Attempting uninstall: python-dateutil + Found existing installation: python-dateutil 2.9.0.post0 + Uninstalling python-dateutil-2.9.0.post0: + Successfully uninstalled python-dateutil-2.9.0.post0 + Attempting uninstall: numpy + Found existing installation: numpy 2.0.2 + Uninstalling numpy-2.0.2: + Successfully uninstalled numpy-2.0.2 + Attempting uninstall: pandas + Found existing installation: pandas 1.5.3 + Uninstalling pandas-1.5.3: + Successfully uninstalled pandas-1.5.3 +Successfully installed numpy-2.0.2 pandas-2.3.1 python-dateutil-2.9.0.post0 pytz-2025.2 six-1.17.0 tzdata-2025.2 +BEGIN: Testing +Success on pip install -e . +Sucess on imports +Operating in dir: /u/ethanmr3/halo/PyHealth +VOCAB SIZE: 6984 +MAX LEN: 42 +AVG LEN: 1.267755803955288 +MAX VISIT LEN: 39 +AVG VISIT LEN: 11.037371134020619 +NUM RECORDS: 46520 +NUM LONGITUDINAL RECORDS: 7537 +Saving Everything +6984 +test data head 2: +[{'visits': [[3625, 827, 6510, 1663]], 'labels': array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., + 0., 0., 0., 0., 0., 0., 0., 0.])}, {'visits': [[5856, 673, 1280, 3459, 3020, 3121, 4337, 119, 6104], [3714, 901, 1807, 3351, 4385, 5410, 292, 5030, 5544, 3756, 940, 1603, 5318, 2249, 2252, 3277, 4686, 5842, 5586, 606, 2271, 4578, 3582, 4849, 5112, 505, 4221, 5374], [129, 6662, 4359, 650, 4748, 796, 2850, 940, 2353, 3121, 4145, 820, 5176, 4666, 3776, 1603, 2123, 3277, 6607, 3295, 6884, 3698, 505], [6658, 6150, 263, 1807, 2961, 5778, 3351, 2712, 5787, 796, 2464, 673, 2725, 940, 5177, 1603, 3012, 967, 74, 2123, 592, 5842, 6108, 2275, 4967, 505, 3698, 2169, 6138, 4222, 5247]], 'labels': array([1., 1., 1., 1., 1., 0., 0., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., + 1., 0., 0., 0., 1., 1., 1., 0.])}, {'visits': [[3776, 489, 1902, 4113, 598, 4183]], 'labels': array([0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 1., 1., 1., 0., 0., 0., + 0., 0., 0., 0., 0., 0., 0., 0.])}, {'visits': [[1152, 4130, 2596, 5956, 2252, 111, 1776, 2708, 4062]], 'labels': array([0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., + 0., 0., 0., 0., 1., 1., 0., 1.])}, {'visits': [[6844, 1222, 2903]], 'labels': array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., + 0., 0., 0., 0., 0., 0., 0., 0.])}] +Success on model setup diff --git a/halo-testing-logs/halo_test_3713150.err b/halo-testing-logs/halo_test_3713150.err new file mode 100644 index 000000000..26bdd2dd4 --- /dev/null +++ b/halo-testing-logs/halo_test_3713150.err @@ -0,0 +1,27 @@ + WARNING: Value for scheme.platlib does not match. Please report this to + distutils: /u/ethanmr3/.local/lib/python3.9/site-packages + sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages + WARNING: Additional context: + user = True + home = None + root = None + prefix = None + WARNING: Value for scheme.platlib does not match. Please report this to + distutils: /u/ethanmr3/.local/lib/python3.9/site-packages + sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages + WARNING: Additional context: + user = True + home = None + root = None + prefix = None + WARNING: Value for scheme.platlib does not match. Please report this to + distutils: /u/ethanmr3/.local/lib/python3.9/site-packages + sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages + WARNING: Additional context: + user = True + home = None + root = None + prefix = None +ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. +pyhealth 1.1.4 requires pandas<2,>=1.3.2, but you have pandas 2.3.1 which is incompatible. + 0%| | 0/58976 [00:00=1.8.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2.7.1) +Requirement already satisfied: torchvision>=0.9.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (0.22.1) +Requirement already satisfied: rdkit>=2022.03.4 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2025.3.3) +Requirement already satisfied: scikit-learn>=0.24.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.6.1) +Requirement already satisfied: networkx>=2.6.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (3.2.1) +Collecting pandas<2,>=1.3.2 + Using cached pandas-1.5.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.2 MB) +Requirement already satisfied: pandarallel>=1.5.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.6.5) +Requirement already satisfied: mne>=1.0.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.8.0) +Requirement already satisfied: urllib3<=1.26.15 in /usr/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.26.5) +Requirement already satisfied: numpy in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2.0.2) +Requirement already satisfied: tqdm in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (4.67.1) +Requirement already satisfied: polars in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.31.0) +Requirement already satisfied: transformers in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (4.52.4) +Requirement already satisfied: lazy-loader>=0.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (0.4) +Requirement already satisfied: jinja2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (3.1.6) +Requirement already satisfied: packaging in /usr/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (20.9) +Requirement already satisfied: decorator in /usr/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (4.4.2) +Requirement already satisfied: scipy>=1.9 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (1.13.1) +Requirement already satisfied: matplotlib>=3.6 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (3.9.4) +Requirement already satisfied: pooch>=1.5 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (1.8.2) +Requirement already satisfied: pyparsing>=2.3.1 in /usr/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (2.4.7) +Requirement already satisfied: contourpy>=1.0.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.3.0) +Requirement already satisfied: importlib-resources>=3.2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (6.5.2) +Requirement already satisfied: pillow>=8 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (11.2.1) +Requirement already satisfied: kiwisolver>=1.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.4.7) +Requirement already satisfied: fonttools>=4.22.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (4.58.4) +Requirement already satisfied: python-dateutil>=2.7 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (2.9.0.post0) +Requirement already satisfied: cycler>=0.10 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (0.12.1) +Requirement already satisfied: zipp>=3.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from importlib-resources>=3.2.0->matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (3.23.0) +Requirement already satisfied: dill>=0.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandarallel>=1.5.3->pyhealth==1.1.4) (0.4.0) +Requirement already satisfied: psutil in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandarallel>=1.5.3->pyhealth==1.1.4) (7.0.0) +Requirement already satisfied: pytz>=2020.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandas<2,>=1.3.2->pyhealth==1.1.4) (2025.2) +Requirement already satisfied: platformdirs>=2.5.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (4.3.8) +Requirement already satisfied: requests>=2.19.0 in /usr/lib/python3.9/site-packages (from pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (2.25.1) +Requirement already satisfied: six>=1.5 in /u/ethanmr3/.local/lib/python3.9/site-packages (from python-dateutil>=2.7->matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.17.0) +Requirement already satisfied: chardet<5,>=3.0.2 in /usr/lib/python3.9/site-packages (from requests>=2.19.0->pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (4.0.0) +Requirement already satisfied: idna<3,>=2.5 in /usr/lib/python3.9/site-packages (from requests>=2.19.0->pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (2.10) +Requirement already satisfied: joblib>=1.2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from scikit-learn>=0.24.2->pyhealth==1.1.4) (1.5.1) +Requirement already satisfied: threadpoolctl>=3.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from scikit-learn>=0.24.2->pyhealth==1.1.4) (3.6.0) +Requirement already satisfied: sympy>=1.13.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (1.14.0) +Requirement already satisfied: nvidia-nvtx-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) +Requirement already satisfied: nvidia-cuda-runtime-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) +Requirement already satisfied: nvidia-cublas-cu12==12.6.4.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.4.1) +Requirement already satisfied: nvidia-curand-cu12==10.3.7.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (10.3.7.77) +Requirement already satisfied: nvidia-cufile-cu12==1.11.1.6 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (1.11.1.6) +Requirement already satisfied: typing-extensions>=4.10.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (4.14.0) +Requirement already satisfied: nvidia-cufft-cu12==11.3.0.4 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (11.3.0.4) +Requirement already satisfied: nvidia-cudnn-cu12==9.5.1.17 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (9.5.1.17) +Requirement already satisfied: nvidia-cusparselt-cu12==0.6.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (0.6.3) +Requirement already satisfied: nvidia-cusparse-cu12==12.5.4.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.5.4.2) +Requirement already satisfied: fsspec in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (2025.5.1) +Requirement already satisfied: filelock in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (3.18.0) +Requirement already satisfied: nvidia-cusolver-cu12==11.7.1.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (11.7.1.2) +Requirement already satisfied: triton==3.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (3.3.1) +Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) +Requirement already satisfied: nvidia-cuda-cupti-cu12==12.6.80 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.80) +Requirement already satisfied: nvidia-nccl-cu12==2.26.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (2.26.2) +Requirement already satisfied: nvidia-nvjitlink-cu12==12.6.85 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.85) +Requirement already satisfied: setuptools>=40.8.0 in /usr/lib/python3.9/site-packages (from triton==3.3.1->torch>=1.8.0->pyhealth==1.1.4) (53.0.0) +Requirement already satisfied: mpmath<1.4,>=1.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from sympy>=1.13.3->torch>=1.8.0->pyhealth==1.1.4) (1.3.0) +Requirement already satisfied: MarkupSafe>=2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from jinja2->mne>=1.0.3->pyhealth==1.1.4) (3.0.2) +Requirement already satisfied: safetensors>=0.4.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.5.3) +Requirement already satisfied: tokenizers<0.22,>=0.21 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.21.1) +Requirement already satisfied: pyyaml>=5.1 in /usr/lib64/python3.9/site-packages (from transformers->pyhealth==1.1.4) (5.4.1) +Requirement already satisfied: regex!=2019.12.17 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (2024.11.6) +Requirement already satisfied: huggingface-hub<1.0,>=0.30.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.33.0) +Requirement already satisfied: hf-xet<2.0.0,>=1.1.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from huggingface-hub<1.0,>=0.30.0->transformers->pyhealth==1.1.4) (1.1.5) +Installing collected packages: pandas, pyhealth + Attempting uninstall: pandas + Found existing installation: pandas 2.3.1 + Uninstalling pandas-2.3.1: + Successfully uninstalled pandas-2.3.1 + Attempting uninstall: pyhealth + Found existing installation: pyhealth 1.1.4 + Uninstalling pyhealth-1.1.4: + Successfully uninstalled pyhealth-1.1.4 + Running setup.py develop for pyhealth +Successfully installed pandas-1.5.3 pyhealth +Defaulting to user installation because normal site-packages is not writeable +Collecting numpy + Using cached numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (19.5 MB) +Installing collected packages: numpy + Attempting uninstall: numpy + Found existing installation: numpy 2.0.2 + Uninstalling numpy-2.0.2: + Successfully uninstalled numpy-2.0.2 +Successfully installed numpy-2.0.2 +Defaulting to user installation because normal site-packages is not writeable +Collecting pandas + Using cached pandas-2.3.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.4 MB) +Collecting python-dateutil>=2.8.2 + Using cached python_dateutil-2.9.0.post0-py2.py3-none-any.whl (229 kB) +Collecting numpy>=1.22.4 + Using cached numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (19.5 MB) +Collecting tzdata>=2022.7 + Using cached tzdata-2025.2-py2.py3-none-any.whl (347 kB) +Collecting pytz>=2020.1 + Using cached pytz-2025.2-py2.py3-none-any.whl (509 kB) +Collecting six>=1.5 + Using cached six-1.17.0-py2.py3-none-any.whl (11 kB) +Installing collected packages: six, tzdata, pytz, python-dateutil, numpy, pandas + Attempting uninstall: six + Found existing installation: six 1.17.0 + Uninstalling six-1.17.0: + Successfully uninstalled six-1.17.0 + Attempting uninstall: tzdata + Found existing installation: tzdata 2025.2 + Uninstalling tzdata-2025.2: + Successfully uninstalled tzdata-2025.2 + Attempting uninstall: pytz + Found existing installation: pytz 2025.2 + Uninstalling pytz-2025.2: + Successfully uninstalled pytz-2025.2 + Attempting uninstall: python-dateutil + Found existing installation: python-dateutil 2.9.0.post0 + Uninstalling python-dateutil-2.9.0.post0: + Successfully uninstalled python-dateutil-2.9.0.post0 + Attempting uninstall: numpy + Found existing installation: numpy 2.0.2 + Uninstalling numpy-2.0.2: + Successfully uninstalled numpy-2.0.2 + Attempting uninstall: pandas + Found existing installation: pandas 1.5.3 + Uninstalling pandas-1.5.3: + Successfully uninstalled pandas-1.5.3 +Successfully installed numpy-2.0.2 pandas-2.3.1 python-dateutil-2.9.0.post0 pytz-2025.2 six-1.17.0 tzdata-2025.2 +BEGIN: Testing +Success on pip install -e . +Sucess on imports +Operating in dir: /u/ethanmr3/halo/PyHealth diff --git a/halo-testing-logs/halo_test_3713164.err b/halo-testing-logs/halo_test_3713164.err new file mode 100644 index 000000000..6a3300cd0 --- /dev/null +++ b/halo-testing-logs/halo_test_3713164.err @@ -0,0 +1,30 @@ + WARNING: Value for scheme.platlib does not match. Please report this to + distutils: /u/ethanmr3/.local/lib/python3.9/site-packages + sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages + WARNING: Additional context: + user = True + home = None + root = None + prefix = None + WARNING: Value for scheme.platlib does not match. Please report this to + distutils: /u/ethanmr3/.local/lib/python3.9/site-packages + sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages + WARNING: Additional context: + user = True + home = None + root = None + prefix = None + WARNING: Value for scheme.platlib does not match. Please report this to + distutils: /u/ethanmr3/.local/lib/python3.9/site-packages + sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages + WARNING: Additional context: + user = True + home = None + root = None + prefix = None +ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. +pyhealth 1.1.4 requires pandas<2,>=1.3.2, but you have pandas 2.3.1 which is incompatible. + 0%| | 0/58976 [00:00=1.8.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2.7.1) +Requirement already satisfied: torchvision>=0.9.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (0.22.1) +Requirement already satisfied: rdkit>=2022.03.4 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2025.3.3) +Requirement already satisfied: scikit-learn>=0.24.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.6.1) +Requirement already satisfied: networkx>=2.6.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (3.2.1) +Collecting pandas<2,>=1.3.2 + Using cached pandas-1.5.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.2 MB) +Requirement already satisfied: pandarallel>=1.5.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.6.5) +Requirement already satisfied: mne>=1.0.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.8.0) +Requirement already satisfied: urllib3<=1.26.15 in /usr/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.26.5) +Requirement already satisfied: numpy in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2.0.2) +Requirement already satisfied: tqdm in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (4.67.1) +Requirement already satisfied: polars in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.31.0) +Requirement already satisfied: transformers in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (4.52.4) +Requirement already satisfied: scipy>=1.9 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (1.13.1) +Requirement already satisfied: lazy-loader>=0.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (0.4) +Requirement already satisfied: matplotlib>=3.6 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (3.9.4) +Requirement already satisfied: packaging in /usr/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (20.9) +Requirement already satisfied: decorator in /usr/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (4.4.2) +Requirement already satisfied: jinja2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (3.1.6) +Requirement already satisfied: pooch>=1.5 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (1.8.2) +Requirement already satisfied: python-dateutil>=2.7 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (2.9.0.post0) +Requirement already satisfied: pillow>=8 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (11.2.1) +Requirement already satisfied: pyparsing>=2.3.1 in /usr/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (2.4.7) +Requirement already satisfied: contourpy>=1.0.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.3.0) +Requirement already satisfied: fonttools>=4.22.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (4.58.4) +Requirement already satisfied: kiwisolver>=1.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.4.7) +Requirement already satisfied: cycler>=0.10 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (0.12.1) +Requirement already satisfied: importlib-resources>=3.2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (6.5.2) +Requirement already satisfied: zipp>=3.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from importlib-resources>=3.2.0->matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (3.23.0) +Requirement already satisfied: dill>=0.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandarallel>=1.5.3->pyhealth==1.1.4) (0.4.0) +Requirement already satisfied: psutil in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandarallel>=1.5.3->pyhealth==1.1.4) (7.0.0) +Requirement already satisfied: pytz>=2020.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandas<2,>=1.3.2->pyhealth==1.1.4) (2025.2) +Requirement already satisfied: platformdirs>=2.5.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (4.3.8) +Requirement already satisfied: requests>=2.19.0 in /usr/lib/python3.9/site-packages (from pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (2.25.1) +Requirement already satisfied: six>=1.5 in /u/ethanmr3/.local/lib/python3.9/site-packages (from python-dateutil>=2.7->matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.17.0) +Requirement already satisfied: chardet<5,>=3.0.2 in /usr/lib/python3.9/site-packages (from requests>=2.19.0->pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (4.0.0) +Requirement already satisfied: idna<3,>=2.5 in /usr/lib/python3.9/site-packages (from requests>=2.19.0->pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (2.10) +Requirement already satisfied: threadpoolctl>=3.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from scikit-learn>=0.24.2->pyhealth==1.1.4) (3.6.0) +Requirement already satisfied: joblib>=1.2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from scikit-learn>=0.24.2->pyhealth==1.1.4) (1.5.1) +Requirement already satisfied: nvidia-cuda-runtime-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) +Requirement already satisfied: nvidia-cusolver-cu12==11.7.1.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (11.7.1.2) +Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) +Requirement already satisfied: triton==3.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (3.3.1) +Requirement already satisfied: nvidia-curand-cu12==10.3.7.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (10.3.7.77) +Requirement already satisfied: sympy>=1.13.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (1.14.0) +Requirement already satisfied: nvidia-nccl-cu12==2.26.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (2.26.2) +Requirement already satisfied: typing-extensions>=4.10.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (4.14.0) +Requirement already satisfied: nvidia-cusparse-cu12==12.5.4.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.5.4.2) +Requirement already satisfied: nvidia-cufile-cu12==1.11.1.6 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (1.11.1.6) +Requirement already satisfied: nvidia-nvjitlink-cu12==12.6.85 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.85) +Requirement already satisfied: nvidia-cudnn-cu12==9.5.1.17 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (9.5.1.17) +Requirement already satisfied: nvidia-nvtx-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) +Requirement already satisfied: nvidia-cusparselt-cu12==0.6.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (0.6.3) +Requirement already satisfied: nvidia-cuda-cupti-cu12==12.6.80 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.80) +Requirement already satisfied: fsspec in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (2025.5.1) +Requirement already satisfied: nvidia-cufft-cu12==11.3.0.4 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (11.3.0.4) +Requirement already satisfied: nvidia-cublas-cu12==12.6.4.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.4.1) +Requirement already satisfied: filelock in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (3.18.0) +Requirement already satisfied: setuptools>=40.8.0 in /usr/lib/python3.9/site-packages (from triton==3.3.1->torch>=1.8.0->pyhealth==1.1.4) (53.0.0) +Requirement already satisfied: mpmath<1.4,>=1.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from sympy>=1.13.3->torch>=1.8.0->pyhealth==1.1.4) (1.3.0) +Requirement already satisfied: MarkupSafe>=2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from jinja2->mne>=1.0.3->pyhealth==1.1.4) (3.0.2) +Requirement already satisfied: pyyaml>=5.1 in /usr/lib64/python3.9/site-packages (from transformers->pyhealth==1.1.4) (5.4.1) +Requirement already satisfied: huggingface-hub<1.0,>=0.30.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.33.0) +Requirement already satisfied: safetensors>=0.4.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.5.3) +Requirement already satisfied: regex!=2019.12.17 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (2024.11.6) +Requirement already satisfied: tokenizers<0.22,>=0.21 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.21.1) +Requirement already satisfied: hf-xet<2.0.0,>=1.1.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from huggingface-hub<1.0,>=0.30.0->transformers->pyhealth==1.1.4) (1.1.5) +Installing collected packages: pandas, pyhealth + Attempting uninstall: pandas + Found existing installation: pandas 2.3.1 + Uninstalling pandas-2.3.1: + Successfully uninstalled pandas-2.3.1 + Attempting uninstall: pyhealth + Found existing installation: pyhealth 1.1.4 + Uninstalling pyhealth-1.1.4: + Successfully uninstalled pyhealth-1.1.4 + Running setup.py develop for pyhealth +Successfully installed pandas-1.5.3 pyhealth +Defaulting to user installation because normal site-packages is not writeable +Collecting numpy + Using cached numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (19.5 MB) +Installing collected packages: numpy + Attempting uninstall: numpy + Found existing installation: numpy 2.0.2 + Uninstalling numpy-2.0.2: + Successfully uninstalled numpy-2.0.2 +Successfully installed numpy-2.0.2 +Defaulting to user installation because normal site-packages is not writeable +Collecting pandas + Using cached pandas-2.3.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.4 MB) +Collecting tzdata>=2022.7 + Using cached tzdata-2025.2-py2.py3-none-any.whl (347 kB) +Collecting numpy>=1.22.4 + Using cached numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (19.5 MB) +Collecting pytz>=2020.1 + Using cached pytz-2025.2-py2.py3-none-any.whl (509 kB) +Collecting python-dateutil>=2.8.2 + Using cached python_dateutil-2.9.0.post0-py2.py3-none-any.whl (229 kB) +Collecting six>=1.5 + Using cached six-1.17.0-py2.py3-none-any.whl (11 kB) +Installing collected packages: six, tzdata, pytz, python-dateutil, numpy, pandas + Attempting uninstall: six + Found existing installation: six 1.17.0 + Uninstalling six-1.17.0: + Successfully uninstalled six-1.17.0 + Attempting uninstall: tzdata + Found existing installation: tzdata 2025.2 + Uninstalling tzdata-2025.2: + Successfully uninstalled tzdata-2025.2 + Attempting uninstall: pytz + Found existing installation: pytz 2025.2 + Uninstalling pytz-2025.2: + Successfully uninstalled pytz-2025.2 + Attempting uninstall: python-dateutil + Found existing installation: python-dateutil 2.9.0.post0 + Uninstalling python-dateutil-2.9.0.post0: + Successfully uninstalled python-dateutil-2.9.0.post0 + Attempting uninstall: numpy + Found existing installation: numpy 2.0.2 + Uninstalling numpy-2.0.2: + Successfully uninstalled numpy-2.0.2 + Attempting uninstall: pandas + Found existing installation: pandas 1.5.3 + Uninstalling pandas-1.5.3: + Successfully uninstalled pandas-1.5.3 +Successfully installed numpy-2.0.2 pandas-2.3.1 python-dateutil-2.9.0.post0 pytz-2025.2 six-1.17.0 tzdata-2025.2 +BEGIN: Testing +Success on pip install -e . +Sucess on imports +Operating in dir: /u/ethanmr3/halo/PyHealth +VOCAB SIZE: 6984 +MAX LEN: 42 +AVG LEN: 1.267755803955288 +MAX VISIT LEN: 39 +AVG VISIT LEN: 11.037371134020619 +NUM RECORDS: 46520 +NUM LONGITUDINAL RECORDS: 7537 +Saving Everything +6984 +test data head 2: +[{'visits': [[4185, 5429, 2182, 63]], 'labels': array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., + 0., 0., 0., 0., 0., 0., 0., 0.])}, {'visits': [[544, 6081, 1440, 5772, 5485, 5552, 6066, 1463, 1661], [1412, 2443, 5519, 3344, 1168, 1809, 1807, 3731, 6172, 4644, 804, 5676, 5947, 6077, 450, 324, 1989, 6471, 4170, 983, 224, 3557, 1637, 5737, 5354, 3185, 2546, 886], [4485, 6540, 1807, 2704, 544, 3750, 44, 5682, 2486, 3896, 184, 6471, 6608, 5329, 6226, 2903, 5719, 5737, 1773, 6384, 1522, 886, 6011], [6784, 2183, 6539, 270, 4623, 2704, 6801, 1807, 802, 5797, 6699, 4524, 5552, 4145, 5555, 950, 3896, 5947, 1217, 4679, 6983, 2903, 983, 2010, 3557, 4967, 5737, 886, 3322, 1916, 3454]], 'labels': array([1., 1., 1., 1., 1., 0., 0., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., + 1., 0., 0., 0., 1., 1., 1., 0.])}, {'visits': [[6600, 4105, 5329, 3513, 1596, 4765]], 'labels': array([0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 1., 1., 1., 0., 0., 0., + 0., 0., 0., 0., 0., 0., 0., 0.])}, {'visits': [[3008, 1989, 6214, 2535, 5162, 2870, 5208, 4953, 4540]], 'labels': array([0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., + 0., 0., 0., 0., 1., 1., 0., 1.])}, {'visits': [[4982, 1375, 2879]], 'labels': array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., + 0., 0., 0., 0., 0., 0., 0., 0.])}] +Success on model setup +Epoch 0, Iter 0: Training Loss:0.003035 +Epoch 0, Iter 24000: Training Loss:0.002145 +Epoch 0 Validation Loss:0.0002305 + +------------ Save best model ------------ + +Epoch 1, Iter 0: Training Loss:0.001832 +Epoch 1, Iter 24000: Training Loss:0.001805 +Epoch 1 Validation Loss:0.0002257 + +------------ Save best model ------------ + +Epoch 2, Iter 0: Training Loss:0.001855 +Epoch 2, Iter 24000: Training Loss:0.001980 +Epoch 2 Validation Loss:0.0002229 + +------------ Save best model ------------ + +Epoch 3, Iter 0: Training Loss:0.001336 +Epoch 3, Iter 24000: Training Loss:0.001547 +Epoch 3 Validation Loss:0.0002187 + +------------ Save best model ------------ + +Epoch 4, Iter 0: Training Loss:0.001439 +Epoch 4, Iter 24000: Training Loss:0.001622 +Epoch 4 Validation Loss:0.0002177 + +------------ Save best model ------------ + +Epoch 5, Iter 0: Training Loss:0.001393 +Epoch 5, Iter 24000: Training Loss:0.001476 +Epoch 5 Validation Loss:0.0002161 + +------------ Save best model ------------ + +Epoch 6, Iter 0: Training Loss:0.001374 +Epoch 6, Iter 24000: Training Loss:0.001541 +Epoch 6 Validation Loss:0.0002136 + +------------ Save best model ------------ + +Epoch 7, Iter 0: Training Loss:0.001676 +Epoch 7, Iter 24000: Training Loss:0.001921 +Epoch 7 Validation Loss:0.0002123 + +------------ Save best model ------------ + +Epoch 8, Iter 0: Training Loss:0.001604 +Epoch 8, Iter 24000: Training Loss:0.001605 +Epoch 8 Validation Loss:0.0002100 + +------------ Save best model ------------ + +Epoch 9, Iter 0: Training Loss:0.001878 +Epoch 9, Iter 24000: Training Loss:0.001517 +Epoch 9 Validation Loss:0.0002089 + +------------ Save best model ------------ + +Epoch 10, Iter 0: Training Loss:0.001447 +Epoch 10, Iter 24000: Training Loss:0.001354 +Epoch 10 Validation Loss:0.0002079 + +------------ Save best model ------------ + +Epoch 11, Iter 0: Training Loss:0.001610 +Epoch 11, Iter 24000: Training Loss:0.001414 +Epoch 11 Validation Loss:0.0002068 + +------------ Save best model ------------ + +Epoch 12, Iter 0: Training Loss:0.001676 +Epoch 12, Iter 24000: Training Loss:0.001512 +Epoch 12 Validation Loss:0.0002053 + +------------ Save best model ------------ + +Epoch 13, Iter 0: Training Loss:0.001766 +Epoch 13, Iter 24000: Training Loss:0.001499 +Epoch 13 Validation Loss:0.0002039 + +------------ Save best model ------------ + +Epoch 14, Iter 0: Training Loss:0.001794 +Epoch 14, Iter 24000: Training Loss:0.001430 +Epoch 14 Validation Loss:0.0002030 + +------------ Save best model ------------ + +Epoch 15, Iter 0: Training Loss:0.001648 +Epoch 15, Iter 24000: Training Loss:0.001802 +Epoch 15 Validation Loss:0.0002019 + +------------ Save best model ------------ + +Epoch 16, Iter 0: Training Loss:0.001942 +Epoch 16, Iter 24000: Training Loss:0.001631 +Epoch 16 Validation Loss:0.0002016 + +------------ Save best model ------------ + +Epoch 17, Iter 0: Training Loss:0.001647 +Epoch 17, Iter 24000: Training Loss:0.001143 +Epoch 17 Validation Loss:0.0002010 + +------------ Save best model ------------ + +Epoch 18, Iter 0: Training Loss:0.001599 +Epoch 18, Iter 24000: Training Loss:0.001771 +Epoch 18 Validation Loss:0.0002006 + +------------ Save best model ------------ + +Epoch 19, Iter 0: Training Loss:0.001910 +Epoch 19, Iter 24000: Training Loss:0.001550 +Epoch 19 Validation Loss:0.0002003 + +------------ Save best model ------------ + +Epoch 20, Iter 0: Training Loss:0.001422 +Epoch 20, Iter 24000: Training Loss:0.001366 +Epoch 20 Validation Loss:0.0002001 + +------------ Save best model ------------ + +Epoch 21, Iter 0: Training Loss:0.001291 +Epoch 21, Iter 24000: Training Loss:0.001427 +Epoch 21 Validation Loss:0.0001996 + +------------ Save best model ------------ + +Epoch 22, Iter 0: Training Loss:0.001385 +Epoch 22, Iter 24000: Training Loss:0.001431 +Epoch 22 Validation Loss:0.0001995 + +------------ Save best model ------------ + +Epoch 23, Iter 0: Training Loss:0.001395 +Epoch 23, Iter 24000: Training Loss:0.001319 +Epoch 23 Validation Loss:0.0001989 + +------------ Save best model ------------ + +Epoch 24, Iter 0: Training Loss:0.001320 +Epoch 24, Iter 24000: Training Loss:0.001303 +Epoch 24 Validation Loss:0.0001991 +Epoch 25, Iter 0: Training Loss:0.001388 +Epoch 25, Iter 24000: Training Loss:0.001399 +Epoch 25 Validation Loss:0.0001993 +Epoch 26, Iter 0: Training Loss:0.001399 +Epoch 26, Iter 24000: Training Loss:0.001139 +Epoch 26 Validation Loss:0.0001995 +Epoch 27, Iter 0: Training Loss:0.001415 +Epoch 27, Iter 24000: Training Loss:0.001281 +Epoch 27 Validation Loss:0.0001997 +Epoch 28, Iter 0: Training Loss:0.001672 +Epoch 28, Iter 24000: Training Loss:0.001224 +Epoch 28 Validation Loss:0.0001993 +Epoch 29, Iter 0: Training Loss:0.001540 +Epoch 29, Iter 24000: Training Loss:0.001223 +Epoch 29 Validation Loss:0.0001995 +Epoch 30, Iter 0: Training Loss:0.001573 +Epoch 30, Iter 24000: Training Loss:0.001106 +Epoch 30 Validation Loss:0.0002004 +Epoch 31, Iter 0: Training Loss:0.001323 +Epoch 31, Iter 24000: Training Loss:0.001443 +Epoch 31 Validation Loss:0.0002011 +Epoch 32, Iter 0: Training Loss:0.001441 +Epoch 32, Iter 24000: Training Loss:0.001345 +Epoch 32 Validation Loss:0.0002013 +Epoch 33, Iter 0: Training Loss:0.001364 +Epoch 33, Iter 24000: Training Loss:0.001100 +Epoch 33 Validation Loss:0.0002019 +Epoch 34, Iter 0: Training Loss:0.001306 +Epoch 34, Iter 24000: Training Loss:0.001148 +Epoch 34 Validation Loss:0.0002029 +Epoch 35, Iter 0: Training Loss:0.001301 +Epoch 35, Iter 24000: Training Loss:0.001408 +Epoch 35 Validation Loss:0.0002038 +Epoch 36, Iter 0: Training Loss:0.001098 +Epoch 36, Iter 24000: Training Loss:0.001162 +Epoch 36 Validation Loss:0.0002050 +Epoch 37, Iter 0: Training Loss:0.001330 +Epoch 37, Iter 24000: Training Loss:0.001073 +Epoch 37 Validation Loss:0.0002063 +Epoch 38, Iter 0: Training Loss:0.001253 +Epoch 38, Iter 24000: Training Loss:0.001129 +Epoch 38 Validation Loss:0.0002069 +Epoch 39, Iter 0: Training Loss:0.001055 +Epoch 39, Iter 24000: Training Loss:0.001310 +Epoch 39 Validation Loss:0.0002079 +Epoch 40, Iter 0: Training Loss:0.001125 +Epoch 40, Iter 24000: Training Loss:0.001104 +Epoch 40 Validation Loss:0.0002095 +Epoch 41, Iter 0: Training Loss:0.001377 +Epoch 41, Iter 24000: Training Loss:0.001138 +Epoch 41 Validation Loss:0.0002109 +Epoch 42, Iter 0: Training Loss:0.001018 +Epoch 42, Iter 24000: Training Loss:0.000935 +Epoch 42 Validation Loss:0.0002124 +Epoch 43, Iter 0: Training Loss:0.001339 +Epoch 43, Iter 24000: Training Loss:0.001016 +Epoch 43 Validation Loss:0.0002149 +Epoch 44, Iter 0: Training Loss:0.001004 +Epoch 44, Iter 24000: Training Loss:0.001025 +Epoch 44 Validation Loss:0.0002163 +Epoch 45, Iter 0: Training Loss:0.001063 +Epoch 45, Iter 24000: Training Loss:0.000845 +Epoch 45 Validation Loss:0.0002190 +Epoch 46, Iter 0: Training Loss:0.000964 +Epoch 46, Iter 24000: Training Loss:0.001032 +Epoch 46 Validation Loss:0.0002212 +Epoch 47, Iter 0: Training Loss:0.001137 +Epoch 47, Iter 24000: Training Loss:0.001174 +Epoch 47 Validation Loss:0.0002232 +Epoch 48, Iter 0: Training Loss:0.000985 +Epoch 48, Iter 24000: Training Loss:0.001066 +Epoch 48 Validation Loss:0.0002258 +Epoch 49, Iter 0: Training Loss:0.000938 +Epoch 49, Iter 24000: Training Loss:0.001042 +Epoch 49 Validation Loss:0.0002287 +Sucess on model train +Success on model test +Success on dataset synthesis +END: Testing success!!! diff --git a/halo_testing_script.py b/halo_testing_script.py new file mode 100644 index 000000000..84b29bfd6 --- /dev/null +++ b/halo_testing_script.py @@ -0,0 +1,28 @@ +print("BEGIN: Testing") +import subprocess, sys, os +subprocess.check_call([sys.executable, "-m", "pip", "install", "-e", "."]) +subprocess.check_call([sys.executable, "-m", "pip", "install", "numpy", "--force-reinstall"]) +subprocess.check_call([sys.executable, "-m", "pip", "install", "pandas", "--force-reinstall"]) +print("Success on pip install -e .") +from pyhealth.models.generators.halo import HALO +from pyhealth.models.generators.halo_resources.halo_model import HALOModel +from pyhealth.models.generators.halo_resources.halo_config import HALOConfig +from pyhealth.datasets.halo_mimic3 import HALO_MIMIC3Dataset +print("Sucess on imports") +print(f"Operating in dir: {os.getcwd()}") + +halo_config = HALOConfig() +halo_dataset = HALO_MIMIC3Dataset(mimic3_dir="../../../../scratch_old/ethanmr3/mimic3/physionet.org/files/mimiciii/1.4/", pkl_data_dir="../../halo_pkl/", gzip=True) +model = HALO(dataset=halo_dataset, config=halo_config, save_dir="../../halo_save/", train_on_init=False) +print("Success on model setup") + +model.train() +print("Sucess on model train") + +model.test(testing_results_dir = "../../halo_results/") +print("Success on model test") + +model.synthesize_dataset(pkl_save_dir = "../../halo_results/") +print("Success on dataset synthesis") + +print("END: Testing success!!!") \ No newline at end of file diff --git a/pyhealth/datasets/__init__.py b/pyhealth/datasets/__init__.py index f0e4f53e7..2d1f4f4a0 100644 --- a/pyhealth/datasets/__init__.py +++ b/pyhealth/datasets/__init__.py @@ -32,6 +32,7 @@ def __init__(self, *args, **kwargs): from .eicu import eICUDataset from .isruc import ISRUCDataset from .medical_transcriptions import MedicalTranscriptionsDataset +from .halo_mimic3 import HALO_MIMIC3Dataset from .mimic3 import MIMIC3Dataset from .mimic4 import MIMIC4CXRDataset, MIMIC4Dataset, MIMIC4EHRDataset, MIMIC4NoteDataset from .mimicextract import MIMICExtractDataset diff --git a/pyhealth/datasets/halo_mimic3.py b/pyhealth/datasets/halo_mimic3.py index b23452fed..6dc95f1c7 100644 --- a/pyhealth/datasets/halo_mimic3.py +++ b/pyhealth/datasets/halo_mimic3.py @@ -37,6 +37,7 @@ def __init__( self, mimic3_dir: str = "./", pkl_data_dir: str = "./", + gzip: bool = False ) -> None: """ Initializes the MIMIC4Dataset with the specified parameters. @@ -47,13 +48,14 @@ def __init__( dataset_name (Optional[str]): The name of the dataset. Defaults to "mimic3". config_path (Optional[str]): The path to the configuration file. If not provided, a default config is used. """ + self.gzip = gzip self.mimic3_dir = mimic3_dir self.pkl_data_dir = pkl_data_dir self.build_dataset() def build_dataset(self) -> None: - admissionFile = self.mimic3_dir + "ADMISSIONS.csv" - diagnosisFile = self.mimic3_dir + "DIAGNOSES_ICD.csv" + admissionFile = self.mimic3_dir + f"ADMISSIONS.csv{'.gz' if self.gzip else ''}" + diagnosisFile = self.mimic3_dir + f"DIAGNOSES_ICD.csv{'.gz' if self.gzip else ''}" admissionDf = pd.read_csv(admissionFile, dtype=str) admissionDf['ADMITTIME'] = pd.to_datetime(admissionDf['ADMITTIME']) @@ -77,47 +79,49 @@ def build_dataset(self) -> None: # Building the hospital admission data point if subject_id not in data: - data[subject_id] = {'visits': [diagnoses]} + data[subject_id] = {'visits': [diagnoses]} else: - data[subject_id]['visits'].append(diagnoses) + data[subject_id]['visits'].append(diagnoses) code_to_index = {} all_codes = list(set([c for p in data.values() for v in p['visits'] for c in v])) np.random.shuffle(all_codes) for c in all_codes: code_to_index[c] = len(code_to_index) - # print(f"VOCAB SIZE: {len(code_to_index)}") + print(f"VOCAB SIZE: {len(code_to_index)}") index_to_code = {v: k for k, v in code_to_index.items()} data = list(data.values()) # print("Adding Labels") - with open("./configs/hcup_ccs_2015_definitions_benchmark.yaml") as definitions_file: + with open("/u/ethanmr3/halo/PyHealth/pyhealth/datasets/configs/hcup_ccs_2015_definitions_benchmark.yaml") as definitions_file: definitions = yaml.full_load(definitions_file) code_to_group = {} for group in definitions: - if definitions[group]['use_in_benchmark'] == False: - continue - codes = definitions[group]['codes'] - for code in codes: - if code not in code_to_group: - code_to_group[code] = group - else: - assert code_to_group[code] == group + if definitions[group]['use_in_benchmark'] == False: + continue + codes = definitions[group]['codes'] + for code in codes: + if code not in code_to_group: + code_to_group[code] = group + else: + assert code_to_group[code] == group id_to_group = sorted([k for k in definitions.keys() if definitions[k]['use_in_benchmark'] == True]) group_to_id = dict((x, i) for (i, x) in enumerate(id_to_group)) # Add Labels + # print(f"DATA LEN IS: {len(data)}") for p in data: - label = np.zeros(len(group_to_id)) - for v in p['visits']: - for c in v: - if c in code_to_group: - label[group_to_id[code_to_group[c]]] = 1 - - p['labels'] = label + label = np.zeros(len(group_to_id)) + for v in p['visits']: + for c in v: + if c in code_to_group: + label[group_to_id[code_to_group[c]]] = 1 + # print(f"MIMIC LABELS = {label}") + + p['labels'] = label # print("Converting Visits") for p in data: @@ -131,12 +135,12 @@ def build_dataset(self) -> None: p['visits'] = new_visits - # print(f"MAX LEN: {max([len(p['visits']) for p in data])}") - # print(f"AVG LEN: {np.mean([len(p['visits']) for p in data])}") - # print(f"MAX VISIT LEN: {max([len(v) for p in data for v in p['visits']])}") - # print(f"AVG VISIT LEN: {np.mean([len(v) for p in data for v in p['visits']])}") - # print(f"NUM RECORDS: {len(data)}") - # print(f"NUM LONGITUDINAL RECORDS: {len([p for p in data if len(p['visits']) > 1])}") + print(f"MAX LEN: {max([len(p['visits']) for p in data])}") + print(f"AVG LEN: {np.mean([len(p['visits']) for p in data])}") + print(f"MAX VISIT LEN: {max([len(v) for p in data for v in p['visits']])}") + print(f"AVG VISIT LEN: {np.mean([len(v) for p in data for v in p['visits']])}") + print(f"NUM RECORDS: {len(data)}") + print(f"NUM LONGITUDINAL RECORDS: {len([p for p in data if len(p['visits']) > 1])}") # Train-Val-Test Split # print("Splitting Datasets") @@ -144,8 +148,8 @@ def build_dataset(self) -> None: train_dataset, val_dataset = train_test_split(train_dataset, test_size=0.1, random_state=4, shuffle=True) # Save Everything - # print("Saving Everything") - # print(len(index_to_code)) + print("Saving Everything") + print(len(index_to_code)) pickle.dump(code_to_index, open(f"{self.pkl_data_dir}codeToIndex.pkl", "wb")) pickle.dump(index_to_code, open(f"{self.pkl_data_dir}indexToCode.pkl", "wb")) pickle.dump(id_to_group, open(f"{self.pkl_data_dir}idToLabel.pkl", "wb")) diff --git a/pyhealth/models/generators/halo.py b/pyhealth/models/generators/halo.py index e5ecd4eaa..d2ec7a145 100644 --- a/pyhealth/models/generators/halo.py +++ b/pyhealth/models/generators/halo.py @@ -14,7 +14,7 @@ # Import the HALO transformer implementation from pyhealth.models.generators.halo_resources.halo_model import HALOModel -from pyhealth.models.generators.halo_resources.halo_model import HALOConfig +from pyhealth.models.generators.halo_resources.halo_config import HALOConfig class HALO: @@ -22,7 +22,8 @@ def __init__( self, dataset: HALO_MIMIC3Dataset, config: HALOConfig, - save_dir: str = "./save/" + save_dir: str = "./save/", + train_on_init: bool = True ) -> None: SEED = 4 random.seed(SEED) @@ -49,11 +50,17 @@ def __init__( self.train_ehr_dataset = pickle.load(open(f'{self.dataset.pkl_data_dir}trainDataset.pkl', 'rb')) self.val_ehr_dataset = pickle.load(open(f'{self.dataset.pkl_data_dir}valDataset.pkl', 'rb')) self.index_to_code = pickle.load(open(f"{self.dataset.pkl_data_dir}indexToCode.pkl", "rb")) + self.id_to_label = pickle.load(open(f"{self.dataset.pkl_data_dir}idToLabel.pkl", "rb")) test_ehr_dataset = pickle.load(open(f'{self.dataset.pkl_data_dir}testDataset.pkl', 'rb')) - train_c = set([c for p in train_ehr_dataset for v in p['visits'] for c in v]) + try: + print(f"test data head 1: \n{test_ehr_dataset.head()}") + except: + print(f"test data head 2: \n{test_ehr_dataset[:5]}") + train_c = set([c for p in self.train_ehr_dataset for v in p['visits'] for c in v]) self.test_ehr_dataset = [{'labels': p['labels'], 'visits': [[c for c in v if c in train_c] for v in p['visits']]} for p in test_ehr_dataset] - self.train() + if train_on_init: + self.train() def train(self) -> None: @@ -76,8 +83,8 @@ def get_batch(loc, batch_size, mode): for i, p in enumerate(ehr): visits = p['visits'] for j, v in enumerate(visits): - batch_ehr[i,j+2][v] = 1 - batch_mask[i,j+2] = 1 + batch_ehr[i,j+2][v] = 1 + batch_mask[i,j+2] = 1 batch_ehr[i,1,self.config.code_vocab_size:self.config.code_vocab_size+self.config.label_vocab_size] = np.array(p['labels']) # Set the patient labels batch_ehr[i,len(visits)+1,self.config.code_vocab_size+self.config.label_vocab_size+1] = 1 # Set the final visit to have the end token batch_ehr[i,len(visits)+2:,self.config.code_vocab_size+self.config.label_vocab_size+2] = 1 # Set the rest to the padded visit token @@ -118,34 +125,34 @@ def shuffle_training_data(train_ehr_dataset): loss.backward() self.optimizer.step() - # if i % (500*self.config.batch_size) == 0: - # print("Epoch %d, Iter %d: Training Loss:%.6f"%(e, i, loss * 8)) + if i % (500*self.config.batch_size) == 0: + print("Epoch %d, Iter %d: Training Loss:%.6f"%(e, i, loss * 8)) if i % (500*self.config.batch_size) == 0: if i == 0: continue - self.model.eval() - with torch.no_grad(): - val_l = [] - for v_i in range(0, len(self.val_ehr_dataset), self.config.batch_size): - batch_ehr, batch_mask = get_batch(v_i, self.config.batch_size, 'valid') - batch_ehr = torch.tensor(batch_ehr, dtype=torch.float32).to(self.device) - batch_mask = torch.tensor(batch_mask, dtype=torch.float32).to(self.device) - - val_loss, _, _ = self.model(batch_ehr, position_ids=None, ehr_labels=batch_ehr, ehr_masks=batch_mask, pos_loss_weight=self.config.pos_loss_weight) - val_l.append((val_loss).cpu().detach().numpy()) + self.model.eval() + with torch.no_grad(): + val_l = [] + for v_i in range(0, len(self.val_ehr_dataset), self.config.batch_size): + batch_ehr, batch_mask = get_batch(v_i, self.config.batch_size, 'valid') + batch_ehr = torch.tensor(batch_ehr, dtype=torch.float32).to(self.device) + batch_mask = torch.tensor(batch_mask, dtype=torch.float32).to(self.device) + + val_loss, _, _ = self.model(batch_ehr, position_ids=None, ehr_labels=batch_ehr, ehr_masks=batch_mask, pos_loss_weight=self.config.pos_loss_weight) + val_l.append((val_loss).cpu().detach().numpy()) - cur_val_loss = np.mean(val_l) - # print("Epoch %d Validation Loss:%.7f"%(e, cur_val_loss)) - if cur_val_loss < global_loss: - global_loss = cur_val_loss - state = { - 'model': self.model.state_dict(), - 'optimizer': self.optimizer.state_dict(), - 'iteration': i - } - torch.save(state, f'{self.save_dir}halo_model') - # print('\n------------ Save best model ------------\n') + cur_val_loss = np.mean(val_l) + print("Epoch %d Validation Loss:%.7f"%(e, cur_val_loss)) + if cur_val_loss < global_loss: + global_loss = cur_val_loss + state = { + 'model': self.model.state_dict(), + 'optimizer': self.optimizer.state_dict(), + 'iteration': i + } + torch.save(state, f'{self.save_dir}halo_model') + print('\n------------ Save best model ------------\n') def test(self, testing_results_dir: str = "./results/testing_stats/") -> None: @@ -167,8 +174,8 @@ def get_batch(loc, batch_size, mode): for i, p in enumerate(ehr): visits = p['visits'] for j, v in enumerate(visits): - batch_ehr[i,j+2][v] = 1 - batch_mask[i,j+2] = 1 + batch_ehr[i,j+2][v] = 1 + batch_mask[i,j+2] = 1 batch_ehr[i,1,self.config.code_vocab_size:self.config.code_vocab_size+self.config.label_vocab_size] = np.array(p['labels']) # Set the patient labels batch_ehr[i,len(visits)+1,self.config.code_vocab_size+self.config.label_vocab_size+1] = 1 # Set the final visit to have the end token batch_ehr[i,len(visits)+2:,self.config.code_vocab_size+self.config.label_vocab_size+2] = 1 # Set the rest to the padded visit token @@ -303,7 +310,7 @@ def convert_ehr(self, ehrs, index_to_code=None): return ehr_outputs - def synthesize_dataset(self, pkl_save_dir: str = "./results/datasets/haloDataset.pkl") -> None: + def synthesize_dataset(self, pkl_save_dir: str = "./results/datasets/") -> None: ## HELPER: def sample_sequence(model, length, context, batch_size, device='cuda', sample=True): @@ -331,7 +338,7 @@ def sample_sequence(model, length, context, batch_size, device='cuda', sample=Tr batch_synthetic_ehrs = self.convert_ehr(batch_synthetic_ehrs) synthetic_ehr_dataset += batch_synthetic_ehrs - pickle.dump(synthetic_ehr_dataset, open(pkl_save_dir, 'wb')) + pickle.dump(synthetic_ehr_dataset, open(f"{pkl_save_dir}haloDataset.pkl", 'wb')) diff --git a/test_halo_model.slurm b/test_halo_model.slurm new file mode 100644 index 000000000..82b921e70 --- /dev/null +++ b/test_halo_model.slurm @@ -0,0 +1,34 @@ +#!/bin/bash +#SBATCH --account=ethanmr3-ic +#SBATCH --job-name=pyhealth-halo-testing +#SBATCH --output=halo-testing-logs/halo_test_%j.out +#SBATCH --error=halo-testing-logs/halo_test_%j.err +#SBATCH --partition=IllinoisComputes-GPU # Change to appropriate partition +#SBATCH --gres=gpu:1 # Request 1 GPU +#SBATCH --cpus-per-task=4 +#SBATCH --mem=64G +#SBATCH --time=48:00:00 + +# Change to the directory where you submitted the job +cd "$SLURM_SUBMIT_DIR" + +# Print useful Slurm environment variables for debugging +echo "SLURM_JOB_ID: $SLURM_JOB_ID" +echo "SLURM_JOB_NODELIST: $SLURM_JOB_NODELIST" +echo "SLURM_NTASKS: $SLURM_NTASKS" +echo "SLURM_CPUS_ON_NODE: $SLURM_CPUS_ON_NODE" +echo "SLURM_GPUS_ON_NODE: $SLURM_GPUS_ON_NODE" +echo "SLURM_GPUS: $SLURM_GPUS" +echo "CUDA_VISIBLE_DEVICES: $CUDA_VISIBLE_DEVICES" + +# Optional: check what GPU(s) is/are actually visible +echo "Running nvidia-smi to confirm GPU availability:" +nvidia-smi + +# Load modules or activate environment +# module load python/3.10 +# module load cuda/11.7 +# conda activate your-env + +# Run your Python training script +python /u/ethanmr3/halo/PyHealth/halo_testing_script.py \ No newline at end of file From ec4f23dd6c6c07a3efc5f6cca4d75fabbfd6a9b7 Mon Sep 17 00:00:00 2001 From: Rasmussen Date: Sun, 27 Jul 2025 10:22:36 -0500 Subject: [PATCH 05/10] Remove testing logs --- halo-testing-logs/halo_test_3706609.err | 4 - halo-testing-logs/halo_test_3706609.out | 29 -- halo-testing-logs/halo_test_3706620.err | 33 -- halo-testing-logs/halo_test_3706620.out | 161 ---------- halo-testing-logs/halo_test_3706738.err | 35 -- halo-testing-logs/halo_test_3706738.out | 161 ---------- halo-testing-logs/halo_test_3707040.err | 35 -- halo-testing-logs/halo_test_3707040.out | 161 ---------- halo-testing-logs/halo_test_3707410.err | 35 -- halo-testing-logs/halo_test_3707410.out | 172 ---------- halo-testing-logs/halo_test_3708721.err | 35 -- halo-testing-logs/halo_test_3708721.out | 175 ---------- halo-testing-logs/halo_test_3708911.err | 28 -- halo-testing-logs/halo_test_3708911.out | 177 ----------- halo-testing-logs/halo_test_3709442.err | 37 --- halo-testing-logs/halo_test_3709442.out | 178 ----------- halo-testing-logs/halo_test_3709762.err | 28 -- halo-testing-logs/halo_test_3709762.out | 178 ----------- halo-testing-logs/halo_test_3713150.err | 27 -- halo-testing-logs/halo_test_3713150.out | 161 ---------- halo-testing-logs/halo_test_3713164.err | 30 -- halo-testing-logs/halo_test_3713164.out | 404 ------------------------ 22 files changed, 2284 deletions(-) delete mode 100644 halo-testing-logs/halo_test_3706609.err delete mode 100644 halo-testing-logs/halo_test_3706609.out delete mode 100644 halo-testing-logs/halo_test_3706620.err delete mode 100644 halo-testing-logs/halo_test_3706620.out delete mode 100644 halo-testing-logs/halo_test_3706738.err delete mode 100644 halo-testing-logs/halo_test_3706738.out delete mode 100644 halo-testing-logs/halo_test_3707040.err delete mode 100644 halo-testing-logs/halo_test_3707040.out delete mode 100644 halo-testing-logs/halo_test_3707410.err delete mode 100644 halo-testing-logs/halo_test_3707410.out delete mode 100644 halo-testing-logs/halo_test_3708721.err delete mode 100644 halo-testing-logs/halo_test_3708721.out delete mode 100644 halo-testing-logs/halo_test_3708911.err delete mode 100644 halo-testing-logs/halo_test_3708911.out delete mode 100644 halo-testing-logs/halo_test_3709442.err delete mode 100644 halo-testing-logs/halo_test_3709442.out delete mode 100644 halo-testing-logs/halo_test_3709762.err delete mode 100644 halo-testing-logs/halo_test_3709762.out delete mode 100644 halo-testing-logs/halo_test_3713150.err delete mode 100644 halo-testing-logs/halo_test_3713150.out delete mode 100644 halo-testing-logs/halo_test_3713164.err delete mode 100644 halo-testing-logs/halo_test_3713164.out diff --git a/halo-testing-logs/halo_test_3706609.err b/halo-testing-logs/halo_test_3706609.err deleted file mode 100644 index ed385264a..000000000 --- a/halo-testing-logs/halo_test_3706609.err +++ /dev/null @@ -1,4 +0,0 @@ -Traceback (most recent call last): - File "/u/ethanmr3/halo/PyHealth/halo_testing_script.py", line 3, in - definitions = yaml.full_load(definitions_file) -NameError: name 'yaml' is not defined diff --git a/halo-testing-logs/halo_test_3706609.out b/halo-testing-logs/halo_test_3706609.out deleted file mode 100644 index 9f0fa8354..000000000 --- a/halo-testing-logs/halo_test_3706609.out +++ /dev/null @@ -1,29 +0,0 @@ -SLURM_JOB_ID: 3706609 -SLURM_JOB_NODELIST: ccc0388 -SLURM_NTASKS: -SLURM_CPUS_ON_NODE: 4 -SLURM_GPUS_ON_NODE: 1 -SLURM_GPUS: -CUDA_VISIBLE_DEVICES: 0 -Running nvidia-smi to confirm GPU availability: -Sat Jul 19 19:12:28 2025 -+-----------------------------------------------------------------------------------------+ -| NVIDIA-SMI 575.57.08 Driver Version: 575.57.08 CUDA Version: 12.9 | -|-----------------------------------------+------------------------+----------------------+ -| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC | -| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. | -| | | MIG M. | -|=========================================+========================+======================| -| 0 NVIDIA A100-SXM4-80GB Off | 00000000:81:00.0 Off | 0 | -| N/A 28C P0 61W / 500W | 0MiB / 81920MiB | 0% Default | -| | | Disabled | -+-----------------------------------------+------------------------+----------------------+ - -+-----------------------------------------------------------------------------------------+ -| Processes: | -| GPU GI CI PID Type Process name GPU Memory | -| ID ID Usage | -|=========================================================================================| -| No running processes found | -+-----------------------------------------------------------------------------------------+ -BEGIN: Testing diff --git a/halo-testing-logs/halo_test_3706620.err b/halo-testing-logs/halo_test_3706620.err deleted file mode 100644 index 1daf24cc4..000000000 --- a/halo-testing-logs/halo_test_3706620.err +++ /dev/null @@ -1,33 +0,0 @@ - WARNING: Value for scheme.platlib does not match. Please report this to - distutils: /u/ethanmr3/.local/lib/python3.9/site-packages - sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages - WARNING: Additional context: - user = True - home = None - root = None - prefix = None - WARNING: Value for scheme.platlib does not match. Please report this to - distutils: /u/ethanmr3/.local/lib/python3.9/site-packages - sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages - WARNING: Additional context: - user = True - home = None - root = None - prefix = None - WARNING: Value for scheme.platlib does not match. Please report this to - distutils: /u/ethanmr3/.local/lib/python3.9/site-packages - sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages - WARNING: Additional context: - user = True - home = None - root = None - prefix = None -ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. -pyhealth 1.1.4 requires pandas<2,>=1.3.2, but you have pandas 2.3.1 which is incompatible. - 0%| | 0/58976 [00:00 - model = HALO(dataset=halo_dataset, config=halo_config, save_dir="../../halo_save/") - File "/u/ethanmr3/halo/PyHealth/pyhealth/models/generators/halo.py", line 53, in __init__ - train_c = set([c for p in train_ehr_dataset for v in p['visits'] for c in v]) -NameError: name 'train_ehr_dataset' is not defined diff --git a/halo-testing-logs/halo_test_3706620.out b/halo-testing-logs/halo_test_3706620.out deleted file mode 100644 index 9405dc56f..000000000 --- a/halo-testing-logs/halo_test_3706620.out +++ /dev/null @@ -1,161 +0,0 @@ -SLURM_JOB_ID: 3706620 -SLURM_JOB_NODELIST: ccc0388 -SLURM_NTASKS: -SLURM_CPUS_ON_NODE: 4 -SLURM_GPUS_ON_NODE: 1 -SLURM_GPUS: -CUDA_VISIBLE_DEVICES: 0 -Running nvidia-smi to confirm GPU availability: -Sat Jul 19 19:13:44 2025 -+-----------------------------------------------------------------------------------------+ -| NVIDIA-SMI 575.57.08 Driver Version: 575.57.08 CUDA Version: 12.9 | -|-----------------------------------------+------------------------+----------------------+ -| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC | -| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. | -| | | MIG M. | -|=========================================+========================+======================| -| 0 NVIDIA A100-SXM4-80GB Off | 00000000:81:00.0 Off | 0 | -| N/A 28C P0 61W / 500W | 0MiB / 81920MiB | 0% Default | -| | | Disabled | -+-----------------------------------------+------------------------+----------------------+ - -+-----------------------------------------------------------------------------------------+ -| Processes: | -| GPU GI CI PID Type Process name GPU Memory | -| ID ID Usage | -|=========================================================================================| -| No running processes found | -+-----------------------------------------------------------------------------------------+ -Defaulting to user installation because normal site-packages is not writeable -Obtaining file:///u/ethanmr3/halo/PyHealth -Requirement already satisfied: torch>=1.8.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2.7.1) -Requirement already satisfied: torchvision>=0.9.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (0.22.1) -Requirement already satisfied: rdkit>=2022.03.4 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2025.3.3) -Requirement already satisfied: scikit-learn>=0.24.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.6.1) -Requirement already satisfied: networkx>=2.6.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (3.2.1) -Collecting pandas<2,>=1.3.2 - Using cached pandas-1.5.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.2 MB) -Requirement already satisfied: pandarallel>=1.5.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.6.5) -Requirement already satisfied: mne>=1.0.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.8.0) -Requirement already satisfied: urllib3<=1.26.15 in /usr/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.26.5) -Requirement already satisfied: numpy in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2.0.2) -Requirement already satisfied: tqdm in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (4.67.1) -Requirement already satisfied: polars in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.31.0) -Requirement already satisfied: transformers in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (4.52.4) -Requirement already satisfied: pooch>=1.5 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (1.8.2) -Requirement already satisfied: packaging in /usr/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (20.9) -Requirement already satisfied: jinja2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (3.1.6) -Requirement already satisfied: lazy-loader>=0.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (0.4) -Requirement already satisfied: scipy>=1.9 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (1.13.1) -Requirement already satisfied: matplotlib>=3.6 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (3.9.4) -Requirement already satisfied: decorator in /usr/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (4.4.2) -Requirement already satisfied: contourpy>=1.0.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.3.0) -Requirement already satisfied: pyparsing>=2.3.1 in /usr/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (2.4.7) -Requirement already satisfied: importlib-resources>=3.2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (6.5.2) -Requirement already satisfied: pillow>=8 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (11.2.1) -Requirement already satisfied: fonttools>=4.22.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (4.58.4) -Requirement already satisfied: python-dateutil>=2.7 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (2.9.0.post0) -Requirement already satisfied: kiwisolver>=1.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.4.7) -Requirement already satisfied: cycler>=0.10 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (0.12.1) -Requirement already satisfied: zipp>=3.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from importlib-resources>=3.2.0->matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (3.23.0) -Requirement already satisfied: dill>=0.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandarallel>=1.5.3->pyhealth==1.1.4) (0.4.0) -Requirement already satisfied: psutil in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandarallel>=1.5.3->pyhealth==1.1.4) (7.0.0) -Requirement already satisfied: pytz>=2020.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandas<2,>=1.3.2->pyhealth==1.1.4) (2025.2) -Requirement already satisfied: platformdirs>=2.5.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (4.3.8) -Requirement already satisfied: requests>=2.19.0 in /usr/lib/python3.9/site-packages (from pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (2.25.1) -Requirement already satisfied: six>=1.5 in /u/ethanmr3/.local/lib/python3.9/site-packages (from python-dateutil>=2.7->matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.17.0) -Requirement already satisfied: idna<3,>=2.5 in /usr/lib/python3.9/site-packages (from requests>=2.19.0->pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (2.10) -Requirement already satisfied: chardet<5,>=3.0.2 in /usr/lib/python3.9/site-packages (from requests>=2.19.0->pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (4.0.0) -Requirement already satisfied: joblib>=1.2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from scikit-learn>=0.24.2->pyhealth==1.1.4) (1.5.1) -Requirement already satisfied: threadpoolctl>=3.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from scikit-learn>=0.24.2->pyhealth==1.1.4) (3.6.0) -Requirement already satisfied: nvidia-nvtx-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) -Requirement already satisfied: nvidia-cuda-cupti-cu12==12.6.80 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.80) -Requirement already satisfied: nvidia-nvjitlink-cu12==12.6.85 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.85) -Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) -Requirement already satisfied: nvidia-curand-cu12==10.3.7.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (10.3.7.77) -Requirement already satisfied: nvidia-cufft-cu12==11.3.0.4 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (11.3.0.4) -Requirement already satisfied: nvidia-cusparse-cu12==12.5.4.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.5.4.2) -Requirement already satisfied: nvidia-cuda-runtime-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) -Requirement already satisfied: nvidia-cusolver-cu12==11.7.1.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (11.7.1.2) -Requirement already satisfied: nvidia-cusparselt-cu12==0.6.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (0.6.3) -Requirement already satisfied: triton==3.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (3.3.1) -Requirement already satisfied: nvidia-nccl-cu12==2.26.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (2.26.2) -Requirement already satisfied: nvidia-cufile-cu12==1.11.1.6 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (1.11.1.6) -Requirement already satisfied: typing-extensions>=4.10.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (4.14.0) -Requirement already satisfied: filelock in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (3.18.0) -Requirement already satisfied: fsspec in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (2025.5.1) -Requirement already satisfied: sympy>=1.13.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (1.14.0) -Requirement already satisfied: nvidia-cudnn-cu12==9.5.1.17 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (9.5.1.17) -Requirement already satisfied: nvidia-cublas-cu12==12.6.4.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.4.1) -Requirement already satisfied: setuptools>=40.8.0 in /usr/lib/python3.9/site-packages (from triton==3.3.1->torch>=1.8.0->pyhealth==1.1.4) (53.0.0) -Requirement already satisfied: mpmath<1.4,>=1.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from sympy>=1.13.3->torch>=1.8.0->pyhealth==1.1.4) (1.3.0) -Requirement already satisfied: MarkupSafe>=2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from jinja2->mne>=1.0.3->pyhealth==1.1.4) (3.0.2) -Requirement already satisfied: pyyaml>=5.1 in /usr/lib64/python3.9/site-packages (from transformers->pyhealth==1.1.4) (5.4.1) -Requirement already satisfied: tokenizers<0.22,>=0.21 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.21.1) -Requirement already satisfied: regex!=2019.12.17 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (2024.11.6) -Requirement already satisfied: safetensors>=0.4.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.5.3) -Requirement already satisfied: huggingface-hub<1.0,>=0.30.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.33.0) -Requirement already satisfied: hf-xet<2.0.0,>=1.1.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from huggingface-hub<1.0,>=0.30.0->transformers->pyhealth==1.1.4) (1.1.5) -Installing collected packages: pandas, pyhealth - Attempting uninstall: pandas - Found existing installation: pandas 2.3.1 - Uninstalling pandas-2.3.1: - Successfully uninstalled pandas-2.3.1 - Attempting uninstall: pyhealth - Found existing installation: pyhealth 1.1.4 - Uninstalling pyhealth-1.1.4: - Successfully uninstalled pyhealth-1.1.4 - Running setup.py develop for pyhealth -Successfully installed pandas-1.5.3 pyhealth -Defaulting to user installation because normal site-packages is not writeable -Collecting numpy - Using cached numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (19.5 MB) -Installing collected packages: numpy - Attempting uninstall: numpy - Found existing installation: numpy 2.0.2 - Uninstalling numpy-2.0.2: - Successfully uninstalled numpy-2.0.2 -Successfully installed numpy-2.0.2 -Defaulting to user installation because normal site-packages is not writeable -Collecting pandas - Using cached pandas-2.3.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.4 MB) -Collecting numpy>=1.22.4 - Using cached numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (19.5 MB) -Collecting python-dateutil>=2.8.2 - Using cached python_dateutil-2.9.0.post0-py2.py3-none-any.whl (229 kB) -Collecting pytz>=2020.1 - Using cached pytz-2025.2-py2.py3-none-any.whl (509 kB) -Collecting tzdata>=2022.7 - Using cached tzdata-2025.2-py2.py3-none-any.whl (347 kB) -Collecting six>=1.5 - Using cached six-1.17.0-py2.py3-none-any.whl (11 kB) -Installing collected packages: six, tzdata, pytz, python-dateutil, numpy, pandas - Attempting uninstall: six - Found existing installation: six 1.17.0 - Uninstalling six-1.17.0: - Successfully uninstalled six-1.17.0 - Attempting uninstall: tzdata - Found existing installation: tzdata 2025.2 - Uninstalling tzdata-2025.2: - Successfully uninstalled tzdata-2025.2 - Attempting uninstall: pytz - Found existing installation: pytz 2025.2 - Uninstalling pytz-2025.2: - Successfully uninstalled pytz-2025.2 - Attempting uninstall: python-dateutil - Found existing installation: python-dateutil 2.9.0.post0 - Uninstalling python-dateutil-2.9.0.post0: - Successfully uninstalled python-dateutil-2.9.0.post0 - Attempting uninstall: numpy - Found existing installation: numpy 2.0.2 - Uninstalling numpy-2.0.2: - Successfully uninstalled numpy-2.0.2 - Attempting uninstall: pandas - Found existing installation: pandas 1.5.3 - Uninstalling pandas-1.5.3: - Successfully uninstalled pandas-1.5.3 -Successfully installed numpy-2.0.2 pandas-2.3.1 python-dateutil-2.9.0.post0 pytz-2025.2 six-1.17.0 tzdata-2025.2 -BEGIN: Testing -Success on pip install -e . -Sucess on imports -Operating in dir: /u/ethanmr3/halo/PyHealth diff --git a/halo-testing-logs/halo_test_3706738.err b/halo-testing-logs/halo_test_3706738.err deleted file mode 100644 index ace8ddcb5..000000000 --- a/halo-testing-logs/halo_test_3706738.err +++ /dev/null @@ -1,35 +0,0 @@ - WARNING: Value for scheme.platlib does not match. Please report this to - distutils: /u/ethanmr3/.local/lib/python3.9/site-packages - sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages - WARNING: Additional context: - user = True - home = None - root = None - prefix = None - WARNING: Value for scheme.platlib does not match. Please report this to - distutils: /u/ethanmr3/.local/lib/python3.9/site-packages - sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages - WARNING: Additional context: - user = True - home = None - root = None - prefix = None - WARNING: Value for scheme.platlib does not match. Please report this to - distutils: /u/ethanmr3/.local/lib/python3.9/site-packages - sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages - WARNING: Additional context: - user = True - home = None - root = None - prefix = None -ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. -pyhealth 1.1.4 requires pandas<2,>=1.3.2, but you have pandas 2.3.1 which is incompatible. - 0%| | 0/58976 [00:00 - model = HALO(dataset=halo_dataset, config=halo_config, save_dir="../../halo_save/") - File "/u/ethanmr3/halo/PyHealth/pyhealth/models/generators/halo.py", line 54, in __init__ - self.test_ehr_dataset = [{'labels': p['labels'], 'visits': [[c for c in v if c in train_c] for v in p['visits']]} for p in test_ehr_dataset] - File "/u/ethanmr3/halo/PyHealth/pyhealth/models/generators/halo.py", line 54, in - self.test_ehr_dataset = [{'labels': p['labels'], 'visits': [[c for c in v if c in train_c] for v in p['visits']]} for p in test_ehr_dataset] -KeyError: 'labels' diff --git a/halo-testing-logs/halo_test_3706738.out b/halo-testing-logs/halo_test_3706738.out deleted file mode 100644 index b9921cdbd..000000000 --- a/halo-testing-logs/halo_test_3706738.out +++ /dev/null @@ -1,161 +0,0 @@ -SLURM_JOB_ID: 3706738 -SLURM_JOB_NODELIST: ccc0388 -SLURM_NTASKS: -SLURM_CPUS_ON_NODE: 4 -SLURM_GPUS_ON_NODE: 1 -SLURM_GPUS: -CUDA_VISIBLE_DEVICES: 0 -Running nvidia-smi to confirm GPU availability: -Sat Jul 19 19:33:53 2025 -+-----------------------------------------------------------------------------------------+ -| NVIDIA-SMI 575.57.08 Driver Version: 575.57.08 CUDA Version: 12.9 | -|-----------------------------------------+------------------------+----------------------+ -| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC | -| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. | -| | | MIG M. | -|=========================================+========================+======================| -| 0 NVIDIA A100-SXM4-80GB Off | 00000000:41:00.0 Off | 0 | -| N/A 26C P0 61W / 500W | 0MiB / 81920MiB | 0% Default | -| | | Disabled | -+-----------------------------------------+------------------------+----------------------+ - -+-----------------------------------------------------------------------------------------+ -| Processes: | -| GPU GI CI PID Type Process name GPU Memory | -| ID ID Usage | -|=========================================================================================| -| No running processes found | -+-----------------------------------------------------------------------------------------+ -Defaulting to user installation because normal site-packages is not writeable -Obtaining file:///u/ethanmr3/halo/PyHealth -Requirement already satisfied: torch>=1.8.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2.7.1) -Requirement already satisfied: torchvision>=0.9.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (0.22.1) -Requirement already satisfied: rdkit>=2022.03.4 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2025.3.3) -Requirement already satisfied: scikit-learn>=0.24.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.6.1) -Requirement already satisfied: networkx>=2.6.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (3.2.1) -Collecting pandas<2,>=1.3.2 - Using cached pandas-1.5.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.2 MB) -Requirement already satisfied: pandarallel>=1.5.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.6.5) -Requirement already satisfied: mne>=1.0.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.8.0) -Requirement already satisfied: urllib3<=1.26.15 in /usr/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.26.5) -Requirement already satisfied: numpy in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2.0.2) -Requirement already satisfied: tqdm in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (4.67.1) -Requirement already satisfied: polars in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.31.0) -Requirement already satisfied: transformers in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (4.52.4) -Requirement already satisfied: decorator in /usr/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (4.4.2) -Requirement already satisfied: scipy>=1.9 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (1.13.1) -Requirement already satisfied: packaging in /usr/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (20.9) -Requirement already satisfied: jinja2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (3.1.6) -Requirement already satisfied: pooch>=1.5 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (1.8.2) -Requirement already satisfied: lazy-loader>=0.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (0.4) -Requirement already satisfied: matplotlib>=3.6 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (3.9.4) -Requirement already satisfied: cycler>=0.10 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (0.12.1) -Requirement already satisfied: python-dateutil>=2.7 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (2.9.0.post0) -Requirement already satisfied: fonttools>=4.22.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (4.58.4) -Requirement already satisfied: kiwisolver>=1.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.4.7) -Requirement already satisfied: pillow>=8 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (11.2.1) -Requirement already satisfied: contourpy>=1.0.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.3.0) -Requirement already satisfied: pyparsing>=2.3.1 in /usr/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (2.4.7) -Requirement already satisfied: importlib-resources>=3.2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (6.5.2) -Requirement already satisfied: zipp>=3.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from importlib-resources>=3.2.0->matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (3.23.0) -Requirement already satisfied: dill>=0.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandarallel>=1.5.3->pyhealth==1.1.4) (0.4.0) -Requirement already satisfied: psutil in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandarallel>=1.5.3->pyhealth==1.1.4) (7.0.0) -Requirement already satisfied: pytz>=2020.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandas<2,>=1.3.2->pyhealth==1.1.4) (2025.2) -Requirement already satisfied: requests>=2.19.0 in /usr/lib/python3.9/site-packages (from pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (2.25.1) -Requirement already satisfied: platformdirs>=2.5.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (4.3.8) -Requirement already satisfied: six>=1.5 in /u/ethanmr3/.local/lib/python3.9/site-packages (from python-dateutil>=2.7->matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.17.0) -Requirement already satisfied: chardet<5,>=3.0.2 in /usr/lib/python3.9/site-packages (from requests>=2.19.0->pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (4.0.0) -Requirement already satisfied: idna<3,>=2.5 in /usr/lib/python3.9/site-packages (from requests>=2.19.0->pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (2.10) -Requirement already satisfied: joblib>=1.2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from scikit-learn>=0.24.2->pyhealth==1.1.4) (1.5.1) -Requirement already satisfied: threadpoolctl>=3.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from scikit-learn>=0.24.2->pyhealth==1.1.4) (3.6.0) -Requirement already satisfied: triton==3.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (3.3.1) -Requirement already satisfied: nvidia-nccl-cu12==2.26.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (2.26.2) -Requirement already satisfied: nvidia-cufft-cu12==11.3.0.4 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (11.3.0.4) -Requirement already satisfied: nvidia-cusparse-cu12==12.5.4.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.5.4.2) -Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) -Requirement already satisfied: nvidia-nvtx-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) -Requirement already satisfied: nvidia-nvjitlink-cu12==12.6.85 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.85) -Requirement already satisfied: filelock in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (3.18.0) -Requirement already satisfied: sympy>=1.13.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (1.14.0) -Requirement already satisfied: nvidia-cufile-cu12==1.11.1.6 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (1.11.1.6) -Requirement already satisfied: nvidia-curand-cu12==10.3.7.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (10.3.7.77) -Requirement already satisfied: nvidia-cusolver-cu12==11.7.1.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (11.7.1.2) -Requirement already satisfied: fsspec in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (2025.5.1) -Requirement already satisfied: nvidia-cuda-runtime-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) -Requirement already satisfied: nvidia-cuda-cupti-cu12==12.6.80 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.80) -Requirement already satisfied: nvidia-cudnn-cu12==9.5.1.17 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (9.5.1.17) -Requirement already satisfied: typing-extensions>=4.10.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (4.14.0) -Requirement already satisfied: nvidia-cublas-cu12==12.6.4.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.4.1) -Requirement already satisfied: nvidia-cusparselt-cu12==0.6.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (0.6.3) -Requirement already satisfied: setuptools>=40.8.0 in /usr/lib/python3.9/site-packages (from triton==3.3.1->torch>=1.8.0->pyhealth==1.1.4) (53.0.0) -Requirement already satisfied: mpmath<1.4,>=1.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from sympy>=1.13.3->torch>=1.8.0->pyhealth==1.1.4) (1.3.0) -Requirement already satisfied: MarkupSafe>=2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from jinja2->mne>=1.0.3->pyhealth==1.1.4) (3.0.2) -Requirement already satisfied: pyyaml>=5.1 in /usr/lib64/python3.9/site-packages (from transformers->pyhealth==1.1.4) (5.4.1) -Requirement already satisfied: huggingface-hub<1.0,>=0.30.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.33.0) -Requirement already satisfied: regex!=2019.12.17 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (2024.11.6) -Requirement already satisfied: safetensors>=0.4.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.5.3) -Requirement already satisfied: tokenizers<0.22,>=0.21 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.21.1) -Requirement already satisfied: hf-xet<2.0.0,>=1.1.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from huggingface-hub<1.0,>=0.30.0->transformers->pyhealth==1.1.4) (1.1.5) -Installing collected packages: pandas, pyhealth - Attempting uninstall: pandas - Found existing installation: pandas 2.3.1 - Uninstalling pandas-2.3.1: - Successfully uninstalled pandas-2.3.1 - Attempting uninstall: pyhealth - Found existing installation: pyhealth 1.1.4 - Uninstalling pyhealth-1.1.4: - Successfully uninstalled pyhealth-1.1.4 - Running setup.py develop for pyhealth -Successfully installed pandas-1.5.3 pyhealth -Defaulting to user installation because normal site-packages is not writeable -Collecting numpy - Using cached numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (19.5 MB) -Installing collected packages: numpy - Attempting uninstall: numpy - Found existing installation: numpy 2.0.2 - Uninstalling numpy-2.0.2: - Successfully uninstalled numpy-2.0.2 -Successfully installed numpy-2.0.2 -Defaulting to user installation because normal site-packages is not writeable -Collecting pandas - Using cached pandas-2.3.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.4 MB) -Collecting pytz>=2020.1 - Using cached pytz-2025.2-py2.py3-none-any.whl (509 kB) -Collecting tzdata>=2022.7 - Using cached tzdata-2025.2-py2.py3-none-any.whl (347 kB) -Collecting python-dateutil>=2.8.2 - Using cached python_dateutil-2.9.0.post0-py2.py3-none-any.whl (229 kB) -Collecting numpy>=1.22.4 - Using cached numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (19.5 MB) -Collecting six>=1.5 - Using cached six-1.17.0-py2.py3-none-any.whl (11 kB) -Installing collected packages: six, tzdata, pytz, python-dateutil, numpy, pandas - Attempting uninstall: six - Found existing installation: six 1.17.0 - Uninstalling six-1.17.0: - Successfully uninstalled six-1.17.0 - Attempting uninstall: tzdata - Found existing installation: tzdata 2025.2 - Uninstalling tzdata-2025.2: - Successfully uninstalled tzdata-2025.2 - Attempting uninstall: pytz - Found existing installation: pytz 2025.2 - Uninstalling pytz-2025.2: - Successfully uninstalled pytz-2025.2 - Attempting uninstall: python-dateutil - Found existing installation: python-dateutil 2.9.0.post0 - Uninstalling python-dateutil-2.9.0.post0: - Successfully uninstalled python-dateutil-2.9.0.post0 - Attempting uninstall: numpy - Found existing installation: numpy 2.0.2 - Uninstalling numpy-2.0.2: - Successfully uninstalled numpy-2.0.2 - Attempting uninstall: pandas - Found existing installation: pandas 1.5.3 - Uninstalling pandas-1.5.3: - Successfully uninstalled pandas-1.5.3 -Successfully installed numpy-2.0.2 pandas-2.3.1 python-dateutil-2.9.0.post0 pytz-2025.2 six-1.17.0 tzdata-2025.2 -BEGIN: Testing -Success on pip install -e . -Sucess on imports -Operating in dir: /u/ethanmr3/halo/PyHealth diff --git a/halo-testing-logs/halo_test_3707040.err b/halo-testing-logs/halo_test_3707040.err deleted file mode 100644 index 191aa9575..000000000 --- a/halo-testing-logs/halo_test_3707040.err +++ /dev/null @@ -1,35 +0,0 @@ - WARNING: Value for scheme.platlib does not match. Please report this to - distutils: /u/ethanmr3/.local/lib/python3.9/site-packages - sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages - WARNING: Additional context: - user = True - home = None - root = None - prefix = None - WARNING: Value for scheme.platlib does not match. Please report this to - distutils: /u/ethanmr3/.local/lib/python3.9/site-packages - sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages - WARNING: Additional context: - user = True - home = None - root = None - prefix = None - WARNING: Value for scheme.platlib does not match. Please report this to - distutils: /u/ethanmr3/.local/lib/python3.9/site-packages - sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages - WARNING: Additional context: - user = True - home = None - root = None - prefix = None -ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. -pyhealth 1.1.4 requires pandas<2,>=1.3.2, but you have pandas 2.3.1 which is incompatible. - 0%| | 0/58976 [00:00 - model = HALO(dataset=halo_dataset, config=halo_config, save_dir="../../halo_save/") - File "/u/ethanmr3/halo/PyHealth/pyhealth/models/generators/halo.py", line 55, in __init__ - self.test_ehr_dataset = [{'labels': p['labels'], 'visits': [[c for c in v if c in train_c] for v in p['visits']]} for p in test_ehr_dataset] - File "/u/ethanmr3/halo/PyHealth/pyhealth/models/generators/halo.py", line 55, in - self.test_ehr_dataset = [{'labels': p['labels'], 'visits': [[c for c in v if c in train_c] for v in p['visits']]} for p in test_ehr_dataset] -KeyError: 'labels' diff --git a/halo-testing-logs/halo_test_3707040.out b/halo-testing-logs/halo_test_3707040.out deleted file mode 100644 index 4ced89a60..000000000 --- a/halo-testing-logs/halo_test_3707040.out +++ /dev/null @@ -1,161 +0,0 @@ -SLURM_JOB_ID: 3707040 -SLURM_JOB_NODELIST: ccc0388 -SLURM_NTASKS: -SLURM_CPUS_ON_NODE: 4 -SLURM_GPUS_ON_NODE: 1 -SLURM_GPUS: -CUDA_VISIBLE_DEVICES: 0 -Running nvidia-smi to confirm GPU availability: -Sat Jul 19 20:05:54 2025 -+-----------------------------------------------------------------------------------------+ -| NVIDIA-SMI 575.57.08 Driver Version: 575.57.08 CUDA Version: 12.9 | -|-----------------------------------------+------------------------+----------------------+ -| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC | -| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. | -| | | MIG M. | -|=========================================+========================+======================| -| 0 NVIDIA A100-SXM4-80GB Off | 00000000:41:00.0 Off | 0 | -| N/A 26C P0 61W / 500W | 0MiB / 81920MiB | 0% Default | -| | | Disabled | -+-----------------------------------------+------------------------+----------------------+ - -+-----------------------------------------------------------------------------------------+ -| Processes: | -| GPU GI CI PID Type Process name GPU Memory | -| ID ID Usage | -|=========================================================================================| -| No running processes found | -+-----------------------------------------------------------------------------------------+ -Defaulting to user installation because normal site-packages is not writeable -Obtaining file:///u/ethanmr3/halo/PyHealth -Requirement already satisfied: torch>=1.8.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2.7.1) -Requirement already satisfied: torchvision>=0.9.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (0.22.1) -Requirement already satisfied: rdkit>=2022.03.4 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2025.3.3) -Requirement already satisfied: scikit-learn>=0.24.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.6.1) -Requirement already satisfied: networkx>=2.6.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (3.2.1) -Collecting pandas<2,>=1.3.2 - Using cached pandas-1.5.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.2 MB) -Requirement already satisfied: pandarallel>=1.5.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.6.5) -Requirement already satisfied: mne>=1.0.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.8.0) -Requirement already satisfied: urllib3<=1.26.15 in /usr/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.26.5) -Requirement already satisfied: numpy in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2.0.2) -Requirement already satisfied: tqdm in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (4.67.1) -Requirement already satisfied: polars in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.31.0) -Requirement already satisfied: transformers in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (4.52.4) -Requirement already satisfied: packaging in /usr/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (20.9) -Requirement already satisfied: lazy-loader>=0.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (0.4) -Requirement already satisfied: jinja2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (3.1.6) -Requirement already satisfied: pooch>=1.5 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (1.8.2) -Requirement already satisfied: scipy>=1.9 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (1.13.1) -Requirement already satisfied: decorator in /usr/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (4.4.2) -Requirement already satisfied: matplotlib>=3.6 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (3.9.4) -Requirement already satisfied: contourpy>=1.0.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.3.0) -Requirement already satisfied: fonttools>=4.22.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (4.58.4) -Requirement already satisfied: pyparsing>=2.3.1 in /usr/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (2.4.7) -Requirement already satisfied: cycler>=0.10 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (0.12.1) -Requirement already satisfied: python-dateutil>=2.7 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (2.9.0.post0) -Requirement already satisfied: importlib-resources>=3.2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (6.5.2) -Requirement already satisfied: pillow>=8 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (11.2.1) -Requirement already satisfied: kiwisolver>=1.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.4.7) -Requirement already satisfied: zipp>=3.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from importlib-resources>=3.2.0->matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (3.23.0) -Requirement already satisfied: dill>=0.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandarallel>=1.5.3->pyhealth==1.1.4) (0.4.0) -Requirement already satisfied: psutil in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandarallel>=1.5.3->pyhealth==1.1.4) (7.0.0) -Requirement already satisfied: pytz>=2020.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandas<2,>=1.3.2->pyhealth==1.1.4) (2025.2) -Requirement already satisfied: requests>=2.19.0 in /usr/lib/python3.9/site-packages (from pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (2.25.1) -Requirement already satisfied: platformdirs>=2.5.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (4.3.8) -Requirement already satisfied: six>=1.5 in /u/ethanmr3/.local/lib/python3.9/site-packages (from python-dateutil>=2.7->matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.17.0) -Requirement already satisfied: idna<3,>=2.5 in /usr/lib/python3.9/site-packages (from requests>=2.19.0->pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (2.10) -Requirement already satisfied: chardet<5,>=3.0.2 in /usr/lib/python3.9/site-packages (from requests>=2.19.0->pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (4.0.0) -Requirement already satisfied: joblib>=1.2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from scikit-learn>=0.24.2->pyhealth==1.1.4) (1.5.1) -Requirement already satisfied: threadpoolctl>=3.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from scikit-learn>=0.24.2->pyhealth==1.1.4) (3.6.0) -Requirement already satisfied: nvidia-cufile-cu12==1.11.1.6 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (1.11.1.6) -Requirement already satisfied: nvidia-nvjitlink-cu12==12.6.85 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.85) -Requirement already satisfied: nvidia-cuda-cupti-cu12==12.6.80 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.80) -Requirement already satisfied: nvidia-cusolver-cu12==11.7.1.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (11.7.1.2) -Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) -Requirement already satisfied: typing-extensions>=4.10.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (4.14.0) -Requirement already satisfied: nvidia-nvtx-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) -Requirement already satisfied: nvidia-cusparse-cu12==12.5.4.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.5.4.2) -Requirement already satisfied: nvidia-cublas-cu12==12.6.4.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.4.1) -Requirement already satisfied: nvidia-cufft-cu12==11.3.0.4 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (11.3.0.4) -Requirement already satisfied: nvidia-cudnn-cu12==9.5.1.17 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (9.5.1.17) -Requirement already satisfied: sympy>=1.13.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (1.14.0) -Requirement already satisfied: filelock in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (3.18.0) -Requirement already satisfied: nvidia-nccl-cu12==2.26.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (2.26.2) -Requirement already satisfied: triton==3.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (3.3.1) -Requirement already satisfied: nvidia-curand-cu12==10.3.7.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (10.3.7.77) -Requirement already satisfied: nvidia-cusparselt-cu12==0.6.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (0.6.3) -Requirement already satisfied: fsspec in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (2025.5.1) -Requirement already satisfied: nvidia-cuda-runtime-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) -Requirement already satisfied: setuptools>=40.8.0 in /usr/lib/python3.9/site-packages (from triton==3.3.1->torch>=1.8.0->pyhealth==1.1.4) (53.0.0) -Requirement already satisfied: mpmath<1.4,>=1.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from sympy>=1.13.3->torch>=1.8.0->pyhealth==1.1.4) (1.3.0) -Requirement already satisfied: MarkupSafe>=2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from jinja2->mne>=1.0.3->pyhealth==1.1.4) (3.0.2) -Requirement already satisfied: regex!=2019.12.17 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (2024.11.6) -Requirement already satisfied: pyyaml>=5.1 in /usr/lib64/python3.9/site-packages (from transformers->pyhealth==1.1.4) (5.4.1) -Requirement already satisfied: tokenizers<0.22,>=0.21 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.21.1) -Requirement already satisfied: safetensors>=0.4.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.5.3) -Requirement already satisfied: huggingface-hub<1.0,>=0.30.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.33.0) -Requirement already satisfied: hf-xet<2.0.0,>=1.1.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from huggingface-hub<1.0,>=0.30.0->transformers->pyhealth==1.1.4) (1.1.5) -Installing collected packages: pandas, pyhealth - Attempting uninstall: pandas - Found existing installation: pandas 2.3.1 - Uninstalling pandas-2.3.1: - Successfully uninstalled pandas-2.3.1 - Attempting uninstall: pyhealth - Found existing installation: pyhealth 1.1.4 - Uninstalling pyhealth-1.1.4: - Successfully uninstalled pyhealth-1.1.4 - Running setup.py develop for pyhealth -Successfully installed pandas-1.5.3 pyhealth -Defaulting to user installation because normal site-packages is not writeable -Collecting numpy - Using cached numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (19.5 MB) -Installing collected packages: numpy - Attempting uninstall: numpy - Found existing installation: numpy 2.0.2 - Uninstalling numpy-2.0.2: - Successfully uninstalled numpy-2.0.2 -Successfully installed numpy-2.0.2 -Defaulting to user installation because normal site-packages is not writeable -Collecting pandas - Using cached pandas-2.3.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.4 MB) -Collecting pytz>=2020.1 - Using cached pytz-2025.2-py2.py3-none-any.whl (509 kB) -Collecting numpy>=1.22.4 - Using cached numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (19.5 MB) -Collecting python-dateutil>=2.8.2 - Using cached python_dateutil-2.9.0.post0-py2.py3-none-any.whl (229 kB) -Collecting tzdata>=2022.7 - Using cached tzdata-2025.2-py2.py3-none-any.whl (347 kB) -Collecting six>=1.5 - Using cached six-1.17.0-py2.py3-none-any.whl (11 kB) -Installing collected packages: six, tzdata, pytz, python-dateutil, numpy, pandas - Attempting uninstall: six - Found existing installation: six 1.17.0 - Uninstalling six-1.17.0: - Successfully uninstalled six-1.17.0 - Attempting uninstall: tzdata - Found existing installation: tzdata 2025.2 - Uninstalling tzdata-2025.2: - Successfully uninstalled tzdata-2025.2 - Attempting uninstall: pytz - Found existing installation: pytz 2025.2 - Uninstalling pytz-2025.2: - Successfully uninstalled pytz-2025.2 - Attempting uninstall: python-dateutil - Found existing installation: python-dateutil 2.9.0.post0 - Uninstalling python-dateutil-2.9.0.post0: - Successfully uninstalled python-dateutil-2.9.0.post0 - Attempting uninstall: numpy - Found existing installation: numpy 2.0.2 - Uninstalling numpy-2.0.2: - Successfully uninstalled numpy-2.0.2 - Attempting uninstall: pandas - Found existing installation: pandas 1.5.3 - Uninstalling pandas-1.5.3: - Successfully uninstalled pandas-1.5.3 -Successfully installed numpy-2.0.2 pandas-2.3.1 python-dateutil-2.9.0.post0 pytz-2025.2 six-1.17.0 tzdata-2025.2 -BEGIN: Testing -Success on pip install -e . -Sucess on imports -Operating in dir: /u/ethanmr3/halo/PyHealth diff --git a/halo-testing-logs/halo_test_3707410.err b/halo-testing-logs/halo_test_3707410.err deleted file mode 100644 index e5a0d5f28..000000000 --- a/halo-testing-logs/halo_test_3707410.err +++ /dev/null @@ -1,35 +0,0 @@ - WARNING: Value for scheme.platlib does not match. Please report this to - distutils: /u/ethanmr3/.local/lib/python3.9/site-packages - sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages - WARNING: Additional context: - user = True - home = None - root = None - prefix = None - WARNING: Value for scheme.platlib does not match. Please report this to - distutils: /u/ethanmr3/.local/lib/python3.9/site-packages - sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages - WARNING: Additional context: - user = True - home = None - root = None - prefix = None - WARNING: Value for scheme.platlib does not match. Please report this to - distutils: /u/ethanmr3/.local/lib/python3.9/site-packages - sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages - WARNING: Additional context: - user = True - home = None - root = None - prefix = None -ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. -pyhealth 1.1.4 requires pandas<2,>=1.3.2, but you have pandas 2.3.1 which is incompatible. - 0%| | 0/58976 [00:00 - model = HALO(dataset=halo_dataset, config=halo_config, save_dir="../../halo_save/") - File "/u/ethanmr3/halo/PyHealth/pyhealth/models/generators/halo.py", line 59, in __init__ - self.test_ehr_dataset = [{'labels': p['labels'], 'visits': [[c for c in v if c in train_c] for v in p['visits']]} for p in test_ehr_dataset] - File "/u/ethanmr3/halo/PyHealth/pyhealth/models/generators/halo.py", line 59, in - self.test_ehr_dataset = [{'labels': p['labels'], 'visits': [[c for c in v if c in train_c] for v in p['visits']]} for p in test_ehr_dataset] -KeyError: 'labels' diff --git a/halo-testing-logs/halo_test_3707410.out b/halo-testing-logs/halo_test_3707410.out deleted file mode 100644 index 5f6316c80..000000000 --- a/halo-testing-logs/halo_test_3707410.out +++ /dev/null @@ -1,172 +0,0 @@ -SLURM_JOB_ID: 3707410 -SLURM_JOB_NODELIST: ccc0388 -SLURM_NTASKS: -SLURM_CPUS_ON_NODE: 4 -SLURM_GPUS_ON_NODE: 1 -SLURM_GPUS: -CUDA_VISIBLE_DEVICES: 0 -Running nvidia-smi to confirm GPU availability: -Sat Jul 19 20:31:37 2025 -+-----------------------------------------------------------------------------------------+ -| NVIDIA-SMI 575.57.08 Driver Version: 575.57.08 CUDA Version: 12.9 | -|-----------------------------------------+------------------------+----------------------+ -| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC | -| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. | -| | | MIG M. | -|=========================================+========================+======================| -| 0 NVIDIA A100-SXM4-80GB Off | 00000000:41:00.0 Off | 0 | -| N/A 26C P0 61W / 500W | 0MiB / 81920MiB | 0% Default | -| | | Disabled | -+-----------------------------------------+------------------------+----------------------+ - -+-----------------------------------------------------------------------------------------+ -| Processes: | -| GPU GI CI PID Type Process name GPU Memory | -| ID ID Usage | -|=========================================================================================| -| No running processes found | -+-----------------------------------------------------------------------------------------+ -Defaulting to user installation because normal site-packages is not writeable -Obtaining file:///u/ethanmr3/halo/PyHealth -Requirement already satisfied: torch>=1.8.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2.7.1) -Requirement already satisfied: torchvision>=0.9.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (0.22.1) -Requirement already satisfied: rdkit>=2022.03.4 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2025.3.3) -Requirement already satisfied: scikit-learn>=0.24.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.6.1) -Requirement already satisfied: networkx>=2.6.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (3.2.1) -Collecting pandas<2,>=1.3.2 - Using cached pandas-1.5.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.2 MB) -Requirement already satisfied: pandarallel>=1.5.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.6.5) -Requirement already satisfied: mne>=1.0.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.8.0) -Requirement already satisfied: urllib3<=1.26.15 in /usr/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.26.5) -Requirement already satisfied: numpy in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2.0.2) -Requirement already satisfied: tqdm in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (4.67.1) -Requirement already satisfied: polars in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.31.0) -Requirement already satisfied: transformers in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (4.52.4) -Requirement already satisfied: scipy>=1.9 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (1.13.1) -Requirement already satisfied: pooch>=1.5 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (1.8.2) -Requirement already satisfied: lazy-loader>=0.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (0.4) -Requirement already satisfied: matplotlib>=3.6 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (3.9.4) -Requirement already satisfied: jinja2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (3.1.6) -Requirement already satisfied: packaging in /usr/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (20.9) -Requirement already satisfied: decorator in /usr/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (4.4.2) -Requirement already satisfied: kiwisolver>=1.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.4.7) -Requirement already satisfied: cycler>=0.10 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (0.12.1) -Requirement already satisfied: python-dateutil>=2.7 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (2.9.0.post0) -Requirement already satisfied: pyparsing>=2.3.1 in /usr/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (2.4.7) -Requirement already satisfied: importlib-resources>=3.2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (6.5.2) -Requirement already satisfied: fonttools>=4.22.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (4.58.4) -Requirement already satisfied: pillow>=8 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (11.2.1) -Requirement already satisfied: contourpy>=1.0.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.3.0) -Requirement already satisfied: zipp>=3.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from importlib-resources>=3.2.0->matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (3.23.0) -Requirement already satisfied: dill>=0.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandarallel>=1.5.3->pyhealth==1.1.4) (0.4.0) -Requirement already satisfied: psutil in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandarallel>=1.5.3->pyhealth==1.1.4) (7.0.0) -Requirement already satisfied: pytz>=2020.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandas<2,>=1.3.2->pyhealth==1.1.4) (2025.2) -Requirement already satisfied: platformdirs>=2.5.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (4.3.8) -Requirement already satisfied: requests>=2.19.0 in /usr/lib/python3.9/site-packages (from pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (2.25.1) -Requirement already satisfied: six>=1.5 in /u/ethanmr3/.local/lib/python3.9/site-packages (from python-dateutil>=2.7->matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.17.0) -Requirement already satisfied: idna<3,>=2.5 in /usr/lib/python3.9/site-packages (from requests>=2.19.0->pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (2.10) -Requirement already satisfied: chardet<5,>=3.0.2 in /usr/lib/python3.9/site-packages (from requests>=2.19.0->pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (4.0.0) -Requirement already satisfied: threadpoolctl>=3.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from scikit-learn>=0.24.2->pyhealth==1.1.4) (3.6.0) -Requirement already satisfied: joblib>=1.2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from scikit-learn>=0.24.2->pyhealth==1.1.4) (1.5.1) -Requirement already satisfied: nvidia-cudnn-cu12==9.5.1.17 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (9.5.1.17) -Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) -Requirement already satisfied: nvidia-curand-cu12==10.3.7.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (10.3.7.77) -Requirement already satisfied: fsspec in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (2025.5.1) -Requirement already satisfied: nvidia-cusolver-cu12==11.7.1.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (11.7.1.2) -Requirement already satisfied: nvidia-cusparselt-cu12==0.6.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (0.6.3) -Requirement already satisfied: sympy>=1.13.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (1.14.0) -Requirement already satisfied: triton==3.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (3.3.1) -Requirement already satisfied: nvidia-nvjitlink-cu12==12.6.85 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.85) -Requirement already satisfied: nvidia-nvtx-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) -Requirement already satisfied: nvidia-nccl-cu12==2.26.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (2.26.2) -Requirement already satisfied: filelock in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (3.18.0) -Requirement already satisfied: nvidia-cuda-runtime-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) -Requirement already satisfied: nvidia-cublas-cu12==12.6.4.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.4.1) -Requirement already satisfied: nvidia-cuda-cupti-cu12==12.6.80 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.80) -Requirement already satisfied: nvidia-cusparse-cu12==12.5.4.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.5.4.2) -Requirement already satisfied: nvidia-cufft-cu12==11.3.0.4 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (11.3.0.4) -Requirement already satisfied: typing-extensions>=4.10.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (4.14.0) -Requirement already satisfied: nvidia-cufile-cu12==1.11.1.6 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (1.11.1.6) -Requirement already satisfied: setuptools>=40.8.0 in /usr/lib/python3.9/site-packages (from triton==3.3.1->torch>=1.8.0->pyhealth==1.1.4) (53.0.0) -Requirement already satisfied: mpmath<1.4,>=1.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from sympy>=1.13.3->torch>=1.8.0->pyhealth==1.1.4) (1.3.0) -Requirement already satisfied: MarkupSafe>=2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from jinja2->mne>=1.0.3->pyhealth==1.1.4) (3.0.2) -Requirement already satisfied: huggingface-hub<1.0,>=0.30.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.33.0) -Requirement already satisfied: safetensors>=0.4.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.5.3) -Requirement already satisfied: tokenizers<0.22,>=0.21 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.21.1) -Requirement already satisfied: pyyaml>=5.1 in /usr/lib64/python3.9/site-packages (from transformers->pyhealth==1.1.4) (5.4.1) -Requirement already satisfied: regex!=2019.12.17 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (2024.11.6) -Requirement already satisfied: hf-xet<2.0.0,>=1.1.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from huggingface-hub<1.0,>=0.30.0->transformers->pyhealth==1.1.4) (1.1.5) -Installing collected packages: pandas, pyhealth - Attempting uninstall: pandas - Found existing installation: pandas 2.3.1 - Uninstalling pandas-2.3.1: - Successfully uninstalled pandas-2.3.1 - Attempting uninstall: pyhealth - Found existing installation: pyhealth 1.1.4 - Uninstalling pyhealth-1.1.4: - Successfully uninstalled pyhealth-1.1.4 - Running setup.py develop for pyhealth -Successfully installed pandas-1.5.3 pyhealth -Defaulting to user installation because normal site-packages is not writeable -Collecting numpy - Using cached numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (19.5 MB) -Installing collected packages: numpy - Attempting uninstall: numpy - Found existing installation: numpy 2.0.2 - Uninstalling numpy-2.0.2: - Successfully uninstalled numpy-2.0.2 -Successfully installed numpy-2.0.2 -Defaulting to user installation because normal site-packages is not writeable -Collecting pandas - Using cached pandas-2.3.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.4 MB) -Collecting pytz>=2020.1 - Using cached pytz-2025.2-py2.py3-none-any.whl (509 kB) -Collecting python-dateutil>=2.8.2 - Using cached python_dateutil-2.9.0.post0-py2.py3-none-any.whl (229 kB) -Collecting numpy>=1.22.4 - Using cached numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (19.5 MB) -Collecting tzdata>=2022.7 - Using cached tzdata-2025.2-py2.py3-none-any.whl (347 kB) -Collecting six>=1.5 - Using cached six-1.17.0-py2.py3-none-any.whl (11 kB) -Installing collected packages: six, tzdata, pytz, python-dateutil, numpy, pandas - Attempting uninstall: six - Found existing installation: six 1.17.0 - Uninstalling six-1.17.0: - Successfully uninstalled six-1.17.0 - Attempting uninstall: tzdata - Found existing installation: tzdata 2025.2 - Uninstalling tzdata-2025.2: - Successfully uninstalled tzdata-2025.2 - Attempting uninstall: pytz - Found existing installation: pytz 2025.2 - Uninstalling pytz-2025.2: - Successfully uninstalled pytz-2025.2 - Attempting uninstall: python-dateutil - Found existing installation: python-dateutil 2.9.0.post0 - Uninstalling python-dateutil-2.9.0.post0: - Successfully uninstalled python-dateutil-2.9.0.post0 - Attempting uninstall: numpy - Found existing installation: numpy 2.0.2 - Uninstalling numpy-2.0.2: - Successfully uninstalled numpy-2.0.2 - Attempting uninstall: pandas - Found existing installation: pandas 1.5.3 - Uninstalling pandas-1.5.3: - Successfully uninstalled pandas-1.5.3 -Successfully installed numpy-2.0.2 pandas-2.3.1 python-dateutil-2.9.0.post0 pytz-2025.2 six-1.17.0 tzdata-2025.2 -BEGIN: Testing -Success on pip install -e . -Sucess on imports -Operating in dir: /u/ethanmr3/halo/PyHealth -VOCAB SIZE: 6984 -MAX LEN: 42 -AVG LEN: 1.267755803955288 -MAX VISIT LEN: 39 -AVG VISIT LEN: 11.037371134020619 -NUM RECORDS: 46520 -NUM LONGITUDINAL RECORDS: 7537 -Saving Everything -6984 -test data head 2: -[{'visits': [[3224, 1966, 294, 6423]]}, {'visits': [[1697, 5357, 1839, 4848, 3439, 5042, 2490, 4315, 767], [2176, 3585, 3208, 5384, 651, 1807, 5392, 1935, 6420, 279, 4123, 3359, 6946, 1059, 809, 3895, 1721, 2234, 3004, 839, 588, 6480, 5203, 597, 2008, 2274, 739, 5224], [5381, 2181, 3208, 3336, 3723, 5392, 4886, 2075, 6946, 5043, 2996, 695, 3895, 2749, 4545, 3266, 1482, 850, 2518, 3671, 3288, 4315, 757], [1025, 3203, 1542, 3208, 1807, 5392, 1697, 5922, 1059, 552, 2353, 2866, 5683, 2996, 3895, 6455, 1341, 4804, 2893, 5326, 5965, 6480, 6737, 3288, 4576, 4064, 6114, 2404, 998, 757, 4221]]}, {'visits': [[4545, 5061, 1642, 3246, 3157, 2492]]}, {'visits': [[5888, 3714, 4708, 3557, 390, 1800, 5935, 597, 1366]]}, {'visits': [[1264, 2827, 5535]]}] diff --git a/halo-testing-logs/halo_test_3708721.err b/halo-testing-logs/halo_test_3708721.err deleted file mode 100644 index 5b5e03499..000000000 --- a/halo-testing-logs/halo_test_3708721.err +++ /dev/null @@ -1,35 +0,0 @@ - WARNING: Value for scheme.platlib does not match. Please report this to - distutils: /u/ethanmr3/.local/lib/python3.9/site-packages - sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages - WARNING: Additional context: - user = True - home = None - root = None - prefix = None - WARNING: Value for scheme.platlib does not match. Please report this to - distutils: /u/ethanmr3/.local/lib/python3.9/site-packages - sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages - WARNING: Additional context: - user = True - home = None - root = None - prefix = None - WARNING: Value for scheme.platlib does not match. Please report this to - distutils: /u/ethanmr3/.local/lib/python3.9/site-packages - sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages - WARNING: Additional context: - user = True - home = None - root = None - prefix = None -ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. -pyhealth 1.1.4 requires pandas<2,>=1.3.2, but you have pandas 2.3.1 which is incompatible. - 0%| | 0/58976 [00:00 - model = HALO(dataset=halo_dataset, config=halo_config, save_dir="../../halo_save/") - File "/u/ethanmr3/halo/PyHealth/pyhealth/models/generators/halo.py", line 59, in __init__ - self.test_ehr_dataset = [{'labels': p['labels'], 'visits': [[c for c in v if c in train_c] for v in p['visits']]} for p in test_ehr_dataset] - File "/u/ethanmr3/halo/PyHealth/pyhealth/models/generators/halo.py", line 59, in - self.test_ehr_dataset = [{'labels': p['labels'], 'visits': [[c for c in v if c in train_c] for v in p['visits']]} for p in test_ehr_dataset] -KeyError: 'labels' diff --git a/halo-testing-logs/halo_test_3708721.out b/halo-testing-logs/halo_test_3708721.out deleted file mode 100644 index 4f156892a..000000000 --- a/halo-testing-logs/halo_test_3708721.out +++ /dev/null @@ -1,175 +0,0 @@ -SLURM_JOB_ID: 3708721 -SLURM_JOB_NODELIST: ccc0388 -SLURM_NTASKS: -SLURM_CPUS_ON_NODE: 4 -SLURM_GPUS_ON_NODE: 1 -SLURM_GPUS: -CUDA_VISIBLE_DEVICES: 0 -Running nvidia-smi to confirm GPU availability: -Sat Jul 19 22:03:32 2025 -+-----------------------------------------------------------------------------------------+ -| NVIDIA-SMI 575.57.08 Driver Version: 575.57.08 CUDA Version: 12.9 | -|-----------------------------------------+------------------------+----------------------+ -| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC | -| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. | -| | | MIG M. | -|=========================================+========================+======================| -| 0 NVIDIA A100-SXM4-80GB Off | 00000000:81:00.0 Off | 0 | -| N/A 31C P0 62W / 500W | 0MiB / 81920MiB | 0% Default | -| | | Disabled | -+-----------------------------------------+------------------------+----------------------+ - -+-----------------------------------------------------------------------------------------+ -| Processes: | -| GPU GI CI PID Type Process name GPU Memory | -| ID ID Usage | -|=========================================================================================| -| No running processes found | -+-----------------------------------------------------------------------------------------+ -Defaulting to user installation because normal site-packages is not writeable -Obtaining file:///u/ethanmr3/halo/PyHealth -Requirement already satisfied: torch>=1.8.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2.7.1) -Requirement already satisfied: torchvision>=0.9.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (0.22.1) -Requirement already satisfied: rdkit>=2022.03.4 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2025.3.3) -Requirement already satisfied: scikit-learn>=0.24.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.6.1) -Requirement already satisfied: networkx>=2.6.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (3.2.1) -Collecting pandas<2,>=1.3.2 - Using cached pandas-1.5.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.2 MB) -Requirement already satisfied: pandarallel>=1.5.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.6.5) -Requirement already satisfied: mne>=1.0.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.8.0) -Requirement already satisfied: urllib3<=1.26.15 in /usr/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.26.5) -Requirement already satisfied: numpy in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2.0.2) -Requirement already satisfied: tqdm in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (4.67.1) -Requirement already satisfied: polars in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.31.0) -Requirement already satisfied: transformers in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (4.52.4) -Requirement already satisfied: matplotlib>=3.6 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (3.9.4) -Requirement already satisfied: jinja2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (3.1.6) -Requirement already satisfied: packaging in /usr/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (20.9) -Requirement already satisfied: pooch>=1.5 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (1.8.2) -Requirement already satisfied: scipy>=1.9 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (1.13.1) -Requirement already satisfied: lazy-loader>=0.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (0.4) -Requirement already satisfied: decorator in /usr/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (4.4.2) -Requirement already satisfied: python-dateutil>=2.7 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (2.9.0.post0) -Requirement already satisfied: pillow>=8 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (11.2.1) -Requirement already satisfied: importlib-resources>=3.2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (6.5.2) -Requirement already satisfied: kiwisolver>=1.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.4.7) -Requirement already satisfied: contourpy>=1.0.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.3.0) -Requirement already satisfied: pyparsing>=2.3.1 in /usr/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (2.4.7) -Requirement already satisfied: fonttools>=4.22.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (4.58.4) -Requirement already satisfied: cycler>=0.10 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (0.12.1) -Requirement already satisfied: zipp>=3.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from importlib-resources>=3.2.0->matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (3.23.0) -Requirement already satisfied: dill>=0.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandarallel>=1.5.3->pyhealth==1.1.4) (0.4.0) -Requirement already satisfied: psutil in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandarallel>=1.5.3->pyhealth==1.1.4) (7.0.0) -Requirement already satisfied: pytz>=2020.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandas<2,>=1.3.2->pyhealth==1.1.4) (2025.2) -Requirement already satisfied: requests>=2.19.0 in /usr/lib/python3.9/site-packages (from pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (2.25.1) -Requirement already satisfied: platformdirs>=2.5.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (4.3.8) -Requirement already satisfied: six>=1.5 in /u/ethanmr3/.local/lib/python3.9/site-packages (from python-dateutil>=2.7->matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.17.0) -Requirement already satisfied: idna<3,>=2.5 in /usr/lib/python3.9/site-packages (from requests>=2.19.0->pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (2.10) -Requirement already satisfied: chardet<5,>=3.0.2 in /usr/lib/python3.9/site-packages (from requests>=2.19.0->pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (4.0.0) -Requirement already satisfied: joblib>=1.2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from scikit-learn>=0.24.2->pyhealth==1.1.4) (1.5.1) -Requirement already satisfied: threadpoolctl>=3.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from scikit-learn>=0.24.2->pyhealth==1.1.4) (3.6.0) -Requirement already satisfied: nvidia-cusolver-cu12==11.7.1.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (11.7.1.2) -Requirement already satisfied: nvidia-cuda-runtime-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) -Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) -Requirement already satisfied: nvidia-cuda-cupti-cu12==12.6.80 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.80) -Requirement already satisfied: nvidia-nccl-cu12==2.26.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (2.26.2) -Requirement already satisfied: typing-extensions>=4.10.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (4.14.0) -Requirement already satisfied: nvidia-nvtx-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) -Requirement already satisfied: nvidia-cusparselt-cu12==0.6.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (0.6.3) -Requirement already satisfied: nvidia-cusparse-cu12==12.5.4.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.5.4.2) -Requirement already satisfied: nvidia-cublas-cu12==12.6.4.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.4.1) -Requirement already satisfied: filelock in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (3.18.0) -Requirement already satisfied: sympy>=1.13.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (1.14.0) -Requirement already satisfied: nvidia-curand-cu12==10.3.7.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (10.3.7.77) -Requirement already satisfied: nvidia-cufile-cu12==1.11.1.6 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (1.11.1.6) -Requirement already satisfied: nvidia-cufft-cu12==11.3.0.4 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (11.3.0.4) -Requirement already satisfied: fsspec in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (2025.5.1) -Requirement already satisfied: triton==3.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (3.3.1) -Requirement already satisfied: nvidia-cudnn-cu12==9.5.1.17 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (9.5.1.17) -Requirement already satisfied: nvidia-nvjitlink-cu12==12.6.85 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.85) -Requirement already satisfied: setuptools>=40.8.0 in /usr/lib/python3.9/site-packages (from triton==3.3.1->torch>=1.8.0->pyhealth==1.1.4) (53.0.0) -Requirement already satisfied: mpmath<1.4,>=1.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from sympy>=1.13.3->torch>=1.8.0->pyhealth==1.1.4) (1.3.0) -Requirement already satisfied: MarkupSafe>=2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from jinja2->mne>=1.0.3->pyhealth==1.1.4) (3.0.2) -Requirement already satisfied: huggingface-hub<1.0,>=0.30.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.33.0) -Requirement already satisfied: regex!=2019.12.17 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (2024.11.6) -Requirement already satisfied: safetensors>=0.4.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.5.3) -Requirement already satisfied: pyyaml>=5.1 in /usr/lib64/python3.9/site-packages (from transformers->pyhealth==1.1.4) (5.4.1) -Requirement already satisfied: tokenizers<0.22,>=0.21 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.21.1) -Requirement already satisfied: hf-xet<2.0.0,>=1.1.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from huggingface-hub<1.0,>=0.30.0->transformers->pyhealth==1.1.4) (1.1.5) -Installing collected packages: pandas, pyhealth - Attempting uninstall: pandas - Found existing installation: pandas 2.3.1 - Uninstalling pandas-2.3.1: - Successfully uninstalled pandas-2.3.1 - Attempting uninstall: pyhealth - Found existing installation: pyhealth 1.1.4 - Uninstalling pyhealth-1.1.4: - Successfully uninstalled pyhealth-1.1.4 - Running setup.py develop for pyhealth -Successfully installed pandas-1.5.3 pyhealth -Defaulting to user installation because normal site-packages is not writeable -Collecting numpy - Using cached numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (19.5 MB) -Installing collected packages: numpy - Attempting uninstall: numpy - Found existing installation: numpy 2.0.2 - Uninstalling numpy-2.0.2: - Successfully uninstalled numpy-2.0.2 -Successfully installed numpy-2.0.2 -Defaulting to user installation because normal site-packages is not writeable -Collecting pandas - Using cached pandas-2.3.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.4 MB) -Collecting tzdata>=2022.7 - Using cached tzdata-2025.2-py2.py3-none-any.whl (347 kB) -Collecting python-dateutil>=2.8.2 - Using cached python_dateutil-2.9.0.post0-py2.py3-none-any.whl (229 kB) -Collecting numpy>=1.22.4 - Using cached numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (19.5 MB) -Collecting pytz>=2020.1 - Using cached pytz-2025.2-py2.py3-none-any.whl (509 kB) -Collecting six>=1.5 - Using cached six-1.17.0-py2.py3-none-any.whl (11 kB) -Installing collected packages: six, tzdata, pytz, python-dateutil, numpy, pandas - Attempting uninstall: six - Found existing installation: six 1.17.0 - Uninstalling six-1.17.0: - Successfully uninstalled six-1.17.0 - Attempting uninstall: tzdata - Found existing installation: tzdata 2025.2 - Uninstalling tzdata-2025.2: - Successfully uninstalled tzdata-2025.2 - Attempting uninstall: pytz - Found existing installation: pytz 2025.2 - Uninstalling pytz-2025.2: - Successfully uninstalled pytz-2025.2 - Attempting uninstall: python-dateutil - Found existing installation: python-dateutil 2.9.0.post0 - Uninstalling python-dateutil-2.9.0.post0: - Successfully uninstalled python-dateutil-2.9.0.post0 - Attempting uninstall: numpy - Found existing installation: numpy 2.0.2 - Uninstalling numpy-2.0.2: - Successfully uninstalled numpy-2.0.2 - Attempting uninstall: pandas - Found existing installation: pandas 1.5.3 - Uninstalling pandas-1.5.3: - Successfully uninstalled pandas-1.5.3 -Successfully installed numpy-2.0.2 pandas-2.3.1 python-dateutil-2.9.0.post0 pytz-2025.2 six-1.17.0 tzdata-2025.2 -BEGIN: Testing -Success on pip install -e . -Sucess on imports -Operating in dir: /u/ethanmr3/halo/PyHealth -VOCAB SIZE: 6984 -DATA LEN IS: 46520 -MIMIC LABELS = [1. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 1. 0. 1. 1. 0. 0. 0. 0. 0. 0. 1. 1. 1. - 1.] -MAX LEN: 42 -AVG LEN: 1.267755803955288 -MAX VISIT LEN: 39 -AVG VISIT LEN: 11.037371134020619 -NUM RECORDS: 46520 -NUM LONGITUDINAL RECORDS: 7537 -Saving Everything -6984 -test data head 2: -[{'visits': [[6144, 3305, 3165, 1430]]}, {'visits': [[2177, 834, 5378, 3431, 2473, 4554, 5098, 592, 624], [384, 3204, 398, 530, 5909, 2584, 6553, 6685, 798, 6431, 6947, 2471, 679, 2093, 1838, 429, 3762, 5308, 1472, 579, 5699, 1482, 4045, 4948, 2267, 4577, 5361, 4220], [2311, 398, 530, 6934, 2584, 5410, 6947, 3491, 3752, 2859, 5812, 5428, 2494, 1601, 2883, 1749, 3544, 1631, 5602, 624, 1395, 6008, 2299], [2177, 5890, 2181, 5638, 3984, 530, 5909, 3736, 2584, 4380, 158, 6947, 3491, 167, 41, 5557, 3511, 3522, 2003, 4948, 1749, 4821, 4827, 6109, 1004, 6130, 243, 756, 6008, 5370, 4220]]}, {'visits': [[1057, 963, 4619, 240, 6934, 5049]]}, {'visits': [[4096, 5090, 5699, 3685, 1515, 4820, 1653, 3479, 5050]]}, {'visits': [[5482, 3995, 3839]]}] diff --git a/halo-testing-logs/halo_test_3708911.err b/halo-testing-logs/halo_test_3708911.err deleted file mode 100644 index f01e0ce4c..000000000 --- a/halo-testing-logs/halo_test_3708911.err +++ /dev/null @@ -1,28 +0,0 @@ - WARNING: Value for scheme.platlib does not match. Please report this to - distutils: /u/ethanmr3/.local/lib/python3.9/site-packages - sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages - WARNING: Additional context: - user = True - home = None - root = None - prefix = None - WARNING: Value for scheme.platlib does not match. Please report this to - distutils: /u/ethanmr3/.local/lib/python3.9/site-packages - sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages - WARNING: Additional context: - user = True - home = None - root = None - prefix = None - WARNING: Value for scheme.platlib does not match. Please report this to - distutils: /u/ethanmr3/.local/lib/python3.9/site-packages - sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages - WARNING: Additional context: - user = True - home = None - root = None - prefix = None -ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. -pyhealth 1.1.4 requires pandas<2,>=1.3.2, but you have pandas 2.3.1 which is incompatible. - 0%| | 0/58976 [00:00=1.8.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2.7.1) -Requirement already satisfied: torchvision>=0.9.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (0.22.1) -Requirement already satisfied: rdkit>=2022.03.4 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2025.3.3) -Requirement already satisfied: scikit-learn>=0.24.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.6.1) -Requirement already satisfied: networkx>=2.6.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (3.2.1) -Collecting pandas<2,>=1.3.2 - Using cached pandas-1.5.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.2 MB) -Requirement already satisfied: pandarallel>=1.5.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.6.5) -Requirement already satisfied: mne>=1.0.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.8.0) -Requirement already satisfied: urllib3<=1.26.15 in /usr/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.26.5) -Requirement already satisfied: numpy in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2.0.2) -Requirement already satisfied: tqdm in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (4.67.1) -Requirement already satisfied: polars in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.31.0) -Requirement already satisfied: transformers in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (4.52.4) -Requirement already satisfied: decorator in /usr/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (4.4.2) -Requirement already satisfied: pooch>=1.5 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (1.8.2) -Requirement already satisfied: jinja2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (3.1.6) -Requirement already satisfied: lazy-loader>=0.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (0.4) -Requirement already satisfied: scipy>=1.9 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (1.13.1) -Requirement already satisfied: matplotlib>=3.6 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (3.9.4) -Requirement already satisfied: packaging in /usr/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (20.9) -Requirement already satisfied: python-dateutil>=2.7 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (2.9.0.post0) -Requirement already satisfied: importlib-resources>=3.2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (6.5.2) -Requirement already satisfied: fonttools>=4.22.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (4.58.4) -Requirement already satisfied: contourpy>=1.0.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.3.0) -Requirement already satisfied: cycler>=0.10 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (0.12.1) -Requirement already satisfied: pyparsing>=2.3.1 in /usr/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (2.4.7) -Requirement already satisfied: kiwisolver>=1.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.4.7) -Requirement already satisfied: pillow>=8 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (11.2.1) -Requirement already satisfied: zipp>=3.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from importlib-resources>=3.2.0->matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (3.23.0) -Requirement already satisfied: dill>=0.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandarallel>=1.5.3->pyhealth==1.1.4) (0.4.0) -Requirement already satisfied: psutil in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandarallel>=1.5.3->pyhealth==1.1.4) (7.0.0) -Requirement already satisfied: pytz>=2020.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandas<2,>=1.3.2->pyhealth==1.1.4) (2025.2) -Requirement already satisfied: requests>=2.19.0 in /usr/lib/python3.9/site-packages (from pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (2.25.1) -Requirement already satisfied: platformdirs>=2.5.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (4.3.8) -Requirement already satisfied: six>=1.5 in /u/ethanmr3/.local/lib/python3.9/site-packages (from python-dateutil>=2.7->matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.17.0) -Requirement already satisfied: idna<3,>=2.5 in /usr/lib/python3.9/site-packages (from requests>=2.19.0->pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (2.10) -Requirement already satisfied: chardet<5,>=3.0.2 in /usr/lib/python3.9/site-packages (from requests>=2.19.0->pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (4.0.0) -Requirement already satisfied: threadpoolctl>=3.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from scikit-learn>=0.24.2->pyhealth==1.1.4) (3.6.0) -Requirement already satisfied: joblib>=1.2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from scikit-learn>=0.24.2->pyhealth==1.1.4) (1.5.1) -Requirement already satisfied: typing-extensions>=4.10.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (4.14.0) -Requirement already satisfied: nvidia-cuda-runtime-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) -Requirement already satisfied: nvidia-cusparselt-cu12==0.6.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (0.6.3) -Requirement already satisfied: nvidia-cudnn-cu12==9.5.1.17 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (9.5.1.17) -Requirement already satisfied: nvidia-cufft-cu12==11.3.0.4 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (11.3.0.4) -Requirement already satisfied: nvidia-cusparse-cu12==12.5.4.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.5.4.2) -Requirement already satisfied: triton==3.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (3.3.1) -Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) -Requirement already satisfied: nvidia-cublas-cu12==12.6.4.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.4.1) -Requirement already satisfied: nvidia-nccl-cu12==2.26.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (2.26.2) -Requirement already satisfied: nvidia-cuda-cupti-cu12==12.6.80 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.80) -Requirement already satisfied: nvidia-cusolver-cu12==11.7.1.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (11.7.1.2) -Requirement already satisfied: sympy>=1.13.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (1.14.0) -Requirement already satisfied: nvidia-curand-cu12==10.3.7.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (10.3.7.77) -Requirement already satisfied: fsspec in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (2025.5.1) -Requirement already satisfied: filelock in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (3.18.0) -Requirement already satisfied: nvidia-nvtx-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) -Requirement already satisfied: nvidia-nvjitlink-cu12==12.6.85 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.85) -Requirement already satisfied: nvidia-cufile-cu12==1.11.1.6 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (1.11.1.6) -Requirement already satisfied: setuptools>=40.8.0 in /usr/lib/python3.9/site-packages (from triton==3.3.1->torch>=1.8.0->pyhealth==1.1.4) (53.0.0) -Requirement already satisfied: mpmath<1.4,>=1.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from sympy>=1.13.3->torch>=1.8.0->pyhealth==1.1.4) (1.3.0) -Requirement already satisfied: MarkupSafe>=2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from jinja2->mne>=1.0.3->pyhealth==1.1.4) (3.0.2) -Requirement already satisfied: pyyaml>=5.1 in /usr/lib64/python3.9/site-packages (from transformers->pyhealth==1.1.4) (5.4.1) -Requirement already satisfied: huggingface-hub<1.0,>=0.30.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.33.0) -Requirement already satisfied: regex!=2019.12.17 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (2024.11.6) -Requirement already satisfied: safetensors>=0.4.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.5.3) -Requirement already satisfied: tokenizers<0.22,>=0.21 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.21.1) -Requirement already satisfied: hf-xet<2.0.0,>=1.1.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from huggingface-hub<1.0,>=0.30.0->transformers->pyhealth==1.1.4) (1.1.5) -Installing collected packages: pandas, pyhealth - Attempting uninstall: pandas - Found existing installation: pandas 2.3.1 - Uninstalling pandas-2.3.1: - Successfully uninstalled pandas-2.3.1 - Attempting uninstall: pyhealth - Found existing installation: pyhealth 1.1.4 - Uninstalling pyhealth-1.1.4: - Successfully uninstalled pyhealth-1.1.4 - Running setup.py develop for pyhealth -Successfully installed pandas-1.5.3 pyhealth -Defaulting to user installation because normal site-packages is not writeable -Collecting numpy - Using cached numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (19.5 MB) -Installing collected packages: numpy - Attempting uninstall: numpy - Found existing installation: numpy 2.0.2 - Uninstalling numpy-2.0.2: - Successfully uninstalled numpy-2.0.2 -Successfully installed numpy-2.0.2 -Defaulting to user installation because normal site-packages is not writeable -Collecting pandas - Using cached pandas-2.3.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.4 MB) -Collecting tzdata>=2022.7 - Using cached tzdata-2025.2-py2.py3-none-any.whl (347 kB) -Collecting pytz>=2020.1 - Using cached pytz-2025.2-py2.py3-none-any.whl (509 kB) -Collecting python-dateutil>=2.8.2 - Using cached python_dateutil-2.9.0.post0-py2.py3-none-any.whl (229 kB) -Collecting numpy>=1.22.4 - Using cached numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (19.5 MB) -Collecting six>=1.5 - Using cached six-1.17.0-py2.py3-none-any.whl (11 kB) -Installing collected packages: six, tzdata, pytz, python-dateutil, numpy, pandas - Attempting uninstall: six - Found existing installation: six 1.17.0 - Uninstalling six-1.17.0: - Successfully uninstalled six-1.17.0 - Attempting uninstall: tzdata - Found existing installation: tzdata 2025.2 - Uninstalling tzdata-2025.2: - Successfully uninstalled tzdata-2025.2 - Attempting uninstall: pytz - Found existing installation: pytz 2025.2 - Uninstalling pytz-2025.2: - Successfully uninstalled pytz-2025.2 - Attempting uninstall: python-dateutil - Found existing installation: python-dateutil 2.9.0.post0 - Uninstalling python-dateutil-2.9.0.post0: - Successfully uninstalled python-dateutil-2.9.0.post0 - Attempting uninstall: numpy - Found existing installation: numpy 2.0.2 - Uninstalling numpy-2.0.2: - Successfully uninstalled numpy-2.0.2 - Attempting uninstall: pandas - Found existing installation: pandas 1.5.3 - Uninstalling pandas-1.5.3: - Successfully uninstalled pandas-1.5.3 -Successfully installed numpy-2.0.2 pandas-2.3.1 python-dateutil-2.9.0.post0 pytz-2025.2 six-1.17.0 tzdata-2025.2 -BEGIN: Testing -Success on pip install -e . -Sucess on imports -Operating in dir: /u/ethanmr3/halo/PyHealth -VOCAB SIZE: 6984 -MAX LEN: 42 -AVG LEN: 1.267755803955288 -MAX VISIT LEN: 39 -AVG VISIT LEN: 11.037371134020619 -NUM RECORDS: 46520 -NUM LONGITUDINAL RECORDS: 7537 -Saving Everything -6984 -test data head 2: -[{'visits': [[6970, 3668, 5101, 1455]], 'labels': array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., - 0., 0., 0., 0., 0., 0., 0., 0.])}, {'visits': [[5349, 6533, 5641, 2732, 335, 370, 5780, 3125, 4699], [6528, 3841, 1034, 5771, 4876, 6548, 280, 412, 6179, 4650, 941, 6959, 5560, 2751, 2623, 6858, 2250, 4817, 2261, 1238, 4442, 859, 6237, 4064, 1255, 5103, 1913, 4479], [5122, 263, 1034, 1681, 5780, 1821, 2342, 295, 4650, 4916, 1852, 450, 5956, 582, 4300, 2003, 2906, 4064, 1122, 4327, 1263, 502, 4479], [3584, 3841, 1034, 3210, 282, 6948, 4650, 2732, 1837, 430, 5683, 2623, 3520, 6465, 5447, 6858, 4300, 3535, 5843, 595, 4565, 3681, 1250, 1122, 3045, 2662, 4327, 4336, 6907, 3964, 4479]], 'labels': array([1., 1., 1., 1., 1., 0., 0., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., - 1., 0., 0., 0., 1., 1., 1., 0.])}, {'visits': [[966, 4808, 746, 3379, 502, 6776]], 'labels': array([0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 1., 1., 1., 0., 0., 0., - 0., 0., 0., 0., 0., 0., 0., 0.])}, {'visits': [[2241, 1734, 360, 6959, 2033, 5813, 6518, 984, 4125]], 'labels': array([0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., - 0., 0., 0., 0., 1., 1., 0., 1.])}, {'visits': [[1001, 4779, 3966]], 'labels': array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., - 0., 0., 0., 0., 0., 0., 0., 0.])}] diff --git a/halo-testing-logs/halo_test_3709442.err b/halo-testing-logs/halo_test_3709442.err deleted file mode 100644 index 790c447cd..000000000 --- a/halo-testing-logs/halo_test_3709442.err +++ /dev/null @@ -1,37 +0,0 @@ - WARNING: Value for scheme.platlib does not match. Please report this to - distutils: /u/ethanmr3/.local/lib/python3.9/site-packages - sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages - WARNING: Additional context: - user = True - home = None - root = None - prefix = None - WARNING: Value for scheme.platlib does not match. Please report this to - distutils: /u/ethanmr3/.local/lib/python3.9/site-packages - sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages - WARNING: Additional context: - user = True - home = None - root = None - prefix = None - WARNING: Value for scheme.platlib does not match. Please report this to - distutils: /u/ethanmr3/.local/lib/python3.9/site-packages - sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages - WARNING: Additional context: - user = True - home = None - root = None - prefix = None -ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. -pyhealth 1.1.4 requires pandas<2,>=1.3.2, but you have pandas 2.3.1 which is incompatible. - 0%| | 0/58976 [00:00 - model.train() - File "/u/ethanmr3/halo/PyHealth/pyhealth/models/generators/halo.py", line 108, in train - checkpoint = torch.load(f'{self.save_dir}halo_model', map_location=torch.device(self.device)) - File "/u/ethanmr3/.local/lib/python3.9/site-packages/torch/serialization.py", line 1486, in load - with _open_zipfile_reader(opened_file) as opened_zipfile: - File "/u/ethanmr3/.local/lib/python3.9/site-packages/torch/serialization.py", line 771, in __init__ - super().__init__(torch._C.PyTorchFileReader(name_or_buffer)) -RuntimeError: PytorchStreamReader failed reading zip archive: failed finding central directory diff --git a/halo-testing-logs/halo_test_3709442.out b/halo-testing-logs/halo_test_3709442.out deleted file mode 100644 index 2f98c5181..000000000 --- a/halo-testing-logs/halo_test_3709442.out +++ /dev/null @@ -1,178 +0,0 @@ -SLURM_JOB_ID: 3709442 -SLURM_JOB_NODELIST: ccc0388 -SLURM_NTASKS: -SLURM_CPUS_ON_NODE: 4 -SLURM_GPUS_ON_NODE: 1 -SLURM_GPUS: -CUDA_VISIBLE_DEVICES: 0 -Running nvidia-smi to confirm GPU availability: -Sat Jul 19 23:14:02 2025 -+-----------------------------------------------------------------------------------------+ -| NVIDIA-SMI 575.57.08 Driver Version: 575.57.08 CUDA Version: 12.9 | -|-----------------------------------------+------------------------+----------------------+ -| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC | -| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. | -| | | MIG M. | -|=========================================+========================+======================| -| 0 NVIDIA A100-SXM4-80GB Off | 00000000:81:00.0 Off | 0 | -| N/A 31C P0 62W / 500W | 0MiB / 81920MiB | 0% Default | -| | | Disabled | -+-----------------------------------------+------------------------+----------------------+ - -+-----------------------------------------------------------------------------------------+ -| Processes: | -| GPU GI CI PID Type Process name GPU Memory | -| ID ID Usage | -|=========================================================================================| -| No running processes found | -+-----------------------------------------------------------------------------------------+ -Defaulting to user installation because normal site-packages is not writeable -Obtaining file:///u/ethanmr3/halo/PyHealth -Requirement already satisfied: torch>=1.8.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2.7.1) -Requirement already satisfied: torchvision>=0.9.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (0.22.1) -Requirement already satisfied: rdkit>=2022.03.4 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2025.3.3) -Requirement already satisfied: scikit-learn>=0.24.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.6.1) -Requirement already satisfied: networkx>=2.6.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (3.2.1) -Collecting pandas<2,>=1.3.2 - Using cached pandas-1.5.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.2 MB) -Requirement already satisfied: pandarallel>=1.5.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.6.5) -Requirement already satisfied: mne>=1.0.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.8.0) -Requirement already satisfied: urllib3<=1.26.15 in /usr/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.26.5) -Requirement already satisfied: numpy in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2.0.2) -Requirement already satisfied: tqdm in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (4.67.1) -Requirement already satisfied: polars in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.31.0) -Requirement already satisfied: transformers in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (4.52.4) -Requirement already satisfied: packaging in /usr/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (20.9) -Requirement already satisfied: scipy>=1.9 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (1.13.1) -Requirement already satisfied: decorator in /usr/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (4.4.2) -Requirement already satisfied: matplotlib>=3.6 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (3.9.4) -Requirement already satisfied: pooch>=1.5 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (1.8.2) -Requirement already satisfied: lazy-loader>=0.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (0.4) -Requirement already satisfied: jinja2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (3.1.6) -Requirement already satisfied: kiwisolver>=1.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.4.7) -Requirement already satisfied: fonttools>=4.22.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (4.58.4) -Requirement already satisfied: python-dateutil>=2.7 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (2.9.0.post0) -Requirement already satisfied: importlib-resources>=3.2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (6.5.2) -Requirement already satisfied: cycler>=0.10 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (0.12.1) -Requirement already satisfied: pillow>=8 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (11.2.1) -Requirement already satisfied: contourpy>=1.0.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.3.0) -Requirement already satisfied: pyparsing>=2.3.1 in /usr/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (2.4.7) -Requirement already satisfied: zipp>=3.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from importlib-resources>=3.2.0->matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (3.23.0) -Requirement already satisfied: dill>=0.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandarallel>=1.5.3->pyhealth==1.1.4) (0.4.0) -Requirement already satisfied: psutil in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandarallel>=1.5.3->pyhealth==1.1.4) (7.0.0) -Requirement already satisfied: pytz>=2020.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandas<2,>=1.3.2->pyhealth==1.1.4) (2025.2) -Requirement already satisfied: requests>=2.19.0 in /usr/lib/python3.9/site-packages (from pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (2.25.1) -Requirement already satisfied: platformdirs>=2.5.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (4.3.8) -Requirement already satisfied: six>=1.5 in /u/ethanmr3/.local/lib/python3.9/site-packages (from python-dateutil>=2.7->matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.17.0) -Requirement already satisfied: idna<3,>=2.5 in /usr/lib/python3.9/site-packages (from requests>=2.19.0->pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (2.10) -Requirement already satisfied: chardet<5,>=3.0.2 in /usr/lib/python3.9/site-packages (from requests>=2.19.0->pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (4.0.0) -Requirement already satisfied: joblib>=1.2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from scikit-learn>=0.24.2->pyhealth==1.1.4) (1.5.1) -Requirement already satisfied: threadpoolctl>=3.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from scikit-learn>=0.24.2->pyhealth==1.1.4) (3.6.0) -Requirement already satisfied: nvidia-cusparse-cu12==12.5.4.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.5.4.2) -Requirement already satisfied: nvidia-nvjitlink-cu12==12.6.85 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.85) -Requirement already satisfied: triton==3.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (3.3.1) -Requirement already satisfied: nvidia-cufile-cu12==1.11.1.6 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (1.11.1.6) -Requirement already satisfied: nvidia-cusparselt-cu12==0.6.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (0.6.3) -Requirement already satisfied: nvidia-cudnn-cu12==9.5.1.17 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (9.5.1.17) -Requirement already satisfied: nvidia-cuda-cupti-cu12==12.6.80 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.80) -Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) -Requirement already satisfied: typing-extensions>=4.10.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (4.14.0) -Requirement already satisfied: nvidia-cuda-runtime-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) -Requirement already satisfied: nvidia-nvtx-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) -Requirement already satisfied: nvidia-curand-cu12==10.3.7.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (10.3.7.77) -Requirement already satisfied: sympy>=1.13.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (1.14.0) -Requirement already satisfied: fsspec in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (2025.5.1) -Requirement already satisfied: nvidia-cublas-cu12==12.6.4.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.4.1) -Requirement already satisfied: nvidia-cusolver-cu12==11.7.1.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (11.7.1.2) -Requirement already satisfied: nvidia-cufft-cu12==11.3.0.4 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (11.3.0.4) -Requirement already satisfied: filelock in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (3.18.0) -Requirement already satisfied: nvidia-nccl-cu12==2.26.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (2.26.2) -Requirement already satisfied: setuptools>=40.8.0 in /usr/lib/python3.9/site-packages (from triton==3.3.1->torch>=1.8.0->pyhealth==1.1.4) (53.0.0) -Requirement already satisfied: mpmath<1.4,>=1.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from sympy>=1.13.3->torch>=1.8.0->pyhealth==1.1.4) (1.3.0) -Requirement already satisfied: MarkupSafe>=2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from jinja2->mne>=1.0.3->pyhealth==1.1.4) (3.0.2) -Requirement already satisfied: safetensors>=0.4.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.5.3) -Requirement already satisfied: tokenizers<0.22,>=0.21 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.21.1) -Requirement already satisfied: regex!=2019.12.17 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (2024.11.6) -Requirement already satisfied: pyyaml>=5.1 in /usr/lib64/python3.9/site-packages (from transformers->pyhealth==1.1.4) (5.4.1) -Requirement already satisfied: huggingface-hub<1.0,>=0.30.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.33.0) -Requirement already satisfied: hf-xet<2.0.0,>=1.1.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from huggingface-hub<1.0,>=0.30.0->transformers->pyhealth==1.1.4) (1.1.5) -Installing collected packages: pandas, pyhealth - Attempting uninstall: pandas - Found existing installation: pandas 2.3.1 - Uninstalling pandas-2.3.1: - Successfully uninstalled pandas-2.3.1 - Attempting uninstall: pyhealth - Found existing installation: pyhealth 1.1.4 - Uninstalling pyhealth-1.1.4: - Successfully uninstalled pyhealth-1.1.4 - Running setup.py develop for pyhealth -Successfully installed pandas-1.5.3 pyhealth -Defaulting to user installation because normal site-packages is not writeable -Collecting numpy - Using cached numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (19.5 MB) -Installing collected packages: numpy - Attempting uninstall: numpy - Found existing installation: numpy 2.0.2 - Uninstalling numpy-2.0.2: - Successfully uninstalled numpy-2.0.2 -Successfully installed numpy-2.0.2 -Defaulting to user installation because normal site-packages is not writeable -Collecting pandas - Using cached pandas-2.3.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.4 MB) -Collecting pytz>=2020.1 - Using cached pytz-2025.2-py2.py3-none-any.whl (509 kB) -Collecting tzdata>=2022.7 - Using cached tzdata-2025.2-py2.py3-none-any.whl (347 kB) -Collecting numpy>=1.22.4 - Using cached numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (19.5 MB) -Collecting python-dateutil>=2.8.2 - Using cached python_dateutil-2.9.0.post0-py2.py3-none-any.whl (229 kB) -Collecting six>=1.5 - Using cached six-1.17.0-py2.py3-none-any.whl (11 kB) -Installing collected packages: six, tzdata, pytz, python-dateutil, numpy, pandas - Attempting uninstall: six - Found existing installation: six 1.17.0 - Uninstalling six-1.17.0: - Successfully uninstalled six-1.17.0 - Attempting uninstall: tzdata - Found existing installation: tzdata 2025.2 - Uninstalling tzdata-2025.2: - Successfully uninstalled tzdata-2025.2 - Attempting uninstall: pytz - Found existing installation: pytz 2025.2 - Uninstalling pytz-2025.2: - Successfully uninstalled pytz-2025.2 - Attempting uninstall: python-dateutil - Found existing installation: python-dateutil 2.9.0.post0 - Uninstalling python-dateutil-2.9.0.post0: - Successfully uninstalled python-dateutil-2.9.0.post0 - Attempting uninstall: numpy - Found existing installation: numpy 2.0.2 - Uninstalling numpy-2.0.2: - Successfully uninstalled numpy-2.0.2 - Attempting uninstall: pandas - Found existing installation: pandas 1.5.3 - Uninstalling pandas-1.5.3: - Successfully uninstalled pandas-1.5.3 -Successfully installed numpy-2.0.2 pandas-2.3.1 python-dateutil-2.9.0.post0 pytz-2025.2 six-1.17.0 tzdata-2025.2 -BEGIN: Testing -Success on pip install -e . -Sucess on imports -Operating in dir: /u/ethanmr3/halo/PyHealth -VOCAB SIZE: 6984 -MAX LEN: 42 -AVG LEN: 1.267755803955288 -MAX VISIT LEN: 39 -AVG VISIT LEN: 11.037371134020619 -NUM RECORDS: 46520 -NUM LONGITUDINAL RECORDS: 7537 -Saving Everything -6984 -test data head 2: -[{'visits': [[5040, 3617, 5458, 5984]], 'labels': array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., - 0., 0., 0., 0., 0., 0., 0., 0.])}, {'visits': [[3686, 2887, 1319, 6919, 3596, 4334, 950, 636, 6847], [4478, 3972, 1029, 3980, 1555, 6812, 1180, 4126, 3101, 286, 6054, 5161, 1072, 5680, 692, 4673, 2768, 2770, 3283, 4445, 3454, 2787, 3710, 2667, 3311, 757, 2678, 5758], [1158, 6919, 5386, 11, 4620, 3607, 1434, 4764, 1184, 3616, 289, 6201, 5319, 2768, 3283, 4568, 3422, 2787, 1252, 3052, 4078, 2678, 6268], [2817, 5890, 1158, 11, 6797, 786, 2066, 153, 6938, 1180, 1053, 939, 950, 2487, 3521, 4673, 5707, 2768, 6101, 5333, 2906, 5087, 2787, 2661, 2156, 4204, 4078, 6767, 5494, 2678, 4478]], 'labels': array([1., 1., 1., 1., 1., 0., 0., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., - 1., 0., 0., 0., 1., 1., 1., 0.])}, {'visits': [[2444, 717, 3052, 274, 693, 4790]], 'labels': array([0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 1., 1., 1., 0., 0., 0., - 0., 0., 0., 0., 0., 0., 0., 0.])}, {'visits': [[6626, 4739, 5924, 4292, 5645, 494, 1072, 5008, 6259]], 'labels': array([0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., - 0., 0., 0., 0., 1., 1., 0., 1.])}, {'visits': [[4298, 5515, 1540]], 'labels': array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., - 0., 0., 0., 0., 0., 0., 0., 0.])}] -Success on model setup diff --git a/halo-testing-logs/halo_test_3709762.err b/halo-testing-logs/halo_test_3709762.err deleted file mode 100644 index 30dbe2ee6..000000000 --- a/halo-testing-logs/halo_test_3709762.err +++ /dev/null @@ -1,28 +0,0 @@ - WARNING: Value for scheme.platlib does not match. Please report this to - distutils: /u/ethanmr3/.local/lib/python3.9/site-packages - sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages - WARNING: Additional context: - user = True - home = None - root = None - prefix = None - WARNING: Value for scheme.platlib does not match. Please report this to - distutils: /u/ethanmr3/.local/lib/python3.9/site-packages - sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages - WARNING: Additional context: - user = True - home = None - root = None - prefix = None - WARNING: Value for scheme.platlib does not match. Please report this to - distutils: /u/ethanmr3/.local/lib/python3.9/site-packages - sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages - WARNING: Additional context: - user = True - home = None - root = None - prefix = None -ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. -pyhealth 1.1.4 requires pandas<2,>=1.3.2, but you have pandas 2.3.1 which is incompatible. - 0%| | 0/58976 [00:00=1.8.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2.7.1) -Requirement already satisfied: torchvision>=0.9.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (0.22.1) -Requirement already satisfied: rdkit>=2022.03.4 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2025.3.3) -Requirement already satisfied: scikit-learn>=0.24.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.6.1) -Requirement already satisfied: networkx>=2.6.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (3.2.1) -Collecting pandas<2,>=1.3.2 - Using cached pandas-1.5.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.2 MB) -Requirement already satisfied: pandarallel>=1.5.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.6.5) -Requirement already satisfied: mne>=1.0.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.8.0) -Requirement already satisfied: urllib3<=1.26.15 in /usr/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.26.5) -Requirement already satisfied: numpy in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2.0.2) -Requirement already satisfied: tqdm in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (4.67.1) -Requirement already satisfied: polars in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.31.0) -Requirement already satisfied: transformers in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (4.52.4) -Requirement already satisfied: packaging in /usr/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (20.9) -Requirement already satisfied: pooch>=1.5 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (1.8.2) -Requirement already satisfied: matplotlib>=3.6 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (3.9.4) -Requirement already satisfied: jinja2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (3.1.6) -Requirement already satisfied: scipy>=1.9 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (1.13.1) -Requirement already satisfied: decorator in /usr/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (4.4.2) -Requirement already satisfied: lazy-loader>=0.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (0.4) -Requirement already satisfied: kiwisolver>=1.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.4.7) -Requirement already satisfied: contourpy>=1.0.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.3.0) -Requirement already satisfied: pillow>=8 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (11.2.1) -Requirement already satisfied: fonttools>=4.22.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (4.58.4) -Requirement already satisfied: cycler>=0.10 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (0.12.1) -Requirement already satisfied: python-dateutil>=2.7 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (2.9.0.post0) -Requirement already satisfied: pyparsing>=2.3.1 in /usr/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (2.4.7) -Requirement already satisfied: importlib-resources>=3.2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (6.5.2) -Requirement already satisfied: zipp>=3.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from importlib-resources>=3.2.0->matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (3.23.0) -Requirement already satisfied: dill>=0.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandarallel>=1.5.3->pyhealth==1.1.4) (0.4.0) -Requirement already satisfied: psutil in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandarallel>=1.5.3->pyhealth==1.1.4) (7.0.0) -Requirement already satisfied: pytz>=2020.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandas<2,>=1.3.2->pyhealth==1.1.4) (2025.2) -Requirement already satisfied: requests>=2.19.0 in /usr/lib/python3.9/site-packages (from pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (2.25.1) -Requirement already satisfied: platformdirs>=2.5.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (4.3.8) -Requirement already satisfied: six>=1.5 in /u/ethanmr3/.local/lib/python3.9/site-packages (from python-dateutil>=2.7->matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.17.0) -Requirement already satisfied: chardet<5,>=3.0.2 in /usr/lib/python3.9/site-packages (from requests>=2.19.0->pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (4.0.0) -Requirement already satisfied: idna<3,>=2.5 in /usr/lib/python3.9/site-packages (from requests>=2.19.0->pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (2.10) -Requirement already satisfied: threadpoolctl>=3.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from scikit-learn>=0.24.2->pyhealth==1.1.4) (3.6.0) -Requirement already satisfied: joblib>=1.2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from scikit-learn>=0.24.2->pyhealth==1.1.4) (1.5.1) -Requirement already satisfied: nvidia-cuda-runtime-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) -Requirement already satisfied: nvidia-cufft-cu12==11.3.0.4 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (11.3.0.4) -Requirement already satisfied: nvidia-cufile-cu12==1.11.1.6 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (1.11.1.6) -Requirement already satisfied: typing-extensions>=4.10.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (4.14.0) -Requirement already satisfied: triton==3.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (3.3.1) -Requirement already satisfied: nvidia-curand-cu12==10.3.7.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (10.3.7.77) -Requirement already satisfied: filelock in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (3.18.0) -Requirement already satisfied: nvidia-nvjitlink-cu12==12.6.85 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.85) -Requirement already satisfied: fsspec in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (2025.5.1) -Requirement already satisfied: nvidia-cusolver-cu12==11.7.1.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (11.7.1.2) -Requirement already satisfied: nvidia-cudnn-cu12==9.5.1.17 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (9.5.1.17) -Requirement already satisfied: nvidia-cublas-cu12==12.6.4.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.4.1) -Requirement already satisfied: sympy>=1.13.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (1.14.0) -Requirement already satisfied: nvidia-nccl-cu12==2.26.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (2.26.2) -Requirement already satisfied: nvidia-cusparse-cu12==12.5.4.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.5.4.2) -Requirement already satisfied: nvidia-cuda-cupti-cu12==12.6.80 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.80) -Requirement already satisfied: nvidia-cusparselt-cu12==0.6.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (0.6.3) -Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) -Requirement already satisfied: nvidia-nvtx-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) -Requirement already satisfied: setuptools>=40.8.0 in /usr/lib/python3.9/site-packages (from triton==3.3.1->torch>=1.8.0->pyhealth==1.1.4) (53.0.0) -Requirement already satisfied: mpmath<1.4,>=1.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from sympy>=1.13.3->torch>=1.8.0->pyhealth==1.1.4) (1.3.0) -Requirement already satisfied: MarkupSafe>=2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from jinja2->mne>=1.0.3->pyhealth==1.1.4) (3.0.2) -Requirement already satisfied: safetensors>=0.4.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.5.3) -Requirement already satisfied: huggingface-hub<1.0,>=0.30.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.33.0) -Requirement already satisfied: tokenizers<0.22,>=0.21 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.21.1) -Requirement already satisfied: regex!=2019.12.17 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (2024.11.6) -Requirement already satisfied: pyyaml>=5.1 in /usr/lib64/python3.9/site-packages (from transformers->pyhealth==1.1.4) (5.4.1) -Requirement already satisfied: hf-xet<2.0.0,>=1.1.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from huggingface-hub<1.0,>=0.30.0->transformers->pyhealth==1.1.4) (1.1.5) -Installing collected packages: pandas, pyhealth - Attempting uninstall: pandas - Found existing installation: pandas 2.3.1 - Uninstalling pandas-2.3.1: - Successfully uninstalled pandas-2.3.1 - Attempting uninstall: pyhealth - Found existing installation: pyhealth 1.1.4 - Uninstalling pyhealth-1.1.4: - Successfully uninstalled pyhealth-1.1.4 - Running setup.py develop for pyhealth -Successfully installed pandas-1.5.3 pyhealth -Defaulting to user installation because normal site-packages is not writeable -Collecting numpy - Using cached numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (19.5 MB) -Installing collected packages: numpy - Attempting uninstall: numpy - Found existing installation: numpy 2.0.2 - Uninstalling numpy-2.0.2: - Successfully uninstalled numpy-2.0.2 -Successfully installed numpy-2.0.2 -Defaulting to user installation because normal site-packages is not writeable -Collecting pandas - Using cached pandas-2.3.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.4 MB) -Collecting python-dateutil>=2.8.2 - Using cached python_dateutil-2.9.0.post0-py2.py3-none-any.whl (229 kB) -Collecting numpy>=1.22.4 - Using cached numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (19.5 MB) -Collecting pytz>=2020.1 - Using cached pytz-2025.2-py2.py3-none-any.whl (509 kB) -Collecting tzdata>=2022.7 - Using cached tzdata-2025.2-py2.py3-none-any.whl (347 kB) -Collecting six>=1.5 - Using cached six-1.17.0-py2.py3-none-any.whl (11 kB) -Installing collected packages: six, tzdata, pytz, python-dateutil, numpy, pandas - Attempting uninstall: six - Found existing installation: six 1.17.0 - Uninstalling six-1.17.0: - Successfully uninstalled six-1.17.0 - Attempting uninstall: tzdata - Found existing installation: tzdata 2025.2 - Uninstalling tzdata-2025.2: - Successfully uninstalled tzdata-2025.2 - Attempting uninstall: pytz - Found existing installation: pytz 2025.2 - Uninstalling pytz-2025.2: - Successfully uninstalled pytz-2025.2 - Attempting uninstall: python-dateutil - Found existing installation: python-dateutil 2.9.0.post0 - Uninstalling python-dateutil-2.9.0.post0: - Successfully uninstalled python-dateutil-2.9.0.post0 - Attempting uninstall: numpy - Found existing installation: numpy 2.0.2 - Uninstalling numpy-2.0.2: - Successfully uninstalled numpy-2.0.2 - Attempting uninstall: pandas - Found existing installation: pandas 1.5.3 - Uninstalling pandas-1.5.3: - Successfully uninstalled pandas-1.5.3 -Successfully installed numpy-2.0.2 pandas-2.3.1 python-dateutil-2.9.0.post0 pytz-2025.2 six-1.17.0 tzdata-2025.2 -BEGIN: Testing -Success on pip install -e . -Sucess on imports -Operating in dir: /u/ethanmr3/halo/PyHealth -VOCAB SIZE: 6984 -MAX LEN: 42 -AVG LEN: 1.267755803955288 -MAX VISIT LEN: 39 -AVG VISIT LEN: 11.037371134020619 -NUM RECORDS: 46520 -NUM LONGITUDINAL RECORDS: 7537 -Saving Everything -6984 -test data head 2: -[{'visits': [[3625, 827, 6510, 1663]], 'labels': array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., - 0., 0., 0., 0., 0., 0., 0., 0.])}, {'visits': [[5856, 673, 1280, 3459, 3020, 3121, 4337, 119, 6104], [3714, 901, 1807, 3351, 4385, 5410, 292, 5030, 5544, 3756, 940, 1603, 5318, 2249, 2252, 3277, 4686, 5842, 5586, 606, 2271, 4578, 3582, 4849, 5112, 505, 4221, 5374], [129, 6662, 4359, 650, 4748, 796, 2850, 940, 2353, 3121, 4145, 820, 5176, 4666, 3776, 1603, 2123, 3277, 6607, 3295, 6884, 3698, 505], [6658, 6150, 263, 1807, 2961, 5778, 3351, 2712, 5787, 796, 2464, 673, 2725, 940, 5177, 1603, 3012, 967, 74, 2123, 592, 5842, 6108, 2275, 4967, 505, 3698, 2169, 6138, 4222, 5247]], 'labels': array([1., 1., 1., 1., 1., 0., 0., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., - 1., 0., 0., 0., 1., 1., 1., 0.])}, {'visits': [[3776, 489, 1902, 4113, 598, 4183]], 'labels': array([0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 1., 1., 1., 0., 0., 0., - 0., 0., 0., 0., 0., 0., 0., 0.])}, {'visits': [[1152, 4130, 2596, 5956, 2252, 111, 1776, 2708, 4062]], 'labels': array([0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., - 0., 0., 0., 0., 1., 1., 0., 1.])}, {'visits': [[6844, 1222, 2903]], 'labels': array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., - 0., 0., 0., 0., 0., 0., 0., 0.])}] -Success on model setup diff --git a/halo-testing-logs/halo_test_3713150.err b/halo-testing-logs/halo_test_3713150.err deleted file mode 100644 index 26bdd2dd4..000000000 --- a/halo-testing-logs/halo_test_3713150.err +++ /dev/null @@ -1,27 +0,0 @@ - WARNING: Value for scheme.platlib does not match. Please report this to - distutils: /u/ethanmr3/.local/lib/python3.9/site-packages - sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages - WARNING: Additional context: - user = True - home = None - root = None - prefix = None - WARNING: Value for scheme.platlib does not match. Please report this to - distutils: /u/ethanmr3/.local/lib/python3.9/site-packages - sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages - WARNING: Additional context: - user = True - home = None - root = None - prefix = None - WARNING: Value for scheme.platlib does not match. Please report this to - distutils: /u/ethanmr3/.local/lib/python3.9/site-packages - sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages - WARNING: Additional context: - user = True - home = None - root = None - prefix = None -ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. -pyhealth 1.1.4 requires pandas<2,>=1.3.2, but you have pandas 2.3.1 which is incompatible. - 0%| | 0/58976 [00:00=1.8.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2.7.1) -Requirement already satisfied: torchvision>=0.9.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (0.22.1) -Requirement already satisfied: rdkit>=2022.03.4 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2025.3.3) -Requirement already satisfied: scikit-learn>=0.24.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.6.1) -Requirement already satisfied: networkx>=2.6.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (3.2.1) -Collecting pandas<2,>=1.3.2 - Using cached pandas-1.5.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.2 MB) -Requirement already satisfied: pandarallel>=1.5.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.6.5) -Requirement already satisfied: mne>=1.0.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.8.0) -Requirement already satisfied: urllib3<=1.26.15 in /usr/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.26.5) -Requirement already satisfied: numpy in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2.0.2) -Requirement already satisfied: tqdm in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (4.67.1) -Requirement already satisfied: polars in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.31.0) -Requirement already satisfied: transformers in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (4.52.4) -Requirement already satisfied: lazy-loader>=0.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (0.4) -Requirement already satisfied: jinja2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (3.1.6) -Requirement already satisfied: packaging in /usr/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (20.9) -Requirement already satisfied: decorator in /usr/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (4.4.2) -Requirement already satisfied: scipy>=1.9 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (1.13.1) -Requirement already satisfied: matplotlib>=3.6 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (3.9.4) -Requirement already satisfied: pooch>=1.5 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (1.8.2) -Requirement already satisfied: pyparsing>=2.3.1 in /usr/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (2.4.7) -Requirement already satisfied: contourpy>=1.0.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.3.0) -Requirement already satisfied: importlib-resources>=3.2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (6.5.2) -Requirement already satisfied: pillow>=8 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (11.2.1) -Requirement already satisfied: kiwisolver>=1.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.4.7) -Requirement already satisfied: fonttools>=4.22.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (4.58.4) -Requirement already satisfied: python-dateutil>=2.7 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (2.9.0.post0) -Requirement already satisfied: cycler>=0.10 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (0.12.1) -Requirement already satisfied: zipp>=3.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from importlib-resources>=3.2.0->matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (3.23.0) -Requirement already satisfied: dill>=0.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandarallel>=1.5.3->pyhealth==1.1.4) (0.4.0) -Requirement already satisfied: psutil in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandarallel>=1.5.3->pyhealth==1.1.4) (7.0.0) -Requirement already satisfied: pytz>=2020.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandas<2,>=1.3.2->pyhealth==1.1.4) (2025.2) -Requirement already satisfied: platformdirs>=2.5.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (4.3.8) -Requirement already satisfied: requests>=2.19.0 in /usr/lib/python3.9/site-packages (from pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (2.25.1) -Requirement already satisfied: six>=1.5 in /u/ethanmr3/.local/lib/python3.9/site-packages (from python-dateutil>=2.7->matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.17.0) -Requirement already satisfied: chardet<5,>=3.0.2 in /usr/lib/python3.9/site-packages (from requests>=2.19.0->pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (4.0.0) -Requirement already satisfied: idna<3,>=2.5 in /usr/lib/python3.9/site-packages (from requests>=2.19.0->pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (2.10) -Requirement already satisfied: joblib>=1.2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from scikit-learn>=0.24.2->pyhealth==1.1.4) (1.5.1) -Requirement already satisfied: threadpoolctl>=3.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from scikit-learn>=0.24.2->pyhealth==1.1.4) (3.6.0) -Requirement already satisfied: sympy>=1.13.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (1.14.0) -Requirement already satisfied: nvidia-nvtx-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) -Requirement already satisfied: nvidia-cuda-runtime-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) -Requirement already satisfied: nvidia-cublas-cu12==12.6.4.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.4.1) -Requirement already satisfied: nvidia-curand-cu12==10.3.7.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (10.3.7.77) -Requirement already satisfied: nvidia-cufile-cu12==1.11.1.6 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (1.11.1.6) -Requirement already satisfied: typing-extensions>=4.10.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (4.14.0) -Requirement already satisfied: nvidia-cufft-cu12==11.3.0.4 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (11.3.0.4) -Requirement already satisfied: nvidia-cudnn-cu12==9.5.1.17 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (9.5.1.17) -Requirement already satisfied: nvidia-cusparselt-cu12==0.6.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (0.6.3) -Requirement already satisfied: nvidia-cusparse-cu12==12.5.4.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.5.4.2) -Requirement already satisfied: fsspec in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (2025.5.1) -Requirement already satisfied: filelock in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (3.18.0) -Requirement already satisfied: nvidia-cusolver-cu12==11.7.1.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (11.7.1.2) -Requirement already satisfied: triton==3.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (3.3.1) -Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) -Requirement already satisfied: nvidia-cuda-cupti-cu12==12.6.80 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.80) -Requirement already satisfied: nvidia-nccl-cu12==2.26.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (2.26.2) -Requirement already satisfied: nvidia-nvjitlink-cu12==12.6.85 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.85) -Requirement already satisfied: setuptools>=40.8.0 in /usr/lib/python3.9/site-packages (from triton==3.3.1->torch>=1.8.0->pyhealth==1.1.4) (53.0.0) -Requirement already satisfied: mpmath<1.4,>=1.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from sympy>=1.13.3->torch>=1.8.0->pyhealth==1.1.4) (1.3.0) -Requirement already satisfied: MarkupSafe>=2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from jinja2->mne>=1.0.3->pyhealth==1.1.4) (3.0.2) -Requirement already satisfied: safetensors>=0.4.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.5.3) -Requirement already satisfied: tokenizers<0.22,>=0.21 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.21.1) -Requirement already satisfied: pyyaml>=5.1 in /usr/lib64/python3.9/site-packages (from transformers->pyhealth==1.1.4) (5.4.1) -Requirement already satisfied: regex!=2019.12.17 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (2024.11.6) -Requirement already satisfied: huggingface-hub<1.0,>=0.30.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.33.0) -Requirement already satisfied: hf-xet<2.0.0,>=1.1.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from huggingface-hub<1.0,>=0.30.0->transformers->pyhealth==1.1.4) (1.1.5) -Installing collected packages: pandas, pyhealth - Attempting uninstall: pandas - Found existing installation: pandas 2.3.1 - Uninstalling pandas-2.3.1: - Successfully uninstalled pandas-2.3.1 - Attempting uninstall: pyhealth - Found existing installation: pyhealth 1.1.4 - Uninstalling pyhealth-1.1.4: - Successfully uninstalled pyhealth-1.1.4 - Running setup.py develop for pyhealth -Successfully installed pandas-1.5.3 pyhealth -Defaulting to user installation because normal site-packages is not writeable -Collecting numpy - Using cached numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (19.5 MB) -Installing collected packages: numpy - Attempting uninstall: numpy - Found existing installation: numpy 2.0.2 - Uninstalling numpy-2.0.2: - Successfully uninstalled numpy-2.0.2 -Successfully installed numpy-2.0.2 -Defaulting to user installation because normal site-packages is not writeable -Collecting pandas - Using cached pandas-2.3.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.4 MB) -Collecting python-dateutil>=2.8.2 - Using cached python_dateutil-2.9.0.post0-py2.py3-none-any.whl (229 kB) -Collecting numpy>=1.22.4 - Using cached numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (19.5 MB) -Collecting tzdata>=2022.7 - Using cached tzdata-2025.2-py2.py3-none-any.whl (347 kB) -Collecting pytz>=2020.1 - Using cached pytz-2025.2-py2.py3-none-any.whl (509 kB) -Collecting six>=1.5 - Using cached six-1.17.0-py2.py3-none-any.whl (11 kB) -Installing collected packages: six, tzdata, pytz, python-dateutil, numpy, pandas - Attempting uninstall: six - Found existing installation: six 1.17.0 - Uninstalling six-1.17.0: - Successfully uninstalled six-1.17.0 - Attempting uninstall: tzdata - Found existing installation: tzdata 2025.2 - Uninstalling tzdata-2025.2: - Successfully uninstalled tzdata-2025.2 - Attempting uninstall: pytz - Found existing installation: pytz 2025.2 - Uninstalling pytz-2025.2: - Successfully uninstalled pytz-2025.2 - Attempting uninstall: python-dateutil - Found existing installation: python-dateutil 2.9.0.post0 - Uninstalling python-dateutil-2.9.0.post0: - Successfully uninstalled python-dateutil-2.9.0.post0 - Attempting uninstall: numpy - Found existing installation: numpy 2.0.2 - Uninstalling numpy-2.0.2: - Successfully uninstalled numpy-2.0.2 - Attempting uninstall: pandas - Found existing installation: pandas 1.5.3 - Uninstalling pandas-1.5.3: - Successfully uninstalled pandas-1.5.3 -Successfully installed numpy-2.0.2 pandas-2.3.1 python-dateutil-2.9.0.post0 pytz-2025.2 six-1.17.0 tzdata-2025.2 -BEGIN: Testing -Success on pip install -e . -Sucess on imports -Operating in dir: /u/ethanmr3/halo/PyHealth diff --git a/halo-testing-logs/halo_test_3713164.err b/halo-testing-logs/halo_test_3713164.err deleted file mode 100644 index 6a3300cd0..000000000 --- a/halo-testing-logs/halo_test_3713164.err +++ /dev/null @@ -1,30 +0,0 @@ - WARNING: Value for scheme.platlib does not match. Please report this to - distutils: /u/ethanmr3/.local/lib/python3.9/site-packages - sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages - WARNING: Additional context: - user = True - home = None - root = None - prefix = None - WARNING: Value for scheme.platlib does not match. Please report this to - distutils: /u/ethanmr3/.local/lib/python3.9/site-packages - sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages - WARNING: Additional context: - user = True - home = None - root = None - prefix = None - WARNING: Value for scheme.platlib does not match. Please report this to - distutils: /u/ethanmr3/.local/lib/python3.9/site-packages - sysconfig: /u/ethanmr3/.local/lib64/python3.9/site-packages - WARNING: Additional context: - user = True - home = None - root = None - prefix = None -ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. -pyhealth 1.1.4 requires pandas<2,>=1.3.2, but you have pandas 2.3.1 which is incompatible. - 0%| | 0/58976 [00:00=1.8.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2.7.1) -Requirement already satisfied: torchvision>=0.9.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (0.22.1) -Requirement already satisfied: rdkit>=2022.03.4 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2025.3.3) -Requirement already satisfied: scikit-learn>=0.24.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.6.1) -Requirement already satisfied: networkx>=2.6.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (3.2.1) -Collecting pandas<2,>=1.3.2 - Using cached pandas-1.5.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.2 MB) -Requirement already satisfied: pandarallel>=1.5.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.6.5) -Requirement already satisfied: mne>=1.0.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.8.0) -Requirement already satisfied: urllib3<=1.26.15 in /usr/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.26.5) -Requirement already satisfied: numpy in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (2.0.2) -Requirement already satisfied: tqdm in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (4.67.1) -Requirement already satisfied: polars in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (1.31.0) -Requirement already satisfied: transformers in /u/ethanmr3/.local/lib/python3.9/site-packages (from pyhealth==1.1.4) (4.52.4) -Requirement already satisfied: scipy>=1.9 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (1.13.1) -Requirement already satisfied: lazy-loader>=0.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (0.4) -Requirement already satisfied: matplotlib>=3.6 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (3.9.4) -Requirement already satisfied: packaging in /usr/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (20.9) -Requirement already satisfied: decorator in /usr/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (4.4.2) -Requirement already satisfied: jinja2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (3.1.6) -Requirement already satisfied: pooch>=1.5 in /u/ethanmr3/.local/lib/python3.9/site-packages (from mne>=1.0.3->pyhealth==1.1.4) (1.8.2) -Requirement already satisfied: python-dateutil>=2.7 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (2.9.0.post0) -Requirement already satisfied: pillow>=8 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (11.2.1) -Requirement already satisfied: pyparsing>=2.3.1 in /usr/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (2.4.7) -Requirement already satisfied: contourpy>=1.0.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.3.0) -Requirement already satisfied: fonttools>=4.22.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (4.58.4) -Requirement already satisfied: kiwisolver>=1.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.4.7) -Requirement already satisfied: cycler>=0.10 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (0.12.1) -Requirement already satisfied: importlib-resources>=3.2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (6.5.2) -Requirement already satisfied: zipp>=3.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from importlib-resources>=3.2.0->matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (3.23.0) -Requirement already satisfied: dill>=0.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandarallel>=1.5.3->pyhealth==1.1.4) (0.4.0) -Requirement already satisfied: psutil in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandarallel>=1.5.3->pyhealth==1.1.4) (7.0.0) -Requirement already satisfied: pytz>=2020.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pandas<2,>=1.3.2->pyhealth==1.1.4) (2025.2) -Requirement already satisfied: platformdirs>=2.5.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (4.3.8) -Requirement already satisfied: requests>=2.19.0 in /usr/lib/python3.9/site-packages (from pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (2.25.1) -Requirement already satisfied: six>=1.5 in /u/ethanmr3/.local/lib/python3.9/site-packages (from python-dateutil>=2.7->matplotlib>=3.6->mne>=1.0.3->pyhealth==1.1.4) (1.17.0) -Requirement already satisfied: chardet<5,>=3.0.2 in /usr/lib/python3.9/site-packages (from requests>=2.19.0->pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (4.0.0) -Requirement already satisfied: idna<3,>=2.5 in /usr/lib/python3.9/site-packages (from requests>=2.19.0->pooch>=1.5->mne>=1.0.3->pyhealth==1.1.4) (2.10) -Requirement already satisfied: threadpoolctl>=3.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from scikit-learn>=0.24.2->pyhealth==1.1.4) (3.6.0) -Requirement already satisfied: joblib>=1.2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from scikit-learn>=0.24.2->pyhealth==1.1.4) (1.5.1) -Requirement already satisfied: nvidia-cuda-runtime-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) -Requirement already satisfied: nvidia-cusolver-cu12==11.7.1.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (11.7.1.2) -Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) -Requirement already satisfied: triton==3.3.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (3.3.1) -Requirement already satisfied: nvidia-curand-cu12==10.3.7.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (10.3.7.77) -Requirement already satisfied: sympy>=1.13.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (1.14.0) -Requirement already satisfied: nvidia-nccl-cu12==2.26.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (2.26.2) -Requirement already satisfied: typing-extensions>=4.10.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (4.14.0) -Requirement already satisfied: nvidia-cusparse-cu12==12.5.4.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.5.4.2) -Requirement already satisfied: nvidia-cufile-cu12==1.11.1.6 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (1.11.1.6) -Requirement already satisfied: nvidia-nvjitlink-cu12==12.6.85 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.85) -Requirement already satisfied: nvidia-cudnn-cu12==9.5.1.17 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (9.5.1.17) -Requirement already satisfied: nvidia-nvtx-cu12==12.6.77 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.77) -Requirement already satisfied: nvidia-cusparselt-cu12==0.6.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (0.6.3) -Requirement already satisfied: nvidia-cuda-cupti-cu12==12.6.80 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.80) -Requirement already satisfied: fsspec in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (2025.5.1) -Requirement already satisfied: nvidia-cufft-cu12==11.3.0.4 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (11.3.0.4) -Requirement already satisfied: nvidia-cublas-cu12==12.6.4.1 in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (12.6.4.1) -Requirement already satisfied: filelock in /u/ethanmr3/.local/lib/python3.9/site-packages (from torch>=1.8.0->pyhealth==1.1.4) (3.18.0) -Requirement already satisfied: setuptools>=40.8.0 in /usr/lib/python3.9/site-packages (from triton==3.3.1->torch>=1.8.0->pyhealth==1.1.4) (53.0.0) -Requirement already satisfied: mpmath<1.4,>=1.1.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from sympy>=1.13.3->torch>=1.8.0->pyhealth==1.1.4) (1.3.0) -Requirement already satisfied: MarkupSafe>=2.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from jinja2->mne>=1.0.3->pyhealth==1.1.4) (3.0.2) -Requirement already satisfied: pyyaml>=5.1 in /usr/lib64/python3.9/site-packages (from transformers->pyhealth==1.1.4) (5.4.1) -Requirement already satisfied: huggingface-hub<1.0,>=0.30.0 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.33.0) -Requirement already satisfied: safetensors>=0.4.3 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.5.3) -Requirement already satisfied: regex!=2019.12.17 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (2024.11.6) -Requirement already satisfied: tokenizers<0.22,>=0.21 in /u/ethanmr3/.local/lib/python3.9/site-packages (from transformers->pyhealth==1.1.4) (0.21.1) -Requirement already satisfied: hf-xet<2.0.0,>=1.1.2 in /u/ethanmr3/.local/lib/python3.9/site-packages (from huggingface-hub<1.0,>=0.30.0->transformers->pyhealth==1.1.4) (1.1.5) -Installing collected packages: pandas, pyhealth - Attempting uninstall: pandas - Found existing installation: pandas 2.3.1 - Uninstalling pandas-2.3.1: - Successfully uninstalled pandas-2.3.1 - Attempting uninstall: pyhealth - Found existing installation: pyhealth 1.1.4 - Uninstalling pyhealth-1.1.4: - Successfully uninstalled pyhealth-1.1.4 - Running setup.py develop for pyhealth -Successfully installed pandas-1.5.3 pyhealth -Defaulting to user installation because normal site-packages is not writeable -Collecting numpy - Using cached numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (19.5 MB) -Installing collected packages: numpy - Attempting uninstall: numpy - Found existing installation: numpy 2.0.2 - Uninstalling numpy-2.0.2: - Successfully uninstalled numpy-2.0.2 -Successfully installed numpy-2.0.2 -Defaulting to user installation because normal site-packages is not writeable -Collecting pandas - Using cached pandas-2.3.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.4 MB) -Collecting tzdata>=2022.7 - Using cached tzdata-2025.2-py2.py3-none-any.whl (347 kB) -Collecting numpy>=1.22.4 - Using cached numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (19.5 MB) -Collecting pytz>=2020.1 - Using cached pytz-2025.2-py2.py3-none-any.whl (509 kB) -Collecting python-dateutil>=2.8.2 - Using cached python_dateutil-2.9.0.post0-py2.py3-none-any.whl (229 kB) -Collecting six>=1.5 - Using cached six-1.17.0-py2.py3-none-any.whl (11 kB) -Installing collected packages: six, tzdata, pytz, python-dateutil, numpy, pandas - Attempting uninstall: six - Found existing installation: six 1.17.0 - Uninstalling six-1.17.0: - Successfully uninstalled six-1.17.0 - Attempting uninstall: tzdata - Found existing installation: tzdata 2025.2 - Uninstalling tzdata-2025.2: - Successfully uninstalled tzdata-2025.2 - Attempting uninstall: pytz - Found existing installation: pytz 2025.2 - Uninstalling pytz-2025.2: - Successfully uninstalled pytz-2025.2 - Attempting uninstall: python-dateutil - Found existing installation: python-dateutil 2.9.0.post0 - Uninstalling python-dateutil-2.9.0.post0: - Successfully uninstalled python-dateutil-2.9.0.post0 - Attempting uninstall: numpy - Found existing installation: numpy 2.0.2 - Uninstalling numpy-2.0.2: - Successfully uninstalled numpy-2.0.2 - Attempting uninstall: pandas - Found existing installation: pandas 1.5.3 - Uninstalling pandas-1.5.3: - Successfully uninstalled pandas-1.5.3 -Successfully installed numpy-2.0.2 pandas-2.3.1 python-dateutil-2.9.0.post0 pytz-2025.2 six-1.17.0 tzdata-2025.2 -BEGIN: Testing -Success on pip install -e . -Sucess on imports -Operating in dir: /u/ethanmr3/halo/PyHealth -VOCAB SIZE: 6984 -MAX LEN: 42 -AVG LEN: 1.267755803955288 -MAX VISIT LEN: 39 -AVG VISIT LEN: 11.037371134020619 -NUM RECORDS: 46520 -NUM LONGITUDINAL RECORDS: 7537 -Saving Everything -6984 -test data head 2: -[{'visits': [[4185, 5429, 2182, 63]], 'labels': array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., - 0., 0., 0., 0., 0., 0., 0., 0.])}, {'visits': [[544, 6081, 1440, 5772, 5485, 5552, 6066, 1463, 1661], [1412, 2443, 5519, 3344, 1168, 1809, 1807, 3731, 6172, 4644, 804, 5676, 5947, 6077, 450, 324, 1989, 6471, 4170, 983, 224, 3557, 1637, 5737, 5354, 3185, 2546, 886], [4485, 6540, 1807, 2704, 544, 3750, 44, 5682, 2486, 3896, 184, 6471, 6608, 5329, 6226, 2903, 5719, 5737, 1773, 6384, 1522, 886, 6011], [6784, 2183, 6539, 270, 4623, 2704, 6801, 1807, 802, 5797, 6699, 4524, 5552, 4145, 5555, 950, 3896, 5947, 1217, 4679, 6983, 2903, 983, 2010, 3557, 4967, 5737, 886, 3322, 1916, 3454]], 'labels': array([1., 1., 1., 1., 1., 0., 0., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., - 1., 0., 0., 0., 1., 1., 1., 0.])}, {'visits': [[6600, 4105, 5329, 3513, 1596, 4765]], 'labels': array([0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 1., 1., 1., 0., 0., 0., - 0., 0., 0., 0., 0., 0., 0., 0.])}, {'visits': [[3008, 1989, 6214, 2535, 5162, 2870, 5208, 4953, 4540]], 'labels': array([0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., - 0., 0., 0., 0., 1., 1., 0., 1.])}, {'visits': [[4982, 1375, 2879]], 'labels': array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., - 0., 0., 0., 0., 0., 0., 0., 0.])}] -Success on model setup -Epoch 0, Iter 0: Training Loss:0.003035 -Epoch 0, Iter 24000: Training Loss:0.002145 -Epoch 0 Validation Loss:0.0002305 - ------------- Save best model ------------ - -Epoch 1, Iter 0: Training Loss:0.001832 -Epoch 1, Iter 24000: Training Loss:0.001805 -Epoch 1 Validation Loss:0.0002257 - ------------- Save best model ------------ - -Epoch 2, Iter 0: Training Loss:0.001855 -Epoch 2, Iter 24000: Training Loss:0.001980 -Epoch 2 Validation Loss:0.0002229 - ------------- Save best model ------------ - -Epoch 3, Iter 0: Training Loss:0.001336 -Epoch 3, Iter 24000: Training Loss:0.001547 -Epoch 3 Validation Loss:0.0002187 - ------------- Save best model ------------ - -Epoch 4, Iter 0: Training Loss:0.001439 -Epoch 4, Iter 24000: Training Loss:0.001622 -Epoch 4 Validation Loss:0.0002177 - ------------- Save best model ------------ - -Epoch 5, Iter 0: Training Loss:0.001393 -Epoch 5, Iter 24000: Training Loss:0.001476 -Epoch 5 Validation Loss:0.0002161 - ------------- Save best model ------------ - -Epoch 6, Iter 0: Training Loss:0.001374 -Epoch 6, Iter 24000: Training Loss:0.001541 -Epoch 6 Validation Loss:0.0002136 - ------------- Save best model ------------ - -Epoch 7, Iter 0: Training Loss:0.001676 -Epoch 7, Iter 24000: Training Loss:0.001921 -Epoch 7 Validation Loss:0.0002123 - ------------- Save best model ------------ - -Epoch 8, Iter 0: Training Loss:0.001604 -Epoch 8, Iter 24000: Training Loss:0.001605 -Epoch 8 Validation Loss:0.0002100 - ------------- Save best model ------------ - -Epoch 9, Iter 0: Training Loss:0.001878 -Epoch 9, Iter 24000: Training Loss:0.001517 -Epoch 9 Validation Loss:0.0002089 - ------------- Save best model ------------ - -Epoch 10, Iter 0: Training Loss:0.001447 -Epoch 10, Iter 24000: Training Loss:0.001354 -Epoch 10 Validation Loss:0.0002079 - ------------- Save best model ------------ - -Epoch 11, Iter 0: Training Loss:0.001610 -Epoch 11, Iter 24000: Training Loss:0.001414 -Epoch 11 Validation Loss:0.0002068 - ------------- Save best model ------------ - -Epoch 12, Iter 0: Training Loss:0.001676 -Epoch 12, Iter 24000: Training Loss:0.001512 -Epoch 12 Validation Loss:0.0002053 - ------------- Save best model ------------ - -Epoch 13, Iter 0: Training Loss:0.001766 -Epoch 13, Iter 24000: Training Loss:0.001499 -Epoch 13 Validation Loss:0.0002039 - ------------- Save best model ------------ - -Epoch 14, Iter 0: Training Loss:0.001794 -Epoch 14, Iter 24000: Training Loss:0.001430 -Epoch 14 Validation Loss:0.0002030 - ------------- Save best model ------------ - -Epoch 15, Iter 0: Training Loss:0.001648 -Epoch 15, Iter 24000: Training Loss:0.001802 -Epoch 15 Validation Loss:0.0002019 - ------------- Save best model ------------ - -Epoch 16, Iter 0: Training Loss:0.001942 -Epoch 16, Iter 24000: Training Loss:0.001631 -Epoch 16 Validation Loss:0.0002016 - ------------- Save best model ------------ - -Epoch 17, Iter 0: Training Loss:0.001647 -Epoch 17, Iter 24000: Training Loss:0.001143 -Epoch 17 Validation Loss:0.0002010 - ------------- Save best model ------------ - -Epoch 18, Iter 0: Training Loss:0.001599 -Epoch 18, Iter 24000: Training Loss:0.001771 -Epoch 18 Validation Loss:0.0002006 - ------------- Save best model ------------ - -Epoch 19, Iter 0: Training Loss:0.001910 -Epoch 19, Iter 24000: Training Loss:0.001550 -Epoch 19 Validation Loss:0.0002003 - ------------- Save best model ------------ - -Epoch 20, Iter 0: Training Loss:0.001422 -Epoch 20, Iter 24000: Training Loss:0.001366 -Epoch 20 Validation Loss:0.0002001 - ------------- Save best model ------------ - -Epoch 21, Iter 0: Training Loss:0.001291 -Epoch 21, Iter 24000: Training Loss:0.001427 -Epoch 21 Validation Loss:0.0001996 - ------------- Save best model ------------ - -Epoch 22, Iter 0: Training Loss:0.001385 -Epoch 22, Iter 24000: Training Loss:0.001431 -Epoch 22 Validation Loss:0.0001995 - ------------- Save best model ------------ - -Epoch 23, Iter 0: Training Loss:0.001395 -Epoch 23, Iter 24000: Training Loss:0.001319 -Epoch 23 Validation Loss:0.0001989 - ------------- Save best model ------------ - -Epoch 24, Iter 0: Training Loss:0.001320 -Epoch 24, Iter 24000: Training Loss:0.001303 -Epoch 24 Validation Loss:0.0001991 -Epoch 25, Iter 0: Training Loss:0.001388 -Epoch 25, Iter 24000: Training Loss:0.001399 -Epoch 25 Validation Loss:0.0001993 -Epoch 26, Iter 0: Training Loss:0.001399 -Epoch 26, Iter 24000: Training Loss:0.001139 -Epoch 26 Validation Loss:0.0001995 -Epoch 27, Iter 0: Training Loss:0.001415 -Epoch 27, Iter 24000: Training Loss:0.001281 -Epoch 27 Validation Loss:0.0001997 -Epoch 28, Iter 0: Training Loss:0.001672 -Epoch 28, Iter 24000: Training Loss:0.001224 -Epoch 28 Validation Loss:0.0001993 -Epoch 29, Iter 0: Training Loss:0.001540 -Epoch 29, Iter 24000: Training Loss:0.001223 -Epoch 29 Validation Loss:0.0001995 -Epoch 30, Iter 0: Training Loss:0.001573 -Epoch 30, Iter 24000: Training Loss:0.001106 -Epoch 30 Validation Loss:0.0002004 -Epoch 31, Iter 0: Training Loss:0.001323 -Epoch 31, Iter 24000: Training Loss:0.001443 -Epoch 31 Validation Loss:0.0002011 -Epoch 32, Iter 0: Training Loss:0.001441 -Epoch 32, Iter 24000: Training Loss:0.001345 -Epoch 32 Validation Loss:0.0002013 -Epoch 33, Iter 0: Training Loss:0.001364 -Epoch 33, Iter 24000: Training Loss:0.001100 -Epoch 33 Validation Loss:0.0002019 -Epoch 34, Iter 0: Training Loss:0.001306 -Epoch 34, Iter 24000: Training Loss:0.001148 -Epoch 34 Validation Loss:0.0002029 -Epoch 35, Iter 0: Training Loss:0.001301 -Epoch 35, Iter 24000: Training Loss:0.001408 -Epoch 35 Validation Loss:0.0002038 -Epoch 36, Iter 0: Training Loss:0.001098 -Epoch 36, Iter 24000: Training Loss:0.001162 -Epoch 36 Validation Loss:0.0002050 -Epoch 37, Iter 0: Training Loss:0.001330 -Epoch 37, Iter 24000: Training Loss:0.001073 -Epoch 37 Validation Loss:0.0002063 -Epoch 38, Iter 0: Training Loss:0.001253 -Epoch 38, Iter 24000: Training Loss:0.001129 -Epoch 38 Validation Loss:0.0002069 -Epoch 39, Iter 0: Training Loss:0.001055 -Epoch 39, Iter 24000: Training Loss:0.001310 -Epoch 39 Validation Loss:0.0002079 -Epoch 40, Iter 0: Training Loss:0.001125 -Epoch 40, Iter 24000: Training Loss:0.001104 -Epoch 40 Validation Loss:0.0002095 -Epoch 41, Iter 0: Training Loss:0.001377 -Epoch 41, Iter 24000: Training Loss:0.001138 -Epoch 41 Validation Loss:0.0002109 -Epoch 42, Iter 0: Training Loss:0.001018 -Epoch 42, Iter 24000: Training Loss:0.000935 -Epoch 42 Validation Loss:0.0002124 -Epoch 43, Iter 0: Training Loss:0.001339 -Epoch 43, Iter 24000: Training Loss:0.001016 -Epoch 43 Validation Loss:0.0002149 -Epoch 44, Iter 0: Training Loss:0.001004 -Epoch 44, Iter 24000: Training Loss:0.001025 -Epoch 44 Validation Loss:0.0002163 -Epoch 45, Iter 0: Training Loss:0.001063 -Epoch 45, Iter 24000: Training Loss:0.000845 -Epoch 45 Validation Loss:0.0002190 -Epoch 46, Iter 0: Training Loss:0.000964 -Epoch 46, Iter 24000: Training Loss:0.001032 -Epoch 46 Validation Loss:0.0002212 -Epoch 47, Iter 0: Training Loss:0.001137 -Epoch 47, Iter 24000: Training Loss:0.001174 -Epoch 47 Validation Loss:0.0002232 -Epoch 48, Iter 0: Training Loss:0.000985 -Epoch 48, Iter 24000: Training Loss:0.001066 -Epoch 48 Validation Loss:0.0002258 -Epoch 49, Iter 0: Training Loss:0.000938 -Epoch 49, Iter 24000: Training Loss:0.001042 -Epoch 49 Validation Loss:0.0002287 -Sucess on model train -Success on model test -Success on dataset synthesis -END: Testing success!!! From 4ce8e215ff3f2c927abe6249bde7509f9eac5da5 Mon Sep 17 00:00:00 2001 From: Rasmussen Date: Sun, 27 Jul 2025 10:27:57 -0500 Subject: [PATCH 06/10] Clean up things a bit --- pyhealth/datasets/halo_mimic3.py | 15 --- pyhealth/models/generators/halo.py | 163 +---------------------------- 2 files changed, 2 insertions(+), 176 deletions(-) diff --git a/pyhealth/datasets/halo_mimic3.py b/pyhealth/datasets/halo_mimic3.py index 6dc95f1c7..4f95b9617 100644 --- a/pyhealth/datasets/halo_mimic3.py +++ b/pyhealth/datasets/halo_mimic3.py @@ -1,12 +1,4 @@ import logging -import warnings -from pathlib import Path -from typing import List, Optional - -import polars as pl - -from .base_dataset import BaseDataset - import yaml import pickle import numpy as np @@ -16,8 +8,6 @@ logger = logging.getLogger(__name__) -# TODO: Clean up imports & update docs correctly - class HALO_MIMIC3Dataset: """ @@ -93,7 +83,6 @@ def build_dataset(self) -> None: data = list(data.values()) - # print("Adding Labels") with open("/u/ethanmr3/halo/PyHealth/pyhealth/datasets/configs/hcup_ccs_2015_definitions_benchmark.yaml") as definitions_file: definitions = yaml.full_load(definitions_file) @@ -112,18 +101,15 @@ def build_dataset(self) -> None: group_to_id = dict((x, i) for (i, x) in enumerate(id_to_group)) # Add Labels - # print(f"DATA LEN IS: {len(data)}") for p in data: label = np.zeros(len(group_to_id)) for v in p['visits']: for c in v: if c in code_to_group: label[group_to_id[code_to_group[c]]] = 1 - # print(f"MIMIC LABELS = {label}") p['labels'] = label - # print("Converting Visits") for p in data: new_visits = [] for v in p['visits']: @@ -143,7 +129,6 @@ def build_dataset(self) -> None: print(f"NUM LONGITUDINAL RECORDS: {len([p for p in data if len(p['visits']) > 1])}") # Train-Val-Test Split - # print("Splitting Datasets") train_dataset, test_dataset = train_test_split(data, test_size=0.2, random_state=4, shuffle=True) train_dataset, val_dataset = train_test_split(train_dataset, test_size=0.1, random_state=4, shuffle=True) diff --git a/pyhealth/models/generators/halo.py b/pyhealth/models/generators/halo.py index d2ec7a145..9223d6d9f 100644 --- a/pyhealth/models/generators/halo.py +++ b/pyhealth/models/generators/halo.py @@ -1,16 +1,11 @@ import torch -import torch.nn as nn -from typing import Any, Dict, List, Optional - import os import numpy as np import random import pickle from tqdm import tqdm -from sklearn import metrics -from pyhealth.datasets import SampleEHRDataset, HALO_MIMIC3Dataset -from pyhealth.models.base_model import BaseModel +from pyhealth.datasets import HALO_MIMIC3Dataset # Import the HALO transformer implementation from pyhealth.models.generators.halo_resources.halo_model import HALOModel @@ -52,10 +47,7 @@ def __init__( self.index_to_code = pickle.load(open(f"{self.dataset.pkl_data_dir}indexToCode.pkl", "rb")) self.id_to_label = pickle.load(open(f"{self.dataset.pkl_data_dir}idToLabel.pkl", "rb")) test_ehr_dataset = pickle.load(open(f'{self.dataset.pkl_data_dir}testDataset.pkl', 'rb')) - try: - print(f"test data head 1: \n{test_ehr_dataset.head()}") - except: - print(f"test data head 2: \n{test_ehr_dataset[:5]}") + train_c = set([c for p in self.train_ehr_dataset for v in p['visits'] for c in v]) self.test_ehr_dataset = [{'labels': p['labels'], 'visits': [[c for c in v if c in train_c] for v in p['visits']]} for p in test_ehr_dataset] @@ -104,7 +96,6 @@ def shuffle_training_data(train_ehr_dataset): self.model = HALOModel(self.config).to(self.device) self.optimizer = torch.optim.Adam(self.model.parameters(), lr=self.config.lr) if os.path.exists(f"{self.save_dir}halo_model"): - # print("Loading previous model") checkpoint = torch.load(f'{self.save_dir}halo_model', map_location=torch.device(self.device)) self.model.load_state_dict(checkpoint['model']) self.optimizer.load_state_dict(checkpoint['optimizer']) @@ -339,153 +330,3 @@ def sample_sequence(model, length, context, batch_size, device='cuda', sample=Tr synthetic_ehr_dataset += batch_synthetic_ehrs pickle.dump(synthetic_ehr_dataset, open(f"{pkl_save_dir}haloDataset.pkl", 'wb')) - - - - - - -# class HALO(BaseModel): -# """ -# HALO model wrapper for PyHealth, leveraging the BaseModel interface. - -# Args: -# dataset: SampleEHRDataset for EHR sequences (visits of codes). -# config: Transformer configuration object for HALOModel. -# feature_key: key in dataset samples for code-based visits (e.g., "list_codes"). -# mode: one of "binary", "multiclass", or "multilabel" for loss function. -# label_key: key in dataset samples for target visits; defaults to feature_key for next-visit prediction. -# pos_loss_weight: weight applied to positive labels in BCE loss. -# """ -# def __init__( -# self, -# dataset: SampleEHRDataset, -# config: HALOConfig, -# feature_key: str = "list_codes", -# mode: str = "multilabel", -# label_key: Optional[str] = None, -# pos_loss_weight: float = 1.0, -# ): -# super(HALO, self).__init__(dataset) -# self.feature_key = feature_key -# self.label_key = label_key or feature_key -# self.pos_loss_weight = pos_loss_weight - -# # Set mode for loss and evaluation -# self.mode = mode - -# # Tokenizer for the code-based input -# self.tokenizer = dataset.input_processors[feature_key] -# self.vocab_size = self.tokenizer.size() - -# # Instantiate the underlying HALO transformer model -# self.halo = HALOModel(config) - -# def _prepare_input_visits(self, codes: List[List[Any]]) -> torch.Tensor: -# """ -# Convert list of visits of codes into multi-hot tensor. - -# Args: -# codes: nested list of shape (batch, num_visits, codes_in_visit) - -# Returns: -# Tensor of shape (batch, num_visits, vocab_size) with 0/1 entries. -# """ -# # batch_encode_3d returns List[List[List[int]]] -# token_ids = self.tokenizer.batch_encode_3d(codes) -# batch_size = len(token_ids) -# max_visits = len(token_ids[0]) - -# visits = torch.zeros( -# batch_size, max_visits, self.vocab_size, device=self.device -# ) -# for i in range(batch_size): -# for t, visit_ids in enumerate(token_ids[i]): -# for cid in visit_ids: -# if cid is None: -# continue -# visits[i, t, cid] = 1.0 -# return visits - -# def forward( -# self, -# **kwargs: Any -# ) -> Dict[str, torch.Tensor]: -# """ -# Forward propagation for HALO within PyHealth. - -# Expects kwargs to contain: -# - feature_key: raw visit code lists. -# - optional label_key: same format for next-visit targets. -# - optional masks: tensor or nested lists for masking visits. - -# Returns a dict with keys: -# - loss: training loss (if labels provided). -# - y_prob: predicted probabilities of codes per visit. -# - y_true: ground-truth multi-hot labels (shifted by one visit). -# - logits: raw logits from the HALO transformer. -# """ -# # Prepare input tensor -# raw_codes = kwargs[self.feature_key] -# input_visits = self._prepare_input_visits(raw_codes) - -# # Gather optional training labels and masks -# ehr_labels = None -# ehr_masks = None -# if self.label_key in kwargs: -# # similarly convert label visits to multi-hot -# ehr_labels = self._prepare_input_visits(kwargs[self.label_key]) -# if "masks" in kwargs: -# ehr_masks = torch.tensor(kwargs["masks"], device=self.device, dtype=torch.float) - -# # Call HALOModel: returns loss & probabilities if labels, else probabilities -# if ehr_labels is not None: -# loss, code_probs, shift_labels = self.halo( -# input_visits, -# ehr_labels=ehr_labels, -# ehr_masks=ehr_masks, -# pos_loss_weight=self.pos_loss_weight, -# ) -# results = {"loss": loss, "y_prob": code_probs, "y_true": shift_labels} -# else: -# code_probs = self.halo(input_visits) -# results = {"y_prob": code_probs} - -# # Attach logits if needed -# if hasattr(self.halo, 'last_logits'): -# results['logits'] = self.halo.last_logits - -# return results - -# # Example usage: -# if __name__ == "__main__": -# from pyhealth.datasets import SampleEHRDataset -# # define samples with visits of codes as nested lists -# samples = [ -# {"patient_id": "p0", "visit_id": "v0", "list_codes": [["A", "B"], ["C"]]}, -# {"patient_id": "p1", "visit_id": "v0", "list_codes": [["D"], ["E", "F"]]}, -# ] -# dataset = SampleEHRDataset(samples=samples, dataset_name="halo_test") - -# # Build transformer config -# config = HALOConfig( -# n_layer=4, -# n_head=8, -# n_embd=128, -# total_vocab_size=dataset.input_processors['list_codes'].size(), -# n_positions=dataset.max_visit_length, -# n_ctx=dataset.max_visit_length, -# layer_norm_epsilon=1e-5, -# ) - -# model = HALO( -# dataset=dataset, -# config=config, -# feature_key='list_codes', -# mode='multilabel', -# ) -# from pyhealth.datasets import get_dataloader -# loader = get_dataloader(dataset, batch_size=2, shuffle=False) -# batch = next(iter(loader)) -# output = model(**batch) -# print(output) From b1584fda9710982ebdb4937f7198182726b92503 Mon Sep 17 00:00:00 2001 From: Rasmussen Date: Sun, 27 Jul 2025 10:38:28 -0500 Subject: [PATCH 07/10] Clean up hardcoded file path --- .gitignore | 6 +++++- pyhealth/datasets/halo_mimic3.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 85a6aa6df..8777ff37f 100644 --- a/.gitignore +++ b/.gitignore @@ -130,4 +130,8 @@ leaderboard/credentials.json leaderboard/rtd_token.txt # locally pre-trained models -pyhealth/medcode/pretrained_embeddings/kg_emb/examples/pretrained_model \ No newline at end of file +pyhealth/medcode/pretrained_embeddings/kg_emb/examples/pretrained_model + +# local testing files +halo_testing_script.py +test_halo_model.slurm \ No newline at end of file diff --git a/pyhealth/datasets/halo_mimic3.py b/pyhealth/datasets/halo_mimic3.py index 4f95b9617..3c38771e8 100644 --- a/pyhealth/datasets/halo_mimic3.py +++ b/pyhealth/datasets/halo_mimic3.py @@ -83,7 +83,7 @@ def build_dataset(self) -> None: data = list(data.values()) - with open("/u/ethanmr3/halo/PyHealth/pyhealth/datasets/configs/hcup_ccs_2015_definitions_benchmark.yaml") as definitions_file: + with open("./configs/hcup_ccs_2015_definitions_benchmark.yaml") as definitions_file: definitions = yaml.full_load(definitions_file) code_to_group = {} From d374603502e079589ab46751e11eccdcb02086b1 Mon Sep 17 00:00:00 2001 From: Rasmussen Date: Sun, 27 Jul 2025 10:39:41 -0500 Subject: [PATCH 08/10] Remove testing files from PR --- .gitignore | 1 + halo_testing_script.py | 28 ---------------------------- test_halo_model.slurm | 34 ---------------------------------- 3 files changed, 1 insertion(+), 62 deletions(-) delete mode 100644 halo_testing_script.py delete mode 100644 test_halo_model.slurm diff --git a/.gitignore b/.gitignore index 8777ff37f..5f887de5e 100644 --- a/.gitignore +++ b/.gitignore @@ -133,5 +133,6 @@ leaderboard/rtd_token.txt pyhealth/medcode/pretrained_embeddings/kg_emb/examples/pretrained_model # local testing files +halo_testing/ halo_testing_script.py test_halo_model.slurm \ No newline at end of file diff --git a/halo_testing_script.py b/halo_testing_script.py deleted file mode 100644 index 84b29bfd6..000000000 --- a/halo_testing_script.py +++ /dev/null @@ -1,28 +0,0 @@ -print("BEGIN: Testing") -import subprocess, sys, os -subprocess.check_call([sys.executable, "-m", "pip", "install", "-e", "."]) -subprocess.check_call([sys.executable, "-m", "pip", "install", "numpy", "--force-reinstall"]) -subprocess.check_call([sys.executable, "-m", "pip", "install", "pandas", "--force-reinstall"]) -print("Success on pip install -e .") -from pyhealth.models.generators.halo import HALO -from pyhealth.models.generators.halo_resources.halo_model import HALOModel -from pyhealth.models.generators.halo_resources.halo_config import HALOConfig -from pyhealth.datasets.halo_mimic3 import HALO_MIMIC3Dataset -print("Sucess on imports") -print(f"Operating in dir: {os.getcwd()}") - -halo_config = HALOConfig() -halo_dataset = HALO_MIMIC3Dataset(mimic3_dir="../../../../scratch_old/ethanmr3/mimic3/physionet.org/files/mimiciii/1.4/", pkl_data_dir="../../halo_pkl/", gzip=True) -model = HALO(dataset=halo_dataset, config=halo_config, save_dir="../../halo_save/", train_on_init=False) -print("Success on model setup") - -model.train() -print("Sucess on model train") - -model.test(testing_results_dir = "../../halo_results/") -print("Success on model test") - -model.synthesize_dataset(pkl_save_dir = "../../halo_results/") -print("Success on dataset synthesis") - -print("END: Testing success!!!") \ No newline at end of file diff --git a/test_halo_model.slurm b/test_halo_model.slurm deleted file mode 100644 index 82b921e70..000000000 --- a/test_halo_model.slurm +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -#SBATCH --account=ethanmr3-ic -#SBATCH --job-name=pyhealth-halo-testing -#SBATCH --output=halo-testing-logs/halo_test_%j.out -#SBATCH --error=halo-testing-logs/halo_test_%j.err -#SBATCH --partition=IllinoisComputes-GPU # Change to appropriate partition -#SBATCH --gres=gpu:1 # Request 1 GPU -#SBATCH --cpus-per-task=4 -#SBATCH --mem=64G -#SBATCH --time=48:00:00 - -# Change to the directory where you submitted the job -cd "$SLURM_SUBMIT_DIR" - -# Print useful Slurm environment variables for debugging -echo "SLURM_JOB_ID: $SLURM_JOB_ID" -echo "SLURM_JOB_NODELIST: $SLURM_JOB_NODELIST" -echo "SLURM_NTASKS: $SLURM_NTASKS" -echo "SLURM_CPUS_ON_NODE: $SLURM_CPUS_ON_NODE" -echo "SLURM_GPUS_ON_NODE: $SLURM_GPUS_ON_NODE" -echo "SLURM_GPUS: $SLURM_GPUS" -echo "CUDA_VISIBLE_DEVICES: $CUDA_VISIBLE_DEVICES" - -# Optional: check what GPU(s) is/are actually visible -echo "Running nvidia-smi to confirm GPU availability:" -nvidia-smi - -# Load modules or activate environment -# module load python/3.10 -# module load cuda/11.7 -# conda activate your-env - -# Run your Python training script -python /u/ethanmr3/halo/PyHealth/halo_testing_script.py \ No newline at end of file From 4f456f9dd08a9d0791229845497df960870dadf3 Mon Sep 17 00:00:00 2001 From: Rasmussen Date: Sun, 27 Jul 2025 10:41:14 -0500 Subject: [PATCH 09/10] Init model properly --- pyhealth/models/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyhealth/models/__init__.py b/pyhealth/models/__init__.py index 69dc60eb8..eb3766ccc 100644 --- a/pyhealth/models/__init__.py +++ b/pyhealth/models/__init__.py @@ -25,3 +25,4 @@ from .transformer import Transformer, TransformerLayer from .transformers_model import TransformersModel from .vae import VAE +from .generators.halo import HALO \ No newline at end of file From 56380f687f3cb94441cdc9e0df5b1fbf7fed3d52 Mon Sep 17 00:00:00 2001 From: Rasmussen Date: Sun, 27 Jul 2025 10:44:20 -0500 Subject: [PATCH 10/10] Update comments --- pyhealth/datasets/halo_mimic3.py | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/pyhealth/datasets/halo_mimic3.py b/pyhealth/datasets/halo_mimic3.py index 3c38771e8..f08eb8a86 100644 --- a/pyhealth/datasets/halo_mimic3.py +++ b/pyhealth/datasets/halo_mimic3.py @@ -17,10 +17,9 @@ class HALO_MIMIC3Dataset: which includes tables such as patients, admissions, and icustays. Attributes: - root (str): The root directory where the dataset is stored. - tables (List[str]): A list of tables to be included in the dataset. - dataset_name (Optional[str]): The name of the dataset. - config_path (Optional[str]): The path to the configuration file. + mimic3_dir (str): The root directory where the dataset is stored. + pkl_data_dir (str): The directory in which .pkl files related to the dataset object will be stored. + gzip (Optional[bool]): Determines whether the object will look for ".csv.gz" (True) or ".csv" (False) files in mimic3_dir. """ def __init__( @@ -29,15 +28,6 @@ def __init__( pkl_data_dir: str = "./", gzip: bool = False ) -> None: - """ - Initializes the MIMIC4Dataset with the specified parameters. - - Args: - root (str): The root directory where the dataset is stored. - tables (List[str]): A list of additional tables to include. - dataset_name (Optional[str]): The name of the dataset. Defaults to "mimic3". - config_path (Optional[str]): The path to the configuration file. If not provided, a default config is used. - """ self.gzip = gzip self.mimic3_dir = mimic3_dir self.pkl_data_dir = pkl_data_dir