We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent 6ae8350 commit b8d30cfCopy full SHA for b8d30cf
delphi/utils.py
@@ -48,11 +48,15 @@ def load_tokenized_data(
48
49
try:
50
from datasets import Column
51
+
52
if isinstance(tokens, Column):
53
from datasets.table import table_iter
54
55
tokens = torch.cat(
56
[
- torch.from_numpy(np.stack(table_chunk["input_ids"].to_numpy(), axis=0))
57
+ torch.from_numpy(
58
+ np.stack(table_chunk["input_ids"].to_numpy(), axis=0)
59
+ )
60
for table_chunk in table_iter(
61
tokens.source._data, convert_to_tensor_chunk_size
62
)
0 commit comments