File tree Expand file tree Collapse file tree 1 file changed +2
-3
lines changed Expand file tree Collapse file tree 1 file changed +2
-3
lines changed Original file line number Diff line number Diff line change @@ -204,12 +204,11 @@ def _create_prompt(
204
204
205
205
left = start_index
206
206
right = start_index + 4 * prompt_tokens
207
- start_tokens = list [int ]
208
207
209
208
while left < right :
210
209
mid = (left + right ) // 2
211
210
test_prompt = self .text_creator .create_text (start_index , mid - start_index )
212
- test_tokens = start_tokens + self .processor .encode (test_prompt )
211
+ test_tokens = self .processor .encode (test_prompt )
213
212
214
213
if len (test_tokens ) == prompt_tokens :
215
214
return test_tokens
@@ -219,7 +218,7 @@ def _create_prompt(
219
218
right = mid
220
219
221
220
final_text = self .text_creator .create_text (start_index , left - start_index )
222
- return start_tokens + self .processor .encode (final_text )
221
+ return self .processor .encode (final_text )
223
222
224
223
225
224
class SyntheticDatasetCreator (DatasetCreator ):
You can’t perform that action at this time.
0 commit comments