From 9ec0f3e86826020c685c8455623e7a0333cdb3ec Mon Sep 17 00:00:00 2001 From: AC Date: Tue, 19 May 2020 00:08:59 -0700 Subject: [PATCH] ignore PAD during dynamic masking --- pretrain/pretrain_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pretrain/pretrain_helpers.py b/pretrain/pretrain_helpers.py index 7329bec..a60f471 100644 --- a/pretrain/pretrain_helpers.py +++ b/pretrain/pretrain_helpers.py @@ -118,7 +118,7 @@ def scatter_update(sequence, updates, positions): def _get_candidates_mask(inputs: pretrain_data.Inputs, vocab, disallow_from_mask=None): """Returns a mask tensor of positions in the input that can be masked out.""" - ignore_ids = [vocab["[SEP]"], vocab["[CLS]"], vocab["[MASK]"]] + ignore_ids = [vocab["[SEP]"], vocab["[CLS]"], vocab["[MASK]"], vocab["[PAD]"]] candidates_mask = tf.ones_like(inputs.input_ids, tf.bool) for ignore_id in ignore_ids: candidates_mask &= tf.not_equal(inputs.input_ids, ignore_id)