From 0625958eff5fbe92365c92ed5fdd1feb224c572b Mon Sep 17 00:00:00 2001 From: clarencechen Date: Fri, 3 Feb 2023 12:18:41 -0800 Subject: [PATCH 1/5] Update README.md --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 16283bc..a73e7ad 100644 --- a/README.md +++ b/README.md @@ -27,4 +27,11 @@ If our work is useful for your own, you can cite us with the following BibTex en title = {DetectGPT: Zero-Shot Machine-Generated Text Detection using Probability Curvature}, publisher = {arXiv}, year = {2023}, - } \ No newline at end of file + } + @article{meister+al.pre22, + author = {Meister, Clara and Pimentel, Tiago and Wiher, Gian and Cotterell, Ryan}, + publisher = {arXiv}, + title = {Locally Typical Sampling}, + url = {https://arxiv.org/abs/2202.00666}, + year = {2022} +} From 3fb76c48c6ca80107027191a8806a17e993375a4 Mon Sep 17 00:00:00 2001 From: clarencechen Date: Fri, 3 Feb 2023 12:19:30 -0800 Subject: [PATCH 2/5] Update README.md --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index a73e7ad..2ed780e 100644 --- a/README.md +++ b/README.md @@ -28,10 +28,10 @@ If our work is useful for your own, you can cite us with the following BibTex en publisher = {arXiv}, year = {2023}, } - @article{meister+al.pre22, + @article{meister2022typical, + url = {https://arxiv.org/abs/2202.00666}, author = {Meister, Clara and Pimentel, Tiago and Wiher, Gian and Cotterell, Ryan}, - publisher = {arXiv}, title = {Locally Typical Sampling}, - url = {https://arxiv.org/abs/2202.00666}, + publisher = {arXiv}, year = {2022} -} + } From 170a4dccf9e1507be1e5bff200c29251bca09dd4 Mon Sep 17 00:00:00 2001 From: Clarence Chen Date: Wed, 15 Feb 2023 15:41:41 -0800 Subject: [PATCH 3/5] add typical sampling --- run.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/run.py b/run.py index cc50274..7dbdaa3 100644 --- a/run.py +++ b/run.py @@ -236,6 +236,8 @@ def sample_from_model(texts, min_words=55, prompt_tokens=30): sampling_kwargs['top_p'] = args.top_p elif args.do_top_k: sampling_kwargs['top_k'] = args.top_k + elif args.do_typical_p: + sampling_kwargs['typical_p'] = args.typical_p min_length = 50 if args.dataset in ['pubmed'] else 150 outputs = base_model.generate(**all_encoded, min_length=min_length, max_length=200, do_sample=True, **sampling_kwargs, pad_token_id=base_tokenizer.eos_token_id, eos_token_id=base_tokenizer.eos_token_id) decoded = base_tokenizer.batch_decode(outputs, skip_special_tokens=True) @@ -766,6 +768,8 @@ def eval_supervised(data, model): parser.add_argument('--top_k', type=int, default=40) parser.add_argument('--do_top_p', action='store_true') parser.add_argument('--top_p', type=float, default=0.96) + parser.add_argument('--do_typical_p', action='store_true') + parser.add_argument('--typical_p', type=float, default=0.96) parser.add_argument('--output_name', type=str, default="") parser.add_argument('--openai_model', type=str, default=None) parser.add_argument('--openai_key', type=str) From ea118dd7e0aa03710b6a84a63835355aa5d70886 Mon Sep 17 00:00:00 2001 From: Clarence Chen Date: Wed, 15 Feb 2023 16:06:44 -0800 Subject: [PATCH 4/5] add typical sampling tests --- paper_scripts/typical.sh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 paper_scripts/typical.sh diff --git a/paper_scripts/typical.sh b/paper_scripts/typical.sh new file mode 100644 index 0000000..e1cf6c8 --- /dev/null +++ b/paper_scripts/typical.sh @@ -0,0 +1,18 @@ +python run.py --output_name main_typical_p --base_model_name gpt2-xl --mask_filling_model_name t5-3b --n_perturbation_list 1,10,100 --n_samples 500 --pct_words_masked 0.3 --span_length 2 --do_typical_p +python run.py --output_name main_typical_p --base_model_name EleutherAI/gpt-neo-2.7B --mask_filling_model_name t5-3b --n_perturbation_list 1,10,100 --n_samples 500 --pct_words_masked 0.3 --span_length 2 --do_typical_p +python run.py --output_name main_typical_p --base_model_name EleutherAI/gpt-j-6B --mask_filling_model_name t5-3b --n_perturbation_list 1,10,100 --n_samples 500 --pct_words_masked 0.3 --span_length 2 --do_typical_p +python run.py --output_name main_typical_p --base_model_name facebook/opt-2.7b --mask_filling_model_name t5-3b --n_perturbation_list 1,10,100 --n_samples 500 --pct_words_masked 0.3 --span_length 2 --do_typical_p +python run.py --output_name main_typical_p --batch_size 20 --base_model_name EleutherAI/gpt-neox-20b --mask_filling_model_name t5-11b --n_perturbation_list 1,10,100 --n_samples 500 --pct_words_masked 0.3 --span_length 2 --do_typical_p + +python run.py --output_name main_typical_p --base_model_name gpt2-xl --mask_filling_model_name t5-3b --n_perturbation_list 1,10,100 --n_samples 312 --pct_words_masked 0.3 --span_length 2 --dataset squad --dataset_key context --do_typical_p +python run.py --output_name main_typical_p --base_model_name EleutherAI/gpt-neo-2.7B --mask_filling_model_name t5-3b --n_perturbation_list 1,10,100 --n_samples 312 --pct_words_masked 0.3 --span_length 2 --dataset squad --dataset_key context --do_typical_p +python run.py --output_name main_typical_p --base_model_name EleutherAI/gpt-j-6B --mask_filling_model_name t5-3b --n_perturbation_list 1,10,100 --n_samples 312 --pct_words_masked 0.3 --span_length 2 --dataset squad --dataset_key context --do_typical_p +python run.py --output_name main_typical_p --base_model_name facebook/opt-2.7b --mask_filling_model_name t5-3b --n_perturbation_list 1,10,100 --n_samples 312 --pct_words_masked 0.3 --span_length 2 --dataset squad --dataset_key context --do_typical_p +python run.py --output_name main_typical_p --batch_size 20 --base_model_name EleutherAI/gpt-neox-20b --mask_filling_model_name t5-11b --n_perturbation_list 1,10,100 --n_samples 312 --pct_words_masked 0.3 --span_length 2 --dataset squad --dataset_key context --do_typical_p + +python run.py --output_name main_typical_p --base_model_name gpt2-xl --mask_filling_model_name t5-3b --n_perturbation_list 1,10,100 --n_samples 500 --pct_words_masked 0.3 --span_length 2 --dataset writing --do_typical_p +python run.py --output_name main_typical_p --base_model_name EleutherAI/gpt-neo-2.7B --mask_filling_model_name t5-3b --n_perturbation_list 1,10,100 --n_samples 500 --pct_words_masked 0.3 --span_length 2 --dataset writing --do_typical_p +python run.py --output_name main_typical_p --base_model_name EleutherAI/gpt-j-6B --mask_filling_model_name t5-3b --n_perturbation_list 1,10,100 --n_samples 500 --pct_words_masked 0.3 --span_length 2 --dataset writing --do_typical_p +python run.py --output_name main_typical_p --base_model_name facebook/opt-2.7b --mask_filling_model_name t5-3b --n_perturbation_list 1,10,100 --n_samples 500 --pct_words_masked 0.3 --span_length 2 --dataset writing --do_typical_p +python run.py --output_name main_typical_p --batch_size 20 --base_model_name EleutherAI/gpt-neox-20b --mask_filling_model_name t5-11b --n_perturbation_list 1,10,100 --n_samples 500 --pct_words_masked 0.3 --span_length 2 --dataset writing --do_typical_p + From 629342ad4f4f46d33a036f277ea9d057f90e7e62 Mon Sep 17 00:00:00 2001 From: Clarence Chen Date: Wed, 15 Feb 2023 16:56:47 -0800 Subject: [PATCH 5/5] decrease typical p and restrict to story writing --- paper_scripts/typical.sh | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/paper_scripts/typical.sh b/paper_scripts/typical.sh index e1cf6c8..f54124c 100644 --- a/paper_scripts/typical.sh +++ b/paper_scripts/typical.sh @@ -1,18 +1,5 @@ -python run.py --output_name main_typical_p --base_model_name gpt2-xl --mask_filling_model_name t5-3b --n_perturbation_list 1,10,100 --n_samples 500 --pct_words_masked 0.3 --span_length 2 --do_typical_p -python run.py --output_name main_typical_p --base_model_name EleutherAI/gpt-neo-2.7B --mask_filling_model_name t5-3b --n_perturbation_list 1,10,100 --n_samples 500 --pct_words_masked 0.3 --span_length 2 --do_typical_p -python run.py --output_name main_typical_p --base_model_name EleutherAI/gpt-j-6B --mask_filling_model_name t5-3b --n_perturbation_list 1,10,100 --n_samples 500 --pct_words_masked 0.3 --span_length 2 --do_typical_p -python run.py --output_name main_typical_p --base_model_name facebook/opt-2.7b --mask_filling_model_name t5-3b --n_perturbation_list 1,10,100 --n_samples 500 --pct_words_masked 0.3 --span_length 2 --do_typical_p -python run.py --output_name main_typical_p --batch_size 20 --base_model_name EleutherAI/gpt-neox-20b --mask_filling_model_name t5-11b --n_perturbation_list 1,10,100 --n_samples 500 --pct_words_masked 0.3 --span_length 2 --do_typical_p - -python run.py --output_name main_typical_p --base_model_name gpt2-xl --mask_filling_model_name t5-3b --n_perturbation_list 1,10,100 --n_samples 312 --pct_words_masked 0.3 --span_length 2 --dataset squad --dataset_key context --do_typical_p -python run.py --output_name main_typical_p --base_model_name EleutherAI/gpt-neo-2.7B --mask_filling_model_name t5-3b --n_perturbation_list 1,10,100 --n_samples 312 --pct_words_masked 0.3 --span_length 2 --dataset squad --dataset_key context --do_typical_p -python run.py --output_name main_typical_p --base_model_name EleutherAI/gpt-j-6B --mask_filling_model_name t5-3b --n_perturbation_list 1,10,100 --n_samples 312 --pct_words_masked 0.3 --span_length 2 --dataset squad --dataset_key context --do_typical_p -python run.py --output_name main_typical_p --base_model_name facebook/opt-2.7b --mask_filling_model_name t5-3b --n_perturbation_list 1,10,100 --n_samples 312 --pct_words_masked 0.3 --span_length 2 --dataset squad --dataset_key context --do_typical_p -python run.py --output_name main_typical_p --batch_size 20 --base_model_name EleutherAI/gpt-neox-20b --mask_filling_model_name t5-11b --n_perturbation_list 1,10,100 --n_samples 312 --pct_words_masked 0.3 --span_length 2 --dataset squad --dataset_key context --do_typical_p - -python run.py --output_name main_typical_p --base_model_name gpt2-xl --mask_filling_model_name t5-3b --n_perturbation_list 1,10,100 --n_samples 500 --pct_words_masked 0.3 --span_length 2 --dataset writing --do_typical_p -python run.py --output_name main_typical_p --base_model_name EleutherAI/gpt-neo-2.7B --mask_filling_model_name t5-3b --n_perturbation_list 1,10,100 --n_samples 500 --pct_words_masked 0.3 --span_length 2 --dataset writing --do_typical_p -python run.py --output_name main_typical_p --base_model_name EleutherAI/gpt-j-6B --mask_filling_model_name t5-3b --n_perturbation_list 1,10,100 --n_samples 500 --pct_words_masked 0.3 --span_length 2 --dataset writing --do_typical_p -python run.py --output_name main_typical_p --base_model_name facebook/opt-2.7b --mask_filling_model_name t5-3b --n_perturbation_list 1,10,100 --n_samples 500 --pct_words_masked 0.3 --span_length 2 --dataset writing --do_typical_p -python run.py --output_name main_typical_p --batch_size 20 --base_model_name EleutherAI/gpt-neox-20b --mask_filling_model_name t5-11b --n_perturbation_list 1,10,100 --n_samples 500 --pct_words_masked 0.3 --span_length 2 --dataset writing --do_typical_p - +python run.py --output_name main_typical_p --base_model_name gpt2-xl --mask_filling_model_name t5-3b --n_perturbation_list 1,10,100 --n_samples 500 --pct_words_masked 0.3 --span_length 2 --dataset writing --do_typical_p --typical_p 0.2 +python run.py --output_name main_typical_p --base_model_name EleutherAI/gpt-neo-2.7B --mask_filling_model_name t5-3b --n_perturbation_list 1,10,100 --n_samples 500 --pct_words_masked 0.3 --span_length 2 --dataset writing --do_typical_p --typical_p 0.2 +python run.py --output_name main_typical_p --base_model_name EleutherAI/gpt-j-6B --mask_filling_model_name t5-3b --n_perturbation_list 1,10,100 --n_samples 500 --pct_words_masked 0.3 --span_length 2 --dataset writing --do_typical_p --typical_p 0.2 +python run.py --output_name main_typical_p --base_model_name facebook/opt-2.7b --mask_filling_model_name t5-3b --n_perturbation_list 1,10,100 --n_samples 500 --pct_words_masked 0.3 --span_length 2 --dataset writing --do_typical_p --typical_p 0.2 +python run.py --output_name main_typical_p --batch_size 20 --base_model_name EleutherAI/gpt-neox-20b --mask_filling_model_name t5-11b --n_perturbation_list 1,10,100 --n_samples 500 --pct_words_masked 0.3 --span_length 2 --dataset writing --do_typical_p --typical_p 0.2