update BERTopic test

Signed-off-by: Tim Schopf <tim.schopf@t-online.de>

Author: TimSchopf

tests/requirements.txt

@@ -3,5 +3,5 @@ keybert>=0.5.0
 flair==0.11.3
 scipy==1.7.3
 bertopic>=0.16.1
-datasets==2.13.2
+scikit-learn>=1.0.1
 umap-learn==0.5.4

tests/test_vectorizers.py

@@ -3,10 +3,10 @@
 import flair
 import spacy
 from bertopic import BERTopic
-from datasets import load_dataset
 from flair.models import SequenceTagger
 from flair.tokenization import SegtokSentenceSplitter
 from keybert import KeyBERT
+from sklearn.datasets import fetch_20newsgroups
 
 import tests.utils as utils
 from keyphrase_vectorizers import KeyphraseCountVectorizer, KeyphraseTfidfVectorizer
@@ -172,9 +172,8 @@ def test_online_vectorizer():
 
 
 def test_bertopic():
-    data = load_dataset("ag_news")
-    texts = data['train']['text']
-    texts = texts[:100]
+    data = fetch_20newsgroups(subset='train')
+    texts = data.data[:100]
     topic_model = BERTopic(vectorizer_model=KeyphraseCountVectorizer())
     topics, probs = topic_model.fit_transform(documents=texts)
     new_topics = topic_model.reduce_outliers(texts, topics)