Meeting protocol

build.gradle

@@ -32,6 +32,8 @@ dependencies {
     implementation 'de.cyklon:ReflectionUtils:192873e'
     implementation 'de.cyklon:JEvent:879a575'
 
+    implementation 'ai.picovoice:leopard-java:2.0.2'
+
     implementation 'se.michaelthelin.spotify:spotify-web-api-java:9.0.0-RC1'
     implementation 'com.github.minndevelopment:emoji-java:master-SNAPSHOT'
     implementation 'org.kohsuke:github-api:1.321'

run_template/credentials

@@ -16,4 +16,7 @@ SPOTIFY_CLIENT_SECRET=-- Spotify Client Secret --
 # Optional. If configured, notifications about new youtube videos and livestreams are sent
 # you can enter an infinite number of api keys according to the same principle as below
 YOUTUBE_API_KEY_0=-- Youtube API Key --
-YOUTUBE_API_KEY_1=-- other Youtube API Key --
+YOUTUBE_API_KEY_1=-- other Youtube API Key --
+
+# Optional. If configured, meeting protocol will automatically be created
+PICOVOICE_ACCESS_KEY=-- PicoVoiceAI Access Key --

src/main/java/de/slimecloud/slimeball/config/GuildConfig.java

@@ -10,7 +10,7 @@
 import de.slimecloud.slimeball.features.fdmds.FdmdsConfig;
 import de.slimecloud.slimeball.features.level.GuildLevelConfig;
 import de.slimecloud.slimeball.features.moderation.AutoDeleteFlag;
-import de.slimecloud.slimeball.features.staff.MeetingConfig;
+import de.slimecloud.slimeball.features.staff.meeting.MeetingConfig;
 import de.slimecloud.slimeball.features.staff.StaffConfig;
 import de.slimecloud.slimeball.features.staff.absence.AbsenceConfig;
 import de.slimecloud.slimeball.features.statistic.StatisticConfig;

src/main/java/de/slimecloud/slimeball/features/staff/MeetingConfig.java → src/main/java/de/slimecloud/slimeball/features/staff/meeting/MeetingConfig.java

@@ -1,4 +1,4 @@
-package de.slimecloud.slimeball.features.staff;
+package de.slimecloud.slimeball.features.staff.meeting;
 
 import de.slimecloud.slimeball.config.ConfigCategory;
 import de.slimecloud.slimeball.config.engine.ConfigField;

src/main/java/de/slimecloud/slimeball/features/staff/TeamMeeting.java → src/main/java/de/slimecloud/slimeball/features/staff/meeting/TeamMeeting.java

@@ -1,4 +1,4 @@
-package de.slimecloud.slimeball.features.staff;
+package de.slimecloud.slimeball.features.staff.meeting;
 
 import de.slimecloud.slimeball.config.GuildConfig;
 import de.slimecloud.slimeball.main.Main;

src/main/java/de/slimecloud/slimeball/features/staff/meeting/commands/MeetingCommand.java

@@ -0,0 +1,29 @@
+package de.slimecloud.slimeball.features.staff.meeting.commands;
+
+import de.mineking.discordutils.commands.ApplicationCommand;
+import de.mineking.discordutils.commands.Command;
+import de.mineking.discordutils.commands.Setup;
+import de.mineking.discordutils.commands.condition.IRegistrationCondition;
+import de.mineking.discordutils.commands.condition.Scope;
+import de.mineking.discordutils.commands.context.ICommandContext;
+import de.mineking.discordutils.list.ListManager;
+import de.slimecloud.slimeball.config.GuildConfig;
+import de.slimecloud.slimeball.features.staff.meeting.protocol.commands.MeetingProtocolStartCommand;
+import de.slimecloud.slimeball.features.staff.meeting.protocol.commands.MeetingProtocolStopCommand;
+import de.slimecloud.slimeball.main.CommandPermission;
+import de.slimecloud.slimeball.main.SlimeBot;
+import org.jetbrains.annotations.NotNull;
+
+@ApplicationCommand(name = "meeting", description = "Verwalte Team Meetings", scope = Scope.GUILD)
+public class MeetingCommand {
+
+	public final CommandPermission permission = CommandPermission.TEAM;
+	public final IRegistrationCondition<ICommandContext> condition = (manager, guild, cache) -> cache.<GuildConfig>getState("config").getMeeting().isPresent();
+
+	@Setup
+	public static void setup(@NotNull SlimeBot bot, @NotNull Command<ICommandContext> command, @NotNull ListManager<ICommandContext> manager) {
+		command.addSubcommand(MeetingProtocolStartCommand.class);
+		command.addSubcommand(MeetingProtocolStopCommand.class);
+	}
+
+}

src/main/java/de/slimecloud/slimeball/features/staff/meeting/protocol/AudioReceiver.java

@@ -0,0 +1,47 @@
+package de.slimecloud.slimeball.features.staff.meeting.protocol;
+
+import net.dv8tion.jda.api.audio.AudioReceiveHandler;
+import net.dv8tion.jda.api.audio.UserAudio;
+
+import javax.sound.sampled.AudioInputStream;
+import java.io.ByteArrayInputStream;
+import java.util.*;
+
+public class AudioReceiver implements AudioReceiveHandler
+{
+	private static final double VOLUME = 1.0;
+	private final Map<Long, List<byte[]>> received = new LinkedHashMap<>();
+
+	@Override
+	public boolean canReceiveUser() {
+		return true;
+	}
+
+	@Override
+	public void handleUserAudio(UserAudio userAudio) {
+		received.computeIfAbsent(userAudio.getUser().getIdLong(), id -> new LinkedList<>()).add(userAudio.getAudioData(VOLUME));
+	}
+
+	public Set<Long> getUsers() {
+		return received.keySet();
+	}
+
+	public byte[] getBytes(Long id) {
+		int size = 0;
+		for (byte[] bytes : received.get(id)) size += bytes.length;
+		byte[] data = new byte[size];
+		int i = 0;
+		for (byte[] bytes : received.get(id)) {
+			for (byte b : bytes) {
+				data[i++] = b;
+			}
+		}
+		return data;
+	}
+
+	public AudioInputStream getAudioStream(Long id) {
+		byte[] data = getBytes(id);
+		return new AudioInputStream(new ByteArrayInputStream(data), OUTPUT_FORMAT, data.length);
+	}
+
+}

src/main/java/de/slimecloud/slimeball/features/staff/meeting/protocol/MeetingProtocol.java

@@ -0,0 +1,107 @@
+package de.slimecloud.slimeball.features.staff.meeting.protocol;
+
+import ai.picovoice.leopard.Leopard;
+import ai.picovoice.leopard.LeopardException;
+import ai.picovoice.leopard.LeopardTranscript;
+import de.mineking.discordutils.commands.ApplicationCommand;
+import de.mineking.discordutils.commands.ApplicationCommandMethod;
+import de.slimecloud.slimeball.main.SlimeBot;
+import lombok.Getter;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import net.dv8tion.jda.api.JDA;
+import net.dv8tion.jda.api.entities.Guild;
+import net.dv8tion.jda.api.entities.GuildVoiceState;
+import net.dv8tion.jda.api.entities.User;
+import net.dv8tion.jda.api.entities.channel.concrete.VoiceChannel;
+import net.dv8tion.jda.api.entities.channel.unions.AudioChannelUnion;
+import net.dv8tion.jda.api.events.interaction.command.SlashCommandInteractionEvent;
+import net.dv8tion.jda.api.managers.AudioManager;
+import org.jetbrains.annotations.NotNull;
+
+import javax.sound.sampled.AudioFileFormat;
+import javax.sound.sampled.AudioInputStream;
+import javax.sound.sampled.AudioSystem;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+
+@Slf4j
+public class MeetingProtocol {
+
+	private static final File protocolDirectory = new File("protocol");
+
+	static {
+		protocolDirectory.mkdirs();
+	}
+
+	private final String picovoice_access_key;
+	private final File leopard_model;
+	@Getter
+	private AudioReceiver receiver;
+
+	public MeetingProtocol(String picovoice_access_key) {
+		this.picovoice_access_key = picovoice_access_key;
+		URL url = MeetingProtocol.class.getClassLoader().getResource("leopard_params_de.pv");
+		if (url == null || url.getPath().isBlank()) throw new RuntimeException("Leopard model not found");
+		this.leopard_model = new File(url.getPath());
+	}
+
+	public void start(VoiceChannel vc) {
+		receiver = new AudioReceiver();
+		Guild guild = vc.getGuild();
+		AudioManager audioManager = guild.getAudioManager();
+		audioManager.openAudioConnection(vc);
+		audioManager.setReceivingHandler(receiver);
+	}
+
+	@SneakyThrows(LeopardException.class)
+	public void stop(Guild guild) {
+		JDA jda = guild.getJDA();
+
+		List<User> users = new ArrayList<>();
+		for (Long id : receiver.getUsers()) users.add(jda.getUserById(id));
+
+		for (User user : users) {
+			try (AudioInputStream ais = receiver.getAudioStream(user.getIdLong())) {
+				AudioSystem.write(ais, AudioFileFormat.Type.WAVE, new File(protocolDirectory, String.format("%s.wav", user.getName())));
+			} catch (IOException e) {
+				logger.error("failed to save protocol as wav from " + user.getName(), e);
+			}
+		}
+
+		Leopard leopard = new Leopard.Builder()
+				.setAccessKey(picovoice_access_key)
+				.setModelPath(leopard_model.getAbsolutePath())
+				.build();
+
+		for (User user : users) {
+			try {
+				File file = new File(protocolDirectory, String.format("%s.wav", user.getName()));
+				if (!file.exists()) {
+					logger.debug("skip user " + user.getName() + " for processing because the " + file.getName() + " file is missing");
+					continue;
+				}
+				LeopardTranscript transcript = leopard.processFile(file.getAbsolutePath());
+				file.delete();
+
+				try (FileOutputStream fos = new FileOutputStream(new File(protocolDirectory, user.getName() + ".txt"))) {
+					fos.write(transcript.getTranscriptString().getBytes());
+				} catch (IOException e) {
+					logger.error("failed to save protocol transcript from " + user.getName(), e);
+				}
+			} catch (LeopardException e) {
+				logger.error("failed to process protocol from " + user.getName(), e);
+			}
+		}
+		leopard.delete();
+
+		AudioManager audioManager = guild.getAudioManager();
+		audioManager.setReceivingHandler(null);
+		audioManager.closeAudioConnection();
+		receiver = null;
+	}
+}

src/main/java/de/slimecloud/slimeball/features/staff/meeting/protocol/commands/MeetingProtocolStartCommand.java

@@ -0,0 +1,27 @@
+package de.slimecloud.slimeball.features.staff.meeting.protocol.commands;
+
+import de.mineking.discordutils.commands.ApplicationCommand;
+import de.mineking.discordutils.commands.ApplicationCommandMethod;
+import de.slimecloud.slimeball.main.SlimeBot;
+import net.dv8tion.jda.api.entities.GuildVoiceState;
+import net.dv8tion.jda.api.entities.channel.unions.AudioChannelUnion;
+import net.dv8tion.jda.api.events.interaction.command.SlashCommandInteractionEvent;
+import org.jetbrains.annotations.NotNull;
+
+@ApplicationCommand(name = "protocol-start", description = "Startet das Automatische Meeting Protokoll", defer = true)
+public class MeetingProtocolStartCommand {
+
+	@ApplicationCommandMethod
+	public void performCommand(@NotNull SlimeBot bot, @NotNull SlashCommandInteractionEvent event) {
+		if (bot.getMeetingProtocol().getReceiver () != null) event.getHook().sendMessage("stop meeting recording first").queue();
+		GuildVoiceState state = event.getMember().getVoiceState();
+		if (state != null) {
+			AudioChannelUnion union = state.getChannel();
+			if (union != null) {
+				bot.getMeetingProtocol().start(union.asVoiceChannel());
+				event.getHook().sendMessage("meeting recording started").queue();
+			} else event.getHook().sendMessage("connect first to a voice channel").queue();
+		} else event.getHook().sendMessage("connect first to a voice channel").queue();
+	}
+
+}

src/main/java/de/slimecloud/slimeball/features/staff/meeting/protocol/commands/MeetingProtocolStopCommand.java

@@ -0,0 +1,19 @@
+package de.slimecloud.slimeball.features.staff.meeting.protocol.commands;
+
+import de.mineking.discordutils.commands.ApplicationCommand;
+import de.mineking.discordutils.commands.ApplicationCommandMethod;
+import de.slimecloud.slimeball.main.SlimeBot;
+import net.dv8tion.jda.api.events.interaction.command.SlashCommandInteractionEvent;
+import org.jetbrains.annotations.NotNull;
+
+@ApplicationCommand(name = "protocol-start", description = "Startet das Automatische Meeting Protokoll", defer = true)
+public class MeetingProtocolStopCommand {
+
+	@ApplicationCommandMethod
+	public void performCommand(@NotNull SlimeBot bot, @NotNull SlashCommandInteractionEvent event) {
+		if (bot.getMeetingProtocol().getReceiver() == null) event.getHook().sendMessage("start meeting recording first").queue();
+		bot.getMeetingProtocol().stop(event.getGuild());
+		event.getHook().sendMessage("meeting recording stopped!").queue();
+	}
+
+}

src/main/java/de/slimecloud/slimeball/main/SlimeBot.java

@@ -51,7 +51,9 @@
 import de.slimecloud.slimeball.features.report.commands.UserReportCommand;
 import de.slimecloud.slimeball.features.report.commands.UserReportSlashCommand;
 import de.slimecloud.slimeball.features.staff.StaffMessage;
-import de.slimecloud.slimeball.features.staff.TeamMeeting;
+import de.slimecloud.slimeball.features.staff.meeting.TeamMeeting;
+import de.slimecloud.slimeball.features.staff.meeting.commands.MeetingCommand;
+import de.slimecloud.slimeball.features.staff.meeting.protocol.MeetingProtocol;
 import de.slimecloud.slimeball.features.staff.absence.Absence;
 import de.slimecloud.slimeball.features.staff.absence.AbsenceCommand;
 import de.slimecloud.slimeball.features.staff.absence.AbsenceScheduler;
@@ -130,6 +132,7 @@ public class SlimeBot extends ListenerAdapter {
 	private final GitHubAPI github;
 	private final Spotify spotify;
 	private final Youtube youtube;
+	private final MeetingProtocol meetingProtocol;
 
 	private final MemberCount memberCount;
 	private final RoleMemberCount roleMemberCount;
@@ -203,7 +206,7 @@ public SlimeBot(@NotNull Config config, @NotNull Dotenv credentials) throws IOEx
 			spotify = null;
 		}
 
-		//Initalize Youtube API
+		//Initialize Youtube API
 		if (config.getYoutube().isPresent()) {
 			String[] keys = getCredentialsArray("YOUTUBE_API_KEY");
 			if (keys.length > 0) youtube = new Youtube(keys, this, config.getYoutube().get());
@@ -216,6 +219,13 @@ public SlimeBot(@NotNull Config config, @NotNull Dotenv credentials) throws IOEx
 			youtube = null;
 		}
 
+		//Initialize Meeting Protocol
+		if (credentials.get("PICOVOICE_ACCESS_KEY") != null) meetingProtocol = new MeetingProtocol(credentials.get("PICOVOICE_ACCESS_KEY"));
+		else {
+			logger.warn("Meeting Protocol disabled due to missing picovoice credentials");
+			meetingProtocol = null;
+		}
+
 
 		//Setup JDA
 		JDABuilder builder = JDABuilder.createDefault(credentials.get("DISCORD_TOKEN"))
@@ -270,6 +280,8 @@ public SlimeBot(@NotNull Config config, @NotNull Dotenv credentials) throws IOEx
 					//old mee6 custom commands
 					manager.registerCommand(SocialsCommand.class);
 
+					manager.registerCommand(MeetingCommand.class);
+
 					//Register report commands
 					if (reports != null) {
 						manager.registerCommand(ReportCommand.class);