diff --git a/build.gradle b/build.gradle index 9fadca50..1a76a398 100644 --- a/build.gradle +++ b/build.gradle @@ -32,6 +32,8 @@ dependencies { implementation 'de.cyklon:ReflectionUtils:192873e' implementation 'de.cyklon:JEvent:879a575' + implementation 'ai.picovoice:leopard-java:2.0.2' + implementation 'se.michaelthelin.spotify:spotify-web-api-java:9.0.0-RC1' implementation 'com.github.minndevelopment:emoji-java:master-SNAPSHOT' implementation 'org.kohsuke:github-api:1.321' diff --git a/run_template/credentials b/run_template/credentials index 59d6dca8..2bd0b2e3 100644 --- a/run_template/credentials +++ b/run_template/credentials @@ -16,4 +16,7 @@ SPOTIFY_CLIENT_SECRET=-- Spotify Client Secret -- # Optional. If configured, notifications about new youtube videos and livestreams are sent # you can enter an infinite number of api keys according to the same principle as below YOUTUBE_API_KEY_0=-- Youtube API Key -- -YOUTUBE_API_KEY_1=-- other Youtube API Key -- \ No newline at end of file +YOUTUBE_API_KEY_1=-- other Youtube API Key -- + +# Optional. If configured, meeting protocol will automatically be created +PICOVOICE_ACCESS_KEY=-- PicoVoiceAI Access Key -- \ No newline at end of file diff --git a/src/main/java/de/slimecloud/slimeball/config/GuildConfig.java b/src/main/java/de/slimecloud/slimeball/config/GuildConfig.java index b37346d0..0e71f213 100644 --- a/src/main/java/de/slimecloud/slimeball/config/GuildConfig.java +++ b/src/main/java/de/slimecloud/slimeball/config/GuildConfig.java @@ -10,7 +10,7 @@ import de.slimecloud.slimeball.features.fdmds.FdmdsConfig; import de.slimecloud.slimeball.features.level.GuildLevelConfig; import de.slimecloud.slimeball.features.moderation.AutoDeleteFlag; -import de.slimecloud.slimeball.features.staff.MeetingConfig; +import de.slimecloud.slimeball.features.staff.meeting.MeetingConfig; import de.slimecloud.slimeball.features.staff.StaffConfig; import de.slimecloud.slimeball.features.staff.absence.AbsenceConfig; import de.slimecloud.slimeball.features.statistic.StatisticConfig; diff --git a/src/main/java/de/slimecloud/slimeball/features/staff/MeetingConfig.java b/src/main/java/de/slimecloud/slimeball/features/staff/meeting/MeetingConfig.java similarity index 99% rename from src/main/java/de/slimecloud/slimeball/features/staff/MeetingConfig.java rename to src/main/java/de/slimecloud/slimeball/features/staff/meeting/MeetingConfig.java index 163f23f8..4fe7174c 100644 --- a/src/main/java/de/slimecloud/slimeball/features/staff/MeetingConfig.java +++ b/src/main/java/de/slimecloud/slimeball/features/staff/meeting/MeetingConfig.java @@ -1,4 +1,4 @@ -package de.slimecloud.slimeball.features.staff; +package de.slimecloud.slimeball.features.staff.meeting; import de.slimecloud.slimeball.config.ConfigCategory; import de.slimecloud.slimeball.config.engine.ConfigField; diff --git a/src/main/java/de/slimecloud/slimeball/features/staff/TeamMeeting.java b/src/main/java/de/slimecloud/slimeball/features/staff/meeting/TeamMeeting.java similarity index 99% rename from src/main/java/de/slimecloud/slimeball/features/staff/TeamMeeting.java rename to src/main/java/de/slimecloud/slimeball/features/staff/meeting/TeamMeeting.java index 40e3e711..38cc963e 100644 --- a/src/main/java/de/slimecloud/slimeball/features/staff/TeamMeeting.java +++ b/src/main/java/de/slimecloud/slimeball/features/staff/meeting/TeamMeeting.java @@ -1,4 +1,4 @@ -package de.slimecloud.slimeball.features.staff; +package de.slimecloud.slimeball.features.staff.meeting; import de.slimecloud.slimeball.config.GuildConfig; import de.slimecloud.slimeball.main.Main; diff --git a/src/main/java/de/slimecloud/slimeball/features/staff/meeting/commands/MeetingCommand.java b/src/main/java/de/slimecloud/slimeball/features/staff/meeting/commands/MeetingCommand.java new file mode 100644 index 00000000..0fa1c9cc --- /dev/null +++ b/src/main/java/de/slimecloud/slimeball/features/staff/meeting/commands/MeetingCommand.java @@ -0,0 +1,29 @@ +package de.slimecloud.slimeball.features.staff.meeting.commands; + +import de.mineking.discordutils.commands.ApplicationCommand; +import de.mineking.discordutils.commands.Command; +import de.mineking.discordutils.commands.Setup; +import de.mineking.discordutils.commands.condition.IRegistrationCondition; +import de.mineking.discordutils.commands.condition.Scope; +import de.mineking.discordutils.commands.context.ICommandContext; +import de.mineking.discordutils.list.ListManager; +import de.slimecloud.slimeball.config.GuildConfig; +import de.slimecloud.slimeball.features.staff.meeting.protocol.commands.MeetingProtocolStartCommand; +import de.slimecloud.slimeball.features.staff.meeting.protocol.commands.MeetingProtocolStopCommand; +import de.slimecloud.slimeball.main.CommandPermission; +import de.slimecloud.slimeball.main.SlimeBot; +import org.jetbrains.annotations.NotNull; + +@ApplicationCommand(name = "meeting", description = "Verwalte Team Meetings", scope = Scope.GUILD) +public class MeetingCommand { + + public final CommandPermission permission = CommandPermission.TEAM; + public final IRegistrationCondition condition = (manager, guild, cache) -> cache.getState("config").getMeeting().isPresent(); + + @Setup + public static void setup(@NotNull SlimeBot bot, @NotNull Command command, @NotNull ListManager manager) { + command.addSubcommand(MeetingProtocolStartCommand.class); + command.addSubcommand(MeetingProtocolStopCommand.class); + } + +} diff --git a/src/main/java/de/slimecloud/slimeball/features/staff/meeting/protocol/AudioReceiver.java b/src/main/java/de/slimecloud/slimeball/features/staff/meeting/protocol/AudioReceiver.java new file mode 100644 index 00000000..dda73577 --- /dev/null +++ b/src/main/java/de/slimecloud/slimeball/features/staff/meeting/protocol/AudioReceiver.java @@ -0,0 +1,47 @@ +package de.slimecloud.slimeball.features.staff.meeting.protocol; + +import net.dv8tion.jda.api.audio.AudioReceiveHandler; +import net.dv8tion.jda.api.audio.UserAudio; + +import javax.sound.sampled.AudioInputStream; +import java.io.ByteArrayInputStream; +import java.util.*; + +public class AudioReceiver implements AudioReceiveHandler +{ + private static final double VOLUME = 1.0; + private final Map> received = new LinkedHashMap<>(); + + @Override + public boolean canReceiveUser() { + return true; + } + + @Override + public void handleUserAudio(UserAudio userAudio) { + received.computeIfAbsent(userAudio.getUser().getIdLong(), id -> new LinkedList<>()).add(userAudio.getAudioData(VOLUME)); + } + + public Set getUsers() { + return received.keySet(); + } + + public byte[] getBytes(Long id) { + int size = 0; + for (byte[] bytes : received.get(id)) size += bytes.length; + byte[] data = new byte[size]; + int i = 0; + for (byte[] bytes : received.get(id)) { + for (byte b : bytes) { + data[i++] = b; + } + } + return data; + } + + public AudioInputStream getAudioStream(Long id) { + byte[] data = getBytes(id); + return new AudioInputStream(new ByteArrayInputStream(data), OUTPUT_FORMAT, data.length); + } + +} diff --git a/src/main/java/de/slimecloud/slimeball/features/staff/meeting/protocol/MeetingProtocol.java b/src/main/java/de/slimecloud/slimeball/features/staff/meeting/protocol/MeetingProtocol.java new file mode 100644 index 00000000..e79876a2 --- /dev/null +++ b/src/main/java/de/slimecloud/slimeball/features/staff/meeting/protocol/MeetingProtocol.java @@ -0,0 +1,107 @@ +package de.slimecloud.slimeball.features.staff.meeting.protocol; + +import ai.picovoice.leopard.Leopard; +import ai.picovoice.leopard.LeopardException; +import ai.picovoice.leopard.LeopardTranscript; +import de.mineking.discordutils.commands.ApplicationCommand; +import de.mineking.discordutils.commands.ApplicationCommandMethod; +import de.slimecloud.slimeball.main.SlimeBot; +import lombok.Getter; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import net.dv8tion.jda.api.JDA; +import net.dv8tion.jda.api.entities.Guild; +import net.dv8tion.jda.api.entities.GuildVoiceState; +import net.dv8tion.jda.api.entities.User; +import net.dv8tion.jda.api.entities.channel.concrete.VoiceChannel; +import net.dv8tion.jda.api.entities.channel.unions.AudioChannelUnion; +import net.dv8tion.jda.api.events.interaction.command.SlashCommandInteractionEvent; +import net.dv8tion.jda.api.managers.AudioManager; +import org.jetbrains.annotations.NotNull; + +import javax.sound.sampled.AudioFileFormat; +import javax.sound.sampled.AudioInputStream; +import javax.sound.sampled.AudioSystem; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; + +@Slf4j +public class MeetingProtocol { + + private static final File protocolDirectory = new File("protocol"); + + static { + protocolDirectory.mkdirs(); + } + + private final String picovoice_access_key; + private final File leopard_model; + @Getter + private AudioReceiver receiver; + + public MeetingProtocol(String picovoice_access_key) { + this.picovoice_access_key = picovoice_access_key; + URL url = MeetingProtocol.class.getClassLoader().getResource("leopard_params_de.pv"); + if (url == null || url.getPath().isBlank()) throw new RuntimeException("Leopard model not found"); + this.leopard_model = new File(url.getPath()); + } + + public void start(VoiceChannel vc) { + receiver = new AudioReceiver(); + Guild guild = vc.getGuild(); + AudioManager audioManager = guild.getAudioManager(); + audioManager.openAudioConnection(vc); + audioManager.setReceivingHandler(receiver); + } + + @SneakyThrows(LeopardException.class) + public void stop(Guild guild) { + JDA jda = guild.getJDA(); + + List users = new ArrayList<>(); + for (Long id : receiver.getUsers()) users.add(jda.getUserById(id)); + + for (User user : users) { + try (AudioInputStream ais = receiver.getAudioStream(user.getIdLong())) { + AudioSystem.write(ais, AudioFileFormat.Type.WAVE, new File(protocolDirectory, String.format("%s.wav", user.getName()))); + } catch (IOException e) { + logger.error("failed to save protocol as wav from " + user.getName(), e); + } + } + + Leopard leopard = new Leopard.Builder() + .setAccessKey(picovoice_access_key) + .setModelPath(leopard_model.getAbsolutePath()) + .build(); + + for (User user : users) { + try { + File file = new File(protocolDirectory, String.format("%s.wav", user.getName())); + if (!file.exists()) { + logger.debug("skip user " + user.getName() + " for processing because the " + file.getName() + " file is missing"); + continue; + } + LeopardTranscript transcript = leopard.processFile(file.getAbsolutePath()); + file.delete(); + + try (FileOutputStream fos = new FileOutputStream(new File(protocolDirectory, user.getName() + ".txt"))) { + fos.write(transcript.getTranscriptString().getBytes()); + } catch (IOException e) { + logger.error("failed to save protocol transcript from " + user.getName(), e); + } + } catch (LeopardException e) { + logger.error("failed to process protocol from " + user.getName(), e); + } + } + leopard.delete(); + + AudioManager audioManager = guild.getAudioManager(); + audioManager.setReceivingHandler(null); + audioManager.closeAudioConnection(); + receiver = null; + } +} diff --git a/src/main/java/de/slimecloud/slimeball/features/staff/meeting/protocol/commands/MeetingProtocolStartCommand.java b/src/main/java/de/slimecloud/slimeball/features/staff/meeting/protocol/commands/MeetingProtocolStartCommand.java new file mode 100644 index 00000000..8e8cc9ed --- /dev/null +++ b/src/main/java/de/slimecloud/slimeball/features/staff/meeting/protocol/commands/MeetingProtocolStartCommand.java @@ -0,0 +1,27 @@ +package de.slimecloud.slimeball.features.staff.meeting.protocol.commands; + +import de.mineking.discordutils.commands.ApplicationCommand; +import de.mineking.discordutils.commands.ApplicationCommandMethod; +import de.slimecloud.slimeball.main.SlimeBot; +import net.dv8tion.jda.api.entities.GuildVoiceState; +import net.dv8tion.jda.api.entities.channel.unions.AudioChannelUnion; +import net.dv8tion.jda.api.events.interaction.command.SlashCommandInteractionEvent; +import org.jetbrains.annotations.NotNull; + +@ApplicationCommand(name = "protocol-start", description = "Startet das Automatische Meeting Protokoll", defer = true) +public class MeetingProtocolStartCommand { + + @ApplicationCommandMethod + public void performCommand(@NotNull SlimeBot bot, @NotNull SlashCommandInteractionEvent event) { + if (bot.getMeetingProtocol().getReceiver () != null) event.getHook().sendMessage("stop meeting recording first").queue(); + GuildVoiceState state = event.getMember().getVoiceState(); + if (state != null) { + AudioChannelUnion union = state.getChannel(); + if (union != null) { + bot.getMeetingProtocol().start(union.asVoiceChannel()); + event.getHook().sendMessage("meeting recording started").queue(); + } else event.getHook().sendMessage("connect first to a voice channel").queue(); + } else event.getHook().sendMessage("connect first to a voice channel").queue(); + } + +} diff --git a/src/main/java/de/slimecloud/slimeball/features/staff/meeting/protocol/commands/MeetingProtocolStopCommand.java b/src/main/java/de/slimecloud/slimeball/features/staff/meeting/protocol/commands/MeetingProtocolStopCommand.java new file mode 100644 index 00000000..9337fdba --- /dev/null +++ b/src/main/java/de/slimecloud/slimeball/features/staff/meeting/protocol/commands/MeetingProtocolStopCommand.java @@ -0,0 +1,19 @@ +package de.slimecloud.slimeball.features.staff.meeting.protocol.commands; + +import de.mineking.discordutils.commands.ApplicationCommand; +import de.mineking.discordutils.commands.ApplicationCommandMethod; +import de.slimecloud.slimeball.main.SlimeBot; +import net.dv8tion.jda.api.events.interaction.command.SlashCommandInteractionEvent; +import org.jetbrains.annotations.NotNull; + +@ApplicationCommand(name = "protocol-start", description = "Startet das Automatische Meeting Protokoll", defer = true) +public class MeetingProtocolStopCommand { + + @ApplicationCommandMethod + public void performCommand(@NotNull SlimeBot bot, @NotNull SlashCommandInteractionEvent event) { + if (bot.getMeetingProtocol().getReceiver() == null) event.getHook().sendMessage("start meeting recording first").queue(); + bot.getMeetingProtocol().stop(event.getGuild()); + event.getHook().sendMessage("meeting recording stopped!").queue(); + } + +} diff --git a/src/main/java/de/slimecloud/slimeball/main/SlimeBot.java b/src/main/java/de/slimecloud/slimeball/main/SlimeBot.java index 218f4556..08d9dd29 100644 --- a/src/main/java/de/slimecloud/slimeball/main/SlimeBot.java +++ b/src/main/java/de/slimecloud/slimeball/main/SlimeBot.java @@ -51,7 +51,9 @@ import de.slimecloud.slimeball.features.report.commands.UserReportCommand; import de.slimecloud.slimeball.features.report.commands.UserReportSlashCommand; import de.slimecloud.slimeball.features.staff.StaffMessage; -import de.slimecloud.slimeball.features.staff.TeamMeeting; +import de.slimecloud.slimeball.features.staff.meeting.TeamMeeting; +import de.slimecloud.slimeball.features.staff.meeting.commands.MeetingCommand; +import de.slimecloud.slimeball.features.staff.meeting.protocol.MeetingProtocol; import de.slimecloud.slimeball.features.staff.absence.Absence; import de.slimecloud.slimeball.features.staff.absence.AbsenceCommand; import de.slimecloud.slimeball.features.staff.absence.AbsenceScheduler; @@ -130,6 +132,7 @@ public class SlimeBot extends ListenerAdapter { private final GitHubAPI github; private final Spotify spotify; private final Youtube youtube; + private final MeetingProtocol meetingProtocol; private final MemberCount memberCount; private final RoleMemberCount roleMemberCount; @@ -203,7 +206,7 @@ public SlimeBot(@NotNull Config config, @NotNull Dotenv credentials) throws IOEx spotify = null; } - //Initalize Youtube API + //Initialize Youtube API if (config.getYoutube().isPresent()) { String[] keys = getCredentialsArray("YOUTUBE_API_KEY"); if (keys.length > 0) youtube = new Youtube(keys, this, config.getYoutube().get()); @@ -216,6 +219,13 @@ public SlimeBot(@NotNull Config config, @NotNull Dotenv credentials) throws IOEx youtube = null; } + //Initialize Meeting Protocol + if (credentials.get("PICOVOICE_ACCESS_KEY") != null) meetingProtocol = new MeetingProtocol(credentials.get("PICOVOICE_ACCESS_KEY")); + else { + logger.warn("Meeting Protocol disabled due to missing picovoice credentials"); + meetingProtocol = null; + } + //Setup JDA JDABuilder builder = JDABuilder.createDefault(credentials.get("DISCORD_TOKEN")) @@ -270,6 +280,8 @@ public SlimeBot(@NotNull Config config, @NotNull Dotenv credentials) throws IOEx //old mee6 custom commands manager.registerCommand(SocialsCommand.class); + manager.registerCommand(MeetingCommand.class); + //Register report commands if (reports != null) { manager.registerCommand(ReportCommand.class); diff --git a/src/main/resources/leopard_params_de.pv b/src/main/resources/leopard_params_de.pv new file mode 100644 index 00000000..dd4f68b8 Binary files /dev/null and b/src/main/resources/leopard_params_de.pv differ