diff --git a/README.md b/README.md index c33500e67b..f6335070ba 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,10 @@ This library is available on maven central. The latest version is always shown i The minimum java version supported by JDA is **Java SE 8**. JDA also uses JSR 305 to support solid interoperability with Kotlin out of the box. +> [!NOTE] +> If you wish to support sending raw audio (and not Opus directly), +> you will also have to add the [`JDA-opus-jna`](opus-jna) dependency. + ### Gradle ```gradle @@ -63,12 +67,10 @@ repositories { dependencies { implementation("net.dv8tion:JDA:$version") { // replace $version with the latest version - // Optionally disable audio natives to reduce jar size by excluding `opus-java` and `tink` + // Optionally disable audio natives to reduce jar size by excluding `tink` // Gradle DSL: - // exclude module: 'opus-java' // required for encoding audio into opus, not needed if audio is already provided in opus encoding // exclude module: 'tink' // required for encrypting and decrypting audio // Kotlin DSL: - // exclude(module="opus-java") // required for encoding audio into opus, not needed if audio is already provided in opus encoding // exclude(module="tink") // required for encrypting and decrypting audio } } @@ -81,13 +83,8 @@ dependencies { net.dv8tion JDA $version - + - + + runtime + +``` diff --git a/opus-jna/build.gradle.kts b/opus-jna/build.gradle.kts new file mode 100644 index 0000000000..cb8214547d --- /dev/null +++ b/opus-jna/build.gradle.kts @@ -0,0 +1,127 @@ +/* + * Copyright 2015 Austin Keener, Michael Ritter, Florian Spieß, and the JDA contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import net.dv8tion.jda.tasks.configureJavadoc +import net.dv8tion.jda.tasks.registerPublication + +plugins { + environment + `java-library` + `jda-publish` +} + +//////////////////////////////////// +// // +// Module Configuration // +// // +//////////////////////////////////// + +group = rootProject.group +version = rootProject.version +val fullProjectName = "${rootProject.name}-${project.name}" + +base { + archivesName.set(fullProjectName) +} + +java { + withSourcesJar() +} + + +//////////////////////////////////// +// // +// Dependency Configuration // +// // +//////////////////////////////////// + +repositories { + mavenCentral() +} + +dependencies { + + /* Internal dependencies */ + + // JDA + implementation(rootProject) + + //Logger + implementation(libs.slf4j) + + //Opus library support + implementation(libs.opus) + + //we use this only together with opus-java + // if that dependency is excluded it also doesn't need jna anymore + // since jna is a transitive runtime dependency of opus-java we don't include it explicitly as dependency + implementation(libs.jna) + + /* Annotations */ + + //Code safety + compileOnly(libs.findbugs) +} + +//////////////////////////////////// +// // +// Build Task Configuration // +// // +//////////////////////////////////// + +val jar by tasks.getting(Jar::class) { + archiveBaseName.set(fullProjectName) + manifest.attributes("Implementation-Version" to project.version, "Automatic-Module-Name" to "net.dv8tion.jda") +} + +val javadoc by configureJavadoc( + targetVersion = JavaVersion.VERSION_1_8, + failOnError = projectEnvironment.isGithubAction, + overviewFile = null, +) + +val javadocJar by tasks.registering(Jar::class) { + dependsOn(javadoc) + archiveClassifier.set("javadoc") + from(javadoc.destinationDir) +} + +tasks.withType { + options.encoding = "UTF-8" + options.isIncremental = true + + val args = mutableListOf("-Xlint:deprecation", "-Xlint:unchecked") + + options.release = 8 + + options.compilerArgs.addAll(args) +} + +//////////////////////////////////// +// // +// Publishing And Signing // +// // +//////////////////////////////////// + +registerPublication( + name = fullProjectName, + description = "Opus support for JDA, based on JNA", + url = "https://github.com/discord-jda/JDA/tree/master/opus-jna", +) { + from(components["java"]) + + artifact(javadocJar) +} diff --git a/opus-jna/src/main/java/net/dv8tion/jda/internal/audio/OpusJnaCodecFactory.java b/opus-jna/src/main/java/net/dv8tion/jda/internal/audio/OpusJnaCodecFactory.java new file mode 100644 index 0000000000..317d12047d --- /dev/null +++ b/opus-jna/src/main/java/net/dv8tion/jda/internal/audio/OpusJnaCodecFactory.java @@ -0,0 +1,82 @@ +/* + * Copyright 2015 Austin Keener, Michael Ritter, Florian Spieß, and the JDA contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.dv8tion.jda.internal.audio; + +import club.minnced.opus.util.OpusLibrary; +import com.sun.jna.ptr.PointerByReference; +import net.dv8tion.jda.api.audio.OpusPacket; +import net.dv8tion.jda.internal.utils.JDALogger; +import org.slf4j.Logger; +import tomp2p.opuswrapper.Opus; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.nio.IntBuffer; + +public class OpusJnaCodecFactory implements OpusCodecFactory { + + public static final Logger LOG = JDALogger.getLog(OpusJnaCodecFactory.class); + + private static boolean initialized = false; + private static boolean audioSupported = false; + + @Override + public synchronized boolean initialize() throws Exception + { + if (initialized) + return audioSupported; + initialized = true; + + if (OpusLibrary.isInitialized()) + return audioSupported = true; + OpusLibrary.loadFromJar(); + return audioSupported = true; + } + + @Nonnull + @Override + public OpusDecoder createDecoder(int ssrc) + { + if (!initialized) + throw new IllegalStateException("Opus is not initialized"); + + IntBuffer error = IntBuffer.allocate(1); + PointerByReference opusDecoder = Opus.INSTANCE.opus_decoder_create(OpusPacket.OPUS_SAMPLE_RATE, OpusPacket.OPUS_CHANNEL_COUNT, error); + if (error.get() != Opus.OPUS_OK && opusDecoder == null) + throw new IllegalStateException("Received error code from opus_decoder_create(...): " + error.get()); + + return new OpusJnaDecoder(ssrc, opusDecoder); + } + + @Nullable + @Override + public OpusEncoder createEncoder() + { + if (!initialized) + throw new IllegalStateException("Opus is not initialized"); + + IntBuffer error = IntBuffer.allocate(1); + PointerByReference opusEncoder = Opus.INSTANCE.opus_encoder_create(OpusPacket.OPUS_SAMPLE_RATE, OpusPacket.OPUS_CHANNEL_COUNT, Opus.OPUS_APPLICATION_AUDIO, error); + if (error.get() != Opus.OPUS_OK && opusEncoder == null) + { + LOG.error("Received error status from opus_encoder_create(...): {}", error.get()); + return null; + } + + return new OpusJnaEncoder(opusEncoder); + } +} diff --git a/opus-jna/src/main/java/net/dv8tion/jda/internal/audio/OpusJnaDecoder.java b/opus-jna/src/main/java/net/dv8tion/jda/internal/audio/OpusJnaDecoder.java new file mode 100644 index 0000000000..4d237eb0cb --- /dev/null +++ b/opus-jna/src/main/java/net/dv8tion/jda/internal/audio/OpusJnaDecoder.java @@ -0,0 +1,140 @@ +/* + * Copyright 2015 Austin Keener, Michael Ritter, Florian Spieß, and the JDA contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.dv8tion.jda.internal.audio; + +import com.sun.jna.ptr.PointerByReference; +import net.dv8tion.jda.api.audio.OpusPacket; +import tomp2p.opuswrapper.Opus; + +import javax.annotation.Nullable; +import java.nio.ByteBuffer; +import java.nio.ShortBuffer; + +public class OpusJnaDecoder implements OpusDecoder +{ + protected int ssrc; + protected char lastSeq; + protected int lastTimestamp; + protected PointerByReference opusDecoder; + + protected OpusJnaDecoder(int ssrc, PointerByReference opusDecoder) + { + this.ssrc = ssrc; + this.lastSeq = (char) -1; + this.lastTimestamp = -1; + this.opusDecoder = opusDecoder; + } + + @Override + public boolean isInOrder(char newSeq) + { + return lastSeq == (char) -1 || newSeq > lastSeq || lastSeq - newSeq > 10; + } + + @Override + public short[] decode(@Nullable AudioPacket decryptedPacket) + { + int result; + ShortBuffer decoded = ShortBuffer.allocate(4096); + synchronized (this) + { + if (opusDecoder == null) + throw new IllegalStateException("Decoder is closed."); + + if (decryptedPacket == null) //Flag for packet-loss + { + result = Opus.INSTANCE.opus_decode(opusDecoder, null, 0, decoded, OpusPacket.OPUS_FRAME_SIZE, 0); + lastSeq = (char) -1; + lastTimestamp = -1; + } + else + { + this.lastSeq = decryptedPacket.getSequence(); + this.lastTimestamp = decryptedPacket.getTimestamp(); + + ByteBuffer encodedAudio = decryptedPacket.getEncodedAudio(); + int length = encodedAudio.remaining(); + int offset = encodedAudio.arrayOffset() + encodedAudio.position(); + byte[] buf = new byte[length]; + byte[] data = encodedAudio.array(); + System.arraycopy(data, offset, buf, 0, length); + result = Opus.INSTANCE.opus_decode(opusDecoder, buf, buf.length, decoded, OpusPacket.OPUS_FRAME_SIZE, 0); + } + } + + //If we get a result that is less than 0, then there was an error. Return null as a signifier. + if (result < 0) + { + handleDecodeError(result); + return null; + } + + short[] audio = new short[result * 2]; + decoded.get(audio); + return audio; + } + + private void handleDecodeError(int result) + { + StringBuilder b = new StringBuilder("Decoder failed to decode audio from user with code "); + switch (result) + { + case Opus.OPUS_BAD_ARG: //-1 + b.append("OPUS_BAD_ARG"); + break; + case Opus.OPUS_BUFFER_TOO_SMALL: //-2 + b.append("OPUS_BUFFER_TOO_SMALL"); + break; + case Opus.OPUS_INTERNAL_ERROR: //-3 + b.append("OPUS_INTERNAL_ERROR"); + break; + case Opus.OPUS_INVALID_PACKET: //-4 + b.append("OPUS_INVALID_PACKET"); + break; + case Opus.OPUS_UNIMPLEMENTED: //-5 + b.append("OPUS_UNIMPLEMENTED"); + break; + case Opus.OPUS_INVALID_STATE: //-6 + b.append("OPUS_INVALID_STATE"); + break; + case Opus.OPUS_ALLOC_FAIL: //-7 + b.append("OPUS_ALLOC_FAIL"); + break; + default: + b.append(result); + } + AudioConnection.LOG.debug("{}", b); + } + + @Override + public synchronized void close() + { + if (opusDecoder != null) + { + Opus.INSTANCE.opus_decoder_destroy(opusDecoder); + opusDecoder = null; + } + } + + @Override + @SuppressWarnings("deprecation") /* If this was in JDK9 we would be using java.lang.ref.Cleaner instead! */ + protected void finalize() throws Throwable + { + super.finalize(); + close(); + } +} diff --git a/opus-jna/src/main/java/net/dv8tion/jda/internal/audio/OpusJnaEncoder.java b/opus-jna/src/main/java/net/dv8tion/jda/internal/audio/OpusJnaEncoder.java new file mode 100644 index 0000000000..ef4c1a8982 --- /dev/null +++ b/opus-jna/src/main/java/net/dv8tion/jda/internal/audio/OpusJnaEncoder.java @@ -0,0 +1,88 @@ +/* + * Copyright 2015 Austin Keener, Michael Ritter, Florian Spieß, and the JDA contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.dv8tion.jda.internal.audio; + +import com.sun.jna.ptr.PointerByReference; +import net.dv8tion.jda.api.audio.OpusPacket; +import net.dv8tion.jda.internal.utils.JDALogger; +import org.slf4j.Logger; +import tomp2p.opuswrapper.Opus; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.nio.Buffer; +import java.nio.ByteBuffer; +import java.nio.ShortBuffer; + +public class OpusJnaEncoder implements OpusEncoder +{ + public static final Logger LOG = JDALogger.getLog(OpusJnaEncoder.class); + + private PointerByReference opusEncoder; + + protected OpusJnaEncoder(PointerByReference opusEncoder) + { + this.opusEncoder = opusEncoder; + } + + @Nullable + @Override + public ByteBuffer encode(@Nonnull ByteBuffer rawAudio) + { + int result; + ByteBuffer encoded = ByteBuffer.allocate(4096); + synchronized (this) + { + if (opusEncoder == null) + throw new IllegalStateException("Encoder is closed."); + + ShortBuffer nonEncodedBuffer = ShortBuffer.allocate(rawAudio.remaining() / 2); + for (int i = rawAudio.position(); i < rawAudio.limit(); i += 2) + { + int firstByte = (0x000000FF & rawAudio.get(i)); //Promotes to int and handles the fact that it was unsigned. + int secondByte = (0x000000FF & rawAudio.get(i + 1)); + + //Combines the 2 bytes into a short. Opus deals with unsigned shorts, not bytes. + short toShort = (short) ((firstByte << 8) | secondByte); + + nonEncodedBuffer.put(toShort); + } + ((Buffer) nonEncodedBuffer).flip(); + + result = Opus.INSTANCE.opus_encode(opusEncoder, nonEncodedBuffer, OpusPacket.OPUS_FRAME_SIZE, encoded, encoded.capacity()); + } + + if (result <= 0) + { + LOG.error("Received error code from opus_encode(...): {}", result); + return null; + } + + ((Buffer) encoded).position(0).limit(result); + return encoded; + } + + @Override + public void close() + { + if (opusEncoder != null) + { + Opus.INSTANCE.opus_encoder_destroy(opusEncoder); + opusEncoder = null; + } + } +} diff --git a/opus-jna/src/main/resources/META-INF/services/net.dv8tion.jda.internal.audio.OpusCodecFactory b/opus-jna/src/main/resources/META-INF/services/net.dv8tion.jda.internal.audio.OpusCodecFactory new file mode 100644 index 0000000000..aac73ee594 --- /dev/null +++ b/opus-jna/src/main/resources/META-INF/services/net.dv8tion.jda.internal.audio.OpusCodecFactory @@ -0,0 +1 @@ +net.dv8tion.jda.internal.audio.OpusJnaCodecFactory diff --git a/settings.gradle.kts b/settings.gradle.kts index 926fb9e63a..33ffa14ac5 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -3,3 +3,7 @@ plugins { } rootProject.name = "JDA" + +include( + ":opus-jna", +) diff --git a/src/examples/java/AudioEchoExample.java b/src/examples/java/AudioEchoExample.java index 3b4a404bde..c4271c4a13 100644 --- a/src/examples/java/AudioEchoExample.java +++ b/src/examples/java/AudioEchoExample.java @@ -37,6 +37,9 @@ public class AudioEchoExample extends ListenerAdapter { + /** + * NOTE: If you copy this code somewhere else, remember to add the Opus dependency! + */ public static void main(String[] args) { if (args.length == 0) diff --git a/src/main/java/net/dv8tion/jda/api/audio/AudioNatives.java b/src/main/java/net/dv8tion/jda/api/audio/AudioNatives.java index a6e1cfb397..41c85f1f18 100644 --- a/src/main/java/net/dv8tion/jda/api/audio/AudioNatives.java +++ b/src/main/java/net/dv8tion/jda/api/audio/AudioNatives.java @@ -16,7 +16,8 @@ package net.dv8tion.jda.api.audio; -import club.minnced.opus.util.OpusLibrary; +import net.dv8tion.jda.internal.audio.MissingOpusException; +import net.dv8tion.jda.internal.audio.OpusCodecFactoryProvider; import net.dv8tion.jda.internal.utils.JDALogger; import org.slf4j.Logger; @@ -72,9 +73,7 @@ public static synchronized boolean ensureOpus() initialized = true; try { - if (OpusLibrary.isInitialized()) - return audioSupported = true; - audioSupported = OpusLibrary.loadFromJar(); + audioSupported = OpusCodecFactoryProvider.getInstance().initialize(); } catch (Throwable e) { @@ -96,7 +95,7 @@ private static void handleException(Throwable e) { LOG.error("Sorry, JDA's audio system doesn't support this system.\n{}", e.getMessage()); } - else if (e instanceof NoClassDefFoundError) + else if (e instanceof MissingOpusException) { LOG.error("Missing opus dependency, unable to initialize audio!"); } diff --git a/src/main/java/net/dv8tion/jda/api/audio/OpusPacket.java b/src/main/java/net/dv8tion/jda/api/audio/OpusPacket.java index 4644000b82..52cd3367ea 100644 --- a/src/main/java/net/dv8tion/jda/api/audio/OpusPacket.java +++ b/src/main/java/net/dv8tion/jda/api/audio/OpusPacket.java @@ -17,7 +17,7 @@ package net.dv8tion.jda.api.audio; import net.dv8tion.jda.internal.audio.AudioPacket; -import net.dv8tion.jda.internal.audio.Decoder; +import net.dv8tion.jda.internal.audio.OpusDecoder; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -45,13 +45,13 @@ public final class OpusPacket implements Comparable private final long userId; private final byte[] opusAudio; - private final Decoder decoder; + private final OpusDecoder decoder; private final AudioPacket rawPacket; private short[] decoded; private boolean triedDecode; - public OpusPacket(@Nonnull AudioPacket packet, long userId, @Nullable Decoder decoder) + public OpusPacket(@Nonnull AudioPacket packet, long userId, @Nullable OpusDecoder decoder) { this.rawPacket = packet; this.userId = userId; @@ -156,7 +156,7 @@ public synchronized short[] decode() if (!decoder.isInOrder(getSequence())) throw new IllegalStateException("Packet is not in order"); triedDecode = true; - return decoded = decoder.decodeFromOpus(rawPacket); // null if failed to decode + return decoded = decoder.decode(rawPacket); // null if failed to decode } /** diff --git a/src/main/java/net/dv8tion/jda/internal/audio/AudioConnection.java b/src/main/java/net/dv8tion/jda/internal/audio/AudioConnection.java index 7e9d65ab63..6e128c33d0 100644 --- a/src/main/java/net/dv8tion/jda/internal/audio/AudioConnection.java +++ b/src/main/java/net/dv8tion/jda/internal/audio/AudioConnection.java @@ -17,7 +17,6 @@ package net.dv8tion.jda.internal.audio; import com.neovisionaries.ws.client.WebSocket; -import com.sun.jna.ptr.PointerByReference; import gnu.trove.map.TIntLongMap; import gnu.trove.map.TIntObjectMap; import gnu.trove.map.hash.TIntLongHashMap; @@ -37,14 +36,11 @@ import net.dv8tion.jda.internal.managers.AudioManagerImpl; import net.dv8tion.jda.internal.utils.JDALogger; import org.slf4j.Logger; -import tomp2p.opuswrapper.Opus; import javax.annotation.Nonnull; import java.net.*; import java.nio.Buffer; import java.nio.ByteBuffer; -import java.nio.IntBuffer; -import java.nio.ShortBuffer; import java.util.*; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.Executors; @@ -66,7 +62,7 @@ public class AudioConnection protected volatile DatagramSocket udpSocket; private final TIntLongMap ssrcMap = new TIntLongHashMap(); - private final TIntObjectMap opusDecoders = new TIntObjectHashMap<>(); + private final TIntObjectMap opusDecoders = new TIntObjectHashMap<>(); private final HashMap> combinedQueue = new HashMap<>(); private final String threadIdentifier; private final AudioWebSocket webSocket; @@ -76,7 +72,7 @@ public class AudioConnection protected final Condition readyCondvar = readyLock.newCondition(); private AudioChannel channel; - private PointerByReference opusEncoder; + private OpusEncoder opusEncoder; private ScheduledExecutorService combinedAudioExecutor; private IAudioSendSystem sendSystem; private Thread receiveThread; @@ -188,11 +184,11 @@ public synchronized void shutdown() } if (opusEncoder != null) { - Opus.INSTANCE.opus_encoder_destroy(opusEncoder); + opusEncoder.close(); opusEncoder = null; } - opusDecoders.valueCollection().forEach(Decoder::close); + opusDecoders.valueCollection().forEach(OpusDecoder::close); opusDecoders.clear(); MiscUtil.locked(readyLock, readyCondvar::signalAll); @@ -266,7 +262,7 @@ protected void removeUserSSRC(long userId) }); if (!modified) return; - final Decoder decoder = opusDecoders.remove(ssrcRef.get()); + final OpusDecoder decoder = opusDecoders.remove(ssrcRef.get()); if (decoder != null) // cleanup decoder decoder.close(); } @@ -290,7 +286,7 @@ protected void updateUserSSRC(int ssrc, long userId) //Only create a decoder if we are actively handling received audio. if (receiveThread != null && AudioNatives.ensureOpus()) - opusDecoders.put(ssrc, new Decoder(ssrc)); + opusDecoders.put(ssrc, getCodecFactory().createDecoder(ssrc)); } } @@ -313,7 +309,7 @@ else if (sendHandler == null && sendSystem != null) if (opusEncoder != null) { - Opus.INSTANCE.opus_encoder_destroy(opusEncoder); + opusEncoder.close(); opusEncoder = null; } } @@ -336,7 +332,7 @@ else if (receiveHandler == null && receiveThread != null) combinedAudioExecutor = null; } - opusDecoders.valueCollection().forEach(Decoder::close); + opusDecoders.valueCollection().forEach(OpusDecoder::close); opusDecoders.clear(); } else if (receiveHandler != null && !receiveHandler.canReceiveCombined() && combinedAudioExecutor != null) @@ -379,7 +375,7 @@ private synchronized void setupReceiveThread() int ssrc = decryptedPacket.getSSRC(); final long userId = ssrcMap.get(ssrc); - Decoder decoder = opusDecoders.get(ssrc); + OpusDecoder decoder = opusDecoders.get(ssrc); if (userId == ssrcMap.getNoEntryValue()) { ByteBuffer audio = decryptedPacket.getEncodedAudio(); @@ -395,7 +391,7 @@ private synchronized void setupReceiveThread() { if (AudioNatives.ensureOpus()) { - opusDecoders.put(ssrc, decoder = new Decoder(ssrc)); + opusDecoders.put(ssrc, decoder = getCodecFactory().createDecoder(ssrc)); } else if (!receiveHandler.canReceiveEncoded()) { @@ -565,33 +561,6 @@ else if (sample < Short.MIN_VALUE) } } - private ByteBuffer encodeToOpus(ByteBuffer rawAudio) - { - ShortBuffer nonEncodedBuffer = ShortBuffer.allocate(rawAudio.remaining() / 2); - ByteBuffer encoded = ByteBuffer.allocate(4096); - for (int i = rawAudio.position(); i < rawAudio.limit(); i += 2) - { - int firstByte = (0x000000FF & rawAudio.get(i)); //Promotes to int and handles the fact that it was unsigned. - int secondByte = (0x000000FF & rawAudio.get(i + 1)); - - //Combines the 2 bytes into a short. Opus deals with unsigned shorts, not bytes. - short toShort = (short) ((firstByte << 8) | secondByte); - - nonEncodedBuffer.put(toShort); - } - ((Buffer) nonEncodedBuffer).flip(); - - int result = Opus.INSTANCE.opus_encode(opusEncoder, nonEncodedBuffer, OpusPacket.OPUS_FRAME_SIZE, encoded, encoded.capacity()); - if (result <= 0) - { - LOG.error("Received error code from opus_encode(...): {}", result); - return null; - } - - ((Buffer) encoded).position(0).limit(result); - return encoded; - } - private void setSpeaking(int raw) { DataObject obj = DataObject.empty() @@ -601,6 +570,11 @@ private void setSpeaking(int raw) webSocket.send(VoiceCode.USER_SPEAKING_UPDATE, obj); } + /** Lazy load optional Opus dependency */ + private static OpusCodecFactory getCodecFactory() + { + return OpusCodecFactoryProvider.getInstance(); + } @Override @SuppressWarnings("deprecation") /* If this was in JDK9 we would be using java.lang.ref.Cleaner instead! */ @@ -706,15 +680,13 @@ private ByteBuffer encodeAudio(ByteBuffer rawAudio) printedError = true; return null; } - IntBuffer error = IntBuffer.allocate(1); - opusEncoder = Opus.INSTANCE.opus_encoder_create(OpusPacket.OPUS_SAMPLE_RATE, OpusPacket.OPUS_CHANNEL_COUNT, Opus.OPUS_APPLICATION_AUDIO, error); - if (error.get() != Opus.OPUS_OK && opusEncoder == null) + opusEncoder = getCodecFactory().createEncoder(); + if (opusEncoder == null) { - LOG.error("Received error status from opus_encoder_create(...): {}", error.get()); return null; } } - return encodeToOpus(rawAudio); + return opusEncoder.encode(rawAudio); } private DatagramPacket getDatagramPacket(ByteBuffer b) diff --git a/src/main/java/net/dv8tion/jda/internal/audio/Decoder.java b/src/main/java/net/dv8tion/jda/internal/audio/Decoder.java deleted file mode 100644 index 47c9fdc5a1..0000000000 --- a/src/main/java/net/dv8tion/jda/internal/audio/Decoder.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright 2015 Austin Keener, Michael Ritter, Florian Spieß, and the JDA contributors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package net.dv8tion.jda.internal.audio; - -import com.sun.jna.ptr.PointerByReference; -import net.dv8tion.jda.api.audio.OpusPacket; -import tomp2p.opuswrapper.Opus; - -import java.nio.ByteBuffer; -import java.nio.IntBuffer; -import java.nio.ShortBuffer; - -/** - * Class that wraps functionality around the Opus decoder. - */ -public class Decoder -{ - protected int ssrc; - protected char lastSeq; - protected int lastTimestamp; - protected PointerByReference opusDecoder; - - protected Decoder(int ssrc) - { - this.ssrc = ssrc; - this.lastSeq = (char) -1; - this.lastTimestamp = -1; - - IntBuffer error = IntBuffer.allocate(1); - opusDecoder = Opus.INSTANCE.opus_decoder_create(OpusPacket.OPUS_SAMPLE_RATE, OpusPacket.OPUS_CHANNEL_COUNT, error); - if (error.get() != Opus.OPUS_OK && opusDecoder == null) - throw new IllegalStateException("Received error code from opus_decoder_create(...): " + error.get()); - } - - public boolean isInOrder(char newSeq) - { - return lastSeq == (char) -1 || newSeq > lastSeq || lastSeq - newSeq > 10; - } - - public boolean wasPacketLost(char newSeq) - { - return newSeq > lastSeq + 1; - } - - public short[] decodeFromOpus(AudioPacket decryptedPacket) - { - int result; - ShortBuffer decoded = ShortBuffer.allocate(4096); - if (decryptedPacket == null) //Flag for packet-loss - { - result = Opus.INSTANCE.opus_decode(opusDecoder, null, 0, decoded, OpusPacket.OPUS_FRAME_SIZE, 0); - lastSeq = (char) -1; - lastTimestamp = -1; - } - else - { - this.lastSeq = decryptedPacket.getSequence(); - this.lastTimestamp = decryptedPacket.getTimestamp(); - - ByteBuffer encodedAudio = decryptedPacket.getEncodedAudio(); - int length = encodedAudio.remaining(); - int offset = encodedAudio.arrayOffset() + encodedAudio.position(); - byte[] buf = new byte[length]; - byte[] data = encodedAudio.array(); - System.arraycopy(data, offset, buf, 0, length); - result = Opus.INSTANCE.opus_decode(opusDecoder, buf, buf.length, decoded, OpusPacket.OPUS_FRAME_SIZE, 0); - } - - //If we get a result that is less than 0, then there was an error. Return null as a signifier. - if (result < 0) - { - handleDecodeError(result); - return null; - } - - short[] audio = new short[result * 2]; - decoded.get(audio); - return audio; - } - - private void handleDecodeError(int result) - { - StringBuilder b = new StringBuilder("Decoder failed to decode audio from user with code "); - switch (result) - { - case Opus.OPUS_BAD_ARG: //-1 - b.append("OPUS_BAD_ARG"); - break; - case Opus.OPUS_BUFFER_TOO_SMALL: //-2 - b.append("OPUS_BUFFER_TOO_SMALL"); - break; - case Opus.OPUS_INTERNAL_ERROR: //-3 - b.append("OPUS_INTERNAL_ERROR"); - break; - case Opus.OPUS_INVALID_PACKET: //-4 - b.append("OPUS_INVALID_PACKET"); - break; - case Opus.OPUS_UNIMPLEMENTED: //-5 - b.append("OPUS_UNIMPLEMENTED"); - break; - case Opus.OPUS_INVALID_STATE: //-6 - b.append("OPUS_INVALID_STATE"); - break; - case Opus.OPUS_ALLOC_FAIL: //-7 - b.append("OPUS_ALLOC_FAIL"); - break; - default: - b.append(result); - } - AudioConnection.LOG.debug("{}", b); - } - - protected synchronized void close() - { - if (opusDecoder != null) - { - Opus.INSTANCE.opus_decoder_destroy(opusDecoder); - opusDecoder = null; - } - } - - @Override - @SuppressWarnings("deprecation") /* If this was in JDK9 we would be using java.lang.ref.Cleaner instead! */ - protected void finalize() throws Throwable - { - super.finalize(); - close(); - } -} diff --git a/src/main/java/net/dv8tion/jda/internal/audio/MissingOpusException.java b/src/main/java/net/dv8tion/jda/internal/audio/MissingOpusException.java new file mode 100644 index 0000000000..9fbefa9c6c --- /dev/null +++ b/src/main/java/net/dv8tion/jda/internal/audio/MissingOpusException.java @@ -0,0 +1,21 @@ +/* + * Copyright 2015 Austin Keener, Michael Ritter, Florian Spieß, and the JDA contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.dv8tion.jda.internal.audio; + +public class MissingOpusException extends RuntimeException +{ +} diff --git a/src/main/java/net/dv8tion/jda/internal/audio/OpusCodecFactory.java b/src/main/java/net/dv8tion/jda/internal/audio/OpusCodecFactory.java new file mode 100644 index 0000000000..838faa6385 --- /dev/null +++ b/src/main/java/net/dv8tion/jda/internal/audio/OpusCodecFactory.java @@ -0,0 +1,31 @@ +/* + * Copyright 2015 Austin Keener, Michael Ritter, Florian Spieß, and the JDA contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.dv8tion.jda.internal.audio; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +public interface OpusCodecFactory +{ + boolean initialize() throws Exception; + + @Nonnull + OpusDecoder createDecoder(int ssrc); + + @Nullable + OpusEncoder createEncoder(); +} diff --git a/src/main/java/net/dv8tion/jda/internal/audio/OpusCodecFactoryProvider.java b/src/main/java/net/dv8tion/jda/internal/audio/OpusCodecFactoryProvider.java new file mode 100644 index 0000000000..4e36dcf899 --- /dev/null +++ b/src/main/java/net/dv8tion/jda/internal/audio/OpusCodecFactoryProvider.java @@ -0,0 +1,54 @@ +/* + * Copyright 2015 Austin Keener, Michael Ritter, Florian Spieß, and the JDA contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.dv8tion.jda.internal.audio; + +import net.dv8tion.jda.internal.utils.JDALogger; +import org.slf4j.Logger; + +import javax.annotation.Nonnull; +import java.util.ServiceLoader; + +public class OpusCodecFactoryProvider +{ + private static final Logger LOG = JDALogger.getLog(OpusCodecFactoryProvider.class); + + private static OpusCodecFactory codecFactory; + + @Nonnull + public static synchronized OpusCodecFactory getInstance() + { + if (codecFactory == null) + { + final ServiceLoader codecFactories = ServiceLoader.load(OpusCodecFactory.class); + for (OpusCodecFactory factory : codecFactories) + { + if (codecFactory != null) + { + LOG.trace("Ignoring {} for opus support as {} is used already", factory.getClass().getName(), codecFactory.getClass().getName()); + continue; + } + LOG.debug("Using {} for Opus support", factory.getClass().getName()); + codecFactory = factory; + } + + if (codecFactory == null) + throw new MissingOpusException(); + } + + return codecFactory; + } +} diff --git a/src/main/java/net/dv8tion/jda/internal/audio/OpusDecoder.java b/src/main/java/net/dv8tion/jda/internal/audio/OpusDecoder.java new file mode 100644 index 0000000000..33a8b52fb8 --- /dev/null +++ b/src/main/java/net/dv8tion/jda/internal/audio/OpusDecoder.java @@ -0,0 +1,28 @@ +/* + * Copyright 2015 Austin Keener, Michael Ritter, Florian Spieß, and the JDA contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.dv8tion.jda.internal.audio; + +import javax.annotation.Nullable; + +public interface OpusDecoder +{ + boolean isInOrder(char newSeq); + + short[] decode(@Nullable AudioPacket decryptedPacket); + + void close(); +} diff --git a/src/main/java/net/dv8tion/jda/internal/audio/OpusEncoder.java b/src/main/java/net/dv8tion/jda/internal/audio/OpusEncoder.java new file mode 100644 index 0000000000..4d21392e3a --- /dev/null +++ b/src/main/java/net/dv8tion/jda/internal/audio/OpusEncoder.java @@ -0,0 +1,29 @@ +/* + * Copyright 2015 Austin Keener, Michael Ritter, Florian Spieß, and the JDA contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.dv8tion.jda.internal.audio; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.nio.ByteBuffer; + +public interface OpusEncoder +{ + @Nullable + ByteBuffer encode(@Nonnull ByteBuffer rawAudio); + + void close(); +}