media/java/android/media/MediaSync.java

/*
 * Copyright (C) 2015 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package android.media;

import android.annotation.IntDef;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.media.AudioTrack;
import android.media.PlaybackParams;
import android.os.Handler;
import android.os.Looper;
import android.os.Message;
import android.util.Log;
import android.view.Surface;

import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.nio.ByteBuffer;
import java.util.concurrent.TimeUnit;
import java.util.LinkedList;
import java.util.List;

/**
 * MediaSync class can be used to synchronously play audio and video streams.
 * It can be used to play audio-only or video-only stream, too.
 *
 * <p>MediaSync is generally used like this:
 * <pre>
 * MediaSync sync = new MediaSync();
 * sync.setSurface(surface);
 * Surface inputSurface = sync.createInputSurface();
 * ...
 * // MediaCodec videoDecoder = ...;
 * videoDecoder.configure(format, inputSurface, ...);
 * ...
 * sync.setAudioTrack(audioTrack);
 * sync.setCallback(new MediaSync.Callback() {
 *     {@literal @Override}
 *     public void onAudioBufferConsumed(MediaSync sync, ByteBuffer audioBuffer, int bufferId) {
 *         ...
 *     }
 * }, null);
 * // This needs to be done since sync is paused on creation.
 * sync.setPlaybackParams(new PlaybackParams().setSpeed(1.f));
 *
 * for (;;) {
 *   ...
 *   // send video frames to surface for rendering, e.g., call
 *   // videoDecoder.releaseOutputBuffer(videoOutputBufferIx, videoPresentationTimeNs);
 *   // More details are available as below.
 *   ...
 *   sync.queueAudio(audioByteBuffer, bufferId, audioPresentationTimeUs); // non-blocking.
 *   // The audioByteBuffer and bufferId will be returned via callback.
 *   // More details are available as below.
 *   ...
 *     ...
 * }
 * sync.setPlaybackParams(new PlaybackParams().setSpeed(0.f));
 * sync.release();
 * sync = null;
 *
 * // The following code snippet illustrates how video/audio raw frames are created by
 * // MediaCodec's, how they are fed to MediaSync and how they are returned by MediaSync.
 * // This is the callback from MediaCodec.
 * onOutputBufferAvailable(MediaCodec codec, int bufferId, BufferInfo info) {
 *     // ...
 *     if (codec == videoDecoder) {
 *         // surface timestamp must contain media presentation time in nanoseconds.
 *         codec.releaseOutputBuffer(bufferId, 1000 * info.presentationTime);
 *     } else {
 *         ByteBuffer audioByteBuffer = codec.getOutputBuffer(bufferId);
 *         sync.queueAudio(audioByteBuffer, bufferId, info.presentationTime);
 *     }
 *     // ...
 * }
 *
 * // This is the callback from MediaSync.
 * onAudioBufferConsumed(MediaSync sync, ByteBuffer buffer, int bufferId) {
 *     // ...
 *     audioDecoder.releaseBuffer(bufferId, false);
 *     // ...
 * }
 *
 * </pre>
 *
 * The client needs to configure corresponding sink by setting the Surface and/or AudioTrack
 * based on the stream type it will play.
 * <p>
 * For video, the client needs to call {@link #createInputSurface} to obtain a surface on
 * which it will render video frames.
 * <p>
 * For audio, the client needs to set up audio track correctly, e.g., using {@link
 * AudioTrack#MODE_STREAM}. The audio buffers are sent to MediaSync directly via {@link
 * #queueAudio}, and are returned to the client via {@link Callback#onAudioBufferConsumed}
 * asynchronously. The client should not modify an audio buffer till it's returned.
 * <p>
 * The client can optionally pre-fill audio/video buffers by setting playback rate to 0.0,
 * and then feed audio/video buffers to corresponding components. This can reduce possible
 * initial underrun.
 * <p>
 */
public final class MediaSync {
    /**
     * MediaSync callback interface. Used to notify the user asynchronously
     * of various MediaSync events.
     */
    public static abstract class Callback {
        /**
         * Called when returning an audio buffer which has been consumed.
         *
         * @param sync The MediaSync object.
         * @param audioBuffer The returned audio buffer.
         * @param bufferId The ID associated with audioBuffer as passed into
         *     {@link MediaSync#queueAudio}.
         */
        public abstract void onAudioBufferConsumed(
                @NonNull MediaSync sync, @NonNull ByteBuffer audioBuffer, int bufferId);
    }

    /** Audio track failed.
     * @see android.media.MediaSync.OnErrorListener
     */
    public static final int MEDIASYNC_ERROR_AUDIOTRACK_FAIL = 1;

    /** The surface failed to handle video buffers.
     * @see android.media.MediaSync.OnErrorListener
     */
    public static final int MEDIASYNC_ERROR_SURFACE_FAIL = 2;

    /**
     * Interface definition of a callback to be invoked when there
     * has been an error during an asynchronous operation (other errors
     * will throw exceptions at method call time).
     */
    public interface OnErrorListener {
        /**
         * Called to indicate an error.
         *
         * @param sync The MediaSync the error pertains to
         * @param what The type of error that has occurred:
         * <ul>
         * <li>{@link #MEDIASYNC_ERROR_AUDIOTRACK_FAIL}
         * <li>{@link #MEDIASYNC_ERROR_SURFACE_FAIL}
         * </ul>
         * @param extra an extra code, specific to the error. Typically
         * implementation dependent.
         */
        void onError(@NonNull MediaSync sync, int what, int extra);
    }

    private static final String TAG = "MediaSync";

    private static final int EVENT_CALLBACK = 1;
    private static final int EVENT_SET_CALLBACK = 2;

    private static final int CB_RETURN_AUDIO_BUFFER = 1;

    private static class AudioBuffer {
        public ByteBuffer mByteBuffer;
        public int mBufferIndex;
        long mPresentationTimeUs;

        public AudioBuffer(@NonNull ByteBuffer byteBuffer, int bufferId,
                           long presentationTimeUs) {
            mByteBuffer = byteBuffer;
            mBufferIndex = bufferId;
            mPresentationTimeUs = presentationTimeUs;
        }
    }

    private final Object mCallbackLock = new Object();
    private Handler mCallbackHandler = null;
    private MediaSync.Callback mCallback = null;

    private final Object mOnErrorListenerLock = new Object();
    private Handler mOnErrorListenerHandler = null;
    private MediaSync.OnErrorListener mOnErrorListener = null;

    private Thread mAudioThread = null;
    // Created on mAudioThread when mAudioThread is started. When used on user thread, they should
    // be guarded by checking mAudioThread.
    private Handler mAudioHandler = null;
    private Looper mAudioLooper = null;

    private final Object mAudioLock = new Object();
    private AudioTrack mAudioTrack = null;
    private List<AudioBuffer> mAudioBuffers = new LinkedList<AudioBuffer>();
    // this is only used for paused/running decisions, so it is not affected by clock drift
    private float mPlaybackRate = 0.0f;

    private long mNativeContext;

    /**
     * Class constructor. On creation, MediaSync is paused, i.e., playback rate is 0.0f.
     */
    public MediaSync() {
        native_setup();
    }

    private native final void native_setup();

    @Override
    protected void finalize() {
        native_finalize();
    }

    private native final void native_finalize();

    /**
     * Make sure you call this when you're done to free up any opened
     * component instance instead of relying on the garbage collector
     * to do this for you at some point in the future.
     */
    public final void release() {
        returnAudioBuffers();
        if (mAudioThread != null) {
            if (mAudioLooper != null) {
                mAudioLooper.quit();
            }
        }
        setCallback(null, null);
        native_release();
    }

    private native final void native_release();

    /**
     * Sets an asynchronous callback for actionable MediaSync events.
     * <p>
     * This method can be called multiple times to update a previously set callback. If the
     * handler is changed, undelivered notifications scheduled for the old handler may be dropped.
     * <p>
     * <b>Do not call this inside callback.</b>
     *
     * @param cb The callback that will run. Use {@code null} to stop receiving callbacks.
     * @param handler The Handler that will run the callback. Use {@code null} to use MediaSync's
     *     internal handler if it exists.
     */
    public void setCallback(@Nullable /* MediaSync. */ Callback cb, @Nullable Handler handler) {
        synchronized(mCallbackLock) {
            if (handler != null) {
                mCallbackHandler = handler;
            } else {
                Looper looper;
                if ((looper = Looper.myLooper()) == null) {
                    looper = Looper.getMainLooper();
                }
                if (looper == null) {
                    mCallbackHandler = null;
                } else {
                    mCallbackHandler = new Handler(looper);
                }
            }

            mCallback = cb;
        }
    }

    /**
     * Sets an asynchronous callback for error events.
     * <p>
     * This method can be called multiple times to update a previously set listener. If the
     * handler is changed, undelivered notifications scheduled for the old handler may be dropped.
     * <p>
     * <b>Do not call this inside callback.</b>
     *
     * @param listener The callback that will run. Use {@code null} to stop receiving callbacks.
     * @param handler The Handler that will run the callback. Use {@code null} to use MediaSync's
     *     internal handler if it exists.
     */
    public void setOnErrorListener(@Nullable /* MediaSync. */ OnErrorListener listener,
            @Nullable Handler handler) {
        synchronized(mOnErrorListenerLock) {
            if (handler != null) {
                mOnErrorListenerHandler = handler;
            } else {
                Looper looper;
                if ((looper = Looper.myLooper()) == null) {
                    looper = Looper.getMainLooper();
                }
                if (looper == null) {
                    mOnErrorListenerHandler = null;
                } else {
                    mOnErrorListenerHandler = new Handler(looper);
                }
            }

            mOnErrorListener = listener;
        }
    }

    /**
     * Sets the output surface for MediaSync.
     * <p>
     * Currently, this is only supported in the Initialized state.
     *
     * @param surface Specify a surface on which to render the video data.
     * @throws IllegalArgumentException if the surface has been released, is invalid,
     *     or can not be connected.
     * @throws IllegalStateException if setting the surface is not supported, e.g.
     *     not in the Initialized state, or another surface has already been set.
     */
    public void setSurface(@Nullable Surface surface) {
        native_setSurface(surface);
    }

    private native final void native_setSurface(@Nullable Surface surface);

    /**
     * Sets the audio track for MediaSync.
     * <p>
     * Currently, this is only supported in the Initialized state.
     *
     * @param audioTrack Specify an AudioTrack through which to render the audio data.
     * @throws IllegalArgumentException if the audioTrack has been released, or is invalid.
     * @throws IllegalStateException if setting the audio track is not supported, e.g.
     *     not in the Initialized state, or another audio track has already been set.
     */
    public void setAudioTrack(@Nullable AudioTrack audioTrack) {
        native_setAudioTrack(audioTrack);
        mAudioTrack = audioTrack;
        if (audioTrack != null && mAudioThread == null) {
            createAudioThread();
        }
    }

    private native final void native_setAudioTrack(@Nullable AudioTrack audioTrack);

    /**
     * Requests a Surface to use as the input. This may only be called after
     * {@link #setSurface}.
     * <p>
     * The application is responsible for calling release() on the Surface when
     * done.
     * @throws IllegalStateException if not set, or another input surface has
     *     already been created.
     */
    @NonNull
    public native final Surface createInputSurface();

    /**
     * Sets playback rate using {@link PlaybackParams}.
     * <p>
     * When using MediaSync with {@link AudioTrack}, set playback params using this
     * call instead of calling it directly on the track, so that the sync is aware of
     * the params change.
     * <p>
     * This call also works if there is no audio track.
     *
     * @param params the playback params to use. {@link PlaybackParams#getSpeed
     *     Speed} is the ratio between desired playback rate and normal one. 1.0 means
     *     normal playback speed. 0.0 means pause. Value larger than 1.0 means faster playback,
     *     while value between 0.0 and 1.0 for slower playback. <b>Note:</b> the normal rate
     *     does not change as a result of this call. To restore the original rate at any time,
     *     use speed of 1.0.
     *
     * @throws IllegalStateException if the internal sync engine or the audio track has not
     *     been initialized.
     * @throws IllegalArgumentException if the params are not supported.
     */
    public void setPlaybackParams(@NonNull PlaybackParams params) {
        synchronized(mAudioLock) {
            mPlaybackRate = native_setPlaybackParams(params);;
        }
        if (mPlaybackRate != 0.0 && mAudioThread != null) {
            postRenderAudio(0);
        }
    }

    /**
     * Gets the playback rate using {@link PlaybackParams}.
     *
     * @return the playback rate being used.
     *
     * @throws IllegalStateException if the internal sync engine or the audio track has not
     *     been initialized.
     */
    @NonNull
    public native PlaybackParams getPlaybackParams();

    private native float native_setPlaybackParams(@NonNull PlaybackParams params);

    /**
     * Sets A/V sync mode.
     *
     * @param params the A/V sync params to apply
     *
     * @throws IllegalStateException if the internal player engine has not been
     * initialized.
     * @throws IllegalArgumentException if params are not supported.
     */
    public void setSyncParams(@NonNull SyncParams params) {
        synchronized(mAudioLock) {
            mPlaybackRate = native_setSyncParams(params);;
        }
        if (mPlaybackRate != 0.0 && mAudioThread != null) {
            postRenderAudio(0);
        }
    }

    private native float native_setSyncParams(@NonNull SyncParams params);

    /**
     * Gets the A/V sync mode.
     *
     * @return the A/V sync params
     *
     * @throws IllegalStateException if the internal player engine has not been
     * initialized.
     */
    @NonNull
    public native SyncParams getSyncParams();

    /**
     * Flushes all buffers from the sync object.
     * <p>
     * All pending unprocessed audio and video buffers are discarded. If an audio track was
     * configured, it is flushed and stopped. If a video output surface was configured, the
     * last frame queued to it is left on the frame. Queue a blank video frame to clear the
     * surface,
     * <p>
     * No callbacks are received for the flushed buffers.
     *
     * @throws IllegalStateException if the internal player engine has not been
     * initialized.
     */
    public void flush() {
        synchronized(mAudioLock) {
            mAudioBuffers.clear();
            mCallbackHandler.removeCallbacksAndMessages(null);
        }
        if (mAudioTrack != null) {
            mAudioTrack.pause();
            mAudioTrack.flush();
            // Call stop() to signal to the AudioSink to completely fill the
            // internal buffer before resuming playback.
            mAudioTrack.stop();
        }
        native_flush();
    }

    private native final void native_flush();

    /**
     * Get current playback position.
     * <p>
     * The MediaTimestamp represents how the media time correlates to the system time in
     * a linear fashion using an anchor and a clock rate. During regular playback, the media
     * time moves fairly constantly (though the anchor frame may be rebased to a current
     * system time, the linear correlation stays steady). Therefore, this method does not
     * need to be called often.
     * <p>
     * To help users get current playback position, this method always anchors the timestamp
     * to the current {@link System#nanoTime system time}, so
     * {@link MediaTimestamp#getAnchorMediaTimeUs} can be used as current playback position.
     *
     * @return a MediaTimestamp object if a timestamp is available, or {@code null} if no timestamp
     *         is available, e.g. because the media player has not been initialized.
     *
     * @see MediaTimestamp
     */
    @Nullable
    public MediaTimestamp getTimestamp()
    {
        try {
            // TODO: create the timestamp in native
            MediaTimestamp timestamp = new MediaTimestamp();
            if (native_getTimestamp(timestamp)) {
                return timestamp;
            } else {
                return null;
            }
        } catch (IllegalStateException e) {
            return null;
        }
    }

    private native final boolean native_getTimestamp(@NonNull MediaTimestamp timestamp);

    /**
     * Queues the audio data asynchronously for playback (AudioTrack must be in streaming mode).
     * If the audio track was flushed as a result of {@link #flush}, it will be restarted.
     * @param audioData the buffer that holds the data to play. This buffer will be returned
     *     to the client via registered callback.
     * @param bufferId an integer used to identify audioData. It will be returned to
     *     the client along with audioData. This helps applications to keep track of audioData,
     *     e.g., it can be used to store the output buffer index used by the audio codec.
     * @param presentationTimeUs the presentation timestamp in microseconds for the first frame
     *     in the buffer.
     * @throws IllegalStateException if audio track is not set or internal configureation
     *     has not been done correctly.
     */
    public void queueAudio(
            @NonNull ByteBuffer audioData, int bufferId, long presentationTimeUs) {
        if (mAudioTrack == null || mAudioThread == null) {
            throw new IllegalStateException(
                    "AudioTrack is NOT set or audio thread is not created");
        }

        synchronized(mAudioLock) {
            mAudioBuffers.add(new AudioBuffer(audioData, bufferId, presentationTimeUs));
        }

        if (mPlaybackRate != 0.0) {
            postRenderAudio(0);
        }
    }

    // When called on user thread, make sure to check mAudioThread != null.
    private void postRenderAudio(long delayMillis) {
        mAudioHandler.postDelayed(new Runnable() {
            public void run() {
                synchronized(mAudioLock) {
                    if (mPlaybackRate == 0.0) {
                        return;
                    }

                    if (mAudioBuffers.isEmpty()) {
                        return;
                    }

                    AudioBuffer audioBuffer = mAudioBuffers.get(0);
                    int size = audioBuffer.mByteBuffer.remaining();
                    // restart audio track after flush
                    if (size > 0 && mAudioTrack.getPlayState() != AudioTrack.PLAYSTATE_PLAYING) {
                        try {
                            mAudioTrack.play();
                        } catch (IllegalStateException e) {
                            Log.w(TAG, "could not start audio track");
                        }
                    }
                    int sizeWritten = mAudioTrack.write(
                            audioBuffer.mByteBuffer,
                            size,
                            AudioTrack.WRITE_NON_BLOCKING);
                    if (sizeWritten > 0) {
                        if (audioBuffer.mPresentationTimeUs != -1) {
                            native_updateQueuedAudioData(
                                    size, audioBuffer.mPresentationTimeUs);
                            audioBuffer.mPresentationTimeUs = -1;
                        }

                        if (sizeWritten == size) {
                            postReturnByteBuffer(audioBuffer);
                            mAudioBuffers.remove(0);
                            if (!mAudioBuffers.isEmpty()) {
                                postRenderAudio(0);
                            }
                            return;
                        }
                    }
                    long pendingTimeMs = TimeUnit.MICROSECONDS.toMillis(
                            native_getPlayTimeForPendingAudioFrames());
                    postRenderAudio(pendingTimeMs / 2);
                }
            }
        }, delayMillis);
    }

    private native final void native_updateQueuedAudioData(
            int sizeInBytes, long presentationTimeUs);

    private native final long native_getPlayTimeForPendingAudioFrames();

    private final void postReturnByteBuffer(@NonNull final AudioBuffer audioBuffer) {
        synchronized(mCallbackLock) {
            if (mCallbackHandler != null) {
                final MediaSync sync = this;
                mCallbackHandler.post(new Runnable() {
                    public void run() {
                        Callback callback;
                        synchronized(mCallbackLock) {
                            callback = mCallback;
                            if (mCallbackHandler == null
                                    || mCallbackHandler.getLooper().getThread()
                                            != Thread.currentThread()) {
                                // callback handler has been changed.
                                return;
                            }
                        }
                        if (callback != null) {
                            callback.onAudioBufferConsumed(sync, audioBuffer.mByteBuffer,
                                    audioBuffer.mBufferIndex);
                        }
                    }
                });
            }
        }
    }

    private final void returnAudioBuffers() {
        synchronized(mAudioLock) {
            for (AudioBuffer audioBuffer: mAudioBuffers) {
                postReturnByteBuffer(audioBuffer);
            }
            mAudioBuffers.clear();
        }
    }

    private void createAudioThread() {
        mAudioThread = new Thread() {
            @Override
            public void run() {
                Looper.prepare();
                synchronized(mAudioLock) {
                    mAudioLooper = Looper.myLooper();
                    mAudioHandler = new Handler();
                    mAudioLock.notify();
                }
                Looper.loop();
            }
        };
        mAudioThread.start();

        synchronized(mAudioLock) {
            try {
                mAudioLock.wait();
            } catch(InterruptedException e) {
            }
        }
    }

    static {
        System.loadLibrary("media_jni");
        native_init();
    }

    private static native final void native_init();
}