×

Android Studio 2.2+ opensl FFmpeg音频解码播放

96
JasonXiao
2016.10.17 20:59* 字数 478

Android FFmpeg音频播放

本文介绍了使用opensl es和FFmpeg在Android平台上实现音频解码播放功能的方法。

opensl es简介

Android NDK中包含了平台特有的opensl es。它包含了一系列底层的音频接口,允许开发者使用纯底层代码实现音频播放,录制等功能。相比于AudioTrack,opensl es具有低延迟,高性能等多项优点。

准备工作

  1. 搭建Android Studio NDK开发环境
  2. 编译FFmpeg库并将之集成到Android Studio中
  3. 在工程的AndroidManifest中加入权限
<!-- RECORD_AUDIO is needed to create an audio recorder -->
<uses-permission android:name="android.permission.RECORD_AUDIO"/>
<!-- MODIFY_AUDIO_SETTINGS is needed to use audio effects such as environmental reverb -->
<uses-permission android:name="android.permission.MODIFY_AUDIO_SETTINGS"/>
<!-- INTERNET is needed to use a URI-based audio player, depending on the URI -->
<uses-permission android:name="android.permission.INTERNET"/>
<uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE" />
  1. CMakeLists.txt中加入链接库opensl es
target_link_libraries(native-lib
log
android
OpenSLES
avcodec-57_lib
avformat-57_lib
avutil-55_lib
swresample-2_lib
swscale-4_lib)

FFmpeg音频解码

  1. 在native-lib中加入相关的库
extern "C"{
#include "libavcodec/avcodec.h"
#include "libavformat/avformat.h"
#include "libswscale/swscale.h"
#include "libswresample/swresample.h"
#include "libavutil/opt.h"
};
  1. 声明全局变量
static void *nextBuffer;
static int nextSize;
static AVPacket *packet;
static AVFrame *pFrame;
static AVCodecContext *pCodecCtx;
static SwrContext *swr;
static AVFormatContext *pFormatCtx;
static int audioindex;uint8_t *outputBuffer;
  1. 初始化FFmpeg,读取音频文件,创建解码器和输出缓存
extern "C"
int Java_cn_jx_audiotest_MainActivity_play(JNIEnv* env, jclass clazz, jstring url) {
    int i;    
    AVCodec *pCodec;
    char input_str[500]={0};
    //读取输入的音频文件地址
    sprintf(input_str, "%s", env->GetStringUTFChars(url, NULL)); 
    //初始化
    av_register_all();
    //分配一个AVFormatContext结构
    pFormatCtx = avformat_alloc_context();
    //打开文件
    if(avformat_open_input(&pFormatCtx,input_str,NULL,NULL)!=0) {
        LOGD("Couldn't open input stream.\n");
        return -1;    
    }    
    //查找文件的流信息
    if(avformat_find_stream_info(pFormatCtx,NULL)<0) {
        LOGD("Couldn't find stream information.\n");
        return -1;    
    }
    //在流信息中找到音频流
    audioindex = -1;
    for(i=0; i<pFormatCtx->nb_streams; i++) {
        if (pFormatCtx->streams[i]->codecpar->codec_type==AVMEDIA_TYPE_AUDIO) {
            audioindex = i;
            break;
        }
    }
    if(audioindex == -1){
        LOGD("Couldn't find a video stream.\n");
        return -1;
    }
    //获取相应音频流的解码器
    pCodecCtx=pFormatCtx->streams[audioindex]->codec;
    pCodec = avcodec_find_decoder(pCodecCtx->codec_id);
    assert(pCodec != NULL);
    //分配一个帧指针,指向解码后的原始帧
    pFrame = av_frame_alloc();
    packet=(AVPacket *)av_malloc(sizeof(AVPacket));
    //打开解码器
    if(avcodec_open2(pCodecCtx, pCodec,NULL)<0){
        LOGD("Couldn't open codec.\n");
        return -1;
    }
    //设置格式转换
    swr = swr_alloc();
    av_opt_set_int(swr, "in_channel_layout",  pCodecCtx->channel_layout, 0);
    av_opt_set_int(swr, "out_channel_layout", pCodecCtx->channel_layout,  0);
    av_opt_set_int(swr, "in_sample_rate",     pCodecCtx->sample_rate, 0);
    av_opt_set_int(swr, "out_sample_rate",    pCodecCtx->sample_rate, 0);
    av_opt_set_sample_fmt(swr, "in_sample_fmt",  pCodecCtx->sample_fmt, 0);
    av_opt_set_sample_fmt(swr, "out_sample_fmt", AV_SAMPLE_FMT_S16,  0);
    swr_init(swr);
    //分配输入缓存
    int outputBufferSize = 8196;
    outputBuffer = (uint8_t *) malloc(sizeof(uint8_t) * outputBufferSize);
    //解码音频文件
    getPCM();
    //将解码后的buffer使用opensl es播放
    SLresult result;
    if (nextSize > 0) {
        result = (*bqPlayerBufferQueue)->Enqueue(bqPlayerBufferQueue, nextBuffer, nextSize);
        if (SL_RESULT_SUCCESS != result) {
            pthread_mutex_unlock(&audioEngineLock);
            return -1;
            }
        }
        return 0;
    }
};
  1. 解码音频文件
int getPCM(){
    while(av_read_frame(pFormatCtx, packet)>=0) {
        if (packet->stream_index == audioindex) {
            int ret = avcodec_send_packet(pCodecCtx, packet);
            if (ret < 0 && ret != AVERROR(EAGAIN) && ret != AVERROR_EOF)
                return -1;

            ret = avcodec_receive_frame(pCodecCtx, pFrame);
            if (ret < 0 && ret != AVERROR_EOF)
                return -1;
            //处理不同的格式
            if (pCodecCtx->sample_fmt == AV_SAMPLE_FMT_S16P) {
                nextSize = av_samples_get_buffer_size(pFrame->linesize, pCodecCtx->channels,pCodecCtx->frame_size,pCodecCtx->sample_fmt, 1);
            }else {
                av_samples_get_buffer_size(&nextSize, pCodecCtx->channels,pCodecCtx->frame_size,pCodecCtx->sample_fmt, 1);
            }
            // 音频格式转换
            swr_convert(swr, &outputBuffer, nextSize,
                        (uint8_t const **) (pFrame->extended_data),
                        pFrame->nb_samples);
            nextBuffer = outputBuffer;
            return 0;
        }
    }
}

opensl es音频播放

  1. 定义相关全局变量
// engine interfaces
static SLObjectItf engineObject = NULL;
static SLEngineItf engineEngine;
static SLObjectItf outputMixObject = NULL;
static SLEnvironmentalReverbItf outputMixEnvironmentalReverb = NULL;
static SLObjectItf bqPlayerObject = NULL;
static SLPlayItf bqPlayerPlay;
static SLAndroidSimpleBufferQueueItf bqPlayerBufferQueue;
static SLEffectSendItf bqPlayerEffectSend;
static SLMuteSoloItf bqPlayerMuteSolo;
static SLVolumeItf bqPlayerVolume;
static SLmilliHertz bqPlayerSampleRate = 0;
static jint   bqPlayerBufSize = 0;
static short *resampleBuf = NULL;
static pthread_mutex_t  audioEngineLock = PTHREAD_MUTEX_INITIALIZER;
// aux effect on the output mix, used by the buffer queue player
static const SLEnvironmentalReverbSettings reverbSettings =
        SL_I3DL2_ENVIRONMENT_PRESET_STONECORRIDOR;
  1. 创建引擎
// create the engine and output mix objects
extern "C"
void
Java_cn_jx_audiotest_MainActivity_createEngine(JNIEnv* env, jclass clazz) {
    SLresult result;
    // create engine
    result = slCreateEngine(&engineObject, 0, NULL, 0, NULL, NULL);
    assert(SL_RESULT_SUCCESS == result);
    (void)result;
    // realize the engine
    result = (*engineObject)->Realize(engineObject, SL_BOOLEAN_FALSE);
    assert(SL_RESULT_SUCCESS == result);
    (void)result;
    // get the engine interface, which is needed in order to create other objects
    result = (*engineObject)->GetInterface(engineObject, SL_IID_ENGINE, &engineEngine);
    assert(SL_RESULT_SUCCESS == result);
    (void)result;
    // create output mix, with environmental reverb specified as a non-required interface
    const SLInterfaceID ids[1] = {SL_IID_ENVIRONMENTALREVERB};
    const SLboolean req[1] = {SL_BOOLEAN_FALSE};
    result = (*engineEngine)->CreateOutputMix(engineEngine, &outputMixObject, 1, ids, req);
    assert(SL_RESULT_SUCCESS == result);
    (void)result;
    // realize the output mix
    result = (*outputMixObject)->Realize(outputMixObject, SL_BOOLEAN_FALSE);
    assert(SL_RESULT_SUCCESS == result);
    (void)result;
    // get the environmental reverb interface
    // this could fail if the environmental reverb effect is not available,
    // either because the feature is not present, excessive CPU load, or
    // the required MODIFY_AUDIO_SETTINGS permission was not requested and granted
    result = (*outputMixObject)->GetInterface(outputMixObject, SL_IID_ENVIRONMENTALREVERB,
                                              &outputMixEnvironmentalReverb);
    if (SL_RESULT_SUCCESS == result) {
        result = (*outputMixEnvironmentalReverb)->SetEnvironmentalReverbProperties(
                outputMixEnvironmentalReverb, &reverbSettings);
        (void)result;
    }
    // ignore unsuccessful result codes for environmental reverb, as it is optional for this example
}
  1. 创建BufferQueueAudioPlayer。注意到代码中注册的bqPlayerCallback,每次buffer被处理后,将会调用这个callback。我们在这个callback中再次解码一帧数据塞给BufferQueueAudioPlayer处理。
// create buffer queue audio player
extern "C"
void
Java_cn_jx_audiotest_MainActivity_createBufferQueueAudioPlayer(JNIEnv* env, jclass clazz, jint sampleRate, jint bufSize)
{
    SLresult result;
    if (sampleRate >= 0 && bufSize >= 0 ) {
        bqPlayerSampleRate = sampleRate * 1000;
        /*
         * device native buffer size is another factor to minimize audio latency, not used in this
         * sample: we only play one giant buffer here
         */
        bqPlayerBufSize = bufSize;
    }
    // configure audio source
    SLDataLocator_AndroidSimpleBufferQueue loc_bufq = {SL_DATALOCATOR_ANDROIDSIMPLEBUFFERQUEUE, 2};
    SLDataFormat_PCM format_pcm = {SL_DATAFORMAT_PCM, 1, SL_SAMPLINGRATE_8,
                                   SL_PCMSAMPLEFORMAT_FIXED_16, SL_PCMSAMPLEFORMAT_FIXED_16,
                                   SL_SPEAKER_FRONT_CENTER, SL_BYTEORDER_LITTLEENDIAN};
    /*
     * Enable Fast Audio when possible:  once we set the same rate to be the native, fast audio path
     * will be triggered
     */
    if(bqPlayerSampleRate) {
        format_pcm.samplesPerSec = bqPlayerSampleRate;       //sample rate in mili second
    }
    SLDataSource audioSrc = {&loc_bufq, &format_pcm};

    // configure audio sink
    SLDataLocator_OutputMix loc_outmix = {SL_DATALOCATOR_OUTPUTMIX, outputMixObject};
    SLDataSink audioSnk = {&loc_outmix, NULL};
    /*
     * create audio player:
     *     fast audio does not support when SL_IID_EFFECTSEND is required, skip it
     *     for fast audio case
     */
    const SLInterfaceID ids[3] = {SL_IID_BUFFERQUEUE, SL_IID_VOLUME, SL_IID_EFFECTSEND,
            /*SL_IID_MUTESOLO,*/};
    const SLboolean req[3] = {SL_BOOLEAN_TRUE, SL_BOOLEAN_TRUE, SL_BOOLEAN_TRUE,
            /*SL_BOOLEAN_TRUE,*/ };
    result = (*engineEngine)->CreateAudioPlayer(engineEngine, &bqPlayerObject, &audioSrc, &audioSnk,
                                                bqPlayerSampleRate? 2 : 3, ids, req);
    assert(SL_RESULT_SUCCESS == result);
    (void)result;
    // realize the player
    result = (*bqPlayerObject)->Realize(bqPlayerObject, SL_BOOLEAN_FALSE);
    assert(SL_RESULT_SUCCESS == result);
    (void)result;
    // get the play interface
    result = (*bqPlayerObject)->GetInterface(bqPlayerObject, SL_IID_PLAY, &bqPlayerPlay);
    assert(SL_RESULT_SUCCESS == result);
    (void)result;
    // get the buffer queue interface
    result = (*bqPlayerObject)->GetInterface(bqPlayerObject, SL_IID_BUFFERQUEUE,
                                             &bqPlayerBufferQueue);
    assert(SL_RESULT_SUCCESS == result);
    (void)result;
    // register callback on the buffer queue
    result = (*bqPlayerBufferQueue)->RegisterCallback(bqPlayerBufferQueue, bqPlayerCallback, NULL);
    assert(SL_RESULT_SUCCESS == result);
    (void)result;
    // get the effect send interface
    bqPlayerEffectSend = NULL;
    if( 0 == bqPlayerSampleRate) {
        result = (*bqPlayerObject)->GetInterface(bqPlayerObject, SL_IID_EFFECTSEND,
                                                 &bqPlayerEffectSend);
        assert(SL_RESULT_SUCCESS == result);
        (void)result;
    }
#if 0   // mute/solo is not supported for sources that are known to be mono, as this is
    // get the mute/solo interface
    result = (*bqPlayerObject)->GetInterface(bqPlayerObject, SL_IID_MUTESOLO, &bqPlayerMuteSolo);
    assert(SL_RESULT_SUCCESS == result);
    (void)result;
#endif
    // get the volume interface
    result = (*bqPlayerObject)->GetInterface(bqPlayerObject, SL_IID_VOLUME, &bqPlayerVolume);
    assert(SL_RESULT_SUCCESS == result);
    (void)result;
    // set the player's state to playing
    result = (*bqPlayerPlay)->SetPlayState(bqPlayerPlay, SL_PLAYSTATE_PLAYING);
    assert(SL_RESULT_SUCCESS == result);
    (void)result;
}
  1. 实现bqPlayerCallback
// this callback handler is called every time a buffer finishes playing
void
bqPlayerCallback(SLAndroidSimpleBufferQueueItf bq, void *context)
{
    assert(bq == bqPlayerBufferQueue);
    assert(NULL == context);
    // for streaming playback, replace this test by logic to find and fill the next buffer
    getPCM();
    if ( NULL != nextBuffer && 0 != nextSize) {
        SLresult result;
        // enqueue another buffer
        result = (*bqPlayerBufferQueue)->Enqueue(bqPlayerBufferQueue, nextBuffer, nextSize);
        // the most likely other result is SL_RESULT_BUFFER_INSUFFICIENT,
        // which for this code example would indicate a programming error
        if (SL_RESULT_SUCCESS != result) {
            pthread_mutex_unlock(&audioEngineLock);
        }
        (void)result;
    } else {
        releaseResampleBuf();
        pthread_mutex_unlock(&audioEngineLock);
    }
}

JAVA层调用

我们在JAVA层定义好JNI接口,并按照顺序调用即可开始播放音频文件了。

  1. 加载so库,定义JNI接口
static {
    System.loadLibrary("native-lib");
}
public static native void createEngine();
public static native boolean createBufferQueueAudioPlayer(int sampleRate, int samplesPerBuf);
public native void play(String url);
  1. 依次调用接口,创建引擎开始播放
createEngine();
int sampleRate = 0;
int bufSize = 0;
/*
 * retrieve fast audio path sample rate and buf size; if we have it, we pass to native
 * side to create a player with fast audio enabled [ fast audio == low latency audio ];
 * IF we do not have a fast audio path, we pass 0 for sampleRate, which will force native
 * side to pick up the 8Khz sample rate.
 */
if(Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN_MR1) {
    AudioManager myAudioMgr = (AudioManager) getSystemService(Context.AUDIO_SERVICE);
    String nativeParam = myAudioMgr.getProperty(AudioManager.PROPERTY_OUTPUT_SAMPLE_RATE);
    sampleRate = Integer.parseInt(nativeParam);
    nativeParam = myAudioMgr.getProperty(AudioManager.PROPERTY_OUTPUT_FRAMES_PER_BUFFER);
    bufSize = Integer.parseInt(nativeParam);
}
createBufferQueueAudioPlayer(sampleRate, bufSize);
//获取文件地址
String folderurl = Environment.getExternalStorageDirectory().getPath();
String inputurl = folderurl+"/background.mp3";
play(inputurl);

遇到的坑

  1. 不同的音视频文件解析出来的音频采样数据格式不一样,有的MP3文件解出来是AV_SAMPLE_FMT_S16P,有的MP4文件是AV_SAMPLE_FMT_FLTP,这两种文件的outputBuffer大小设置不太一样。没处理好就会有大量杂音,处理方式详见代码getPCM中的解码部分。(处理格式如有遗漏欢迎举报)

参考资料

  1. 最简单的基于FFMPEG+SDL的音频播放器
  2. NDK sample(native audio)
  3. AAC to PCM produced a lot of noise
Android FFmpeg播放器制作
Web note ad 1