Android 获取视频缩略图(获取视频每帧数据)的优化方案

代码位置 MediaMetadataRetriever Wrapper

速度对比

左边的图片是通过方式1
右边的图片是通过方式2

speed.gif

速度优化,效果拔群。

  • 在缩小2倍的Bitmap输出情况下
    使用MediaMetadataRetriever 抽帧的速度,每帧稳定在 300ms左右。
    使用MediaCodec+ImageReader 第一次抽帧。大概是200ms ,后续每帧则是50ms左右。

注意:如果不缩小图片的话,建议还是使用MediaMetadataRetriever。
使用当前库的话,调用metadataRetriever.forceFallBack(true);

方案

1. 通过MediaMetaRetrivier来进行获取

代码较为简单,就是一个循环

 MediaMetadataRetriever metadataRetriever = new MediaMetadataRetriever();
                metadataRetriever.setDataSource(fileName);

                String duration = metadataRetriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_DURATION);
                Log.d(TAG, "duration = " + duration);
                int durationMs = Integer.parseInt(duration);

                //每秒取一次
                for (int i = 0; i < durationMs; i += 1000) {
                    long start = System.nanoTime();
                    Log.d(TAG, "getFrameAtTime time = " + i);
                    //这里传入的是ms
                    Bitmap frameAtIndex = metadataRetriever.getFrameAtTime(i * 1000);
                    Bitmap frame=Bitmap.createScaledBitmap(frameAtIndex,frameAtIndex.getWidth()/8,frameAtIndex.getHeight()/8,false);
                    frameAtIndex.recycle();
                    long end = System.nanoTime();
                    long cost = end - start;
                    Log.d(TAG, "cost time in millis = " + (cost * 1f / 1000000));

                    if (callBack != null) {
                        callBack.onComplete(frame);
                    }
                }
                metadataRetriever.release();

2. 通过MediaCodec和ImageReader进行获取

就是通过通过Surface,用MediaExtrator,将MediaCodec解码后的数据,传递给ImageReader。来进行显示。

 MediaExtractor extractor = null;
                MediaCodec codec = null;
                try {
                    extractor = new MediaExtractor();
                    extractor.setDataSource(fileName);
                    int trackCount = extractor.getTrackCount();
                    MediaFormat videoFormat = null;
                    for (int i = 0; i < trackCount; i++) {
                        MediaFormat trackFormat = extractor.getTrackFormat(i);
                        if (trackFormat.getString(MediaFormat.KEY_MIME).contains("video")) {
                            videoFormat = trackFormat;
                            extractor.selectTrack(i);
                            break;
                        }
                    }
                    if (videoFormat == null) {
                        Log.d(TAG, "Can not get video format");
                        return;
                    }

                    int imageFormat = ImageFormat.YUV_420_888;
                    int colorFormat = COLOR_FormatYUV420Flexible;
                    videoFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, colorFormat);
                    videoFormat.setInteger(MediaFormat.KEY_WIDTH, videoFormat.getInteger(MediaFormat.KEY_WIDTH) / 4);
                    videoFormat.setInteger(MediaFormat.KEY_HEIGHT, videoFormat.getInteger(MediaFormat.KEY_HEIGHT) / 4);

                    long duration = videoFormat.getLong(MediaFormat.KEY_DURATION);

                    codec = MediaCodec.createDecoderByType(videoFormat.getString(MediaFormat.KEY_MIME));
                    ImageReader imageReader = ImageReader
                            .newInstance(
                                    videoFormat.getInteger(MediaFormat.KEY_WIDTH),
                                    videoFormat.getInteger(MediaFormat.KEY_HEIGHT),
                                    imageFormat,
                                    3);
                    final ImageReaderHandlerThread imageReaderHandlerThread = new ImageReaderHandlerThread();

                    imageReader.setOnImageAvailableListener(new MyOnImageAvailableListener(callBack), imageReaderHandlerThread.getHandler());
                    codec.configure(videoFormat, imageReader.getSurface(), null, 0);
                    codec.start();
                    MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
                    long timeOut = 5 * 1000;//10ms
                    boolean inputDone = false;
                    boolean outputDone = false;
                    ByteBuffer[] inputBuffers = null;
                    if (android.os.Build.VERSION.SDK_INT < android.os.Build.VERSION_CODES.LOLLIPOP) {
                        inputBuffers = codec.getInputBuffers();
                    }
                    //开始进行解码。
                    int count = 1;
                    while (!outputDone) {
                        if (requestStop) {
                            return;
                        }
                        if (!inputDone) {
                            //feed data
                            int inputBufferIndex = codec.dequeueInputBuffer(timeOut);
                            if (inputBufferIndex >= 0) {
                                ByteBuffer inputBuffer;
                                if (android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.LOLLIPOP) {
                                    inputBuffer = codec.getInputBuffer(inputBufferIndex);
                                } else {
                                    inputBuffer = inputBuffers[inputBufferIndex];
                                }
                                int sampleData = extractor.readSampleData(inputBuffer, 0);
                                if (sampleData > 0) {
                                    long sampleTime = extractor.getSampleTime();
                                    codec.queueInputBuffer(inputBufferIndex, 0, sampleData, sampleTime, 0);
                                    //继续
                                    if (interval == 0) {
                                        extractor.advance();
                                    } else {
                                        extractor.seekTo(count * interval * 1000, MediaExtractor.SEEK_TO_PREVIOUS_SYNC);
                                        count++;
//                                        extractor.advance();
                                    }
                                } else {
                                    //小于0,说明读完了
                                    codec.queueInputBuffer(inputBufferIndex, 0, 0, 0L, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
                                    inputDone = true;
                                    Log.d(TAG, "end of stream");
                                }
                            }
                        }
                        if (!outputDone) {
                            //get data
                            int status = codec.dequeueOutputBuffer(bufferInfo, timeOut);
                            if (status ==
                                    MediaCodec.INFO_TRY_AGAIN_LATER) {
                                //继续
                            } else if (status == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
                                //开始进行解码
                            } else if (status == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
                                //同样啥都不做
                            } else {
                                //在这里判断,当前编码器的状态
                                if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
                                    Log.d(TAG, "output EOS");
                                    outputDone = true;
                                }
                                boolean doRender = (bufferInfo.size != 0);
                                long presentationTimeUs = bufferInfo.presentationTimeUs;
                                if (lastPresentationTimeUs == 0) {
                                    lastPresentationTimeUs = presentationTimeUs;
                                } else {
                                    long diff = presentationTimeUs - lastPresentationTimeUs;
                                    if (interval != 0) {
                                        if (diff < interval * 1000) {
                                            doRender = false;
                                        } else {
                                            lastPresentationTimeUs = presentationTimeUs;
                                        }
                                        Log.d(TAG,
                                                "diff time in ms =" + diff / 1000);
                                    }
                                }
                                //有数据了.因为会直接传递给Surface,所以说明都不做好了
                                Log.d(TAG, "surface decoder given buffer " + status +
                                        " (size=" + bufferInfo.size + ")" + ",doRender = " + doRender + ", presentationTimeUs=" + presentationTimeUs);
                                //直接送显就可以了
                                codec.releaseOutputBuffer(status, doRender);
                            }
                        }
                    }
                } catch (IOException e) {
                    e.printStackTrace();
                } finally {
                    if (codec != null) {
                        codec.stop();
                        codec.release();
                    }
                    if (extractor != null) {
                        extractor.release();
                    }
                }
            }

通过libyuv进行数据的转换

private static class MyOnImageAvailableListener implements ImageReader.OnImageAvailableListener {
        private final BitmapCallBack callBack;

        private MyOnImageAvailableListener(BitmapCallBack callBack) {
            this.callBack = callBack;
        }

        @Override
        public void onImageAvailable(ImageReader reader) {
            Log.i(TAG, "in OnImageAvailable");
            Image img = null;
            try {
                img = reader.acquireLatestImage();
                if (img != null) {
                    //这里得到的YUV的数据。需要将YUV的数据变成Bitmap
                    Image.Plane[] planes = img.getPlanes();
                    if (planes[0].getBuffer() == null) {
                        return;
                    }

//                    Bitmap bitmap = getBitmap(img);
                    Bitmap bitmap = getBitmapScale(img, 8);
//                    Bitmap bitmap = getBitmapFromNv21(img);
                    if (callBack != null && bitmap != null) {
                        Log.d(TAG, "onComplete bitmap ");
                        callBack.onComplete(bitmap);
                    }
                }
            } catch (Exception e) {
                e.printStackTrace();
            } finally {
                if (img != null) {
                    img.close();
                }
            }

        }
        
        @NonNull
        private Bitmap getBitmapScale(Image img, int scale) {
            int width = img.getWidth() / scale;
            int height = img.getHeight() / scale;
            final byte[] bytesImage = getDataFromYUV420Scale(img, scale);
            Bitmap bitmap = null;
            bitmap = Bitmap.createBitmap(height, width, Bitmap.Config.ARGB_8888);
            bitmap.copyPixelsFromBuffer(ByteBuffer.wrap(bytesImage));
            return bitmap;
        }

        private byte[] getDataFromYUV420Scale(Image image, int scale) {
            int width = image.getWidth();
            int height = image.getHeight();
            // Read image data
            Image.Plane[] planes = image.getPlanes();

            byte[] argb = new byte[width / scale * height / scale * 4];

            //值得注意的是在Java层传入byte[]以RGBA顺序排列时,libyuv是用ABGR来表示这个排列
            //libyuv表示的排列顺序和Bitmap的RGBA表示的顺序是反向的。
            // 所以实际要调用libyuv::ABGRToI420才能得到正确的结果。
            YuvUtils.yuvI420ToABGRWithScale(
                    argb,
                    planes[0].getBuffer(), planes[0].getRowStride(),
                    planes[1].getBuffer(), planes[1].getRowStride(),
                    planes[2].getBuffer(), planes[2].getRowStride(),
                    width, height,
                    scale
            );
            return argb;
        }
}

libyuv


extern "C"
JNIEXPORT void JNICALL
Java_com_example_yuv_YuvUtils_yuvI420ToABGRWithScale(JNIEnv *env, jclass type, jbyteArray argb_,
                                                     jobject y_buffer, jint y_rowStride,
                                                     jobject u_buffer, jint u_rowStride,
                                                     jobject v_buffer, jint v_rowStride,
                                                     jint width, jint height,
                                                     jint scale) {
    jbyte *argb = env->GetByteArrayElements(argb_, NULL);

    uint8_t *srcYPtr = reinterpret_cast<uint8_t *>(env->GetDirectBufferAddress(y_buffer));
    uint8_t *srcUPtr = reinterpret_cast<uint8_t *>(env->GetDirectBufferAddress(u_buffer));
    uint8_t *srcVPtr = reinterpret_cast<uint8_t *>(env->GetDirectBufferAddress(v_buffer));

    int scaleW = width / scale;
    int scaleH = height / scale;
    int scaleSize = scaleW * scaleH;
    jbyte *temp_y_scale = new jbyte[scaleSize * 3 / 2];
    jbyte *temp_u_scale = temp_y_scale + scaleSize;
    jbyte *temp_v_scale = temp_y_scale + scaleSize + scaleSize / 4;

    libyuv::I420Scale(
            srcYPtr, y_rowStride,
            srcUPtr, u_rowStride,
            srcVPtr, v_rowStride,
            width, height,
            (uint8_t *) temp_y_scale, scaleW,
            (uint8_t *) temp_u_scale, scaleW >> 1,
            (uint8_t *) temp_v_scale, scaleW >> 1,
            scaleW, scaleH,
            libyuv::kFilterNone
    );

    width = scaleW;
    height = scaleH;
    jbyte *temp_y = new jbyte[width * height * 3 / 2];
    jbyte *temp_u = temp_y + width * height;
    jbyte *temp_v = temp_y + width * height + width * height / 4;

    libyuv::I420Rotate(
            (uint8_t *) temp_y_scale, scaleW,
            (uint8_t *) temp_u_scale, scaleW >> 1,
            (uint8_t *) temp_v_scale, scaleW >> 1,
//
            (uint8_t *) temp_y, height,
            (uint8_t *) temp_u, height >> 1,
            (uint8_t *) temp_v, height >> 1,

            width, height,
            libyuv::kRotate90
    );

    libyuv::I420ToABGR(
            (uint8_t *) temp_y, height,
            (uint8_t *) temp_u, height >> 1,
            (uint8_t *) temp_v, height >> 1,

            (uint8_t *) argb, height * 4,
            height, width
    );



    env->ReleaseByteArrayElements(argb_, argb, 0);
}

后续

将文件通过MediaCodec解码。 输出到ImageReader当中。来获取截图。

使用MediaMetadataRetriever的方式,因为无法配置输出的图片的大小。

但当我们只需要生成小图预览的时候, 如果我们实现做了缩放的处理。就能得到很快的速度。

不足

需要对原来MediaMetadataRetriever的原理探究

推荐阅读更多精彩内容