Android采集并使用OpenGL ES预览摄像头画面

╰半橙微兮° 2022-05-22 06:43 102阅读 0赞

Android的Camera现在有新的api，但是要5.0之后才能用，书上的源码也是用老版本api，简单分析一下预览流程。为了篇幅部分中间的调用流程将会省略，只贴出核心代码。

#### EGL环境初始化 ####

首先我们肯定是要初始化egl，Native层使用hanlder的形式来循环处理消息。

void MVRecordingPreviewController::prepareEGLContext(ANativeWindow* window, JavaVM *g_jvm, jobject obj, int screenWidth, int screenHeight, int cameraFacingId){
        LOGI("Creating MVRecordingPreviewController thread");
        this->g_jvm = g_jvm;
        this->obj = obj;
        this->_window = window;
        this->screenWidth = screenWidth;
        this->screenHeight = screenHeight;
        this->facingId = cameraFacingId;
        handler->postMessage(new Message(MSG_EGL_THREAD_CREATE));
        //开启线程循环处理消息，没有消息会阻塞wait()
        pthread_create(&_threadId, 0, threadStartCallback, this);
    }

class MVRecordingPreviewHandler: public Handler {
        private:
            MVRecordingPreviewController* previewController;
    
        public:
            MVRecordingPreviewHandler(MVRecordingPreviewController* previewController, MessageQueue* queue) :
                    Handler(queue) {
                this->previewController = previewController;
            }
            void handleMessage(Message* msg) {
                int what = msg->getWhat();
                switch (what) {
                case MSG_EGL_THREAD_CREATE:
                    previewController->initialize();
                    break;
                case MSG_EGL_CREATE_PREVIEW_SURFACE:
                    previewController->createPreviewSurface();
                    break;
                case MSG_SWITCH_CAMERA_FACING:
                    previewController->switchCamera();
                    break;
                case MSG_EGL_DESTROY_PREVIEW_SURFACE:
                    previewController->destroyPreviewSurface();
                    break;
                case MSG_EGL_THREAD_EXIT:
                    previewController->destroy();
                    break;
                case MSG_RENDER_FRAME:
                    previewController->renderFrame();
                    break;
                }
            }
        };

真正的初始化逻辑

bool MVRecordingPreviewController::initialize() {
        const EGLint attribs[] = { EGL_BUFFER_SIZE, 32, EGL_ALPHA_SIZE, 8, EGL_BLUE_SIZE, 8, EGL_GREEN_SIZE, 8, EGL_RED_SIZE, 8, EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT,
                EGL_SURFACE_TYPE, EGL_WINDOW_BIT, EGL_NONE };
        LOGI("Initializing context");
        eglCore = new EGLCore();
        eglCore->init();
        this->createPreviewSurface();
        //到此初始化egl环境已经完成
        ...
        return true;
    }

#### 配置Camera的参数 ####

配置这个参数会从Native层调用Java的Camera的API来获取配置。

@TargetApi(Build.VERSION_CODES.ICE_CREAM_SANDWICH)
        private CameraConfigInfo setUpCamera(final int id) throws CameraParamSettingException {
             forcePreviewSize_640_480();
    //      forcePreviewSize_1280_720();
            // printStackTrace(CameraLoader.class);
            try {
                // 1、开启Camera
                try {
                    mCamera = getCameraInstance(id);
                } catch (CameraParamSettingException e) {
                    throw e;
                }
                boolean mHasPermission = hasPermission();
                if (!mHasPermission) {
                    throw new CameraParamSettingException("拍摄权限被禁用或被其他程序占用, 请确认后再录制");
                }
                Parameters parameters = mCamera.getParameters();
    
                // 2、设置预览照片的图像格式
                List<Integer> supportedPreviewFormats = parameters.getSupportedPreviewFormats();
                if (supportedPreviewFormats.contains(ImageFormat.NV21)) {
                    parameters.setPreviewFormat(ImageFormat.NV21);
                } else {
                    throw new CameraParamSettingException("视频参数设置错误:设置预览图像格式异常");
                }
    
                // 3、设置预览照片的尺寸
                List<Size> supportedPreviewSizes = parameters.getSupportedPreviewSizes();
                int previewWidth = VIDEO_WIDTH;
                int previewHeight = VIDEO_HEIGHT;
                boolean isSupportPreviewSize = isSupportPreviewSize(supportedPreviewSizes, previewWidth, previewHeight);
                if (isSupportPreviewSize) {
                    parameters.setPreviewSize(previewWidth, previewHeight);
                } else {
                    previewWidth = DEFAULT_VIDEO_WIDTH;
                    previewHeight = DEFAULT_VIDEO_HEIGHT;
                    isSupportPreviewSize = isSupportPreviewSize(
                            supportedPreviewSizes, previewWidth, previewHeight);
                    if (isSupportPreviewSize) {
                        VIDEO_WIDTH = DEFAULT_VIDEO_WIDTH;
                        VIDEO_HEIGHT = DEFAULT_VIDEO_HEIGHT;
                        parameters.setPreviewSize(previewWidth, previewHeight);
                    } else {
                        throw new CameraParamSettingException("视频参数设置错误:设置预览的尺寸异常");
                    }
                }
                //下面这行设置 有可能导致 返回的图像尺寸和预期不一致
    //          parameters.setRecordingHint(true);
    
                // 4、设置视频记录的连续自动对焦模式
                if (parameters.getSupportedFocusModes().contains(Camera.Parameters.FOCUS_MODE_CONTINUOUS_VIDEO)) {
                    parameters.setFocusMode(Camera.Parameters.FOCUS_MODE_CONTINUOUS_VIDEO);
                }
    
                try {
                    mCamera.setParameters(parameters);
                } catch (Exception e) {
                    throw new CameraParamSettingException("视频参数设置错误");
                }
    
                int degress = getCameraDisplayOrientation((Activity) mContext, id);
                int cameraFacing = getCameraFacing(id);
                return new CameraConfigInfo(degress, previewWidth, previewHeight, cameraFacing);
            } catch (Exception e) {
                throw new CameraParamSettingException(e.getMessage());
            }
        }

配置Camera的流程

1.  打开对应id的摄像头
2.  设置图像格式，基本上都只支持NV21，NV21也是YUV420的一种模式，然而UV是交叉排放的。
3.  设置预览图像的宽高，我们通用获取支持的宽高集合来进行遍历，寻找适合的宽高来进行设置
4.  设置摄像头的对焦模式，设置连续对焦，这个好像在某些厂商无法连续对焦可以看这里。[https://stackoverflow.com/questions/23357121/android-camera-autofocus-and-focus-mode-continuous-picture][https_stackoverflow.com_questions_23357121_android-camera-autofocus-and-focus-mode-continuous-picture]
5.  旋转摄像头，摄像头的方向和我们理解的稍微有点差别，所以需要根据屏幕的旋转角度来旋转摄像头，然后让我们的预览画面看起来更合理，具体算法可以看源码。(本文使用opengl来处理图像旋转，没有直接旋转摄像头)  
    以上设置基本上就可以等待预览了。

#### 初始化OpenGL绘制参数(纹理、fbo、顶点坐标等) ####

void RecordingPreviewRenderer::init(int degress, bool isVFlip, int textureWidth, int textureHeight, int cameraWidth, int cameraHeight) {
        LOGI("enter RecordingPreviewRenderer::init() textureWidth is %d, textureHeight is %d", textureWidth, textureHeight);
        this->degress = degress;
        this->isVFlip = isVFlip;
        textureCoords = new GLfloat[textureCoordsSize];
        //下面这个方法通过旋转的角度设置对应的坐标
        //如果是前置摄像头再进行镜像左右翻转
        //看一个没有旋转的矩阵坐标吧
        /*
        static GLfloat CAMERA_TEXTURE_NO_ROTATION[8] = {
            0.0f, 1.0f,
            1.0f, 1.0f,
            0.0f, 0.0f,
            1.0f, 0.0f
        };
        */  
        this->fillTextureCoords();
        this->textureWidth = textureWidth;
        this->textureHeight = textureHeight;
        this->cameraWidth = cameraWidth;
        this->cameraHeight = cameraHeight;
        //1、初始化fbo绘制的program
        mCopier = new GPUTextureFrameCopier();
        mCopier->init();
        //初始化屏幕绘制的program
        mRenderer = new VideoGLSurfaceRender();
        mRenderer->init(textureWidth, textureHeight);
        //初始化扩展纹理
        cameraTexFrame = new GPUTextureFrame();
        cameraTexFrame->createTexture();
        //初始化fbo的输入纹理
        glGenTextures(1, &inputTexId);
        checkGlError("glGenTextures inputTexId");
        glBindTexture(GL_TEXTURE_2D, inputTexId);
        checkGlError("glBindTexture inputTexId");
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
        GLint internalFormat = GL_RGBA;
        glTexImage2D(GL_TEXTURE_2D, 0, internalFormat, (GLsizei) textureWidth, (GLsizei) textureHeight, 0, internalFormat, GL_UNSIGNED_BYTE, 0);
        glBindTexture(GL_TEXTURE_2D, 0);
        //初始化fbo的输出纹理
        glGenTextures(1, &outputTexId);
        checkGlError("glGenTextures outputTexId");
        glBindTexture(GL_TEXTURE_2D, outputTexId);
        checkGlError("glBindTexture outputTexId");
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
        glTexImage2D(GL_TEXTURE_2D, 0, internalFormat, (GLsizei) textureWidth, (GLsizei) textureHeight, 0, internalFormat, GL_UNSIGNED_BYTE, 0);
        glBindTexture(GL_TEXTURE_2D, 0);
        //初始化fbo
        glGenFramebuffers(1, &FBO);
        checkGlError("glGenFramebuffers");
        //初始化旋转纹理
        glGenTextures(1, &rotateTexId);
        checkGlError("glGenTextures rotateTexId");
        glBindTexture(GL_TEXTURE_2D, rotateTexId);
        checkGlError("glBindTexture rotateTexId");
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
    
        if (degress == 90 || degress == 270)
            glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, cameraHeight, cameraWidth, 0, GL_RGBA, GL_UNSIGNED_BYTE, 0);
        else
            glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, cameraWidth, cameraHeight, 0, GL_RGBA, GL_UNSIGNED_BYTE, 0);
    
        glBindTexture(GL_TEXTURE_2D, 0);
        //暂停时候的纹理
        mixFilterId = -1;
        glGenTextures(1, &pausedTexId);
        checkGlError("glGenTextures pausedTexId");
        glBindTexture(GL_TEXTURE_2D, pausedTexId);
        checkGlError("glBindTexture pausedTexId");
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
        glTexImage2D(GL_TEXTURE_2D, 0, internalFormat, (GLsizei) textureWidth, (GLsizei) textureHeight, 0, internalFormat, GL_UNSIGNED_BYTE, 0);
        glBindTexture(GL_TEXTURE_2D, 0);
        LOGI("leave RecordingPreviewRenderer::init()");
    }

关于上面的初始化主要需要注意：

片段着色器的纹理需要使用OpenGL ES扩展纹理类型，最终的着色器代码

static char* GPU_FRAME_FRAGMENT_SHADER =
        "#extension GL_OES_EGL_image_external : require\n"
        "precision mediump float;\n"
        "uniform samplerExternalOES yuvTexSampler;\n"
        "varying vec2 yuvTexCoords;\n"
        "void main() {\n"
        "  gl_FragColor = texture2D(yuvTexSampler, yuvTexCoords);\n"
        "}\n";

对应的生成的纹理也需要改变类型

int GPUTextureFrame::initTexture() {
        glGenTextures(1, &decodeTexId);
        glBindTexture(GL_TEXTURE_EXTERNAL_OES, decodeTexId);
        if (checkGlError("glBindTexture")) {
            return -1;
        }
        glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
        if (checkGlError("glTexParameteri")) {
            return -1;
        }
        glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
        if (checkGlError("glTexParameteri")) {
            return -1;
        }
        glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
        if (checkGlError("glTexParameteri")) {
            return -1;
        }
        glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
        if (checkGlError("glTexParameteri")) {
            return -1;
        }
        return 1;
    }

#### 开始循环绘制 ####

准备工作做完之后就可以启动摄像头的绘制了，这里通用从Native调用Java层开启绘制，并将扩展纹理的id传过来。

/** 当底层EGLThread创建初纹理之后,设置给Camera **/
        public void startPreviewFromNative(int textureId) {
            mCamera.setCameraPreviewTexture(textureId);
        }

@TargetApi(Build.VERSION_CODES.JELLY_BEAN)
        public void setCameraPreviewTexture(int textureId) {
            Log.i(TAG, "setCameraPreviewTexture...");
            mCameraSurfaceTexture = new SurfaceTexture(textureId);
            try {
                mCamera.setPreviewTexture(mCameraSurfaceTexture);
                mCameraSurfaceTexture.setOnFrameAvailableListener(new SurfaceTexture.OnFrameAvailableListener() {
                    @Override
                    public void onFrameAvailable(SurfaceTexture surfaceTexture) {
                        if (null != mCallback) {
    //                      Log.d("RecordingPublisher", "surfaceTexture time stamp is "+surfaceTexture.getTimestamp()/1000000000.0f);
                            mCallback.notifyFrameAvailable();
                        }
                    }
                });
                mCamera.startPreview();
            } catch (Exception e) {
                e.printStackTrace();
            }
        }

我们使用扩展纹理的Id生成一个SurfaceTexture，给Camera设置一个回调，在Camera获取到一帧数据之后会调用这个接口，我们要做的就是通知Native层进行绘制。然而这里还有一点需要做，就是在绘制之前需要调用SurfaceTexture先更新纹理的数据。

void MVRecordingPreviewController::renderFrame() {
        if (NULL != eglCore && !isInSwitchingCamera) {
            if (startTime == -1) {
                startTime = getCurrentTime();
            }
            float position = ((float) (getCurrentTime() - startTime)) / 1000.0f;
            //这个方法会先回调java层来更新SurfaceTexture
            this->processVideoFrame(position);
            if (previewSurface != EGL_NO_SURFACE) {
                this->draw();
            }
        }
    }

接下来看这个方法，在扩展纹理有数据之后，

void RecordingPreviewRenderer::processFrame(float position) {
        glBindFramebuffer(GL_FRAMEBUFFER, FBO);
        checkGlError("glBindFramebuffer FBO");
    
        if (degress == 90 || degress == 270)
            glViewport(0, 0, cameraHeight, cameraWidth);
        else
            glViewport(0, 0, cameraWidth, cameraHeight);
    
        GLfloat* vertexCoords = this->getVertexCoords();
        //这个方法将扩展纹理绘制到了旋转的纹理上，经过一系列的矩阵变换
        mCopier->renderWithCoords(cameraTexFrame, rotateTexId, vertexCoords, textureCoords);
    
        int rotateTexWidth = cameraWidth;
        int rotateTexHeight = cameraHeight;
        if (degress == 90 || degress == 270){
            rotateTexWidth = cameraHeight;
            rotateTexHeight = cameraWidth;
        }
        //这个方法将旋转纹理绘制到了输入纹理上
        //注意目前都是在fbo上绘制，没有绘制到屏幕
        mRenderer->renderToAutoFitTexture(rotateTexId, rotateTexWidth, rotateTexHeight, inputTexId);
        //解绑Fbo
        glBindFramebuffer(GL_FRAMEBUFFER, 0);
    }

处理完成之后我们的inputTexId纹理是有数据的，然后我们再绘制到屏幕上。绘制的逻辑基本上大同小异，都是矩阵，顶点坐标以及纹理坐标。至于为什么要这些变换，当然是为了做视频的处理了。

void MVRecordingPreviewController::draw() {
        eglCore->makeCurrent(previewSurface);
        renderer->drawToViewWithAutofit(screenWidth, screenHeight, textureWidth, textureHeight);
        if (!eglCore->swapBuffers(previewSurface)) {
            LOGE("eglSwapBuffers(previewSurface) returned error %d", eglGetError());
        }
    }

最后总结一下，相机预览纹理需要是用扩展纹理而不是普通的2D纹理，对应的着色器也需要有变化，摄像头需要根据屏幕来做对应的旋转，然后就是使用了大量的fbo绘制到纹理的做法来进行图像的处理。  
[参考源码][Link 1]

[https_stackoverflow.com_questions_23357121_android-camera-autofocus-and-focus-mode-continuous-picture]: https://stackoverflow.com/questions/23357121/android-camera-autofocus-and-focus-mode-continuous-picture
[Link 1]: https://github.com/zhanxiaokai/Android-CameraPreview