AR——在摄像头上叠加一个视频

小灰灰 2022-08-20 05:13 9阅读 0赞

今天接到的新任务，看能不能在摄像头的视场叠加一个视频和声音。

经过3个小时的努力还是成功了，openframework中的examples 真是无尽的宝藏啊，我在里面的一个opencvdemo中看到了一个面板中有3个视频的不同处理，这个代码给我启发，让我进一步了解了openframework这个开发包的套路，我打算在后面，多做几个例子后再仔细写一个openframework的工作机制和汉明码的识别与媒体叠加的工作过程，这里也就先放着，能够一步步在别人的代码上摸索出新的东西，虽然还不完全懂地依样画葫芦，但自己的热情也渐渐上涨，对它们的工作过程也渐渐有了深入的认识。

兴趣很重要。

这是源代码:

ofApp.cpp

#include "ofApp.h"
    #include <opencv2/core/core.hpp>
    #include <opencv2/imgproc/imgproc.hpp>
    #include <opencv2/highgui/highgui.hpp>
    
    using namespace cv;
    
    //--------------------------------------------------------------
    void ofApp::setup(){
    	markNum = 0;
    	loopNum = 0;
    	markDuration = 0;
    	isFindMark = false;
    	ofSetLogLevel(OF_LOG_VERBOSE);
        ofBackground(50, 0);
    	//ofSetWindowShape(640, 480);
    	ofSetWindowShape(640, 480);//设置面板(窗口)大小
    	bAnimate = true;
    	//ofSetVerticalSync(true);
    	/* initialize lighting */
    
    
    
    	ofEnableDepthTest();
        ofDisableArbTex(); // we need GL_TEXTURE_2D for our models coords.
    
    	model.loadModel("sphere.3ds", true);
    	model.setScale(0.01f, 0.01f, 0.01f);
    	model.setRotation(0, 90, -2, 0, 0);
    	model.setLoopStateForAllAnimations(OF_LOOP_NORMAL);
    	model.playAllAnimations();
    	//if (bAnimate) {
    		//model.setPausedForAllAnimations(true);
    	//}
    	//m_models.push_back(model);
    //	}
    	//m_meshes.resize(model_names.size(), ofMesh());
    	
    	m_video.setDeviceID(0);
    	m_video.initGrabber(640, 480);
    
    	Point3f corners_3d[] = 
    	{
    		Point3f(-0.5f, -0.5f, 0),
    		Point3f(-0.5f,  0.5f, 0),
    		Point3f( 0.5f,  0.5f, 0),
    		Point3f( 0.5f, -0.5f, 0)
    	};
    	float camera_matrix[] = 
    	{
    		848.069f, 0.0f, 268.697f,
    		0.0f, 847.687f, 264.266f,
    		0.0f, 0.0f, 1.0f
    	};
    	float dist_coeff[] = {-0.445957f, 0.278828f, -0.002213f, -0.000656f};
    
    	m_corners_3d = vector<Point3f>(corners_3d, corners_3d + 4);
    	//Mat构造函数不会拷贝数据，只会将指针指向数据区域，所以对于局部变量内存，需要clone
    	m_camera_matrix = Mat(3, 3, CV_32FC1, camera_matrix).clone();
    	m_dist_coeff = Mat(1, 4, CV_32FC1, dist_coeff).clone();
    	/new add movie
    	vidPlayer.load("2.mp4");
    	vidPlayer.play();
    	vidPlayer.setLoopState(OF_LOOP_NORMAL);//播放是否循
    	/new add movie
    }
    
    //--------------------------------------------------------------
    void ofApp::update() {
    	/*for (int i = 0; i < m_models.size(); ++i)
    	{
    		m_models[i].update();
    		m_meshes[i] = m_models[i].getCurrentAnimatedMesh(0);
    	}*/
    	vidPlayer.update();
    	colorImg.setFromPixels(vidPlayer.getPixels());
    	model.update();
    	m_video.update();
    
    
    	if (m_video.isFrameNew())
    	{
    		bool good_format = true;
    		int width = m_video.getWidth();
    		int height = m_video.getHeight();
    		m_img_gray.create(height, width, CV_8UC1);
    
    		ofPixelFormat format = m_video.getPixelFormat();
    		switch (format)
    		{
    		case OF_PIXELS_RGB:
    			m_img_color.create(height, width, CV_8UC3);
    			memcpy(m_img_color.data, m_video.getPixels(), height*width * 3);
    			cvtColor(m_img_color, m_img_gray, CV_RGB2GRAY);
    			break;
    		case OF_PIXELS_RGBA:
    			m_img_color.create(height, width, CV_8UC4);
    			memcpy(m_img_color.data, m_video.getPixels(), height*width * 4);
    			cvtColor(m_img_color, m_img_gray, CV_RGBA2GRAY);
    			break;
    		case OF_PIXELS_BGRA:
    			m_img_color.create(height, width, CV_8UC4);
    			memcpy(m_img_color.data, m_video.getPixels(), height*width * 4);
    			cvtColor(m_img_color, m_img_gray, CV_BGRA2GRAY);
    			break;
    		default:
    			good_format = false;
    			cout << "Unsupported video format!" << endl;
    			break;
    		}
    		if (!good_format) return;
    
    		m_recognizer.update(m_img_gray, 100);
    	}
    	
    	if ((!isFindMark)&&(loopNum%50==0)){  //对视频帧采样以减小错误
    		markDuration = 0;               
    	}
    	
    }
    void ofApp::draw() {
    	
    	//ofSetColor(255);
        
    	
    	float view_width = ofGetViewportWidth();
    	float view_height = ofGetViewportHeight();
    	//m_video.draw(0, 0, view_width, view_height);//这个是在面板上将摄像头视野占满
    	m_video.draw(0, 0, 640, 480);//摄像头的大小为640*480
    	colorImg.resize(200, 200);//设置视频的长宽为200*200 /*画视频的函数不能放在画摄像头的前面，不然会放覆盖掉*/
    	colorImg.draw(0, 0);//括号中是视频左上角的坐标 0,0就放在了视野的左上角
    	
    	//Set camera matrix to the opengl projection matrix;
    	intrinsicMatrix2ProjectionMatrix(m_camera_matrix, 640, 480, 0.01f, 100.0f, m_projection_matrix);
    	ofSetMatrixMode(OF_MATRIX_PROJECTION);
    	//Openframeworks里将(-1, -1)映射到屏幕左上角，而非一般的左下角，所以需要一个矩阵进行垂直镜像
    	static float reflect[] =
    	{
    		1,  0, 0, 0,
    		0, -1, 0, 0,
    		0,  0, 1, 0,
    		0,  0, 0, 1
    	};
    	ofLoadMatrix(reflect);
    	//OpenGL默认为右乘
    	ofMultMatrix(m_projection_matrix);
    
    	//Reset model view matrix to identity;
    	ofSetMatrixMode(OF_MATRIX_MODELVIEW);
    	ofLoadIdentityMatrix();
    
    	//ofDrawPlane(-1,-1,-1,2,2);
    
    	//Set opengl parameters
    	ofSetColor(255);
    	ofEnableBlendMode(OF_BLENDMODE_ALPHA);
    	ofEnableDepthTest();
    	glShadeModel(GL_SMOOTH); //some model / light stuff
    	m_light.enable();
    	ofEnableSeparateSpecularLight();
    
    	vector<Marker>& markers = m_recognizer.getMarkers();
    	Mat r, t;
    
    	isFindMark = false;
    	loopNum++;
    	if (markers.size() != 0) {
    		markNum = markers.size(); 
    	}
    
    	for (int i = 0; i < markers.size(); ++i)
    		//if(markers.size()>0)
    	{
    		isFindMark = true;
    
    		//求出的旋转矩阵r的行列式为+1，即为刚体变换，所以不改变坐标系的手性
    		markers[i].estimateTransformToCamera(m_corners_3d, m_camera_matrix, m_dist_coeff, r, t);
    		extrinsicMatrix2ModelViewMatrix(r, t, m_model_view_matrix);
    		ofLoadMatrix(m_model_view_matrix);
    		ofxAssimpAnimation & animation = model.getAnimation(1);
    	   
    			model.drawFaces();
    	}
    	
    	//Reset parameters
    		ofDisableDepthTest();
    		m_light.disable();
    		ofDisableLighting();
    		ofDisableSeparateSpecularLight();
    
    		ofSetMatrixMode(OF_MATRIX_MODELVIEW);
    		ofLoadIdentityMatrix();
    		ofSetMatrixMode(OF_MATRIX_PROJECTION);
    		ofLoadIdentityMatrix();
    		//colorImg.draw(400, 400);
    }
    
    void ofApp::intrinsicMatrix2ProjectionMatrix(cv::Mat& camera_matrix, float width, float height, float near_plane, float far_plane, float* projection_matrix)
    {
    	float f_x = camera_matrix.at<float>(0,0);
    	float f_y = camera_matrix.at<float>(1,1);
    
    	float c_x = camera_matrix.at<float>(0,2);
    	float c_y = camera_matrix.at<float>(1,2);
    	
    	projection_matrix[0] = 2*f_x/width;
    	projection_matrix[1] = 0.0f;
    	projection_matrix[2] = 0.0f;
    	projection_matrix[3] = 0.0f;
    
    	projection_matrix[4] = 0.0f;
    	projection_matrix[5] = 2*f_y/height;
    	projection_matrix[6] = 0.0f;
    	projection_matrix[7] = 0.0f;
    
    	projection_matrix[8] = 1.0f - 2*c_x/width;
    	projection_matrix[9] = 2*c_y/height - 1.0f;
    	projection_matrix[10] = -(far_plane + near_plane)/(far_plane - near_plane);
    	projection_matrix[11] = -1.0f;
    
    	projection_matrix[12] = 0.0f;
    	projection_matrix[13] = 0.0f;
    	projection_matrix[14] = -2.0f*far_plane*near_plane/(far_plane - near_plane);
    	projection_matrix[15] = 0.0f;
    }
    
    void ofApp::extrinsicMatrix2ModelViewMatrix(cv::Mat& rotation, cv::Mat& translation, float* model_view_matrix)
    {
    	//绕X轴旋转180度，从OpenCV坐标系变换为OpenGL坐标系
    	static double d[] = 
    	{
    		1, 0, 0,
    		0, -1, 0,
    		0, 0, -1
    	};
    	Mat_<double> rx(3, 3, d);
    
    	rotation = rx*rotation;
    	translation = rx*translation;
    
    	model_view_matrix[0] = rotation.at<double>(0,0);
    	model_view_matrix[1] = rotation.at<double>(1,0);
    	model_view_matrix[2] = rotation.at<double>(2,0);
    	model_view_matrix[3] = 0.0f;
    
    	model_view_matrix[4] = rotation.at<double>(0,1);
    	model_view_matrix[5] = rotation.at<double>(1,1);
    	model_view_matrix[6] = rotation.at<double>(2,1);
    	model_view_matrix[7] = 0.0f;
    
    	model_view_matrix[8] = rotation.at<double>(0,2);
    	model_view_matrix[9] = rotation.at<double>(1,2);
    	model_view_matrix[10] = rotation.at<double>(2,2);
    	model_view_matrix[11] = 0.0f;
    
    	model_view_matrix[12] = translation.at<double>(0, 0);
    	model_view_matrix[13] = translation.at<double>(1, 0);
    	model_view_matrix[14] = translation.at<double>(2, 0);
    	model_view_matrix[15] = 1.0f;
    }

我把视频叠加在了摄像头视野的左上角，并且有声音，其大小，位置均可调。

ofApp.h中添加这几句:

ofVideoPlayer 		vidPlayer;
    		ofxCvColorImage			colorImg;