虚拟视点图像生成006

柔情只为你懂 2022-08-08 11:50 149阅读 0赞

这一段时间的研究成果有：

（1）对于映射到虚拟视点后直接取整这个问题，对他进行了改进。

方法：首先开辟存储空间进行存储映射后的深度信息，而不是存储量化后的深度值；

其次开辟存储空间存储四舍五入后的整数像素点到映射后的精确位置的欧拉值；

再后对于映射后的像素点不是简单地取整，而是四舍五入进行取整；

最后，对于映射到同一位置的像素点，比较深度信息，深度值大的进行显示，深度值相同的，比较欧拉值，欧拉值小的进行显示。

结果：psnr值和ssim都有一定程度的提高，成功！！！！！

（2）对于映射后两张图片亮度不同的情况，进行色彩空间转换，rgb空间到Lab和hsv空间，然后在Lab和hsv空间进行插值。对亮度取合成后图像的周围八像素的平均值，其他通道取两张图片的平均值。

结果：psnr值和ssim值在不同的空间进行插值后没有明显的提高，失败！！！！

（3）对于只取一张图片进行分析的情况存在随机误差的可能性较大，所以遍历所有图片计算ssim和psnr，计算所有图片的平均值。

结果：程序实现成功！！！！

源程序

#ifndef _wrapingOf3D1
    #define _wrapingOf3D1
    #include<iostream>
    #include<opencv2\opencv.hpp>
    #include <math.h> 
    #include <stdio.h> 
    #include <cv.h>	
    #include <highgui.h>
    #include<io.h>
    using std::cout;
    using std::endl;
    using namespace std;
    #include <cmath>
    #include<string.h>
    #include <iostream>
    #include <fstream>
    using namespace std;
    double round_off(double x)
    {
    	double temp = floor(x + 0.5);
    	return temp;
    }
    double max(double x, double y) {
    	return ((x > y) ? x : y);
    }
    double ssim(char *ref_image, char *obj_image)
    {
    	// default settings
    	double C1 = 6.5025, C2 = 58.5225;
    
    	IplImage
    		*img1 = NULL, *img2 = NULL, *img1_img2 = NULL,
    		*img1_temp = NULL, *img2_temp = NULL,
    		*img1_sq = NULL, *img2_sq = NULL,
    		*mu1 = NULL, *mu2 = NULL,
    		*mu1_sq = NULL, *mu2_sq = NULL, *mu1_mu2 = NULL,
    		*sigma1_sq = NULL, *sigma2_sq = NULL, *sigma12 = NULL,
    		*ssim_map = NULL, *temp1 = NULL, *temp2 = NULL, *temp3 = NULL;
    
    
    	/***************************** INITS **********************************/
    	img1_temp = cvLoadImage(ref_image);
    	img2_temp = cvLoadImage(obj_image);
    
    	if (img1_temp == NULL || img2_temp == NULL)
    		return -1;
    
    	int x = img1_temp->width, y = img1_temp->height;
    	int nChan = img1_temp->nChannels, d = IPL_DEPTH_32F;
    	CvSize size = cvSize(x, y);
    
    	img1 = cvCreateImage(size, d, nChan);
    	img2 = cvCreateImage(size, d, nChan);
    
    	cvConvert(img1_temp, img1);
    	cvConvert(img2_temp, img2);
    	cvReleaseImage(&img1_temp);
    	cvReleaseImage(&img2_temp);
    
    
    	img1_sq = cvCreateImage(size, d, nChan);
    	img2_sq = cvCreateImage(size, d, nChan);
    	img1_img2 = cvCreateImage(size, d, nChan);
    
    	cvPow(img1, img1_sq, 2);
    	cvPow(img2, img2_sq, 2);
    	cvMul(img1, img2, img1_img2, 1);
    
    	mu1 = cvCreateImage(size, d, nChan);
    	mu2 = cvCreateImage(size, d, nChan);
    
    	mu1_sq = cvCreateImage(size, d, nChan);
    	mu2_sq = cvCreateImage(size, d, nChan);
    	mu1_mu2 = cvCreateImage(size, d, nChan);
    
    
    	sigma1_sq = cvCreateImage(size, d, nChan);
    	sigma2_sq = cvCreateImage(size, d, nChan);
    	sigma12 = cvCreateImage(size, d, nChan);
    
    	temp1 = cvCreateImage(size, d, nChan);
    	temp2 = cvCreateImage(size, d, nChan);
    	temp3 = cvCreateImage(size, d, nChan);
    
    	ssim_map = cvCreateImage(size, d, nChan);
    	/*************************** END INITS **********************************/
    
    
    	//
    	// PRELIMINARY COMPUTING
    	cvSmooth(img1, mu1, CV_GAUSSIAN, 11, 11, 1.5);
    	cvSmooth(img2, mu2, CV_GAUSSIAN, 11, 11, 1.5);
    
    	cvPow(mu1, mu1_sq, 2);
    	cvPow(mu2, mu2_sq, 2);
    	cvMul(mu1, mu2, mu1_mu2, 1);
    
    
    	cvSmooth(img1_sq, sigma1_sq, CV_GAUSSIAN, 11, 11, 1.5);
    	cvAddWeighted(sigma1_sq, 1, mu1_sq, -1, 0, sigma1_sq);
    
    	cvSmooth(img2_sq, sigma2_sq, CV_GAUSSIAN, 11, 11, 1.5);
    	cvAddWeighted(sigma2_sq, 1, mu2_sq, -1, 0, sigma2_sq);
    
    	cvSmooth(img1_img2, sigma12, CV_GAUSSIAN, 11, 11, 1.5);
    	cvAddWeighted(sigma12, 1, mu1_mu2, -1, 0, sigma12);
    
    
    	//
    	// FORMULA
    
    	// (2*mu1_mu2 + C1)
    	cvScale(mu1_mu2, temp1, 2);
    	cvAddS(temp1, cvScalarAll(C1), temp1);
    
    	// (2*sigma12 + C2)
    	cvScale(sigma12, temp2, 2);
    	cvAddS(temp2, cvScalarAll(C2), temp2);
    
    	// ((2*mu1_mu2 + C1).*(2*sigma12 + C2))
    	cvMul(temp1, temp2, temp3, 1);
    
    	// (mu1_sq + mu2_sq + C1)
    	cvAdd(mu1_sq, mu2_sq, temp1);
    	cvAddS(temp1, cvScalarAll(C1), temp1);
    
    	// (sigma1_sq + sigma2_sq + C2)
    	cvAdd(sigma1_sq, sigma2_sq, temp2);
    	cvAddS(temp2, cvScalarAll(C2), temp2);
    
    	// ((mu1_sq + mu2_sq + C1).*(sigma1_sq + sigma2_sq + C2))
    	cvMul(temp1, temp2, temp1, 1);
    
    	// ((2*mu1_mu2 + C1).*(2*sigma12 + C2))./((mu1_sq + mu2_sq + C1).*(sigma1_sq + sigma2_sq + C2))
    	cvDiv(temp3, temp1, ssim_map, 1);
    
    
    	CvScalar index_scalar = cvAvg(ssim_map);
    
    	// through observation, there is approximately 
    	// 1% error max with the original matlab program
    
    	cout << "(R, G & B SSIM index)" << std::endl;
    	cout << index_scalar.val[2] << endl;
    	cout << index_scalar.val[1] << endl;
    	cout << index_scalar.val[0] << endl;
    
    	cvReleaseImage(&img1_sq);
    	cvReleaseImage(&img2_sq);
    	cvReleaseImage(&img1_img2);
    	cvReleaseImage(&mu1);
    	cvReleaseImage(&mu2);
    	cvReleaseImage(&mu1_sq);
    	cvReleaseImage(&mu2_sq);
    	cvReleaseImage(&mu1_mu2);
    	cvReleaseImage(&sigma1_sq);
    	cvReleaseImage(&sigma2_sq);
    	cvReleaseImage(&sigma12);
    	cvReleaseImage(&temp1);
    	cvReleaseImage(&temp2);
    	cvReleaseImage(&temp3);
    	cvReleaseImage(&ssim_map);
    	//double ssim=max(max(index_scalar.val[0], index_scalar.val[1]), index_scalar.val[2]);
    	double ssim = (index_scalar.val[0] + index_scalar.val[1] + index_scalar.val[2]) / 3;
    	return ssim;
    }
    
    double psnr(char *ref_image, char *obj_image)
    {
    	cv::Mat image_ref = cv::imread(ref_image);
    	cv::Mat image_obj = cv::imread(obj_image);
    	double mse = 0;
    	double div_r = 0;
    	double div_g = 0;
    	double div_b = 0;
    	int width = image_ref.cols;
    	int height = image_ref.rows;
    	double psnr = 0;
    	for (int v = 0; v < height; v++)
    	{
    		for (int u = 0; u < width; u++)
    		{
    			div_r = image_ref.at<cv::Vec3b>(v, u)[0] - image_obj.at<cv::Vec3b>(v, u)[0];
    			div_g = image_ref.at<cv::Vec3b>(v, u)[1] - image_obj.at<cv::Vec3b>(v, u)[1];
    			div_b = image_ref.at<cv::Vec3b>(v, u)[2] - image_obj.at<cv::Vec3b>(v, u)[2];
    			mse += ((div_r*div_r + div_b*div_b + div_g*div_g) / 3);
    
    		}
    	}
    	mse = mse / (width*height);
    	psnr = 10 * log10(255 * 255 / mse);
    	printf("%lf\n", mse);
    	printf("%lf\n", psnr);
    	return psnr;
    }
    
    class Histogram1D{
    private:
    	int histSize[1];
    	float hranges[2];
    	const float *ranges[1];
    	int channels[1];
    public:
    	Histogram1D(){
    		histSize[0] = 256;
    		hranges[0] = 0.0;
    		hranges[1] = 255.0;
    		ranges[0] = hranges;
    		channels[0] = 0;
    	}
    	cv::MatND getHistogram(const cv::Mat &image){
    		cv::MatND hist;
    		cv::calcHist(&image, 1, channels, cv::Mat(), hist, 1, histSize, ranges);
    		return hist;
    	}
    	cv::Mat getHistogramImage(const cv::Mat &image){
    		cv::MatND hist = getHistogram(image);
    		double maxVal = 0;
    		double minVal = 0;
    		cv::minMaxLoc(hist, &minVal, &maxVal, 0, 0);
    		maxVal = 25500*2;
    		minVal = 0;
    		cv::Mat histImg(histSize[0], histSize[0], CV_8U, cv::Scalar(255));
    		int hpt = static_cast<int>(0.9*histSize[0]);
    		for (int h = 0; h < histSize[0]; h++){
    			float binVal = hist.at<float>(h);
    			int intensity = static_cast<int>(binVal*hpt / maxVal);
    			cv::line(histImg, cv::Point(h, histSize[0]), cv::Point(h, histSize[0] - intensity), cv::Scalar::all(100), 1);
    		}
    		return histImg;
    	}
    };
    class Histogram1D1{
    private:
    	int histSize[1];
    	float hranges[2];
    	const float *ranges[1];
    	int channels[1];
    public:
    	Histogram1D1(){
    		histSize[0] = 256;
    		hranges[0] = 0.0;
    		hranges[1] = 255.0;
    		ranges[0] = hranges;
    		channels[0] = 1;
    	}
    	cv::MatND getHistogram(const cv::Mat &image){
    		cv::MatND hist;
    		cv::calcHist(&image, 1, channels, cv::Mat(), hist, 1, histSize, ranges);
    		return hist;
    	}
    	cv::Mat getHistogramImage(const cv::Mat &image){
    		cv::MatND hist = getHistogram(image);
    		double maxVal = 0;
    		double minVal = 0;
    		cv::minMaxLoc(hist, &minVal, &maxVal, 0, 0);
    		maxVal = 25500*2;
    		minVal = 0;
    		cv::Mat histImg(histSize[0], histSize[0], CV_8U, cv::Scalar(255));
    		int hpt = static_cast<int>(0.9*histSize[0]);
    		for (int h = 0; h < histSize[0]; h++){
    			float binVal = hist.at<float>(h);
    			int intensity = static_cast<int>(binVal*hpt / maxVal);
    			cv::line(histImg, cv::Point(h, histSize[0]), cv::Point(h, histSize[0] - intensity), cv::Scalar::all(100), 1);
    		}
    		return histImg;
    	}
    };
    class Histogram1D2{
    private:
    	int histSize[1];
    	float hranges[2];
    	const float *ranges[1];
    	int channels[1];
    public:
    	Histogram1D2(){
    		histSize[0] = 256;
    		hranges[0] = 0.0;
    		hranges[1] = 255.0;
    		ranges[0] = hranges;
    		channels[0] = 2;
    	}
    	cv::MatND getHistogram(const cv::Mat &image){
    		cv::MatND hist;
    		cv::calcHist(&image, 1, channels, cv::Mat(), hist, 1, histSize, ranges);
    		return hist;
    	}
    	cv::Mat getHistogramImage(const cv::Mat &image){
    		cv::MatND hist = getHistogram(image);
    		double maxVal = 0;
    		double minVal = 0;
    		cv::minMaxLoc(hist, &minVal, &maxVal, 0, 0);
    		maxVal = 25500*2;
    		minVal = 0;
    		cv::Mat histImg(histSize[0], histSize[0], CV_8U, cv::Scalar(255));
    		int hpt = static_cast<int>(0.9*histSize[0]);
    		for (int h = 0; h < histSize[0]; h++){
    			float binVal = hist.at<float>(h);
    			int intensity = static_cast<int>(binVal*hpt / maxVal);
    			cv::line(histImg, cv::Point(h, histSize[0]), cv::Point(h, histSize[0] - intensity), cv::Scalar::all(100), 1);
    		}
    		return histImg;
    	}
    };
    class ColorHistogram{
    private:
    	int histSize[3];
    	float hranges[2];
    	const float *ranges[3];
    	int channels[3];
    public:
    	ColorHistogram(){
    		histSize[0] = histSize[1] = histSize[2] = 256;
    		hranges[0] = 0.0;
    		hranges[1] = 255.0;
    		ranges[0] = hranges;
    		ranges[1] = hranges;
    		ranges[2] = hranges;
    		channels[0] = 0;
    		channels[1] = 1;
    		channels[2] = 2;
    	}
    	cv::MatND getHistogram(const cv::Mat &image){
    		cv::MatND hist;
    		cv::calcHist(&image, 1, channels, cv::Mat(), hist, 3, histSize, ranges);
    		return hist;
    	}
    	cv::SparseMat getSparseHistogram(const cv::Mat &image){
    		cv::SparseMat hist(3, histSize, CV_32F);
    		cv::calcHist(&image, 1, channels, cv::Mat(), hist, 3, histSize, ranges);
    		return hist;
    	}
    };
    /*
    **define a struct included intrinsic and extrinsic args
    */
    typedef struct {
    	double m_K[3][3]; // 3x3 intrinsic matrix
    	double m_RotMatrix[3][3]; // rotation matrix
    	double m_Trans[3]; // translation vector
    
    	double m_ProjMatrix[4][4]; // projection matrix
    } CalibStruct;
    /*
    **define globle variables
    */
    CalibStruct m_CalibParams[8];
    int m_NumCameras = 8;
    int m_Width = 1024, m_Height = 768; // camera resolution is 1024x768
    double pts[8][3];
    
    
    /*
    **declare function
    */
    void InitializeFromFile(char *fileName);
    double DepthLevelToZ(unsigned char d);
    unsigned char ZToDepthLever(double z);
    double projXYZtoUV(double projMatrix[4][4], double x, double y, double z, double *u, double *v);
    void projUVZtoXY(double projMatrix[4][4], double u, double v, double z, double *x, double *y, double pt[8][2]);
    void wrapingImage(int ref, int proj, cv::Mat &imageColor, cv::Mat &imageDepth, cv::Mat &imageColorOut);
    void pointProject_from_ref_to_otherView(double pts[8][2], int ref, int u, int v, unsigned char d);
    /*
    **define function
    */
    
    /*
    ** read text file and write args to struct of globle variable
    */
    void readCalibrationFile(char *fileName)
    {
    	int i, j, k;
    	FILE *pIn;
    	double dIn; // dummy variable
    	int camIdx;
    
    	if (pIn = fopen(fileName, "r"))
    	{
    		for (k = 0; k<m_NumCameras; k++)
    		{
    			// camera index
    			fscanf(pIn, "%d", &camIdx);
    			//std::cout << camIdx << std::endl;
    
    			// camera intrinsics
    			for (i = 0; i < 3; i++){
    				fscanf(pIn, "%lf\t%lf\t%lf", &(m_CalibParams[camIdx].m_K[i][0]), &(m_CalibParams[camIdx].m_K[i][1]), &(m_CalibParams[camIdx].m_K[i][2]));
    				//std::cout << (m_CalibParams[camIdx].m_K[i][0])<<"\t"<<(m_CalibParams[camIdx].m_K[i][1]) <<"\t"<< (m_CalibParams[camIdx].m_K[i][2]) << std::endl;
    			}
    
    			// read barrel distortion params (assume 0)
    			fscanf(pIn, "%lf", &dIn);
    			fscanf(pIn, "%lf", &dIn);
    
    			// read extrinsics
    			for (i = 0; i<3; i++)
    			{
    				for (j = 0; j<3; j++)
    				{
    					fscanf(pIn, "%lf", &dIn);
    					m_CalibParams[camIdx].m_RotMatrix[i][j] = dIn;
    					//std::cout << m_CalibParams[camIdx].m_RotMatrix[i][j] << std::endl;
    				}
    
    				fscanf(pIn, "%lf", &dIn);
    				m_CalibParams[camIdx].m_Trans[i] = dIn;
    			}
    
    		}
    
    		fclose(pIn);
    	}
    }// readCalibrationFile
    
    /*
    **calcular all projectionMatrices depended on struct variables
    */
    void computeProjectionMatrices()
    {
    	int i, j, k, camIdx;
    	double(*inMat)[3];
    	double exMat[3][4];
    
    	for (camIdx = 0; camIdx<m_NumCameras; camIdx++)
    	{
    		// The intrinsic matrix
    		inMat = m_CalibParams[camIdx].m_K;
    
    		// The extrinsic matrix
    		for (i = 0; i<3; i++)
    		{
    			for (j = 0; j<3; j++)
    			{
    				exMat[i][j] = m_CalibParams[camIdx].m_RotMatrix[i][j];
    			}
    		}
    
    		for (i = 0; i<3; i++)
    		{
    			exMat[i][3] = m_CalibParams[camIdx].m_Trans[i];
    		}
    
    		// Multiply the intrinsic matrix by the extrinsic matrix to find our projection matrix
    		for (i = 0; i<3; i++)
    		{
    			for (j = 0; j<4; j++)
    			{
    				m_CalibParams[camIdx].m_ProjMatrix[i][j] = 0.0;
    
    				for (k = 0; k<3; k++)
    				{
    					m_CalibParams[camIdx].m_ProjMatrix[i][j] += inMat[i][k] * exMat[k][j];
    				}
    			}
    		}
    
    		m_CalibParams[camIdx].m_ProjMatrix[3][0] = 0.0;
    		m_CalibParams[camIdx].m_ProjMatrix[3][1] = 0.0;
    		m_CalibParams[camIdx].m_ProjMatrix[3][2] = 0.0;
    		m_CalibParams[camIdx].m_ProjMatrix[3][3] = 1.0;
    	}
    }
    
    /**
    **init projection matrix
    */
    void InitializeFromFile(char *fileName)
    {
    	readCalibrationFile(fileName);
    	computeProjectionMatrices();
    }
    /**
    **calcular z depended on d of depth image
    */
    double DepthLevelToZ(unsigned char d)
    {
    	double z;
    	double MinZ = 44.0, MaxZ = 120.0;
    
    	z = 1.0 / ((d / 255.0)*(1.0 / MinZ - 1.0 / MaxZ) + 1.0 / MaxZ);
    	return z;
    }
    
    /**
    **calcular d of depth image depended on z
    */
    unsigned char ZToDepthLever(double z)
    {
    	double MinZ = 44.0, MaxZ = 120.0;
    	unsigned char d;
    	d = (unsigned char)(255.0*(1 / (double)z - 1 / MaxZ) / (1 / MinZ - 1 / MaxZ));
    	return d;
    }
    
    /**
    **calcular x,y depended on u,v,z and projection Matrix
    **for example,projection Matrix is m_CalibParams[i].m_ProjMatrix which is index of camera
    */
    void projUVZtoXY(double projMatrix[4][4], double u, double v, double z, double *x, double *y)
    {
    	double c0, c1, c2;
    
    	// image (0,0) is bottom lefthand corner
    	v = (double)m_Height - v - 1.0;
    
    	c0 = z*projMatrix[0][2] + projMatrix[0][3];
    	c1 = z*projMatrix[1][2] + projMatrix[1][3];
    	c2 = z*projMatrix[2][2] + projMatrix[2][3];
    
    	*y = u*(c1*projMatrix[2][0] - projMatrix[1][0] * c2) +
    		v*(c2*projMatrix[0][0] - projMatrix[2][0] * c0) +
    		projMatrix[1][0] * c0 - c1*projMatrix[0][0];
    
    	*y /= v*(projMatrix[2][0] * projMatrix[0][1] - projMatrix[2][1] * projMatrix[0][0]) +
    		u*(projMatrix[1][0] * projMatrix[2][1] - projMatrix[1][1] * projMatrix[2][0]) +
    		projMatrix[0][0] * projMatrix[1][1] - projMatrix[1][0] * projMatrix[0][1];
    
    	*x = (*y)*(projMatrix[0][1] - projMatrix[2][1] * u) + c0 - c2*u;
    	*x /= projMatrix[2][0] * u - projMatrix[0][0];
    } // projUVZtoXY
    
    /**
    **calcular u,v,z1 depended on x,y,z
    **z1 is after projection and z is before projection.z1 is return value
    **/
    
    double projXYZtoUV(double projMatrix[4][4], double x, double y, double z, double *u, double *v)
    {
    	double w;
    
    	*u = projMatrix[0][0] * x +
    		projMatrix[0][1] * y +
    		projMatrix[0][2] * z +
    		projMatrix[0][3];
    
    	*v = projMatrix[1][0] * x +
    		projMatrix[1][1] * y +
    		projMatrix[1][2] * z +
    		projMatrix[1][3];
    
    	w = projMatrix[2][0] * x +
    		projMatrix[2][1] * y +
    		projMatrix[2][2] * z +
    		projMatrix[2][3];
    
    	*u /= w;
    	*v /= w;
    
    	// image (0,0) is bottom lefthand corner
    	*v = (double)m_Height - *v - 1.0;
    
    	return w;
    
    } // projXYZtoUV
    
    /**
    
    **/
    
    void pointProject_from_ref_to_otherView(double pts[8][3], int ref, int u, int v, unsigned char d)
    {
    	double x, y, z = DepthLevelToZ(d);
    
    	//printf("Testing projection of pt (%d,%d) in camera 0 with d = %d (z = %f) to other cameras\n", u, v, d, z);
    
    	projUVZtoXY(m_CalibParams[ref].m_ProjMatrix, (double)u, (double)v, z, &x, &y);
    	//printf("3D pt = (%f, %f, %f) [coordinates wrt reference camera]\n", x, y, z);
    	for (int cam = 0; cam<8; cam++)
    	{
    		double *pt = pts[cam];
    		pt[0] = 0;
    		pt[1] = 0;
    		pt[2] = 0;
    	}
    	for (int cam = 0; cam<8; cam++)
    	{
    		double *pt = pts[cam];
    
    		pt[2] = projXYZtoUV(m_CalibParams[cam].m_ProjMatrix, x, y, z, &pt[0], &pt[1]);
    		//printf("Camera %d: (%f, %f)\n", cam, pt[0], pt[1]);
    		pt[2] = ZToDepthLever(pt[2]);
    	}
    }
    
    /**
    **wraping image,ref represent reference cam,proj represent projection cam
    **the kernal code
    **/
    void wrapingImage(int ref, int proj, cv::Mat &imageColor, cv::Mat &imageDepth, cv::Mat &imageColorOut, cv::Mat &imageDepthOut)
    {
    	for (int v = 0; v < imageColor.rows; v++)
    		for (int u = 0; u < imageColor.cols; u++)
    		{
    			double d = imageDepth.at<cv::Vec3b>(v, u)[0];
    			pointProject_from_ref_to_otherView(pts, ref, u, v, d);
    			int u1 = (int)pts[proj][0];
    			int v1 = (int)pts[proj][1];
    			int k1 = (int)pts[proj][2];
    			if (u1 < 0 || u1 >= imageColor.cols - 1 || v1 < 0 || v1 >= imageColor.rows - 1)
    				continue;
    			if (k1 < imageDepthOut.at<cv::Vec3b>(v1, u1)[0])
    				continue;
    			imageColorOut.at<cv::Vec3b>(v1, u1) = imageColor.at<cv::Vec3b>(v, u);
    			imageDepthOut.at<cv::Vec3b>(v1, u1)[0] = k1;
    			imageDepthOut.at<cv::Vec3b>(v1, u1)[1] = k1;
    			imageDepthOut.at<cv::Vec3b>(v1, u1)[2] = k1;
    		}
    }
    void wrapingImageGai1(int ref, int proj, cv::Mat &imageColor, cv::Mat &imageDepth, cv::Mat &imageColorOut, cv::Mat &imageDepthOut)
    {
    	cv::Mat matBj;
    	matBj.create(imageColor.rows, imageColor.cols, CV_64FC3);
    	for (int v = 0; v < imageColor.rows; v++)
    	{
    		for (int u = 0; u < imageColor.cols; u++)
    		{
    			matBj.at<cv::Vec3d>(v, u)[0] = 0;
    			matBj.at<cv::Vec3d>(v, u)[1] = 0;
    			matBj.at<cv::Vec3d>(v, u)[2] = 0;
    		}
    	}
    	for (int v = 0; v < imageColor.rows; v++)
    		for (int u = 0; u < imageColor.cols; u++)
    		{
    			double d = imageDepth.at<cv::Vec3b>(v, u)[0];
    			pointProject_from_ref_to_otherView(pts, ref, u, v, d);
    			int u1 = (int)round_off(pts[proj][0]);
    			int v1 = (int)round_off(pts[proj][1]);
    			double xy = (u1 - pts[proj][0])*(u1 - pts[proj][0]) + (v1 - pts[proj][1])*(v1 - pts[proj][1]);
    			double z = pts[proj][2];
    			int k1 = (int)round_off(pts[proj][2]);
    			if (u1 < 0 || u1 >= imageColor.cols - 1 || v1 < 0 || v1 >= imageColor.rows - 1)
    				continue;
    			
    				
    			if (int(matBj.at<cv::Vec3d>(v1, u1)[2]) != 0 && z < matBj.at<cv::Vec3d>(v1, u1)[1])
    				continue;
    
    			if (z == matBj.at<cv::Vec3d>(v1, u1)[1] && int(matBj.at<cv::Vec3d>(v1, u1)[2]) != 0 && xy > matBj.at<cv::Vec3d>(v1, u1)[0])
    				continue;
    			imageColorOut.at<cv::Vec3b>(v1, u1) = imageColor.at<cv::Vec3b>(v, u);
    			imageDepthOut.at<cv::Vec3b>(v1, u1)[0] = k1;
    			imageDepthOut.at<cv::Vec3b>(v1, u1)[1] = k1;
    			imageDepthOut.at<cv::Vec3b>(v1, u1)[2] = k1;
    			matBj.at<cv::Vec3d>(v1, u1)[0] = xy;
    			matBj.at<cv::Vec3d>(v1, u1)[1] = z;
    			matBj.at<cv::Vec3d>(v1, u1)[2] = 1;
    		}
    }
    void wrapingImage_inverse(int ref, int proj, cv::Mat &imageColor, cv::Mat &imageDepth, cv::Mat &imageColorOut, cv::Mat &imageDepthOut)
    {
    	for (int v = 0; v < imageColor.rows; v++)
    		for (int u = 0; u < imageColor.cols; u++)
    		{
    
    			double d = imageDepthOut.at<cv::Vec3b>(v, u)[0];
    			if (d == 0)
    				continue;
    			if (imageColorOut.at<cv::Vec3b>(v, u)[0] != 0 && imageColorOut.at<cv::Vec3b>(v, u)[1] != 0 && imageColorOut.at<cv::Vec3b>(v, u)[2] != 0)
    				continue;
    			pointProject_from_ref_to_otherView(pts, proj, u, v, d);
    			int u1 = (int)pts[ref][0];
    			int v1 = (int)pts[ref][1];
    			int k1 = (int)pts[ref][2];
    			if (u1 < 0 || u1 >= imageColor.cols - 1 || v1 < 0 || v1 >= imageColor.rows - 1)
    				continue;
    			imageColorOut.at<cv::Vec3b>(v, u) = imageColor.at<cv::Vec3b>(v1, u1);
    		}
    }
    
    void dipslay(char *calibParam, char *refColor, char *refDepth, char *refColor2, char *refDepth2, char *actColor)
    {
    	//initialize projection_Matrix
    	InitializeFromFile(calibParam);
    	//display projection_Matrix
    	/*for (int i = 0; i < m_NumCameras; i++){
    		for (int j = 0; j < 3; j++){
    			for (int k = 0; k < 3; k++)
    				std::cout << m_CalibParams[i].m_K[j][k] << "\t";
    			std::cout << std::endl;
    		}
    
    		for (int j = 0; j < 3; j++){
    			for (int k = 0; k < 3; k++)
    				std::cout << m_CalibParams[i].m_RotMatrix[j][k] << "\t";
    			std::cout << std::endl;
    		}
    		for (int k = 0; k < 3; k++)
    			std::cout << m_CalibParams[i].m_Trans[k] << "\t";
    		std::cout << std::endl;
    		std::cout << std::endl;
    		std::cout << std::endl;
    		std::cout << std::endl;
    	}*/
    	//suspend and users enter a digit
    	int aa;
    	//std::cin >> aa;
    	//read color image and depth image of refrence
    	cv::Mat imageColor = cv::imread(refColor);
    	cv::Mat imageDepth = cv::imread(refDepth);
    	cv::Mat imageColor2 = cv::imread(refColor2);
    	cv::Mat imageDepth2 = cv::imread(refDepth2);
    	//read true image used to compare
    	cv::Mat imageColor_actual = cv::imread(actColor);
    	//set reference cam
    	int ref = 5;
    	int ref2 = 7;
    	//set projection cam
    	int proj = 6;
    	//test code
    	pointProject_from_ref_to_otherView(pts, ref, 700, 700, imageDepth.at<cv::Vec3b>(700, 700)[0]);
    	for (int i = 0; i < 8; i++)
    	{
    		std::cout << pts[i][0] << "\t" << pts[i][1] << std::endl;
    	}
    	//std::cin >> aa;
    	//define two variable of output
    	cv::Mat imageColorOut;
    	cv::Mat imageColorOut2;
    	cv::Mat imageColorOut3;
    	cv::Mat imageDepthOut;
    	cv::Mat imageDepthOut2;
    	
    	imageColorOut.create(imageColor.rows, imageColor.cols, imageColor.type());
    	imageColorOut2.create(imageColor.rows, imageColor.cols, imageColor.type());
    	imageColorOut3.create(imageColor.rows, imageColor.cols, imageColor.type());
    	imageDepthOut.create(imageColor.rows, imageColor.cols, imageColor.type());
    	imageDepthOut2.create(imageColor.rows, imageColor.cols, imageColor.type());
    	for (int v = 0; v < imageColor.rows; v++)
    	{
    		for (int u = 0; u < imageColor.cols; u++)
    		{
    			imageColorOut.at<cv::Vec3b>(v, u)[0] = 0;
    			imageColorOut.at<cv::Vec3b>(v, u)[1] = 0;
    			imageColorOut.at<cv::Vec3b>(v, u)[2] = 0;
    			imageColorOut2.at<cv::Vec3b>(v, u)[0] = 0;
    			imageColorOut2.at<cv::Vec3b>(v, u)[1] = 0;
    			imageColorOut2.at<cv::Vec3b>(v, u)[2] = 0;
    			imageColorOut3.at<cv::Vec3b>(v, u)[0] = 0;
    			imageColorOut3.at<cv::Vec3b>(v, u)[1] = 0;
    			imageColorOut3.at<cv::Vec3b>(v, u)[2] = 0;
    			imageDepthOut.at<cv::Vec3b>(v, u)[0] = 0;
    			imageDepthOut.at<cv::Vec3b>(v, u)[1] = 0;
    			imageDepthOut.at<cv::Vec3b>(v, u)[2] = 0;
    			imageDepthOut2.at<cv::Vec3b>(v, u)[0] = 0;
    			imageDepthOut2.at<cv::Vec3b>(v, u)[1] = 0;
    			imageDepthOut2.at<cv::Vec3b>(v, u)[2] = 0;
    
    		}
    	}
    	//save_dir
    	char *save_dir = "C:\\Users\\jiang\\Desktop\\experientdata\\experiencePictrue\\";
    	//wrapingImage(ref, proj, imageColor, imageDepth, imageColorOut, imageDepthOut);
    	//wrapingImage(ref2, proj, imageColor2, imageDepth2, imageColorOut2, imageDepthOut2);
    	wrapingImageGai1(ref, proj, imageColor, imageDepth, imageColorOut, imageDepthOut);
    	wrapingImageGai1(ref2, proj, imageColor2, imageDepth2, imageColorOut2, imageDepthOut2);
    	cv::cvtColor(imageColorOut, imageColorOut, CV_BGR2HSV);
    	cv::cvtColor(imageColorOut2, imageColorOut2, CV_BGR2HSV);
    	cv::cvtColor(imageColorOut3, imageColorOut3, CV_BGR2HSV);
    	Histogram1D h;
    	cv::MatND histo = h.getHistogramImage(imageColorOut);
    	cv::MatND histo3 = h.getHistogramImage(imageColorOut2);
    	cv::Mat histoVsHisto2;
    	cv::addWeighted(histo, 0.5, histo3, 0.5, 0.0, histoVsHisto2);
    	/*cv::imshow("histoVsHisto2gram", histoVsHisto2);
    	cv::imshow("L1", histo);
    	cv::imshow("L2", histo3);*/
    	Histogram1D1 h1;
    	cv::MatND histo1 = h1.getHistogramImage(imageColorOut);
    	cv::MatND histo21 = h1.getHistogramImage(imageColorOut2);
    	cv::Mat histoVsHisto21;
    	cv::addWeighted(histo1, 0.5, histo21, 0.5, 0.0, histoVsHisto21);
    	/*cv::imshow("histoVsHisto2gram1", histoVsHisto21);
    	cv::imshow("a1", histo1);
    	cv::imshow("a2", histo21);*/
    	Histogram1D2 h2;
    	cv::MatND histo2 = h2.getHistogramImage(imageColorOut);
    	cv::MatND histo22 = h2.getHistogramImage(imageColorOut2);
    	cv::Mat histoVsHisto22;
    	cv::addWeighted(histo2, 0.5, histo22, 0.5, 0.0, histoVsHisto22);
    	/*cv::imshow("histoVsHisto2gram2", histoVsHisto22);
    	cv::imshow("b1", histo2);
    	cv::imshow("b2", histo22);*/
    	
    	for (int v = 0; v < imageColor.rows; v++)
    	{
    		for (int u = 0; u < imageColor.cols; u++)
    		{
    			if (imageColorOut.at<cv::Vec3b>(v, u)[2] == 0 && imageColorOut.at<cv::Vec3b>(v, u)[1] == 0 && imageColorOut.at<cv::Vec3b>(v, u)[0] == 0)
    			{
    				continue;
    			}
    			if (imageColorOut2.at<cv::Vec3b>(v, u)[2] == 0 && imageColorOut2.at<cv::Vec3b>(v, u)[1] == 0 && imageColorOut2.at<cv::Vec3b>(v, u)[0] == 0)
    			{
    				continue;
    			}
    			imageColorOut3.at<cv::Vec3b>(v, u)[0] = (imageColorOut.at<cv::Vec3b>(v, u)[0] + imageColorOut2.at<cv::Vec3b>(v, u)[0]) / 2;
    			imageColorOut3.at<cv::Vec3b>(v, u)[1] = (imageColorOut.at<cv::Vec3b>(v, u)[1] + imageColorOut2.at<cv::Vec3b>(v, u)[1]) / 2;
    			imageColorOut3.at<cv::Vec3b>(v, u)[2] = (imageColorOut.at<cv::Vec3b>(v, u)[2] + imageColorOut2.at<cv::Vec3b>(v, u)[2]) / 2;
    
    		}
    	}
    	for (int v = 0; v < imageColor.rows; v++)
    	{
    		for (int u = 0; u < imageColor.cols; u++)
    		{
    			if (imageColorOut.at<cv::Vec3b>(v, u)[2] == 0 && imageColorOut.at<cv::Vec3b>(v, u)[1] == 0 && imageColorOut.at<cv::Vec3b>(v, u)[0] == 0 && imageColorOut2.at<cv::Vec3b>(v, u)[2] != 0 && imageColorOut2.at<cv::Vec3b>(v, u)[1] != 0 && imageColorOut2.at<cv::Vec3b>(v, u)[0] != 0)
    			{
    				imageColorOut3.at<cv::Vec3b>(v, u)[0] = imageColorOut2.at<cv::Vec3b>(v, u)[0];
    				imageColorOut3.at<cv::Vec3b>(v, u)[1] = imageColorOut2.at<cv::Vec3b>(v, u)[1];
    				imageColorOut3.at<cv::Vec3b>(v, u)[2] = imageColorOut2.at<cv::Vec3b>(v, u)[2];
    
    			}
    			else if (imageColorOut2.at<cv::Vec3b>(v, u)[2] == 0 && imageColorOut2.at<cv::Vec3b>(v, u)[1] == 0 && imageColorOut2.at<cv::Vec3b>(v, u)[0] == 0 && imageColorOut.at<cv::Vec3b>(v, u)[2] != 0 && imageColorOut.at<cv::Vec3b>(v, u)[1] != 0 && imageColorOut.at<cv::Vec3b>(v, u)[0] != 0)
    			{
    				imageColorOut3.at<cv::Vec3b>(v, u)[0] = imageColorOut.at<cv::Vec3b>(v, u)[0];
    				imageColorOut3.at<cv::Vec3b>(v, u)[1] = imageColorOut.at<cv::Vec3b>(v, u)[1];
    				imageColorOut3.at<cv::Vec3b>(v, u)[2] = imageColorOut.at<cv::Vec3b>(v, u)[2];
    			}
    		}
    	}
    	/*int t = 1;
    	int s = 1;
    	for (int v = 0+s; v < imageColor.rows-s; v++)
    	{
    		for (int u = 0+s; u < imageColor.cols-s; u++)
    		{
    			if (imageColorOut.at<cv::Vec3b>(v, u)[2] == 0 && imageColorOut.at<cv::Vec3b>(v, u)[1] == 0 && imageColorOut.at<cv::Vec3b>(v, u)[0] == 0 && imageColorOut2.at<cv::Vec3b>(v, u)[2] != 0 && imageColorOut2.at<cv::Vec3b>(v, u)[1] != 0 && imageColorOut2.at<cv::Vec3b>(v, u)[0] != 0)
    			{
    				int n01=0;
    				int aa = 0;
    				for (int i = v - t; i <= v + t; i++)
    					for (int j = u - t; j <= u + t; j++){
    						if (imageColorOut3.at<cv::Vec3b>(i, j)[2] != 0 && imageColorOut3.at<cv::Vec3b>(i, j)[1] != 0 && imageColorOut3.at<cv::Vec3b>(i, j)[0] != 0){
    							aa += imageColorOut3.at<cv::Vec3b>(i, j)[1];
    							n01++;
    						}
    					}
    				if (n01 == 0)
    					continue;
    				imageColorOut3.at<cv::Vec3b>(v, u)[0] = imageColorOut2.at<cv::Vec3b>(v, u)[0];
    				imageColorOut3.at<cv::Vec3b>(v, u)[1] = aa/(n01);
    				imageColorOut3.at<cv::Vec3b>(v, u)[2] = imageColorOut2.at<cv::Vec3b>(v, u)[2];
    				
    			}
    			else if (imageColorOut2.at<cv::Vec3b>(v, u)[2] == 0 && imageColorOut2.at<cv::Vec3b>(v, u)[1] == 0 && imageColorOut2.at<cv::Vec3b>(v, u)[0] == 0 && imageColorOut.at<cv::Vec3b>(v, u)[2] != 0 && imageColorOut.at<cv::Vec3b>(v, u)[1] != 0 && imageColorOut.at<cv::Vec3b>(v, u)[0] != 0)
    			{
    				int n02 = 0;
    				int aa = 0;
    				for (int i = v - t; i <= v + t; i++)
    					for (int j = u - t; j <= u + t; j++){
    						if (imageColorOut3.at<cv::Vec3b>(i, j)[2] != 0 && imageColorOut3.at<cv::Vec3b>(i, j)[1] != 0 && imageColorOut3.at<cv::Vec3b>(i, j)[0] != 0){
    								aa += imageColorOut3.at<cv::Vec3b>(i, j)[1];
    								n02++;
    						}
    					}
    				if (n02 == 0)
    					continue;
    				imageColorOut3.at<cv::Vec3b>(v, u)[0] = imageColorOut.at<cv::Vec3b>(v, u)[0];
    				imageColorOut3.at<cv::Vec3b>(v, u)[1] = aa/(n02);
    				imageColorOut3.at<cv::Vec3b>(v, u)[2] = imageColorOut.at<cv::Vec3b>(v, u)[2];
    			}		
    		}
    	}*/
    	/*int t = 1;
    	int s = 1;
    	for (int v = 0 + s; v < imageColor.rows - s; v++)
    	{
    		for (int u = 0 + s; u < imageColor.cols - s; u++)
    		{
    			if (imageColorOut.at<cv::Vec3b>(v, u)[2] == 0 && imageColorOut.at<cv::Vec3b>(v, u)[1] == 0 && imageColorOut.at<cv::Vec3b>(v, u)[0] == 0 && imageColorOut2.at<cv::Vec3b>(v, u)[2] != 0 && imageColorOut2.at<cv::Vec3b>(v, u)[1] != 0 && imageColorOut2.at<cv::Vec3b>(v, u)[0] != 0)
    			{
    				int n01 = 0;
    				int aa = 0;
    				for (int i = v - t; i <= v + t; i++)
    					for (int j = u - t; j <= u + t; j++){
    						if (imageColorOut3.at<cv::Vec3b>(i, j)[2] != 0 && imageColorOut3.at<cv::Vec3b>(i, j)[1] != 0 && imageColorOut3.at<cv::Vec3b>(i, j)[0] != 0){
    							aa += imageColorOut3.at<cv::Vec3b>(i, j)[2];
    							n01++;
    						}
    					}
    				if (n01 == 0)
    					continue;
    				imageColorOut3.at<cv::Vec3b>(v, u)[0] = imageColorOut2.at<cv::Vec3b>(v, u)[0];
    				imageColorOut3.at<cv::Vec3b>(v, u)[2] = aa / (n01);
    				imageColorOut3.at<cv::Vec3b>(v, u)[1] = imageColorOut2.at<cv::Vec3b>(v, u)[1];
    
    			}
    			if (imageColorOut2.at<cv::Vec3b>(v, u)[2] == 0 && imageColorOut2.at<cv::Vec3b>(v, u)[1] == 0 && imageColorOut2.at<cv::Vec3b>(v, u)[0] == 0 && imageColorOut.at<cv::Vec3b>(v, u)[2] != 0 && imageColorOut.at<cv::Vec3b>(v, u)[1] != 0 && imageColorOut.at<cv::Vec3b>(v, u)[0] != 0)
    			{
    				int n02 = 0;
    				int aa = 0;
    				for (int i = v - t; i <= v + t; i++)
    					for (int j = u - t; j <= u + t; j++){
    						if (imageColorOut3.at<cv::Vec3b>(i, j)[2] != 0 && imageColorOut3.at<cv::Vec3b>(i, j)[1] != 0 && imageColorOut3.at<cv::Vec3b>(i, j)[0] != 0){
    							aa += imageColorOut3.at<cv::Vec3b>(i, j)[2];
    							n02++;
    						}
    					}
    				if (n02 == 0)
    					continue;
    				imageColorOut3.at<cv::Vec3b>(v, u)[0] = imageColorOut.at<cv::Vec3b>(v, u)[0];
    				imageColorOut3.at<cv::Vec3b>(v, u)[2] = aa / (n02);
    				imageColorOut3.at<cv::Vec3b>(v, u)[1] = imageColorOut.at<cv::Vec3b>(v, u)[1];
    			}
    		}
    	}*/
    	/*int t = 1;
    	int s = 1;
    	for (int v = 0 + s; v < imageColor.rows - s; v++)
    	{
    		for (int u = 0 + s; u < imageColor.cols - s; u++)
    		{
    			if (imageColorOut.at<cv::Vec3b>(v, u)[2] == 0 && imageColorOut.at<cv::Vec3b>(v, u)[1] == 0 && imageColorOut.at<cv::Vec3b>(v, u)[0] == 0 && imageColorOut2.at<cv::Vec3b>(v, u)[2] != 0 && imageColorOut2.at<cv::Vec3b>(v, u)[1] != 0 && imageColorOut2.at<cv::Vec3b>(v, u)[0] != 0)
    			{
    				int n01 = 0;
    				int aa = 0;
    				for (int i = v - t; i <= v + t; i++)
    					for (int j = u - t; j <= u + t; j++){
    						if (imageColorOut3.at<cv::Vec3b>(i, j)[2] != 0 && imageColorOut3.at<cv::Vec3b>(i, j)[1] != 0 && imageColorOut3.at<cv::Vec3b>(i, j)[0] != 0){
    							aa += imageColorOut3.at<cv::Vec3b>(i, j)[0];
    							n01++;
    						}
    					}
    				if (n01 == 0)
    					continue;
    				imageColorOut3.at<cv::Vec3b>(v, u)[0] = aa/n01;
    				imageColorOut3.at<cv::Vec3b>(v, u)[1] = imageColorOut2.at<cv::Vec3b>(v, u)[1];
    				imageColorOut3.at<cv::Vec3b>(v, u)[2] = imageColorOut2.at<cv::Vec3b>(v, u)[2];
    
    			}
    			else if (imageColorOut2.at<cv::Vec3b>(v, u)[2] == 0 && imageColorOut2.at<cv::Vec3b>(v, u)[1] == 0 && imageColorOut2.at<cv::Vec3b>(v, u)[0] == 0 && imageColorOut.at<cv::Vec3b>(v, u)[2] != 0 && imageColorOut.at<cv::Vec3b>(v, u)[1] != 0 && imageColorOut.at<cv::Vec3b>(v, u)[0] != 0)
    			{
    				int n01 = 0;
    				int aa = 0;
    				for (int i = v - t; i <= v + t; i++)
    					for (int j = u - t; j <= u + t; j++){
    						if (imageColorOut3.at<cv::Vec3b>(i, j)[2] != 0 && imageColorOut3.at<cv::Vec3b>(i, j)[1] != 0 && imageColorOut3.at<cv::Vec3b>(i, j)[0] != 0){
    							aa += imageColorOut3.at<cv::Vec3b>(i, j)[0];
    							n01++;
    						}
    					}
    				if (n01 == 0)
    					continue;
    				imageColorOut3.at<cv::Vec3b>(v, u)[0] = aa / n01;
    				imageColorOut3.at<cv::Vec3b>(v, u)[1] = imageColorOut.at<cv::Vec3b>(v, u)[1];
    				imageColorOut3.at<cv::Vec3b>(v, u)[2] = imageColorOut.at<cv::Vec3b>(v, u)[2];
    			}
    		}
    	}*/
    	cv::cvtColor(imageColorOut, imageColorOut, CV_HSV2BGR);
    	cv::cvtColor(imageColorOut2, imageColorOut2, CV_HSV2BGR);
    	cv::cvtColor(imageColorOut3, imageColorOut3, CV_HSV2BGR);
    	cv::imshow("virtruel_Color_image01", imageColorOut);
    	cv::imwrite("C:\\Users\\jiang\\Desktop\\experientdata\\experiencePictrue\\virtruel_Color_image01.jpg", imageColorOut);
    	cv::imshow("virtruel_Color_image21", imageColorOut2);
    	cv::imwrite("C:\\Users\\jiang\\Desktop\\experientdata\\experiencePictrue\\virtruel_Color_image21.jpg", imageColorOut2);
    	cv::imshow("virtruel_Color_image11", imageColorOut3);
    	cv::imwrite("C:\\Users\\jiang\\Desktop\\experientdata\\experiencePictrue\\virtruel_Color_image11.jpg", imageColorOut3);
    	//cv::imshow("virtruel_Depth_image", imageDepthOut);
    	cv::imwrite("C:\\Users\\jiang\\Desktop\\experientdata\\experiencePictrue\\virtruel_Depth_image01.jpg", imageDepthOut);
    	//cv::imshow("virtruel_Depth_image2", imageDepthOut2);
    	cv::imwrite("C:\\Users\\jiang\\Desktop\\experientdata\\experiencePictrue\\virtruel_Depth_image21.jpg", imageDepthOut2);
    	//display actruel_image
    	cv::imshow("actruel_image", imageColor_actual);
    	cv::imwrite("C:\\Users\\jiang\\Desktop\\experientdata\\experiencePictrue\\actruel_image.jpg", imageColor_actual);
    	//mix actual and virtruel and check changes
    	cv::Mat imageColor01ActVSProj;
    	cv::addWeighted(imageColor_actual, 0.5, imageColorOut, 0.5, 0.0, imageColor01ActVSProj);
    	cv::imshow("imageColorActVSProj01", imageColor01ActVSProj);
    	cv::imwrite("C:\\Users\\jiang\\Desktop\\experientdata\\experiencePictrue\\imageColorActVSProj01.jpg", imageColor01ActVSProj);
    	cv::Mat imageColor21ActVSProj;
    	cv::addWeighted(imageColor_actual, 0.5, imageColorOut3, 0.5, 0.0, imageColor21ActVSProj);
    	cv::imshow("imageColorActVSProj21", imageColor21ActVSProj);
    	cv::imwrite("C:\\Users\\jiang\\Desktop\\experientdata\\experiencePictrue\\imageColorActVSProj21.jpg", imageColor21ActVSProj);
    	cv::Mat imageColor02ActVSProj;
    	cv::addWeighted(imageColorOut, 0.5, imageColorOut2, 0.5, 0.0, imageColor02ActVSProj);
    	cv::imshow("imageColorActVSProj02", imageColor02ActVSProj);
    	cv::imwrite("C:\\Users\\jiang\\Desktop\\experientdata\\experiencePictrue\\imageColorActVSProj02.jpg", imageColor02ActVSProj);
    	//cv::waitKey();
    
    }
    
    void main()
    {
    	//char refColor[200];
    	//char refDepth[200];
    	FILE *fp;
    	FILE *fp2;
    	FILE *fp3;
    	char filename[] = "d:\\psnr_ssim.txt"; // 此处写入文件名 
    	ofstream fout(filename);
    	
    	char str0[200][100];
    	char str2[200][100];
    	char str3[200][100];
    	int i, n = 0;
    	system("dir /a-d /b C:\\Users\\jiang\\Desktop\\experientdata\\3DVideos-distrib\\MSR3DVideo-Breakdancers\\cam5 >d:\\cam5.txt");
    	system("dir /a-d /b C:\\Users\\jiang\\Desktop\\experientdata\\3DVideos-distrib\\MSR3DVideo-Breakdancers\\cam7 >d:\\cam7.txt");
    	system("dir /a-d /b C:\\Users\\jiang\\Desktop\\experientdata\\3DVideos-distrib\\MSR3DVideo-Breakdancers\\cam6 >d:\\cam6.txt");
    	fp = fopen("d:\\cam5.txt", "r");
    	fp2 = fopen("d:\\cam7.txt", "r");
    	fp3 = fopen("d:\\cam6.txt", "r");
    	while (1){
    		if (fgets(str0[n], 50, fp) == NULL) break;
    		str0[n][strlen(str0[n]) - 1] = '\0';   // 加一个字符串结束符
    		if (fgets(str2[n], 50, fp2) == NULL) break;
    		str2[n][strlen(str2[n]) - 1] = '\0';   // 加一个字符串结束符
    		if (fgets(str3[n], 50, fp3) == NULL) break;
    		str3[n][strlen(str3[n]) - 1] = '\0';   // 加一个字符串结束符
    		n++;
    	}
    	n--;
    	fclose(fp);
    	fclose(fp2);
    	fclose(fp3);
    	double value_psnr01 = 0;
    	double value_ssim01 = 0;
    	double value_psnr21 = 0;
    	double value_ssim21 = 0;
    	double value_psnr11 = 0;
    	double value_ssim11 = 0;
    	for (i = 0; i < 100; i++){
    		char refColor[200] = "C:\\Users\\jiang\\Desktop\\experientdata\\3DVideos-distrib\\MSR3DVideo-Breakdancers\\cam5\\";
    		char refDepth[200] = "C:\\Users\\jiang\\Desktop\\experientdata\\3DVideos-distrib\\MSR3DVideo-Breakdancers\\cam5\\";
    		char refColor2[200] = "C:\\Users\\jiang\\Desktop\\experientdata\\3DVideos-distrib\\MSR3DVideo-Breakdancers\\cam7\\";
    		char refDepth2[200] = "C:\\Users\\jiang\\Desktop\\experientdata\\3DVideos-distrib\\MSR3DVideo-Breakdancers\\cam7\\";
    		char calibParam[200] = "C:\\Users\\jiang\\Desktop\\experientdata\\3DVideos-distrib\\MSR3DVideo-Breakdancers\\calibParams-breakdancers.txt";
    		char actColor[200] = "C:\\Users\\jiang\\Desktop\\experientdata\\3DVideos-distrib\\MSR3DVideo-Breakdancers\\cam6\\";
    		strcat(refColor, str0[i]);
    		strcat(refDepth, str0[i+100]);
    		strcat(refColor2, str2[i]);
    		strcat(refDepth2, str2[i + 100]);
    		strcat(actColor, str3[i]);
    		/*printf("%s\n",refColor);
    		printf("%s\n",refDepth);
    		printf("%s\n", refColor2);
    		printf("%s\n", refDepth2);
    		printf("%s\n", actColor);*/
    		dipslay(calibParam, refColor, refDepth, refColor2, refDepth2, actColor);
    		char *ref_img01 = "C:\\Users\\jiang\\Desktop\\experientdata\\experiencePictrue\\virtruel_Color_image01.jpg";
    		char *ref_img21 = "C:\\Users\\jiang\\Desktop\\experientdata\\experiencePictrue\\virtruel_Color_image21.jpg";
    		char *ref_img11 = "C:\\Users\\jiang\\Desktop\\experientdata\\experiencePictrue\\virtruel_Color_image11.jpg";
    		char *act_img = "C:\\Users\\jiang\\Desktop\\experientdata\\experiencePictrue\\actruel_image.jpg";
    		value_psnr01 += psnr(ref_img01, act_img);
    		printf("psnr01=%lf\n", value_psnr01 / (i + 1));
    		fout << value_psnr01 / (i + 1)<<"\t"; // fout用法和cout一致, 不过是写到文件里面去
    		value_ssim01 += ssim(ref_img01, act_img);
    		printf("ssim01=%lf\n", value_ssim01 / (i + 1));
    		fout << value_ssim01 / (i + 1) << "\t"; // fout用法和cout一致, 不过是写到文件里面去
    		value_psnr21 += psnr(ref_img21, act_img);
    		printf("psnr21=%lf\n", value_psnr21 / (i + 1));
    		fout << value_psnr21 / (i + 1) << "\t"; // fout用法和cout一致, 不过是写到文件里面去
    		value_ssim21 += ssim(ref_img21, act_img);
    		printf("ssim21=%lf\n", value_ssim21 / (i + 1));
    		fout << value_ssim21 / (i + 1) << "\t"; // fout用法和cout一致, 不过是写到文件里面去
    		value_psnr11 += psnr(ref_img11, act_img);
    		printf("psnr11=%lf\n", value_psnr11 / (i + 1));
    		fout << value_psnr11 / (i + 1) << "\t"; // fout用法和cout一致, 不过是写到文件里面去
    		value_ssim11 += ssim(ref_img11, act_img);
    		printf("ssim11=%lf\n", value_ssim11 / (i + 1));
    		fout << value_ssim11 / (i + 1) << "\t"; // fout用法和cout一致, 不过是写到文件里面去
    		fout << endl;
    	}
    	cin >> i;
    	/*char *refColor = "C:\\Users\\jiang\\Desktop\\experientdata\\3DVideos-distrib\\MSR3DVideo-Ballet\\cam5\\color-cam5-f000.jpg";
    	char *refDepth = "C:\\Users\\jiang\\Desktop\\experientdata\\3DVideos-distrib\\MSR3DVideo-Ballet\\cam5\\depth-cam5-f000.png";
    	char *refColor2 = "C:\\Users\\jiang\\Desktop\\experientdata\\3DVideos-distrib\\MSR3DVideo-Ballet\\cam7\\color-cam7-f000.jpg";
    	char *refDepth2 = "C:\\Users\\jiang\\Desktop\\experientdata\\3DVideos-distrib\\MSR3DVideo-Ballet\\cam7\\depth-cam7-f000.png";
    	char *calibParam = "C:\\Users\\jiang\\Desktop\\experientdata\\3DVideos-distrib\\MSR3DVideo-Ballet\\calibParams-ballet.txt";
    	char *actColor = "C:\\Users\\jiang\\Desktop\\experientdata\\3DVideos-distrib\\MSR3DVideo-Ballet\\cam6\\color-cam6-f000.jpg";
    	char *ref_img01 = "C:\\Users\\jiang\\Desktop\\experientdata\\experiencePictrue\\virtruel_Color_image01.jpg";
    	char *ref_img21 = "C:\\Users\\jiang\\Desktop\\experientdata\\experiencePictrue\\virtruel_Color_image21.jpg";
    	char *ref_img11 = "C:\\Users\\jiang\\Desktop\\experientdata\\experiencePictrue\\virtruel_Color_image11.jpg";
    	char *act_img = "C:\\Users\\jiang\\Desktop\\experientdata\\experiencePictrue\\actruel_image.jpg";
    	dipslay(calibParam, refColor, refDepth, refColor2, refDepth2, actColor);
    	double value_psnr01 = psnr(ref_img01, act_img);
    	printf("psnr01=%lf\n", value_psnr01);
    	double value_ssim01 = ssim(ref_img01, act_img);
    	printf("ssim01=%lf\n", value_ssim01);
    	double value_psnr21 = psnr(ref_img21, act_img);
    	printf("psnr21=%lf\n", value_psnr21);
    	double value_ssim21 = ssim(ref_img21, act_img);
    	printf("ssim21=%lf\n", value_ssim21);
    	double value_psnr11 = psnr(ref_img11, act_img);
    	printf("psnr11=%lf\n", value_psnr11);
    	double value_ssim11 = ssim(ref_img11, act_img);
    	printf("ssim11=%lf\n", value_ssim11);*/
    	getchar();
    	int aa;
    	std::cin >> aa;
    }
    #endif