Ubuntu 14.04上使用CMake编译MXNet源码操作步骤(C++)

╰半夏微凉° 2022-04-03 09:22 237阅读 0赞

MXNet源码版本号为1.3.0，其它依赖库的版本号可参考：[https://blog.csdn.net/fengbingchun/article/details/84997490][https_blog.csdn.net_fengbingchun_article_details_84997490]

build.sh脚本内容为：

#! /bin/bash
    
    real_path=$(realpath $0)
    dir_name=`dirname "${real_path}"`
    echo "real_path: ${real_path}, dir_name: ${dir_name}"
    
    data_dir="data"
    if [ -d ${dir_name}/${data_dir} ]; then
    	rm -rf ${dir_name}/${data_dir}
    fi
    
    ln -s ${dir_name}/./../../${data_dir} ${dir_name}
    
    new_dir_name=${dir_name}/build
    mkdir -p ${new_dir_name}
    cd ${new_dir_name}
    echo "pos: ${new_dir_name}"
    if [ "$(ls -A ${new_dir_name})" ]; then
    	echo "directory is not empty: ${new_dir_name}"
    	#rm -r *
    else
    	echo "directory is empty: ${new_dir_name}"
    fi
    
    cd -
    # build openblas
    echo "========== start build openblas =========="
    openblas_path=${dir_name}/../../src/openblas
    if [ -f ${openblas_path}/build/lib/libopenblas.so ]; then
    	echo "openblas dynamic library already exists without recompiling"
    else
    	mkdir -p ${openblas_path}/build
    	cd ${openblas_path}/build
    	cmake  -DBUILD_SHARED_LIBS=ON ..
    	make
    fi
    
    ln -s ${openblas_path}/build/lib/libopenblas* ${new_dir_name}
    echo "========== finish build openblas =========="
    
    cd -
    # build dmlc-core
    echo "========== start build dmlc-core =========="
    dmlc_path=${dir_name}/../../src/dmlc-core
    if [ -f ${dmlc_path}/build/libdmlc.a ]; then
    	echo "dmlc static library already exists without recompiling"
    else
    	mkdir -p ${dmlc_path}/build
    	cd ${dmlc_path}/build
    	cmake ..
    	make
    fi
    
    ln -s ${dmlc_path}/build/libdmlc.a ${new_dir_name}
    echo "========== finish build dmlc-core =========="
    
    rc=$?
    if [[ ${rc} != 0 ]]; then
    	echo "########## Error: some of thess commands have errors above, please check"
    	exit ${rc}
    fi
    
    cd -
    cd ${new_dir_name}
    cmake ..
    make
    
    cd -

CMakeLists.txt文件内容为：

PROJECT(MXNet_Test)
    CMAKE_MINIMUM_REQUIRED(VERSION 3.0)
    
    # support C++11
    SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c11")
    SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
    # support C++14, when gcc version > 5.1, use -std=c++14 instead of c++1y
    SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++1y")
    
    IF(NOT CMAKE_BUILD_TYPE)
    	SET(CMAKE_BUILD_TYPE "Release")
    	SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -O2")
    	SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -O2")
    ELSE()
    	SET(CMAKE_BUILD_TYPE "Debug")
    	SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -Wall -O2")
    	SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -Wall -O2")
    ENDIF()
    MESSAGE(STATUS "cmake build type: ${CMAKE_BUILD_TYPE}")
    
    MESSAGE(STATUS "cmake current source dir: ${CMAKE_CURRENT_SOURCE_DIR}")
    SET(PATH_TEST_FILES ${CMAKE_CURRENT_SOURCE_DIR}/./../../demo/MXNet_Test)
    SET(PATH_SRC_DMLC_FILES ${CMAKE_CURRENT_SOURCE_DIR}/./../../src/dmlc-core)
    SET(PATH_SRC_MSHADOW_FILES ${CMAKE_CURRENT_SOURCE_DIR}/./../../src/mshadow)
    SET(PATH_SRC_OPENBLAS_FILES ${CMAKE_CURRENT_SOURCE_DIR}/./../../src/openblas)
    SET(PATH_SRC_MXNET_FILES ${CMAKE_CURRENT_SOURCE_DIR}/./../../src/mxnet)
    SET(PATH_SRC_TVM_FILES ${CMAKE_CURRENT_SOURCE_DIR}/./../../src/tvm)
    SET(PATH_SRC_DLPACK_FILES ${CMAKE_CURRENT_SOURCE_DIR}/./../../src/dlpack)
    MESSAGE(STATUS "path test files: ${PATH_TEST_FILES}")
    
    # don't use opencv in mxnet
    ADD_DEFINITIONS(-DMXNET_USE_OPENCV=0)
    ADD_DEFINITIONS(-DMSHADOW_USE_F16C=0)
    
    SET(PATH_OPENCV /opt/opencv3.4.2)
    IF(EXISTS ${PATH_OPENCV})
    	MESSAGE(STATUS "Found OpenCV: ${PATH_OPENCV}")
    ELSE()
    	MESSAGE(FATAL_ERROR "Can not find OpenCV in ${PATH_OPENCV}")
    ENDIF()
    
    # head file search path
    INCLUDE_DIRECTORIES(
    	${PATH_SRC_OPENBLAS_FILES}
    	${PATH_SRC_OPENBLAS_FILES}/build # include openblas config.h
    	${PATH_SRC_DLPACK_FILES}/include
    	${PATH_SRC_MSHADOW_FILES}
    	${PATH_SRC_DMLC_FILES}/include
    	${PATH_SRC_TVM_FILES}/include
    	${PATH_SRC_TVM_FILES}/nnvm/include
    	${PATH_SRC_MXNET_FILES}/src
    	${PATH_SRC_MXNET_FILES}/include
    	${PATH_SRC_MXNET_FILES}/cpp-package/include
    	${PATH_OPENCV}/include
    	${PATH_TEST_FILES}
    )
    
    # build mxnet dynamic library
    SET(MXNET_SRC_LIST )
    
    # tvm
    FILE(GLOB_RECURSE SRC_TVM_NNVM_C_API ${PATH_SRC_TVM_FILES}/nnvm/src/c_api/*.cc)
    FILE(GLOB_RECURSE SRC_TVM_NNVM_CORE ${PATH_SRC_TVM_FILES}/nnvm/src/core/*.cc)
    FILE(GLOB_RECURSE SRC_TVM_NNVM_PASS ${PATH_SRC_TVM_FILES}/nnvm/src/pass/*.cc)
    
    FILE(GLOB_RECURSE SRC_MXNET ${PATH_SRC_MXNET_FILES}/src/*.cc)
    
    LIST(APPEND MXNET_SRC_LIST
    	${SRC_TVM_NNVM_C_API}
    	${SRC_TVM_NNVM_CORE}
    	${SRC_TVM_NNVM_PASS}
    	${SRC_MXNET}
    )
    #MESSAGE(STATUS "mxnet src: ${MXNET_SRC_LIST}")
    
    ADD_LIBRARY(mxnet SHARED ${MXNET_SRC_LIST})
    
    # find opencv library
    FIND_LIBRARY(opencv_core NAMES opencv_core PATHS ${PATH_OPENCV}/lib NO_DEFAULT_PATH)
    FIND_LIBRARY(opencv_imgproc NAMES opencv_imgproc PATHS ${PATH_OPENCV}/lib NO_DEFAULT_PATH)
    FIND_LIBRARY(opencv_highgui NAMES opencv_highgui PATHS ${PATH_OPENCV}/lib NO_DEFAULT_PATH)
    FIND_LIBRARY(opencv_imgcodecs NAMES opencv_imgcodecs PATHS ${PATH_OPENCV}/lib NO_DEFAULT_PATH)
    FIND_LIBRARY(opencv_video NAMES opencv_video PATHS ${PATH_OPENCV}/lib NO_DEFAULT_PATH)
    FIND_LIBRARY(opencv_videoio NAMES opencv_videoio PATHS ${PATH_OPENCV}/lib NO_DEFAULT_PATH)
    FIND_LIBRARY(opencv_objdetect NAMES opencv_objdetect PATHS ${PATH_OPENCV}/lib NO_DEFAULT_PATH)
    FIND_LIBRARY(opencv_ml NAMES opencv_ml PATHS ${PATH_OPENCV}/lib NO_DEFAULT_PATH)
    MESSAGE(STATUS "opencv libraries: ${opencv_core} ${opencv_imgproc} ${opencv_highgui} ${opencv_imgcodecs} ${opencv_video}" ${opencv_videoio} ${opencv_objdetect} ${opencv_ml})
    
    # find dep library
    SET(DEP_LIB_DIR ${CMAKE_CURRENT_SOURCE_DIR}/build CACHE PATH "dep library path")
    MESSAGE(STATUS "dep library dir: ${DEP_LIB_DIR}")
    LINK_DIRECTORIES(${DEP_LIB_DIR})
    
    # recursive query match files :*.cpp
    FILE(GLOB_RECURSE TEST_CPP_LIST ${PATH_TEST_FILES}/*.cpp)
    FILE(GLOB_RECURSE TEST_CC_LIST ${PATH_TEST_FILES}/*.cc)
    MESSAGE(STATUS "test cpp list: ${TEST_CPP_LIST} ${TEST_C_LIST}")
    
    # build executable program
    ADD_EXECUTABLE(MXNet_Test ${TEST_CPP_LIST} ${TEST_CC_LIST})
    # add dependent library: static and dynamic
    TARGET_LINK_LIBRARIES(MXNet_Test
    	mxnet
    	${DEP_LIB_DIR}/libdmlc.a
    	${DEP_LIB_DIR}/libopenblas.so
    	pthread
    	rt # undefined reference to shm_open
    	${opencv_core}
    	${opencv_imgproc}
    	${opencv_highgui}
    	${opencv_imgcodecs}
    	${opencv_video}
    	${opencv_videoio}
    	${opencv_objdetect}
    	${opencv_ml}
    )

注：对源码有两处修改：

1. OpenBLAS注释掉common.h中的第681行；

2. dmlc-core再用CMake编译时，关闭OpenMP的支持，即将CMakeLists.txt中的第18行由ON调整为OFF.

以下是MNIST train的测试代码：

#include "funset.hpp"
    #include <chrono>
    #include <string>
    #include <fstream>
    #include <vector>
    #include "mxnet-cpp/MxNetCpp.h"
    
    namespace {
    
    bool isFileExists(const std::string &filename)
    {
    	std::ifstream fhandle(filename.c_str());
    	return fhandle.good();
    }
    
    bool check_datafiles(const std::vector<std::string> &data_files)
    {
    	for (size_t index = 0; index < data_files.size(); index++) {
    		if (!(isFileExists(data_files[index]))) {
    			LG << "Error: File does not exist: " << data_files[index];
    			return false;
    		}
    	}
    	return true;
    }
    
    bool setDataIter(mxnet::cpp::MXDataIter *iter, std::string useType, const std::vector<std::string> &data_files, int batch_size)
    {
    	if (!check_datafiles(data_files))
    		return false;
    
    	iter->SetParam("batch_size", batch_size);
    	iter->SetParam("shuffle", 1);
    	iter->SetParam("flat", 1);
    
    	if (useType == "Train") {
    		iter->SetParam("image", data_files[0]);
    		iter->SetParam("label", data_files[1]);
    	} else if (useType == "Label") {
    		iter->SetParam("image", data_files[2]);
    		iter->SetParam("label", data_files[3]);
    	}
    
    	iter->CreateDataIter();
    	return true;
    }
    
    } // namespace
    
    // mnist 
    /* reference: 
    	https://mxnet.incubator.apache.org/tutorials/c%2B%2B/basics.html
    	mxnet_source/cpp-package/example/mlp_cpu.cpp
    */
    namespace {
    
    mxnet::cpp::Symbol mlp(const std::vector<int> &layers)
    {
    	auto x = mxnet::cpp::Symbol::Variable("X");
    	auto label = mxnet::cpp::Symbol::Variable("label");
    
    	std::vector<mxnet::cpp::Symbol> weights(layers.size());
    	std::vector<mxnet::cpp::Symbol> biases(layers.size());
    	std::vector<mxnet::cpp::Symbol> outputs(layers.size());
    
    	for (size_t i = 0; i < layers.size(); ++i) {
    		weights[i] = mxnet::cpp::Symbol::Variable("w" + std::to_string(i));
    		biases[i] = mxnet::cpp::Symbol::Variable("b" + std::to_string(i));
    		mxnet::cpp::Symbol fc = mxnet::cpp::FullyConnected(i == 0 ? x : outputs[i - 1], weights[i], biases[i], layers[i]);
    		outputs[i] = i == layers.size() - 1 ? fc : mxnet::cpp::Activation(fc, mxnet::cpp::ActivationActType::kRelu);
    	}
    
    	return mxnet::cpp::SoftmaxOutput(outputs.back(), label);
    }
    
    } // namespace
    
    int test_mnist_train()
    {
    	const int image_size = 28;
    	const std::vector<int> layers{ 128, 64, 10 };
    	const int batch_size = 100;
    	const int max_epoch = 10;
    	const float learning_rate = 0.1;
    	const float weight_decay = 1e-2;
    
    #ifdef _MSC_VER
    	std::vector<std::string> data_files = { "E:/GitCode/MXNet_Test/data/mnist/train-images.idx3-ubyte",
    						"E:/GitCode/MXNet_Test/data/mnist/train-labels.idx1-ubyte",
    						"E:/GitCode/MXNet_Test/data/mnist/t10k-images.idx3-ubyte",
    						"E:/GitCode/MXNet_Test/data/mnist/t10k-labels.idx1-ubyte"};
    #else
    	std::vector<std::string> data_files = { "data/mnist/train-images.idx3-ubyte",
    						"data/mnist/train-labels.idx1-ubyte",
    						"data/mnist/t10k-images.idx3-ubyte",
    						"data/mnist/t10k-labels.idx1-ubyte"};
    
    #endif
    
    	auto train_iter = mxnet::cpp::MXDataIter("MNISTIter");
    	setDataIter(&train_iter, "Train", data_files, batch_size);
    
    	auto val_iter = mxnet::cpp::MXDataIter("MNISTIter");
    	setDataIter(&val_iter, "Label", data_files, batch_size);
    
    	auto net = mlp(layers);
    
    	mxnet::cpp::Context ctx = mxnet::cpp::Context::cpu();  // Use CPU for training
    
    	std::map<std::string, mxnet::cpp::NDArray> args;
    	args["X"] = mxnet::cpp::NDArray(mxnet::cpp::Shape(batch_size, image_size*image_size), ctx);
    	args["label"] = mxnet::cpp::NDArray(mxnet::cpp::Shape(batch_size), ctx);
    	// Let MXNet infer shapes other parameters such as weights
    	net.InferArgsMap(ctx, &args, args);
    
    	// Initialize all parameters with uniform distribution U(-0.01, 0.01)
    	auto initializer = mxnet::cpp::Uniform(0.01);
    	for (auto& arg : args) {
    		// arg.first is parameter name, and arg.second is the value
    		initializer(arg.first, &arg.second);
    	}
    
    	// Create sgd optimizer
    	mxnet::cpp::Optimizer* opt = mxnet::cpp::OptimizerRegistry::Find("sgd");
    	opt->SetParam("rescale_grad", 1.0 / batch_size)->SetParam("lr", learning_rate)->SetParam("wd", weight_decay);
    
    	// Create executor by binding parameters to the model
    	auto *exec = net.SimpleBind(ctx, args);
    	auto arg_names = net.ListArguments();
    
    	// Start training
    	for (int iter = 0; iter < max_epoch; ++iter) {
    		int samples = 0;
    		train_iter.Reset();
    
    		auto tic = std::chrono::system_clock::now();
    		while (train_iter.Next()) {
    			samples += batch_size;
    			auto data_batch = train_iter.GetDataBatch();
    			// Set data and label
    			data_batch.data.CopyTo(&args["X"]);
    			data_batch.label.CopyTo(&args["label"]);
    
    			// Compute gradients
    			exec->Forward(true);
    			exec->Backward();
    			// Update parameters
    			for (size_t i = 0; i < arg_names.size(); ++i) {
    				if (arg_names[i] == "X" || arg_names[i] == "label") continue;
    				opt->Update(i, exec->arg_arrays[i], exec->grad_arrays[i]);
    			}
    		}
    		auto toc = std::chrono::system_clock::now();
    
    		mxnet::cpp::Accuracy acc;
    		val_iter.Reset();
    		while (val_iter.Next()) {
    			auto data_batch = val_iter.GetDataBatch();
    			data_batch.data.CopyTo(&args["X"]);
    			data_batch.label.CopyTo(&args["label"]);
    			// Forward pass is enough as no gradient is needed when evaluating
    			exec->Forward(false);
    			acc.Update(data_batch.label, exec->outputs[0]);
    		}
    		float duration = std::chrono::duration_cast<std::chrono::milliseconds>
    			(toc - tic).count() / 1000.0;
    		LG << "Epoch: " << iter << " " << samples / duration << " samples/sec Accuracy: " << acc.Get();
    	}
    
    #ifdef _MSC_VER
    	std::string json_file{ "E:/GitCode/MXNet_Test/data/mnist.json" };
    	std::string param_file{"E:/GitCode/MXNet_Test/data/mnist.params"};
    #else
    	std::string json_file{ "data/mnist.json" };
    	std::string param_file{"data/mnist.params"};
    #endif
    	net.Save(json_file);
    	mxnet::cpp::NDArray::Save(param_file, exec->arg_arrays);
    
    	delete exec;
    	MXNotifyShutdown();
    
    	return 0;
    }

执行结果如下：

![20181221153826454.png][]

**GitHub**：[https://github.com/fengbingchun/MXNet\_Test][https_github.com_fengbingchun_MXNet_Test]

[https_blog.csdn.net_fengbingchun_article_details_84997490]: https://blog.csdn.net/fengbingchun/article/details/84997490
[20181221153826454.png]: /images/20220403/4a261ef3e4fb4f0d91b8b1a80f95bcf0.png
[https_github.com_fengbingchun_MXNet_Test]: https://github.com/fengbingchun/MXNet_Test