Hello,
When I use trained caffe model to predict with the c++ code changed according to the "cpp_classification" example in Caffe project, the result of this program is "blob",and the values are very small, kline 0.0009; Are there some lines need to be corrected?
Thanks.
`#include <caffe/caffe.hpp>
#ifdef USE_OPENCV
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#endif // USE_OPENCV
#include
#include
#include
#include
#include
#include
#ifdef USE_OPENCV
using namespace caffe; // NOLINT(build/namespaces)
using std::string;
class MyClassifier {
public:
MyClassifier(const string& model_file,
const string& trained_file,
const string& mean_file);
cv::Mat Predict(const cv::Mat& rgbImg, const cv::Mat& depthImg);
private:
void SetMean(const string& mean_file);
void WrapInputLayer(std::vectorcv::Mat* input_channels);
void Preprocess(const cv::Mat& rgbImg, const cv::Mat& depthImg,std::vectorcv::Mat* input_channels);
private:
shared_ptr<Net > net_;
cv::Size input_geometry_rgb;
cv::Size input_geometry_depth;
int num_channels_rgb;
int num_channels_depth;
cv::Mat mean_rgb;
cv::Mat mean_depth;
};
MyClassifier::MyClassifier(const string& model_file,
const string& trained_file,
const string& mean_file) {
#ifdef CPU_ONLY
Caffe::set_mode(Caffe::CPU);
#else
Caffe::set_mode(Caffe::GPU);
#endif
/* Load the network. */
net_.reset(new Net<float>(model_file, TEST));
net_->CopyTrainedLayersFrom(trained_file);
CHECK_EQ(net_->num_inputs(), 2) << "Network should have exactly two input.";
CHECK_EQ(net_->num_outputs(), 1) << "Network should have exactly one output.";
Blob<float>* input_layer_rgb = net_->input_blobs()[0];
Blob<float>* input_layer_depth = net_->input_blobs()[1];
num_channels_rgb = input_layer_rgb->channels();
num_channels_depth = input_layer_depth->channels();
CHECK(num_channels_rgb == 3)
<< "RGB Input layer should have 3 channels.";
CHECK(num_channels_depth == 1)
<< "Depth Input layer should have 1 channels.";
input_geometry_rgb = cv::Size(input_layer_rgb->width(), input_layer_rgb->height());
input_geometry_depth = cv::Size(input_layer_depth->width(), input_layer_depth->height());
/* Load the binaryproto mean file. */
SetMean(mean_file);
}
/* Load the mean file in binaryproto format. */
void MyClassifier::SetMean(const string& mean_file) {
BlobProto blob_proto;
ReadProtoFromBinaryFileOrDie(mean_file.c_str(), &blob_proto);
/* Convert from BlobProto to Blob<float> */
Blob<float> mean_blob;
mean_blob.FromProto(blob_proto);
//mean prototxt include 1label channel: shoudl subtact 1
CHECK_EQ(mean_blob.channels()-1, (num_channels_rgb+num_channels_depth))
<< "Number of channels of mean file doesn't match input layer.";
/* The format of the mean file is planar cv_8uc4 float BGR + grayscale. */
std::vector<cv::Mat> channels_rgb;
std::vector<cv::Mat> channels_depth;
float* data = mean_blob.mutable_cpu_data();
for (int i = 0; i < (num_channels_rgb+num_channels_depth); ++i) {
/* Extract an individual channel. */
cv::Mat channel(mean_blob.height(), mean_blob.width(), CV_8UC1, data);
if(i<num_channels_rgb) {
channels_rgb.push_back(channel);
}else {
channels_depth.push_back(channel);
}
data += mean_blob.height() * mean_blob.width();
}
/* Merge the separate channels into a single image. */
cv::Mat meanRgb,meanDepth;
cv::merge(channels_rgb, meanRgb);
cv::merge(channels_depth, meanDepth);
/* Compute the global mean pixel value and create a mean image
* filled with this value. */
cv::Scalar channel_mean_rgb = cv::mean(meanRgb);
cv::Scalar channel_mean_depth = cv::mean(meanDepth);
mean_rgb = cv::Mat(input_geometry_rgb, meanRgb.type(), channel_mean_rgb);
mean_depth = cv::Mat(input_geometry_depth, meanDepth.type(), channel_mean_depth);
}
cv::Mat MyClassifier::Predict(const cv::Mat& rgbImg, const cv::Mat& depthImg) {
Blob* input_layer_rgb = net_->input_blobs()[0];
Blob* input_layer_depth = net_->input_blobs()[1];
input_layer_rgb->Reshape(1, num_channels_rgb,
input_geometry_rgb.height, input_geometry_rgb.width);
input_layer_depth->Reshape(1, num_channels_depth,
input_geometry_depth.height, input_geometry_depth.width);
/* Forward dimension change to all layers. */
net_->Reshape();
std::vector<cv::Mat> input_channels;
WrapInputLayer(&input_channels);
Preprocess(rgbImg,depthImg, &input_channels);
net_->Forward();
/* Copy the output layer to a std::vector */
Blob<float>* output_layer = net_->output_blobs()[0];
const float * result=output_layer->cpu_data();
cv::Mat resultMat(output_layer->height(),output_layer->width(), CV_32FC1,(float *)result);
cv::Mat resultMatUC;
return resultMat;
}
void MyClassifier::WrapInputLayer(std::vectorcv::Mat* input_channels) {
Blob* input_layer_rgb = net_->input_blobs()[0];
Blob* input_layer_depth = net_->input_blobs()[1];
int width = input_layer_rgb->width();
int height = input_layer_rgb->height();
float* input_data = input_layer_rgb->mutable_cpu_data();
for (int i = 0; i < input_layer_rgb->channels(); ++i) {
cv::Mat channel(height, width, CV_8UC3, input_data);
input_channels->push_back(channel);
input_data += width * height;
}
width = input_layer_depth->width();
height = input_layer_depth->height();
input_data = input_layer_depth->mutable_cpu_data();
for (int i = 0; i < input_layer_depth->channels(); ++i) {
cv::Mat channel(height, width, CV_8UC1, input_data);
input_channels->push_back(channel);
input_data += width * height;
}
}
void MyClassifier::Preprocess(const cv::Mat& rgbImg, const cv::Mat& depthImg,
std::vectorcv::Mat* input_channels) {
/* Convert the input image to the input image format of the network. */
cv::Mat sample_rgb,sample_depth;
if (rgbImg.channels() == 4)
cv::cvtColor(rgbImg, sample_rgb, cv::COLOR_BGRA2BGR);
else if (rgbImg.channels() == 1 )
cv::cvtColor(rgbImg, sample_rgb, cv::COLOR_GRAY2BGR);
else
sample_rgb = rgbImg;
//depth image
if (depthImg.channels() == 4)
cv::cvtColor(depthImg, sample_depth, cv::COLOR_BGRA2GRAY);
else if (depthImg.channels() == 3 )
cv::cvtColor(depthImg, sample_depth, cv::COLOR_BGR2GRAY);
else if (depthImg.channels() == 1){
if(depthImg.type()!=CV_8UC1){
depthImg.convertTo(sample_depth,CV_8UC1,1.0/255);
}else {
sample_depth = depthImg;
}
}
cv::Mat sample_resized_rgb;
if (sample_rgb.size() != input_geometry_rgb)
cv::resize(sample_rgb, sample_resized_rgb, input_geometry_rgb);
else
sample_resized_rgb = sample_rgb;
cv::Mat sample_resized_depth;
if (sample_depth.size() != input_geometry_depth)
cv::resize(sample_depth, sample_resized_depth, input_geometry_depth);
else
sample_resized_depth = sample_depth;
cv::Mat sample_normalized_rgb,sample_normalized_depth;
cv::subtract(sample_resized_rgb, mean_rgb, sample_normalized_rgb);
cv::subtract(sample_resized_depth, mean_depth, sample_normalized_depth);
/* This operation will write the separate BGR planes directly to the
* input layer of the network because it is wrapped by the cv::Mat
* objects in input_channels. */
cv::split(sample_normalized_rgb, *input_channels);
//depth image is 1 channel
input_channels->push_back(sample_resized_depth);
}
int main(int argc, char** argv) {
// if (argc != 6) {
// std::cerr << "Usage: " << argv[0]
// << " deploy.prototxt network.caffemodel"
// << " mean.binaryproto img.jpg depth.png" << std::endl;
// return 1;
// }
::google::InitGoogleLogging(argv[0]);
// string model_file = argv[1];//deploy.prototxt
// string trained_file = argv[2];//caffemodel
// string mean_file = argv[3];//mean.prototxt
// string file = argv[4];
// string fileDepth = argv[5];//mean.prototxt
string model_file="segmentation/nyu40-sf1/deploy.prototxt";
string trained_file="caffemodels_iter_75000.caffemodel";
string mean_file="db/mean/mean.prototxt";
string file="NYU0015/fullres/NYU0015.jpg";
string fileDepth="NYU0015/fullres/NYU0015.png";
MyClassifier classifier(model_file, trained_file, mean_file);
std::cout << "---------- Prediction for "
<< file << " ----------" << std::endl;
cv::Mat img = cv::imread(file, -1);
cv::Mat imgDepth = cv::imread(fileDepth, -1);
CHECK(!img.empty()) << "Unable to decode image " << file;
cv::Mat result=classifier.Predict(img,imgDepth);
// std::cout<<result.type() <<std::endl;
// std::cout<<result.cols <<std::endl;
// std::cout<<result.rows <<std::endl;
}
#else
int main(int argc, char** argv) {
LOG(FATAL) << "This example requires OpenCV; compile with USE_OPENCV.";
}
#endif // USE_OPENCV
`