python3.6 mo_tf.py  --input_model=/home/gsj/object-detection/test_models/ssd_inception_v2_coco_2018_01_28/frozen_inference_graph.pb --tensorflow_use_custom_operations_config /opt/intel/openvino_2019.2.201/deployment_tools/model_optimizer/extensions/front/tf/ssd_v2_support.json --tensorflow_object_detection_api_pipeline_config /home/gsj/object-detection/test_models/ssd_inception_v2_coco_2018_01_28/pipeline.config --reverse_input_channels --batch 32

main.cpp   self_object_detection_engine_head.h 
CMakeLists.txt  README.md  self_object_detection.h

  1 // Copyright (C) 2018-2019 Intel Corporation
  2 // SPDX-License-Identifier: Apache-2.0
  3 //
  4 /*******************************************************************
  5 * Copyright:2019-2030, CC
  6 * File Name: Main.cpp
  7 * Description: main function includes ObjectDetection,
  8 *                          Initialize_Check, readInputimagesNames,
  9 *                          load_inference_engine, readIRfiles,
 10 *                          prepare_input_blobs, load_and_create_request,*                          
 11 *                          prepare_input , process
 12 *Author: Gao Shengjun
 13 *Date: 2019-07-19
 14 *******************************************************************/
 15 #include <gflags/gflags.h>
 16 #include <iostream>
 17 #include <string>
 18 #include <memory>
 19 #include <vector>
 20 #include <algorithm>
 21 #include <map>
 22 
 23 #include <format_reader_ptr.h>
 24 #include <inference_engine.hpp>
 25 #include <ext_list.hpp>
 26 
 27 #include <samples/common.hpp>
 28 #include <samples/slog.hpp>
 29 #include <samples/args_helper.hpp>
 30 
 31 #include <vpu/vpu_tools_common.hpp>
 32 #include <vpu/vpu_plugin_config.hpp>
 33 
 34 #include "self_object_detection_engine_head.h"
 35 
 36 
 37 bool ParseAndCheckCommandLine(int argc, char *argv[]) {
 38     gflags::ParseCommandLineNonHelpFlags(&argc, &argv, true);
 39     if (FLAGS_h) {
 40         showUsage();
 41         showAvailableDevices();
 42         return false;
 43     }
 44 
 45     slog::info << "Parsing input parameters" << slog::endl;
 46 
 47     if (FLAGS_i.empty()) {
 48         throw std::logic_error("Parameter -i is not set");
 49     }
 50 
 51     if (FLAGS_m.empty()) {
 52         throw std::logic_error("Parameter -m is not set");
 53     }
 54 
 55     return true;
 56 }
 57 
 58 
 59 
 60 static std::map<std::string, std::string> configure(const std::string& confFileName) {
 61     auto config = parseConfig(confFileName);
 62 
 63     return config;
 64 }
 65 
 66 
 67 
 68 void Initialize_Check_params(int argc,char* argv[]){
 69       slog::info << "InferenceEngine: " << GetInferenceEngineVersion()<<"\n";
 70       if(!ParseAndCheckCommandLine(argc,argv)){
 71           slog::info<<"Check successfully"<<"\n";
 72           return ;
 73       }
 74       return;
 75 }
 76 
 77 
 78 
 79 std::vector<std::string> readInputimagesNames(){
 80         std::vector<std::string>images;
 81         parseInputFilesArguments(images);
 82         if(images.empty()) throw std::logic_error("No suitable images were found");
 83         return images;
 84 }
 85 
 86 
 87 void load_inference_engine(Core &ie){
 88        slog::info <<"Loading Inference Engine"<<slog::endl;
 89        slog::info <<"Device info:" <<slog::endl;
 90        std::cout<<ie.GetVersions(FLAGS_d);
 91        if(FLAGS_p_msg){
 92        ie.SetLogCallback(error_listener);
 93        }
 94        if (FLAGS_d.find("CPU")!=std::string::npos){
 95           ie.AddExtension(std::make_shared<Extensions::Cpu::CpuExtensions>(),"CPU");
 96        }
 97        if(!FLAGS_l.empty()){
 98           IExtensionPtr  extension_ptr = make_so_pointer<IExtension>(FLAGS_l);
 99           ie.AddExtension(extension_ptr,"CPU");
100           slog::info <<"CPU Extension loaded: "<<FLAGS_l<<slog::endl;
101        }
102        if(!FLAGS_c.empty()){
103           ie.SetConfig({{PluginConfigParams::KEY_CONFIG_FILE,FLAGS_c}},"GPU");
104           slog::info<<"GPU Extension loaded: "<<FLAGS_c<<slog::endl;
105        }
106 
107 }
108 
109 
110 struct NetworkReader_networkinfo readIRfiles(){
111       struct NetworkReader_networkinfo nettools;
112       std::string binFileName = fileNameNoExt(FLAGS_m) +".bin";
113       slog::info << "Loading network files:"
114             "\n\t" << FLAGS_m <<
115             "\n\t" << binFileName <<
116             slog::endl;
117       CNNNetReader  networkReader;
118       networkReader.ReadNetwork(FLAGS_m);
119       networkReader.ReadWeights(binFileName);
120       CNNNetwork network = networkReader.getNetwork();
121       nettools.networkReader = networkReader;
122       nettools.network = network;
123       return nettools;
124 }
125 
126 
127 
128 struct inputInfo_imageName prepare_input_blobs( CNNNetwork &network,CNNNetReader  &networkReader, InputsDataMap &inputsInfo){
129        slog::info << "Preparing input blobs" << slog::endl;
130        struct inputInfo_imageName res;
131        if (inputsInfo.size() != 1 && inputsInfo.size() != 2) throw std::logic_error("Sample supports topologies only with 1 or 2 inputs");
132        std::string imageInputName,imInfoInputName;
133        InputInfo::Ptr inputInfo  = nullptr;
134        SizeVector inputImageDims;
135        for(auto &item:inputsInfo){
136          if(item.second->getInputData()->getTensorDesc().getDims().size()==4){
137             imageInputName = item.first;
138             inputInfo = item.second;
139             slog::info<<"Batch size is "<<std::to_string(networkReader.getNetwork().getBatchSize())<<slog::endl;
140             Precision inputPrecision =Precision::U8;
141             item.second->setPrecision(inputPrecision);
142             }else if(item.second->getInputData()->getTensorDesc().getDims().size()==2){
143              imInfoInputName = item.first;
144              Precision inputPrecision = Precision::FP32;
145              item.second->setPrecision(inputPrecision);
146              if((item.second->getTensorDesc().getDims()[1]!=3 && item.second->getTensorDesc().getDims()[1]!=6)){
147                   throw std::logic_error("Invalid input info. Should be 3 or 6 values length");
148                }
149             }
150        }
151        if(inputInfo == nullptr){
152            inputInfo = inputsInfo.begin()->second;
153    }
154    res.inputInfo = inputInfo;
155    res.InputName = imageInputName;
156    res.imInfoInputName=imInfoInputName;
157    return res;
158 }
159 
160 
161 
162 struct outputInfoStruct prepare_output_blobs(CNNNetwork &network){
163    struct outputInfoStruct res_output;
164    slog::info << "Preparing output blobs" << slog::endl;
165    OutputsDataMap outputsInfo(network.getOutputsInfo());
166    std::string outputName;
167    DataPtr outputInfo;
168    for(const auto& out : outputsInfo){
169        if(out.second->getCreatorLayer().lock()->type=="DetectionOutput"){
170              outputName = out.first;
171              outputInfo = out.second;
172        }
173    }
174    if(outputInfo == nullptr){
175        throw std::logic_error("Can't find a DetectionOutput layers in the topology");
176    }
177    const SizeVector outputDims = outputInfo->getTensorDesc().getDims();
178    res_output.maxProposalCount=outputDims[2];
179    res_output.objectSize=outputDims[3];
180    res_output.outputName=outputName;
181    if (res_output.objectSize != 7) {
182             throw std::logic_error("Output item should have 7 as a last dimension");
183         }
184 
185    if (outputDims.size() != 4) {
186             throw std::logic_error("Incorrect output dimensions for SSD model");
187         }
188 
189    outputInfo->setPrecision(Precision::FP32);
190    return res_output;
191 }
192 
193 
194 
195 struct exenet_requests load_and_create_request(CNNNetwork& network,Core &ie){
196    struct exenet_requests res;
197    slog::info << "Loading model to the device" << slog::endl;
198    res.executable_network = ie.LoadNetwork(network, FLAGS_d, configure(FLAGS_config));
199    slog::info << "Create infer request" << slog::endl;
200    res.infer_request = res.executable_network.CreateInferRequest();
201    return res;
202 }
203 
204 
205 
206 
207 struct res_outputStruct prepare_input(std::vector<std::string>& images,CNNNetwork& network, struct inputInfo_imageName& res,InferRequest& infer_request,InputsDataMap& inputsInfo){
208      struct res_outputStruct output_res2;
209      std::vector<std::shared_ptr<unsigned char>> imageData,originalImagesData;
210      std::vector<size_t>imageWidths,imageHeights;
211      for(auto &i : images){
212          FormatReader::ReaderPtr reader(i.c_str());
213          if(reader.get()==nullptr){
214             slog::warn << "Image" + i + "cannot be read!" <<slog::endl;
215             continue;
216          }
217      std::shared_ptr<unsigned char>originalData(reader->getData());
218      std::shared_ptr<unsigned char>data(reader->getData(res.inputInfo->getTensorDesc().getDims()[3],res.inputInfo->getTensorDesc().getDims()[2]));
219      if(data.get()!=nullptr){
220           originalImagesData.push_back(originalData);
221           imageData.push_back(data);
222           imageWidths.push_back(reader->width());
223           imageHeights.push_back(reader->height());
224 
225         }
226 
227      }
228      if(imageData.empty())throw std::logic_error("Valid input images were not found!");
229      size_t batchSize = network.getBatchSize();
230      slog::info << "Batch Size is "<<std::to_string(batchSize)<<slog::endl;
231      if(batchSize!=imageData.size()){
232        slog::warn << "Number of images " + std::to_string(imageData.size()) + \
233          "dosen't match batch size "+std::to_string(batchSize)<<slog::endl;
234      batchSize = std::min(batchSize,imageData.size());
235      slog::warn <<"Number of images to be processed is "<<std::to_string(batchSize)<<slog::endl;
236     }
237     Blob::Ptr imageInput = infer_request.GetBlob(res.InputName);
238     size_t num_channels = imageInput->getTensorDesc().getDims()[1];
239     size_t image_size=imageInput->getTensorDesc().getDims()[3]*imageInput->getTensorDesc().getDims()[2];
240     unsigned char* data = static_cast<unsigned char*>(imageInput->buffer());
241     for(size_t image_id = 0 ; image_id < std::min(imageData.size(),batchSize);++image_id){
242        for(size_t pid = 0; pid < image_size; pid++){
243           for(size_t ch = 0 ; ch  < num_channels;++ch){
244              data[image_id*image_size*num_channels + ch*image_size+pid] = imageData.at(image_id).get()[pid*num_channels + ch];
245           }
246        }
247     }
248     if(res.imInfoInputName!=""){
249         Blob::Ptr input2 = infer_request.GetBlob(res.imInfoInputName);
250         auto imInfoDim = inputsInfo.find(res.imInfoInputName)->second->getTensorDesc().getDims()[1];
251         float *p = input2->buffer().as<PrecisionTrait<Precision::FP32>::value_type*>();
252         for(size_t image_id=0;image_id<std::min(imageData.size(),batchSize);++image_id){
253             p[image_id*imInfoDim+0] = static_cast<float>(inputsInfo[res.InputName]->getTensorDesc().getDims()[2]);
254             p[image_id*imInfoDim+1] = static_cast<float>(inputsInfo[res.InputName]->getTensorDesc().getDims()[3]);
255             for(size_t k = 2; k < imInfoDim; ++k){
256                p[image_id*imInfoDim+k]=1.0f;
257             }
258         }
259     }
260      output_res2.originalImagesData=originalImagesData;
261      output_res2.imageWidths=imageWidths;
262      output_res2.imageHeights=imageHeights;
263      output_res2.batchSize=batchSize;
264      slog::info<<"Start inference"<<slog::endl;
265      infer_request.Infer();
266      return output_res2;
267 }
268 
269 
270 
271 void process(InferRequest& infer_request,std::string& outputName,size_t& batchSize,const int& maxProposalCount,const int& objectSize,std::vector<size_t>& imageWidths,std::vector<size_t>& imageHeights,std::vector<std::shared_ptr<unsigned char>>& originalImagesData){
272     slog::info << "Processing output blobs" <<slog::endl;
273     const Blob::Ptr output_blob = infer_request.GetBlob(outputName);
274     const float* detection = static_cast<PrecisionTrait<Precision::FP32>::value_type*>(output_blob->buffer());
275     std::vector<std::vector<int>>boxes(batchSize);
276     std::vector<std::vector<int>>classes(batchSize);
277     std::cout<<imageWidths[0]<<"--"<<imageHeights[0]<<"   "<<detection[3]<<std::endl;
278     
279     for(int curProposal = 0; curProposal < maxProposalCount;curProposal++){
280        auto image_id = static_cast<int>(detection[curProposal * objectSize +0]);
281        if(image_id < 0){break;}
282        float confidence =detection[curProposal * objectSize + 2];
283        auto label = static_cast<int>(detection[curProposal * objectSize + 1]);
284        auto xmin  = static_cast<int>(detection[curProposal * objectSize + 3] * imageWidths[image_id]);
285        auto ymin  = static_cast<int>(detection[curProposal * objectSize + 4] * imageHeights[image_id]);
286        auto xmax  = static_cast<int>(detection[curProposal * objectSize + 5] * imageWidths[image_id]);
287        auto ymax  = static_cast<int>(detection[curProposal * objectSize + 6] * imageHeights[image_id]);
288        std::cout << "[" << curProposal << "," << label << "] element, prob = " << confidence <<
289                 "    (" << xmin << "," << ymin << ")-(" << xmax << "," << ymax << ")" << " batch id : " << image_id;
290        if(confidence > 0.3){
291            classes[image_id].push_back(label);
292            boxes[image_id].push_back(xmin);
293            boxes[image_id].push_back(ymin);
294            boxes[image_id].push_back(xmax - xmin);
295            boxes[image_id].push_back(ymax - ymin);
296            std::cout << " WILL BE PRINTED!";
297        }
298        std::cout<<std::endl;
299    }
300    for(size_t batch_id = 0 ; batch_id < batchSize; ++batch_id){
301         addRectangles(originalImagesData[batch_id].get(),imageHeights[batch_id],imageWidths[batch_id],boxes[batch_id],classes[batch_id],BBOX_THICKNESS);
302    const std::string image_path = "out_" + std::to_string(batch_id) + ".bmp";
303    if (writeOutputBmp(image_path, originalImagesData[batch_id].get(), imageHeights[batch_id], imageWidths[batch_id])) {
304           slog::info << "Image " + image_path + " created!" << slog::endl;
305             } else {
306                 throw std::logic_error(std::string("Can't create a file: ") + image_path);
307             }
308 
309 }
310 
311 }
312 
313 /****************************************MAIN***************************************************/
314 
315 int main(int argc, char *argv[]) {
316     try {
317         /** This sample covers certain topology and cannot be generalized for any object detection one **/
318         // --------------------------- 1. Parsing and validation of input args ---------------------------------
319         Initialize_Check_params(argc,argv);
320         // --------------------------- 2. Read input -----------------------------------------------------------
321         /** This vector stores paths to the processed images **/
322         std::vector<std::string> images = readInputimagesNames();
323         // -----------------------------------------------------------------------------------------------------
324 
325         // --------------------------- 3. Load inference engine -------------------------------------
326         Core ie;
327         load_inference_engine(ie);
328 
329         // --------------------------- 4. Read IR Generated by ModelOptimizer (.xml and .bin files) ------------
330         CNNNetwork network = readIRfiles().network;
331         CNNNetReader networkReader = readIRfiles().networkReader;
332         // -----------------------------------------------------------------------------------------------------
333 
334         // --------------------------- 5. Prepare input blobs --------------------------------------------------
335           /** Taking information about all topology inputs **/
336           InputsDataMap inputsInfo(network.getInputsInfo());
337 
338           struct inputInfo_imageName res =  prepare_input_blobs(network,networkReader, inputsInfo);
339           InputInfo::Ptr inputInfo = res.inputInfo;
340           std::string imageInputName = res.InputName;
341           std::string imInfoInputName = res.imInfoInputName;
342 
343