-
Notifications
You must be signed in to change notification settings - Fork 68
/
bagofwords_classification.cpp
2625 lines (2317 loc) · 114 KB
/
bagofwords_classification.cpp
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#include "opencv2/opencv_modules.hpp"
#include "opencv2/highgui/highgui.hpp"
#include "opencv2/imgproc/imgproc.hpp"
#include "opencv2/features2d/features2d.hpp"
#include "opencv2/nonfree/nonfree.hpp"
#include "opencv2/ml/ml.hpp"
#ifdef HAVE_OPENCV_OCL
#define _OCL_SVM_ 1 //select whether using ocl::svm method or not, default is using
#include "opencv2/ocl/ocl.hpp"
#endif
#include <fstream>
#include <iostream>
#include <memory>
#include <functional>
#if defined WIN32 || defined _WIN32
#define WIN32_LEAN_AND_MEAN
#include <windows.h>
#undef min
#undef max
#include "sys/types.h"
#endif
#include <sys/stat.h>
#define DEBUG_DESC_PROGRESS
using namespace cv;
using namespace std;
const string paramsFile = "params.xml";
const string vocabularyFile = "vocabulary.xml.gz";
const string bowImageDescriptorsDir = "/bowImageDescriptors";
const string svmsDir = "/svms";
const string plotsDir = "/plots";
static void help(char** argv)
{
cout << "\nThis program shows how to read in, train on and produce test results for the PASCAL VOC (Visual Object Challenge) data. \n"
<< "It shows how to use detectors, descriptors and recognition methods \n"
"Using OpenCV version %s\n" << CV_VERSION << "\n"
<< "Call: \n"
<< "Format:\n ./" << argv[0] << " [VOC path] [result directory] \n"
<< " or: \n"
<< " ./" << argv[0] << " [VOC path] [result directory] [feature detector] [descriptor extractor] [descriptor matcher] \n"
<< "\n"
<< "Input parameters: \n"
<< "[VOC path] Path to Pascal VOC data (e.g. /home/my/VOCdevkit/VOC2010). Note: VOC2007-VOC2010 are supported. \n"
<< "[result directory] Path to result diractory. Following folders will be created in [result directory]: \n"
<< " bowImageDescriptors - to store image descriptors, \n"
<< " svms - to store trained svms, \n"
<< " plots - to store files for plots creating. \n"
<< "[feature detector] Feature detector name (e.g. SURF, FAST...) - see createFeatureDetector() function in detectors.cpp \n"
<< " Currently 12/2010, this is FAST, STAR, SIFT, SURF, MSER, GFTT, HARRIS \n"
<< "[descriptor extractor] Descriptor extractor name (e.g. SURF, SIFT) - see createDescriptorExtractor() function in descriptors.cpp \n"
<< " Currently 12/2010, this is SURF, OpponentSIFT, SIFT, OpponentSURF, BRIEF \n"
<< "[descriptor matcher] Descriptor matcher name (e.g. BruteForce) - see createDescriptorMatcher() function in matchers.cpp \n"
<< " Currently 12/2010, this is BruteForce, BruteForce-L1, FlannBased, BruteForce-Hamming, BruteForce-HammingLUT \n"
<< "\n";
}
static void makeDir( const string& dir )
{
#if defined WIN32 || defined _WIN32
CreateDirectory( dir.c_str(), 0 );
#else
mkdir( dir.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH );
#endif
}
static void makeUsedDirs( const string& rootPath )
{
makeDir(rootPath + bowImageDescriptorsDir);
makeDir(rootPath + svmsDir);
makeDir(rootPath + plotsDir);
}
/****************************************************************************************\
* Classes to work with PASCAL VOC dataset *
\****************************************************************************************/
//
// TODO: refactor this part of the code
//
//used to specify the (sub-)dataset over which operations are performed
enum ObdDatasetType {CV_OBD_TRAIN, CV_OBD_TEST};
class ObdObject
{
public:
string object_class;
Rect boundingBox;
};
//extended object data specific to VOC
enum VocPose {CV_VOC_POSE_UNSPECIFIED, CV_VOC_POSE_FRONTAL, CV_VOC_POSE_REAR, CV_VOC_POSE_LEFT, CV_VOC_POSE_RIGHT};
class VocObjectData
{
public:
bool difficult;
bool occluded;
bool truncated;
VocPose pose;
};
//enum VocDataset {CV_VOC2007, CV_VOC2008, CV_VOC2009, CV_VOC2010};
enum VocPlotType {CV_VOC_PLOT_SCREEN, CV_VOC_PLOT_PNG};
enum VocGT {CV_VOC_GT_NONE, CV_VOC_GT_DIFFICULT, CV_VOC_GT_PRESENT};
enum VocConfCond {CV_VOC_CCOND_RECALL, CV_VOC_CCOND_SCORETHRESH};
enum VocTask {CV_VOC_TASK_CLASSIFICATION, CV_VOC_TASK_DETECTION};
class ObdImage
{
public:
ObdImage(string p_id, string p_path) : id(p_id), path(p_path) {}
string id;
string path;
};
//used by getDetectorGroundTruth to sort a two dimensional list of floats in descending order
class ObdScoreIndexSorter
{
public:
float score;
int image_idx;
int obj_idx;
bool operator < (const ObdScoreIndexSorter& compare) const {return (score < compare.score);}
};
class VocData
{
public:
VocData( const string& vocPath, bool useTestDataset )
{ initVoc( vocPath, useTestDataset ); }
~VocData(){}
/* functions for returning classification/object data for multiple images given an object class */
void getClassImages(const string& obj_class, const ObdDatasetType dataset, vector<ObdImage>& images, vector<char>& object_present);
void getClassObjects(const string& obj_class, const ObdDatasetType dataset, vector<ObdImage>& images, vector<vector<ObdObject> >& objects);
void getClassObjects(const string& obj_class, const ObdDatasetType dataset, vector<ObdImage>& images, vector<vector<ObdObject> >& objects, vector<vector<VocObjectData> >& object_data, vector<VocGT>& ground_truth);
/* functions for returning object data for a single image given an image id */
ObdImage getObjects(const string& id, vector<ObdObject>& objects);
ObdImage getObjects(const string& id, vector<ObdObject>& objects, vector<VocObjectData>& object_data);
ObdImage getObjects(const string& obj_class, const string& id, vector<ObdObject>& objects, vector<VocObjectData>& object_data, VocGT& ground_truth);
/* functions for returning the ground truth (present/absent) for groups of images */
void getClassifierGroundTruth(const string& obj_class, const vector<ObdImage>& images, vector<char>& ground_truth);
void getClassifierGroundTruth(const string& obj_class, const vector<string>& images, vector<char>& ground_truth);
int getDetectorGroundTruth(const string& obj_class, const ObdDatasetType dataset, const vector<ObdImage>& images, const vector<vector<Rect> >& bounding_boxes, const vector<vector<float> >& scores, vector<vector<char> >& ground_truth, vector<vector<char> >& detection_difficult, bool ignore_difficult = true);
/* functions for writing VOC-compatible results files */
void writeClassifierResultsFile(const string& out_dir, const string& obj_class, const ObdDatasetType dataset, const vector<ObdImage>& images, const vector<float>& scores, const int competition = 1, const bool overwrite_ifexists = false);
/* functions for calculating metrics from a set of classification/detection results */
string getResultsFilename(const string& obj_class, const VocTask task, const ObdDatasetType dataset, const int competition = -1, const int number = -1);
void calcClassifierPrecRecall(const string& obj_class, const vector<ObdImage>& images, const vector<float>& scores, vector<float>& precision, vector<float>& recall, float& ap, vector<size_t>& ranking);
void calcClassifierPrecRecall(const string& obj_class, const vector<ObdImage>& images, const vector<float>& scores, vector<float>& precision, vector<float>& recall, float& ap);
void calcClassifierPrecRecall(const string& input_file, vector<float>& precision, vector<float>& recall, float& ap, bool outputRankingFile = false);
/* functions for calculating confusion matrices */
void calcClassifierConfMatRow(const string& obj_class, const vector<ObdImage>& images, const vector<float>& scores, const VocConfCond cond, const float threshold, vector<string>& output_headers, vector<float>& output_values);
void calcDetectorConfMatRow(const string& obj_class, const ObdDatasetType dataset, const vector<ObdImage>& images, const vector<vector<float> >& scores, const vector<vector<Rect> >& bounding_boxes, const VocConfCond cond, const float threshold, vector<string>& output_headers, vector<float>& output_values, bool ignore_difficult = true);
/* functions for outputting gnuplot output files */
void savePrecRecallToGnuplot(const string& output_file, const vector<float>& precision, const vector<float>& recall, const float ap, const string title = string(), const VocPlotType plot_type = CV_VOC_PLOT_SCREEN);
/* functions for reading in result/ground truth files */
void readClassifierGroundTruth(const string& obj_class, const ObdDatasetType dataset, vector<ObdImage>& images, vector<char>& object_present);
void readClassifierResultsFile(const std:: string& input_file, vector<ObdImage>& images, vector<float>& scores);
void readDetectorResultsFile(const string& input_file, vector<ObdImage>& images, vector<vector<float> >& scores, vector<vector<Rect> >& bounding_boxes);
/* functions for getting dataset info */
const vector<string>& getObjectClasses();
string getResultsDirectory();
protected:
void initVoc( const string& vocPath, const bool useTestDataset );
void initVoc2007to2010( const string& vocPath, const bool useTestDataset);
void readClassifierGroundTruth(const string& filename, vector<string>& image_codes, vector<char>& object_present);
void readClassifierResultsFile(const string& input_file, vector<string>& image_codes, vector<float>& scores);
void readDetectorResultsFile(const string& input_file, vector<string>& image_codes, vector<vector<float> >& scores, vector<vector<Rect> >& bounding_boxes);
void extractVocObjects(const string filename, vector<ObdObject>& objects, vector<VocObjectData>& object_data);
string getImagePath(const string& input_str);
void getClassImages_impl(const string& obj_class, const string& dataset_str, vector<ObdImage>& images, vector<char>& object_present);
void calcPrecRecall_impl(const vector<char>& ground_truth, const vector<float>& scores, vector<float>& precision, vector<float>& recall, float& ap, vector<size_t>& ranking, int recall_normalization = -1);
//test two bounding boxes to see if they meet the overlap criteria defined in the VOC documentation
float testBoundingBoxesForOverlap(const Rect detection, const Rect ground_truth);
//extract class and dataset name from a VOC-standard classification/detection results filename
void extractDataFromResultsFilename(const string& input_file, string& class_name, string& dataset_name);
//get classifier ground truth for a single image
bool getClassifierGroundTruthImage(const string& obj_class, const string& id);
//utility functions
void getSortOrder(const vector<float>& values, vector<size_t>& order, bool descending = true);
int stringToInteger(const string input_str);
void readFileToString(const string filename, string& file_contents);
string integerToString(const int input_int);
string checkFilenamePathsep(const string filename, bool add_trailing_slash = false);
void convertImageCodesToObdImages(const vector<string>& image_codes, vector<ObdImage>& images);
int extractXMLBlock(const string src, const string tag, const int searchpos, string& tag_contents);
//utility sorter
struct orderingSorter
{
bool operator ()(std::pair<size_t, vector<float>::const_iterator> const& a, std::pair<size_t, vector<float>::const_iterator> const& b)
{
return (*a.second) > (*b.second);
}
};
//data members
string m_vocPath;
string m_vocName;
//string m_resPath;
string m_annotation_path;
string m_image_path;
string m_imageset_path;
string m_class_imageset_path;
vector<string> m_classifier_gt_all_ids;
vector<char> m_classifier_gt_all_present;
string m_classifier_gt_class;
//data members
string m_train_set;
string m_test_set;
vector<string> m_object_classes;
float m_min_overlap;
bool m_sampled_ap;
};
//Return the classification ground truth data for all images of a given VOC object class
//--------------------------------------------------------------------------------------
//INPUTS:
// - obj_class The VOC object class identifier string
// - dataset Specifies whether to extract images from the training or test set
//OUTPUTS:
// - images An array of ObdImage containing info of all images extracted from the ground truth file
// - object_present An array of bools specifying whether the object defined by 'obj_class' is present in each image or not
//NOTES:
// This function is primarily useful for the classification task, where only
// whether a given object is present or not in an image is required, and not each object instance's
// position etc.
void VocData::getClassImages(const string& obj_class, const ObdDatasetType dataset, vector<ObdImage>& images, vector<char>& object_present)
{
string dataset_str;
//generate the filename of the classification ground-truth textfile for the object class
if (dataset == CV_OBD_TRAIN)
{
dataset_str = m_train_set;
} else {
dataset_str = m_test_set;
}
getClassImages_impl(obj_class, dataset_str, images, object_present);
}
void VocData::getClassImages_impl(const string& obj_class, const string& dataset_str, vector<ObdImage>& images, vector<char>& object_present)
{
//generate the filename of the classification ground-truth textfile for the object class
string gtFilename = m_class_imageset_path;
gtFilename.replace(gtFilename.find("%s"),2,obj_class);
gtFilename.replace(gtFilename.find("%s"),2,dataset_str);
//parse the ground truth file, storing in two separate vectors
//for the image code and the ground truth value
vector<string> image_codes;
readClassifierGroundTruth(gtFilename, image_codes, object_present);
//prepare output arrays
images.clear();
convertImageCodesToObdImages(image_codes, images);
}
//Return the object data for all images of a given VOC object class
//-----------------------------------------------------------------
//INPUTS:
// - obj_class The VOC object class identifier string
// - dataset Specifies whether to extract images from the training or test set
//OUTPUTS:
// - images An array of ObdImage containing info of all images in chosen dataset (tag, path etc.)
// - objects Contains the extended object info (bounding box etc.) for each object instance in each image
// - object_data Contains VOC-specific extended object info (marked difficult etc.)
// - ground_truth Specifies whether there are any difficult/non-difficult instances of the current
// object class within each image
//NOTES:
// This function returns extended object information in addition to the absent/present
// classification data returned by getClassImages. The objects returned for each image in the 'objects'
// array are of all object classes present in the image, and not just the class defined by 'obj_class'.
// 'ground_truth' can be used to determine quickly whether an object instance of the given class is present
// in an image or not.
void VocData::getClassObjects(const string& obj_class, const ObdDatasetType dataset, vector<ObdImage>& images, vector<vector<ObdObject> >& objects)
{
vector<vector<VocObjectData> > object_data;
vector<VocGT> ground_truth;
getClassObjects(obj_class,dataset,images,objects,object_data,ground_truth);
}
void VocData::getClassObjects(const string& obj_class, const ObdDatasetType dataset, vector<ObdImage>& images, vector<vector<ObdObject> >& objects, vector<vector<VocObjectData> >& object_data, vector<VocGT>& ground_truth)
{
//generate the filename of the classification ground-truth textfile for the object class
string gtFilename = m_class_imageset_path;
gtFilename.replace(gtFilename.find("%s"),2,obj_class);
if (dataset == CV_OBD_TRAIN)
{
gtFilename.replace(gtFilename.find("%s"),2,m_train_set);
} else {
gtFilename.replace(gtFilename.find("%s"),2,m_test_set);
}
//parse the ground truth file, storing in two separate vectors
//for the image code and the ground truth value
vector<string> image_codes;
vector<char> object_present;
readClassifierGroundTruth(gtFilename, image_codes, object_present);
//prepare output arrays
images.clear();
objects.clear();
object_data.clear();
ground_truth.clear();
string annotationFilename;
vector<ObdObject> image_objects;
vector<VocObjectData> image_object_data;
VocGT image_gt;
//transfer to output arrays and read in object data for each image
for (size_t i = 0; i < image_codes.size(); ++i)
{
ObdImage image = getObjects(obj_class, image_codes[i], image_objects, image_object_data, image_gt);
images.push_back(image);
objects.push_back(image_objects);
object_data.push_back(image_object_data);
ground_truth.push_back(image_gt);
}
}
//Return ground truth data for the objects present in an image with a given UID
//-----------------------------------------------------------------------------
//INPUTS:
// - id VOC Dataset unique identifier (string code in form YYYY_XXXXXX where YYYY is the year)
//OUTPUTS:
// - obj_class (*3) Specifies the object class to use to resolve 'ground_truth'
// - objects Contains the extended object info (bounding box etc.) for each object in the image
// - object_data (*2,3) Contains VOC-specific extended object info (marked difficult etc.)
// - ground_truth (*3) Specifies whether there are any difficult/non-difficult instances of the current
// object class within the image
//RETURN VALUE:
// ObdImage containing path and other details of image file with given code
//NOTES:
// There are three versions of this function
// * One returns a simple array of objects given an id [1]
// * One returns the same as (1) plus VOC specific object data [2]
// * One returns the same as (2) plus the ground_truth flag. This also requires an extra input obj_class [3]
ObdImage VocData::getObjects(const string& id, vector<ObdObject>& objects)
{
vector<VocObjectData> object_data;
ObdImage image = getObjects(id, objects, object_data);
return image;
}
ObdImage VocData::getObjects(const string& id, vector<ObdObject>& objects, vector<VocObjectData>& object_data)
{
//first generate the filename of the annotation file
string annotationFilename = m_annotation_path;
annotationFilename.replace(annotationFilename.find("%s"),2,id);
//extract objects contained in the current image from the xml
extractVocObjects(annotationFilename,objects,object_data);
//generate image path from extracted string code
string path = getImagePath(id);
ObdImage image(id, path);
return image;
}
ObdImage VocData::getObjects(const string& obj_class, const string& id, vector<ObdObject>& objects, vector<VocObjectData>& object_data, VocGT& ground_truth)
{
//extract object data (except for ground truth flag)
ObdImage image = getObjects(id,objects,object_data);
//pregenerate a flag to indicate whether the current class is present or not in the image
ground_truth = CV_VOC_GT_NONE;
//iterate through all objects in current image
for (size_t j = 0; j < objects.size(); ++j)
{
if (objects[j].object_class == obj_class)
{
if (object_data[j].difficult == false)
{
//if at least one non-difficult example is present, this flag is always set to CV_VOC_GT_PRESENT
ground_truth = CV_VOC_GT_PRESENT;
break;
} else {
//set if at least one object instance is present, but it is marked difficult
ground_truth = CV_VOC_GT_DIFFICULT;
}
}
}
return image;
}
//Return ground truth data for the presence/absence of a given object class in an arbitrary array of images
//---------------------------------------------------------------------------------------------------------
//INPUTS:
// - obj_class The VOC object class identifier string
// - images An array of ObdImage OR strings containing the images for which ground truth
// will be computed
//OUTPUTS:
// - ground_truth An output array indicating the presence/absence of obj_class within each image
void VocData::getClassifierGroundTruth(const string& obj_class, const vector<ObdImage>& images, vector<char>& ground_truth)
{
vector<char>(images.size()).swap(ground_truth);
vector<ObdObject> objects;
vector<VocObjectData> object_data;
vector<char>::iterator gt_it = ground_truth.begin();
for (vector<ObdImage>::const_iterator it = images.begin(); it != images.end(); ++it, ++gt_it)
{
//getObjects(obj_class, it->id, objects, object_data, voc_ground_truth);
(*gt_it) = (getClassifierGroundTruthImage(obj_class, it->id));
}
}
void VocData::getClassifierGroundTruth(const string& obj_class, const vector<string>& images, vector<char>& ground_truth)
{
vector<char>(images.size()).swap(ground_truth);
vector<ObdObject> objects;
vector<VocObjectData> object_data;
vector<char>::iterator gt_it = ground_truth.begin();
for (vector<string>::const_iterator it = images.begin(); it != images.end(); ++it, ++gt_it)
{
//getObjects(obj_class, (*it), objects, object_data, voc_ground_truth);
(*gt_it) = (getClassifierGroundTruthImage(obj_class, (*it)));
}
}
//Return ground truth data for the accuracy of detection results
//--------------------------------------------------------------
//INPUTS:
// - obj_class The VOC object class identifier string
// - images An array of ObdImage containing the images for which ground truth
// will be computed
// - bounding_boxes A 2D input array containing the bounding box rects of the objects of
// obj_class which were detected in each image
//OUTPUTS:
// - ground_truth A 2D output array indicating whether each object detection was accurate
// or not
// - detection_difficult A 2D output array indicating whether the detection fired on an object
// marked as 'difficult'. This allows it to be ignored if necessary
// (the voc documentation specifies objects marked as difficult
// have no effects on the results and are effectively ignored)
// - (ignore_difficult) If set to true, objects marked as difficult will be ignored when returning
// the number of hits for p-r normalization (default = true)
//RETURN VALUE:
// Returns the number of object hits in total in the gt to allow proper normalization
// of a p-r curve
//NOTES:
// As stated in the VOC documentation, multiple detections of the same object in an image are
// considered FALSE detections e.g. 5 detections of a single object is counted as 1 correct
// detection and 4 false detections - it is the responsibility of the participant's system
// to filter multiple detections from its output
int VocData::getDetectorGroundTruth(const string& obj_class, const ObdDatasetType dataset, const vector<ObdImage>& images, const vector<vector<Rect> >& bounding_boxes, const vector<vector<float> >& scores, vector<vector<char> >& ground_truth, vector<vector<char> >& detection_difficult, bool ignore_difficult)
{
int recall_normalization = 0;
/* first create a list of indices referring to the elements of bounding_boxes and scores in
* descending order of scores */
vector<ObdScoreIndexSorter> sorted_ids;
{
/* first count how many objects to allow preallocation */
size_t obj_count = 0;
CV_Assert(images.size() == bounding_boxes.size());
CV_Assert(scores.size() == bounding_boxes.size());
for (size_t im_idx = 0; im_idx < scores.size(); ++im_idx)
{
CV_Assert(scores[im_idx].size() == bounding_boxes[im_idx].size());
obj_count += scores[im_idx].size();
}
/* preallocate id vector */
sorted_ids.resize(obj_count);
/* now copy across scores and indexes to preallocated vector */
int flat_pos = 0;
for (size_t im_idx = 0; im_idx < scores.size(); ++im_idx)
{
for (size_t ob_idx = 0; ob_idx < scores[im_idx].size(); ++ob_idx)
{
sorted_ids[flat_pos].score = scores[im_idx][ob_idx];
sorted_ids[flat_pos].image_idx = (int)im_idx;
sorted_ids[flat_pos].obj_idx = (int)ob_idx;
++flat_pos;
}
}
/* and sort the vector in descending order of score */
std::sort(sorted_ids.begin(),sorted_ids.end());
std::reverse(sorted_ids.begin(),sorted_ids.end());
}
/* prepare ground truth + difficult vector (1st dimension) */
vector<vector<char> >(images.size()).swap(ground_truth);
vector<vector<char> >(images.size()).swap(detection_difficult);
vector<vector<char> > detected(images.size());
vector<vector<ObdObject> > img_objects(images.size());
vector<vector<VocObjectData> > img_object_data(images.size());
/* preload object ground truth bounding box data */
{
vector<vector<ObdObject> > img_objects_all(images.size());
vector<vector<VocObjectData> > img_object_data_all(images.size());
for (size_t image_idx = 0; image_idx < images.size(); ++image_idx)
{
/* prepopulate ground truth bounding boxes */
getObjects(images[image_idx].id, img_objects_all[image_idx], img_object_data_all[image_idx]);
/* meanwhile, also set length of target ground truth + difficult vector to same as number of object detections (2nd dimension) */
ground_truth[image_idx].resize(bounding_boxes[image_idx].size());
detection_difficult[image_idx].resize(bounding_boxes[image_idx].size());
}
/* save only instances of the object class concerned */
for (size_t image_idx = 0; image_idx < images.size(); ++image_idx)
{
for (size_t obj_idx = 0; obj_idx < img_objects_all[image_idx].size(); ++obj_idx)
{
if (img_objects_all[image_idx][obj_idx].object_class == obj_class)
{
img_objects[image_idx].push_back(img_objects_all[image_idx][obj_idx]);
img_object_data[image_idx].push_back(img_object_data_all[image_idx][obj_idx]);
}
}
detected[image_idx].resize(img_objects[image_idx].size(), false);
}
}
/* calculate the total number of objects in the ground truth for the current dataset */
{
vector<ObdImage> gt_images;
vector<char> gt_object_present;
getClassImages(obj_class, dataset, gt_images, gt_object_present);
for (size_t image_idx = 0; image_idx < gt_images.size(); ++image_idx)
{
vector<ObdObject> gt_img_objects;
vector<VocObjectData> gt_img_object_data;
getObjects(gt_images[image_idx].id, gt_img_objects, gt_img_object_data);
for (size_t obj_idx = 0; obj_idx < gt_img_objects.size(); ++obj_idx)
{
if (gt_img_objects[obj_idx].object_class == obj_class)
{
if ((gt_img_object_data[obj_idx].difficult == false) || (ignore_difficult == false))
++recall_normalization;
}
}
}
}
#ifdef PR_DEBUG
int printed_count = 0;
#endif
/* now iterate through detections in descending order of score, assigning to ground truth bounding boxes if possible */
for (size_t detect_idx = 0; detect_idx < sorted_ids.size(); ++detect_idx)
{
//read in indexes to make following code easier to read
int im_idx = sorted_ids[detect_idx].image_idx;
int ob_idx = sorted_ids[detect_idx].obj_idx;
//set ground truth for the current object to false by default
ground_truth[im_idx][ob_idx] = false;
detection_difficult[im_idx][ob_idx] = false;
float maxov = -1.0;
bool max_is_difficult = false;
int max_gt_obj_idx = -1;
//-- for each detected object iterate through objects present in the bounding box ground truth --
for (size_t gt_obj_idx = 0; gt_obj_idx < img_objects[im_idx].size(); ++gt_obj_idx)
{
if (detected[im_idx][gt_obj_idx] == false)
{
//check if the detected object and ground truth object overlap by a sufficient margin
float ov = testBoundingBoxesForOverlap(bounding_boxes[im_idx][ob_idx], img_objects[im_idx][gt_obj_idx].boundingBox);
if (ov != -1.0)
{
//if all conditions are met store the overlap score and index (as objects are assigned to the highest scoring match)
if (ov > maxov)
{
maxov = ov;
max_gt_obj_idx = (int)gt_obj_idx;
//store whether the maximum detection is marked as difficult or not
max_is_difficult = (img_object_data[im_idx][gt_obj_idx].difficult);
}
}
}
}
//-- if a match was found, set the ground truth of the current object to true --
if (maxov != -1.0)
{
CV_Assert(max_gt_obj_idx != -1);
ground_truth[im_idx][ob_idx] = true;
//store whether the maximum detection was marked as 'difficult' or not
detection_difficult[im_idx][ob_idx] = max_is_difficult;
//remove the ground truth object so it doesn't match with subsequent detected objects
//** this is the behaviour defined by the voc documentation **
detected[im_idx][max_gt_obj_idx] = true;
}
#ifdef PR_DEBUG
if (printed_count < 10)
{
cout << printed_count << ": id=" << images[im_idx].id << ", score=" << scores[im_idx][ob_idx] << " (" << ob_idx << ") [" << bounding_boxes[im_idx][ob_idx].x << "," <<
bounding_boxes[im_idx][ob_idx].y << "," << bounding_boxes[im_idx][ob_idx].width + bounding_boxes[im_idx][ob_idx].x <<
"," << bounding_boxes[im_idx][ob_idx].height + bounding_boxes[im_idx][ob_idx].y << "] detected=" << ground_truth[im_idx][ob_idx] <<
", difficult=" << detection_difficult[im_idx][ob_idx] << endl;
++printed_count;
/* print ground truth */
for (int gt_obj_idx = 0; gt_obj_idx < img_objects[im_idx].size(); ++gt_obj_idx)
{
cout << " GT: [" << img_objects[im_idx][gt_obj_idx].boundingBox.x << "," <<
img_objects[im_idx][gt_obj_idx].boundingBox.y << "," << img_objects[im_idx][gt_obj_idx].boundingBox.width + img_objects[im_idx][gt_obj_idx].boundingBox.x <<
"," << img_objects[im_idx][gt_obj_idx].boundingBox.height + img_objects[im_idx][gt_obj_idx].boundingBox.y << "]";
if (gt_obj_idx == max_gt_obj_idx) cout << " <--- (" << maxov << " overlap)";
cout << endl;
}
}
#endif
}
return recall_normalization;
}
//Write VOC-compliant classifier results file
//-------------------------------------------
//INPUTS:
// - obj_class The VOC object class identifier string
// - dataset Specifies whether working with the training or test set
// - images An array of ObdImage containing the images for which data will be saved to the result file
// - scores A corresponding array of confidence scores given a query
// - (competition) If specified, defines which competition the results are for (see VOC documentation - default 1)
//NOTES:
// The result file path and filename are determined automatically using m_results_directory as a base
void VocData::writeClassifierResultsFile( const string& out_dir, const string& obj_class, const ObdDatasetType dataset, const vector<ObdImage>& images, const vector<float>& scores, const int competition, const bool overwrite_ifexists)
{
CV_Assert(images.size() == scores.size());
string output_file_base, output_file;
if (dataset == CV_OBD_TRAIN)
{
output_file_base = out_dir + "/comp" + integerToString(competition) + "_cls_" + m_train_set + "_" + obj_class;
} else {
output_file_base = out_dir + "/comp" + integerToString(competition) + "_cls_" + m_test_set + "_" + obj_class;
}
output_file = output_file_base + ".txt";
//check if file exists, and if so create a numbered new file instead
if (overwrite_ifexists == false)
{
struct stat stFileInfo;
if (stat(output_file.c_str(),&stFileInfo) == 0)
{
string output_file_new;
int filenum = 0;
do
{
++filenum;
output_file_new = output_file_base + "_" + integerToString(filenum);
output_file = output_file_new + ".txt";
} while (stat(output_file.c_str(),&stFileInfo) == 0);
}
}
//output data to file
std::ofstream result_file(output_file.c_str());
if (result_file.is_open())
{
for (size_t i = 0; i < images.size(); ++i)
{
result_file << images[i].id << " " << scores[i] << endl;
}
result_file.close();
} else {
string err_msg = "could not open classifier results file '" + output_file + "' for writing. Before running for the first time, a 'results' subdirectory should be created within the VOC dataset base directory. e.g. if the VOC data is stored in /VOC/VOC2010 then the path /VOC/results must be created.";
CV_Error(CV_StsError,err_msg.c_str());
}
}
//---------------------------------------
//CALCULATE METRICS FROM VOC RESULTS DATA
//---------------------------------------
//Utility function to construct a VOC-standard classification results filename
//----------------------------------------------------------------------------
//INPUTS:
// - obj_class The VOC object class identifier string
// - task Specifies whether to generate a filename for the classification or detection task
// - dataset Specifies whether working with the training or test set
// - (competition) If specified, defines which competition the results are for (see VOC documentation
// default of -1 means this is set to 1 for the classification task and 3 for the detection task)
// - (number) If specified and above 0, defines which of a number of duplicate results file produced for a given set of
// of settings should be used (this number will be added as a postfix to the filename)
//NOTES:
// This is primarily useful for returning the filename of a classification file previously computed using writeClassifierResultsFile
// for example when calling calcClassifierPrecRecall
string VocData::getResultsFilename(const string& obj_class, const VocTask task, const ObdDatasetType dataset, const int competition, const int number)
{
if ((competition < 1) && (competition != -1))
CV_Error(CV_StsBadArg,"competition argument should be a positive non-zero number or -1 to accept the default");
if ((number < 1) && (number != -1))
CV_Error(CV_StsBadArg,"number argument should be a positive non-zero number or -1 to accept the default");
string dset, task_type;
if (dataset == CV_OBD_TRAIN)
{
dset = m_train_set;
} else {
dset = m_test_set;
}
int comp = competition;
if (task == CV_VOC_TASK_CLASSIFICATION)
{
task_type = "cls";
if (comp == -1) comp = 1;
} else {
task_type = "det";
if (comp == -1) comp = 3;
}
stringstream ss;
if (number < 1)
{
ss << "comp" << comp << "_" << task_type << "_" << dset << "_" << obj_class << ".txt";
} else {
ss << "comp" << comp << "_" << task_type << "_" << dset << "_" << obj_class << "_" << number << ".txt";
}
string filename = ss.str();
return filename;
}
//Calculate metrics for classification results
//--------------------------------------------
//INPUTS:
// - ground_truth A vector of booleans determining whether the currently tested class is present in each input image
// - scores A vector containing the similarity score for each input image (higher is more similar)
//OUTPUTS:
// - precision A vector containing the precision calculated at each datapoint of a p-r curve generated from the result set
// - recall A vector containing the recall calculated at each datapoint of a p-r curve generated from the result set
// - ap The ap metric calculated from the result set
// - (ranking) A vector of the same length as 'ground_truth' and 'scores' containing the order of the indices in both of
// these arrays when sorting by the ranking score in descending order
//NOTES:
// The result file path and filename are determined automatically using m_results_directory as a base
void VocData::calcClassifierPrecRecall(const string& obj_class, const vector<ObdImage>& images, const vector<float>& scores, vector<float>& precision, vector<float>& recall, float& ap, vector<size_t>& ranking)
{
vector<char> res_ground_truth;
getClassifierGroundTruth(obj_class, images, res_ground_truth);
calcPrecRecall_impl(res_ground_truth, scores, precision, recall, ap, ranking);
}
void VocData::calcClassifierPrecRecall(const string& obj_class, const vector<ObdImage>& images, const vector<float>& scores, vector<float>& precision, vector<float>& recall, float& ap)
{
vector<char> res_ground_truth;
getClassifierGroundTruth(obj_class, images, res_ground_truth);
vector<size_t> ranking;
calcPrecRecall_impl(res_ground_truth, scores, precision, recall, ap, ranking);
}
//< Overloaded version which accepts VOC classification result file input instead of array of scores/ground truth >
//INPUTS:
// - input_file The path to the VOC standard results file to use for calculating precision/recall
// If a full path is not specified, it is assumed this file is in the VOC standard results directory
// A VOC standard filename can be retrieved (as used by writeClassifierResultsFile) by calling getClassifierResultsFilename
void VocData::calcClassifierPrecRecall(const string& input_file, vector<float>& precision, vector<float>& recall, float& ap, bool outputRankingFile)
{
//read in classification results file
vector<string> res_image_codes;
vector<float> res_scores;
string input_file_std = checkFilenamePathsep(input_file);
readClassifierResultsFile(input_file_std, res_image_codes, res_scores);
//extract the object class and dataset from the results file filename
string class_name, dataset_name;
extractDataFromResultsFilename(input_file_std, class_name, dataset_name);
//generate the ground truth for the images extracted from the results file
vector<char> res_ground_truth;
getClassifierGroundTruth(class_name, res_image_codes, res_ground_truth);
if (outputRankingFile)
{
/* 1. store sorting order by score (descending) in 'order' */
vector<std::pair<size_t, vector<float>::const_iterator> > order(res_scores.size());
size_t n = 0;
for (vector<float>::const_iterator it = res_scores.begin(); it != res_scores.end(); ++it, ++n)
order[n] = make_pair(n, it);
std::sort(order.begin(),order.end(),orderingSorter());
/* 2. save ranking results to text file */
string input_file_std1 = checkFilenamePathsep(input_file);
size_t fnamestart = input_file_std1.rfind("/");
string scoregt_file_str = input_file_std1.substr(0,fnamestart+1) + "scoregt_" + class_name + ".txt";
std::ofstream scoregt_file(scoregt_file_str.c_str());
if (scoregt_file.is_open())
{
for (size_t i = 0; i < res_scores.size(); ++i)
{
scoregt_file << res_image_codes[order[i].first] << " " << res_scores[order[i].first] << " " << res_ground_truth[order[i].first] << endl;
}
scoregt_file.close();
} else {
string err_msg = "could not open scoregt file '" + scoregt_file_str + "' for writing.";
CV_Error(CV_StsError,err_msg.c_str());
}
}
//finally, calculate precision+recall+ap
vector<size_t> ranking;
calcPrecRecall_impl(res_ground_truth,res_scores,precision,recall,ap,ranking);
}
//< Protected implementation of Precision-Recall calculation used by both calcClassifierPrecRecall and calcDetectorPrecRecall >
void VocData::calcPrecRecall_impl(const vector<char>& ground_truth, const vector<float>& scores, vector<float>& precision, vector<float>& recall, float& ap, vector<size_t>& ranking, int recall_normalization)
{
CV_Assert(ground_truth.size() == scores.size());
//add extra element for p-r at 0 recall (in case that first retrieved is positive)
vector<float>(scores.size()+1).swap(precision);
vector<float>(scores.size()+1).swap(recall);
// SORT RESULTS BY THEIR SCORE
/* 1. store sorting order in 'order' */
VocData::getSortOrder(scores, ranking);
#ifdef PR_DEBUG
std::ofstream scoregt_file("D:/pr.txt");
if (scoregt_file.is_open())
{
for (int i = 0; i < scores.size(); ++i)
{
scoregt_file << scores[ranking[i]] << " " << ground_truth[ranking[i]] << endl;
}
scoregt_file.close();
}
#endif
// CALCULATE PRECISION+RECALL
int retrieved_hits = 0;
int recall_norm;
if (recall_normalization != -1)
{
recall_norm = recall_normalization;
} else {
recall_norm = (int)std::count_if(ground_truth.begin(),ground_truth.end(),std::bind2nd(std::equal_to<char>(),(char)1));
}
ap = 0;
recall[0] = 0;
for (size_t idx = 0; idx < ground_truth.size(); ++idx)
{
if (ground_truth[ranking[idx]] != 0) ++retrieved_hits;
precision[idx+1] = static_cast<float>(retrieved_hits)/static_cast<float>(idx+1);
recall[idx+1] = static_cast<float>(retrieved_hits)/static_cast<float>(recall_norm);
if (idx == 0)
{
//add further point at 0 recall with the same precision value as the first computed point
precision[idx] = precision[idx+1];
}
if (recall[idx+1] == 1.0)
{
//if recall = 1, then end early as all positive images have been found
recall.resize(idx+2);
precision.resize(idx+2);
break;
}
}
/* ap calculation */
if (m_sampled_ap == false)
{
// FOR VOC2010+ AP IS CALCULATED FROM ALL DATAPOINTS
/* make precision monotonically decreasing for purposes of calculating ap */
vector<float> precision_monot(precision.size());
vector<float>::iterator prec_m_it = precision_monot.begin();
for (vector<float>::iterator prec_it = precision.begin(); prec_it != precision.end(); ++prec_it, ++prec_m_it)
{
vector<float>::iterator max_elem;
max_elem = std::max_element(prec_it,precision.end());
(*prec_m_it) = (*max_elem);
}
/* calculate ap */
for (size_t idx = 0; idx < (recall.size()-1); ++idx)
{
ap += (recall[idx+1] - recall[idx])*precision_monot[idx+1] + //no need to take min of prec - is monotonically decreasing
0.5f*(recall[idx+1] - recall[idx])*std::abs(precision_monot[idx+1] - precision_monot[idx]);
}
} else {
// FOR BEFORE VOC2010 AP IS CALCULATED BY SAMPLING PRECISION AT RECALL 0.0,0.1,..,1.0
for (float recall_pos = 0.f; recall_pos <= 1.f; recall_pos += 0.1f)
{
//find iterator of the precision corresponding to the first recall >= recall_pos
vector<float>::iterator recall_it = recall.begin();
vector<float>::iterator prec_it = precision.begin();
while ((*recall_it) < recall_pos)
{
++recall_it;
++prec_it;
if (recall_it == recall.end()) break;
}
/* if no recall >= recall_pos found, this level of recall is never reached so stop adding to ap */
if (recall_it == recall.end()) break;
/* if the prec_it is valid, compute the max precision at this level of recall or higher */
vector<float>::iterator max_prec = std::max_element(prec_it,precision.end());
ap += (*max_prec)/11;
}
}
}
/* functions for calculating confusion matrix rows */
//Calculate rows of a confusion matrix
//------------------------------------
//INPUTS:
// - obj_class The VOC object class identifier string for the confusion matrix row to compute
// - images An array of ObdImage containing the images to use for the computation
// - scores A corresponding array of confidence scores for the presence of obj_class in each image
// - cond Defines whether to use a cut off point based on recall (CV_VOC_CCOND_RECALL) or score
// (CV_VOC_CCOND_SCORETHRESH) the latter is useful for classifier detections where positive
// values are positive detections and negative values are negative detections
// - threshold Threshold value for cond. In case of CV_VOC_CCOND_RECALL, is proportion recall (e.g. 0.5).
// In the case of CV_VOC_CCOND_SCORETHRESH is the value above which to count results.
//OUTPUTS:
// - output_headers An output vector of object class headers for the confusion matrix row
// - output_values An output vector of values for the confusion matrix row corresponding to the classes
// defined in output_headers
//NOTES:
// The methodology used by the classifier version of this function is that true positives have a single unit
// added to the obj_class column in the confusion matrix row, whereas false positives have a single unit
// distributed in proportion between all the columns in the confusion matrix row corresponding to the objects
// present in the image.
void VocData::calcClassifierConfMatRow(const string& obj_class, const vector<ObdImage>& images, const vector<float>& scores, const VocConfCond cond, const float threshold, vector<string>& output_headers, vector<float>& output_values)
{
CV_Assert(images.size() == scores.size());
// SORT RESULTS BY THEIR SCORE
/* 1. store sorting order in 'ranking' */
vector<size_t> ranking;
VocData::getSortOrder(scores, ranking);
// CALCULATE CONFUSION MATRIX ENTRIES
/* prepare object category headers */
output_headers = m_object_classes;
vector<float>(output_headers.size(),0.0).swap(output_values);
/* find the index of the target object class in the headers for later use */
int target_idx;
{
vector<string>::iterator target_idx_it = std::find(output_headers.begin(),output_headers.end(),obj_class);
/* if the target class can not be found, raise an exception */
if (target_idx_it == output_headers.end())
{
string err_msg = "could not find the target object class '" + obj_class + "' in list of valid classes.";
CV_Error(CV_StsError,err_msg.c_str());
}
/* convert iterator to index */
target_idx = (int)std::distance(output_headers.begin(),target_idx_it);
}
/* prepare variables related to calculating recall if using the recall threshold */
int retrieved_hits = 0;
int total_relevant = 0;
if (cond == CV_VOC_CCOND_RECALL)
{
vector<char> ground_truth;
/* in order to calculate the total number of relevant images for normalization of recall
it's necessary to extract the ground truth for the images under consideration */
getClassifierGroundTruth(obj_class, images, ground_truth);
total_relevant = (int)std::count_if(ground_truth.begin(),ground_truth.end(),std::bind2nd(std::equal_to<char>(),(char)1));
}
/* iterate through images */
vector<ObdObject> img_objects;
vector<VocObjectData> img_object_data;