diff --git a/experiments/scripts/faster_rcnn_end2end.sh b/experiments/scripts/faster_rcnn_end2end.sh index 79770aa56..54f8d5678 100755 --- a/experiments/scripts/faster_rcnn_end2end.sh +++ b/experiments/scripts/faster_rcnn_end2end.sh @@ -29,6 +29,18 @@ case $DATASET in PT_DIR="pascal_voc" ITERS=70000 ;; + traffic) + TRAIN_IMDB="traffic_2016_train" + TEST_IMDB="traffic_2016_test" + PT_DIR="traffic" + ITERS=60000 + ;; + traffic_india) + TRAIN_IMDB="traffic_india_2016_trainval" + TEST_IMDB="traffic_india_2016_test" + PT_DIR="traffic_india" + ITERS=20000 + ;; coco) # This is a very long and slow training schedule # You can probably use fewer iterations and reduce the @@ -48,18 +60,20 @@ LOG="experiments/logs/faster_rcnn_end2end_${NET}_${EXTRA_ARGS_SLUG}.txt.`date +' exec &> >(tee -a "$LOG") echo Logging output to "$LOG" -time ./tools/train_net.py --gpu ${GPU_ID} \ - --solver models/${PT_DIR}/${NET}/faster_rcnn_end2end/solver.prototxt \ - --weights data/imagenet_models/${NET}.v2.caffemodel \ - --imdb ${TRAIN_IMDB} \ - --iters ${ITERS} \ - --cfg experiments/cfgs/faster_rcnn_end2end.yml \ - ${EXTRA_ARGS} +#time ./tools/train_net.py --gpu ${GPU_ID} \ + #--solver models/${PT_DIR}/${NET}/faster_rcnn_end2end/solver.prototxt \ + #--weights data/imagenet_models/${NET}.v2.caffemodel \ + #--imdb ${TRAIN_IMDB} \ + #--iters ${ITERS} \ + #--cfg experiments/cfgs/faster_rcnn_end2end.yml \ + #${EXTRA_ARGS} set +x NET_FINAL=`grep -B 1 "done solving" ${LOG} | grep "Wrote snapshot" | awk '{print $4}'` set -x +#temp +NET_FINAL=/home/ubuntu/py-faster-rcnn/output/faster_rcnn_end2end/traffic_2016_train/zf_faster_rcnn_iter_60000.caffemodel time ./tools/test_net.py --gpu ${GPU_ID} \ --def models/${PT_DIR}/${NET}/faster_rcnn_end2end/test.prototxt \ --net ${NET_FINAL} \ diff --git a/lib/datasets/factory.py b/lib/datasets/factory.py index 8c3fdb898..14a52b821 100644 --- a/lib/datasets/factory.py +++ b/lib/datasets/factory.py @@ -11,8 +11,22 @@ from datasets.pascal_voc import pascal_voc from datasets.coco import coco +from datasets.traffic import traffic +from datasets.traffic_india import traffic_india import numpy as np +# cars-test dataset +for year in ['2016']: + for split in ['train', 'val', 'trainval', 'test']: + name = 'traffic_{}_{}'.format(year, split) + __sets[name] = (lambda split=split, year=year: traffic(split, year)) + +# traffic india dataset +for year in ['2016']: + for split in ['train', 'val', 'trainval', 'test']: + name = 'traffic_india_{}_{}'.format(year, split) + __sets[name] = (lambda split=split, year=year: traffic_india(split, year)) + # Set up voc__ using selective search "fast" mode for year in ['2007', '2012']: for split in ['train', 'val', 'trainval', 'test']: diff --git a/lib/datasets/traffic.py b/lib/datasets/traffic.py new file mode 100644 index 000000000..55a433a18 --- /dev/null +++ b/lib/datasets/traffic.py @@ -0,0 +1,340 @@ +# -------------------------------------------------------- +# Fast R-CNN +# Copyright (c) 2015 Microsoft +# Licensed under The MIT License [see LICENSE for details] +# Written by Ross Girshick +# -------------------------------------------------------- + +import os +from datasets.imdb import imdb +import datasets.ds_utils as ds_utils +import xml.etree.ElementTree as ET +import numpy as np +import scipy.sparse +import scipy.io as sio +import utils.cython_bbox +import cPickle +import subprocess +import uuid +from voc_eval import voc_eval +from fast_rcnn.config import cfg + +class traffic(imdb): + def __init__(self, image_set, year, devkit_path=None): + imdb.__init__(self, 'traffic_' + year + '_' + image_set) + self._year = year + self._image_set = image_set + self._data_path = os.path.join(cfg.DATA_DIR, 'traffic') + self._classes = ('__background__', # always index 0 + 'cyclist', 'bus', 'auto', 'car', 'hgv', 'lgv', + 'motorbike', 'pedestrian') + self._class_to_ind = dict(zip(self.classes, xrange(self.num_classes))) + self._image_ext = '.jpg' + self._image_index = self._load_image_set_index() + # Default to roidb handler + self._roidb_handler = self.selective_search_roidb + self._salt = str(uuid.uuid4()) + self._comp_id = 'comp4' + self._results_dir = os.path.join(self._data_path, 'results') + if not os.path.isdir(self._results_dir): + os.mkdir(self._results_dir) + + + assert os.path.exists(self._data_path), \ + 'Path does not exist: {}'.format(self._data_path) + + def image_path_at(self, i): + """ + Return the absolute path to image i in the image sequence. + """ + return self.image_path_from_index(self._image_index[i]) + + def image_path_from_index(self, index): + """ + Construct an image path from the image's "index" identifier. + """ + image_path = os.path.join(self._data_path, 'JPEGImages', + index + self._image_ext) + assert os.path.exists(image_path), \ + 'Path does not exist: {}'.format(image_path) + return image_path + + def _load_image_set_index(self): + """ + Load the indexes listed in this dataset's image set file. + """ + # Example path to image set file: + # self._devkit_path + /VOCdevkit2007/VOC2007/ImageSets/Main/val.txt + image_set_file = os.path.join(self._data_path, 'ImageSets', 'Main', + self._image_set + '.txt') + assert os.path.exists(image_set_file), \ + 'Path does not exist: {}'.format(image_set_file) + with open(image_set_file) as f: + image_index = [x.strip() for x in f.readlines()] + return image_index + + def gt_roidb(self): + """ + Return the database of ground-truth regions of interest. + + This function loads/saves from/to a cache file to speed up future calls. + """ + cache_file = os.path.join(self.cache_path, self.name + '_gt_roidb.pkl') + if os.path.exists(cache_file): + with open(cache_file, 'rb') as fid: + roidb = cPickle.load(fid) + print '{} gt roidb loaded from {}'.format(self.name, cache_file) + return roidb + + gt_roidb = [self._load_traffic_annotation(index) + for index in self.image_index] + with open(cache_file, 'wb') as fid: + cPickle.dump(gt_roidb, fid, cPickle.HIGHEST_PROTOCOL) + print 'wrote gt roidb to {}'.format(cache_file) + + return gt_roidb + + def selective_search_roidb(self): + """ + Return the database of selective search regions of interest. + Ground-truth ROIs are also included. + + This function loads/saves from/to a cache file to speed up future calls. + """ + cache_file = os.path.join(self.cache_path, + self.name + '_selective_search_roidb.pkl') + + if os.path.exists(cache_file): + with open(cache_file, 'rb') as fid: + roidb = cPickle.load(fid) + print '{} ss roidb loaded from {}'.format(self.name, cache_file) + return roidb + + if int(self._year) == 2007 or self._image_set != 'test': + gt_roidb = self.gt_roidb() + ss_roidb = self._load_selective_search_roidb(gt_roidb) + roidb = imdb.merge_roidbs(gt_roidb, ss_roidb) + else: + roidb = self._load_selective_search_roidb(None) + with open(cache_file, 'wb') as fid: + cPickle.dump(roidb, fid, cPickle.HIGHEST_PROTOCOL) + print 'wrote ss roidb to {}'.format(cache_file) + + return roidb + + def rpn_roidb(self): + if int(self._year) == 2007 or self._image_set != 'test': + gt_roidb = self.gt_roidb() + rpn_roidb = self._load_rpn_roidb(gt_roidb) + roidb = imdb.merge_roidbs(gt_roidb, rpn_roidb) + else: + roidb = self._load_rpn_roidb(None) + + return roidb + + def _load_rpn_roidb(self, gt_roidb): + filename = self.config['rpn_file'] + print 'loading {}'.format(filename) + assert os.path.exists(filename), \ + 'rpn data not found at: {}'.format(filename) + with open(filename, 'rb') as f: + box_list = cPickle.load(f) + return self.create_roidb_from_box_list(box_list, gt_roidb) + + def _load_selective_search_roidb(self, gt_roidb): + filename = os.path.abspath(os.path.join(cfg.DATA_DIR, + 'selective_search_data', + self.name + '.mat')) + assert os.path.exists(filename), \ + 'Selective search data not found at: {}'.format(filename) + raw_data = sio.loadmat(filename)['boxes'].ravel() + + box_list = [] + for i in xrange(raw_data.shape[0]): + boxes = raw_data[i][:, (1, 0, 3, 2)] - 1 + keep = ds_utils.unique_boxes(boxes) + boxes = boxes[keep, :] + keep = ds_utils.filter_small_boxes(boxes, self.config['min_size']) + boxes = boxes[keep, :] + box_list.append(boxes) + + return self.create_roidb_from_box_list(box_list, gt_roidb) + + def _load_traffic_annotation(self, index): + """ + Load image and bounding boxes info from XML file in the PASCAL VOC + format. + """ + filename = os.path.join(self._data_path, 'Annotations', index + '.xml') + tree = ET.parse(filename) + objs = tree.findall('object') + + # if not self.config['use_diff']: + # Exclude the samples labeled as occluded + non_diff_objs = [] + + for obj in objs: + occ = obj.find('occluded') + if occ is None: + continue + + if occ.text == 'no': + non_diff_objs.append(obj) + + # non_diff_objs = [ + # obj for obj in objs if obj.find('occluded').text == 'yes'] + if len(non_diff_objs) != len(objs): + print 'Removed {} occluded objects'.format( + len(objs) - len(non_diff_objs)) + objs = non_diff_objs + num_objs = len(objs) + + boxes = np.zeros((num_objs, 4), dtype=np.uint16) + gt_classes = np.zeros((num_objs), dtype=np.int32) + overlaps = np.zeros((num_objs, self.num_classes), dtype=np.float32) + # "Seg" area for pascal is just the box area + seg_areas = np.zeros((num_objs), dtype=np.float32) + + # Load object bounding boxes into a data frame. + for ix, obj in enumerate(objs): + bbox = obj.find('bndbox') + # Make pixel indexes 0-based + x1 = float(bbox.find('xmin').text) - 1 + y1 = float(bbox.find('ymin').text) - 1 + x2 = float(bbox.find('xmax').text) - 1 + y2 = float(bbox.find('ymax').text) - 1 + cls = self._class_to_ind[obj.find('name').text.lower().strip()] + boxes[ix, :] = [x1, y1, x2, y2] + gt_classes[ix] = cls + overlaps[ix, cls] = 1.0 + seg_areas[ix] = (x2 - x1 + 1) * (y2 - y1 + 1) + + overlaps = scipy.sparse.csr_matrix(overlaps) + + return {'boxes' : boxes, + 'gt_classes': gt_classes, + 'gt_overlaps' : overlaps, + 'flipped' : False, + 'seg_areas' : seg_areas} + + def _get_comp_id(self): + comp_id = (self._comp_id + '_' + self._salt if self.config['use_salt'] + else self._comp_id) + return comp_id + + def _get_voc_results_file_template(self): + # VOCdevkit/results/VOC2007/Main/_det_test_aeroplane.txt + filename = self._get_comp_id() + '_det_' + self._image_set + '_{:s}.txt' + path = os.path.join( + self._devkit_path, + 'results', + 'VOC' + self._year, + 'Main', + filename) + return path + + def _get_results_file_template(self): + return os.path.join(self._results_dir, '{}.txt') + + def _write_voc_results_file(self, all_boxes): + for cls_ind, cls in enumerate(self.classes): + if cls == '__background__': + continue + print 'Writing {} results file'.format(cls) + filename = self._get_results_file_template().format(cls) + with open(filename, 'wt') as f: + for im_ind, index in enumerate(self.image_index): + dets = all_boxes[cls_ind][im_ind] + if dets == []: + continue + # the VOCdevkit expects 1-based indices + for k in xrange(dets.shape[0]): + f.write('{:s} {:.3f} {:.1f} {:.1f} {:.1f} {:.1f}\n'. + format(index, dets[k, -1], + dets[k, 0] + 1, dets[k, 1] + 1, + dets[k, 2] + 1, dets[k, 3] + 1)) + + def _do_python_eval(self, output_dir = 'output'): + annopath = os.path.join( + self._data_path, + 'Annotations', + '{:s}.xml') + imagesetfile = os.path.join( + self._data_path, + 'ImageSets', + 'Main', + self._image_set + '.txt') + cachedir = os.path.join(self.cache_path, 'annotations_cache') + aps = [] + # The PASCAL VOC metric changed in 2010 + use_07_metric = True if int(self._year) < 2010 else False + print 'VOC07 metric? ' + ('Yes' if use_07_metric else 'No') + if not os.path.isdir(output_dir): + os.mkdir(output_dir) + for i, cls in enumerate(self._classes): + if cls == '__background__': + continue + filename = self._get_results_file_template().format(cls) + rec, prec, ap = voc_eval( + filename, annopath, imagesetfile, cls, cachedir, ovthresh=0.5, + use_07_metric=use_07_metric) + aps += [ap] + print('AP for {} = {:.4f}'.format(cls, ap)) + with open(os.path.join(output_dir, cls + '_pr.pkl'), 'w') as f: + cPickle.dump({'rec': rec, 'prec': prec, 'ap': ap}, f) + print('Mean AP = {:.4f}'.format(np.mean(aps))) + print('~~~~~~~~') + print('Results:') + for ap in aps: + print('{:.3f}'.format(ap)) + print('{:.3f}'.format(np.mean(aps))) + print('~~~~~~~~') + print('') + print('--------------------------------------------------------------') + print('Results computed with the **unofficial** Python eval code.') + print('Results should be very close to the official MATLAB eval code.') + print('Recompute with `./tools/reval.py --matlab ...` for your paper.') + print('-- Thanks, The Management') + print('--------------------------------------------------------------') + + def _do_matlab_eval(self, output_dir='output'): + print '-----------------------------------------------------' + print 'Computing results with the official MATLAB eval code.' + print '-----------------------------------------------------' + path = os.path.join(cfg.ROOT_DIR, 'lib', 'datasets', + 'VOCdevkit-matlab-wrapper') + cmd = 'cd {} && '.format(path) + cmd += '{:s} -nodisplay -nodesktop '.format(cfg.MATLAB) + cmd += '-r "dbstop if error; ' + cmd += 'voc_eval(\'{:s}\',\'{:s}\',\'{:s}\',\'{:s}\'); quit;"' \ + .format(self._devkit_path, self._get_comp_id(), + self._image_set, output_dir) + print('Running:\n{}'.format(cmd)) + status = subprocess.call(cmd, shell=True) + + def evaluate_detections(self, all_boxes, output_dir): + self._write_voc_results_file(all_boxes) + self._do_python_eval(output_dir) + # if self.config['matlab_eval']: + # self._do_matlab_eval(output_dir) + # if self.config['cleanup']: + + for cls in self._classes: + if cls == '__background__': + continue + filename = self._get_results_file_template().format(cls) + os.remove(filename) + + def competition_mode(self, on): + if on: + self.config['use_salt'] = False + self.config['cleanup'] = False + else: + self.config['use_salt'] = True + self.config['cleanup'] = True + +if __name__ == '__main__': + from datasets.pascal_voc import pascal_voc + d = pascal_voc('trainval', '2007') + res = d.roidb + from IPython import embed; embed() diff --git a/lib/datasets/voc_eval.py b/lib/datasets/voc_eval.py index 8d0a83076..3757e6b20 100644 --- a/lib/datasets/voc_eval.py +++ b/lib/datasets/voc_eval.py @@ -16,14 +16,14 @@ def parse_rec(filename): for obj in tree.findall('object'): obj_struct = {} obj_struct['name'] = obj.find('name').text - obj_struct['pose'] = obj.find('pose').text - obj_struct['truncated'] = int(obj.find('truncated').text) - obj_struct['difficult'] = int(obj.find('difficult').text) + # obj_struct['pose'] = obj.find('pose').text + # obj_struct['truncated'] = int(obj.find('truncated').text) + # obj_struct['difficult'] = int(obj.find('difficult').text) bbox = obj.find('bndbox') - obj_struct['bbox'] = [int(bbox.find('xmin').text), - int(bbox.find('ymin').text), - int(bbox.find('xmax').text), - int(bbox.find('ymax').text)] + obj_struct['bbox'] = [int(float(bbox.find('xmin').text)), + int(float(bbox.find('ymin').text)), + int(float(bbox.find('xmax').text)), + int(float(bbox.find('ymax').text))] objects.append(obj_struct) return objects @@ -125,13 +125,17 @@ def voc_eval(detpath, for imagename in imagenames: R = [obj for obj in recs[imagename] if obj['name'] == classname] bbox = np.array([x['bbox'] for x in R]) - difficult = np.array([x['difficult'] for x in R]).astype(np.bool) + # difficult = np.array([x['difficult'] for x in R]).astype(np.bool) det = [False] * len(R) - npos = npos + sum(~difficult) + # npos = npos + sum(~difficult) + npos = npos + len(R) class_recs[imagename] = {'bbox': bbox, - 'difficult': difficult, + # 'difficult': difficult, 'det': det} + # print 'Ground truth objects for {}'.format(classname) + # print class_recs + # read dets detfile = detpath.format(classname) with open(detfile, 'r') as f: @@ -179,12 +183,12 @@ def voc_eval(detpath, jmax = np.argmax(overlaps) if ovmax > ovthresh: - if not R['difficult'][jmax]: - if not R['det'][jmax]: - tp[d] = 1. - R['det'][jmax] = 1 - else: - fp[d] = 1. + # if not R['difficult'][jmax]: + if not R['det'][jmax]: + tp[d] = 1. + R['det'][jmax] = 1 + else: + fp[d] = 1. else: fp[d] = 1. @@ -197,4 +201,6 @@ def voc_eval(detpath, prec = tp / np.maximum(tp + fp, np.finfo(np.float64).eps) ap = voc_ap(rec, prec, use_07_metric) + # print '{}, TP: {}'.format(classname, tp) + # print '{}, FP: {}'.format(classname, fp) return rec, prec, ap diff --git a/lib/setup.py b/lib/setup.py index 0f4615f70..d84fff99a 100644 --- a/lib/setup.py +++ b/lib/setup.py @@ -132,7 +132,7 @@ def build_extensions(self): # we're only going to use certain compiler args with nvcc and not with # gcc the implementation of this trick is in customize_compiler() below extra_compile_args={'gcc': ["-Wno-unused-function"], - 'nvcc': ['-arch=sm_35', + 'nvcc': ['-arch=sm_30', '--ptxas-options=-v', '-c', '--compiler-options',