From 0c9053733c8c2d8309a776376cc4019e9ed01698 Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Tue, 23 Aug 2022 14:59:13 +0100 Subject: [PATCH 01/54] Update train.py --- detectree2/models/train.py | 532 +++++++++++++++++++++++-------------- 1 file changed, 333 insertions(+), 199 deletions(-) diff --git a/detectree2/models/train.py b/detectree2/models/train.py index 3686ec6c..dc56f171 100644 --- a/detectree2/models/train.py +++ b/detectree2/models/train.py @@ -32,238 +32,372 @@ class LossEvalHook(HookBase): - """Do inference and get the loss metric. - + """Do inference and get the loss metric Class to: - Do inference of dataset like an Evaluator does - Get the loss metric like the trainer does https://github.com/facebookresearch/detectron2/blob/master/detectron2/evaluation/evaluator.py https://github.com/facebookresearch/detectron2/blob/master/detectron2/engine/train_loop.py See https://gist.github.com/ortegatron/c0dad15e49c2b74de8bb09a5615d9f6b - Attributes: model: - period: - data_loader: - patience: number of evaluation periods to wait for improvement + period + data loader + """ + """The algorithm of early-stopping is from of Goodfellow section 7.8. + The main calculation of early_stopping is in after_step and + then the best weight recorded is loaded in the current model """ - def __init__(self, eval_period, model, data_loader, patience): - """Inits LossEvalHook.""" - self._model = model - self._period = eval_period - self._data_loader = data_loader - self.patience = patience - self.iter = 0 - self.max_ap = 0 - self.best_iter = 0 - - def _do_loss_eval(self): - """Copying inference_on_dataset from evaluator.py. - - Returns: - _type_: _description_ - """ - total = len(self._data_loader) - num_warmup = min(5, total - 1) + def __init__(self, eval_period, model, data_loader, patience, out_dir): + self._model = model + self._period = eval_period + self._data_loader = data_loader + self.patience = patience + self.iter = 0 + self.max_value = 0 + self.best_iter = 0 + #self.checkpointer = DetectionCheckpointer(self._model, save_dir=out_dir) + + def _do_loss_eval(self): + total = len(self._data_loader) + num_warmup = min(5, total - 1) + + start_time = time.perf_counter() + total_compute_time = 0 + losses = [] + for idx, inputs in enumerate(self._data_loader): + if idx == num_warmup: start_time = time.perf_counter() total_compute_time = 0 - losses = [] - for idx, inputs in enumerate(self._data_loader): - if idx == num_warmup: - start_time = time.perf_counter() - total_compute_time = 0 - start_compute_time = time.perf_counter() - if torch.cuda.is_available(): - torch.cuda.synchronize() - total_compute_time += time.perf_counter() - start_compute_time - iters_after_start = idx + 1 - num_warmup * int(idx >= num_warmup) - seconds_per_img = total_compute_time / iters_after_start - if idx >= num_warmup * 2 or seconds_per_img > 5: - total_seconds_per_img = (time.perf_counter() - - start_time) / iters_after_start - eta = datetime.timedelta(seconds=int(total_seconds_per_img - * (total - idx - 1))) - log_every_n_seconds( - logging.INFO, - "Loss on Validation done {}/{}. {:.4f} s / img. ETA={}". - format(idx + 1, total, seconds_per_img, str(eta)), - n=5, - ) - loss_batch = self._get_loss(inputs) - losses.append(loss_batch) - mean_loss = np.mean(losses) - # print(self.trainer.cfg.DATASETS.TEST) - # Combine the AP50s of the different datasets - if len(self.trainer.cfg.DATASETS.TEST) > 1: - APs = [] - for dataset in self.trainer.cfg.DATASETS.TEST: - APs.append( - self.trainer.test( - self.trainer.cfg, - self.trainer.model)[dataset]['segm']['AP50']) - AP = sum(APs) / len(APs) - else: - AP = self.trainer.test(self.trainer.cfg, - self.trainer.model)['segm']['AP50'] - print("Av. AP50 =", AP) - self.trainer.APs.append(AP) - self.trainer.storage.put_scalar("validation_loss", mean_loss) - self.trainer.storage.put_scalar("validation_ap", AP) - comm.synchronize() - - return losses - - def _get_loss(self, data): - """Calculate loss in train_loop. - + start_compute_time = time.perf_counter() + if torch.cuda.is_available(): + torch.cuda.synchronize() + total_compute_time += time.perf_counter() - start_compute_time + iters_after_start = idx + 1 - num_warmup * int(idx >= num_warmup) + seconds_per_img = total_compute_time / iters_after_start + if idx >= num_warmup * 2 or seconds_per_img > 5: + total_seconds_per_img = (time.perf_counter() - + start_time) / iters_after_start + eta = datetime.timedelta(seconds=int(total_seconds_per_img * + (total - idx - 1))) + log_every_n_seconds( + logging.INFO, + "Loss on Validation done {}/{}. {:.4f} s / img. ETA={}".format( + idx + 1, total, seconds_per_img, str(eta)), + n=5, + ) + loss_batch = self._get_loss(inputs) + losses.append(loss_batch) + mean_loss = np.mean(losses) + self.trainer.storage.put_scalar("validation_loss", mean_loss, smoothing_hint = False) + + comm.synchronize() + + #return losses + def _get_loss(self, data): + """How loss is calculated on train_loop Args: data (_type_): _description_ - Returns: _type_: _description_ """ - metrics_dict = self._model(data) - metrics_dict = { - k: - v.detach().cpu().item() if isinstance(v, torch.Tensor) else float(v) - for k, v in metrics_dict.items() - } - total_losses_reduced = sum(loss for loss in metrics_dict.values()) - return total_losses_reduced - - def after_step(self): - next_iter = self.trainer.iter + 1 - is_final = next_iter == self.trainer.max_iter - if is_final or (self._period > 0 and next_iter % self._period == 0): - self._do_loss_eval() - if self.max_ap < self.trainer.APs[-1]: - self.iter = 0 - self.max_ap = self.trainer.APs[-1] - self.trainer.checkpointer.save('model_' - + str(len(self.trainer.APs))) - self.best_iter = self.trainer.iter - else: - self.iter += 1 - if self.iter == self.patience: - self.trainer.early_stop = True - print("Early stopping occurs in iter {}, max ap is {}".format( - self.best_iter, self.max_ap)) - self.trainer.storage.put_scalars(timetest=12) - - def after_train(self): - # Select the model with the best AP50 - index = self.trainer.APs.index(max(self.trainer.APs)) + 1 - self.trainer.checkpointer.load(self.trainer.cfg.OUTPUT_DIR + '/model_' - + str(index) + '.pth') - + # + metrics_dict = self._model(data) + metrics_dict = { + k: v.detach().cpu().item() if isinstance(v, torch.Tensor) else float(v) + for k, v in metrics_dict.items() + } + total_losses_reduced = sum(loss for loss in metrics_dict.values()) + return total_losses_reduced + + + '''early stop see of goodfellow''' + def after_step(self): + next_iter = self.trainer.iter + 1 + is_final = next_iter == self.trainer.max_iter + if is_final or (self._period > 0 and next_iter % self._period == 0): + if len(self.trainer.cfg.DATASETS.TEST) > 1: + APs = [] + for dataset in self.trainer.cfg.DATASETS.TEST: + APs.append( + self.trainer.test( + self.trainer.cfg, + self.trainer.model)[dataset]['segm']['AP50']) + AP = sum(APs) / len(APs) + else: + AP = self.trainer.test(self.trainer.cfg, self.trainer.model)['segm']['AP50'] + print("Av. AP50 =", AP) + self.trainer.values.append(AP) + self.trainer.storage.put_scalar("validation_AP", AP, smoothing_hint = False) + if self.trainer.metrix == 'AP50': + if len(self.trainer.cfg.DATASETS.TEST) > 1: + APs = [] + for dataset in self.trainer.cfg.DATASETS.TRAIN: + APs.append( + self.trainer.test_train( + self.trainer.cfg, + self.trainer.model)[dataset]['segm']['AP50']) + AP = sum(APs) / len(APs) + else: + AP = self.trainer.test(self.trainer.cfg, self.trainer.model)['segm']['AP50'] + self.trainer.storage.put_scalar("training_AP", AP, smoothing_hint = False) + if self.trainer.metrix == 'loss': + self._do_loss_eval() + else: + if len(self.trainer.cfg.DATASETS.TEST) > 1: + APs = [] + for dataset in self.trainer.cfg.DATASETS.TRAIN: + APs.append( + self.trainer.test_train( + self.trainer.cfg, + self.trainer.model)[dataset]['segm']['AP50']) + AP = sum(APs) / len(APs) + else: + AP = self.trainer.test(self.trainer.cfg, self.trainer.model)['segm']['AP50'] + self.trainer.storage.put_scalar("training_AP", AP, smoothing_hint = False) + loss = self._do_loss_eval() + if self.max_value < self.trainer.values[-1]: + self.iter = 0 + self.max_value = self.trainer.values[-1] + #self.checkpointer.save('model_' + str(len(self.trainer.values))) + torch.save(self._model.state_dict(), self.trainer.cfg.OUTPUT_DIR + '/Model_' + str(len(self.trainer.values)) + '.pth') + self.best_iter = self.trainer.iter + else: + self.iter += 1 + if self.iter == self.patience: + self.trainer.early_stop = True + print("Early stopping occurs in iter {}, max ap is {}".format(self.best_iter, self.max_value)) + self.trainer.storage.put_scalars(timetest=12) + + def after_train(self): + print('train done !!!') + if len(self.trainer.values) != 0: + index = self.trainer.values.index(max(self.trainer.values)) + 1 + print(self.trainer.early_stop,"best model is", index) + trainer.cfg.MODEL.WEIGHTS = self.trainer.cfg.OUTPUT_DIR + '/Model_' + str(index) + '.pth' + else: + print('train fails') # See https://jss367.github.io/data-augmentation-in-detectron2.html for data augmentation advice class MyTrainer(DefaultTrainer): - """Summary. - + """_summary_ Args: DefaultTrainer (_type_): _description_ - Returns: _type_: _description_ """ + # add a judge on if early-stopping in train function + # train is inherited from TrainerBase https://detectron2.readthedocs.io/en/latest/_modules/detectron2/engine/train_loop.html + def __init__(self, cfg, patience = 5, training_metrix = 'loss'): + self.patience = patience + self.metrix = training_metrix + super().__init__(cfg) + def train(self): + """ + Run training. - def __init__(self, cfg, patience): - self.patience = patience - super().__init__(cfg) - - def train(self): - """Run training. + Returns: + OrderedDict of results, if evaluation is enabled. Otherwise None. + """ + """ + Args: + start_iter, max_iter (int): See docs above + """ + start_iter = self.start_iter + max_iter = self.max_iter + logger = logging.getLogger(__name__) + logger.info("Starting training from iteration {}".format(start_iter)) + + self.iter = self.start_iter = start_iter + self.max_iter = max_iter + self.early_stop = False + self.values = [] + self.reweight = False ### used to decide when to increase the weight of classification loss + + with EventStorage(start_iter) as self.storage: + try: + self.before_train() + for self.iter in range(start_iter, max_iter): + self.before_step() + self.run_step() + self.after_step() + if self.early_stop: + break + # self.iter == max_iter can be used by `after_train` to + # tell whether the training successfully finished or failed + # due to exceptions. + self.iter += 1 + except Exception: + logger.exception("Exception during training:") + raise + finally: + self.after_train() + if len(self.cfg.TEST.EXPECTED_RESULTS) and comm.is_main_process(): + assert hasattr( + self, "_last_eval_results" + ), "No evaluation results obtained during training!" + verify_results(self.cfg, self._last_eval_results) + return self._last_eval_results + + def run_step(self): + self._trainer.iter = self.iter + """ + Implement the standard training logic described above. + """ + assert self._trainer.model.training, "[SimpleTrainer] model was changed to eval mode!" + start = time.perf_counter() + """ + If you want to do something with the data, you can wrap the dataloader. + """ + data = next(self._trainer._data_loader_iter) + data_time = time.perf_counter() - start - Args: - start_iter, max_iter (int): See docs above + """ + If you want to do something with the losses, you can wrap the model. + """ + loss_dict = self._trainer.model(data) + if isinstance(loss_dict, torch.Tensor): + losses = loss_dict + loss_dict = {"total_loss": loss_dict} + else: + # loss_dict['cls'] = torch.tensor(0) + # loss_dict['loss_rpn_cls'] = torch.tensor(0) + # if self.iter > 1000: + # self.reweight = True + # if self.reweight: + # loss_dict['loss_mask'] *= 4 + loss_dict['loss_mask'] *= 2 + losses = sum(loss_dict.values()) - Returns: - OrderedDict of results, if evaluation is enabled. Otherwise None. - """ + """ + If you need to accumulate gradients or do something similar, you can + wrap the optimizer with your custom `zero_grad()` method. + """ + self.optimizer.zero_grad() + losses.backward() - start_iter = self.start_iter - max_iter = self.max_iter - logger = logging.getLogger(__name__) - logger.info("Starting training from iteration {}".format(start_iter)) - - self.iter = self.start_iter = start_iter - self.max_iter = max_iter - self.early_stop = False - self.APs = [] - - with EventStorage(start_iter) as self.storage: - try: - self.before_train() - for self.iter in range(start_iter, max_iter): - self.before_step() - self.run_step() - self.after_step() - if self.early_stop: - break - # self.iter == max_iter can be used by `after_train` to - # tell whether the training successfully finished or failed - # due to exceptions. - self.iter += 1 - except Exception: - logger.exception("Exception during training:") - raise - finally: - self.after_train() - if len(self.cfg.TEST.EXPECTED_RESULTS) and comm.is_main_process(): - assert hasattr(self, "_last_eval_results" - ), "No evaluation results obtained during training!" - verify_results(self.cfg, self._last_eval_results) - return self._last_eval_results - - @classmethod - def build_evaluator(cls, cfg, dataset_name, output_folder=None): - if output_folder is None: - os.makedirs("eval_2", exist_ok=True) - output_folder = "eval_2" - return COCOEvaluator(dataset_name, cfg, True, output_folder) - - def build_hooks(self): - hooks = super().build_hooks() - hooks.insert( - -1, - LossEvalHook( - self.cfg.TEST.EVAL_PERIOD, - self.model, - build_detection_test_loader(self.cfg, self.cfg.DATASETS.TEST, - DatasetMapper(self.cfg, True)), - self.patience, - ), - ) - return hooks + self._trainer._write_metrics(loss_dict, data_time) - def build_train_loader(cls, cfg): - """Summary. - Args: - cfg (_type_): _description_ + """ + If you need gradient clipping/scaling or other processing, you can + wrap the optimizer with your custom `step()` method. But it is + suboptimal as explained in https://arxiv.org/abs/2006.15704 Sec 3.2.4 + """ + self._trainer.optimizer.step() + + + @classmethod + def build_evaluator(cls, cfg, dataset_name, output_folder=None): + if output_folder is None: + os.makedirs("eval_2", exist_ok=True) + output_folder = "eval_2" + return COCOEvaluator(dataset_name, cfg, output_dir = output_folder) + + + def build_hooks(self): + hooks = super().build_hooks() + hooks.insert( + -1, + LossEvalHook( + self.cfg.TEST.EVAL_PERIOD, + self.model, + build_detection_test_loader(self.cfg, self.cfg.DATASETS.TEST[0], + DatasetMapper(self.cfg, True)), + self.patience, + self.cfg.OUTPUT_DIR, + ), + ) + return hooks - Returns: - _type_: _description_ - """ - return build_detection_train_loader( + def build_train_loader(cls, cfg): + """_summary_ + Args: + cfg (_type_): _description_ + Returns: + _type_: _description_ + """ + for i, datas in enumerate(DatasetCatalog.get(cfg.DATASETS.TRAIN[0])): + location = datas['file_name'] + size = cv2.imread(location).shape[0] + break + return build_detection_train_loader( + cfg, + mapper=DatasetMapper( cfg, - mapper=DatasetMapper( - cfg, - is_train=True, - augmentations=[ - T.Resize((800, 800)), # is this necessary/helpful? - T.RandomBrightness(0.8, 1.8), - T.RandomContrast(0.6, 1.3), - T.RandomSaturation(0.8, 1.4), - T.RandomRotation(angle=[90, 90], expand=False), - T.RandomLighting(0.7), - T.RandomFlip(prob=0.4, horizontal=True, vertical=False), - T.RandomFlip(prob=0.4, horizontal=False, vertical=True), - ], - ), - ) + is_train=True, + augmentations=[ + #T.Resize((800, 800)), + #T.Resize((random_size, random_size)), + T.ResizeScale(0.6, 1.4, size, size), + #T.RandomCrop('relative',(0.5,0.5)), + T.RandomBrightness(0.8, 1.8), + T.RandomContrast(0.6, 1.3), + T.RandomSaturation(0.8, 1.4), + T.RandomRotation(angle=[90, 90], expand=False), + T.RandomLighting(0.7), + T.RandomFlip(prob=0.4, horizontal=True, vertical=False), + T.RandomFlip(prob=0.4, horizontal=False, vertical=True), + ], + ), + ) + + @classmethod + def test_train(cls, cfg, model, evaluators=None): + """ + Evaluate the given model. The given model is expected to already contain + weights to evaluate. + + Args: + cfg (CfgNode): + model (nn.Module): + evaluators (list[DatasetEvaluator] or None): if None, will call + :meth:`build_evaluator`. Otherwise, must have the same length as + ``cfg.DATASETS.TEST``. + + Returns: + dict: a dict of result metrics + """ + logger = logging.getLogger(__name__) + if isinstance(evaluators, DatasetEvaluator): + evaluators = [evaluators] + if evaluators is not None: + assert len(cfg.DATASETS.TRAIN) == len(evaluators), "{} != {}".format( + len(cfg.DATASETS.TRAIN), len(evaluators) + ) + + results = OrderedDict() + for idx, dataset_name in enumerate(cfg.DATASETS.TRAIN): + data_loader = cls.build_test_loader(cfg, dataset_name) + # When evaluators are passed in as arguments, + # implicitly assume that evaluators can be created before data_loader. + if evaluators is not None: + evaluator = evaluators[idx] + else: + try: + evaluator = cls.build_evaluator(cfg, dataset_name) + except NotImplementedError: + logger.warn( + "No evaluator found. Use `DefaultTrainer.test(evaluators=)`, " + "or implement its `build_evaluator` method." + ) + results[dataset_name] = {} + continue + results_i = inference_on_dataset(model, data_loader, evaluator) + results[dataset_name] = results_i + if comm.is_main_process(): + assert isinstance( + results_i, dict + ), "Evaluator must return a dict on the main process. Got {} instead.".format( + results_i + ) + logger.info("Evaluation results for {} in csv format:".format(dataset_name)) + print_csv_format(results_i) + + if len(results) == 1: + results = list(results.values())[0] + return results def get_tree_dicts(directory: str, classes: List[str] = None) -> List[Dict]: From 68389f725452e32c179ac0b27848e3dd2d415fef Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Tue, 23 Aug 2022 15:54:36 +0100 Subject: [PATCH 02/54] Update MyPredictor --- detectree2/models/predict.py | 251 +++++++++++++++++++++++++++++++++++ 1 file changed, 251 insertions(+) diff --git a/detectree2/models/predict.py b/detectree2/models/predict.py index 52c64877..55a5f10f 100644 --- a/detectree2/models/predict.py +++ b/detectree2/models/predict.py @@ -12,10 +12,137 @@ from shapely.geometry import box, shape from detectree2.models.train import get_filenames +from detectron2.data.build import DatasetMapper +import time +from PIL import Image +import pycocotools.mask as mask_util # Code to convert RLE data from the output instances into Polygons, a small about of info is lost but is fine. # https://github.com/hazirbas/coco-json-converter/blob/master/generate_coco_json.py <-- found here +def reproject_to_geojson_spatially(data, + output_fold=None, + pred_fold=None, + EPSG="32650"): # noqa:N803 + """Reprojects the coordinates back so the crowns can be overlaid with the original tif file of the entire region. + Takes a json and changes it to a geojson so it can overlay with crowns. + Another copy is produced to overlay with PNGs. + """ + + Path(output_fold).mkdir(parents=True, exist_ok=True) + entries = os.listdir(pred_fold) + #print(entries) + # scale to deal with the resolution + scalingx = data.transform[0] + scalingy = -data.transform[4] + + for file in entries: + if ".json" in file: + # create a geofile for each tile --> the EPSG value might need to be changed. + geofile = { + "type": "FeatureCollection", + "crs": { + "type": "name", + "properties": { + "name": "urn:ogc:def:crs:EPSG::" + EPSG + } + }, + "features": [] + } + + # create a dictionary for each file to store data used multiple times + img_dict = {} + img_dict["filename"] = file + #print(img_dict["filename"]) + + file_mins = file.replace(".json", "") + file_mins_split = file_mins.split("_") + minx = int(file_mins_split[-4]) + miny = int(file_mins_split[-3]) + tile_height = int(file_mins_split[-2]) + buffer = int(file_mins_split[-1]) + height = (tile_height + 2 * buffer) / scalingx + + # update the image dictionary to store all information cleanly + img_dict.update({ + "minx": minx, + "miny": miny, + "height": height, + "buffer": buffer + }) + # print("Img dict:", img_dict) + + # load the json file we need to convert into a geojson + with open(pred_fold + img_dict["filename"]) as prediction_file: + datajson = json.load(prediction_file) + # print("data_json:",datajson) + + # json file is formated as a list of segmentation polygons so cycle through each one + for crown_data in datajson: + # just a check that the crown image is correct + if str(minx) + '_' + str(miny) in crown_data["image_id"]: + crown = crown_data["segmentation"] + confidence_score = crown_data['score'] + + # changing the coords from RLE format so can be read as numbers, here the numbers are + # integers so a bit of info on position is lost + mask_of_coords = mask_util.decode(crown) + crown_coords = polygon_from_mask(mask_of_coords) + moved_coords = [] + + # coords from json are in a list of [x1, y1, x2, y2,... ] so convert them to [[x1, y1], ...] + # format and at the same time rescale them so they are in the correct position for QGIS + for c in range(0, len(crown_coords), 2): + x_coord = crown_coords[c] + y_coord = crown_coords[c + 1] + + # print("ycoord:", y_coord) + # print("height:", height) + + # rescaling the coords depending on where the tile is in the original image, note the + # correction factors have been manually added as outputs did not line up with predictions + # from training script + if minx == int(data.bounds[0]) and miny == int(data.bounds[1]): + #print("Bottom Corner") + x_coord = (x_coord) * scalingx + minx + y_coord = (height - y_coord) * scalingy + miny + elif minx == int(data.bounds[0]): + #print("Left Edge") + x_coord = (x_coord) * scalingx + minx + y_coord = (height + - y_coord) * scalingy - buffer + miny + elif miny == int(data.bounds[1]): + #print("Bottom Edge") + x_coord = (x_coord) * scalingx - buffer + minx + y_coord = (height + - y_coord) * scalingy - buffer + miny + else: + # print("Anywhere else") + x_coord = (x_coord) * scalingx - buffer + minx + y_coord = (height + - y_coord) * scalingy - buffer + miny + + moved_coords.append([x_coord, y_coord]) + + geofile["features"].append({ + "type": "Feature", + "properties": { + "Confidence score": confidence_score + }, + "geometry": { + "type": "Polygon", + "coordinates": [moved_coords] + } + }) + + # Check final form is correct - compare to a known geojson file if error appears. + # print("geofile",geofile) + + output_geo_file = output_fold + img_dict["filename"].replace( + '.json', "_" + EPSG + '_lidar.geojson') + # print("output location:", output_geo_file) + with open(output_geo_file, "w") as dest: + json.dump(geofile, dest) def polygonFromMask(maskedArr): """ @@ -185,6 +312,130 @@ def clean_crowns(crowns: gpd.GeoDataFrame): crowns_out = crowns_out.append(match) return crowns_out.reset_index() +class MyPredictor(DefaultPredictor): + def __init__(self, cfg, mode): + self.cfg = cfg.clone() # cfg can be modified by model + self.model = build_model(self.cfg) + self.model.eval() + self.mode = mode + cfg.DATASETS.TEST = ('pigs',) + if len(cfg.DATASETS.TEST): + self.metadata = MetadataCatalog.get(cfg.DATASETS.TEST[0]) + + checkpointer = DetectionCheckpointer(self.model) + checkpointer.load(cfg.MODEL.WEIGHTS) + + self.aug = self.augmentation() + + self.input_format = cfg.INPUT.FORMAT + assert self.input_format in ["RGB", "BGR"], self.input_format + + def __call__(self, original_image): + """ + Args: + original_image (np.ndarray): an image of shape (H, W, C) (in BGR order). + + Returns: + predictions (dict): + the output of the model for one image only. + See :doc:`/tutorials/models` for details about the format. + """ + with torch.no_grad(): # https://github.com/sphinx-doc/sphinx/issues/4258 + # Apply pre-processing to image. + if self.input_format == "RGB": + # whether the model expects BGR inputs or RGB + original_image = original_image[:, :, ::-1] + height, width = original_image.shape[:2] + if self.aug != None: + image = self.aug.get_transform(original_image).apply_image(original_image) + else: + image = original_image + image = torch.as_tensor(image.astype("float32").transpose(2, 0, 1)) + + inputs = {"image": image, "height": height, "width": width} + predictions = self.model([inputs])[0] + + + def _predict(self, IN_DIR, save = True): + dataset_dicts = [] + files = glob.glob(IN_DIR + "*.png") + for filename in [file for file in files]: + file = {} + filename = os.path.join(IN_DIR, filename) + file["file_name"] = filename + dataset_dicts.append(file) + + # Works out if all items in folder should be predicted on + + num_to_pred = len(dataset_dicts) + + pred_dir = IN_DIR + "predictions" + + Path(pred_dir).mkdir(parents=True, exist_ok=True) + + for data in random.sample(dataset_dicts,num_to_pred): + with torch.no_grad(): + print(data["file_name"]) + img = cv2.imread(data["file_name"]) + if self.input_format == "RGB": + # whether the model expects BGR inputs or RGB + img = img[:, :, ::-1] + height, width = img.shape[:2] + if self.aug != None: + image = self.aug.get_transform(img).apply_image(img) + else: + image = img + image = img + image = torch.as_tensor(image.astype("float32").transpose(2, 0, 1)) + + inputs = {"image": image, "height": height, "width": width} + predictions = self.model([inputs])[0] + file_name_path = data["file_name"] + file_name = os.path.basename(os.path.normpath(file_name_path)) #Strips off all slashes so just final file name left + file_name = file_name.replace("png","json") + + output_file = pred_dir + "/predictions_" + file_name + + if save: + ## Converting the predictions to json files and saving them in the specfied output file. + evaluations= instances_to_coco_json(predictions["instances"].to("cpu"),data["file_name"]) + with open(output_file, "w") as dest: + json.dump(evaluations,dest) + + + def predict(self, save=True): + + for i in range(len(self.cfg.IN_DIR)): + + self._predict(self.cfg.IN_DIR[i]) + files = glob.glob(self.cfg.IN_DIR[i] + "*.tif") + data = rasterio.open(files[0]) + reproject_to_geojson_spatially(data, self.cfg.OUT_DIR[-1], self.cfg.IN_DIR[i] + "predictions/", EPSG = "32650") + + folder = self.cfg.OUT_DIR[-1] + + crowns = stitch_crowns(folder, 1) + + crowns = clean_crowns(crowns) + + x = crowns.buffer(0.0001) + tolerance = 0.03 + simplified = x.simplify(tolerance, preserve_topology=True) + + crowns.to_file(folder + "crowns_out.gpkg") + + + def augmentation(self): + if self.mode == 'resize_fixed': + return T.ResizeShortestEdge( + [self.cfg.INPUT.MIN_SIZE_TEST, self.cfg.INPUT.MIN_SIZE_TEST], self.cfg.INPUT.MAX_SIZE_TEST) + if self.mode == 'No_resize': + return None + else: + print('No such a mode') + return T.ResizeShortestEdge( + [self.cfg.INPUT.MIN_SIZE_TEST, self.cfg.INPUT.MIN_SIZE_TEST], self.cfg.INPUT.MAX_SIZE_TEST) + if __name__ == "__main__": print("something") From 017eb7c4fa6b73229f49f78443f952ddd217c93f Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Sun, 11 Sep 2022 21:13:28 +0100 Subject: [PATCH 03/54] update train.py after intern --- detectree2/models/train.py | 207 +++++++++++++++++++++---------------- 1 file changed, 116 insertions(+), 91 deletions(-) diff --git a/detectree2/models/train.py b/detectree2/models/train.py index dc56f171..42b290b5 100644 --- a/detectree2/models/train.py +++ b/detectree2/models/train.py @@ -120,11 +120,11 @@ def after_step(self): if is_final or (self._period > 0 and next_iter % self._period == 0): if len(self.trainer.cfg.DATASETS.TEST) > 1: APs = [] - for dataset in self.trainer.cfg.DATASETS.TEST: - APs.append( - self.trainer.test( + AP_datasets = self.trainer.test( self.trainer.cfg, - self.trainer.model)[dataset]['segm']['AP50']) + self.trainer.model) + for dataset in self.trainer.cfg.DATASETS.TEST: + APs.append(AP_datasets[dataset]['segm']['AP50']) AP = sum(APs) / len(APs) else: AP = self.trainer.test(self.trainer.cfg, self.trainer.model)['segm']['AP50'] @@ -134,25 +134,25 @@ def after_step(self): if self.trainer.metrix == 'AP50': if len(self.trainer.cfg.DATASETS.TEST) > 1: APs = [] - for dataset in self.trainer.cfg.DATASETS.TRAIN: - APs.append( - self.trainer.test_train( + AP_datasets = self.trainer.test_train( self.trainer.cfg, - self.trainer.model)[dataset]['segm']['AP50']) + self.trainer.model) + for dataset in self.trainer.cfg.DATASETS.TEST: + APs.append(AP_datasets[dataset]['segm']['AP50']) AP = sum(APs) / len(APs) else: - AP = self.trainer.test(self.trainer.cfg, self.trainer.model)['segm']['AP50'] + AP = self.trainer.test_train(self.trainer.cfg, self.trainer.model)['segm']['AP50'] self.trainer.storage.put_scalar("training_AP", AP, smoothing_hint = False) - if self.trainer.metrix == 'loss': + elif self.trainer.metrix == 'loss': self._do_loss_eval() else: if len(self.trainer.cfg.DATASETS.TEST) > 1: APs = [] - for dataset in self.trainer.cfg.DATASETS.TRAIN: - APs.append( - self.trainer.test_train( + AP_datasets = self.trainer.test_train( self.trainer.cfg, - self.trainer.model)[dataset]['segm']['AP50']) + self.trainer.model) + for dataset in self.trainer.cfg.DATASETS.TRAIN: + APs.append(AP_datasets[dataset]['segm']['AP50']) AP = sum(APs) / len(APs) else: AP = self.trainer.test(self.trainer.cfg, self.trainer.model)['segm']['AP50'] @@ -241,6 +241,7 @@ def train(self): verify_results(self.cfg, self._last_eval_results) return self._last_eval_results + def run_step(self): self._trainer.iter = self.iter """ @@ -268,7 +269,7 @@ def run_step(self): # self.reweight = True # if self.reweight: # loss_dict['loss_mask'] *= 4 - loss_dict['loss_mask'] *= 2 + loss_dict['loss_mask'] *= 0.8 losses = sum(loss_dict.values()) """ @@ -287,18 +288,53 @@ def run_step(self): """ self._trainer.optimizer.step() - - @classmethod - def build_evaluator(cls, cfg, dataset_name, output_folder=None): - if output_folder is None: - os.makedirs("eval_2", exist_ok=True) - output_folder = "eval_2" - return COCOEvaluator(dataset_name, cfg, output_dir = output_folder) - - + def build_hooks(self): - hooks = super().build_hooks() - hooks.insert( + """ + Build a list of default hooks, including timing, evaluation, + checkpointing, lr scheduling, precise BN, writing events. + Returns: + list[HookBase]: + """ + cfg = self.cfg.clone() + cfg.defrost() + cfg.DATALOADER.NUM_WORKERS = 0 # save some memory and time for PreciseBN + + ret = [ + hooks.IterationTimer(), + hooks.LRScheduler(), + hooks.PreciseBN( + # Run at the same freq as (but before) evaluation. + cfg.TEST.EVAL_PERIOD, + self.model, + # Build a new data loader to not affect training + self.build_train_loader(cfg), + cfg.TEST.PRECISE_BN.NUM_ITER, + ) + if cfg.TEST.PRECISE_BN.ENABLED and get_bn_modules(self.model) + else None, + ] + + # Do PreciseBN before checkpointer, because it updates the model and need to + # be saved by checkpointer. + # This is not always the best: if checkpointing has a different frequency, + # some checkpoints may have more precise statistics than others. + if comm.is_main_process(): + ret.append(hooks.PeriodicCheckpointer(self.checkpointer, cfg.SOLVER.CHECKPOINT_PERIOD)) + + # def test_and_save_results(): + # self._last_eval_results = self.test(self.cfg, self.model) + # return self._last_eval_results + + # # Do evaluation after checkpointer, because then if it fails, + # # we can use the saved checkpoint to debug. + # ret.append(hooks.EvalHook(cfg.TEST.EVAL_PERIOD, test_and_save_results)) + + if comm.is_main_process(): + # Here the default print/log frequency of each writer is used. + # run writers in the end, so that evaluation metrics are written + ret.append(hooks.PeriodicWriter(self.build_writers(), period=20)) + ret.insert( -1, LossEvalHook( self.cfg.TEST.EVAL_PERIOD, @@ -309,7 +345,16 @@ def build_hooks(self): self.cfg.OUTPUT_DIR, ), ) - return hooks + return ret + + + @classmethod + def build_evaluator(cls, cfg, dataset_name, output_folder=None): + if output_folder is None: + os.makedirs("eval_2", exist_ok=True) + output_folder = "eval_2" + return COCOEvaluator(dataset_name, cfg, output_dir = output_folder) + def build_train_loader(cls, cfg): """_summary_ @@ -568,70 +613,50 @@ def load_json_arr(json_path): return lines -def setup_cfg(base_model: - str = "COCO-InstanceSegmentation/mask_rcnn_R_101_FPN_3x.yaml", - trains=("trees_train",), - tests=("trees_val",), - update_model=None, - workers=2, - ims_per_batch=2, - gamma=0.1, - backbone_freeze=3, - warm_iter=120, - momentum=0.9, - batch_size_per_im=1024, - base_lr=0.001, - max_iter=1000, - num_classes=1, - eval_period=100, - out_dir="/content/drive/Shareddrives/detectree2/train_outputs"): - """Set up config object - - Args: - base_model: - trains: - tests: - update_model: - workers: - ims_per_batch: - gamma: - backbone_freeze: - warm_iter: - momentum: - batch_size_per_im: - base_lr: - max_iter: - num_classes: - eval_period: - out_dir: - """ - cfg = get_cfg() - cfg.merge_from_file(model_zoo.get_config_file(base_model)) - cfg.DATASETS.TRAIN = trains - cfg.DATASETS.TEST = tests - cfg.DATALOADER.NUM_WORKERS = workers - #cfg.SOLVER.IMS_PER_BATCH = ims_per_batch - cfg.SOLVER.GAMMA = gamma - cfg.MODEL.BACKBONE.FREEZE_AT = backbone_freeze - cfg.SOLVER.WARMUP_ITERS = warm_iter - cfg.SOLVER.MOMENTUM = momentum - #cfg.MODEL.RPN.BATCH_SIZE_PER_IMAGE = batch_size_per_im - #cfg.SOLVER.WEIGHT_DECAY = 0.001 - cfg.SOLVER.BASE_LR = base_lr - cfg.OUTPUT_DIR = out_dir - os.makedirs(cfg.OUTPUT_DIR, exist_ok=True) - if update_model is not None: - cfg.MODEL.WEIGHTS = update_model - else: - cfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url(base_model) - - cfg.SOLVER.IMS_PER_BATCH = ims_per_batch - cfg.SOLVER.BASE_LR = base_lr - cfg.SOLVER.MAX_ITER = max_iter - cfg.MODEL.ROI_HEADS.NUM_CLASSES = num_classes - cfg.TEST.EVAL_PERIOD = eval_period - return cfg - +def setup_cfg( + base_model="COCO-InstanceSegmentation/mask_rcnn_R_101_FPN_3x.yaml", + trains=("trees_train",), + tests=("trees_val",), + update_model=None, + workers=2, + ims_per_batch=1, + base_lr=0.0003, + max_iter=1000, + num_classes=1, + eval_period=100, + out_dir="/content/drive/Shareddrives/detectree2/train_outputs"): + """ + To set up config object + """ + cfg = get_cfg() + cfg.merge_from_file(model_zoo.get_config_file(base_model)) + cfg.DATASETS.TRAIN = trains + cfg.DATASETS.TEST = tests + cfg.DATALOADER.NUM_WORKERS = workers + cfg.OUTPUT_DIR = out_dir + cfg.SOLVER.IMS_PER_BATCH = 2 + cfg.SOLVER.GAMMA = 0.1 + cfg.MODEL.BACKBONE.FREEZE_AT = 3 + cfg.SOLVER.WARMUP_ITERS = 120 + cfg.SOLVER.MOMENTUM = 0.9 + cfg.MODEL.RPN.BATCH_SIZE_PER_IMAGE = 128 + cfg.SOLVER.WEIGHT_DECAY = 0 + cfg.SOLVER.BASE_LR = 0.001 + cfg.betas = (0.9, 0.999) + os.makedirs(cfg.OUTPUT_DIR, exist_ok=True) + if update_model is not None: + cfg.MODEL.WEIGHTS = update_model # DOESN'T WORK + else: + cfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url(base_model) + + cfg.SOLVER.IMS_PER_BATCH = ims_per_batch + cfg.SOLVER.BASE_LR = base_lr + cfg.SOLVER.MAX_ITER = max_iter + cfg.MODEL.ROI_HEADS.NUM_CLASSES = num_classes + cfg.TEST.EVAL_PERIOD = eval_period + cfg.MODEL.BACKBONE.FREEZE_AT = 2 + cfg.MODEL.ROI_BOX_HEAD.BBOX_REG_LOSS_TYPE = 'diou' + return cfg def predictions_on_data(directory=None, predictor=DefaultTrainer, From edc8c3ccdd0a2d8af918128d59db0e209adff99f Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Sun, 11 Sep 2022 23:00:51 +0100 Subject: [PATCH 04/54] Update train.py --- detectree2/models/train.py | 1 + 1 file changed, 1 insertion(+) diff --git a/detectree2/models/train.py b/detectree2/models/train.py index 42b290b5..12b23fbd 100644 --- a/detectree2/models/train.py +++ b/detectree2/models/train.py @@ -15,6 +15,7 @@ import torch from detectron2 import model_zoo from detectron2.checkpoint import DetectionCheckpointer +from detectron2.engine import DefaultPredictor, DefaultTrainer, hooks from detectron2.config import get_cfg from detectron2.data import (DatasetCatalog, DatasetMapper, MetadataCatalog, build_detection_test_loader, From 37bbbc93d762412bbb934b2bef928c4c16e07528 Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Sun, 11 Sep 2022 23:28:35 +0100 Subject: [PATCH 05/54] Update train.py --- detectree2/models/train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/detectree2/models/train.py b/detectree2/models/train.py index 12b23fbd..a6588476 100644 --- a/detectree2/models/train.py +++ b/detectree2/models/train.py @@ -177,7 +177,7 @@ def after_train(self): if len(self.trainer.values) != 0: index = self.trainer.values.index(max(self.trainer.values)) + 1 print(self.trainer.early_stop,"best model is", index) - trainer.cfg.MODEL.WEIGHTS = self.trainer.cfg.OUTPUT_DIR + '/Model_' + str(index) + '.pth' + self.trainer.cfg.MODEL.WEIGHTS = self.trainer.cfg.OUTPUT_DIR + '/Model_' + str(index) + '.pth' else: print('train fails') From 07fd704397fcb6bbe7ecdfa031619fa37de12544 Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Tue, 25 Oct 2022 09:55:52 +0100 Subject: [PATCH 06/54] Create custom_rpn.py custom rpn head for custom nms which considers intersection over area and overlapping between different speices is avoided --- detectree2/models/custom_rpn.py | 177 ++++++++++++++++++++++++++++++++ 1 file changed, 177 insertions(+) create mode 100644 detectree2/models/custom_rpn.py diff --git a/detectree2/models/custom_rpn.py b/detectree2/models/custom_rpn.py new file mode 100644 index 00000000..9c2bdd92 --- /dev/null +++ b/detectree2/models/custom_rpn.py @@ -0,0 +1,177 @@ +from typing import Tuple + +import torch +import torchvision +from torch import Tensor +from torchvision.extension import _assert_has_ops +import logging +import math +from typing import List, Tuple, Union + +from detectron2.layers import batched_nms, cat +from detectron2.structures import Boxes, Instances +from typing import Dict, List, Optional, Tuple, Union +import torch.nn.functional as F +from torch import nn + +from detectron2.config import configurable +from detectron2.layers import Conv2d, ShapeSpec, cat +from detectron2.structures import Boxes, ImageList, Instances, pairwise_iou +from detectron2.utils.events import get_event_storage +from detectron2.utils.memory import retry_if_cuda_oom +from detectron2.utils.registry import Registry +from detectron2.modeling.proposal_generator.rpn import RPN, build_rpn_head + +from detectron2.modeling.anchor_generator import build_anchor_generator +from detectron2.modeling.box_regression import Box2BoxTransform, _dense_box_regression_loss +from detectron2.modeling.matcher import Matcher +from detectron2.modeling.sampling import subsample_labels +from detectron2.modeling.proposal_generator.build import PROPOSAL_GENERATOR_REGISTRY + + +@torch.jit.script_if_tracing +def move_device_like(src: torch.Tensor, dst: torch.Tensor) -> torch.Tensor: + """ + Tracing friendly way to cast tensor to another tensor's device. Device will be treated + as constant during tracing, scripting the casting process as whole can workaround this issue. + """ + return src.to(dst.device) + + +@torch.jit.script_if_tracing +def move_device_like(src: torch.Tensor, dst: torch.Tensor) -> torch.Tensor: + """ + Tracing friendly way to cast tensor to another tensor's device. Device will be treated + as constant during tracing, scripting the casting process as whole can workaround this issue. + """ + return src.to(dst.device) + + +@PROPOSAL_GENERATOR_REGISTRY.register() +class custom_RPN(RPN): + + @configurable + def __init__(self, + *, + in_features: List[str], + head: nn.Module, + anchor_generator: nn.Module, + anchor_matcher: Matcher, + box2box_transform: Box2BoxTransform, + batch_size_per_image: int, + positive_fraction: float, + pre_nms_topk: Tuple[float, float], + post_nms_topk: Tuple[float, float], + nms_thresh: float = 0.7, + nms_thresh_union: float = 0.7, + min_box_size: float = 0.0, + anchor_boundary_thresh: float = -1.0, + loss_weight: Union[float, Dict[str, float]] = 1.0, + box_reg_loss_type: str = "smooth_l1", + smooth_l1_beta: float = 0.0,): + super().__init__() + self.nms_thresh_union = nms_thresh_union + + @classmethod + def from_config(cls, cfg, input_shape: Dict[str, ShapeSpec]): + in_features = cfg.MODEL.RPN.IN_FEATURES + ret = { + "in_features": in_features, + "min_box_size": cfg.MODEL.PROPOSAL_GENERATOR.MIN_SIZE, + "nms_thresh": cfg.MODEL.RPN.NMS_THRESH, + "nms_thresh_union": cfg.nms_thresh_union, + "batch_size_per_image": cfg.MODEL.RPN.BATCH_SIZE_PER_IMAGE, + "positive_fraction": cfg.MODEL.RPN.POSITIVE_FRACTION, + "loss_weight": { + "loss_rpn_cls": cfg.MODEL.RPN.LOSS_WEIGHT, + "loss_rpn_loc": cfg.MODEL.RPN.BBOX_REG_LOSS_WEIGHT * cfg.MODEL.RPN.LOSS_WEIGHT, + }, + "anchor_boundary_thresh": cfg.MODEL.RPN.BOUNDARY_THRESH, + "box2box_transform": Box2BoxTransform(weights=cfg.MODEL.RPN.BBOX_REG_WEIGHTS), + "box_reg_loss_type": cfg.MODEL.RPN.BBOX_REG_LOSS_TYPE, + "smooth_l1_beta": cfg.MODEL.RPN.SMOOTH_L1_BETA, + } + + ret["pre_nms_topk"] = (cfg.MODEL.RPN.PRE_NMS_TOPK_TRAIN, cfg.MODEL.RPN.PRE_NMS_TOPK_TEST) + ret["post_nms_topk"] = (cfg.MODEL.RPN.POST_NMS_TOPK_TRAIN, cfg.MODEL.RPN.POST_NMS_TOPK_TEST) + + ret["anchor_generator"] = build_anchor_generator(cfg, [input_shape[f] for f in in_features]) + ret["anchor_matcher"] = Matcher( + cfg.MODEL.RPN.IOU_THRESHOLDS, cfg.MODEL.RPN.IOU_LABELS, allow_low_quality_matches=True + ) + ret["head"] = build_rpn_head(cfg, [input_shape[f] for f in in_features]) + return ret + + def find_top_rpn_proposals( + proposals: List[torch.Tensor], + pred_objectness_logits: List[torch.Tensor], + image_sizes: List[Tuple[int, int]], + nms_thresh: float, + nms_thresh_union: float, + pre_nms_topk: int, + post_nms_topk: int, + min_box_size: float, + training: bool, +): + # here the box refinement has been done when proposals are inputted + num_images = len(image_sizes) + device = ( + proposals[0].device + if torch.jit.is_scripting() + else ("cpu" if torch.jit.is_tracing() else proposals[0].device) + ) + + # 1. Select top-k anchor for every level and every image + topk_scores = [] # #lvl Tensor, each of shape N x topk + topk_proposals = [] + batch_idx = move_device_like(torch.arange(num_images, device=device), proposals[0]) + for level_id, (proposals_i, logits_i) in enumerate(zip(proposals, pred_objectness_logits)): + Hi_Wi_A = logits_i.shape[1] + if isinstance(Hi_Wi_A, torch.Tensor): # it's a tensor in tracing + num_proposals_i = torch.clamp(Hi_Wi_A, max=pre_nms_topk) + else: + num_proposals_i = min(Hi_Wi_A, pre_nms_topk) + + topk_scores_i, topk_idx = logits_i.topk(num_proposals_i, dim=1) + + # each is N x topk + topk_proposals_i = proposals_i[batch_idx[:, None], topk_idx] # N x topk x 4 + + topk_proposals.append(topk_proposals_i) + topk_scores.append(topk_scores_i) + + + # 2. Concat all levels together + topk_scores = cat(topk_scores, dim=1) + topk_proposals = cat(topk_proposals, dim=1) + + # 3. For each image, run a per-level NMS, and choose topk results. + results: List[Instances] = [] + for n, image_size in enumerate(image_sizes): + boxes = Boxes(topk_proposals[n]) + scores_per_img = topk_scores[n] + + valid_mask = torch.isfinite(boxes.tensor).all(dim=1) & torch.isfinite(scores_per_img) + if not valid_mask.all(): + if training: + raise FloatingPointError( + "Predicted boxes or scores contain Inf/NaN. Training has diverged." + ) + boxes = boxes[valid_mask] + scores_per_img = scores_per_img[valid_mask] + boxes.clip(image_size) + + # filter empty boxes + keep = boxes.nonempty(threshold=min_box_size) + if _is_tracing() or keep.sum().item() != len(boxes): + boxes, scores_per_img= boxes[keep], scores_per_img[keep] + + keep = custom_nms(boxes.tensor, scores_per_img, nms_thresh, nms_thresh_union) + + keep = keep[:post_nms_topk] # keep is already sorted + + res = Instances(image_size) + res.proposal_boxes = boxes[keep] + res.objectness_logits = scores_per_img[keep] + results.append(res) + return results From 18cc8904e5dcdb85f44d3560764d71b32586af0f Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Tue, 25 Oct 2022 09:56:49 +0100 Subject: [PATCH 07/54] Update custom_rpn.py --- detectree2/models/custom_rpn.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/detectree2/models/custom_rpn.py b/detectree2/models/custom_rpn.py index 9c2bdd92..90833ab0 100644 --- a/detectree2/models/custom_rpn.py +++ b/detectree2/models/custom_rpn.py @@ -1,5 +1,3 @@ -from typing import Tuple - import torch import torchvision from torch import Tensor From a63e3773eea02f3f582b8b62d67ff3f1396b6fee Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Tue, 25 Oct 2022 09:58:34 +0100 Subject: [PATCH 08/54] Update custom_rpn.py --- detectree2/models/custom_rpn.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/detectree2/models/custom_rpn.py b/detectree2/models/custom_rpn.py index 90833ab0..ddbf094d 100644 --- a/detectree2/models/custom_rpn.py +++ b/detectree2/models/custom_rpn.py @@ -36,13 +36,13 @@ def move_device_like(src: torch.Tensor, dst: torch.Tensor) -> torch.Tensor: return src.to(dst.device) -@torch.jit.script_if_tracing -def move_device_like(src: torch.Tensor, dst: torch.Tensor) -> torch.Tensor: - """ - Tracing friendly way to cast tensor to another tensor's device. Device will be treated - as constant during tracing, scripting the casting process as whole can workaround this issue. - """ - return src.to(dst.device) +def _is_tracing(): + # (fixed in TORCH_VERSION >= 1.9) + if torch.jit.is_scripting(): + # https://github.com/pytorch/pytorch/issues/47379 + return False + else: + return torch.jit.is_tracing() @PROPOSAL_GENERATOR_REGISTRY.register() From b3c709f396c988f1ec121074d03c970f1803cb5f Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Tue, 25 Oct 2022 10:00:21 +0100 Subject: [PATCH 09/54] Update custom_rpn.py --- detectree2/models/custom_rpn.py | 102 +++++++++++++++++++++++++++++++- 1 file changed, 101 insertions(+), 1 deletion(-) diff --git a/detectree2/models/custom_rpn.py b/detectree2/models/custom_rpn.py index ddbf094d..bb83f0c4 100644 --- a/detectree2/models/custom_rpn.py +++ b/detectree2/models/custom_rpn.py @@ -43,7 +43,7 @@ def _is_tracing(): return False else: return torch.jit.is_tracing() - + @PROPOSAL_GENERATOR_REGISTRY.register() class custom_RPN(RPN): @@ -173,3 +173,103 @@ def find_top_rpn_proposals( res.objectness_logits = scores_per_img[keep] results.append(res) return results + + +def custom_nms(P : torch.tensor ,scores: torch.tensor, thresh_iou : float, thresh_iou_o : float): + """ + Apply non-maximum suppression to avoid detecting too many + overlapping bounding boxes for a given object. + Args: + boxes: (tensor) The location preds for the image + along with the class predscores, Shape: [num_boxes,5]. + thresh_iou: (float) The overlap thresh for suppressing unnecessary boxes. + Returns: + A list of filtered boxes index, Shape: [ , 1] + """ + + # we extract coordinates for every + # prediction box present in P + x1 = P[:, 0] + y1 = P[:, 1] + x2 = P[:, 2] + y2 = P[:, 3] + + # we extract the confidence scores as well + scores = scores + + # calculate area of every block in P + areas = (x2 - x1) * (y2 - y1) + + # sort the prediction boxes in P + # according to their confidence scores + order = scores.argsort() + + # initialise an empty list for + # filtered prediction boxes + keep = [] + + + while len(order) > 0: + + # extract the index of the + # prediction with highest score + # we call this prediction S + idx = order[-1] + + # push S in filtered predictions list + keep.append(idx) + + # remove S from P + order = order[:-1] + + # sanity check + if len(order) == 0: + break + + # select coordinates of BBoxes according to + # the indices in order + xx1 = torch.index_select(x1,dim = 0, index = order) + xx2 = torch.index_select(x2,dim = 0, index = order) + yy1 = torch.index_select(y1,dim = 0, index = order) + yy2 = torch.index_select(y2,dim = 0, index = order) + + # find the coordinates of the intersection boxes + xx1 = torch.max(xx1, x1[idx]) + yy1 = torch.max(yy1, y1[idx]) + xx2 = torch.min(xx2, x2[idx]) + yy2 = torch.min(yy2, y2[idx]) + + # find height and width of the intersection boxes + w = xx2 - xx1 + h = yy2 - yy1 + + # take max with 0.0 to avoid negative w and h + # due to non-overlapping boxes + w = torch.clamp(w, min=0.0) + h = torch.clamp(h, min=0.0) + + # find the intersection area + inter = w*h + + # find the areas of BBoxes according the indices in order + rem_areas = torch.index_select(areas, dim = 0, index = order) + + # find the union of every prediction T in P + # with the prediction S + # Note that areas[idx] represents area of S + union = (rem_areas - inter) + areas[idx] + + # find the IoU of every prediction in P with S + IoU = inter / union + + # find the interaction over S + IoU_S = inter / areas[idx] + + # find the interaction over prediction + IoU_P = inter / areas + + # keep the boxes with IoU less than thresh_iou + mask = (IoU < thresh_iou)&(IoU_S < thresh_iou_o)&(IoU_P < thresh_iou_o) + order = order[mask] + + return keep From 0b55726069abbce1e40eb7deeb268f91ae72dc84 Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Wed, 28 Dec 2022 22:09:33 +0000 Subject: [PATCH 10/54] Create custom_nms.py this file includes a custom nms layer - which uses inter over area instead of inter over union as the anchor selection metric. The issue of crowns of different types overlapping each other is also solved. This change inceases the speed and mask quality a bit. Another function called nms_mask is used in prediction part and is part of the proprocessing where instead of eliminating all crowns with a low threshold, the nms is used and the nms metric is no longer box iou but mask iou. This change increases the output quality a lot. --- detectree2/models/custom_nms.py | 334 ++++++++++++++++++++++++++++++++ 1 file changed, 334 insertions(+) create mode 100644 detectree2/models/custom_nms.py diff --git a/detectree2/models/custom_nms.py b/detectree2/models/custom_nms.py new file mode 100644 index 00000000..5c40a966 --- /dev/null +++ b/detectree2/models/custom_nms.py @@ -0,0 +1,334 @@ +from typing import Tuple + +import torch +import torchvision +from torch import Tensor +from torchvision.extension import _assert_has_ops +import logging +import math +from typing import List, Tuple, Union + +from detectron2.layers import batched_nms, cat +from detectron2.structures import Boxes, Instances + +from detectron2.structures import Boxes, Instances +from typing import Dict, List, Optional, Tuple, Union +import torch +import torch.nn.functional as F +from torch import nn + +from detectron2.config import configurable +from detectron2.layers import Conv2d, ShapeSpec, cat +from detectron2.structures import Boxes, ImageList, Instances, pairwise_iou +from detectron2.utils.events import get_event_storage +from detectron2.utils.memory import retry_if_cuda_oom +from detectron2.utils.registry import Registry +from detectron2.modeling.proposal_generator.rpn import RPN, build_rpn_head + +from detectron2.modeling.anchor_generator import build_anchor_generator +from detectron2.modeling.box_regression import Box2BoxTransform, _dense_box_regression_loss +from detectron2.modeling.matcher import Matcher +from detectron2.modeling.sampling import subsample_labels +from detectron2.modeling.proposal_generator.build import PROPOSAL_GENERATOR_REGISTRY + + +@torch.jit.script_if_tracing +def move_device_like(src: torch.Tensor, dst: torch.Tensor) -> torch.Tensor: + """ + Tracing friendly way to cast tensor to another tensor's device. Device will be treated + as constant during tracing, scripting the casting process as whole can workaround this issue. + """ + return src.to(dst.device) + +def _is_tracing(): + # (fixed in TORCH_VERSION >= 1.9) + if torch.jit.is_scripting(): + # https://github.com/pytorch/pytorch/issues/47379 + return False + else: + return torch.jit.is_tracing() + +@PROPOSAL_GENERATOR_REGISTRY.register() +class custom_RPN(RPN): + + @configurable + def __init__(self, + *, + in_features: List[str], + head: nn.Module, + anchor_generator: nn.Module, + anchor_matcher: Matcher, + box2box_transform: Box2BoxTransform, + batch_size_per_image: int, + positive_fraction: float, + pre_nms_topk: Tuple[float, float], + post_nms_topk: Tuple[float, float], + nms_thresh: float = 0.7, + nms_thresh_union: float = 0.2, + min_box_size: float = 0.0, + anchor_boundary_thresh: float = -1.0, + loss_weight: Union[float, Dict[str, float]] = 1.0, + box_reg_loss_type: str = "smooth_l1", + smooth_l1_beta: float = 0.0,): + super().__init__() + self.nms_thresh_union = nms_thresh_union + + @classmethod + def from_config(cls, cfg, input_shape: Dict[str, ShapeSpec]): + in_features = cfg.MODEL.RPN.IN_FEATURES + ret = { + "in_features": in_features, + "min_box_size": cfg.MODEL.PROPOSAL_GENERATOR.MIN_SIZE, + "nms_thresh": cfg.MODEL.RPN.NMS_THRESH, + "nms_thresh_union": cfg.nms_thresh_union, + "batch_size_per_image": cfg.MODEL.RPN.BATCH_SIZE_PER_IMAGE, + "positive_fraction": cfg.MODEL.RPN.POSITIVE_FRACTION, + "loss_weight": { + "loss_rpn_cls": cfg.MODEL.RPN.LOSS_WEIGHT, + "loss_rpn_loc": cfg.MODEL.RPN.BBOX_REG_LOSS_WEIGHT * cfg.MODEL.RPN.LOSS_WEIGHT, + }, + "anchor_boundary_thresh": cfg.MODEL.RPN.BOUNDARY_THRESH, + "box2box_transform": Box2BoxTransform(weights=cfg.MODEL.RPN.BBOX_REG_WEIGHTS), + "box_reg_loss_type": cfg.MODEL.RPN.BBOX_REG_LOSS_TYPE, + "smooth_l1_beta": cfg.MODEL.RPN.SMOOTH_L1_BETA, + } + + ret["pre_nms_topk"] = (cfg.MODEL.RPN.PRE_NMS_TOPK_TRAIN, cfg.MODEL.RPN.PRE_NMS_TOPK_TEST) + ret["post_nms_topk"] = (cfg.MODEL.RPN.POST_NMS_TOPK_TRAIN, cfg.MODEL.RPN.POST_NMS_TOPK_TEST) + + ret["anchor_generator"] = build_anchor_generator(cfg, [input_shape[f] for f in in_features]) + ret["anchor_matcher"] = Matcher( + cfg.MODEL.RPN.IOU_THRESHOLDS, cfg.MODEL.RPN.IOU_LABELS, allow_low_quality_matches=True + ) + ret["head"] = build_rpn_head(cfg, [input_shape[f] for f in in_features]) + return ret + + def find_top_rpn_proposals( + proposals: List[torch.Tensor], + pred_objectness_logits: List[torch.Tensor], + image_sizes: List[Tuple[int, int]], + nms_thresh: float, + nms_thresh_union: float, + pre_nms_topk: int, + post_nms_topk: int, + min_box_size: float, + training: bool, +): + # here the box refinement has been done when proposals are inputted + num_images = len(image_sizes) + device = ( + proposals[0].device + if torch.jit.is_scripting() + else ("cpu" if torch.jit.is_tracing() else proposals[0].device) + ) + + # 1. Select top-k anchor for every level and every image + topk_scores = [] # #lvl Tensor, each of shape N x topk + topk_proposals = [] + batch_idx = move_device_like(torch.arange(num_images, device=device), proposals[0]) + for level_id, (proposals_i, logits_i) in enumerate(zip(proposals, pred_objectness_logits)): + Hi_Wi_A = logits_i.shape[1] + if isinstance(Hi_Wi_A, torch.Tensor): # it's a tensor in tracing + num_proposals_i = torch.clamp(Hi_Wi_A, max=pre_nms_topk) + else: + num_proposals_i = min(Hi_Wi_A, pre_nms_topk) + + topk_scores_i, topk_idx = logits_i.topk(num_proposals_i, dim=1) + + # each is N x topk + topk_proposals_i = proposals_i[batch_idx[:, None], topk_idx] # N x topk x 4 + + topk_proposals.append(topk_proposals_i) + topk_scores.append(topk_scores_i) + + + # 2. Concat all levels together + topk_scores = cat(topk_scores, dim=1) + topk_proposals = cat(topk_proposals, dim=1) + + # 3. For each image, run a per-level NMS, and choose topk results. + results: List[Instances] = [] + for n, image_size in enumerate(image_sizes): + boxes = Boxes(topk_proposals[n]) + scores_per_img = topk_scores[n] + + valid_mask = torch.isfinite(boxes.tensor).all(dim=1) & torch.isfinite(scores_per_img) + if not valid_mask.all(): + if training: + raise FloatingPointError( + "Predicted boxes or scores contain Inf/NaN. Training has diverged." + ) + boxes = boxes[valid_mask] + scores_per_img = scores_per_img[valid_mask] + boxes.clip(image_size) + + # filter empty boxes + keep = boxes.nonempty(threshold=min_box_size) + if _is_tracing() or keep.sum().item() != len(boxes): + boxes, scores_per_img= boxes[keep], scores_per_img[keep] + + keep = custom_nms(boxes.tensor, scores_per_img, nms_thresh_union) + + keep = keep[:post_nms_topk] # keep is already sorted + + res = Instances(image_size) + res.proposal_boxes = boxes[keep] + res.objectness_logits = scores_per_img[keep] + results.append(res) + return results + +def custom_nms(P : torch.tensor ,scores: torch.tensor, thresh_iou_o : float): + """ + Apply non-maximum suppression to avoid detecting too many + overlapping bounding boxes for a given object. + Args: + boxes: (tensor) The location preds for the image + along with the class predscores, Shape: [num_boxes,5]. + thresh_iou: (float) The overlap thresh for suppressing unnecessary boxes. + Returns: + A list of filtered boxes index, Shape: [ , 1] + """ + + # we extract coordinates for every + # prediction box present in P + x1 = P[:, 0] + y1 = P[:, 1] + x2 = P[:, 2] + y2 = P[:, 3] + + # we extract the confidence scores as well + scores = scores + + # calculate area of every block in P + areas = (x2 - x1) * (y2 - y1) + + # sort the prediction boxes in P + # according to their confidence scores + order = scores.argsort() + + # initialise an empty list for + # filtered prediction boxes + keep = [] + + + while len(order) > 0: + + # extract the index of the + # prediction with highest score + # we call this prediction S + idx = order[-1] + + # push S in filtered predictions list + keep.append(idx) + + # remove S from P + order = order[:-1] + + # sanity check + if len(order) == 0: + break + + # select coordinates of BBoxes according to + # the indices in order + xx1 = torch.index_select(x1,dim = 0, index = order) + xx2 = torch.index_select(x2,dim = 0, index = order) + yy1 = torch.index_select(y1,dim = 0, index = order) + yy2 = torch.index_select(y2,dim = 0, index = order) + + # find the coordinates of the intersection boxes + xx1 = torch.max(xx1, x1[idx]) + yy1 = torch.max(yy1, y1[idx]) + xx2 = torch.min(xx2, x2[idx]) + yy2 = torch.min(yy2, y2[idx]) + + # find height and width of the intersection boxes + w = xx2 - xx1 + h = yy2 - yy1 + + # take max with 0.0 to avoid negative w and h + # due to non-overlapping boxes + w = torch.clamp(w, min=0.0) + h = torch.clamp(h, min=0.0) + + # find the intersection area + inter = w*h + + # find the areas of BBoxes according the indices in order + rem_areas = torch.index_select(areas, dim = 0, index = order) + + # find the interaction over S + IoU_S = inter / areas[idx] + + # find the interaction over prediction + IoU_P = inter / areas + + # keep the boxes with IoU less than thresh_iou + mask = (IoU_S < thresh_iou_o)&(IoU_P < thresh_iou_o) + order = order[mask] + + return keep + +def custom_nms_mask(P : torch.tensor ,scores: torch.tensor, thresh_iou_o : float): + """ + Apply non-maximum suppression to avoid detecting too many + overlapping bounding boxes for a given object. + Args: + masks: (tensor) The location preds for the image + along with the class predscores, Shape: [n,image_shape,image_shape]. + thresh_iou: (float) The overlap thresh for suppressing unnecessary boxes. + Returns: + A list of filtered boxes index, Shape: [ , 1] + """ + + # we turn masks into ndarray + masks = P.reshape(len(P),-1,1) + + # we extract the confidence scores as well + scores = scores + + # calculate area of every block in P + areas = torch.sum(masks, axis = 1) + + # sort the prediction boxes in P + # according to their confidence scores + order = scores.argsort() + + # initialise an empty list for + # filtered prediction boxes + keep = [] + + + while len(order) > 0: + + # extract the index of the + # prediction with highest score + # we call this prediction S + idx = order[-1] + + # push S in filtered predictions list + keep.append(idx) + + # remove S from P + order = order[:-1] + + # sanity check + if len(order) == 0: + break + + # find the areas of BBoxes according the indices in order + rem_areas = areas[order] + + # find the intersection area + inter = torch.sum(masks[idx] * masks[order], axis=1) + + # find the interaction over S + IoU_S = inter / areas[idx] + + # find the interaction over prediction + IoU_P = inter / rem_areas + + # keep the masks with IoU less than thresh_iou + mask = (IoU_S < thresh_iou_o)&(IoU_P < thresh_iou_o).reshape(-1,1) + order = order.reshape(-1,1)[mask] + + return keep From 7d6cfd389f74399a67d620508471416f3d4b462e Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Wed, 28 Dec 2022 22:10:06 +0000 Subject: [PATCH 11/54] Delete custom_rpn.py --- detectree2/models/custom_rpn.py | 275 -------------------------------- 1 file changed, 275 deletions(-) delete mode 100644 detectree2/models/custom_rpn.py diff --git a/detectree2/models/custom_rpn.py b/detectree2/models/custom_rpn.py deleted file mode 100644 index bb83f0c4..00000000 --- a/detectree2/models/custom_rpn.py +++ /dev/null @@ -1,275 +0,0 @@ -import torch -import torchvision -from torch import Tensor -from torchvision.extension import _assert_has_ops -import logging -import math -from typing import List, Tuple, Union - -from detectron2.layers import batched_nms, cat -from detectron2.structures import Boxes, Instances -from typing import Dict, List, Optional, Tuple, Union -import torch.nn.functional as F -from torch import nn - -from detectron2.config import configurable -from detectron2.layers import Conv2d, ShapeSpec, cat -from detectron2.structures import Boxes, ImageList, Instances, pairwise_iou -from detectron2.utils.events import get_event_storage -from detectron2.utils.memory import retry_if_cuda_oom -from detectron2.utils.registry import Registry -from detectron2.modeling.proposal_generator.rpn import RPN, build_rpn_head - -from detectron2.modeling.anchor_generator import build_anchor_generator -from detectron2.modeling.box_regression import Box2BoxTransform, _dense_box_regression_loss -from detectron2.modeling.matcher import Matcher -from detectron2.modeling.sampling import subsample_labels -from detectron2.modeling.proposal_generator.build import PROPOSAL_GENERATOR_REGISTRY - - -@torch.jit.script_if_tracing -def move_device_like(src: torch.Tensor, dst: torch.Tensor) -> torch.Tensor: - """ - Tracing friendly way to cast tensor to another tensor's device. Device will be treated - as constant during tracing, scripting the casting process as whole can workaround this issue. - """ - return src.to(dst.device) - - -def _is_tracing(): - # (fixed in TORCH_VERSION >= 1.9) - if torch.jit.is_scripting(): - # https://github.com/pytorch/pytorch/issues/47379 - return False - else: - return torch.jit.is_tracing() - - -@PROPOSAL_GENERATOR_REGISTRY.register() -class custom_RPN(RPN): - - @configurable - def __init__(self, - *, - in_features: List[str], - head: nn.Module, - anchor_generator: nn.Module, - anchor_matcher: Matcher, - box2box_transform: Box2BoxTransform, - batch_size_per_image: int, - positive_fraction: float, - pre_nms_topk: Tuple[float, float], - post_nms_topk: Tuple[float, float], - nms_thresh: float = 0.7, - nms_thresh_union: float = 0.7, - min_box_size: float = 0.0, - anchor_boundary_thresh: float = -1.0, - loss_weight: Union[float, Dict[str, float]] = 1.0, - box_reg_loss_type: str = "smooth_l1", - smooth_l1_beta: float = 0.0,): - super().__init__() - self.nms_thresh_union = nms_thresh_union - - @classmethod - def from_config(cls, cfg, input_shape: Dict[str, ShapeSpec]): - in_features = cfg.MODEL.RPN.IN_FEATURES - ret = { - "in_features": in_features, - "min_box_size": cfg.MODEL.PROPOSAL_GENERATOR.MIN_SIZE, - "nms_thresh": cfg.MODEL.RPN.NMS_THRESH, - "nms_thresh_union": cfg.nms_thresh_union, - "batch_size_per_image": cfg.MODEL.RPN.BATCH_SIZE_PER_IMAGE, - "positive_fraction": cfg.MODEL.RPN.POSITIVE_FRACTION, - "loss_weight": { - "loss_rpn_cls": cfg.MODEL.RPN.LOSS_WEIGHT, - "loss_rpn_loc": cfg.MODEL.RPN.BBOX_REG_LOSS_WEIGHT * cfg.MODEL.RPN.LOSS_WEIGHT, - }, - "anchor_boundary_thresh": cfg.MODEL.RPN.BOUNDARY_THRESH, - "box2box_transform": Box2BoxTransform(weights=cfg.MODEL.RPN.BBOX_REG_WEIGHTS), - "box_reg_loss_type": cfg.MODEL.RPN.BBOX_REG_LOSS_TYPE, - "smooth_l1_beta": cfg.MODEL.RPN.SMOOTH_L1_BETA, - } - - ret["pre_nms_topk"] = (cfg.MODEL.RPN.PRE_NMS_TOPK_TRAIN, cfg.MODEL.RPN.PRE_NMS_TOPK_TEST) - ret["post_nms_topk"] = (cfg.MODEL.RPN.POST_NMS_TOPK_TRAIN, cfg.MODEL.RPN.POST_NMS_TOPK_TEST) - - ret["anchor_generator"] = build_anchor_generator(cfg, [input_shape[f] for f in in_features]) - ret["anchor_matcher"] = Matcher( - cfg.MODEL.RPN.IOU_THRESHOLDS, cfg.MODEL.RPN.IOU_LABELS, allow_low_quality_matches=True - ) - ret["head"] = build_rpn_head(cfg, [input_shape[f] for f in in_features]) - return ret - - def find_top_rpn_proposals( - proposals: List[torch.Tensor], - pred_objectness_logits: List[torch.Tensor], - image_sizes: List[Tuple[int, int]], - nms_thresh: float, - nms_thresh_union: float, - pre_nms_topk: int, - post_nms_topk: int, - min_box_size: float, - training: bool, -): - # here the box refinement has been done when proposals are inputted - num_images = len(image_sizes) - device = ( - proposals[0].device - if torch.jit.is_scripting() - else ("cpu" if torch.jit.is_tracing() else proposals[0].device) - ) - - # 1. Select top-k anchor for every level and every image - topk_scores = [] # #lvl Tensor, each of shape N x topk - topk_proposals = [] - batch_idx = move_device_like(torch.arange(num_images, device=device), proposals[0]) - for level_id, (proposals_i, logits_i) in enumerate(zip(proposals, pred_objectness_logits)): - Hi_Wi_A = logits_i.shape[1] - if isinstance(Hi_Wi_A, torch.Tensor): # it's a tensor in tracing - num_proposals_i = torch.clamp(Hi_Wi_A, max=pre_nms_topk) - else: - num_proposals_i = min(Hi_Wi_A, pre_nms_topk) - - topk_scores_i, topk_idx = logits_i.topk(num_proposals_i, dim=1) - - # each is N x topk - topk_proposals_i = proposals_i[batch_idx[:, None], topk_idx] # N x topk x 4 - - topk_proposals.append(topk_proposals_i) - topk_scores.append(topk_scores_i) - - - # 2. Concat all levels together - topk_scores = cat(topk_scores, dim=1) - topk_proposals = cat(topk_proposals, dim=1) - - # 3. For each image, run a per-level NMS, and choose topk results. - results: List[Instances] = [] - for n, image_size in enumerate(image_sizes): - boxes = Boxes(topk_proposals[n]) - scores_per_img = topk_scores[n] - - valid_mask = torch.isfinite(boxes.tensor).all(dim=1) & torch.isfinite(scores_per_img) - if not valid_mask.all(): - if training: - raise FloatingPointError( - "Predicted boxes or scores contain Inf/NaN. Training has diverged." - ) - boxes = boxes[valid_mask] - scores_per_img = scores_per_img[valid_mask] - boxes.clip(image_size) - - # filter empty boxes - keep = boxes.nonempty(threshold=min_box_size) - if _is_tracing() or keep.sum().item() != len(boxes): - boxes, scores_per_img= boxes[keep], scores_per_img[keep] - - keep = custom_nms(boxes.tensor, scores_per_img, nms_thresh, nms_thresh_union) - - keep = keep[:post_nms_topk] # keep is already sorted - - res = Instances(image_size) - res.proposal_boxes = boxes[keep] - res.objectness_logits = scores_per_img[keep] - results.append(res) - return results - - -def custom_nms(P : torch.tensor ,scores: torch.tensor, thresh_iou : float, thresh_iou_o : float): - """ - Apply non-maximum suppression to avoid detecting too many - overlapping bounding boxes for a given object. - Args: - boxes: (tensor) The location preds for the image - along with the class predscores, Shape: [num_boxes,5]. - thresh_iou: (float) The overlap thresh for suppressing unnecessary boxes. - Returns: - A list of filtered boxes index, Shape: [ , 1] - """ - - # we extract coordinates for every - # prediction box present in P - x1 = P[:, 0] - y1 = P[:, 1] - x2 = P[:, 2] - y2 = P[:, 3] - - # we extract the confidence scores as well - scores = scores - - # calculate area of every block in P - areas = (x2 - x1) * (y2 - y1) - - # sort the prediction boxes in P - # according to their confidence scores - order = scores.argsort() - - # initialise an empty list for - # filtered prediction boxes - keep = [] - - - while len(order) > 0: - - # extract the index of the - # prediction with highest score - # we call this prediction S - idx = order[-1] - - # push S in filtered predictions list - keep.append(idx) - - # remove S from P - order = order[:-1] - - # sanity check - if len(order) == 0: - break - - # select coordinates of BBoxes according to - # the indices in order - xx1 = torch.index_select(x1,dim = 0, index = order) - xx2 = torch.index_select(x2,dim = 0, index = order) - yy1 = torch.index_select(y1,dim = 0, index = order) - yy2 = torch.index_select(y2,dim = 0, index = order) - - # find the coordinates of the intersection boxes - xx1 = torch.max(xx1, x1[idx]) - yy1 = torch.max(yy1, y1[idx]) - xx2 = torch.min(xx2, x2[idx]) - yy2 = torch.min(yy2, y2[idx]) - - # find height and width of the intersection boxes - w = xx2 - xx1 - h = yy2 - yy1 - - # take max with 0.0 to avoid negative w and h - # due to non-overlapping boxes - w = torch.clamp(w, min=0.0) - h = torch.clamp(h, min=0.0) - - # find the intersection area - inter = w*h - - # find the areas of BBoxes according the indices in order - rem_areas = torch.index_select(areas, dim = 0, index = order) - - # find the union of every prediction T in P - # with the prediction S - # Note that areas[idx] represents area of S - union = (rem_areas - inter) + areas[idx] - - # find the IoU of every prediction in P with S - IoU = inter / union - - # find the interaction over S - IoU_S = inter / areas[idx] - - # find the interaction over prediction - IoU_P = inter / areas - - # keep the boxes with IoU less than thresh_iou - mask = (IoU < thresh_iou)&(IoU_S < thresh_iou_o)&(IoU_P < thresh_iou_o) - order = order[mask] - - return keep From 587976623de2bd015b112e082dc312ef6748c375 Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Wed, 28 Dec 2022 22:12:41 +0000 Subject: [PATCH 12/54] Update predict.py --- detectree2/models/predict.py | 449 +++++------------------------------ 1 file changed, 65 insertions(+), 384 deletions(-) diff --git a/detectree2/models/predict.py b/detectree2/models/predict.py index 55a5f10f..5120718a 100644 --- a/detectree2/models/predict.py +++ b/detectree2/models/predict.py @@ -1,191 +1,102 @@ +"""Generate predictions.""" import json import os import random -from http.client import REQUEST_URI_TOO_LONG from pathlib import Path import cv2 -import geopandas as gpd -from detectron2.engine import DefaultPredictor from detectron2.evaluation.coco_evaluation import instances_to_coco_json -from fiona.crs import from_epsg -from shapely.geometry import box, shape -from detectree2.models.train import get_filenames -from detectron2.data.build import DatasetMapper -import time -from PIL import Image -import pycocotools.mask as mask_util +from detectree2.models.train import get_filenames, get_tree_dicts -# Code to convert RLE data from the output instances into Polygons, a small about of info is lost but is fine. +from custom_nms import custom_nms_mask + +# Code to convert RLE data from the output instances into Polygons, +# a small amout of info is lost but is fine. # https://github.com/hazirbas/coco-json-converter/blob/master/generate_coco_json.py <-- found here -def reproject_to_geojson_spatially(data, - output_fold=None, - pred_fold=None, - EPSG="32650"): # noqa:N803 - """Reprojects the coordinates back so the crowns can be overlaid with the original tif file of the entire region. - Takes a json and changes it to a geojson so it can overlay with crowns. - Another copy is produced to overlay with PNGs. +class DefaultPredictor1: """ + intersection over area and nms is added here - Path(output_fold).mkdir(parents=True, exist_ok=True) - entries = os.listdir(pred_fold) - #print(entries) - # scale to deal with the resolution - scalingx = data.transform[0] - scalingy = -data.transform[4] - - for file in entries: - if ".json" in file: - # create a geofile for each tile --> the EPSG value might need to be changed. - geofile = { - "type": "FeatureCollection", - "crs": { - "type": "name", - "properties": { - "name": "urn:ogc:def:crs:EPSG::" + EPSG - } - }, - "features": [] - } - - # create a dictionary for each file to store data used multiple times - img_dict = {} - img_dict["filename"] = file - #print(img_dict["filename"]) - - file_mins = file.replace(".json", "") - file_mins_split = file_mins.split("_") - minx = int(file_mins_split[-4]) - miny = int(file_mins_split[-3]) - tile_height = int(file_mins_split[-2]) - buffer = int(file_mins_split[-1]) - height = (tile_height + 2 * buffer) / scalingx - - # update the image dictionary to store all information cleanly - img_dict.update({ - "minx": minx, - "miny": miny, - "height": height, - "buffer": buffer - }) - # print("Img dict:", img_dict) - - # load the json file we need to convert into a geojson - with open(pred_fold + img_dict["filename"]) as prediction_file: - datajson = json.load(prediction_file) - # print("data_json:",datajson) - - # json file is formated as a list of segmentation polygons so cycle through each one - for crown_data in datajson: - # just a check that the crown image is correct - if str(minx) + '_' + str(miny) in crown_data["image_id"]: - crown = crown_data["segmentation"] - confidence_score = crown_data['score'] - - # changing the coords from RLE format so can be read as numbers, here the numbers are - # integers so a bit of info on position is lost - mask_of_coords = mask_util.decode(crown) - crown_coords = polygon_from_mask(mask_of_coords) - moved_coords = [] - - # coords from json are in a list of [x1, y1, x2, y2,... ] so convert them to [[x1, y1], ...] - # format and at the same time rescale them so they are in the correct position for QGIS - for c in range(0, len(crown_coords), 2): - x_coord = crown_coords[c] - y_coord = crown_coords[c + 1] - - # print("ycoord:", y_coord) - # print("height:", height) + """ + def __init__(self, cfg): + self.cfg = cfg.clone() # cfg can be modified by model + self.model = build_model(self.cfg) + self.model.eval() + if len(cfg.DATASETS.TEST): + self.metadata = MetadataCatalog.get(cfg.DATASETS.TEST[0]) - # rescaling the coords depending on where the tile is in the original image, note the - # correction factors have been manually added as outputs did not line up with predictions - # from training script - if minx == int(data.bounds[0]) and miny == int(data.bounds[1]): - #print("Bottom Corner") - x_coord = (x_coord) * scalingx + minx - y_coord = (height - y_coord) * scalingy + miny - elif minx == int(data.bounds[0]): - #print("Left Edge") - x_coord = (x_coord) * scalingx + minx - y_coord = (height - - y_coord) * scalingy - buffer + miny - elif miny == int(data.bounds[1]): - #print("Bottom Edge") - x_coord = (x_coord) * scalingx - buffer + minx - y_coord = (height - - y_coord) * scalingy - buffer + miny - else: - # print("Anywhere else") - x_coord = (x_coord) * scalingx - buffer + minx - y_coord = (height - - y_coord) * scalingy - buffer + miny + checkpointer = DetectionCheckpointer(self.model) + checkpointer.load(cfg.MODEL.WEIGHTS) - moved_coords.append([x_coord, y_coord]) + self.aug = T.ResizeShortestEdge( + [cfg.INPUT.MIN_SIZE_TEST, cfg.INPUT.MIN_SIZE_TEST], cfg.INPUT.MAX_SIZE_TEST + ) - geofile["features"].append({ - "type": "Feature", - "properties": { - "Confidence score": confidence_score - }, - "geometry": { - "type": "Polygon", - "coordinates": [moved_coords] - } - }) + self.input_format = cfg.INPUT.FORMAT + assert self.input_format in ["RGB", "BGR"], self.input_format - # Check final form is correct - compare to a known geojson file if error appears. - # print("geofile",geofile) + def __call__(self, original_image): + """ + Args: + original_image (np.ndarray): an image of shape (H, W, C) (in BGR order). + Returns: + predictions (dict): + the output of the model for one image only. + See :doc:`/tutorials/models` for details about the format. + """ + with torch.no_grad(): # https://github.com/sphinx-doc/sphinx/issues/4258 + # Apply pre-processing to image. + if self.input_format == "RGB": + # whether the model expects BGR inputs or RGB + original_image = original_image[:, :, ::-1] + height, width = original_image.shape[:2] + image = self.aug.get_transform(original_image).apply_image(original_image) + image = torch.as_tensor(image.astype("float32").transpose(2, 0, 1)) - output_geo_file = output_fold + img_dict["filename"].replace( - '.json', "_" + EPSG + '_lidar.geojson') - # print("output location:", output_geo_file) - with open(output_geo_file, "w") as dest: - json.dump(geofile, dest) + inputs = {"image": image, "height": height, "width": width} + predictions = self.model([inputs])[0] -def polygonFromMask(maskedArr): - """ - Turn mask into polygons - """ - contours, _ = cv2.findContours(maskedArr, cv2.RETR_TREE, - cv2.CHAIN_APPROX_SIMPLE) + boxes = predictions['instances'].get_fields()['pred_boxes'] + scores = predictions['instances'].get_fields()['scores'] + pred_classes = predictions['instances'].get_fields()['pred_classes'] + pred_masks = predictions['instances'].get_fields()['pred_masks'] - segmentation = [] - for contour in contours: - # Valid polygons have >= 6 coordinates (3 points) - if contour.size >= 6: - segmentation.append(contour.flatten().tolist()) - # RLEs = mask_util.frPyObjects(segmentation, maskedArr.shape[0], - # maskedArr.shape[1]) - #RLE = mask_util.merge(RLEs) - # RLE = mask.encode(np.asfortranarray(maskedArr)) - # area = mask_util.area(RLE) - # [x, y, w, h] = cv2.boundingRect(maskedArr) + keep = custom_nms_mask(pred_masks ,scores, thresh_iou_o = 0.3) - return segmentation[0] # , [x, y, w, h], area + predictions['instances'].get_fields()['pred_boxes'] = boxes[keep] + predictions['instances'].get_fields()['scores'] = scores[keep] + predictions['instances'].get_fields()['pred_classes'] = pred_classes[keep] + predictions['instances'].get_fields()['pred_masks'] = pred_masks[keep] + return predictions def predict_on_data( directory: str = "./", - predictor=DefaultPredictor, + predictor=DefaultPredictor1, + eval=False, save: bool = True, + num_predictions=0, ): - """Make predictions on tiled data - - Predicts crowns for all png images present in a directory and outputs masks - as jsons + """Make predictions on tiled data. + Predicts crowns for all png images present in a directory and outputs masks as jsons. """ pred_dir = os.path.join(directory, "predictions") Path(pred_dir).mkdir(parents=True, exist_ok=True) - dataset_dicts = get_filenames(directory) + if eval: + dataset_dicts = get_tree_dicts(directory) + else: + dataset_dicts = get_filenames(directory) # Works out if all items in folder should be predicted on - - num_to_pred = len(dataset_dicts) + if num_predictions == 0: + num_to_pred = len(dataset_dicts) + else: + num_to_pred = num_predictions for d in random.sample(dataset_dicts, num_to_pred): img = cv2.imread(d["file_name"]) @@ -202,240 +113,10 @@ def predict_on_data( if save: # Converting the predictions to json files and saving them in the # specfied output file. - evaluations = instances_to_coco_json(outputs["instances"].to("cpu"), - d["file_name"]) + evaluations = instances_to_coco_json(outputs["instances"].to("cpu"), d["file_name"]) with open(output_file, "w") as dest: json.dump(evaluations, dest) -def filename_geoinfo(filename): - """Return geographic info of a tile from its filename - """ - parts = os.path.basename(filename).split("_") - - parts = [int(part) for part in parts[-6:-1]] # type: ignore - minx = parts[0] - miny = parts[1] - width = parts[2] - buffer = parts[3] - crs = parts[4] - return (minx, miny, width, buffer, crs) - - -def box_filter(filename, shift: int = 0): - """Create a bounding box from a file name to filter edge crowns - """ - minx, miny, width, buffer, crs = filename_geoinfo(filename) - bounding_box = box_make(minx, miny, width, buffer, crs, shift) - return bounding_box - - -def box_make(minx: int, - miny: int, - width: int, - buffer: int, - crs, - shift: int = 0): - """Generate bounding box from geographic specifications - """ - bbox = box( - minx - buffer + shift, - miny - buffer + shift, - minx + width + buffer - shift, - miny + width + buffer - shift, - ) - geo = gpd.GeoDataFrame({"geometry": bbox}, index=[0], crs=from_epsg(crs)) - return geo - - -def stitch_crowns(folder: str, shift: int = 1): - """Stitch together predicted crowns - """ - crowns_path = Path(folder) - files = crowns_path.glob("*geojson") - _, _, _, _, crs = filename_geoinfo(list(files)[0]) - files = crowns_path.glob("*geojson") - crowns = gpd.GeoDataFrame(columns=["Confidence score", "geometry"], - geometry="geometry", - crs=from_epsg(crs)) # initiate an empty gpd.GDF - for file in files: - crowns_tile = gpd.read_file(file) - #crowns_tile.crs = "epsg:32622" - #crowns_tile = crowns_tile.set_crs(from_epsg(32622)) - # print(crowns_tile) - - geo = box_filter(file, shift) - # geo.plot() - crowns_tile = gpd.sjoin(crowns_tile, geo, "inner", "within") - # print(crowns_tile) - crowns = crowns.append(crowns_tile) - # print(crowns) - return crowns - - -def calc_iou(shape1, shape2): - """Calculate the IoU of two shapes - """ - iou = shape1.intersection(shape2).area / shape1.union(shape2).area - return iou - -def clean_crowns(crowns: gpd.GeoDataFrame): - """Clean overlapping crowns - - Outputs can contain highly overlapping crowns including in the buffer region. - This function removes crowns with a high degree of overlap with others but a - lower Confidence Score. - """ - crowns_out = gpd.GeoDataFrame() - for index, row in crowns.iterrows(): #iterate over each crown - if index % 1000 == 0: - print(str(index) + " / " + str(len(crowns)) + " cleaned") - if crowns.intersects(shape(row.geometry)).sum() == 1: # if there is not a crown interesects with the row (other than itself) - crowns_out = crowns_out.append(row) # retain it - else: - intersecting = crowns.loc[crowns.intersects(shape(row.geometry))] # Find those crowns that intersect with it - intersecting = intersecting.reset_index().drop("index", axis=1) - iou = [] - for index1, row1 in intersecting.iterrows(): # iterate over those intersecting crowns - #print(row1.geometry) - iou.append(calc_iou(row.geometry, row1.geometry)) # Calculate the IoU with each of those crowns - #print(iou) - intersecting['iou'] = iou - matches = intersecting[intersecting['iou'] > 0.75] # Remove those crowns with a poor match - matches = matches.sort_values('Confidence score', ascending=False).reset_index().drop('index', axis=1) - match = matches.loc[[0]] # Of the remaining crowns select the crown with the highest confidence - if match['iou'][0] < 1: # If the most confident is not the initial crown - continue - else: - match = match.drop('iou', axis=1) - #print(index) - crowns_out = crowns_out.append(match) - return crowns_out.reset_index() - -class MyPredictor(DefaultPredictor): - def __init__(self, cfg, mode): - self.cfg = cfg.clone() # cfg can be modified by model - self.model = build_model(self.cfg) - self.model.eval() - self.mode = mode - cfg.DATASETS.TEST = ('pigs',) - if len(cfg.DATASETS.TEST): - self.metadata = MetadataCatalog.get(cfg.DATASETS.TEST[0]) - - checkpointer = DetectionCheckpointer(self.model) - checkpointer.load(cfg.MODEL.WEIGHTS) - - self.aug = self.augmentation() - - self.input_format = cfg.INPUT.FORMAT - assert self.input_format in ["RGB", "BGR"], self.input_format - - def __call__(self, original_image): - """ - Args: - original_image (np.ndarray): an image of shape (H, W, C) (in BGR order). - - Returns: - predictions (dict): - the output of the model for one image only. - See :doc:`/tutorials/models` for details about the format. - """ - with torch.no_grad(): # https://github.com/sphinx-doc/sphinx/issues/4258 - # Apply pre-processing to image. - if self.input_format == "RGB": - # whether the model expects BGR inputs or RGB - original_image = original_image[:, :, ::-1] - height, width = original_image.shape[:2] - if self.aug != None: - image = self.aug.get_transform(original_image).apply_image(original_image) - else: - image = original_image - image = torch.as_tensor(image.astype("float32").transpose(2, 0, 1)) - - inputs = {"image": image, "height": height, "width": width} - predictions = self.model([inputs])[0] - - - def _predict(self, IN_DIR, save = True): - dataset_dicts = [] - files = glob.glob(IN_DIR + "*.png") - for filename in [file for file in files]: - file = {} - filename = os.path.join(IN_DIR, filename) - file["file_name"] = filename - dataset_dicts.append(file) - - # Works out if all items in folder should be predicted on - - num_to_pred = len(dataset_dicts) - - pred_dir = IN_DIR + "predictions" - - Path(pred_dir).mkdir(parents=True, exist_ok=True) - - for data in random.sample(dataset_dicts,num_to_pred): - with torch.no_grad(): - print(data["file_name"]) - img = cv2.imread(data["file_name"]) - if self.input_format == "RGB": - # whether the model expects BGR inputs or RGB - img = img[:, :, ::-1] - height, width = img.shape[:2] - if self.aug != None: - image = self.aug.get_transform(img).apply_image(img) - else: - image = img - image = img - image = torch.as_tensor(image.astype("float32").transpose(2, 0, 1)) - - inputs = {"image": image, "height": height, "width": width} - predictions = self.model([inputs])[0] - file_name_path = data["file_name"] - file_name = os.path.basename(os.path.normpath(file_name_path)) #Strips off all slashes so just final file name left - file_name = file_name.replace("png","json") - - output_file = pred_dir + "/predictions_" + file_name - - if save: - ## Converting the predictions to json files and saving them in the specfied output file. - evaluations= instances_to_coco_json(predictions["instances"].to("cpu"),data["file_name"]) - with open(output_file, "w") as dest: - json.dump(evaluations,dest) - - - def predict(self, save=True): - - for i in range(len(self.cfg.IN_DIR)): - - self._predict(self.cfg.IN_DIR[i]) - files = glob.glob(self.cfg.IN_DIR[i] + "*.tif") - data = rasterio.open(files[0]) - reproject_to_geojson_spatially(data, self.cfg.OUT_DIR[-1], self.cfg.IN_DIR[i] + "predictions/", EPSG = "32650") - - folder = self.cfg.OUT_DIR[-1] - - crowns = stitch_crowns(folder, 1) - - crowns = clean_crowns(crowns) - - x = crowns.buffer(0.0001) - tolerance = 0.03 - simplified = x.simplify(tolerance, preserve_topology=True) - - crowns.to_file(folder + "crowns_out.gpkg") - - - def augmentation(self): - if self.mode == 'resize_fixed': - return T.ResizeShortestEdge( - [self.cfg.INPUT.MIN_SIZE_TEST, self.cfg.INPUT.MIN_SIZE_TEST], self.cfg.INPUT.MAX_SIZE_TEST) - if self.mode == 'No_resize': - return None - else: - print('No such a mode') - return T.ResizeShortestEdge( - [self.cfg.INPUT.MIN_SIZE_TEST, self.cfg.INPUT.MIN_SIZE_TEST], self.cfg.INPUT.MAX_SIZE_TEST) - - if __name__ == "__main__": print("something") From 7068875ff55c1949f05ada59024c236b483c7965 Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Wed, 28 Dec 2022 22:16:39 +0000 Subject: [PATCH 13/54] Update train.py --- detectree2/models/train.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/detectree2/models/train.py b/detectree2/models/train.py index a6588476..9fb315b8 100644 --- a/detectree2/models/train.py +++ b/detectree2/models/train.py @@ -31,6 +31,9 @@ from IPython.display import display from PIL import Image +from custom_ms import custom_RPN +from predict import DefaultPredictor1 + class LossEvalHook(HookBase): """Do inference and get the loss metric @@ -656,11 +659,11 @@ def setup_cfg( cfg.MODEL.ROI_HEADS.NUM_CLASSES = num_classes cfg.TEST.EVAL_PERIOD = eval_period cfg.MODEL.BACKBONE.FREEZE_AT = 2 - cfg.MODEL.ROI_BOX_HEAD.BBOX_REG_LOSS_TYPE = 'diou' + cfg.MODEL.PROPOSAL_GENERATOR.NAME = 'custom_RPN' return cfg def predictions_on_data(directory=None, - predictor=DefaultTrainer, + predictor=DefaultPredictor1, trees_metadata=None, save=True, scale=1, From d625d09d9272242fe2a00ef9b7643f22b9ceb9d5 Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Wed, 28 Dec 2022 22:23:54 +0000 Subject: [PATCH 14/54] Update train.py --- detectree2/models/train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/detectree2/models/train.py b/detectree2/models/train.py index 9fb315b8..e8068f97 100644 --- a/detectree2/models/train.py +++ b/detectree2/models/train.py @@ -31,7 +31,7 @@ from IPython.display import display from PIL import Image -from custom_ms import custom_RPN +from custom_nms import custom_RPN from predict import DefaultPredictor1 From 347d013455d35ad76b9dcb496da982aeb41c44e6 Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Wed, 28 Dec 2022 22:32:33 +0000 Subject: [PATCH 15/54] Update train.py --- detectree2/models/train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/detectree2/models/train.py b/detectree2/models/train.py index e8068f97..b4ba4ca8 100644 --- a/detectree2/models/train.py +++ b/detectree2/models/train.py @@ -31,7 +31,7 @@ from IPython.display import display from PIL import Image -from custom_nms import custom_RPN +from .custom_nms import custom_RPN from predict import DefaultPredictor1 From 8f78d1cbbb2265b031b3fa6643f68374eb98dfdc Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Wed, 28 Dec 2022 22:33:13 +0000 Subject: [PATCH 16/54] Update train.py --- detectree2/models/train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/detectree2/models/train.py b/detectree2/models/train.py index b4ba4ca8..42ee51bf 100644 --- a/detectree2/models/train.py +++ b/detectree2/models/train.py @@ -32,7 +32,7 @@ from PIL import Image from .custom_nms import custom_RPN -from predict import DefaultPredictor1 +from .predict import DefaultPredictor1 class LossEvalHook(HookBase): From 9788473b63511861fa5d7d8c10eb0b2ad1b7c982 Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Wed, 28 Dec 2022 22:38:14 +0000 Subject: [PATCH 17/54] Update custom_nms.py --- detectree2/models/custom_nms.py | 57 +++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/detectree2/models/custom_nms.py b/detectree2/models/custom_nms.py index 5c40a966..15991f34 100644 --- a/detectree2/models/custom_nms.py +++ b/detectree2/models/custom_nms.py @@ -30,6 +30,7 @@ from detectron2.modeling.matcher import Matcher from detectron2.modeling.sampling import subsample_labels from detectron2.modeling.proposal_generator.build import PROPOSAL_GENERATOR_REGISTRY +from detectron2.modeling import build_model @torch.jit.script_if_tracing @@ -332,3 +333,59 @@ def custom_nms_mask(P : torch.tensor ,scores: torch.tensor, thresh_iou_o : float order = order.reshape(-1,1)[mask] return keep + +class DefaultPredictor1: + """ + intersection over area and nms is added here + """ + def __init__(self, cfg): + self.cfg = cfg.clone() # cfg can be modified by model + self.model = build_model(self.cfg) + self.model.eval() + if len(cfg.DATASETS.TEST): + self.metadata = MetadataCatalog.get(cfg.DATASETS.TEST[0]) + + checkpointer = DetectionCheckpointer(self.model) + checkpointer.load(cfg.MODEL.WEIGHTS) + + self.aug = T.ResizeShortestEdge( + [cfg.INPUT.MIN_SIZE_TEST, cfg.INPUT.MIN_SIZE_TEST], cfg.INPUT.MAX_SIZE_TEST + ) + + self.input_format = cfg.INPUT.FORMAT + assert self.input_format in ["RGB", "BGR"], self.input_format + + def __call__(self, original_image): + """ + Args: + original_image (np.ndarray): an image of shape (H, W, C) (in BGR order). + Returns: + predictions (dict): + the output of the model for one image only. + See :doc:`/tutorials/models` for details about the format. + """ + with torch.no_grad(): # https://github.com/sphinx-doc/sphinx/issues/4258 + # Apply pre-processing to image. + if self.input_format == "RGB": + # whether the model expects BGR inputs or RGB + original_image = original_image[:, :, ::-1] + height, width = original_image.shape[:2] + image = self.aug.get_transform(original_image).apply_image(original_image) + image = torch.as_tensor(image.astype("float32").transpose(2, 0, 1)) + + inputs = {"image": image, "height": height, "width": width} + predictions = self.model([inputs])[0] + + boxes = predictions['instances'].get_fields()['pred_boxes'] + scores = predictions['instances'].get_fields()['scores'] + pred_classes = predictions['instances'].get_fields()['pred_classes'] + pred_masks = predictions['instances'].get_fields()['pred_masks'] + + keep = custom_nms_mask(pred_masks ,scores, thresh_iou_o = 0.3) + + predictions['instances'].get_fields()['pred_boxes'] = boxes[keep] + predictions['instances'].get_fields()['scores'] = scores[keep] + predictions['instances'].get_fields()['pred_classes'] = pred_classes[keep] + predictions['instances'].get_fields()['pred_masks'] = pred_masks[keep] + + return predictions From 6449d96e3baf008dc7a5f26727a4f9f624b177f6 Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Wed, 28 Dec 2022 22:38:57 +0000 Subject: [PATCH 18/54] Update predict.py --- detectree2/models/predict.py | 58 +----------------------------------- 1 file changed, 1 insertion(+), 57 deletions(-) diff --git a/detectree2/models/predict.py b/detectree2/models/predict.py index 5120718a..b4131517 100644 --- a/detectree2/models/predict.py +++ b/detectree2/models/predict.py @@ -9,68 +9,12 @@ from detectree2.models.train import get_filenames, get_tree_dicts -from custom_nms import custom_nms_mask +from .custom_nms import DefaultPredictor1 # Code to convert RLE data from the output instances into Polygons, # a small amout of info is lost but is fine. # https://github.com/hazirbas/coco-json-converter/blob/master/generate_coco_json.py <-- found here -class DefaultPredictor1: - """ - intersection over area and nms is added here - - """ - def __init__(self, cfg): - self.cfg = cfg.clone() # cfg can be modified by model - self.model = build_model(self.cfg) - self.model.eval() - if len(cfg.DATASETS.TEST): - self.metadata = MetadataCatalog.get(cfg.DATASETS.TEST[0]) - - checkpointer = DetectionCheckpointer(self.model) - checkpointer.load(cfg.MODEL.WEIGHTS) - - self.aug = T.ResizeShortestEdge( - [cfg.INPUT.MIN_SIZE_TEST, cfg.INPUT.MIN_SIZE_TEST], cfg.INPUT.MAX_SIZE_TEST - ) - - self.input_format = cfg.INPUT.FORMAT - assert self.input_format in ["RGB", "BGR"], self.input_format - - def __call__(self, original_image): - """ - Args: - original_image (np.ndarray): an image of shape (H, W, C) (in BGR order). - Returns: - predictions (dict): - the output of the model for one image only. - See :doc:`/tutorials/models` for details about the format. - """ - with torch.no_grad(): # https://github.com/sphinx-doc/sphinx/issues/4258 - # Apply pre-processing to image. - if self.input_format == "RGB": - # whether the model expects BGR inputs or RGB - original_image = original_image[:, :, ::-1] - height, width = original_image.shape[:2] - image = self.aug.get_transform(original_image).apply_image(original_image) - image = torch.as_tensor(image.astype("float32").transpose(2, 0, 1)) - - inputs = {"image": image, "height": height, "width": width} - predictions = self.model([inputs])[0] - - boxes = predictions['instances'].get_fields()['pred_boxes'] - scores = predictions['instances'].get_fields()['scores'] - pred_classes = predictions['instances'].get_fields()['pred_classes'] - pred_masks = predictions['instances'].get_fields()['pred_masks'] - - keep = custom_nms_mask(pred_masks ,scores, thresh_iou_o = 0.3) - - predictions['instances'].get_fields()['pred_boxes'] = boxes[keep] - predictions['instances'].get_fields()['scores'] = scores[keep] - predictions['instances'].get_fields()['pred_classes'] = pred_classes[keep] - predictions['instances'].get_fields()['pred_masks'] = pred_masks[keep] - - return predictions def predict_on_data( directory: str = "./", From fa66c04e7d35ae83751cbca5a0216e710e63dc1a Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Wed, 28 Dec 2022 22:39:22 +0000 Subject: [PATCH 19/54] Update train.py --- detectree2/models/train.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/detectree2/models/train.py b/detectree2/models/train.py index 42ee51bf..fa878e9e 100644 --- a/detectree2/models/train.py +++ b/detectree2/models/train.py @@ -31,8 +31,7 @@ from IPython.display import display from PIL import Image -from .custom_nms import custom_RPN -from .predict import DefaultPredictor1 +from .custom_nms import custom_RPN, DefaultPredictor1 class LossEvalHook(HookBase): From 5f1f255ce41e1ee6c93c347f5bb26630fe9d4afc Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Wed, 28 Dec 2022 22:56:25 +0000 Subject: [PATCH 20/54] Update custom_nms.py --- detectree2/models/custom_nms.py | 132 ++++++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) diff --git a/detectree2/models/custom_nms.py b/detectree2/models/custom_nms.py index 15991f34..552fadb9 100644 --- a/detectree2/models/custom_nms.py +++ b/detectree2/models/custom_nms.py @@ -177,6 +177,138 @@ def find_top_rpn_proposals( res.objectness_logits = scores_per_img[keep] results.append(res) return results + +@PROPOSAL_GENERATOR_REGISTRY.register() +class custom_RPN(RPN): + + @configurable + def __init__(self, + *, + in_features: List[str], + head: nn.Module, + anchor_generator: nn.Module, + anchor_matcher: Matcher, + box2box_transform: Box2BoxTransform, + batch_size_per_image: int, + positive_fraction: float, + pre_nms_topk: Tuple[float, float], + post_nms_topk: Tuple[float, float], + nms_thresh: float = 0.4, + nms_thresh_union: float = 0.4, + min_box_size: float = 0.0, + anchor_boundary_thresh: float = -1.0, + loss_weight: Union[float, Dict[str, float]] = 1.0, + box_reg_loss_type: str = "smooth_l1", + smooth_l1_beta: float = 0.0,): + super().__init__(in_features=in_features, head=head, anchor_generator=anchor_generator, + anchor_matcher=anchor_matcher, box2box_transform=box2box_transform, batch_size_per_image=batch_size_per_image, + positive_fraction=positive_fraction, pre_nms_topk=pre_nms_topk, post_nms_topk=post_nms_topk) + self.nms_thresh_union = nms_thresh_union + + @classmethod + def from_config(cls, cfg, input_shape: Dict[str, ShapeSpec]): + in_features = cfg.MODEL.RPN.IN_FEATURES + ret = { + "in_features": in_features, + "min_box_size": cfg.MODEL.PROPOSAL_GENERATOR.MIN_SIZE, + "nms_thresh": cfg.MODEL.RPN.NMS_THRESH, + "nms_thresh_union": cfg.nms_thresh_union, + "batch_size_per_image": cfg.MODEL.RPN.BATCH_SIZE_PER_IMAGE, + "positive_fraction": cfg.MODEL.RPN.POSITIVE_FRACTION, + "loss_weight": { + "loss_rpn_cls": cfg.MODEL.RPN.LOSS_WEIGHT, + "loss_rpn_loc": cfg.MODEL.RPN.BBOX_REG_LOSS_WEIGHT * cfg.MODEL.RPN.LOSS_WEIGHT, + }, + "anchor_boundary_thresh": cfg.MODEL.RPN.BOUNDARY_THRESH, + "box2box_transform": Box2BoxTransform(weights=cfg.MODEL.RPN.BBOX_REG_WEIGHTS), + "box_reg_loss_type": cfg.MODEL.RPN.BBOX_REG_LOSS_TYPE, + "smooth_l1_beta": cfg.MODEL.RPN.SMOOTH_L1_BETA, + } + + ret["pre_nms_topk"] = (cfg.MODEL.RPN.PRE_NMS_TOPK_TRAIN, cfg.MODEL.RPN.PRE_NMS_TOPK_TEST) + ret["post_nms_topk"] = (cfg.MODEL.RPN.POST_NMS_TOPK_TRAIN, cfg.MODEL.RPN.POST_NMS_TOPK_TEST) + + ret["anchor_generator"] = build_anchor_generator(cfg, [input_shape[f] for f in in_features]) + ret["anchor_matcher"] = Matcher( + cfg.MODEL.RPN.IOU_THRESHOLDS, cfg.MODEL.RPN.IOU_LABELS, allow_low_quality_matches=True + ) + ret["head"] = build_rpn_head(cfg, [input_shape[f] for f in in_features]) + return ret + + def find_top_rpn_proposals( + proposals: List[torch.Tensor], + pred_objectness_logits: List[torch.Tensor], + image_sizes: List[Tuple[int, int]], + nms_thresh: float, + nms_thresh_union: float, + pre_nms_topk: int, + post_nms_topk: int, + min_box_size: float, + training: bool, +): + # here the box refinement has been done when proposals are inputted + num_images = len(image_sizes) + device = ( + proposals[0].device + if torch.jit.is_scripting() + else ("cpu" if torch.jit.is_tracing() else proposals[0].device) + ) + + # 1. Select top-k anchor for every level and every image + topk_scores = [] # #lvl Tensor, each of shape N x topk + topk_proposals = [] + batch_idx = move_device_like(torch.arange(num_images, device=device), proposals[0]) + for level_id, (proposals_i, logits_i) in enumerate(zip(proposals, pred_objectness_logits)): + Hi_Wi_A = logits_i.shape[1] + if isinstance(Hi_Wi_A, torch.Tensor): # it's a tensor in tracing + num_proposals_i = torch.clamp(Hi_Wi_A, max=pre_nms_topk) + else: + num_proposals_i = min(Hi_Wi_A, pre_nms_topk) + + topk_scores_i, topk_idx = logits_i.topk(num_proposals_i, dim=1) + + # each is N x topk + topk_proposals_i = proposals_i[batch_idx[:, None], topk_idx] # N x topk x 4 + + topk_proposals.append(topk_proposals_i) + topk_scores.append(topk_scores_i) + + + # 2. Concat all levels together + topk_scores = cat(topk_scores, dim=1) + topk_proposals = cat(topk_proposals, dim=1) + + # 3. For each image, run a per-level NMS, and choose topk results. + results: List[Instances] = [] + for n, image_size in enumerate(image_sizes): + print("----------------------------------------", image_size) + boxes = Boxes(topk_proposals[n]) + scores_per_img = topk_scores[n] + + valid_mask = torch.isfinite(boxes.tensor).all(dim=1) & torch.isfinite(scores_per_img) + if not valid_mask.all(): + if training: + raise FloatingPointError( + "Predicted boxes or scores contain Inf/NaN. Training has diverged." + ) + boxes = boxes[valid_mask] + scores_per_img = scores_per_img[valid_mask] + boxes.clip(image_size) + + # filter empty boxes + keep = boxes.nonempty(threshold=min_box_size) + if _is_tracing() or keep.sum().item() != len(boxes): + boxes, scores_per_img= boxes[keep], scores_per_img[keep] + + keep = custom_nms(boxes.tensor, scores_per_img, nms_thresh_union) + + keep = keep[:post_nms_topk] # keep is already sorted + + res = Instances(image_size) + res.proposal_boxes = boxes[keep] + res.objectness_logits = scores_per_img[keep] + results.append(res) + return results def custom_nms(P : torch.tensor ,scores: torch.tensor, thresh_iou_o : float): """ From 01b5affc418eeeee3e7d837a6c59f5ee6cc3c097 Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Wed, 28 Dec 2022 22:57:15 +0000 Subject: [PATCH 21/54] Update train.py --- detectree2/models/train.py | 1 + 1 file changed, 1 insertion(+) diff --git a/detectree2/models/train.py b/detectree2/models/train.py index fa878e9e..6eea22c1 100644 --- a/detectree2/models/train.py +++ b/detectree2/models/train.py @@ -659,6 +659,7 @@ def setup_cfg( cfg.TEST.EVAL_PERIOD = eval_period cfg.MODEL.BACKBONE.FREEZE_AT = 2 cfg.MODEL.PROPOSAL_GENERATOR.NAME = 'custom_RPN' + cfg.nms_thresh_union = 0.2 return cfg def predictions_on_data(directory=None, From b19d2f410da5c35ce5cae20f79b7f63b238daee4 Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Wed, 28 Dec 2022 23:01:41 +0000 Subject: [PATCH 22/54] Update custom_nms.py --- detectree2/models/custom_nms.py | 129 -------------------------------- 1 file changed, 129 deletions(-) diff --git a/detectree2/models/custom_nms.py b/detectree2/models/custom_nms.py index 552fadb9..80d7c970 100644 --- a/detectree2/models/custom_nms.py +++ b/detectree2/models/custom_nms.py @@ -49,135 +49,6 @@ def _is_tracing(): else: return torch.jit.is_tracing() -@PROPOSAL_GENERATOR_REGISTRY.register() -class custom_RPN(RPN): - - @configurable - def __init__(self, - *, - in_features: List[str], - head: nn.Module, - anchor_generator: nn.Module, - anchor_matcher: Matcher, - box2box_transform: Box2BoxTransform, - batch_size_per_image: int, - positive_fraction: float, - pre_nms_topk: Tuple[float, float], - post_nms_topk: Tuple[float, float], - nms_thresh: float = 0.7, - nms_thresh_union: float = 0.2, - min_box_size: float = 0.0, - anchor_boundary_thresh: float = -1.0, - loss_weight: Union[float, Dict[str, float]] = 1.0, - box_reg_loss_type: str = "smooth_l1", - smooth_l1_beta: float = 0.0,): - super().__init__() - self.nms_thresh_union = nms_thresh_union - - @classmethod - def from_config(cls, cfg, input_shape: Dict[str, ShapeSpec]): - in_features = cfg.MODEL.RPN.IN_FEATURES - ret = { - "in_features": in_features, - "min_box_size": cfg.MODEL.PROPOSAL_GENERATOR.MIN_SIZE, - "nms_thresh": cfg.MODEL.RPN.NMS_THRESH, - "nms_thresh_union": cfg.nms_thresh_union, - "batch_size_per_image": cfg.MODEL.RPN.BATCH_SIZE_PER_IMAGE, - "positive_fraction": cfg.MODEL.RPN.POSITIVE_FRACTION, - "loss_weight": { - "loss_rpn_cls": cfg.MODEL.RPN.LOSS_WEIGHT, - "loss_rpn_loc": cfg.MODEL.RPN.BBOX_REG_LOSS_WEIGHT * cfg.MODEL.RPN.LOSS_WEIGHT, - }, - "anchor_boundary_thresh": cfg.MODEL.RPN.BOUNDARY_THRESH, - "box2box_transform": Box2BoxTransform(weights=cfg.MODEL.RPN.BBOX_REG_WEIGHTS), - "box_reg_loss_type": cfg.MODEL.RPN.BBOX_REG_LOSS_TYPE, - "smooth_l1_beta": cfg.MODEL.RPN.SMOOTH_L1_BETA, - } - - ret["pre_nms_topk"] = (cfg.MODEL.RPN.PRE_NMS_TOPK_TRAIN, cfg.MODEL.RPN.PRE_NMS_TOPK_TEST) - ret["post_nms_topk"] = (cfg.MODEL.RPN.POST_NMS_TOPK_TRAIN, cfg.MODEL.RPN.POST_NMS_TOPK_TEST) - - ret["anchor_generator"] = build_anchor_generator(cfg, [input_shape[f] for f in in_features]) - ret["anchor_matcher"] = Matcher( - cfg.MODEL.RPN.IOU_THRESHOLDS, cfg.MODEL.RPN.IOU_LABELS, allow_low_quality_matches=True - ) - ret["head"] = build_rpn_head(cfg, [input_shape[f] for f in in_features]) - return ret - - def find_top_rpn_proposals( - proposals: List[torch.Tensor], - pred_objectness_logits: List[torch.Tensor], - image_sizes: List[Tuple[int, int]], - nms_thresh: float, - nms_thresh_union: float, - pre_nms_topk: int, - post_nms_topk: int, - min_box_size: float, - training: bool, -): - # here the box refinement has been done when proposals are inputted - num_images = len(image_sizes) - device = ( - proposals[0].device - if torch.jit.is_scripting() - else ("cpu" if torch.jit.is_tracing() else proposals[0].device) - ) - - # 1. Select top-k anchor for every level and every image - topk_scores = [] # #lvl Tensor, each of shape N x topk - topk_proposals = [] - batch_idx = move_device_like(torch.arange(num_images, device=device), proposals[0]) - for level_id, (proposals_i, logits_i) in enumerate(zip(proposals, pred_objectness_logits)): - Hi_Wi_A = logits_i.shape[1] - if isinstance(Hi_Wi_A, torch.Tensor): # it's a tensor in tracing - num_proposals_i = torch.clamp(Hi_Wi_A, max=pre_nms_topk) - else: - num_proposals_i = min(Hi_Wi_A, pre_nms_topk) - - topk_scores_i, topk_idx = logits_i.topk(num_proposals_i, dim=1) - - # each is N x topk - topk_proposals_i = proposals_i[batch_idx[:, None], topk_idx] # N x topk x 4 - - topk_proposals.append(topk_proposals_i) - topk_scores.append(topk_scores_i) - - - # 2. Concat all levels together - topk_scores = cat(topk_scores, dim=1) - topk_proposals = cat(topk_proposals, dim=1) - - # 3. For each image, run a per-level NMS, and choose topk results. - results: List[Instances] = [] - for n, image_size in enumerate(image_sizes): - boxes = Boxes(topk_proposals[n]) - scores_per_img = topk_scores[n] - - valid_mask = torch.isfinite(boxes.tensor).all(dim=1) & torch.isfinite(scores_per_img) - if not valid_mask.all(): - if training: - raise FloatingPointError( - "Predicted boxes or scores contain Inf/NaN. Training has diverged." - ) - boxes = boxes[valid_mask] - scores_per_img = scores_per_img[valid_mask] - boxes.clip(image_size) - - # filter empty boxes - keep = boxes.nonempty(threshold=min_box_size) - if _is_tracing() or keep.sum().item() != len(boxes): - boxes, scores_per_img= boxes[keep], scores_per_img[keep] - - keep = custom_nms(boxes.tensor, scores_per_img, nms_thresh_union) - - keep = keep[:post_nms_topk] # keep is already sorted - - res = Instances(image_size) - res.proposal_boxes = boxes[keep] - res.objectness_logits = scores_per_img[keep] - results.append(res) - return results - @PROPOSAL_GENERATOR_REGISTRY.register() class custom_RPN(RPN): From fe9bcdef3cb6708da8e45b0688f1dbcdc86c6f22 Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Wed, 28 Dec 2022 23:36:48 +0000 Subject: [PATCH 23/54] Update train.py --- detectree2/models/train.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/detectree2/models/train.py b/detectree2/models/train.py index 6eea22c1..357b3e70 100644 --- a/detectree2/models/train.py +++ b/detectree2/models/train.py @@ -658,8 +658,16 @@ def setup_cfg( cfg.MODEL.ROI_HEADS.NUM_CLASSES = num_classes cfg.TEST.EVAL_PERIOD = eval_period cfg.MODEL.BACKBONE.FREEZE_AT = 2 + #cfg.MODEL.ROI_BOX_HEAD.BBOX_REG_LOSS_TYPE = 'diou' cfg.MODEL.PROPOSAL_GENERATOR.NAME = 'custom_RPN' cfg.nms_thresh_union = 0.2 + cfg.MODEL.RPN.NMS_THRESH = 0 + + #cfg.nms_thresh_union = 0 + #cfg.MODEL.ROI_HEADS.NMS_THRESH_TEST = 0.1 + + #cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5 + cfg.MODEL.ROI_HEADS.NUM_CLASSES = 1 # only has one class (pnumonia) return cfg def predictions_on_data(directory=None, From c876df826c74b3e615e526c94edbfb018c38dfaf Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Wed, 28 Dec 2022 23:39:18 +0000 Subject: [PATCH 24/54] Update train.py --- detectree2/models/train.py | 167 +++++++++++++++++++++++++++---------- 1 file changed, 123 insertions(+), 44 deletions(-) diff --git a/detectree2/models/train.py b/detectree2/models/train.py index 357b3e70..095d6469 100644 --- a/detectree2/models/train.py +++ b/detectree2/models/train.py @@ -391,6 +391,61 @@ def build_train_loader(cls, cfg): ), ) + # def build_optimizer(cls, cfg, model): + # """ + # Returns: + # torch.optim.Optimizer: + + # It now calls :func:`detectron2.solver.build_optimizer`. + # Overwrite it if you'd like a different optimizer. + # """ + # def build_optimizer_adam(cfg, model): + # """ + # Build an optimizer from config. + # """ + # norm_module_types = ( + # torch.nn.BatchNorm1d, + # torch.nn.BatchNorm2d, + # torch.nn.BatchNorm3d, + # torch.nn.SyncBatchNorm, + # # NaiveSyncBatchNorm inherits from BatchNorm2d + # torch.nn.GroupNorm, + # torch.nn.InstanceNorm1d, + # torch.nn.InstanceNorm2d, + # torch.nn.InstanceNorm3d, + # torch.nn.LayerNorm, + # torch.nn.LocalResponseNorm, + # ) + # params: List[Dict[str, Any]] = [] + # memo: Set[torch.nn.parameter.Parameter] = set() + # for module in model.modules(): + # for key, value in module.named_parameters(recurse=False): + # if not value.requires_grad: + # continue + # # Avoid duplicating parameters + # if value in memo: + # continue + # memo.add(value) + # lr = cfg.SOLVER.BASE_LR + # weight_decay = cfg.SOLVER.WEIGHT_DECAY + # if isinstance(module, norm_module_types): + # weight_decay = cfg.SOLVER.WEIGHT_DECAY_NORM + # elif key == "bias": + # # NOTE: unlike Detectron v1, we now default BIAS_LR_FACTOR to 1.0 + # # and WEIGHT_DECAY_BIAS to WEIGHT_DECAY so that bias optimizer + # # hyperparameters are by default exactly the same as for regular + # # weights. + # lr = cfg.SOLVER.BASE_LR * cfg.SOLVER.BIAS_LR_FACTOR + # #weight_decay = cfg.SOLVER.WEIGHT_DECAY_BIAS + # weight_decay = 0 + # params += [{"params": [value], "lr": lr, "weight_decay": weight_decay}] + + # optimizer = torch.optim.Adam(params, cfg.SOLVER.BASE_LR, cfg.betas, weight_decay = weight_decay) + # optimizer = maybe_add_gradient_clipping(cfg, optimizer) + # return optimizer + + # return build_optimizer_adam(cfg, model) + @classmethod def test_train(cls, cfg, model, evaluators=None): """ @@ -448,13 +503,11 @@ def test_train(cls, cfg, model, evaluators=None): return results -def get_tree_dicts(directory: str, classes: List[str] = None) -> List[Dict]: +def get_tree_dicts(directory: str, classes: List[str] = None, classes_at: str = None) -> List[Dict]: """Get the tree dictionaries. - Args: directory: Path to directory classes: Signifies which column (if any) corresponds to the class labels - Returns: List of dictionaries corresponding to segmentations of trees. Each dictionary includes bounding box around tree and points tracing a polygon around a tree. @@ -467,8 +520,7 @@ def get_tree_dicts(directory: str, classes: List[str] = None) -> List[Dict]: # if classes is not None: # list_of_classes = crowns[variable].unique().tolist() - list_of_classes = ["CIRAD", "CNES", "INRA"] - classes = list_of_classes + classes = classes else: classes = ["tree"] # classes = Genus_Species_UniqueList #['tree'] # genus_species list @@ -478,16 +530,16 @@ def get_tree_dicts(directory: str, classes: List[str] = None) -> List[Dict]: # if file.endswith(".geojson"): # print(os.path.join(root, file)) - for filename in [ - file for file in os.listdir(directory) if file.endswith(".geojson") - ]: + for filename in [file for file in os.listdir(directory) if file.endswith(".geojson")]: json_file = os.path.join(directory, filename) with open(json_file) as f: img_anns = json.load(f) # Turn off type checking for annotations until we have a better solution - record: dict[str, Any] = {} + record: Dict[str, Any] = {} + # filename = os.path.join(directory, img_anns["imagePath"]) filename = img_anns["imagePath"] + # Make sure we have the correct height and width height, width = cv2.imread(filename).shape[:2] @@ -511,29 +563,19 @@ def get_tree_dicts(directory: str, classes: List[str] = None) -> List[Dict]: # print("#### HERE ARE SOME POLYS #####", poly) if classes != ["tree"]: obj = { - "bbox": [np.min(px), - np.min(py), - np.max(px), - np.max(py)], - "bbox_mode": - BoxMode.XYXY_ABS, + "bbox": [np.min(px), np.min(py), np.max(px), np.max(py)], + "bbox_mode": BoxMode.XYXY_ABS, "segmentation": [poly], - "category_id": - classes.index(features["properties"]["PlotOrg"] - ), # id + "category_id": classes.index(features["properties"][classes_at]), # id # "category_id": 0, #id - "iscrowd": - 0, + "iscrowd": 0, } else: obj = { - "bbox": [np.min(px), - np.min(py), - np.max(px), - np.max(py)], + "bbox": [np.min(px), np.min(py), np.max(px), np.max(py)], "bbox_mode": BoxMode.XYXY_ABS, "segmentation": [poly], - "category_id": 0, # id + "category_id": 0, # id "iscrowd": 0, } # pdb.set_trace() @@ -546,13 +588,13 @@ def get_tree_dicts(directory: str, classes: List[str] = None) -> List[Dict]: def combine_dicts(root_dir: str, val_dir: int, - mode: str = "train") -> List[Dict]: + mode: str = "train", + classes: List[str] = None, + classes_at: str = None) -> List[Dict]: """Join tree dicts from different directories. - Args: root_dir: val_dir: - Returns: Concatenated array of dictionaries over all directories """ @@ -561,18 +603,19 @@ def combine_dicts(root_dir: str, del train_dirs[(val_dir - 1)] tree_dicts = [] for d in train_dirs: - tree_dicts += get_tree_dicts(d) - return tree_dicts - else: - tree_dicts = get_tree_dicts(train_dirs[(val_dir - 1)]) - return tree_dicts + tree_dicts += get_tree_dicts(d, classes=classes, classes_at=classes_at) + elif mode == "val": + tree_dicts = get_tree_dicts(train_dirs[(val_dir - 1)], classes=classes, classes_at=classes_at) + elif mode == "full": + tree_dicts = [] + for d in train_dirs: + tree_dicts += get_tree_dicts(d, classes=classes, classes_at=classes_at) + return tree_dicts def get_filenames(directory: str): """Get the file names if no geojson is present. - Allows for predictions where no delinations have been manually produced. - Args: directory (str): directory of images to be predicted on """ @@ -587,12 +630,50 @@ def get_filenames(directory: str): return dataset_dicts -def register_train_data(train_location, name="tree", val_fold=1): - for d in ["train", "val"]: - DatasetCatalog.register( - name + "_" + d, - lambda d=d: combine_dicts(train_location, val_fold, d)) - MetadataCatalog.get(name + "_" + d).set(thing_classes=["tree"]) +def register_train_data(train_location, + name: str = "tree", + val_fold=None, + classes=None, + classes_at=None): + """Register data for training and (optionally) validation. + Args: + train_location: directory containing training folds + name: string to name data + val_fold: fold assigned for validation and tuning. If not given, + will take place on all folds. + """ + if val_fold is not None: + for d in ["train", "val"]: + DatasetCatalog.register(name + "_" + d, lambda d=d: combine_dicts(train_location, + val_fold, d, + classes=classes, classes_at=classes_at)) + if classes is None: + MetadataCatalog.get(name + "_" + d).set(thing_classes=["tree"]) + else: + MetadataCatalog.get(name + "_" + d).set(thing_classes=classes) + else: + DatasetCatalog.register(name + "_" + "full", lambda d=d: combine_dicts(train_location, + 0, "full", + classes=classes, classes_at=classes_at)) + if classes is None: + MetadataCatalog.get(name + "_" + "full").set(thing_classes=["tree"]) + else: + MetadataCatalog.get(name + "_" + "full").set(thing_classes=classes) + + +def read_data(out_dir): + """Function that will read the classes that are recorded during tiling.""" + list = [] + out_tif = out_dir + 'classes.txt' + # open file and read the content in a list + with open(out_tif, 'r') as fp: + for line in fp: + # remove linebreak from a current name + # linebreak is the last character of each line + x = line[:-1] + # add current item to the list + list.append(x) + return (list) def remove_registered_data(name="tree"): @@ -603,8 +684,7 @@ def remove_registered_data(name="tree"): def register_test_data(test_location, name="tree"): d = "test" - DatasetCatalog.register(name + "_" + d, - lambda d=d: get_tree_dicts(test_location)) + DatasetCatalog.register(name + "_" + d, lambda d=d: get_tree_dicts(test_location)) MetadataCatalog.get(name + "_" + d).set(thing_classes=["tree"]) @@ -615,7 +695,6 @@ def load_json_arr(json_path): lines.append(json.loads(line)) return lines - def setup_cfg( base_model="COCO-InstanceSegmentation/mask_rcnn_R_101_FPN_3x.yaml", trains=("trees_train",), From 1298f28659527774112f81d0a923529935907908 Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Sun, 8 Jan 2023 01:03:15 +0000 Subject: [PATCH 25/54] Update predict.py --- detectree2/models/predict.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/detectree2/models/predict.py b/detectree2/models/predict.py index b4131517..df8e2cd5 100644 --- a/detectree2/models/predict.py +++ b/detectree2/models/predict.py @@ -5,10 +5,14 @@ from pathlib import Path import cv2 +from detectron2.engine import DefaultPredictor from detectron2.evaluation.coco_evaluation import instances_to_coco_json - from detectree2.models.train import get_filenames, get_tree_dicts +# Code to convert RLE data from the output instances into Polygons, +# a small amout of info is lost but is fine. +# https://github.com/hazirbas/coco-json-converter/blob/master/generate_coco_json.py <-- found here + from .custom_nms import DefaultPredictor1 # Code to convert RLE data from the output instances into Polygons, From fc574f5d62dcbe8930402b56c477cea4bd722a00 Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Sun, 8 Jan 2023 01:31:32 +0000 Subject: [PATCH 26/54] Update train.py --- detectree2/models/train.py | 717 ++++++++++++++----------------------- 1 file changed, 264 insertions(+), 453 deletions(-) diff --git a/detectree2/models/train.py b/detectree2/models/train.py index 095d6469..4e13c282 100644 --- a/detectree2/models/train.py +++ b/detectree2/models/train.py @@ -1,3 +1,8 @@ +"""Train a model. + +Classes and functions to train a model based on othomosaics and corresponding +manual crown data. +""" import datetime import glob import json @@ -17,19 +22,25 @@ from detectron2.checkpoint import DetectionCheckpointer from detectron2.engine import DefaultPredictor, DefaultTrainer, hooks from detectron2.config import get_cfg -from detectron2.data import (DatasetCatalog, DatasetMapper, MetadataCatalog, - build_detection_test_loader, - build_detection_train_loader) +from detectron2.data import ( + DatasetCatalog, + DatasetMapper, + MetadataCatalog, + build_detection_test_loader, + build_detection_train_loader, +) from detectron2.engine import DefaultTrainer from detectron2.engine.hooks import HookBase from detectron2.evaluation import COCOEvaluator, verify_results from detectron2.evaluation.coco_evaluation import instances_to_coco_json from detectron2.structures import BoxMode -from detectron2.utils.events import EventStorage, get_event_storage +from detectron2.utils.events import get_event_storage # noqa:F401 +from detectron2.utils.events import EventStorage from detectron2.utils.logger import log_every_n_seconds from detectron2.utils.visualizer import ColorMode, Visualizer -from IPython.display import display -from PIL import Image + +# from IPython.display import display +# from PIL import Image from .custom_nms import custom_RPN, DefaultPredictor1 @@ -74,114 +85,89 @@ def _do_loss_eval(self): if idx == num_warmup: start_time = time.perf_counter() total_compute_time = 0 - start_compute_time = time.perf_counter() - if torch.cuda.is_available(): - torch.cuda.synchronize() - total_compute_time += time.perf_counter() - start_compute_time - iters_after_start = idx + 1 - num_warmup * int(idx >= num_warmup) - seconds_per_img = total_compute_time / iters_after_start - if idx >= num_warmup * 2 or seconds_per_img > 5: - total_seconds_per_img = (time.perf_counter() - - start_time) / iters_after_start - eta = datetime.timedelta(seconds=int(total_seconds_per_img * - (total - idx - 1))) - log_every_n_seconds( - logging.INFO, - "Loss on Validation done {}/{}. {:.4f} s / img. ETA={}".format( - idx + 1, total, seconds_per_img, str(eta)), - n=5, - ) - loss_batch = self._get_loss(inputs) - losses.append(loss_batch) - mean_loss = np.mean(losses) - self.trainer.storage.put_scalar("validation_loss", mean_loss, smoothing_hint = False) - - comm.synchronize() + losses = [] + for idx, inputs in enumerate(self._data_loader): + if idx == num_warmup: + start_time = time.perf_counter() + total_compute_time = 0 + start_compute_time = time.perf_counter() + if torch.cuda.is_available(): + torch.cuda.synchronize() + total_compute_time += time.perf_counter() - start_compute_time + iters_after_start = idx + 1 - num_warmup * int(idx >= num_warmup) + seconds_per_img = total_compute_time / iters_after_start + if idx >= num_warmup * 2 or seconds_per_img > 5: + total_seconds_per_img = (time.perf_counter() - start_time) / iters_after_start + eta = datetime.timedelta(seconds=int(total_seconds_per_img * (total - idx - 1))) + log_every_n_seconds( + logging.INFO, + "Loss on Validation done {}/{}. {:.4f} s / img. ETA={}".format(idx + 1, total, seconds_per_img, + str(eta)), + n=5, + ) + loss_batch = self._get_loss(inputs) + losses.append(loss_batch) + mean_loss = np.mean(losses) + # print(self.trainer.cfg.DATASETS.TEST) + # Combine the AP50s of the different datasets + if len(self.trainer.cfg.DATASETS.TEST) > 1: + APs = [] + for dataset in self.trainer.cfg.DATASETS.TEST: + APs.append(self.trainer.test(self.trainer.cfg, self.trainer.model)[dataset]["segm"]["AP50"]) + AP = sum(APs) / len(APs) + else: + AP = self.trainer.test(self.trainer.cfg, self.trainer.model)["segm"]["AP50"] + print("Av. AP50 =", AP) + self.trainer.APs.append(AP) + self.trainer.storage.put_scalar("validation_loss", mean_loss) + self.trainer.storage.put_scalar("validation_ap", AP) + comm.synchronize() + + return losses - #return losses - def _get_loss(self, data): - """How loss is calculated on train_loop + def _get_loss(self, data): + """Calculate loss in train_loop. Args: data (_type_): _description_ Returns: _type_: _description_ """ - # - metrics_dict = self._model(data) - metrics_dict = { - k: v.detach().cpu().item() if isinstance(v, torch.Tensor) else float(v) - for k, v in metrics_dict.items() - } - total_losses_reduced = sum(loss for loss in metrics_dict.values()) - return total_losses_reduced - - - '''early stop see of goodfellow''' - def after_step(self): - next_iter = self.trainer.iter + 1 - is_final = next_iter == self.trainer.max_iter - if is_final or (self._period > 0 and next_iter % self._period == 0): - if len(self.trainer.cfg.DATASETS.TEST) > 1: - APs = [] - AP_datasets = self.trainer.test( - self.trainer.cfg, - self.trainer.model) - for dataset in self.trainer.cfg.DATASETS.TEST: - APs.append(AP_datasets[dataset]['segm']['AP50']) - AP = sum(APs) / len(APs) - else: - AP = self.trainer.test(self.trainer.cfg, self.trainer.model)['segm']['AP50'] - print("Av. AP50 =", AP) - self.trainer.values.append(AP) - self.trainer.storage.put_scalar("validation_AP", AP, smoothing_hint = False) - if self.trainer.metrix == 'AP50': - if len(self.trainer.cfg.DATASETS.TEST) > 1: - APs = [] - AP_datasets = self.trainer.test_train( - self.trainer.cfg, - self.trainer.model) - for dataset in self.trainer.cfg.DATASETS.TEST: - APs.append(AP_datasets[dataset]['segm']['AP50']) - AP = sum(APs) / len(APs) - else: - AP = self.trainer.test_train(self.trainer.cfg, self.trainer.model)['segm']['AP50'] - self.trainer.storage.put_scalar("training_AP", AP, smoothing_hint = False) - elif self.trainer.metrix == 'loss': - self._do_loss_eval() - else: - if len(self.trainer.cfg.DATASETS.TEST) > 1: - APs = [] - AP_datasets = self.trainer.test_train( - self.trainer.cfg, - self.trainer.model) - for dataset in self.trainer.cfg.DATASETS.TRAIN: - APs.append(AP_datasets[dataset]['segm']['AP50']) - AP = sum(APs) / len(APs) - else: - AP = self.trainer.test(self.trainer.cfg, self.trainer.model)['segm']['AP50'] - self.trainer.storage.put_scalar("training_AP", AP, smoothing_hint = False) - loss = self._do_loss_eval() - if self.max_value < self.trainer.values[-1]: - self.iter = 0 - self.max_value = self.trainer.values[-1] - #self.checkpointer.save('model_' + str(len(self.trainer.values))) - torch.save(self._model.state_dict(), self.trainer.cfg.OUTPUT_DIR + '/Model_' + str(len(self.trainer.values)) + '.pth') - self.best_iter = self.trainer.iter - else: - self.iter += 1 - if self.iter == self.patience: - self.trainer.early_stop = True - print("Early stopping occurs in iter {}, max ap is {}".format(self.best_iter, self.max_value)) - self.trainer.storage.put_scalars(timetest=12) - - def after_train(self): - print('train done !!!') - if len(self.trainer.values) != 0: - index = self.trainer.values.index(max(self.trainer.values)) + 1 - print(self.trainer.early_stop,"best model is", index) - self.trainer.cfg.MODEL.WEIGHTS = self.trainer.cfg.OUTPUT_DIR + '/Model_' + str(index) + '.pth' - else: - print('train fails') + metrics_dict = self._model(data) + metrics_dict = { + k: v.detach().cpu().item() if isinstance(v, torch.Tensor) else float(v) + for k, v in metrics_dict.items() + } + total_losses_reduced = sum(loss for loss in metrics_dict.values()) + return total_losses_reduced + + def after_step(self): + next_iter = self.trainer.iter + 1 + is_final = next_iter == self.trainer.max_iter + if is_final or (self._period > 0 and next_iter % self._period == 0): + self._do_loss_eval() + if self.max_ap < self.trainer.APs[-1]: + self.iter = 0 + self.max_ap = self.trainer.APs[-1] + self.trainer.checkpointer.save("model_" + str(len(self.trainer.APs))) + self.best_iter = self.trainer.iter + else: + self.iter += 1 + if self.iter == self.patience: + self.trainer.early_stop = True + print("Early stopping occurs in iter {}, max ap is {}".format(self.best_iter, self.max_ap)) + self.trainer.storage.put_scalars(timetest=12) + + def after_train(self): + # Select the model with the best AP50 + index = self.trainer.APs.index(max(self.trainer.APs)) + 1 + # Error in demo: + # AssertionError: Checkpoint /__w/detectree2/detectree2/detectree2-data/paracou-out/train_outputs-1/model_1.pth + # not found! + # Therefore sleep is attempt to allow CI to pass, but it often still fails. + time.sleep(15) + self.trainer.checkpointer.load(self.trainer.cfg.OUTPUT_DIR + '/model_' + str(index) + '.pth') + +>>>>>>> master # See https://jss367.github.io/data-augmentation-in-detectron2.html for data augmentation advice class MyTrainer(DefaultTrainer): @@ -193,18 +179,18 @@ class MyTrainer(DefaultTrainer): """ # add a judge on if early-stopping in train function # train is inherited from TrainerBase https://detectron2.readthedocs.io/en/latest/_modules/detectron2/engine/train_loop.html + def __init__(self, cfg, patience = 5, training_metrix = 'loss'): self.patience = patience self.metrix = training_metrix super().__init__(cfg) + def train(self): """ Run training. - + Returns: OrderedDict of results, if evaluation is enabled. Otherwise None. - """ - """ Args: start_iter, max_iter (int): See docs above """ @@ -291,6 +277,7 @@ def run_step(self): """ self._trainer.optimizer.step() +<<<<<<< master def build_hooks(self): """ @@ -350,292 +337,101 @@ def build_hooks(self): ) return ret +def build_train_loader(cls, cfg): + """Summary. - @classmethod - def build_evaluator(cls, cfg, dataset_name, output_folder=None): - if output_folder is None: - os.makedirs("eval_2", exist_ok=True) - output_folder = "eval_2" - return COCOEvaluator(dataset_name, cfg, output_dir = output_folder) + Args: + cfg (_type_): _description_ + Returns: + _type_: _description_ + """ + augmentations = [ + T.RandomBrightness(0.8, 1.8), + T.RandomContrast(0.6, 1.3), + T.RandomSaturation(0.8, 1.4), + T.RandomRotation(angle=[90, 90], expand=False), + T.RandomLighting(0.7), + T.RandomFlip(prob=0.4, horizontal=True, vertical=False), + T.RandomFlip(prob=0.4, horizontal=False, vertical=True), + ] - def build_train_loader(cls, cfg): - """_summary_ - Args: - cfg (_type_): _description_ - Returns: - _type_: _description_ - """ - for i, datas in enumerate(DatasetCatalog.get(cfg.DATASETS.TRAIN[0])): - location = datas['file_name'] - size = cv2.imread(location).shape[0] - break + if cfg.RESIZE: + augmentations.append(T.Resize((1000, 1000))) + elif cfg.RESIZE == "random": + for i, datas in enumerate(DatasetCatalog.get(cfg.DATASETS.TRAIN[0])): + location = datas['file_name'] + size = cv2.imread(location).shape[0] + break + print("ADD RANDOM RESIZE WITH SIZE = ", size) + augmentations.append(T.ResizeScale(0.6, 1.4, size, size)) return build_detection_train_loader( cfg, mapper=DatasetMapper( cfg, is_train=True, - augmentations=[ - #T.Resize((800, 800)), - #T.Resize((random_size, random_size)), - T.ResizeScale(0.6, 1.4, size, size), - #T.RandomCrop('relative',(0.5,0.5)), - T.RandomBrightness(0.8, 1.8), - T.RandomContrast(0.6, 1.3), - T.RandomSaturation(0.8, 1.4), - T.RandomRotation(angle=[90, 90], expand=False), - T.RandomLighting(0.7), - T.RandomFlip(prob=0.4, horizontal=True, vertical=False), - T.RandomFlip(prob=0.4, horizontal=False, vertical=True), - ], + augmentations=augmentations, ), ) - # def build_optimizer(cls, cfg, model): - # """ - # Returns: - # torch.optim.Optimizer: - - # It now calls :func:`detectron2.solver.build_optimizer`. - # Overwrite it if you'd like a different optimizer. - # """ - # def build_optimizer_adam(cfg, model): - # """ - # Build an optimizer from config. - # """ - # norm_module_types = ( - # torch.nn.BatchNorm1d, - # torch.nn.BatchNorm2d, - # torch.nn.BatchNorm3d, - # torch.nn.SyncBatchNorm, - # # NaiveSyncBatchNorm inherits from BatchNorm2d - # torch.nn.GroupNorm, - # torch.nn.InstanceNorm1d, - # torch.nn.InstanceNorm2d, - # torch.nn.InstanceNorm3d, - # torch.nn.LayerNorm, - # torch.nn.LocalResponseNorm, - # ) - # params: List[Dict[str, Any]] = [] - # memo: Set[torch.nn.parameter.Parameter] = set() - # for module in model.modules(): - # for key, value in module.named_parameters(recurse=False): - # if not value.requires_grad: - # continue - # # Avoid duplicating parameters - # if value in memo: - # continue - # memo.add(value) - # lr = cfg.SOLVER.BASE_LR - # weight_decay = cfg.SOLVER.WEIGHT_DECAY - # if isinstance(module, norm_module_types): - # weight_decay = cfg.SOLVER.WEIGHT_DECAY_NORM - # elif key == "bias": - # # NOTE: unlike Detectron v1, we now default BIAS_LR_FACTOR to 1.0 - # # and WEIGHT_DECAY_BIAS to WEIGHT_DECAY so that bias optimizer - # # hyperparameters are by default exactly the same as for regular - # # weights. - # lr = cfg.SOLVER.BASE_LR * cfg.SOLVER.BIAS_LR_FACTOR - # #weight_decay = cfg.SOLVER.WEIGHT_DECAY_BIAS - # weight_decay = 0 - # params += [{"params": [value], "lr": lr, "weight_decay": weight_decay}] - - # optimizer = torch.optim.Adam(params, cfg.SOLVER.BASE_LR, cfg.betas, weight_decay = weight_decay) - # optimizer = maybe_add_gradient_clipping(cfg, optimizer) - # return optimizer - - # return build_optimizer_adam(cfg, model) - - @classmethod - def test_train(cls, cfg, model, evaluators=None): - """ - Evaluate the given model. The given model is expected to already contain - weights to evaluate. - - Args: - cfg (CfgNode): - model (nn.Module): - evaluators (list[DatasetEvaluator] or None): if None, will call - :meth:`build_evaluator`. Otherwise, must have the same length as - ``cfg.DATASETS.TEST``. - - Returns: - dict: a dict of result metrics - """ - logger = logging.getLogger(__name__) - if isinstance(evaluators, DatasetEvaluator): - evaluators = [evaluators] - if evaluators is not None: - assert len(cfg.DATASETS.TRAIN) == len(evaluators), "{} != {}".format( - len(cfg.DATASETS.TRAIN), len(evaluators) - ) - - results = OrderedDict() - for idx, dataset_name in enumerate(cfg.DATASETS.TRAIN): - data_loader = cls.build_test_loader(cfg, dataset_name) - # When evaluators are passed in as arguments, - # implicitly assume that evaluators can be created before data_loader. - if evaluators is not None: - evaluator = evaluators[idx] - else: - try: - evaluator = cls.build_evaluator(cfg, dataset_name) - except NotImplementedError: - logger.warn( - "No evaluator found. Use `DefaultTrainer.test(evaluators=)`, " - "or implement its `build_evaluator` method." - ) - results[dataset_name] = {} - continue - results_i = inference_on_dataset(model, data_loader, evaluator) - results[dataset_name] = results_i - if comm.is_main_process(): - assert isinstance( - results_i, dict - ), "Evaluator must return a dict on the main process. Got {} instead.".format( - results_i - ) - logger.info("Evaluation results for {} in csv format:".format(dataset_name)) - print_csv_format(results_i) - - if len(results) == 1: - results = list(results.values())[0] - return results - - -def get_tree_dicts(directory: str, classes: List[str] = None, classes_at: str = None) -> List[Dict]: - """Get the tree dictionaries. - Args: - directory: Path to directory - classes: Signifies which column (if any) corresponds to the class labels - Returns: - List of dictionaries corresponding to segmentations of trees. Each dictionary includes - bounding box around tree and points tracing a polygon around a tree. +@classmethod + def test_train(cls, cfg, model, evaluators=None): """ - # filepath = '/content/drive/MyDrive/forestseg/paracou_data/Panayiotis_Outputs/220303_AllSpLabelled.gpkg' - # datagpd = gpd.read_file(filepath) - # List_Genus = datagpd.Genus_Species.to_list() - # Genus_Species_UniqueList = list(set(List_Genus)) - - # - if classes is not None: - # list_of_classes = crowns[variable].unique().tolist() - classes = classes - else: - classes = ["tree"] - # classes = Genus_Species_UniqueList #['tree'] # genus_species list - dataset_dicts = [] - # for root, dirs, files in os.walk(train_location): - # for file in files: - # if file.endswith(".geojson"): - # print(os.path.join(root, file)) - - for filename in [file for file in os.listdir(directory) if file.endswith(".geojson")]: - json_file = os.path.join(directory, filename) - with open(json_file) as f: - img_anns = json.load(f) - # Turn off type checking for annotations until we have a better solution - record: Dict[str, Any] = {} - - # filename = os.path.join(directory, img_anns["imagePath"]) - filename = img_anns["imagePath"] - - # Make sure we have the correct height and width - height, width = cv2.imread(filename).shape[:2] - - record["file_name"] = filename - record["height"] = height - record["width"] = width - record["image_id"] = filename[0:400] - record["annotations"] = {} - # print(filename[0:400]) - - objs = [] - for features in img_anns["features"]: - anno = features["geometry"] - # pdb.set_trace() - # GenusSpecies = features['properties']['Genus_Species'] - px = [a[0] for a in anno["coordinates"][0]] - py = [np.array(height) - a[1] for a in anno["coordinates"][0]] - # print("### HERE IS PY ###", py) - poly = [(x, y) for x, y in zip(px, py)] - poly = [p for x in poly for p in x] - # print("#### HERE ARE SOME POLYS #####", poly) - if classes != ["tree"]: - obj = { - "bbox": [np.min(px), np.min(py), np.max(px), np.max(py)], - "bbox_mode": BoxMode.XYXY_ABS, - "segmentation": [poly], - "category_id": classes.index(features["properties"][classes_at]), # id - # "category_id": 0, #id - "iscrowd": 0, - } - else: - obj = { - "bbox": [np.min(px), np.min(py), np.max(px), np.max(py)], - "bbox_mode": BoxMode.XYXY_ABS, - "segmentation": [poly], - "category_id": 0, # id - "iscrowd": 0, - } - # pdb.set_trace() - objs.append(obj) - # print("#### HERE IS OBJS #####", objs) - record["annotations"] = objs - dataset_dicts.append(record) - return dataset_dicts - - -def combine_dicts(root_dir: str, - val_dir: int, - mode: str = "train", - classes: List[str] = None, - classes_at: str = None) -> List[Dict]: - """Join tree dicts from different directories. + Evaluate the given model. The given model is expected to already contain + weights to evaluate. + Args: - root_dir: - val_dir: + cfg (CfgNode): + model (nn.Module): + evaluators (list[DatasetEvaluator] or None): if None, will call + :meth:`build_evaluator`. Otherwise, must have the same length as + ``cfg.DATASETS.TEST``. + Returns: - Concatenated array of dictionaries over all directories + dict: a dict of result metrics """ - train_dirs = [os.path.join(root_dir, dir) for dir in os.listdir(root_dir)] - if mode == "train": - del train_dirs[(val_dir - 1)] - tree_dicts = [] - for d in train_dirs: - tree_dicts += get_tree_dicts(d, classes=classes, classes_at=classes_at) - elif mode == "val": - tree_dicts = get_tree_dicts(train_dirs[(val_dir - 1)], classes=classes, classes_at=classes_at) - elif mode == "full": - tree_dicts = [] - for d in train_dirs: - tree_dicts += get_tree_dicts(d, classes=classes, classes_at=classes_at) - return tree_dicts - - -def get_filenames(directory: str): - """Get the file names if no geojson is present. - Allows for predictions where no delinations have been manually produced. - Args: - directory (str): directory of images to be predicted on - """ - dataset_dicts = [] - files = glob.glob(directory + "*.png") - for filename in [file for file in files]: - file = {} - filename = os.path.join(directory, filename) - file["file_name"] = filename - - dataset_dicts.append(file) - return dataset_dicts - - -def register_train_data(train_location, - name: str = "tree", - val_fold=None, - classes=None, - classes_at=None): - """Register data for training and (optionally) validation. + logger = logging.getLogger(__name__) + if isinstance(evaluators, DatasetEvaluator): + evaluators = [evaluators] + if evaluators is not None: + assert len(cfg.DATASETS.TRAIN) == len(evaluators), "{} != {}".format( + len(cfg.DATASETS.TRAIN), len(evaluators) + ) + + results = OrderedDict() + for idx, dataset_name in enumerate(cfg.DATASETS.TRAIN): + data_loader = cls.build_test_loader(cfg, dataset_name) + # When evaluators are passed in as arguments, + # implicitly assume that evaluators can be created before data_loader. + if evaluators is not None: + evaluator = evaluators[idx] + else: + try: + evaluator = cls.build_evaluator(cfg, dataset_name) + except NotImplementedError: + logger.warn( + "No evaluator found. Use `DefaultTrainer.test(evaluators=)`, " + "or implement its `build_evaluator` method." + ) + results[dataset_name] = {} + continue + results_i = inference_on_dataset(model, data_loader, evaluator) + results[dataset_name] = results_i + if comm.is_main_process(): + assert isinstance( + results_i, dict + ), "Evaluator must return a dict on the main process. Got {} instead.".format( + results_i + ) + logger.info("Evaluation results for {} in csv format:".format(dataset_name)) + print_csv_format(results_i) + + if len(results) == 1: + results = list(results.values())[0] + return results + +def get_tree_dicts(directory: str, classes: List[str] = None, classes_at: str = None) -> List[Dict]: + """Get the tree dictionaries. Args: train_location: directory containing training folds name: string to name data @@ -696,58 +492,85 @@ def load_json_arr(json_path): return lines def setup_cfg( - base_model="COCO-InstanceSegmentation/mask_rcnn_R_101_FPN_3x.yaml", - trains=("trees_train",), - tests=("trees_val",), + base_model: str = "COCO-InstanceSegmentation/mask_rcnn_R_101_FPN_3x.yaml", + trains=("trees_train", ), + tests=("trees_val", ), update_model=None, workers=2, - ims_per_batch=1, - base_lr=0.0003, + ims_per_batch=2, + gamma=0.1, + backbone_freeze=3, + warm_iter=120, + momentum=0.9, + batch_size_per_im=1024, + base_lr=0.0003389, + weight_decay=0.001, max_iter=1000, num_classes=1, eval_period=100, - out_dir="/content/drive/Shareddrives/detectree2/train_outputs"): - """ - To set up config object - """ - cfg = get_cfg() - cfg.merge_from_file(model_zoo.get_config_file(base_model)) - cfg.DATASETS.TRAIN = trains - cfg.DATASETS.TEST = tests - cfg.DATALOADER.NUM_WORKERS = workers - cfg.OUTPUT_DIR = out_dir - cfg.SOLVER.IMS_PER_BATCH = 2 - cfg.SOLVER.GAMMA = 0.1 - cfg.MODEL.BACKBONE.FREEZE_AT = 3 - cfg.SOLVER.WARMUP_ITERS = 120 - cfg.SOLVER.MOMENTUM = 0.9 - cfg.MODEL.RPN.BATCH_SIZE_PER_IMAGE = 128 - cfg.SOLVER.WEIGHT_DECAY = 0 - cfg.SOLVER.BASE_LR = 0.001 - cfg.betas = (0.9, 0.999) - os.makedirs(cfg.OUTPUT_DIR, exist_ok=True) - if update_model is not None: - cfg.MODEL.WEIGHTS = update_model # DOESN'T WORK - else: - cfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url(base_model) - - cfg.SOLVER.IMS_PER_BATCH = ims_per_batch - cfg.SOLVER.BASE_LR = base_lr - cfg.SOLVER.MAX_ITER = max_iter - cfg.MODEL.ROI_HEADS.NUM_CLASSES = num_classes - cfg.TEST.EVAL_PERIOD = eval_period - cfg.MODEL.BACKBONE.FREEZE_AT = 2 - #cfg.MODEL.ROI_BOX_HEAD.BBOX_REG_LOSS_TYPE = 'diou' - cfg.MODEL.PROPOSAL_GENERATOR.NAME = 'custom_RPN' - cfg.nms_thresh_union = 0.2 - cfg.MODEL.RPN.NMS_THRESH = 0 - - #cfg.nms_thresh_union = 0 - #cfg.MODEL.ROI_HEADS.NMS_THRESH_TEST = 0.1 - - #cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5 - cfg.MODEL.ROI_HEADS.NUM_CLASSES = 1 # only has one class (pnumonia) - return cfg + nms_thresh_union = 0.1, + rpn_nms_thresh = 0, + nms_thresh_test = 0.2, + out_dir="/content/drive/Shareddrives/detectree2/train_outputs", + resize=True, +): + """Set up config object # noqa: D417. + + Args: + base_model: base pre-trained model from detectron2 model_zoo + trains: names of registered data to use for training + tests: names of registered data to use for evaluating models + update_model: updated pre-trained model from detectree2 model_garden + workers: + ims_per_batch: + gamma: + backbone_freeze: + warm_iter: + momentum: + batch_size_per_im: + base_lr: + weight_decay + max_iter: + num_classes: + eval_period: + out_dir: + """ + cfg = get_cfg() + cfg.merge_from_file(model_zoo.get_config_file(base_model)) + cfg.DATASETS.TRAIN = trains + cfg.DATASETS.TEST = tests + cfg.DATALOADER.NUM_WORKERS = workers + cfg.SOLVER.IMS_PER_BATCH = ims_per_batch + cfg.SOLVER.GAMMA = gamma + cfg.MODEL.BACKBONE.FREEZE_AT = backbone_freeze + cfg.SOLVER.WARMUP_ITERS = warm_iter + cfg.SOLVER.MOMENTUM = momentum + cfg.MODEL.RPN.BATCH_SIZE_PER_IMAGE = batch_size_per_im + cfg.SOLVER.WEIGHT_DECAY = weight_decay + cfg.SOLVER.BASE_LR = base_lr + cfg.OUTPUT_DIR = out_dir + os.makedirs(cfg.OUTPUT_DIR, exist_ok=True) + if update_model is not None: + cfg.MODEL.WEIGHTS = update_model + else: + cfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url(base_model) + + cfg.SOLVER.IMS_PER_BATCH = ims_per_batch + cfg.SOLVER.BASE_LR = base_lr + cfg.SOLVER.MAX_ITER = max_iter + cfg.MODEL.ROI_HEADS.NUM_CLASSES = num_classes + cfg.TEST.EVAL_PERIOD = eval_period + cfg.RESIZE = resize + cfg.INPUT.MIN_SIZE_TRAIN = 1000 + + cfg.MODEL.BACKBONE.FREEZE_AT = 2 + #cfg.MODEL.ROI_BOX_HEAD.BBOX_REG_LOSS_TYPE = 'diou' + cfg.MODEL.PROPOSAL_GENERATOR.NAME = 'custom_RPN' + cfg.nms_thresh_union = rpn_nms_thresh + cfg.MODEL.RPN.NMS_THRESH = rpn_nms_thresh + cfg.MODEL.ROI_HEADS.NMS_THRESH_TEST = nms_thresh_test + + return cfg def predictions_on_data(directory=None, predictor=DefaultPredictor1, @@ -756,12 +579,10 @@ def predictions_on_data(directory=None, scale=1, geos_exist=True, num_predictions=0): - """ - Prediction produced from a test folder and outputted to predictions folder - """ + """Prediction produced from a test folder and outputted to predictions folder.""" - test_location = directory + "test" - pred_dir = directory + "predictions" + test_location = directory + "/test" + pred_dir = test_location + "/predictions" Path(pred_dir).mkdir(parents=True, exist_ok=True) @@ -785,10 +606,10 @@ def predictions_on_data(directory=None, metadata=trees_metadata, scale=scale, instance_mode=ColorMode.SEGMENTATION, - ) # remove the colors of unsegmented pixels + ) # remove the colors of unsegmented pixels v = v.draw_instance_predictions(outputs["instances"].to("cpu")) - image = cv2.cvtColor(v.get_image()[:, :, ::-1], cv2.COLOR_BGR2RGB) - display(Image.fromarray(image)) + # image = cv2.cvtColor(v.get_image()[:, :, ::-1], cv2.COLOR_BGR2RGB) + # display(Image.fromarray(image)) # Creating the file name of the output file file_name_path = d["file_name"] @@ -797,34 +618,29 @@ def predictions_on_data(directory=None, file_name = file_name.replace("png", "json") output_file = pred_dir + "/Prediction_" + file_name - print(output_file) if save: # Converting the predictions to json files and saving them in the specfied output file. - evaluations = instances_to_coco_json(outputs["instances"].to("cpu"), - d["file_name"]) + evaluations = instances_to_coco_json(outputs["instances"].to("cpu"), d["file_name"]) with open(output_file, "w") as dest: json.dump(evaluations, dest) if __name__ == "__main__": train_location = "/content/drive/Shareddrives/detectree2/data/Paracou/tiles/train/" - register_train_data(train_location, "Paracou", - 1) # folder, name, validation fold + register_train_data(train_location, "Paracou", 1) # folder, name, validation fold name = "Paracou2019" train_location = "/content/drive/Shareddrives/detectree2/data/Paracou/tiles2019/train/" dataset_dicts = combine_dicts(train_location, 1) trees_metadata = MetadataCatalog.get(name + "_train") - #dataset_dicts = get_tree_dicts("./") + # dataset_dicts = get_tree_dicts("./") for d in dataset_dicts: img = cv2.imread(d["file_name"]) - visualizer = Visualizer(img[:, :, ::-1], - metadata=trees_metadata, - scale=0.5) + visualizer = Visualizer(img[:, :, ::-1], metadata=trees_metadata, scale=0.5) out = visualizer.draw_dataset_dict(d) image = cv2.cvtColor(out.get_image()[:, :, ::-1], cv2.COLOR_BGR2RGB) - display(Image.fromarray(image)) + # display(Image.fromarray(image)) # Set the base (pre-trained) model from the detectron2 model_zoo model = "COCO-InstanceSegmentation/mask_rcnn_R_101_FPN_3x.yaml" # Set the names of the registered train and test sets @@ -849,12 +665,7 @@ def predictions_on_data(directory=None, out_dir = "/content/drive/Shareddrives/detectree2/220703_train_outputs" # update_model arg can be used to load in trained model - cfg = setup_cfg(model, - trains, - tests, - eval_period=100, - max_iter=3000, - out_dir=out_dir) + cfg = setup_cfg(model, trains, tests, eval_period=100, max_iter=3000, out_dir=out_dir) trainer = MyTrainer(cfg, patience=4) trainer.resume_or_load(resume=False) trainer.train() From 32676d2b4e6fe1abc89f7135176f78bbdfebfb94 Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Sun, 8 Jan 2023 01:40:08 +0000 Subject: [PATCH 27/54] Update train.py --- detectree2/models/train.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/detectree2/models/train.py b/detectree2/models/train.py index 4e13c282..aed2fb0c 100644 --- a/detectree2/models/train.py +++ b/detectree2/models/train.py @@ -167,8 +167,6 @@ def after_train(self): time.sleep(15) self.trainer.checkpointer.load(self.trainer.cfg.OUTPUT_DIR + '/model_' + str(index) + '.pth') ->>>>>>> master - # See https://jss367.github.io/data-augmentation-in-detectron2.html for data augmentation advice class MyTrainer(DefaultTrainer): """_summary_ @@ -375,7 +373,7 @@ def build_train_loader(cls, cfg): ) @classmethod - def test_train(cls, cfg, model, evaluators=None): +def test_train(cls, cfg, model, evaluators=None): """ Evaluate the given model. The given model is expected to already contain weights to evaluate. From c150ee03d58cf4eec6de495afcc0535ac041a3fb Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Sun, 8 Jan 2023 23:37:15 +0000 Subject: [PATCH 28/54] Update train.py --- detectree2/models/train.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/detectree2/models/train.py b/detectree2/models/train.py index aed2fb0c..0b5a555d 100644 --- a/detectree2/models/train.py +++ b/detectree2/models/train.py @@ -274,8 +274,6 @@ def run_step(self): suboptimal as explained in https://arxiv.org/abs/2006.15704 Sec 3.2.4 """ self._trainer.optimizer.step() - -<<<<<<< master def build_hooks(self): """ From bf731132df973f5f48ee7b8a0f983c9acb9876df Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Mon, 9 Jan 2023 01:33:04 +0000 Subject: [PATCH 29/54] Update train.py --- detectree2/models/train.py | 129 +++++++++++++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) diff --git a/detectree2/models/train.py b/detectree2/models/train.py index 0b5a555d..ae95057c 100644 --- a/detectree2/models/train.py +++ b/detectree2/models/train.py @@ -428,6 +428,135 @@ def test_train(cls, cfg, model, evaluators=None): def get_tree_dicts(directory: str, classes: List[str] = None, classes_at: str = None) -> List[Dict]: """Get the tree dictionaries. + Args: + directory: Path to directory + classes: Signifies which column (if any) corresponds to the class labels + Returns: + List of dictionaries corresponding to segmentations of trees. Each dictionary includes + bounding box around tree and points tracing a polygon around a tree. + """ + # filepath = '/content/drive/MyDrive/forestseg/paracou_data/Panayiotis_Outputs/220303_AllSpLabelled.gpkg' + # datagpd = gpd.read_file(filepath) + # List_Genus = datagpd.Genus_Species.to_list() + # Genus_Species_UniqueList = list(set(List_Genus)) + + # + if classes is not None: + # list_of_classes = crowns[variable].unique().tolist() + classes = classes + else: + classes = ["tree"] + # classes = Genus_Species_UniqueList #['tree'] # genus_species list + dataset_dicts = [] + # for root, dirs, files in os.walk(train_location): + # for file in files: + # if file.endswith(".geojson"): + # print(os.path.join(root, file)) + + for filename in [file for file in os.listdir(directory) if file.endswith(".geojson")]: + json_file = os.path.join(directory, filename) + with open(json_file) as f: + img_anns = json.load(f) + # Turn off type checking for annotations until we have a better solution + record: Dict[str, Any] = {} + + # filename = os.path.join(directory, img_anns["imagePath"]) + filename = img_anns["imagePath"] + + # Make sure we have the correct height and width + height, width = cv2.imread(filename).shape[:2] + + record["file_name"] = filename + record["height"] = height + record["width"] = width + record["image_id"] = filename[0:400] + record["annotations"] = {} + # print(filename[0:400]) + + objs = [] + for features in img_anns["features"]: + anno = features["geometry"] + # pdb.set_trace() + # GenusSpecies = features['properties']['Genus_Species'] + px = [a[0] for a in anno["coordinates"][0]] + py = [np.array(height) - a[1] for a in anno["coordinates"][0]] + # print("### HERE IS PY ###", py) + poly = [(x, y) for x, y in zip(px, py)] + poly = [p for x in poly for p in x] + # print("#### HERE ARE SOME POLYS #####", poly) + if classes != ["tree"]: + obj = { + "bbox": [np.min(px), np.min(py), np.max(px), np.max(py)], + "bbox_mode": BoxMode.XYXY_ABS, + "segmentation": [poly], + "category_id": classes.index(features["properties"][classes_at]), # id + # "category_id": 0, #id + "iscrowd": 0, + } + else: + obj = { + "bbox": [np.min(px), np.min(py), np.max(px), np.max(py)], + "bbox_mode": BoxMode.XYXY_ABS, + "segmentation": [poly], + "category_id": 0, # id + "iscrowd": 0, + } + # pdb.set_trace() + objs.append(obj) + # print("#### HERE IS OBJS #####", objs) + record["annotations"] = objs + dataset_dicts.append(record) + return dataset_dicts + + +def combine_dicts(root_dir: str, + val_dir: int, + mode: str = "train", + classes: List[str] = None, + classes_at: str = None) -> List[Dict]: + """Join tree dicts from different directories. + Args: + root_dir: + val_dir: + Returns: + Concatenated array of dictionaries over all directories + """ + train_dirs = [os.path.join(root_dir, dir) for dir in os.listdir(root_dir)] + if mode == "train": + del train_dirs[(val_dir - 1)] + tree_dicts = [] + for d in train_dirs: + tree_dicts += get_tree_dicts(d, classes=classes, classes_at=classes_at) + elif mode == "val": + tree_dicts = get_tree_dicts(train_dirs[(val_dir - 1)], classes=classes, classes_at=classes_at) + elif mode == "full": + tree_dicts = [] + for d in train_dirs: + tree_dicts += get_tree_dicts(d, classes=classes, classes_at=classes_at) + return tree_dicts + +def get_filenames(directory: str): + """Get the file names if no geojson is present. + Allows for predictions where no delinations have been manually produced. + Args: + directory (str): directory of images to be predicted on + """ + dataset_dicts = [] + files = glob.glob(directory + "*.png") + for filename in [file for file in files]: + file = {} + filename = os.path.join(directory, filename) + file["file_name"] = filename + + dataset_dicts.append(file) + return dataset_dicts + +def register_train_data(train_location, + name: str = "tree", + val_fold=None, + classes=None, + classes_at=None): + """Register data for training and (optionally) validation. Args: train_location: directory containing training folds name: string to name data From 233bb6cd7519bca53f7d50edefdf7f03a827c174 Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Fri, 13 Jan 2023 00:28:50 +0000 Subject: [PATCH 30/54] Update train.py --- detectree2/models/train.py | 76 ++++++++++---------------------------- 1 file changed, 19 insertions(+), 57 deletions(-) diff --git a/detectree2/models/train.py b/detectree2/models/train.py index ae95057c..f70030ad 100644 --- a/detectree2/models/train.py +++ b/detectree2/models/train.py @@ -275,63 +275,25 @@ def run_step(self): """ self._trainer.optimizer.step() - def build_hooks(self): - """ - Build a list of default hooks, including timing, evaluation, - checkpointing, lr scheduling, precise BN, writing events. - Returns: - list[HookBase]: - """ - cfg = self.cfg.clone() - cfg.defrost() - cfg.DATALOADER.NUM_WORKERS = 0 # save some memory and time for PreciseBN - - ret = [ - hooks.IterationTimer(), - hooks.LRScheduler(), - hooks.PreciseBN( - # Run at the same freq as (but before) evaluation. - cfg.TEST.EVAL_PERIOD, - self.model, - # Build a new data loader to not affect training - self.build_train_loader(cfg), - cfg.TEST.PRECISE_BN.NUM_ITER, - ) - if cfg.TEST.PRECISE_BN.ENABLED and get_bn_modules(self.model) - else None, - ] - - # Do PreciseBN before checkpointer, because it updates the model and need to - # be saved by checkpointer. - # This is not always the best: if checkpointing has a different frequency, - # some checkpoints may have more precise statistics than others. - if comm.is_main_process(): - ret.append(hooks.PeriodicCheckpointer(self.checkpointer, cfg.SOLVER.CHECKPOINT_PERIOD)) - - # def test_and_save_results(): - # self._last_eval_results = self.test(self.cfg, self.model) - # return self._last_eval_results - - # # Do evaluation after checkpointer, because then if it fails, - # # we can use the saved checkpoint to debug. - # ret.append(hooks.EvalHook(cfg.TEST.EVAL_PERIOD, test_and_save_results)) - - if comm.is_main_process(): - # Here the default print/log frequency of each writer is used. - # run writers in the end, so that evaluation metrics are written - ret.append(hooks.PeriodicWriter(self.build_writers(), period=20)) - ret.insert( - -1, - LossEvalHook( - self.cfg.TEST.EVAL_PERIOD, - self.model, - build_detection_test_loader(self.cfg, self.cfg.DATASETS.TEST[0], - DatasetMapper(self.cfg, True)), - self.patience, - self.cfg.OUTPUT_DIR, - ), - ) - return ret +def build_hooks(self): + hooks = super().build_hooks() + # augmentations = [T.ResizeShortestEdge(short_edge_length=(1000, 1000), + # max_size=1333, + # sample_style='choice')] + hooks.insert( + -1, + LossEvalHook( + self.cfg.TEST.EVAL_PERIOD, + self.model, + build_detection_test_loader( + self.cfg, + self.cfg.DATASETS.TEST, + DatasetMapper(self.cfg, True) + ), + self.patience, + ), + ) + return hooks def build_train_loader(cls, cfg): """Summary. From 9e4b26642484d52f98c0bb9d3f2493aa8f62a41f Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Fri, 13 Jan 2023 00:34:56 +0000 Subject: [PATCH 31/54] Update train.py --- detectree2/models/train.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/detectree2/models/train.py b/detectree2/models/train.py index f70030ad..092b5273 100644 --- a/detectree2/models/train.py +++ b/detectree2/models/train.py @@ -250,13 +250,7 @@ def run_step(self): losses = loss_dict loss_dict = {"total_loss": loss_dict} else: - # loss_dict['cls'] = torch.tensor(0) - # loss_dict['loss_rpn_cls'] = torch.tensor(0) - # if self.iter > 1000: - # self.reweight = True - # if self.reweight: - # loss_dict['loss_mask'] *= 4 - loss_dict['loss_mask'] *= 0.8 + loss_dict['loss_mask'] *= self.cfg.mask_weight losses = sum(loss_dict.values()) """ @@ -274,7 +268,14 @@ def run_step(self): suboptimal as explained in https://arxiv.org/abs/2006.15704 Sec 3.2.4 """ self._trainer.optimizer.step() - + +@classmethod +def build_evaluator(cls, cfg, dataset_name, output_folder=None): + if output_folder is None: + os.makedirs("eval", exist_ok=True) + output_folder = "eval" + return COCOEvaluator(dataset_name, cfg, True, output_folder) + def build_hooks(self): hooks = super().build_hooks() # augmentations = [T.ResizeShortestEdge(short_edge_length=(1000, 1000), From c03d628dc1281372a50d56fbb711a4d2138948ca Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Fri, 13 Jan 2023 00:37:07 +0000 Subject: [PATCH 32/54] Update train.py --- detectree2/models/train.py | 58 ++------------------------------------ 1 file changed, 2 insertions(+), 56 deletions(-) diff --git a/detectree2/models/train.py b/detectree2/models/train.py index 092b5273..c1dbf6d1 100644 --- a/detectree2/models/train.py +++ b/detectree2/models/train.py @@ -333,62 +333,6 @@ def build_train_loader(cls, cfg): ), ) -@classmethod -def test_train(cls, cfg, model, evaluators=None): - """ - Evaluate the given model. The given model is expected to already contain - weights to evaluate. - - Args: - cfg (CfgNode): - model (nn.Module): - evaluators (list[DatasetEvaluator] or None): if None, will call - :meth:`build_evaluator`. Otherwise, must have the same length as - ``cfg.DATASETS.TEST``. - - Returns: - dict: a dict of result metrics - """ - logger = logging.getLogger(__name__) - if isinstance(evaluators, DatasetEvaluator): - evaluators = [evaluators] - if evaluators is not None: - assert len(cfg.DATASETS.TRAIN) == len(evaluators), "{} != {}".format( - len(cfg.DATASETS.TRAIN), len(evaluators) - ) - - results = OrderedDict() - for idx, dataset_name in enumerate(cfg.DATASETS.TRAIN): - data_loader = cls.build_test_loader(cfg, dataset_name) - # When evaluators are passed in as arguments, - # implicitly assume that evaluators can be created before data_loader. - if evaluators is not None: - evaluator = evaluators[idx] - else: - try: - evaluator = cls.build_evaluator(cfg, dataset_name) - except NotImplementedError: - logger.warn( - "No evaluator found. Use `DefaultTrainer.test(evaluators=)`, " - "or implement its `build_evaluator` method." - ) - results[dataset_name] = {} - continue - results_i = inference_on_dataset(model, data_loader, evaluator) - results[dataset_name] = results_i - if comm.is_main_process(): - assert isinstance( - results_i, dict - ), "Evaluator must return a dict on the main process. Got {} instead.".format( - results_i - ) - logger.info("Evaluation results for {} in csv format:".format(dataset_name)) - print_csv_format(results_i) - - if len(results) == 1: - results = list(results.values())[0] - return results - def get_tree_dicts(directory: str, classes: List[str] = None, classes_at: str = None) -> List[Dict]: """Get the tree dictionaries. Args: @@ -599,6 +543,7 @@ def setup_cfg( nms_thresh_union = 0.1, rpn_nms_thresh = 0, nms_thresh_test = 0.2, + mask_weight = 1, out_dir="/content/drive/Shareddrives/detectree2/train_outputs", resize=True, ): @@ -652,6 +597,7 @@ def setup_cfg( cfg.INPUT.MIN_SIZE_TRAIN = 1000 cfg.MODEL.BACKBONE.FREEZE_AT = 2 + cfg.mask_weight = mask_weight #cfg.MODEL.ROI_BOX_HEAD.BBOX_REG_LOSS_TYPE = 'diou' cfg.MODEL.PROPOSAL_GENERATOR.NAME = 'custom_RPN' cfg.nms_thresh_union = rpn_nms_thresh From f31c032d807032e53788a037946de62a42874a4b Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Fri, 13 Jan 2023 01:43:29 +0000 Subject: [PATCH 33/54] Update train.py --- detectree2/models/train.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/detectree2/models/train.py b/detectree2/models/train.py index c1dbf6d1..385a3781 100644 --- a/detectree2/models/train.py +++ b/detectree2/models/train.py @@ -599,9 +599,10 @@ def setup_cfg( cfg.MODEL.BACKBONE.FREEZE_AT = 2 cfg.mask_weight = mask_weight #cfg.MODEL.ROI_BOX_HEAD.BBOX_REG_LOSS_TYPE = 'diou' - cfg.MODEL.PROPOSAL_GENERATOR.NAME = 'custom_RPN' - cfg.nms_thresh_union = rpn_nms_thresh + #cfg.MODEL.PROPOSAL_GENERATOR.NAME = 'custom_RPN' + cfg.nms_thresh_union = nms_thresh_union cfg.MODEL.RPN.NMS_THRESH = rpn_nms_thresh + #cfg.MODEL.RPN.IOA_THRESHOLDS = ? cfg.MODEL.ROI_HEADS.NMS_THRESH_TEST = nms_thresh_test return cfg From eef70a03ff7c6bba77320926b41e70e03a6bdf29 Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Fri, 13 Jan 2023 01:43:36 +0000 Subject: [PATCH 34/54] Update custom_nms.py --- detectree2/models/custom_nms.py | 223 ++++++++++++++++++++++++++++++-- 1 file changed, 212 insertions(+), 11 deletions(-) diff --git a/detectree2/models/custom_nms.py b/detectree2/models/custom_nms.py index 80d7c970..dfdd1774 100644 --- a/detectree2/models/custom_nms.py +++ b/detectree2/models/custom_nms.py @@ -47,8 +47,149 @@ def _is_tracing(): # https://github.com/pytorch/pytorch/issues/47379 return False else: - return torch.jit.is_tracing() - + return torch.jit.is_tracing() + +def pairwise_ioa(boxes1: Boxes, boxes2: Boxes) -> torch.Tensor: + """ + Similar to :func:`pariwise_iou` but compute the IoA (intersection over boxes2 area). + Args: + boxes1,boxes2 (Boxes): two `Boxes`. Contains N & M boxes, respectively. + Returns: + Tensor: IoA, sized [N,M]. + """ + area2 = boxes2.area() # [M] + area1 = boxes1.area() # [M] + inter = pairwise_intersection(boxes1, boxes2) + + # handle empty boxes + ioa_dif = torch.where( + inter > 0, abs(inter / area2 - inter / area1), torch.zeros(1, dtype=inter.dtype, device=inter.device) + ) + return ioa_dif + +class Ioa_Matcher(object): + """ + This class assigns to each predicted "element" (e.g., a box) a ground-truth + element. Each predicted element will have exactly zero or one matches; each + ground-truth element may be matched to zero or more predicted elements. + The matching is determined by the MxN match_quality_matrix, that characterizes + how well each (ground-truth, prediction)-pair match each other. For example, + if the elements are boxes, this matrix may contain box intersection-over-union + overlap values. + The matcher returns (a) a vector of length N containing the index of the + ground-truth element m in [0, M) that matches to prediction n in [0, N). + (b) a vector of length N containing the labels for each prediction. + """ + + def __init__( + self, thresholds: List[float], thresholds_ioa: List[float], labels: List[int], allow_low_quality_matches: bool = False + ): + """ + Args: + thresholds (list): a list of thresholds used to stratify predictions + into levels. + thresholds_ioa (list): a list of thresholds used to stratify predictions + into levels. + labels (list): a list of values to label predictions belonging at + each level. A label can be one of {-1, 0, 1} signifying + {ignore, negative class, positive class}, respectively. + allow_low_quality_matches (bool): if True, produce additional matches + for predictions with maximum match quality lower than high_threshold. + See set_low_quality_matches_ for more details. + For example, + thresholds = [0.3, 0.5] + thresholds_ioa = 0.5 + labels = [0, -1, 1] + All predictions with iou < 0.3 and ioa_dif > 0.5 will be marked with 0 and + thus will be considered as false positives while training. + All predictions with 0.3 <= iou < 0.5 will be marked with -1 and + thus will be ignored. + All predictions with 0.5 <= iou and ioa_dif < 0.5 will be marked with 1 and + thus will be considered as true positives. + """ + # Add -inf and +inf to first and last position in thresholds + thresholds = thresholds[:] + assert thresholds[0] > 0 + thresholds.insert(0, -float("inf")) + thresholds.append(float("inf")) + # Currently torchscript does not support all + generator + assert all([low <= high for (low, high) in zip(thresholds[:-1], thresholds[1:])]) + assert all([l in [-1, 0, 1] for l in labels]) + assert len(labels) == len(thresholds) - 1 + self.thresholds = thresholds + self.labels = labels + self.allow_low_quality_matches = allow_low_quality_matches + + def __call__(self, match_quality_matrix, match_quality_matrix_ioa): + """ + Args: + match_quality_matrix (Tensor[float]): an MxN tensor, containing the + pairwise quality between M ground-truth elements and N predicted + elements. All elements must be >= 0 (due to the us of `torch.nonzero` + for selecting indices in :meth:`set_low_quality_matches_`). + match_quality_matrix (Tensor[float]): an MxN tensor, containing the + two ioa difference between M ground-truth elements and N predicted + elements. All elements must be >= 0 (due to the us of `torch.nonzero` + for selecting indices in :meth:`set_low_quality_matches_`). + Returns: + matches (Tensor[int64]): a vector of length N, where matches[i] is a matched + ground-truth index in [0, M) + match_labels (Tensor[int8]): a vector of length N, where pred_labels[i] indicates + whether a prediction is a true or false positive or ignored + """ + assert match_quality_matrix.dim() == 2 + if match_quality_matrix.numel() == 0: + default_matches = match_quality_matrix.new_full( + (match_quality_matrix.size(1),), 0, dtype=torch.int64 + ) + # When no gt boxes exist, we define IOU = 0 and therefore set labels + # to `self.labels[0]`, which usually defaults to background class 0 + # To choose to ignore instead, can make labels=[-1,0,-1,1] + set appropriate thresholds + default_match_labels = match_quality_matrix.new_full( + (match_quality_matrix.size(1),), self.labels[0], dtype=torch.int8 + ) + return default_matches, default_match_labels + + assert torch.all(match_quality_matrix >= 0) + + # match_quality_matrix is M (gt) x N (predicted) + # Max over gt elements (dim 0) to find best gt candidate for each prediction + matched_vals, matches = match_quality_matrix.max(dim=0) + + match_labels = matches.new_full(matches.size(), 1, dtype=torch.int8) + + for (l, low, high) in zip(self.labels, self.thresholds[:-1], self.thresholds[1:]): + low_high = (matched_vals >= low) & (matched_vals < high) + match_labels[low_high] = l + + if self.allow_low_quality_matches: + self.set_low_quality_matches_(match_labels, match_quality_matrix) + + return matches, match_labels + + def set_low_quality_matches_(self, match_labels, match_quality_matrix): + """ + Produce additional matches for predictions that have only low-quality matches. + Specifically, for each ground-truth G find the set of predictions that have + maximum overlap with it (including ties); for each prediction in that set, if + it is unmatched, then match it to the ground-truth G. + This function implements the RPN assignment case (i) in Sec. 3.1.2 of + :paper:`Faster R-CNN`. + """ + # For each gt, find the prediction with which it has highest quality + highest_quality_foreach_gt, _ = match_quality_matrix.max(dim=1) + # Find the highest quality match available, even if it is low, including ties. + # Note that the matches qualities must be positive due to the use of + # `torch.nonzero`. + _, pred_inds_with_highest_quality = nonzero_tuple( + match_quality_matrix == highest_quality_foreach_gt[:, None] + ) + # If an anchor was labeled positive only due to a low-quality match + # with gt_A, but it has larger overlap with gt_B, it's matched index will still be gt_B. + # This follows the implementation in Detectron, and is found to have no significant impact. + match_labels[pred_inds_with_highest_quality] = 1 + + @PROPOSAL_GENERATOR_REGISTRY.register() class custom_RPN(RPN): @@ -58,17 +199,17 @@ def __init__(self, in_features: List[str], head: nn.Module, anchor_generator: nn.Module, - anchor_matcher: Matcher, + anchor_matcher: Ioa_Matcher, box2box_transform: Box2BoxTransform, batch_size_per_image: int, positive_fraction: float, pre_nms_topk: Tuple[float, float], post_nms_topk: Tuple[float, float], - nms_thresh: float = 0.4, - nms_thresh_union: float = 0.4, - min_box_size: float = 0.0, - anchor_boundary_thresh: float = -1.0, - loss_weight: Union[float, Dict[str, float]] = 1.0, + nms_thresh: float, + nms_thresh_union: float, + min_box_size: float, + anchor_boundary_thresh: float, + loss_weight: Union[float, Dict[str, float]], box_reg_loss_type: str = "smooth_l1", smooth_l1_beta: float = 0.0,): super().__init__(in_features=in_features, head=head, anchor_generator=anchor_generator, @@ -100,12 +241,72 @@ def from_config(cls, cfg, input_shape: Dict[str, ShapeSpec]): ret["post_nms_topk"] = (cfg.MODEL.RPN.POST_NMS_TOPK_TRAIN, cfg.MODEL.RPN.POST_NMS_TOPK_TEST) ret["anchor_generator"] = build_anchor_generator(cfg, [input_shape[f] for f in in_features]) - ret["anchor_matcher"] = Matcher( - cfg.MODEL.RPN.IOU_THRESHOLDS, cfg.MODEL.RPN.IOU_LABELS, allow_low_quality_matches=True + ret["anchor_matcher"] = Ioa_Matcher( + cfg.MODEL.RPN.IOU_THRESHOLDS, cfg.MODEL.RPN.IOA_THRESHOLDS, cfg.MODEL.RPN.IOU_LABELS, allow_low_quality_matches=True ) ret["head"] = build_rpn_head(cfg, [input_shape[f] for f in in_features]) return ret + @torch.jit.unused + @torch.no_grad() + def label_and_sample_anchors( + self, anchors: List[Boxes], gt_instances: List[Instances] + ) -> Tuple[List[torch.Tensor], List[torch.Tensor]]: + """ + Args: + anchors (list[Boxes]): anchors for each feature map. + gt_instances: the ground-truth instances for each image. + Returns: + list[Tensor]: + List of #img tensors. i-th element is a vector of labels whose length is + the total number of anchors across all feature maps R = sum(Hi * Wi * A). + Label values are in {-1, 0, 1}, with meanings: -1 = ignore; 0 = negative + class; 1 = positive class. + list[Tensor]: + i-th element is a Rx4 tensor. The values are the matched gt boxes for each + anchor. Values are undefined for those anchors not labeled as 1. + """ + anchors = Boxes.cat(anchors) + + gt_boxes = [x.gt_boxes for x in gt_instances] + image_sizes = [x.image_size for x in gt_instances] + del gt_instances + + gt_labels = [] + matched_gt_boxes = [] + for image_size_i, gt_boxes_i in zip(image_sizes, gt_boxes): + """ + image_size_i: (h, w) for the i-th image + gt_boxes_i: ground-truth boxes for i-th image + """ + + match_quality_matrix = retry_if_cuda_oom(pairwise_iou)(gt_boxes_i, anchors) + match_quality_matrix_ioa = retry_if_cuda_oom(pairwise_ioa)(gt_boxes_i, anchors) + matched_idxs, gt_labels_i = retry_if_cuda_oom(self.anchor_matcher)(match_quality_matrix, match_quality_matrix_ioa) + # Matching is memory-expensive and may result in CPU tensors. But the result is small + gt_labels_i = gt_labels_i.to(device=gt_boxes_i.device) + del match_quality_matrix + + if self.anchor_boundary_thresh >= 0: + # Discard anchors that go out of the boundaries of the image + # NOTE: This is legacy functionality that is turned off by default in Detectron2 + anchors_inside_image = anchors.inside_box(image_size_i, self.anchor_boundary_thresh) + gt_labels_i[~anchors_inside_image] = -1 + + # A vector of labels (-1, 0, 1) for each anchor + gt_labels_i = self._subsample_labels(gt_labels_i) + + if len(gt_boxes_i) == 0: + # These values won't be used anyway since the anchor is labeled as background + matched_gt_boxes_i = torch.zeros_like(anchors.tensor) + else: + # TODO wasted indexing computation for ignored boxes + matched_gt_boxes_i = gt_boxes_i[matched_idxs].tensor + + gt_labels.append(gt_labels_i) # N,AHW + matched_gt_boxes.append(matched_gt_boxes_i) + return gt_labels, matched_gt_boxes + def find_top_rpn_proposals( proposals: List[torch.Tensor], pred_objectness_logits: List[torch.Tensor], @@ -384,7 +585,7 @@ def __call__(self, original_image): pred_classes = predictions['instances'].get_fields()['pred_classes'] pred_masks = predictions['instances'].get_fields()['pred_masks'] - keep = custom_nms_mask(pred_masks ,scores, thresh_iou_o = 0.3) + keep = custom_nms_mask(pred_masks ,scores, thresh_iou_o = self.cfg.MODEL.ROI_HEADS.NMS_THRESH_TEST) predictions['instances'].get_fields()['pred_boxes'] = boxes[keep] predictions['instances'].get_fields()['scores'] = scores[keep] From 2ffbf88ebba11f908162a2a289ab429e509399d6 Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Fri, 13 Jan 2023 02:06:01 +0000 Subject: [PATCH 35/54] Update custom_nms.py --- detectree2/models/custom_nms.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/detectree2/models/custom_nms.py b/detectree2/models/custom_nms.py index dfdd1774..cb5e8c47 100644 --- a/detectree2/models/custom_nms.py +++ b/detectree2/models/custom_nms.py @@ -51,7 +51,7 @@ def _is_tracing(): def pairwise_ioa(boxes1: Boxes, boxes2: Boxes) -> torch.Tensor: """ - Similar to :func:`pariwise_iou` but compute the IoA (intersection over boxes2 area). + Compute the IoA difference (intersection over boxes2 area - intersection over boxes2 area). Args: boxes1,boxes2 (Boxes): two `Boxes`. Contains N & M boxes, respectively. Returns: @@ -249,7 +249,7 @@ def from_config(cls, cfg, input_shape: Dict[str, ShapeSpec]): @torch.jit.unused @torch.no_grad() - def label_and_sample_anchors( + def label_and_sample_anchors( self, anchors: List[Boxes], gt_instances: List[Instances] ) -> Tuple[List[torch.Tensor], List[torch.Tensor]]: """ From 73b548a67d13a8a29739fa6ab7f31a2228431d3e Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Fri, 13 Jan 2023 02:06:10 +0000 Subject: [PATCH 36/54] Update train.py --- detectree2/models/train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/detectree2/models/train.py b/detectree2/models/train.py index 385a3781..bc86cfda 100644 --- a/detectree2/models/train.py +++ b/detectree2/models/train.py @@ -602,7 +602,7 @@ def setup_cfg( #cfg.MODEL.PROPOSAL_GENERATOR.NAME = 'custom_RPN' cfg.nms_thresh_union = nms_thresh_union cfg.MODEL.RPN.NMS_THRESH = rpn_nms_thresh - #cfg.MODEL.RPN.IOA_THRESHOLDS = ? + cfg.MODEL.RPN.IOA_THRESHOLDS = 0.5 cfg.MODEL.ROI_HEADS.NMS_THRESH_TEST = nms_thresh_test return cfg From 629e286c01454b8314eef0fae5dcc4e1419716b5 Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Fri, 13 Jan 2023 02:45:21 +0000 Subject: [PATCH 37/54] Update train.py --- detectree2/models/train.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/detectree2/models/train.py b/detectree2/models/train.py index bc86cfda..5e40f263 100644 --- a/detectree2/models/train.py +++ b/detectree2/models/train.py @@ -269,14 +269,14 @@ def run_step(self): """ self._trainer.optimizer.step() -@classmethod -def build_evaluator(cls, cfg, dataset_name, output_folder=None): + @classmethod + def build_evaluator(cls, cfg, dataset_name, output_folder=None): if output_folder is None: os.makedirs("eval", exist_ok=True) output_folder = "eval" return COCOEvaluator(dataset_name, cfg, True, output_folder) -def build_hooks(self): + def build_hooks(self): hooks = super().build_hooks() # augmentations = [T.ResizeShortestEdge(short_edge_length=(1000, 1000), # max_size=1333, @@ -296,7 +296,7 @@ def build_hooks(self): ) return hooks -def build_train_loader(cls, cfg): + def build_train_loader(cls, cfg): """Summary. Args: From 84234a4b331b3398183110aa7043e901b41864f7 Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Fri, 13 Jan 2023 03:36:35 +0000 Subject: [PATCH 38/54] Update and rename custom_nms.py to custom_rpn.py --- .../models/{custom_nms.py => custom_rpn.py} | 29 +++++++++++++------ 1 file changed, 20 insertions(+), 9 deletions(-) rename detectree2/models/{custom_nms.py => custom_rpn.py} (95%) diff --git a/detectree2/models/custom_nms.py b/detectree2/models/custom_rpn.py similarity index 95% rename from detectree2/models/custom_nms.py rename to detectree2/models/custom_rpn.py index cb5e8c47..ec23ee7f 100644 --- a/detectree2/models/custom_nms.py +++ b/detectree2/models/custom_rpn.py @@ -1,3 +1,6 @@ +''' change iou metric to ioa in rpn nms process for better mask quality and combine these two in anchor matching for better box regression''' +''' a predictor using mask_iou for proprocessing''' + from typing import Tuple import torch @@ -17,6 +20,7 @@ import torch.nn.functional as F from torch import nn +from detectron2.layers import nonzero_tuple from detectron2.config import configurable from detectron2.layers import Conv2d, ShapeSpec, cat from detectron2.structures import Boxes, ImageList, Instances, pairwise_iou @@ -72,10 +76,10 @@ class Ioa_Matcher(object): This class assigns to each predicted "element" (e.g., a box) a ground-truth element. Each predicted element will have exactly zero or one matches; each ground-truth element may be matched to zero or more predicted elements. - The matching is determined by the MxN match_quality_matrix, that characterizes - how well each (ground-truth, prediction)-pair match each other. For example, + The matching is determined by the MxN match_quality_matrix and match_quality_matrix_ioa, + they characterize how well each (ground-truth, prediction)-pair match each other. For example, if the elements are boxes, this matrix may contain box intersection-over-union - overlap values. + overlap values and intersection-over-area difference values. The matcher returns (a) a vector of length N containing the index of the ground-truth element m in [0, M) that matches to prediction n in [0, N). (b) a vector of length N containing the labels for each prediction. @@ -98,13 +102,13 @@ def __init__( See set_low_quality_matches_ for more details. For example, thresholds = [0.3, 0.5] - thresholds_ioa = 0.5 + thresholds_ioa = 0.6 labels = [0, -1, 1] - All predictions with iou < 0.3 and ioa_dif > 0.5 will be marked with 0 and + All predictions with iou < 0.3 will be marked with 0 and thus will be considered as false positives while training. All predictions with 0.3 <= iou < 0.5 will be marked with -1 and thus will be ignored. - All predictions with 0.5 <= iou and ioa_dif < 0.5 will be marked with 1 and + All predictions with 0.5 <= iou and ioa_dif < 0.6 will be marked with 1 and thus will be considered as true positives. """ # Add -inf and +inf to first and last position in thresholds @@ -117,6 +121,7 @@ def __init__( assert all([l in [-1, 0, 1] for l in labels]) assert len(labels) == len(thresholds) - 1 self.thresholds = thresholds + self.thresholds_ioa = thresholds_ioa self.labels = labels self.allow_low_quality_matches = allow_low_quality_matches @@ -155,11 +160,13 @@ def __call__(self, match_quality_matrix, match_quality_matrix_ioa): # match_quality_matrix is M (gt) x N (predicted) # Max over gt elements (dim 0) to find best gt candidate for each prediction matched_vals, matches = match_quality_matrix.max(dim=0) + + matched_vals_ioa = match_quality_matrix_ioa[matches, np.arange(0, match_quality_matrix.size(1))] - match_labels = matches.new_full(matches.size(), 1, dtype=torch.int8) + match_labels = matches.new_full(matches.size(), 0, dtype=torch.int8) for (l, low, high) in zip(self.labels, self.thresholds[:-1], self.thresholds[1:]): - low_high = (matched_vals >= low) & (matched_vals < high) + low_high = (matched_vals >= low) & (matched_vals < high) & (matched_vals_ioa*l < self.thresholds_ioa) match_labels[low_high] = l if self.allow_low_quality_matches: @@ -192,7 +199,11 @@ def set_low_quality_matches_(self, match_labels, match_quality_matrix): @PROPOSAL_GENERATOR_REGISTRY.register() class custom_RPN(RPN): - + ''' + from detectron2-rpn.py + the main change is to use ioa instead of iou in nms and combine these two in anchor matching + another change is that before nms will be processed for each class, but now images from all classes wil be processed together + ''' @configurable def __init__(self, *, From 9865b8858aaaeee355b08d803009c063adf07089 Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Fri, 13 Jan 2023 03:37:07 +0000 Subject: [PATCH 39/54] Update train.py --- detectree2/models/train.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/detectree2/models/train.py b/detectree2/models/train.py index 5e40f263..1ed53114 100644 --- a/detectree2/models/train.py +++ b/detectree2/models/train.py @@ -42,7 +42,7 @@ # from IPython.display import display # from PIL import Image -from .custom_nms import custom_RPN, DefaultPredictor1 +from .custom_rpn import custom_RPN, DefaultPredictor1 class LossEvalHook(HookBase): @@ -599,10 +599,10 @@ def setup_cfg( cfg.MODEL.BACKBONE.FREEZE_AT = 2 cfg.mask_weight = mask_weight #cfg.MODEL.ROI_BOX_HEAD.BBOX_REG_LOSS_TYPE = 'diou' - #cfg.MODEL.PROPOSAL_GENERATOR.NAME = 'custom_RPN' + cfg.MODEL.PROPOSAL_GENERATOR.NAME = 'custom_RPN' cfg.nms_thresh_union = nms_thresh_union cfg.MODEL.RPN.NMS_THRESH = rpn_nms_thresh - cfg.MODEL.RPN.IOA_THRESHOLDS = 0.5 + cfg.MODEL.RPN.IOA_THRESHOLDS = 0.4 cfg.MODEL.ROI_HEADS.NMS_THRESH_TEST = nms_thresh_test return cfg From e25ad7e30c39057fed7000cd6a0a9a83fb9de1b2 Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Sun, 15 Jan 2023 01:33:53 +0000 Subject: [PATCH 40/54] Update custom_rpn.py --- detectree2/models/custom_rpn.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/detectree2/models/custom_rpn.py b/detectree2/models/custom_rpn.py index ec23ee7f..7fa9eccf 100644 --- a/detectree2/models/custom_rpn.py +++ b/detectree2/models/custom_rpn.py @@ -24,6 +24,7 @@ from detectron2.config import configurable from detectron2.layers import Conv2d, ShapeSpec, cat from detectron2.structures import Boxes, ImageList, Instances, pairwise_iou +from detectron2.structures.boxes import pairwise_intersection from detectron2.utils.events import get_event_storage from detectron2.utils.memory import retry_if_cuda_oom from detectron2.utils.registry import Registry @@ -62,12 +63,11 @@ def pairwise_ioa(boxes1: Boxes, boxes2: Boxes) -> torch.Tensor: Tensor: IoA, sized [N,M]. """ area2 = boxes2.area() # [M] - area1 = boxes1.area() # [M] + area1 = boxes1.area() # [N] inter = pairwise_intersection(boxes1, boxes2) - # handle empty boxes ioa_dif = torch.where( - inter > 0, abs(inter / area2 - inter / area1), torch.zeros(1, dtype=inter.dtype, device=inter.device) + inter > 0, abs(inter / area2 - (inter.T / area1).T), torch.zeros(1, dtype=inter.dtype, device=inter.device) ) return ioa_dif From 339ea5f4c7a047ec9ae6fd19c3b1132d2bfea61a Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Sun, 15 Jan 2023 01:35:37 +0000 Subject: [PATCH 41/54] Update train.py --- detectree2/models/train.py | 46 ++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/detectree2/models/train.py b/detectree2/models/train.py index 1ed53114..9b22222c 100644 --- a/detectree2/models/train.py +++ b/detectree2/models/train.py @@ -39,11 +39,10 @@ from detectron2.utils.logger import log_every_n_seconds from detectron2.utils.visualizer import ColorMode, Visualizer -# from IPython.display import display -# from PIL import Image - from .custom_rpn import custom_RPN, DefaultPredictor1 +# from IPython.display import display +# from PIL import Image class LossEvalHook(HookBase): """Do inference and get the loss metric @@ -64,25 +63,24 @@ class LossEvalHook(HookBase): """ - def __init__(self, eval_period, model, data_loader, patience, out_dir): - self._model = model - self._period = eval_period - self._data_loader = data_loader - self.patience = patience - self.iter = 0 - self.max_value = 0 - self.best_iter = 0 - #self.checkpointer = DetectionCheckpointer(self._model, save_dir=out_dir) + def __init__(self, eval_period, model, data_loader, patience): + self._model = model + self._period = eval_period + self._data_loader = data_loader + self.patience = patience + self.iter = 0 + self.max_value = 0 + self.best_iter = 0 + #self.checkpointer = DetectionCheckpointer(self._model, save_dir=out_dir) def _do_loss_eval(self): - total = len(self._data_loader) - num_warmup = min(5, total - 1) - - start_time = time.perf_counter() - total_compute_time = 0 - losses = [] - for idx, inputs in enumerate(self._data_loader): - if idx == num_warmup: + """Copying inference_on_dataset from evaluator.py. + Returns: + _type_: _description_ + """ + total = len(self._data_loader) + num_warmup = min(5, total - 1) + start_time = time.perf_counter() total_compute_time = 0 losses = [] @@ -165,7 +163,10 @@ def after_train(self): # not found! # Therefore sleep is attempt to allow CI to pass, but it often still fails. time.sleep(15) - self.trainer.checkpointer.load(self.trainer.cfg.OUTPUT_DIR + '/model_' + str(index) + '.pth') + if len(self.trainer.values) != 0: + index = self.trainer.values.index(max(self.trainer.values)) + 1 + print(self.trainer.early_stop,"best model is", index) + self.trainer.checkpointer.load(self.trainer.cfg.OUTPUT_DIR + '/model_' + str(index) + '.pth') # See https://jss367.github.io/data-augmentation-in-detectron2.html for data augmentation advice class MyTrainer(DefaultTrainer): @@ -298,10 +299,8 @@ def build_hooks(self): def build_train_loader(cls, cfg): """Summary. - Args: cfg (_type_): _description_ - Returns: _type_: _description_ """ @@ -548,7 +547,6 @@ def setup_cfg( resize=True, ): """Set up config object # noqa: D417. - Args: base_model: base pre-trained model from detectron2 model_zoo trains: names of registered data to use for training From c0135b0756ab3091f4903b1faab78ca6385143f6 Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Sun, 15 Jan 2023 17:51:01 +0000 Subject: [PATCH 42/54] Update train.py --- detectree2/models/train.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/detectree2/models/train.py b/detectree2/models/train.py index 9b22222c..7a16a2f8 100644 --- a/detectree2/models/train.py +++ b/detectree2/models/train.py @@ -123,7 +123,7 @@ def _do_loss_eval(self): return losses - def _get_loss(self, data): + def _get_loss(self, data): """Calculate loss in train_loop. Args: data (_type_): _description_ @@ -138,7 +138,7 @@ def _get_loss(self, data): total_losses_reduced = sum(loss for loss in metrics_dict.values()) return total_losses_reduced - def after_step(self): + def after_step(self): next_iter = self.trainer.iter + 1 is_final = next_iter == self.trainer.max_iter if is_final or (self._period > 0 and next_iter % self._period == 0): @@ -155,7 +155,7 @@ def after_step(self): print("Early stopping occurs in iter {}, max ap is {}".format(self.best_iter, self.max_ap)) self.trainer.storage.put_scalars(timetest=12) - def after_train(self): + def after_train(self): # Select the model with the best AP50 index = self.trainer.APs.index(max(self.trainer.APs)) + 1 # Error in demo: From d920d228d86ede7f44116156275f0151755b8912 Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Sun, 15 Jan 2023 18:01:08 +0000 Subject: [PATCH 43/54] Update train.py --- detectree2/models/train.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/detectree2/models/train.py b/detectree2/models/train.py index 7a16a2f8..d7548af1 100644 --- a/detectree2/models/train.py +++ b/detectree2/models/train.py @@ -138,7 +138,7 @@ def _get_loss(self, data): total_losses_reduced = sum(loss for loss in metrics_dict.values()) return total_losses_reduced - def after_step(self): + def after_step(self): next_iter = self.trainer.iter + 1 is_final = next_iter == self.trainer.max_iter if is_final or (self._period > 0 and next_iter % self._period == 0): @@ -155,7 +155,7 @@ def after_step(self): print("Early stopping occurs in iter {}, max ap is {}".format(self.best_iter, self.max_ap)) self.trainer.storage.put_scalars(timetest=12) - def after_train(self): + def after_train(self): # Select the model with the best AP50 index = self.trainer.APs.index(max(self.trainer.APs)) + 1 # Error in demo: From 9c109d18039042dd6ef29cd88bc664fb691575a7 Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Sun, 15 Jan 2023 19:17:55 +0000 Subject: [PATCH 44/54] Update custom_rpn.py --- detectree2/models/custom_rpn.py | 1 + 1 file changed, 1 insertion(+) diff --git a/detectree2/models/custom_rpn.py b/detectree2/models/custom_rpn.py index 7fa9eccf..8d3e2836 100644 --- a/detectree2/models/custom_rpn.py +++ b/detectree2/models/custom_rpn.py @@ -3,6 +3,7 @@ from typing import Tuple +import numpy as np import torch import torchvision from torch import Tensor From 5c7018ead8c6b7c6a02120999d3f15c8b1e4d991 Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Sun, 15 Jan 2023 19:22:14 +0000 Subject: [PATCH 45/54] Update train.py --- detectree2/models/train.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/detectree2/models/train.py b/detectree2/models/train.py index d7548af1..15ba1ebe 100644 --- a/detectree2/models/train.py +++ b/detectree2/models/train.py @@ -71,7 +71,6 @@ def __init__(self, eval_period, model, data_loader, patience): self.iter = 0 self.max_value = 0 self.best_iter = 0 - #self.checkpointer = DetectionCheckpointer(self._model, save_dir=out_dir) def _do_loss_eval(self): """Copying inference_on_dataset from evaluator.py. @@ -116,7 +115,7 @@ def _do_loss_eval(self): else: AP = self.trainer.test(self.trainer.cfg, self.trainer.model)["segm"]["AP50"] print("Av. AP50 =", AP) - self.trainer.APs.append(AP) + self.trainer.values.append(AP) self.trainer.storage.put_scalar("validation_loss", mean_loss) self.trainer.storage.put_scalar("validation_ap", AP) comm.synchronize() @@ -143,10 +142,10 @@ def after_step(self): is_final = next_iter == self.trainer.max_iter if is_final or (self._period > 0 and next_iter % self._period == 0): self._do_loss_eval() - if self.max_ap < self.trainer.APs[-1]: + if self.max_ap < self.trainer.values[-1]: self.iter = 0 - self.max_ap = self.trainer.APs[-1] - self.trainer.checkpointer.save("model_" + str(len(self.trainer.APs))) + self.max_ap = self.trainer.values[-1] + self.trainer.checkpointer.save("model_" + str(len(self.trainer.values))) self.best_iter = self.trainer.iter else: self.iter += 1 @@ -156,8 +155,6 @@ def after_step(self): self.trainer.storage.put_scalars(timetest=12) def after_train(self): - # Select the model with the best AP50 - index = self.trainer.APs.index(max(self.trainer.APs)) + 1 # Error in demo: # AssertionError: Checkpoint /__w/detectree2/detectree2/detectree2-data/paracou-out/train_outputs-1/model_1.pth # not found! From b4ec20214ff8fcc645aefa43a1642c11b224a510 Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Sun, 15 Jan 2023 20:20:40 +0000 Subject: [PATCH 46/54] Update train.py --- detectree2/models/train.py | 1 + 1 file changed, 1 insertion(+) diff --git a/detectree2/models/train.py b/detectree2/models/train.py index 15ba1ebe..f5c22dde 100644 --- a/detectree2/models/train.py +++ b/detectree2/models/train.py @@ -71,6 +71,7 @@ def __init__(self, eval_period, model, data_loader, patience): self.iter = 0 self.max_value = 0 self.best_iter = 0 + self.max_ap = 0 def _do_loss_eval(self): """Copying inference_on_dataset from evaluator.py. From f05c755e7605eb998b8d8c90823c3c81b23b58c1 Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Sun, 15 Jan 2023 20:21:35 +0000 Subject: [PATCH 47/54] Update train.py --- detectree2/models/train.py | 1 - 1 file changed, 1 deletion(-) diff --git a/detectree2/models/train.py b/detectree2/models/train.py index f5c22dde..e0dd6ac2 100644 --- a/detectree2/models/train.py +++ b/detectree2/models/train.py @@ -69,7 +69,6 @@ def __init__(self, eval_period, model, data_loader, patience): self._data_loader = data_loader self.patience = patience self.iter = 0 - self.max_value = 0 self.best_iter = 0 self.max_ap = 0 From 3340b5bf55963e88c1bea188b888b559eaef1be2 Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Sun, 15 Jan 2023 20:23:28 +0000 Subject: [PATCH 48/54] Update train.py --- detectree2/models/train.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/detectree2/models/train.py b/detectree2/models/train.py index e0dd6ac2..41ab0af3 100644 --- a/detectree2/models/train.py +++ b/detectree2/models/train.py @@ -115,7 +115,7 @@ def _do_loss_eval(self): else: AP = self.trainer.test(self.trainer.cfg, self.trainer.model)["segm"]["AP50"] print("Av. AP50 =", AP) - self.trainer.values.append(AP) + self.trainer.APs.append(AP) self.trainer.storage.put_scalar("validation_loss", mean_loss) self.trainer.storage.put_scalar("validation_ap", AP) comm.synchronize() @@ -142,10 +142,10 @@ def after_step(self): is_final = next_iter == self.trainer.max_iter if is_final or (self._period > 0 and next_iter % self._period == 0): self._do_loss_eval() - if self.max_ap < self.trainer.values[-1]: + if self.max_ap < self.trainer.APs[-1]: self.iter = 0 - self.max_ap = self.trainer.values[-1] - self.trainer.checkpointer.save("model_" + str(len(self.trainer.values))) + self.max_ap = self.trainer.APs[-1] + self.trainer.checkpointer.save("model_" + str(len(self.trainer.APs))) self.best_iter = self.trainer.iter else: self.iter += 1 @@ -160,8 +160,8 @@ def after_train(self): # not found! # Therefore sleep is attempt to allow CI to pass, but it often still fails. time.sleep(15) - if len(self.trainer.values) != 0: - index = self.trainer.values.index(max(self.trainer.values)) + 1 + if len(self.trainer.APs) != 0: + index = self.trainer.values.index(max(self.trainer.APs)) + 1 print(self.trainer.early_stop,"best model is", index) self.trainer.checkpointer.load(self.trainer.cfg.OUTPUT_DIR + '/model_' + str(index) + '.pth') @@ -198,7 +198,7 @@ def train(self): self.iter = self.start_iter = start_iter self.max_iter = max_iter self.early_stop = False - self.values = [] + self.APs = [] self.reweight = False ### used to decide when to increase the weight of classification loss with EventStorage(start_iter) as self.storage: From f4377abedeb2e85a966ac842f862d9e706877552 Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Sun, 15 Jan 2023 20:51:41 +0000 Subject: [PATCH 49/54] Update custom_rpn.py --- detectree2/models/custom_rpn.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/detectree2/models/custom_rpn.py b/detectree2/models/custom_rpn.py index 8d3e2836..4ae8ad2c 100644 --- a/detectree2/models/custom_rpn.py +++ b/detectree2/models/custom_rpn.py @@ -58,6 +58,8 @@ def _is_tracing(): def pairwise_ioa(boxes1: Boxes, boxes2: Boxes) -> torch.Tensor: """ Compute the IoA difference (intersection over boxes2 area - intersection over boxes2 area). + This metric is used for anchor choice and the chosen anchor is processsed through box2box transformation. + The box2box transformation is assumed to be linear so we use IOA difference to restrict distances between anchors and gt bounding boxes Args: boxes1,boxes2 (Boxes): two `Boxes`. Contains N & M boxes, respectively. Returns: From c3506d00d1bce6fee507d2b4537c1987a8bd717f Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Sun, 15 Jan 2023 21:34:09 +0000 Subject: [PATCH 50/54] Update train.py --- detectree2/models/train.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/detectree2/models/train.py b/detectree2/models/train.py index 41ab0af3..fc2c8747 100644 --- a/detectree2/models/train.py +++ b/detectree2/models/train.py @@ -161,7 +161,7 @@ def after_train(self): # Therefore sleep is attempt to allow CI to pass, but it often still fails. time.sleep(15) if len(self.trainer.APs) != 0: - index = self.trainer.values.index(max(self.trainer.APs)) + 1 + index = self.trainer.APs.index(max(self.trainer.APs)) + 1 print(self.trainer.early_stop,"best model is", index) self.trainer.checkpointer.load(self.trainer.cfg.OUTPUT_DIR + '/model_' + str(index) + '.pth') @@ -594,7 +594,7 @@ def setup_cfg( cfg.MODEL.BACKBONE.FREEZE_AT = 2 cfg.mask_weight = mask_weight #cfg.MODEL.ROI_BOX_HEAD.BBOX_REG_LOSS_TYPE = 'diou' - cfg.MODEL.PROPOSAL_GENERATOR.NAME = 'custom_RPN' + #cfg.MODEL.PROPOSAL_GENERATOR.NAME = 'custom_RPN' cfg.nms_thresh_union = nms_thresh_union cfg.MODEL.RPN.NMS_THRESH = rpn_nms_thresh cfg.MODEL.RPN.IOA_THRESHOLDS = 0.4 From ed249a91574971e2f2547027359b972de39c5dbd Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Mon, 16 Jan 2023 03:38:47 +0000 Subject: [PATCH 51/54] Update train.py --- detectree2/models/train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/detectree2/models/train.py b/detectree2/models/train.py index fc2c8747..4c0b566c 100644 --- a/detectree2/models/train.py +++ b/detectree2/models/train.py @@ -594,7 +594,7 @@ def setup_cfg( cfg.MODEL.BACKBONE.FREEZE_AT = 2 cfg.mask_weight = mask_weight #cfg.MODEL.ROI_BOX_HEAD.BBOX_REG_LOSS_TYPE = 'diou' - #cfg.MODEL.PROPOSAL_GENERATOR.NAME = 'custom_RPN' + cfg.MODEL.PROPOSAL_GENERATOR.NAME = 'custom_RPN' cfg.nms_thresh_union = nms_thresh_union cfg.MODEL.RPN.NMS_THRESH = rpn_nms_thresh cfg.MODEL.RPN.IOA_THRESHOLDS = 0.4 From 66670229c2bf20097ead19e5c86a9781537ca0bb Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Mon, 16 Jan 2023 03:47:24 +0000 Subject: [PATCH 52/54] Update train.py --- detectree2/models/train.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/detectree2/models/train.py b/detectree2/models/train.py index 4c0b566c..1eb95511 100644 --- a/detectree2/models/train.py +++ b/detectree2/models/train.py @@ -592,13 +592,13 @@ def setup_cfg( cfg.INPUT.MIN_SIZE_TRAIN = 1000 cfg.MODEL.BACKBONE.FREEZE_AT = 2 - cfg.mask_weight = mask_weight + '''cfg.mask_weight = mask_weight #cfg.MODEL.ROI_BOX_HEAD.BBOX_REG_LOSS_TYPE = 'diou' cfg.MODEL.PROPOSAL_GENERATOR.NAME = 'custom_RPN' cfg.nms_thresh_union = nms_thresh_union cfg.MODEL.RPN.NMS_THRESH = rpn_nms_thresh cfg.MODEL.RPN.IOA_THRESHOLDS = 0.4 - cfg.MODEL.ROI_HEADS.NMS_THRESH_TEST = nms_thresh_test + cfg.MODEL.ROI_HEADS.NMS_THRESH_TEST = nms_thresh_test''' return cfg From 5fe518466e94c80a6ddb39c410c6a0278bd6f5e9 Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Mon, 16 Jan 2023 04:04:51 +0000 Subject: [PATCH 53/54] Update train.py --- detectree2/models/train.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/detectree2/models/train.py b/detectree2/models/train.py index 1eb95511..eedc5c51 100644 --- a/detectree2/models/train.py +++ b/detectree2/models/train.py @@ -539,7 +539,6 @@ def setup_cfg( nms_thresh_union = 0.1, rpn_nms_thresh = 0, nms_thresh_test = 0.2, - mask_weight = 1, out_dir="/content/drive/Shareddrives/detectree2/train_outputs", resize=True, ): @@ -592,13 +591,12 @@ def setup_cfg( cfg.INPUT.MIN_SIZE_TRAIN = 1000 cfg.MODEL.BACKBONE.FREEZE_AT = 2 - '''cfg.mask_weight = mask_weight #cfg.MODEL.ROI_BOX_HEAD.BBOX_REG_LOSS_TYPE = 'diou' cfg.MODEL.PROPOSAL_GENERATOR.NAME = 'custom_RPN' cfg.nms_thresh_union = nms_thresh_union cfg.MODEL.RPN.NMS_THRESH = rpn_nms_thresh cfg.MODEL.RPN.IOA_THRESHOLDS = 0.4 - cfg.MODEL.ROI_HEADS.NMS_THRESH_TEST = nms_thresh_test''' + cfg.MODEL.ROI_HEADS.NMS_THRESH_TEST = nms_thresh_test return cfg From 183529c961c9373aa1e05fde60ba89f104265138 Mon Sep 17 00:00:00 2001 From: WangLuran <94464527+WangLuran@users.noreply.github.com> Date: Mon, 16 Jan 2023 04:05:39 +0000 Subject: [PATCH 54/54] Update train.py --- detectree2/models/train.py | 41 -------------------------------------- 1 file changed, 41 deletions(-) diff --git a/detectree2/models/train.py b/detectree2/models/train.py index eedc5c51..9f4ba1d0 100644 --- a/detectree2/models/train.py +++ b/detectree2/models/train.py @@ -226,47 +226,6 @@ def train(self): verify_results(self.cfg, self._last_eval_results) return self._last_eval_results - - def run_step(self): - self._trainer.iter = self.iter - """ - Implement the standard training logic described above. - """ - assert self._trainer.model.training, "[SimpleTrainer] model was changed to eval mode!" - start = time.perf_counter() - """ - If you want to do something with the data, you can wrap the dataloader. - """ - data = next(self._trainer._data_loader_iter) - data_time = time.perf_counter() - start - - """ - If you want to do something with the losses, you can wrap the model. - """ - loss_dict = self._trainer.model(data) - if isinstance(loss_dict, torch.Tensor): - losses = loss_dict - loss_dict = {"total_loss": loss_dict} - else: - loss_dict['loss_mask'] *= self.cfg.mask_weight - losses = sum(loss_dict.values()) - - """ - If you need to accumulate gradients or do something similar, you can - wrap the optimizer with your custom `zero_grad()` method. - """ - self.optimizer.zero_grad() - losses.backward() - - self._trainer._write_metrics(loss_dict, data_time) - - """ - If you need gradient clipping/scaling or other processing, you can - wrap the optimizer with your custom `step()` method. But it is - suboptimal as explained in https://arxiv.org/abs/2006.15704 Sec 3.2.4 - """ - self._trainer.optimizer.step() - @classmethod def build_evaluator(cls, cfg, dataset_name, output_folder=None): if output_folder is None: