Dev (#28)

* Hypercolumn (#16)

* fixed lovash loss, added helpers for loss weighing (#14)

* updated results exploration, added unet with hypercolumn

* updated with lighter hypercolumn setup

* Model average (#17)

* added prediction average notebook

* added simple average notebook

* added replication pad instead of zero pad (#18)

* changed to heng-like arch, added channel and spatial squeeze and excite, extended hypercolumn (#19)

* Update unet_models.py

typo in resnet unet fixed

* added resnet 18 an50 pretrained options, unified hyper and vanilla in one class (#20)

* Update models.py

Changed old class import and namings

* Loss design (#21)

* local

* initial

* formated results

* added focal, added border weighing, added size weighing added focus, added loss desing notebook

* fixed wrong focal definition, updated loss api

* exp with dropped borders

* set best params, not using weighing for now

* Dev depth experiments (#23)

* add depth layer in input

* reduce lr on plateau scheduler

* depth channels transformer

* fix reduce lr

* bugfix

* change default config

* added adaptive threshold in callbacks (#24)

* added adaptive threshold in callbacks

* fix

* added initial lr selector (#25)

* Initial lb selector (#26)

* added initial lr selector

* small refactor

* Auxiliary data small masks (#27)

* exping

* auxiliary data for border masks generated

Author: jakubczakon

common_blocks/callbacks.py

@@ -7,7 +7,7 @@
 from PIL import Image
 import neptune
 from torch.autograd import Variable
-from torch.optim.lr_scheduler import ExponentialLR
+from torch.optim.lr_scheduler import ExponentialLR, ReduceLROnPlateau
 from tempfile import TemporaryDirectory
 
 from steppy.base import Step, IdentityOperation
@@ -200,6 +200,83 @@ def on_batch_end(self, *args, **kwargs):
         self.batch_id += 1
 
 
+class ReduceLROnPlateauScheduler(Callback):
+    def __init__(self, metric_name, minimize, reduce_factor, reduce_patience, min_lr):
+        super().__init__()
+        self.ctx = neptune.Context()
+        self.metric_name = metric_name
+        self.minimize = minimize
+        self.reduce_factor = reduce_factor
+        self.reduce_patience = reduce_patience
+        self.min_lr = min_lr
+
+    def set_params(self, transformer, validation_datagen, *args, **kwargs):
+        super().set_params(transformer, validation_datagen)
+        self.validation_datagen = validation_datagen
+        self.model = transformer.model
+        self.optimizer = transformer.optimizer
+        self.loss_function = transformer.loss_function
+        self.lr_scheduler = ReduceLROnPlateau(optimizer=self.optimizer,
+                                              mode='min' if self.minimize else 'max',
+                                              factor=self.reduce_factor,
+                                              patience=self.reduce_patience,
+                                              min_lr=self.min_lr)
+
+    def on_train_begin(self, *args, **kwargs):
+        self.epoch_id = 0
+        self.batch_id = 0
+
+    def on_epoch_end(self, *args, **kwargs):
+        self.model.eval()
+        val_loss = self.get_validation_loss()
+        metric = val_loss[self.metric_name]
+        metric = metric.data.cpu().numpy()[0]
+        self.model.train()
+
+        self.lr_scheduler.step(metrics=metric, epoch=self.epoch_id)
+        logger.info('epoch {0} current lr: {1}'.format(self.epoch_id + 1,
+                                                       self.optimizer.state_dict()['param_groups'][0]['lr']))
+        self.ctx.channel_send('Learning Rate', x=self.epoch_id,
+                              y=self.optimizer.state_dict()['param_groups'][0]['lr'])
+
+        self.epoch_id += 1
+
+
+class InitialLearningRateFinder(Callback):
+    def __init__(self, min_lr=1e-8, multipy_factor=1.05, add_factor=0.0):
+        super().__init__()
+        self.ctx = neptune.Context()
+        self.min_lr = min_lr
+        self.multipy_factor = multipy_factor
+        self.add_factor = add_factor
+
+    def set_params(self, transformer, validation_datagen, *args, **kwargs):
+        super().set_params(transformer, validation_datagen)
+        self.validation_datagen = validation_datagen
+        self.model = transformer.model
+        self.optimizer = transformer.optimizer
+        self.loss_function = transformer.loss_function
+
+    def on_train_begin(self, *args, **kwargs):
+        self.epoch_id = 0
+        self.batch_id = 0
+
+        for param_group in self.optimizer.param_groups:
+            param_group['lr'] = self.min_lr
+
+    def on_batch_end(self, metrics, *args, **kwargs):
+        for name, loss in metrics.items():
+            loss = loss.data.cpu().numpy()[0]
+        current_lr = self.optimizer.state_dict()['param_groups'][0]['lr']
+        logger.info('Learning Rate {} Loss {})'.format(current_lr, loss))
+        self.ctx.channel_send('Learning Rate', x=self.batch_id, y=current_lr)
+        self.ctx.channel_send('Loss', x=self.batch_id, y=loss)
+
+        for param_group in self.optimizer.param_groups:
+            param_group['lr'] = current_lr * self.multipy_factor + self.add_factor
+        self.batch_id += 1
+
+
 class ExperimentTiming(Callback):
     def __init__(self, epoch_every=None, batch_every=None):
         super().__init__()
@@ -340,11 +417,24 @@ def on_epoch_end(self, *args, **kwargs):
 
     def _get_validation_loss(self):
         output, epoch_loss = self._transform()
-        y_pred = self._generate_prediction(output)
+        logger.info('Selecting best threshold')
+
+        iout_best, threshold_best = 0.0, 0.5
+        for threshold in np.linspace(0.5, 0.3, 21):
+            y_pred = self._generate_prediction(output, threshold)
+            iout_score = intersection_over_union_thresholds(self.y_true, y_pred)
+            logger.info('threshold {} IOUT {}'.format(threshold, iout_score))
+            if iout_score > iout_best:
+                iout_best = iout_score
+                threshold_best = threshold
+            else:
+                break
+        logger.info('Selected best threshold {} IOUT {}'.format(threshold_best, iout_best))
 
         logger.info('Calculating IOU and IOUT Scores')
-        iou_score = intersection_over_union(self.y_true, y_pred)
+        y_pred = self._generate_prediction(output, threshold_best)
         iout_score = intersection_over_union_thresholds(self.y_true, y_pred)
+        iou_score = intersection_over_union(self.y_true, y_pred)
         logger.info('IOU score on validation is {}'.format(iou_score))
         logger.info('IOUT score on validation is {}'.format(iout_score))
 
@@ -407,14 +497,14 @@ def _transform(self):
 
         return outputs, average_losses
 
-    def _generate_prediction(self, outputs):
+    def _generate_prediction(self, outputs, threshold):
         data = {'callback_input': {'meta': self.meta_valid,
                                    'meta_valid': None,
                                    },
                 'unet_output': {**outputs}
                 }
         with TemporaryDirectory() as cache_dirpath:
-            pipeline = self.validation_pipeline(cache_dirpath, self.loader_mode)
+            pipeline = self.validation_pipeline(cache_dirpath, self.loader_mode, threshold)
             output = pipeline.transform(data)
         y_pred = output['y_pred']
         return y_pred
@@ -494,7 +584,7 @@ def on_epoch_end(self, *args, **kwargs):
         self.epoch_id += 1
 
 
-def postprocessing_pipeline_simplified(cache_dirpath, loader_mode):
+def postprocessing_pipeline_simplified(cache_dirpath, loader_mode, threshold):
     if loader_mode == 'resize_and_pad':
         size_adjustment_function = partial(crop_image, target_size=ORIGINAL_SIZE)
     elif loader_mode == 'resize':
@@ -513,7 +603,7 @@ def postprocessing_pipeline_simplified(cache_dirpath, loader_mode):
 
     binarizer = Step(name='binarizer',
                      transformer=make_apply_transformer(
-                         partial(binarize, threshold=THRESHOLD),
+                         partial(binarize, threshold=threshold),
                          output_name='binarized_images',
                          apply_on=['images']),
                      input_steps=[mask_resize],

common_blocks/loaders.py

@@ -14,7 +14,7 @@
 import json
 from steppy.base import BaseTransformer
 
-from .utils import from_pil, to_pil, binary_from_rle, ImgAug
+from .utils import from_pil, to_pil, binary_from_rle, ImgAug, AddDepthChannels
 
 
 class ImageReader(BaseTransformer):
@@ -337,6 +337,7 @@ def __init__(self, train_mode, loader_params, dataset_params, augmentation_param
                                                    transforms.ToTensor(),
                                                    transforms.Normalize(mean=self.dataset_params.MEAN,
                                                                         std=self.dataset_params.STD),
+                                                   AddDepthChannels()
                                                    ])
         self.mask_transform = transforms.Compose([transforms.Lambda(to_array),
                                                   transforms.Lambda(to_tensor),
@@ -364,6 +365,7 @@ def __init__(self, loader_params, dataset_params, augmentation_params):
                                                    transforms.ToTensor(),
                                                    transforms.Normalize(mean=self.dataset_params.MEAN,
                                                                         std=self.dataset_params.STD),
+                                                   AddDepthChannels()
                                                    ])
         self.mask_transform = transforms.Compose([transforms.Lambda(to_array),
                                                   transforms.Lambda(to_tensor),