[Orbit] Add min_delta parameter to NewBestMetric

orbit/actions/new_best_metric.py

@@ -68,7 +68,8 @@ def __init__(self,
                metric: Union[str, MetricFn],
                higher_is_better: bool = True,
                filename: Optional[str] = None,
-               write_metric=True):
+               write_metric=True,
+               min_delta: float = 0.0):
     """Initializes the instance.
 
     Args:
@@ -87,6 +88,9 @@ def __init__(self,
         file to obtain the initial value. Setting this to `False` for most
         clients in some multi-client setups can avoid unnecessary file writes.
         Has no effect if `filename` is `None`.
+      min_delta: Minimum change in the monitored quantity to qualify as an
+        improvement, i.e. an absolute change of less than min_delta, will count
+        as no improvement.
     """
     self.metric = metric
     self.higher_is_better = higher_is_better
@@ -95,6 +99,7 @@ def __init__(self,
         initial_value=-float_max if higher_is_better else float_max,
         filename=filename,
         write_value=write_metric)
+    self._min_delta = abs(min_delta)
 
   def __call__(self, output: runner.Output) -> bool:
     """Tests `output` and updates the current best value if necessary.
@@ -137,10 +142,10 @@ def test(self, output: runner.Output) -> bool:
     """
     metric_value = self.metric_value(output)
     if self.higher_is_better:
-      if metric_value > self.best_value:
+      if metric_value > self.best_value + self._min_delta:
         return True
     else:  # Lower is better.
-      if metric_value < self.best_value:
+      if metric_value < self.best_value - self._min_delta:
         return True
     return False
 

orbit/actions/new_best_metric_test.py

@@ -89,6 +89,17 @@ def test_json_persisted_value_create_dirs(self):
     actions.JSONPersistedValue(value, tempfile)
     self.assertTrue(tf.io.gfile.exists(tempfile))
 
+  def test_new_best_metric_with_min_delta(self):
+    new_best_metric = actions.NewBestMetric(
+      lambda x: x['value'], higher_is_better=True, min_delta=0.1)
+    self.assertTrue(new_best_metric.commit({'value': 0.5}))
+    self.assertFalse(new_best_metric.test({'value': 0.55}))
+    self.assertTrue(new_best_metric.test({'value': 0.61}))
+    new_best_metric_loss = actions.NewBestMetric(
+      'value', higher_is_better=False, min_delta=0.1)
+    self.assertTrue(new_best_metric_loss.commit({'value': 1.0}))
+    self.assertFalse(new_best_metric_loss.test({'value': 0.95}))
+    self.assertTrue(new_best_metric_loss.test({'value': 0.89}))
 
 if __name__ == '__main__':
   tf.test.main()