python: improve handling of !d conversion specifier in output formats

doc/man1/flux-jobs.rst

@@ -221,8 +221,8 @@ the following conversion flags are supported by *flux-jobs*:
    convert a timestamp to a Python datetime object. This allows datetime
    specific format to be used, e.g. *{t_inactive!d:%H:%M:%S}*. Additionally,
    width and alignment can be specified after the time format by using
-   two colons (``::``), e.g. *{t_inactive!d:%H:%M:%S::>20}*. Defaults to
-   datetime of epoch if timestamp field does not exist.
+   two colons (``::``), e.g. *{t_inactive!d:%H:%M:%S::>20}*. Returns an
+   empty string (or "-" if the *h* suffix is used) for an unset timestamp.
 
 **!F**
    convert a time duration in floating point seconds to Flux Standard

src/bindings/python/flux/util.py

@@ -312,16 +312,28 @@ class UtilDatetime(datetime):
     def __format__(self, fmt):
         # The string "::" is used to split the strftime() format from
         # any Python format spec:
-        vals = fmt.split("::", 1)
+        timefmt, *spec = fmt.split("::", 1)
 
-        # Call strftime() to get the formatted datetime as a string
-        result = self.strftime(vals[0])
+        if self == datetime.fromtimestamp(0.0):
+            result = ""
+        else:
+            # Call strftime() to get the formatted datetime as a string
+            result = self.strftime(timefmt or "%FT%T")
+
+        spec = spec[0] if spec else ""
+
+        # Handling of the 'h' suffix on spec is required here, since the
+        # UtilDatetime format() is called _after_ UtilFormatter.format_field()
+        # (where this option is handled for other types)
+        if spec.endswith("h"):
+            if not result:
+                result = "-"
+            spec = spec[:-1] + "s"
 
         # If there was a format spec, apply it here:
-        try:
-            return f"{{0:{vals[1]}}}".format(result)
-        except IndexError:
-            return result
+        if spec:
+            return f"{{0:{spec}}}".format(result)
+        return result
 
 
 def fsd(secs):
@@ -421,7 +433,10 @@ def format_field(self, value, spec):
             denote_truncation = True
             spec = spec[:-1]
 
-        if spec.endswith("h"):
+        # Note: handling of the 'h' suffix for UtilDatetime objects
+        # must be deferred to the UtilDatetetime format() method, since
+        # that method will be called after this one:
+        if spec.endswith("h") and not isinstance(value, UtilDatetime):
             localepoch = datetime.fromtimestamp(0.0).strftime("%FT%T")
             basecases = ("", "0s", "0.0", "0:00:00", "1970-01-01T00:00:00", localepoch)
             value = "-" if str(value) in basecases else str(value)

t/python/t0024-util.py

@@ -15,7 +15,7 @@
 from pycotap import TAPTestRunner
 
 from datetime import datetime, timedelta
-from flux.util import parse_datetime
+from flux.util import parse_datetime, UtilDatetime
 
 
 def ts(year, month, day, hour=0, minute=0, sec=0, us=0):
@@ -70,6 +70,25 @@ def test_nlp(self):
         self.assertEqual(self.parse("noon tomorrow"), ts(2021, 6, 11, 12))
         self.assertEqual(self.parse("last wed"), ts(2021, 6, 9))
 
+class TestUtilDatetime(unittest.TestCase):
+    @classmethod
+    def setUpClass(self):
+        self.zero = UtilDatetime.fromtimestamp(0.)
+        self.ts = UtilDatetime(2021, 6, 10, 8, 0, 0)
+
+    def test_zero(self):
+        self.assertEqual(f"{self.zero}", "")
+        self.assertEqual(f"{self.zero:::h}", "-")
+        self.assertEqual(f"{self.zero:%FT%T::<6}",  "      ")
+        self.assertEqual(f"{self.zero:%FT%T::<6h}", "-     ")
+
+    def test_fmt(self):
+        self.assertEqual(f"{self.ts}", "2021-06-10T08:00:00")
+        self.assertEqual(f"{self.ts:::h}", "2021-06-10T08:00:00")
+        self.assertEqual(f"{self.ts:%b%d %R}", "Jun10 08:00")
+        self.assertEqual(f"{self.ts:%b%d %R::h}", "Jun10 08:00")
+        self.assertEqual(f"{self.ts:%b%d %R::>12}", " Jun10 08:00")
+        self.assertEqual(f"{self.ts:%b%d %R::>12h}", " Jun10 08:00")
 
 if __name__ == "__main__":
     unittest.main(testRunner=TAPTestRunner())

t/t2800-jobs-cmd.t

@@ -937,9 +937,20 @@ test_expect_success 'flux-jobs --format={expiration!d:%FT%T::>20.20} works' '
 	cat expiration.in | \
 	    flux jobs --from-stdin -o "{expiration!d:%b%d %R::>20.20}" \
 	    >exp-fmt.out &&
+	cat expiration.in | \
+	    flux jobs --from-stdin -o "{expiration!d:%b%d %R::>20.20h}" \
+	    >exp-fmth.out &&
 	test_debug "cat exp-fmt.out" &&
 	grep "          EXPIRATION" exp-fmt.out &&
-	grep "         $(date --date=@${exp} +%b%d\ %R)" exp-fmt.out
+	grep "         $(date --date=@${exp} +%b%d\ %R)" exp-fmt.out &&
+	test_debug "cat exp-fmth.out" &&
+	grep "          EXPIRATION" exp-fmth.out &&
+	grep "         $(date --date=@${exp} +%b%d\ %R)" exp-fmth.out
+'
+test_expect_success 'flux-jobs --format={expiration!d} works' '
+	cat expiration.in | flux jobs --from-stdin -o "{expiration!d}" \
+	    >exp-fmtd.out &&
+	grep "$(date --date=@${exp} +%FT%T)" exp-fmtd.out
 '
 test_expect_success 'flux-jobs --format={expiration!d:%FT%T::=^20} works' '
 	cat expiration.in | \
@@ -1002,13 +1013,14 @@ test_expect_success 'flux-jobs emits empty string for special case t_estimate' '
 	fmt="${fmt},{annotations.sched.t_estimate!D}" &&
 	fmt="${fmt},{annotations.sched.t_estimate!F}" &&
 	fmt="${fmt},{annotations.sched.t_estimate!H}" &&
+	fmt="${fmt},{annotations.sched.t_estimate!d:%H:%M::h}" &&
 	fmt="${fmt},{annotations.sched.t_estimate!D:h}" &&
 	fmt="${fmt},{annotations.sched.t_estimate!F:h}" &&
 	fmt="${fmt},{annotations.sched.t_estimate!H:h}" &&
 	flux jobs -no "${fmt}" >t_estimate_annotations.out 2>&1 &&
 	test_debug "cat t_estimate_annotations.out" &&
 	for i in `seq 1 $(state_count active)`; do
-		echo ",00:00,,,,-,-,-" >> t_estimate_annotations.exp
+		echo ",,,,,-,-,-,-" >> t_estimate_annotations.exp
 	done &&
 	test_cmp t_estimate_annotations.out t_estimate_annotations.exp
 '