all: use FFDIFFSIGN to resolve possible undefined behavior in comparators

FFDIFFSIGN was created explicitly for this purpose, since the common
return a - b idiom is unsafe regarding overflow on signed integers. It
optimizes to branchless code on common compilers.

FFDIFFSIGN also has the subjective benefit of being easier to read due
to lack of ternary operators.

Tested with FATE.

Things not covered by this are unsigned integers, for which overflows
are well defined, and also places where overflow is clearly impossible,
e.g an instance where the a - b was being done on 24 bit values.

Reviewed-by: Michael Niedermayer <[email protected]>
Reviewed-by: Clément Bœsch <[email protected]>
Signed-off-by: Ganesh Ajjanagadde <[email protected]>

Author: Ganesh Ajjanagadde

cmdutils.c

@@ -1421,7 +1421,7 @@ static int compare_codec_desc(const void *a, const void *b)
     const AVCodecDescriptor * const *da = a;
     const AVCodecDescriptor * const *db = b;
 
-    return (*da)->type != (*db)->type ? (*da)->type - (*db)->type :
+    return (*da)->type != (*db)->type ? FFDIFFSIGN((*da)->type, (*db)->type) :
            strcmp((*da)->name, (*db)->name);
 }
 

cmdutils_opencl.c

@@ -206,7 +206,9 @@ static int64_t run_opencl_bench(AVOpenCLExternalEnv *ext_opencl_env)
 
 static int compare_ocl_device_desc(const void *a, const void *b)
 {
-    return ((const OpenCLDeviceBenchmark*)a)->runtime - ((const OpenCLDeviceBenchmark*)b)->runtime;
+    const OpenCLDeviceBenchmark* va = (const OpenCLDeviceBenchmark*)a;
+    const OpenCLDeviceBenchmark* vb = (const OpenCLDeviceBenchmark*)b;
+    return FFDIFFSIGN(va->runtime , vb->runtime);
 }
 
 int opt_opencl_bench(void *optctx, const char *opt, const char *arg)

ffmpeg.c

@@ -2578,8 +2578,7 @@ static InputStream *get_input_stream(OutputStream *ost)
 
 static int compare_int64(const void *a, const void *b)
 {
-    int64_t va = *(int64_t *)a, vb = *(int64_t *)b;
-    return va < vb ? -1 : va > vb ? +1 : 0;
+    return FFDIFFSIGN(*(const int64_t *)a, *(const int64_t *)b);
 }
 
 static int init_output_stream(OutputStream *ost, char *error, int error_len)

libavfilter/f_sendcmd.c

@@ -364,11 +364,7 @@ static int cmp_intervals(const void *a, const void *b)
 {
     const Interval *i1 = a;
     const Interval *i2 = b;
-    int64_t ts_diff = i1->start_ts - i2->start_ts;
-    int ret;
-
-    ret = ts_diff > 0 ? 1 : ts_diff < 0 ? -1 : 0;
-    return ret == 0 ? i1->index - i2->index : ret;
+    return 2 * FFDIFFSIGN(i1->start_ts, i2->start_ts) + FFDIFFSIGN(i1->index, i2->index);
 }
 
 static av_cold int init(AVFilterContext *ctx)

libavfilter/vf_deshake.c

@@ -94,8 +94,7 @@ AVFILTER_DEFINE_CLASS(deshake);
 
 static int cmp(const void *a, const void *b)
 {
-    const double va = *(const double *)a, vb = *(const double *)b;
-    return va < vb ? -1 : ( va > vb ? 1 : 0 );
+    return FFDIFFSIGN(*(const double *)a, *(const double *)b);
 }
 
 /**

libavfilter/vf_palettegen.c

@@ -130,7 +130,7 @@ static int cmp_color(const void *a, const void *b)
 {
     const struct range_box *box1 = a;
     const struct range_box *box2 = b;
-    return box1->color - box2->color;
+    return FFDIFFSIGN(box1->color , box2->color);
 }
 
 static av_always_inline int diff(const uint32_t a, const uint32_t b)

libavfilter/vf_removegrain.c

@@ -83,10 +83,9 @@ static int mode01(int c, int a1, int a2, int a3, int a4, int a5, int a6, int a7,
 
 static int cmp_int(const void *p1, const void *p2)
 {
-    int left = *(const int *)p1;
+    int left  = *(const int *)p1;
     int right = *(const int *)p2;
-
-    return ((left > right) - (left < right));
+    return FFDIFFSIGN(left, right);
 }
 
 static int mode02(int c, int a1, int a2, int a3, int a4, int a5, int a6, int a7, int a8)

libavformat/subtitles.c

@@ -146,12 +146,9 @@ static int cmp_pkt_sub_ts_pos(const void *a, const void *b)
 {
     const AVPacket *s1 = a;
     const AVPacket *s2 = b;
-    if (s1->pts == s2->pts) {
-        if (s1->pos == s2->pos)
-            return 0;
-        return s1->pos > s2->pos ? 1 : -1;
-    }
-    return s1->pts > s2->pts ? 1 : -1;
+    if (s1->pts == s2->pts)
+        return FFDIFFSIGN(s1->pos, s2->pos);
+    return FFDIFFSIGN(s1->pts , s2->pts);
 }
 
 static int cmp_pkt_sub_pos_ts(const void *a, const void *b)