From 8f58a1bac7a2aff41a3b13d00d5888e13ac46f5d Mon Sep 17 00:00:00 2001
From: Ell <ell_se@yahoo.com>
Date: Mon, 19 Aug 2019 17:33:29 +0300
Subject: [PATCH] avx2, two-table: don't segfault for NaN input

In the AVX2 and two-table linear-float => gamma-int8 conversions,
tweak the input bounds-check to handle NaN values.  NaN would
previously lead to an out-of-bounds table lookup, and a segfault
(see issue #43).
---
 extensions/avx2-int8.c | 36 ++++++++++++++++++------------------
 extensions/two-table.c | 36 ++++++++++++++++++++++--------------
 2 files changed, 40 insertions(+), 32 deletions(-)

diff --git a/extensions/avx2-int8.c b/extensions/avx2-int8.c
index 3eaf225..b6d5165 100644
--- a/extensions/avx2-int8.c
+++ b/extensions/avx2-int8.c
@@ -41,31 +41,31 @@
                                                           \
       if (x < 0.0f)                                       \
         *dst = 0;                                         \
-      else if (x > 1.0f)                                  \
-        *dst = 255;                                       \
-      else                                                \
+      else if (x <= 1.0f)                                 \
         *dst = linear_to_gamma[(int) (SCALE * x + 0.5f)]; \
+      else /* x > 1.0f || isnan (x) */                    \
+        *dst = 255;                                       \
                                                           \
       src++;                                              \
       dst++;                                              \
     }                                                     \
   while (0)
 
-#define CVTA1(src, dst)           \
-  do                              \
-    {                             \
-      float x = *src;             \
-                                  \
-      if (x < 0.0f)               \
-        *dst = 0;                 \
-      else if (x > 1.0f)          \
-        *dst = 255;               \
-      else                        \
-        *dst = 255.0f * x + 0.5f; \
-                                  \
-      src++;                      \
-      dst++;                      \
-    }                             \
+#define CVTA1(src, dst)                                   \
+  do                                                      \
+    {                                                     \
+      float x = *src;                                     \
+                                                          \
+      if (x < 0.0f)                                       \
+        *dst = 0;                                         \
+      else if (x <= 1.0f)                                 \
+        *dst = 255.0f * x + 0.5f;                         \
+      else /* x > 1.0f || isnan (x) */                    \
+        *dst = 255;                                       \
+                                                          \
+      src++;                                              \
+      dst++;                                              \
+    }                                                     \
   while (0)
 
 static inline void
diff --git a/extensions/two-table.c b/extensions/two-table.c
index efdfa70..8becfee 100644
--- a/extensions/two-table.c
+++ b/extensions/two-table.c
@@ -25,21 +25,29 @@
 static inline unsigned char
 conv_float_u8_two_table_map (float value)
 {
-  unsigned short index;
-  unsigned char result;
   if (value < 0.0f)
-    return 0;
-  else if (value > 1.0f)
-    return 0xFF;
-  index = (unsigned short)(value * 0xFFFF);
-  result = linear_to_gamma[index];
-
-  if (value < u8_gamma_minimums[result])
-    result -= 1;
-  else if (value >= u8_gamma_minimums[result+1])
-    result += 1;
-
-  return result;
+    {
+      return 0;
+    }
+  else if (value <= 1.0f)
+    {
+      unsigned short index;
+      unsigned char  result;
+
+      index  = (unsigned short) (value * 0xFFFF);
+      result = linear_to_gamma[index];
+
+      if (value < u8_gamma_minimums[result])
+        result -= 1;
+      else if (value >= u8_gamma_minimums[result+1])
+        result += 1;
+
+      return result;
+    }
+  else /* value > 1.0f || isnan (value) */
+    {
+      return 0xFF;
+    }
 }
 
 static void
-- 
2.30.2