add interop test against parquet-testing

Author: wgtmac

parquet-column/src/main/java/org/apache/parquet/column/statistics/Statistics.java

@@ -387,6 +387,7 @@ public int hashCode() {
     return 31 * type.hashCode()
         + 31 * Arrays.hashCode(getMaxBytes())
         + 17 * Arrays.hashCode(getMinBytes())
+        + 13 * Long.valueOf(this.getNanCount()).hashCode()
         + Long.valueOf(this.getNumNulls()).hashCode();
   }
 

parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/ColumnIndexBuilder.java

@@ -32,7 +32,6 @@
 import it.unimi.dsi.fastutil.longs.LongLists;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Formatter;
 import java.util.List;
 import java.util.PrimitiveIterator;
@@ -255,10 +254,6 @@ boolean isNullPage(int pageIndex) {
       return nullPages[pageIndex];
     }
 
-    private int getArrayIndex(int pageIndex) {
-      return Arrays.binarySearch(pageIndexes, pageIndex);
-    }
-
     // Returns true if this ColumnIndex may have NaN-polluted min/max values that cannot be trusted.
     // If true, we need to conservatively include pages with NaN in min/max for some predicates.
     boolean mayHaveNaNPollutedMinMax() {
@@ -285,12 +280,6 @@ boolean hasNaNMinMax(int arrayIndex) {
       return isMinNaN(arrayIndex) || isMaxNaN(arrayIndex);
     }
 
-    // Returns true if this page is a confirmed all-NaN page:
-    // min==max==NaN and nanCounts confirms at least one NaN value.
-    private boolean isAllNaNs(int arrayIndex, int pageIndex) {
-      return nanCounts != null && nanCounts[pageIndex] > 0 && isMinNaN(arrayIndex) && isMaxNaN(arrayIndex);
-    }
-
     /*
      * Returns the min value for arrayIndex as a ByteBuffer. (Min values are not stored for null-pages so arrayIndex
      * might not equal to pageIndex.)
@@ -428,9 +417,12 @@ public <T extends Comparable<T>> PrimitiveIterator.OfInt visit(NotEq<T> notEq) {
       }
 
       if (isNaNLiteral(value)) {
-        return nanCounts == null
-            ? IndexIterator.all(getPageCount())
-            : IndexIterator.filter(getPageCount(), pageIndex -> nanCounts[pageIndex] == 0);
+        // v != NaN is satisfied by every null value, every non-NaN value, and (under IEEE754 total
+        // order) every NaN whose bit pattern differs from the literal. The page-level min/max and
+        // nan_count are not sufficient to prove that *all* values of a page equal the NaN literal
+        // (e.g. a page may mix NaNs with nulls, or hold NaNs with different payloads), so we
+        // conservatively keep every page.
+        return IndexIterator.all(getPageCount());
       }
 
       if (nullCounts == null) {

parquet-column/src/test/java/org/apache/parquet/internal/column/columnindex/TestColumnIndexBuilderNaN.java

@@ -276,7 +276,7 @@ public void testNaNFloatMixed() {
     assertEquals(List.of(0), toList(ci.visit(FilterApi.in(FLOAT_COL, new HashSet<>(List.of(1.5f))))));
 
     assertEquals(List.of(1), toList(ci.visit(FilterApi.eq(FLOAT_COL, Float.NaN))));
-    assertEquals(List.of(0, 2), toList(ci.visit(FilterApi.notEq(FLOAT_COL, Float.NaN))));
+    assertEquals(List.of(0, 1, 2), toList(ci.visit(FilterApi.notEq(FLOAT_COL, Float.NaN))));
     assertEquals(List.of(0, 1, 2), toList(ci.visit(FilterApi.lt(FLOAT_COL, Float.NaN))));
     assertEquals(List.of(0, 1, 2), toList(ci.visit(FilterApi.ltEq(FLOAT_COL, Float.NaN))));
     assertEquals(List.of(0, 1, 2), toList(ci.visit(FilterApi.gt(FLOAT_COL, Float.NaN))));
@@ -331,7 +331,7 @@ public void testNaNDoubleMixed() {
     assertEquals(List.of(0), toList(ci.visit(FilterApi.in(DOUBLE_COL, new HashSet<>(List.of(1.5))))));
 
     assertEquals(List.of(1), toList(ci.visit(FilterApi.eq(DOUBLE_COL, Double.NaN))));
-    assertEquals(List.of(0, 2), toList(ci.visit(FilterApi.notEq(DOUBLE_COL, Double.NaN))));
+    assertEquals(List.of(0, 1, 2), toList(ci.visit(FilterApi.notEq(DOUBLE_COL, Double.NaN))));
     assertEquals(List.of(0, 1, 2), toList(ci.visit(FilterApi.lt(DOUBLE_COL, Double.NaN))));
     assertEquals(List.of(0, 1, 2), toList(ci.visit(FilterApi.ltEq(DOUBLE_COL, Double.NaN))));
     assertEquals(List.of(0, 1, 2), toList(ci.visit(FilterApi.gt(DOUBLE_COL, Double.NaN))));
@@ -386,7 +386,7 @@ public void testNaNFloat16Mixed() {
     assertEquals(List.of(0), toList(ci.visit(FilterApi.in(FLOAT16_COL, new HashSet<>(List.of(FLOAT16_ONE))))));
 
     assertEquals(List.of(1), toList(ci.visit(FilterApi.eq(FLOAT16_COL, FLOAT16_NAN))));
-    assertEquals(List.of(0, 2), toList(ci.visit(FilterApi.notEq(FLOAT16_COL, FLOAT16_NAN))));
+    assertEquals(List.of(0, 1, 2), toList(ci.visit(FilterApi.notEq(FLOAT16_COL, FLOAT16_NAN))));
     assertEquals(List.of(0, 1, 2), toList(ci.visit(FilterApi.lt(FLOAT16_COL, FLOAT16_NAN))));
     assertEquals(List.of(0, 1, 2), toList(ci.visit(FilterApi.ltEq(FLOAT16_COL, FLOAT16_NAN))));
     assertEquals(List.of(0, 1, 2), toList(ci.visit(FilterApi.gt(FLOAT16_COL, FLOAT16_NAN))));

parquet-hadoop/src/main/java/org/apache/parquet/filter2/statisticslevel/StatisticsFilter.java

@@ -371,7 +371,10 @@ public <T extends Comparable<T>> Boolean visit(NotEq<T> notEq) {
     }
 
     if (isNaNLiteral(meta, value)) {
-      return (stats.isNanCountSet() && stats.getNanCount() == 0) ? BLOCK_CANNOT_MATCH : BLOCK_MIGHT_MATCH;
+      // We are looking for records where v != NaN. Every non-NaN value (and every null) satisfies
+      // this predicate, and the chunk-level statistics cannot prove that all values are NaN in a
+      // way that lets us drop the block safely, so we conservatively keep it.
+      return BLOCK_MIGHT_MATCH;
     }
 
     if (isAllNaNs(meta)) {

parquet-hadoop/src/test/java/org/apache/parquet/filter2/statisticslevel/TestStatisticsFilter.java

@@ -708,7 +708,7 @@ public void testNaNDoubleZeroNaNCount() {
     assertFalse(canDrop(in(doubleColumn, new HashSet<>(List.of(50.0))), metas));
 
     assertTrue(canDrop(eq(doubleColumn, Double.NaN), metas));
-    assertTrue(canDrop(notEq(doubleColumn, Double.NaN), metas));
+    assertFalse(canDrop(notEq(doubleColumn, Double.NaN), metas));
     assertFalse(canDrop(lt(doubleColumn, Double.NaN), metas));
     assertFalse(canDrop(ltEq(doubleColumn, Double.NaN), metas));
     assertFalse(canDrop(gt(doubleColumn, Double.NaN), metas));
@@ -825,7 +825,7 @@ public void testNaNFloatZeroNaNCount() {
     assertFalse(canDrop(in(floatCol, new HashSet<>(List.of(50.0f))), metas));
 
     assertTrue(canDrop(eq(floatCol, Float.NaN), metas));
-    assertTrue(canDrop(notEq(floatCol, Float.NaN), metas));
+    assertFalse(canDrop(notEq(floatCol, Float.NaN), metas));
     assertFalse(canDrop(lt(floatCol, Float.NaN), metas));
     assertFalse(canDrop(ltEq(floatCol, Float.NaN), metas));
     assertFalse(canDrop(gt(floatCol, Float.NaN), metas));
@@ -947,7 +947,7 @@ public void testNaNFloat16ZeroNaNCount() {
     assertFalse(canDrop(in(float16Column, new HashSet<>(List.of(FLOAT16_FIFTY))), metas));
 
     assertTrue(canDrop(eq(float16Column, FLOAT16_NAN), metas));
-    assertTrue(canDrop(notEq(float16Column, FLOAT16_NAN), metas));
+    assertFalse(canDrop(notEq(float16Column, FLOAT16_NAN), metas));
     assertFalse(canDrop(lt(float16Column, FLOAT16_NAN), metas));
     assertFalse(canDrop(ltEq(float16Column, FLOAT16_NAN), metas));
     assertFalse(canDrop(gt(float16Column, FLOAT16_NAN), metas));

parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestInterOpReadFloatingPointNanCount.java

@@ -50,6 +50,15 @@
 
 public class TestInterOpReadFloatingPointNanCount {
 
+  // Canonical parquet-testing fixture exercising the same scenarios as the local generator, as a
+  // single file with five row groups. parquet-java's writer produces the same statistics and
+  // column index layout, so the reference file is verified with the exact same expectations as the
+  // locally generated merged file.
+  private static final String REFERENCE_FILE = "floating_orders_nan_count.parquet";
+  private static final String REFERENCE_CHANGESET = "ffdcbb5e22828186c7461e56dbd26a0fe3caee56";
+
+  private final InterOpTester interop = new InterOpTester();
+
   @Rule
   public TemporaryFolder temp = new TemporaryFolder();
 
@@ -74,6 +83,25 @@ public void testReadStatsAndColumnIndex() throws IOException {
         Scenario.ZERO_MAX);
   }
 
+  /**
+   * Reads the canonical {@code floating_orders_nan_count.parquet} from the parquet-testing repo and
+   * verifies it against the same expectations as the locally generated files, confirming that
+   * parquet-java reads the reference file (produced by another writer) consistently.
+   */
+  @Test
+  public void testReadReferenceFile() throws IOException {
+    Configuration conf = new Configuration();
+    Path file = interop.GetInterOpFile(REFERENCE_FILE, REFERENCE_CHANGESET);
+    verifyFile(
+        file,
+        conf,
+        Scenario.NO_NAN,
+        Scenario.MIXED_NAN,
+        Scenario.ALL_NAN,
+        Scenario.ZERO_MIN,
+        Scenario.ZERO_MAX);
+  }
+
   private static void verifyFile(Path file, Configuration conf, Scenario... scenarios) throws IOException {
     List<Float> expectFloats = concatFloatValues(scenarios);
     List<Double> expectDoubles = concatDoubleValues(scenarios);