Merge branch 'prestodb:master' into wanweitao/fix-presto-ui

Author: wanweitao

.github/workflows/prestocpp-macos-build.yml

@@ -3,15 +3,33 @@ name: prestocpp-macos-build
 on:
   workflow_dispatch:
   pull_request:
-    paths:
-      - 'presto-native-execution/**'
-      - '.github/workflows/prestocpp-macos-build.yml'
+
 jobs:
+  changes:
+    runs-on: macos-15
+    # Required permissions
+    permissions:
+      pull-requests: read
+    # Set job outputs to values from filter step
+    outputs:
+      codechange: ${{ steps.filter.outputs.codechange }}
+    steps:
+      # For pull requests it's not necessary to checkout the code
+      - name: Run changes check for PRs
+        uses: dorny/paths-filter@7267a8516b6f92bdb098633497bad573efdbf271
+        id: filter
+        with:
+          filters: |
+            codechange:
+              - 'presto-native-execution/**'
+              - '.github/workflows/prestocpp-macos-build.yml'
+
   prestocpp-macos-build-engine:
     strategy:
       matrix:
         os: [macos-13, macos-15]
     runs-on: ${{ matrix.os }}
+    needs: changes
     env:
       CCACHE_DIR: "${{ github.workspace }}/ccache"
       CMAKE_POLICY_VERSION_MINIMUM: "3.5"
@@ -24,18 +42,26 @@ jobs:
       cancel-in-progress: true
     steps:
       - uses: actions/checkout@v4
+        if: |
+          needs.changes.outputs.codechange == 'true'
 
       - name: Fix git permissions
+        if: |
+          needs.changes.outputs.codechange == 'true'
         # Usually actions/checkout does this but as we run in a container
         # it doesn't work
         run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
 
       - name: Update submodules
+        if: |
+          needs.changes.outputs.codechange == 'true'
         run: |
           cd presto-native-execution
           make submodules
 
       - name: "Setup MacOS"
+        if: |
+          needs.changes.outputs.codechange == 'true'
         run: |
           set -xu
           source presto-native-execution/scripts/setup-macos.sh
@@ -68,18 +94,26 @@ jobs:
           brew link --force protobuf@21
 
       - name: Install Github CLI for using apache/infrastructure-actions/stash
+        if: |
+          needs.changes.outputs.codechange == 'true'
         run: |
           brew install gh
 
       - uses: apache/infrastructure-actions/stash/restore@4ab8682fbd4623d2b4fc1c98db38aba5091924c3
+        if: |
+          needs.changes.outputs.codechange == 'true'
         with:
           path: '${{ env.CCACHE_DIR }}'
           key: ccache-prestocpp-macos-build-engine-${{ matrix.os }}
 
       - name: Zero ccache statistics
+        if: |
+          needs.changes.outputs.codechange == 'true'
         run: PATH=${INSTALL_PREFIX}/bin:${PATH} ccache -sz
 
       - name: "Build presto_cpp on MacOS"
+        if: |
+          needs.changes.outputs.codechange == 'true'
         run: |
           clang --version
           export PATH=$(brew --prefix m4)/bin:$(brew --prefix bison)/bin:${PATH}
@@ -99,9 +133,13 @@ jobs:
           ninja -C _build/${BUILD_TYPE} -j ${NJOBS}
 
       - name: Ccache after
+        if: |
+          needs.changes.outputs.codechange == 'true'
         run: PATH=${INSTALL_PREFIX}/bin:${PATH} ccache -s
 
       - uses: apache/infrastructure-actions/stash/save@4ab8682fbd4623d2b4fc1c98db38aba5091924c3
+        if: |
+          needs.changes.outputs.codechange == 'true'
         with:
           path: '${{ env.CCACHE_DIR }}'
           key: ccache-prestocpp-macos-build-engine-${{ matrix.os }}

CONTRIBUTING.md

@@ -342,7 +342,7 @@ Note: Each PR/commit should have a single primary type. If your changes span mul
 * `spi` - Service Provider Interface changes
 * `scheduler` - Task scheduling and execution
 * `protocol` - Wire protocol and serialization
-* `connector` - Changes to broader connector functionality or conector SPI
+* `connector` - Changes to broader connector functionality or connector SPI
 * `resource` - Resource management (memory manager, resource groups)
 * `security` - Authentication and authorization
 * `function` - Built-in functions and operators

pom.xml

@@ -83,7 +83,7 @@
         <dep.commons.compress.version>1.26.2</dep.commons.compress.version>
         <dep.protobuf-java.version>4.29.0</dep.protobuf-java.version>
         <dep.jetty.version>12.0.18</dep.jetty.version>
-        <dep.netty.version>4.1.126.Final</dep.netty.version>
+        <dep.netty.version>4.1.128.Final</dep.netty.version>
         <dep.reactor-netty.version>1.2.8</dep.reactor-netty.version>
         <dep.snakeyaml.version>2.5</dep.snakeyaml.version>
         <dep.gson.version>2.12.1</dep.gson.version>

presto-docs/src/main/sphinx/connector/iceberg.rst

@@ -347,9 +347,9 @@ Property Name                                           Description
 
 ``iceberg.compression-codec``                           The compression codec to use when writing files. The          ``GZIP``                           Yes                 No, write is not supported yet
                                                         available values are ``NONE``, ``SNAPPY``, ``GZIP``,
-                                                        and ``ZSTD``.
+                                                        ``LZ4``, and ``ZSTD``.
 
-                                                        Note: ``ZSTD`` is only available when 
+                                                        Note: ``LZ4`` is only available when
                                                         ``iceberg.file-format=ORC``.
 
 

presto-docs/src/main/sphinx/functions/math.rst

@@ -266,9 +266,14 @@ Probability Functions: cdf
     Compute the Poisson cdf with given lambda (mean) parameter:  P(N <= value; lambda).
     The lambda parameter must be a positive real number (of type DOUBLE) and value must be a non-negative integer.
 
+.. function:: t_cdf(df, value) -> double
+
+    Compute the Student's t cdf with given degrees of freedom:  P(N < value; df).
+    The degrees of freedom must be a positive real number and value must be a real value.
+
 .. function:: weibull_cdf(a, b, value) -> double
 
-    Compute the Weibull cdf with given parameters a, b: P(N <= value). The ``a``
+    Compute the Weibull cdf with given parameters a, b:  P(N <= value). The ``a``
     and ``b`` parameters must be positive doubles and ``value`` must also be a double.
 
 
@@ -332,6 +337,12 @@ Probability Functions: inverse_cdf
     The lambda parameter must be a positive real number (of type DOUBLE).
     The probability p must lie on the interval [0, 1).
 
+.. function:: inverse_t_cdf(df, p) -> double
+
+    Compute the inverse of the Student's t cdf with given degrees of freedom for the cumulative
+    probability (p): P(N < n). The degrees of freedom must be a positive real value.
+    The probability p must lie on the interval [0, 1].
+
 .. function:: inverse_weibull_cdf(a, b, p) -> double
 
     Compute the inverse of the Weibull cdf with given parameters ``a``, ``b`` for the probability ``p``.

presto-docs/src/main/sphinx/presto-cpp.rst

@@ -84,4 +84,11 @@ For more information see :doc:`/connector/iceberg` documentation.
 TPCH Connector
 ^^^^^^^^^^^^^^
 
-* TPCH connector, with ``tpch.naming=standard`` catalog property.
+* TPCH connector, with ``tpch.naming=standard`` catalog property.
+
+TPC-DS Connector
+^^^^^^^^^^^^^^^^
+
+* TPC-DS connector, with ``tpcds.use-varchar-type=true`` in the coordinator's TPCDS catalog file.
+
+For more information see :doc:`/connector/tpcds` documentation.

presto-hive/src/main/java/com/facebook/presto/hive/HiveCompressionCodec.java

@@ -16,6 +16,7 @@
 import com.facebook.presto.orc.metadata.CompressionKind;
 import org.apache.hadoop.io.compress.CompressionCodec;
 import org.apache.hadoop.io.compress.GzipCodec;
+import org.apache.hadoop.io.compress.Lz4Codec;
 import org.apache.hadoop.io.compress.SnappyCodec;
 import org.apache.parquet.hadoop.metadata.CompressionCodecName;
 
@@ -25,19 +26,20 @@
 import static com.facebook.presto.hive.HiveStorageFormat.DWRF;
 import static com.facebook.presto.hive.HiveStorageFormat.ORC;
 import static com.facebook.presto.hive.HiveStorageFormat.PAGEFILE;
+import static com.facebook.presto.hive.HiveStorageFormat.PARQUET;
 import static java.util.Objects.requireNonNull;
 
 public enum HiveCompressionCodec
 {
     NONE(null, CompressionKind.NONE, CompressionCodecName.UNCOMPRESSED, f -> true),
     SNAPPY(SnappyCodec.class, CompressionKind.SNAPPY, CompressionCodecName.SNAPPY, f -> true),
     GZIP(GzipCodec.class, CompressionKind.ZLIB, CompressionCodecName.GZIP, f -> true),
-    LZ4(null, CompressionKind.NONE, null, f -> f == PAGEFILE),
-    ZSTD(null, CompressionKind.ZSTD, null, f -> f == ORC || f == DWRF || f == PAGEFILE);
+    LZ4(Lz4Codec.class, CompressionKind.LZ4, CompressionCodecName.UNCOMPRESSED, f -> f == PAGEFILE || f == ORC),
+    ZSTD(null, CompressionKind.ZSTD, CompressionCodecName.ZSTD, f -> f == ORC || f == DWRF || f == PAGEFILE || f == PARQUET);
 
     private final Optional<Class<? extends CompressionCodec>> codec;
     private final CompressionKind orcCompressionKind;
-    private final Optional<CompressionCodecName> parquetCompressionCodec;
+    private final CompressionCodecName parquetCompressionCodec;
     private final Predicate<HiveStorageFormat> supportedStorageFormats;
 
     HiveCompressionCodec(
@@ -48,7 +50,7 @@ public enum HiveCompressionCodec
     {
         this.codec = Optional.ofNullable(codec);
         this.orcCompressionKind = requireNonNull(orcCompressionKind, "orcCompressionKind is null");
-        this.parquetCompressionCodec = Optional.ofNullable(parquetCompressionCodec);
+        this.parquetCompressionCodec = requireNonNull(parquetCompressionCodec, "parquetCompressionCodec is null");
         this.supportedStorageFormats = requireNonNull(supportedStorageFormats, "supportedStorageFormats is null");
     }
 
@@ -62,7 +64,7 @@ public CompressionKind getOrcCompressionKind()
         return orcCompressionKind;
     }
 
-    public Optional<CompressionCodecName> getParquetCompressionCodec()
+    public CompressionCodecName getParquetCompressionCodec()
     {
         return parquetCompressionCodec;
     }

presto-hive/src/main/java/com/facebook/presto/hive/HiveSessionProperties.java

@@ -59,7 +59,7 @@ public final class HiveSessionProperties
     private static final String ORC_OPTIMIZED_WRITER_COMPRESSION_LEVEL = "orc_optimized_writer_compression_level";
     private static final String PAGEFILE_WRITER_MAX_STRIPE_SIZE = "pagefile_writer_max_stripe_size";
     public static final String HIVE_STORAGE_FORMAT = "hive_storage_format";
-    private static final String COMPRESSION_CODEC = "compression_codec";
+    static final String COMPRESSION_CODEC = "compression_codec";
     private static final String ORC_COMPRESSION_CODEC = "orc_compression_codec";
     public static final String RESPECT_TABLE_FORMAT = "respect_table_format";
     private static final String CREATE_EMPTY_BUCKET_FILES = "create_empty_bucket_files";

presto-hive/src/main/java/com/facebook/presto/hive/util/ConfigurationUtils.java

@@ -120,7 +120,7 @@ private static void setCompressionProperties(Configuration config, HiveCompressi
             config.unset(FileOutputFormat.COMPRESS_CODEC);
         }
         // For Parquet
-        compression.getParquetCompressionCodec().ifPresent(codec -> config.set(ParquetOutputFormat.COMPRESSION, codec.name()));
+        config.set(ParquetOutputFormat.COMPRESSION, compression.getParquetCompressionCodec().name());
         // For SequenceFile
         config.set(FileOutputFormat.COMPRESS_TYPE, BLOCK.toString());
         // For PageFile

presto-hive/src/test/java/com/facebook/presto/hive/TestHiveIntegrationSmokeTest.java

@@ -74,6 +74,7 @@
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 import java.util.stream.LongStream;
+import java.util.stream.Stream;
 
 import static com.facebook.airlift.json.JsonCodec.jsonCodec;
 import static com.facebook.presto.SystemSessionProperties.COLOCATED_JOIN;
@@ -106,6 +107,7 @@
 import static com.facebook.presto.hive.HiveQueryRunner.TPCH_SCHEMA;
 import static com.facebook.presto.hive.HiveQueryRunner.createBucketedSession;
 import static com.facebook.presto.hive.HiveQueryRunner.createMaterializeExchangesSession;
+import static com.facebook.presto.hive.HiveSessionProperties.COMPRESSION_CODEC;
 import static com.facebook.presto.hive.HiveSessionProperties.FILE_RENAMING_ENABLED;
 import static com.facebook.presto.hive.HiveSessionProperties.MANIFEST_VERIFICATION_ENABLED;
 import static com.facebook.presto.hive.HiveSessionProperties.OPTIMIZED_PARTITION_UPDATE_SERIALIZATION_ENABLED;
@@ -159,6 +161,7 @@
 import static io.airlift.tpch.TpchTable.PART_SUPPLIER;
 import static java.lang.String.format;
 import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.util.Locale.ROOT;
 import static java.util.Objects.requireNonNull;
 import static java.util.stream.Collectors.joining;
 import static org.assertj.core.api.Assertions.assertThat;
@@ -5267,17 +5270,6 @@ public void testPageFileFormatSmallSplitSize()
         assertUpdate("DROP TABLE test_pagefile_small_split");
     }
 
-    @Test
-    public void testPageFileCompression()
-    {
-        for (HiveCompressionCodec compression : HiveCompressionCodec.values()) {
-            if (!compression.isSupportedStorageFormat(PAGEFILE)) {
-                continue;
-            }
-            testPageFileCompression(compression.name());
-        }
-    }
-
     @Test
     public void testPartialAggregatePushdownORC()
     {
@@ -5703,31 +5695,35 @@ public void testParquetSelectivePageSourceFails()
         assertQueryFails(parquetFilterPushdownSession, "SELECT a FROM test_parquet_filter_pushdoown WHERE b = false", "Parquet reader doesn't support filter pushdown yet");
     }
 
-    private void testPageFileCompression(String compression)
+    @DataProvider(name = "testFormatAndCompressionCodecs")
+    public Object[][] compressionCodecs()
     {
-        Session testSession = Session.builder(getQueryRunner().getDefaultSession())
-                .setCatalogSessionProperty(catalog, "compression_codec", compression)
-                .setCatalogSessionProperty(catalog, "pagefile_writer_max_stripe_size", "100B")
-                .setCatalogSessionProperty(catalog, "max_split_size", "1kB")
-                .setCatalogSessionProperty(catalog, "max_initial_split_size", "1kB")
-                .build();
-
-        assertUpdate(
-                testSession,
-                "CREATE TABLE test_pagefile_compression\n" +
-                        "WITH (\n" +
-                        "format = 'PAGEFILE'\n" +
-                        ") AS\n" +
-                        "SELECT\n" +
-                        "*\n" +
-                        "FROM tpch.orders",
-                "SELECT count(*) FROM orders");
-
-        assertQuery(testSession, "SELECT count(*) FROM test_pagefile_compression", "SELECT count(*) FROM orders");
-
-        assertQuery(testSession, "SELECT sum(custkey) FROM test_pagefile_compression", "SELECT sum(custkey) FROM orders");
+        return Stream.of(PARQUET, ORC, PAGEFILE)
+                .flatMap(format -> Arrays.stream(HiveCompressionCodec.values())
+                        .map(codec -> new Object[] {codec, format}))
+                .toArray(Object[][]::new);
+    }
 
-        assertUpdate("DROP TABLE test_pagefile_compression");
+    @Test(dataProvider = "testFormatAndCompressionCodecs")
+    public void testFormatAndCompressionCodecs(HiveCompressionCodec codec, HiveStorageFormat format)
+    {
+        String tableName = "test_" + format.name().toLowerCase(ROOT) + "_compression_codec_" + codec.name().toLowerCase(ROOT);
+        Session session = Session.builder(getSession())
+                .setCatalogSessionProperty("hive", COMPRESSION_CODEC, codec.name()).build();
+        if (codec.isSupportedStorageFormat(format == PARQUET ? HiveStorageFormat.PARQUET : HiveStorageFormat.ORC)) {
+            assertUpdate(session,
+                    format("CREATE TABLE %s WITH (format = '%s') AS SELECT * FROM orders",
+                            tableName, format.name()),
+                    "SELECT count(*) FROM orders");
+            assertQuery(format("SELECT count(*) FROM %s", tableName), "SELECT count(*) FROM orders");
+            assertQuery(format("SELECT sum(custkey) FROM %s", tableName), "SELECT sum(custkey) FROM orders");
+            assertQuerySucceeds(format("DROP TABLE %s", tableName));
+        }
+        else {
+            assertQueryFails(session, format("CREATE TABLE %s WITH (format = '%s') AS SELECT * FROM orders",
+                            tableName, format.name()),
+                    format("%s compression is not supported with %s", codec, format));
+        }
     }
 
     private static Consumer<Plan> assertTableWriterMergeNodeIsPresent()