preferredOrderingColumns,
@JsonProperty("tableStorageFormat") HiveStorageFormat tableStorageFormat,
@JsonProperty("partitionStorageFormat") HiveStorageFormat partitionStorageFormat,
+ @JsonProperty("actualStorageFormat") HiveStorageFormat actualStorageFormat,
@JsonProperty("compressionCodec") HiveCompressionCodec compressionCodec)
{
super(
@@ -51,6 +52,7 @@ public HiveInsertTableHandle(
preferredOrderingColumns,
tableStorageFormat,
partitionStorageFormat,
+ actualStorageFormat,
compressionCodec);
}
}
diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java
index a3c832fd37003..6d67612439e94 100644
--- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java
+++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java
@@ -14,6 +14,14 @@
package com.facebook.presto.hive;
import com.facebook.airlift.json.JsonCodec;
+import com.facebook.presto.common.Subfield;
+import com.facebook.presto.common.block.Block;
+import com.facebook.presto.common.predicate.Domain;
+import com.facebook.presto.common.predicate.NullableValue;
+import com.facebook.presto.common.predicate.TupleDomain;
+import com.facebook.presto.common.type.Type;
+import com.facebook.presto.common.type.TypeManager;
+import com.facebook.presto.common.type.VarcharType;
import com.facebook.presto.hive.HdfsEnvironment.HdfsContext;
import com.facebook.presto.hive.LocationService.WriteInfo;
import com.facebook.presto.hive.PartitionUpdate.FileWriteInfo;
@@ -55,21 +63,15 @@
import com.facebook.presto.spi.SchemaTableName;
import com.facebook.presto.spi.SchemaTablePrefix;
import com.facebook.presto.spi.StandardErrorCode;
-import com.facebook.presto.spi.Subfield;
import com.facebook.presto.spi.SystemTable;
import com.facebook.presto.spi.TableNotFoundException;
import com.facebook.presto.spi.ViewNotFoundException;
-import com.facebook.presto.spi.block.Block;
import com.facebook.presto.spi.connector.ConnectorOutputMetadata;
import com.facebook.presto.spi.connector.ConnectorPartitioningHandle;
import com.facebook.presto.spi.connector.ConnectorPartitioningMetadata;
import com.facebook.presto.spi.connector.ConnectorTransactionHandle;
-import com.facebook.presto.spi.function.FunctionMetadataManager;
import com.facebook.presto.spi.function.StandardFunctionResolution;
import com.facebook.presto.spi.plan.FilterStatsCalculatorService;
-import com.facebook.presto.spi.predicate.Domain;
-import com.facebook.presto.spi.predicate.NullableValue;
-import com.facebook.presto.spi.predicate.TupleDomain;
import com.facebook.presto.spi.relation.RowExpression;
import com.facebook.presto.spi.relation.RowExpressionService;
import com.facebook.presto.spi.relation.SpecialFormExpression;
@@ -84,9 +86,6 @@
import com.facebook.presto.spi.statistics.TableStatisticType;
import com.facebook.presto.spi.statistics.TableStatistics;
import com.facebook.presto.spi.statistics.TableStatisticsMetadata;
-import com.facebook.presto.spi.type.Type;
-import com.facebook.presto.spi.type.TypeManager;
-import com.facebook.presto.spi.type.VarcharType;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Functions;
import com.google.common.base.Joiner;
@@ -118,8 +117,8 @@
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
-import java.util.ArrayList;
import java.util.Collection;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
@@ -137,6 +136,8 @@
import java.util.stream.Stream;
import static com.facebook.airlift.concurrent.MoreFutures.toCompletableFuture;
+import static com.facebook.presto.common.predicate.TupleDomain.withColumnDomains;
+import static com.facebook.presto.common.type.BigintType.BIGINT;
import static com.facebook.presto.expressions.LogicalRowExpressions.TRUE_CONSTANT;
import static com.facebook.presto.expressions.LogicalRowExpressions.binaryExpression;
import static com.facebook.presto.hive.HiveAnalyzeProperties.getPartitionList;
@@ -161,6 +162,7 @@
import static com.facebook.presto.hive.HivePartition.UNPARTITIONED_ID;
import static com.facebook.presto.hive.HiveSessionProperties.getCompressionCodec;
import static com.facebook.presto.hive.HiveSessionProperties.getHiveStorageFormat;
+import static com.facebook.presto.hive.HiveSessionProperties.getOrcCompressionCodec;
import static com.facebook.presto.hive.HiveSessionProperties.getTemporaryTableCompressionCodec;
import static com.facebook.presto.hive.HiveSessionProperties.getTemporaryTableSchema;
import static com.facebook.presto.hive.HiveSessionProperties.getTemporaryTableStorageFormat;
@@ -174,6 +176,10 @@
import static com.facebook.presto.hive.HiveSessionProperties.isSortedWriteToTempPathEnabled;
import static com.facebook.presto.hive.HiveSessionProperties.isSortedWritingEnabled;
import static com.facebook.presto.hive.HiveSessionProperties.isStatisticsEnabled;
+import static com.facebook.presto.hive.HiveStorageFormat.AVRO;
+import static com.facebook.presto.hive.HiveStorageFormat.DWRF;
+import static com.facebook.presto.hive.HiveStorageFormat.ORC;
+import static com.facebook.presto.hive.HiveStorageFormat.values;
import static com.facebook.presto.hive.HiveTableProperties.AVRO_SCHEMA_URL;
import static com.facebook.presto.hive.HiveTableProperties.BUCKETED_BY_PROPERTY;
import static com.facebook.presto.hive.HiveTableProperties.BUCKET_COUNT_PROPERTY;
@@ -235,10 +241,8 @@
import static com.facebook.presto.spi.StandardErrorCode.INVALID_TABLE_PROPERTY;
import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED;
import static com.facebook.presto.spi.StandardErrorCode.SCHEMA_NOT_EMPTY;
-import static com.facebook.presto.spi.predicate.TupleDomain.withColumnDomains;
import static com.facebook.presto.spi.security.PrincipalType.USER;
import static com.facebook.presto.spi.statistics.TableStatisticType.ROW_COUNT;
-import static com.facebook.presto.spi.type.BigintType.BIGINT;
import static com.google.common.base.MoreObjects.firstNonNull;
import static com.google.common.base.MoreObjects.toStringHelper;
import static com.google.common.base.Preconditions.checkArgument;
@@ -270,14 +274,6 @@ public class HiveMetadata
private static final String ORC_BLOOM_FILTER_FPP_KEY = "orc.bloom.filter.fpp";
private static final String PRESTO_TEMPORARY_TABLE_NAME_PREFIX = "__presto_temporary_table_";
- private static final ConnectorTableLayout EMPTY_TABLE_LAYOUT = new ConnectorTableLayout(
- new ConnectorTableLayoutHandle() {},
- Optional.empty(),
- TupleDomain.none(),
- Optional.empty(),
- Optional.empty(),
- Optional.empty(),
- emptyList());
// Comma is not a reserved keyword with or without quote
// See https://cwiki.apache.org/confluence/display/Hive/LanguageManual+DDL#LanguageManualDDL-Keywords,Non-reservedKeywordsandReservedKeywords
@@ -296,7 +292,6 @@ public class HiveMetadata
private final TypeManager typeManager;
private final LocationService locationService;
private final StandardFunctionResolution functionResolution;
- private final FunctionMetadataManager functionMetadataManager;
private final RowExpressionService rowExpressionService;
private final FilterStatsCalculatorService filterStatsCalculatorService;
private final TableParameterCodec tableParameterCodec;
@@ -323,7 +318,6 @@ public HiveMetadata(
TypeManager typeManager,
LocationService locationService,
StandardFunctionResolution functionResolution,
- FunctionMetadataManager functionMetadataManager,
RowExpressionService rowExpressionService,
FilterStatsCalculatorService filterStatsCalculatorService,
TableParameterCodec tableParameterCodec,
@@ -344,7 +338,6 @@ public HiveMetadata(
this.typeManager = requireNonNull(typeManager, "typeManager is null");
this.locationService = requireNonNull(locationService, "locationService is null");
this.functionResolution = requireNonNull(functionResolution, "functionResolution is null");
- this.functionMetadataManager = requireNonNull(functionMetadataManager, "functionMetadataManager is null");
this.rowExpressionService = requireNonNull(rowExpressionService, "rowExpressionService is null");
this.filterStatsCalculatorService = requireNonNull(filterStatsCalculatorService, "filterStatsCalculatorService is null");
this.tableParameterCodec = requireNonNull(tableParameterCodec, "tableParameterCodec is null");
@@ -730,7 +723,7 @@ public ColumnMetadata getColumnMetadata(ConnectorSession session, ConnectorTable
/**
* Returns a TupleDomain of constraints that is suitable for Explain (Type IO)
- *
+ *
* Only Hive partition columns that are used in IO planning.
*/
@Override
@@ -819,7 +812,7 @@ public void createTable(ConnectorSession session, ConnectorTableMetadata tableMe
}
else {
tableType = MANAGED_TABLE;
- LocationHandle locationHandle = locationService.forNewTable(metastore, session, schemaName, tableName, isTempPathRequired(session, bucketProperty));
+ LocationHandle locationHandle = locationService.forNewTable(metastore, session, schemaName, tableName, isTempPathRequired(session, bucketProperty, preferredOrderingColumns));
targetPath = locationService.getQueryWriteInfo(locationHandle).getTargetPath();
}
@@ -905,7 +898,7 @@ public ConnectorTableHandle createTemporaryTable(ConnectorSession session, List<
private void validateColumns(HiveStorageFormat hiveStorageFormat, List handles)
{
- if (hiveStorageFormat == HiveStorageFormat.AVRO) {
+ if (hiveStorageFormat == AVRO) {
for (HiveColumnHandle handle : handles) {
if (!handle.isPartitionKey()) {
validateAvroType(handle.getHiveType().getTypeInfo(), handle.getName());
@@ -962,7 +955,7 @@ private Map getEmptyTableProperties(ConnectorTableMetadata table
String avroSchemaUrl = getAvroSchemaUrl(tableMetadata.getProperties());
if (avroSchemaUrl != null) {
HiveStorageFormat hiveStorageFormat = getHiveStorageFormat(tableMetadata.getProperties());
- if (hiveStorageFormat != HiveStorageFormat.AVRO) {
+ if (hiveStorageFormat != AVRO) {
throw new PrestoException(INVALID_TABLE_PROPERTY, format("Cannot specify %s table property for storage format: %s", AVRO_SCHEMA_URL, hiveStorageFormat));
}
tableProperties.put(AVRO_SCHEMA_URL_KEY, validateAndNormalizeAvroSchemaUrl(avroSchemaUrl, hdfsContext));
@@ -1273,7 +1266,7 @@ public HiveOutputTableHandle beginCreateTable(ConnectorSession session, Connecto
.collect(toList());
checkPartitionTypesSupported(partitionColumns);
- LocationHandle locationHandle = locationService.forNewTable(metastore, session, schemaName, tableName, isTempPathRequired(session, bucketProperty));
+ LocationHandle locationHandle = locationService.forNewTable(metastore, session, schemaName, tableName, isTempPathRequired(session, bucketProperty, preferredOrderingColumns));
HiveOutputTableHandle result = new HiveOutputTableHandle(
schemaName,
tableName,
@@ -1283,7 +1276,8 @@ public HiveOutputTableHandle beginCreateTable(ConnectorSession session, Connecto
locationHandle,
tableStorageFormat,
partitionStorageFormat,
- getCompressionCodec(session),
+ actualStorageFormat,
+ getHiveCompressionCodec(session, false, actualStorageFormat),
partitionedBy,
bucketProperty,
preferredOrderingColumns,
@@ -1512,7 +1506,10 @@ public HiveInsertTableHandle beginInsert(ConnectorSession session, ConnectorTabl
HiveStorageFormat tableStorageFormat = extractHiveStorageFormat(table.get());
LocationHandle locationHandle;
boolean isTemporaryTable = table.get().getTableType().equals(TEMPORARY_TABLE);
- boolean tempPathRequired = isTempPathRequired(session, table.map(Table::getStorage).flatMap(Storage::getBucketProperty));
+ boolean tempPathRequired = isTempPathRequired(
+ session,
+ table.map(Table::getStorage).flatMap(Storage::getBucketProperty),
+ decodePreferredOrderingColumnsFromStorage(table.get().getStorage()));
if (isTemporaryTable) {
locationHandle = locationService.forTemporaryTable(metastore, session, table.get(), tempPathRequired);
}
@@ -1520,6 +1517,8 @@ public HiveInsertTableHandle beginInsert(ConnectorSession session, ConnectorTabl
locationHandle = locationService.forExistingTable(metastore, session, table.get(), tempPathRequired);
}
+ HiveStorageFormat partitionStorageFormat = isRespectTableFormat(session) ? tableStorageFormat : HiveSessionProperties.getHiveStorageFormat(session);
+ HiveStorageFormat actualStorageFormat = table.get().getPartitionColumns().isEmpty() ? tableStorageFormat : partitionStorageFormat;
HiveInsertTableHandle result = new HiveInsertTableHandle(
tableName.getSchemaName(),
tableName.getTableName(),
@@ -1530,8 +1529,9 @@ public HiveInsertTableHandle beginInsert(ConnectorSession session, ConnectorTabl
table.get().getStorage().getBucketProperty(),
decodePreferredOrderingColumnsFromStorage(table.get().getStorage()),
tableStorageFormat,
- isRespectTableFormat(session) ? tableStorageFormat : HiveSessionProperties.getHiveStorageFormat(session),
- isTemporaryTable ? getTemporaryTableCompressionCodec(session) : getCompressionCodec(session));
+ partitionStorageFormat,
+ actualStorageFormat,
+ getHiveCompressionCodec(session, isTemporaryTable, actualStorageFormat));
WriteInfo writeInfo = locationService.getQueryWriteInfo(locationHandle);
metastore.declareIntentionToWrite(
@@ -1545,6 +1545,17 @@ public HiveInsertTableHandle beginInsert(ConnectorSession session, ConnectorTabl
return result;
}
+ private HiveCompressionCodec getHiveCompressionCodec(ConnectorSession session, boolean isTemporaryTable, HiveStorageFormat storageFormat)
+ {
+ if (isTemporaryTable) {
+ return getTemporaryTableCompressionCodec(session);
+ }
+ if (storageFormat == ORC || storageFormat == DWRF) {
+ return getOrcCompressionCodec(session);
+ }
+ return getCompressionCodec(session);
+ }
+
@Override
public Optional finishInsert(ConnectorSession session, ConnectorInsertTableHandle insertHandle, Collection fragments, Collection computedStatistics)
{
@@ -1656,9 +1667,10 @@ else if (partitionUpdate.getUpdateMode() == NEW || partitionUpdate.getUpdateMode
.collect(Collectors.toList())));
}
- private static boolean isTempPathRequired(ConnectorSession session, Optional bucketProperty)
+ private static boolean isTempPathRequired(ConnectorSession session, Optional bucketProperty, List preferredOrderingColumns)
{
- return isSortedWriteToTempPathEnabled(session) && bucketProperty.map(property -> !property.getSortedBy().isEmpty()).orElse(false);
+ boolean hasSortedWrite = bucketProperty.map(property -> !property.getSortedBy().isEmpty()).orElse(false) || !preferredOrderingColumns.isEmpty();
+ return isSortedWriteToTempPathEnabled(session) && hasSortedWrite;
}
private List getTargetFileNames(List fileWriteInfos)
@@ -1955,7 +1967,7 @@ private String createTableLayoutString(
.add("buckets", bucketHandle.map(HiveBucketHandle::getReadBucketCount).orElse(null))
.add("bucketsToKeep", bucketFilter.map(HiveBucketFilter::getBucketsToKeep).orElse(null))
.add("filter", TRUE_CONSTANT.equals(remainingPredicate) ? null : rowExpressionService.formatRowExpression(session, remainingPredicate))
- .add("domains", domainPredicate.isAll() ? null : domainPredicate.toString(session))
+ .add("domains", domainPredicate.isAll() ? null : domainPredicate.toString(session.getSqlFunctionProperties()))
.toString();
}
@@ -2019,7 +2031,7 @@ private boolean isPushdownFilterEnabled(ConnectorSession session, ConnectorTable
boolean pushdownFilterEnabled = HiveSessionProperties.isPushdownFilterEnabled(session);
if (pushdownFilterEnabled) {
HiveStorageFormat hiveStorageFormat = getHiveStorageFormat(getTableMetadata(session, tableHandle).getProperties());
- if (hiveStorageFormat == HiveStorageFormat.ORC || hiveStorageFormat == HiveStorageFormat.DWRF) {
+ if (hiveStorageFormat == ORC || hiveStorageFormat == DWRF) {
return true;
}
}
@@ -2226,7 +2238,7 @@ private static Domain buildColumnDomain(ColumnHandle column, List
checkArgument(!partitions.isEmpty(), "partitions cannot be empty");
boolean hasNull = false;
- List