+ * a) combining the two part execution with CachedDataFrame where first execution + * does the caching ("prepare" phase) along with the actual execution while subsequent + * executions only do the latter + *
+ * b) shortens the SQL string to display properly in the UI (CachedDataFrame already + * takes care of posting the SQL string rather than method name unlike Spark). + *
+ * This is invoked before initialization of SharedState for Spark releases + * where listener is attached independently of SharedState before latter is created + * while it is invoked after initialization of SharedState for newer Spark versions. + */ + def createAndAttachSQLListener(sparkContext: SparkContext): Unit + + /** + * Create a new global instance of [[SnappySharedState]]. + */ + def newSharedState(sparkContext: SparkContext): SnappySharedState + + /** + * Clear any global SQL listener. + */ + def clearSQLListener(): Unit + + /** + * Create a SQL string appropriate for a persisted VIEW plan and storage in catalog + * from a given [[LogicalPlan]] for the VIEW. + */ + def createViewSQL(session: SparkSession, plan: LogicalPlan, + originalText: Option[String]): String + + /** + * Create a [[LogicalPlan]] for CREATE VIEW. + */ + def createView(desc: CatalogTable, output: Seq[Attribute], child: LogicalPlan): LogicalPlan + + /** + * Create a [[LogicalPlan]] for CREATE FUNCTION. + */ + def newCreateFunctionCommand(schemaName: Option[String], functionName: String, + className: String, resources: Seq[FunctionResource], isTemp: Boolean, + ignoreIfExists: Boolean, replace: Boolean): LogicalPlan + + /** + * Create a [[LogicalPlan]] for DESCRIBE TABLE. + */ + def newDescribeTableCommand(table: TableIdentifier, partitionSpec: Map[String, String], + isExtended: Boolean, isFormatted: Boolean): RunnableCommand + + /** + * Create a [[LogicalPlan]] for CLEAR CACHE. + */ + def newClearCacheCommand(): LogicalPlan + + /** + * Create a [[LogicalPlan]] for CREATE TABLE ... LIKE + */ + def newCreateTableLikeCommand(targetIdent: TableIdentifier, sourceIdent: TableIdentifier, + location: Option[String], allowExisting: Boolean): RunnableCommand + + /** + * Lookup a relation in catalog. + */ + def lookupRelation(catalog: SessionCatalog, name: TableIdentifier, + alias: Option[String]): LogicalPlan + + /** + * Resolve Maven coordinates for a package, cache the jars and return the required CLASSPATH. + */ + def resolveMavenCoordinates(coordinates: String, remoteRepos: Option[String], + ivyPath: Option[String], exclusions: Seq[String]): String + + /** + * Create a copy of [[Attribute]] as [[AttributeReference]] with given arguments. + */ + def toAttributeReference(attr: Attribute)(name: String = attr.name, + dataType: DataType = attr.dataType, nullable: Boolean = attr.nullable, + metadata: Metadata = attr.metadata, exprId: ExprId = attr.exprId): AttributeReference + + /** + * Create a new instance of [[AttributeReference]] + */ + def newAttributeReference(name: String, dataType: DataType, nullable: Boolean, + metadata: Metadata, exprId: ExprId, qualifier: Seq[String], + isGenerated: Boolean = false): AttributeReference + + /** + * Create a new concrete instance of [[ErrorEstimateAttribute]]. + */ + def newErrorEstimateAttribute(name: String, dataType: DataType, + nullable: Boolean, metadata: Metadata, realExprId: ExprId, + exprId: ExprId = NamedExpression.newExprId, + qualifier: Seq[String] = Nil): ErrorEstimateAttribute + + /** + * Create a new concrete instance of [[ApproxColumnExtractor]]. + */ + def newApproxColumnExtractor(child: Expression, name: String, ordinal: Int, + dataType: DataType, nullable: Boolean, exprId: ExprId = NamedExpression.newExprId, + qualifier: Seq[String] = Nil): ApproxColumnExtractor + + /** + * Create a new concrete instance of [[TaggedAttribute]]. + */ + def newTaggedAttribute(tag: Tag, name: String, dataType: DataType, nullable: Boolean, + metadata: Metadata, exprId: ExprId = NamedExpression.newExprId, + qualifier: Seq[String] = Nil): TaggedAttribute + + /** + * Create a new concrete instance of [[TaggedAlias]]. + */ + def newTaggedAlias(tag: TransformableTag, child: Expression, name: String, + exprId: ExprId = NamedExpression.newExprId, qualifier: Seq[String] = Nil): TaggedAlias + + // scalastyle:off + + /** + * Create a new concrete instance of [[ClosedFormColumnExtractor]]. + */ + def newClosedFormColumnExtractor(child: Expression, name: String, confidence: Double, + confFactor: Double, aggType: ErrorAggregate.Type, error: Double, dataType: DataType, + behavior: HAC.Type, nullable: Boolean, exprId: ExprId = NamedExpression.newExprId, + qualifier: Seq[String] = Nil): ClosedFormColumnExtractor + + // scalastyle:on + + /** + * Create a copy of [[InsertIntoTable]] plan with a new child. + */ + def withNewChild(insert: InsertIntoTable, newChild: LogicalPlan): InsertIntoTable + + /** + * Create a new [[InsertIntoTable]] plan. + */ + def newInsertIntoTable(table: LogicalPlan, partition: Map[String, Option[String]], + child: LogicalPlan, overwrite: Boolean, ifNotExists: Boolean): InsertIntoTable + + /** + * Return true if overwrite is enabled in the insert plan else false. + */ + def getOverwriteOption(insert: InsertIntoTable): Boolean + + /** + * Create an expression for GROUPING SETS. + */ + def newGroupingSet(groupingSets: Seq[Seq[Expression]], groupByExprs: Seq[Expression], + child: LogicalPlan, aggregations: Seq[NamedExpression]): LogicalPlan + + /** + * Create a new unresolved relation (Table/View/Alias). + */ + def newUnresolvedRelation(tableIdentifier: TableIdentifier, alias: Option[String]): LogicalPlan + + /** + * Get alias if specified in UnresolvedRelation else None. + */ + def unresolvedRelationAlias(u: UnresolvedRelation): Option[String] + + /** + * Create an alias for a sub-query. + */ + def newSubqueryAlias(alias: String, child: LogicalPlan, + view: Option[TableIdentifier] = None): SubqueryAlias + + /** + * Get view, if defined, or else alias name of a SubqueryAlias. + */ + def getViewFromAlias(q: SubqueryAlias): Option[TableIdentifier] + + /** + * Create an alias with given parameters and optionally copying other fields from existing Alias. + */ + def newAlias(child: Expression, name: String, copyAlias: Option[NamedExpression], + exprId: ExprId = NamedExpression.newExprId, qualifier: Seq[String] = Nil): Alias + + /** + * Create a plan for column aliases in a table/sub-query/... + * Not supported by older Spark versions. + */ + def newUnresolvedColumnAliases(outputColumnNames: Seq[String], + child: LogicalPlan): LogicalPlan + + /** + * Create a [[SortOrder]]. + */ + def newSortOrder(child: Expression, direction: SortDirection, + nullOrdering: NullOrdering): SortOrder + + /** + * Create a new [[LogicalPlan]] for REPARTITION. + */ + def newRepartitionByExpression(partitionExpressions: Seq[Expression], + numPartitions: Int, child: LogicalPlan): RepartitionByExpression + + /** + * Create a new unresolved table value function. + */ + def newUnresolvedTableValuedFunction(functionName: String, functionArgs: Seq[Expression], + outputNames: Seq[String]): UnresolvedTableValuedFunction + + /** + * Create a new frame boundary. This is a FrameBoundary is older Spark versions + * while newer ones use an Expression instead. + */ + def newFrameBoundary(boundaryType: FrameBoundaryType.Type, + num: Option[Expression] = None): Any + + /** + * Create a new [[SpecifiedWindowFrame]] given the [[FrameType]] and start/end frame + * boundaries as returned by [[newFrameBoundary]]. + */ + def newSpecifiedWindowFrame(frameType: FrameType, + frameStart: Any, frameEnd: Any): SpecifiedWindowFrame + + /** + * Create a new wrapper [[LogicalPlan]] that encapsulates an arbitrary set of hints. + */ + def newLogicalPlanWithHints(child: LogicalPlan, + hints: Map[QueryHint.Type, HintName.Type]): LogicalPlan + + /** + * Create a new TABLESAMPLE operator. + */ + def newTableSample(lowerBound: Double, upperBound: Double, withReplacement: Boolean, + seed: Long, child: LogicalPlan): Sample + + /** + * Return true if the given LogicalPlan encapsulates a child plan with query hint(s). + */ + def isHintPlan(plan: LogicalPlan): Boolean + + /** + * If the given plan encapsulates query hints, then return the hint type and name pairs. + */ + def getHints(plan: LogicalPlan): Map[QueryHint.Type, HintName.Type] + + /** + * Return true if current plan has been explicitly marked for broadcast and false otherwise. + */ + def isBroadcastable(plan: LogicalPlan): Boolean + + /** + * Create a new OneRowRelation. + */ + def newOneRowRelation(): LogicalPlan + + /** + * Create a new [[LogicalPlan]] for GENERATE. + */ + def newGeneratePlan(generator: Generator, outer: Boolean, qualifier: Option[String], + generatorOutput: Seq[Attribute], child: LogicalPlan): LogicalPlan + + /** + * Write a DataFrame to a DataSource. + */ + def writeToDataSource(ds: DataSource, mode: SaveMode, data: Dataset[Row]): BaseRelation + + /** + * Create a new [[LogicalRelation]]. + */ + def newLogicalRelation(relation: BaseRelation, + expectedOutputAttributes: Option[Seq[AttributeReference]], + catalogTable: Option[CatalogTable], isStreaming: Boolean): LogicalRelation + + /** + * Create a DataFrame out of an RDD of InternalRows. + */ + def internalCreateDataFrame(session: SparkSession, catalystRows: RDD[InternalRow], + schema: StructType, isStreaming: Boolean = false): Dataset[Row] + + /** + * Create a new [[RowDataSourceScanExec]] with the given parameters. + */ + def newRowDataSourceScanExec(fullOutput: Seq[Attribute], requiredColumnsIndex: Seq[Int], + filters: Seq[Filter], handledFilters: Seq[Filter], rdd: RDD[InternalRow], + metadata: Map[String, String], relation: BaseRelation, + tableIdentifier: Option[TableIdentifier]): RowDataSourceScanExec + + /** + * Create a new [[CodegenSparkFallback]] with the given child. + */ + def newCodegenSparkFallback(child: SparkPlan, session: SnappySession): CodegenSparkFallback + + /** + * Create a new [[LogicalDStreamPlan]] with the given parameters. + */ + def newLogicalDStreamPlan(output: Seq[Attribute], stream: DStream[InternalRow], + streamingSnappy: SnappyStreamingContext): LogicalDStreamPlan + + /** + * Create a new CatalogDatabase given the parameters. Newer Spark releases require a URI + * for locationUri so the given string will be converted to URI for those Spark versions. + */ + def newCatalogDatabase(name: String, description: String, + locationUri: String, properties: Map[String, String]): CatalogDatabase + + /** Get the locationURI for CatalogDatabase in String format. */ + def catalogDatabaseLocationURI(database: CatalogDatabase): String + + // scalastyle:off + + /** + * Create a new CatalogTable given the parameters. The primary constructor + * of the class has seen major changes across Spark versions. + */ + def newCatalogTable(identifier: TableIdentifier, tableType: CatalogTableType, + storage: CatalogStorageFormat, schema: StructType, provider: Option[String], + partitionColumnNames: Seq[String], bucketSpec: Option[BucketSpec], + owner: String, createTime: Long, lastAccessTime: Long, properties: Map[String, String], + stats: Option[AnyRef], viewOriginalText: Option[String], viewText: Option[String], + comment: Option[String], unsupportedFeatures: Seq[String], + tracksPartitionsInCatalog: Boolean, schemaPreservesCase: Boolean, + ignoredProperties: Map[String, String]): CatalogTable + + // scalastyle:on + + /** Get the viewOriginalText of CataLogTable or None if not present. */ + def catalogTableViewOriginalText(catalogTable: CatalogTable): Option[String] + + /** Get the ignoredProperties map of CataLogTable or empty map if not present. */ + def catalogTableIgnoredProperties(catalogTable: CatalogTable): Map[String, String] + + /** Return a new CatalogTable with updated viewOriginalText if possible. */ + def newCatalogTableWithViewOriginalText(catalogTable: CatalogTable, + viewOriginalText: Option[String]): CatalogTable + + /** + * Create a new CatalogStorageFormat given the parameters. + */ + def newCatalogStorageFormat(locationUri: Option[String], inputFormat: Option[String], + outputFormat: Option[String], serde: Option[String], compressed: Boolean, + properties: Map[String, String]): CatalogStorageFormat + + /** Get the string representation of locationUri field of CatalogStorageFormat. */ + def catalogStorageFormatLocationUri(storageFormat: CatalogStorageFormat): Option[String] + + /** Serialize a CatalogTablePartition to InternalRow */ + def catalogTablePartitionToRow(partition: CatalogTablePartition, + partitionSchema: StructType, defaultTimeZoneId: String): InternalRow + + /** Query catalog to load dynamic partitions defined in given Spark table. */ + def loadDynamicPartitions(externalCatalog: ExternalCatalog, schema: String, + table: String, loadPath: String, partition: TablePartitionSpec, replace: Boolean, + numDP: Int, holdDDLTime: Boolean): Unit + + /** Alter table schema in the ExternalCatalog if possible else throw an exception */ + def alterTableSchema(externalCatalog: ExternalCatalog, schemaName: String, + table: String, newSchema: StructType): Unit + + /** + * Alter table statistics in the ExternalCatalog if possible else throw an exception. + * The `stats` argument is an optional Statistics (for Spark < 2.2) or CatalogStatistics object. + */ + def alterTableStats(externalCatalog: ExternalCatalog, schema: String, table: String, + stats: Option[AnyRef]): Unit + + /** Alter function definition in the ExternalCatalog if possible else throw an exception */ + def alterFunction(externalCatalog: ExternalCatalog, schema: String, + function: CatalogFunction): Unit + + /** Convert a ColumnStat (or CatalogColumnStat for Spark >= 2.4) to a map. */ + def columnStatToMap(stat: Any, colName: String, dataType: DataType): Map[String, String] + + /** Convert a map created by [[columnStatToMap]] to ColumnStat or CatalogColumnStat. */ + def columnStatFromMap(table: String, field: StructField, + map: Map[String, String]): Option[AnyRef] + + /** + * Create a Statistics/CatalogStatistics object from given arguments. The `colStats` argument + * is a map of string to ColumnStat(Spark < 2.4)/CatalogColumnStat + */ + def toCatalogStatistics(sizeInBytes: BigInt, rowCount: Option[BigInt], + colStats: Map[String, AnyRef]): AnyRef + + /** + * Create a new instance of SnappyHiveExternalCatalog. The method overrides in + * ExternalCatalog have changed across Spark versions. + */ + def newEmbeddedHiveCatalog(conf: SparkConf, hadoopConf: Configuration, + createTime: Long): SnappyHiveExternalCatalog + + /** + * Create a new instance of SmartConnectorExternalCatalog. The method overrides in + * ExternalCatalog have changed across Spark versions. + */ + def newSmartConnectorExternalCatalog(session: SparkSession): SnappyExternalCatalog + + /** Lookup the data source for a given provider. */ + def lookupDataSource(provider: String, conf: => SQLConf): Class[_] + + /** + * Create a new shuffle exchange plan. + */ + def newShuffleExchange(newPartitioning: Partitioning, child: SparkPlan): Exchange + + /** + * Return true if the given plan is a ShuffleExchange. + */ + def isShuffleExchange(plan: SparkPlan): Boolean + + /** + * Get the classOf ShuffleExchange operator. + */ + def classOfShuffleExchange(): Class[_] + + /** + * Get the [[Statistics]] for a given [[LogicalPlan]]. + */ + def getStatistics(plan: LogicalPlan): Statistics + + /** + * Return true if the given [[AggregateFunction]] support partial result aggregation. + */ + def supportsPartial(aggregate: AggregateFunction): Boolean + + /** + * Create a physical [[SparkPlan]] for an [[AggregateFunction]] that does not support + * partial result aggregation ([[supportsPartial]] is false). + */ + def planAggregateWithoutPartial(groupingExpressions: Seq[NamedExpression], + aggregateExpressions: Seq[AggregateExpression], + resultExpressions: Seq[NamedExpression], planChild: () => SparkPlan): Seq[SparkPlan] + + /** + * Compile given generated code assuming it results in an implemenation of [[GeneratedClass]]. + */ + def compile(code: CodeAndComment): GeneratedClass + + /** + * Create a new [[JSONOptions]] object given the parameters. + */ + def newJSONOptions(parameters: Map[String, String], + session: Option[SparkSession]): JSONOptions + + /** + * Create a new instance of [[SnappySessionState]] appropriate for the current Spark version. + */ + def newSnappySessionState(snappySession: SnappySession): SnappySessionState + + /** + * Return the Spark plan for check pre-conditions before a write operation. + */ + def newPreWriteCheck(sessionState: SnappySessionState): LogicalPlan => Unit + + /** + * Return list of HiveConditionalStrategies to be applied when hive external catalog is enabled. + */ + def hiveConditionalStrategies(sessionState: SnappySessionState): Seq[Strategy] + + /** + * Create a new SnappyData extended CacheManager to clear cached plans on cached data changes. + */ + def newCacheManager(): CacheManager + + /** + * Create a new SQLConf entry with registration actions for the given key. + */ + def buildConf(key: String): ConfigBuilder + + /** + * Get the global list of cached RDDs (as list of [[RDDStorageInfo]]). + */ + def getCachedRDDInfos(context: SparkContext): Seq[RDDStorageInfo] + + /** + * Get the return data type of given java method. + * A result of NullType indicates a possible StructType, so caller should check for the same. + */ + def getReturnDataType(method: Method): DataType + + /** + * Create a new ExprCode with given arguments. + */ + def newExprCode(code: String, isNull: String, value: String, dt: DataType): ExprCode + + /** + * Make a copy of ExprCode with given new arguments. + */ + def copyExprCode(ev: ExprCode, code: String = null, isNull: String = null, + value: String = null, dt: DataType = null): ExprCode + + /** + * Reset the code field of [[ExprCode]] to empty code block. + */ + def resetCode(ev: ExprCode): Unit + + /** + * Get the string for isNull field of [[ExprCode]]. + */ + def exprCodeIsNull(ev: ExprCode): String + + /** + * Set the isNull field of [[ExprCode]]. + */ + def setExprCodeIsNull(ev: ExprCode, isNull: String): Unit + + /** + * Get the string for value field of [[ExprCode]]. + */ + def exprCodeValue(ev: ExprCode): String + + /** + * Get the string for java type for given [[DataType]]. + */ + def javaType(dt: DataType, ctx: CodegenContext): String + + /** + * Get the java type of boxed type for given type. + */ + def boxedType(javaType: String, ctx: CodegenContext): String + + /** + * Get the string form of default value for given [[DataType]]. + */ + def defaultValue(dt: DataType, ctx: CodegenContext): String + + /** + * Returns true if the Java type has a special accessor and setter in [[InternalRow]]. + */ + def isPrimitiveType(javaType: String, ctx: CodegenContext): Boolean + + /** + * Returns the name used in accessor and setter for a Java primitive type. + */ + def primitiveTypeName(javaType: String, ctx: CodegenContext): String + + /** + * Returns the specialized code to access a value from `inputRow` at `ordinal`. + */ + def getValue(input: String, dataType: DataType, ordinal: String, ctx: CodegenContext): String + + /** + * List of any optional plans to be executed in the QueryExecution.preparations phase. + */ + def optionalQueryPreparations(session: SparkSession): Seq[Rule[SparkPlan]] + + /** + * Create a new instance of [[Pivot]] plan. + */ + def newPivot(groupByExprs: Seq[NamedExpression], pivotColumn: Expression, + pivotValues: Seq[Expression], aggregates: Seq[Expression], child: LogicalPlan): Pivot + + /** + * Create a copy of [[Pivot]] plan with a new set of groupBy expressions. + */ + def copyPivot(pivot: Pivot, groupByExprs: Seq[NamedExpression]): Pivot + + /** + * Create a new instance of [[Intersect]] plan. + */ + def newIntersect(left: LogicalPlan, right: LogicalPlan, isAll: Boolean): Intersect + + /** + * Create a new instance of [[Except]] plan. + */ + def newExcept(left: LogicalPlan, right: LogicalPlan, isAll: Boolean): Except + + /** + * Create a plan for explain command. + */ + def newExplainCommand(logicalPlan: LogicalPlan, extended: Boolean, + codegen: Boolean, cost: Boolean): LogicalPlan + + /** + * Get the internal cached RDD for an in-memory relation. + */ + def cachedColumnBuffers(relation: InMemoryRelation): RDD[_] + + /** + * Add SnappyData custom string promotion rules to deal with ParamLiterals. + */ + def addStringPromotionRules(rules: Seq[Rule[LogicalPlan]], + analyzer: SnappyAnalyzer, conf: SQLConf): Seq[Rule[LogicalPlan]] + + /** + * Create table definition in the catalog. + */ + def createTable(catalog: SessionCatalog, tableDefinition: CatalogTable, + ignoreIfExists: Boolean, validateLocation: Boolean): Unit = { + catalog.createTable(tableDefinition, ignoreIfExists) + } + + /** + * Transform down a [[LogicalPlan]] during analysis phase. + * This translates to resolveOperatorsDown in Spark 2.4.x + * while it uses transformDown in earlier versions. + */ + def logicalPlanResolveDown(plan: LogicalPlan)( + rule: PartialFunction[LogicalPlan, LogicalPlan]): LogicalPlan = plan.transformDown(rule) + + /** + * Transform up a [[LogicalPlan]] during analysis phase. + * This translates to resolveOperatorsUp in Spark 2.4.x + * while it uses transformUp in earlier versions. + */ + def logicalPlanResolveUp(plan: LogicalPlan)( + rule: PartialFunction[LogicalPlan, LogicalPlan]): LogicalPlan = plan.transformUp(rule) + + /** + * Transform all expressions in a [[LogicalPlan]] during analysis phase. + * This translates to resolveExpressions in Spark 2.4.x + * while it uses transformAllExpressions in earlier versions. + */ + def logicalPlanResolveExpressions(plan: LogicalPlan)( + rule: PartialFunction[Expression, Expression]): LogicalPlan = { + plan.transformAllExpressions(rule) + } +} + +/** + * Enumeration for frame boundary type to provie a common way of expressing it due to + * major change in frame boundary handling across Spark versions. + */ +object FrameBoundaryType extends Enumeration { + type Type = Value + + val CurrentRow, UnboundedPreceding, UnboundedFollowing, ValuePreceding, ValueFollowing = Value +} diff --git a/core/src/main/scala/org/apache/spark/sql/SparkSupport.scala b/core/src/main/scala/org/apache/spark/sql/SparkSupport.scala new file mode 100644 index 0000000000..958efc3588 --- /dev/null +++ b/core/src/main/scala/org/apache/spark/sql/SparkSupport.scala @@ -0,0 +1,155 @@ +/* + * Copyright (c) 2017-2020 TIBCO Software Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +package org.apache.spark.sql + +import scala.util.control.NonFatal + +import com.gemstone.gemfire.internal.GemFireVersion +import com.gemstone.gemfire.internal.cache.GemFireCacheImpl +import com.pivotal.gemfirexd.internal.GemFireXDVersion +import com.pivotal.gemfirexd.internal.shared.common.SharedUtils + +import org.apache.spark.util.Utils +import org.apache.spark.{Logging, SparkContext, SparkException} + +/** + * Helper trait for easy access to [[SparkInternals]] using the "internals" method. + */ +trait SparkSupport { + protected final def internals: SparkInternals = SparkSupport.internals +} + +/** + * Load appropriate Spark version support as per the current Spark version. + */ +object SparkSupport extends Logging { + + /** + * The default Spark version for which core will be built and must exactly match + * the version of the embedded SnappyData Spark since this will be used on executors. + */ + final val DEFAULT_VERSION = "2.4.5" + + private[this] val EXTENDED_VERSION_PATTERN = "([0-9]\\.[0-9]\\.[0-9])\\.[0-9]".r + + @volatile private[this] var internalImpl: SparkInternals = _ + + private val INTERNAL_PACKAGE = "org.apache.spark.sql.internal" + + lazy val isEnterpriseEdition: Boolean = { + GemFireCacheImpl.setGFXDSystem(true) + GemFireVersion.getInstance(classOf[GemFireXDVersion], SharedUtils.GFXD_VERSION_PROPERTIES) + GemFireVersion.isEnterpriseEdition + } + + private lazy val aqpOverridesClass: Option[Class[_]] = { + if (isEnterpriseEdition) { + try { + Some(Utils.classForName("org.apache.spark.sql.execution.SnappyContextAQPFunctions")) + } catch { + case NonFatal(e) => + // Let the user know if it failed to load AQP classes. + logWarning(s"Failed to load AQP classes in Enterprise edition: $e") + None + } + } else None + } + + private[sql] def newContextFunctions(session: SnappySession): SnappyContextFunctions = { + aqpOverridesClass match { + case None => new SnappyContextFunctions(session) + case Some(c) => c.getConstructor(classOf[SnappySession]).newInstance(session) + .asInstanceOf[SnappyContextFunctions] + } + } + + /** + * An instance of [[SnappyContextFunctions]] with null session meaning any of the methods + * that require a session instance will fail with an NPE. + */ + lazy val contextFunctionsStateless: SnappyContextFunctions = newContextFunctions(session = null) + + /** + * List all the supported Spark versions below. All implementations are required to + * have a public constructor having current SparkContext as the one argument. + */ + private val implementations: Map[String, String] = Map( + "2.4.5" -> s"$INTERNAL_PACKAGE.Spark24Internals", + "2.4.4" -> s"$INTERNAL_PACKAGE.Spark24Internals", + "2.4.3" -> s"$INTERNAL_PACKAGE.Spark24Internals", + "2.4.2" -> s"$INTERNAL_PACKAGE.Spark24Internals", + "2.4.1" -> s"$INTERNAL_PACKAGE.Spark24Internals", + "2.4.0" -> s"$INTERNAL_PACKAGE.Spark24Internals", + "2.3.4" -> s"$INTERNAL_PACKAGE.Spark23Internals", + "2.3.3" -> s"$INTERNAL_PACKAGE.Spark23Internals", + "2.3.2" -> s"$INTERNAL_PACKAGE.Spark23Internals", + "2.3.1" -> s"$INTERNAL_PACKAGE.Spark23Internals", + "2.3.0" -> s"$INTERNAL_PACKAGE.Spark23Internals", + "2.1.3" -> s"$INTERNAL_PACKAGE.Spark21Internals", + "2.1.2" -> s"$INTERNAL_PACKAGE.Spark21Internals", + "2.1.1" -> s"$INTERNAL_PACKAGE.Spark21Internals" + ) + + /** + * Get the appropriate [[SparkInternals]] for current SparkContext version. + */ + def internals: SparkInternals = { + val impl = internalImpl + if (impl ne null) impl + else synchronized { + val impl = internalImpl + if (impl ne null) impl + else { + val sparkVersion = org.apache.spark.SPARK_VERSION match { + case EXTENDED_VERSION_PATTERN(v) => v + case v => v + } + val implClassName = implementations.get(sparkVersion) match { + case Some(v) => v + case None => throw new SparkException(s"Unsupported Spark version $sparkVersion") + } + val implClass: Class[_] = Utils.classForName(implClassName) + internalImpl = implClass.getConstructor(classOf[String]) + .newInstance(sparkVersion).asInstanceOf[SparkInternals] + internalImpl + } + } + } + + def internals(context: SparkContext): SparkInternals = { + val impl = internals + val version = context.version match { + case EXTENDED_VERSION_PATTERN(v) => v + case v => v + } + if (impl.version != version) { + throw new IllegalStateException(s"SparkVersion mismatch: " + + s"runtime version = ${context.version}. " + + s"Compile version = ${impl.version}") + } + impl + } + + private[sql] def clear(): Unit = synchronized { + val impl = internalImpl + if (impl ne null) { + impl.clearSQLListener() + internalImpl = null + } + } +} diff --git a/core/src/main/scala/org/apache/spark/sql/catalyst/expressions/DynamicFoldableExpression.scala b/core/src/main/scala/org/apache/spark/sql/catalyst/expressions/DynamicFoldableExpression.scala index bc64f06410..b15e6d4325 100644 --- a/core/src/main/scala/org/apache/spark/sql/catalyst/expressions/DynamicFoldableExpression.scala +++ b/core/src/main/scala/org/apache/spark/sql/catalyst/expressions/DynamicFoldableExpression.scala @@ -68,7 +68,7 @@ case class DynamicFoldableExpression(var expr: Expression) extends UnaryExpressi override def toString: String = { def removeCast(expr: Expression): Expression = expr match { - case Cast(child, _) => removeCast(child) + case c: Cast => removeCast(c.child) case _ => expr } diff --git a/core/src/main/scala/org/apache/spark/sql/catalyst/expressions/DynamicInSet.scala b/core/src/main/scala/org/apache/spark/sql/catalyst/expressions/DynamicInSet.scala index 29aa15f18a..e52a46e613 100644 --- a/core/src/main/scala/org/apache/spark/sql/catalyst/expressions/DynamicInSet.scala +++ b/core/src/main/scala/org/apache/spark/sql/catalyst/expressions/DynamicInSet.scala @@ -17,6 +17,7 @@ package org.apache.spark.sql.catalyst.expressions +import org.apache.spark.sql.SparkSupport import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode} /** @@ -24,7 +25,7 @@ import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCo * change dynamically in executions. */ case class DynamicInSet(child: Expression, hset: IndexedSeq[Expression]) - extends UnaryExpression with Predicate { + extends UnaryExpression with Predicate with SparkSupport { require((hset ne null) && hset.nonEmpty, "hset cannot be null or empty") // all expressions must be constant types @@ -66,12 +67,10 @@ case class DynamicInSet(child: Expression, hset: IndexedSeq[Expression]) val exprClass = classOf[Expression].getName val elements = new Array[AnyRef](hset.length) val childGen = child.genCode(ctx) - val hsetTerm = ctx.freshName("hset") val elementsTerm = ctx.freshName("elements") val idxTerm = ctx.freshName("idx") val idx = ctx.references.length ctx.references += elements - val hasNullTerm = ctx.freshName("hasNull") for (i <- hset.indices) { val e = hset(i) @@ -82,34 +81,36 @@ case class DynamicInSet(child: Expression, hset: IndexedSeq[Expression]) elements(i) = v } - ctx.addMutableState("boolean", hasNullTerm, "") - ctx.addMutableState(setName, hsetTerm, + val hasNullTerm = internals.addClassField(ctx, "boolean", "hasNull") + val hsetTerm = internals.addClassField(ctx, setName, "hset", hsetVar => s""" |Object[] $elementsTerm = (Object[])references[$idx]; - |$hsetTerm = new $setName($elementsTerm.length, 0.7f); + |$hsetVar = new $setName($elementsTerm.length, 0.7f); |for (int $idxTerm = 0; $idxTerm < $elementsTerm.length; $idxTerm++) { | Object e = $elementsTerm[$idxTerm]; | if (e instanceof $exprClass) e = (($exprClass)e).eval(null); | if (e != null) { - | $hsetTerm.put(e, e); + | $hsetVar.put(e, e); | } else if (!$hasNullTerm) { | $hasNullTerm = true; | } |} """.stripMargin) - ev.copy(code = + val evIsNull = internals.exprCodeIsNull(ev) + val evValue = internals.exprCodeValue(ev) + internals.copyExprCode(ev, code = s""" - ${childGen.code} - boolean ${ev.isNull} = ${childGen.isNull}; - boolean ${ev.value} = false; - if (!${ev.isNull}) { - ${ev.value} = $hsetTerm.containsKey(${childGen.value}); - if (!${ev.value} && $hasNullTerm) { - ${ev.isNull} = true; - } - } - """) + ${childGen.code.toString} + boolean $evIsNull = ${internals.exprCodeIsNull(childGen)}; + boolean $evValue = false; + if (!$evIsNull) { + $evValue = $hsetTerm.containsKey(${internals.exprCodeValue(childGen)}); + if (!$evValue && $hasNullTerm) { + $evIsNull = true; + } + } + """) } override def sql: String = { diff --git a/core/src/main/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpression.scala b/core/src/main/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpression.scala index d0674988e6..72daa24d20 100644 --- a/core/src/main/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpression.scala +++ b/core/src/main/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpression.scala @@ -17,6 +17,7 @@ package org.apache.spark.sql.catalyst.expressions +import org.apache.spark.sql.SparkSupport import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis.TypeCheckResult import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode} @@ -24,7 +25,7 @@ import org.apache.spark.sql.types.{AbstractDataType, CalendarIntervalType, DataT import org.apache.spark.unsafe.types.CalendarInterval case class IntervalExpression(children: Seq[Expression], units: Seq[Long]) - extends Expression with ImplicitCastInputTypes { + extends Expression with ImplicitCastInputTypes with SparkSupport { override def inputTypes: Seq[AbstractDataType] = if (children.length == 1) LongType :: Nil else Seq.fill(children.length)(LongType) @@ -53,7 +54,7 @@ case class IntervalExpression(children: Seq[Expression], units: Seq[Long]) override def foldable: Boolean = if (children.length == 1) children.head.foldable else children.forall(_.foldable) - override def deterministic: Boolean = + override lazy val deterministic: Boolean = if (children.length == 1) children.head.deterministic else children.forall(_.deterministic) override def nullable: Boolean = @@ -92,20 +93,22 @@ case class IntervalExpression(children: Seq[Expression], units: Seq[Long]) val micros = ctx.freshName("micros") val intervalClass = classOf[CalendarInterval].getName val nullable = this.nullable + val evIsNull = internals.exprCodeIsNull(ev) + val evValue = internals.exprCodeValue(ev) if (children.length == 1) { val childGen = children.head.genCode(ctx) - val childIsNull = if (nullable) childGen.isNull else "false" + val childIsNull = if (nullable) internals.exprCodeIsNull(childGen) else "false" val code = s""" - |${childGen.code} - |$intervalClass ${ev.value}; - |${doGenCodeSingle(childGen.value, childIsNull, ev.value, + |${childGen.code.toString} + |$intervalClass $evValue; + |${doGenCodeSingle(internals.exprCodeValue(childGen), childIsNull, evValue, units.head.toString, months, micros, intervalClass)} """.stripMargin if (childIsNull == "false") { - ev.copy(code = code, isNull = "false") + internals.copyExprCode(ev, code = code, isNull = "false") } else { - ev.copy(code = code + s"boolean ${ev.isNull} = ${ev.value} == null;\n") + internals.copyExprCode(ev, code = code + s"boolean $evIsNull = $evValue == null;\n") } } else { val index = ctx.freshName("i") @@ -117,31 +120,33 @@ case class IntervalExpression(children: Seq[Expression], units: Seq[Long]) val size = childGens.length val initArr = childGens.indices.map { i => s""" - |$childValueArr[$i] = ${childGens(i).value}; - |${if (nullable) s"$childIsNullArr[$i] = ${childGens(i).isNull};" else ""} + |$childValueArr[$i] = ${internals.exprCodeValue(childGens(i))}; + |${if (nullable) s"$childIsNullArr[$i] = ${internals.exprCodeIsNull(childGens(i))};" + else ""} """.stripMargin }.mkString("") val childIsNull = if (nullable) s"$childIsNullArr[$index]" else "false" val code = s""" - |${childGens.map(_.code).mkString("\n")} + |${childGens.map(_.code.toString).mkString("\n")} |long[] $childValueArr = new long[$size]; |${if (nullable) s"boolean[] $childIsNullArr = new boolean[$size];" else ""} - |$intervalClass ${ev.value} = null; + |$intervalClass $evValue = null; |$initArr |for (int $index = 0; $index < $size; $index++) { | $intervalClass $result; | ${doGenCodeSingle(s"$childValueArr[$index]", childIsNull, result, s"$unitsArr[$index]", months, micros, intervalClass)} | if ($result == null) { - | ${ev.value} = null; + | $evValue = null; | break; | } - | ${ev.value} = ${ev.value} != null ? ${ev.value}.add($result) : $result; + | $evValue = $evValue != null ? $evValue.add($result) : $result; |} """.stripMargin - if (nullable) ev.copy(code = code + s"boolean ${ev.isNull} = ${ev.value} == null;\n") - else ev.copy(code = code, isNull = "false") + if (nullable) { + internals.copyExprCode(ev, code = code + s"boolean $evIsNull = $evValue == null;\n") + } else internals.copyExprCode(ev, code = code, isNull = "false") } } diff --git a/core/src/main/scala/org/apache/spark/sql/catalyst/expressions/ParamLiteral.scala b/core/src/main/scala/org/apache/spark/sql/catalyst/expressions/ParamLiteral.scala index ef3c01429c..876034508f 100644 --- a/core/src/main/scala/org/apache/spark/sql/catalyst/expressions/ParamLiteral.scala +++ b/core/src/main/scala/org/apache/spark/sql/catalyst/expressions/ParamLiteral.scala @@ -30,6 +30,7 @@ import org.json4s.JsonAST.JField import org.apache.spark.memory.{MemoryMode, TaskMemoryManager} import org.apache.spark.serializer.StructTypeSerializer +import org.apache.spark.sql.SparkSupport import org.apache.spark.sql.catalyst.CatalystTypeConverters._ import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode} @@ -41,7 +42,7 @@ case class TermValues(literalValueRef: String, isNull: String, valueTerm: String // A marker interface to extend usage of Literal case matching. // A literal that can change across multiple query execution. -trait DynamicReplacableConstant extends Expression { +trait DynamicReplacableConstant extends Expression with SparkSupport { @transient private lazy val termMap = java.util.Collections.synchronizedMap(new util.HashMap[CodegenContext, TermValues]()) @@ -58,7 +59,7 @@ trait DynamicReplacableConstant extends Expression { value } - override final def deterministic: Boolean = true + override final lazy val deterministic: Boolean = true private def checkValueType(value: Any, expectedClass: Class[_]): Unit = { val valueClass = if (value != null) value.getClass else null @@ -86,10 +87,10 @@ trait DynamicReplacableConstant extends Expression { // temporary variable for storing value() result for cases where it can be // potentially expensive (e.g. for DynamicFoldableExpression) val valueResult = ctx.freshName("valueResult") - val isNullLocal = ev.isNull - val valueLocal = ev.value + val isNullLocal = internals.exprCodeIsNull(ev) + val valueLocal = internals.exprCodeValue(ev) val dataType = Utils.getSQLDataType(this.dataType) - val javaType = ctx.javaType(dataType) + val javaType = internals.javaType(dataType, ctx) // get values from map val isNull = termValues.isNull val valueTerm = termValues.valueTerm @@ -102,10 +103,11 @@ trait DynamicReplacableConstant extends Expression { if (!addMutableState) { // use the already added fields - return ev.copy(initCode, isNullLocal, valueLocal) + return internals.copyExprCode(ev, initCode, isNullLocal, valueLocal, dataType) } val valueRef = literalValueRef - val box = ctx.boxedType(javaType) + val box = internals.boxedType(javaType, ctx) + val defValue = internals.defaultValue(dataType, ctx) val unbox = dataType match { case BooleanType => @@ -137,11 +139,11 @@ trait DynamicReplacableConstant extends Expression { val memoryManagerClass = classOf[TaskMemoryManager].getName val memoryModeClass = classOf[MemoryMode].getName val consumerClass = classOf[DirectStringConsumer].getName - ctx.addMutableState(javaType, valueTerm, + internals.addClassField(ctx, javaType, valueTerm, _ => s""" |Object $valueResult = $valueRef.value(); |if (($isNull = ($valueResult == null))) { - | $valueTerm = ${ctx.defaultValue(dataType)}; + | $valueTerm = $defValue; |} else { | $valueTerm = ($box)$valueResult; | if (com.gemstone.gemfire.internal.cache.GemFireCacheImpl.hasNewOffHeap() && @@ -154,21 +156,21 @@ trait DynamicReplacableConstant extends Expression { | } | } |} - """.stripMargin) + """.stripMargin, forceInline = true, useFreshName = false) // indicate that code for valueTerm has already been generated null.asInstanceOf[String] case _ => "" } - ctx.addMutableState("boolean", isNull, "") + internals.addClassField(ctx, "boolean", isNull, forceInline = true, useFreshName = false) if (unbox ne null) { - ctx.addMutableState(javaType, valueTerm, + internals.addClassField(ctx, javaType, valueTerm, _ => s""" |Object $valueResult = $valueRef.value(); |$isNull = $valueResult == null; - |$valueTerm = $isNull ? ${ctx.defaultValue(dataType)} : (($box)$valueResult)$unbox; - """.stripMargin) + |$valueTerm = $isNull ? $defValue : (($box)$valueResult)$unbox; + """.stripMargin, forceInline = true, useFreshName = false) } - ev.copy(initCode, isNullLocal, valueLocal) + internals.copyExprCode(ev, initCode, isNullLocal, valueLocal, dataType) } } @@ -206,6 +208,8 @@ trait TokenizedLiteral extends LeafExpression with DynamicReplacableConstant { final class TokenLiteral(_value: Any, _dataType: DataType) extends Literal(_value, _dataType) with TokenizedLiteral with KryoSerializable { + _foldable = true + override def valueString: String = toString() override def jsonFields: List[JField] = super.jsonFields @@ -408,12 +412,12 @@ object TokenLiteral { def isConstant(expression: Expression): Boolean = expression match { case _: DynamicReplacableConstant | _: Literal => true - case Cast(child, dataType) => - val isConstant = child match { + case c: Cast => + val isConstant = c.child match { case _: DynamicReplacableConstant | _: Literal => true case _ => false } - isConstant & dataType.isInstanceOf[AtomicType] + isConstant && c.dataType.isInstanceOf[AtomicType] case _ => false } diff --git a/core/src/main/scala/org/apache/spark/sql/collection/MultiColumnOpenHashSet.scala b/core/src/main/scala/org/apache/spark/sql/collection/MultiColumnOpenHashSet.scala index a4cf0bdf1e..305e685d05 100644 --- a/core/src/main/scala/org/apache/spark/sql/collection/MultiColumnOpenHashSet.scala +++ b/core/src/main/scala/org/apache/spark/sql/collection/MultiColumnOpenHashSet.scala @@ -22,13 +22,13 @@ import scala.collection.mutable.ArrayBuffer import scala.collection.{IterableLike, mutable} import scala.util.hashing.MurmurHash3 -import org.apache.spark.sql.Row import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.catalyst.expressions.codegen.{CodeAndComment, CodeGenerator, GeneratedClass} +import org.apache.spark.sql.catalyst.expressions.codegen.{CodeAndComment, GeneratedClass} import org.apache.spark.sql.collection.MultiColumnOpenHashSet.ColumnHandler import org.apache.spark.sql.execution.BufferedRowIterator import org.apache.spark.sql.types._ +import org.apache.spark.sql.{Row, SparkSupport} import org.apache.spark.util.collection.BitSet /** @@ -1178,24 +1178,27 @@ object QCSSQLColumnHandler { def newSqlHandler(qcsPlan: (CodeAndComment, ArrayBuffer[Any], Array[DataType], Array[DataType]), hashColHandler: ColumnHandler): ColumnHandler = { - new QCSSQLColumnHandler( (CodeGenerator.compile(qcsPlan._1), qcsPlan._2, qcsPlan._3, qcsPlan._4), hashColHandler) + new QCSSQLColumnHandler((SparkSupport.internals.compile(qcsPlan._1), + qcsPlan._2, qcsPlan._3, qcsPlan._4), hashColHandler) } - val func: (Int, Iterator[InternalRow], GeneratedClass, ArrayBuffer[Any]) => Iterator[InternalRow] = { + val func: (Int, Iterator[InternalRow], + GeneratedClass, ArrayBuffer[Any]) => Iterator[InternalRow] = { (index, iter, clazz, bufferArr) => val buffer = clazz.generate(bufferArr.toArray).asInstanceOf[BufferedRowIterator] buffer.init(index, Array(iter)) new Iterator[InternalRow] { - override def hasNext(): Boolean = buffer.hasNext + override def hasNext(): Boolean = buffer.hasNext - override def next: InternalRow =buffer.next + override def next: InternalRow = buffer.next } } val iter = new Iterator[InternalRow]() { - def next: InternalRow = RowToInternalRow - def hasNext = RowToInternalRow.rowHolder.get() != null + def hasNext: Boolean = RowToInternalRow.rowHolder.get() != null + + def next(): InternalRow = RowToInternalRow } } @@ -1211,7 +1214,8 @@ object RowToInternalRow extends BaseGenericInternalRow { converters(ordinal)(row.getAs(ordinal)) } - override def copy(): InternalRow = throw new UnsupportedOperationException("Not implemented") + override def copy(): GenericInternalRow = + throw new UnsupportedOperationException("Not implemented") override def setNullAt(i: Int): Unit = {} diff --git a/core/src/main/scala/org/apache/spark/sql/collection/Utils.scala b/core/src/main/scala/org/apache/spark/sql/collection/Utils.scala index ff521870f3..4c9c18c4bb 100644 --- a/core/src/main/scala/org/apache/spark/sql/collection/Utils.scala +++ b/core/src/main/scala/org/apache/spark/sql/collection/Utils.scala @@ -34,8 +34,10 @@ import com.esotericsoftware.kryo.io.{Input, Output} import com.esotericsoftware.kryo.{Kryo, KryoSerializable} import com.gemstone.gemfire.internal.cache.PartitionedRegion import com.gemstone.gemfire.internal.shared.unsafe.UnsafeHolder +import com.pivotal.gemfirexd.Attribute.{PASSWORD_ATTR, USERNAME_ATTR} import com.pivotal.gemfirexd.internal.engine.Misc import com.pivotal.gemfirexd.internal.engine.jdbc.GemFireXDRuntimeException +import io.snappydata.Constant.{SPARK_STORE_PREFIX, STORE_PROPERTY_PREFIX} import io.snappydata.{Constant, ToolsCallback} import org.apache.commons.math3.distribution.NormalDistribution import org.eclipse.collections.impl.map.mutable.UnifiedMap @@ -49,7 +51,7 @@ import org.apache.spark.scheduler.local.LocalSchedulerBackend import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.expressions.codegen.CodegenContext import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, EqualNullSafe, EqualTo, Expression, GenericRow, SpecificInternalRow, TokenLiteral, UnsafeProjection} -import org.apache.spark.sql.catalyst.json.{JSONOptions, JacksonGenerator, JacksonUtils} +import org.apache.spark.sql.catalyst.json.{JacksonGenerator, JacksonUtils} import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan} import org.apache.spark.sql.catalyst.plans.physical.{HashPartitioning, Partitioning, PartitioningCollection} import org.apache.spark.sql.catalyst.util.DateTimeUtils @@ -63,12 +65,11 @@ import org.apache.spark.sql.sources.{CastLongTime, JdbcExtendedUtils} import org.apache.spark.sql.store.StoreUtils import org.apache.spark.sql.types._ import org.apache.spark.storage.{BlockId, BlockManager, BlockManagerId} -import org.apache.spark.ui.exec.ExecutorsListener import org.apache.spark.util.AccumulatorV2 import org.apache.spark.util.collection.BitSet import org.apache.spark.util.io.ChunkedByteBuffer -object Utils extends Logging { +object Utils extends Logging with SparkSupport { final val EMPTY_STRING_ARRAY = SharedUtils.EMPTY_STRING_ARRAY final val WEIGHTAGE_COLUMN_NAME = "snappy_sampler_weightage" @@ -249,10 +250,10 @@ object Utils extends Logging { private final val timeIntervalSpec = "([0-9]+)(ms|s|m|h)".r /** - * Parse the given time interval value as long milliseconds. - * - * @see timeIntervalSpec for the allowed string specification - */ + * Parse the given time interval value as long milliseconds. + * + * @see timeIntervalSpec for the allowed string specification + */ def parseTimeInterval(optV: Any, module: String): Long = { optV match { case tii: Int => tii.toLong @@ -432,7 +433,7 @@ object Utils extends Logging { * field is stored (and rendered) as VARCHAR by SnappyStore. * * @param size the size parameter of the VARCHAR() column type - * @param md optional Metadata object to be merged into the result + * @param md optional Metadata object to be merged into the result * @return the result Metadata object to use for StructField */ def varcharMetadata(size: Int, md: Metadata): Metadata = { @@ -470,7 +471,7 @@ object Utils extends Logging { * field is stored (and rendered) as CHAR by SnappyStore. * * @param size the size parameter of the CHAR() column type - * @param md optional Metadata object to be merged into the result + * @param md optional Metadata object to be merged into the result * @return the result Metadata object to use for StructField */ def charMetadata(size: Int, md: Metadata): Metadata = { @@ -512,9 +513,9 @@ object Utils extends Logging { } /** - * Get the result schema given an optional explicit schema and base table. - * In case both are specified, then check compatibility between the two. - */ + * Get the result schema given an optional explicit schema and base table. + * In case both are specified, then check compatibility between the two. + */ def getSchemaAndPlanFromBase(schemaOpt: Option[StructType], baseTableOpt: Option[String], catalog: SnappySessionCatalog, asSelect: Boolean, table: String, @@ -578,8 +579,8 @@ object Utils extends Logging { } /** - * Register given driver class with Spark's loader. - */ + * Register given driver class with Spark's loader. + */ def registerDriver(driver: String): Unit = { try { DriverRegistry.register(driver) @@ -590,22 +591,14 @@ object Utils extends Logging { } /** - * Register driver for given JDBC URL and return the driver class name. - */ + * Register driver for given JDBC URL and return the driver class name. + */ def registerDriverUrl(url: String): String = { val driver = getDriverClassName(url) registerDriver(driver) driver } - /** - * Wrap a DataFrame action to track all Spark jobs in the body so that - * we can connect them with an execution. - */ - def withNewExecutionId[T](df: DataFrame, body: => T): T = { - df.withNewExecutionId(body) - } - def immutableMap[A, B](m: mutable.Map[A, B]): Map[A, B] = new Map[A, B] { private[this] val map = m @@ -671,6 +664,16 @@ object Utils extends Logging { def getInternalSparkConf(sc: SparkContext): SparkConf = sc.conf + def getUserPassword(sparkConf: SparkConf): Option[(String, String)] = { + sparkConf.getOption(SPARK_STORE_PREFIX + USERNAME_ATTR) match { + case None => sparkConf.getOption(STORE_PROPERTY_PREFIX + USERNAME_ATTR) match { + case None => None + case Some(user) => Some(user -> sparkConf.get(STORE_PROPERTY_PREFIX + PASSWORD_ATTR, "")) + } + case Some(user) => Some(user -> sparkConf.get(SPARK_STORE_PREFIX + PASSWORD_ATTR, "")) + } + } + def newClusterSparkConf(): SparkConf = newClusterSparkConf(Misc.getMemStoreBooting.getBootProperties) @@ -767,7 +770,7 @@ object Utils extends Logging { writer: java.io.Writer): AnyRef = { val schema = StructType(Seq(StructField(columnName, dataType))) JacksonUtils.verifySchema(schema) - new JacksonGenerator(schema, writer, new JSONOptions(Map.empty[String, String])) + new JacksonGenerator(schema, writer, internals.newJSONOptions(Map.empty, None)) } def generateJson(gen: AnyRef, row: InternalRow, columnIndex: Int, @@ -788,27 +791,21 @@ object Utils extends Logging { def genTaskContextFunction(ctx: CodegenContext): String = { // use common taskContext variable so it is obtained only once for a plan - if (!ctx.addedFunctions.contains(TASKCONTEXT_FUNCTION)) { - val taskContextVar = ctx.freshName("taskContext") + if (!internals.isFunctionAddedToOuterClass(ctx, TASKCONTEXT_FUNCTION)) { val contextClass = classOf[TaskContext].getName - ctx.addMutableState(contextClass, taskContextVar, "") - ctx.addNewFunction(TASKCONTEXT_FUNCTION, + val taskContextVar = internals.addClassField(ctx, contextClass, "taskContext") + internals.addFunction(ctx, TASKCONTEXT_FUNCTION, s""" |private $contextClass $TASKCONTEXT_FUNCTION() { | final $contextClass context = $taskContextVar; | if (context != null) return context; | return ($taskContextVar = $contextClass.get()); |} - """.stripMargin) + """.stripMargin, inlineToOuterClass = true) } TASKCONTEXT_FUNCTION } - def executorsListener(sc: SparkContext): Option[ExecutorsListener] = sc.ui match { - case Some(ui) => Some(ui.executorsListener) - case _ => None - } - def getActiveSession: Option[SparkSession] = SparkSession.getActiveSession def sqlInternal(snappy: SnappySession, sqlText: String): CachedDataFrame = @@ -827,21 +824,21 @@ object Utils extends Logging { } def getPrunedPartition(partitionColumns: Seq[String], - filters: Array[Expression], schema: StructType, - numBuckets: Int, partitionColumnCount: Int): Int = { + filters: Array[Expression], schema: StructType, + numBuckets: Int, partitionColumnCount: Int): Int = { // this will yield partitioning column ordered Array of Expression (Literals/ParamLiterals). // RDDs needn't have to care for orderless hashing scheme at invocation point. val (pruningExpressions, fields) = partitionColumns.map { pc => filters.collectFirst { case EqualTo(a: Attribute, v) if TokenLiteral.isConstant(v) && - pc.equalsIgnoreCase(a.name) => (v, schema(a.name)) + pc.equalsIgnoreCase(a.name) => (v, schema(a.name)) case EqualTo(v, a: Attribute) if TokenLiteral.isConstant(v) && - pc.equalsIgnoreCase(a.name) => (v, schema(a.name)) + pc.equalsIgnoreCase(a.name) => (v, schema(a.name)) case EqualNullSafe(a: Attribute, v) if TokenLiteral.isConstant(v) && - pc.equalsIgnoreCase(a.name) => (v, schema(a.name)) + pc.equalsIgnoreCase(a.name) => (v, schema(a.name)) case EqualNullSafe(v, a: Attribute) if TokenLiteral.isConstant(v) && - pc.equalsIgnoreCase(a.name) => (v, schema(a.name)) + pc.equalsIgnoreCase(a.name) => (v, schema(a.name)) } }.filter(_.nonEmpty).map(_.get).unzip @@ -849,10 +846,10 @@ object Utils extends Logging { val mutableRow = new SpecificInternalRow(pcFields.map(_.dataType)) val bucketIdGeneration = UnsafeProjection.create( HashPartitioning(pcFields, numBuckets) - .partitionIdExpression :: Nil, pcFields) + .partitionIdExpression :: Nil, pcFields) if (pruningExpressions.nonEmpty && - // verify all the partition columns are provided as filters - pruningExpressions.length == partitionColumnCount) { + // verify all the partition columns are provided as filters + pruningExpressions.length == partitionColumnCount) { pruningExpressions.zipWithIndex.foreach { case (e, i) => mutableRow(i) = e.eval(null) } @@ -991,11 +988,13 @@ final class MultiBucketExecutorPartition(private[this] var _index: Int, private[this] var bucket = bucketSet.nextSetBit(0) override def hasNext: Boolean = bucket >= 0 + override def next(): Integer = { val b = Int.box(bucket) bucket = bucketSet.nextSetBit(bucket + 1) b } + override def remove(): Unit = throw new UnsupportedOperationException } @@ -1070,15 +1069,15 @@ private[spark] case class NarrowExecutorLocalSplitDep( } /** - * Stores information about the narrow dependencies used by a StoreRDD. - * - * @param narrowDep maps to the dependencies variable in the parent RDD: - * for each one to one dependency in dependencies, - * narrowDeps has a NarrowExecutorLocalSplitDep (describing - * the partition for that dependency) at the corresponding - * index. The size of narrowDeps should always be equal to - * the number of parents. - */ + * Stores information about the narrow dependencies used by a StoreRDD. + * + * @param narrowDep maps to the dependencies variable in the parent RDD: + * for each one to one dependency in dependencies, + * narrowDeps has a NarrowExecutorLocalSplitDep (describing + * the partition for that dependency) at the corresponding + * index. The size of narrowDeps should always be equal to + * the number of parents. + */ private[spark] class CoGroupExecutorLocalPartition( idx: Int, val blockId: BlockManagerId, val narrowDep: Option[NarrowExecutorLocalSplitDep]) diff --git a/core/src/main/scala/org/apache/spark/sql/dataFrames.scala b/core/src/main/scala/org/apache/spark/sql/dataFrames.scala index 942422e951..00c0fb90ab 100644 --- a/core/src/main/scala/org/apache/spark/sql/dataFrames.scala +++ b/core/src/main/scala/org/apache/spark/sql/dataFrames.scala @@ -16,6 +16,8 @@ */ package org.apache.spark.sql +import scala.collection.mutable + import io.snappydata.Constant import org.apache.spark.sql.SampleDataFrameContract.ErrorRow @@ -23,11 +25,8 @@ import org.apache.spark.sql.catalyst.encoders.{ExpressionEncoder, RowEncoder} import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.collection.MultiColumnOpenHashMap import org.apache.spark.sql.execution.QueryExecution - import org.apache.spark.sql.sources.StatCounter -import scala.collection.mutable - final class SampleDataFrame(@transient val snappySession: SnappySession, @transient override val logicalPlan: LogicalPlan) extends DataFrame(snappySession, logicalPlan, DataFrameUtil.encoder(snappySession, @@ -48,8 +47,7 @@ final class SampleDataFrame(@transient val snappySession: SnappySession, implementor.errorEstimateAverage(columnName, confidence, groupByColumns) private def createSampleDataFrameContract = - snappySession.snappyContextFunctions.createSampleDataFrameContract(snappySession, - this, logicalPlan) + snappySession.contextFunctions.createSampleDataFrameContract(this, logicalPlan) } final class DataFrameWithTime(_snappySession: SnappySession, @@ -59,13 +57,12 @@ final class DataFrameWithTime(_snappySession: SnappySession, case class AQPDataFrame(@transient snappySession: SnappySession, @transient qe: QueryExecution) extends DataFrame(snappySession, qe, - DataFrameUtil.encoder(snappySession, qe)) { + DataFrameUtil.encoder(snappySession, qe)) { def withError(error: Double, confidence: Double = Constant.DEFAULT_CONFIDENCE, behavior: String = Constant.DEFAULT_BEHAVIOR): DataFrame = - snappySession.snappyContextFunctions.withErrorDataFrame(this, error, - confidence, behavior) + snappySession.contextFunctions.withErrorDataFrame(this, error, confidence, behavior) } object DataFrameUtil { diff --git a/core/src/main/scala/org/apache/spark/sql/execution/CodegenSparkFallback.scala b/core/src/main/scala/org/apache/spark/sql/execution/CodegenSparkFallback.scala index 3892040e24..d8e573e5d8 100644 --- a/core/src/main/scala/org/apache/spark/sql/execution/CodegenSparkFallback.scala +++ b/core/src/main/scala/org/apache/spark/sql/execution/CodegenSparkFallback.scala @@ -24,7 +24,6 @@ import org.apache.spark.rdd.RDD import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.{Attribute, SortOrder} import org.apache.spark.sql.catalyst.plans.physical.Partitioning -import org.apache.spark.sql.execution.command.ExecutedCommandExec import org.apache.spark.sql.execution.metric.SQLMetric import org.apache.spark.sql.internal.CodeGenerationException import org.apache.spark.sql.{CachedDataFrame, SnappySession} @@ -33,7 +32,7 @@ import org.apache.spark.sql.{CachedDataFrame, SnappySession} * Catch exceptions in code generation of SnappyData plans and fallback * to Spark plans as last resort (including non-code generated paths). */ -case class CodegenSparkFallback(var child: SparkPlan, +abstract case class CodegenSparkFallback(var child: SparkPlan, @transient session: SnappySession) extends UnaryExecNode { override def output: Seq[Attribute] = child.output @@ -98,7 +97,7 @@ case class CodegenSparkFallback(var child: SparkPlan, } logInfo(s"SnappyData code generation failed due to $msg." + s" Falling back to Spark plans.") - session.sessionState.disableStoreOptimizations = true + session.snappySessionState.disableStoreOptimizations = true } try { val plan = exec().executedPlan.transform { @@ -114,7 +113,7 @@ case class CodegenSparkFallback(var child: SparkPlan, SnappySession.clearAllCache() throw CachedDataFrame.catalogStaleFailure(t, session) } finally { - session.sessionState.disableStoreOptimizations = false + session.snappySessionState.disableStoreOptimizations = false } case _ => throw t } @@ -129,7 +128,7 @@ case class CodegenSparkFallback(var child: SparkPlan, SnappySession.clearAllCache() // fail immediate for insert/update/delete, else retry entire query val action = plan.find { - case _: ExecutePlan | _: ExecutedCommandExec => true + case p if SnappySession.isCommandExec(p) => true case _ => false } if (action.isDefined) throw CachedDataFrame.catalogStaleFailure(t, session) @@ -161,10 +160,6 @@ case class CodegenSparkFallback(var child: SparkPlan, def execute(plan: SparkPlan): RDD[InternalRow] = executeWithFallback(_.execute(), plan) - override def generateTreeString(depth: Int, lastChildren: Seq[Boolean], - builder: StringBuilder, verbose: Boolean, prefix: String): StringBuilder = - child.generateTreeString(depth, lastChildren, builder, verbose, prefix) - // override def children: Seq[SparkPlan] = child.children // override private[sql] def metadata = child.metadata diff --git a/core/src/main/scala/org/apache/spark/sql/execution/DictionaryOptimizedMapAccessor.scala b/core/src/main/scala/org/apache/spark/sql/execution/DictionaryOptimizedMapAccessor.scala index 9c4768f761..13843d829a 100644 --- a/core/src/main/scala/org/apache/spark/sql/execution/DictionaryOptimizedMapAccessor.scala +++ b/core/src/main/scala/org/apache/spark/sql/execution/DictionaryOptimizedMapAccessor.scala @@ -18,11 +18,11 @@ package org.apache.spark.sql.execution import io.snappydata.collection.ObjectHashSet -import org.apache.spark.sql.SnappySession import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode} import org.apache.spark.sql.execution.columnar.encoding.ColumnEncoding import org.apache.spark.sql.types.StringType +import org.apache.spark.sql.{SnappySession, SparkSupport} /** * Makes use of dictionary indexes for strings if any. @@ -67,7 +67,7 @@ import org.apache.spark.sql.types.StringType * the effort (and could possibly even reduce overall performance in some * cases), hence this optimization is currently only for string type. */ -object DictionaryOptimizedMapAccessor { +object DictionaryOptimizedMapAccessor extends SparkSupport { def canHaveSingleKeyCase(keyExpressions: Seq[Expression]): Boolean = { keyExpressions.length == 1 && @@ -78,7 +78,7 @@ object DictionaryOptimizedMapAccessor { keyVars: => Seq[ExprCode], ctx: CodegenContext, session: SnappySession): Option[DictionaryCode] = { if (canHaveSingleKeyCase(keyExpressions)) { - session.getDictionaryCode(ctx, keyVars.head.value) + session.getDictionaryCode(ctx, internals.exprCodeValue(keyVars.head)) } else None } @@ -87,9 +87,11 @@ object DictionaryOptimizedMapAccessor { resultVar: String, valueInit: String, continueOnNull: Boolean, accessor: ObjectHashMapAccessor): String = { val key = ctx.freshName("dictionaryKey") - val keyIndex = keyDictVar.dictionaryIndex.value - val keyNull = keyVar.isNull != "false" - val keyEv = ExprCode("", if (keyNull) s"($key == null)" else "false", key) + val keyIndex = internals.exprCodeValue(keyDictVar.dictionaryIndex) + val keyNull = internals.exprCodeIsNull(keyVar) != "false" + val keyValue = internals.exprCodeValue(keyVar) + val keyEv = internals.copyExprCode(keyVar, code = "", + isNull = if (keyNull) s"($key == null)" else "false", key, StringType) val className = accessor.getClassName // for the case when there is no entry in map (hash join), insert a token @@ -115,7 +117,7 @@ object DictionaryOptimizedMapAccessor { val hashExprCode = if (keyNull) s"$key != null ? $key.hashCode() : -1" else s"$key.hashCode()" // if hash has already been calculated then use it - val hashExpr = accessor.session.getHashVar(ctx, keyVar.value :: Nil) match { + val hashExpr = accessor.session.getHashVar(ctx, keyValue :: Nil) match { case Some(h) => hash = h s"if ($h == 0) $h = $hashExprCode;" @@ -123,9 +125,9 @@ object DictionaryOptimizedMapAccessor { } // if keyVar code has not been consumed, then use dictionary - val keyAssign = if (keyVar.code.isEmpty) s"final UTF8String $key = ${keyVar.value};" + val keyAssign = if (keyVar.code.isEmpty) s"final UTF8String $key = $keyValue;" else { - val dictionaryVar = keyDictVar.dictionary.value + val dictionaryVar = internals.exprCodeValue(keyDictVar.dictionary) val stringAssignCode = ColumnEncoding.stringFromDictionaryCode( dictionaryVar, keyDictVar.bufferVar, keyIndex) s"final UTF8String $key = $stringAssignCode;" @@ -133,7 +135,7 @@ object DictionaryOptimizedMapAccessor { val indexCode = keyDictVar.evaluateIndexCode() val dictionaryIndexInit = if (indexCode.isEmpty) "" else { - s"int ${keyDictVar.dictionaryIndex.value} = -1;" + s"int $keyIndex = -1;" } s""" diff --git a/core/src/main/scala/org/apache/spark/sql/execution/EncoderScanExec.scala b/core/src/main/scala/org/apache/spark/sql/execution/EncoderScanExec.scala index 2079e3e287..50322b264f 100644 --- a/core/src/main/scala/org/apache/spark/sql/execution/EncoderScanExec.scala +++ b/core/src/main/scala/org/apache/spark/sql/execution/EncoderScanExec.scala @@ -17,15 +17,17 @@ package org.apache.spark.sql.execution import org.apache.spark.rdd.RDD -import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.analysis.MultiInstanceRelation import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode} import org.apache.spark.sql.catalyst.expressions.objects.StaticInvoke import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, BindReferences, Expression} +import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, Statistics} import org.apache.spark.sql.catalyst.util.DateTimeUtils import org.apache.spark.sql.collection.Utils -import org.apache.spark.sql.types.DateType +import org.apache.spark.sql.types.{DateType, ObjectType} +import org.apache.spark.sql.{SparkSession, SparkSupport} /** * Efficient SparkPlan with code generation support to consume an RDD @@ -33,7 +35,7 @@ import org.apache.spark.sql.types.DateType */ case class EncoderScanExec(rdd: RDD[Any], encoder: ExpressionEncoder[Any], isFlat: Boolean, output: Seq[Attribute]) - extends LeafExecNode with CodegenSupport { + extends LeafExecNode with CodegenSupport with SparkSupport { override protected def doExecute(): RDD[InternalRow] = { rdd.mapPartitionsInternal(_.map(encoder.toRow)) @@ -45,13 +47,12 @@ case class EncoderScanExec(rdd: RDD[Any], encoder: ExpressionEncoder[Any], override protected def doProduce(ctx: CodegenContext): String = { val dateTimeClass = DateTimeUtils.getClass.getName.replace("$", "") - val iterator = ctx.freshName("iterator") - ctx.addMutableState("scala.collection.Iterator", iterator, - s"$iterator = inputs[0];") + val iterator = internals.addClassField(ctx, "scala.collection.Iterator", "iterator", + v => s"$v = inputs[0];") val javaClass = encoder.clsTag.runtimeClass val javaTypeName = - if (javaClass.isPrimitive) ctx.boxedType(javaClass.getTypeName) + if (javaClass.isPrimitive) internals.boxedType(javaClass.getTypeName, ctx) else javaClass.getTypeName val objVar = ctx.freshName("object") @@ -71,53 +72,56 @@ case class EncoderScanExec(rdd: RDD[Any], encoder: ExpressionEncoder[Any], | throw new RuntimeException("top level null input object"); |}""") } - ctx.currentVars = Seq(ExprCode("", nullVar, objVar)) + ctx.currentVars = internals.newExprCode(code = "", nullVar, objVar, + ObjectType(javaClass)) :: Nil val declarations = new StringBuilder def optimizeDate(expr: Expression): ExprCode = expr match { - case s@StaticInvoke(_, _, "fromJavaDate", inputValue :: Nil, _) => + case s: StaticInvoke if s.functionName == "fromJavaDate" && s.arguments.length == 1 => // optimization to re-use previous date since it may remain // same for a while in many cases val prevJavaDate = ctx.freshName("prevJavaDate") val prevDate = ctx.freshName("prevDate") declarations.append(s"java.sql.Date $prevJavaDate = null;\n") declarations.append(s"int $prevDate = 0;\n") - val inputDate = inputValue.genCode(ctx) - val javaDate = inputDate.value + val inputDate = s.arguments.head.genCode(ctx) + val javaDate = internals.exprCodeValue(inputDate) val ev = s.genCode(ctx) - val code = if (ev.isNull == "false") { + val evIsNull = internals.exprCodeIsNull(ev) + val evValue = internals.exprCodeValue(ev) + val code = if (evIsNull == "false") { s""" - |${inputDate.code} - |int ${ev.value} = -1; + |${inputDate.code.toString} + |int $evValue = -1; |if ($prevJavaDate != null && | $prevJavaDate.getTime() == $javaDate.getTime()) { - | ${ev.value} = $prevDate; + | $evValue = $prevDate; |} else { | $prevJavaDate = $javaDate; | $prevDate = $dateTimeClass.fromJavaDate($javaDate); - | ${ev.value} = $prevDate; + | $evValue = $prevDate; |} """.stripMargin } else { s""" - |${inputDate.code} - |boolean ${ev.isNull}; - |int ${ev.value} = -1; - |if (${inputDate.isNull}) { - | ${ev.isNull} = true; + |${inputDate.code.toString} + |boolean $evIsNull; + |int $evValue = -1; + |if (${internals.exprCodeIsNull(inputDate)}) { + | $evIsNull = true; |} else if ($prevJavaDate != null && | $prevJavaDate.getTime() == $javaDate.getTime()) { - | ${ev.value} = $prevDate; - | ${ev.isNull} = false; + | $evValue = $prevDate; + | $evIsNull = false; |} else { | $prevJavaDate = $javaDate; | $prevDate = $dateTimeClass.fromJavaDate($javaDate); - | ${ev.value} = $prevDate; - | ${ev.isNull} = false; + | $evValue = $prevDate; + | $evIsNull = false; |} """.stripMargin } - ev.copy(code = code) + internals.copyExprCode(ev, code = code) case Alias(child, _) => optimizeDate(child) @@ -138,7 +142,7 @@ case class EncoderScanExec(rdd: RDD[Any], encoder: ExpressionEncoder[Any], // Hence the below code was erronous and after fixing null handing in above date field // it works for all cases. /* if (ctx.isPrimitiveType(dataType)) { - ev.copy(isNull = "false") + internals.copyExprCode(ev, isNull = "false") } else { ev } */ @@ -156,13 +160,23 @@ case class EncoderScanExec(rdd: RDD[Any], encoder: ExpressionEncoder[Any], } } -class EncoderPlan[T](rdd: RDD[T], val encoder: ExpressionEncoder[T], - val isFlat: Boolean, output: Seq[Attribute], session: SparkSession) - extends LogicalRDD(output, rdd.asInstanceOf[RDD[InternalRow]])(session) { +case class EncoderPlan[T](rdd: RDD[T], encoder: ExpressionEncoder[T], + isFlat: Boolean, output: Seq[Attribute])(session: SparkSession) + extends LeafNode with MultiInstanceRelation with LogicalPlanLike { + + override protected def otherCopyArgs: Seq[AnyRef] = session :: Nil override def newInstance(): EncoderPlan.this.type = { - val newRDD = super.newInstance().asInstanceOf[LogicalRDD] - new EncoderPlan(rdd, encoder, isFlat, - newRDD.output, session).asInstanceOf[this.type] + EncoderPlan(rdd, encoder, isFlat, output.map(_.newInstance()))(session).asInstanceOf[this.type] } + + override protected def stringArgs: Iterator[Any] = Iterator(output) + + override def computeStats(): Statistics = Statistics( + // TODO: Instead of returning a default value here, find a way to return a meaningful size + // estimate for RDDs. See PR 1238 for more discussions. + sizeInBytes = BigInt(session.sessionState.conf.defaultSizeInBytes) + ) + + @transient override lazy val statistics: Statistics = computeStats() } diff --git a/core/src/main/scala/org/apache/spark/sql/execution/ExistingPlans.scala b/core/src/main/scala/org/apache/spark/sql/execution/ExistingPlans.scala index 308dcf479f..262488df20 100644 --- a/core/src/main/scala/org/apache/spark/sql/execution/ExistingPlans.scala +++ b/core/src/main/scala/org/apache/spark/sql/execution/ExistingPlans.scala @@ -19,27 +19,26 @@ package org.apache.spark.sql.execution import scala.collection.mutable.ArrayBuffer import com.gemstone.gemfire.internal.cache.LocalRegion + import org.apache.spark.SparkContext import org.apache.spark.rdd.{RDD, ZippedPartitionsBaseRDD} -import org.apache.spark.sql.catalyst.errors.attachTree import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode} import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression, _} +import org.apache.spark.sql.catalyst.plans.logical.Statistics import org.apache.spark.sql.catalyst.plans.physical._ import org.apache.spark.sql.catalyst.util.{ArrayData, MapData} -import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow, TableIdentifier} +import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow} import org.apache.spark.sql.collection.Utils +import org.apache.spark.sql.execution.columnar.ConnectionType import org.apache.spark.sql.execution.columnar.impl.{BaseColumnFormatRelation, ColumnarStorePartitionedRDD, IndexColumnFormatRelation, SmartConnectorColumnRDD} -import org.apache.spark.sql.execution.columnar.{ColumnTableScan, ConnectionType} -import org.apache.spark.sql.execution.exchange.{ReusedExchangeExec, ShuffleExchange} +import org.apache.spark.sql.execution.exchange.ReusedExchangeExec import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetricInfo, SQLMetrics} -import org.apache.spark.sql.execution.row.{RowFormatRelation, RowFormatScanRDD, RowTableScan} +import org.apache.spark.sql.execution.row.{RowFormatRelation, RowFormatScanRDD} import org.apache.spark.sql.sources.{BaseRelation, PrunedUnsafeFilteredScan, SamplingRelation} import org.apache.spark.sql.types._ -import org.apache.spark.sql.{AnalysisException, CachedDataFrame, SnappySession} +import org.apache.spark.sql.{AnalysisException, CachedDataFrame, SnappySession, SparkSupport} import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String} - - /** * Physical plan node for scanning data from an DataSource scan RDD. * If user knows that the data is partitioned or replicated across @@ -53,10 +52,9 @@ private[sql] abstract class PartitionedPhysicalScan( numBuckets: Int, partitionColumns: Seq[Expression], partitionColumnAliases: Seq[Seq[Attribute]], - @transient override val relation: BaseRelation, - // not used currently (if need to use then get from relation.table) - override val metastoreTableIdentifier: Option[TableIdentifier] = None) - extends DataSourceScanExec with CodegenSupportOnExecutor { + @transient val relation: BaseRelation) + extends LeafExecNode with CodegenSupportOnExecutor + with NonRecursivePlans with SparkSupport { def getMetrics: Map[String, SQLMetric] = { if (sqlContext eq null) Map.empty @@ -91,10 +89,6 @@ private[sql] abstract class PartitionedPhysicalScan( rdd :: Nil } - protected override def doExecute(): RDD[InternalRow] = { - WholeStageCodegenExec(this).execute() - } - /** Specifies how data is partitioned across different nodes in the cluster. */ override lazy val outputPartitioning: Partitioning = { // when buckets are linked to partitions then actual buckets needs to be considered. @@ -146,7 +140,7 @@ private[sql] abstract class PartitionedPhysicalScan( } } -private[sql] object PartitionedPhysicalScan { +private[sql] object PartitionedPhysicalScan extends SparkSupport { private[sql] val CT_BLOB_POSITION = 4 private val EMPTY_PARAMS = Array.empty[ParamLiteral] @@ -165,7 +159,7 @@ private[sql] object PartitionedPhysicalScan { relation match { case i: IndexColumnFormatRelation => val caseSensitive = i.sqlContext.conf.caseSensitiveAnalysis - val columnScan = ColumnTableScan(output, rdd, otherRDDs, numBuckets, + val columnScan = internals.columnTableScan(output, rdd, otherRDDs, numBuckets, partitionColumns, partitionColumnAliases, relation, relation.schema, allFilters, schemaAttributes, caseSensitive) val table = i.getBaseTableRelation @@ -176,7 +170,7 @@ private[sql] object PartitionedPhysicalScan { def resolveCol(left: Attribute, right: AttributeReference) = columnScan.sqlContext.sessionState.analyzer.resolver(left.name, right.name) - val rowBufferScan = RowTableScan(output, StructType.fromAttributes( + val rowBufferScan = internals.rowTableScan(output, StructType.fromAttributes( output), baseTableRDD, numBuckets, Nil, Nil, table.table, table, caseSensitive) val otherPartKeys = partitionColumns.map(_.transform { case a: AttributeReference => rowBufferScan.output.find(resolveCol(_, a)).getOrElse { @@ -189,22 +183,22 @@ private[sql] object PartitionedPhysicalScan { ZipPartitionScan(columnScan, columnScan.partitionColumns, rowBufferScan, otherPartKeys) case c: BaseColumnFormatRelation => - ColumnTableScan(output, rdd, otherRDDs, numBuckets, + internals.columnTableScan(output, rdd, otherRDDs, numBuckets, partitionColumns, partitionColumnAliases, relation, relation.schema, allFilters, schemaAttributes, c.sqlContext.conf.caseSensitiveAnalysis) case r: SamplingRelation => if (r.isReservoirAsRegion) { - ColumnTableScan(output, rdd, Nil, numBuckets, partitionColumns, + internals.columnTableScan(output, rdd, Nil, numBuckets, partitionColumns, partitionColumnAliases, relation, relation.schema, allFilters, schemaAttributes, r.sqlContext.conf.caseSensitiveAnalysis, - isForSampleReservoirAsRegion = true) + isSampleReservoirAsRegion = true) } else { - ColumnTableScan(output, rdd, otherRDDs, numBuckets, + internals.columnTableScan(output, rdd, otherRDDs, numBuckets, partitionColumns, partitionColumnAliases, relation, relation.schema, allFilters, schemaAttributes, r.sqlContext.conf.caseSensitiveAnalysis) } case r: RowFormatRelation => - RowTableScan(output, StructType.fromAttributes(output), rdd, numBuckets, + internals.rowTableScan(output, StructType.fromAttributes(output), rdd, numBuckets, partitionColumns, partitionColumnAliases, relation.table, relation, r.sqlContext.conf.caseSensitiveAnalysis) } @@ -225,8 +219,13 @@ private[sql] object PartitionedPhysicalScan { val simpleString = SnappySession.replaceParamLiterals( plan.simpleString, paramLiterals, paramsId) + val metadata = plan match { + case s: FileSourceScanExec => s.metadata + case s: RowDataSourceScanExec => s.metadata + case _ => Map.empty[String, String] + } new SparkPlanInfo(plan.nodeName, simpleString, - children.map(getSparkPlanInfo(_, paramLiterals, paramsId)), plan.metadata, metrics) + children.map(getSparkPlanInfo(_, paramLiterals, paramsId)), metadata, metrics) } private[sql] def updatePlanInfo(planInfo: SparkPlanInfo, @@ -270,15 +269,15 @@ case class ExecutePlan(child: SparkPlan, preAction: () => Unit = () => ()) val (queryStringShortForm, queryStr, queryExecStr, planInfo) = if (key eq null) { val callSite = sqlContext.sparkContext.getCallSite() (callSite.shortForm, callSite.longForm, treeString(verbose = true), - PartitionedPhysicalScan.getSparkPlanInfo(this)) + PartitionedPhysicalScan.getSparkPlanInfo(this)) } else { val paramLiterals = key.currentLiterals val paramsId = key.currentParamsId (key.sqlText, key.sqlText, SnappySession.replaceParamLiterals( treeString(verbose = true), paramLiterals, paramsId), PartitionedPhysicalScan - .getSparkPlanInfo(this, paramLiterals, paramsId)) + .getSparkPlanInfo(this, paramLiterals, paramsId)) } - CachedDataFrame.withNewExecutionId(session, queryStringShortForm, + CachedDataFrame.withNewExecutionId(session, child, queryStringShortForm, queryStr, queryExecStr, planInfo) { preAction() val rdd = child.execute() @@ -301,7 +300,7 @@ case class ExecutePlan(child: SparkPlan, preAction: () => Unit = () => ()) } finally { logDebug(s" Unlocking the table in execute of ExecutePlan:" + - s" ${child.treeString(false)}") + s" ${child.treeString(verbose = false)}") session.clearWriteLockOnTable() } } @@ -345,15 +344,16 @@ trait PartitionedDataSourceScan extends PrunedUnsafeFilteredScan { private[sql] final case class ZipPartitionScan(basePlan: CodegenSupport, basePartKeys: Seq[Expression], otherPlan: SparkPlan, - otherPartKeys: Seq[Expression]) extends SparkPlan with CodegenSupport { + otherPartKeys: Seq[Expression]) extends SparkPlan with CodegenSupport + with NonRecursivePlans with SparkSupport { private var consumedCode: String = _ private val consumedVars: ArrayBuffer[ExprCode] = ArrayBuffer.empty - private val inputCode = basePlan.asInstanceOf[CodegenSupport] - private val withShuffle = ShuffleExchange(HashPartitioning( - ClusteredDistribution(otherPartKeys) - .clustering, inputCode.inputRDDs().head.getNumPartitions), otherPlan) + private val withShuffle = internals.newShuffleExchange(HashPartitioning( + otherPartKeys, basePlan.inputRDDs().head.getNumPartitions), otherPlan) + + override def needCopyResult: Boolean = false override def children: Seq[SparkPlan] = basePlan :: withShuffle :: Nil @@ -361,27 +361,29 @@ private[sql] final case class ZipPartitionScan(basePlan: CodegenSupport, ClusteredDistribution(basePartKeys) :: ClusteredDistribution(otherPartKeys) :: Nil override def inputRDDs(): Seq[RDD[InternalRow]] = - inputCode.inputRDDs ++ Some(withShuffle.execute()) + basePlan.inputRDDs ++ Some(withShuffle.execute()) override protected def doProduce(ctx: CodegenContext): String = { - val child1Produce = inputCode.produce(ctx, this) - val input = ctx.freshName("input") - ctx.addMutableState("scala.collection.Iterator", input, s" $input = inputs[1]; ") + val child1Produce = basePlan.produce(ctx, this) + val input = internals.addClassField(ctx, "scala.collection.Iterator", "input", + v => s"$v = inputs[1];") val row = ctx.freshName("row") val columnsInputEval = otherPlan.output.zipWithIndex.map { case (ref, ordinal) => val baseIndex = ordinal val ev = consumedVars(ordinal) + val evIsNull = internals.exprCodeIsNull(ev) + val evValue = internals.exprCodeValue(ev) val dataType = ref.dataType - val javaType = ctx.javaType(dataType) - val value = ctx.getValue(row, dataType, baseIndex.toString) + val javaType = internals.javaType(dataType, ctx) + val value = internals.getValue(row, dataType, baseIndex.toString, ctx) if (ref.nullable) { s""" - boolean ${ev.isNull} = $row.isNullAt($ordinal); - $javaType ${ev.value} = ${ev.isNull} ? ${ctx.defaultValue(dataType)} : ($value); + boolean $evIsNull = $row.isNullAt($ordinal); + $javaType $evValue = $evIsNull ? ${internals.defaultValue(dataType, ctx)} : ($value); """ } else { - s"""$javaType ${ev.value} = $value;""" + s"""$javaType $evValue = $value;""" } }.mkString("\n") @@ -405,10 +407,6 @@ private[sql] final case class ZipPartitionScan(basePlan: CodegenSupport, consumeInput + "\n" + consumedCode } - override protected def doExecute(): RDD[InternalRow] = attachTree(this, "execute") { - WholeStageCodegenExec(this).execute() - } - override def output: Seq[Attribute] = basePlan.output } @@ -420,8 +418,10 @@ private[sql] final case class ZipPartitionScan(basePlan: CodegenSupport, final class TokenizedScalarSubquery(_plan: SubqueryExec, _exprId: ExprId) extends ScalarSubquery(_plan, _exprId) { - override def withNewPlan(query: SubqueryExec): ScalarSubquery = - new TokenizedScalarSubquery(query, exprId) + override def copy(plan: SubqueryExec = plan, exprId: ExprId = exprId): ScalarSubquery = + new TokenizedScalarSubquery(plan, exprId) + + override def withNewPlan(query: SubqueryExec): ScalarSubquery = copy(plan = query) override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { val result = CatalystTypeConverters.convertToCatalyst(super.eval(null)) @@ -440,7 +440,7 @@ class StratumInternalRow(val weight: Long) extends InternalRow { def copy(): InternalRow = throw new UnsupportedOperationException("not implemented") - def anyNull: Boolean = throw new UnsupportedOperationException("not implemented") + override def anyNull: Boolean = throw new UnsupportedOperationException("not implemented") def isNullAt(ordinal: Int): Boolean = throw new UnsupportedOperationException("not implemented") @@ -495,7 +495,7 @@ trait BatchConsumer extends CodegenSupport { /** * Generate Java source code to do any processing before a batch is consumed - * by a [[DataSourceScanExec]] that does batch processing (e.g. per-batch + * by a [[PartitionedPhysicalScan]] that does batch processing (e.g. per-batch * optimizations, initializations etc). *
* Implementations should use this for additional optimizations that can be @@ -517,13 +517,14 @@ trait BatchConsumer extends CodegenSupport { * Extended information for ExprCode variable to also hold the variable having * dictionary reference and its index when dictionary encoding is being used. */ -case class DictionaryCode(dictionary: ExprCode, bufferVar: String, dictionaryIndex: ExprCode) { +case class DictionaryCode(dictionary: ExprCode, bufferVar: String, + dictionaryIndex: ExprCode) extends SparkSupport { private def evaluate(ev: ExprCode): String = { - if (ev.code.isEmpty) "" + val code = ev.code.toString + if (code.isEmpty) "" else { - val code = ev.code - ev.code = "" + internals.resetCode(ev) code } } @@ -532,3 +533,13 @@ case class DictionaryCode(dictionary: ExprCode, bufferVar: String, dictionaryInd def evaluateIndexCode(): String = evaluate(dictionaryIndex) } + +/** + * Intermediate trait to accommodate differences in statistics method in Spark versions. + */ +trait LogicalPlanLike { + + def statistics: Statistics + + def computeStats(): Statistics +} diff --git a/core/src/main/scala/org/apache/spark/sql/execution/NonRecursivePlans.scala b/core/src/main/scala/org/apache/spark/sql/execution/NonRecursivePlans.scala index c14644c922..cfeab3f6f6 100644 --- a/core/src/main/scala/org/apache/spark/sql/execution/NonRecursivePlans.scala +++ b/core/src/main/scala/org/apache/spark/sql/execution/NonRecursivePlans.scala @@ -17,6 +17,7 @@ package org.apache.spark.sql.execution import org.apache.spark.rdd.RDD +import org.apache.spark.sql.SparkSupport import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.internal.CodeGenerationException @@ -25,7 +26,7 @@ import org.apache.spark.sql.internal.CodeGenerationException * version and use the same for non-codegenerated case. For that case this * prevents recursive calls into code generation in case it fails for some reason. */ -abstract class NonRecursivePlans extends SparkPlan { +trait NonRecursivePlans extends SparkPlan with SparkSupport { /** * Variable to disallow recursive generation so will mark the case of @@ -33,12 +34,16 @@ abstract class NonRecursivePlans extends SparkPlan { */ protected final var nonCodeGeneratedPlanCalls: Int = _ + // from CodegenSupport in newer Spark releases that allows child classes + // to override and still compile fine in older releases + def needCopyResult: Boolean + override protected def doExecute(): RDD[InternalRow] = { if (nonCodeGeneratedPlanCalls > 4) { throw new CodeGenerationException("Code generation failed for some of the child plans") } nonCodeGeneratedPlanCalls += 1 - WholeStageCodegenExec(this).execute() + internals.newWholeStagePlan(this).execute() } override def makeCopy(newArgs: Array[AnyRef]): NonRecursivePlans = { diff --git a/core/src/main/scala/org/apache/spark/sql/execution/ObjectHashMapAccessor.scala b/core/src/main/scala/org/apache/spark/sql/execution/ObjectHashMapAccessor.scala index b9ccf6b8a3..da6753cb0a 100644 --- a/core/src/main/scala/org/apache/spark/sql/execution/ObjectHashMapAccessor.scala +++ b/core/src/main/scala/org/apache/spark/sql/execution/ObjectHashMapAccessor.scala @@ -22,15 +22,15 @@ import com.gemstone.gemfire.internal.shared.ClientResolverUtils import io.snappydata.collection.ObjectHashSet import org.apache.spark.rdd.RDD -import org.apache.spark.sql.SnappySession import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode} -import org.apache.spark.sql.catalyst.expressions.{Attribute, BindReferences, Expression, NamedExpression} +import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeSet, BindReferences, Expression, NamedExpression} import org.apache.spark.sql.catalyst.plans._ import org.apache.spark.sql.execution.columnar.encoding.StringDictionary import org.apache.spark.sql.execution.joins.{BuildLeft, BuildRight, BuildSide, HashJoinExec} import org.apache.spark.sql.execution.row.RowTableScan import org.apache.spark.sql.types._ +import org.apache.spark.sql.{SnappySession, SparkSupport} import org.apache.spark.unsafe.array.ByteArrayMethods /** @@ -85,7 +85,7 @@ case class ObjectHashMapAccessor(@transient session: SnappySession, hashMapTerm: String, dataTerm: String, maskTerm: String, multiMap: Boolean, @transient consumer: CodegenSupport, @transient cParent: CodegenSupport, override val child: SparkPlan) - extends UnaryExecNode with CodegenSupport { + extends UnaryExecNode with CodegenSupport with SparkSupport { override def output: Seq[Attribute] = child.output @@ -131,8 +131,7 @@ case class ObjectHashMapAccessor(@transient session: SnappySession, private type ClassVar = (DataType, String, ExprCode, Int) - @transient private[this] val (className, valueClassName, classVars, - numNullVars) = initClass() + @transient private[this] val (className, valueClassName, classVars, numNullVars) = initClass() private def initClass(): (String, String, IndexedSeq[ClassVar], Int) = { @@ -171,7 +170,9 @@ case class ObjectHashMapAccessor(@transient session: SnappySession, // Generate equals code for key columns only. val keyVars = entryVars.take(valueIndex) val equalsCode = keyVars.map { - case (dataType, _, ExprCode(_, nullVar, varName), nullIndex) => + case (dataType, _, ev, nullIndex) => + val nullVar = internals.exprCodeIsNull(ev) + val varName = internals.exprCodeValue(ev) genEqualsCode("this", varName, nullVar, other, varName, nullVar, nullIndex, isPrimitiveType(dataType), dataType) }.mkString(" &&\n") @@ -180,7 +181,8 @@ case class ObjectHashMapAccessor(@transient session: SnappySession, (s""" public static class $valueClass { $nullDecls - ${valClassVars.map(e => s"${e._2} ${e._3.value};").mkString("\n")} + ${valClassVars.map(e => s"${e._2} ${internals.exprCodeValue(e._3)};") + .mkString("\n")} $valueClass $nextValueVar; } """, s" extends $valueClass", "", "") @@ -191,7 +193,7 @@ case class ObjectHashMapAccessor(@transient session: SnappySession, s""" public static final class $entryClass$extendsCode { $nulls - ${entryVars.map(e => s"${e._2} ${e._3.value};").mkString("\n")} + ${entryVars.map(e => s"${e._2} ${internals.exprCodeValue(e._3)};").mkString("\n")} $multiValues final int hash; @@ -211,12 +213,12 @@ case class ObjectHashMapAccessor(@transient session: SnappySession, } } """ - // using addNewFunction to register the class since there is nothing - // function specific in the addNewFunction method + // using addFunction to register the class since there is nothing + // function specific in the addFunction method if (!valueClassCode.isEmpty) { - ctx.addNewFunction(valueClass, valueClassCode) + internals.addFunction(ctx, valueClass, valueClassCode, inlineToOuterClass = true) } - ctx.addNewFunction(entryClass, classCode) + internals.addFunction(ctx, entryClass, classCode, inlineToOuterClass = true) session.addClass(ctx, valClassTypes, keyTypes, entryTypes, valueClass, entryClass, multiMap) } @@ -245,7 +247,7 @@ case class ObjectHashMapAccessor(@transient session: SnappySession, val javaType = dataType match { // use raw byte arrays for strings to minimize overhead case StringType if !multiMap => "byte[]" - case _ => ctx.javaType(dataType) + case _ => internals.javaType(dataType, ctx) } val (nullVar, nullIndex) = if (nullable) { if (isPrimitiveType(dataType)) { @@ -261,11 +263,11 @@ case class ObjectHashMapAccessor(@transient session: SnappySession, } else ("", NULL_NON_PRIM) // field itself is nullable } else ("", -1) if (index < numEntryVars) { - entryVars += ((dataType, javaType, ExprCode("", nullVar, varName), - nullIndex)) + entryVars += ((dataType, javaType, internals.newExprCode(code = "", nullVar, varName, + dataType), nullIndex)) } else { - valClassVars += ((dataType, javaType, ExprCode("", nullVar, varName), - nullIndex)) + valClassVars += ((dataType, javaType, internals.newExprCode(code = "", nullVar, varName, + dataType), nullIndex)) } } val numNullVars = if (numNulls >= 0) (numNulls / 64) + 1 else 0 @@ -291,8 +293,8 @@ case class ObjectHashMapAccessor(@transient session: SnappySession, override protected def doProduce(ctx: CodegenContext): String = throw new UnsupportedOperationException("unexpected invocation") - override def doConsume(ctx: CodegenContext, input: Seq[ExprCode], - row: ExprCode): String = { + private def doConsume(ctx: CodegenContext, keyExpressions: Seq[Expression], + valueExpressions: Seq[Expression], input: Seq[ExprCode]): String = { // consume the data and populate the map val entryVar = "mapEntry" // local variable val hashVar = Array(ctx.freshName("hash")) @@ -301,13 +303,12 @@ case class ObjectHashMapAccessor(@transient session: SnappySession, val keyVars = getExpressionVars(keyExpressions, input) // skip expressions already in key variables (that are also skipped // in the value class fields in class generation) - val valueVars = getExpressionVars( - valueExprIndexes.filter(_._2 >= 0).map(_._1), input) + val valueVars = getExpressionVars(valueExpressions, input) // Update min/max code for primitive type columns. Avoiding additional // index mapping here for mix of integral and non-integral keys // rather using key index since overhead of blanks will be negligible. val updateMinMax = integralKeys.map { index => - s"$hashMapTerm.updateLimits(${keyVars(index).value}, $index);" + s"$hashMapTerm.updateLimits(${internals.exprCodeValue(keyVars(index))}, $index);" }.mkString("\n") val doCopy = !ObjectHashMapAccessor.providesImmutableObjects(child) @@ -328,13 +329,17 @@ case class ObjectHashMapAccessor(@transient session: SnappySession, // mark map as not unique on multiple inserts for same key $hashMapTerm.setKeyIsUnique(false);""" } + val nullableKeys = keyVars.map(internals.exprCodeIsNull).filter(_ != "false") + val (nullCheckStart, nullCheckEnd) = + if (nullableKeys.isEmpty) ("", "") + else { + (s"// skip if a key is null\nif (${nullableKeys.mkString("!", " &&\n!", "")}) {\n", "\n}") + } s""" // evaluate the key and value expressions ${evaluateVariables(keyVars)}${evaluateVariables(valueVars)} - // skip if any key is null - if (${keyVars.map(_.isNull).mkString(" ||\n")}) continue; - // generate hash code - ${generateHashCode(hashVar, keyVars, keyExpressions, register = false)} + $nullCheckStart// generate hash code + ${generateHashCode(hashVar, keyVars, register = false)} // lookup or insert the grouping key in map // using inline get call so that equals() is inline using // existing register variables instead of having to fill up @@ -370,10 +375,73 @@ case class ObjectHashMapAccessor(@transient session: SnappySession, break; } - } + }$nullCheckEnd """ } + override def doConsume(ctx: CodegenContext, input: Seq[ExprCode], row: ExprCode): String = { + val valueExpressions = valueExprIndexes.filter(_._2 >= 0).map(_._1) + val output = this.output + // try to create a separate function for doConsume to reduce outer function size + if (calculateParamLength(ctx, output) <= 255) { + val doConsumeFunction = ctx.freshName("doConsume") + val usedInput = AttributeSet(keyExpressions) ++ AttributeSet(valueExpressions) + val usedInputCode = new mutable.ArrayBuffer[String] + val args = new mutable.ArrayBuffer[String] + val params = new mutable.ArrayBuffer[String] + val newInput = new mutable.ArrayBuffer[ExprCode]() + for (i <- input.indices) { + val attr = output(i) + val ev = input(i) + if (usedInput.contains(attr)) { + val varName = ctx.freshName("arg") + val dataType = attr.dataType + val evCode = ev.code.toString + if (!evCode.isEmpty) usedInputCode += evCode + args += internals.exprCodeValue(ev) + params += s"${internals.javaType(dataType, ctx)} $varName" + var isNull = internals.exprCodeIsNull(ev) + if (isNull != "false") { + args += isNull + isNull = ctx.freshName("isNull") + params += s"boolean $isNull" + } + newInput += internals.newExprCode(code = "", isNull, varName, dataType) + } else { + newInput += ev + } + } + val functionName = internals.addFunction(ctx, doConsumeFunction, + s""" + |private void $doConsumeFunction(${params.mkString(", ")}) throws java.io.IOException { + | ${doConsume(ctx, keyExpressions, valueExpressions, newInput)} + |} + """.stripMargin) + s""" + |${usedInputCode.mkString("\n")} + |$functionName(${args.mkString(", ")}); + """.stripMargin + } else { + doConsume(ctx, keyExpressions, valueExpressions, input) + } + } + + /** + * Taken from CodeGenerator.calculateParamLength in Spark 2.4.x + */ + private def calculateParamLength(ctx: CodegenContext, params: Seq[Expression]): Int = { + def paramLengthForExpr(input: Expression): Int = { + val javaParamLength = internals.javaType(input.dataType, ctx) match { + case "long" | "double" => 2 + case _ => 1 + } + // For a nullable expression, we need to pass in an extra boolean parameter. + (if (input.nullable) 1 else 0) + javaParamLength + } + // Initial value is 1 for `this`. + 1 + params.map(paramLengthForExpr).sum + } + /** get the generated class name */ def getClassName: String = className @@ -382,13 +450,12 @@ case class ObjectHashMapAccessor(@transient session: SnappySession, * correspond to the key columns in this class. */ def generateHashCode(hashVar: Array[String], keyVars: Seq[ExprCode], - keyExpressions: Seq[Expression], skipDeclaration: Boolean = false, - register: Boolean = true): String = { + skipDeclaration: Boolean = false, register: Boolean = true): String = { var hash = hashVar(0) val hashDeclaration = if (skipDeclaration) "" else s"int $hash;\n" // check if hash has already been generated for keyExpressions var doRegister = register - val vars = keyVars.map(_.value) + val vars = keyVars.map(internals.exprCodeValue) val (prefix, suffix) = session.getHashVar(ctx, vars) match { case Some(h) => hashVar(0) = h @@ -404,9 +471,8 @@ case class ObjectHashMapAccessor(@transient session: SnappySession, } // optimize for first column to use fast hashing - val expr = keyVars.head - val colVar = expr.value - val nullVar = expr.isNull + val nullVar = internals.exprCodeIsNull(keyVars.head) + val colVar = internals.exprCodeValue(keyVars.head) val firstColumnHash = classVars(0)._1 match { case BooleanType => hashSingleInt(s"($colVar) ? 1 : 0", nullVar, hash) @@ -428,22 +494,23 @@ case class ObjectHashMapAccessor(@transient session: SnappySession, hashSingleInt(s"$colVar.hashCode()", nullVar, hash) } if (keyVars.length > 1) { - classVars.tail.zip(keyVars.tail).map { - case ((BooleanType, _, _, _), ev) => - addHashInt(s"${ev.value} ? 1 : 0", ev.isNull, hash) - case ((ByteType | ShortType | IntegerType | DateType, _, _, _), ev) => - addHashInt(ev.value, ev.isNull, hash) - case ((LongType | TimestampType, _, _, _), ev) => - addHashLong(ev.value, ev.isNull, hash) - case ((FloatType, _, _, _), ev) => - addHashInt(s"Float.floatToIntBits(${ev.value})", ev.isNull, hash) - case ((DoubleType, _, _, _), ev) => - addHashLong(s"Double.doubleToLongBits(${ev.value})", ev.isNull, + classVars.tail.zip(keyVars.tail).map(p => (p._1._1, + internals.exprCodeIsNull(p._2), internals.exprCodeValue(p._2))).map { + case (BooleanType, evIsNull, evValue) => + addHashInt(s"$evValue ? 1 : 0", evIsNull, hash) + case (ByteType | ShortType | IntegerType | DateType, evIsNull, evValue) => + addHashInt(evValue, evIsNull, hash) + case (LongType | TimestampType, evIsNull, evValue) => + addHashLong(evValue, evIsNull, hash) + case (FloatType, evIsNull, evValue) => + addHashInt(s"Float.floatToIntBits($evValue)", evIsNull, hash) + case (DoubleType, evIsNull, evValue) => + addHashLong(s"Double.doubleToLongBits($evValue)", evIsNull, hash) - case ((_: DecimalType, _, _, _), ev) => - addHashInt(s"${ev.value}.fastHashCode()", ev.isNull, hash) - case (_, ev) => - addHashInt(s"${ev.value}.hashCode()", ev.isNull, hash) + case (_: DecimalType, evIsNull, evValue) => + addHashInt(s"$evValue.fastHashCode()", evIsNull, hash) + case (_, evIsNull, evValue) => + addHashInt(s"$evValue.hashCode()", evIsNull, hash) }.mkString(prefix + firstColumnHash, "", suffix) } else prefix + firstColumnHash + suffix } @@ -456,9 +523,10 @@ case class ObjectHashMapAccessor(@transient session: SnappySession, */ def generateEquals(objVar: String, keyVars: Seq[ExprCode]): String = classVars.zip(keyVars).map { - case ((dataType, _, ExprCode(_, nullVar, varName), nullIndex), colVar) => - genEqualsCode("", colVar.value, colVar.isNull, objVar, varName, - nullVar, nullIndex, isPrimitiveType(dataType), dataType) + case ((dataType, _, ev, nullIndex), colVar) => + genEqualsCode("", internals.exprCodeValue(colVar), internals.exprCodeIsNull(colVar), + objVar, internals.exprCodeValue(ev), internals.exprCodeIsNull(ev), nullIndex, + isPrimitiveType(dataType), dataType) }.mkString(" &&\n") /** @@ -492,7 +560,7 @@ case class ObjectHashMapAccessor(@transient session: SnappySession, } declarations.append(s"long $nullValMaskVar = $nullMaskVar;\n") nullValMaskVars(index) = nullValMaskVar - nullVar -> (nullMaskVar, nullValMaskVar) + (nullVar, (nullMaskVar, nullValMaskVar)) }.toMap val vars = if (onlyKeyVars) classVars.take(valueIndex) @@ -518,6 +586,7 @@ case class ObjectHashMapAccessor(@transient session: SnappySession, // nullIndex contains index of referenced key variable in this case case null if !onlyValueVars => columnVars += columnVars(nullIndex) case _ => + val evValue = internals.exprCodeValue(ev) val (localVar, localDeclaration) = { dataType match { case StringType if !multiMap => @@ -526,29 +595,30 @@ case class ObjectHashMapAccessor(@transient session: SnappySession, (lv, new StringBuilder().append(s"final UTF8String $lv = ").append( if (checkNullObj) { s"($objVar != null ? UTF8String.fromBytes(" + - s"$objVar.${ev.value}) : null);" + s"$objVar.$evValue) : null);" } else { - s"UTF8String.fromBytes($objVar.${ev.value});" + s"UTF8String.fromBytes($objVar.$evValue);" })) case _ => val lv = ctx.freshName("localField") (lv, new StringBuilder().append(s"final $javaType $lv = ").append( if (checkNullObj) { - s"($objVar != null ? $objVar.${ev.value} " + - s" : ${ctx.defaultValue(dataType)});" + s"($objVar != null ? $objVar.$evValue " + + s" : ${internals.defaultValue(dataType, ctx)});" } else { - s"$objVar.${ev.value};" + s"$objVar.$evValue;" })) } } - val nullExpr = nullMaskVarMap.get(ev.isNull) + val nullExpr = nullMaskVarMap.get(internals.exprCodeIsNull(ev)) .map(p => if (isKeyVar) genNullCode(p._1, nullIndex) else genNullCode(p._2, nullIndex)).getOrElse( if (nullIndex == NULL_NON_PRIM) s"($localVar == null)" else "false") val nullVar = ctx.freshName("isNull") localDeclaration.append(s"\nboolean $nullVar = $nullExpr;") - columnVars += ExprCode(localDeclaration.toString, nullVar, localVar) + columnVars += internals.newExprCode(localDeclaration.toString, nullVar, + localVar, dataType) } } (declarations.toString(), columnVars, nullValMaskVars) @@ -564,25 +634,28 @@ case class ObjectHashMapAccessor(@transient session: SnappySession, // generate the variables for each of the key terms with proper types val (keyDecls, keyCalls, newKeyVars) = keyExpressions .zip(keyVars).map { case (expr, ev) => - val javaType = ctx.javaType(expr.dataType) + val javaType = internals.javaType(expr.dataType, ctx) val newKeyVar = ctx.freshName("keyCol") - if (ev.isNull == "false") { - (s"final $javaType $newKeyVar", ev.value, ev.copy(value = newKeyVar)) + val evIsNull = internals.exprCodeIsNull(ev) + val evValue = internals.exprCodeValue(ev) + if (evIsNull == "false") { + (s"final $javaType $newKeyVar", evValue, + internals.copyExprCode(ev, value = newKeyVar, dt = expr.dataType)) } else { // new variable for nullability since isNull can be an expression val newNullVar = ctx.freshName("keyIsNull") (s"final $javaType $newKeyVar, final boolean $newNullVar", - s"${ev.value}, ${ev.isNull}", - ev.copy(isNull = newNullVar, value = newKeyVar)) + s"$evValue, $evIsNull", + internals.copyExprCode(ev, isNull = newNullVar, value = newKeyVar, dt = expr.dataType)) } }.unzip3 val keyDeclarations = keyDecls.mkString(", ") val skipInit = valueInit eq null // check for existing function with matching null vars and skipInit - val fnKey = className -> keyVars.map(_.isNull == "false") - val fn = session.getContextObject[(String, Boolean)](ctx, "F", fnKey) match { - case Some((functionName, skip)) if skipInit || !skip => functionName + val fnKey = className -> keyVars.map(internals.exprCodeIsNull(_) == "false") + val fn = session.getContextObject[(String, String, Boolean)](ctx, "F", fnKey) match { + case Some((_, functionName, skip)) if skipInit || !skip => functionName case f => // re-use function for non-matching skipInit but change its body // to also handle insertion of new blank entry @@ -590,6 +663,7 @@ case class ObjectHashMapAccessor(@transient session: SnappySession, case None => ctx.freshName("mapLookup") case Some(p) => p._1 } + val hashMapArg = ctx.freshName("hashMap") val insertCode = if (skipInit) { s"""else { | // key not found so return entry as null for consumption @@ -610,7 +684,7 @@ case class ObjectHashMapAccessor(@transient session: SnappySession, | ${generateUpdate(objVar, Nil, newKeyVars, forKey = true)} | // insert into the map and rehash if required | $dataTerm[$pos] = $objVar; - | if ($hashMapTerm.handleNewInsert($pos)) { + | if ($hashMapArg.handleNewInsert($pos)) { | // return null to indicate map was rehashed | return null; | } else { @@ -618,11 +692,11 @@ case class ObjectHashMapAccessor(@transient session: SnappySession, | } |}""".stripMargin } - ctx.addNewFunction(function, + val functionName = internals.addFunction(ctx, function, s""" |private $className $function(final int $hash, $keyDeclarations, | final $className[] $dataTerm, final int $maskTerm, - | final ${classOf[ObjectHashSet[_]].getName} $hashMapTerm, + | final ${classOf[ObjectHashSet[_]].getName} $hashMapArg, | final boolean skipInit) { | // Lookup or insert the key in map (for group by). | // Using inline get call so that equals() is inline using @@ -647,8 +721,8 @@ case class ObjectHashMapAccessor(@transient session: SnappySession, """.stripMargin) // register the new function - session.addContextObject(ctx, "F", fnKey, function -> skipInit) - function + session.addContextObject(ctx, "F", fnKey, (function, functionName, skipInit)) + functionName } val keyArgs = keyCalls.mkString(", ") @@ -677,21 +751,23 @@ case class ObjectHashMapAccessor(@transient session: SnappySession, output), ctx, session) dictionaryKey match { case Some(d@DictionaryCode(dictionary, _, _)) => + val dictValue = internals.exprCodeValue(dictionary) // initialize or reuse the array at batch level for join // null key will be placed at the last index of dictionary // and dictionary index will be initialized to that by ColumnTableScan - ctx.addMutableState(classOf[StringDictionary].getName, dictionary.value, "") - ctx.addNewFunction(dictionaryArrayInit, + internals.addClassField(ctx, classOf[StringDictionary].getName, + dictValue, forceInline = true, useFreshName = false) + internals.addFunction(ctx, dictionaryArrayInit, s""" |public $className[] $dictionaryArrayInit() { | ${d.evaluateDictionaryCode()} - | if (${dictionary.value} != null) { - | return new $className[${dictionary.value}.size() + 1]; + | if ($dictValue != null) { + | return new $className[$dictValue.size() + 1]; | } else { | return null; | } |} - """.stripMargin) + """.stripMargin, inlineToOuterClass = true) true case None => false } @@ -730,7 +806,7 @@ case class ObjectHashMapAccessor(@transient session: SnappySession, // evaluate the key expressions ${evaluateVariables(keyVars)} // evaluate hash code of the lookup key - ${generateHashCode(hashVar, keyVars, keyExpressions, register = false)} + ${generateHashCode(hashVar, keyVars, register = false)} ${mapLookupCode(keyVars)} } """ @@ -742,7 +818,7 @@ case class ObjectHashMapAccessor(@transient session: SnappySession, $inputEvals ${evaluateVariables(keyVars)} // evaluate hash code of the lookup key - ${generateHashCode(hashVar, keyVars, keyExpressions)} + ${generateHashCode(hashVar, keyVars)} $className $objVar; ${mapLookupCode(keyVars)} """ @@ -754,15 +830,13 @@ case class ObjectHashMapAccessor(@transient session: SnappySession, s"$numRows++;\n${consumer.consume(ctx, resultVars)}" // scalastyle:off - def generateMapLookup(entryVar: String, localValueVar: String, - mapSize: String, keyIsUnique: String, initMap: String, - initMapCode: String, numRows: String, nullMaskVars: Array[String], - initCode: String, checkCond: (Option[ExprCode], String, Option[Expression]), - streamKeys: Seq[Expression], streamKeyVars: Seq[ExprCode], - streamOutput: Seq[Attribute], buildKeyVars: Seq[ExprCode], - buildVars: Seq[ExprCode], input: Seq[ExprCode], - resultVars: Seq[ExprCode], dictArrayVar: String, dictArrayInitVar: String, - joinType: JoinType, buildSide: BuildSide): String = { + def generateMapLookup(entryVar: String, localValueVar: String, mapSize: String, + keyIsUnique: String, initMap: String, initMapCode: String, numRows: String, + nullMaskVars: Array[String], initCode: String, checkCond: (Option[ExprCode], String, + Option[Expression]), streamKeys: Seq[Expression], streamOutput: Seq[Attribute], + buildKeyVars: Seq[ExprCode], buildVars: Seq[ExprCode], input: Seq[ExprCode], + dictArrayVar: String, dictArrayInitVar: String, joinType: JoinType, + buildSide: BuildSide): String = { // scalastyle:on val hash = ctx.freshName("hash") @@ -784,25 +858,44 @@ case class ObjectHashMapAccessor(@transient session: SnappySession, } val mapKeyCodes = s"$initCode\n${evaluateVariables(mapKeyVars)}" + // continue to next entry on no match + val continueOnNull = joinType match { + case Inner | LeftSemi => true + case _ => false + } + // initialize dictionaryKey + initDictionaryCodeForSingleKeyCase(dictArrayInitVar, input, streamKeys, streamOutput) + + // check if streamKeyVars need to be evaluated in the outer block in which case pre-evaluate + // the used input variables in appropriate positions to avoid double variable initialization + val inputKeysCode = if (dictionaryKey.isEmpty || + // determine if initFilters will be empty or not + !continueOnNull || integralKeys.nonEmpty || streamKeys.exists(_.nullable)) { + evaluateRequiredVariables(streamOutput, input, AttributeSet(streamKeys)) + } else "" + val resultVars = buildSide match { + case BuildLeft => buildVars ++ input + case BuildRight => input ++ buildVars + } + + ctx.INPUT_ROW = null + ctx.currentVars = input + val boundStreamKeys = streamKeys.map(BindReferences.bindReference(_, streamOutput)) + val streamKeyVars = ctx.generateExpressions(boundStreamKeys) + // invoke generateHashCode before consume so that hash variables // can be re-used by consume if possible - val streamHashCode = generateHashCode(hashVar, streamKeyVars, streamKeys, - skipDeclaration = true) + val streamHashCode = generateHashCode(hashVar, streamKeyVars, skipDeclaration = true) // if previous hash variable is being used then skip declaration val hashInit = if (hashVar(0) eq hash) s"int $hash = 0;" else "" // if a stream-side key is null then skip (or null for outer join) - val nullStreamKey = streamKeyVars.filter(_.isNull != "false") - .map(v => s"!${v.isNull}") - // continue to next entry on no match - val continueOnNull = joinType match { - case Inner | LeftSemi => true - case _ => false + val nullStreamKeys = streamKeys.indices.collect { + case i if streamKeys(i).nullable => s"!${internals.exprCodeIsNull(streamKeyVars(i))}" } // filter as per min/max if provided; the min/max variables will be // initialized by the caller outside the loop after creating the map - val minMaxFilter = integralKeys.zipWithIndex.map { - case (indexKey, index) => - val keyVar = streamKeyVars(indexKey).value + val minMaxFilter = integralKeys.zipWithIndex.map { case (indexKey, index) => + val keyVar = internals.exprCodeValue(streamKeyVars(indexKey)) val minVar = integralKeysMinVars(index) val maxVar = integralKeysMaxVars(index) s"$keyVar >= $minVar && $keyVar <= $maxVar" @@ -810,11 +903,10 @@ case class ObjectHashMapAccessor(@transient session: SnappySession, // generate the initial filter condition from above two // also add a mapSize check but when continueOnNull is true, then emit a continue immediately val (checkMapSize, initFilters) = if (continueOnNull) { - (s"if ($mapSize == 0) continue;\n", nullStreamKey ++ minMaxFilter) - } - else ("", s"$mapSize != 0" +: (nullStreamKey ++ minMaxFilter)) - val initFilterCode = if (initFilters.isEmpty) "" - else initFilters.mkString("if (", " &&\n", ")") + (s"if ($mapSize == 0) continue;\n", nullStreamKeys ++ minMaxFilter) + } else ("", s"$mapSize != 0" +: (nullStreamKeys ++ minMaxFilter)) + val initFilterCode = + if (initFilters.isEmpty) "" else initFilters.mkString("if (", " &&\n", ")") // common multi-value iteration code fragments val entryIndexVar = ctx.freshName("entryIndex") @@ -866,31 +958,28 @@ case class ObjectHashMapAccessor(@transient session: SnappySession, } // optimized path for single key string column if dictionary is present - val lookup = mapLookup(entryVar, hashVar(0), streamKeys, streamKeyVars, - valueInit = null) - val preEvalKeys = if (initFilterCode.isEmpty) "" - else evaluateVariables(streamKeyVars) - initDictionaryCodeForSingleKeyCase(dictArrayInitVar, input, - streamKeys, streamOutput) + val lookup = mapLookup(entryVar, hashVar(0), boundStreamKeys, streamKeyVars, valueInit = null) + val preEvalKeys = if (initFilterCode.isEmpty) "" else evaluateVariables(streamKeyVars) var mapLookupCode = dictionaryKey match { case Some(dictKey) => val keyVar = streamKeyVars.head + val keyCode = keyVar.code.toString // don't call evaluateVariables for streamKeyVars for the else // part below because it is in else block and should be re-evaluated // if required outside the block val code = s""" ${DictionaryOptimizedMapAccessor.dictionaryArrayGetOrInsert(ctx, - streamKeys, keyVar, dictKey, dictArrayVar, entryVar, - valueInit = null, continueOnNull, this)} else { - // evaluate the key expressions - ${if (keyVar.code.isEmpty) "" else keyVar.code.trim} + boundStreamKeys, keyVar, dictKey, dictArrayVar, entryVar, + valueInit = null, continueOnNull, accessor = this)} else { + // evaluate the string key expression + ${if (keyCode.isEmpty) "" else keyCode.trim} // generate hash code from stream side key columns $streamHashCode $lookup } """ // copy back the updated code to input if present - if (keyVar.code.nonEmpty) input.find(_.value == keyVar.value) + if (keyCode.nonEmpty) input.find(_.value == keyVar.value) .foreach(_.code = keyVar.code) code case None => @@ -959,22 +1048,26 @@ case class ObjectHashMapAccessor(@transient session: SnappySession, val existsVar = ctx.freshName("exists") genExistenceJoinCodes(entryVar, existsVar, mapKeyCodes, checkCondition, checkCode, numRows, getConsumeResultCode(numRows, - input :+ ExprCode("", "false", existsVar)), keyIsUnique, + input :+ internals.newExprCode("", "false", existsVar, BooleanType)), keyIsUnique, declareLocalVars, moveNextValue, inputCodes) case _ => throw new IllegalArgumentException( s"HashJoin should not take $joinType as the JoinType") } + // wrap in "do {...} while(false)" so that the code inside can break out with continue s""" - if (!$initMap) { - $initMapCode - } - $checkMapSize$className $entryVar = null; - $hashInit - $mapLookupCode - $entryConsume - """ + |if (!$initMap) { + | $initMapCode + |} + |do { + | $checkMapSize$className $entryVar = null; + | $inputKeysCode + | $hashInit + | $mapLookupCode + | $entryConsume + |} while (false); + """.stripMargin } /** @@ -1000,10 +1093,10 @@ case class ObjectHashMapAccessor(@transient session: SnappySession, val nullLocalVars = if (columnVars.isEmpty) { // get nullability from object fields - fieldVars.map(e => genNullCode(s"$objVar.${e._3.isNull}", e._4)) + fieldVars.map(e => genNullCode(s"$objVar.${internals.exprCodeIsNull(e._3)}", e._4)) } else { // get nullability from already set local vars passed in columnVars - columnVars.map(_.isNull) + columnVars.map(internals.exprCodeIsNull) } fieldVars.zip(nullLocalVars).zip(resultVars).map { case (((dataType, _, @@ -1011,18 +1104,19 @@ case class ObjectHashMapAccessor(@transient session: SnappySession, if (nullIdx == -1) { // if incoming variable is null, then default will get assigned // because the variable will be initialized with the default - genVarAssignCode(objVar, resultVar, fieldVar.value, dataType, doCopy) + genVarAssignCode(objVar, resultVar, internals.exprCodeValue(fieldVar), + dataType, doCopy) } else if (nullIdx == NULL_NON_PRIM) { - val varName = fieldVar.value + val varName = internals.exprCodeValue(fieldVar) s""" - if (${resultVar.isNull}) { + if (${internals.exprCodeIsNull(resultVar)}) { $objVar.$varName = null; } else { ${genVarAssignCode(objVar, resultVar, varName, dataType, doCopy)} } """ } else { - val nullVar = fieldVar.isNull + val nullVar = internals.exprCodeIsNull(fieldVar) // when initializing the object, no need to clear null mask val nullClear = if (forInit) "" else { @@ -1033,11 +1127,11 @@ case class ObjectHashMapAccessor(@transient session: SnappySession, """ } s""" - if (${resultVar.isNull}) { + if (${internals.exprCodeIsNull(resultVar)}) { $objVar.$nullVar |= ${genNullBitMask(nullIdx)}; } else { $nullClear - ${genVarAssignCode(objVar, resultVar, fieldVar.value, + ${genVarAssignCode(objVar, resultVar, internals.exprCodeValue(fieldVar), dataType, doCopy)} } """ @@ -1053,8 +1147,8 @@ case class ObjectHashMapAccessor(@transient session: SnappySession, val consumeCode = checkCondition match { case None => consumeResult case Some(ev) => - s"""${ev.code} - if (!${ev.isNull} && ${ev.value}) { + s"""${ev.code.toString} + if (!${internals.exprCodeIsNull(ev)} && ${internals.exprCodeValue(ev)}) { $consumeResult }""" } @@ -1092,17 +1186,16 @@ case class ObjectHashMapAccessor(@transient session: SnappySession, s"""$buildInitCode if ($entryVar == null) { // set null variables for outer join in failed match - ${buildVars.map(ev => s"${ev.isNull} = true;").mkString("\n")} + ${buildVars.map(ev => s"${internals.exprCodeIsNull(ev)} = true;").mkString("\n")} } $consumeResult""" case Some(ev) => // assign null to entryVar if checkCondition fails so that it is // treated like an empty outer join match by subsequent code - s""" - ${ev.code} - if (${ev.isNull} || !${ev.value}) { + ${ev.code.toString} + if (${internals.exprCodeIsNull(ev)} || !${internals.exprCodeValue(ev)}) { if ($localValueVar.$nextValueVar != null) { continue; } @@ -1116,7 +1209,7 @@ case class ObjectHashMapAccessor(@transient session: SnappySession, // the outer join needs to be converted to inner join if ($entryVar == null || $matchFailedCompletely) { // set null variables for outer join in failed match - ${buildVars.map(ev => s"${ev.isNull} = true;").mkString("\n")} + ${buildVars.map(ev => s"${internals.exprCodeIsNull(ev)} = true;").mkString("\n")} } $consumeResult""" } @@ -1147,7 +1240,7 @@ case class ObjectHashMapAccessor(@transient session: SnappySession, case None => // no key/value assignments required - s"if ($entryVar == null) continue;\n$consumeResult" + s"if ($entryVar == null) continue;\n$inputCodes\n$consumeResult" case Some(ev) => val breakLoop = ctx.freshName("breakLoop") @@ -1162,9 +1255,9 @@ case class ObjectHashMapAccessor(@transient session: SnappySession, $breakLoop: while (true) { $checkCode do { // single iteration loop meant for breaking out with "continue" - ${ev.code} + ${ev.code.toString} // consume only one result - if (!${ev.isNull} && ${ev.value}) { + if (!${internals.exprCodeIsNull(ev)} && ${internals.exprCodeValue(ev)}) { $consumeResult break $breakLoop; } @@ -1185,7 +1278,7 @@ case class ObjectHashMapAccessor(@transient session: SnappySession, case None => // success if no match for an anti-join (no value iteration) - s"if ($entryVar != null) continue;\n$consumeResult" + s"if ($entryVar != null) continue;\n$inputCodes\n$consumeResult" case Some(ev) => val breakLoop = ctx.freshName("breakLoop") @@ -1203,8 +1296,8 @@ case class ObjectHashMapAccessor(@transient session: SnappySession, $checkCode do { // single iteration loop meant for breaking out with "continue" // fail if condition matches for any row - ${ev.code} - if (!${ev.isNull} && ${ev.value}) { + ${ev.code.toString} + if (!${internals.exprCodeIsNull(ev)} && ${internals.exprCodeValue(ev)}) { $matched = true; break $breakLoop; } @@ -1233,6 +1326,7 @@ case class ObjectHashMapAccessor(@transient session: SnappySession, case None => // only one match needed, so no value iteration s"""final boolean $existsVar = ($entryVar != null); + $inputCodes $consumeResult""" case Some(ev) => @@ -1247,8 +1341,8 @@ case class ObjectHashMapAccessor(@transient session: SnappySession, $breakLoop: while (true) { $checkCode do { // single iteration loop meant for breaking out with "continue" - ${ev.code} - if (!${ev.isNull} && ${ev.value}) { + ${ev.code.toString} + if (!${internals.exprCodeIsNull(ev)} && ${internals.exprCodeValue(ev)}) { // consume only one result $existsVar = true; break $breakLoop; @@ -1288,37 +1382,36 @@ case class ObjectHashMapAccessor(@transient session: SnappySession, // check for object field or local variable val colVar = if (varName.isEmpty) objVar else s"$objVar.$varName" - genVarAssignCode(colVar, resultVar, dataType, doCopy) + genVarAssignCode(colVar, internals.exprCodeValue(resultVar), dataType, doCopy) } - private def genVarAssignCode(colVar: String, resultVar: ExprCode, + private def genVarAssignCode(colVar: String, resultVar: String, dataType: DataType, doCopy: Boolean): String = dataType match { // if doCopy is true, then create a copy of some non-primitives that just // holds a reference to UnsafeRow bytes (and can change under the hood) case StringType if doCopy && !multiMap => - s"$colVar = ${resultVar.value}.getBytes();" + s"$colVar = $resultVar.getBytes();" case StringType if !multiMap => // copy just reference of the object if underlying byte[] is immutable - val stringVar = resultVar.value val bytes = ctx.freshName("stringBytes") s"""byte[] $bytes = null; - if ($stringVar == null || ($stringVar.getBaseOffset() == Platform.BYTE_ARRAY_OFFSET - && ($bytes = (byte[])$stringVar.getBaseObject()).length == $stringVar.numBytes())) { + if ($resultVar == null || ($resultVar.getBaseOffset() == Platform.BYTE_ARRAY_OFFSET + && ($bytes = (byte[])$resultVar.getBaseObject()).length == $resultVar.numBytes())) { $colVar = $bytes; } else { - $colVar = $stringVar.getBytes(); + $colVar = $resultVar.getBytes(); }""" // multimap holds a reference to UTF8String itself case StringType => // copy just reference of the object if underlying byte[] is immutable - ObjectHashMapAccessor.cloneStringIfRequired(resultVar.value, colVar, doCopy) + ObjectHashMapAccessor.cloneStringIfRequired(resultVar, colVar, doCopy) case _: ArrayType | _: MapType | _: StructType if doCopy => - val javaType = ctx.javaType(dataType) - s"$colVar = ($javaType)(${resultVar.value} != null ? ${resultVar.value}.copy() : null);" + val javaType = internals.javaType(dataType, ctx) + s"$colVar = ($javaType)($resultVar != null ? $resultVar.copy() : null);" case _: BinaryType if doCopy => - s"$colVar = (byte[])(${resultVar.value} != null ? ${resultVar.value}.clone() : null);" + s"$colVar = (byte[])($resultVar != null ? $resultVar.clone() : null);" case _ => - s"$colVar = ${resultVar.value};" + s"$colVar = $resultVar;" } private def genNullBitMask(nullIdx: Int): String = diff --git a/core/src/main/scala/org/apache/spark/sql/execution/SHAMapAccessor.scala b/core/src/main/scala/org/apache/spark/sql/execution/SHAMapAccessor.scala index 3ed64ae89a..ec8ca0b3e6 100644 --- a/core/src/main/scala/org/apache/spark/sql/execution/SHAMapAccessor.scala +++ b/core/src/main/scala/org/apache/spark/sql/execution/SHAMapAccessor.scala @@ -22,11 +22,10 @@ import scala.reflect.runtime.universe._ import com.gemstone.gemfire.internal.shared.{BufferSizeLimitExceededException, ClientResolverUtils} import io.snappydata.Property -import io.snappydata.collection.{ByteBufferData, SHAMap} - +import io.snappydata.collection.SHAMap import org.apache.spark.rdd.RDD -import org.apache.spark.sql.SnappySession +import org.apache.spark.sql.{SnappySession, SparkSupport} import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode} import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression, GenericInternalRow, UnsafeArrayData, UnsafeRow} @@ -44,14 +43,14 @@ case class SHAMapAccessor(@transient session: SnappySession, vdBaseObjectTerm: String, vdBaseOffsetTerm: String, nullKeysBitsetTerm: String, numBytesForNullKeyBits: Int, allocatorTerm: String, numBytesForNullAggBits: Int, - nullAggsBitsetTerm: String, sizeAndNumNotNullFuncForStringArr: String, + nullAggsBitsetTerm: String, sizeAndNumNotNullFuncForArray: String, keyBytesHolderVarTerm: String, baseKeyObject: String, baseKeyHolderOffset: String, keyExistedTerm: String, skipLenForAttribIndex: Int, codeForLenOfSkippedTerm: String, valueDataCapacityTerm: String, storedAggNullBitsTerm: Option[String], storedKeyNullBitsTerm: Option[String], aggregateBufferVars: Seq[String], keyHolderCapacityTerm: String) - extends CodegenSupport { + extends CodegenSupport with SparkSupport { private val alwaysExplode = Property.TestExplodeComplexDataTypeInSHA. get(session.sessionState.conf) @@ -84,9 +83,7 @@ case class SHAMapAccessor(@transient session: SnappySession, s"${org.apache.spark.sql.types.TypeUtilities.getClass.getName}.MODULE$$" val bigDecimalClass = classOf[java.math.BigDecimal].getName val bigIntegerClass = classOf[java.math.BigInteger].getName - val byteBufferClass = classOf[ByteBuffer].getName val unsafeClass = classOf[UnsafeRow].getName - val castTerm = SHAMapAccessor.getNullBitsCastTerm(numBytesForNullBits) dataTypes.zip(varNames).zipWithIndex.map { case ((dt, varName), i) => val nullVar = if (isKey) { if (nestingLevel == 0 && skipNullBitsCode) { @@ -152,7 +149,7 @@ case class SHAMapAccessor(@transient session: SnappySession, | $varName, ${Platform.BYTE_ARRAY_OFFSET}, $varName.length); | $currentValueOffsetTerm += $varName.length; """.stripMargin - case x: AtomicType => { + case x: AtomicType => (typeOf(x.tag) match { case t if t =:= typeOf[Boolean] => s"""$varName = $plaformClass.getBoolean($vdBaseObjectTerm, $currentValueOffsetTerm); @@ -184,23 +181,21 @@ case class SHAMapAccessor(@transient session: SnappySession, } else { val tempByteArrayTerm = ctx.freshName("tempByteArray") - val len = ctx.freshName("len") s""" - |byte[] $tempByteArrayTerm = new byte[${dt.asInstanceOf[DecimalType]. - defaultSize}]; + |byte[] $tempByteArrayTerm = + | new byte[${dt.asInstanceOf[DecimalType].defaultSize}]; |$plaformClass.copyMemory($vdBaseObjectTerm, $currentValueOffsetTerm, |$tempByteArrayTerm, ${Platform.BYTE_ARRAY_OFFSET} , $tempByteArrayTerm.length); |$varName = $bigDecimalObjectClass.apply(new $bigDecimalClass( |new $bigIntegerClass($tempByteArrayTerm), - |${dt.asInstanceOf[DecimalType].scale}, - | $typeUtiltiesObjectClass.mathContextCache()[${dt.asInstanceOf[DecimalType].precision - 1}])); + | ${dt.asInstanceOf[DecimalType].scale}, $typeUtiltiesObjectClass. + | mathContextCache()[${dt.asInstanceOf[DecimalType].precision - 1}])); """.stripMargin } case _ => throw new UnsupportedOperationException("unknown type " + dt) }) + s"""$currentValueOffsetTerm += ${dt.defaultSize};""" - } case ArrayType(elementType, containsNull) => val isExploded = ctx.freshName("isExplodedArray") val arraySize = ctx.freshName("arraySize") @@ -212,8 +207,8 @@ case class SHAMapAccessor(@transient session: SnappySession, val objectClass = classOf[Object].getName val counter = ctx.freshName("counter") val readingCodeExprs = getBufferVars(Seq(elementType), Seq(s"$objectArray[$counter]"), - currentValueOffsetTerm, true, "", -1, - true, nestingLevel) + currentValueOffsetTerm, isKey = true, "", -1, + skipNullBitsCode = true, nestingLevel) val varWidthNumNullBytes = ctx.freshName("numNullBytes") val varWidthNullBits = ctx.freshName("nullBits") val remainder = ctx.freshName("remainder") @@ -236,12 +231,12 @@ case class SHAMapAccessor(@transient session: SnappySession, |int $remainder = $counter % 8; |int $indx = $counter / 8; |if ( ($varWidthNullBits[$indx] & (0x01 << $remainder)) == 0) { - |${readingCodeExprs.map(_.code).mkString("\n")} + |${readingCodeExprs.map(_.code.toString).mkString("\n")} |} |} |} else { |for (int $counter = 0; $counter < $arraySize; ++$counter ) { - |${readingCodeExprs.map(_.code).mkString("\n")} + |${readingCodeExprs.map(_.code.toString).mkString("\n")} |} |} @@ -260,7 +255,6 @@ case class SHAMapAccessor(@transient session: SnappySession, case st: StructType => val objectArray = ctx.freshName("objectArray") val byteBufferClass = classOf[ByteBuffer].getName - val currentOffset = ctx.freshName("currentOffset") val nullBitSetTermForStruct = SHAMapAccessor.generateNullKeysBitTermForStruct( varName) val numNullKeyBytesForStruct = SHAMapAccessor.calculateNumberOfBytesForNullBits(st.length) @@ -297,9 +291,9 @@ case class SHAMapAccessor(@transient session: SnappySession, } ${ getBufferVars(st.map(_.dataType), keyVarNamesWithStructFlags.unzip._1, - currentValueOffsetTerm, true, nullBitSetTermForStruct, - numNullKeyBytesForStruct, false, nestingLevel + 1). - map(_.code).mkString("\n") + currentValueOffsetTerm, isKey = true, nullBitSetTermForStruct, + numNullKeyBytesForStruct, skipNullBitsCode = false, nestingLevel + 1). + map(_.code.toString).mkString("\n") } //add child Internal Rows to parent struct's object array ${ @@ -344,7 +338,7 @@ case class SHAMapAccessor(@transient session: SnappySession, } }""".stripMargin } - ExprCode(exprCode, nullVar, varName) + internals.newExprCode(exprCode, nullVar, varName, dt) } } @@ -372,15 +366,26 @@ case class SHAMapAccessor(@transient session: SnappySession, } } + def initKeyOrBufferVal(dataTypes: Seq[DataType], varNames: Seq[String], + genClassField: Boolean = false): String = { + dataTypes.zip(varNames).map { case (dt, varName) => + if (genClassField) { + internals.addClassField(ctx, internals.javaType(dt, ctx), varName, + forceInline = true, useFreshName = false) + s"$varName = ${internals.defaultValue(dt, ctx)};" + } else s"${internals.javaType(dt, ctx)} $varName = ${internals.defaultValue(dt, ctx)};" + }.mkString("\n") + } - def initKeyOrBufferVal(dataTypes: Seq[DataType], varNames: Seq[String]): - String = dataTypes.zip(varNames).map { case (dt, varName) => - s"${ctx.javaType(dt)} $varName = ${ctx.defaultValue(dt)};" - }.mkString("\n") + def declareNullVarsForAggBuffer(varNames: Seq[String], genClassField: Boolean = false): String = + varNames.map { varName => + if (genClassField) { + internals.addClassField(ctx, "boolean", s"$varName${SHAMapAccessor.nullVarSuffix}", + forceInline = true, useFreshName = false) + s"$varName${SHAMapAccessor.nullVarSuffix} = false;" + } else s"boolean $varName${SHAMapAccessor.nullVarSuffix} = false;" + }.mkString("\n") - def declareNullVarsForAggBuffer(varNames: Seq[String]): String = - varNames.map(varName => s"boolean ${varName}${SHAMapAccessor.nullVarSuffix} = false;"). - mkString("\n") /** * Generate code to lookup the map or insert a new key, value if not found. */ @@ -391,10 +396,9 @@ case class SHAMapAccessor(@transient session: SnappySession, val tempValueData = ctx.freshName("tempValueData") val linkedListClass = classOf[java.util.LinkedList[SHAMap]].getName val exceptionName = classOf[BufferSizeLimitExceededException].getName - val bbDataClass = classOf[ByteBufferData].getName val shaMapClassName = classOf[SHAMap].getName // val valueInit = valueInitCode + '\n' - val insertDoneTerm = ctx.freshName("insertDone"); + val insertDoneTerm = ctx.freshName("insertDone") /* generateUpdate(objVar, Nil, valueInitVars, forKey = false, doCopy = false) */ @@ -406,11 +410,11 @@ case class SHAMapAccessor(@transient session: SnappySession, // evaluate key vars |${evaluateVariables(keyVars)} |${keyVars.zip(keysDataType).filter(_._2 match { - case x: StructType => true + case _: StructType => true case _ => false }).map { - case (exprCode, dt) => explodeStruct(exprCode.value, exprCode.isNull, - dt.asInstanceOf[StructType]) + case (exprCode, dt) => explodeStruct(internals.exprCodeValue(exprCode), + internals.exprCodeIsNull(exprCode), dt.asInstanceOf[StructType]) }.mkString("\n") } // evaluate hash code of the lookup key @@ -446,9 +450,9 @@ case class SHAMapAccessor(@transient session: SnappySession, |} catch ($exceptionName bsle) { |$overflowHashMapsTerm = new $linkedListClass<$shaMapClassName>(); |$overflowHashMapsTerm.add($hashMapTerm); - |$hashMapTerm = new $shaMapClassName(${Property.initialCapacityOfSHABBMap.get(session.sessionState.conf)}, - |$keyValSize, - |${Property.ApproxMaxCapacityOfBBMap.get(session.sessionState.conf)}); + |$hashMapTerm = new $shaMapClassName( + | ${Property.initialCapacityOfSHABBMap.get(session.sessionState.conf)}, $keyValSize, + | ${Property.ApproxMaxCapacityOfBBMap.get(session.sessionState.conf)}); |$overflowHashMapsTerm.add($hashMapTerm); |$valueOffsetTerm = $hashMapTerm.putBufferIfAbsent($baseKeyObject, |$baseKeyHolderOffset, $numKeyBytesTerm, $numValueBytes + $numKeyBytesTerm, @@ -481,9 +485,9 @@ case class SHAMapAccessor(@transient session: SnappySession, |} |} |if (!$insertDoneTerm) { - |$hashMapTerm = new $shaMapClassName(${Property.initialCapacityOfSHABBMap.get(session.sessionState.conf)}, - | $keyValSize, - | ${Property.ApproxMaxCapacityOfBBMap.get(session.sessionState.conf)}); + |$hashMapTerm = new $shaMapClassName( + | ${Property.initialCapacityOfSHABBMap.get(session.sessionState.conf)}, $keyValSize, + | ${Property.ApproxMaxCapacityOfBBMap.get(session.sessionState.conf)}); |$overflowHashMapsTerm.add($hashMapTerm); |$valueOffsetTerm = $hashMapTerm.putBufferIfAbsent($baseKeyObject, |$baseKeyHolderOffset, $numKeyBytesTerm, $numValueBytes + $numKeyBytesTerm, @@ -537,7 +541,7 @@ case class SHAMapAccessor(@transient session: SnappySession, s"""|boolean $nullVarName = $structNullVarName || | (!$alwaysExplode && $structVarName instanceof $unsafeRowClass) || | $structVarName.isNullAt($index); - | ${ctx.javaType(dt)} $varName = ${ctx.defaultValue(dt)}; + | ${internals.javaType(dt, ctx)} $varName = ${internals.defaultValue(dt, ctx)}; | if ($alwaysExplode|| !($structVarName instanceof $unsafeRowClass)) { |if (!$nullVarName) { |$varName = $valueExtractCode; @@ -555,7 +559,7 @@ case class SHAMapAccessor(@transient session: SnappySession, ${ SHAMapAccessor.initNullBitsetCode( SHAMapAccessor.generateNullKeysBitTermForStruct(structVarName), - SHAMapAccessor.calculateNumberOfBytesForNullBits(structType.length)) + SHAMapAccessor.calculateNumberOfBytesForNullBits(structType.length), ctx) } $explodedStructCode """.stripMargin @@ -567,7 +571,6 @@ case class SHAMapAccessor(@transient session: SnappySession, def generateUpdate(bufferVars: Seq[ExprCode], aggBufferDataType: Seq[DataType]): String = { - val plaformClass = classOf[Platform].getName val setStoredAggNullBitsTerm = storedAggNullBitsTerm.map(storedNullBit => { s"""// If key did not exist, make cachedAggBit -1 , so that the update will always write // the right state of agg bit , else it will be that stored Agg Bit will match the @@ -587,14 +590,11 @@ case class SHAMapAccessor(@transient session: SnappySession, ${ writeKeyOrValue(vdBaseObjectTerm, currentOffSetForMapLookupUpdt, aggBufferDataType, bufferVars, nullAggsBitsetTerm, numBytesForNullAggBits, - false, false) + isKey = false, skipNullEvalCode = false) } """.stripMargin - } - - def writeKeyOrValue(baseObjectTerm: String, offsetTerm: String, dataTypes: Seq[DataType], varsToWrite: Seq[ExprCode], nullBitsTerm: String, numBytesForNullBits: Int, isKey: Boolean, skipNullEvalCode: Boolean, @@ -616,7 +616,7 @@ case class SHAMapAccessor(@transient session: SnappySession, s"""$storeNullBitStartOffsetAndRepositionOffset |${dataTypes.zip(varsToWrite).zipWithIndex.map { case ((dt, expr), i) => - val variable = expr.value + val variable = internals.exprCodeValue(expr) val writingCode = (dt match { case x: AtomicType => val snippet = typeOf(x.tag) match { @@ -711,8 +711,8 @@ case class SHAMapAccessor(@transient session: SnappySession, |$offsetTerm += 1; |${ writeKeyOrValue(baseObjectTerm, offsetTerm, childDataTypes, childExprCodes, - newNullBitTerm, newNumBytesForNullBits, true, false, - nestingLevel + 1) + newNullBitTerm, newNumBytesForNullBits, isKey = true, + skipNullEvalCode = false, nestingLevel + 1) } """.stripMargin val unexplodedStructSnippet = @@ -736,13 +736,11 @@ case class SHAMapAccessor(@transient session: SnappySession, } - case at@ArrayType(elementType, containsNull) => + case ArrayType(elementType, containsNull) => val varWidthNullBitStartPos = ctx.freshName("nullBitBeginPos") val varWidthNumNullBytes = ctx.freshName("numNullBytes") val varWidthNullBits = ctx.freshName("nullBits") val arrElement = ctx.freshName("arrElement") - val tempObj = ctx.freshName("temp") - val array = ctx.freshName("array") val counter = ctx.freshName("counter") val remainder = ctx.freshName("remainder") val arrIndex = ctx.freshName("arrIndex") @@ -751,8 +749,9 @@ case class SHAMapAccessor(@transient session: SnappySession, val dataType = ctx.freshName("dataType") val dataTypeClass = classOf[DataType].getName val elementWitingCode = writeKeyOrValue(baseObjectTerm, offsetTerm, Seq(elementType), - Seq(ExprCode("", "false", arrElement)), "", -1, - true, true, nestingLevel) + Seq(internals.newExprCode("", "false", arrElement, elementType)), "", -1, + isKey = true, skipNullEvalCode = true, nestingLevel) + val elType = internals.javaType(elementType, ctx) val explodeArraySnippet = s"""|$plaformClass.putBoolean($baseObjectTerm, $offsetTerm, true); |$offsetTerm += 1; @@ -782,8 +781,8 @@ case class SHAMapAccessor(@transient session: SnappySession, | throw new IllegalStateException("Not null Array element contains null"); |} |} else { - |${ctx.javaType(elementType)} $arrElement = - |(${ctx.boxedType(elementType)}) $variable.get($counter, $dataType); + |$elType $arrElement = + |(${internals.boxedType(elType, ctx)}) $variable.get($counter, $dataType); |$elementWitingCode |} |} @@ -883,9 +882,8 @@ case class SHAMapAccessor(@transient session: SnappySession, long $currentOffset = $baseKeyHolderOffset; // first write key data - ${ writeKeyOrValue(baseKeyObject, currentOffset, keysDataType, keyVars, - nullKeysBitsetTerm, numBytesForNullKeyBits, true, numBytesForNullKeyBits == 0) - } + ${writeKeyOrValue(baseKeyObject, currentOffset, keysDataType, keyVars, nullKeysBitsetTerm, + numBytesForNullKeyBits, isKey = true, skipNullEvalCode = numBytesForNullKeyBits == 0)} // write value data ${"" /* writeKeyOrValue(baseKeyObject, currentOffset, aggregatesDataType, valueInitVars, nullAggsBitsetTerm, numBytesForNullAggBits, false, false) */ @@ -920,35 +918,37 @@ case class SHAMapAccessor(@transient session: SnappySession, val unsafeArrayDataClass = classOf[UnsafeArrayData].getName keysDataType.zip(keyVars).zipWithIndex.map { case ((dt, expr), i) => - val nullVar = expr.isNull + val nullVar = internals.exprCodeIsNull(expr) val notNullSizeExpr = if (TypeUtilities.isFixedWidth(dt)) { dt.defaultSize.toString } else { + val exprValue = internals.exprCodeValue(expr) dt match { case StringType => - val strPart = s"${expr.value}.numBytes()" + val strPart = s"$exprValue.numBytes()" if (nestingLevel == 0 && i == skipLenForAttribIndex) { strPart } else { s"($strPart + 4)" } - case BinaryType => s"(${expr.value}.length + 4) " - case st: StructType => val (childKeysVars, childDataTypes) = - getExplodedExprCodeAndDataTypeForStruct(expr.value, st, nestingLevel) + case BinaryType => s"(${internals.exprCodeValue(expr)}.length + 4) " + case st: StructType => + val (childKeysVars, childDataTypes) = + getExplodedExprCodeAndDataTypeForStruct(exprValue, st, nestingLevel) val explodedStructSizeCode = generateKeySizeCode(childKeysVars, childDataTypes, SHAMapAccessor.calculateNumberOfBytesForNullBits(st.length), nestingLevel + 1) - val unexplodedStructSizeCode = s"(($unsafeRowClass) ${expr.value}).getSizeInBytes() + 4" + val unexplodedStructSizeCode = s"(($unsafeRowClass) $exprValue).getSizeInBytes() + 4" "1 + " + (if (alwaysExplode) { explodedStructSizeCode } else { - s"""(${expr.value} instanceof $unsafeRowClass ? $unexplodedStructSizeCode + s"""($exprValue instanceof $unsafeRowClass ? $unexplodedStructSizeCode |: $explodedStructSizeCode) """.stripMargin } ) - case at@ArrayType(elementType, containsNull) => + case ArrayType(elementType, containsNull) => // The array serialization format is following /** * Boolean (exploded or not) @@ -971,18 +971,18 @@ case class SHAMapAccessor(@transient session: SnappySession, (false, 0) } val snippetNullBitsSizeCode = - s"""${expr.value}.numElements()/8 + (${expr.value}.numElements() % 8 > 0 ? 1 : 0) + s"""$exprValue.numElements()/8 + ($exprValue.numElements() % 8 > 0 ? 1 : 0) """.stripMargin - val snippetNotNullFixedWidth = s"4 + ${expr.value}.numElements() * $unitSize" + val snippetNotNullFixedWidth = s"4 + $exprValue.numElements() * $unitSize" val snippetNotNullVarWidth = - s"""4 + (int)($sizeAndNumNotNullFuncForStringArr(${expr.value}, true) >>> 32L) + s"""4 + (int)($sizeAndNumNotNullFuncForArray($exprValue, true) >>> 32L) """.stripMargin val snippetNullVarWidth = s" $snippetNullBitsSizeCode + $snippetNotNullVarWidth" val snippetNullFixedWidth = s"""4 + $snippetNullBitsSizeCode + - |$unitSize * (int)($sizeAndNumNotNullFuncForStringArr( - |${expr.value}, false) & 0xffffffffL) + |$unitSize * (int)($sizeAndNumNotNullFuncForArray( + |$exprValue, false) & 0xffffffffL) """.stripMargin "( 1 + " + (if (alwaysExplode) { @@ -1000,8 +1000,8 @@ case class SHAMapAccessor(@transient session: SnappySession, } } } else { - s"""(${expr.value} instanceof $unsafeArrayDataClass ? - |(($unsafeArrayDataClass) ${expr.value}).getSizeInBytes() + 4 + s"""($exprValue instanceof $unsafeArrayDataClass ? + |(($unsafeArrayDataClass) $exprValue).getSizeInBytes() + 4 |: ${ if (isFixedWidth) { s"""$containsNull ? ($snippetNullFixedWidth) |: ($snippetNotNullFixedWidth)) @@ -1029,12 +1029,9 @@ case class SHAMapAccessor(@transient session: SnappySession, nestingLevel: Int): (Seq[ExprCode], Seq[DataType]) = st.zipWithIndex.map { case (sf, index) => val (varName, nullVarName) = SHAMapAccessor.generateExplodedStructFieldVars(parentStructVarName, nestingLevel, index) - ExprCode("", nullVarName, varName) -> sf.dataType + internals.newExprCode("", nullVarName, varName, sf.dataType) -> sf.dataType }.unzip - - - /** * Generate code to calculate the hash code for given column variables that * correspond to the key columns in this class. @@ -1046,7 +1043,7 @@ case class SHAMapAccessor(@transient session: SnappySession, val hashDeclaration = if (skipDeclaration) "" else s"int $hash = 0;\n" // check if hash has already been generated for keyExpressions var doRegister = register - val vars = keyVars.map(_.value) + val vars = keyVars.map(internals.exprCodeValue) val (prefix, suffix) = session.getHashVar(ctx, vars) match { case Some(h) => hashVar(0) = h @@ -1063,8 +1060,8 @@ case class SHAMapAccessor(@transient session: SnappySession, // optimize for first column to use fast hashing val expr = keyVars.head - val colVar = expr.value - val nullVar = expr.isNull + val nullVar = internals.exprCodeIsNull(expr) + val colVar = internals.exprCodeValue(expr) val firstColumnHash = keysDataType.head match { case BooleanType => hashSingleInt(s"($colVar) ? 1 : 0", nullVar, hash) @@ -1088,24 +1085,25 @@ case class SHAMapAccessor(@transient session: SnappySession, hashSingleInt(s"$colVar.hashCode()", nullVar, hash) } if (keyVars.length > 1) { - keysDataType.tail.zip(keyVars.tail).map { - case (BooleanType, ev) => - addHashInt(s"${ev.value} ? 1 : 0", ev.isNull, hash) - case (ByteType | ShortType | IntegerType | DateType, ev) => - addHashInt(ev.value, ev.isNull, hash) - case (BinaryType, ev) => - hashBinary(ev.value, ev.isNull, hash) - case (LongType | TimestampType, ev) => - addHashLong(ctx, ev.value, ev.isNull, hash) - case (FloatType, ev) => - addHashInt(s"Float.floatToIntBits(${ev.value})", ev.isNull, hash) - case (DoubleType, ev) => - addHashLong(ctx, s"Double.doubleToLongBits(${ev.value})", ev.isNull, + keysDataType.tail.zip(keyVars.tail).map(p => (p._1, internals.exprCodeIsNull(p._2), + internals.exprCodeValue(p._2))).map { + case (BooleanType, evIsNull, evValue) => + addHashInt(s"$evValue ? 1 : 0", evIsNull, hash) + case (ByteType | ShortType | IntegerType | DateType, evIsNull, evValue) => + addHashInt(evValue, evIsNull, hash) + case (BinaryType, evIsNull, evValue) => + hashBinary(evValue, evIsNull, hash) + case (LongType | TimestampType, evIsNull, evValue) => + addHashLong(ctx, evValue, evIsNull, hash) + case (FloatType, evIsNull, evValue) => + addHashInt(s"Float.floatToIntBits($evValue)", evIsNull, hash) + case (DoubleType, evIsNull, evValue) => + addHashLong(ctx, s"Double.doubleToLongBits($evValue)", evIsNull, hash) - case (_: DecimalType, ev) => - addHashInt(s"${ev.value}.fastHashCode()", ev.isNull, hash) - case (_, ev) => - addHashInt(s"${ev.value}.hashCode()", ev.isNull, hash) + case (_: DecimalType, evIsNull, evValue) => + addHashInt(s"$evValue.fastHashCode()", evIsNull, hash) + case (_, evIsNull, evValue) => + addHashInt(s"$evValue.hashCode()", evIsNull, hash) }.mkString(prefix + firstColumnHash, "", suffix) } else prefix + firstColumnHash + suffix } @@ -1179,37 +1177,51 @@ case class SHAMapAccessor(@transient session: SnappySession, } -object SHAMapAccessor { +object SHAMapAccessor extends SparkSupport { val nullVarSuffix = "_isNull" - val supportedDataTypes: DataType => Boolean = dt => - dt match { - case _: MapType => false - case _: UserDefinedType[_] => false - case CalendarIntervalType => false - case NullType => false - case _: ObjectType => false - case ArrayType(elementType, _) => elementType match { - case _: StructType => false - case _ => true - } + val supportedDataTypes: DataType => Boolean = { + case _: MapType => false + case _: UserDefinedType[_] => false + case CalendarIntervalType => false + case NullType => false + case _: ObjectType => false + case ArrayType(elementType, _) => elementType match { + case _: StructType => false case _ => true - - // includes atomic types, string type, array type - // ( depends on element type) , struct type ( depends on fields) } + case _ => true + // includes atomic types, string type, array type + // ( depends on element type) , struct type ( depends on fields) + } - def initNullBitsetCode(nullBitsetTerm: String, - numBytesForNullBits: Int): String = if (numBytesForNullBits == 0) { + def initNullBitsetCode(nullBitsetTerm: String, numBytesForNullBits: Int, ctx: CodegenContext, + genClassField: Boolean = false): String = if (numBytesForNullBits == 0) { "" } else if (numBytesForNullBits == 1) { - s"byte $nullBitsetTerm = 0;" + if (genClassField) { + internals.addClassField(ctx, "byte", nullBitsetTerm, + forceInline = true, useFreshName = false) + s"$nullBitsetTerm = 0;" + } else s"byte $nullBitsetTerm = 0;" } else if (numBytesForNullBits == 2) { - s"short $nullBitsetTerm = 0;" + if (genClassField) { + internals.addClassField(ctx, "short", nullBitsetTerm, + forceInline = true, useFreshName = false) + s"$nullBitsetTerm = 0;" + } else s"short $nullBitsetTerm = 0;" } else if (numBytesForNullBits <= 4) { - s"int $nullBitsetTerm = 0;" + if (genClassField) { + internals.addClassField(ctx, "int", nullBitsetTerm, + forceInline = true, useFreshName = false) + s"$nullBitsetTerm = 0;" + } else s"int $nullBitsetTerm = 0;" } else if (numBytesForNullBits <= 8) { - s"long $nullBitsetTerm = 0l;" + if (genClassField) { + internals.addClassField(ctx, "long", nullBitsetTerm, + forceInline = true, useFreshName = false) + s"$nullBitsetTerm = 0L;" + } else s"long $nullBitsetTerm = 0L;" } else { s""" |for( int i = 0 ; i < $numBytesForNullBits; ++i) { @@ -1233,7 +1245,10 @@ object SHAMapAccessor { def calculateNumberOfBytesForNullBits(numAttributes: Int): Int = (numAttributes + 7 )/ 8 - def generateNullKeysBitTermForStruct(structName: String): String = s"${structName}_nullKeysBitset" + def generateNullKeysBitTermForStruct(structName: String): String = { + if (structName.indexOf('[') == -1) s"${structName}_nullKeysBitset" + else s"${structName.replace('[', '_').replace(']', '_')}_nullKeysBitset" + } def generateVarNameForStructField(parentVar: String, nestingLevel: Int, index: Int): String = s"${parentVar}_${nestingLevel}_$index" @@ -1294,7 +1309,7 @@ object SHAMapAccessor { i: Int, nullBitsTerm: String, offsetTerm: String, dt: DataType, isKey: Boolean, writingCodeToEmbed: String): String = { val castTerm = SHAMapAccessor.getNullBitsCastTerm(numBytesForNullBits) - val nullVar = expr.isNull + val nullVar = internals.exprCodeIsNull(expr) if (numBytesForNullBits > 8) { val remainder = i % 8 val index = i / 8 @@ -1349,5 +1364,4 @@ object SHAMapAccessor { } } } - -} \ No newline at end of file +} diff --git a/core/src/main/scala/org/apache/spark/sql/execution/SnappySortExec.scala b/core/src/main/scala/org/apache/spark/sql/execution/SnappySortExec.scala index 0eb5943b56..49643a24fa 100644 --- a/core/src/main/scala/org/apache/spark/sql/execution/SnappySortExec.scala +++ b/core/src/main/scala/org/apache/spark/sql/execution/SnappySortExec.scala @@ -16,13 +16,14 @@ */ package org.apache.spark.sql.execution +import scala.collection.AbstractIterator + import org.apache.spark.TaskContext import org.apache.spark.rdd.RDD import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode} import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeSet, SortOrder, UnsafeRow} import org.apache.spark.sql.catalyst.plans.physical.{Distribution, Partitioning} -import org.apache.spark.sql.catalyst.util.AbstractScalaRowIterator import org.apache.spark.sql.execution.metric.SQLMetric /** @@ -55,9 +56,9 @@ case class SnappySortExec(sortPlan: SortExec, child: SparkPlan) child.execute().mapPartitionsPreserveInternal(itr => - new AbstractScalaRowIterator[UnsafeRow] { + new AbstractIterator[InternalRow] { - private lazy val sortedIterator: AbstractScalaRowIterator[UnsafeRow] = { + private lazy val sortedIterator: Iterator[InternalRow] = { val sorter = sortPlan.createSorter() val metrics = TaskContext.get().taskMetrics() // Remember spill data size of this task before execute this operator so that we can @@ -68,12 +69,12 @@ case class SnappySortExec(sortPlan: SortExec, child: SparkPlan) peakMemory += sorter.getPeakMemoryUsage spillSize += metrics.memoryBytesSpilled - spillSizeBefore metrics.incPeakExecutionMemory(sorter.getPeakMemoryUsage) - sortedIterator.asInstanceOf[AbstractScalaRowIterator[UnsafeRow]] + sortedIterator } override def hasNext: Boolean = sortedIterator.hasNext - override def next(): UnsafeRow = sortedIterator.next() + override def next(): InternalRow = sortedIterator.next() }) } diff --git a/core/src/main/scala/org/apache/spark/sql/execution/TableExec.scala b/core/src/main/scala/org/apache/spark/sql/execution/TableExec.scala index 542769c2c2..78ed2c6c36 100644 --- a/core/src/main/scala/org/apache/spark/sql/execution/TableExec.scala +++ b/core/src/main/scala/org/apache/spark/sql/execution/TableExec.scala @@ -25,15 +25,16 @@ import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, import org.apache.spark.sql.catalyst.plans.physical._ import org.apache.spark.sql.collection.{SmartExecutorBucketPartition, Utils} import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics} -import org.apache.spark.sql.sources.{DestroyRelation, JdbcExtendedUtils, NativeTableRowLevelSecurityRelation} +import org.apache.spark.sql.sources.{DestroyRelation, JdbcExtendedUtils, SnappyTableRelation} import org.apache.spark.sql.store.StoreUtils import org.apache.spark.sql.types.{LongType, StructType} -import org.apache.spark.sql.{DelegateRDD, SnappyContext, SnappySession, ThinClientConnectorMode} +import org.apache.spark.sql.{DelegateRDD, SnappyContext, SnappySession, SparkSupport, ThinClientConnectorMode} /** * Base class for bulk insert/mutation operations for column and row tables. */ -trait TableExec extends UnaryExecNode with CodegenSupportOnExecutor { +trait TableExec extends UnaryExecNode with CodegenSupportOnExecutor + with NonRecursivePlans with SparkSupport { def partitionColumns: Seq[String] @@ -59,7 +60,7 @@ trait TableExec extends UnaryExecNode with CodegenSupportOnExecutor { if (!onExecutor) { val catalogVersion: Option[Long] = Utils.executeIfSmartConnector(sqlContext.sparkContext) { relation match { - case Some(r: NativeTableRowLevelSecurityRelation) => r.relationInfo.catalogSchemaVersion + case Some(r: SnappyTableRelation) => r.relationInfo.catalogSchemaVersion case _ => -1 } @@ -79,7 +80,7 @@ trait TableExec extends UnaryExecNode with CodegenSupportOnExecutor { // Only one insert plan possible in the plan tree, so no clashes. if (partitioned) { val session = sqlContext.sparkSession.asInstanceOf[SnappySession] - session.sessionState.conf.setExecutionShufflePartitions(numBuckets) + session.snappySessionState.snappyConf.setExecutionShufflePartitions(numBuckets) } /** Specifies how data is partitioned for the table. */ @@ -111,11 +112,6 @@ trait TableExec extends UnaryExecNode with CodegenSupportOnExecutor { s"number of ${opType.toLowerCase} rows")) } - override protected def doExecute(): RDD[InternalRow] = { - // don't expect code generation to fail - WholeStageCodegenExec(this).execute() - } - override def inputRDDs(): Seq[RDD[InternalRow]] = { val inputRDDs = child.asInstanceOf[CodegenSupport].inputRDDs() if (partitioned) { @@ -157,6 +153,7 @@ trait TableExec extends UnaryExecNode with CodegenSupportOnExecutor { }) locations } + inputRDDs.map { rdd => // if the two are different then its partition pruning case if (numBuckets == rdd.getNumPartitions) { @@ -174,15 +171,15 @@ trait TableExec extends UnaryExecNode with CodegenSupportOnExecutor { case _ => throw new UnsupportedOperationException( s"Expected a child supporting code generation. Got: $child") } - if (!ctx.addedFunctions.contains("shouldStop")) { + if (!internals.isFunctionAddedToOuterClass(ctx, "shouldStop")) { // no need to stop in iteration at any point - ctx.addNewFunction("shouldStop", + internals.addFunction(ctx, "shouldStop", s""" |@Override |protected final boolean shouldStop() { | return false; |} - """.stripMargin) + """.stripMargin, inlineToOuterClass = true) } childProduce } diff --git a/core/src/main/scala/org/apache/spark/sql/execution/aggregate/CollectAggregateExec.scala b/core/src/main/scala/org/apache/spark/sql/execution/aggregate/CollectAggregateExec.scala index e0f3793739..629ac65acb 100644 --- a/core/src/main/scala/org/apache/spark/sql/execution/aggregate/CollectAggregateExec.scala +++ b/core/src/main/scala/org/apache/spark/sql/execution/aggregate/CollectAggregateExec.scala @@ -19,20 +19,19 @@ package org.apache.spark.sql.execution.aggregate import scala.collection.mutable.ArrayBuffer import org.apache.spark.rdd.RDD -import org.apache.spark.sql.CachedDataFrame import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.Attribute -import org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator import org.apache.spark.sql.catalyst.plans.physical.{Distribution, UnspecifiedDistribution} -import org.apache.spark.sql.execution.{BufferedRowIterator, InputAdapter, PlanLater, SparkPlan, UnaryExecNode, WholeStageCodegenExec} +import org.apache.spark.sql.execution.{BufferedRowIterator, InputAdapter, PlanLater, SparkPlan, UnaryExecNode} import org.apache.spark.sql.hive.SnappySessionState +import org.apache.spark.sql.{CachedDataFrame, SparkSupport} /** * Special plan to collect top-level aggregation on driver itself and avoid * an exchange for simple aggregates. */ case class CollectAggregateExec(child: SparkPlan)( - @transient val basePlan: SnappyHashAggregateExec) extends UnaryExecNode { + @transient val basePlan: SnappyHashAggregateExec) extends UnaryExecNode with SparkSupport { override val output: Seq[Attribute] = basePlan.output @@ -49,13 +48,13 @@ case class CollectAggregateExec(child: SparkPlan)( // temporarily switch producer to an InputAdapter for rows as normal // Iterator[UnsafeRow] which will be set explicitly in executeCollect() basePlan.childProducer = InputAdapter(child) - val (ctx, cleanedSource) = WholeStageCodegenExec(basePlan).doCodeGen() + val (ctx, cleanedSource) = internals.newWholeStagePlan(basePlan).doCodeGen() basePlan.childProducer = child (cleanedSource, ctx.references.toArray) } @transient private[sql] lazy val generatedClass = { - CodeGenerator.compile(generatedSource) + internals.compile(generatedSource) } /** diff --git a/core/src/main/scala/org/apache/spark/sql/execution/aggregate/SnappyHashAggregateExec.scala b/core/src/main/scala/org/apache/spark/sql/execution/aggregate/SnappyHashAggregateExec.scala index 4e2e384e91..969cf6fe21 100644 --- a/core/src/main/scala/org/apache/spark/sql/execution/aggregate/SnappyHashAggregateExec.scala +++ b/core/src/main/scala/org/apache/spark/sql/execution/aggregate/SnappyHashAggregateExec.scala @@ -55,7 +55,7 @@ import org.apache.spark.sql.execution._ import org.apache.spark.sql.execution.columnar.encoding.ColumnEncoding import org.apache.spark.sql.execution.metric.SQLMetrics import org.apache.spark.sql.types._ -import org.apache.spark.sql.{SnappySession, collection} +import org.apache.spark.sql.{SnappySession, SparkSupport, collection} import org.apache.spark.unsafe.Platform import org.apache.spark.unsafe.types.UTF8String import org.apache.spark.util.Utils @@ -77,7 +77,7 @@ case class SnappyHashAggregateExec( __resultExpressions: Seq[NamedExpression], child: SparkPlan, hasDistinct: Boolean) - extends NonRecursivePlans with UnaryExecNode with BatchConsumer { + extends NonRecursivePlans with UnaryExecNode with BatchConsumer with SparkSupport { val useByteBufferMapBasedAggregation: Boolean = { val conf = sqlContext.sparkSession.sessionState.conf @@ -96,6 +96,8 @@ case class SnappyHashAggregateExec( override def nodeName: String = if (useByteBufferMapBasedAggregation) "BufferMapHashAggregate" else "SnappyHashAggregate" + override def needCopyResult: Boolean = false + @transient def resultExpressions: Seq[NamedExpression] = __resultExpressions @transient lazy private[this] val aggregateBufferAttributes = { @@ -162,29 +164,24 @@ case class SnappyHashAggregateExec( case g: GroupAggregate => g.aggBufferAttributesForGroup case sum: Sum if !sum.child.nullable => val sumAttr = sum.aggBufferAttributes.head - sumAttr.copy(nullable = false)(sumAttr.exprId, sumAttr.qualifier, - sumAttr.isGenerated) :: Nil + internals.toAttributeReference(sumAttr)(nullable = false) :: Nil case avg: Average if !avg.child.nullable => val sumAttr = avg.aggBufferAttributes.head - sumAttr.copy(nullable = false)(sumAttr.exprId, sumAttr.qualifier, - sumAttr.isGenerated) :: avg.aggBufferAttributes(1) :: Nil + internals.toAttributeReference(sumAttr)(nullable = false) :: + avg.aggBufferAttributes(1) :: Nil case max: Max if !max.child.nullable => val maxAttr = max.aggBufferAttributes.head - maxAttr.copy(nullable = false)(maxAttr.exprId, maxAttr.qualifier, - maxAttr.isGenerated) :: Nil + internals.toAttributeReference(maxAttr)(nullable = false) :: Nil case min: Min if !min.child.nullable => val minAttr = min.aggBufferAttributes.head - minAttr.copy(nullable = false)(minAttr.exprId, minAttr.qualifier, - minAttr.isGenerated) :: Nil + internals.toAttributeReference(minAttr)(nullable = false) :: Nil case last: Last if !last.child.nullable => val lastAttr = last.aggBufferAttributes.head val tail = if (last.aggBufferAttributes.length == 2) { val valueSetAttr = last.aggBufferAttributes(1) - valueSetAttr.copy(nullable = false)(valueSetAttr.exprId, - valueSetAttr.qualifier, valueSetAttr.isGenerated) :: Nil + internals.toAttributeReference(valueSetAttr)(nullable = false) :: Nil } else Nil - lastAttr.copy(nullable = false)(lastAttr.exprId, lastAttr.qualifier, - lastAttr.isGenerated) :: tail + internals.toAttributeReference(lastAttr)(nullable = false) :: tail case _ => aggregate.aggBufferAttributes } @@ -272,7 +269,7 @@ case class SnappyHashAggregateExec( // this array will be used at batch level for grouping if possible dictionaryArrayTerm = ctx.freshName("dictionaryArray") dictionaryArrayInit = ctx.freshName("dictionaryArrayInit") - ctx.addNewFunction(dictionaryArrayInit, + dictionaryArrayInit = internals.addFunction(ctx, dictionaryArrayInit, s""" |private $className[] $dictionaryArrayInit() { | return null; @@ -282,30 +279,13 @@ case class SnappyHashAggregateExec( } } - override def beforeStop(ctx: CodegenContext, plan: SparkPlan, - input: Seq[ExprCode]): String = { - if (bufVars eq null) "" - else { - bufVarUpdates = bufVars.indices.map { i => - val ev = bufVars(i) - s""" - |// update the member result variables from local variables - |this.${ev.isNull} = ${ev.isNull}; - |this.${ev.value} = ${ev.value}; - """.stripMargin - }.mkString("\n").trim - bufVarUpdates - } - } - // The variables used as aggregation buffer @transient protected var bufVars: Seq[ExprCode] = _ // code to update buffer variables with current values @transient protected var bufVarUpdates: String = _ private def doProduceWithoutKeys(ctx: CodegenContext): String = { - val initAgg = ctx.freshName("initAgg") - ctx.addMutableState("boolean", initAgg, s"$initAgg = false;") + val initAgg = internals.addClassField(ctx, "boolean", "initAgg", v => s"$v = false;") // generate variables for aggregation buffer val functions = aggregateExpressions.map(_.aggregateFunction @@ -314,20 +294,18 @@ case class SnappyHashAggregateExec( ctx.INPUT_ROW = null ctx.currentVars = null bufVars = initExpr.map { e => - val isNull = ctx.freshName("bufIsNull") - val value = ctx.freshName("bufValue") - ctx.addMutableState("boolean", isNull, "") - ctx.addMutableState(ctx.javaType(e.dataType), value, "") + val isNull = internals.addClassField(ctx, "boolean", "bufIsNull") + val value = internals.addClassField(ctx, internals.javaType(e.dataType, ctx), "bufValue") // The initial expression should not access any column val ev = e.genCode(ctx) val initVars = s""" - | $isNull = ${ev.isNull}; - | $value = ${ev.value}; + | $isNull = ${internals.exprCodeIsNull(ev)}; + | $value = ${internals.exprCodeValue(ev)}; """.stripMargin - ExprCode(ev.code + initVars, isNull, value) + internals.newExprCode(ev.code.toString + initVars, isNull, value, e.dataType) } - var initBufVar = evaluateVariables(bufVars) + val initBufVar = evaluateVariables(bufVars) // generate variables for output val (resultVars, genResult) = if (modes.contains(Final) || @@ -357,21 +335,10 @@ case class SnappyHashAggregateExec( (resultVars, evaluateVariables(resultVars)) } - val doAgg = ctx.freshName("doAggregateWithoutKey") - var produceOutput = getChildProducer.asInstanceOf[CodegenSupport].produce( + var doAgg = ctx.freshName("doAggregateWithoutKey") + val produceOutput = getChildProducer.asInstanceOf[CodegenSupport].produce( ctx, this) - if (bufVarUpdates ne null) { - // use local variables while member variables are updated at the end - initBufVar = bufVars.indices.map { i => - val ev = bufVars(i) - s""" - |boolean ${ev.isNull} = this.${ev.isNull}; - |${ctx.javaType(initExpr(i).dataType)} ${ev.value} = this.${ev.value}; - """.stripMargin - }.mkString("", "\n", initBufVar).trim - produceOutput = s"$produceOutput\n$bufVarUpdates" - } - ctx.addNewFunction(doAgg, + doAgg = internals.addFunction(ctx, doAgg, s""" |private void $doAgg() throws java.io.IOException { | // initialize aggregation buffer @@ -402,18 +369,20 @@ case class SnappyHashAggregateExec( protected def genAssignCodeForWithoutKeys(ctx: CodegenContext, ev: ExprCode, i: Int, doCopy: Boolean, inputAttrs: Seq[Attribute]): String = { + val evValue = internals.exprCodeValue(ev) + val bufValue = internals.exprCodeValue(bufVars(i)) if (doCopy) { inputAttrs(i).dataType match { case StringType => - ObjectHashMapAccessor.cloneStringIfRequired(ev.value, bufVars(i).value, doCopy = true) + ObjectHashMapAccessor.cloneStringIfRequired(evValue, bufValue, doCopy = true) case d@(_: ArrayType | _: MapType | _: StructType) => - val javaType = ctx.javaType(d) - s"${bufVars(i).value} = ($javaType)(${ev.value} != null ? ${ev.value}.copy() : null);" + val javaType = internals.javaType(d, ctx) + s"$bufValue = ($javaType)($evValue != null ? $evValue.copy() : null);" case _: BinaryType => - s"${bufVars(i).value} = (byte[])(${ev.value} != null ? ${ev.value}.clone() : null);" - case _ => s"${bufVars(i).value} = ${ev.value};" + s"$bufValue = (byte[])($evValue != null ? $evValue.clone() : null);" + case _ => s"$bufValue = $evValue;" } - } else s"${bufVars(i).value} = ${ev.value};" + } else s"$bufValue = $evValue;" } private def doConsumeWithoutKeys(ctx: CodegenContext, @@ -444,7 +413,7 @@ case class SnappyHashAggregateExec( val doCopy = !ObjectHashMapAccessor.providesImmutableObjects(child) val updates = aggVals.zipWithIndex.map { case (ev, i) => s""" - | ${bufVars(i).isNull} = ${ev.isNull}; + | ${internals.exprCodeIsNull(bufVars(i))} = ${internals.exprCodeIsNull(ev)}; | ${genAssignCodeForWithoutKeys(ctx, ev, i, doCopy, inputAttrs)} """.stripMargin } @@ -531,14 +500,14 @@ case class SnappyHashAggregateExec( private def generateResultCodeForSHAMap( ctx: CodegenContext, keyBufferVars: Seq[ExprCode], aggBufferVars: Seq[ExprCode], iterValueOffsetTerm: String): String = { - /* Asif: It appears that we have to put the code of materilization of each grouping column - & aggreagte before we can send it to parent. The reason is following: - 1) In the byte buffer hashmap data is written consecitively i.e key1, key2 agg1 etc. + /* Asif: It appears that we have to put the code of materialization of each grouping column + & aggregate before we can send it to parent. The reason is following: + 1) In the byte buffer hashmap data is written consecutively i.e key1, key2 agg1 etc. Now the pointer cannot jump arbitrarily to just read key2 without reading key1 - So suppose we have a nested query such that inner query produces code for outputting key1 , key2, - while outer query is going to use only key2. If we do not write the code of materialzing key1, - the pointer will not move forward, as the outer query is going to try to materialzie only key2, - but the pointer will not move to key2 unleass key1 has been consumed. + So suppose we have a nested query such that inner query produces code for outputting key1/2, + while outer query is going to use only key2. If we do not write the code of materializing key1, + the pointer will not move forward, as the outer query is going to try to materialize only key2, + but the pointer will not move to key2 unless key1 has been consumed. We need to resolve this issue. I suppose we can declare local variable pointers pointing to start location of each key/aggregate & use those declared pointers in the materialization code for each key */ @@ -581,17 +550,20 @@ case class SnappyHashAggregateExec( } else if (modes.contains(Partial) || modes.contains(PartialMerge)) { // Combined grouping keys and aggregate values in buffer + var evaluateKeyVars = evaluateVariables(keyBufferVars) ctx.INPUT_ROW = null ctx.currentVars = keyBufferVars val keyVars = groupingExpressions.zipWithIndex.map { case (e, i) => BoundReference(i, e.dataType, e.nullable).genCode(ctx) } - val evaluateKeyVars = evaluateVariables(keyVars) + evaluateKeyVars += evaluateVariables(keyVars) + + var evaluateBufferVars = evaluateVariables(aggBufferVars) ctx.currentVars = aggBufferVars val bufferVars = aggregateBufferAttributesForGroup.zipWithIndex.map { case (e, i) => BoundReference(i, e.dataType, e.nullable).genCode(ctx) } - val evaluateBufferVars = evaluateVariables(bufferVars) + evaluateBufferVars += evaluateVariables(bufferVars) s""" ${byteBufferAccessor.readNullBitsCode(iterValueOffsetTerm, byteBufferAccessor.nullKeysBitsetTerm, byteBufferAccessor.numBytesForNullKeyBits)} @@ -601,8 +573,6 @@ case class SnappyHashAggregateExec( $evaluateBufferVars ${consume(ctx, keyBufferVars ++ aggBufferVars)} """ - - } else { // generate result based on grouping key ctx.INPUT_ROW = null @@ -626,18 +596,14 @@ case class SnappyHashAggregateExec( } private def doProduceWithKeysForSHAMap(ctx: CodegenContext): String = { - val initAgg = ctx.freshName("initAgg") - ctx.addMutableState("boolean", initAgg, s"$initAgg = false;") + val initAgg = internals.addClassField(ctx, "boolean", "initAgg", v => s"$v = false;") // Create a name for iterator from HashMap val endIterValueOffset = ctx.freshName("endIterValueOffset") val localIterValueOffsetTerm = ctx.freshName("localIterValueOffsetTerm") val localIterValueStartOffsetTerm = ctx.freshName("localIterValueStartOffsetTerm") - val iterValueOffsetTerm = ctx.freshName("iterValueOffsetTerm") - ctx.addMutableState("long", iterValueOffsetTerm, s"$iterValueOffsetTerm = 0;") - - val nullKeysBitsetTerm = ctx.freshName("nullKeysBitset") - val nullAggsBitsetTerm = ctx.freshName("nullAggsBitset") + val iterValueOffsetTerm = internals.addClassField(ctx, "long", "iterValueOffsetTerm", + v => s"$v = 0;") val numBytesForNullKeyBits = if (this.groupingAttributes.forall(!_.nullable)) { 0 @@ -648,15 +614,17 @@ case class SnappyHashAggregateExec( val numBytesForNullAggsBits = SHAMapAccessor.calculateNumberOfBytesForNullBits( this.aggregateBufferAttributesForGroup.length) - if (SHAMapAccessor.isByteArrayNeededForNullBits(numBytesForNullKeyBits)) { - ctx.addMutableState("byte[]", nullKeysBitsetTerm, - s"$nullKeysBitsetTerm = new byte[$numBytesForNullKeyBits];") - } - - if (SHAMapAccessor.isByteArrayNeededForNullBits(numBytesForNullAggsBits)) { - ctx.addMutableState("byte[]", nullAggsBitsetTerm, - s"$nullKeysBitsetTerm = new byte[$numBytesForNullAggsBits];") - } + val nullKeysBitsetTerm = if (SHAMapAccessor.isByteArrayNeededForNullBits( + numBytesForNullKeyBits)) { + internals.addClassField(ctx, "byte[]", "nullKeysBitset", v => + s"$v = new byte[$numBytesForNullKeyBits];") + } else ctx.freshName("nullKeysBitset") + + val nullAggsBitsetTerm = if (SHAMapAccessor.isByteArrayNeededForNullBits( + numBytesForNullAggsBits)) { + internals.addClassField(ctx, "byte[]", "nullAggsBitset", + v => s"$v = new byte[$numBytesForNullAggsBits];") + } else ctx.freshName("nullAggsBitset") val probableSkipLen = this.groupingAttributes. lastIndexWhere(attr => !TypeUtilities.isFixedWidth(attr.dataType)) @@ -672,15 +640,15 @@ case class SnappyHashAggregateExec( val arrayDataClass = classOf[ArrayData].getName val platformClass = classOf[Platform].getName - val sizeAndNumNotNullFuncForStringArr = ctx.freshName("calculateStringArrSizeAndNumNotNulls") + var sizeAndNumNotNullFuncForArray = ctx.freshName("calculateArraySizeAndNumNotNulls") - if (groupingAttributes.exists(attrib => attrib.dataType.existsRecursively(_ match { + if (groupingAttributes.exists(_.dataType.existsRecursively { case ArrayType(StringType, _) | ArrayType(_, true) => true case _ => false - }))) { - ctx.addNewFunction(sizeAndNumNotNullFuncForStringArr, + })) { + sizeAndNumNotNullFuncForArray = internals.addFunction(ctx, sizeAndNumNotNullFuncForArray, s""" - private long $sizeAndNumNotNullFuncForStringArr($arrayDataClass arrayData, + private long $sizeAndNumNotNullFuncForArray($arrayDataClass arrayData, boolean isStringData) { long size = 0L; int numNulls = 0; @@ -699,25 +667,17 @@ case class SnappyHashAggregateExec( """) } - - val valueOffsetTerm = ctx.freshName("valueOffset") val currentValueOffSetTerm = ctx.freshName("currentValueOffSet") - val valueDataTerm = ctx.freshName("valueData") - val vdBaseObjectTerm = ctx.freshName("vdBaseObjectTerm") - val vdBaseOffsetTerm = ctx.freshName("vdBaseOffsetTerm") - val valueDataCapacityTerm = ctx.freshName("valueDataCapacity") - val doAgg = ctx.freshName("doAggregateWithKeys") - val setBBMap = ctx.freshName("setBBMap") - // generate variable name for hash map for use here and in consume - hashMapTerm = ctx.freshName("hashMap") - val hashSetClassName = classOf[SHAMap].getName + val valueDataTerm = internals.addClassField(ctx, bbDataClass, "valueData") + val vdBaseObjectTerm = internals.addClassField(ctx, "Object", "vdBaseObjectTerm") + val vdBaseOffsetTerm = internals.addClassField(ctx, "long", "vdBaseOffsetTerm") + val valueDataCapacityTerm = internals.addClassField(ctx, "int", "valueDataCapacity") + + var doAgg = ctx.freshName("doAggregateWithKeys") + var setBBMap = ctx.freshName("setBBMap") - val overflowHashMapsTerm = ctx.freshName("overflowHashMaps") - val listClassName = classOf[java.util.List[SHAMap]].getName - val overflowMapIter = ctx.freshName("overflowMapIter") - val iterClassName = classOf[java.util.Iterator[SHAMap]].getName // generate variable names for holding data from the Map buffer val aggregateBufferVars = for (i <- this.aggregateBufferAttributesForGroup.indices) yield { ctx.freshName(s"buffer_$i") @@ -728,54 +688,46 @@ case class SnappyHashAggregateExec( } val keysDataType = this.groupingAttributes.map(_.dataType) + // noinspection ScalaUnnecessaryParentheses // declare nullbitset terms for nested structs if required - val nestedStructNullBitsTermCreator: ((String, StructType, Int) => Any) => (String, StructType, Int) => Any = - (f: (String, StructType, Int) => Any) => - (structVarName: String, structType: StructType, nestingLevel: Int) => { - val numBytesForNullBits = SHAMapAccessor. - calculateNumberOfBytesForNullBits(structType.length) - if (SHAMapAccessor.isByteArrayNeededForNullBits(numBytesForNullBits)) { - val nullBitTerm = SHAMapAccessor. - generateNullKeysBitTermForStruct(structVarName) - ctx.addMutableState("byte[]", nullBitTerm, - s"$nullBitTerm = new byte[$numBytesForNullBits];") - } - structType.zipWithIndex.foreach { case (sf, index) => sf.dataType match { - case stt: StructType => val structtVarName = SHAMapAccessor. + val nestedStructNullBitsTermCreator: ((String, StructType, Int) => Any) => + (String, StructType, Int) => Any = (f: (String, StructType, Int) => Any) => + (structVarName: String, structType: StructType, nestingLevel: Int) => { + structType.zipWithIndex.foreach { case (sf, index) => sf.dataType match { + case stt: StructType => val structtVarName = SHAMapAccessor. generateExplodedStructFieldVars(structVarName, nestingLevel + 1, index)._1 - f(structtVarName, stt, nestingLevel + 1) - null - case _ => null - } + f(structtVarName, stt, nestingLevel + 1) + null + case _ => null + } - } } + } + // noinspection ScalaUnnecessaryParentheses val nestedStructNullBitsTermInitializer: ((String, StructType, Int) => Any) => - (String, StructType, Int) => Any = - (f: (String, StructType, Int) => Any) => - (structVarName: String, structType: StructType, nestingLevel: Int) => { - val numBytesForNullBits = SHAMapAccessor. + (String, StructType, Int) => Any = (f: (String, StructType, Int) => Any) => + (structVarName: String, structType: StructType, nestingLevel: Int) => { + val numBytesForNullBits = SHAMapAccessor. calculateNumberOfBytesForNullBits(structType.length) - val nullBitTerm = SHAMapAccessor. - generateNullKeysBitTermForStruct(structVarName) - val snippet1 = SHAMapAccessor.initNullBitsetCode(nullBitTerm, numBytesForNullBits) - - val snippet2 = structType.zipWithIndex.map { case (sf, index) => sf.dataType match { - case stt: StructType => val structtVarName = SHAMapAccessor. - generateVarNameForStructField(structVarName, nestingLevel , index) - f(structtVarName, stt, nestingLevel + 1).toString - case _ => "" - } - }.mkString("\n") - s""" - ${snippet1} - $snippet2 - """.stripMargin + val nullBitTerm = SHAMapAccessor.generateNullKeysBitTermForStruct(structVarName) + val snippet1 = SHAMapAccessor.initNullBitsetCode(nullBitTerm, numBytesForNullBits, ctx) + + val snippet2 = structType.zipWithIndex.map { case (sf, index) => sf.dataType match { + case stt: StructType => val structtVarName = SHAMapAccessor. + generateVarNameForStructField(structVarName, nestingLevel, index) + f(structtVarName, stt, nestingLevel + 1).toString + case _ => "" } + }.mkString("\n") + s""" + $snippet1 + $snippet2 + """.stripMargin + } - def recursiveApply(f: - ((String, StructType, Int) => Any) => (String, StructType, Int) => Any): - (String, StructType, Int) => Any = f(recursiveApply(f))(_, _, _) + // noinspection ScalaUnnecessaryParentheses + def recursiveApply(f: ((String, StructType, Int) => Any) => (String, StructType, Int) => + Any): (String, StructType, Int) => Any = f(recursiveApply(f))(_, _, _) // Now create nullBitTerms KeyBufferVars.zip(keysDataType).foreach { @@ -792,10 +744,11 @@ case class SnappyHashAggregateExec( val gfeCacheImplClass = classOf[GemFireCacheImpl].getName val byteBufferClass = classOf[ByteBuffer].getName - val keyBytesHolderVar = ctx.freshName("keyBytesHolder") - val baseKeyHolderOffset = ctx.freshName("baseKeyHolderOffset") - val baseKeyObject = ctx.freshName("baseKeyHolderObject") - val keyHolderCapacityTerm = ctx.freshName("keyholderCapacity") + val keyBytesHolderVar = internals.addClassField(ctx, byteBufferClass, "keyBytesHolder") + val baseKeyHolderOffset = internals.addClassField(ctx, "long", "baseKeyHolderOffset") + val baseKeyObject = internals.addClassField(ctx, "Object", "baseKeyHolderObject") + val keyHolderCapacityTerm = internals.addClassField(ctx, "int", "keyholderCapacity") + val keyExistedTerm = ctx.freshName("keyExisted") val codeForLenOfSkippedTerm = if (skipLenForAttrib != -1) { @@ -807,26 +760,28 @@ case class SnappyHashAggregateExec( }.toString } else { keysToProcessSize.zipWithIndex.map { - case(attrib, i) => { + case(attrib, i) => val sizeTerm = attrib.dataType.defaultSize s"""(int)(${SHAMapAccessor.getExpressionForNullEvalFromMask(i + numToDrop, numBytesForNullKeyBits, nullKeysBitsetTerm)} ? 0 : $sizeTerm) """ - } }.mkString("+") } s"""$keyLengthTerm - |(int)($localIterValueOffsetTerm - $localIterValueStartOffsetTerm) - |${ if (keysToProcessSize.length > 0) s" - ($suffixSize)" else ""};""".stripMargin + |${ if (keysToProcessSize.nonEmpty) s" - ($suffixSize)" else ""};""".stripMargin } else "" - - ctx.addMutableState(hashSetClassName, hashMapTerm, s"$hashMapTerm = null;") - ctx.addMutableState(listClassName + s"<$hashSetClassName>", overflowHashMapsTerm, - s"$overflowHashMapsTerm = null;") - ctx.addMutableState(iterClassName + s"<$hashSetClassName>", overflowMapIter, - s"$overflowMapIter = null;") + val hashSetClassName = classOf[SHAMap].getName + val listClassName = classOf[java.util.List[SHAMap]].getName + val iterClassName = classOf[java.util.Iterator[SHAMap]].getName + // generate variable name for hash map for use here and in consume + hashMapTerm = internals.addClassField(ctx, hashSetClassName, "hashMap", v => s"$v = null;") + val overflowHashMapsTerm = internals.addClassField(ctx, listClassName + s"<$hashSetClassName>", + "overflowHashMaps", v => s"$v = null;") + val overflowMapIter = internals.addClassField(ctx, iterClassName + s"<$hashSetClassName>", + "overflowMapIter", v => s"$v = null;") val storedAggNullBitsTerm = ctx.freshName("storedAggNullBit") val cacheStoredAggNullBits = !SHAMapAccessor.isByteArrayNeededForNullBits( @@ -839,7 +794,7 @@ case class SnappyHashAggregateExec( // generate the map accessor to generate key/value class // and get map access methods val session = sqlContext.sparkSession.asInstanceOf[SnappySession] - val numKeyBytesTerm = ctx.freshName("numKeyBytes") + val numKeyBytesTerm = internals.addClassField(ctx, "int", "numKeyBytes") val numValueBytes = SHAMapAccessor.getSizeOfValueBytes(aggBuffDataTypes, numBytesForNullAggsBits) @@ -855,7 +810,7 @@ case class SnappyHashAggregateExec( keyValSize, valueOffsetTerm, numKeyBytesTerm, numValueBytes, currentValueOffSetTerm, valueDataTerm, vdBaseObjectTerm, vdBaseOffsetTerm, nullKeysBitsetTerm, numBytesForNullKeyBits, allocatorTerm, - numBytesForNullAggsBits, nullAggsBitsetTerm, sizeAndNumNotNullFuncForStringArr, + numBytesForNullAggsBits, nullAggsBitsetTerm, sizeAndNumNotNullFuncForArray, keyBytesHolderVar, baseKeyObject, baseKeyHolderOffset, keyExistedTerm, skipLenForAttrib, codeForLenOfSkippedTerm, valueDataCapacityTerm, if (cacheStoredAggNullBits) Some(storedAggNullBitsTerm) else None, @@ -867,7 +822,7 @@ case class SnappyHashAggregateExec( val childProduce = childProducer.asInstanceOf[CodegenSupport].produce(ctx, this) - ctx.addNewFunction(doAgg, + doAgg = internals.addFunction(ctx, doAgg, s"""private void $doAgg() throws java.io.IOException { |$hashMapTerm = new $hashSetClassName(${Property.initialCapacityOfSHABBMap.get( sqlContext.sparkSession.asInstanceOf[SnappySession].sessionState.conf)}, @@ -875,27 +830,33 @@ case class SnappyHashAggregateExec( asInstanceOf[SnappySession].sessionState.conf)}); |$allocatorClass $allocatorTerm = $gfeCacheImplClass. |getCurrentBufferAllocator(); - |$byteBufferClass $keyBytesHolderVar = null; - |int $keyHolderCapacityTerm = 0; - |Object $baseKeyObject = null; - |long $baseKeyHolderOffset = -1L; - |$bbDataClass $valueDataTerm = $hashMapTerm.getValueData(); - |Object $vdBaseObjectTerm = $valueDataTerm.baseObject(); - |long $vdBaseOffsetTerm = $valueDataTerm.baseOffset(); - |int $valueDataCapacityTerm = $valueDataTerm.capacity(); - |${SHAMapAccessor.initNullBitsetCode(nullKeysBitsetTerm, numBytesForNullKeyBits)} - |${SHAMapAccessor.initNullBitsetCode(nullAggsBitsetTerm, numBytesForNullAggsBits)} - |${byteBufferAccessor.initKeyOrBufferVal(aggBuffDataTypes, aggregateBufferVars)} - |${byteBufferAccessor.declareNullVarsForAggBuffer(aggregateBufferVars)} + |$keyBytesHolderVar = null; + |$keyHolderCapacityTerm = 0; + |$baseKeyObject = null; + |$baseKeyHolderOffset = -1L; + |$valueDataTerm = $hashMapTerm.getValueData(); + |$vdBaseObjectTerm = $valueDataTerm.baseObject(); + |$vdBaseOffsetTerm = $valueDataTerm.baseOffset(); + |$valueDataCapacityTerm = $valueDataTerm.capacity(); + |${SHAMapAccessor.initNullBitsetCode(nullKeysBitsetTerm, numBytesForNullKeyBits, + ctx, genClassField = true)} + |${SHAMapAccessor.initNullBitsetCode(nullAggsBitsetTerm, numBytesForNullAggsBits, + ctx, genClassField = true)} + |${byteBufferAccessor.initKeyOrBufferVal(aggBuffDataTypes, aggregateBufferVars, + genClassField = true)} + |${byteBufferAccessor.declareNullVarsForAggBuffer(aggregateBufferVars, + genClassField = true)} |${ if (cacheStoredAggNullBits) { - SHAMapAccessor.initNullBitsetCode(storedAggNullBitsTerm, numBytesForNullAggsBits) + SHAMapAccessor.initNullBitsetCode(storedAggNullBitsTerm, + numBytesForNullAggsBits, ctx, genClassField = true) } else "" } |${ if (cacheStoredKeyNullBits) { - SHAMapAccessor.initNullBitsetCode(storedKeyNullBitsTerm, numBytesForNullKeyBits) + SHAMapAccessor.initNullBitsetCode(storedKeyNullBitsTerm, + numBytesForNullKeyBits, ctx, genClassField = true) } else "" } - |int $numKeyBytesTerm = 0; + |$numKeyBytesTerm = 0; |$childProduce |if ($overflowHashMapsTerm == null) { | long $maxMemory = $hashMapTerm.maxMemory(); @@ -916,15 +877,15 @@ case class SnappyHashAggregateExec( |} |}""".stripMargin) - ctx.addNewFunction(setBBMap, + setBBMap = internals.addFunction(ctx, setBBMap, s"""private boolean $setBBMap() { |if ($hashMapTerm != null) { |return true; |} else { |if ($overflowMapIter.hasNext()) { |$hashMapTerm = ($hashSetClassName)$overflowMapIter.next(); - |$bbDataClass $valueDataTerm = $hashMapTerm.getValueData(); - |Object $vdBaseObjectTerm = $valueDataTerm.baseObject(); + |$valueDataTerm = $hashMapTerm.getValueData(); + |$vdBaseObjectTerm = $valueDataTerm.baseObject(); |$iterValueOffsetTerm = $valueDataTerm.baseOffset(); return true; |} else { @@ -939,17 +900,19 @@ case class SnappyHashAggregateExec( keyBufferTerm, keyBufferTerm, onlyKeyVars = false, onlyValueVars = false) */ val keysExpr = byteBufferAccessor.getBufferVars(keysDataType, KeyBufferVars, - localIterValueOffsetTerm, true, byteBufferAccessor.nullKeysBitsetTerm, - byteBufferAccessor.numBytesForNullKeyBits, byteBufferAccessor.numBytesForNullKeyBits == 0) + localIterValueOffsetTerm, isKey = true, byteBufferAccessor.nullKeysBitsetTerm, + byteBufferAccessor.numBytesForNullKeyBits, + skipNullBitsCode = byteBufferAccessor.numBytesForNullKeyBits == 0) val aggsExpr = byteBufferAccessor.getBufferVars(aggBuffDataTypes, - aggregateBufferVars, localIterValueOffsetTerm, false, byteBufferAccessor.nullAggsBitsetTerm, - byteBufferAccessor.numBytesForNullAggBits, false) + aggregateBufferVars, localIterValueOffsetTerm, isKey = false, + byteBufferAccessor.nullAggsBitsetTerm, byteBufferAccessor.numBytesForNullAggBits, + skipNullBitsCode = false) val outputCode = generateResultCodeForSHAMap(ctx, keysExpr, aggsExpr, localIterValueOffsetTerm) val numOutput = metricTerm(ctx, "numOutputRows") val localNumRowsIterated = ctx.freshName("localNumRowsIterated") // The child could change `copyResult` to true, but we had already // consumed all the rows, so `copyResult` should be reset to `false`. - ctx.copyResult = false + internals.resetCopyResult(ctx) val aggTime = metricTerm(ctx, "aggTime") val beforeAgg = ctx.freshName("beforeAgg") @@ -983,8 +946,8 @@ case class SnappyHashAggregateExec( $overflowMapIter = $overflowHashMapsTerm.iterator(); $overflowMapIter.next(); } - $bbDataClass $valueDataTerm = $hashMapTerm.getValueData(); - Object $vdBaseObjectTerm = $valueDataTerm.baseObject(); + $valueDataTerm = $hashMapTerm.getValueData(); + $vdBaseObjectTerm = $valueDataTerm.baseObject(); $iterValueOffsetTerm += $valueDataTerm.baseOffset(); } if ($hashMapTerm == null) { @@ -994,8 +957,8 @@ case class SnappyHashAggregateExec( getCurrentBufferAllocator(); ${byteBufferAccessor.initKeyOrBufferVal(aggBuffDataTypes, aggregateBufferVars)} ${byteBufferAccessor.initKeyOrBufferVal(keysDataType, KeyBufferVars)} - ${SHAMapAccessor.initNullBitsetCode(nullKeysBitsetTerm, numBytesForNullKeyBits)} - ${SHAMapAccessor.initNullBitsetCode(nullAggsBitsetTerm, numBytesForNullAggsBits)} + ${SHAMapAccessor.initNullBitsetCode(nullKeysBitsetTerm, numBytesForNullKeyBits, ctx)} + ${SHAMapAccessor.initNullBitsetCode(nullAggsBitsetTerm, numBytesForNullAggsBits, ctx)} ${KeyBufferVars.zip(keysDataType).map { case (varName, dataType) => dataType match { case st: StructType => @@ -1006,8 +969,8 @@ case class SnappyHashAggregateExec( // output the result while($setBBMap()) { - $bbDataClass $valueDataTerm = $hashMapTerm.getValueData(); - Object $vdBaseObjectTerm = $valueDataTerm.baseObject(); + $valueDataTerm = $hashMapTerm.getValueData(); + $vdBaseObjectTerm = $valueDataTerm.baseObject(); long $endIterValueOffset = $hashMapTerm.valueDataSize() + $valueDataTerm.baseOffset(); long $localIterValueOffsetTerm = $iterValueOffsetTerm; ${byteBufferAccessor.declareNullVarsForAggBuffer(aggregateBufferVars)} @@ -1040,23 +1003,19 @@ case class SnappyHashAggregateExec( } private def doProduceWithKeys(ctx: CodegenContext): String = { - val initAgg = ctx.freshName("initAgg") - ctx.addMutableState("boolean", initAgg, s"$initAgg = false;") + val initAgg = internals.addClassField(ctx, "boolean", "initAgg", v => s"$v = false;") // Create a name for iterator from HashMap - val iterTerm = ctx.freshName("mapIter") val iter = ctx.freshName("mapIter") val iterObj = ctx.freshName("iterObj") val iterClass = "java.util.Iterator" - ctx.addMutableState(iterClass, iterTerm, "") + val iterTerm = internals.addClassField(ctx, iterClass, "mapIter") - val doAgg = ctx.freshName("doAggregateWithKeys") + var doAgg = ctx.freshName("doAggregateWithKeys") // generate variable name for hash map for use here and in consume - hashMapTerm = ctx.freshName("hashMap") val hashSetClassName = classOf[ObjectHashSet[_]].getName - - ctx.addMutableState(hashSetClassName, hashMapTerm, "") + hashMapTerm = internals.addClassField(ctx, hashSetClassName, "hashMap") // generate variables for HashMap data array and mask mapDataTerm = ctx.freshName("mapData") @@ -1072,19 +1031,23 @@ case class SnappyHashAggregateExec( aggregateBufferAttributesForGroup, "KeyBuffer", hashMapTerm, mapDataTerm, maskTerm, multiMap = false, this, this.parent, child) - val entryClass = keyBufferAccessor.getClassName val numKeyColumns = groupingExpressions.length + internals.addClassField(ctx, s"$entryClass[]", mapDataTerm, + forceInline = true, useFreshName = false) + internals.addClassField(ctx, "int", maskTerm, + forceInline = true, useFreshName = false) + val childProduce = childProducer.asInstanceOf[CodegenSupport].produce(ctx, this) - ctx.addNewFunction(doAgg, + doAgg = internals.addFunction(ctx, doAgg, s""" private void $doAgg() throws java.io.IOException { $hashMapTerm = new $hashSetClassName(128, 0.6, $numKeyColumns, false, scala.reflect.ClassTag$$.MODULE$$.apply($entryClass.class)); - $entryClass[] $mapDataTerm = ($entryClass[])$hashMapTerm.data(); - int $maskTerm = $hashMapTerm.mask(); + $mapDataTerm = ($entryClass[])$hashMapTerm.data(); + $maskTerm = $hashMapTerm.mask(); $childProduce @@ -1107,7 +1070,7 @@ case class SnappyHashAggregateExec( // The child could change `copyResult` to true, but we had already // consumed all the rows, so `copyResult` should be reset to `false`. - ctx.copyResult = false + internals.resetCopyResult(ctx) val aggTime = metricTerm(ctx, "aggTime") val beforeAgg = ctx.freshName("beforeAgg") @@ -1174,16 +1137,19 @@ case class SnappyHashAggregateExec( val bufferVars = byteBufferAccessor.getBufferVars(aggBuffDataTypes, byteBufferAccessor.aggregateBufferVars, - byteBufferAccessor.currentOffSetForMapLookupUpdt, - false, byteBufferAccessor.nullAggsBitsetTerm, byteBufferAccessor.numBytesForNullAggBits, - false) + byteBufferAccessor.currentOffSetForMapLookupUpdt, isKey = false, + byteBufferAccessor.nullAggsBitsetTerm, byteBufferAccessor.numBytesForNullAggBits, + skipNullBitsCode = false) val bufferEval = evaluateVariables(bufferVars) - val bufferVarsFromInitVars = byteBufferAccessor.aggregateBufferVars.zip(initVars).map { - case (bufferVarName, initExpr) => ExprCode( - s""" - |$bufferVarName${SHAMapAccessor.nullVarSuffix} = ${initExpr.isNull}; - |$bufferVarName = ${initExpr.value};""".stripMargin, - s"$bufferVarName${SHAMapAccessor.nullVarSuffix}", bufferVarName) + val bufferVarsFromInitVars = byteBufferAccessor.aggregateBufferVars.indices.map { i => + val bufferVarName = byteBufferAccessor.aggregateBufferVars(i) + val initEv = initVars(i) + internals.newExprCode(code = + s""" + |$bufferVarName${SHAMapAccessor.nullVarSuffix} = ${internals.exprCodeIsNull(initEv)}; + |$bufferVarName = ${internals.exprCodeValue(initEv)};""".stripMargin, + isNull = s"$bufferVarName${SHAMapAccessor.nullVarSuffix}", value = bufferVarName, + aggBuffDataTypes(i)) } val bufferEvalFromInitVars = evaluateVariables(bufferVarsFromInitVars) ctx.currentVars = bufferVars ++ input diff --git a/core/src/main/scala/org/apache/spark/sql/execution/bootstrap/ApproxColumnExtractor.scala b/core/src/main/scala/org/apache/spark/sql/execution/bootstrap/ApproxColumnExtractor.scala new file mode 100644 index 0000000000..21132e2f0a --- /dev/null +++ b/core/src/main/scala/org/apache/spark/sql/execution/bootstrap/ApproxColumnExtractor.scala @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2017-2020 TIBCO Software Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ +package org.apache.spark.sql.execution.bootstrap + +import org.apache.spark.sql.SparkSupport +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute +import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode} +import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, NamedExpression, UnaryExpression} +import org.apache.spark.sql.types.Metadata + +trait ApproxColumnExtractor extends UnaryExpression with NamedExpression with SparkSupport { + + val ordinal: Int + + override lazy val resolved: Boolean = true + + override def eval(input: InternalRow): Any = + throw new UnsupportedOperationException("not implemented") + + override protected def doGenCode(ctx: CodegenContext, + ev: ExprCode): ExprCode = { + + val childEval = child.genCode(ctx) + val evIsNull = internals.exprCodeIsNull(ev) + val evVal = internals.exprCodeValue(ev) + val childVal = internals.exprCodeValue(childEval) + val code = + s""" + ${childEval.code} + double $evVal = 0d; + boolean $evIsNull = ((InternalRow) $childVal).isNullAt($ordinal); + if (!$evIsNull) { + $evVal = ((InternalRow) $childVal).getDouble($ordinal); + } + """ + internals.copyExprCode(ev, code = code) + } + + override def metadata: Metadata = Metadata.empty + + override def toAttribute: Attribute = { + if (resolved) { + internals.newAttributeReference(name, dataType, nullable, metadata, exprId, qualifier.toSeq) + } else { + UnresolvedAttribute(name) + } + } + + override def toString: String = s"$child AS $name#${exprId.id}$typeSuffix" + + override protected final def otherCopyArgs: Seq[AnyRef] = { + exprId :: qualifier :: Nil + } + + override def equals(other: Any): Boolean = other match { + case a: Alias => + name == a.name && exprId == a.exprId && child == a.child + + case _ => false + } + + /** Returns a copy of this expression with a new `exprId`. */ + override def newInstance(): NamedExpression = + internals.newApproxColumnExtractor(child, name, ordinal, dataType, nullable, + qualifier = qualifier.toSeq) +} diff --git a/core/src/main/scala/org/apache/spark/sql/execution/bootstrap/lazyExpressions.scala b/core/src/main/scala/org/apache/spark/sql/execution/bootstrap/lazyExpressions.scala new file mode 100644 index 0000000000..ecd69a37c6 --- /dev/null +++ b/core/src/main/scala/org/apache/spark/sql/execution/bootstrap/lazyExpressions.scala @@ -0,0 +1,167 @@ +/* + * Copyright (c) 2017-2020 TIBCO Software Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ +package org.apache.spark.sql.execution.bootstrap + +import org.apache.spark.sql.SparkSupport +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute +import org.apache.spark.sql.catalyst.errors.TreeNodeException +import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode} +import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeReference, ExprId, Expression, NamedExpression, Unevaluable} +import org.apache.spark.sql.types.{DataType, Metadata} + +trait Tag { + + def symbol: String + + def simpleString: String = "" +} + +trait TransformableTag extends Tag { + def toTag: Tag +} + +object Seed extends TransformableTag { + + val symbol = ":" + + def toTag: TransformableTag = this +} + +object Bootstrap extends TransformableTag { + + val symbol = ":" + + def toTag: TransformableTag = this + + override lazy val simpleString = "No Op" // s"^${branches.mkString("[", ", ", "]")}" +} + +trait TaggedAttribute extends Attribute with Unevaluable with SparkSupport { + + val tag: Tag + + override def equals(other: Any): Boolean = other match { + case ar: TaggedAttribute => tag == ar.tag && name == ar.name && + exprId == ar.exprId && dataType == ar.dataType + case _ => false + } + + + override def hashCode(): Int = { + // See http://stackoverflow.com/questions/113511/hash-code-implementation + var h = 17 + h = h * 37 + exprId.hashCode() + h = h * 37 + dataType.hashCode() + h = h * 37 + metadata.hashCode() + h = h * 37 + tag.hashCode() + h + } + + override def newInstance(): TaggedAttribute = internals.newTaggedAttribute(tag, name, + dataType, nullable, metadata, qualifier = qualifier.toSeq) + + /** + * Returns a copy of this [[TaggedAttribute]] with changed nullability. + */ + override def withNullability(newNullability: Boolean): TaggedAttribute = { + if (nullable == newNullability) { + this + } else { + internals.newTaggedAttribute(tag, name, dataType, newNullability, metadata, + exprId, qualifier.toSeq) + } + } + + override def withName(newName: String): TaggedAttribute = { + if (name == newName) { + this + } else { + internals.newTaggedAttribute(tag, newName, dataType, nullable, metadata, + exprId, qualifier.toSeq) + } + } + + def withExprId(newExprId: ExprId): TaggedAttribute = { + if (exprId == newExprId) { + this + } else { + internals.newTaggedAttribute(tag, name, dataType, nullable, metadata, + newExprId, qualifier.toSeq) + } + } + + def toAttributeReference: AttributeReference = internals.newAttributeReference(name, + dataType, nullable, metadata, exprId, qualifier.toSeq) + + override def withMetadata(newMetadata: Metadata): Attribute = { + internals.newTaggedAttribute(tag, name, dataType, nullable, metadata, + exprId, qualifier.toSeq) + } + + override protected final def otherCopyArgs: Seq[AnyRef] = exprId :: qualifier :: Nil +} + +trait TaggedAlias extends NamedExpression with SparkSupport { + + val child: Expression + + val tag: TransformableTag + + // override type EvaluatedType = Any + /** Just a simple passthrough for code generation. */ + override def genCode(ctx: CodegenContext): ExprCode = child.genCode(ctx) + + override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = + internals.copyExprCode(ev, code = "") + + override def eval(input: InternalRow): Any = throw new TreeNodeException( + this, s"No function to evaluate expression. type: ${this.nodeName}") + + override def dataType: DataType = child.dataType + + override def nullable: Boolean = child.nullable + + override def metadata: Metadata = { + child match { + case named: NamedExpression => named.metadata + case _ => Metadata.empty + } + } + + def children: Seq[Expression] = child :: Nil + + override def toAttribute: Attribute = { + if (resolved) { + internals.newTaggedAttribute(tag.toTag, name, child.dataType, child.nullable, + metadata, exprId, qualifier.toSeq) + } else { + UnresolvedAttribute(name) + } + } + + override def toString: String = + s"$child${tag.simpleString} AS ${tag.symbol}$name#${exprId.id}$typeSuffix" + + override protected final def otherCopyArgs: Seq[AnyRef] = exprId :: qualifier :: Nil + + def toAlias: Alias = internals.newAlias(child, name, copyAlias = None, exprId, qualifier.toSeq) + + /** Returns a copy of this expression with a new `exprId`. */ + override def newInstance(): NamedExpression = internals.newTaggedAlias(tag, child, + name, qualifier = qualifier.toSeq) +} diff --git a/core/src/main/scala/org/apache/spark/sql/execution/closedform/ClosedFormColumnExtractor.scala b/core/src/main/scala/org/apache/spark/sql/execution/closedform/ClosedFormColumnExtractor.scala new file mode 100644 index 0000000000..00dde1e8ac --- /dev/null +++ b/core/src/main/scala/org/apache/spark/sql/execution/closedform/ClosedFormColumnExtractor.scala @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2017-2020 TIBCO Software Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ +package org.apache.spark.sql.execution.closedform + +import org.apache.spark.sql.SparkSupport +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute +import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode} +import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, NamedExpression, UnaryExpression} +import org.apache.spark.sql.execution.common.HAC +import org.apache.spark.sql.types.Metadata + +trait ClosedFormColumnExtractor extends UnaryExpression with NamedExpression with SparkSupport { + + val confidence: Double + + val confFactor: Double + + val aggType: ErrorAggregate.Type + + val error: Double + + val behavior: HAC.Type + + // Alias(Generator, xx) need to be transformed into Generate(generator, ...) + override lazy val resolved = true + + override def eval(input: InternalRow): Any = { + val errorStats = child.eval(input).asInstanceOf[ClosedFormStats] + val retVal: Double = SparkSupport.contextFunctionsStateless.finalizeEvaluation( + errorStats, confidence, confFactor, aggType, error, behavior) + if (retVal.isNaN) null else retVal + } + + protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { + val childEval = child.genCode(ctx) + val statClass = classOf[ClosedFormStats].getName + val statVar = ctx.freshName("errorStats") + val returnValue = ctx.freshName("returnValue") + val statCounterUDTF = "org.apache.spark.sql.execution.closedform.StatCounterUDTCF" + val behaviorString = HAC.getBehaviorAsString(behavior) + val hacClass = HAC.getClass.getName + val aggTypeStr = aggType.toString + val aggTypeClass = ErrorAggregate.getClass.getName + + val code = childEval.code.toString + + s""" + $statClass $statVar = ($statClass)${internals.exprCodeValue(childEval)}; + double $returnValue = $statCounterUDTF.MODULE$$.finalizeEvaluation($statVar, + $confidence, $confFactor,$aggTypeClass.MODULE$$.withName("$aggTypeStr"), $error, + $hacClass.MODULE$$.getBehavior("$behaviorString")); + boolean ${internals.exprCodeIsNull(ev)} = Double.isNaN($returnValue); + double ${internals.exprCodeValue(ev)} = $returnValue; + """ + internals.copyExprCode(ev, code = code) + } + + override def metadata: Metadata = Metadata.empty + + override def toAttribute: Attribute = + if (resolved) { + internals.newAttributeReference(name, dataType, nullable, metadata, exprId, qualifier.toSeq) + } else { + UnresolvedAttribute(name) + } + + override def toString: String = s"$child AS $name#${exprId.id}$typeSuffix" + + override protected final def otherCopyArgs: Seq[AnyRef] = exprId :: qualifier :: Nil + + override def equals(other: Any): Boolean = other match { + case a: Alias => + name == a.name && exprId == a.exprId && child == a.child + + case _ => false + } + + /** Returns a copy of this expression with a new `exprId`. */ + override def newInstance(): NamedExpression = internals.newClosedFormColumnExtractor( + child, name, confidence, confFactor, aggType, error, dataType, behavior, + nullable, qualifier = qualifier.toSeq) +} diff --git a/core/src/main/scala/org/apache/spark/sql/execution/closedform/ClosedFormStats.scala b/core/src/main/scala/org/apache/spark/sql/execution/closedform/ClosedFormStats.scala new file mode 100644 index 0000000000..82adc81d84 --- /dev/null +++ b/core/src/main/scala/org/apache/spark/sql/execution/closedform/ClosedFormStats.scala @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2017-2020 TIBCO Software Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ +package org.apache.spark.sql.execution.closedform + +import org.apache.spark.sql.catalyst.expressions.BaseGenericInternalRow +import org.apache.spark.sql.sources.StatVarianceCounter + +trait ClosedFormStats extends StatVarianceCounter { + + self: BaseGenericInternalRow => + + // New variance as per closed form formula + var weightedCount: Double + var trueSum: Double + + override def numFields: Int = 5 + + def mergeTrueSum(other: ClosedFormStats): Unit = { + (trueSum.isNaN, other.trueSum.isNaN) match { + case (false, false) => trueSum += other.trueSum + case (true, false) => trueSum = other.trueSum + case (false, true) => if (other.count > 0) trueSum = other.trueSum + case _ => + } + } + + protected override def genericGet(ordinal: Int): Any = { + triggerSerialization() + ordinal match { + case 0 => count + case 1 => mean + case 2 => nvariance + case 3 => weightedCount + case 4 => trueSum + } + } + + override def getLong(ordinal: Int): Long = { + triggerSerialization() + if (ordinal == 0) { + count + } else { + throw new ClassCastException("cannot cast double to long") + } + } + + override def getDouble(ordinal: Int): Double = { + triggerSerialization() + ordinal match { + case 1 => mean + case 2 => nvariance + case 3 => weightedCount + case 0 => count + case 4 => trueSum + } + } + + def triggerSerialization(): Unit + + def copy(other: ClosedFormStats): Unit = { + other.count = count + other.mean = mean + other.nvariance = nvariance + other.weightedCount = weightedCount + other.trueSum = trueSum + } + + + def merge(other: ClosedFormStats) { + if (other ne this) { + this.mergeDistinctCounter(other) + weightedCount += other.weightedCount + mergeTrueSum(other) + } else { + merge(other.copy()) // Avoid overwriting fields in a weird order + } + } + + /////////////// + + protected def mergeDistinctCounter(other: ClosedFormStats) { + if (count == 0) { + mean = other.mean + count = other.count + } else if (other.count != 0) { + val delta = other.mean - mean + if (other.count * 10 < count) { + mean = mean + (delta * other.count) / (count + other.count) + } else if (count * 10 < other.count) { + mean = other.mean - (delta * count) / (count + other.count) + } else { + mean = (mean * count + other.mean * other.count) / + (count + other.count) + } + count += other.count + } + nvariance += other.nvariance + } +} diff --git a/core/src/main/scala/org/apache/spark/sql/execution/closedform/ErrorAggregate.scala b/core/src/main/scala/org/apache/spark/sql/execution/closedform/ErrorAggregate.scala new file mode 100644 index 0000000000..a08325b6cc --- /dev/null +++ b/core/src/main/scala/org/apache/spark/sql/execution/closedform/ErrorAggregate.scala @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2017-2020 TIBCO Software Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ +package org.apache.spark.sql.execution.closedform + +object ErrorAggregate extends Enumeration { + + type Type = Value + + val separator = '_' + + val Avg: Type = Value("Avg") + val Sum: Type = Value("Sum") + val Count: Type = Value("Count") + + val Sum_Lower: Type = Value(Sum.toString + separator + "Lower") + val Avg_Lower: Type = Value(Avg.toString + separator + "Lower") + val Count_Lower: Type = Value(Count.toString + separator + "Lower") + + val Sum_Upper: Type = Value(Sum.toString + separator + "Upper") + val Avg_Upper: Type = Value(Avg.toString + separator + "Upper") + val Count_Upper: Type = Value(Count.toString + separator + "Upper") + + // relative error + val Sum_Relative: Type = Value(Sum.toString + separator + "Relative") + val Avg_Relative: Type = Value(Avg.toString + separator + "Relative") + val Count_Relative: Type = Value(Count.toString + separator + "Relative") + + // absolute error + val Sum_Absolute: Type = Value(Sum.toString + separator + "Absolute") + val Avg_Absolute: Type = Value(Avg.toString + separator + "Absolute") + val Count_Absolute: Type = Value(Count.toString + separator + "Absolute") + + def getBaseAggregateType(param: ErrorAggregate.Type): ErrorAggregate.Type = { + val name = param.toString + val sepIndex = name.indexOf(separator) + if (sepIndex == -1) { + param + } else { + val baseName = name.substring(0, sepIndex) + ErrorAggregate.withName(baseName) + } + } + + def getRelativeErrorTypeForBaseType(baseAggregateType: Type): Type = { + val relErrorName = baseAggregateType.toString + separator + "Relative" + ErrorAggregate.withName(relErrorName) + } + + def isBaseAggType(aggType: Type): Boolean = { + val name = aggType.toString + val sepIndex = name.indexOf(separator) + sepIndex == -1 + } + + private def getSuffix(name: String): Option[String] = { + val sepIndex = name.indexOf(separator) + if (sepIndex == -1) { + None + } else { + Some(name.substring(sepIndex + 1)) + } + } + + private def getPrefix(name: String): Option[String] = { + val sepIndex = name.indexOf(separator) + if (sepIndex == -1) { + None + } else { + Some(name.substring(0, sepIndex)) + } + } + + def checkFor(suffix: String, aggType: Type): Boolean = { + getSuffix(aggType.toString) match { + case Some(x) => x == suffix + case None => false + } + } + + def checkFor(prefix: String, errorEstimateFuncName: String): Boolean = { + getPrefix(errorEstimateFuncName) match { + case Some(x) => x == prefix + case None => false + } + } + + def isLowerAggType(aggType: Type): Boolean = checkFor("Lower", aggType) + + def isUpperAggType(aggType: Type): Boolean = checkFor("Upper", aggType) + + def isRelativeErrorAggType(aggType: Type): Boolean = + checkFor("Relative", aggType) + + def isAbsoluteErrorAggType(aggType: Type): Boolean = + checkFor("Absolute", aggType) + + def isLowerAggType(errorEstimateFuncName: String): Boolean = + checkFor("Lower", errorEstimateFuncName) + + def isUpperAggType(errorEstimateFuncName: String): Boolean = + checkFor("Upper", errorEstimateFuncName) + + def isRelativeErrorAggType(errorEstimateFuncName: String): Boolean = + checkFor("Relative", errorEstimateFuncName) + + def isAbsoluteErrorAggType(errorEstimateFuncName: String): Boolean = + checkFor("Absolute", errorEstimateFuncName) +} diff --git a/core/src/main/scala/org/apache/spark/sql/execution/closedform/ErrorEstimateAttribute.scala b/core/src/main/scala/org/apache/spark/sql/execution/closedform/ErrorEstimateAttribute.scala new file mode 100644 index 0000000000..73256f4d75 --- /dev/null +++ b/core/src/main/scala/org/apache/spark/sql/execution/closedform/ErrorEstimateAttribute.scala @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2017-2020 TIBCO Software Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ +package org.apache.spark.sql.execution.closedform + +import org.apache.spark.sql.SparkSupport +import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, AttributeSet, ExprId, Expression, Unevaluable} +import org.apache.spark.sql.catalyst.plans.logical.EventTimeWatermark +import org.apache.spark.sql.catalyst.util.quoteIdentifier +import org.apache.spark.sql.types.Metadata + +trait ErrorEstimateAttribute extends Attribute with Unevaluable with SparkSupport { + + def realExprId: ExprId + + /** + * Returns true iff the expression id is the same for both attributes. + */ + def sameRef(other: AttributeReference): Boolean = this.exprId == other.exprId + + override def equals(other: Any): Boolean = other match { + case ar: AttributeReference => name == ar.name && dataType == ar.dataType && + nullable == ar.nullable && metadata == ar.metadata && exprId == ar.exprId && + qualifier == ar.qualifier + case eea: ErrorEstimateAttribute => (eea eq this) || (name == eea.name && + dataType == eea.dataType && nullable == eea.nullable && metadata == eea.metadata && + exprId == eea.exprId && qualifier == eea.qualifier) + case _ => false + } + + override def semanticEquals(other: Expression): Boolean = other match { + case ar: AttributeReference => sameRef(ar) + case _ => false + } + + override def semanticHash(): Int = { + this.exprId.hashCode() + } + + override def hashCode(): Int = { + // See http://stackoverflow.com/questions/113511/hash-code-implementation + var h = 17 + h = h * 37 + name.hashCode() + h = h * 37 + dataType.hashCode() + h = h * 37 + nullable.hashCode() + h = h * 37 + metadata.hashCode() + h = h * 37 + exprId.hashCode() + h = h * 37 + qualifier.hashCode() + h + } + + override def newInstance(): ErrorEstimateAttribute = { + internals.newErrorEstimateAttribute(name, dataType, nullable, metadata, realExprId, + qualifier = qualifier.toSeq) + } + + /** + * Returns a copy of this [[ErrorEstimateAttribute]] with changed nullability. + */ + override def withNullability(newNullability: Boolean): ErrorEstimateAttribute = { + if (nullable == newNullability) { + this + } else { + internals.newErrorEstimateAttribute(name, dataType, newNullability, metadata, realExprId, + exprId, qualifier.toSeq) + } + } + + override def withName(newName: String): ErrorEstimateAttribute = { + if (name == newName) { + this + } else { + internals.newErrorEstimateAttribute(newName, dataType, nullable, metadata, realExprId, + exprId, qualifier.toSeq) + } + } + + def withExprId(newExprId: ExprId): ErrorEstimateAttribute = { + if (exprId == newExprId) { + this + } else { + internals.newErrorEstimateAttribute(name, dataType, nullable, metadata, realExprId, + newExprId, qualifier.toSeq) + } + } + + override def references: AttributeSet = AttributeSet(internals.toAttributeReference(this)()) + + override def withMetadata(newMetadata: Metadata): Attribute = { + internals.newErrorEstimateAttribute(name, dataType, nullable, newMetadata, realExprId, + exprId, qualifier.toSeq) + } + + /** Used to signal the column used to calculate an eventTime watermark (e.g. a#1-T{delayMs}) */ + private def delaySuffix = if (metadata.contains(EventTimeWatermark.delayKey)) { + s"-T${metadata.getLong(EventTimeWatermark.delayKey)}ms" + } else { + "" + } + + override protected final def otherCopyArgs: Seq[AnyRef] = exprId :: qualifier :: Nil + + override def toString: String = s"$name#${exprId.id}$typeSuffix$delaySuffix" + + // Since the expression id is not in the first constructor it is missing from the default + // tree string. + override def simpleString: String = s"$name#${exprId.id}: ${dataType.simpleString}" + + override def sql: String = { + val qualifierPrefix = if (qualifier.isEmpty) "" else qualifier.head + '.' + s"$qualifierPrefix${quoteIdentifier(name)}" + } +} diff --git a/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnBatchCreator.scala b/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnBatchCreator.scala index 722769f43c..2e8744af6f 100644 --- a/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnBatchCreator.scala +++ b/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnBatchCreator.scala @@ -26,12 +26,12 @@ import org.eclipse.collections.impl.set.mutable.UnifiedSet import org.apache.spark.Logging import org.apache.spark.rdd.RDD +import org.apache.spark.sql.SparkSupport import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.codegen.CodegenContext import org.apache.spark.sql.catalyst.expressions.{Attribute, BoundReference} import org.apache.spark.sql.execution.columnar.impl.ColumnFormatRelation -import org.apache.spark.sql.execution.row.RowTableScan -import org.apache.spark.sql.execution.{BufferedRowIterator, CodegenSupportOnExecutor, LeafExecNode, WholeStageCodegenExec} +import org.apache.spark.sql.execution.{BufferedRowIterator, CodegenSupportOnExecutor, LeafExecNode} import org.apache.spark.sql.store.CodeGeneration import org.apache.spark.sql.types._ @@ -41,7 +41,7 @@ final class ColumnBatchCreator( val columnTableName: String, val schema: StructType, val externalStore: ExternalStore, - val compressionCodec: String) extends Logging { + val compressionCodec: String) extends Logging with SparkSupport { def createAndStoreBatch(sc: ScanController, row: AbstractCompactExecRow, batchID: Long, bucketID: Int, @@ -81,7 +81,7 @@ final class ColumnBatchCreator( // the lookup key does not depend on tableName since the generated // code does not (which is passed in the references separately) val gen = CodeGeneration.compileCode("columnTable.batch", schema.fields, () => { - val tableScan = RowTableScan(schema.toAttributes, schema, + val tableScan = internals.rowTableScan(schema.toAttributes, schema, dataRDD = null, numBuckets = -1, partitionColumns = Nil, partitionColumnAliases = Nil, tableName, baseRelation = null, caseSensitive = true) // sending negative values for batch size and delta rows will create @@ -94,7 +94,7 @@ final class ColumnBatchCreator( // this is only used for local code generation while its RDD semantics // and related methods are all ignored val (ctx, code) = ExternalStoreUtils.codeGenOnExecutor( - WholeStageCodegenExec(insertPlan), insertPlan) + internals.newWholeStagePlan(insertPlan), insertPlan) val references = ctx.references // also push the index of batchId reference at the end which can be // used by caller to update the reference objects before execution @@ -149,7 +149,7 @@ final class ColumnBatchCreator( // this is only used for local code generation while its RDD semantics // and related methods are all ignored val (ctx, code) = ExternalStoreUtils.codeGenOnExecutor( - WholeStageCodegenExec(insertPlan), insertPlan) + internals.newWholeStagePlan(insertPlan), insertPlan) val references = ctx.references.toArray (code, references) }) @@ -176,7 +176,7 @@ trait ColumnBatchRowsBuffer { * code to closure callbacks model as required by StratifiedSampler.append */ case class CallbackColumnInsert(_schema: StructType) - extends LeafExecNode with CodegenSupportOnExecutor { + extends LeafExecNode with CodegenSupportOnExecutor with SparkSupport { override def output: Seq[Attribute] = _schema.toAttributes @@ -190,34 +190,32 @@ case class CallbackColumnInsert(_schema: StructType) override protected def doProduce(ctx: CodegenContext): String = { val row = ctx.freshName("row") - val hasResults = ctx.freshName("hasResults") - val clearResults = ctx.freshName("clearResults") - val rowsBuffer = ctx.freshName("rowsBuffer") + var hasResults = ctx.freshName("hasResults") + var clearResults = ctx.freshName("clearResults") val rowsBufferClass = classOf[ColumnBatchRowsBuffer].getName - ctx.addMutableState(rowsBufferClass, rowsBuffer, "") + val rowsBuffer = internals.addClassField(ctx, rowsBufferClass, "rowsBuffer") // add bucketId variable set to -1 by default - bucketIdTerm = ctx.freshName("bucketId") + bucketIdTerm = internals.addClassField(ctx, "int", "bucketId", v => s"$v = -1;") resetInsertions = ctx.freshName("resetInsertionsCount") - ctx.addMutableState("int", bucketIdTerm, s"$bucketIdTerm = -1;") val columnsExpr = output.zipWithIndex.map { case (a, i) => BoundReference(i, a.dataType, a.nullable) } ctx.INPUT_ROW = row ctx.currentVars = null val columnsInput = ctx.generateExpressions(columnsExpr) - ctx.addNewFunction(hasResults, + hasResults = internals.addFunction(ctx, hasResults, s""" |public final boolean $hasResults() { | return !currentRows.isEmpty(); |} """.stripMargin) - ctx.addNewFunction(clearResults, + clearResults = internals.addFunction(ctx, clearResults, s""" |public final void $clearResults() { | currentRows.clear(); |} """.stripMargin) - ctx.addNewFunction("getRowsBuffer", + internals.addFunction(ctx, "getRowsBuffer", s""" |public $rowsBufferClass getRowsBuffer() throws java.io.IOException { | $clearResults(); // clear any old results @@ -229,7 +227,7 @@ case class CallbackColumnInsert(_schema: StructType) | } | return this.$rowsBuffer; |} - """.stripMargin) + """.stripMargin, inlineToOuterClass = true) // create the rows buffer implementation as an inner anonymous // class so that it can be fit easily in the iterator model of // doProduce/doConsume having access to all the final local variables diff --git a/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnDeleteExec.scala b/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnDeleteExec.scala index fa07563fc1..9e42b31408 100644 --- a/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnDeleteExec.scala +++ b/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnDeleteExec.scala @@ -89,29 +89,23 @@ case class ColumnDeleteExec(child: SparkPlan, columnTable: String, override def doConsume(ctx: CodegenContext, input: Seq[ExprCode], row: ExprCode): String = { - val position = ctx.freshName("position") - val lastColumnBatchId = ctx.freshName("lastColumnBatchId") - val lastBucketId = ctx.freshName("lastBucketId") - val lastNumRows = ctx.freshName("lastNumRows") - val deleteEncoder = ctx.freshName("deleteEncoder") - batchOrdinal = ctx.freshName("batchOrdinal") finishDelete = ctx.freshName("finishDelete") deleteMetric = if (onExecutor) null else metricTerm(ctx, "numDeleteColumnBatchRows") val deleteEncoderClass = classOf[ColumnDeleteEncoder].getName - val initializeEncoder = + val deleteEncoder = internals.addClassField(ctx, deleteEncoderClass, "deleteEncoder") + val initializeEncoder: String => String = position => s""" |$deleteEncoder = new $deleteEncoderClass(); |$position = $deleteEncoder.initialize(8); // start with a default size """.stripMargin - - ctx.addMutableState(deleteEncoderClass, deleteEncoder, "") - ctx.addMutableState("int", position, initializeEncoder) - ctx.addMutableState("int", batchOrdinal, "") - ctx.addMutableState("long", lastColumnBatchId, s"$lastColumnBatchId = $invalidUUID;") - ctx.addMutableState("int", lastBucketId, "") - ctx.addMutableState("int", lastNumRows, "") + val position = internals.addClassField(ctx, "int", "position", initializeEncoder) + batchOrdinal = internals.addClassField(ctx, "int", "batchOrdinal") + val lastColumnBatchId = internals.addClassField(ctx, "long", "lastColumnBatchId", + v => s"$v = $invalidUUID;") + val lastBucketId = internals.addClassField(ctx, "int", "lastBucketId") + val lastNumRows = internals.addClassField(ctx, "int", "lastNumRows") val tableName = ctx.addReferenceObj("columnTable", columnTable, "java.lang.String") @@ -130,17 +124,17 @@ case class ColumnDeleteExec(child: SparkPlan, columnTable: String, ctx.currentVars = null val keyVars = keysInput.takeRight(4) - val ordinalIdVar = keyVars.head.value - val batchIdVar = keyVars(1).value - val bucketVar = keyVars(2).value - val numRowsVar = keyVars(3).value + val ordinalIdVar = internals.exprCodeValue(keyVars.head) + val batchIdVar = internals.exprCodeValue(keyVars(1)) + val bucketVar = internals.exprCodeValue(keyVars(2)) + val numRowsVar = internals.exprCodeValue(keyVars(3)) val externalStoreTerm = ctx.addReferenceObj("externalStore", externalStore) val keyVarsCode = evaluateVariables(keysInput) // row buffer needs to select the rowId and partitioning columns so drop last three val rowConsume = super.doConsume(ctx, keysInput.dropRight(3), StructType(getUpdateSchema(keyColumns.dropRight(3)))) - ctx.addNewFunction(finishDelete, + finishDelete = internals.addFunction(ctx, finishDelete, s""" |private void $finishDelete(long batchId, int bucketId, int numRows) { | if (batchId == $invalidUUID || batchId != $lastColumnBatchId) { @@ -157,7 +151,7 @@ case class ColumnDeleteExec(child: SparkPlan, columnTable: String, | $lastColumnBatchId, ${compressionCodec.id}, new scala.Some($connTerm)); | $result += $batchOrdinal; | ${if (deleteMetric eq null) "" else s"$deleteMetric.${metricAdd(batchOrdinal)};"} - | $initializeEncoder + | ${initializeEncoder(position)} | $lastColumnBatchId = batchId; | $lastBucketId = bucketId; | $lastNumRows = numRows; diff --git a/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnExec.scala b/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnExec.scala index 886b65beac..d903cae937 100644 --- a/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnExec.scala +++ b/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnExec.scala @@ -26,7 +26,6 @@ import org.apache.spark.sql.catalyst.expressions.codegen.CodegenContext import org.apache.spark.sql.catalyst.expressions.{Attribute, SortOrder} import org.apache.spark.sql.catalyst.plans.physical.{ClusteredDistribution, Distribution} import org.apache.spark.sql.collection.Utils -import org.apache.spark.sql.execution.WholeStageCodegenExec import org.apache.spark.sql.execution.columnar.impl.{JDBCSourceAsColumnarStore, SnapshotConnectionListener} import org.apache.spark.sql.execution.row.RowExec import org.apache.spark.sql.store.StoreUtils @@ -69,12 +68,10 @@ trait ColumnExec extends RowExec { val externalStoreTerm = ctx.addReferenceObj("externalStore", externalStore) val listenerClass = classOf[SnapshotConnectionListener].getName val storeClass = classOf[JDBCSourceAsColumnarStore].getName - taskListener = ctx.freshName("taskListener") - connTerm = ctx.freshName("connection") val getContext = Utils.genTaskContextFunction(ctx) - ctx.addMutableState(listenerClass, taskListener, "") - ctx.addMutableState(connectionClass, connTerm, "") + taskListener = internals.addClassField(ctx, listenerClass, "taskListener") + connTerm = internals.addClassField(ctx, connectionClass, "connection") val initCode = s""" @@ -90,7 +87,7 @@ trait ColumnExec extends RowExec { override protected def doExecute(): RDD[InternalRow] = { // don't expect code generation to fail try { - WholeStageCodegenExec(this).execute() + internals.newWholeStagePlan(this).execute() } finally { sqlContext.sparkSession.asInstanceOf[SnappySession].clearWriteLockOnTable() diff --git a/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnInsertExec.scala b/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnInsertExec.scala index e3eaea041c..37904c2edd 100644 --- a/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnInsertExec.scala +++ b/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnInsertExec.scala @@ -23,7 +23,7 @@ import org.eclipse.collections.impl.set.mutable.UnifiedSet import org.apache.spark.TaskContext import org.apache.spark.rdd.RDD -import org.apache.spark.sql.SnappySession +import org.apache.spark.sql.{SnappySession, SparkSupport} import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode, GenerateUnsafeProjection} import org.apache.spark.sql.catalyst.expressions.{Attribute, BoundReference, Expression, Literal} @@ -31,7 +31,7 @@ import org.apache.spark.sql.catalyst.util.{SerializedArray, SerializedMap, Seria import org.apache.spark.sql.collection.Utils import org.apache.spark.sql.execution.columnar.encoding.{BitSet, ColumnEncoder, ColumnEncoding, ColumnStatsSchema} import org.apache.spark.sql.execution.columnar.impl.BaseColumnFormatRelation -import org.apache.spark.sql.execution.{SparkPlan, TableExec, WholeStageCodegenExec} +import org.apache.spark.sql.execution.{SparkPlan, TableExec} import org.apache.spark.sql.sources.DestroyRelation import org.apache.spark.sql.store.CompressionCodecId import org.apache.spark.sql.types._ @@ -56,7 +56,7 @@ case class ColumnInsertExec(child: SparkPlan, partitionColumns: Seq[String], onExecutor = false, relation.schema, relation.externalStore, useMemberVariables = false) } - @transient private var encoderCursorTerms: Seq[(String, String)] = _ + @transient private var encoderCursorTerms: Array[(String, String)] = _ @transient private var maxDeltaRowsTerm: String = _ @transient private var batchSizeTerm: String = _ @transient private var defaultBatchSizeTerm: String = _ @@ -74,8 +74,6 @@ case class ColumnInsertExec(child: SparkPlan, partitionColumns: Seq[String], @transient private var initEncoders: String = _ @transient private val MAX_CURSOR_DECLARATIONS = 30 - @transient private var cursorsArrayTerm: String = _ - @transient private var cursorsArrayCreate: String = _ @transient private var encoderArrayTerm: String = _ @transient private var cursorArrayTerm: String = _ @transient private var catalogVersion: String = _ @@ -119,8 +117,8 @@ case class ColumnInsertExec(child: SparkPlan, partitionColumns: Seq[String], */ private def addBatchSizeAndCloseEncoders(ctx: CodegenContext, closeEncoders: String): String = { - val closeEncodersFunction = ctx.freshName("closeEncoders") - ctx.addNewFunction(closeEncodersFunction, + var closeEncodersFunction = ctx.freshName("closeEncoders") + closeEncodersFunction = internals.addFunction(ctx, closeEncodersFunction, s""" |private void $closeEncodersFunction() { | $closeEncoders @@ -131,7 +129,7 @@ case class ColumnInsertExec(child: SparkPlan, partitionColumns: Seq[String], val listenerClass = classOf[TaskCompletionListener].getName val getContext = Utils.genTaskContextFunction(ctx) - ctx.addMutableState("int", defaultBatchSizeTerm, + internals.addClassField(ctx, "int", defaultBatchSizeTerm, _ => s""" |if ($getContext() != null) { | $getContext().addTaskCompletionListener(new $listenerClass() { @@ -141,7 +139,7 @@ case class ColumnInsertExec(child: SparkPlan, partitionColumns: Seq[String], | } | }); |} - """.stripMargin) + """.stripMargin, useFreshName = false) s""" |if ($numInsertions >= 0 && $getContext() == null) { | $closeEncodersFunction(); @@ -163,59 +161,43 @@ case class ColumnInsertExec(child: SparkPlan, partitionColumns: Seq[String], classOf[StructType].getName) val schemaLength = tableSchema.length - encoderArrayTerm = ctx.freshName("encoderArray") - cursorArrayTerm = ctx.freshName("cursorArray") - numInsertions = ctx.freshName("numInsertions") - ctx.addMutableState("long", numInsertions, s"$numInsertions = -1L;") + cursorArrayTerm = internals.addClassField(ctx, "long[]", "cursorArray", + cur => s"this.$cur = new long[$schemaLength];") + numInsertions = internals.addClassField(ctx, "long", "numInsertions", v => s"$v = -1L;") maxDeltaRowsTerm = ctx.freshName("maxDeltaRows") - batchSizeTerm = ctx.freshName("currentBatchSize") - txIdConnArray = ctx.freshName("txIdConnArray") + txIdConnArray = internals.addClassField(ctx, "Object[]", "txIdConnArray") txId = ctx.freshName("txId") conn = ctx.freshName("conn") - val batchSizeDeclaration = if (true) { - ctx.addMutableState("int", batchSizeTerm, s"$batchSizeTerm = 0;") - "" - } else { - s"int $batchSizeTerm = 0;" - } defaultBatchSizeTerm = ctx.freshName("defaultBatchSize") + batchSizeTerm = internals.addClassField(ctx, "int", "currentBatchSize", v => s"$v = 0;") val defaultRowSize = ctx.freshName("defaultRowSize") + + val initEncoderCode: String => String = encoderArray => + s""" + |this.$encoderArray[i] = $encodingClass.getColumnEncoder( + | $schemaTerm.fields()[i]); + """.stripMargin + encoderArrayTerm = internals.addClassField(ctx, s"$encoderClass[]", "encoderArray", enc => + s""" + |this.$enc = new $encoderClass[$schemaLength]; + |${loop(initEncoderCode(enc), schemaLength)} + """.stripMargin) + val childProduce = doChildProduce(ctx) child match { case c: CallbackColumnInsert => - ctx.addNewFunction(c.resetInsertions, + internals.addFunction(ctx, c.resetInsertions, s""" |public final void ${c.resetInsertions}() { | $batchSizeTerm = 0; | $numInsertions = -1; |} - """.stripMargin) + """.stripMargin, inlineToOuterClass = true) batchBucketIdTerm = Some(c.bucketIdTerm) case _ => } - val initEncoderCode = - s""" - |this.$encoderArrayTerm[i] = $encodingClass.getColumnEncoder( - | $schemaTerm.fields()[i]); - """.stripMargin - - val initEncoderArray = loop(initEncoderCode, schemaLength) - - ctx.addMutableState(s"$encoderClass[]", - encoderArrayTerm, - s""" - |this.$encoderArrayTerm = - | new $encoderClass[$schemaLength]; - |$initEncoderArray - """.stripMargin) - - ctx.addMutableState("long[]", cursorArrayTerm, - s""" - |this.$cursorArrayTerm = new long[$schemaLength]; - """.stripMargin) - val encoderLoopCode = s"$defaultRowSize += " + s"$encoderArrayTerm[i].defaultSize($schemaTerm.fields()[i].dataType());" @@ -227,13 +209,13 @@ case class ColumnInsertExec(child: SparkPlan, partitionColumns: Seq[String], s"if ($numInsertions >= 0) return" } // no need to stop in iteration at any point - ctx.addNewFunction("shouldStop", + internals.addFunction(ctx, "shouldStop", s""" |@Override |protected final boolean shouldStop() { | return false; |} - """.stripMargin) + """.stripMargin, inlineToOuterClass = true) val closeEncoders = loop( s"if ($encoderArrayTerm[i] != null) $encoderArrayTerm[i].close();", @@ -243,11 +225,10 @@ case class ColumnInsertExec(child: SparkPlan, partitionColumns: Seq[String], s""" |$checkEnd; // already done | - |final Object[] $txIdConnArray = $beginSnapshotTx(); + |$txIdConnArray = $beginSnapshotTx(); | |boolean success = false; |try { - |$batchSizeDeclaration |if ($numInsertions < 0) { | $numInsertions = 0; | int $defaultRowSize = 0; @@ -269,7 +250,7 @@ case class ColumnInsertExec(child: SparkPlan, partitionColumns: Seq[String], |$closeForNoContext |${if (numInsertedRowsMetric eq null) "" else s"$numInsertedRowsMetric.${metricAdd(numInsertions)};"} - |${consume(ctx, Seq(ExprCode("", "false", numInsertions)))} + |${consume(ctx, Seq(internals.newExprCode("", "false", numInsertions, LongType)))} |success = true; |} |finally { @@ -315,72 +296,55 @@ case class ColumnInsertExec(child: SparkPlan, partitionColumns: Seq[String], else metricTerm(ctx, "numInsertedRows") schemaTerm = ctx.addReferenceObj("schema", tableSchema, classOf[StructType].getName) - encoderCursorTerms = tableSchema.map { _ => - (ctx.freshName("encoder"), ctx.freshName("cursor")) - } - numInsertions = ctx.freshName("numInsertions") - ctx.addMutableState("long", numInsertions, s"$numInsertions = -1L;") + encoderCursorTerms = new Array[(String, String)](tableSchema.length) + numInsertions = internals.addClassField(ctx, "long", "numInsertions", v => s"$v = -1L;") maxDeltaRowsTerm = ctx.freshName("maxDeltaRows") - batchSizeTerm = ctx.freshName("currentBatchSize") - txIdConnArray = ctx.freshName("txIdConnArray") + txIdConnArray = internals.addClassField(ctx, "Object[]", "txIdConnArray") txId = ctx.freshName("txId") conn = ctx.freshName("conn") - val batchSizeDeclaration = if (useMemberVariables) { - ctx.addMutableState("int", batchSizeTerm, s"$batchSizeTerm = 0;") - "" - } else { - s"int $batchSizeTerm = 0;" - } + batchSizeTerm = internals.addClassField(ctx, "int", "currentBatchSize", v => s"$v = 0;") defaultBatchSizeTerm = ctx.freshName("defaultBatchSize") val defaultRowSize = ctx.freshName("defaultRowSize") + val closeEncoders = new StringBuilder + val declarations = tableSchema.indices.map { i => + val encoder = internals.addClassField(ctx, encoderClass, "encoder", + enc => s"this.$enc = $encodingClass.getColumnEncoder($schemaTerm.fields()[$i]);") + val cursor = internals.addClassField(ctx, "long", "cursor", v => s"$v = 0L;") + encoderCursorTerms(i) = (encoder, cursor) + val declaration = + s"$defaultRowSize += $encoder.defaultSize($schemaTerm.fields()[$i].dataType());" + closeEncoders.append(s"if ($encoder != null) $encoder.close();\n") + declaration + } + val childProduce = doChildProduce(ctx) child match { case c: CallbackColumnInsert => - ctx.addNewFunction(c.resetInsertions, + internals.addFunction(ctx, c.resetInsertions, s""" |public final void ${c.resetInsertions}() { | $batchSizeTerm = 0; | $numInsertions = -1; |} - """.stripMargin) + """.stripMargin, inlineToOuterClass = true) batchBucketIdTerm = Some(c.bucketIdTerm) case _ => } - val closeEncoders = new StringBuilder - val (declarations, cursorDeclarations) = encoderCursorTerms.indices.map { i => - val (encoder, cursor) = encoderCursorTerms(i) - ctx.addMutableState(encoderClass, encoder, - s""" - |this.$encoder = $encodingClass.getColumnEncoder( - | $schemaTerm.fields()[$i]); - """.stripMargin) - val cursorDeclaration = if (useMemberVariables) { - ctx.addMutableState("long", cursor, s"$cursor = 0L;") - "" - } else s"long $cursor = 0L;" - val declaration = - s""" - |final $encoderClass $encoder = this.$encoder; - |$defaultRowSize += $encoder.defaultSize($schemaTerm.fields()[$i].dataType()); - """.stripMargin - closeEncoders.append(s"if ($encoder != null) $encoder.close();\n") - (declaration, cursorDeclaration) - }.unzip val checkEnd = if (useMemberVariables) { "if (!currentRows.isEmpty()) return" } else { s"if ($numInsertions >= 0) return" } // no need to stop in iteration at any point - ctx.addNewFunction("shouldStop", + internals.addFunction(ctx, "shouldStop", s""" |@Override |protected final boolean shouldStop() { | return false; |} - """.stripMargin) + """.stripMargin, inlineToOuterClass = true) val closeForNoContext = addBatchSizeAndCloseEncoders(ctx, closeEncoders.toString()) val useBatchSize = if (columnBatchSize > 0) columnBatchSize else ExternalStoreUtils.sizeAsBytes(Property.ColumnBatchSize.defaultValue.get, @@ -388,11 +352,9 @@ case class ColumnInsertExec(child: SparkPlan, partitionColumns: Seq[String], val resetConnectionAttributes = resetConnectionAttributesCode() s""" |$checkEnd; // already done - |final Object[] $txIdConnArray = $beginSnapshotTx(); + |$txIdConnArray = $beginSnapshotTx(); |boolean success = false; |try { - |$batchSizeDeclaration - |${cursorDeclarations.mkString("\n")} |if ($numInsertions < 0) { | $numInsertions = 0; | int $defaultRowSize = 0; @@ -406,7 +368,6 @@ case class ColumnInsertExec(child: SparkPlan, partitionColumns: Seq[String], | $childProduce |} |if ($batchSizeTerm > 0) { - | $cursorsArrayCreate | $storeColumnBatch($columnMaxDeltaRows, $storeColumnBatchArgs, | new scala.Some((java.sql.Connection)$txIdConnArray[0])); | $batchSizeTerm = 0; @@ -414,7 +375,7 @@ case class ColumnInsertExec(child: SparkPlan, partitionColumns: Seq[String], |$closeForNoContext |${if (numInsertedRowsMetric eq null) "" else s"$numInsertedRowsMetric.${metricAdd(numInsertions)};"} - |${consume(ctx, Seq(ExprCode("", "false", numInsertions)))} + |${consume(ctx, Seq(internals.newExprCode("", "false", numInsertions, LongType)))} |success = true; |} |finally { @@ -478,8 +439,7 @@ case class ColumnInsertExec(child: SparkPlan, partitionColumns: Seq[String], | $body |} """.stripMargin - ctx.addNewFunction(name, code) - name + internals.addFunction(ctx, name, code) } s""" |${functions.map(name => s"$name();").mkString("\n")} @@ -491,10 +451,10 @@ case class ColumnInsertExec(child: SparkPlan, partitionColumns: Seq[String], */ private def setColumn(ctx: CodegenContext, row: String, dataType: DataType, ordinal: Int, value: String): String = { - val jt = ctx.javaType(dataType) + val jt = internals.javaType(dataType, ctx) dataType match { - case _ if ctx.isPrimitiveType(jt) => - s"$row.set${ctx.primitiveTypeName(jt)}($ordinal, $value)" + case _ if internals.isPrimitiveType(jt, ctx) => + s"$row.set${internals.primitiveTypeName(jt, ctx)}($ordinal, $value)" case t: DecimalType => s"$row.setDecimal($ordinal, $value, ${t.precision})" case udt: UserDefinedType[_] => setColumn(ctx, row, udt.sqlType, ordinal, value) case _ => s"$row.update($ordinal, $value)" @@ -508,32 +468,29 @@ case class ColumnInsertExec(child: SparkPlan, partitionColumns: Seq[String], val columnBatch = ctx.freshName("columnBatch") val sizeTerm = ctx.freshName("size") val sizeExceededTerm = ctx.freshName("sizeExceeded") - cursorsArrayTerm = ctx.freshName("cursors") - - val mutableRow = ctx.freshName("mutableRow") - ctx.addMutableState("SpecificInternalRow", mutableRow, - s"$mutableRow = new SpecificInternalRow($schemaTerm);") + val mutableRow = internals.addClassField(ctx, "SpecificInternalRow", "mutableRow", + v => s"$v = new SpecificInternalRow($schemaTerm);") val rowWriteExprs = schema.indices.map { i => val field = schema(i) val dataType = field.dataType val evaluationCode = input(i) - evaluationCode.code + + evaluationCode.code.toString + s""" - if (${evaluationCode.isNull}) { + if (${internals.exprCodeIsNull(evaluationCode)}) { $mutableRow.setNullAt($i); } else { - ${setColumn(ctx, mutableRow, dataType, i, evaluationCode.value)}; + ${setColumn(ctx, mutableRow, dataType, i, internals.exprCodeValue(evaluationCode))}; } """ } - val allRowWriteExprs = ctx.splitExpressions(ctx.INPUT_ROW, rowWriteExprs) + val allRowWriteExprs = internals.splitExpressions(ctx, rowWriteExprs) ctx.INPUT_ROW = mutableRow val rowReadExprs = schema.zipWithIndex.map { case (field, ordinal) => - ExprCode("", s"${ctx.INPUT_ROW}.isNullAt($ordinal)", - ctx.getValue(ctx.INPUT_ROW, field.dataType, ordinal.toString)) + internals.newExprCode("", s"${ctx.INPUT_ROW}.isNullAt($ordinal)", + internals.getValue(ctx.INPUT_ROW, field.dataType, ordinal.toString, ctx), IntegerType) } val columnWrite = schema.indices.map { i => @@ -573,21 +530,22 @@ case class ColumnInsertExec(child: SparkPlan, partitionColumns: Seq[String], val tableName = ctx.addReferenceObj("columnTable", columnTable, "java.lang.String") + val cursorArray = ctx.freshName("cursorArray") val bufferLoopCode = - s"""$buffers[i] = $encoderArrayTerm[i].finish($cursorArrayTerm[i]);\n""".stripMargin + s"$buffers[i] = $encoderArrayTerm[i].finish($cursorArray[i]);\n" val buffersCode = loop(bufferLoopCode, schema.length) val (statsSchema, stats) = columnStats.unzip val statsEv = ColumnWriter.genStatsRow(ctx, batchSizeTerm, stats, statsSchema) - val statsRow = statsEv.value + val statsRow = internals.exprCodeValue(statsEv) storeColumnBatch = ctx.freshName("storeColumnBatch") - ctx.addNewFunction(storeColumnBatch, + storeColumnBatch = internals.addFunction(ctx, storeColumnBatch, s""" |private final void $storeColumnBatch(int $maxDeltaRowsTerm, - | int $batchSizeTerm, long[] $cursorArrayTerm, scala.Option $conn) { + | int $batchSizeTerm, long[] $cursorArray, scala.Option $conn) { | // create statistics row - | ${statsEv.code.trim} + | ${statsEv.code.toString.trim} | // create ColumnBatch and insert | final java.nio.ByteBuffer[] $buffers = | new java.nio.ByteBuffer[${schema.length}]; @@ -604,21 +562,21 @@ case class ColumnInsertExec(child: SparkPlan, partitionColumns: Seq[String], generateBeginSnapshotTx(ctx, externalStoreTerm) commitSnapshotTx = ctx.freshName("commitSnapshotTx") - ctx.addNewFunction(commitSnapshotTx, + commitSnapshotTx = internals.addFunction(ctx, commitSnapshotTx, s""" |private final void $commitSnapshotTx(String $txId, scala.Option $conn) { | $externalStoreTerm.commitTx($txId, false, $conn); |} """.stripMargin) rollbackSnapshotTx = ctx.freshName("rollbackSnapshotTx") - ctx.addNewFunction(rollbackSnapshotTx, + rollbackSnapshotTx = internals.addFunction(ctx, rollbackSnapshotTx, s""" |private final void $rollbackSnapshotTx(String $txId, scala.Option $conn) { | $externalStoreTerm.rollbackTx($txId, $conn); |} """.stripMargin) closeConnection = ctx.freshName("closeConnection") - ctx.addNewFunction(closeConnection, + closeConnection = internals.addFunction(ctx, closeConnection, s""" |private final void $closeConnection(scala.Option $conn) { | $externalStoreTerm.closeConnection($conn); @@ -658,14 +616,14 @@ case class ColumnInsertExec(child: SparkPlan, partitionColumns: Seq[String], catalogVersion = ctx.addReferenceObj("catalogVersion", catalogSchemaVersion) if (!onExecutor && Utils.isSmartConnectorMode(sqlContext.sparkContext)) { // on smart connector also set connection attributes to check catalog schema version - ctx.addNewFunction(beginSnapshotTx, + beginSnapshotTx = internals.addFunction(ctx, beginSnapshotTx, s""" |private final Object[] $beginSnapshotTx() throws java.io.IOException { | return $externalStoreTerm.beginTxSmartConnector(false, $catalogVersion); |} """.stripMargin) } else { - ctx.addNewFunction(beginSnapshotTx, + beginSnapshotTx = internals.addFunction(ctx, beginSnapshotTx, s""" |private final Object[] $beginSnapshotTx() { | return $externalStoreTerm.beginTx(false); @@ -687,9 +645,7 @@ case class ColumnInsertExec(child: SparkPlan, partitionColumns: Seq[String], val sizeTerm = ctx.freshName("size") val sizeExceededTerm = ctx.freshName("sizeExceeded") - val encoderClass = classOf[ColumnEncoder].getName val buffersCode = new StringBuilder - val encoderCursorDeclarations = new StringBuilder val batchFunctionDeclarations = new StringBuilder val batchFunctionCall = new StringBuilder val calculateSize = new StringBuilder @@ -700,8 +656,6 @@ case class ColumnInsertExec(child: SparkPlan, partitionColumns: Seq[String], s"$schemaTerm.fields()[$i], $defaultBatchSizeTerm, true);" buffersCode.append( s"$buffers[$i] = $encoderTerm.finish($cursorTerm);\n") - encoderCursorDeclarations.append( - s"final $encoderClass $encoderTerm = this.$encoderTerm;\n") batchFunctionDeclarations.append(s"long $cursorTerm,\n") batchFunctionCall.append(s"$cursorTerm,\n") @@ -716,7 +670,6 @@ case class ColumnInsertExec(child: SparkPlan, partitionColumns: Seq[String], batchFunctionDeclarations.setLength( batchFunctionDeclarations.length - 2) batchFunctionCall.setLength(batchFunctionCall.length - 2) - cursorsArrayCreate = "" val columnBatchClass = classOf[ColumnBatch].getName val externalStoreTerm = ctx.addReferenceObj("externalStore", externalStore) @@ -734,15 +687,14 @@ case class ColumnInsertExec(child: SparkPlan, partitionColumns: Seq[String], "java.lang.String") val (statsSchema, stats) = columnStats.unzip val statsEv = ColumnWriter.genStatsRow(ctx, batchSizeTerm, stats, statsSchema) - val statsRow = statsEv.value + val statsRow = internals.exprCodeValue(statsEv) storeColumnBatch = ctx.freshName("storeColumnBatch") - ctx.addNewFunction(storeColumnBatch, + storeColumnBatch = internals.addFunction(ctx, storeColumnBatch, s""" |private final void $storeColumnBatch(int $maxDeltaRowsTerm, | int $batchSizeTerm, ${batchFunctionDeclarations.toString()}, scala.Some $conn) { - | $encoderCursorDeclarations | // create statistics row - | ${statsEv.code.trim} + | ${statsEv.code.toString.trim} | // create ColumnBatch and insert | final java.nio.ByteBuffer[] $buffers = | new java.nio.ByteBuffer[${schema.length}]; @@ -757,21 +709,21 @@ case class ColumnInsertExec(child: SparkPlan, partitionColumns: Seq[String], """.stripMargin) generateBeginSnapshotTx(ctx, externalStoreTerm) commitSnapshotTx = ctx.freshName("commitSnapshotTx") - ctx.addNewFunction(commitSnapshotTx, + commitSnapshotTx = internals.addFunction(ctx, commitSnapshotTx, s""" |private final void $commitSnapshotTx(String $txId, scala.Option $conn) { | $externalStoreTerm.commitTx($txId, false, $conn); |} """.stripMargin) rollbackSnapshotTx = ctx.freshName("rollbackSnapshotTx") - ctx.addNewFunction(rollbackSnapshotTx, + rollbackSnapshotTx = internals.addFunction(ctx, rollbackSnapshotTx, s""" |private final void $rollbackSnapshotTx(String $txId, scala.Option $conn) { | $externalStoreTerm.rollbackTx($txId, $conn); |} """.stripMargin) closeConnection = ctx.freshName("closeConnection") - ctx.addNewFunction(closeConnection, + closeConnection = internals.addFunction(ctx, closeConnection, s""" |private final void $closeConnection(scala.Option $conn) { | $externalStoreTerm.closeConnection($conn); @@ -789,7 +741,6 @@ case class ColumnInsertExec(child: SparkPlan, partitionColumns: Seq[String], | $sizeExceededTerm = $sizeTerm >= $columnBatchSize; | } | if ($sizeExceededTerm) { - | $cursorsArrayCreate | $storeColumnBatch(-1, $storeColumnBatchArgs, | new scala.Some((java.sql.Connection)$txIdConnArray[0])); | $batchSizeTerm = 0; @@ -805,14 +756,12 @@ case class ColumnInsertExec(child: SparkPlan, partitionColumns: Seq[String], override protected def doExecute(): RDD[InternalRow] = { // don't expect code generation to fail try { - WholeStageCodegenExec(this).execute() - } - finally { + internals.newWholeStagePlan(this).execute() + } finally { sqlContext.sparkSession.asInstanceOf[SnappySession].clearWriteLockOnTable() } } - private def genCodeColumnWrite(ctx: CodegenContext, dataType: DataType, nullable: Boolean, encoder: String, cursorTerm: String, ev: ExprCode): String = { @@ -825,7 +774,7 @@ case class ColumnInsertExec(child: SparkPlan, partitionColumns: Seq[String], s"batchSize=$columnBatchSize maxDeltaRows=$columnMaxDeltaRows compression=$compressionCodec" } -object ColumnWriter { +object ColumnWriter extends SparkSupport { /** * Supported types for which column statistics are maintained and can be used @@ -844,7 +793,7 @@ object ColumnWriter { var canBeNull = false val nullCount = ctx.freshName("nullCount") val sqlType = Utils.getSQLDataType(field.dataType) - val jt = ctx.javaType(sqlType) + val jt = internals.javaType(sqlType, ctx) val (lCode, uCode) = sqlType match { case BooleanType => (s"final boolean $lower = $encoder.lowerLong() > 0;", @@ -880,19 +829,21 @@ object ColumnWriter { } else (lCode, uCode) (ColumnStatsSchema(field.name, field.dataType, nullCountNullable).schema, Seq( - ExprCode(lowerCode, lowerIsNull, lower), - ExprCode(upperCode, upperIsNull, upper), - ExprCode(s"final int $nullCount = $encoder.nullCount();", "false", nullCount))) + internals.newExprCode(lowerCode, lowerIsNull, lower, field.dataType), + internals.newExprCode(upperCode, upperIsNull, upper, field.dataType), + internals.newExprCode(s"final int $nullCount = $encoder.nullCount();", "false", + nullCount, IntegerType))) } def genStatsRow(ctx: CodegenContext, batchSizeTerm: String, stats: Seq[Seq[ExprCode]], statsSchema: Seq[Seq[Attribute]]): ExprCode = { - val statsVars = ExprCode("", "false", batchSizeTerm) +: stats.flatten + val statsVars = internals.newExprCode(code = "", isNull = "false", batchSizeTerm, + IntegerType) +: stats.flatten val statsExprs = (ColumnStatsSchema.COUNT_ATTRIBUTE +: statsSchema.flatten) .zipWithIndex.map { case (a, i) => a.dataType match { // some types will always be null so avoid unnecessary generated code - case _ if statsVars(i).isNull == "true" => Literal(null, NullType) + case _ if internals.exprCodeIsNull(statsVars(i)) == "true" => Literal(null, NullType) case _ => BoundReference(i, a.dataType, a.nullable) } } @@ -906,12 +857,12 @@ object ColumnWriter { ev: ExprCode, batchSizeTerm: String, offsetTerm: String = null, baseOffsetTerm: String = null): String = { val sqlType = Utils.getSQLDataType(dataType) - val jt = ctx.javaType(sqlType) - var isNull = ev.isNull - val input = ev.value + val jt = internals.javaType(sqlType, ctx) + var isNull = internals.exprCodeIsNull(ev) + val input = internals.exprCodeValue(ev) val writeValue = sqlType match { - case _ if ctx.isPrimitiveType(jt) => - val typeName = ctx.primitiveTypeName(jt) + case _ if internals.isPrimitiveType(jt, ctx) => + val typeName = internals.primitiveTypeName(jt, ctx) if (offsetTerm eq null) { s"$cursorTerm = $encoder.write$typeName($cursorTerm, $input);" } else { @@ -1135,7 +1086,7 @@ object ColumnWriter { baseDataOffset: String, skipBytes: Int): String = { // scalastyle:on - val getter = ctx.getValue(input, dt, index) + val getter = internals.getValue(input, dt, index, ctx) val bitSetClass = BitSet.getClass.getName val fieldOffset = ctx.freshName("fieldOffset") val value = ctx.freshName("value") @@ -1144,8 +1095,8 @@ object ColumnWriter { s""" |final long $fieldOffset = $baseDataOffset + ($index << 3); |${genCodeColumnWrite(ctx, dt, nullable = false, encoder, encoder, - cursorTerm, ExprCode("", "false", value), batchSizeTerm, - fieldOffset, baseOffset)} + cursorTerm, internals.newExprCode("", "false", value, IntegerType), + batchSizeTerm, fieldOffset, baseOffset)} """.stripMargin val (checkNull, assignValue) = dt match { case d: DecimalType => val checkNull = @@ -1159,14 +1110,14 @@ object ColumnWriter { } if (canBeNull) { s""" - |final ${ctx.javaType(dt)} $value; + |final ${internals.javaType(dt, ctx)} $value; |if ($checkNull) { | $bitSetClass.MODULE$$.set($encoder.buffer(), | $encoder.baseOffset() + $baseOffset, $index + ${skipBytes << 3}); |} else {$assignValue$serializeValue} """.stripMargin } else { - s"final ${ctx.javaType(dt)} $value = $getter;$serializeValue" + s"final ${internals.javaType(dt, ctx)} $value = $getter;$serializeValue" } } } diff --git a/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnTableScan.scala b/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnTableScan.scala index 899b5b1f06..1dbd5c6b7a 100644 --- a/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnTableScan.scala +++ b/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnTableScan.scala @@ -41,7 +41,7 @@ import scala.reflect.ClassTag import io.snappydata.ResultSetWithNull import org.apache.spark.rdd.{RDD, UnionPartition} -import org.apache.spark.sql.SnappySession +import org.apache.spark.sql.{SnappySession, SparkSupport} import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.dsl.expressions._ import org.apache.spark.sql.catalyst.expressions._ @@ -64,7 +64,7 @@ import org.apache.spark.{Dependency, Logging, Partition, RangeDependency, SparkC * This plan overrides outputPartitioning and makes it inline with the * partitioning of the underlying DataSource. */ -private[sql] final case class ColumnTableScan( +abstract case class ColumnTableScan( output: Seq[Attribute], dataRDD: RDD[Any], otherRDDs: Seq[RDD[InternalRow]], @@ -86,10 +86,9 @@ private[sql] final case class ColumnTableScan( else "ColumnTableScan" } - override def sameResult(plan: SparkPlan): Boolean = plan match { - case r: ColumnTableScan => r.baseRelation.table == baseRelation.table && - r.numBuckets == numBuckets && r.schema == schema - case _ => false + @transient private lazy val session: Option[SnappySession] = sqlContext match { + case null => None + case c => Some(c.sparkSession.asInstanceOf[SnappySession]) } @transient private val MAX_SCHEMA_LENGTH = 40 @@ -135,10 +134,6 @@ private[sql] final case class ColumnTableScan( private lazy val otherRDDsPartitionIndex = rdd.getNumPartitions - - @transient private val session = - Option(sqlContext).map(_.sparkSession.asInstanceOf[SnappySession]) - override def inputRDDs(): Seq[RDD[InternalRow]] = { allRDDs.asInstanceOf[RDD[InternalRow]] :: Nil } @@ -153,8 +148,7 @@ private[sql] final case class ColumnTableScan( | $body |} """.stripMargin - ctx.addNewFunction(name, code) - name + internals.addFunction(ctx, name, code) } functions.map(name => s"$name();").mkString("\n") } @@ -162,25 +156,24 @@ private[sql] final case class ColumnTableScan( def convertExprToMethodCall(ctx: CodegenContext, expr: ExprCode, attr: Attribute, index: Int, batchOrdinal: String): ExprCode = { val retValName = ctx.freshName(s"col$index") - val nullVarForCol = ctx.freshName(s"nullVarForCol$index") - ctx.addMutableState("boolean", nullVarForCol, "") + val nullVarForCol = internals.addClassField(ctx, "boolean", s"nullVarForCol$index") val sqlType = Utils.getSQLDataType(attr.dataType) - val jt = ctx.javaType(sqlType) - val name = s"readValue_$index" + val jt = internals.javaType(sqlType, ctx) + var name = s"readValue_$index" val code = s""" |private $jt $name(int $batchOrdinal) { - | ${expr.code} - | $nullVarForCol = ${expr.isNull}; - | return ${expr.value}; + | ${expr.code.toString} + | $nullVarForCol = ${internals.exprCodeIsNull(expr)}; + | return ${internals.exprCodeValue(expr)}; |} """.stripMargin - ctx.addNewFunction(name, code) + name = internals.addFunction(ctx, name, code) val exprCode = s""" |$jt $retValName = $name($batchOrdinal); """.stripMargin - ExprCode(exprCode, s"$nullVarForCol", s"$retValName") + internals.newExprCode(exprCode, nullVarForCol, retValName, sqlType) } override def doProduce(ctx: CodegenContext): String = { @@ -194,31 +187,30 @@ private[sql] final case class ColumnTableScan( // It returns an iterator of iterators (row + column) // except when doing union with multiple RDDs where other // RDDs return iterator of UnsafeRows. - val rowInput = ctx.freshName("rowInput") - val colInput = ctx.freshName("colInput") - val rowInputSRR = ctx.freshName("rowInputSRR") - val input = ctx.freshName("input") - val inputIsRow = s"${input}IsRow" - val inputIsRowSRR = s"${input}IsRowSRR" - val inputIsOtherRDD = s"${input}IsOtherRDD" - val rs = ctx.freshName("resultSet") + var rowInput: String = null + var colInput: String = null + var rowInputSRR: String = null + var input: String = null + var inputIsRow: String = null + var inputIsRowSRR: String = null + var inputIsOtherRDD: String = null + var rs: String = null val rsIterClass = classOf[ResultSetTraversal].getName - val unsafeHolder = if (otherRDDs.isEmpty && !isForSampleReservoirAsRegion) null + var unsafeHolder = if (otherRDDs.isEmpty && !isForSampleReservoirAsRegion) null else ctx.freshName("unsafeHolder") val updatedColumnCount = metricTerm(ctx, "updatedColumnCount") val deletedBatchCount = metricTerm(ctx, "deletedBatchCount") val unsafeHolderClass = classOf[UnsafeRowHolder].getName val stratumRowClass = classOf[StratumInternalRow].getName - // TODO [sumedh]: Asif, why this special treatment for weightage column + // TODO [sumedh]: why this special treatment for weightage column // in the code here? Why not as a normal AttributeReference in the plan // (or an extension of it if some special treatment is required)? val wrappedRow = if (isForSampleReservoirAsRegion) ctx.freshName("wrappedRow") else null val (weightVarName, weightAssignCode) = if (output.exists(_.name.equalsIgnoreCase( Utils.WEIGHTAGE_COLUMN_NAME))) { - val varName = ctx.freshName("weightage") - ctx.addMutableState("long", varName, s"$varName = 0;") + val varName = internals.addClassField(ctx, "long", "weightage", v => s"$v = 0;") (varName, s"$varName = $wrappedRow.weight();") } else ("", "") @@ -227,35 +219,35 @@ private[sql] final case class ColumnTableScan( else classOf[ColumnBatchIteratorOnRS].getName if (otherRDDs.isEmpty) { if (isForSampleReservoirAsRegion) { - ctx.addMutableState(iteratorClass, rowInputSRR, - s"$rowInputSRR = ($iteratorClass)inputs[0].next();") - ctx.addMutableState(unsafeHolderClass, unsafeHolder, - s"$unsafeHolder = new $unsafeHolderClass();") - ctx.addMutableState("boolean", inputIsRowSRR, s"$inputIsRowSRR = true;") + rowInputSRR = internals.addClassField(ctx, iteratorClass, "rowInputSRR", + v => s"$v = ($iteratorClass)inputs[0].next();") + unsafeHolder = internals.addClassField(ctx, unsafeHolderClass, "unsafeHolder", + v => s"$v = new $unsafeHolderClass();") + inputIsRowSRR = internals.addClassField(ctx, "boolean", "inputIsRowSRR", + v => s"$v = true;") } - ctx.addMutableState(iteratorClass, rowInput, - s"$rowInput = ($iteratorClass)inputs[0].next();") - ctx.addMutableState(colIteratorClass, colInput, - s"$colInput = ($colIteratorClass)inputs[0].next();") - ctx.addMutableState("java.sql.ResultSet", rs, - s"$rs = (($rsIterClass)$rowInput).rs();") + rowInput = internals.addClassField(ctx, iteratorClass, "rowInput", + v => s"$v = ($iteratorClass)inputs[0].next();") + colInput = internals.addClassField(ctx, colIteratorClass, "colInput", + v => s"$v = ($colIteratorClass)inputs[0].next();") + rs = internals.addClassField(ctx, "java.sql.ResultSet", "resultSet", + v => s"$v = (($rsIterClass)$rowInput).rs();") } else { - ctx.addMutableState("boolean", inputIsOtherRDD, - s"$inputIsOtherRDD = (partitionIndex >= $otherRDDsPartitionIndex);") - ctx.addMutableState(iteratorClass, rowInput, - s"$rowInput = $inputIsOtherRDD ? inputs[0] " + + inputIsOtherRDD = internals.addClassField(ctx, "boolean", "inputIsOtherRDD", + v => s"$v = (partitionIndex >= $otherRDDsPartitionIndex);") + rowInput = internals.addClassField(ctx, iteratorClass, "rowInput", + v => s"$v = $inputIsOtherRDD ? inputs[0] " + s": ($iteratorClass)inputs[0].next();") - ctx.addMutableState(colIteratorClass, colInput, - s"$colInput = $inputIsOtherRDD ? null : ($colIteratorClass)inputs[0].next();") - ctx.addMutableState("java.sql.ResultSet", rs, - s"$rs = $inputIsOtherRDD ? null : (($rsIterClass)$rowInput).rs();") - ctx.addMutableState(unsafeHolderClass, unsafeHolder, - s"$unsafeHolder = new $unsafeHolderClass();") + colInput = internals.addClassField(ctx, colIteratorClass, "colInput", + v => s"$v = $inputIsOtherRDD ? null : ($colIteratorClass)inputs[0].next();") + rs = internals.addClassField(ctx, "java.sql.ResultSet", "resultSet", + v => s"$v = $inputIsOtherRDD ? null : (($rsIterClass)$rowInput).rs();") + unsafeHolder = internals.addClassField(ctx, unsafeHolderClass, "unsafeHolder", + v => s"$v = new $unsafeHolderClass();") } - ctx.addMutableState(iteratorClass, input, - if (isForSampleReservoirAsRegion) s"$input = $rowInputSRR;" - else s"$input = $rowInput;") - ctx.addMutableState("boolean", inputIsRow, s"$inputIsRow = true;") + input = internals.addClassField(ctx, iteratorClass, "input", + v => if (isForSampleReservoirAsRegion) s"$v = $rowInputSRR;" else s"$v = $rowInput;") + inputIsRow = internals.addClassField(ctx, "boolean", "inputIsRow", v => s"$v = true;") ctx.currentVars = null val encodingClass = ColumnEncoding.encodingClassName @@ -266,25 +258,20 @@ private[sql] final case class ColumnTableScan( val rowDecoderClass = classOf[UnsafeRowDecoder].getName val deletedDecoderClass = classOf[ColumnDeleteDecoder].getName val batch = ctx.freshName("batch") - val numBatchRows = s"${batch}NumRows" val numFullRows = s"${batch}NumFullRows" val numDeltaRows = s"${batch}NumDeltaRows" - val batchIndex = s"${batch}Index" - val buffers = s"${batch}Buffers" val numRows = ctx.freshName("numRows") val batchOrdinal = ctx.freshName("batchOrdinal") - val deletedDecoder = s"${batch}Deleted" - val deletedDecoderLocal = s"${deletedDecoder}Local" + val deletedDecoderLocal = ctx.freshName("deletedDecoderLocal") var deletedDeclaration = "" var deletedCheck = "" - val deletedCount = ctx.freshName("deletedCount") var deletedCountCheck = "" - ctx.addMutableState("java.nio.ByteBuffer", buffers, "") - ctx.addMutableState("int", numBatchRows, "") - ctx.addMutableState("int", batchIndex, "") - ctx.addMutableState(deletedDecoderClass, deletedDecoder, "") - ctx.addMutableState("int", deletedCount, "") + val buffers = internals.addClassField(ctx, "java.nio.ByteBuffer", "buffers") + val numBatchRows = internals.addClassField(ctx, "int", "numBatchRows") + val batchIndex = internals.addClassField(ctx, "int", "batchIndex") + val deletedDecoder = internals.addClassField(ctx, deletedDecoderClass, "deletedDecoder") + val deletedCount = internals.addClassField(ctx, "int", "deletedCount") // need DataType and nullable to get decoder in generated code // shipping as StructType for efficient serialization @@ -346,51 +333,46 @@ private[sql] final case class ColumnTableScan( // this mapper is for the physical columns in the table val columnsInputMapper = (attr: Attribute, index: Int, rsIndex: Int) => { - val decoder = ctx.freshName("decoder") - val decoderLocal = s"${decoder}Local" - val updatedDecoder = s"${decoder}Updated" - val updatedDecoderLocal = s"${decoder}UpdatedLocal" - val numNullsVar = s"${decoder}NumNulls" - val buffer = s"${decoder}Buffer" - val bufferVar = s"${buffer}Object" - val initBufferFunction = s"${buffer}Init" - val closeDecoderFunction = s"${decoder}Close" - if (isWideSchema) { - ctx.addMutableState("Object", bufferVar, "") - } + val decoderLocal = ctx.freshName("decoderLocal") + val updatedDecoderLocal = ctx.freshName("decoderUpdatedLocal") + val buffer = internals.addClassField(ctx, "java.nio.ByteBuffer", "buffer") + val numNullsVar = internals.addClassField(ctx, "int", "numNulls") + var initBufferFunction = ctx.freshName("bufferInit") + val bufferVar = if (isWideSchema) { + internals.addClassField(ctx, "Object", "bufferObject") + } else ctx.freshName("bufferObject") // projections are not pushed in embedded mode for optimized access val baseIndex = Utils.fieldIndex(schemaAttributes, attr.name, caseSensitive) val rsPosition = if (embedded) baseIndex + 1 else rsIndex + 1 val incrementUpdatedColumnCount = if (updatedColumnCount eq null) "" else s"\n$updatedColumnCount.${metricAdd("1")};" - ctx.addMutableState("java.nio.ByteBuffer", buffer, "") - ctx.addMutableState("int", numNullsVar, "") - - val rowDecoderCode = + val rowDecoderCode: String => String = decoder => s"$decoder = new $rsDecoderClass(($rsWithNullClass)$rs, $rsPosition);" - if (otherRDDs.isEmpty) { + val decoder = if (otherRDDs.isEmpty) { if (isForSampleReservoirAsRegion) { - ctx.addMutableState(decoderClass, decoder, - s"$decoder = new $rowDecoderClass($unsafeHolder, $baseIndex);") - initRowTableDecoders.append(rowDecoderCode).append('\n') + val decoderVar = internals.addClassField(ctx, decoderClass, "decoder", + v => s"$v = new $rowDecoderClass($unsafeHolder, $baseIndex);") + initRowTableDecoders.append(rowDecoderCode(decoderVar)).append('\n') + decoderVar } else { - ctx.addMutableState(decoderClass, decoder, rowDecoderCode) + internals.addClassField(ctx, decoderClass, "decoder", rowDecoderCode) } } else { - ctx.addMutableState(decoderClass, decoder, + internals.addClassField(ctx, decoderClass, "decoder", decoder => s""" if ($inputIsOtherRDD) { $decoder = new $rowDecoderClass($unsafeHolder, $baseIndex); } else { - $rowDecoderCode + ${rowDecoderCode(decoder)} } """ ) } - ctx.addMutableState(updatedDecoderClass, updatedDecoder, "") + val updatedDecoder = internals.addClassField(ctx, updatedDecoderClass, "updatedDecoder") + var closeDecoderFunction = ctx.freshName("decoderClose") - ctx.addNewFunction(initBufferFunction, + initBufferFunction = internals.addFunction(ctx, initBufferFunction, s""" |private void $initBufferFunction() { | $buffer = $colInput.getColumnLob($baseIndex); @@ -407,7 +389,7 @@ private[sql] final case class ColumnTableScan( """.stripMargin) columnBufferInit.append(s"$initBufferFunction();\n") - ctx.addNewFunction(closeDecoderFunction, + closeDecoderFunction = internals.addFunction(ctx, closeDecoderFunction, s""" |private void $closeDecoderFunction() { | if ($decoder != null) { @@ -453,14 +435,14 @@ private[sql] final case class ColumnTableScan( ColumnDelta.mutableKeyNames.indexOf(attr.name) match { case 0 => ordinalIdTerm = ctx.freshName("ordinalId") - ExprCode("", "false", ordinalIdTerm) + internals.newExprCode("", "false", ordinalIdTerm, LongType) case 1 => columnBatchIdTerm = ctx.freshName("columnBatchId") - ExprCode("", "false", columnBatchIdTerm) + internals.newExprCode("", "false", columnBatchIdTerm, LongType) case 2 => bucketIdTerm = ctx.freshName("bucketId") - ExprCode("", "false", bucketIdTerm) - case 3 => ExprCode("", "false", numBatchRows) + internals.newExprCode("", "false", bucketIdTerm, IntegerType) + case 3 => internals.newExprCode("", "false", numBatchRows, IntegerType) case _ => throw new IllegalStateException(s"Unexpected internal attribute $attr") } case (attr, index) => rsIndex += 1; columnsInputMapper(attr, index, rsIndex) @@ -547,7 +529,7 @@ private[sql] final case class ColumnTableScan( if (!$colInput.hasNext()) return false; }""" } - val nextBatch = ctx.freshName("nextBatch") + var nextBatch = ctx.freshName("nextBatch") val closeDecodersFunction = ctx.freshName("closeAllDecoders") val switchSRR = if (isForSampleReservoirAsRegion) { // triple switch between rowInputSRR, rowInput, colInput @@ -568,7 +550,7 @@ private[sql] final case class ColumnTableScan( """.stripMargin } else "" - ctx.addNewFunction(nextBatch, + nextBatch = internals.addFunction(ctx, nextBatch, s""" |private boolean $nextBatch() throws Exception { | if ($buffers != null) return true; @@ -603,16 +585,15 @@ private[sql] final case class ColumnTableScan( | return true; |} """.stripMargin) - ctx.addNewFunction(closeDecodersFunction, + internals.addFunction(ctx, closeDecodersFunction, s""" |private void $closeDecodersFunction() { | ${closeDecoders.toString()} |} - """.stripMargin) + """.stripMargin, inlineToOuterClass = true) val (assignBatchId, assignOrdinalId) = if (ordinalIdTerm ne null) ( s""" - |final boolean $inputIsRow = this.$inputIsRow; |final long $columnBatchIdTerm; |final int $bucketIdTerm; |if ($inputIsRow) { @@ -693,24 +674,25 @@ private[sql] final case class ColumnTableScan( val nonNullPosition = if (attr.nullable) s"$batchOrdinal - $numNullsVar" else batchOrdinal val col = ctx.freshName("col") val sqlType = Utils.getSQLDataType(attr.dataType) - val jt = ctx.javaType(sqlType) + val jt = internals.javaType(sqlType, ctx) var colAssign = "" var updatedAssign = "" val typeName = sqlType match { case DateType => "Date" case TimestampType => "Timestamp" - case _ if ctx.isPrimitiveType(jt) => ctx.primitiveTypeName(jt) + case _ if internals.isPrimitiveType(jt, ctx) => internals.primitiveTypeName(jt, ctx) case StringType => val dictionaryVar = ctx.freshName("dictionary") val dictionaryIndexVar = ctx.freshName("dictionaryIndex") - val dictionary = ExprCode( + val dictionary = internals.newExprCode( s""" |$dictionaryVar = $mutableDecoderGlobal == null | ? $decoderGlobal.getStringDictionary() | : $mutableDecoderGlobal.getStringDictionary(); - """.stripMargin, s"($dictionaryVar == null)", dictionaryVar) + """.stripMargin, s"($dictionaryVar == null)", dictionaryVar, + ObjectType(classOf[StringDictionary])) val dictionaryIndex = if (attr.nullable) { - ExprCode( + internals.newExprCode( s""" |${genIfNonNullCode(ctx, decoder, buffer, batchOrdinal, numNullsVar)} { | $dictionaryIndexVar = $updateDecoder == null @@ -719,14 +701,14 @@ private[sql] final case class ColumnTableScan( |} else { | $dictionaryIndexVar = $dictionaryVar.size(); |} - """.stripMargin, "false", dictionaryIndexVar) + """.stripMargin, "false", dictionaryIndexVar, IntegerType) } else { - ExprCode( + internals.newExprCode( s""" |$dictionaryIndexVar = $updateDecoder == null | ? $decoder.readDictionaryIndex($buffer, $nonNullPosition) | : $updateDecoder.readDictionaryIndex(); - """.stripMargin, "false", dictionaryIndexVar) + """.stripMargin, "false", dictionaryIndexVar, IntegerType) } session.foreach(_.addDictionaryCode(ctx, col, DictionaryCode(dictionary, buffer, dictionaryIndex))) @@ -763,7 +745,7 @@ private[sql] final case class ColumnTableScan( val unchangedCode = s"$updateDecoder == null || $updateDecoder.unchanged($batchOrdinal)" if (attr.nullable) { val isNullVar = ctx.freshName("isNull") - val defaultValue = ctx.defaultValue(jt) + val defaultValue = internals.defaultValue(sqlType, ctx) val code = s""" |final $jt $col; @@ -782,7 +764,7 @@ private[sql] final case class ColumnTableScan( | $isNullVar = true; |} """.stripMargin - ExprCode(code, isNullVar, col) + internals.newExprCode(code, isNullVar, col, sqlType) } else { var code = s""" @@ -793,7 +775,7 @@ private[sql] final case class ColumnTableScan( if (weightVar != null && attr.name.equalsIgnoreCase(Utils.WEIGHTAGE_COLUMN_NAME)) { code += s"if ($col == 1) $col = $weightVar;\n" } - ExprCode(code, "false", col) + internals.newExprCode(code, "false", col, sqlType) } } @@ -821,7 +803,7 @@ private[sql] final case class ColumnTableScan( } } -object ColumnTableScan extends Logging { +object ColumnTableScan extends Logging with SparkSupport { def generateStatPredicate(ctx: CodegenContext, isColumnTable: Boolean, schemaAttrs: Seq[AttributeReference], allFilters: Seq[Expression], numRowsTerm: String, @@ -940,9 +922,11 @@ object ColumnTableScan extends Logging { ctx.INPUT_ROW = statsRow ctx.currentVars = null val predicateEval = predicate.genCode(ctx) + val predicateIsNull = internals.exprCodeIsNull(predicateEval) + val predicateVal = internals.exprCodeValue(predicateEval) // skip filtering if nothing is to be applied - if (predicateEval.value == "true" && predicateEval.isNull == "false") { + if (predicateVal == "true" && predicateIsNull == "false") { return "" } val columnBatchesSkipped = if (metricTerm ne null) { @@ -951,16 +935,16 @@ object ColumnTableScan extends Logging { val addBatchMetric = if (columnBatchesSkipped ne null) { s"$columnBatchesSkipped.${metricAdd("1")};" } else "" - val filterFunction = ctx.freshName("columnBatchFilter") - ctx.addNewFunction(filterFunction, + var filterFunction = ctx.freshName("columnBatchFilter") + filterFunction = internals.addFunction(ctx, filterFunction, s""" |private boolean $filterFunction(UnsafeRow $statsRow, int $numRowsTerm, | boolean isLastStatsRow, boolean isDelta) { | // Skip the column batches based on the predicate - | ${predicateEval.code} - | if (isDelta && (${predicateEval.isNull} || ${predicateEval.value})) { + | ${predicateEval.code.toString} + | if (isDelta && ($predicateIsNull|| $predicateVal)) { | return true; - | } else if (!${predicateEval.isNull} && ${predicateEval.value}) { + | } else if (!$predicateIsNull && $predicateVal) { | return true; | } else { | // add to skipped metric only if both stats say so @@ -1020,14 +1004,14 @@ private[sql] final class UnionScanRDD[T: ClassTag]( } } -case class NumBatchRows(varName: String) extends LeafExpression { +case class NumBatchRows(varName: String) extends LeafExpression with SparkSupport { override def nullable: Boolean = false override def dataType: DataType = IntegerType override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { - ExprCode("", "false", varName) + internals.newExprCode("", "false", varName, IntegerType) } override def eval(input: InternalRow): Any = @@ -1038,7 +1022,7 @@ case class NumBatchRows(varName: String) extends LeafExpression { } case class StartsWithForStats(upper: Expression, lower: Expression, - pattern: Expression) extends Expression { + pattern: Expression) extends Expression with SparkSupport { // pattern must be a string constant for stats row evaluation assert(TokenLiteral.isConstant(pattern)) @@ -1054,49 +1038,53 @@ case class StartsWithForStats(upper: Expression, lower: Expression, override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { val upperExpr = upper.genCode(ctx) + val upperIsNull = internals.exprCodeIsNull(upperExpr) + val upperVal = internals.exprCodeValue(upperExpr) val lowerExpr = lower.genCode(ctx) val patternExpr = pattern.genCode(ctx) + val patternIsNull = internals.exprCodeIsNull(patternExpr) + val patternVal = internals.exprCodeValue(patternExpr) val str = ctx.freshName("str") val len = str + "Len" val lastCharPos = str + "LastPos" val upperBytes = str + "Upper" val upperStr = str + "UpperUTF8" - val result = ev.value + val result = internals.exprCodeValue(ev) val code = s""" - |${patternExpr.code} + |${patternExpr.code.toString} |boolean $result = true; - |if (!${patternExpr.isNull}) { - | ${lowerExpr.code} - | ${upperExpr.code} + |if (!$patternIsNull) { + | ${lowerExpr.code.toString} + | ${upperExpr.code.toString} | // upper bound for column (i.e. LessThan) can be found by going to | // next value of the last character of literal - | int $len = ${patternExpr.value}.numBytes(); + | int $len = $patternVal.numBytes(); | byte[] $upperBytes = new byte[$len]; - | ${patternExpr.value}.writeToMemory($upperBytes, Platform.BYTE_ARRAY_OFFSET); + | $patternVal.writeToMemory($upperBytes, Platform.BYTE_ARRAY_OFFSET); | int $lastCharPos = $len - 1; | // check for maximum unsigned value 0xff | while ($lastCharPos >= 0 && $upperBytes[$lastCharPos] == (byte)-1) { | $lastCharPos--; | } - | if ($lastCharPos < 0 || (${lowerExpr.isNull})) { // all bytes are 0xff + | if ($lastCharPos < 0 || (${internals.exprCodeIsNull(lowerExpr)})) { // all bytes 0xff | // a >= startsWithPREFIX - | if (!${upperExpr.isNull}) { - | $result = ${patternExpr.value}.compareTo(${upperExpr.value}) <= 0; + | if (!$upperIsNull) { + | $result = $patternVal.compareTo($upperVal) <= 0; | } | } else { | $upperBytes[$lastCharPos] = (byte)($upperBytes[$lastCharPos] + 1); | UTF8String $upperStr = UTF8String.fromAddress($upperBytes, | Platform.BYTE_ARRAY_OFFSET, $len); | // a >= startsWithPREFIX && a < startsWithPREFIX+1 - | $result = ((${upperExpr.isNull}) || - | ${patternExpr.value}.compareTo(${upperExpr.value}) <= 0) && - | ${lowerExpr.value}.compareTo($upperStr) < 0; + | $result = (($upperIsNull) || + | $patternVal.compareTo($upperVal) <= 0) && + | ${internals.exprCodeValue(lowerExpr)}.compareTo($upperStr) < 0; | } |} | """.stripMargin - ev.copy(code, "false", result) + internals.copyExprCode(ev, code = code, isNull = "false", value = result, BooleanType) } override def eval(input: InternalRow): Any = diff --git a/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnUpdateExec.scala b/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnUpdateExec.scala index 384b193936..88895e2a00 100644 --- a/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnUpdateExec.scala +++ b/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnUpdateExec.scala @@ -30,7 +30,7 @@ import org.apache.spark.sql.execution.row.RowExec import org.apache.spark.sql.sources.JdbcExtendedUtils.quotedName import org.apache.spark.sql.sources.{ConnectionProperties, DestroyRelation, JdbcExtendedUtils} import org.apache.spark.sql.store.{CompressionCodecId, StoreUtils} -import org.apache.spark.sql.types.StructType +import org.apache.spark.sql.types.{IntegerType, StructType} /** * Generated code plan for updates into a column table. @@ -129,13 +129,7 @@ case class ColumnUpdateExec(child: SparkPlan, columnTable: String, override def doConsume(ctx: CodegenContext, input: Seq[ExprCode], row: ExprCode): String = { // use an array of delta encoders and cursors - val deltaEncoders = ctx.freshName("deltaEncoders") - val cursors = ctx.freshName("cursors") val index = ctx.freshName("index") - batchOrdinal = ctx.freshName("batchOrdinal") - val lastColumnBatchId = ctx.freshName("lastColumnBatchId") - val lastBucketId = ctx.freshName("lastBucketId") - val lastNumRows = ctx.freshName("lastNumRows") finishUpdate = ctx.freshName("finishUpdate") val initializeEncoders = ctx.freshName("initializeEncoders") @@ -152,17 +146,18 @@ case class ColumnUpdateExec(child: SparkPlan, columnTable: String, val encoderClass = classOf[ColumnEncoder].getName val columnBatchClass = classOf[ColumnBatch].getName - ctx.addMutableState(s"$deltaEncoderClass[]", deltaEncoders, "") - ctx.addMutableState("long[]", cursors, + val deltaEncoders = internals.addClassField(ctx, s"$deltaEncoderClass[]", "deltaEncoders") + val cursors = internals.addClassField(ctx, "long[]", "cursors", v => s""" |$deltaEncoders = new $deltaEncoderClass[$numColumns]; - |$cursors = new long[$numColumns]; + |$v = new long[$numColumns]; |$initializeEncoders(); """.stripMargin) - ctx.addMutableState("int", batchOrdinal, "") - ctx.addMutableState("long", lastColumnBatchId, s"$lastColumnBatchId = $invalidUUID;") - ctx.addMutableState("int", lastBucketId, "") - ctx.addMutableState("int", lastNumRows, "") + batchOrdinal = internals.addClassField(ctx, "int", "batchOrdinal") + val lastColumnBatchId = internals.addClassField(ctx, "long", "lastColumnBatchId", + v => s"$v = $invalidUUID;") + val lastBucketId = internals.addClassField(ctx, "int", "lastBucketId") + val lastNumRows = internals.addClassField(ctx, "int", "lastNumRows") // last three columns in keyColumns should be internal ones val keyCols = keyColumns.takeRight(4) @@ -185,17 +180,17 @@ case class ColumnUpdateExec(child: SparkPlan, columnTable: String, ctx.currentVars = null val keyVars = updateInput.takeRight(4) - val ordinalIdVar = keyVars.head.value - val batchIdVar = keyVars(1).value - val bucketVar = keyVars(2).value - val numRowsVar = keyVars(3).value + val ordinalIdVar = internals.exprCodeValue(keyVars.head) + val batchIdVar = internals.exprCodeValue(keyVars(1)) + val bucketVar = internals.exprCodeValue(keyVars(2)) + val numRowsVar = internals.exprCodeValue(keyVars(3)) val updateVarsCode = evaluateVariables(updateInput) // row buffer needs to select the rowId and partitioning columns so drop last three val rowConsume = super.doConsume(ctx, updateInput.dropRight(3), StructType(getUpdateSchema(allExpressions.dropRight(3)))) - ctx.addNewFunction(initializeEncoders, + internals.addFunction(ctx, initializeEncoders, s""" |private void $initializeEncoders() { | for (int $index = 0; $index < $numColumns; $index++) { @@ -204,12 +199,12 @@ case class ColumnUpdateExec(child: SparkPlan, columnTable: String, | ${classOf[ColumnDelta].getName}.INIT_SIZE(), true); | } |} - """.stripMargin) + """.stripMargin, inlineToOuterClass = true) // Creating separate encoder write functions instead of inlining for wide-schemas // in updates (especially with support for putInto being added). Performance should // be about the same since JVM inlines where it determines will help performance. val callEncoders = updateColumns.zipWithIndex.map { case (col, i) => - val function = ctx.freshName("encoderFunction") + var function = ctx.freshName("encoderFunction") val ordinal = ctx.freshName("ordinal") val isNull = ctx.freshName("isNull") val field = ctx.freshName("field") @@ -218,19 +213,21 @@ case class ColumnUpdateExec(child: SparkPlan, columnTable: String, val realEncoderTerm = s"${encoderTerm}_realEncoder" val cursorTerm = s"$cursors[$i]" val ev = updateInput(i) - ctx.addNewFunction(function, + function = internals.addFunction(ctx, function, s""" |private void $function(int $ordinal, int $ordinalIdVar, - | boolean $isNull, ${ctx.javaType(dataType)} $field) { + | boolean $isNull, ${internals.javaType(dataType, ctx)} $field) { | final $deltaEncoderClass $encoderTerm = $deltaEncoders[$i]; | final $encoderClass $realEncoderTerm = $encoderTerm.getRealEncoder(); | $encoderTerm.setUpdatePosition($ordinalIdVar); | ${ColumnWriter.genCodeColumnWrite(ctx, dataType, col.nullable, realEncoderTerm, - encoderTerm, cursorTerm, ev.copy(isNull = isNull, value = field), ordinal)} + encoderTerm, cursorTerm, internals.copyExprCode(ev, isNull = isNull, + value = field, dt = dataType), ordinal)} |} """.stripMargin) // code for invoking the function - s"$function($batchOrdinal, (int)$ordinalIdVar, ${ev.isNull}, ${ev.value});" + s"$function($batchOrdinal, (int)$ordinalIdVar, ${internals.exprCodeIsNull(ev)}, " + + s"${internals.exprCodeValue(ev)});" }.mkString("\n") // Old code(Keeping the comment for better understanding) // Write the delta stats row for all table columns at the end of a batch. @@ -253,14 +250,16 @@ case class ColumnUpdateExec(child: SparkPlan, columnTable: String, // equals to 1 i.e LZ4 compression codec id ). // Hence setting each 3rd bit( null count stats) with not null flag. This will never cause // the word to be read as negative number. - val allNullsExprs = Seq(ExprCode("", "true", ""), - ExprCode("", "true", ""), ExprCode("", "false", "-1")) val (statsSchema, stats) = tableSchema.indices.map { i => val field = tableSchema(i) tableToUpdateIndex.get(i) match { case null => + val dataType = field.dataType + val allNullsExprs = Seq(internals.newExprCode("", "true", "", dataType), + internals.newExprCode("", "true", "", dataType), + internals.newExprCode("", "false", "-1", IntegerType)) // write null for unchanged columns apart from null count field (by this update) - (ColumnStatsSchema(field.name, field.dataType, + (ColumnStatsSchema(field.name, dataType, nullCountNullable = false).schema, allNullsExprs) case u => ColumnWriter.genCodeColumnStats(ctx, field, s"$deltaEncoders[$u].getRealEncoder()") @@ -270,7 +269,7 @@ case class ColumnUpdateExec(child: SparkPlan, columnTable: String, // methods if required so no need to add separate functions explicitly. // Count is hardcoded as zero which will change for "insert" index deltas. val statsEv = ColumnWriter.genStatsRow(ctx, "0", stats, statsSchema) - ctx.addNewFunction(finishUpdate, + finishUpdate = internals.addFunction(ctx, finishUpdate, s""" |private void $finishUpdate(long batchId, int bucketId, int numRows) { | if (batchId == $invalidUUID || batchId != $lastColumnBatchId) { @@ -287,10 +286,10 @@ case class ColumnUpdateExec(child: SparkPlan, columnTable: String, | buffers[$index] = $deltaEncoders[$index].finish($cursors[$index], $lastNumRows); | } | // create delta statistics row - | ${statsEv.code} + | ${statsEv.code.toString} | // store the delta column batch - | final $columnBatchClass columnBatch = $columnBatchClass.apply( - | $batchOrdinal, buffers, ${statsEv.value}.getBytes(), $deltaIndexes); + | final $columnBatchClass columnBatch = $columnBatchClass.apply($batchOrdinal, + | buffers, ${internals.exprCodeValue(statsEv)}.getBytes(), $deltaIndexes); | // maxDeltaRows is -1 so that insert into row buffer is never considered | $externalStoreTerm.storeColumnBatch($tableName, columnBatch, $lastBucketId, | $lastColumnBatchId, -1, ${compressionCodec.id}, new scala.Some($connTerm)); diff --git a/core/src/main/scala/org/apache/spark/sql/execution/columnar/ExternalStoreUtils.scala b/core/src/main/scala/org/apache/spark/sql/execution/columnar/ExternalStoreUtils.scala index 809e8314de..865c1fc67c 100644 --- a/core/src/main/scala/org/apache/spark/sql/execution/columnar/ExternalStoreUtils.scala +++ b/core/src/main/scala/org/apache/spark/sql/execution/columnar/ExternalStoreUtils.scala @@ -18,7 +18,6 @@ package org.apache.spark.sql.execution.columnar import java.sql.{Connection, PreparedStatement, SQLException, Statement, Types} import java.util.Properties -import java.util.concurrent.atomic.AtomicReference import javax.naming.NameNotFoundException import scala.collection.JavaConverters._ @@ -44,12 +43,9 @@ import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.expressions import org.apache.spark.sql.catalyst.expressions.codegen.{CodeAndComment, CodeFormatter, CodegenContext} import org.apache.spark.sql.catalyst.expressions.{Attribute, BinaryExpression, DynamicInSet, Expression, TokenLiteral} -import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap import org.apache.spark.sql.collection.Utils import org.apache.spark.sql.execution.columnar.impl.JDBCSourceAsColumnarStore -import org.apache.spark.sql.execution.datasources.DataSource import org.apache.spark.sql.execution.datasources.jdbc.DriverRegistry -import org.apache.spark.sql.execution.ui.SQLListener import org.apache.spark.sql.execution.{BufferedRowIterator, CodegenSupport, CodegenSupportOnExecutor, ConnectionPool, RefreshMetadata} import org.apache.spark.sql.jdbc.{JdbcDialect, JdbcDialects} import org.apache.spark.sql.row.SnappyStoreDialect @@ -61,7 +57,7 @@ import org.apache.spark.util.{Utils => SparkUtils} /** * Utility methods used by external storage layers. */ -object ExternalStoreUtils { +object ExternalStoreUtils extends SparkSupport { private[spark] final lazy val (defaultTableBuckets, defaultSampleTableBuckets) = { val sc = Option(SnappyContext.globalSparkContext) @@ -105,6 +101,9 @@ object ExternalStoreUtils { COLUMN_BATCH_SIZE_TRANSIENT, COLUMN_MAX_DELTA_ROWS, COLUMN_MAX_DELTA_ROWS_TRANSIENT, COMPRESSION_CODEC, RELATION_FOR_SAMPLE, KEY_COLUMNS) + private[this] val storePropertyPrefixes = Array("", Constant.STORE_PROPERTY_PREFIX, + Constant.SPARK_STORE_PREFIX, Constant.PROPERTY_PREFIX, Constant.SPARK_SNAPPY_PREFIX) + registerBuiltinDrivers() def registerBuiltinDrivers(): Unit = { @@ -219,7 +218,7 @@ object ExternalStoreUtils { case None => // Do nothing } }) - new CaseInsensitiveMap(optMap.toMap) + internals.newCaseInsensitiveMap(optMap.toMap) } def getLdapGroupsForUser(userId: String): Array[String] = { @@ -387,12 +386,18 @@ object ExternalStoreUtils { } def getCredentials(session: SparkSession, prefix: String = ""): (String, String) = { - val prefix = SnappyContext.getClusterMode(session.sparkContext) match { - case ThinClientConnectorMode(_, _) => Constant.SPARK_STORE_PREFIX - case _ => "" + for (prefix <- storePropertyPrefixes) { + val userProperty = + if (prefix.isEmpty) ClientAttribute.USERNAME + else prefix + ClientAttribute.USERNAME + if (session.conf.contains(userProperty)) { + val passwordProperty = + if (prefix.isEmpty) ClientAttribute.PASSWORD + else prefix + ClientAttribute.PASSWORD + return (session.conf.get(userProperty), session.conf.get(passwordProperty, "")) + } } - (session.conf.get(prefix + ClientAttribute.USERNAME, ""), - session.conf.get(prefix + ClientAttribute.PASSWORD, "")) + ("", "") } def getConnection(id: String, connProperties: ConnectionProperties, @@ -416,10 +421,10 @@ object ExternalStoreUtils { } /** check if the DataSource implements ExternalSchemaRelationProvider */ - def isExternalSchemaRelationProvider(provider: String): Boolean = { + def isExternalSchemaRelationProvider(provider: String, session: SparkSession): Boolean = { try { classOf[ExternalSchemaRelationProvider].isAssignableFrom( - DataSource.lookupDataSource(provider)) + internals.lookupDataSource(provider, session.sessionState.conf)) } catch { case NonFatal(_) => false } @@ -734,10 +739,6 @@ object ExternalStoreUtils { Property.ColumnMaxDeltaRows.name) } - def getSQLListener: AtomicReference[SQLListener] = { - SparkSession.sqlListener - } - def setSchemaVersionOnConnection(catalogVersion: Long, conn: Connection): Unit = { var clientStmt: Option[Statement] = None if (catalogVersion != -1) { diff --git a/core/src/main/scala/org/apache/spark/sql/execution/columnar/JDBCAppendableRelation.scala b/core/src/main/scala/org/apache/spark/sql/execution/columnar/JDBCAppendableRelation.scala index 955b9807dc..7f3d099b60 100644 --- a/core/src/main/scala/org/apache/spark/sql/execution/columnar/JDBCAppendableRelation.scala +++ b/core/src/main/scala/org/apache/spark/sql/execution/columnar/JDBCAppendableRelation.scala @@ -19,20 +19,20 @@ package org.apache.spark.sql.execution.columnar import java.sql.Connection import java.util.concurrent.locks.ReentrantReadWriteLock -import com.gemstone.gemfire.internal.shared.ClientResolverUtils - import scala.collection.JavaConverters._ + +import com.gemstone.gemfire.internal.shared.ClientResolverUtils import com.pivotal.gemfirexd.Attribute import io.snappydata.{Constant, SnappyTableStatsProviderService} import org.eclipse.collections.impl.map.mutable.primitive.ObjectLongHashMap + import org.apache.spark.Logging import org.apache.spark.rdd.RDD import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.expressions.{Expression, SortDirection} -import org.apache.spark.sql.catalyst.plans.logical.OverwriteOptions -import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap import org.apache.spark.sql.collection.Utils +import org.apache.spark.sql.execution.columnar.ExternalStoreUtils.CaseInsensitiveMutableHashMap import org.apache.spark.sql.execution.datasources.LogicalRelation import org.apache.spark.sql.execution.datasources.jdbc.{JDBCOptions, JdbcUtils} import org.apache.spark.sql.jdbc.JdbcDialect @@ -49,7 +49,7 @@ abstract case class JDBCAppendableRelation( provider: String, mode: SaveMode, userSchema: StructType, - origOptions: CaseInsensitiveMap, + override val origOptions: CaseInsensitiveMutableHashMap[String], externalStore: ExternalStore, @transient override val sqlContext: SQLContext) extends BaseRelation with PrunedUnsafeFilteredScan @@ -58,7 +58,8 @@ abstract case class JDBCAppendableRelation( with DestroyRelation with IndexableRelation with Logging - with NativeTableRowLevelSecurityRelation + with SnappyTableRelation + with SparkSupport with Serializable { self => @@ -129,11 +130,11 @@ abstract case class JDBCAppendableRelation( // use the Insert plan for best performance // that will use the getInsertPlan above (in StoreStrategy) sqlContext.sessionState.executePlan( - new Insert( + internals.newInsertIntoTable( table = LogicalRelation(this), partition = Map.empty[String, Option[String]], child = data.logicalPlan, - OverwriteOptions(overwrite), + overwrite, ifNotExists = false)).toRdd } @@ -198,12 +199,8 @@ abstract case class JDBCAppendableRelation( override def equals(that: Any): Boolean = { that match { - case r: JDBCAppendableRelation => { - (this eq r) || ( - hashCode() == r.hashCode() - && r.schemaName.equalsIgnoreCase(schemaName) - && r.tableName.equalsIgnoreCase(tableName)) - } + case r: JDBCAppendableRelation => (this eq r) || + (r.schemaName.equalsIgnoreCase(schemaName) && r.tableName.equalsIgnoreCase(tableName)) case _ => false } } diff --git a/core/src/main/scala/org/apache/spark/sql/execution/columnar/impl/ColumnFormatRelation.scala b/core/src/main/scala/org/apache/spark/sql/execution/columnar/impl/ColumnFormatRelation.scala index 507233bc17..80a32af096 100644 --- a/core/src/main/scala/org/apache/spark/sql/execution/columnar/impl/ColumnFormatRelation.scala +++ b/core/src/main/scala/org/apache/spark/sql/execution/columnar/impl/ColumnFormatRelation.scala @@ -28,7 +28,6 @@ import io.snappydata.{Constant, Property} import org.apache.spark.rdd.RDD import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, Descending, Expression, SortDirection} -import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier, analysis} import org.apache.spark.sql.collection.Utils import org.apache.spark.sql.execution.columnar.ExternalStoreUtils.CaseInsensitiveMutableHashMap @@ -68,7 +67,7 @@ abstract class BaseColumnFormatRelation( _userSchema: StructType, val schemaExtensions: String, val ddlExtensionForShadowTable: String, - _origOptions: CaseInsensitiveMap, + _origOptions: CaseInsensitiveMutableHashMap[String], _externalStore: ExternalStore, val partitioningColumns: Seq[String], _context: SQLContext, @@ -275,7 +274,7 @@ abstract class BaseColumnFormatRelation( val snc = sqlContext.sparkSession.asInstanceOf[SnappySession] val lockOption = snc.getContextObject[(Option[TableIdentifier], PartitionedRegion.RegionLock)]( SnappySession.PUTINTO_LOCK) match { - case None if (Property.SerializeWrites.get(snc.sessionState.conf)) => + case None if Property.SerializeWrites.get(snc.sessionState.conf) => snc.grabLock(table, schemaName, connProperties) case _ => None // Do nothing as putInto will release lock } @@ -302,10 +301,9 @@ abstract class BaseColumnFormatRelation( } finally { lockOption match { - case Some(lock) => { + case Some(lock) => logDebug(s"Releasing the $lock object in InsertRows") snc.releaseLock(lock) - } case None => // do Nothing } } @@ -317,7 +315,7 @@ abstract class BaseColumnFormatRelation( val lockOption = snc.getContextObject[(Option[TableIdentifier], PartitionedRegion.RegionLock)]( SnappySession.PUTINTO_LOCK) match { - case None if (Property.SerializeWrites.get(snc.sessionState.conf)) => + case None if Property.SerializeWrites.get(snc.sessionState.conf) => snc.grabLock(table, schemaName, connProperties) case _ => None // Do nothing as putInto will release lock } @@ -326,11 +324,10 @@ abstract class BaseColumnFormatRelation( } finally { lockOption match { - case Some(lock) => { + case Some(lock) => logDebug(s"Added the $lock object to the context for $table") snc.addContextObject( SnappySession.BULKWRITE_LOCK, lock) - } case None => // do nothing } } @@ -482,7 +479,7 @@ class ColumnFormatRelation( _userSchema: StructType, _schemaExtensions: String, _ddlExtensionForShadowTable: String, - _origOptions: CaseInsensitiveMap, + _origOptions: CaseInsensitiveMutableHashMap[String], _externalStore: ExternalStore, _partitioningColumns: Seq[String], _context: SQLContext, @@ -517,8 +514,8 @@ class ColumnFormatRelation( cr.origOptions, cr.externalStore, cr.partitioningColumns, cr.sqlContext, _relationInfoAndRegion) newRelation.delayRollover = true - relation.copy(relation = newRelation, - expectedOutputAttributes = Some(relation.output ++ ColumnDelta.mutableKeyAttributes)) + internals.newLogicalRelation(newRelation, Some(relation.output ++ + ColumnDelta.mutableKeyAttributes), relation.catalogTable, isStreaming = false) } override def dropIndex(indexIdent: TableIdentifier, @@ -600,7 +597,7 @@ class ColumnFormatRelation( indexTblName, "column", tableRelation.schema, - indexOptions) + indexOptions.toMap) } override def createIndex(indexIdent: TableIdentifier, @@ -662,7 +659,7 @@ class IndexColumnFormatRelation( _userSchema: StructType, _schemaExtensions: String, _ddlExtensionForShadowTable: String, - _origOptions: CaseInsensitiveMap, + _origOptions: CaseInsensitiveMutableHashMap[String], _externalStore: ExternalStore, _partitioningColumns: Seq[String], _context: SQLContext, @@ -693,15 +690,16 @@ class IndexColumnFormatRelation( cr.externalStore, cr.partitioningColumns, cr.sqlContext, baseTableName, _relationInfoAndRegion) newRelation.delayRollover = true - relation.copy(relation = newRelation, - expectedOutputAttributes = Some(relation.output ++ ColumnDelta.mutableKeyAttributes)) + internals.newLogicalRelation(newRelation, Some(relation.output ++ + ColumnDelta.mutableKeyAttributes), relation.catalogTable, isStreaming = false) } def getBaseTableRelation: ColumnFormatRelation = { val session = sqlContext.sparkSession.asInstanceOf[SnappySession] - val catalog = session.sessionState.catalog + val catalog = session.snappySessionState.catalog catalog.resolveRelation(session.tableIdentifier(baseTableName)) match { - case LogicalRelation(cr: ColumnFormatRelation, _, _) => cr + case lr: LogicalRelation if lr.relation.isInstanceOf[ColumnFormatRelation] => + lr.relation.asInstanceOf[ColumnFormatRelation] case _ => throw new UnsupportedOperationException("Index scan other than Column table unsupported") } diff --git a/core/src/main/scala/org/apache/spark/sql/execution/columnar/impl/DefaultSource.scala b/core/src/main/scala/org/apache/spark/sql/execution/columnar/impl/DefaultSource.scala index 312cd86b79..67531123d3 100644 --- a/core/src/main/scala/org/apache/spark/sql/execution/columnar/impl/DefaultSource.scala +++ b/core/src/main/scala/org/apache/spark/sql/execution/columnar/impl/DefaultSource.scala @@ -20,14 +20,13 @@ import io.snappydata.Constant import io.snappydata.sql.catalog.SnappyExternalCatalog import org.apache.spark.Logging -import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap import org.apache.spark.sql.collection.Utils import org.apache.spark.sql.execution.columnar.ExternalStoreUtils import org.apache.spark.sql.execution.columnar.ExternalStoreUtils.CaseInsensitiveMutableHashMap import org.apache.spark.sql.sources.{CreatableRelationProvider, DataSourceRegister, ExternalSchemaRelationProvider, JdbcExtendedUtils, SchemaRelationProvider} import org.apache.spark.sql.store.StoreUtils import org.apache.spark.sql.types.StructType -import org.apache.spark.sql.{AnalysisException, DataFrame, SQLContext, SaveMode, SnappyParserConsts, SnappySession} +import org.apache.spark.sql.{AnalysisException, DataFrame, SQLContext, SaveMode, SnappyParserConsts, SnappySession, SparkSupport} /** * Column tables don't support any extensions over regular Spark schema syntax, @@ -39,7 +38,7 @@ import org.apache.spark.sql.{AnalysisException, DataFrame, SQLContext, SaveMode, * which is parsed locally in the CreatableRelationProvider implementation. */ final class DefaultSource extends ExternalSchemaRelationProvider with SchemaRelationProvider - with CreatableRelationProvider with DataSourceRegister with Logging { + with CreatableRelationProvider with DataSourceRegister with Logging with SparkSupport { override def shortName(): String = SnappyParserConsts.COLUMN_SOURCE @@ -85,7 +84,7 @@ final class DefaultSource extends ExternalSchemaRelationProvider with SchemaRela // on the servers to determine table properties like compression etc. // SnappyExternalCatalog will alter the definition for final entry if required. session.sessionCatalog.createTableForBuiltin(relation.resolvedName, - getClass.getCanonicalName, relation.schema, relation.origOptions, + getClass.getCanonicalName, relation.schema, relation.origOptions.toMap, mode != SaveMode.ErrorIfExists) relation.insert(data, mode == SaveMode.Overwrite) success = true @@ -93,7 +92,7 @@ final class DefaultSource extends ExternalSchemaRelationProvider with SchemaRela } finally { if (!success && relation.tableCreated) { // remove the catalog entry - session.sessionCatalog.externalCatalog.dropTable(relation.schemaName, + session.sessionCatalog.snappyExternalCatalog.dropTable(relation.schemaName, relation.tableName, ignoreIfNotExists = true, purge = false) // destroy the relation relation.destroy(ifExists = true) @@ -134,7 +133,7 @@ final class DefaultSource extends ExternalSchemaRelationProvider with SchemaRela } val partitioningColumns = StoreUtils.getAndSetPartitioningAndKeyColumns(session, schema, parameters) - val tableOptions = new CaseInsensitiveMap(parameters.toMap) + val tableOptions = new CaseInsensitiveMutableHashMap[String](parameters.toMap) val ddlExtension = StoreUtils.ddlExtensionString(parameters, isRowTable = false, isShadowTable = false) diff --git a/core/src/main/scala/org/apache/spark/sql/execution/columnar/impl/JDBCSourceAsColumnarStore.scala b/core/src/main/scala/org/apache/spark/sql/execution/columnar/impl/JDBCSourceAsColumnarStore.scala index 36d594eeb2..9a6aa4775e 100644 --- a/core/src/main/scala/org/apache/spark/sql/execution/columnar/impl/JDBCSourceAsColumnarStore.scala +++ b/core/src/main/scala/org/apache/spark/sql/execution/columnar/impl/JDBCSourceAsColumnarStore.scala @@ -48,12 +48,12 @@ import org.apache.spark.sql.execution.columnar._ import org.apache.spark.sql.execution.columnar.encoding.ColumnDeleteDelta import org.apache.spark.sql.execution.row.{ResultSetTraversal, RowFormatScanRDD, RowInsertExec} import org.apache.spark.sql.execution.sources.StoreDataSourceStrategy.translateToFilter -import org.apache.spark.sql.execution.{BufferedRowIterator, ConnectionPool, RDDKryo, WholeStageCodegenExec} +import org.apache.spark.sql.execution.{BufferedRowIterator, ConnectionPool, RDDKryo} import org.apache.spark.sql.sources.JdbcExtendedUtils.quotedName import org.apache.spark.sql.sources.{ConnectionProperties, JdbcExtendedUtils} import org.apache.spark.sql.store.CodeGeneration import org.apache.spark.sql.types.StructType -import org.apache.spark.sql.{SnappySession, SparkSession} +import org.apache.spark.sql.{SnappySession, SparkSession, SparkSupport} import org.apache.spark.util.TaskCompletionListener import org.apache.spark.{Partition, TaskContext, TaskKilledException} @@ -62,7 +62,7 @@ import org.apache.spark.{Partition, TaskContext, TaskKilledException} */ class JDBCSourceAsColumnarStore(private var _connProperties: ConnectionProperties, var numPartitions: Int, private var _tableName: String, var schema: StructType) - extends ExternalStore with KryoSerializable { + extends ExternalStore with KryoSerializable with SparkSupport { self => @@ -597,7 +597,7 @@ class JDBCSourceAsColumnarStore(private var _connProperties: ConnectionPropertie val gen = CodeGeneration.compileCode( tableName + ".columnTable.decompress", schema.fields, () => { val schemaAttrs = schema.toAttributes - val tableScan = ColumnTableScan(schemaAttrs, dataRDD = null, + val tableScan = internals.columnTableScan(schemaAttrs, dataRDD = null, otherRDDs = Nil, numBuckets = -1, partitionColumns = Nil, partitionColumnAliases = Nil, baseRelation = null, schema, allFilters = Nil, schemaAttrs, @@ -609,7 +609,7 @@ class JDBCSourceAsColumnarStore(private var _connProperties: ConnectionPropertie // this is only used for local code generation while its RDD // semantics and related methods are all ignored val (ctx, code) = ExternalStoreUtils.codeGenOnExecutor( - WholeStageCodegenExec(insertPlan), insertPlan) + internals.newWholeStagePlan(insertPlan), insertPlan) val references = ctx.references // also push the index of connection reference at the end which // will be used below to update connection before execution @@ -707,12 +707,12 @@ final class ColumnarStorePartitionedRDD( case -1 if allPartitions != null => allPartitions case -1 => - allPartitions = session.sessionState.getTablePartitions( + allPartitions = session.snappySessionState.getTablePartitions( region.asInstanceOf[PartitionedRegion]) allPartitions case bucketId: Int => if (!session.partitionPruning) { - allPartitions = session.sessionState.getTablePartitions( + allPartitions = session.snappySessionState.getTablePartitions( region.asInstanceOf[PartitionedRegion]) allPartitions } else { @@ -796,7 +796,7 @@ final class SmartConnectorColumnRDD( private var serializedFilters: Array[Byte] = _ - private var preferHostName = SmartConnectorHelper.preferHostName(session) + private var preferHostName = SmartConnectorHelper.preferHostName override def compute(split: Partition, context: TaskContext): Iterator[ByteBuffer] = { @@ -919,7 +919,7 @@ class SmartConnectorRowRDD(_session: SnappySession, _filters, _partEval, _partitionPruner, _commitTx, _delayRollover, projection = Array.emptyIntArray, None) { - private var preferHostName = SmartConnectorHelper.preferHostName(session) + private var preferHostName = SmartConnectorHelper.preferHostName override def commitTxBeforeTaskCompletion(conn: Option[Connection], context: TaskContext): Unit = { diff --git a/core/src/main/scala/org/apache/spark/sql/execution/columnar/impl/StoreCallbacksImpl.scala b/core/src/main/scala/org/apache/spark/sql/execution/columnar/impl/StoreCallbacksImpl.scala index 1e35ad48f5..e9a93a8e4e 100644 --- a/core/src/main/scala/org/apache/spark/sql/execution/columnar/impl/StoreCallbacksImpl.scala +++ b/core/src/main/scala/org/apache/spark/sql/execution/columnar/impl/StoreCallbacksImpl.scala @@ -52,7 +52,7 @@ import org.apache.spark.Logging import org.apache.spark.memory.{MemoryManagerCallback, MemoryMode} import org.apache.spark.serializer.KryoSerializerPool import org.apache.spark.sql._ -import org.apache.spark.sql.catalyst.expressions.codegen.{CodeAndComment, CodeFormatter, CodeGenerator, CodegenContext} +import org.apache.spark.sql.catalyst.expressions.codegen.{CodeAndComment, CodeFormatter, CodegenContext} import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression, Literal, TokenLiteral, UnsafeRow} import org.apache.spark.sql.catalyst.{CatalystTypeConverters, expressions} import org.apache.spark.sql.collection.{SharedUtils, ToolsCallbackInit, Utils} @@ -65,7 +65,7 @@ import org.apache.spark.sql.store.{CodeGeneration, StoreHashFunction} import org.apache.spark.sql.types._ import org.apache.spark.unsafe.types.UTF8String -object StoreCallbacksImpl extends StoreCallbacks with Logging with Serializable { +object StoreCallbacksImpl extends StoreCallbacks with SparkSupport with Logging with Serializable { private val partitioner = new StoreHashFunction @@ -227,8 +227,7 @@ object StoreCallbacksImpl extends StoreCallbacks with Logging with Serializable val ctx = new CodegenContext val rowClass = classOf[UnsafeRow].getName // create the code snippet for applying the filters - val numRows = ctx.freshName("numRows") - ctx.addMutableState("int", numRows, "") + val numRows = internals.addClassField(ctx, "int", "numRows") val filterFunction = ColumnTableScan.generateStatPredicate(ctx, isColumnTable = true, schemaAttrs, batchFilterExprs, numRows, metricTerm = null, metricAdd = null) val filterPredicate = if (filterFunction.isEmpty) null @@ -271,7 +270,7 @@ object StoreCallbacksImpl extends StoreCallbacks with Logging with Serializable CodeGeneration.logDebug(s"\n${CodeFormatter.format(cleanedSource)}") - val clazz = CodeGenerator.compile(cleanedSource) + val clazz = internals.compile(cleanedSource) clazz.generate(ctx.references.toArray).asInstanceOf[StatsPredicate] } val batchIterator = ColumnBatchIterator(region, bucketIds, projection, diff --git a/core/src/main/scala/org/apache/spark/sql/execution/common/HAC.scala b/core/src/main/scala/org/apache/spark/sql/execution/common/HAC.scala new file mode 100644 index 0000000000..8e9ca6cc9b --- /dev/null +++ b/core/src/main/scala/org/apache/spark/sql/execution/common/HAC.scala @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2017-2020 TIBCO Software Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ +package org.apache.spark.sql.execution.common + +import io.snappydata.{Constant, Property} + +import org.apache.spark.sql.catalyst.expressions.{Expression, Literal, ParamLiteral} +import org.apache.spark.sql.collection.Utils +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.types.StringType + +object HAC extends Enumeration { + + type Type = Value + + val DO_NOTHING: Type = Value(0) + val SPECIAL_SYMBOL: Type = Value(1) + val THROW_EXCEPTION: Type = Value(2) + val REROUTE_TO_BASE: Type = Value(3) + val PARTIAL_ROUTING: Type = Value(4) + + override def toString(): String = { + s" 1)DO_NOTHING 2)LOCAL_OMIT 3)STRICT 4)RUN_ON_FULL_TABLE 5)PARTIAL_RUN_ON_BASE_TABLE" + } + + def getBehavior(expr: Expression): HAC.Type = { + expr match { + case lp: ParamLiteral => getBehavior(lp.valueString) + case _ => getBehavior(expr.simpleString) + } + } + + + def getBehavior(name: String): HAC.Type = { + Utils.toUpperCase(name) match { + case Constant.BEHAVIOR_DO_NOTHING => DO_NOTHING + case Constant.BEHAVIOR_LOCAL_OMIT => SPECIAL_SYMBOL + case Constant.BEHAVIOR_STRICT => THROW_EXCEPTION + case Constant.BEHAVIOR_RUN_ON_FULL_TABLE => REROUTE_TO_BASE + case Constant.DEFAULT_BEHAVIOR => getDefaultBehavior() + case Constant.BEHAVIOR_PARTIAL_RUN_ON_BASE_TABLE => PARTIAL_ROUTING + + case x@_ => throw new UnsupportedOperationException( + s"Please specify valid HAC from below:\n$HAC\nGiven: $x") + } + } + + def getBehaviorAsString(value: HAC.Type): String = { + value match { + case DO_NOTHING => Constant.BEHAVIOR_DO_NOTHING + case SPECIAL_SYMBOL => Constant.BEHAVIOR_LOCAL_OMIT + case THROW_EXCEPTION => Constant.BEHAVIOR_STRICT + case REROUTE_TO_BASE => Constant.BEHAVIOR_RUN_ON_FULL_TABLE + case PARTIAL_ROUTING => Constant.BEHAVIOR_PARTIAL_RUN_ON_BASE_TABLE + case _ => "INVALID" + } + } + + def getDefaultBehavior(conf: SQLConf = null): HAC.Type = { + if (System.getProperty(Constant.defaultBehaviorAsDO_NOTHING, "false").toBoolean) { + DO_NOTHING + } + else if (conf != null) { + try { + HAC.getBehavior(Literal.create(Property.Behavior.getOption(conf).getOrElse( + Constant.BEHAVIOR_RUN_ON_FULL_TABLE), + StringType)) + } catch { + case e: UnsupportedOperationException => Property.Behavior.set(conf, + Constant.BEHAVIOR_RUN_ON_FULL_TABLE) + throw e + } + } else REROUTE_TO_BASE + } +} diff --git a/core/src/main/scala/org/apache/spark/sql/execution/ddl.scala b/core/src/main/scala/org/apache/spark/sql/execution/ddl.scala index a11ef58d84..49f90fcee6 100644 --- a/core/src/main/scala/org/apache/spark/sql/execution/ddl.scala +++ b/core/src/main/scala/org/apache/spark/sql/execution/ddl.scala @@ -18,7 +18,6 @@ package org.apache.spark.sql.execution import java.io.File -import java.lang import java.nio.file.{Files, Paths} import java.util.Map.Entry import java.util.function.Consumer @@ -35,7 +34,6 @@ import io.snappydata.Property import io.snappydata.util.ServiceUtils import org.apache.spark.SparkContext -import org.apache.spark.deploy.SparkSubmitUtils import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec @@ -45,7 +43,7 @@ import org.apache.spark.sql.catalyst.plans.QueryPlan import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.collection.{ToolsCallbackInit, Utils} import org.apache.spark.sql.execution.columnar.InMemoryTableScanExec -import org.apache.spark.sql.execution.command.{DescribeTableCommand, DropTableCommand, RunnableCommand, SetCommand, ShowTablesCommand} +import org.apache.spark.sql.execution.command.{DropTableCommand, RunnableCommand, SetCommand, ShowTablesCommand} import org.apache.spark.sql.execution.datasources.LogicalRelation import org.apache.spark.sql.functions._ import org.apache.spark.sql.internal.{BypassRowLevelSecurity, ContextJarUtils, StaticSQLConf} @@ -66,13 +64,15 @@ case class CreateTableUsingCommand( partitionColumns: Array[String], bucketSpec: Option[BucketSpec], query: Option[LogicalPlan], - isBuiltIn: Boolean) extends RunnableCommand { + isExternal: Boolean, + comment: Option[String] = None, + location: Option[String] = None) extends RunnableCommand { override def run(sparkSession: SparkSession): Seq[Row] = { val session = sparkSession.asInstanceOf[SnappySession] val allOptions = session.addBaseTableOption(baseTable, options) - session.createTableInternal(tableIdent, provider, userSpecifiedSchema, - schemaDDL, mode, allOptions, isBuiltIn, partitionColumns, bucketSpec, query) + session.createTableInternal(tableIdent, provider, userSpecifiedSchema, schemaDDL, mode, + allOptions, isExternal, partitionColumns, bucketSpec, query, comment, location) Nil } } @@ -132,7 +132,7 @@ case class DropPolicyCommand(ifExists: Boolean, } case class TruncateManagedTableCommand(ifExists: Boolean, - table: TableIdentifier) extends RunnableCommand { + table: TableIdentifier) extends RunnableCommand with SparkSupport { override def run(session: SparkSession): Seq[Row] = { val catalog = session.asInstanceOf[SnappySession].sessionCatalog @@ -144,7 +144,8 @@ case class TruncateManagedTableCommand(ifExists: Boolean, case plan => throw new AnalysisException( s"Table '$table' must be a DestroyRelation for truncate. Found plan: $plan") } - session.sharedState.cacheManager.uncacheQuery(session.table(table)) + internals.uncacheQuery(session, session.table(table).logicalPlan, + cascade = true, blocking = true) } Nil } @@ -290,7 +291,7 @@ case class SnappyStreamingActionsCommand(action: Int, * in the GUI rather than count() plan for InMemoryRelation. */ case class SnappyCacheTableCommand(tableIdent: TableIdentifier, queryString: String, - plan: Option[LogicalPlan], isLazy: Boolean) extends RunnableCommand { + plan: Option[LogicalPlan], isLazy: Boolean) extends RunnableCommand with SparkSupport { require(plan.isEmpty || tableIdent.database.isEmpty, "Schema name is not allowed in CACHE TABLE AS SELECT") @@ -324,28 +325,28 @@ case class SnappyCacheTableCommand(tableIdent: TableIdentifier, queryString: Str val previousJobDescription = localProperties.getProperty(SparkContext.SPARK_JOB_DESCRIPTION) localProperties.setProperty(SparkContext.SPARK_JOB_DESCRIPTION, queryShortString) try { - session.sessionState.enableExecutionCache = true + session.snappySessionState.enableExecutionCache = true // Get the actual QueryExecution used by InMemoryRelation so that // "withNewExecutionId" runs on the same and shows proper metrics in GUI. val cachedExecution = try { if (isOffHeap) df.persist(StorageLevel.OFF_HEAP) else df.persist() - session.sessionState.getExecution(df.logicalPlan) + session.snappySessionState.getExecution(df.logicalPlan) } finally { - session.sessionState.enableExecutionCache = false - session.sessionState.clearExecutionCache() + session.snappySessionState.enableExecutionCache = false + session.snappySessionState.clearExecutionCache() } val memoryPlan = df.queryExecution.executedPlan.collectFirst { case plan: InMemoryTableScanExec => plan.relation }.get val planInfo = PartitionedPhysicalScan.getSparkPlanInfo(cachedExecution.executedPlan) Row(CachedDataFrame.withCallback(session, df = null, cachedExecution, "cache")(_ => - CachedDataFrame.withNewExecutionId(session, queryShortString, queryString, - cachedExecution.toString(), planInfo)({ + CachedDataFrame.withNewExecutionId(session, cachedExecution.executedPlan, + queryShortString, queryString, cachedExecution.toString(), planInfo)({ val start = System.nanoTime() // Dummy op to materialize the cache. This does the minimal job of count on // the actual cached data (RDD[CachedBatch]) to force materialization of cache // while avoiding creation of any new SparkPlan. - val count = memoryPlan.cachedColumnBuffers.count() + val count = internals.cachedColumnBuffers(memoryPlan).count() (count, System.nanoTime() - start) }))._1) :: Nil } finally { @@ -364,11 +365,13 @@ case class SnappyCacheTableCommand(tableIdent: TableIdentifier, queryString: Str * Also when hive compatibility is turned on, then this does not include the schema name * or "isTemporary" to return hive compatible result. */ -class ShowSnappyTablesCommand(session: SnappySession, schemaOpt: Option[String], - tablePattern: Option[String]) extends ShowTablesCommand(schemaOpt, tablePattern) { +class ShowSnappyTablesCommand(schemaOpt: Option[String], tablePattern: Option[String])( + val hiveCompatible: Boolean) extends ShowTablesCommand(schemaOpt, tablePattern) { - private val hiveCompatible = Property.HiveCompatibility.get( - session.sessionState.conf).equalsIgnoreCase("full") + def this(schemaOpt: Option[String], tablePattern: Option[String], session: SnappySession) { + this(schemaOpt, tablePattern)(Property.HiveCompatibility.get( + session.sessionState.conf).equalsIgnoreCase("full")) + } override val output: Seq[Attribute] = { if (hiveCompatible) AttributeReference("name", StringType, nullable = false)() :: Nil @@ -379,6 +382,8 @@ class ShowSnappyTablesCommand(session: SnappySession, schemaOpt: Option[String], } } + override protected def otherCopyArgs: Seq[AnyRef] = Boolean.box(hiveCompatible) :: Nil + override def run(sparkSession: SparkSession): Seq[Row] = { if (!hiveCompatible) { return super.run(sparkSession) @@ -450,9 +455,13 @@ case class ShowViewsCommand(session: SnappySession, schemaOpt: Option[String], /** * This extends Spark's describe to add support for CHAR and VARCHAR types. */ -class DescribeSnappyTableCommand(table: TableIdentifier, - partitionSpec: TablePartitionSpec, isExtended: Boolean, isFormatted: Boolean) - extends DescribeTableCommand(table, partitionSpec, isExtended, isFormatted) { +case class DescribeSnappyTableCommand(table: TableIdentifier, partitionSpec: TablePartitionSpec, + isExtended: Boolean, isFormatted: Boolean) extends RunnableCommand with SparkSupport { + + private[this] val describeCmd = internals.newDescribeTableCommand( + table, partitionSpec, isExtended, isFormatted) + + override def output: Seq[Attribute] = describeCmd.output override def run(sparkSession: SparkSession): Seq[Row] = { val catalog = sparkSession.asInstanceOf[SnappySession].sessionCatalog @@ -460,7 +469,7 @@ class DescribeSnappyTableCommand(table: TableIdentifier, // set the flag to return CharType/VarcharType if present catalog.convertCharTypesInMetadata = true try { - super.run(sparkSession) + describeCmd.run(sparkSession) } finally { catalog.convertCharTypesInMetadata = false } @@ -484,11 +493,11 @@ case class DeployCommand( alias: String, repos: Option[String], jarCache: Option[String], - restart: Boolean) extends RunnableCommand { + restart: Boolean) extends RunnableCommand with SparkSupport { override def run(sparkSession: SparkSession): Seq[Row] = { try { - val jarsstr = SparkSubmitUtils.resolveMavenCoordinates(coordinates, repos, jarCache) + val jarsstr = internals.resolveMavenCoordinates(coordinates, repos, jarCache, Nil) if (jarsstr.nonEmpty) { val jars = jarsstr.split(",") val sc = sparkSession.sparkContext @@ -592,7 +601,7 @@ case class ListPackageJarsCommand(isJar: Boolean) extends RunnableCommand { } } -case class UnDeployCommand(alias: String) extends RunnableCommand { +case class UnDeployCommand(alias: String) extends RunnableCommand with SparkSupport { override def run(sparkSession: SparkSession): Seq[Row] = { var value = "" @@ -613,8 +622,7 @@ case class UnDeployCommand(alias: String) extends RunnableCommand { val coordinates = value.substring(0, indexOf) val repos = Option(value.substring(indexOf + 1, lastIndexOf)) val jarCache = Option(value.substring(lastIndexOf + 1, value.length)) - val jarsstr = SparkSubmitUtils.resolveMavenCoordinates(coordinates, - repos, jarCache) + val jarsstr = internals.resolveMavenCoordinates(coordinates, repos, jarCache, Nil) if (jarsstr.nonEmpty) { val pkgs = jarsstr.split(",") RefreshMetadata.executeOnAll(sc, RefreshMetadata.REMOVE_URIS_FROM_CLASSLOADER, pkgs) diff --git a/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoinExec.scala b/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoinExec.scala index 9c5d7c76d8..f150200c5d 100644 --- a/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoinExec.scala +++ b/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoinExec.scala @@ -40,7 +40,7 @@ import org.apache.spark.sql.execution._ import org.apache.spark.sql.execution.metric.SQLMetrics import org.apache.spark.sql.streaming.PhysicalDStreamPlan import org.apache.spark.sql.types.TypeUtilities -import org.apache.spark.sql.{DelegateRDD, SnappySession} +import org.apache.spark.sql.{DelegateRDD, SnappySession, SparkSupport} /** * :: DeveloperApi :: @@ -62,10 +62,12 @@ case class HashJoinExec(leftKeys: Seq[Expression], rightSizeInBytes: BigInt, replicatedTableJoin: Boolean) extends NonRecursivePlans with BinaryExecNode with HashJoin - with SnappyJoinLike with BatchConsumer { + with SnappyJoinLike with BatchConsumer with SparkSupport { override def nodeName: String = "SnappyHashJoin" + override def needCopyResult: Boolean = false + @transient private var mapAccessor: ObjectHashMapAccessor = _ @transient private var hashMapTerm: String = _ @transient private var mapDataTerm: String = _ @@ -130,7 +132,6 @@ case class HashJoinExec(leftKeys: Seq[Expression], // return empty here as code of required variables is explicitly instantiated override def usedInputs: AttributeSet = AttributeSet.empty - private def findShuffleDependencies(rdd: RDD[_]): Seq[Dependency[_]] = { rdd.dependencies.flatMap { case s: ShuffleDependency[_, _, _] => if (s.rdd ne rdd) { @@ -283,21 +284,19 @@ case class HashJoinExec(leftKeys: Seq[Expression], } override def doProduce(ctx: CodegenContext): String = { - initMap = ctx.freshName("initMap") - ctx.addMutableState("boolean", initMap, s"$initMap = false;") + initMap = internals.addClassField(ctx, "boolean", "initMap", v => s"$v = false;") val createMap = ctx.freshName("createMap") val createMapClass = ctx.freshName("CreateMap") - val getOrCreateMap = ctx.freshName("getOrCreateMap") + var getOrCreateMap = ctx.freshName("getOrCreateMap") val beforeMap = ctx.freshName("beforeMap") val buildTime = metricTerm(ctx, "buildTime") val numOutputRows = metricTerm(ctx, "numOutputRows") // generate variable name for hash map for use here and in consume - hashMapTerm = ctx.freshName("hashMap") val hashSetClassName = classOf[ObjectHashSet[_]].getName - ctx.addMutableState(hashSetClassName, hashMapTerm, "") + hashMapTerm = internals.addClassField(ctx, hashSetClassName, "hashMap") // using the expression IDs is enough to ensure uniqueness val buildCodeGen = buildPlan.asInstanceOf[CodegenSupport] @@ -312,12 +311,12 @@ case class HashJoinExec(leftKeys: Seq[Expression], val cacheKeyTerm = ctx.addReferenceObj("cacheKey", new CacheKey(exprIds, rdds.head.id)) - // generate local variables for HashMap data array and mask + // generate variables for HashMap data array and mask mapDataTerm = ctx.freshName("mapData") - maskTerm = ctx.freshName("hashMapMask") - mapSize = ctx.freshName("mapSize") - keyIsUniqueTerm = ctx.freshName("keyIsUnique") - numRowsTerm = ctx.freshName("numRows") + maskTerm = internals.addClassField(ctx, "int", "hashMapMask") + mapSize = internals.addClassField(ctx, "int", "mapSize", v => s"$v = -1;") + keyIsUniqueTerm = internals.addClassField(ctx, "boolean", "keyIsUnique", v => s"$v = true;") + numRowsTerm = internals.addClassField(ctx, "long", "numRows", v => s"$v = 0L;") // generate the map accessor to generate key/value class // and get map access methods @@ -327,10 +326,8 @@ case class HashJoinExec(leftKeys: Seq[Expression], multiMap = true, this, this.parent, buildPlan) val entryClass = mapAccessor.getClassName - ctx.addMutableState(s"$entryClass[]", mapDataTerm, "") - ctx.addMutableState("int", maskTerm, "") - ctx.addMutableState("int", mapSize, s"$mapSize = -1;") - ctx.addMutableState("boolean", keyIsUniqueTerm, s"$keyIsUniqueTerm = true;") + internals.addClassField(ctx, s"$entryClass[]", mapDataTerm, + forceInline = true, useFreshName = false) val buildRDDs = ctx.addReferenceObj("buildRDDs", rdds.toArray, s"${classOf[RDD[_]].getName}[]") @@ -338,20 +335,18 @@ case class HashJoinExec(leftKeys: Seq[Expression], val partitionClass = classOf[Partition].getName val buildPartsVar = ctx.addReferenceObj("buildParts", buildParts.toArray, s"$partitionClass[][]") - val allIterators = ctx.freshName("allIterators") val indexVar = ctx.freshName("index") - val contextName = ctx.freshName("context") val taskContextClass = classOf[TaskContext].getName - ctx.addMutableState(taskContextClass, contextName, - s"this.$contextName = $taskContextClass.get();") - + val contextName = internals.addClassField(ctx, taskContextClass, "context", + v => s"this.$v = $taskContextClass.get();") // switch inputs to use the buildPlan RDD iterators - ctx.addMutableState("scala.collection.Iterator[]", allIterators, + val scalaIterorClass = "scala.collection.Iterator" + val allIterators = internals.addClassField(ctx, scalaIterorClass + "[]", "allIterators", v => s""" - |$allIterators = inputs; - |inputs = new scala.collection.Iterator[$buildRDDs.length]; - |$taskContextClass $contextName = $taskContextClass.get(); + |$v = inputs; + |inputs = new $scalaIterorClass[$buildRDDs.length]; + |$contextName = $taskContextClass.get(); |for (int $indexVar = 0; $indexVar < $buildRDDs.length; $indexVar++) { | $partitionClass[] parts = $buildPartsVar[$indexVar]; | // check for replicate table @@ -366,25 +361,24 @@ case class HashJoinExec(leftKeys: Seq[Expression], """.stripMargin) val buildProduce = buildCodeGen.produce(ctx, mapAccessor) - // switch inputs back to streamPlan iterators - val numIterators = ctx.freshName("numIterators") - ctx.addMutableState("int", numIterators, s"inputs = $allIterators;") + // switch inputs back to streamPlan iterators (variable added is a dummy) + internals.addClassField(ctx, "int", "numIterators", _ => s"inputs = $allIterators;") val numKeyColumns = buildSideKeys.length val longLived = replicatedTableJoin val buildSideCreateMap = - s"""$hashSetClassName $hashMapTerm = new $hashSetClassName(128, 0.6, + s"""$hashMapTerm = new $hashSetClassName(128, 0.6, $numKeyColumns, $longLived, scala.reflect.ClassTag$$.MODULE$$.apply( $entryClass.class)); - this.$hashMapTerm = $hashMapTerm; int $maskTerm = $hashMapTerm.mask(); - $entryClass[] $mapDataTerm = ($entryClass[])$hashMapTerm.data(); + this.$maskTerm = $maskTerm; + this.$mapDataTerm = ($entryClass[])$hashMapTerm.data(); $buildProduce""" if (replicatedTableJoin) { var cacheClass = HashedObjectCache.getClass.getName cacheClass = cacheClass.substring(0, cacheClass.length - 1) - ctx.addNewFunction(getOrCreateMap, + getOrCreateMap = internals.addFunction(ctx, getOrCreateMap, s""" public final void $createMap() throws java.io.IOException { $buildSideCreateMap @@ -404,7 +398,7 @@ case class HashJoinExec(leftKeys: Seq[Expression], } """) } else { - ctx.addNewFunction(getOrCreateMap, + getOrCreateMap = internals.addFunction(ctx, getOrCreateMap, s""" public final void $getOrCreateMap() throws java.io.IOException { $buildSideCreateMap @@ -417,15 +411,15 @@ case class HashJoinExec(leftKeys: Seq[Expression], // The child could change `copyResult` to true, but we had already // consumed all the rows, so `copyResult` should be reset to `false`. - ctx.copyResult = false + internals.resetCopyResult(ctx) // initialization of min/max for integral keys val initMinMaxVars = mapAccessor.integralKeys.zipWithIndex.map { case (indexKey, index) => val minVar = mapAccessor.integralKeysMinVars(index) val maxVar = mapAccessor.integralKeysMaxVars(index) - ctx.addMutableState("long", minVar, "") - ctx.addMutableState("long", maxVar, "") + internals.addClassField(ctx, "long", minVar, forceInline = true, useFreshName = false) + internals.addClassField(ctx, "long", maxVar, forceInline = true, useFreshName = false) s""" $minVar = $hashMapTerm.getMinValue($indexKey); $maxVar = $hashMapTerm.getMaxValue($indexKey); @@ -439,19 +433,17 @@ case class HashJoinExec(leftKeys: Seq[Expression], $buildTime.${metricAdd(s"(System.nanoTime() - $beforeMap) / 1000000")}; this.$initMap = true; - this.$mapSize = $hashMapTerm.size(); + $mapSize = $hashMapTerm.size(); this.$keyIsUniqueTerm = $keyIsUniqueTerm = $hashMapTerm.keyIsUnique(); $initMinMaxVars this.$maskTerm = $maskTerm = $hashMapTerm.mask(); - this.$mapDataTerm = $mapDataTerm = ($entryClass[])$hashMapTerm.data();""" + $mapDataTerm = ($entryClass[])$hashMapTerm.data();""" val produced = streamedPlan.asInstanceOf[CodegenSupport].produce(ctx, this) s""" boolean $keyIsUniqueTerm = this.$keyIsUniqueTerm; int $maskTerm = this.$maskTerm; - $entryClass[] $mapDataTerm = this.$mapDataTerm; - long $numRowsTerm = 0L; try { ${session.evaluateFinallyCode(ctx, produced)} } finally { @@ -476,21 +468,10 @@ case class HashJoinExec(leftKeys: Seq[Expression], val buildVars = keyValueVars.drop(buildSideKeys.length) val checkCondition = getJoinCondition(ctx, input, buildVars) - ctx.INPUT_ROW = null - ctx.currentVars = input - val (resultVars, streamKeys) = buildSide match { - case BuildLeft => (buildVars ++ input, - streamSideKeys.map(BindReferences.bindReference(_, right.output))) - case BuildRight => (input ++ buildVars, - streamSideKeys.map(BindReferences.bindReference(_, left.output))) - } - val streamKeyVars = ctx.generateExpressions(streamKeys) - - mapAccessor.generateMapLookup(entryVar, localValueVar, - mapSize, keyIsUniqueTerm, initMap, initMapCode, numRowsTerm, - nullMaskVars, initCode, checkCondition, streamSideKeys, - streamKeyVars, streamedPlan.output, buildKeyVars, buildVars, input, - resultVars, dictionaryArrayTerm, dictionaryArrayInit, joinType, buildSide) + mapAccessor.generateMapLookup(entryVar, localValueVar, mapSize, keyIsUniqueTerm, initMap, + initMapCode, numRowsTerm, nullMaskVars, initCode, checkCondition, streamSideKeys, + streamedPlan.output, buildKeyVars, buildVars, input, dictionaryArrayTerm, + dictionaryArrayInit, joinType, buildSide) } override def canConsume(plan: SparkPlan): Boolean = { @@ -510,7 +491,7 @@ case class HashJoinExec(leftKeys: Seq[Expression], // this array will be used at batch level for grouping if possible dictionaryArrayTerm = ctx.freshName("dictionaryArray") dictionaryArrayInit = ctx.freshName("dictionaryArrayInit") - ctx.addNewFunction(dictionaryArrayInit, + dictionaryArrayInit = internals.addFunction(ctx, dictionaryArrayInit, s""" |private $className[] $dictionaryArrayInit() { | return null; @@ -531,7 +512,7 @@ case class HashJoinExec(leftKeys: Seq[Expression], val eval = evaluateRequiredVariables(buildPlan.output, buildVars, expr.references) // filter the output via condition - ctx.currentVars = input.map(_.copy(code = "")) ++ buildVars + ctx.currentVars = input.map(internals.copyExprCode(_, code = "")) ++ buildVars val ev = BindReferences.bindReference(expr, streamedPlan.output ++ buildPlan.output).genCode(ctx) (Some(ev), eval, condition) diff --git a/core/src/main/scala/org/apache/spark/sql/execution/row/DefaultSource.scala b/core/src/main/scala/org/apache/spark/sql/execution/row/DefaultSource.scala index 3df538f809..8cfe57e258 100644 --- a/core/src/main/scala/org/apache/spark/sql/execution/row/DefaultSource.scala +++ b/core/src/main/scala/org/apache/spark/sql/execution/row/DefaultSource.scala @@ -17,7 +17,6 @@ package org.apache.spark.sql.execution.row import org.apache.spark.sql._ -import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap import org.apache.spark.sql.execution.columnar.ExternalStoreUtils import org.apache.spark.sql.execution.columnar.ExternalStoreUtils.CaseInsensitiveMutableHashMap import org.apache.spark.sql.execution.datasources.jdbc.JDBCPartition @@ -28,7 +27,7 @@ import org.apache.spark.sql.types.StructType import org.apache.spark.{Logging, Partition, SparkContext} final class DefaultSource extends ExternalSchemaRelationProvider with SchemaRelationProvider - with CreatableRelationProvider with DataSourceRegister with Logging { + with CreatableRelationProvider with DataSourceRegister with Logging with SparkSupport { override def shortName(): String = SnappyParserConsts.ROW_SOURCE @@ -72,7 +71,7 @@ final class DefaultSource extends ExternalSchemaRelationProvider with SchemaRela // on the servers to determine table properties like compression etc. // SnappyExternalCatalog will alter the definition for final entry if required. session.sessionCatalog.createTableForBuiltin(relation.resolvedName, - getClass.getCanonicalName, relation.schema, relation.origOptions, + getClass.getCanonicalName, relation.schema, relation.origOptions.toMap, mode != SaveMode.ErrorIfExists) // SaveMode.Overwrite already taken care by createTable to truncate relation.insert(data, overwrite = false) @@ -81,7 +80,7 @@ final class DefaultSource extends ExternalSchemaRelationProvider with SchemaRela } finally { if (!success && relation.tableCreated) { // remove the catalog entry - session.sessionCatalog.externalCatalog.dropTable(relation.schemaName, + session.sessionCatalog.snappyExternalCatalog.dropTable(relation.schemaName, relation.tableName, ignoreIfNotExists = true, purge = false) // destroy the relation relation.destroy(ifExists = true) @@ -97,7 +96,7 @@ final class DefaultSource extends ExternalSchemaRelationProvider with SchemaRela ExternalStoreUtils.getAndSetTotalPartitions(session, parameters, forManagedTable = true, forColumnTable = false) StoreUtils.getAndSetPartitioningAndKeyColumns(session, schema = null, parameters) - val tableOptions = new CaseInsensitiveMap(parameters.toMap) + val tableOptions = new CaseInsensitiveMutableHashMap[String](parameters.toMap) val ddlExtension = StoreUtils.ddlExtensionString(parameters, isRowTable = true, isShadowTable = false) val schemaExtension = s"$schemaString $ddlExtension" diff --git a/core/src/main/scala/org/apache/spark/sql/execution/row/RowExec.scala b/core/src/main/scala/org/apache/spark/sql/execution/row/RowExec.scala index 1e328c083b..5a6233dfa8 100644 --- a/core/src/main/scala/org/apache/spark/sql/execution/row/RowExec.scala +++ b/core/src/main/scala/org/apache/spark/sql/execution/row/RowExec.scala @@ -26,7 +26,7 @@ import org.apache.spark.sql.execution.TableExec import org.apache.spark.sql.execution.columnar.ExternalStoreUtils import org.apache.spark.sql.sources.ConnectionProperties import org.apache.spark.sql.store.CodeGeneration -import org.apache.spark.sql.types.{StructField, StructType} +import org.apache.spark.sql.types.{LongType, StructField, StructType} /** * Base class for bulk row table insert, update, put, delete operations. @@ -39,7 +39,6 @@ trait RowExec extends TableExec { @transient protected var rowCount: String = _ @transient protected var result: String = _ - def resolvedName: String def connProps: ConnectionProperties @@ -49,17 +48,17 @@ trait RowExec extends TableExec { protected def connectionCodes(ctx: CodegenContext): (String, String, String) = { val connectionClass = classOf[Connection].getName - connTerm = ctx.freshName("connection") // onExecutor will never be true in case of ColumnDelete/Update if (onExecutor) { // actual connection will be filled into references before execution connRef = ctx.references.length // connObj position in the array is connRef val connObj = ctx.addReferenceObj("conn", null, connectionClass) + connTerm = ctx.freshName("connection") (s"final $connectionClass $connTerm = $connObj;", "", "") } else { val utilsClass = ExternalStoreUtils.getClass.getName - ctx.addMutableState(connectionClass, connTerm, "") + connTerm = internals.addClassField(ctx, connectionClass, "connection") val props = ctx.addReferenceObj("connectionProperties", connProps) val catalogVersion = ctx.addReferenceObj("catalogVersion", catalogSchemaVersion) val initCode: String = getInitCode(utilsClass, props, catalogVersion) @@ -123,20 +122,18 @@ trait RowExec extends TableExec { protected def doProduce(ctx: CodegenContext, pstmtStr: String, produceAddonCode: () => String = () => ""): String = { + + stmt = internals.addClassField(ctx, "java.sql.PreparedStatement", "statement") + result = internals.addClassField(ctx, "long", "result", v => s"$v = -1L;") + rowCount = internals.addClassField(ctx, "long", "rowCount") + val (initCode, commitCode, endCode) = connectionCodes(ctx) - result = ctx.freshName("result") - stmt = ctx.freshName("statement") - rowCount = ctx.freshName("rowCount") - val numOpRowsMetric = if (onExecutor) null - else metricTerm(ctx, s"num${opType}Rows") + val numOpRowsMetric = if (onExecutor) null else metricTerm(ctx, s"num${opType}Rows") val numOperations = ctx.freshName("numOperations") val childProduce = doChildProduce(ctx) - val mutateTable = ctx.freshName("mutateTable") + var mutateTable = ctx.freshName("mutateTable") - ctx.addMutableState("java.sql.PreparedStatement", stmt, "") - ctx.addMutableState("long", result, s"$result = -1L;") - ctx.addMutableState("long", rowCount, "") - ctx.addNewFunction(mutateTable, + mutateTable = internals.addFunction(ctx, mutateTable, s""" |private void $mutateTable() throws java.io.IOException, java.sql.SQLException { | $childProduce @@ -154,7 +151,7 @@ trait RowExec extends TableExec { | $stmt = $connTerm.prepareStatement("$pstmtStr"); | $result = 0L; | $mutateTable(); - | ${consume(ctx, Seq(ExprCode("", "false", result)))} + | ${consume(ctx, Seq(internals.newExprCode("", "false", result, LongType)))} |} catch (java.sql.SQLException sqle) { | throw new java.io.IOException(sqle.toString(), sqle); |}$commitCode @@ -177,10 +174,9 @@ trait RowExec extends TableExec { protected def doConsume(ctx: CodegenContext, input: Seq[ExprCode], schema: StructType): String = { val schemaTerm = ctx.addReferenceObj("schema", schema) - val schemaFields = ctx.freshName("schemaFields") val structFieldClass = classOf[StructField].getName - ctx.addMutableState(s"$structFieldClass[]", schemaFields, - s"$schemaFields = $schemaTerm.fields();") + val schemaFields = internals.addClassField(ctx, s"$structFieldClass[]", "schemaFields", + v => s"$v = $schemaTerm.fields();") val batchSize = connProps.executorConnProps .getProperty("batchsize", "1000").toInt val numOpRowsMetric = if (onExecutor) null @@ -193,27 +189,38 @@ trait RowExec extends TableExec { val isNull = ctx.freshName("isNull") val field = ctx.freshName("field") val ev = input(col) - val dataType = ctx.javaType(f.dataType) - val columnSetterFunction = ctx.freshName("setColumnOfRow") + val javaType = internals.javaType(f.dataType, ctx) + var columnSetterFunction = ctx.freshName("setColumnOfRow") val columnSetterCode = CodeGeneration.getColumnSetterFragment(col, f.dataType, - connProps.dialect, ev.copy(isNull = isNull, value = field), stmt, schemaFields, ctx) - ctx.addNewFunction(columnSetterFunction, + connProps.dialect, internals.copyExprCode(ev, isNull = isNull, value = field, + dt = f.dataType), stmt, schemaFields, ctx) + columnSetterFunction = internals.addFunction(ctx, columnSetterFunction, s""" |private void $columnSetterFunction(final boolean $isNull, - | final $dataType $field) throws java.sql.SQLException { + | final $javaType $field) throws java.sql.SQLException { | $columnSetterCode |} """.stripMargin) - s"$columnSetterFunction(${ev.isNull}, ${ev.value});" + s"$columnSetterFunction(${internals.exprCodeIsNull(ev)}, ${internals.exprCodeValue(ev)});" }.mkString("\n") s""" - |$inputCode - |$functionCalls - |$rowCount++; - |$stmt.addBatch(); - |if (($rowCount % $batchSize) == 0) { - | ${executeBatchCode(numOperations, numOpRowsMetric)} - | $rowCount = 0; + |try { + | $inputCode + | $functionCalls + | $rowCount++; + | $stmt.addBatch(); + | if (($rowCount % $batchSize) == 0) { + | ${executeBatchCode(numOperations, numOpRowsMetric)} + | $rowCount = 0; + | } + |} catch (RuntimeException re) { + | throw re; + |} catch (Exception e) { + | if (e instanceof java.io.IOException) { + | throw (java.io.IOException)e; + | } else { + | throw new java.io.IOException(e.toString(), e); + | } |} """.stripMargin } diff --git a/core/src/main/scala/org/apache/spark/sql/execution/row/RowFormatRelation.scala b/core/src/main/scala/org/apache/spark/sql/execution/row/RowFormatRelation.scala index db35e4f3f3..8e56c0b6b2 100644 --- a/core/src/main/scala/org/apache/spark/sql/execution/row/RowFormatRelation.scala +++ b/core/src/main/scala/org/apache/spark/sql/execution/row/RowFormatRelation.scala @@ -26,9 +26,9 @@ import org.apache.spark.Partition import org.apache.spark.rdd.RDD import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.expressions.{And, Ascending, Attribute, Descending, EqualTo, Expression, In, SortDirection} -import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap import org.apache.spark.sql.catalyst.{InternalRow, analysis} import org.apache.spark.sql.collection.Utils +import org.apache.spark.sql.execution.columnar.ExternalStoreUtils.CaseInsensitiveMutableHashMap import org.apache.spark.sql.execution.columnar.impl.SmartConnectorRowRDD import org.apache.spark.sql.execution.columnar.{ConnectionType, ExternalStoreUtils} import org.apache.spark.sql.execution.datasources.LogicalRelation @@ -49,7 +49,7 @@ class RowFormatRelation( _mode: SaveMode, _userSpecifiedString: String, _parts: Array[Partition], - _origOptions: CaseInsensitiveMap, + _origOptions: CaseInsensitiveMutableHashMap[String], _context: SQLContext) extends JDBCMutableRelation(_connProperties, _table, diff --git a/core/src/main/scala/org/apache/spark/sql/execution/row/RowFormatScanRDD.scala b/core/src/main/scala/org/apache/spark/sql/execution/row/RowFormatScanRDD.scala index 8277c19e8b..a1db4bb49d 100644 --- a/core/src/main/scala/org/apache/spark/sql/execution/row/RowFormatScanRDD.scala +++ b/core/src/main/scala/org/apache/spark/sql/execution/row/RowFormatScanRDD.scala @@ -61,7 +61,8 @@ class RowFormatScanRDD(@transient val session: SnappySession, protected var connProperties: ConnectionProperties, @transient private[sql] val filters: Array[Expression] = Array.empty[Expression], @transient protected val partitionEvaluator: () => Array[Partition] = () => - Array.empty[Partition], protected val partitionPruner: () => Int = () => -1, + Array.empty[Partition], + @transient protected val partitionPruner: () => Int = () => -1, protected var commitTx: Boolean, protected var delayRollover: Boolean, protected var projection: Array[Int], @transient protected val region: Option[LocalRegion]) @@ -349,8 +350,8 @@ class RowFormatScanRDD(@transient val session: SnappySession, } region match { - case Some(pr: PartitionedRegion) => session.sessionState.getTablePartitions(pr) - case Some(dr: CacheDistributionAdvisee) => session.sessionState.getTablePartitions(dr) + case Some(pr: PartitionedRegion) => session.snappySessionState.getTablePartitions(pr) + case Some(dr: CacheDistributionAdvisee) => session.snappySessionState.getTablePartitions(dr) // system table/VTI is shown as a replicated table having a single partition case _ => Array(new MultiBucketExecutorPartition(0, null, 0, Nil)) } diff --git a/core/src/main/scala/org/apache/spark/sql/execution/row/RowTableScan.scala b/core/src/main/scala/org/apache/spark/sql/execution/row/RowTableScan.scala index 3e321dfce1..407dc8e02e 100644 --- a/core/src/main/scala/org/apache/spark/sql/execution/row/RowTableScan.scala +++ b/core/src/main/scala/org/apache/spark/sql/execution/row/RowTableScan.scala @@ -22,11 +22,13 @@ import com.gemstone.gemfire.internal.shared.ClientSharedData import com.pivotal.gemfirexd.internal.engine.store.{AbstractCompactExecRow, ResultWasNull} import org.apache.spark.rdd.RDD +import org.apache.spark.sql.SnappySession +import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode} import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression} import org.apache.spark.sql.catalyst.util.{SerializedArray, SerializedMap, SerializedRow} import org.apache.spark.sql.collection.Utils -import org.apache.spark.sql.execution.{PartitionedDataSourceScan, PartitionedPhysicalScan, SparkPlan} +import org.apache.spark.sql.execution.{PartitionedDataSourceScan, PartitionedPhysicalScan} import org.apache.spark.sql.sources.BaseRelation import org.apache.spark.sql.types._ @@ -37,7 +39,7 @@ import org.apache.spark.sql.types._ * and Broadcast joins. This plan overrides outputPartitioning and * makes it inline with the partitioning of the underlying DataSource. */ -private[sql] final case class RowTableScan( +abstract case class RowTableScan( output: Seq[Attribute], _schema: StructType, dataRDD: RDD[Any], @@ -55,18 +57,21 @@ private[sql] final case class RowTableScan( override val nodeName: String = "RowTableScan" - override def sameResult(plan: SparkPlan): Boolean = plan match { - case r: RowTableScan => r.table == table && r.numBuckets == numBuckets && r.schema == schema - case _ => false + lazy val tableIdentifier: Option[TableIdentifier] = baseRelation match { + case null => None + case r => sqlContext match { + case null => Some(SnappySession.tableIdentifier(r.table, catalog = null, resolve = false)) + case c => + Some(c.sparkSession.asInstanceOf[SnappySession].tableIdentifier(r.table, resolve = true)) + } } override def doProduce(ctx: CodegenContext): String = { // a parent plan may set a custom input (e.g. HashJoinExec) // for that case no need to add the "shouldStop()" calls // PartitionedPhysicalRDD always has one input - val input = ctx.freshName("input") - ctx.addMutableState("scala.collection.Iterator", - input, s"$input = inputs[0];") + val input = internals.addClassField(ctx, "scala.collection.Iterator", "input", + v => s"$v = inputs[0];") val numOutputRows = if (sqlContext eq null) null else metricTerm(ctx, "numOutputRows") ctx.currentVars = null @@ -153,7 +158,7 @@ private[sql] final case class RowTableScan( private def genCodeCompactRowColumn(ctx: CodegenContext, rowVar: String, holder: String, ordinal: Int, dataType: DataType, nullable: Boolean): ExprCode = { - val javaType = ctx.javaType(dataType) + val javaType = internals.javaType(dataType, ctx) val col = ctx.freshName("col") val pos = ordinal + 1 var useHolder = true @@ -162,7 +167,8 @@ private[sql] final case class RowTableScan( s"final $javaType $col = $rowVar.getAsInt($pos, $holder);" case StringType => useHolder = false - s"final $javaType $col = $rowVar.getAsUTF8String($ordinal);" + val typeUtilsClass = TypeUtilities.getClass.getName.replace("$", "") + s"final $javaType $col = $typeUtilsClass.readUTF8String($rowVar, $ordinal);" case LongType => s"final $javaType $col = $rowVar.getAsLong($pos, $holder);" case BooleanType => @@ -256,21 +262,21 @@ private[sql] final case class RowTableScan( if (nullable) { val isNullVar = ctx.freshName("isNull") if (useHolder) { - ExprCode(s"$code\nfinal boolean $isNullVar = $holder.wasNullAndClear();", - isNullVar, col) + internals.newExprCode(s"$code\nfinal boolean $isNullVar = $holder.wasNullAndClear();", + isNullVar, col, dataType) } else { - ExprCode(s"$code\nfinal boolean $isNullVar = $col == null;", - isNullVar, col) + internals.newExprCode(s"$code\nfinal boolean $isNullVar = $col == null;", + isNullVar, col, dataType) } } else { - ExprCode(code, "false", col) + internals.newExprCode(code, "false", col, dataType) } } private def genCodeResultSetColumn(ctx: CodegenContext, rsVar: String, holder: String, ordinal: Int, dataType: DataType, nullable: Boolean): ExprCode = { - val javaType = ctx.javaType(dataType) + val javaType = internals.javaType(dataType, ctx) val col = ctx.freshName("col") val pos = ordinal + 1 val code = dataType match { @@ -375,10 +381,10 @@ private[sql] final case class RowTableScan( } if (nullable) { val isNullVar = ctx.freshName("isNull") - ExprCode(code + s"\nfinal boolean $isNullVar = $rsVar.wasNull();", - isNullVar, col) + internals.newExprCode(code + s"\nfinal boolean $isNullVar = $rsVar.wasNull();", + isNullVar, col, dataType) } else { - ExprCode(code, "false", col) + internals.newExprCode(code, "false", col, dataType) } } } diff --git a/core/src/main/scala/org/apache/spark/sql/execution/sources/StoreDataSourceStrategy.scala b/core/src/main/scala/org/apache/spark/sql/execution/sources/StoreDataSourceStrategy.scala index e5b701600e..a9a77690a4 100644 --- a/core/src/main/scala/org/apache/spark/sql/execution/sources/StoreDataSourceStrategy.scala +++ b/core/src/main/scala/org/apache/spark/sql/execution/sources/StoreDataSourceStrategy.scala @@ -39,24 +39,24 @@ import scala.collection.mutable import org.apache.spark.rdd.RDD import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeReference, AttributeSet, EmptyRow, Expression, NamedExpression, ParamLiteral, PredicateHelper, TokenLiteral} -import org.apache.spark.sql.catalyst.plans.logical.{BroadcastHint, LogicalPlan, Project, Filter => LFilter} -import org.apache.spark.sql.catalyst.plans.physical.UnknownPartitioning +import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Project, UnaryNode, Filter => LFilter} import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow, analysis, expressions} +import org.apache.spark.sql.execution.PartitionedDataSourceScan import org.apache.spark.sql.execution.datasources.LogicalRelation -import org.apache.spark.sql.execution.{PartitionedDataSourceScan, RowDataSourceScanExec} import org.apache.spark.sql.sources.{Filter, PrunedUnsafeFilteredScan} -import org.apache.spark.sql.{AnalysisException, SnappySession, SparkSession, Strategy, execution, sources} +import org.apache.spark.sql.{AnalysisException, SnappySession, SparkSession, SparkSupport, Strategy, execution, sources} /** * This strategy makes a PartitionedPhysicalRDD out of a PrunedFilterScan based datasource. * Mostly this is a copy of DataSourceStrategy of Spark. But it takes care of the underlying * partitions of the datasource. */ -private[sql] object StoreDataSourceStrategy extends Strategy { +private[sql] object StoreDataSourceStrategy extends Strategy with SparkSupport { def apply(plan: LogicalPlan): Seq[execution.SparkPlan] = plan match { case PhysicalScan(projects, filters, scan) => scan match { - case l@LogicalRelation(t: PartitionedDataSourceScan, _, _) => + case l: LogicalRelation if l.relation.isInstanceOf[PartitionedDataSourceScan] => + val t = l.relation.asInstanceOf[PartitionedDataSourceScan] pruneFilterProject( l, projects, @@ -64,7 +64,8 @@ private[sql] object StoreDataSourceStrategy extends Strategy { t.numBuckets, t.partitionColumns, (a, f) => t.buildUnsafeScan(a.map(_.name).toArray, f.toArray)) :: Nil - case l@LogicalRelation(t: PrunedUnsafeFilteredScan, _, _) => + case l: LogicalRelation if l.relation.isInstanceOf[PrunedUnsafeFilteredScan] => + val t = l.relation.asInstanceOf[PrunedUnsafeFilteredScan] pruneFilterProject( l, projects, @@ -72,7 +73,7 @@ private[sql] object StoreDataSourceStrategy extends Strategy { 0, Nil, (a, f) => t.buildUnsafeScan(a.map(_.name).toArray, f.toArray)) :: Nil - case LogicalRelation(_, _, _) => + case _: LogicalRelation => var foundParamLiteral = false val tp = plan.transformAllExpressions { case pl: ParamLiteral => @@ -156,10 +157,15 @@ private[sql] object StoreDataSourceStrategy extends Strategy { }) } else Nil + var pushedFilters: Seq[Filter] = Nil + var handledFilters: Seq[Filter] = Nil + def getMetadata: Map[String, String] = if (numBuckets > 0) { Map.empty[String, String] } else { - val pushedFilters = candidatePredicates.flatMap(translateToFilter) + pushedFilters = candidatePredicates.flatMap(translateToFilter) + handledFilters = (candidatePredicates.toSet -- unhandledPredicates.toSet) + .flatMap(translateToFilter).toSeq val pairs = mutable.ArrayBuffer.empty[(String, String)] if (pushedFilters.nonEmpty) { pairs += ("PushedFilters" -> @@ -198,11 +204,11 @@ private[sql] object StoreDataSourceStrategy extends Strategy { (requestedColumns, candidatePredicates) ) case baseRelation => - RowDataSourceScanExec( - mappedProjects, + val metadata = getMetadata + internals.newRowDataSourceScanExec( + mappedProjects, mappedProjects.indices, pushedFilters, handledFilters, scanBuilder(requestedColumns, candidatePredicates)._1.asInstanceOf[RDD[InternalRow]], - baseRelation, UnknownPartitioning(0), getMetadata, - relation.catalogTable.map(_.identifier)) + metadata, baseRelation, relation.catalogTable.map(_.identifier)) } filterCondition.map(execution.FilterExec(_, scan)).getOrElse(scan) } else { @@ -226,11 +232,11 @@ private[sql] object StoreDataSourceStrategy extends Strategy { (requestedColumns, candidatePredicates) ) case baseRelation => - RowDataSourceScanExec( - mappedProjects, + val metadata = getMetadata + internals.newRowDataSourceScanExec( + mappedProjects, mappedProjects.indices, pushedFilters, handledFilters, scanBuilder(requestedColumns, candidatePredicates)._1.asInstanceOf[RDD[InternalRow]], - baseRelation, UnknownPartitioning(0), getMetadata, - relation.catalogTable.map(_.identifier)) + metadata, baseRelation, relation.catalogTable.map(_.identifier)) } if (projectOnlyAttributes || allDeterministic || filterCondition.isEmpty) { execution.ProjectExec(projects, @@ -337,7 +343,8 @@ private[sql] object StoreDataSourceStrategy extends Strategy { * [[org.apache.spark.sql.catalyst.expressions.Alias Aliases]] are in-lined/substituted if * necessary. */ -object PhysicalScan extends PredicateHelper { +object PhysicalScan extends PredicateHelper with SparkSupport { + type ReturnType = (Seq[NamedExpression], Seq[Expression], LogicalPlan) def unapply(plan: LogicalPlan): Option[ReturnType] = { @@ -372,7 +379,8 @@ object PhysicalScan extends PredicateHelper { val substitutedCondition = substitute(aliases)(condition) (fields, filters ++ splitConjunctivePredicates(substitutedCondition), other, aliases) - case BroadcastHint(child) => collectProjectsAndFilters(child) + case _ if internals.isHintPlan(plan) => + collectProjectsAndFilters(plan.asInstanceOf[UnaryNode].child) case other => (None, Nil, other, Map.empty) } @@ -383,14 +391,14 @@ object PhysicalScan extends PredicateHelper { private def substitute(aliases: Map[Attribute, Expression])(expr: Expression): Expression = { expr.transform { - case a@Alias(ref: AttributeReference, name) => - aliases.get(ref) - .map(Alias(_, name)(a.exprId, a.qualifier, isGenerated = a.isGenerated)) - .getOrElse(a) - - case a: AttributeReference => - aliases.get(a) - .map(Alias(_, a.name)(a.exprId, a.qualifier, isGenerated = a.isGenerated)).getOrElse(a) + case a@Alias(ref: AttributeReference, name) => aliases.get(ref) match { + case None => a + case Some(e) => internals.newAlias(e, name, Some(a)) + } + case a: AttributeReference => aliases.get(a) match { + case None => a + case Some(e) => internals.newAlias(e, a.name, Some(a)) + } } } } diff --git a/core/src/main/scala/org/apache/spark/sql/hive/HiveClientUtil.scala b/core/src/main/scala/org/apache/spark/sql/hive/HiveClientUtil.scala index 4fe2594db2..45d6f83776 100644 --- a/core/src/main/scala/org/apache/spark/sql/hive/HiveClientUtil.scala +++ b/core/src/main/scala/org/apache/spark/sql/hive/HiveClientUtil.scala @@ -16,9 +16,8 @@ */ package org.apache.spark.sql.hive -import java.util.Properties - import java.nio.file.Paths +import java.util.Properties import com.gemstone.gemfire.internal.shared.SystemProperties import com.pivotal.gemfirexd.Attribute.{PASSWORD_ATTR, USERNAME_ATTR} diff --git a/core/src/main/scala/org/apache/spark/sql/hive/SnappyHiveExternalCatalog.scala b/core/src/main/scala/org/apache/spark/sql/hive/SnappyHiveExternalCatalog.scala index cf0109e968..1f7f0f1e36 100644 --- a/core/src/main/scala/org/apache/spark/sql/hive/SnappyHiveExternalCatalog.scala +++ b/core/src/main/scala/org/apache/spark/sql/hive/SnappyHiveExternalCatalog.scala @@ -49,7 +49,6 @@ import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.{NoSuchDatabaseException, NoSuchTableException, TableAlreadyExistsException} import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec import org.apache.spark.sql.catalyst.catalog._ -import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.catalyst.util.StringUtils import org.apache.spark.sql.collection.Utils.EMPTY_STRING_ARRAY import org.apache.spark.sql.collection.{ToolsCallbackInit, Utils} @@ -63,38 +62,40 @@ import org.apache.spark.sql.sources.JdbcExtendedUtils.normalizeSchema import org.apache.spark.sql.store.CodeGeneration import org.apache.spark.sql.types.LongType -class SnappyHiveExternalCatalog private[hive](val conf: SparkConf, +abstract class SnappyHiveExternalCatalog(val conf: SparkConf, val hadoopConf: Configuration, val createTime: Long) extends SnappyHiveCatalogBase(conf, hadoopConf) with SnappyExternalCatalog { - { + /** A cache of Spark SQL data source tables that have been accessed. */ + protected final val cachedCatalogTables: LoadingCache[(String, String), CatalogTable] = { + + // base initialization first + // fire dummy queries to initialize more components of hive meta-store withHiveExceptionHandling { assert(!client.tableExists(SYS_SCHEMA, "dbs")) assert(!client.functionExists(SYS_SCHEMA, "funcs")) } - } - /** A cache of Spark SQL data source tables that have been accessed. */ - protected val cachedCatalogTables: LoadingCache[(String, String), CatalogTable] = { + // initialize the CacheLoader + val cacheLoader = new CacheLoader[(String, String), CatalogTable]() { override def load(name: (String, String)): CatalogTable = { logDebug(s"Looking up data source for ${name._1}.${name._2}") - try { - withHiveExceptionHandling(SnappyHiveExternalCatalog.super.getTableOption( - name._1, name._2)) match { - case None => + withHiveExceptionHandling { + try { + finalizeCatalogTable(SnappyHiveExternalCatalog.super.getTable(name._1, name._2)) + } catch { + case _: NoSuchTableException => nonExistentTables.put(name, java.lang.Boolean.TRUE) throw new TableNotFoundException(name._1, name._2) - case Some(catalogTable) => finalizeCatalogTable(catalogTable) + case _: NullPointerException => + // dropTableUnsafe() searches for below exception message. check before changing. + throw new AnalysisException( + s"Table ${name._1}.${name._2} might be inconsistent in hive catalog. " + + "Use system procedure SYS.REMOVE_METASTORE_ENTRY to remove inconsistency. " + + "Refer to troubleshooting section of documentation for more details") } - } catch { - case _: NullPointerException => - // dropTableUnsafe() searches for below exception message. check before changing. - throw new AnalysisException( - s"Table ${name._1}.${name._2} might be inconsistent in hive catalog. " + - "Use system procedure SYS.REMOVE_METASTORE_ENTRY to remove inconsistency. " + - "Refer to troubleshooting section of documentation for more details") } } } @@ -102,7 +103,7 @@ class SnappyHiveExternalCatalog private[hive](val conf: SparkConf, } /** A cache of SQL data source tables that are missing in catalog. */ - protected val nonExistentTables: Cache[(String, String), java.lang.Boolean] = { + protected final val nonExistentTables: Cache[(String, String), java.lang.Boolean] = { CacheBuilder.newBuilder().maximumSize(ConnectorExternalCatalog.cacheSize).build() } @@ -123,7 +124,7 @@ class SnappyHiveExternalCatalog private[hive](val conf: SparkConf, /** * Retries on transient disconnect exceptions. */ - private[sql] def withHiveExceptionHandling[T](function: => T, + protected[sql] def withHiveExceptionHandling[T](function: => T, handleDisconnects: Boolean = true): T = synchronized { val skipFlags = GfxdDataDictionary.SKIP_CATALOG_OPS.get() val oldSkipCatalogCalls = skipFlags.skipHiveCatalogCalls @@ -189,11 +190,39 @@ class SnappyHiveExternalCatalog private[hive](val conf: SparkConf, } } + // -------------------------------------------------------------------------- + // Base HiveExternalCatalog calls + // -------------------------------------------------------------------------- + + protected def baseCreateDatabase(schemaDefinition: CatalogDatabase, + ignoreIfExists: Boolean): Unit + + protected def baseDropDatabase(schema: String, ignoreIfNotExists: Boolean, + cascade: Boolean): Unit + + protected def baseCreateTable(tableDefinition: CatalogTable, ignoreIfExists: Boolean): Unit + + protected def baseDropTable(schema: String, table: String, ignoreIfNotExists: Boolean, + purge: Boolean): Unit + + protected def baseAlterTable(table: CatalogTable): Unit + + protected def baseRenameTable(schema: String, oldName: String, newName: String): Unit + + protected def baseLoadDynamicPartitions(schema: String, table: String, loadPath: String, + partition: TablePartitionSpec, replace: Boolean, numDP: Int, holdDDLTime: Boolean): Unit + + protected def baseCreateFunction(schema: String, funcDefinition: CatalogFunction): Unit + + protected def baseDropFunction(schema: String, name: String): Unit + + protected def baseRenameFunction(schema: String, oldName: String, newName: String): Unit + // -------------------------------------------------------------------------- // Databases // -------------------------------------------------------------------------- - override def createDatabase(schemaDefinition: CatalogDatabase, + protected def createDatabaseImpl(schemaDefinition: CatalogDatabase, ignoreIfExists: Boolean): Unit = { // dot is used for schema, name separation and will cause many problems if present if (schemaDefinition.name.indexOf('.') != -1) { @@ -209,21 +238,31 @@ class SnappyHiveExternalCatalog private[hive](val conf: SparkConf, if (ignoreIfExists) return else throw new AnalysisException(s"Schema ${schemaDefinition.name} already exists") } - withHiveExceptionHandling(super.createDatabase(schemaDefinition, ignoreIfExists)) + withHiveExceptionHandling(baseCreateDatabase(schemaDefinition, ignoreIfExists)) } - override def dropDatabase(schema: String, ignoreIfNotExists: Boolean, cascade: Boolean): Unit = { + protected def dropDatabaseImpl(schema: String, ignoreIfNotExists: Boolean, + cascade: Boolean): Unit = { if (schema == SYS_SCHEMA) { throw new AnalysisException(s"$schema is a system preserved database/schema") } try { - withHiveExceptionHandling(super.dropDatabase(schema, ignoreIfNotExists, cascade)) + withHiveExceptionHandling(baseDropDatabase(schema, ignoreIfNotExists, cascade)) } catch { case _: NoSuchDatabaseException | _: NoSuchObjectException => throw SnappyExternalCatalog.schemaNotFoundException(schema) } } + protected def alterDatabaseImpl(schemaDefinition: CatalogDatabase): Unit = { + try { + withHiveExceptionHandling(super.alterDatabase(schemaDefinition)) + } catch { + case _: NoSuchDatabaseException | _: NoSuchObjectException => + throw SnappyExternalCatalog.schemaNotFoundException(schemaDefinition.name) + } + } + // Special in-built SYS schema does not have hive catalog entry so the methods below // add that specifically to the existing schemas. @@ -270,15 +309,6 @@ class SnappyHiveExternalCatalog private[hive](val conf: SparkConf, } } - override def alterDatabase(schemaDefinition: CatalogDatabase): Unit = { - try { - withHiveExceptionHandling(super.alterDatabase(schemaDefinition)) - } catch { - case _: NoSuchDatabaseException | _: NoSuchObjectException => - throw SnappyExternalCatalog.schemaNotFoundException(schemaDefinition.name) - } - } - // -------------------------------------------------------------------------- // Tables // -------------------------------------------------------------------------- @@ -312,12 +342,10 @@ class SnappyHiveExternalCatalog private[hive](val conf: SparkConf, props = JdbcExtendedUtils.addSplitProperty(v, SPLIT_VIEW_TEXT_PROPERTY, props, maxLen) case _ => } - if (catalogTable.viewOriginalText.isEmpty && catalogTable.viewText.isDefined) { - catalogTable = catalogTable.copy(viewOriginalText = catalogTable.viewText) - } - catalogTable.viewOriginalText match { + internals.catalogTableViewOriginalText(catalogTable) match { case Some(v) if v.length > maxLen => - catalogTable = catalogTable.copy(viewOriginalText = Some(v.substring(0, maxLen))) + catalogTable = internals.newCatalogTableWithViewOriginalText( + catalogTable, Some(v.substring(0, maxLen))) props = JdbcExtendedUtils.addSplitProperty(v, SPLIT_VIEW_ORIGINAL_TEXT_PROPERTY, props, maxLen) case _ => @@ -355,9 +383,14 @@ class SnappyHiveExternalCatalog private[hive](val conf: SparkConf, } } - override def createTable(tableDefinition: CatalogTable, ignoreIfExists: Boolean): Unit = { + protected def createTableImpl(tableDefinition: CatalogTable, ignoreIfExists: Boolean): Unit = { val catalogTable = addViewProperties(tableDefinition) - var ifExists = ignoreIfExists + var ifExists = + if (ignoreIfExists) { + val realIfExists = SnappyHiveExternalCatalog.ignoreIfExists.get() + // check if the CTAS flag has been explicitly set else honour the passed flag + (realIfExists eq null) || realIfExists.booleanValue() + } else false // Add dependency on base table if required. This is done before actual table // entry so that if there is a cluster failure between the two steps, then // table will still not be in catalog and base table will simply ignore @@ -390,7 +423,7 @@ class SnappyHiveExternalCatalog private[hive](val conf: SparkConf, } try { - withHiveExceptionHandling(super.createTable(catalogTable, ifExists)) + withHiveExceptionHandling(baseCreateTable(catalogTable, ifExists)) } catch { case e: TableAlreadyExistsException => val objectType = CatalogObjectType.getTableType(tableDefinition) @@ -428,14 +461,14 @@ class SnappyHiveExternalCatalog private[hive](val conf: SparkConf, invalidate(schema -> table) } - override def dropTable(schema: String, table: String, ignoreIfNotExists: Boolean, + protected def dropTableImpl(schema: String, table: String, ignoreIfNotExists: Boolean, purge: Boolean): Unit = { - val tableDefinition = getTableOption(schema, table) match { + val tableDefinition = getTableIfExists(schema, table) match { case None => if (ignoreIfNotExists) return else throw new TableNotFoundException(schema, table) case Some(t) => t } - withHiveExceptionHandling(super.dropTable(schema, table, ignoreIfNotExists, purge)) + withHiveExceptionHandling(baseDropTable(schema, table, ignoreIfNotExists, purge)) // drop all policies for the table if (Misc.getMemStoreBooting.isRLSEnabled) { @@ -443,7 +476,7 @@ class SnappyHiveExternalCatalog private[hive](val conf: SparkConf, if (policies.nonEmpty) for (policy <- policies) { val schemaName = policy.database val policyName = policy.identifier.table - withHiveExceptionHandling(super.dropTable(schemaName, policyName, + withHiveExceptionHandling(baseDropTable(schemaName, policyName, ignoreIfNotExists = true, purge = false)) invalidate(schemaName -> policyName) } @@ -460,7 +493,7 @@ class SnappyHiveExternalCatalog private[hive](val conf: SparkConf, registerCatalogSchemaChange(refreshRelations) } - override def alterTable(tableDefinition: CatalogTable): Unit = { + protected def alterTableImpl(tableDefinition: CatalogTable): Unit = { val catalogTable = addViewProperties(tableDefinition) val schemaName = catalogTable.database val tableName = catalogTable.identifier.table @@ -489,15 +522,15 @@ class SnappyHiveExternalCatalog private[hive](val conf: SparkConf, } } - withHiveExceptionHandling(super.alterTable(catalogTable)) + withHiveExceptionHandling(baseAlterTable(catalogTable)) registerCatalogSchemaChange(schemaName -> tableName :: Nil) } - override def renameTable(schemaName: String, oldName: String, newName: String): Unit = { - withHiveExceptionHandling(super.renameTable(schemaName, oldName, newName)) + protected def renameTableImpl(schema: String, oldName: String, newName: String): Unit = { + withHiveExceptionHandling(baseRenameTable(schema, oldName, newName)) - registerCatalogSchemaChange(schemaName -> oldName :: schemaName -> newName :: Nil) + registerCatalogSchemaChange(schema -> oldName :: schema -> newName :: Nil) } /** @@ -512,13 +545,13 @@ class SnappyHiveExternalCatalog private[hive](val conf: SparkConf, val viewText = JdbcExtendedUtils.readSplitProperty(SPLIT_VIEW_TEXT_PROPERTY, table.properties).orElse(table.viewText) val viewOriginalText = JdbcExtendedUtils.readSplitProperty(SPLIT_VIEW_ORIGINAL_TEXT_PROPERTY, - table.properties).orElse(table.viewOriginalText) + table.properties).orElse(internals.catalogTableViewOriginalText(table)) // update the meta-data from properties ExternalStoreUtils.getTableSchema(table.properties, forView = true) match { - case Some(s) => table.copy(identifier = tableIdent, schema = s, viewText = viewText, - viewOriginalText = viewOriginalText) - case None => table.copy(identifier = tableIdent, viewText = viewText, - viewOriginalText = viewOriginalText) + case Some(s) => internals.newCatalogTableWithViewOriginalText( + table.copy(identifier = tableIdent, schema = s, viewText = viewText), viewOriginalText) + case None => internals.newCatalogTableWithViewOriginalText( + table.copy(identifier = tableIdent, viewText = viewText), viewOriginalText) } } else if (CatalogObjectType.isPolicy(table)) { // explicitly change table name in policy properties to lower-case @@ -563,14 +596,6 @@ class SnappyHiveExternalCatalog private[hive](val conf: SparkConf, else withHiveExceptionHandling(cachedCatalogTables.get(name)) } - override def getTableOption(schema: String, table: String): Option[CatalogTable] = { - try { - Some(getTable(schema, table)) - } catch { - case _: NoSuchTableException => None - } - } - private def toLowerCase(s: Array[String]): Array[String] = { val r = new Array[String](s.length) for (i <- s.indices) { @@ -645,7 +670,7 @@ class SnappyHiveExternalCatalog private[hive](val conf: SparkConf, val expandedApplyTo = ExternalStoreUtils.getExpandedGranteesIterator(applyTo).toSeq val newProperties = table.properties + (PolicyProperties.expandedPolicyApplyTo -> expandedApplyTo.mkString(",")) - withHiveExceptionHandling(super.alterTable(table.copy(properties = newProperties))) + withHiveExceptionHandling(baseAlterTable(table.copy(properties = newProperties))) } } } @@ -729,9 +754,9 @@ class SnappyHiveExternalCatalog private[hive](val conf: SparkConf, isOverwrite, holdDDLTime, inheritTableSpecs)) } - override def loadDynamicPartitions(schema: String, table: String, loadPath: String, + protected def loadDynamicPartitionsImpl(schema: String, table: String, loadPath: String, partition: TablePartitionSpec, replace: Boolean, numDP: Int, holdDDLTime: Boolean): Unit = { - withHiveExceptionHandling(super.loadDynamicPartitions(schema, table, loadPath, partition, + withHiveExceptionHandling(baseLoadDynamicPartitions(schema, table, loadPath, partition, replace, numDP, holdDDLTime)) } @@ -755,27 +780,22 @@ class SnappyHiveExternalCatalog private[hive](val conf: SparkConf, withHiveExceptionHandling(super.listPartitions(schema, table, partialSpec)) } - override def listPartitionsByFilter(schema: String, table: String, - predicates: Seq[Expression]): Seq[CatalogTablePartition] = { - withHiveExceptionHandling(super.listPartitionsByFilter(schema, table, predicates)) - } - // -------------------------------------------------------------------------- // Functions // -------------------------------------------------------------------------- - override def createFunction(schema: String, funcDefinition: CatalogFunction): Unit = { - withHiveExceptionHandling(super.createFunction(schema, funcDefinition)) + protected def createFunctionImpl(schema: String, funcDefinition: CatalogFunction): Unit = { + withHiveExceptionHandling(baseCreateFunction(schema, funcDefinition)) SnappySession.clearAllCache() } - override def dropFunction(schema: String, name: String): Unit = { - withHiveExceptionHandling(super.dropFunction(schema, name)) + protected def dropFunctionImpl(schema: String, name: String): Unit = { + withHiveExceptionHandling(baseDropFunction(schema, name)) SnappySession.clearAllCache() } - override def renameFunction(schema: String, oldName: String, newName: String): Unit = { - withHiveExceptionHandling(super.renameFunction(schema, oldName, newName)) + protected def renameFunctionImpl(schema: String, oldName: String, newName: String): Unit = { + withHiveExceptionHandling(baseRenameFunction(schema, oldName, newName)) SnappySession.clearAllCache() } @@ -834,11 +854,19 @@ class SnappyHiveExternalCatalog private[hive](val conf: SparkConf, } } -object SnappyHiveExternalCatalog { +object SnappyHiveExternalCatalog extends SparkSupport { @GuardedBy("this") private[this] var instance: SnappyHiveExternalCatalog = _ + /** + * Hack for CTAS for builtin tables that need to pre-create the tables before + * insert for the store layer to find them. This flag allows handling of this + * case in the ExternalCatalog.createTable method. + */ + private[sql] val ignoreIfExists: ThreadLocal[java.lang.Boolean] = + new ThreadLocal[java.lang.Boolean]() + def getInstance(sparkConf: SparkConf, hadoopConf: Configuration): SnappyHiveExternalCatalog = synchronized { val catalog = instance @@ -866,7 +894,7 @@ object SnappyHiveExternalCatalog { log4jLogger.setLevel(Level.ERROR) } try { - instance = new SnappyHiveExternalCatalog(sparkConf, hadoopConf, createTime) + instance = internals.newEmbeddedHiveCatalog(sparkConf, hadoopConf, createTime) } finally { logger.setLevel(previousLevel) log4jLogger.setLevel(log4jLevel) diff --git a/core/src/main/scala/org/apache/spark/sql/hive/SnappySessionState.scala b/core/src/main/scala/org/apache/spark/sql/hive/SnappySessionState.scala index 15d16ed41d..b74a832708 100644 --- a/core/src/main/scala/org/apache/spark/sql/hive/SnappySessionState.scala +++ b/core/src/main/scala/org/apache/spark/sql/hive/SnappySessionState.scala @@ -24,14 +24,15 @@ import scala.collection.mutable.ArrayBuffer import com.gemstone.gemfire.internal.cache.{CacheDistributionAdvisee, ColocationHelper, PartitionedRegion} import com.pivotal.gemfirexd.internal.engine.store.GemFireStore import io.snappydata.Property -import io.snappydata.Property.HashAggregateSize import org.apache.spark.Partition import org.apache.spark.sql.catalyst.analysis -import org.apache.spark.sql.catalyst.analysis.TypeCoercion.{PromoteStrings, numericPrecedence} -import org.apache.spark.sql.catalyst.analysis.{Analyzer, CleanupAliases, EliminateUnions, ResolveCreateNamedStruct, ResolveInlineTables, ResolveTableValuedFunctions, Star, SubstituteUnresolvedOrdinals, TimeWindowing, TypeCoercion, UnresolvedAttribute} -import org.apache.spark.sql.catalyst.expressions.{And, BinaryArithmetic, EqualTo, In, ScalarSubquery, _} -import org.apache.spark.sql.catalyst.optimizer.{Optimizer, ReorderJoin} +import org.apache.spark.sql.catalyst.analysis.TypeCoercion.numericPrecedence +import org.apache.spark.sql.catalyst.analysis.{Analyzer, Star, UnresolvedAttribute} +import org.apache.spark.sql.catalyst.catalog.SessionCatalog +import org.apache.spark.sql.catalyst.expressions.{And, BinaryArithmetic, EqualTo, In, _} +import org.apache.spark.sql.catalyst.optimizer.Optimizer +import org.apache.spark.sql.catalyst.parser.ParserInterface import org.apache.spark.sql.catalyst.planning.ExtractEquiJoinKeys import org.apache.spark.sql.catalyst.plans.JoinType import org.apache.spark.sql.catalyst.plans.logical.{Filter => LogicalFilter, _} @@ -41,9 +42,8 @@ import org.apache.spark.sql.execution._ import org.apache.spark.sql.execution.columnar.impl.IndexColumnFormatRelation import org.apache.spark.sql.execution.command.{ExecutedCommandExec, RunnableCommand} import org.apache.spark.sql.execution.datasources._ -import org.apache.spark.sql.execution.exchange.{EnsureRequirements, ReuseExchange} import org.apache.spark.sql.execution.sources.{PhysicalScan, StoreDataSourceStrategy} -import org.apache.spark.sql.hive.execution.{CreateHiveTableAsSelectCommand, HiveTableScanExec, InsertIntoHiveTable} +import org.apache.spark.sql.hive.execution.{CreateHiveTableAsSelectCommand, HiveTableScanExec} import org.apache.spark.sql.internal._ import org.apache.spark.sql.policy.PolicyProperties import org.apache.spark.sql.sources._ @@ -57,27 +57,26 @@ import org.apache.spark.streaming.Duration /** * Holds all session-specific state for a given [[SnappySession]]. */ -class SnappySessionState(val snappySession: SnappySession) - extends SessionState(snappySession) with SnappyStrategies { +trait SnappySessionState extends SessionState with SnappyStrategies with SparkSupport { - @transient - val contextFunctions: SnappyContextFunctions = new SnappyContextFunctions + val snappySession: SnappySession - val sampleSnappyCase: PartialFunction[LogicalPlan, Seq[SparkPlan]] = { - case MarkerForCreateTableAsSelect(child) => PlanLater(child) :: Nil - case BypassRowLevelSecurity(child) => PlanLater(child) :: Nil - case _ => Nil - } + def catalogBuilder(wrapped: Option[SnappySessionCatalog]): SessionCatalog - override lazy val streamingQueryManager: StreamingQueryManager = { - // Disabling `SnappyAggregateStrategy` for streaming queries as it clashes with - // `StatefulAggregationStrategy` which is applied by spark for streaming queries. This - // implies that Snappydata aggregation optimisation will be turned off for any usage of - // this session including non-streaming queries. + def analyzerBuilder(): Analyzer - HashAggregateSize.set(conf, "-1") - new StreamingQueryManager(snappySession) - } + def optimizerBuilder(): Optimizer + + val conf: SQLConf + val sqlParser: ParserInterface + val streamingQueryManager: StreamingQueryManager + + final def snappyConf: SnappyConf = conf.asInstanceOf[SnappyConf] + + final def snappySqlParser: SnappySqlParser = sqlParser.asInstanceOf[SnappySqlParser] + + private[sql] lazy val sampleSnappyCase: PartialFunction[LogicalPlan, Seq[SparkPlan]] = + snappySession.contextFunctions.createSampleSnappyCase() private[sql] lazy val hiveSession: SparkSession = { // disable enableHiveSupport during initialization to avoid calls into SnappyConf @@ -86,15 +85,14 @@ class SnappySessionState(val snappySession: SnappySession) snappySession.hiveInitializing = true val session = SnappyContext.newHiveSession() val hiveConf = session.sessionState.conf - conf.foreach(hiveConf.setConfString) + snappyConf.foreach(hiveConf.setConfString) hiveConf.setConfString(StaticSQLConf.CATALOG_IMPLEMENTATION.key, "hive") snappySession.enableHiveSupport = oldValue snappySession.hiveInitializing = false session } - private[sql] lazy val hiveState: HiveSessionState = - hiveSession.sessionState.asInstanceOf[HiveSessionState] + private[sql] def hiveState: SessionState = hiveSession.sessionState /** * Execute a method switching the session and shared states in the session to external hive. @@ -111,64 +109,16 @@ class SnappySessionState(val snappySession: SnappySession) } } - override lazy val sqlParser: SnappySqlParser = - contextFunctions.newSQLParser(this.snappySession) - private[sql] var disableStoreOptimizations: Boolean = false - def getExtendedResolutionRules(analyzer: Analyzer): Seq[Rule[LogicalPlan]] = - new HiveConditionalRule(_.catalog.ParquetConversions, this) :: - new HiveConditionalRule(_.catalog.OrcConversions, this) :: - AnalyzeCreateTable(snappySession) :: - new PreprocessTable(this) :: - ResolveAliasInGroupBy :: - new FindDataSourceTable(snappySession) :: - DataSourceAnalysis(conf) :: - AnalyzeMutableOperations(snappySession, analyzer) :: - ResolveQueryHints(snappySession) :: - RowLevelSecurity :: - ExternalRelationLimitFetch :: - (if (conf.runSQLonFile) new ResolveDataSource(snappySession) :: - Nil else Nil) - - - def getExtendedCheckRules: Seq[LogicalPlan => Unit] = { - Seq(ConditionalPreWriteCheck(datasources.PreWriteCheck(conf, wrapperCatalog)), PrePutCheck) - } + override lazy val analyzer: Analyzer = analyzerBuilder() - override lazy val analyzer: Analyzer = new SnappyAnalyzer(this) { - - override val extendedCheckRules: Seq[LogicalPlan => Unit] = getExtendedCheckRules - - override val extendedResolutionRules: Seq[Rule[LogicalPlan]] = - getExtendedResolutionRules(this) - } - - override lazy val optimizer: Optimizer = new SparkOptimizer(catalog, conf, experimentalMethods) { - override def batches: Seq[Batch] = { - implicit val ss: SnappySession = snappySession - var insertedSnappyOpts = 0 - val modified = super.batches.map { - case batch if batch.name.equalsIgnoreCase("Operator Optimizations") => - insertedSnappyOpts += 1 - val (left, right) = batch.rules.splitAt(batch.rules.indexOf(ReorderJoin)) - Batch(batch.name, batch.strategy, (left :+ ResolveIndex()) ++ right: _*) - case b => b - } - - if (insertedSnappyOpts != 1) { - throw new AnalysisException("Snappy Optimizations not applied") - } + override lazy val optimizer: Optimizer = optimizerBuilder() - modified :+ - Batch("Streaming SQL Optimizers", Once, PushDownWindowLogicalPlan) :+ - Batch("Link buckets to RDD partitions", Once, new LinkPartitionsToBuckets) :+ - Batch("TokenizedLiteral Folding Optimization", Once, TokenizedLiteralFolding) :+ - Batch("Order join conditions ", Once, OrderJoinConditions) - } + protected[sql] def getExtendedCheckRules: Seq[LogicalPlan => Unit] = { + Seq(ConditionalPreWriteCheck(internals.newPreWriteCheck(this)), PrePutCheck, HiveOnlyCheck) } - // copy of ConstantFolding that will turn a constant up/down cast into // a static value. object TokenizedLiteralFolding extends Rule[LogicalPlan] { @@ -195,7 +145,7 @@ class SnappySessionState(val snappySession: SnappySession) } p // also mark linking for scalar/predicate subqueries and disable plan caching - case s@(_: ScalarSubquery | _: PredicateSubquery) if foldable => + case s: SubqueryExpression if foldable => snappySession.linkPartitionsToBuckets(flag = true) snappySession.planCaching = false s @@ -220,11 +170,11 @@ class SnappySessionState(val snappySession: SnappySession) DynamicFoldableExpression(mark(e, foldable = false)) } - plan transform { + internals.logicalPlanResolveDown(plan) { // transformDown for expression so that top-most node which is foldable gets // selected for wrapping by DynamicFoldableExpression and further sub-expressions // do not since foldExpression will reset inner ParamLiterals as non-foldable - case q: LogicalPlan => q.mapExpressions(expr => unmarkAll(mark(expr).transformDown { + case q: LogicalPlan => q.mapExpressions(ex => unmarkAll(mark(ex).transformDown { // ignore leaf literals case l@(_: Literal | _: DynamicReplacableConstant) => l // Wrap expressions that are foldable. @@ -266,18 +216,16 @@ class SnappySessionState(val snappySession: SnappySession) var duration: Duration = null var slide: Option[Duration] = None var transformed: Boolean = false - plan transformDown { + internals.logicalPlanResolveDown(plan) { case win@WindowLogicalPlan(d, s, child, false) => child match { - case LogicalRelation(_, _, _) | - LogicalDStreamPlan(_, _) => win + case _: LogicalRelation | _: LogicalDStreamPlan => win case _ => duration = d slide = s transformed = true win.child } - case c@(LogicalRelation(_, _, _) | - LogicalDStreamPlan(_, _)) => + case c@(_: LogicalRelation | _: LogicalDStreamPlan) => if (transformed) { transformed = false WindowLogicalPlan(duration, slide, c, transformed = true) @@ -291,7 +239,7 @@ class SnappySessionState(val snappySession: SnappySession) * be created for tables to be the same as number of buckets. This will avoid * exchange on one side of a non-collocated join in many cases. */ - final class LinkPartitionsToBuckets extends Rule[LogicalPlan] { + object LinkPartitionsToBuckets extends Rule[LogicalPlan] { def apply(plan: LogicalPlan): LogicalPlan = { plan.foreach { case _ if Property.ForceLinkPartitionsToBuckets.get(conf) => @@ -300,18 +248,18 @@ class SnappySessionState(val snappySession: SnappySession) case j: Join if !JoinStrategy.isReplicatedJoin(j) => // disable for the entire query for consistency snappySession.linkPartitionsToBuckets(flag = true) - case _: InsertIntoTable | _: TableMutationPlan | - LogicalRelation(_: IndexColumnFormatRelation, _, _) => + case _: InsertIntoTable | _: TableMutationPlan => // disable for inserts/puts to avoid exchanges and indexes to work correctly snappySession.linkPartitionsToBuckets(flag = true) + case l: LogicalRelation if l.relation.isInstanceOf[IndexColumnFormatRelation] => + // disable for indexes + snappySession.linkPartitionsToBuckets(flag = true) case _ => // nothing for others } plan } } - override lazy val conf: SnappyConf = new SnappyConf(snappySession) - /** * The partition mapping selected for the lead partitioned region in * a collocated chain for current execution @@ -327,7 +275,7 @@ class SnappySessionState(val snappySession: SnappySession) * Orders the join keys as per the underlying partitioning keys ordering of the table. */ object OrderJoinConditions extends Rule[LogicalPlan] with JoinQueryPlanning { - def apply(plan: LogicalPlan): LogicalPlan = plan transform { + def apply(plan: LogicalPlan): LogicalPlan = internals.logicalPlanResolveDown(plan) { case ExtractEquiJoinKeys(joinType, leftKeys, rightKeys, otherCondition, left, right) => prepareOrderedCondition(joinType, left, right, leftKeys, rightKeys, otherCondition) } @@ -335,8 +283,9 @@ class SnappySessionState(val snappySession: SnappySession) def getPartCols(plan: LogicalPlan): Seq[NamedExpression] = { plan match { case PhysicalScan(_, _, child) => child match { - case r@LogicalRelation(scan: PartitionedDataSourceScan, _, _) => + case r: LogicalRelation if r.relation.isInstanceOf[PartitionedDataSourceScan] => // send back numPartitions=1 for replicated table since collocated + val scan = r.relation.asInstanceOf[PartitionedDataSourceScan] if (!scan.isPartitioned) return Nil val partCols = scan.partitionColumns.map(colName => r.resolveQuoted(colName, analysis.caseInsensitiveResolution) @@ -393,7 +342,7 @@ class SnappySessionState(val snappySession: SnappySession) } object ResolveAliasInGroupBy extends Rule[LogicalPlan] { - def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators { + def apply(plan: LogicalPlan): LogicalPlan = internals.logicalPlanResolveDown(plan) { // pivot with '*' projection messes up references for some reason // in older versions of Spark case Project(projectList, p: Pivot) @@ -416,7 +365,7 @@ class SnappySessionState(val snappySession: SnappySession) if groupBy.isEmpty && pivotColumn.resolved && aggregates.forall(_.resolved) => val pivotColAndAggRefs = pivotColumn.references ++ AttributeSet(aggregates) val groupByExprs = child.output.filterNot(pivotColAndAggRefs.contains) - p.copy(groupByExprs = groupByExprs) + internals.copyPivot(p, groupByExprs) case o => o } @@ -426,14 +375,14 @@ class SnappySessionState(val snappySession: SnappySession) // noinspection ScalaUnnecessaryParentheses // Y combinator val conditionEvaluator: (Expression => Boolean) => Expression => Boolean = - (f: Expression => Boolean) => - (exp: Expression) => exp.eq(PolicyProperties.rlsAppliedCondition) || - (exp match { - case And(left, _) => f(left) - case EqualTo(l: Literal, r: Literal) => - l.value == r.value && l.value == PolicyProperties.rlsConditionStringUtf8 - case _ => false - }) + (f: Expression => Boolean) => + (exp: Expression) => exp.eq(PolicyProperties.rlsAppliedCondition) || + (exp match { + case And(left, _) => f(left) + case EqualTo(l: Literal, r: Literal) => + l.value == r.value && l.value == PolicyProperties.rlsConditionStringUtf8 + case _ => false + }) // noinspection ScalaUnnecessaryParentheses def rlsConditionChecker(f: (Expression => Boolean) => @@ -451,16 +400,19 @@ class SnappySessionState(val snappySession: SnappySession) // is of type RunnableCommad. Later if it turns out any data operation // is happening via this command we need to handle it case _: RunnableCommand => plan - case _ if !alreadyPolicyApplied(plan) => plan.transformUp { - case lr@LogicalRelation(rlsRelation: RowLevelSecurityRelation, _, _) => - val policyFilter = catalog.getCombinedPolicyFilterForNativeTable(rlsRelation, Some(lr)) + case _ if !alreadyPolicyApplied(plan) => internals.logicalPlanResolveUp(plan) { + case lr: LogicalRelation if lr.relation.isInstanceOf[RowLevelSecurityRelation] => + val policyFilter = catalog.getCombinedPolicyFilterForNativeTable( + lr.relation.asInstanceOf[RowLevelSecurityRelation], Some(lr)) policyFilter match { case Some(filter) => filter.copy(child = lr) case None => lr } - case SubqueryAlias(name, LogicalFilter(condition, child), ti) => LogicalFilter(condition, - SubqueryAlias(name, child, ti)) + case a: SubqueryAlias if a.child.isInstanceOf[LogicalFilter] => + val lf = a.child.asInstanceOf[LogicalFilter] + LogicalFilter(lf.condition, internals.newSubqueryAlias(a.alias, lf.child, + internals.getViewFromAlias(a))) case LogicalFilter(condition1, LogicalFilter(condition2, child)) => if (rlsConditionChecker(conditionEvaluator)(condition1)) { @@ -510,9 +462,9 @@ class SnappySessionState(val snappySession: SnappySession) var externalRelation: ApplyLimitOnExternalRelation = null plan.foreachUp { { - case LogicalRelation(baseRelation: ApplyLimitOnExternalRelation, _, _) => + case lr: LogicalRelation if lr.relation.isInstanceOf[ApplyLimitOnExternalRelation] => boolsArray(extRelation_bool) = true - externalRelation = baseRelation + externalRelation = lr.relation.asInstanceOf[ApplyLimitOnExternalRelation] case _: MarkerForCreateTableAsSelect => boolsArray(create_tv_bool) = true case _: Aggregate => boolsArray(agg_func_bool) = true @@ -549,7 +501,8 @@ class SnappySessionState(val snappySession: SnappySession) plan: LogicalPlan): (Seq[NamedExpression], LogicalPlan, LogicalRelation) = { var tableName = "" val keyColumns = table.collectFirst { - case lr@LogicalRelation(mutable: MutableRelation, _, _) => + case lr: LogicalRelation if lr.relation.isInstanceOf[MutableRelation] => + val mutable = lr.relation.asInstanceOf[MutableRelation] val ks = mutable.getKeyColumns if (ks.isEmpty) { val currentKey = snappySession.currentKey @@ -568,9 +521,10 @@ class SnappySessionState(val snappySession: SnappySession) s"Update/Delete requires a MutableRelation but got $table")) // resolve key columns right away var mutablePlan: Option[LogicalRelation] = None - val newChild = child.transformDown { - case lr@LogicalRelation(mutable: MutableRelation, _, _) - if mutable.table.equalsIgnoreCase(tableName) => + val newChild = internals.logicalPlanResolveDown(child) { + case lr: LogicalRelation if lr.relation.isInstanceOf[MutableRelation] && + lr.relation.asInstanceOf[MutableRelation].table.equalsIgnoreCase(tableName) => + val mutable = lr.relation.asInstanceOf[MutableRelation] mutablePlan = Some(mutable.withKeyColumns(lr, keyColumns)) mutablePlan.get } @@ -590,7 +544,7 @@ class SnappySessionState(val snappySession: SnappySession) } } - def apply(plan: LogicalPlan): LogicalPlan = plan transform { + def apply(plan: LogicalPlan): LogicalPlan = internals.logicalPlanResolveDown(plan) { case u@Update(table, child, keyColumns, updateCols, updateExprs) if keyColumns.isEmpty && u.resolved && child.resolved => // add the key columns to the plan @@ -676,41 +630,17 @@ class SnappySessionState(val snappySession: SnappySession) /** * Internal catalog for managing table and database states. */ - override lazy val catalog: SnappySessionCatalog = { - new SnappySessionCatalog( - snappySession.sharedState.getExternalCatalogInstance(snappySession), - snappySession, - snappySession.sharedState.globalTempViewManager, - functionResourceLoader, - functionRegistry, - conf, - newHadoopConf()) - } + override lazy val catalog: SnappySessionCatalog = + catalogBuilder(None).asInstanceOf[SnappySessionCatalog] - protected lazy val wrapperCatalog: SessionCatalogWrapper = { - new SessionCatalogWrapper( - catalog.externalCatalog, - snappySession, - snappySession.sharedState.globalTempViewManager, - functionResourceLoader, - functionRegistry, - conf, - catalog.hadoopConf, - catalog) - } + lazy val wrapperCatalog: SnappySessionCatalog = + catalogBuilder(Some(catalog)).asInstanceOf[SnappySessionCatalog] - protected[sql] def queryPreparations( - topLevel: Boolean): Seq[Rule[SparkPlan]] = Seq[Rule[SparkPlan]]( - python.ExtractPythonUDFs, - TokenizeSubqueries(snappySession), - EnsureRequirements(conf), - OptimizeSortAndFilePlans(conf), - CollapseCollocatedPlans(snappySession), - CollapseCodegenStages(conf), - InsertCachedPlanFallback(snappySession, topLevel), - ReuseExchange(conf)) + private def queryPreparations(topLevel: Boolean): Seq[Rule[SparkPlan]] = + snappySession.contextFunctions.queryPreparations(topLevel) protected def newQueryExecution(plan: LogicalPlan): QueryExecution = { + initSnappyStrategies new QueryExecution(snappySession, plan) { override protected def preparations: Seq[Rule[SparkPlan]] = { @@ -722,7 +652,6 @@ class SnappySessionState(val snappySession: SnappySession) } override final def executePlan(plan: LogicalPlan): QueryExecution = { - initSnappyStrategies clearExecutionData() beforeExecutePlan(plan) val qe = newQueryExecution(plan) @@ -730,15 +659,13 @@ class SnappySessionState(val snappySession: SnappySession) qe } - private lazy val initSnappyStrategies: Unit = { + private[sql] lazy val initSnappyStrategies: Unit = { val storeOptimizedRules: Seq[Strategy] = Seq(StoreDataSourceStrategy, SnappyAggregation, HashJoinStrategies) experimentalMethods.extraStrategies = experimentalMethods.extraStrategies ++ - Seq(new HiveConditionalStrategy(_.HiveTableScans, this), - new HiveConditionalStrategy(_.DataSinks, this), - new HiveConditionalStrategy(_.Scripts, this), - SnappyStrategies, StoreStrategy, StreamQueryStrategy) ++ storeOptimizedRules + internals.hiveConditionalStrategies(this) ++ + Seq(SnappyStrategies, new StoreStrategy(this), StreamQueryStrategy) ++ storeOptimizedRules } protected def beforeExecutePlan(plan: LogicalPlan): Unit = { @@ -755,7 +682,7 @@ class SnappySessionState(val snappySession: SnappySession) } private[spark] def clearExecutionData(): Unit = { - conf.resetDefaults() + snappyConf.resetDefaults() leaderPartitions.clear() snappySession.clearContext() } @@ -770,7 +697,7 @@ class SnappySessionState(val snappySession: SnappySession) if (linkPartitionsToBuckets || preferPrimaries) { // also set the default shuffle partitions for this execution // to minimize exchange - conf.setExecutionShufflePartitions(region.getTotalNumberOfBuckets) + snappyConf.setExecutionShufflePartitions(region.getTotalNumberOfBuckets) } StoreUtils.getPartitionsPartitionedTable(snappySession, pr, linkPartitionsToBuckets, preferPrimaries) @@ -782,7 +709,7 @@ class SnappySessionState(val snappySession: SnappySession) StoreUtils.getPartitionsReplicatedTable(snappySession, region) } -class HiveConditionalRule(rule: HiveSessionState => Rule[LogicalPlan], state: SnappySessionState) +class HiveConditionalRule(rule: SessionState => Rule[LogicalPlan], state: SnappySessionState) extends Rule[LogicalPlan] { override def apply(plan: LogicalPlan): LogicalPlan = { // Parquet/Orc conversion rules will indirectly read the session state from the session @@ -797,7 +724,7 @@ class HiveConditionalStrategy(strategy: HiveStrategies => Strategy, state: Snapp extends Strategy { override def apply(plan: LogicalPlan): Seq[SparkPlan] = { // some strategies like DataSinks read the session state and expect it to be - // HiveSessionState so switch it before invoking the strategy and restore at the end + // hive-enabled SessionState so switch it before invoking the strategy and restore at the end if (state.snappySession.enableHiveSupport) state.withHiveSession { strategy(state.hiveState.planner.asInstanceOf[HiveStrategies])(plan) } else Nil @@ -805,74 +732,29 @@ class HiveConditionalStrategy(strategy: HiveStrategies => Strategy, state: Snapp } -class SnappyAnalyzer(sessionState: SnappySessionState) - extends Analyzer(sessionState.catalog, sessionState.conf) { - - // This list of rule is exact copy of org.apache.spark.sql.catalyst.analysis.Analyzer.batches - // It is replicated to inject StringPromotionCheckForUpdate rule. Since Analyzer.batches is - // declared as a lazy val, it can not be accessed using super keywork. - private[sql] lazy val ruleBatches = Seq( - Batch("Substitution", fixedPoint, - CTESubstitution, - WindowsSubstitution, - EliminateUnions, - new SubstituteUnresolvedOrdinals(sessionState.conf)), - Batch("Resolution", fixedPoint, - ResolveTableValuedFunctions :: - ResolveRelations :: - ResolveReferences :: - ResolveCreateNamedStruct :: - ResolveDeserializer :: - ResolveNewInstance :: - ResolveUpCast :: - ResolveGroupingAnalytics :: - ResolvePivot :: - ResolveOrdinalInOrderByAndGroupBy :: - ResolveMissingReferences :: - ExtractGenerator :: - ResolveGenerate :: - ResolveFunctions :: - ResolveAliases :: - ResolveSubquery :: - ResolveWindowOrder :: - ResolveWindowFrame :: - ResolveNaturalAndUsingJoin :: - ExtractWindowExpressions :: - GlobalAggregates :: - ResolveAggregateFunctions :: - TimeWindowing :: - ResolveInlineTables :: - TypeCoercion.typeCoercionRules ++ - extendedResolutionRules: _*), - Batch("Nondeterministic", Once, - PullOutNondeterministic), - Batch("UDF", Once, - HandleNullInputsForUDF), - Batch("FixNullability", Once, - FixNullability), - Batch("Cleanup", fixedPoint, - CleanupAliases) - ) - - override lazy val batches: Seq[Batch] = ruleBatches.map { - case batch if batch.name.equalsIgnoreCase("Resolution") => - val rules = batch.rules.flatMap { - case PromoteStrings => - StringPromotionCheckForUpdate.asInstanceOf[Rule[LogicalPlan]] :: SnappyPromoteStrings :: - PromoteStrings :: Nil - case r => r :: Nil - } +trait SnappyAnalyzer extends Analyzer with SparkSupport { + + def session: SnappySession + + val baseAnalyzerInstance: Analyzer - Batch(batch.name, batch.strategy, rules: _*) - case batch => Batch(batch.name, batch.strategy, batch.rules: _*) + override lazy val batches: Seq[Batch] = baseAnalyzerInstance.batches.map { + case batch if batch.name.equalsIgnoreCase("Resolution") => + val rules = internals.addStringPromotionRules(batch.rules, this, session.sessionState.conf) + Batch(batch.name, batch.strategy.asInstanceOf[Strategy], rules: _*) + case batch => Batch(batch.name, batch.strategy.asInstanceOf[Strategy], batch.rules: _*) } + def baseExecute(plan: LogicalPlan): LogicalPlan = super.execute(plan) + + override def execute(plan: LogicalPlan): LogicalPlan = + session.contextFunctions.executePlan(this, plan) // This Rule fails an update query when type of Arithmetic operators doesn't match. This // need to be done because by default spark performs fail safe implicit type // conversion when type of two operands does't match and this can lead to null values getting // populated in the table. - private object StringPromotionCheckForUpdate extends Rule[LogicalPlan] { + object StringPromotionCheckForUpdate extends Rule[LogicalPlan] { override def apply(plan: LogicalPlan): LogicalPlan = { plan match { @@ -897,9 +779,9 @@ class SnappyAnalyzer(sessionState: SnappySessionState) ParamLiteral (or vice-versa) as by default ParamLiteral datatype is NullType. In such a case, this rule converts ParmaLiteral type to StringType to prevent it being replaced by NULL */ - object SnappyPromoteStrings extends Rule[LogicalPlan] { + object SnappyPromoteStrings extends Rule[LogicalPlan] with SparkSupport { override def apply(plan: LogicalPlan): LogicalPlan = { - plan resolveExpressions { + internals.logicalPlanResolveExpressions(plan) { case e if !e.childrenResolved => e case p@BinaryComparison(left@StringType(), right@QuestionMark(_)) if right.dataType == NullType => @@ -913,6 +795,7 @@ class SnappyAnalyzer(sessionState: SnappySessionState) } } } + } /** @@ -923,7 +806,7 @@ case class OptimizeSortAndFilePlans(conf: SnappyConf) extends Rule[SparkPlan] { override def apply(plan: SparkPlan): SparkPlan = plan.transformUp { case join@joins.SortMergeJoinExec(_, _, _, _, _, sort@SortExec(_, _, child, _)) => join.copy(right = SnappySortExec(sort, child)) - case s@(_: FileSourceScanExec | _: HiveTableScanExec | _: InsertIntoHiveTable | + case s@(_: FileSourceScanExec | _: HiveTableScanExec | ExecutedCommandExec(_: InsertIntoHadoopFsRelationCommand | _: CreateHiveTableAsSelectCommand)) => conf.setDynamicCpusPerTask() diff --git a/core/src/main/scala/org/apache/spark/sql/internal/ColumnTableBulkOps.scala b/core/src/main/scala/org/apache/spark/sql/internal/ColumnTableBulkOps.scala index e5987238c2..a62b8787d9 100644 --- a/core/src/main/scala/org/apache/spark/sql/internal/ColumnTableBulkOps.scala +++ b/core/src/main/scala/org/apache/spark/sql/internal/ColumnTableBulkOps.scala @@ -21,21 +21,21 @@ import io.snappydata.Property import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation import org.apache.spark.sql.catalyst.encoders.RowEncoder import org.apache.spark.sql.catalyst.expressions.{And, Attribute, AttributeReference, AttributeSet, EqualTo, Expression} -import org.apache.spark.sql.catalyst.plans.logical.{BinaryNode, Join, LogicalPlan, OverwriteOptions, Project} +import org.apache.spark.sql.catalyst.plans.logical.{BinaryNode, Join, LogicalPlan, Project} import org.apache.spark.sql.catalyst.plans.{Inner, LeftAnti} import org.apache.spark.sql.collection.Utils import org.apache.spark.sql.execution.columnar.ExternalStoreUtils import org.apache.spark.sql.execution.datasources.LogicalRelation import org.apache.spark.sql.sources._ import org.apache.spark.sql.types.{DataType, LongType, StructType} -import org.apache.spark.sql.{AnalysisException, Dataset, Row, SnappySession, SparkSession} +import org.apache.spark.sql.{AnalysisException, Dataset, Row, SnappySession, SparkSession, SparkSupport} /** * Helper object for PutInto operations for column tables. * This class takes the logical plans from SnappyParser * and converts it into another plan. */ -object ColumnTableBulkOps { +object ColumnTableBulkOps extends SparkSupport { def transformPutPlan(session: SnappySession, originalPlan: PutIntoTable): LogicalPlan = { validateOp(originalPlan) @@ -44,7 +44,8 @@ object ColumnTableBulkOps { var transFormedPlan: LogicalPlan = originalPlan table.collectFirst { - case LogicalRelation(mutable: BulkPutRelation, _, _) => + case lr: LogicalRelation if lr.relation.isInstanceOf[BulkPutRelation] => + val mutable = lr.relation.asInstanceOf[BulkPutRelation] val putKeys = mutable.getPutKeys(session) match { case None => throw new AnalysisException( s"PutInto in a column table requires key column(s) but got empty string") @@ -52,10 +53,17 @@ object ColumnTableBulkOps { } val condition = prepareCondition(session, table, subQuery, putKeys) + val conf = session.sessionState.conf + val analyzer = session.sessionState.analyzer + val resolver = analyzer.resolver val keyColumns = getKeyColumns(table) + // JOIN may be optimized to a trivial form (e.g. PUT INTO ... VALUES(...)) + // where condition may be missing after optimization so enable cross join + conf.setConf(SQLConf.CROSS_JOINS_ENABLED, value = true) var updateSubQuery: LogicalPlan = Join(table, subQuery, Inner, condition) - val updateColumns = table.output.filterNot(a => keyColumns.contains(a.name)) - val updateExpressions = subQuery.output.filterNot(a => keyColumns.contains(a.name)) + val updateColumns = table.output.filterNot(a => keyColumns.exists(resolver(_, a.name))) + val updateExpressions = subQuery.output.filterNot( + a => keyColumns.exists(resolver(_, a.name))) if (updateExpressions.isEmpty) { throw new AnalysisException( s"PutInto is attempted without any column which can be updated." + @@ -63,37 +71,41 @@ object ColumnTableBulkOps { } val cacheSize = ExternalStoreUtils.sizeAsBytes( - Property.PutIntoInnerJoinCacheSize.get(session.sqlContext.conf), + Property.PutIntoInnerJoinCacheSize.get(conf), Property.PutIntoInnerJoinCacheSize.name, -1, Long.MaxValue) val updatePlan = Update(table, updateSubQuery, Nil, updateColumns, updateExpressions) - val updateDS = new Dataset(session, updatePlan, RowEncoder(updatePlan.schema)) - var analyzedUpdate = updateDS.queryExecution.analyzed.asInstanceOf[Update] + var analyzedUpdate = analyzer.execute(updatePlan).asInstanceOf[Update] updateSubQuery = analyzedUpdate.child // explicitly project out only the updated expression references and key columns // from the sub-query to minimize cache (if it is selected to be done) - val analyzer = session.sessionState.analyzer val updateReferences = AttributeSet(updateExpressions.flatMap(_.references)) updateSubQuery = Project(updateSubQuery.output.filter(a => - updateReferences.contains(a) || keyColumns.contains(a.name) || - putKeys.exists(k => analyzer.resolver(a.name, k))), updateSubQuery) + updateReferences.contains(a) || keyColumns.exists(resolver(_, a.name)) || + putKeys.exists(resolver(_, a.name))), updateSubQuery) - val insertChild = session.cachePutInto( - subQuery.statistics.sizeInBytes <= cacheSize, updateSubQuery, mutable.table) match { + val insertChild = session.cachePutInto(internals.getStatistics(subQuery) + .sizeInBytes <= cacheSize, updateSubQuery, mutable.table) match { case None => subQuery case Some(newUpdateSubQuery) => if (updateSubQuery ne newUpdateSubQuery) { - analyzedUpdate = analyzedUpdate.copy(child = newUpdateSubQuery) + updateSubQuery = newUpdateSubQuery + analyzedUpdate = analyzedUpdate.copy(child = updateSubQuery) } - Join(subQuery, newUpdateSubQuery, LeftAnti, condition) + // project out the columns already present in subQuery + val subQueryOutput = subQuery.output + if (subQueryOutput.intersect(updateSubQuery.output).nonEmpty) { + updateSubQuery = Project(updateSubQuery.output.filterNot( + subQueryOutput.contains), updateSubQuery) + } + Join(subQuery, updateSubQuery, LeftAnti, condition) } - val insertPlan = new Insert(table, Map.empty[String, + val insertPlan = internals.newInsertIntoTable(table, Map.empty[String, Option[String]], Project(subQuery.output, insertChild), - OverwriteOptions(enabled = false), ifNotExists = false) - - transFormedPlan = PutIntoColumnTable(table, insertPlan, analyzedUpdate) + overwrite = false, ifNotExists = false) + transFormedPlan = PutIntoColumnTable(table, analyzer.execute(insertPlan), analyzedUpdate) case _ => // Do nothing, original putInto plan is enough } transFormedPlan @@ -101,11 +113,11 @@ object ColumnTableBulkOps { def validateOp(originalPlan: PutIntoTable) { originalPlan match { - case PutIntoTable(LogicalRelation(t: BulkPutRelation, _, _), query) => + case PutIntoTable(lr: LogicalRelation, query) if lr.relation.isInstanceOf[BulkPutRelation] => val srcRelations = query.collect { - case LogicalRelation(src: BaseRelation, _, _) => src + case r: LogicalRelation => r.relation } - if (srcRelations.contains(t)) { + if (srcRelations.contains(lr.relation)) { throw Utils.analysisException( "Cannot put into table that is also being read from.") } else { @@ -145,7 +157,8 @@ object ColumnTableBulkOps { def getKeyColumns(table: LogicalPlan): Set[String] = { table.collectFirst { - case LogicalRelation(mutable: MutableRelation, _, _) => mutable.getKeyColumns.toSet + case lr: LogicalRelation if lr.relation.isInstanceOf[MutableRelation] => + lr.relation.asInstanceOf[MutableRelation].getKeyColumns.toSet } match { case None => throw new AnalysisException( s"Update/Delete requires a MutableRelation but got $table") @@ -160,8 +173,8 @@ object ColumnTableBulkOps { var transFormedPlan: LogicalPlan = originalPlan table.collectFirst { - case LogicalRelation(mutable: MutableRelation, _, _) => - val ks = mutable.getPrimaryKeyColumns(session) + case lr: LogicalRelation if lr.relation.isInstanceOf[MutableRelation] => + val ks = lr.relation.asInstanceOf[MutableRelation].getPrimaryKeyColumns(session) if (ks.isEmpty) { throw new AnalysisException( s"DeleteFrom operation requires key columns(s) or primary key defined on table.") @@ -180,18 +193,18 @@ object ColumnTableBulkOps { val session = sparkSession.asInstanceOf[SnappySession] val tableIdent = session.tableIdentifier(resolvedName) val encoder = RowEncoder(schema) - val ds = session.internalCreateDataFrame(session.sparkContext.parallelize( + val ds = internals.internalCreateDataFrame(session, session.sparkContext.parallelize( rows.map(encoder.toRow)), schema) val plan = if (putInto) { PutIntoTable( table = UnresolvedRelation(tableIdent), child = ds.logicalPlan) } else { - new Insert( + internals.newInsertIntoTable( table = UnresolvedRelation(tableIdent), partition = Map.empty[String, Option[String]], child = ds.logicalPlan, - overwrite = OverwriteOptions(enabled = false), + overwrite = false, ifNotExists = false) } session.sessionState.executePlan(plan).executedPlan.executeCollect() @@ -201,7 +214,7 @@ object ColumnTableBulkOps { } case class PutIntoColumnTable(table: LogicalPlan, - insert: Insert, update: Update) extends BinaryNode { + insert: LogicalPlan, update: LogicalPlan) extends BinaryNode { override lazy val output: Seq[Attribute] = AttributeReference( "count", LongType)() :: Nil diff --git a/core/src/main/scala/org/apache/spark/sql/internal/JarUtils.scala b/core/src/main/scala/org/apache/spark/sql/internal/ContextJarUtils.scala similarity index 96% rename from core/src/main/scala/org/apache/spark/sql/internal/JarUtils.scala rename to core/src/main/scala/org/apache/spark/sql/internal/ContextJarUtils.scala index 0d24ef8c76..99ce81c402 100644 --- a/core/src/main/scala/org/apache/spark/sql/internal/JarUtils.scala +++ b/core/src/main/scala/org/apache/spark/sql/internal/ContextJarUtils.scala @@ -49,7 +49,7 @@ object ContextJarUtils extends Logging { val JAR_PATH = "snappy-jars" private val driverJars = new ConcurrentHashMap[String, URLClassLoader]().asScala val functionKeyPrefix = "__FUNC__" - val droppedFunctionsKey = functionKeyPrefix + "DROPPED__" + val droppedFunctionsKey: String = functionKeyPrefix + "DROPPED__" val DELIMITER = "," def addDriverJar(key: String, classLoader: URLClassLoader): Option[URLClassLoader] = { @@ -60,7 +60,7 @@ object ContextJarUtils extends Logging { def removeDriverJar(key: String) : Unit = driverJars.remove(key) - def getDriverJarURLs(): Array[URL] = { + def getDriverJarURLs: Array[URL] = { var urls = new mutable.HashSet[URL]() driverJars.foreach(_._2.getURLs.foreach(urls += _)) urls.toArray @@ -77,7 +77,7 @@ object ContextJarUtils extends Logging { def fetchFile(prefix: String, path: String): URL = { val callbacks = ToolsCallbackInit.toolsCallback val localName = path.split("/").last - val changedFileName = s"${prefix}-${localName}" + val changedFileName = s"$prefix-$localName" logInfo(s"Fetching jar $path to driver local directory $jarDir") val changedFile = new File(jarDir, changedFileName) if (!changedFile.exists()) { @@ -90,7 +90,7 @@ object ContextJarUtils extends Logging { val callbacks = ToolsCallbackInit.toolsCallback if (callbacks != null) { val localName = path.split("/").last - val changedFileName = s"${prefix}-${localName}" + val changedFileName = s"$prefix-$localName" val jarFile = new File(jarDir, changedFileName) try { @@ -126,7 +126,7 @@ object ContextJarUtils extends Logging { case e: AnalysisException => if (!ignoreIfNotExists) { sessionCatalog match { - case Some(ssc) => ssc.failFunctionLookup(functionName) + case Some(ssc) => ssc.functionNotFound(functionName) case None => throw new NoSuchFunctionException(schemaName, identifier.funcName) } } else { // Log, just in case. @@ -181,5 +181,3 @@ object ContextJarUtils extends Logging { value != null && value.contains(item) } } - - diff --git a/core/src/main/scala/org/apache/spark/sql/internal/SnappySessionCatalog.scala b/core/src/main/scala/org/apache/spark/sql/internal/SnappySessionCatalog.scala index 38b4d8ac7d..db336109a3 100644 --- a/core/src/main/scala/org/apache/spark/sql/internal/SnappySessionCatalog.scala +++ b/core/src/main/scala/org/apache/spark/sql/internal/SnappySessionCatalog.scala @@ -29,10 +29,9 @@ import io.snappydata.Constant import io.snappydata.sql.catalog.CatalogObjectType.getTableType import io.snappydata.sql.catalog.SnappyExternalCatalog.{DBTABLE_PROPERTY, getTableWithSchema} import io.snappydata.sql.catalog.{CatalogObjectType, SnappyExternalCatalog} -import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path -import org.apache.spark.annotation.DeveloperApi +import org.apache.spark.rdd.RDD import org.apache.spark.sql._ import org.apache.spark.sql.catalog.Column import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder @@ -43,34 +42,34 @@ import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, SubqueryAlias} import org.apache.spark.sql.catalyst.{FunctionIdentifier, IdentifierWithDatabase, TableIdentifier} import org.apache.spark.sql.collection.{ToolsCallbackInit, Utils} +import org.apache.spark.sql.execution.TopK import org.apache.spark.sql.execution.command.DDLUtils import org.apache.spark.sql.execution.datasources.{DataSource, FindDataSourceTable, LogicalRelation} -import org.apache.spark.sql.hive.HiveSessionCatalog +import org.apache.spark.sql.hive.{HiveSessionCatalog, SnappyHiveExternalCatalog} import org.apache.spark.sql.policy.PolicyProperties import org.apache.spark.sql.sources.{DestroyRelation, JdbcExtendedUtils, MutableRelation, RowLevelSecurityRelation} import org.apache.spark.sql.types._ import org.apache.spark.util.MutableURLClassLoader /** - * ::DeveloperApi:: - * Catalog using Hive for persistence and adding Snappy extensions like + * SessionCatalog implementation using Snappy store for persistence in embedded mode and + * using client API calls for smart connector mode, Adds Snappy extensions like * stream/topK tables and returning LogicalPlan to materialize these entities. */ -@DeveloperApi -class SnappySessionCatalog(val externalCatalog: SnappyExternalCatalog, - val snappySession: SnappySession, - globalTempViewManager: GlobalTempViewManager, - functionResourceLoader: FunctionResourceLoader, - functionRegistry: FunctionRegistry, - sqlConf: SQLConf, - val hadoopConf: Configuration) - extends SessionCatalog( - externalCatalog, - globalTempViewManager, - functionResourceLoader, - functionRegistry, - sqlConf, - hadoopConf) { +trait SnappySessionCatalog extends SessionCatalog with SparkSupport { + + def snappyExternalCatalog: SnappyExternalCatalog + def globalTempManager: GlobalTempViewManager + val functionResourceLoader: FunctionResourceLoader + val functionRegistry: FunctionRegistry + val snappySession: SnappySession + val sqlConf: SQLConf + val parser: SnappySqlParser + val wrappedCatalog: Option[SnappySessionCatalog] + + def functionNotFound(name: String): Unit + + final def contextFunctions: SnappyContextFunctions = snappySession.contextFunctions /** * Can be used to temporarily switch the metadata returned by catalog @@ -112,8 +111,8 @@ class SnappySessionCatalog(val externalCatalog: SnappyExternalCatalog, * Fallback session state to lookup from external hive catalog in case * "snappydata.sql.hive.enabled" is set on the session. */ - protected final lazy val hiveSessionCatalog: HiveSessionCatalog = - snappySession.sessionState.hiveState.catalog + protected[sql] final lazy val hiveSessionCatalog: HiveSessionCatalog = + snappySession.snappySessionState.hiveState.catalog.asInstanceOf[HiveSessionCatalog] /** * Return true if the given table needs to be checked in the builtin catalog @@ -133,8 +132,8 @@ class SnappySessionCatalog(val externalCatalog: SnappyExternalCatalog, val tableIdent = snappySession.tableIdentifier(table) val relation = resolveRelation(tableIdent) val keyColumns = relation match { - case LogicalRelation(mutable: MutableRelation, _, _) => - val keyCols = mutable.getPrimaryKeyColumns(snappySession) + case lr: LogicalRelation if lr.relation.isInstanceOf[MutableRelation] => + val keyCols = lr.relation.asInstanceOf[MutableRelation].getPrimaryKeyColumns(snappySession) if (keyCols.isEmpty) { Nil } else { @@ -189,7 +188,7 @@ class SnappySessionCatalog(val externalCatalog: SnappyExternalCatalog, } else { val catalogTable = getTableMetadata(new TableIdentifier( rlsRelation.tableName, Some(rlsRelation.schemaName))) - val policyFilters = externalCatalog.getPolicies(rlsRelation.schemaName, + val policyFilters = snappyExternalCatalog.getPolicies(rlsRelation.schemaName, rlsRelation.tableName, catalogTable.properties).map { ct => resolveRelation(ct.identifier).asInstanceOf[BypassRowLevelSecurity].child } @@ -225,11 +224,10 @@ class SnappySessionCatalog(val externalCatalog: SnappyExternalCatalog, // remap filter val mappingInfo = storedLR.output.map(_.exprId).zip( queryLR.get.output.map(_.exprId)).toMap - filter.transformAllExpressions { + internals.logicalPlanResolveExpressions(filter) { case ar: AttributeReference if mappingInfo.contains(ar.exprId) => - AttributeReference(ar.name, ar.dataType, ar.nullable, - ar.metadata)(mappingInfo(ar.exprId), ar.qualifier, ar.isGenerated) - } + internals.toAttributeReference(ar)(exprId = mappingInfo(ar.exprId)) + }.asInstanceOf[Filter] } } @@ -267,7 +265,7 @@ class SnappySessionCatalog(val externalCatalog: SnappyExternalCatalog, final def resolveRelationWithAlias(tableIdent: TableIdentifier, alias: Option[String] = None): LogicalPlan = { // resolve the relation right away with alias around - new FindDataSourceTable(snappySession)(lookupRelation(tableIdent, alias)) + new FindDataSourceTable(snappySession)(lookupRelationImpl(tableIdent, alias, wrapped = None)) } /** @@ -292,8 +290,8 @@ class SnappySessionCatalog(val externalCatalog: SnappyExternalCatalog, protected def addMissingGlobalTempSchema(name: TableIdentifier): TableIdentifier = { if (name.database.isEmpty) { val tableName = formatTableName(name.table) - if (globalTempViewManager.get(tableName).isDefined) { - name.copy(table = tableName, database = Some(globalTempViewManager.database)) + if (globalTempManager.get(tableName).isDefined) { + name.copy(table = tableName, database = Some(globalTempManager.database)) } else name } else name } @@ -305,7 +303,7 @@ class SnappySessionCatalog(val externalCatalog: SnappyExternalCatalog, } protected[sql] def validateSchemaName(schemaName: String, checkForDefault: Boolean): Unit = { - if (schemaName == globalTempViewManager.database) { + if (schemaName == globalTempManager.database) { throw new AnalysisException(s"$schemaName is a system preserved database/schema for global " + s"temporary tables. You cannot create, drop or set a schema with this name.") } @@ -314,8 +312,8 @@ class SnappySessionCatalog(val externalCatalog: SnappyExternalCatalog, } } - def isLocalTemporaryView(name: TableIdentifier): Boolean = synchronized { - name.database.isEmpty && tempTables.contains(formatTableName(name.table)) + def isLocalTemporaryView(name: TableIdentifier): Boolean = { + name.database.isEmpty && getTempView(name.table).isDefined } private def schemaDescription(schemaName: String): String = s"User $schemaName schema" @@ -329,7 +327,7 @@ class SnappySessionCatalog(val externalCatalog: SnappyExternalCatalog, validateSchemaName(schemaName, checkForDefault = false) // create schema in catalog first - if (externalCatalog.databaseExists(schemaName)) { + if (snappyExternalCatalog.databaseExists(schemaName)) { if (!ignoreIfExists) throw new AnalysisException(s"Schema '$schemaName' already exists") } else { super.createDatabase(CatalogDatabase(schemaName, schemaDescription(schemaName), @@ -409,7 +407,7 @@ class SnappySessionCatalog(val externalCatalog: SnappyExternalCatalog, throw new AnalysisException(s"$schemaName is a system preserved database/schema") } - if (!externalCatalog.databaseExists(schemaName)) { + if (!snappyExternalCatalog.databaseExists(schemaName)) { if (ignoreIfNotExists) return else throw SnappyExternalCatalog.schemaNotFoundException(schemaName) } @@ -417,8 +415,8 @@ class SnappySessionCatalog(val externalCatalog: SnappyExternalCatalog, if (cascade) { // drop all the tables in order first, dependents followed by others - val allTables = externalCatalog.listTables(schemaName).flatMap( - table => externalCatalog.getTableOption(schemaName, formatTableName(table))) + val allTables = snappyExternalCatalog.listTables(schemaName).flatMap( + table => snappyExternalCatalog.getTableIfExists(schemaName, formatTableName(table))) // keep dropping leaves until empty if (allTables.nonEmpty) { // drop streams at the end @@ -426,7 +424,7 @@ class SnappySessionCatalog(val externalCatalog: SnappyExternalCatalog, var tables = others while (tables.nonEmpty) { val (leaves, remaining) = tables.partition(t => t.tableType == CatalogTableType.VIEW || - externalCatalog.getDependents(t.database, t.identifier.table, t, + snappyExternalCatalog.getDependents(t.database, t.identifier.table, t, Nil, CatalogObjectType.Policy :: Nil).isEmpty) leaves.foreach(t => snappySession.dropTable(t.identifier, ifExists = true, t.tableType == CatalogTableType.VIEW)) @@ -507,7 +505,7 @@ class SnappySessionCatalog(val externalCatalog: SnappyExternalCatalog, if (force || schemaName != getCurrentSchema) { validateSchemaName(schemaName, checkForDefault = false) super.setCurrentDatabase(schemaName) - externalCatalog.setCurrentDatabase(schemaName) + snappyExternalCatalog.setCurrentDatabase(schemaName) // no need to set the current schema in external hive metastore since the // database may not exist and all calls to it will already ensure fully qualified // table names @@ -516,6 +514,7 @@ class SnappySessionCatalog(val externalCatalog: SnappyExternalCatalog, override def getDatabaseMetadata(schema: String): CatalogDatabase = { val schemaName = formatDatabaseName(schema) + val externalCatalog = snappyExternalCatalog if (externalCatalog.databaseExists(schemaName)) externalCatalog.getDatabase(schemaName) else if (snappySession.enableHiveSupport && hiveSessionCatalog.databaseExists(schema)) { hiveSessionCatalog.getDatabaseMetadata(schema) @@ -550,7 +549,11 @@ class SnappySessionCatalog(val externalCatalog: SnappyExternalCatalog, } else super.listDatabases(pattern).distinct.sorted } - override def createTable(table: CatalogTable, ignoreIfExists: Boolean): Unit = { + protected def baseCreateTable(table: CatalogTable, ignoreIfExists: Boolean, + validateTableLocation: Boolean): Unit + + protected final def createTableImpl(table: CatalogTable, ignoreIfExists: Boolean, + validateTableLocation: Boolean): Unit = { // first check required permission to create objects in a schema val schemaName = getSchemaName(table.identifier) val tableName = formatTableName(table.identifier.table) @@ -558,7 +561,7 @@ class SnappySessionCatalog(val externalCatalog: SnappyExternalCatalog, // hive tables will be created in external hive catalog if enabled else will fail table.provider match { - case Some(DDLUtils.HIVE_PROVIDER) => + case Some(p) if p.equalsIgnoreCase(DDLUtils.HIVE_PROVIDER) => if (snappySession.enableHiveSupport) { // check for existing table else for hive table it could create in both catalogs @@ -572,7 +575,8 @@ class SnappySessionCatalog(val externalCatalog: SnappyExternalCatalog, } // resolve table fully as per current schema in this session - hiveSessionCatalog.createTable(resolveCatalogTable(table, schemaName), ignoreIfExists) + internals.createTable(hiveSessionCatalog, resolveCatalogTable(table, schemaName), + ignoreIfExists, validateTableLocation) } else { throw Utils.analysisException( s"External hive support (${StaticSQLConf.CATALOG_IMPLEMENTATION.key} = hive) " + @@ -581,8 +585,19 @@ class SnappySessionCatalog(val externalCatalog: SnappyExternalCatalog, } case _ => createSchema(schemaName, ignoreIfExists = true) - super.createTable(table, ignoreIfExists) + // hack to always pass ignoreIfExists as true so that + // for the case of CTAS for builtin tables which is handled + // in SnappyHiveExternalCatalog but premature exception gets + // thrown in newer SessionCatalog.createTable + SnappyHiveExternalCatalog.ignoreIfExists.set(ignoreIfExists) + try { + baseCreateTable(table, ignoreIfExists = true, validateTableLocation) + } finally { + SnappyHiveExternalCatalog.ignoreIfExists.remove() + } } + + contextFunctions.postCreateTable(table) } /** @@ -602,10 +617,10 @@ class SnappySessionCatalog(val externalCatalog: SnappyExternalCatalog, val catalogTable = CatalogTable(new TableIdentifier(tableName, Some(schemaName)), CatalogTableType.EXTERNAL, DataSource.buildStorageFormatFromOptions( options + (DBTABLE_PROPERTY -> fullTableName)), schema, Some(provider)) - createTable(catalogTable, ignoreIfExists) + createTableImpl(catalogTable, ignoreIfExists, validateTableLocation = false) } - private def convertCharTypes(table: CatalogTable): CatalogTable = { + protected def convertCharTypes(table: CatalogTable): CatalogTable = { if (convertCharTypesInMetadata) table.copy(schema = StructType(table.schema.map { field => field.dataType match { case StringType if field.metadata.contains(Constant.CHAR_TYPE_BASE_PROP) => @@ -631,25 +646,26 @@ class SnappySessionCatalog(val externalCatalog: SnappyExternalCatalog, } else false } - override def getTableMetadata(name: TableIdentifier): CatalogTable = { - super.getTableMetadataOption(name) match { - case None => + def getTableMetadataIfExists(name: TableIdentifier): Option[CatalogTable] = { + try { + Some(convertCharTypes(super.getTableMetadata(name))) + } catch { + case _: Exception => val schemaName = getSchemaName(name) if (snappySession.enableHiveSupport && hiveSessionCatalog.databaseExists(schemaName)) { - hiveSessionCatalog.getTableMetadata(qualifiedTableIdentifier(name, schemaName)) - } else throw new TableNotFoundException(schemaName, name.table) - case Some(table) => convertCharTypes(table) + try { + Some(hiveSessionCatalog.getTableMetadata(qualifiedTableIdentifier(name, schemaName))) + } catch { + case _: Exception => None + } + } else None } } - override def getTableMetadataOption(name: TableIdentifier): Option[CatalogTable] = { - super.getTableMetadataOption(name) match { - case None => - val schemaName = getSchemaName(name) - if (snappySession.enableHiveSupport && hiveSessionCatalog.databaseExists(schemaName)) { - hiveSessionCatalog.getTableMetadataOption(qualifiedTableIdentifier(name, schemaName)) - } else None - case Some(table) => Some(convertCharTypes(table)) + override def getTableMetadata(name: TableIdentifier): CatalogTable = { + getTableMetadataIfExists(name) match { + case Some(table) => table + case None => throw new TableNotFoundException(getSchemaName(name), name.table) } } @@ -664,7 +680,7 @@ class SnappySessionCatalog(val externalCatalog: SnappyExternalCatalog, val table = formatTableName(name.table) checkSchemaPermission(schema, table, defaultUser = null) // resolve the table and destroy underlying storage if possible - externalCatalog.getTableOption(schema, table) match { + snappyExternalCatalog.getTableIfExists(schema, table) match { case None => // check in external hive catalog if (snappySession.enableHiveSupport && hiveSessionCatalog.databaseExists(schema)) { @@ -675,8 +691,8 @@ class SnappySessionCatalog(val externalCatalog: SnappyExternalCatalog, if (ignoreIfNotExists) return else throw new TableNotFoundException(schema, table) case Some(metadata) => // fail if there are any existing dependents except policies - val dependents = externalCatalog.getDependents(schema, table, - externalCatalog.getTable(schema, table), Nil, CatalogObjectType.Policy :: Nil) + val dependents = snappyExternalCatalog.getDependents(schema, table, + snappyExternalCatalog.getTable(schema, table), Nil, CatalogObjectType.Policy :: Nil) if (dependents.nonEmpty) { throw new AnalysisException(s"Object $schema.$table cannot be dropped because of " + s"dependent objects: ${dependents.map(_.identifier.unquotedString).mkString(",")}") @@ -684,18 +700,17 @@ class SnappySessionCatalog(val externalCatalog: SnappyExternalCatalog, // remove from temporary base table if applicable dropFromTemporaryBaseTable(metadata) metadata.provider match { - case Some(provider) if provider != DDLUtils.HIVE_PROVIDER => - val relation = try { + case Some(provider) if !provider.equalsIgnoreCase(DDLUtils.HIVE_PROVIDER) => + try { DataSource(snappySession, provider, userSpecifiedSchema = Some(metadata.schema), partitionColumns = metadata.partitionColumnNames, bucketSpec = metadata.bucketSpec, - options = metadata.storage.properties).resolveRelation() + options = metadata.storage.properties).resolveRelation() match { + case d: DestroyRelation if d ne null => d.destroy(ignoreIfNotExists) + case _ => + } } catch { - case NonFatal(_) => null // ignore any exception in class lookup - } - relation match { - case d: DestroyRelation => d.destroy(ignoreIfNotExists) - case _ => + case NonFatal(_) => // ignore any exception in class lookup } case _ => } @@ -704,9 +719,34 @@ class SnappySessionCatalog(val externalCatalog: SnappyExternalCatalog, super.dropTable(name, ignoreIfNotExists, purge) } - protected def dropTemporaryTable(tableIdent: TableIdentifier): Unit = {} + def addSampleDataFrame(base: LogicalPlan, sample: LogicalPlan, name: String = ""): Unit = + contextFunctions.addSampleDataFrame(base, sample, name) - protected def dropFromTemporaryBaseTable(table: CatalogTable): Unit = {} + /** + * Return the set of temporary samples for a given table that are not tracked in catalog. + */ + def getSamples(base: LogicalPlan): Seq[LogicalPlan] = contextFunctions.getSamples(base) + + /** + * Return the set of samples for a given table that are tracked in catalog and are not temporary. + */ + def getSampleRelations(baseTable: TableIdentifier): Seq[(LogicalPlan, String)] = + contextFunctions.getSampleRelations(baseTable) + + protected def dropTemporaryTable(tableIdent: TableIdentifier): Unit = + contextFunctions.dropTemporaryTable(tableIdent) + + protected def dropFromTemporaryBaseTable(table: CatalogTable): Unit = + contextFunctions.dropFromTemporaryBaseTable(table) + + def lookupTopK(topKName: String): Option[(AnyRef, RDD[(Int, TopK)])] = + contextFunctions.lookupTopK(topKName) + + def registerTopK(topK: AnyRef, rdd: RDD[(Int, TopK)], + ifExists: Boolean, overwrite: Boolean): Boolean = + contextFunctions.registerTopK(topK, rdd, ifExists, overwrite) + + def unregisterTopK(topKName: String): Unit = contextFunctions.unregisterTopK(topKName) override def alterTable(table: CatalogTable): Unit = { // first check required permission to alter objects in a schema @@ -730,8 +770,8 @@ class SnappySessionCatalog(val externalCatalog: SnappyExternalCatalog, override def renameTable(old: TableIdentifier, newName: TableIdentifier): Unit = { val oldName = addMissingGlobalTempSchema(old) if (isTemporaryTable(oldName)) { - if (newName.database.isEmpty && oldName.database.contains(globalTempViewManager.database)) { - super.renameTable(oldName, newName.copy(database = Some(globalTempViewManager.database))) + if (newName.database.isEmpty && oldName.database.contains(globalTempManager.database)) { + super.renameTable(oldName, newName.copy(database = Some(globalTempManager.database))) } else super.renameTable(oldName, newName) } else { // first check required permission to alter objects in a schema @@ -743,7 +783,7 @@ class SnappySessionCatalog(val externalCatalog: SnappyExternalCatalog, } if (checkBuiltinCatalog(oldName)) { - getTableMetadataOption(oldName).flatMap(_.provider) match { + getTableMetadataIfExists(oldName).flatMap(_.provider) match { // in-built tables don't support rename yet case Some(p) if SnappyContext.isBuiltInProvider(p) => throw new UnsupportedOperationException( @@ -794,12 +834,11 @@ class SnappySessionCatalog(val externalCatalog: SnappyExternalCatalog, } createSchema(schemaName, ignoreIfExists = true) - externalCatalog.createPolicy(schemaName, policyName, targetIdent.unquotedString, + snappyExternalCatalog.createPolicy(schemaName, policyName, targetIdent.unquotedString, policyFor, policyApplyTo, expandedPolicyApplyTo, owner, filterString) } private def getPolicyPlan(table: CatalogTable): LogicalPlan = { - val parser = snappySession.sessionState.sqlParser val filterExpression = table.properties.get(PolicyProperties.filterString) match { case Some(e) => parser.parseExpression(e) case None => throw new IllegalStateException("Filter for the policy not found") @@ -808,7 +847,7 @@ class SnappySessionCatalog(val externalCatalog: SnappyExternalCatalog, case Some(t) => snappySession.tableIdentifier(t) case None => throw new IllegalStateException("Target Table for the policy not found") } - /* val targetRelation = snappySession.sessionState.catalog.lookupRelation(tableIdent) + /* val targetRelation = lookupRelationImpl(tableIdent, None) val isTargetExternalRelation = targetRelation.find(x => x match { case _: ExternalRelation => true case _ => false @@ -820,33 +859,38 @@ class SnappySessionCatalog(val externalCatalog: SnappyExternalCatalog, toSeq.filterNot(_.isEmpty)) } - override def lookupRelation(name: TableIdentifier, alias: Option[String]): LogicalPlan = { - synchronized { + def newView(table: CatalogTable, child: LogicalPlan): LogicalPlan + + def newCatalogRelation(schemaName: String, table: CatalogTable): LogicalPlan + + protected final def lookupRelationImpl(name: TableIdentifier, alias: Option[String], + wrapped: Option[SnappySessionCatalog] = wrappedCatalog): LogicalPlan = wrapped match { + case None => synchronized { val tableName = formatTableName(name.table) var view: Option[TableIdentifier] = Some(name) val relationPlan = (if (name.database.isEmpty) { - tempTables.get(tableName) match { - case None => globalTempViewManager.get(tableName) + getTempView(tableName) match { + case None => globalTempManager.get(tableName) case s => s } } else None) match { case None => val schemaName = if (name.database.isEmpty) currentDb else formatDatabaseName(name.database.get) - if (schemaName == globalTempViewManager.database) { - globalTempViewManager.get(tableName) match { + if (schemaName == globalTempManager.database) { + globalTempManager.get(tableName) match { case None => throw new TableNotFoundException(schemaName, tableName) case Some(p) => p } } else { - val table = externalCatalog.getTableOption(schemaName, tableName) match { + val table = snappyExternalCatalog.getTableIfExists(schemaName, tableName) match { case None => if (snappySession.enableHiveSupport) { // lookupRelation uses HiveMetastoreCatalog that looks up the session state and // catalog from the session every time so use withHiveState to switch the catalog - val state = snappySession.sessionState + val state = snappySession.snappySessionState if (hiveSessionCatalog.databaseExists(schemaName)) state.withHiveSession { - return hiveSessionCatalog.lookupRelation( + return internals.lookupRelation(hiveSessionCatalog, TableIdentifier(tableName, Some(schemaName)), alias) } } @@ -855,33 +899,35 @@ class SnappySessionCatalog(val externalCatalog: SnappyExternalCatalog, } if (table.tableType == CatalogTableType.VIEW) { if (table.viewText.isEmpty) sys.error("Invalid view without text.") - new SnappySqlParser(snappySession).parsePlan(table.viewText.get) + newView(table, new SnappySqlParser(snappySession).parsePlan(table.viewText.get)) } else if (CatalogObjectType.isPolicy(table)) { getPolicyPlan(table) } else { view = None - SimpleCatalogRelation(schemaName, table) + newCatalogRelation(schemaName, table) } } case Some(p) => p } - SubqueryAlias(if (alias.isEmpty) tableName else alias.get, relationPlan, view) + internals.newSubqueryAlias(if (alias.isEmpty) tableName else alias.get, relationPlan, view) } + + case Some(c) => c.resolveRelationWithAlias(name, alias) } override def isTemporaryTable(name: TableIdentifier): Boolean = { if (name.database.isEmpty) synchronized { // check both local and global temporary tables val tableName = formatTableName(name.table) - tempTables.contains(tableName) || globalTempViewManager.get(tableName).isDefined - } else if (formatDatabaseName(name.database.get) == globalTempViewManager.database) { - globalTempViewManager.get(formatTableName(name.table)).isDefined + getTempView(tableName).isDefined || globalTempManager.get(tableName).isDefined + } else if (formatDatabaseName(name.database.get) == globalTempManager.database) { + globalTempManager.get(formatTableName(name.table)).isDefined } else false } override def listTables(schema: String, pattern: String): Seq[TableIdentifier] = { val schemaName = formatDatabaseName(schema) - if (schemaName != globalTempViewManager.database && !databaseExists(schemaName)) { + if (schemaName != globalTempManager.database && !databaseExists(schemaName)) { throw SnappyExternalCatalog.schemaNotFoundException(schema) } if (snappySession.enableHiveSupport && hiveSessionCatalog.databaseExists(schema)) { @@ -892,11 +938,10 @@ class SnappySessionCatalog(val externalCatalog: SnappyExternalCatalog, override def refreshTable(name: TableIdentifier): Unit = { val table = addMissingGlobalTempSchema(name) - if (isTemporaryTable(table)) { - super.refreshTable(table) - } else { + super.refreshTable(table) + if (!isTemporaryTable(table)) { val resolved = resolveTableIdentifier(table) - externalCatalog.invalidate(resolved.database.get -> resolved.table) + snappyExternalCatalog.invalidate(resolved.database.get -> resolved.table) if (snappySession.enableHiveSupport) { hiveSessionCatalog.refreshTable(resolved) } @@ -904,7 +949,7 @@ class SnappySessionCatalog(val externalCatalog: SnappyExternalCatalog, } def getDataSourceRelations[T](tableType: CatalogObjectType.Type): Seq[T] = { - externalCatalog.getAllTables().collect { + snappyExternalCatalog.getAllTables().collect { case table if tableType == CatalogObjectType.getTableType(table) => resolveRelation(table.identifier).asInstanceOf[LogicalRelation].relation.asInstanceOf[T] } @@ -1040,7 +1085,7 @@ class SnappySessionCatalog(val externalCatalog: SnappyExternalCatalog, super.listPartitionsByFilter(tableName, predicates) } - // TODO: SW: clean up function creation to be like Spark with backward compatibility + // TODO: SW: clean up function resource loading to be like Spark with backward compatibility override def loadFunctionResources(resources: Seq[FunctionResource]): Unit = { val qualifiedName = SnappyExternalCatalog.currentFunctionIdentifier.get() @@ -1049,15 +1094,11 @@ class SnappySessionCatalog(val externalCatalog: SnappyExternalCatalog, val callbacks = ToolsCallbackInit.toolsCallback val newClassLoader = ContextJarUtils.getDriverJar(functionQualifiedName) match { case None => - val urls = if (callbacks != null) { + val urls = if (callbacks ne null) { resources.map { r => ContextJarUtils.fetchFile(functionQualifiedName, r.uri) } - } else { - resources.map { r => - toUrl(r) - } - } + } else resources.map(toUrl) val newClassLoader = new MutableURLClassLoader(urls.toArray, parentLoader) ContextJarUtils.addDriverJar(functionQualifiedName, newClassLoader) newClassLoader @@ -1081,15 +1122,11 @@ class SnappySessionCatalog(val externalCatalog: SnappyExternalCatalog, checkSchemaPermission(schemaName, name.funcName, defaultUser = null) val qualifiedName = name.copy(database = Some(schemaName)) - ContextJarUtils.removeFunctionArtifacts(externalCatalog, Option(this), + ContextJarUtils.removeFunctionArtifacts(snappyExternalCatalog, Option(this), qualifiedName.database.get, qualifiedName.funcName, isEmbeddedMode, ignoreIfNotExists) super.dropFunction(name, ignoreIfNotExists) } - override def failFunctionLookup(name: String): Nothing = { - super.failFunctionLookup(name) - } - override def createFunction(funcDefinition: CatalogFunction, ignoreIfExists: Boolean): Unit = { val schemaName = getSchemaName(funcDefinition.identifier) // first check required permission to create objects in a schema @@ -1120,15 +1157,16 @@ class SnappySessionCatalog(val externalCatalog: SnappyExternalCatalog, } else false } - override def makeFunctionBuilder(funcName: String, className: String): FunctionBuilder = { - val uRLClassLoader = ContextJarUtils.getDriverJar(funcName) match { + protected def makeFunctionBuilderImpl(funcName: String, className: String): FunctionBuilder = { + val urlClassLoader = ContextJarUtils.getDriverJar(funcName) match { case None => org.apache.spark.util.Utils.getContextOrSparkClassLoader case Some(c) => c } - val (actualClassName, typeName) = className.splitAt(className.lastIndexOf("__")) - UDFFunction.makeFunctionBuilder(funcName, - uRLClassLoader.loadClass(actualClassName), - snappySession.sessionState.sqlParser.parseDataType(typeName.stripPrefix("__"))) + val splitIndex = className.lastIndexOf("__") + val actualClassName = className.substring(0, splitIndex) + val typeName = if (splitIndex != -1) className.substring(splitIndex + 2) else "" + val dataType = if (typeName.isEmpty) None else Some(parser.parseDataType(typeName)) + UDFFunction.makeFunctionBuilder(funcName, urlClassLoader.loadClass(actualClassName), dataType) } /** @@ -1235,25 +1273,3 @@ class SnappySessionCatalog(val externalCatalog: SnappyExternalCatalog, } } } - -final class SessionCatalogWrapper(externalCatalog: SnappyExternalCatalog, - snappySession: SnappySession, - globalTempViewManager: GlobalTempViewManager, - functionResourceLoader: FunctionResourceLoader, - functionRegistry: FunctionRegistry, - sqlConf: SQLConf, - hadoopConf: Configuration, - catalog: SnappySessionCatalog) - extends SnappySessionCatalog( - externalCatalog, - snappySession, - globalTempViewManager, - functionResourceLoader, - functionRegistry, - sqlConf, - hadoopConf) { - - override def lookupRelation(name: TableIdentifier, alias: Option[String]): LogicalPlan = { - catalog.resolveRelationWithAlias(name, alias) - } -} diff --git a/core/src/main/scala/org/apache/spark/sql/internal/UDFFunction.scala b/core/src/main/scala/org/apache/spark/sql/internal/UDFFunction.scala index 04aed81d7e..5ccf2867ed 100644 --- a/core/src/main/scala/org/apache/spark/sql/internal/UDFFunction.scala +++ b/core/src/main/scala/org/apache/spark/sql/internal/UDFFunction.scala @@ -18,25 +18,63 @@ package org.apache.spark.sql.internal import scala.util.control.NonFatal -import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.api.java._ import org.apache.spark.sql.catalyst.analysis.FunctionRegistry._ import org.apache.spark.sql.catalyst.expressions.{Expression, ScalaUDF} import org.apache.spark.sql.execution.aggregate.ScalaUDAF import org.apache.spark.sql.expressions.UserDefinedAggregateFunction -import org.apache.spark.sql.types.DataType +import org.apache.spark.sql.types.{DataType, NullType} +import org.apache.spark.sql.{AnalysisException, SparkSupport} -object UDFFunction { +object UDFFunction extends SparkSupport { - def makeFunctionBuilder(name: String, clazz: Class[_] , returnType: DataType): FunctionBuilder = { - (children: Seq[Expression]) => { - try { + private def scalaUDF(function: AnyRef, dataType: DataType, + children: Seq[Expression], inputTypes: Seq[DataType] = Nil, + udfName: Option[String] = None): ScalaUDF = { + // noinspection RedundantNewCaseClass + new ScalaUDF(function, dataType, children, inputTypes, udfName) + } + def makeFunctionBuilder(name: String, clazz: Class[_], dt: Option[DataType]): FunctionBuilder = { + children: Seq[Expression] => { + try { if (classOf[UserDefinedAggregateFunction].isAssignableFrom(clazz)) { val udaf = clazz.newInstance().asInstanceOf[UserDefinedAggregateFunction] - ScalaUDAF(children, udaf) + val e = ScalaUDAF(children, udaf, 1, 1) + // Check input argument size + if (e.inputTypes.length != children.length) { + throw new AnalysisException(s"Invalid number of arguments for function $name. " + + s"Expected: ${e.inputTypes.size}; Found: ${children.length}") + } + dt match { + case None => + case Some(t) => + if (t.asNullable != e.dataType.asNullable) { + throw new AnalysisException(s"Defined return type (${t.catalogString}) " + + s"does not match the one in function definition (${e.dataType.catalogString})") + } + } + e } else { + // infer the return type and check against the one defined + val inferred = clazz.getMethods.find(_.getName == "call") match { + case None => NullType + case Some(m) => internals.getReturnDataType(m) + } + val returnType = dt match { + case None => inferred + case Some(t) => + if (t.asNullable != inferred.asNullable) { + // an inferred type of NullType can be StructType or any other + if (inferred != NullType) { + throw new AnalysisException(s"Defined return type (${t.catalogString}) " + + s"does not match the one in function definition (${inferred.catalogString})") + } + } + t + } + // noinspection ScalaDocParserErrorInspection children.size match { // scalastyle:off line.size.limit @@ -57,102 +95,100 @@ object UDFFunction { // Script code starts case 1 => val func = clazz.newInstance().asInstanceOf[UDF1[Any, Any]] - ScalaUDF(func.call(_: Any), returnType, children) + scalaUDF(func.call(_: Any), returnType, children) case 2 => val func = clazz.newInstance().asInstanceOf[UDF2[Any, Any, Any]] - ScalaUDF(func.call(_: Any, _: Any), returnType, children) + scalaUDF(func.call(_: Any, _: Any), returnType, children) case 3 => val func = clazz.newInstance().asInstanceOf[UDF3[Any, Any, Any, Any]] - ScalaUDF(func.call(_: Any, _: Any, _: Any), returnType, children) + scalaUDF(func.call(_: Any, _: Any, _: Any), returnType, children) case 4 => val func = clazz.newInstance().asInstanceOf[UDF4[Any, Any, Any, Any, Any]] - ScalaUDF(func.call(_: Any, _: Any, _: Any, _: Any), returnType, children) + scalaUDF(func.call(_: Any, _: Any, _: Any, _: Any), returnType, children) case 5 => val func = clazz.newInstance().asInstanceOf[UDF5[Any, Any, Any, Any, Any, Any]] - ScalaUDF(func.call(_: Any, _: Any, _: Any, _: Any, _: Any), returnType, children) + scalaUDF(func.call(_: Any, _: Any, _: Any, _: Any, _: Any), returnType, children) case 6 => val func = clazz.newInstance().asInstanceOf[UDF6[Any, Any, Any, Any, Any, Any, Any]] - ScalaUDF(func.call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, children) + scalaUDF(func.call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, children) case 7 => val func = clazz.newInstance().asInstanceOf[UDF7[Any, Any, Any, Any, Any, Any, Any, Any]] - ScalaUDF(func.call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, children) + scalaUDF(func.call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, children) case 8 => val func = clazz.newInstance().asInstanceOf[UDF8[Any, Any, Any, Any, Any, Any, Any, Any, Any]] - ScalaUDF(func.call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, children) + scalaUDF(func.call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, children) case 9 => val func = clazz.newInstance().asInstanceOf[UDF9[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]] - ScalaUDF(func.call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, children) + scalaUDF(func.call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, children) case 10 => val func = clazz.newInstance().asInstanceOf[UDF10[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]] - ScalaUDF(func.call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, children) + scalaUDF(func.call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, children) case 11 => val func = clazz.newInstance().asInstanceOf[UDF11[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]] - ScalaUDF(func.call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, children) + scalaUDF(func.call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, children) case 12 => val func = clazz.newInstance().asInstanceOf[UDF12[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]] - ScalaUDF(func.call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, children) + scalaUDF(func.call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, children) case 13 => val func = clazz.newInstance().asInstanceOf[UDF13[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]] - ScalaUDF(func.call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, children) + scalaUDF(func.call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, children) case 14 => val func = clazz.newInstance().asInstanceOf[UDF14[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]] - ScalaUDF(func.call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, children) + scalaUDF(func.call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, children) case 15 => val func = clazz.newInstance().asInstanceOf[UDF15[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]] - ScalaUDF(func.call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, children) + scalaUDF(func.call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, children) case 16 => val func = clazz.newInstance().asInstanceOf[UDF16[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]] - ScalaUDF(func.call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, children) + scalaUDF(func.call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, children) case 17 => val func = clazz.newInstance().asInstanceOf[UDF17[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]] - ScalaUDF(func.call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, children) + scalaUDF(func.call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, children) case 18 => val func = clazz.newInstance().asInstanceOf[UDF18[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]] - ScalaUDF(func.call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, children) + scalaUDF(func.call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, children) case 19 => val func = clazz.newInstance().asInstanceOf[UDF19[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]] - ScalaUDF(func.call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, children) + scalaUDF(func.call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, children) case 20 => val func = clazz.newInstance().asInstanceOf[UDF20[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]] - ScalaUDF(func.call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, children) + scalaUDF(func.call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, children) case 21 => val func = clazz.newInstance().asInstanceOf[UDF21[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]] - ScalaUDF(func.call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, children) + scalaUDF(func.call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, children) case 22 => val func = clazz.newInstance().asInstanceOf[UDF22[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]] - ScalaUDF(func.call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, children) + scalaUDF(func.call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, children) - //Script code end + // Script code end // scalastyle:on line.size.limit - case _ => throw new AnalysisException(s"No handler for SnappyStore UDF '${clazz.getCanonicalName}'") + case _ => throw new AnalysisException( + s"No handler for SnappyStore UDF '${clazz.getCanonicalName}'") } - } - } catch { - case ae: AnalysisException => - throw ae + case ae: AnalysisException => throw ae case NonFatal(e) => val analysisException = new AnalysisException(s"No handler for SnappyStore UDF '${clazz.getCanonicalName}': $e") diff --git a/core/src/main/scala/org/apache/spark/sql/internal/session.scala b/core/src/main/scala/org/apache/spark/sql/internal/session.scala index 11b01de8b7..991c0a9527 100644 --- a/core/src/main/scala/org/apache/spark/sql/internal/session.scala +++ b/core/src/main/scala/org/apache/spark/sql/internal/session.scala @@ -32,22 +32,24 @@ import io.snappydata.{Constant, Property} import org.apache.spark.SparkConf import org.apache.spark.internal.config.{ConfigBuilder, ConfigEntry, TypedConfigBuilder} import org.apache.spark.sql.catalyst.analysis -import org.apache.spark.sql.catalyst.analysis.{EliminateSubqueryAliases, UnresolvedRelation} -import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, Cast, Contains, EndsWith, EqualTo, Expression, Like, Literal, StartsWith} -import org.apache.spark.sql.catalyst.plans.logical.{BroadcastHint, InsertIntoTable, LogicalPlan, OverwriteOptions, Project, UnaryNode, Filter => LogicalFilter} +import org.apache.spark.sql.catalyst.analysis.{EliminateSubqueryAliases, UnresolvedAttribute, UnresolvedTableValuedFunction} +import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeReference, Cast, Contains, EndsWith, EqualTo, Expression, Like, Literal, NamedExpression, StartsWith} +import org.apache.spark.sql.catalyst.optimizer.ReorderJoin +import org.apache.spark.sql.catalyst.plans.QueryPlan +import org.apache.spark.sql.catalyst.plans.logical.{InsertIntoTable, LogicalPlan, Project, UnaryNode, Filter => LogicalFilter} import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.collection.Utils import org.apache.spark.sql.execution.columnar.ExternalStoreUtils import org.apache.spark.sql.execution.columnar.ExternalStoreUtils.CaseInsensitiveMutableHashMap import org.apache.spark.sql.execution.command.DDLUtils import org.apache.spark.sql.execution.datasources.{CreateTable, LogicalRelation, PreprocessTableInsertion} -import org.apache.spark.sql.execution.{SecurityUtils, datasources} +import org.apache.spark.sql.execution.{SecurityUtils, SparkOptimizer} import org.apache.spark.sql.hive.SnappySessionState -import org.apache.spark.sql.internal.SQLConf.SQLConfigBuilder import org.apache.spark.sql.internal.StaticSQLConf.CATALOG_IMPLEMENTATION +import org.apache.spark.sql.row.JDBCMutableRelation import org.apache.spark.sql.sources._ -import org.apache.spark.sql.types.{DecimalType, StringType} -import org.apache.spark.sql.{AnalysisException, SaveMode, SnappyContext, SnappyParser, SnappySession} +import org.apache.spark.sql.types.{DecimalType, LongType, StringType} +import org.apache.spark.sql.{AnalysisException, DMLExternalTable, SaveMode, SnappyContext, SnappyParser, SnappySession, SparkSupport} import org.apache.spark.unsafe.types.UTF8String // Misc helper classes for session handling @@ -76,6 +78,11 @@ class SnappyConf(@transient val session: SnappySession) */ @volatile private[this] var dynamicCpusPerTask: Int = _ + // disable LogicalPlan cache since the ExternalCatalog implementations already have + // a large enough cache and this cache causes lot of trouble with stale data especially + // in smart connector mode which is already handled by SmartConnectorExternalCatalog + setConfString("spark.sql.filesourceTableRelationCacheSize", "0") + SQLConf.SHUFFLE_PARTITIONS.defaultValue match { case Some(d) if (session ne null) && super.numShufflePartitions == d => dynamicShufflePartitions = coreCountForShuffle @@ -183,7 +190,7 @@ class SnappyConf(@transient val session: SnappySession) // initialize hive session upfront if (newValue) { session.hiveInitializing = true - assert(session.sessionState.hiveSession ne null) + assert(session.snappySessionState.hiveSession ne null) session.hiveInitializing = false } session.enableHiveSupport = newValue @@ -256,7 +263,7 @@ class SnappyConf(@transient val session: SnappySession) } else key } - private def hiveConf: SQLConf = session.sessionState.hiveSession.sessionState.conf + private def hiveConf: SQLConf = session.snappySessionState.hiveSession.sessionState.conf private[sql] def resetDefaults(): Unit = synchronized { if (session ne null) { @@ -281,17 +288,23 @@ class SnappyConf(@transient val session: SnappySession) private[sql] def setDynamicCpusPerTask(): Unit = synchronized { if (dynamicCpusPerTask != -1) { + val numExecutors = SnappyContext.numExecutors + val totalUsableHeap = SnappyContext.foldLeftBlockIds(0L)(_ + _.usableHeapBytes) + + // skip for smart connector where there is no information of physical cores or heap + if (numExecutors == 0 || totalUsableHeap <= 0) return + val sparkCores = session.sparkContext.defaultParallelism.toDouble // calculate minimum required heap assuming a block size of 128M val minRequiredHeap = 128.0 * 1024.0 * 1024.0 * sparkCores * 1.2 - val totalUsableHeap = SnappyContext.foldLeftBlockIds(0L)(_ + _.usableHeapBytes) + // select bigger among (required heap / available) and (logical cores / physical) val cpusPerTask0 = math.max(minRequiredHeap / totalUsableHeap, sparkCores / SnappyContext.totalPhysicalCoreCount.get()) // keep a reasonable upper-limit so tasks can at least be scheduled: // used below is average logical cores / 2 val cpusPerTask = math.max(1, math.ceil(math.min(sparkCores / - (2 * SnappyContext.numExecutors), cpusPerTask0)).toInt) + (2 * numExecutors), cpusPerTask0)).toInt) setConfString(Constant.CPUS_PER_TASK_PROP, cpusPerTask.toString) dynamicCpusPerTask = cpusPerTask logDebug(s"Set dynamic ${Constant.CPUS_PER_TASK_PROP} to $cpusPerTask") @@ -376,7 +389,7 @@ class SQLConfigEntry private(private[sql] val entry: ConfigEntry[_]) { override def toString: String = entry.toString } -object SQLConfigEntry { +object SQLConfigEntry extends SparkSupport { private def handleDefault[T](entry: TypedConfigBuilder[T], defaultValue: Option[T]): SQLConfigEntry = defaultValue match { @@ -406,16 +419,16 @@ object SQLConfigEntry { def apply[T: ClassTag](key: String, doc: String, defaultValue: Option[T], isPublic: Boolean = true): SQLConfigEntry = { classTag[T] match { - case ClassTag.Int => handleDefault[Int](SQLConfigBuilder(key) + case ClassTag.Int => handleDefault[Int](internals.buildConf(key) .doc(doc).intConf, defaultValue.asInstanceOf[Option[Int]]) - case ClassTag.Long => handleDefault[Long](SQLConfigBuilder(key) + case ClassTag.Long => handleDefault[Long](internals.buildConf(key) .doc(doc).longConf, defaultValue.asInstanceOf[Option[Long]]) - case ClassTag.Double => handleDefault[Double](SQLConfigBuilder(key) + case ClassTag.Double => handleDefault[Double](internals.buildConf(key) .doc(doc).doubleConf, defaultValue.asInstanceOf[Option[Double]]) - case ClassTag.Boolean => handleDefault[Boolean](SQLConfigBuilder(key) + case ClassTag.Boolean => handleDefault[Boolean](internals.buildConf(key) .doc(doc).booleanConf, defaultValue.asInstanceOf[Option[Boolean]]) case c if c.runtimeClass == classOf[String] => - handleDefault[String](SQLConfigBuilder(key).doc(doc).stringConf, + handleDefault[String](internals.buildConf(key).doc(doc).stringConf, defaultValue.asInstanceOf[Option[String]]) case c => throw new IllegalArgumentException( s"Unknown type of configuration key: $c") @@ -555,11 +568,96 @@ trait SQLAltName[T] extends AltName[T] { } } -private[sql] final class PreprocessTable(state: SnappySessionState) extends Rule[LogicalPlan] { +trait DefaultOptimizer extends SparkOptimizer { + + def state: SnappySessionState + + def batchesImpl: Seq[Batch] = { + implicit val ss: SnappySession = state.snappySession + var insertedSnappyOpts = 0 + val modified = super.batches.map { + case batch if batch.name.startsWith("Operator Optimization") => + insertedSnappyOpts += 1 + val (left, right) = batch.rules.splitAt(batch.rules.indexOf(ReorderJoin)) + Batch(batch.name, batch.strategy, (left :+ ResolveIndex()) ++ right: _*) + case b => b + } + + if (insertedSnappyOpts == 0) { + throw new AnalysisException("Snappy Optimizations not applied") + } + + modified :+ + Batch("Streaming SQL Optimizers", Once, state.PushDownWindowLogicalPlan) :+ + Batch("Link buckets to RDD partitions", Once, state.LinkPartitionsToBuckets) :+ + Batch("TokenizedLiteral Folding Optimization", Once, state.TokenizedLiteralFolding) :+ + Batch("Order join conditions ", Once, state.OrderJoinConditions) + } +} + +private[sql] final class PreprocessTable(state: SnappySessionState) + extends Rule[LogicalPlan] with SparkSupport { private def conf: SQLConf = state.conf - def apply(plan: LogicalPlan): LogicalPlan = plan transform { + private def resolveProjection(u: UnresolvedTableValuedFunction, + child: LogicalPlan, op: String): (LogicalPlan, LogicalPlan) = { + val session = state.snappySession + if (u.functionArgs.forall(_.isInstanceOf[UnresolvedAttribute])) { + val relation = session.sessionCatalog.resolveRelation( + session.tableIdentifier(u.functionName, resolve = true)) + val output = relation.output + val childOutput = child.output + if (childOutput.length != u.functionArgs.length) { + throw new AnalysisException("Query in the INSERT/PUT statement " + + s"(${childOutput.map(_.name).mkString("; ")}) should generate the same number " + + s"of columns as the table projection (${u.functionArgs.mkString("; ")})") + } + // if all columns are being projected then apply the Projections else + // check for row tables and pass them through since those may have + // default values or identity columns + val projection = new Array[NamedExpression](output.length) + val resolver = state.analyzer.resolver + var index = -1 + for (i <- u.functionArgs.indices) { + val e = u.functionArgs(i) + relation.resolve(e.asInstanceOf[UnresolvedAttribute].nameParts, resolver) match { + case Some(attr) if (index = output.indexOf(attr)).isInstanceOf[Unit] && index != -1 => + projection(index) = internals.newAlias(childOutput(i), output(index).name, None) + case None => + throw new AnalysisException(s"Could not resolve $e for $op " + + s"in table ${u.functionName} among (${output.map(_.name).mkString(", ")})") + } + } + val isRowTable = relation match { + case lr: LogicalRelation if lr.relation.isInstanceOf[JDBCMutableRelation] => true + case _ => false + } + val currentKey = session.currentKey + var hasNullValueProjection = false + for (i <- projection.indices) { + if (projection(i) eq null) { + hasNullValueProjection = true + // add NULL of target type + if (!isRowTable || (currentKey eq null)) { + val attr = output(i) + if (!attr.nullable) { + throw new AnalysisException( + s"For $op in ${u.functionName}, ${attr.name} not specified but is NOT NULL") + } + projection(i) = internals.newAlias(Literal(null, attr.dataType), attr.name, None) + } + } + } + if (hasNullValueProjection && isRowTable && (currentKey ne null)) { + // fallback to store-layer SQL to handle possible default and autoincrement columns + // TODO: handle default (using Metadata query) and autoinc (using builtin functions) + (u, DMLExternalTable(relation, currentKey.sqlText)) + } else (relation, Project(projection.toSeq, child)) + } else (u, child) + } + + def apply(plan: LogicalPlan): LogicalPlan = internals.logicalPlanResolveDown(plan) { // Add dbtable property for create table. While other routes can add it in // SnappySession.createTable, the DataFrameWriter path needs to be handled here. @@ -573,9 +671,9 @@ private[sql] final class PreprocessTable(state: SnappySessionState) extends Rule if (mode == SaveMode.Append && queryOpt.isDefined && (isBuiltin || (tableDesc.bucketSpec.isEmpty && tableDesc.partitionColumnNames.isEmpty)) && state.catalog.tableExists(tableIdent)) { - new Insert(table = UnresolvedRelation(tableIdent), - partition = Map.empty, child = queryOpt.get, - overwrite = OverwriteOptions(enabled = false), ifNotExists = false) + internals.newInsertIntoTable( + table = internals.newUnresolvedRelation(tableIdent, None), + partition = Map.empty, child = queryOpt.get, overwrite = false, ifNotExists = false) } else if (isBuiltin) { val tableName = tableIdent.unquotedString // dependent tables are stored as comma-separated so don't allow comma in table name @@ -600,14 +698,50 @@ private[sql] final class PreprocessTable(state: SnappySessionState) extends Rule c.copy(tableDesc.copy(storage = tableDesc.storage.copy(properties = newOptions))) } else c + // resolve INSERT INTO/OVERWRITE TABLE(columns) ... + case i: InsertIntoTable if i.table.isInstanceOf[UnresolvedTableValuedFunction] => + val isOverwrite = internals.getOverwriteOption(i) + val query = i.children.head + resolveProjection(i.table.asInstanceOf[UnresolvedTableValuedFunction], query, + s"INSERT ${if (isOverwrite) "OVERWRITE" else "INTO"}") match { + case (_, d: DMLExternalTable) => + // no support for OVERWRITE or PARTITION for this case + val tableName = d.child match { + case lr: LogicalRelation if lr.relation.isInstanceOf[JDBCMutableRelation] => + " " + lr.relation.asInstanceOf[JDBCMutableRelation].resolvedName + case _ => "" + } + if (isOverwrite) { + throw new AnalysisException(s"INSERT OVERWRITE not supported with " + + s"table column specification on row table$tableName") + } + if (i.partition.nonEmpty) { + throw new AnalysisException(s"INSERT with PARTITION not supported with " + + s"table column specification on row table$tableName") + } + d + case (t, c) => + if ((t eq i.table) && (c eq query)) i + else i.copy(t, i.partition, c, i.overwrite) + } + + // resolve PUT INTO TABLE(columns) ... + case p@PutIntoTable(u: UnresolvedTableValuedFunction, child) => + resolveProjection(u, child, "PUT INTO") match { + case (_, d: DMLExternalTable) => d + case (t, c) => if ((t eq u) && (c eq child)) p else p.copy(table = t, child = c) + } + // Check for SchemaInsertableRelation first - case i@InsertIntoTable(l@LogicalRelation(r: SchemaInsertableRelation, - _, _), _, child, _, _) if l.resolved && child.resolved => + case i@InsertIntoTable(l: LogicalRelation, _, child, _, _) + if l.relation.isInstanceOf[SchemaInsertableRelation] && l.resolved && child.resolved => + val r = l.relation.asInstanceOf[SchemaInsertableRelation] r.insertableRelation(child.output) match { case Some(ir) if r eq ir => i case Some(ir) => val br = ir.asInstanceOf[BaseRelation] - val relation = LogicalRelation(br, catalogTable = l.catalogTable) + val relation = internals.newLogicalRelation(br, + None, l.catalogTable, isStreaming = false) castAndRenameChildOutputForPut(i.copy(table = relation), relation.output, br, null, child) case None => @@ -622,7 +756,7 @@ private[sql] final class PreprocessTable(state: SnappySessionState) extends Rule // ResolveRelations, no such special rule has been added for PUT case p@PutIntoTable(table, child) if table.resolved && child.resolved => EliminateSubqueryAliases(table) match { - case l@LogicalRelation(ir: RowInsertableRelation, _, _) => + case l: LogicalRelation if l.relation.isInstanceOf[RowInsertableRelation] => // First, make sure the data to be inserted have the same number of // fields with the schema of the relation. val expectedOutput = l.output @@ -631,7 +765,7 @@ private[sql] final class PreprocessTable(state: SnappySessionState) extends Rule "SELECT clause of the PUT INTO statement " + "generates the same number of columns as its schema.") } - castAndRenameChildOutputForPut(p, expectedOutput, ir, l, child) + castAndRenameChildOutputForPut(p, expectedOutput, l.relation, l, child) case _ => p } @@ -642,9 +776,9 @@ private[sql] final class PreprocessTable(state: SnappySessionState) extends Rule // ResolveRelations, no such special rule has been added for PUT case d@DeleteFromTable(table, child) if table.resolved && child.resolved => EliminateSubqueryAliases(table) match { - case l@LogicalRelation(dr: MutableRelation, _, _) => - - val keyColumns = dr.getPrimaryKeyColumns(state.snappySession) + case l: LogicalRelation if l.relation.isInstanceOf[MutableRelation] => + val mr = l.relation.asInstanceOf[MutableRelation] + val keyColumns = mr.getPrimaryKeyColumns(state.snappySession) val childOutput = keyColumns.map(col => child.resolveQuoted(col, analysis.caseInsensitiveResolution) match { case Some(a: Attribute) => a @@ -662,7 +796,8 @@ private[sql] final class PreprocessTable(state: SnappySessionState) extends Rule s"Actual schema: ${l.output.mkString(",")}") }) - castAndRenameChildOutputForPut(d, expectedOutput, dr, l, Project(childOutput, child)) + castAndRenameChildOutputForPut(d, expectedOutput, l.relation, + l, Project(childOutput, child)) case _ => d } @@ -710,7 +845,7 @@ private[sql] final class PreprocessTable(state: SnappySessionState) extends Rule child = Project(newChildOutput, child)).asInstanceOf[T] case d: DeleteFromTable => d.copy(table = newRelation, child = Project(newChildOutput, child)).asInstanceOf[T] - case i: InsertIntoTable => i.copy(child = Project(newChildOutput, + case i: InsertIntoTable => internals.withNewChild(i, Project(newChildOutput, child)).asInstanceOf[T] } } @@ -720,12 +855,12 @@ private[sql] case object PrePutCheck extends (LogicalPlan => Unit) { def apply(plan: LogicalPlan): Unit = { plan.foreach { - case PutIntoTable(LogicalRelation(t: RowPutRelation, _, _), query) => + case PutIntoTable(l: LogicalRelation, query) if l.relation.isInstanceOf[RowPutRelation] => // Get all input data source relations of the query. val srcRelations = query.collect { - case LogicalRelation(src: BaseRelation, _, _) => src + case l: LogicalRelation => l.relation } - if (srcRelations.contains(t)) { + if (srcRelations.contains(l.relation)) { throw Utils.analysisException( "Cannot put into table that is also being read from.") } else { @@ -738,7 +873,7 @@ private[sql] case object PrePutCheck extends (LogicalPlan => Unit) { } } -private[sql] case class ConditionalPreWriteCheck(sparkPreWriteCheck: datasources.PreWriteCheck) +private[sql] case class ConditionalPreWriteCheck(sparkPreWriteCheck: LogicalPlan => Unit) extends (LogicalPlan => Unit) { def apply(plan: LogicalPlan): Unit = { plan match { @@ -748,12 +883,51 @@ private[sql] case class ConditionalPreWriteCheck(sparkPreWriteCheck: datasources } } +/** + * Unlike Spark's `InsertIntoTable` this plan provides the count of rows + * inserted as the output. + * + * Note that the underlying BaseRelation should always be a [[PlanInsertableRelation]]. + */ +case class InsertIntoPlan(logicalRelation: LogicalRelation, + query: LogicalPlan, overwrite: Boolean) extends LogicalPlan { + + override lazy val output: Seq[Attribute] = AttributeReference("count", LongType)() :: Nil + + override def children: Seq[LogicalPlan] = Nil + + override protected def innerChildren: Seq[QueryPlan[_]] = query :: Nil + + val relation: PlanInsertableRelation = + logicalRelation.relation.asInstanceOf[PlanInsertableRelation] +} + +private[sql] object ResolveInsertIntoPlan extends Rule[LogicalPlan] with SparkSupport { + + override def apply(plan: LogicalPlan): LogicalPlan = internals.logicalPlanResolveDown(plan) { + case i@InsertIntoTable(l: LogicalRelation, _, query, _, _) + if l.relation.isInstanceOf[PlanInsertableRelation] && l.resolved && query.resolved => + + // check that insert with overwrite does not refer to the source table in the query + val isOverwrite = internals.getOverwriteOption(i) + if (isOverwrite) { + query.foreach { + case lr: LogicalRelation if lr.relation == l.relation => + throw new AnalysisException( + "Cannot insert overwrite into table that is also being read from.") + case _ => + } + } + InsertIntoPlan(l, query, isOverwrite) + } +} + /** * Deals with any escape characters in the LIKE pattern in optimization. * Does not deal with startsAndEndsWith equivalent of Spark's LikeSimplification * so 'a%b' kind of pattern with additional escaped chars will not be optimized. */ -object LikeEscapeSimplification { +object LikeEscapeSimplification extends SparkSupport { private def addTokenizedLiteral(parser: SnappyParser, s: String): Expression = { if (parser ne null) parser.addTokenizedLiteral(UTF8String.fromString(s), StringType) @@ -800,7 +974,7 @@ object LikeEscapeSimplification { } } - def apply(plan: LogicalPlan): LogicalPlan = plan transformAllExpressions { + def apply(plan: LogicalPlan): LogicalPlan = internals.logicalPlanResolveExpressions(plan) { case l@Like(left, Literal(pattern, StringType)) => simplifyLike(null, l, left, pattern.toString) } @@ -813,21 +987,3 @@ case class MarkerForCreateTableAsSelect(child: LogicalPlan) extends UnaryNode { case class BypassRowLevelSecurity(child: LogicalFilter) extends UnaryNode { override def output: Seq[Attribute] = child.output } - -/** - * Wrap plan-specific query hints (like joinType). This extends Spark's BroadcastHint - * so that filters/projections etc can be pushed below this by optimizer. - */ -class LogicalPlanWithHints(_child: LogicalPlan, val hints: Map[String, String]) - extends BroadcastHint(_child) { - - override def productArity: Int = 2 - - override def productElement(n: Int): Any = n match { - case 0 => child - case 1 => hints - } - - override def simpleString: String = - s"LogicalPlanWithHints[hints = $hints; child = ${child.simpleString}]" -} diff --git a/core/src/main/scala/org/apache/spark/sql/policy/policyFunctions.scala b/core/src/main/scala/org/apache/spark/sql/policy/policyFunctions.scala index 5036402391..3cc17f22ed 100644 --- a/core/src/main/scala/org/apache/spark/sql/policy/policyFunctions.scala +++ b/core/src/main/scala/org/apache/spark/sql/policy/policyFunctions.scala @@ -38,7 +38,8 @@ import org.apache.spark.unsafe.types.UTF8String @ExpressionDescription( usage = "_FUNC_() - Returns the name of the user that owns the session executing the " + "current SQL statement.", - extended = """ + extended = + """ Examples: > SELECT _FUNC_(); USER1 @@ -64,11 +65,14 @@ case class CurrentUser() extends LeafExpression with CodegenFallback { /** * Get the LDAP groups of the current user executing the function. + * + * There is no code generation since this expression should get constant folded by the optimizer. */ @ExpressionDescription( usage = "_FUNC_() - Returns all the ldap groups as an ARRAY to which the user " + "who is executing the current SQL statement belongs.", - extended = """ + extended = + """ Examples: > SELECT array_contains(_FUNC_(), 'GROUP1'); true diff --git a/core/src/main/scala/org/apache/spark/sql/rdds.scala b/core/src/main/scala/org/apache/spark/sql/rdds.scala index f9e82ca54d..f20dd04a80 100644 --- a/core/src/main/scala/org/apache/spark/sql/rdds.scala +++ b/core/src/main/scala/org/apache/spark/sql/rdds.scala @@ -60,8 +60,10 @@ class DelegateRDD[T: ClassTag]( preferredLocations: Array[Seq[String]] = null, allDependencies: Seq[Dependency[_]] = null) extends RDD[T](sc, - if (allDependencies == null) baseRdd.dependencies - else allDependencies) + // for some weird reason passing dependencies as such causes deserialization errors + // in tests, so converting to forms (toArray.toList) that deserialize correctly + if (allDependencies == null) baseRdd.dependencies.toArray.toList + else allDependencies.toArray.toList) with Serializable { @transient override val partitioner: Option[Partitioner] = baseRdd.partitioner @@ -86,6 +88,6 @@ case class EmptyIteratorWithRowCount[U](rowCount : Long) extends Iterator[U] { object RDDs { def getIteratorSize[T](iterator: Iterator[T]): Long = iterator match { case EmptyIteratorWithRowCount(rowCount) => rowCount - case _ => Utils.getIteratorSize[T](iterator) + case _ => Utils.getIteratorSize(iterator) } } diff --git a/core/src/main/scala/org/apache/spark/sql/row/JDBCMutableRelation.scala b/core/src/main/scala/org/apache/spark/sql/row/JDBCMutableRelation.scala index f5aa5315a4..0aa20bb193 100644 --- a/core/src/main/scala/org/apache/spark/sql/row/JDBCMutableRelation.scala +++ b/core/src/main/scala/org/apache/spark/sql/row/JDBCMutableRelation.scala @@ -18,19 +18,18 @@ package org.apache.spark.sql.row import java.sql.Connection -import com.gemstone.gemfire.internal.shared.ClientResolverUtils - import scala.collection.JavaConverters._ + +import com.gemstone.gemfire.internal.shared.ClientResolverUtils import io.snappydata.SnappyTableStatsProviderService -import kafka.client.ClientUtils + import org.apache.spark.rdd.RDD import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression, SortDirection} -import org.apache.spark.sql.catalyst.plans.logical.OverwriteOptions -import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier} import org.apache.spark.sql.collection.Utils import org.apache.spark.sql.execution.columnar.ExternalStoreUtils +import org.apache.spark.sql.execution.columnar.ExternalStoreUtils.CaseInsensitiveMutableHashMap import org.apache.spark.sql.execution.datasources.LogicalRelation import org.apache.spark.sql.execution.datasources.jdbc._ import org.apache.spark.sql.execution.row.{RowDeleteExec, RowInsertExec, RowUpdateExec} @@ -54,7 +53,7 @@ abstract case class JDBCMutableRelation( mode: SaveMode, userSpecifiedString: String, parts: Array[Partition], - origOptions: CaseInsensitiveMap, + override val origOptions: CaseInsensitiveMutableHashMap[String], @transient override val sqlContext: SQLContext) extends BaseRelation with PrunedUnsafeFilteredScan @@ -66,7 +65,8 @@ abstract case class JDBCMutableRelation( with DestroyRelation with IndexableRelation with AlterableRelation - with NativeTableRowLevelSecurityRelation + with SnappyTableRelation + with SparkSupport with Logging { override val needConversion: Boolean = false @@ -192,11 +192,11 @@ abstract case class JDBCMutableRelation( // use the Insert plan for best performance // that will use the getInsertPlan above (in StoreStrategy) sqlContext.sessionState.executePlan( - new Insert( + internals.newInsertIntoTable( table = LogicalRelation(this), partition = Map.empty[String, Option[String]], child = data.logicalPlan, - OverwriteOptions(overwrite), + overwrite, ifNotExists = false)).toRdd } @@ -411,12 +411,10 @@ abstract case class JDBCMutableRelation( override def equals(that: Any): Boolean = { that match { - case mutable: JDBCMutableRelation => { - (this eq mutable) || ( - hashCode() == mutable.hashCode() + case mutable: JDBCMutableRelation => + (this eq mutable) || (hashCode() == mutable.hashCode() && mutable.schemaName.equalsIgnoreCase(schemaName) && mutable.tableName.equalsIgnoreCase(tableName)) - } case _ => false } } diff --git a/core/src/main/scala/org/apache/spark/sql/sources/RuleUtils.scala b/core/src/main/scala/org/apache/spark/sql/sources/RuleUtils.scala index 5bf3247018..e4effe5223 100644 --- a/core/src/main/scala/org/apache/spark/sql/sources/RuleUtils.scala +++ b/core/src/main/scala/org/apache/spark/sql/sources/RuleUtils.scala @@ -37,14 +37,15 @@ import org.apache.spark.sql.execution.datasources.LogicalRelation import org.apache.spark.sql.execution.row.RowFormatRelation import org.apache.spark.sql.internal.SnappySessionCatalog import org.apache.spark.sql.sources.Entity.{INDEX, INDEX_RELATION, TABLE} -import org.apache.spark.sql.{AnalysisException, SnappySession} +import org.apache.spark.sql.{AnalysisException, SnappySession, SparkSupport} -object RuleUtils extends PredicateHelper { +object RuleUtils extends PredicateHelper with SparkSupport { private def getIndex(catalog: SnappySessionCatalog, table: CatalogTable): Option[INDEX] = { val relation = catalog.resolveRelation(table.identifier) relation match { - case LogicalRelation(_: IndexColumnFormatRelation, _, _) => Some(relation) + case lr: LogicalRelation if lr.relation.isInstanceOf[IndexColumnFormatRelation] => + Some(relation) case _ => None } } @@ -53,10 +54,10 @@ object RuleUtils extends PredicateHelper { table: LogicalPlan): Seq[(LogicalPlan, Seq[LogicalPlan])] = { val catalog = snappySession.sessionCatalog table.collect { - case l@LogicalRelation(p: PartitionedDataSourceScan, _, _) => + case lr: LogicalRelation if lr.relation.isInstanceOf[PartitionedDataSourceScan] => val (schemaName, table) = JdbcExtendedUtils.getTableWithSchema( - p.table, null, Some(snappySession)) - (l.asInstanceOf[LogicalPlan], catalog.externalCatalog.getDependentsFromProperties( + lr.relation.asInstanceOf[PartitionedDataSourceScan].table, null, Some(snappySession)) + (lr.asInstanceOf[LogicalPlan], catalog.snappyExternalCatalog.getDependentsFromProperties( schemaName, table, includeTypes = CatalogObjectType.Index :: Nil) .flatMap(getIndex(catalog, _))) } @@ -87,7 +88,7 @@ object RuleUtils extends PredicateHelper { case expressions.EqualNullSafe(l, r) if canEvaluate(l, right) && canEvaluate(r, left) => Some((Coalesce(Seq(r, Literal.default(r.dataType))), Coalesce(Seq(l, Literal.default(l.dataType))))) - case other => None + case _ => None } } @@ -96,8 +97,8 @@ object RuleUtils extends PredicateHelper { replicatedReachablePaths: Seq[List[LogicalPlan]]): Boolean = { if (source.isEmpty) { - return false - } else if (source.exists(_ == target)) { + false + } else if (source.contains(target)) { true } else if (replicatedReachablePaths.isEmpty) { false @@ -109,10 +110,10 @@ object RuleUtils extends PredicateHelper { case ((otherKey, current), plan) => plan match { case l :: r :: o if o.isEmpty & (l == rep1) => - ((otherKey ++ Some(r)), current.filterNot(_ == plan)) + (otherKey ++ Some(r), current.filterNot(_ == plan)) case l :: r :: o if o.isEmpty & (r == rep1) => - ((otherKey ++ Some(l)), current.filterNot(_ == plan)) - case _ => ((otherKey, current)) + (otherKey ++ Some(l), current.filterNot(_ == plan)) + case _ => (otherKey, current) } } @@ -124,7 +125,7 @@ object RuleUtils extends PredicateHelper { } } - protected[sql] def applyDefaultAction[A](entity: (PartialPlan, A), withFilters: Boolean) + private[sql] def applyDefaultAction[A](entity: (PartialPlan, A), withFilters: Boolean) (implicit snappySession: SnappySession, addToDefault: (PartialPlan, A) => PartialPlan): PartialPlan = entity match { // handles replicated & non-colocated logical plan @@ -177,7 +178,7 @@ object RuleUtils extends PredicateHelper { addToDefault(newPlan, replacement.asInstanceOf[A]) } - protected[sql] def createJoin(curPlan: LogicalPlan, + private[sql] def createJoin(curPlan: LogicalPlan, planToAdd: LogicalPlan, toJoinWith: Seq[Expression]) = if (curPlan == null) { planToAdd } else { @@ -186,19 +187,19 @@ object RuleUtils extends PredicateHelper { Join(curPlan, planToAdd, Inner, toJoinWith.reduceLeftOption(expressions.And)) } - protected[sql] def partitionBy(allColumns: AttributeSet, expressions: Seq[Expression]): + private[sql] def partitionBy(allColumns: AttributeSet, expressions: Seq[Expression]): (Seq[Expression], Seq[Expression]) = expressions.partition(e => e.references.subsetOf(allColumns) && !SubqueryExpression.hasCorrelatedSubquery(e)) - protected[sql] def returnPlan(partial: PartialPlan) = { + private[sql] def returnPlan(partial: PartialPlan): CompletePlan = { val input = if (partial.curPlan == null) partial.input else Seq(partial.curPlan) ++ partial.input CompletePlan(ReorderJoin.createOrderedJoin(input.map((_, Inner)), partial.conditions), partial.replaced ++ partial.input.map(t => Replacement(t, t))) } - protected[sql] def chooseIndexForFilter(child: LogicalPlan, conditions: Seq[Expression]) - (implicit snappySession: SnappySession) = { + private[sql] def chooseIndexForFilter(child: LogicalPlan, conditions: Seq[Expression]) + (implicit snappySession: SnappySession): Option[Replacement] = { val columnGroups = conditions.collect { case expressions.EqualTo(l, r) => l.collectFirst { case a: AttributeReference => a }.orElse { @@ -210,30 +211,32 @@ object RuleUtils extends PredicateHelper { } }.groupBy(_.map(_.qualifier)).collect { case (table, cols) if table.nonEmpty && table.get.nonEmpty => ( - table.get.get, + table.get.head, cols.collect { case a if a.nonEmpty => a.get }) } + var ir: IndexColumnFormatRelation = null val currentSchema = snappySession.getCurrentSchema val satisfyingPartitionColumns = for { (table, indexes) <- RuleUtils.fetchIndexes(snappySession, child) filterCols <- columnGroups.collectFirst { case (t, predicates) if predicates.nonEmpty => table match { - case LogicalRelation(b: ColumnFormatRelation, _, _) - if b.table.equalsIgnoreCase(t) || b.table.equalsIgnoreCase(s"$currentSchema.$t") => - predicates - case SubqueryAlias(alias, _, _) if alias.equalsIgnoreCase(t) => - predicates + case lr: LogicalRelation if lr.relation.isInstanceOf[ColumnFormatRelation] && + (lr.relation.asInstanceOf[ColumnFormatRelation].table.equalsIgnoreCase(t) || + lr.relation.asInstanceOf[ColumnFormatRelation].table.equalsIgnoreCase( + s"$currentSchema.$t")) => predicates + case s: SubqueryAlias if s.alias.equalsIgnoreCase(t) => predicates case _ => Nil } } if filterCols.nonEmpty matchedIndexes = indexes.collect { - case idx@LogicalRelation(ir: IndexColumnFormatRelation, _, _) - if ir.partitionColumns.length <= filterCols.length & - ir.partitionColumns.forall(p => filterCols.exists(f => - f.name.equalsIgnoreCase(p))) => + case idx: LogicalRelation if idx.relation.isInstanceOf[IndexColumnFormatRelation] && + (ir = idx.relation.asInstanceOf[IndexColumnFormatRelation]).isInstanceOf[Unit] && + ir.partitionColumns.length <= filterCols.length & + ir.partitionColumns.forall(p => filterCols.exists(f => + f.name.equalsIgnoreCase(p))) => (ir.partitionColumns.length, idx.asInstanceOf[LogicalPlan]) } if matchedIndexes.nonEmpty @@ -245,7 +248,7 @@ object RuleUtils extends PredicateHelper { None } else { Some(satisfyingPartitionColumns.maxBy { - r => r.index.statistics.sizeInBytes + r => internals.getStatistics(r.index).sizeInBytes }) } } @@ -276,10 +279,11 @@ object Entity { def unwrapBaseColumnRelation( plan: LogicalPlan): Option[BaseColumnFormatRelation] = plan collectFirst { - case LogicalRelation(relation: BaseColumnFormatRelation, _, _) => - relation - case SubqueryAlias(alias, LogicalRelation(relation: BaseColumnFormatRelation, _, _), _) => - relation + case lr: LogicalRelation if lr.relation.isInstanceOf[BaseColumnFormatRelation] => + lr.relation.asInstanceOf[BaseColumnFormatRelation] + case s: SubqueryAlias if s.child.isInstanceOf[LogicalRelation] && + s.child.asInstanceOf[LogicalRelation].relation.isInstanceOf[BaseColumnFormatRelation] => + s.child.asInstanceOf[LogicalRelation].relation.asInstanceOf[BaseColumnFormatRelation] } private def findR(p: Any) = p match { @@ -311,7 +315,7 @@ object Entity { } } -object HasColocatedEntities { +object HasColocatedEntities extends SparkSupport { type ReturnType = ( Seq[(INDEX_RELATION, INDEX_RELATION)], Seq[ReplacementSet] @@ -356,7 +360,7 @@ object HasColocatedEntities { // assert(leftRightEntityMapping.size <= 1) val mappings = leftRightEntityMapping.flatMap { mappedElements => - val (leftTable, rightTable) = mappedElements(0) // first pairing is always (table, table) + val (leftTable, rightTable) = mappedElements.head // first pairing is always (table, table) for { (leftPlan, rightPlan) <- mappedElements leftRelation = Entity.unwrapBaseColumnRelation(leftPlan) if leftRelation.nonEmpty @@ -365,13 +369,13 @@ object HasColocatedEntities { } yield { val leftReplacement = leftTable match { case _: LogicalRelation => Replacement(leftTable, leftPlan) - case subquery@SubqueryAlias(alias, _, v) => - Replacement(subquery, SubqueryAlias(alias, leftPlan, None)) + case subquery: SubqueryAlias => + Replacement(subquery, internals.newSubqueryAlias(subquery.alias, leftPlan)) } val rightReplacement = rightTable match { case _: LogicalRelation => Replacement(rightTable, rightPlan) - case subquery@SubqueryAlias(alias, _, _) => - Replacement(subquery, SubqueryAlias(alias, rightPlan, None)) + case subquery: SubqueryAlias => + Replacement(subquery, internals.newSubqueryAlias(subquery.alias, rightPlan)) } ((leftRelation.get, rightRelation.get), ReplacementSet(ArrayBuffer(leftReplacement, rightReplacement), Nil)) @@ -391,38 +395,42 @@ object HasColocatedEntities { * Table to table or Table to index replacement. */ case class Replacement(table: TABLE, index: INDEX, isPartitioned: Boolean = true) - extends PredicateHelper { + extends PredicateHelper with SparkSupport { def isReplacable: Boolean = table != index - val indexAttributes = index.output.collect { case ar: AttributeReference => ar } + private[sql] val indexAttributes = index.output.collect { case ar: AttributeReference => ar } - val tableToIndexAttributeMap = AttributeMap(table.output.map { + private[sql] val tableToIndexAttributeMap = AttributeMap(table.output.map { case f: AttributeReference => val newA = indexAttributes.find(_.name.equalsIgnoreCase(f.name)). getOrElse(throw new IllegalStateException( - s"Field $f not found in ${indexAttributes}")) + s"Field $f not found in $indexAttributes")) (f, newA) - case a => throw new AssertionError(s"UnHandled Attribute ${a} in table" + + case a => throw new IllegalStateException(s"Unhandled Attribute $a in table" + s" ${table.output.mkString(",")}") }) - private var _replacedEntity: LogicalPlan = null + private var _replacedEntity: LogicalPlan = _ def numPartitioningCols: Int = index match { - case LogicalRelation(b: BaseColumnFormatRelation, _, _) => b.partitionColumns.length + case lr: LogicalRelation if lr.relation.isInstanceOf[BaseColumnFormatRelation] => + lr.relation.asInstanceOf[BaseColumnFormatRelation].partitionColumns.length case _ => 0 } override def toString: String = { "" + (table match { - case LogicalRelation(b: BaseColumnFormatRelation, _, _) => b.table + case lr: LogicalRelation if lr.relation.isInstanceOf[BaseColumnFormatRelation] => + lr.relation.asInstanceOf[BaseColumnFormatRelation].table case _ => table.toString() }) + " ----> " + (index match { - case LogicalRelation(b: BaseColumnFormatRelation, _, _) => b.table - case LogicalRelation(r: RowFormatRelation, _, _) => r.table + case lr: LogicalRelation if lr.relation.isInstanceOf[BaseColumnFormatRelation] => + lr.relation.asInstanceOf[BaseColumnFormatRelation].table + case lr: LogicalRelation if lr.relation.isInstanceOf[RowFormatRelation] => + lr.relation.asInstanceOf[RowFormatRelation].table case _ => index.toString() }) } @@ -430,7 +438,7 @@ case class Replacement(table: TABLE, index: INDEX, isPartitioned: Boolean = true def mappedConditions(conditions: Seq[Expression]): Seq[Expression] = conditions.map(Entity.replaceAttribute(_, tableToIndexAttributeMap)) - protected[sources] def replacedPlan(conditions: Seq[Expression]): LogicalPlan = { + private[sources] def replacedPlan(conditions: Seq[Expression]): LogicalPlan = { if (_replacedEntity == null) { val tableConditions = conditions.filter(canEvaluate(_, table)) _replacedEntity = if (tableConditions.isEmpty) { @@ -443,8 +451,7 @@ case class Replacement(table: TABLE, index: INDEX, isPartitioned: Boolean = true } def estimatedSize(conditions: Seq[Expression]): BigInt = - replacedPlan(conditions).statistics.sizeInBytes - + internals.getStatistics(replacedPlan(conditions)).sizeInBytes } /** @@ -458,16 +465,16 @@ case class Replacement(table: TABLE, index: INDEX, isPartitioned: Boolean = true */ case class ReplacementSet(chain: ArrayBuffer[Replacement], conditions: Seq[Expression]) - extends Ordered[ReplacementSet] with PredicateHelper { + extends Ordered[ReplacementSet] with PredicateHelper with SparkSupport { lazy val bestJoinOrder: Seq[Replacement] = { val (part, rep) = chain.partition(_.isPartitioned) // pick minimum number of replicated tables required to fulfill colocated join order. val feasibleJoinPlan = Seq.range(0, chain.length - part.length + 1).flatMap(elem => rep.combinations(elem).map(part ++ _). - flatMap(_.permutations).filter(hasJoinConditions)).filter(_.nonEmpty) + flatMap(_.permutations).filter(hasJoinConditions)).filter(_.nonEmpty) - if(feasibleJoinPlan.isEmpty) { + if (feasibleJoinPlan.isEmpty) { Nil } else { val all = feasibleJoinPlan.sortBy { jo => @@ -478,9 +485,9 @@ case class ReplacementSet(chain: ArrayBuffer[Replacement], } } - lazy val bestPlanEstimatedSize = estimateSize(bestJoinOrder) + private[sql] lazy val bestPlanEstimatedSize = estimateSize(bestJoinOrder) - lazy val bestJoinOrderConditions = joinConditions(bestJoinOrder) + private[sql] lazy val bestJoinOrderConditions = joinConditions(bestJoinOrder) private def joinConditions(joinOrder: Seq[Replacement]) = { val refs = joinOrder.map(_.table.outputSet).reduce(_ ++ _) @@ -497,8 +504,8 @@ case class ReplacementSet(chain: ArrayBuffer[Replacement], } val sz = joinOrder.map(_.replacedPlan(conditions)).zipWithIndex.foldLeft(BigInt(0)) { - case (tot, (table, depth)) if depth == 2 => tot + table.statistics.sizeInBytes - case (tot, (table, depth)) => tot + (table.statistics.sizeInBytes * depth) + case (tot, (table, depth)) if depth == 2 => tot + internals.getStatistics(table).sizeInBytes + case (tot, (table, depth)) => tot + (internals.getStatistics(table).sizeInBytes * depth) } sz @@ -560,7 +567,7 @@ object ExtractFiltersAndInnerJoins extends PredicateHelper { val (plans, conditions) = flattenJoin(left) (plans ++ Seq(right), conditions ++ cond.toSeq) - case plans.logical.Filter(filterCondition, j@Join(left, right, Inner, joinCondition)) => + case plans.logical.Filter(filterCondition, j@Join(_, _, Inner, _)) => val (plans, conditions) = flattenJoin(j) (plans, conditions ++ splitConjunctivePredicates(filterCondition)) @@ -570,12 +577,12 @@ object ExtractFiltersAndInnerJoins extends PredicateHelper { def unapply(plan: LogicalPlan): // tables, joinConditions, filterConditions Option[(Seq[LogicalPlan], Seq[Expression])] = plan match { - case f@plans.logical.Filter(filterCondition, j@Join(_, _, Inner, _)) => + case f@plans.logical.Filter(_, Join(_, _, Inner, _)) => Some(flattenJoin(f)) case j@Join(_, _, Inner, _) => Some(flattenJoin(j)) - case f@plans.logical.Filter(filterCondition, child) => - Some(Seq(child), splitConjunctivePredicates(filterCondition)) + case plans.logical.Filter(filterCondition, child) => + Some((Seq(child), splitConjunctivePredicates(filterCondition))) case _ => None } } @@ -613,11 +620,10 @@ case class PartialPlan(curPlan: LogicalPlan, replaced: Seq[Replacement], outputS finalPlan case (finalPlan, replacement: Replacement) if finalPlan.replaced.contains(replacement) => finalPlan - case (partial, table) if specializedHandling.isDefinedAt(partial, table) => - specializedHandling.lift(partial, table).get + case (partial, table) if specializedHandling.isDefinedAt((partial, table)) => + specializedHandling.lift((partial, table)).get } } - } case class CompletePlan(plan: LogicalPlan, replaced: Seq[Replacement]) extends SubPlan diff --git a/core/src/main/scala/org/apache/spark/sql/sources/SnappyOptimizations.scala b/core/src/main/scala/org/apache/spark/sql/sources/SnappyOptimizations.scala index 8366b208d3..0981ca15db 100644 --- a/core/src/main/scala/org/apache/spark/sql/sources/SnappyOptimizations.scala +++ b/core/src/main/scala/org/apache/spark/sql/sources/SnappyOptimizations.scala @@ -23,7 +23,6 @@ import scala.collection.mutable.ArrayBuffer import io.snappydata.QueryHint._ -import org.apache.spark.sql.SnappySession import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression, PredicateHelper} import org.apache.spark.sql.catalyst.optimizer.ReorderJoin import org.apache.spark.sql.catalyst.plans.Inner @@ -33,15 +32,18 @@ import org.apache.spark.sql.catalyst.{expressions, plans} import org.apache.spark.sql.execution.PartitionedDataSourceScan import org.apache.spark.sql.execution.columnar.impl.{BaseColumnFormatRelation, ColumnFormatRelation, IndexColumnFormatRelation} import org.apache.spark.sql.execution.datasources.LogicalRelation +import org.apache.spark.sql.internal.SnappySessionCatalog import org.apache.spark.sql.sources.Entity.{INDEX_RELATION, TABLE} +import org.apache.spark.sql.{SnappySession, SparkSupport} /** * Replace table with index hint */ -case class ResolveQueryHints(snappySession: SnappySession) extends Rule[LogicalPlan] { +case class ResolveQueryHints(snappySession: SnappySession) + extends Rule[LogicalPlan] with SparkSupport { - private def catalog = snappySession.sessionState.catalog + private def catalog: SnappySessionCatalog = snappySession.snappySessionState.catalog private def analyzer = snappySession.sessionState.analyzer @@ -53,20 +55,22 @@ case class ResolveQueryHints(snappySession: SnappySession) extends Rule[LogicalP return plan } - plan transformUp { - case table@LogicalRelation(colRelation: ColumnFormatRelation, _, _) => - explicitIndexHint.getOrElse(colRelation.table, Some(table)).get - case subQuery@SubqueryAlias(alias, lr: LogicalRelation, _) - if !lr.relation.isInstanceOf[IndexColumnFormatRelation] => - explicitIndexHint.get(alias) match { - case Some(Some(index)) => SubqueryAlias(alias, index, None) - case _ => subQuery + val resolved = internals.logicalPlanResolveUp(plan) { + case lr: LogicalRelation if lr.relation.isInstanceOf[ColumnFormatRelation] => + explicitIndexHint.getOrElse(lr.relation.asInstanceOf[ColumnFormatRelation].table, + Some(lr)).get + case s: SubqueryAlias if s.child.isInstanceOf[LogicalRelation] && + !s.child.asInstanceOf[LogicalRelation].relation.isInstanceOf[IndexColumnFormatRelation] => + explicitIndexHint.get(s.alias) match { + case Some(Some(index)) => internals.newSubqueryAlias(s.alias, index) + case _ => s } - } transformUp { + } + internals.logicalPlanResolveUp(resolved) { case q: LogicalPlan => q transformExpressionsUp { case a: AttributeReference => - q.resolveChildren(Seq(a.qualifier.getOrElse(""), a.name), + q.resolveChildren((if (a.qualifier.isEmpty) "" else a.qualifier.head) :: a.name :: Nil, analyzer.resolver).getOrElse(a) } } @@ -74,7 +78,7 @@ case class ResolveQueryHints(snappySession: SnappySession) extends Rule[LogicalP } private def getIndexHints: mutable.Map[String, Option[LogicalPlan]] = { - val indexHint = Index + val indexHint = Index.toString val hints = snappySession.queryHints if (hints.isEmpty) mutable.Map.empty else hints.asScala.collect { @@ -110,10 +114,6 @@ case class ResolveQueryHints(snappySession: SnappySession) extends Rule[LogicalP case class ResolveIndex(implicit val snappySession: SnappySession) extends Rule[LogicalPlan] with PredicateHelper { - lazy val catalog = snappySession.sessionState.catalog - - lazy val analyzer = snappySession.sessionState.analyzer - private def createColocatedJoins(input: Seq[LogicalPlan], conditions: Seq[Expression], visited: mutable.HashSet[LogicalPlan]): CompletePlan = { @@ -137,8 +137,8 @@ case class ResolveIndex(implicit val snappySession: SnappySession) extends Rule[ val (partitioned, replicates, others) = ((new TableList, new TableList, new TableList) /: input) { case (splitted@(part, rep, _), - l@LogicalRelation(b: PartitionedDataSourceScan, _, _)) => - if (b.partitionColumns.nonEmpty) { + l: LogicalRelation) if l.relation.isInstanceOf[PartitionedDataSourceScan] => + if (l.relation.asInstanceOf[PartitionedDataSourceScan].partitionColumns.nonEmpty) { part += l } else { rep += l @@ -189,7 +189,8 @@ case class ResolveIndex(implicit val snappySession: SnappySession) extends Rule[ val nonColocatedWithFilters = ncf.map(r => RuleUtils.chooseIndexForFilter(r, conditions) .getOrElse(Replacement(r, r))) - val replicatesWithColocated = ReplacementSet(replicates.map(r => Replacement(r, r, false)) ++ + val replicatesWithColocated = ReplacementSet(replicates.map( + r => Replacement(r, r, isPartitioned = false)) ++ (if (colocationGroups.nonEmpty) colocationGroups.head.chain else Nil), conditions) val replicatesWithNonColocatedHavingFilters = nonColocatedWithFilters.map(nc => @@ -223,7 +224,7 @@ case class ResolveIndex(implicit val snappySession: SnappySession) extends Rule[ finalJoinOrder ++= nonColocated.map(r => Replacement(r, r)) } else { - for (i <- 0 to smallerNC) { + for (_ <- 0 to smallerNC) { // pack NC tables first. } } @@ -317,7 +318,7 @@ case class ResolveIndex(implicit val snappySession: SnappySession) extends Rule[ case l :: r :: o if o.isEmpty & RuleUtils.getJoinKeys(l, r, joinConditions).nonEmpty => List(replicates.toList) - case l :: o if o.isEmpty => + case _ :: o if o.isEmpty => List(replicates.toList) case _ => List(List.empty[Entity.TABLE]) } @@ -482,7 +483,7 @@ case class ResolveIndex(implicit val snappySession: SnappySession) extends Rule[ } val hints = snappySession.queryHints if (!hints.isEmpty && hints.asScala.exists { - case (hint, _) => hint.startsWith(Index) && + case (hint, _) => hint.startsWith(Index.toString) && !joinOrderHints.contains(ContinueOptimizations) } || Entity.hasUnresolvedReferences(plan)) { return plan @@ -508,7 +509,7 @@ case class ResolveIndex(implicit val snappySession: SnappySession) extends Rule[ case f: AttributeReference => val newA = newAttributes.find(_.name.equalsIgnoreCase(f.name)). getOrElse(throw new IllegalStateException( - s"Field $f not found in ${newAttributes}")) + s"Field $f not found in $newAttributes")) newAttributesMap ++= Some((f, newA)) } case _ => @@ -521,7 +522,7 @@ case class ResolveIndex(implicit val snappySession: SnappySession) extends Rule[ case q: LogicalPlan => q transformExpressionsUp { case a: AttributeReference => newAttributesMap.find({ - case (tableA, indexA) => tableA.exprId == a.exprId + case (tableA, _) => tableA.exprId == a.exprId }).map({ case (t, i) => i.withQualifier(t.qualifier) }).getOrElse(a) } } diff --git a/core/src/main/scala/org/apache/spark/sql/sources/StoreStrategy.scala b/core/src/main/scala/org/apache/spark/sql/sources/StoreStrategy.scala index 56c724f7a7..da8ec96fd5 100644 --- a/core/src/main/scala/org/apache/spark/sql/sources/StoreStrategy.scala +++ b/core/src/main/scala/org/apache/spark/sql/sources/StoreStrategy.scala @@ -18,20 +18,20 @@ package org.apache.spark.sql.sources import scala.reflect.{ClassTag, classTag} -import org.apache.spark.rdd.RDD -import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, Expression} -import org.apache.spark.sql.catalyst.plans.logical.{InsertIntoTable, LogicalPlan, OverwriteOptions} +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.execution._ import org.apache.spark.sql.execution.command.{ExecutedCommandExec, RunnableCommand} import org.apache.spark.sql.execution.datasources.LogicalRelation -import org.apache.spark.sql.internal.PutIntoColumnTable +import org.apache.spark.sql.hive.SnappySessionState +import org.apache.spark.sql.internal.{InsertIntoPlan, PutIntoColumnTable} import org.apache.spark.sql.types.{DataType, LongType} +import org.apache.spark.sql.{Strategy, _} /** * Support for DML and other operations on external tables. */ -object StoreStrategy extends Strategy { +class StoreStrategy(sessionState: SnappySessionState) extends Strategy with SparkSupport { private def findLogicalRelation[T: ClassTag](table: LogicalPlan): Option[LogicalRelation] = { table.find(_.isInstanceOf[LogicalRelation]) match { @@ -44,13 +44,12 @@ object StoreStrategy extends Strategy { def apply(plan: LogicalPlan): Seq[SparkPlan] = plan match { case p: EncoderPlan[_] => val plan = p.asInstanceOf[EncoderPlan[Any]] - EncoderScanExec(plan.rdd.asInstanceOf[RDD[Any]], - plan.encoder, plan.isFlat, plan.output) :: Nil + EncoderScanExec(plan.rdd, plan.encoder, plan.isFlat, plan.output) :: Nil - case InsertIntoTable(l@LogicalRelation(p: PlanInsertableRelation, - _, _), part, query, overwrite, false) if part.isEmpty => - val preAction = if (overwrite.enabled) () => p.truncate() else () => () - ExecutePlan(p.getInsertPlan(l, planLater(query)), preAction) :: Nil + case i@InsertIntoPlan(l, query, overwrite) => + val preAction = if (overwrite) () => i.relation.truncate() else () => () + val childPlan = new QueryExecution(sessionState.snappySession, query).sparkPlan + ExecutePlan(i.relation.getInsertPlan(l, childPlan), preAction) :: Nil case d@DMLExternalTable(table, cmd) => findLogicalRelation[BaseRelation](table) match { case Some(l) => ExecutedCommandExec(ExternalTableDMLCmd(l, cmd, d.output)) :: Nil @@ -130,30 +129,6 @@ case class PutIntoTable(table: LogicalPlan, child: LogicalPlan) } } -/** - * Unlike Spark's InsertIntoTable this plan provides the count of rows - * inserted as the output. - */ -final class Insert( - table: LogicalPlan, - partition: Map[String, Option[String]], - child: LogicalPlan, - overwrite: OverwriteOptions, - ifNotExists: Boolean) - extends InsertIntoTable(table, partition, child, overwrite, ifNotExists) { - - override def output: Seq[Attribute] = AttributeReference( - "count", LongType)() :: Nil - - override def copy(table: LogicalPlan = table, - partition: Map[String, Option[String]] = partition, - child: LogicalPlan = child, - overwrite: OverwriteOptions = overwrite, - ifNotExists: Boolean = ifNotExists): Insert = { - new Insert(table, partition, child, overwrite, ifNotExists) - } -} - /** * Plan for update of a column or row table. The "table" passed should be * a resolved one (by parser and other callers) else there is ambiguity diff --git a/core/src/main/scala/org/apache/spark/sql/sources/interfaces.scala b/core/src/main/scala/org/apache/spark/sql/sources/interfaces.scala index bac7e2c1a3..13a509cf8f 100644 --- a/core/src/main/scala/org/apache/spark/sql/sources/interfaces.scala +++ b/core/src/main/scala/org/apache/spark/sql/sources/interfaces.scala @@ -18,21 +18,20 @@ package org.apache.spark.sql.sources import java.sql.Connection -import scala.collection.JavaConverters._ - import com.gemstone.gemfire.internal.cache.LocalRegion import com.pivotal.gemfirexd.internal.engine.Misc import io.snappydata.sql.catalog.{RelationInfo, SnappyExternalCatalog} import org.apache.spark.annotation.DeveloperApi +import org.apache.spark.jdbc.{ConnectionConf, ConnectionUtil} import org.apache.spark.rdd.RDD import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression, SortDirection} import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier} import org.apache.spark.sql.execution.SparkPlan +import org.apache.spark.sql.execution.columnar.ExternalStoreUtils.CaseInsensitiveMutableHashMap import org.apache.spark.sql.execution.columnar.impl.BaseColumnFormatRelation import org.apache.spark.sql.execution.datasources.LogicalRelation -import org.apache.spark.sql.execution.datasources.jdbc.{JDBCOptions, JDBCRDD} import org.apache.spark.sql.jdbc.JdbcDialect import org.apache.spark.sql.sources.JdbcExtendedUtils.quotedName import org.apache.spark.sql.types.{StructField, StructType} @@ -305,8 +304,21 @@ trait RowLevelSecurityRelation { enableRowLevelSecurity: Boolean) } +/** + * ::DeveloperApi:: + * Marker interface for data sources that allow for extended schema specification + * in CREATE TABLE (like constraints in RDBMS databases). The schema string is passed + * as [[SnappyExternalCatalog.SCHEMADDL_PROPERTY]] in the relation provider parameters. + */ @DeveloperApi -trait NativeTableRowLevelSecurityRelation extends DestroyRelation with RowLevelSecurityRelation { +trait ExternalSchemaRelationProvider extends RelationProvider { + + def getSchemaString(options: Map[String, String]): Option[String] = + JdbcExtendedUtils.readSplitProperty(SnappyExternalCatalog.SCHEMADDL_PROPERTY, options) +} + +@DeveloperApi +trait SnappyTableRelation extends DestroyRelation with RowLevelSecurityRelation { protected val connFactory: () => Connection @@ -314,6 +326,8 @@ trait NativeTableRowLevelSecurityRelation extends DestroyRelation with RowLevelS def connProperties: ConnectionProperties + def origOptions: CaseInsensitiveMutableHashMap[String] + protected def isRowTable: Boolean val sqlContext: SQLContext @@ -370,7 +384,7 @@ trait NativeTableRowLevelSecurityRelation extends DestroyRelation with RowLevelS } } - protected[this] var _schema: StructType = _ + protected[this] var _schema: StructType = JdbcExtendedUtils.EMPTY_SCHEMA @transient protected[this] var _relationInfoAndRegion: (RelationInfo, Option[LocalRegion]) = _ protected def refreshTableSchema(invalidateCached: Boolean, fetchFromStore: Boolean): Unit = { @@ -381,10 +395,14 @@ trait NativeTableRowLevelSecurityRelation extends DestroyRelation with RowLevelS if (invalidateCached) session.externalCatalog.invalidate(schemaName -> tableName) _relationInfoAndRegion = null if (fetchFromStore) { - _schema = JdbcExtendedUtils.normalizeSchema(JDBCRDD.resolveTable(new JDBCOptions( - connProperties.url, table, connProperties.connProps.asScala.toMap))) + val conn = ConnectionUtil.getPooledConnection(schemaName, new ConnectionConf(connProperties)) + try { + _schema = JdbcExtendedUtils.getTableSchema(schemaName, tableName, conn, Some(session)) + } finally { + conn.close() + } } else { - session.externalCatalog.getTableOption(schemaName, tableName) match { + session.externalCatalog.getTableIfExists(schemaName, tableName) match { case None => _schema = JdbcExtendedUtils.EMPTY_SCHEMA case Some(t) => _schema = t.schema; assert(relationInfoAndRegion ne null) } @@ -460,19 +478,6 @@ trait NativeTableRowLevelSecurityRelation extends DestroyRelation with RowLevelS } } -/** - * ::DeveloperApi:: - * Marker interface for data sources that allow for extended schema specification - * in CREATE TABLE (like constraints in RDBMS databases). The schema string is passed - * as [[SnappyExternalCatalog.SCHEMADDL_PROPERTY]] in the relation provider parameters. - */ -@DeveloperApi -trait ExternalSchemaRelationProvider extends RelationProvider { - - def getSchemaString(options: Map[String, String]): Option[String] = - JdbcExtendedUtils.readSplitProperty(SnappyExternalCatalog.SCHEMADDL_PROPERTY, options) -} - /** * ::DeveloperApi:: * A BaseRelation that can eliminate unneeded columns and filter using selected diff --git a/core/src/main/scala/org/apache/spark/sql/sources/subrules.scala b/core/src/main/scala/org/apache/spark/sql/sources/subrules.scala index c24fd6ef10..18ca172a2a 100644 --- a/core/src/main/scala/org/apache/spark/sql/sources/subrules.scala +++ b/core/src/main/scala/org/apache/spark/sql/sources/subrules.scala @@ -220,12 +220,12 @@ case object ApplyRest extends JoinOrderStrategy { * This doesn't require any alteration to joinOrder as such. */ case object ContinueOptimizations extends JoinOrderStrategy { - override def shortName: String = ""// JOS.ContinueOptimizations + override def shortName: String = ""// HintNames.JoinOrder_ContinueOptimizations } /** * This hint too doesn't require any implementation as such. */ case object IncludeGeneratedPaths extends JoinOrderStrategy { - override def shortName: String = ""// JOS.IncludeGeneratedPaths + override def shortName: String = ""// HintNames.JoinOrder_IncludeGeneratedPaths } diff --git a/core/src/main/scala/org/apache/spark/sql/store/CodeGeneration.scala b/core/src/main/scala/org/apache/spark/sql/store/CodeGeneration.scala index f26b54ccdb..6e3813a8fb 100644 --- a/core/src/main/scala/org/apache/spark/sql/store/CodeGeneration.scala +++ b/core/src/main/scala/org/apache/spark/sql/store/CodeGeneration.scala @@ -28,7 +28,6 @@ import com.pivotal.gemfirexd.internal.engine.distributed.GfxdHeapDataOutputStrea import org.codehaus.janino.CompilerFactory import org.apache.spark.metrics.source.CodegenMetrics -import org.apache.spark.sql.Row import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.encoders.RowEncoder import org.apache.spark.sql.catalyst.expressions.codegen._ @@ -40,6 +39,7 @@ import org.apache.spark.sql.jdbc.JdbcDialect import org.apache.spark.sql.row.SnappyStoreDialect import org.apache.spark.sql.sources.JdbcExtendedUtils import org.apache.spark.sql.types._ +import org.apache.spark.sql.{Row, SparkSupport} import org.apache.spark.unsafe.Platform import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String} import org.apache.spark.{Logging, SparkEnv} @@ -52,7 +52,7 @@ import org.apache.spark.{Logging, SparkEnv} * generation of code string itself only if not found in cache * (and using some other lookup key than the code string) */ -object CodeGeneration extends Logging { +object CodeGeneration extends Logging with SparkSupport { override def logInfo(msg: => String): Unit = super.logInfo(msg) @@ -61,7 +61,7 @@ object CodeGeneration extends Logging { lazy val (codeCacheSize, cacheSize) = { val env = SparkEnv.get val size = if (env ne null) { - env.conf.getInt("spark.sql.codegen.cacheSize", 2000) + env.conf.getInt("spark.sql.codegen.cache.maxEntries", 2000) } else 2000 // don't need as big a cache for other caches (size, size >>> 2) @@ -107,7 +107,10 @@ object CodeGeneration extends Logging { CodegenMetrics.METRIC_SOURCE_CODE_SIZE.update(code.body.length) CodegenMetrics.METRIC_COMPILATION_TIME.update(timeMs.toLong) logInfo(s"Local code for ${key.name} generated in $timeMs ms") - (result.asInstanceOf[GeneratedClass], references) + result match { + case (c, _) => (c.asInstanceOf[GeneratedClass], references) + case _ => (result.asInstanceOf[GeneratedClass], references) + } } }) @@ -145,31 +148,31 @@ object CodeGeneration extends Logging { val serArrayClass = classOf[SerializedArray].getName val serMapClass = classOf[SerializedMap].getName val serRowClass = classOf[SerializedRow].getName + val evValue = internals.exprCodeValue(ev) val nonNullCode = Utils.getSQLDataType(dataType) match { - case IntegerType => s"$stmt.setInt(${col + 1}, ${ev.value});" - case LongType => s"$stmt.setLong(${col + 1}, ${ev.value});" - case DoubleType => s"$stmt.setDouble(${col + 1}, ${ev.value});" - case FloatType => s"$stmt.setFloat(${col + 1}, ${ev.value});" - case ShortType => s"$stmt.setInt(${col + 1}, ${ev.value});" - case ByteType => s"$stmt.setInt(${col + 1}, ${ev.value});" - case BooleanType => s"$stmt.setBoolean(${col + 1}, ${ev.value});" - case StringType => s"$stmt.setString(${col + 1}, ${ev.value}.toString());" - case BinaryType => s"$stmt.setBytes(${col + 1}, ${ev.value});" + case IntegerType => s"$stmt.setInt(${col + 1}, $evValue);" + case LongType => s"$stmt.setLong(${col + 1}, $evValue);" + case DoubleType => s"$stmt.setDouble(${col + 1}, $evValue);" + case FloatType => s"$stmt.setFloat(${col + 1}, $evValue);" + case ShortType => s"$stmt.setInt(${col + 1}, $evValue);" + case ByteType => s"$stmt.setInt(${col + 1}, $evValue);" + case BooleanType => s"$stmt.setBoolean(${col + 1}, $evValue);" + case StringType => s"$stmt.setString(${col + 1}, $evValue.toString());" + case BinaryType => s"$stmt.setBytes(${col + 1}, $evValue);" case TimestampType => - s"$stmt.setTimestamp(${col + 1}, $timeUtilsClass.toJavaTimestamp(${ev.value}));" + s"$stmt.setTimestamp(${col + 1}, $timeUtilsClass.toJavaTimestamp($evValue));" case DateType => - s"$stmt.setDate(${col + 1}, $timeUtilsClass.toJavaDate(${ev.value}));" + s"$stmt.setDate(${col + 1}, $timeUtilsClass.toJavaDate($evValue));" case _: DecimalType => - s"$stmt.setBigDecimal(${col + 1}, ${ev.value}.toJavaBigDecimal());" + s"$stmt.setBigDecimal(${col + 1}, $evValue.toJavaBigDecimal());" case a: ArrayType => - val encoderVar = ctx.freshName("encoderObj") val arr = ctx.freshName("arr") val encoder = ctx.freshName("encoder") val cursor = ctx.freshName("cursor") - ctx.addMutableState(encoderClass, encoderVar, - s"$encoderVar = new $encoderClass();") + val encoderVar = internals.addClassField(ctx, encoderClass, "encoderObj", + v => s"$v = new $encoderClass();", forceInline = true) s""" - |final ArrayData $arr = ${ev.value}; + |final ArrayData $arr = $evValue; |if ($arr instanceof $serArrayClass) { | $stmt.setBytes(${col + 1}, (($serArrayClass)$arr).toBytes()); |} else { @@ -182,14 +185,13 @@ object CodeGeneration extends Logging { |} """.stripMargin case m: MapType => - val encoderVar = ctx.freshName("encoderObj") val map = ctx.freshName("mapValue") val encoder = ctx.freshName("encoder") val cursor = ctx.freshName("cursor") - ctx.addMutableState(encoderClass, encoderVar, - s"$encoderVar = new $encoderClass();") + val encoderVar = internals.addClassField(ctx, encoderClass, "encoderObj", + v => s"$v = new $encoderClass();", forceInline = true) s""" - |final MapData $map = ${ev.value}; + |final MapData $map = $evValue; |if ($map instanceof $serMapClass) { | $stmt.setBytes(${col + 1}, (($serMapClass)$map).toBytes()); |} else { @@ -201,14 +203,13 @@ object CodeGeneration extends Logging { |} """.stripMargin case s: StructType => - val encoderVar = ctx.freshName("encoderObj") val struct = ctx.freshName("structValue") val encoder = ctx.freshName("encoder") val cursor = ctx.freshName("cursor") - ctx.addMutableState(encoderClass, encoderVar, - s"$encoderVar = new $encoderClass();") + val encoderVar = internals.addClassField(ctx, encoderClass, "encoderObj", + v => s"$v = new $encoderClass();", forceInline = true) s""" - |final InternalRow $struct = ${ev.value}; + |final InternalRow $struct = $evValue; |if ($struct instanceof $serRowClass) { | $stmt.setBytes(${col + 1}, (($serRowClass)$struct).toBytes()); |} else { @@ -221,17 +222,18 @@ object CodeGeneration extends Logging { |} """.stripMargin case _ => - s"$stmt.setObject(${col + 1}, ${ev.value});" + s"$stmt.setObject(${col + 1}, $evValue);" } - val code = if (ev.code == "") "" + val evCode = ev.code.toString + val code = if (evCode.isEmpty) "" else { - val c = s"${ev.code}\n" - ev.code = "" + val c = s"$evCode\n" + internals.resetCode(ev) c } val jdbcType = JdbcExtendedUtils.getJdbcType(NullType, null, dialect).jdbcNullType s""" - |${code}if (${ev.isNull}) { + |${code}if (${internals.exprCodeIsNull(ev)}) { | $stmt.setNull(${col + 1}, $jdbcType); |} else { | $nonNullCode @@ -251,8 +253,9 @@ object CodeGeneration extends Logging { def getRowSetterFragment(schema: Array[StructField], dialect: JdbcDialect, row: String, stmt: String, schemaTerm: String, ctx: CodegenContext): String = { - val rowInput = (col: Int) => ExprCode("", s"$row.isNullAt($col)", - ctx.getValue(row, schema(col).dataType, Integer.toString(col))) + val rowInput = (col: Int) => internals.newExprCode(code = "", isNull = s"$row.isNullAt($col)", + value = internals.getValue(row, schema(col).dataType, Integer.toString(col), ctx), + schema(col).dataType) genStmtSetters(schema, dialect, rowInput, stmt, schemaTerm, ctx) } @@ -281,13 +284,18 @@ object CodeGeneration extends Logging { val evaluator = new CompilerFactory().newScriptEvaluator() evaluator.setClassName("io.snappydata.execute.GeneratedEvaluation") evaluator.setParentClassLoader(getClass.getClassLoader) - evaluator.setDefaultImports(defaultImports) + evaluator.setDefaultImports(defaultImports: _*) val separator = "\n " - val varDeclarations = ctx.mutableStates.map { case (javaType, name, init) => - s"$javaType $name;$separator${init.replace("this.", "")}" + val mutableStates = internals.getInlinedClassFields(ctx) + val varDeclarations = mutableStates._1.map { case (javaType, name) => + s"$javaType $name;" + } + val initVars = mutableStates._2.map { init => + init.replace("this.", "") } val expression = s""" ${varDeclarations.mkString(separator)} + ${initVars.mkString(separator)} int $rowCount = 0; int $result = 0; while ($rows.hasNext()) { @@ -329,13 +337,18 @@ object CodeGeneration extends Logging { val evaluator = new CompilerFactory().newScriptEvaluator() evaluator.setClassName("io.snappydata.execute.GeneratedIndexEvaluation") evaluator.setParentClassLoader(getClass.getClassLoader) - evaluator.setDefaultImports(defaultImports) + evaluator.setDefaultImports(defaultImports: _*) val separator = "\n " - val varDeclarations = ctx.mutableStates.map { case (javaType, name, init) => - s"$javaType $name;$separator${init.replace("this.", "")}" + val mutableStates = internals.getInlinedClassFields(ctx) + val varDeclarations = mutableStates._1.map { case (javaType, name) => + s"$javaType $name;" + } + val initVars = mutableStates._2.map { init => + init.replace("this.", "") } val expression = s""" ${varDeclarations.mkString(separator)} + ${initVars.mkString(separator)} $code stmt.addBatch(); return 1;""" @@ -421,20 +434,25 @@ object CodeGeneration extends Logging { val evaluator = new CompilerFactory().newScriptEvaluator() evaluator.setClassName("io.snappydata.execute.GeneratedSerialization") evaluator.setParentClassLoader(getClass.getClassLoader) - evaluator.setDefaultImports(Array(classOf[Platform].getName, + evaluator.setDefaultImports(classOf[Platform].getName, classOf[InternalRow].getName, classOf[UTF8String].getName, classOf[Decimal].getName, classOf[CalendarInterval].getName, classOf[ArrayData].getName, classOf[MapData].getName, - classOf[InternalDataSerializer].getName)) + classOf[InternalDataSerializer].getName) val separator = "\n " - val varDeclarations = ctx.mutableStates.map { case (javaType, name, init) => - s"$javaType $name;$separator${init.replace("this.", "")}" + val mutableStates = internals.getInlinedClassFields(ctx) + val varDeclarations = mutableStates._1.map { case (javaType, name) => + s"$javaType $name;" + } + val initVars = mutableStates._2.map { init => + init.replace("this.", "") } val expression = s""" ${varDeclarations.mkString(separator)} + ${initVars.mkString(separator)} $typeConversion""" logDebug(s"DEBUG: For complex type=$dataType, generated code=$expression") diff --git a/core/src/main/scala/org/apache/spark/sql/streaming/LogicalDStreamPlan.scala b/core/src/main/scala/org/apache/spark/sql/streaming/LogicalDStreamPlan.scala index 61f91ff5d1..9b80f9e49f 100644 --- a/core/src/main/scala/org/apache/spark/sql/streaming/LogicalDStreamPlan.scala +++ b/core/src/main/scala/org/apache/spark/sql/streaming/LogicalDStreamPlan.scala @@ -18,26 +18,24 @@ package org.apache.spark.sql.streaming import scala.collection.immutable +import org.apache.spark.sql.SparkSupport import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis.MultiInstanceRelation import org.apache.spark.sql.catalyst.expressions.Attribute -import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Statistics} +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.streaming.SnappyStreamingContext import org.apache.spark.streaming.dstream.DStream - -case class LogicalDStreamPlan(output: Seq[Attribute], +abstract case class LogicalDStreamPlan(output: Seq[Attribute], stream: DStream[InternalRow]) (val streamingSnappy: SnappyStreamingContext) - extends LogicalPlan with MultiInstanceRelation { + extends LogicalPlan with MultiInstanceRelation with SparkSupport { - def newInstance(): LogicalDStreamPlan = - LogicalDStreamPlan(output.map(_.newInstance()), - stream)(streamingSnappy).asInstanceOf[this.type] + override protected def otherCopyArgs: Seq[AnyRef] = streamingSnappy :: Nil - @transient override lazy val statistics = Statistics( - sizeInBytes = BigInt(streamingSnappy.snappySession.sessionState.conf.defaultSizeInBytes) - ) + def newInstance(): LogicalDStreamPlan = + internals.newLogicalDStreamPlan(output.map(_.newInstance()), + stream, streamingSnappy).asInstanceOf[this.type] def children: immutable.Nil.type = Nil } diff --git a/core/src/main/scala/org/apache/spark/sql/streaming/SchemaDStream.scala b/core/src/main/scala/org/apache/spark/sql/streaming/SchemaDStream.scala index 5d57de35a2..4f920f30bd 100644 --- a/core/src/main/scala/org/apache/spark/sql/streaming/SchemaDStream.scala +++ b/core/src/main/scala/org/apache/spark/sql/streaming/SchemaDStream.scala @@ -24,9 +24,8 @@ import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.collection.WrappedInternalRow import org.apache.spark.sql.execution._ -import org.apache.spark.sql.execution.exchange.ShuffleExchange import org.apache.spark.sql.types.StructType -import org.apache.spark.sql.{DataFrame, Row, SnappySession} +import org.apache.spark.sql.{DataFrame, Row, SnappySession, SparkSupport} import org.apache.spark.storage.StorageLevel import org.apache.spark.streaming.dstream.DStream import org.apache.spark.streaming.{Duration, SnappyStreamingContext, Time} @@ -46,7 +45,7 @@ import org.apache.spark.streaming.{Duration, SnappyStreamingContext, Time} */ class SchemaDStream(@transient val snsc: SnappyStreamingContext, @transient val queryExecution: QueryExecution) - extends DStream[Row](snsc) { + extends DStream[Row](snsc) with SparkSupport { @transient private val snappySession: SnappySession = snsc.snappySession @@ -290,7 +289,7 @@ class SchemaDStream(@transient val snsc: SnappyStreamingContext, } private val _cachedField = { - val f = classOf[ShuffleExchange].getDeclaredFields.find( + val f = internals.classOfShuffleExchange().getDeclaredFields.find( _.getName.contains("cachedShuffleRDD")).get f.setAccessible(true) f @@ -298,7 +297,7 @@ class SchemaDStream(@transient val snsc: SnappyStreamingContext, private def executionPlan: SparkPlan = { queryExecution.executedPlan.foreach { - case s: ShuffleExchange => _cachedField.set(s, null) + case s if internals.isShuffleExchange(s) => _cachedField.set(s, null) case _ => } queryExecution.executedPlan diff --git a/core/src/main/scala/org/apache/spark/sql/streaming/SnappySinkCallback.scala b/core/src/main/scala/org/apache/spark/sql/streaming/SnappySinkCallback.scala index 9894531ae7..be90fdc634 100644 --- a/core/src/main/scala/org/apache/spark/sql/streaming/SnappySinkCallback.scala +++ b/core/src/main/scala/org/apache/spark/sql/streaming/SnappySinkCallback.scala @@ -25,14 +25,16 @@ import io.snappydata.Property._ import io.snappydata.util.ServiceUtils import org.apache.spark.Logging +import org.apache.spark.sql._ import org.apache.spark.sql.execution.CatalogStaleException import org.apache.spark.sql.execution.columnar.ExternalStoreUtils +import org.apache.spark.sql.execution.datasources.LogicalRelation import org.apache.spark.sql.execution.streaming.Sink -import org.apache.spark.sql.sources.{DataSourceRegister, StreamSinkProvider} +import org.apache.spark.sql.row.JDBCMutableRelation +import org.apache.spark.sql.sources.{DataSourceRegister, JdbcExtendedUtils, StreamSinkProvider} import org.apache.spark.sql.streaming.SnappyStoreSinkProvider.EventType._ import org.apache.spark.sql.streaming.SnappyStoreSinkProvider._ import org.apache.spark.sql.types.StructType -import org.apache.spark.sql.{DataFrame, Dataset, Row, SnappyContext, SnappySession, _} import org.apache.spark.storage.StorageLevel import org.apache.spark.util.Utils @@ -101,7 +103,7 @@ class SnappyStoreSinkProvider extends StreamSinkProvider with DataSourceRegister private def createSinkStateTableIfNotExist(sqlContext: SQLContext, stateTableSchema: Option[String]) = { - sqlContext.asInstanceOf[SnappyContext].snappySession.sql(s"create table if not exists" + + sqlContext.sql(s"create table if not exists" + s" ${stateTable(stateTableSchema)} (" + s" $QUERY_ID_COLUMN varchar(200)," + s" $BATCH_ID_COLUMN long, " + @@ -137,8 +139,8 @@ private[streaming] object SnappyStoreSinkProvider { .getOrElse(SINK_STATE_TABLE) } -case class SnappyStoreSink(snappySession: SnappySession, - parameters: Map[String, String], sinkCallback: SnappySinkCallback) extends Sink with Logging { +case class SnappyStoreSink(snappySession: SnappySession, parameters: Map[String, String], + sinkCallback: SnappySinkCallback) extends Sink with Logging with SparkSupport { override def addBatch(batchId: Long, data: Dataset[Row]): Unit = { val message = s"queryName must be specified for ${SnappyContext.SNAPPY_SINK_NAME}." @@ -189,16 +191,18 @@ case class SnappyStoreSink(snappySession: SnappySession, private def isPossibleDuplicate(queryName: String, batchId: Long): Boolean = { val stateTableSchema = parameters.get(STATE_TABLE_SCHEMA) - val updated = snappySession.sql(s"update ${stateTable(stateTableSchema)} " + + val relation = snappySession.sessionCatalog.resolveRelation( + snappySession.tableIdentifier(stateTable(stateTableSchema))) + .asInstanceOf[LogicalRelation].relation.asInstanceOf[JDBCMutableRelation] + val updated = relation.executeUpdate(s"update ${stateTable(stateTableSchema)} " + s"set $BATCH_ID_COLUMN=$batchId where $QUERY_ID_COLUMN='$queryName' " + - s"and $BATCH_ID_COLUMN != $batchId") - .collect()(0).getAs("count").asInstanceOf[Long] + s"and $BATCH_ID_COLUMN != $batchId", + JdbcExtendedUtils.toUpperCase(snappySession.getCurrentSchema)) - // TODO: use JDBC connection here var posDup = false if (updated == 0) { try { - snappySession.insert(stateTable(stateTableSchema), Row(queryName, batchId)) + relation.insert(Row(queryName, batchId) :: Nil) posDup = false } catch { @@ -218,7 +222,7 @@ case class SnappyStoreSink(snappySession: SnappySession, * for a detailed discussion. */ private def convert(ds: DataFrame): DataFrame = { - snappySession.internalCreateDataFrame( + internals.internalCreateDataFrame(snappySession, ds.queryExecution.toRdd, StructType(ds.schema.fields)) } @@ -342,4 +346,4 @@ class DefaultSnappySinkCallback extends SnappySinkCallback with Logging { } } } -} \ No newline at end of file +} diff --git a/core/src/main/scala/org/apache/spark/sql/streaming/StreamBaseRelation.scala b/core/src/main/scala/org/apache/spark/sql/streaming/StreamBaseRelation.scala index ed067c6665..32cf8f28b5 100644 --- a/core/src/main/scala/org/apache/spark/sql/streaming/StreamBaseRelation.scala +++ b/core/src/main/scala/org/apache/spark/sql/streaming/StreamBaseRelation.scala @@ -21,24 +21,23 @@ import scala.collection.mutable import io.snappydata.sql.catalog.SnappyExternalCatalog import org.apache.spark.rdd.{EmptyRDD, RDD} -import org.apache.spark.sql.Row import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap import org.apache.spark.sql.collection.Utils import org.apache.spark.sql.sources._ +import org.apache.spark.sql.{Row, SparkSupport} import org.apache.spark.storage.StorageLevel import org.apache.spark.streaming.dstream.{DStream, InputDStream, ReceiverInputDStream} import org.apache.spark.streaming.{SnappyStreamingContext, StreamUtils, StreamingContextState, Time} import org.apache.spark.{Logging, util} -abstract class StreamBaseRelation(opts: Map[String, String]) - extends DestroyRelation with StreamPlan with TableScan with Serializable with Logging { +abstract class StreamBaseRelation(opts: Map[String, String]) extends DestroyRelation + with StreamPlan with TableScan with Serializable with Logging with SparkSupport { final def context: SnappyStreamingContext = SnappyStreamingContext.getInstance().getOrElse( throw new IllegalStateException("No initialized streaming context")) - protected val options = new CaseInsensitiveMap(opts) + protected val options: Map[String, String] = internals.newCaseInsensitiveMap(opts) @transient val tableName = options(SnappyExternalCatalog.DBTABLE_PROPERTY) diff --git a/core/src/main/scala/org/apache/spark/sql/streaming/StreamSqlHelper.scala b/core/src/main/scala/org/apache/spark/sql/streaming/StreamSqlHelper.scala index 31dd9c8005..4afcc25789 100644 --- a/core/src/main/scala/org/apache/spark/sql/streaming/StreamSqlHelper.scala +++ b/core/src/main/scala/org/apache/spark/sql/streaming/StreamSqlHelper.scala @@ -24,12 +24,12 @@ import org.apache.spark.sql.catalyst.{InternalRow, JavaTypeInference} import org.apache.spark.sql.execution.datasources.LogicalRelation import org.apache.spark.sql.sources.SchemaRelationProvider import org.apache.spark.sql.types.StructType -import org.apache.spark.sql.{AnalysisException, Row} +import org.apache.spark.sql.{AnalysisException, Row, SparkSupport} import org.apache.spark.streaming.SnappyStreamingContext import org.apache.spark.streaming.api.java.JavaDStream import org.apache.spark.streaming.dstream.DStream -object StreamSqlHelper { +object StreamSqlHelper extends SparkSupport { def clearStreams(): Unit = { StreamBaseRelation.clearStreams() @@ -46,10 +46,12 @@ object StreamSqlHelper { } def getSchemaDStream(ssc: SnappyStreamingContext, tableName: String): SchemaDStream = { - val catalog = ssc.snappySession.sessionState.catalog + val catalog = ssc.snappySession.snappySessionState.catalog catalog.resolveRelation(ssc.snappySession.tableIdentifier(tableName)) match { - case LogicalRelation(sr: StreamPlan, _, _) => new SchemaDStream(ssc, - LogicalDStreamPlan(sr.schema.toAttributes, sr.rowStream)(ssc)) + case lr: LogicalRelation if lr.relation.isInstanceOf[StreamPlan] => + val sr = lr.relation.asInstanceOf[StreamPlan] + new SchemaDStream(ssc, internals.newLogicalDStreamPlan( + sr.schema.toAttributes, sr.rowStream, ssc)) case _ => throw new AnalysisException(s"Table $tableName not a stream table") } @@ -62,16 +64,16 @@ object StreamSqlHelper { stream: DStream[A]): SchemaDStream = { val encoder = ExpressionEncoder[A]() val schema = encoder.schema - val logicalPlan = LogicalDStreamPlan(schema.toAttributes, - stream.map(encoder.toRow(_).copy()))(ssc) + val logicalPlan = internals.newLogicalDStreamPlan(schema.toAttributes, + stream.map(encoder.toRow(_).copy()), ssc) new SchemaDStream(ssc, logicalPlan) } def createSchemaDStream(ssc: SnappyStreamingContext, rowStream: DStream[Row], schema: StructType): SchemaDStream = { val encoder = RowEncoder(schema) - val logicalPlan = LogicalDStreamPlan(schema.toAttributes, - rowStream.map(encoder.toRow(_).copy()))(ssc) + val logicalPlan = internals.newLogicalDStreamPlan(schema.toAttributes, + rowStream.map(encoder.toRow(_).copy()), ssc) new SchemaDStream(ssc, logicalPlan) } @@ -79,8 +81,8 @@ object StreamSqlHelper { rowStream: JavaDStream[_], beanClass: Class[_]): SchemaDStream = { val encoder = ExpressionEncoder.javaBean(beanClass.asInstanceOf[Class[Any]]) val schema = encoder.schema - val logicalPlan = LogicalDStreamPlan(schema.toAttributes, - rowStream.dstream.map(encoder.toRow(_).copy()))(ssc) + val logicalPlan = internals.newLogicalDStreamPlan(schema.toAttributes, + rowStream.dstream.map(encoder.toRow(_).copy()), ssc) new SchemaDStream(ssc, logicalPlan) } } diff --git a/core/src/main/scala/org/apache/spark/sql/types/TypeUtilities.scala b/core/src/main/scala/org/apache/spark/sql/types/TypeUtilities.scala index 2478cd6ad3..c98b65d2ff 100644 --- a/core/src/main/scala/org/apache/spark/sql/types/TypeUtilities.scala +++ b/core/src/main/scala/org/apache/spark/sql/types/TypeUtilities.scala @@ -23,9 +23,15 @@ import scala.reflect.runtime.universe._ import com.esotericsoftware.kryo.Kryo import com.esotericsoftware.kryo.io.{Input, Output} +import com.pivotal.gemfirexd.internal.engine.store.{AbstractCompactExecRow, RowFormatter} +import com.pivotal.gemfirexd.internal.iapi.sql.dictionary.ColumnDescriptor +import com.pivotal.gemfirexd.internal.impl.jdbc.Util +import com.pivotal.gemfirexd.internal.shared.common.StoredFormatIds +import com.pivotal.gemfirexd.internal.shared.common.reference.SQLState import org.apache.spark.rdd.RDD import org.apache.spark.sql.execution.CodegenSupport +import org.apache.spark.unsafe.Platform import org.apache.spark.unsafe.types.UTF8String @@ -130,6 +136,63 @@ object TypeUtilities { } } + private def assertCharType(cd: ColumnDescriptor): Unit = { + cd.columnType.getTypeId.getTypeFormatId match { + case StoredFormatIds.CHAR_TYPE_ID | StoredFormatIds.LONGVARCHAR_TYPE_ID | + StoredFormatIds.VARCHAR_TYPE_ID | StoredFormatIds.CLOB_TYPE_ID => + case _ => throw Util.generateCsSQLException(SQLState.LANG_FORMAT_EXCEPTION, + "UTF8String", cd.getColumnName) + } + } + + private def readUTF8String(rf: RowFormatter, index: Int, bytes: Array[Byte]): UTF8String = { + val cd = rf.columns(index) + val offsetFromMap = rf.positionMap(index) + val offsetAndWidth = rf.getOffsetAndWidth(index, bytes, offsetFromMap, cd, false) + if (offsetAndWidth >= 0) { + val columnWidth = offsetAndWidth.toInt + val offset = (offsetAndWidth >>> Integer.SIZE).toInt + assertCharType(cd) + // TODO: SW: SQLChar should be full UTF8 else below is broken for > 3-character UTF8 + UTF8String.fromAddress(bytes, Platform.BYTE_ARRAY_OFFSET + offset, columnWidth) + } else { + if (offsetAndWidth == RowFormatter.OFFSET_AND_WIDTH_IS_NULL) null + else { + assert(offsetAndWidth == RowFormatter.OFFSET_AND_WIDTH_IS_DEFAULT) + val defaultBytes = cd.columnDefaultBytes + if (defaultBytes ne null) { + UTF8String.fromAddress(defaultBytes, Platform.BYTE_ARRAY_OFFSET, defaultBytes.length) + } else null + } + } + } + + private def readUTF8String(rf: RowFormatter, index: Int, + byteArrays: Array[Array[Byte]]): UTF8String = { + val cd = rf.columns(index) + if (!cd.isLob) { + readUTF8String(rf, index, byteArrays(0)) + } else { + val offsetFromMap = rf.positionMap(index) + val bytes = + if (offsetFromMap != 0) byteArrays(offsetFromMap) else cd.columnDefaultBytes + if (bytes ne null) { + assertCharType(cd) + UTF8String.fromAddress(bytes, Platform.BYTE_ARRAY_OFFSET, bytes.length) + } else null + } + } + + def readUTF8String(row: AbstractCompactExecRow, index: Int): UTF8String = { + val rf = row.getRowFormatter + row.getBaseByteSource match { + case bytes: Array[Byte] => readUTF8String(rf, index, bytes) + case byteArrays: Array[Array[Byte]] => readUTF8String(rf, index, byteArrays) + case s => throw new UnsupportedOperationException( + s"readUTF8String(AbstractCompactExecRow): unexpected source: $s") + } + } + val mathContextCache: Array[MathContext] = Array.tabulate[MathContext]( DecimalType.MAX_PRECISION)(i => new MathContext(i + 1)) } diff --git a/core/src/main/scala/org/apache/spark/streaming/SnappyStreamingContext.scala b/core/src/main/scala/org/apache/spark/streaming/SnappyStreamingContext.scala index f6d143a9f3..11ab3be83a 100644 --- a/core/src/main/scala/org/apache/spark/streaming/SnappyStreamingContext.scala +++ b/core/src/main/scala/org/apache/spark/streaming/SnappyStreamingContext.scala @@ -79,7 +79,8 @@ class SnappyStreamingContext protected[spark]( /** * Create a SnappyStreamingContext using an existing SparkContext. - * @param sparkContext existing SparkContext + * + * @param sparkContext existing SparkContext * @param batchDuration the time interval at which streaming data will be divided into batches */ def this(sparkContext: SparkContext, batchDuration: Duration) = { @@ -93,7 +94,8 @@ class SnappyStreamingContext protected[spark]( /** * Create a SnappyStreamingContext by providing the configuration necessary * for a new SparkContext. - * @param conf a org.apache.spark.SparkConf object specifying Spark parameters + * + * @param conf a org.apache.spark.SparkConf object specifying Spark parameters * @param batchDuration the time interval at which streaming data will be divided into batches */ def this(conf: SparkConf, batchDuration: Duration) = { @@ -103,7 +105,8 @@ class SnappyStreamingContext protected[spark]( /** * Recreate a SnappyStreamingContext from a checkpoint file. - * @param path Path to the directory that was specified as the checkpoint directory + * + * @param path Path to the directory that was specified as the checkpoint directory * @param hadoopConf Optional, configuration object if necessary for reading from * HDFS compatible filesystems */ @@ -112,13 +115,15 @@ class SnappyStreamingContext protected[spark]( /** * Recreate a SnappyStreamingContext from a checkpoint file. + * * @param path Path to the directory that was specified as the checkpoint directory */ def this(path: String) = this(path, SparkHadoopUtil.get.conf) /** * Recreate a SnappyStreamingContext from a checkpoint file using an existing SparkContext. - * @param path Path to the directory that was specified as the checkpoint directory + * + * @param path Path to the directory that was specified as the checkpoint directory * @param sparkContext Existing SparkContext */ def this(path: String, sparkContext: SparkContext) = { @@ -139,7 +144,7 @@ class SnappyStreamingContext protected[spark]( if (getState() == StreamingContextState.INITIALIZED) { registerStreamTables() // register population of AQP tables from stream tables - snappySession.snappyContextFunctions.aqpTablePopulator(snappySession) + snappySession.contextFunctions.aqpTablePopulator() } SnappyStreamingContext.setActiveContext(self) super.start() @@ -148,7 +153,7 @@ class SnappyStreamingContext protected[spark]( def registerStreamTables(): Unit = { // register dummy output transformations for the stream tables // so that the streaming context starts - snappySession.sessionState.catalog.getDataSourceRelations[StreamBaseRelation]( + snappySession.snappySessionState.catalog.getDataSourceRelations[StreamBaseRelation]( CatalogObjectType.Stream).foreach(_.rowStream.foreachRDD(_ => Unit)) } @@ -330,11 +335,11 @@ object SnappyStreamingContext extends Logging { creatingFunc: () => SnappyStreamingContext, hadoopConf: Configuration = SparkHadoopUtil.get.conf, createOnError: Boolean = false - ): SnappyStreamingContext = { + ): SnappyStreamingContext = { val checkpointOption = CheckpointReader.read( checkpointPath, new SparkConf(), hadoopConf, createOnError) checkpointOption.map(new SnappyStreamingContext(null, _, null)). - getOrElse(creatingFunc()) + getOrElse(creatingFunc()) } /** @@ -354,19 +359,17 @@ object SnappyStreamingContext extends Logging { * thrown on error. */ def getOrCreateWithUseCredential( - checkpointPath: String, - creatingFunc: () => SnappyStreamingContext, - currentSession: SnappySession, - hadoopConf: Configuration = SparkHadoopUtil.get.conf, - createOnError: Boolean = false - ): SnappyStreamingContext = { + checkpointPath: String, + creatingFunc: () => SnappyStreamingContext, + currentSession: SnappySession, + hadoopConf: Configuration = SparkHadoopUtil.get.conf, + createOnError: Boolean = false + ): SnappyStreamingContext = { val checkpointOption = CheckpointReader.read( checkpointPath, new SparkConf(), hadoopConf, createOnError) checkpointOption.map(new SnappyStreamingContext(null, _, null, None, Option(currentSession))). - getOrElse(creatingFunc()) - + getOrElse(creatingFunc()) } - } @@ -376,8 +379,7 @@ private class SnappyStreamingContextPythonHelper { */ def tryRecoverFromCheckpoint(checkpointPath: String): Option[SnappyStreamingContext] = { val checkpointOption = CheckpointReader.read( - checkpointPath, new SparkConf(), SparkHadoopUtil.get.conf, - ignoreReadError = false) + checkpointPath, new SparkConf(), SparkHadoopUtil.get.conf) checkpointOption.map(new SnappyStreamingContext(null, _, null)) } } diff --git a/core/src/test/scala/io/snappydata/ConcurrentOpsTests.scala b/core/src/test/scala/io/snappydata/ConcurrentOpsTests.scala index ed730247ee..8207ad05bc 100644 --- a/core/src/test/scala/io/snappydata/ConcurrentOpsTests.scala +++ b/core/src/test/scala/io/snappydata/ConcurrentOpsTests.scala @@ -29,6 +29,7 @@ import scala.concurrent.{Await, Future} object ConcurrentOpsTests extends Assertions with Logging { + private val maxWait = Duration("180s") def testSimpleLockInsert(session: SnappySession): Unit = { val tableName = "ColumnTable" @@ -190,10 +191,10 @@ object ConcurrentOpsTests extends Assertions with Logging { } val putTasks = Array.fill(10)(doPut()) - putTasks.foreach(Await.result(_, Duration.Inf)) + putTasks.foreach(Await.result(_, maxWait)) val putTasks2 = Array.fill(5)(doPut()) - putTasks2.foreach(Await.result(_, Duration.Inf)) + putTasks2.foreach(Await.result(_, maxWait)) val result = snc.sql("SELECT * FROM " + tableName) val r2 = result.collect @@ -233,7 +234,7 @@ object ConcurrentOpsTests extends Assertions with Logging { } val putTasks = Array.fill(10)(doUpdate()) - putTasks.foreach(Await.result(_, Duration.Inf)) + putTasks.foreach(Await.result(_, maxWait)) val r3 = result.collect assert(r3.length == 2000) @@ -271,7 +272,7 @@ object ConcurrentOpsTests extends Assertions with Logging { } val putTasks = Array.fill(10)(doDelete()) - putTasks.foreach(Await.result(_, Duration.Inf)) + putTasks.foreach(Await.result(_, maxWait)) val r3 = session.sql("SELECT * FROM " + tableName).collect() assert(r3.length == 0) @@ -312,8 +313,8 @@ object ConcurrentOpsTests extends Assertions with Logging { val putTasks = Array.fill(5)(doPut()) val putTasks2 = Array.fill(5)(doUpdate()) - putTasks.foreach(Await.result(_, Duration.Inf)) - putTasks2.foreach(Await.result(_, Duration.Inf)) + putTasks.foreach(Await.result(_, maxWait)) + putTasks2.foreach(Await.result(_, maxWait)) val result = session.sql("SELECT * FROM " + tableName) val r2 = result.collect @@ -381,10 +382,10 @@ object ConcurrentOpsTests extends Assertions with Logging { val updateTasks = Array.fill(5)(doUpdate()) val deleteTasks = Array.fill(5)(doDelete()) - putTasks.foreach(Await.result(_, Duration.Inf)) - insertTasks.foreach(Await.result(_, Duration.Inf)) - deleteTasks.foreach(Await.result(_, Duration.Inf)) - updateTasks.foreach(Await.result(_, Duration.Inf)) + putTasks.foreach(Await.result(_, maxWait)) + insertTasks.foreach(Await.result(_, maxWait)) + deleteTasks.foreach(Await.result(_, maxWait)) + updateTasks.foreach(Await.result(_, maxWait)) val result = session.sql("SELECT * FROM " + tableName) val r2 = result.collect @@ -453,10 +454,10 @@ object ConcurrentOpsTests extends Assertions with Logging { val putTasks4 = Array.fill(5)(doPut(tableName4)) - putTasks.foreach(Await.result(_, Duration.Inf)) - putTasks2.foreach(Await.result(_, Duration.Inf)) - putTasks3.foreach(Await.result(_, Duration.Inf)) - putTasks4.foreach(Await.result(_, Duration.Inf)) + putTasks.foreach(Await.result(_, maxWait)) + putTasks2.foreach(Await.result(_, maxWait)) + putTasks3.foreach(Await.result(_, maxWait)) + putTasks4.foreach(Await.result(_, maxWait)) Seq(tableName, tableName2, tableName3, tableName4).foreach(table => { val result = session.sql("SELECT * FROM " + table).collect() @@ -548,10 +549,10 @@ object ConcurrentOpsTests extends Assertions with Logging { val delTasks4 = Array.fill(5)(doDelete(tableName4, counter.addAndGet(500))) - delTasks.foreach(Await.result(_, Duration.Inf)) - delTasks2.foreach(Await.result(_, Duration.Inf)) - delTasks3.foreach(Await.result(_, Duration.Inf)) - delTasks4.foreach(Await.result(_, Duration.Inf)) + delTasks.foreach(Await.result(_, maxWait)) + delTasks2.foreach(Await.result(_, maxWait)) + delTasks3.foreach(Await.result(_, maxWait)) + delTasks4.foreach(Await.result(_, maxWait)) Seq(tableName, tableName2, tableName3, tableName4).foreach(table => { val result = session.sql("SELECT * FROM " + table).collect() diff --git a/core/src/test/scala/io/snappydata/SnappyFunSuite.scala b/core/src/test/scala/io/snappydata/SnappyFunSuite.scala index 1f488bd3ed..0702d94a03 100644 --- a/core/src/test/scala/io/snappydata/SnappyFunSuite.scala +++ b/core/src/test/scala/io/snappydata/SnappyFunSuite.scala @@ -29,14 +29,14 @@ import io.snappydata.util.TestUtils import org.scalatest.Assertions import org.apache.spark.sql.catalyst.encoders.{ExpressionEncoder, RowEncoder} -import org.apache.spark.sql.catalyst.expressions.{Alias, And, AttributeReference, EqualNullSafe, EqualTo, Exists, ExprId, Expression, ListQuery, PredicateHelper, PredicateSubquery, ScalarSubquery} -import org.apache.spark.sql.catalyst.plans.logical.{Filter, Join, LogicalPlan, OneRowRelation, Sample} +import org.apache.spark.sql.catalyst.expressions.{Alias, And, AttributeReference, EqualNullSafe, EqualTo, Exists, ExprId, Expression, ListQuery, PlanExpression, PredicateHelper, ScalarSubquery} +import org.apache.spark.sql.catalyst.plans.logical.{Filter, Join, LogicalPlan, Sample} import org.apache.spark.sql.catalyst.util.{sideBySide, stackTraceToString} import org.apache.spark.sql.collection.Utils import org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils import org.apache.spark.sql.row.SnappyStoreDialect import org.apache.spark.sql.types.{Metadata, StructField, StructType, TypeUtilities} -import org.apache.spark.sql.{AnalysisException, DataFrame, Dataset, QueryTest, Row, SnappySession} +import org.apache.spark.sql.{AnalysisException, DataFrame, Dataset, QueryTest, Row, SnappySession, SparkSupport} // scalastyle:off import org.scalatest.{BeforeAndAfterAll, FunSuite, Outcome, Retries} // scalastyle:on @@ -63,17 +63,9 @@ abstract class SnappyFunSuite protected var testName: String = _ protected val dirList: ArrayBuffer[String] = ArrayBuffer[String]() - protected def sc: SparkContext = { - val ctx = SnappyContext.globalSparkContext - if (ctx != null && !ctx.isStopped) { - ctx - } else { - cachedContext = null - new SparkContext(newSparkConf()) - } - } + protected final def sc: SparkContext = sc(addOn = null) - protected def sc(addOn: SparkConf => SparkConf): SparkContext = { + protected final def sc(addOn: SparkConf => SparkConf): SparkContext = { val ctx = SnappyContext.globalSparkContext if (ctx != null && !ctx.isStopped) { ctx @@ -162,28 +154,6 @@ abstract class SnappyFunSuite baseCleanup() } - /** - * Wait until given criterion is met - * - * @param check Function criterion to wait on - * @param ms total time to wait, in milliseconds - * @param interval pause interval between waits - * @param throwOnTimeout if false, don't generate an error - */ - def waitForCriterion(check: => Boolean, desc: String, ms: Long, - interval: Long, throwOnTimeout: Boolean): Unit = { - val criterion = new WaitCriterion { - - override def done: Boolean = { - check - } - - override def description(): String = desc - } - DistributedTestBase.waitForCriterion(criterion, ms, interval, - throwOnTimeout) - } - def stopAll(): Unit = { val sc = SnappyContext.globalSparkContext logInfo("Check stop required for spark context = " + sc) @@ -206,7 +176,7 @@ abstract class SnappyFunSuite SnappyFunSuite.checkAnswer(df, expectedAnswer) } -object SnappyFunSuite extends Assertions { +object SnappyFunSuite extends Assertions with SparkSupport { def checkAnswer(df: => DataFrame, expectedAnswer: Seq[Row]): Unit = { val analyzedDF = try df catch { case ae: AnalysisException => @@ -256,12 +226,34 @@ object SnappyFunSuite extends Assertions { val schema = StructType(JdbcUtils.getSchema(rs, SnappyStoreDialect).map(f => StructField( f.name.toLowerCase, f.dataType, f.nullable, withName(f.name.toLowerCase, f.metadata)))) val rows = Utils.resultSetToSparkInternalRows(rs, schema).map(_.copy()).toSeq - session.internalCreateDataFrame(session.sparkContext.makeRDD(rows), schema) + internals.internalCreateDataFrame(session, session.sparkContext.makeRDD(rows), schema) } else { implicit val encoder: ExpressionEncoder[Row] = RowEncoder(StructType(Nil)) session.createDataset[Row](Nil) } } + + /** + * Wait until given criterion is met + * + * @param check Function criterion to wait on + * @param ms total time to wait, in milliseconds + * @param interval pause interval between waits + * @param throwOnTimeout if false, don't generate an error + */ + def waitForCriterion(check: => Boolean, desc: String, ms: Long = 10000, + interval: Long = 500, throwOnTimeout: Boolean = true): Unit = { + val criterion = new WaitCriterion { + + override def done: Boolean = { + check + } + + override def description(): String = desc + } + DistributedTestBase.waitForCriterion(criterion, ms, interval, + throwOnTimeout) + } } /** @@ -272,7 +264,8 @@ object SnappyFunSuite extends Assertions { * itself but its an abstract class & parent to all spark tests. Later we can revisit how best * we can reuse the spark test code. */ -trait PlanTest extends SnappyFunSuite with PredicateHelper { +trait PlanTest extends SnappyFunSuite with PredicateHelper with SparkSupport { + /** * Since attribute references are given globally unique ids during analysis, * we must normalize them to check if two different queries are identical. @@ -285,8 +278,9 @@ trait PlanTest extends SnappyFunSuite with PredicateHelper { e.copy(exprId = ExprId(0)) case l: ListQuery => l.copy(exprId = ExprId(0)) - case p: PredicateSubquery => - p.copy(exprId = ExprId(0)) + case p if internals.isPredicateSubquery(p) => + internals.copyPredicateSubquery(p, + p.asInstanceOf[PlanExpression[LogicalPlan]].plan, ExprId(0)) case a: AttributeReference => AttributeReference(a.name, a.dataType, a.nullable)(exprId = ExprId(0)) case a: Alias => @@ -310,7 +304,8 @@ trait PlanTest extends SnappyFunSuite with PredicateHelper { Filter(splitConjunctivePredicates(condition).map(rewriteEqual).sortBy(_.hashCode()) .reduce(And), child) case sample: Sample => - sample.copy(seed = 0L)(true) + internals.newTableSample(sample.lowerBound, sample.upperBound, + sample.withReplacement, seed = 0L, sample.child) case Join(left, right, joinType, condition) if condition.isDefined => val newCondition = splitConjunctivePredicates(condition.get).map(rewriteEqual).sortBy(_.hashCode()) @@ -348,6 +343,7 @@ trait PlanTest extends SnappyFunSuite with PredicateHelper { /** Fails the test if the two expressions do not match */ protected def compareExpressions(e1: Expression, e2: Expression): Unit = { - comparePlans(Filter(e1, OneRowRelation), Filter(e2, OneRowRelation)) + comparePlans(Filter(e1, internals.newOneRowRelation()), + Filter(e2, internals.newOneRowRelation())) } } diff --git a/core/src/test/scala/io/snappydata/util/TestUtils.scala b/core/src/test/scala/io/snappydata/util/TestUtils.scala index 6a5de125b4..09253baa12 100644 --- a/core/src/test/scala/io/snappydata/util/TestUtils.scala +++ b/core/src/test/scala/io/snappydata/util/TestUtils.scala @@ -59,7 +59,7 @@ object TestUtils extends Logging { val sc = SnappyContext.globalSparkContext if (sc != null && !sc.isStopped) { try { - val catalog = session.sessionState.catalog + val catalog = session.snappySessionState.catalog catalog.destroyAndRegisterBuiltInFunctionsForTests() } catch { case t: Throwable => logError("Failure in dropping function in cleanup", t) diff --git a/core/src/test/scala/org/apache/spark/TestPackageUtils.scala b/core/src/test/scala/org/apache/spark/TestPackageUtils.scala index 473601c3b3..ec4eaf7184 100644 --- a/core/src/test/scala/org/apache/spark/TestPackageUtils.scala +++ b/core/src/test/scala/org/apache/spark/TestPackageUtils.scala @@ -1,12 +1,26 @@ +/* + * Copyright (c) 2017-2019 TIBCO Software Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ package org.apache.spark import java.io.File object TestPackageUtils { - val userDir = System.getProperty("user.dir") - - val pathSeparator = File.pathSeparator + private val userDir = System.getProperty("user.dir") def destDir: File = { val jarDir = new File(s"$userDir/jars") diff --git a/core/src/test/scala/org/apache/spark/sql/SnappyTempTableTest.scala b/core/src/test/scala/org/apache/spark/sql/SnappyTempTableTest.scala index 1576bcffb4..f4f3a60dad 100644 --- a/core/src/test/scala/org/apache/spark/sql/SnappyTempTableTest.scala +++ b/core/src/test/scala/org/apache/spark/sql/SnappyTempTableTest.scala @@ -47,7 +47,7 @@ class SnappyTempTableTest extends SnappyFunSuite val qName = snc.snappySession.tableIdentifier(tableName) val plan = catalog.resolveRelation(qName) plan match { - case LogicalRelation(_, _, _) => fail(" A RDD based temp table " + + case _: LogicalRelation => fail(" A RDD based temp table " + "should have been matched with LogicalPlan") case _ => } @@ -74,7 +74,7 @@ class SnappyTempTableTest extends SnappyFunSuite val qName = snc.snappySession.tableIdentifier(tableName) val plan = catalog.resolveRelation(qName) plan match { - case LogicalRelation(_, _, _) => + case _: LogicalRelation => case _ => fail("A CSV relation temp table should have been " + "matched with LogicalRelation") } diff --git a/core/src/test/scala/org/apache/spark/sql/internal/UpdateStatementTypeCastingSuite.scala b/core/src/test/scala/org/apache/spark/sql/internal/UpdateStatementTypeCastingSuite.scala index 69eef94eae..7a223960f1 100644 --- a/core/src/test/scala/org/apache/spark/sql/internal/UpdateStatementTypeCastingSuite.scala +++ b/core/src/test/scala/org/apache/spark/sql/internal/UpdateStatementTypeCastingSuite.scala @@ -23,12 +23,13 @@ import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll} import org.apache.spark.sql.catalyst.analysis.Analyzer import org.apache.spark.sql.hive.SnappyAnalyzer import org.apache.spark.sql.types.{DataType, DecimalType, FloatType, IntegerType, LongType, StringType} -import org.apache.spark.sql.{AnalysisException, Row} +import org.apache.spark.sql.{AnalysisException, Row, SnappySession} class UpdateStatementTypeCastingSuite extends SnappyFunSuite with BeforeAndAfterAll with BeforeAndAfter { override def beforeAll(): Unit = { + super.beforeAll() // creating table with COLUMN_MAX_DELTA_ROWS = 1 to flush the records immediately on // column table because if all records will be in row buffer then spark's fail safe type // casting doesn't kick in @@ -37,10 +38,6 @@ class UpdateStatementTypeCastingSuite extends SnappyFunSuite with BeforeAndAfter | string_col varchar(20)) using column options(COLUMN_MAX_DELTA_ROWS '1')""".stripMargin) } - override def afterAll(): Unit = { - snc.sql("drop table testTable") - } - before { snc.sql("truncate table testTable") snc.sql("insert into testTable values (1, 1, 1, 1.2, 'abc')") @@ -186,10 +183,18 @@ class UpdateStatementTypeCastingSuite extends SnappyFunSuite with BeforeAndAfter } test("SnappyAnalyzer rules matches the rules from upstream Analyzer") { - val analyzer = new Analyzer(snc.sessionState.catalog, snc.sessionState.conf) - val snappyAnalyzer = new SnappyAnalyzer(snc.sessionState) + val snappySession = snc.snappySession + val state = snappySession.sessionState + val analyzer = new Analyzer(state.catalog, state.conf) + val snappyAnalyzer = new Analyzer(state.catalog, state.conf) + with SnappyAnalyzer { + + override def session: SnappySession = snappySession + + override lazy val baseAnalyzerInstance: Analyzer = analyzer + } assertEquals(analyzer.batches.size, snappyAnalyzer.batches.size) - for ((expBatch, actBatch) <- analyzer.batches zip snappyAnalyzer.ruleBatches) { + for ((expBatch, actBatch) <- analyzer.batches zip snappyAnalyzer.baseAnalyzerInstance.batches) { assertEquals(expBatch.name, actBatch.name) assertEquals(expBatch.strategy.toString, actBatch.strategy.toString) for ((exp, act) <- expBatch.rules zip actBatch.rules) { diff --git a/core/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala b/core/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala deleted file mode 100644 index c75c309972..0000000000 --- a/core/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala +++ /dev/null @@ -1,382 +0,0 @@ -/* - * Copyright (c) 2017-2019 TIBCO Software Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -package org.apache.spark.sql.kafka010 - -import java.io.File -import java.lang.{Integer => JInt} -import java.net.InetSocketAddress -import java.util.concurrent.TimeUnit -import java.util.{Properties, Map => JMap} - -import kafka.admin.AdminUtils -import kafka.api.Request -import kafka.common.TopicAndPartition -import kafka.server.{KafkaConfig, KafkaServer, OffsetCheckpoint} -import kafka.utils.ZkUtils -import org.apache.kafka.clients.consumer.KafkaConsumer -import org.apache.kafka.clients.producer._ -import org.apache.kafka.common.TopicPartition -import org.apache.kafka.common.serialization.{StringDeserializer, StringSerializer} -import org.apache.spark.SparkConf -import org.apache.spark.internal.Logging -import org.apache.spark.util.Utils -import org.apache.zookeeper.server.{NIOServerCnxnFactory, ZooKeeperServer} -import org.scalatest.concurrent.Eventually._ -import org.scalatest.time.SpanSugar._ - -import scala.collection.JavaConverters._ -import scala.language.postfixOps -import scala.util.Random - -/** - * This is a helper class for Kafka test suites. This has the functionality to set up - * and tear down local Kafka servers, and to push data using Kafka producers. - * - * The reason to put Kafka test utility class in src is to test Python related Kafka APIs. - */ -class KafkaTestUtils extends Logging { - - // Zookeeper related configurations - private val zkHost = "localhost" - private var zkPort: Int = 0 - private val zkConnectionTimeout = 60000 - private val zkSessionTimeout = 6000 - - private var zookeeper: EmbeddedZookeeper = _ - - private var zkUtils: kafka.utils.ZkUtils = _ - - // Kafka broker related configurations - private val brokerHost = "localhost" - private var brokerPort = 0 - private var brokerConf: KafkaConfig = _ - - // Kafka broker server - private var server: KafkaServer = _ - - // Kafka producer - private var producer: Producer[String, String] = _ - - // Flag to test whether the system is correctly started - private var zkReady = false - private var brokerReady = false - - def zkAddress: String = { - assert(zkReady, "Zookeeper not setup yet or already torn down, cannot get zookeeper address") - s"$zkHost:$zkPort" - } - - def brokerAddress: String = { - assert(brokerReady, "Kafka not setup yet or already torn down, cannot get broker address") - s"$brokerHost:$brokerPort" - } - - def zookeeperClient: ZkUtils = { - assert(zkReady, "Zookeeper not setup yet or already torn down, cannot get zookeeper client") - Option(zkUtils).getOrElse( - throw new IllegalStateException("Zookeeper client is not yet initialized")) - } - - // Set up the Embedded Zookeeper server and get the proper Zookeeper port - private def setupEmbeddedZookeeper(): Unit = { - // Zookeeper server startup - zookeeper = new EmbeddedZookeeper(s"$zkHost:$zkPort") - // Get the actual zookeeper binding port - zkPort = zookeeper.actualPort - zkUtils = ZkUtils(s"$zkHost:$zkPort", zkSessionTimeout, zkConnectionTimeout, false) - zkReady = true - } - - // Set up the Embedded Kafka server - private def setupEmbeddedKafkaServer(): Unit = { - assert(zkReady, "Zookeeper should be set up beforehand") - - // Kafka broker startup - Utils.startServiceOnPort(brokerPort, port => { - brokerPort = port - brokerConf = new KafkaConfig(brokerConfiguration, doLog = false) - server = new KafkaServer(brokerConf) - server.startup() - brokerPort = server.boundPort() - (server, brokerPort) - }, new SparkConf(), "KafkaBroker") - - brokerReady = true - } - - /** setup the whole embedded servers, including Zookeeper and Kafka brokers */ - def setup(): Unit = { - setupEmbeddedZookeeper() - setupEmbeddedKafkaServer() - } - - /** Teardown the whole servers, including Kafka broker and Zookeeper */ - def teardown(): Unit = { - brokerReady = false - zkReady = false - - if (producer != null) { - producer.close() - producer = null - } - - if (server != null) { - server.shutdown() - server = null - } - - brokerConf.logDirs.foreach { f => Utils.deleteRecursively(new File(f)) } - - if (zkUtils != null) { - zkUtils.close() - zkUtils = null - } - - if (zookeeper != null) { - zookeeper.shutdown() - zookeeper = null - } - } - - /** Create a Kafka topic and wait until it is propagated to the whole cluster */ - def createTopic(topic: String, partitions: Int, overwrite: Boolean = false): Unit = { - var created = false - while (!created) { - try { - AdminUtils.createTopic(zkUtils, topic, partitions, 1) - created = true - } catch { - case e: kafka.common.TopicExistsException if overwrite => deleteTopic(topic) - } - } - // wait until metadata is propagated - (0 until partitions).foreach { p => - waitUntilMetadataIsPropagated(topic, p) - } - } - - def getAllTopicsAndPartitionSize(): Seq[(String, Int)] = { - zkUtils.getPartitionsForTopics(zkUtils.getAllTopics()).mapValues(_.size).toSeq - } - - /** Create a Kafka topic and wait until it is propagated to the whole cluster */ - def createTopic(topic: String): Unit = { - createTopic(topic, 1) - } - - /** Delete a Kafka topic and wait until it is propagated to the whole cluster */ - def deleteTopic(topic: String): Unit = { - val partitions = zkUtils.getPartitionsForTopics(Seq(topic))(topic).size - AdminUtils.deleteTopic(zkUtils, topic) - verifyTopicDeletionWithRetries(zkUtils, topic, partitions, List(this.server)) - } - - /** Add new paritions to a Kafka topic */ - def addPartitions(topic: String, partitions: Int): Unit = { - AdminUtils.addPartitions(zkUtils, topic, partitions) - // wait until metadata is propagated - (0 until partitions).foreach { p => - waitUntilMetadataIsPropagated(topic, p) - } - } - - /** Java-friendly function for sending messages to the Kafka broker */ - def sendMessages(topic: String, messageToFreq: JMap[String, JInt]): Unit = { - sendMessages(topic, Map(messageToFreq.asScala.mapValues(_.intValue()).toSeq: _*)) - } - - /** Send the messages to the Kafka broker */ - def sendMessages(topic: String, messageToFreq: Map[String, Int]): Unit = { - val messages = messageToFreq.flatMap { case (s, freq) => Seq.fill(freq)(s) }.toArray - sendMessages(topic, messages) - } - - /** Send the array of messages to the Kafka broker */ - def sendMessages(topic: String, messages: Array[String]): Seq[(String, RecordMetadata)] = { - sendMessages(topic, messages, None) - } - - /** Send the array of messages to the Kafka broker using specified partition */ - def sendMessages( - topic: String, - messages: Array[String], - partition: Option[Int]): Seq[(String, RecordMetadata)] = { - producer = new KafkaProducer[String, String](producerConfiguration) - val offsets = try { - messages.map { m => - val record = partition match { - case Some(p) => new ProducerRecord[String, String](topic, p, null, m) - case None => new ProducerRecord[String, String](topic, m) - } - val metadata = - producer.send(record).get(10, TimeUnit.SECONDS) - // logInfo(s"\tSent $m to partition ${metadata.partition}, offset ${metadata.offset}") - (m, metadata) - } - } finally { - if (producer != null) { - producer.close() - producer = null - } - } - offsets - } - - def getLatestOffsets(topics: Set[String]): Map[TopicPartition, Long] = { - val kc = new KafkaConsumer[String, String](consumerConfiguration) - logInfo("Created consumer to get latest offsets") - kc.subscribe(topics.asJavaCollection) - kc.poll(0) - val partitions = kc.assignment() - kc.pause(partitions) - kc.seekToEnd(partitions) - val offsets = partitions.asScala.map(p => p -> kc.position(p)).toMap - kc.close() - logInfo("Closed consumer to get latest offsets") - offsets - } - - protected def brokerConfiguration: Properties = { - val props = new Properties() - props.put("broker.id", "0") - props.put("host.name", "localhost") - props.put("advertised.host.name", "localhost") - props.put("port", brokerPort.toString) - props.put("log.dir", Utils.createTempDir().getAbsolutePath) - props.put("zookeeper.connect", zkAddress) - props.put("log.flush.interval.messages", "1") - props.put("replica.socket.timeout.ms", "1500") - props.put("delete.topic.enable", "true") - props - } - - private def producerConfiguration: Properties = { - val props = new Properties() - props.put("bootstrap.servers", brokerAddress) - props.put("value.serializer", classOf[StringSerializer].getName) - props.put("key.serializer", classOf[StringSerializer].getName) - // wait for all in-sync replicas to ack sends - props.put("acks", "all") - props - } - - private def consumerConfiguration: Properties = { - val props = new Properties() - props.put("bootstrap.servers", brokerAddress) - props.put("group.id", "group-KafkaTestUtils-" + Random.nextInt) - props.put("value.deserializer", classOf[StringDeserializer].getName) - props.put("key.deserializer", classOf[StringDeserializer].getName) - props.put("enable.auto.commit", "false") - props - } - - /** Verify topic is deleted in all places, e.g, brokers, zookeeper. */ - private def verifyTopicDeletion( - topic: String, - numPartitions: Int, - servers: Seq[KafkaServer]): Unit = { - val topicAndPartitions = (0 until numPartitions).map(TopicAndPartition(topic, _)) - - import ZkUtils._ - // wait until admin path for delete topic is deleted, signaling completion of topic deletion - assert( - !zkUtils.pathExists(getDeleteTopicPath(topic)), - s"${getDeleteTopicPath(topic)} still exists") - assert(!zkUtils.pathExists(getTopicPath(topic)), s"${getTopicPath(topic)} still exists") - // ensure that the topic-partition has been deleted from all brokers' replica managers - assert(servers.forall(server => topicAndPartitions.forall(tp => - server.replicaManager.getPartition(tp.topic, tp.partition) == None)), - s"topic $topic still exists in the replica manager") - // ensure that logs from all replicas are deleted if delete topic is marked successful - assert(servers.forall(server => topicAndPartitions.forall(tp => - server.getLogManager().getLog(tp).isEmpty)), - s"topic $topic still exists in log mananger") - // ensure that topic is removed from all cleaner offsets - assert(servers.forall(server => topicAndPartitions.forall { tp => - val checkpoints = server.getLogManager().logDirs.map { logDir => - new OffsetCheckpoint(new File(logDir, "cleaner-offset-checkpoint")).read() - } - checkpoints.forall(checkpointsPerLogDir => !checkpointsPerLogDir.contains(tp)) - }), s"checkpoint for topic $topic still exists") - // ensure the topic is gone - assert( - !zkUtils.getAllTopics().contains(topic), - s"topic $topic still exists on zookeeper") - } - - /** Verify topic is deleted. Retry to delete the topic if not. */ - private def verifyTopicDeletionWithRetries( - zkUtils: ZkUtils, - topic: String, - numPartitions: Int, - servers: Seq[KafkaServer]) { - eventually(timeout(60.seconds), interval(200.millis)) { - try { - verifyTopicDeletion(topic, numPartitions, servers) - } catch { - case e: Throwable => - // As pushing messages into Kafka updates Zookeeper asynchronously, there is a small - // chance that a topic will be recreated after deletion due to the asynchronous update. - // Hence, delete the topic and retry. - AdminUtils.deleteTopic(zkUtils, topic) - throw e - } - } - } - - private def waitUntilMetadataIsPropagated(topic: String, partition: Int): Unit = { - def isPropagated = server.apis.metadataCache.getPartitionInfo(topic, partition) match { - case Some(partitionState) => - val leaderAndInSyncReplicas = partitionState.leaderIsrAndControllerEpoch.leaderAndIsr - - zkUtils.getLeaderForPartition(topic, partition).isDefined && - Request.isValidBrokerId(leaderAndInSyncReplicas.leader) && - leaderAndInSyncReplicas.isr.size >= 1 - - case _ => - false - } - - eventually(timeout(60.seconds)) { - assert(isPropagated, s"Partition [$topic, $partition] metadata not propagated after timeout") - } - } - - private class EmbeddedZookeeper(val zkConnect: String) { - val snapshotDir = Utils.createTempDir() - val logDir = Utils.createTempDir() - - val zookeeper = new ZooKeeperServer(snapshotDir, logDir, 500) - val (ip, port) = { - val splits = zkConnect.split(":") - (splits(0), splits(1).toInt) - } - val factory = new NIOServerCnxnFactory() - factory.configure(new InetSocketAddress(ip, port), 16) - factory.startup(zookeeper) - - val actualPort = factory.getLocalPort - - def shutdown() { - factory.shutdown() - Utils.deleteRecursively(snapshotDir) - Utils.deleteRecursively(logDir) - } - } - -} diff --git a/core/src/test/scala/org/apache/spark/sql/store/CatalogConsistencyTest.scala b/core/src/test/scala/org/apache/spark/sql/store/CatalogConsistencyTest.scala index 1a90416d8c..12667f05dd 100644 --- a/core/src/test/scala/org/apache/spark/sql/store/CatalogConsistencyTest.scala +++ b/core/src/test/scala/org/apache/spark/sql/store/CatalogConsistencyTest.scala @@ -88,7 +88,7 @@ class CatalogConsistencyTest snc.createTable("column_table1", "column", dataDF.schema, props) // remove the table entry from Hive store but not from store DD - snc.snappySession.sessionCatalog.externalCatalog.dropTable("app", "column_table1", + snc.snappySession.sessionCatalog.snappyExternalCatalog.dropTable("app", "column_table1", ignoreIfNotExists = false, purge = false) // should throw an exception since the table has been removed from Hive store @@ -114,7 +114,7 @@ class CatalogConsistencyTest dataDF.write.format("column").mode(SaveMode.Append).options(props).saveAsTable("column_table2") // remove the table entry from Hive store but not from store DD - snc.snappySession.sessionCatalog.externalCatalog.dropTable("app", "column_table1", + snc.snappySession.sessionCatalog.snappyExternalCatalog.dropTable("app", "column_table1", ignoreIfNotExists = false, purge = false) // repair the catalog @@ -154,7 +154,7 @@ class CatalogConsistencyTest routeQueryDisabledConn.createStatement().execute("drop table " + ColumnFormatRelation.columnBatchTableName("app.column_table1")) // remove the table entry from Hive store - snc.snappySession.sessionCatalog.externalCatalog.dropTable("app", "column_table1", + snc.snappySession.sessionCatalog.snappyExternalCatalog.dropTable("app", "column_table1", ignoreIfNotExists = false, purge = false) // make sure that the table does not exist in Hive metastore @@ -256,7 +256,7 @@ class CatalogConsistencyTest snc.createTable("row_table1", "row", dataDF.schema, props) // remove the table entry from Hive store but not from store DD - snc.snappySession.sessionCatalog.externalCatalog.dropTable("app", "row_table1", + snc.snappySession.sessionCatalog.snappyExternalCatalog.dropTable("app", "row_table1", ignoreIfNotExists = false, purge = false) // should throw an exception since the table has been removed from Hive store @@ -277,7 +277,7 @@ class CatalogConsistencyTest dataDF.write.format("row").mode(SaveMode.Append).options(props).saveAsTable("row_table2") // remove the table entry from Hive store but not from store DD - snc.snappySession.sessionCatalog.externalCatalog.dropTable("app", "row_table1", + snc.snappySession.sessionCatalog.snappyExternalCatalog.dropTable("app", "row_table1", ignoreIfNotExists = false, purge = false) // repair the catalog diff --git a/core/src/test/scala/org/apache/spark/sql/store/ColumnTableBatchInsertTest.scala b/core/src/test/scala/org/apache/spark/sql/store/ColumnTableBatchInsertTest.scala index 0863e87fd3..381cbcabcb 100644 --- a/core/src/test/scala/org/apache/spark/sql/store/ColumnTableBatchInsertTest.scala +++ b/core/src/test/scala/org/apache/spark/sql/store/ColumnTableBatchInsertTest.scala @@ -16,13 +16,16 @@ */ package org.apache.spark.sql.store +import scala.collection.mutable + +import io.snappydata.SnappyFunSuite.waitForCriterion import io.snappydata.core.{Data, TestData} import io.snappydata.{ConcurrentOpsTests, SnappyFunSuite} -import org.apache.spark.sql._ -import org.apache.spark.{Logging, SparkContext} import org.scalatest.{Assertions, BeforeAndAfter} -import scala.collection.mutable +import org.apache.spark.sql._ +import org.apache.spark.status.api.v1.RDDStorageInfo +import org.apache.spark.{Logging, SparkContext} class ColumnTableBatchInsertTest extends SnappyFunSuite with Logging @@ -40,6 +43,7 @@ class ColumnTableBatchInsertTest extends SnappyFunSuite snc.dropTable(tableName2, ifExists = true) snc.dropTable(tableName3, ifExists = true) snc.dropTable(tableName4, ifExists = true) + snc.dropTable("rowTable", ifExists = true) } test("test the shadow table creation") { @@ -81,8 +85,17 @@ class ColumnTableBatchInsertTest extends SnappyFunSuite "PARTITION_BY 'Col1'," + "BUCKETS '1')") + // check insert statement result + assert(snc.sql(s"insert into $tableName values (1, 2, 3)").collect() === Array(Row(1L))) + assert(snc.sql(s"insert into $tableName values (1, 2, 3), (4, 5, 6)").collect() === + Array(Row(2L))) + assert(snc.sql(s"insert into $tableName select 7, 8, 9").collect() === Array(Row(1L))) + assert(snc.sql(s"insert into $tableName select 7, 8, 9 union all select 1, 2, 3").collect() === + Array(Row(2L))) + snc.sql(s"truncate table $tableName") + val data = Seq(Seq(1, 2, 3), Seq(7, 8, 9), Seq(9, 2, 3), Seq(4, 2, 3), Seq(5, 6, 7)) - val rdd = sc.parallelize(data, data.length).map(s => new Data(s(0), s(1), s(2))) + val rdd = sc.parallelize(data, data.length).map(s => Data(s.head, s(1), s(2))) val dataDF = snc.createDataFrame(rdd) dataDF.write.insertInto(tableName) @@ -129,23 +142,16 @@ class ColumnTableBatchInsertTest extends SnappyFunSuite try { snc.sql(s"insert overwrite $tableName select * from $tableName") fail("Expected AnalysisException while overwriting table which is also being read from") - } - catch { + } catch { case ae: AnalysisException => assert(ae.getMessage().contains("Cannot insert overwrite")) - case t: Throwable => fail("Unexpected Exception ", t) } try { snc.sql(s"insert into $tableName select * from $tableName") - fail("Expected AnalysisException while overwriting table which is also being read from") - } - catch { + } catch { case ae: AnalysisException => assert(ae.getMessage().contains("Cannot insert overwrite")) - case t: Throwable => fail("Unexpected Exception ", t) } - } - test("test the shadow table creation heavy insert") { // snc.sql(s"DROP TABLE IF EXISTS $tableName") @@ -457,26 +463,33 @@ class ColumnTableBatchInsertTest extends SnappyFunSuite } } -object ColumnTableBatchInsertTest extends Assertions { +object ColumnTableBatchInsertTest extends Assertions with SparkSupport { + + private def waitForRDDInfos(sc: SparkContext, expectedSize: Int, + message: String): Seq[RDDStorageInfo] = { + var rddInfos: Seq[RDDStorageInfo] = null + waitForCriterion({ + rddInfos = internals.getCachedRDDInfos(sc) + rddInfos.length == expectedSize + }, message) + rddInfos + } def testSparkCachingUsingSQL(sc: SparkContext, executeSQL: String => Dataset[Row], isTableCached: String => Boolean, isCached: Dataset[Row] => Boolean): Unit = { executeSQL("cache table cachedTable1 as select id, rand() from range(1000000)") // check that table has been cached and materialized assert(isTableCached("cachedTable1")) - var rddInfos = sc.ui.get.storageListener.rddInfoList - assert(rddInfos.length === 1) + var rddInfos = waitForRDDInfos(sc, 1, "cached table should show up") assert(rddInfos.head.name.contains("Range (0, 1000000")) assert(executeSQL("select count(*) from cachedTable1").collect()(0).getLong(0) === 1000000) - rddInfos = sc.ui.get.storageListener.rddInfoList - assert(rddInfos.length === 1) + rddInfos = waitForRDDInfos(sc, 1, "cached table should be present") assert(rddInfos.head.name.contains("Range (0, 1000000")) executeSQL("uncache table cachedTable1") assert(!isTableCached("cachedTable1")) - rddInfos = sc.ui.get.storageListener.rddInfoList - assert(rddInfos.length === 0) + rddInfos = waitForRDDInfos(sc, 0, "cached table should be cleared") // temporary table should still exist assert(executeSQL("select count(*) from cachedTable1").collect()(0).getLong(0) === 1000000) @@ -484,19 +497,17 @@ object ColumnTableBatchInsertTest extends Assertions { executeSQL("cache lazy table cachedTable2 as select id, rand() from range(500000)") assert(isTableCached("cachedTable2")) // check that cache has not been materialized yet - rddInfos = sc.ui.get.storageListener.rddInfoList + rddInfos = internals.getCachedRDDInfos(sc) assert(rddInfos.length === 0) assert(executeSQL("select count(*) from cachedTable2").collect()(0).getLong(0) === 500000) - rddInfos = sc.ui.get.storageListener.rddInfoList - assert(rddInfos.length === 1) + rddInfos = waitForRDDInfos(sc, 1, "lazily cached table should show up after query") assert(rddInfos.head.name.contains("Range (0, 500000")) // drop table directly without explicit uncache should also do it val table = executeSQL("select * from cachedTable2") executeSQL("drop table cachedTable2") assert(!isCached(table)) - rddInfos = sc.ui.get.storageListener.rddInfoList - assert(rddInfos.length === 0) + rddInfos = waitForRDDInfos(sc, 0, "cached table should be cleared") executeSQL("drop table cachedTable1") } diff --git a/core/src/test/scala/org/apache/spark/sql/store/ColumnTableTest.scala b/core/src/test/scala/org/apache/spark/sql/store/ColumnTableTest.scala index eb2337f0c0..ef05e42bf2 100644 --- a/core/src/test/scala/org/apache/spark/sql/store/ColumnTableTest.scala +++ b/core/src/test/scala/org/apache/spark/sql/store/ColumnTableTest.scala @@ -21,7 +21,7 @@ import java.sql.{DriverManager, SQLException} import scala.util.{Failure, Success, Try} import com.gemstone.gemfire.cache.{EvictionAction, EvictionAlgorithm} -import com.gemstone.gemfire.internal.cache.{DistributedRegion, PartitionedRegion} +import com.gemstone.gemfire.internal.cache.{DistributedRegion, GemFireCacheImpl, PartitionedRegion, TXManagerImpl} import com.pivotal.gemfirexd.internal.engine.Misc import com.pivotal.gemfirexd.internal.impl.jdbc.EmbedConnection import com.pivotal.gemfirexd.internal.impl.sql.compile.ParserImpl @@ -183,6 +183,8 @@ class ColumnTableTest try { snc.sql("insert into coltab values (1, 2)") } catch { + case ae: AnalysisException => assert(ae.message.contains( + "data to be inserted have the same number of columns as the target table")) case ex: SQLException => assert("42802".equals(ex.getSQLState)) } snc.sql("drop table coltab") @@ -951,6 +953,13 @@ class ColumnTableTest testRowBufferEviction("testTableWithoutSchema") } + private def commitTX(): Unit = { + val tx = TXManagerImpl.getCurrentSnapshotTXState + val txMgr = GemFireCacheImpl.getExisting.getCacheTransactionManager + txMgr.masqueradeAs(tx) + txMgr.commit() + } + private def testRowBufferEviction(tableName: String): Unit = { val props = Map("BUCKETS" -> "1", "PARTITION_BY" -> "col1") val data = Seq(Seq(1, 2, 3), Seq(7, 8, 9), Seq(9, 2, 3), Seq(4, 2, 3), @@ -974,6 +983,9 @@ class ColumnTableTest assert(rs.getInt(1) <= 3) assert(!rs.next()) rs.close() + // need to do explicit commit on thread-local TX since this creates an implicit + // scan-local snapshot TX which is normally closed by Spark layer commit + commitTX() // also check with the insert API snc.truncateTable(tableName) @@ -983,6 +995,9 @@ class ColumnTableTest assert(rs.getInt(1) <= 3) assert(!rs.next()) rs.close() + // need to do explicit commit on thread-local TX since this creates an implicit + // scan-local snapshot TX which is normally closed by Spark layer commit + commitTX() conn.close() } @@ -1458,7 +1473,7 @@ class ColumnTableTest } test("Test method for getting table type of snappy tables") { - var session = new SnappySession(snc.sparkContext) + val session = new SnappySession(snc.sparkContext) session.sql("drop table if exists temp1") session.sql("drop table if exists temp2") session.sql("drop table if exists temp3") @@ -1481,7 +1496,7 @@ class ColumnTableTest snc.sql(s"insert into t1 values(3,'test3')") val df = snc.sql("select * from t1") df.collect() - val tempPath = System.getProperty("user.dir") + System.currentTimeMillis() + val tempPath = System.getProperty("user.dir") + "/" + System.currentTimeMillis() assert(df.count() == 3) df.write.option("header", "true").csv(tempPath) @@ -1505,6 +1520,13 @@ class ColumnTableTest "Should not have succedded with incorrect options") case Failure(_) => // Do nothing } + + session.sql("drop table if exists temp1") + session.sql("drop table if exists temp2") + session.sql("drop table if exists temp3") + session.sql("drop table if exists temp4") + snc.sql("drop table if exists t1") + FileUtils.deleteDirectory(new java.io.File(tempPath)) } private def getTableType(table: String, session: SnappySession): String = { @@ -1517,6 +1539,7 @@ class ColumnTableTest snc.sql("create table t1(id integer, str string) using column options(key_columns 'id')") snc.sql("put into t1 select 1, 'aa'") snc.sql("put into t1 select 2, 'aa' union all select 3, 'bb'") + // TODO: using values causes serialization error for some reason snc.sql("put into t1 select 1, 'cc'") val rows = snc.sql("select * from t1") assert(rows.count() == 3) diff --git a/core/src/test/scala/org/apache/spark/sql/store/CreateIndexTest.scala b/core/src/test/scala/org/apache/spark/sql/store/CreateIndexTest.scala index 34bbf52b0d..7451ddf591 100644 --- a/core/src/test/scala/org/apache/spark/sql/store/CreateIndexTest.scala +++ b/core/src/test/scala/org/apache/spark/sql/store/CreateIndexTest.scala @@ -299,7 +299,8 @@ class CreateIndexTest extends SnappyFunSuite with BeforeAndAfterEach { dataDF.write.insertInto(table3) } - test("Test two table joins") { + // TODO: fails with Spark 2.4 + ignore("Test two table joins") { val table1 = "tabOne" val table2 = "tabTwo" val table3 = "tabThree" @@ -816,7 +817,8 @@ object CreateIndexTest extends SnappyFunSuite { def validateIndex(index: Seq[String], tables: String*)(df: DataFrame): Unit = { val (indexesMatched, indexesUnMatched) = df.queryExecution.optimizedPlan.collect { - case l@LogicalRelation(idx: IndexColumnFormatRelation, _, _) => idx + case l: LogicalRelation if l.relation.isInstanceOf[IndexColumnFormatRelation] => + l.relation.asInstanceOf[IndexColumnFormatRelation] }.partition(rel => index.exists(i => rel.table.indexOf(i.toUpperCase) > 0)) if (indexesMatched.size != index.size) { @@ -826,8 +828,10 @@ object CreateIndexTest extends SnappyFunSuite { } val tablesAppeared = df.queryExecution.optimizedPlan.collect { - case l@LogicalRelation(columnTable: ColumnFormatRelation, _, _) => columnTable.table - case l@LogicalRelation(rowTable: RowFormatRelation, _, _) => rowTable.table + case l: LogicalRelation if l.relation.isInstanceOf[ColumnFormatRelation] => + l.relation.asInstanceOf[ColumnFormatRelation].table + case l: LogicalRelation if l.relation.isInstanceOf[RowFormatRelation] => + l.relation.asInstanceOf[RowFormatRelation].table } val (tablesFound, tablesNotFound) = tables.partition(tab => diff --git a/core/src/test/scala/org/apache/spark/sql/store/MetadataTest.scala b/core/src/test/scala/org/apache/spark/sql/store/MetadataTest.scala index 664c154e17..d431493db7 100644 --- a/core/src/test/scala/org/apache/spark/sql/store/MetadataTest.scala +++ b/core/src/test/scala/org/apache/spark/sql/store/MetadataTest.scala @@ -90,6 +90,10 @@ object MetadataTest extends Assertions { assert(!rsMap.contains("spark.sql.sources.schema.numParts")) } + private def compare(schema1: StructType, schema2: StructType): Unit = { + assert(schema1.toString() === schema2.toString()) + } + private val expectedSYSTables = Array("ASYNCEVENTLISTENERS", "GATEWAYRECEIVERS", "GATEWAYSENDERS", "SYSALIASES", "SYSCHECKS", "SYSCOLPERMS", "SYSCOLUMNS", "SYSCONGLOMERATES", "SYSCONSTRAINTS", "SYSDEPENDS", "SYSDISKSTORES", "SYSFILES", "SYSFOREIGNKEYS", @@ -166,7 +170,7 @@ object MetadataTest extends Assertions { val expectedSizes = List(256, 256, 24, 12, 32672, 32672) rs = ds.collect() // check schema of the returned Dataset - assert(ds.schema === StructType(expectedColumns.zip(expectedSizes).map(p => + compare(ds.schema, StructType(expectedColumns.zip(expectedSizes).map(p => StructField(p._1, StringType, nullable = false, getMetadata(p._1, p._2))))) checkMembers(rs, forShow = true) @@ -175,7 +179,7 @@ object MetadataTest extends Assertions { ds = executeSQL("select * from sys.sysSchemas") rs = ds.collect() // check schema of the returned Dataset - assert(ds.schema === StructType(sysSchemasColumns.map(p => + compare(ds.schema, StructType(sysSchemasColumns.map(p => StructField(p._1, StringType, nullable = false, getMetadata(p._1, p._2, p._3))))) val expectedDefaultSchemas = List("APP", "DEFAULT", "NULLID", "SNAPPY_HIVE_METASTORE", "SQLJ", "SYS", "SYSCAT", "SYSCS_DIAG", "SYSCS_UTIL", "SYSFUN", "SYSIBM", "SYSPROC", "SYSSTAT") @@ -186,7 +190,7 @@ object MetadataTest extends Assertions { ds = executeSQL("select * from sys.sysTables where tableSchemaName = 'SYS'") rs = ds.collect() // check schema of the returned Dataset - assert(ds.schema === StructType(sysTablesColumns.map { case (name, size, typeName, nullable) => + compare(ds.schema, StructType(sysTablesColumns.map { case (name, size, typeName, nullable) => val dataType = typeName match { case "BOOLEAN" => BooleanType case _ => StringType @@ -474,8 +478,6 @@ object MetadataTest extends Assertions { // check schema of the returned Dataset assert(ds.schema.map(_.copy(metadata = Metadata.empty)) === expectedColumns.zip(nullability) .map(p => StructField(p._1, StringType, p._2))) - // last row is detailed information and an empty row before that (no partitioning information) - assert(rs.length === sysSchemasColumns.length + 2) assert(rs.take(sysSchemasColumns.length).toSeq === sysSchemasColumns.map( p => Row(p._1, s"${p._3.toLowerCase}(${p._2})", null))) assert(rs(sysSchemasColumns.length + 1).getString(0) === "# Detailed Table Information") @@ -496,8 +498,6 @@ object MetadataTest extends Assertions { // check schema of the returned Dataset assert(ds.schema.map(_.copy(metadata = Metadata.empty)) === expectedColumns.zip(nullability) .map(p => StructField(p._1, StringType, p._2))) - // last row is detailed information and an empty row before that (no partitioning information) - assert(rs.length === sysTablesColumns.length + 2) assert(rs.take(sysTablesColumns.length).toSeq === sysTablesColumns.map { case (name, _, "BOOLEAN", _) => Row(name, BooleanType.simpleString, null) case (name, _, "LONGVARCHAR", _) => Row(name, StringType.simpleString, null) @@ -597,8 +597,6 @@ object MetadataTest extends Assertions { assert(rs === Array(Row("id", IntegerType.simpleString, null), Row("data", StringType.simpleString, null))) rs = executeSQL("describe extended columnTable2").collect() - // last row is detailed information and an empty row before that (no partitioning information) - assert(rs.length === 5) assert(rs.take(3) === Array(Row("id", LongType.simpleString, null), Row("data", StringType.simpleString, null), Row("data2", DecimalType.SYSTEM_DEFAULT.simpleString, null))) @@ -628,10 +626,10 @@ object MetadataTest extends Assertions { // check schema of the returned Dataset which should be a single string column // for JDBC it should be a CLOB column if (usingJDBC) { - assert(ds.schema === StructType(Array(StructField("plan", StringType, nullable = true, + compare(ds.schema, StructType(Array(StructField("plan", StringType, nullable = true, getMetadata("plan", 0, "CLOB"))))) } else { - assert(ds.schema === StructType(Array(StructField("plan", StringType, nullable = true)))) + compare(ds.schema, StructType(Array(StructField("plan", StringType)))) } assert(matches(plan, ".*Physical Plan.*Partitioned Scan RowFormatRelation\\[app" + ".rowtable1\\].*numBuckets = 1 numPartitions = 1.*")) @@ -642,10 +640,10 @@ object MetadataTest extends Assertions { assert(rs.length === 1) plan = rs(0).getString(0) if (usingJDBC) { - assert(ds.schema === StructType(Array(StructField("plan", StringType, nullable = true, + compare(ds.schema, StructType(Array(StructField("plan", StringType, nullable = true, getMetadata("plan", 0, "CLOB"))))) } else { - assert(ds.schema === StructType(Array(StructField("plan", StringType, nullable = true)))) + compare(ds.schema, StructType(Array(StructField("plan", StringType)))) } def literalString(value: String): String = { @@ -668,13 +666,13 @@ object MetadataTest extends Assertions { assert(rs.length === 1) plan = rs(0).getString(0) if (usingJDBC) { - assert(ds.schema === StructType(Array(StructField("plan", StringType, nullable = false, + compare(ds.schema, StructType(Array(StructField("plan", StringType, nullable = false, getMetadata("plan", 0, "CLOB"))))) assert(plan.contains("stmt_id")) assert(plan.contains("SQL_stmt select * from rowTable1 where id = 10")) assert(plan.contains("REGION-GET")) } else { - assert(ds.schema === StructType(Array(StructField("plan", StringType, nullable = true)))) + compare(ds.schema, StructType(Array(StructField("plan", StringType)))) expectedPattern = ".*Physical Plan.*Partitioned Scan RowFormatRelation\\[app" + ".rowtable1\\].*numBuckets = 1 numPartitions = 1.*id.* = " + literalString("10") + ".*" assert(matches(plan, expectedPattern)) @@ -685,10 +683,10 @@ object MetadataTest extends Assertions { assert(rs.length === 1) plan = rs(0).getString(0) if (usingJDBC) { - assert(ds.schema === StructType(Array(StructField("plan", StringType, nullable = true, + compare(ds.schema, StructType(Array(StructField("plan", StringType, nullable = true, getMetadata("plan", 0, "CLOB"))))) } else { - assert(ds.schema === StructType(Array(StructField("plan", StringType, nullable = true)))) + compare(ds.schema, StructType(Array(StructField("plan", StringType)))) } expectedPattern = s".*Parsed Logical Plan.*Filter.*id = " + literalString("10") + "" + ".*Analyzed Logical Plan.*Filter.*id#[0-9]* = " + literalString("10") + @@ -705,14 +703,14 @@ object MetadataTest extends Assertions { assert(rs.length === 1) plan = rs(0).getString(0) if (usingJDBC) { - assert(ds.schema === StructType(Array(StructField("plan", StringType, nullable = true, + compare(ds.schema, StructType(Array(StructField("plan", StringType, nullable = true, getMetadata("plan", 0, "CLOB"))))) } else { - assert(ds.schema === StructType(Array(StructField("plan", StringType, nullable = true)))) + compare(ds.schema, StructType(Array(StructField("plan", StringType)))) } expectedPattern = ".*Physical Plan.*Partitioned Scan ColumnFormatRelation" + "\\[app.columntable2\\].*numBuckets = [0-9]* numPartitions = [0-9]*" + - s".*id#[0-9]*L = DynExpr\\(" + literalString("10") + "\\).*" + s".*id#[0-9]*L = .*" + literalString("10") + ".*" assert(matches(plan, expectedPattern)) ds = executeSQL("explain extended select * from columnTable2 where id > 20") @@ -720,17 +718,17 @@ object MetadataTest extends Assertions { assert(rs.length === 1) plan = rs(0).getString(0) if (usingJDBC) { - assert(ds.schema === StructType(Array(StructField("plan", StringType, nullable = true, + compare(ds.schema, StructType(Array(StructField("plan", StringType, nullable = true, getMetadata("plan", 0, "CLOB"))))) } else { - assert(ds.schema === StructType(Array(StructField("plan", StringType, nullable = true)))) + compare(ds.schema, StructType(Array(StructField("plan", StringType)))) } expectedPattern = s".*Parsed Logical Plan.*Filter.*id > ${literalString("20")}" + s".*Analyzed Logical Plan.*Filter.*id#[0-9]*L > cast\\(${literalString("20")} as bigint" + - s".*Optimized Logical Plan.*Filter.*id#[0-9]*L > DynExpr\\(${literalString("20")}\\)" + + s".*Optimized Logical Plan.*Filter.*id#[0-9]*L > .*${literalString("20")}" + ".*ColumnFormatRelation\\[app.columntable2\\].*Physical Plan.*Partitioned Scan" + " ColumnFormatRelation\\[app.columntable2\\].*numBuckets = [0-9]* numPartitions = [0-9]*" + - s".*id#[0-9]*L > DynExpr\\(${literalString("20")}\\).*" + s".*id#[0-9]*L > .*${literalString("20")}.*" assert(matches(plan, expectedPattern)) // ----- check EXPLAIN for DDLs ----- @@ -740,12 +738,12 @@ object MetadataTest extends Assertions { assert(rs.length === 1) plan = rs(0).getString(0) if (usingJDBC) { - assert(ds.schema === StructType(Array(StructField("plan", StringType, nullable = true, + compare(ds.schema, StructType(Array(StructField("plan", StringType, nullable = true, getMetadata("plan", 0, "CLOB"))))) } else { - assert(ds.schema === StructType(Array(StructField("plan", StringType, nullable = true)))) + compare(ds.schema, StructType(Array(StructField("plan", StringType)))) } - assert(matches(plan, ".*Physical Plan.*ExecutedCommand.*CreateTableUsingCommand" + + assert(matches(plan, ".*Physical Plan.*Execute.*CreateTableUsingCommand" + ".*rowtable2.*\\(id int primary key, id2 int\\), row.*")) // create more tables and repeat the checks @@ -789,8 +787,6 @@ object MetadataTest extends Assertions { Row("data", DateType.simpleString, null), Row("data2", StringType.simpleString, null))) rs = executeSQL("describe extended schema2.rowTable2").collect() - // last row is detailed information and an empty row before that (no partitioning information) - assert(rs.length === 4) assert(rs.take(2) === Array(Row("id", IntegerType.simpleString, null), Row("data", StringType.simpleString, null))) assert(rs(3).getString(0) === "# Detailed Table Information") @@ -818,10 +814,10 @@ object MetadataTest extends Assertions { // check schema of the returned Dataset which should be a single string column // for JDBC it should be a CLOB column if (usingJDBC) { - assert(ds.schema === StructType(Array(StructField("plan", StringType, nullable = true, + compare(ds.schema, StructType(Array(StructField("plan", StringType, nullable = true, getMetadata("plan", 0, "CLOB"))))) } else { - assert(ds.schema === StructType(Array(StructField("plan", StringType, nullable = true)))) + compare(ds.schema, StructType(Array(StructField("plan", StringType)))) } assert(matches(plan, ".*Physical Plan.*Partitioned Scan RowFormatRelation\\[schema2" + ".rowtable2\\].*numBuckets = 8 numPartitions = [0-9]*.*")) @@ -832,10 +828,10 @@ object MetadataTest extends Assertions { assert(rs.length === 1) plan = rs(0).getString(0) if (usingJDBC) { - assert(ds.schema === StructType(Array(StructField("plan", StringType, nullable = true, + compare(ds.schema, StructType(Array(StructField("plan", StringType, nullable = true, getMetadata("plan", 0, "CLOB"))))) } else { - assert(ds.schema === StructType(Array(StructField("plan", StringType, nullable = true)))) + compare(ds.schema, StructType(Array(StructField("plan", StringType)))) } expectedPattern = ".*Physical Plan.*Partitioned Scan RowFormatRelation" + "\\[schema2.rowtable2\\].*numBuckets = 8 numPartitions = [0-9]*" + @@ -849,13 +845,13 @@ object MetadataTest extends Assertions { assert(rs.length === 1) plan = rs(0).getString(0) if (usingJDBC) { - assert(ds.schema === StructType(Array(StructField("plan", StringType, nullable = false, + compare(ds.schema, StructType(Array(StructField("plan", StringType, nullable = false, getMetadata("plan", 0, "CLOB"))))) assert(plan.contains("stmt_id")) assert(plan.contains("SQL_stmt select * from schema2.rowTable2 where id = 15")) assert(plan.contains("REGION-GET")) } else { - assert(ds.schema === StructType(Array(StructField("plan", StringType, nullable = true)))) + compare(ds.schema, StructType(Array(StructField("plan", StringType)))) // no pruning for row tables yet expectedPattern = ".*Physical Plan.*Partitioned Scan RowFormatRelation" + "\\[schema2.rowtable2\\].*numBuckets = 8 numPartitions = [0-9]*" + @@ -870,10 +866,10 @@ object MetadataTest extends Assertions { assert(rs.length === 1) plan = rs(0).getString(0) if (usingJDBC) { - assert(ds.schema === StructType(Array(StructField("plan", StringType, nullable = true, + compare(ds.schema, StructType(Array(StructField("plan", StringType, nullable = true, getMetadata("plan", 0, "CLOB"))))) } else { - assert(ds.schema === StructType(Array(StructField("plan", StringType, nullable = true)))) + compare(ds.schema, StructType(Array(StructField("plan", StringType)))) } assert(matches(plan, ".*Physical Plan.*Partitioned Scan ColumnFormatRelation" + "\\[schema1.columntable1\\].*numBuckets = [0-9]* numPartitions = 1" + @@ -884,10 +880,10 @@ object MetadataTest extends Assertions { assert(rs.length === 1) plan = rs(0).getString(0) if (usingJDBC) { - assert(ds.schema === StructType(Array(StructField("plan", StringType, nullable = true, + compare(ds.schema, StructType(Array(StructField("plan", StringType, nullable = true, getMetadata("plan", 0, "CLOB"))))) } else { - assert(ds.schema === StructType(Array(StructField("plan", StringType, nullable = true)))) + compare(ds.schema, StructType(Array(StructField("plan", StringType)))) } // should prune to a single partition diff --git a/core/src/test/scala/org/apache/spark/sql/store/RowTableTest.scala b/core/src/test/scala/org/apache/spark/sql/store/RowTableTest.scala index cb99ea9aa5..f74f92b3c3 100644 --- a/core/src/test/scala/org/apache/spark/sql/store/RowTableTest.scala +++ b/core/src/test/scala/org/apache/spark/sql/store/RowTableTest.scala @@ -303,7 +303,7 @@ class RowTableTest val rdd1 = sc.parallelize(data1, data1.length).map(s => new Data(s(0), s(1), s(2))) val dataDF1 = snc.createDataFrame(rdd1) - dataDF1.write.format("row").mode(SaveMode.Overwrite).options(props).saveAsTable(tableName) + dataDF1.write.insertInto(tableName) snc.sql("PUT INTO TABLE " + tableName + " SELECT * FROM tempTable") diff --git a/core/src/test/scala/org/apache/spark/sql/store/SnappyCatalogSuite.scala b/core/src/test/scala/org/apache/spark/sql/store/SnappyCatalogSuite.scala index d1cc1818a5..a2afa6ed30 100644 --- a/core/src/test/scala/org/apache/spark/sql/store/SnappyCatalogSuite.scala +++ b/core/src/test/scala/org/apache/spark/sql/store/SnappyCatalogSuite.scala @@ -35,16 +35,16 @@ package org.apache.spark.sql.store import io.snappydata.SnappyFunSuite -import org.scalatest.{BeforeAndAfterAll, BeforeAndAfter} +import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll} -import org.apache.spark.sql.types.{StringType, StructField, StructType, IntegerType} -import org.apache.spark.sql.{SnappySession, AnalysisException} -import org.apache.spark.sql.catalog.{Column, Function, Table, Database} -import org.apache.spark.sql.catalyst.{ScalaReflection, FunctionIdentifier, TableIdentifier} +import org.apache.spark.sql.catalog.{Column, Database, Function, Table} import org.apache.spark.sql.catalyst.catalog._ import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionInfo} import org.apache.spark.sql.catalyst.plans.logical.Range +import org.apache.spark.sql.catalyst.{FunctionIdentifier, ScalaReflection, TableIdentifier} import org.apache.spark.sql.internal.CatalogImpl +import org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructType} +import org.apache.spark.sql.{AnalysisException, SnappySession, SparkSupport} import org.apache.spark.util.Utils /** @@ -53,7 +53,7 @@ import org.apache.spark.util.Utils class SnappyCatalogSuite extends SnappyFunSuite with BeforeAndAfter - with BeforeAndAfterAll { + with BeforeAndAfterAll with SparkSupport { var snappySession: SnappySession = _ @@ -105,7 +105,7 @@ class SnappyCatalogSuite extends SnappyFunSuite private def createTempFunction(name: String): Unit = { val info = new ExpressionInfo("className", name) val tempFunc = (e: Seq[Expression]) => e.head - sessionCatalog.createTempFunction(name, info, tempFunc, ignoreIfExists = false) + internals.registerFunction(snappySession, FunctionIdentifier(name, None), info, tempFunc) } private def dropFunction(name: String, db: Option[String] = None): Unit = { @@ -343,7 +343,7 @@ class SnappyCatalogSuite extends SnappyFunSuite /** * A collection of utility fields and methods for tests related to the [[ExternalCatalog]]. */ -abstract class CatalogTestUtils { +abstract class CatalogTestUtils extends SparkSupport { // Unimplemented methods val tableInputFormat: String @@ -400,7 +400,7 @@ abstract class CatalogTestUtils { def newUriForDatabase(): String = Utils.createTempDir().toURI.toString.stripSuffix("/") def newDb(name: String): CatalogDatabase = { - CatalogDatabase(name, name + " description", newUriForDatabase(), Map.empty) + internals.newCatalogDatabase(name, name + " description", newUriForDatabase(), Map.empty) } def newTable(name: String, db: String): CatalogTable = newTable(name, Some(db)) diff --git a/core/src/test/scala/org/apache/spark/sql/store/SnappyJoinSuite.scala b/core/src/test/scala/org/apache/spark/sql/store/SnappyJoinSuite.scala index 8e3512fec2..9434b06f85 100644 --- a/core/src/test/scala/org/apache/spark/sql/store/SnappyJoinSuite.scala +++ b/core/src/test/scala/org/apache/spark/sql/store/SnappyJoinSuite.scala @@ -183,7 +183,7 @@ class SnappyJoinSuite extends SnappyFunSuite with BeforeAndAfterAll { } - test("Check shuffle in operations with partition pruning"){ + test("Check shuffle in operations with partition pruning") { val t1 = "t1" val t2 = "t2" @@ -196,20 +196,19 @@ class SnappyJoinSuite extends SnappyFunSuite with BeforeAndAfterAll { "options( partition_by 'ol_1_int_id', buckets '16')") var df = snc.sql(s"select sum(ol_1_int2_id) from $t1 where ol_1_int_id=1") - checkForShuffle(df.logicalPlan, snc , shuffleExpected = false) + checkForShuffle(df.logicalPlan, snc, shuffleExpected = false) // with limit df = snc.sql(s"select sum(ol_1_int2_id) from $t1 where ol_1_int_id=1 limit 1") - checkForShuffle(df.logicalPlan, snc , shuffleExpected = false) + checkForShuffle(df.logicalPlan, snc, shuffleExpected = false) df = snc.sql(s"update $t1 set ol_1_str_id = '3' where ol_1_int_id in (" + s"select ol_1_int_id from $t2 where $t2.ol_1_int_id=1)") - checkForShuffle(df.logicalPlan, snc , shuffleExpected = false) - - snc.dropTable("t1"); - snc.dropTable("t2"); + checkForShuffle(df.logicalPlan, snc, shuffleExpected = false) + snc.dropTable("t1") + snc.dropTable("t2") } /** @@ -219,7 +218,7 @@ class SnappyJoinSuite extends SnappyFunSuite with BeforeAndAfterAll { def checkForShuffle(plan: LogicalPlan, snc: SnappyContext, shuffleExpected: Boolean): Unit = { - val qe = new QueryExecution(snc.snappySession, plan) + val qe = snc.snappySession.executePlan(plan) // logInfo(qe.executedPlan) val lj = qe.executedPlan collect { case ex: Exchange => ex @@ -228,10 +227,10 @@ class SnappyJoinSuite extends SnappyFunSuite with BeforeAndAfterAll { if (lj.isEmpty) sys.error(s"Shuffle Expected , but was not found") } else { lj.foreach(a => a.child.collect { - // this means no Exhange should have child as PartitionedPhysicalRDD - case p: PartitionedPhysicalScan => sys.error( + // this means no Exchange should have child as PartitionedPhysicalRDD + case _: PartitionedPhysicalScan => sys.error( s"Did not expect exchange with partitioned scan with same partitions") - case p: RowDataSourceScanExec => sys.error( + case _: RowDataSourceScanExec => sys.error( s"Did not expect RowDataSourceScanExec with PartitionedDataSourceScan") case _ => // do nothing, may be some other Exchange and not with scan }) @@ -650,7 +649,7 @@ class SnappyJoinSuite extends SnappyFunSuite with BeforeAndAfterAll { s"where c.cid= f.cid and f.sid = so.sid and c.cid = so.cid" + s" and subTotal >13 and c.cid>3 and f.tid = 1") - assert(df.collect().size === 2) + assert(df.collect().length === 2) df = snc.sql(s" select f.cid, cust_name, f.sid, so.sid," + s" so.qty, subTotal, oid, order_time, ask from" + @@ -659,16 +658,16 @@ class SnappyJoinSuite extends SnappyFunSuite with BeforeAndAfterAll { s" trade.sellorders so" + s" where c.cid= f.cid and f.sid = so.sid and c.cid = so.cid" + s" and subTotal >13 and c.cid>1 and f.tid = 1") - assert(df.collect().size === 4) + assert(df.collect().length === 4) df = snc.sql(s"select n.cid, cust_name, n.securities, n.cash, n.tid, " + s"c.cid from trade.customers c, trade.networth n where n.cid = c.cid" + s" and n.tid = 1 and c.cid > 3") - assert(df.collect().size === 3) + assert(df.collect().length === 3) df = snc.sql(s"select n.cid, cust_name, n.securities, n.cash, n.tid, c.cid" + s" from trade.customers c, trade.networth n where n.cid = c.cid" + s" and n.tid = 1 and c.cid > 5") - assert(df.collect().size === 1) + assert(df.collect().length === 1) } private def dropTables(): Unit = { @@ -699,7 +698,7 @@ class SnappyJoinSuite extends SnappyFunSuite with BeforeAndAfterAll { s" trade.sellorders so" + s" where c.cid= f.cid and f.sid = so.sid and c.cid = so.cid" + s" and subTotal > 4 and c.cid = 1 and f.tid = 1") - assert(df.collect().size === 1) + assert(df.collect().length === 1) df = snc.sql(s" select f.cid, cust_name, f.sid, so.sid," + s" so.qty, subTotal, oid, order_time, ask from" + s" trade.customers c," + @@ -707,7 +706,7 @@ class SnappyJoinSuite extends SnappyFunSuite with BeforeAndAfterAll { s" trade.sellorders so" + s" where c.cid= f.cid and f.sid = so.sid and c.cid = so.cid" + s" and subTotal > 4 and c.cid = 2 and f.tid = 1") - assert(df.collect().size === 1) + assert(df.collect().length === 1) dropTables() loadTables("COLUMN", "", "partition_by 'cid'", ", colocate_with 'trade.customers'") diff --git a/core/src/test/scala/org/apache/spark/sql/store/TokenizationTest.scala b/core/src/test/scala/org/apache/spark/sql/store/TokenizationTest.scala index 59bcd60e68..8982f7f228 100644 --- a/core/src/test/scala/org/apache/spark/sql/store/TokenizationTest.scala +++ b/core/src/test/scala/org/apache/spark/sql/store/TokenizationTest.scala @@ -20,7 +20,6 @@ import scala.collection.mutable.ArrayBuffer import io.snappydata.core.{Data, TestData2} import io.snappydata.{Property, SnappyFunSuite, SnappyTableStatsProviderService} -import jdk.internal.org.objectweb.asm.tree.analysis.AnalyzerException import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll} import org.apache.spark.Logging @@ -696,7 +695,9 @@ class TokenizationTest test("SNAP-1894") { val snap = snc - val row = identity[(java.lang.Integer, java.lang.Double)](_) + + def row(i: java.lang.Integer, d: java.lang.Double): (java.lang.Integer, java.lang.Double) = + (i, d) import snap.implicits._ lazy val l = Seq( @@ -1033,18 +1034,18 @@ class TokenizationTest // null, non-null combinations of updates // implicit int to string cast will cause it to be null (SNAP-2039) - // Update [SNAP-2052]: this behavior is updated to fail the update query if a string expression is - // as part of arithmetic operator in update expression. Explicity casting the srring to int is a - // workaround. However, it is important to note that casting a non-numeric string value to int will - // still end up in a NULL. + // Update [SNAP-2052]: this behavior is updated to fail the update query if a string expression + // is as part of arithmetic operator in update expression. Explicity casting the srring to int + // is a workaround. However, it is important to note that casting a non-numeric string value to + // int will still end up in a NULL. try { res2 = snc.sql(s"update $colTableName set DEST = DEST + 1000 where " + "depdelay = 0 and arrdelay > 0 and airtime > 350").collect() fail("AnalyzerException was expected here") } catch { case ex: AnalysisException => - val expectedMessage = "Implicit type casting of string type to numeric type is not performed" + - " for update statements.;" + val expectedMessage = "Implicit type casting of string type to numeric type is not " + + "performed for update statements.;" assertResult(expectedMessage)(ex.getMessage) } diff --git a/core/src/test/scala/org/apache/spark/sql/streaming/SnappyStoreSinkProviderSecuritySuite.scala b/core/src/test/scala/org/apache/spark/sql/streaming/SnappyStoreSinkProviderSecuritySuite.scala index 4c163d5ab1..91dc5af6e8 100644 --- a/core/src/test/scala/org/apache/spark/sql/streaming/SnappyStoreSinkProviderSecuritySuite.scala +++ b/core/src/test/scala/org/apache/spark/sql/streaming/SnappyStoreSinkProviderSecuritySuite.scala @@ -243,9 +243,12 @@ class SnappyStoreSinkProviderSecuritySuite extends SnappyFunSuite fail("StreamingQueryException was expected") } catch { case x: StreamingQueryException => - val expectedMessage = "User 'GEMFIRE5' does not have SELECT permission on column" + + val expectedMessage1 = "User 'GEMFIRE5' does not have SELECT permission on column" + " 'STREAM_QUERY_ID' of table 'GEMGROUP1'.'SNAPPYSYS_INTERNAL____SINK_STATE_TABLE'." - assert(x.getCause.getCause.getMessage.equals(expectedMessage)) + val expectedMessage2 = "User 'GEMFIRE5' does not have UPDATE permission on column " + + " 'BATCH_ID' of table 'GEMGROUP1'.'SNAPPYSYS_INTERNAL____SINK_STATE_TABLE'." + val cause = if (x.getCause.getCause eq null) x.getCause else x.getCause.getCause + assert(cause.getMessage === expectedMessage1 || cause.getMessage === expectedMessage2) } finally { streamingQuery1.stop() } diff --git a/core/src/test/scala/org/apache/spark/sql/streaming/SnappyStoreSinkProviderSuite.scala b/core/src/test/scala/org/apache/spark/sql/streaming/SnappyStoreSinkProviderSuite.scala index 208bc97469..3628744ad4 100644 --- a/core/src/test/scala/org/apache/spark/sql/streaming/SnappyStoreSinkProviderSuite.scala +++ b/core/src/test/scala/org/apache/spark/sql/streaming/SnappyStoreSinkProviderSuite.scala @@ -26,7 +26,7 @@ import com.pivotal.gemfirexd.internal.shared.common.reference.SQLState.SNAPPY_CA import io.snappydata.SnappyFunSuite import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll} -import org.apache.spark.sql.{Dataset, Row, SnappyContext, SnappySession} +import org.apache.spark.sql.{DataFrame, Dataset, Row, SnappyContext, SnappySession} import org.apache.spark.sql.catalyst.encoders.RowEncoder import org.apache.spark.sql.execution.CatalogStaleException import org.apache.spark.sql.kafka010.KafkaTestUtils @@ -457,7 +457,7 @@ class SnappyStoreSinkProviderSuite extends SnappyFunSuite private def createAndStartStreamingQuery(topic: String, testId: Int, withEventTypeColumn: Boolean = true, withQueryName: Boolean = true, options: Map[String, String] = Map.empty) = { - val streamingDF = session + val streamingDF: DataFrame = session .readStream .format("kafka") .option("kafka.bootstrap.servers", kafkaTestUtils.brokerAddress) diff --git a/core/src/test/scala/org/apache/spark/streaming/SnappyStreamingContextSuite.scala b/core/src/test/scala/org/apache/spark/streaming/SnappyStreamingContextSuite.scala index 7b6feff020..c64d1faf9e 100644 --- a/core/src/test/scala/org/apache/spark/streaming/SnappyStreamingContextSuite.scala +++ b/core/src/test/scala/org/apache/spark/streaming/SnappyStreamingContextSuite.scala @@ -51,9 +51,6 @@ class SnappyStreamingContextSuite extends SnappyFunSuite with Eventually stopAll() } - before { - } - after { val activeSsc = SnappyStreamingContext.getActive activeSsc match { diff --git a/docs/best_practices/important_settings.md b/docs/best_practices/important_settings.md index 81aa22e626..3a945ad752 100644 --- a/docs/best_practices/important_settings.md +++ b/docs/best_practices/important_settings.md @@ -169,7 +169,7 @@ Optionally when using the `-XX:+HeapDumpOnOutOfMemoryError` option, you can spec SnappyData uses generated code for best performance for most of the queries and internal operations. This is done for both Spark-side whole-stage code generation for queries, for example,[Technical Preview of Apache Spark 2.0 blog]( https://databricks.com/blog/2016/05/11/apache-spark-2-0-technical-preview-easier-faster-and-smarter.html), and internally by SnappyData for many operations. For example, rolling over data from row buffer to column store or merging batches among others. The point key lookup queries on row tables, and JDBC inserts bypass this and perform direct operations. However, for all other operations, the product uses code generation for best performance. In many cases, the first query execution is slightly slower than subsequent query executions. This is primarily due to the overhead of compilation of generated code for the query plan and optimized machine code generation by JVM's hotspot JIT. -Each distinct piece of generated code is a separate class which is loaded using its own ClassLoader. To reduce these overheads in multiple runs, this class is reused using a cache whose size is controlled by **spark.sql.codegen.cacheSize** property (default is 2000). Thus when the size limit of the cache is breached, the older classes that are used for a while gets removed from the cache. +Each distinct piece of generated code is a separate class which is loaded using its own ClassLoader. To reduce these overheads in multiple runs, this class is reused using a cache whose size is controlled by **spark.sql.codegen.cache.maxEntries** property (default is 2000). Thus when the size limit of the cache is breached, the older classes that are used for a while gets removed from the cache. Further to minimize the generated plans, SnappyData performs tokenization of the values that are most constant in queries by default. Therefore the queries that differ only in constants can still create the same generated code plan. Thus if an application has a fixed number of query patterns that are used repeatedly, then the effect of the slack during the first execution, due to compilation and JIT, is minimized. @@ -177,7 +177,7 @@ Thus if an application has a fixed number of query patterns that are used repeat !!!note A single query pattern constitutes of queries that differ only in constant values that are embedded in the query string. -For cases where the application has many query patterns, you can increase the value of **spark.sql.codegen.cacheSize** property from the default size of **2000**. +For cases where the application has many query patterns, you can increase the value of **spark.sql.codegen.cache.maxEntries** property from the default size of **2000**. You can also increase the value for JVM's **ReservedCodeCacheSize** property and add additional RAM capacity accordingly. diff --git a/docs/best_practices/setup_cluster.md b/docs/best_practices/setup_cluster.md index a212b17630..0795d72595 100644 --- a/docs/best_practices/setup_cluster.md +++ b/docs/best_practices/setup_cluster.md @@ -69,7 +69,7 @@ Two cores are statically assigned to the low latency pool. Also, the low latency If a query requires all 30 partitions and no low latency queries are running at that time, all 30 cores are assigned to the first query. However, when a low latency query is assigned, the scheduler does its best to allocate cores as soon as tasks from the earlier query finish. -Applications can explicitly configure to use a particular pool for the current session using a SQL configuration property, `snappydata.scheduler.pool`. For example, the `set snappydata.scheduler.pool=lowlatency` command sets the pool as low latency pool for the current session. +Applications can explicitly configure to use a particular pool for the current session using a SQL configuration property, `spark.scheduler.pool`. For example, the `set spark.scheduler.pool=lowlatency` command sets the pool as low latency pool for the current session. New pools can be added and properties of the existing pools can be configured by modifying the **conf/fairscheduler.xml** file. We do not recommend changing the pool names (`default` and `lowlatency`). @@ -80,7 +80,7 @@ The product is configured with two out-of-the-box pools, that is the **Default p The [**Stages**](/monitoring/monitoring.md#stages) tab on the SnappyData Monitoring Console shows the available pools. When you track a job for an SQL query on the [**SQL**](/monitoring/monitoring.md#sql) tab, it shows the pool that is used in the **Pool Name** column. In-built tasks such as ingestion can show lower priority pools by default to give priority to foreground queries. To configure such priority, do the following: 1. Define the pools in **conf/fairscheduler.xml** -2. Set a pool for a job using Spark API or use `set snappydata.scheduler.pool` property in a SnappySession. +2. Set a pool for a job using Spark API or use `set spark.scheduler.pool` property in a SnappySession. To configure the priority based on specific requirements, you can also either permit the users to set the priority for queries or add some pool allocation logic in the application as per client requirements. diff --git a/docs/configuring_cluster/configuring_cluster.md b/docs/configuring_cluster/configuring_cluster.md index 507c2ba767..3306ae5407 100644 --- a/docs/configuring_cluster/configuring_cluster.md +++ b/docs/configuring_cluster/configuring_cluster.md @@ -119,7 +119,7 @@ Refer to the [SnappyData properties](property_description.md) for the complete l |-spark.jobserver.max-jobs-per-context|The number of jobs that can be run simultaneously in the context. The default is 8.| |-spark.local.dir|Directory to use for "scratch" space in SnappyData, including map output files and RDDs that get stored on disk. This should be on a fast, local disk in your system. It can also be a comma-separated list of multiple directories on different disks.| |-spark.network.timeout|The default timeout for all network interactions while running queries. | -|-spark.sql.codegen.cacheSize|Size of the generated code cache that is used by Spark, in the SnappyData Spark distribution, and by SnappyData. The default is 2000.| +|-spark.sql.codegen.cache.maxEntries|Size of the generated code cache that is used by Spark, in the SnappyData Spark distribution, and by SnappyData. The default is 2000.| |-spark.ssl.enabled|Enables or disables Spark layer encryption. The default is false.| |-spark.ssl.keyPassword|The password to the private key in the key store.| |-spark.ssl.keyStore|Path to the key store file. The path can be absolute or relative to the directory in which the process is started.| diff --git a/docs/configuring_cluster/property_description.md b/docs/configuring_cluster/property_description.md index 2c5099984c..c5631d4798 100644 --- a/docs/configuring_cluster/property_description.md +++ b/docs/configuring_cluster/property_description.md @@ -106,7 +106,7 @@ The following list of commonly used configuration properties can be set to confi |-spark.local.dir|Directory to use for "scratch" space in SnappyData, including map output files and RDDs that get stored on disk. This should be on a fast, local disk in your system. It can also be a comma-separated list of multiple directories on different disks. For more information, refer to [Best Practices](../best_practices/important_settings.md#spark-local-dir).|LeadServer| |-spark.network.timeout|The default timeout for all network interactions while running queries.|Lead| |-spark.sql.autoBroadcastJoinThreshold|Configures the maximum size in bytes for a table that is broadcast to all server nodes when performing a join. By setting this value to **-1** broadcasting can be disabled. | -|-spark.sql.codegen.cacheSize|Size of the generated code cache. This effectively controls the maximum number of query plans whose generated code (Classes) is cached. The default is 2000. |Lead| +|-spark.sql.codegen.cache.maxEntries|Size of the generated code cache. This effectively controls the maximum number of query plans whose generated code (Classes) is cached. The default is 2000. |Lead| |-spark.sql.aqp.numBootStrapTrials|Number of bootstrap trials to do for calculating error bounds. The default value is100. This property must be set in the **conf/leads** file.| |-spark.sql.aqp.error|Maximum relative error tolerable in the approximate value calculation. It should be a fractional value not exceeding 1. The default value is0.2. This property can be set as connection property in the Snappy SQL shell.| |-spark.sql.aqp.confidence|Confidence with which the error bounds are calculated for the approximate value. It should be a fractional value not exceeding 1. The default value is0.95. This property can be set as connection property in the Snappy SQL shell.| @@ -187,7 +187,7 @@ node-l -heap-size=4096m -spark.ui.port=9090 -locators=node-b:8888,node-a:9999 -s |-snappydata.sql.partitionPruning|Use this property to set/unset the partition pruning of queries.| |-snappydata.sql.tokenize|Use this property to enable/disable tokenization.| |-snappydata.cache.putIntoInnerJoinResultSize| Use this property with extreme limits such as 1K and 10GB. The default is 100 MB.| -|-snappydata.scheduler.pool|Use this property to define scheduler pool to either default or low latency. You can also assign queries to different pools.| +|-spark.scheduler.pool|Use this property to define scheduler pool to either default or low latency. You can also assign queries to different pools.| |-snappydata.enable-experimental-features|Use this property to enable and disable experimental features. You can call out in case some features are completely broken and need to be removed from the product.| |-snappydata.sql.planCaching|Use this property to enable/disable plan caching. By default it is disabled. |Lead| |sync-commits| See [sync-commits](/reference/configuration_parameters/sync-commits.md)|| diff --git a/docs/monitoring/monitoring.md b/docs/monitoring/monitoring.md index afbcb8f019..dfebfe886b 100644 --- a/docs/monitoring/monitoring.md +++ b/docs/monitoring/monitoring.md @@ -198,7 +198,7 @@ The SQL section shows all the queries and their corresponding details along with | **Colocated** | When colocated tables are joined on the partitioning columns, the join happens locally on the node where data is present, without the need of shuffling the data. This improves the performance of the query significantly instead of broadcasting the data across all the data partitions. | |**Whole-Stage Code Generation** | A whole stage code generation node compiles a sub-tree of plans that support code generation together into a single Java function, which helps improve execution performance. | | **Per node execution timing** | Displays the time required for the execution of each node. If there are too many rows that are not getting filtered or exchanged. | -| **Pool Name** | Default/Low Latency. Applications can explicitly configure the use of this pool using a SQL command `set snappydata.scheduler.pool=lowlatency`. | +| **Pool Name** | Default/Low Latency. Applications can explicitly configure the use of this pool using a SQL command `set spark.scheduler.pool=lowlatency`. | |**Query Node Details**| Hover over a component to view its details. | | **Filter** | Displays the number of rows that are filtered for each node. | | **Joins** | If HashJoin puts pressure on memory, you can change the HashJoin size to use SortMergeJoin to avoid on-heap memory pressure. | diff --git a/dtests/src/test/scala/io/snappydata/hydra/consistency/ConsistencyTest.scala b/dtests/src/test/scala/io/snappydata/hydra/consistency/ConsistencyTest.scala index 4870bea9dd..af2db58da4 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/consistency/ConsistencyTest.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/consistency/ConsistencyTest.scala @@ -96,7 +96,7 @@ class ConsistencyTest { pw.flush() } catch { case se: SQLException => - pw.println(s"${printTime} Got exception while executing select query for $op", se) + pw.println(s"$printTime Got exception while executing select query for $op: $se") pw.flush() } } diff --git a/dtests/src/test/scala/org/apache/spark/sql/DistIndexTestUtils.scala b/dtests/src/test/scala/org/apache/spark/sql/DistIndexTestUtils.scala index 52c2d0b1f0..728e5f6bb9 100644 --- a/dtests/src/test/scala/org/apache/spark/sql/DistIndexTestUtils.scala +++ b/dtests/src/test/scala/org/apache/spark/sql/DistIndexTestUtils.scala @@ -24,7 +24,8 @@ import io.snappydata.benchmark.snappy.tpch.QueryExecutor import io.snappydata.benchmark.snappy.{SnappyAdapter, TPCH} import org.apache.spark.sql.catalyst.plans.logical.Sort -import org.apache.spark.util.Benchmark +import org.apache.spark.sql.execution.benchmark.BenchmarkWithCleanup +import org.apache.spark.sql.execution.benchmark.ColumnCacheBenchmark.addCaseWithCleanup object DistIndexTestUtils { @@ -40,7 +41,8 @@ object DistIndexTestUtils { val size = qryProvider.estimateSizes(query, tableSizes, executor) // scalastyle:off println pw.println(s"$qNum size $size") - val b = new Benchmark(s"JoinOrder optimization", size, minNumIters = 5, output = Some(fos)) + val b = new BenchmarkWithCleanup( + s"JoinOrder optimization", size, minNumIters = 5, output = Some(fos)) def case1(): Unit = snc.setConf(io.snappydata.Property.EnableExperimentalFeatures.name, "false") @@ -62,14 +64,14 @@ object DistIndexTestUtils { def evalBaseTPCH = qryProvider.execute(query, executor) - - b.addCase(s"$qNum baseTPCH index = F", numIters = 0, prepare = case3, cleanup = () => {})( - _ => evalBaseTPCH) + addCaseWithCleanup(b, s"$qNum baseTPCH index = F", numIters = 0, prepare = case3, + cleanup = () => {})(_ => evalBaseTPCH) // b.addCase(s"$qNum baseTPCH joinOrder = T", prepare = case2)(i => evalBaseTPCH) // b.addCase(s"$qNum snappyMods joinOrder = F", prepare = case1)(i => evalSnappyMods(false)) // b.addCase(s"$qNum snappyMods joinOrder = T", prepare = case2)(i => evalSnappyMods(false)) - b.addCase(s"$qNum baseTPCH index = T", numIters = 0, prepare = case3, cleanup = () => {})(_ => - evalBaseTPCH) + addCaseWithCleanup(b, s"$qNum baseTPCH index = T", numIters = 0, prepare = case3, + cleanup = () => {})(_ => evalBaseTPCH) + b.run() } diff --git a/encoders/build.gradle b/encoders/build.gradle index f55506666d..9e5490c218 100644 --- a/encoders/build.gradle +++ b/encoders/build.gradle @@ -34,10 +34,20 @@ dependencies { exclude(group: 'org.scala-lang', module: 'scala-compiler') } - compileOnly "org.apache.spark:spark-core_${scalaBinaryVersion}:${sparkVersion}" - compileOnly "org.apache.spark:spark-catalyst_${scalaBinaryVersion}:${sparkVersion}" - compileOnly "org.apache.spark:spark-sql_${scalaBinaryVersion}:${sparkVersion}" - compileOnly "org.apache.spark:spark-hive_${scalaBinaryVersion}:${sparkVersion}" + // always use stock spark so that snappy extensions don't get accidently + // included here in snappy-jdbc code. + if (System.properties.containsKey('ideaBuild') && new File(rootDir, 'spark/build.gradle').exists()) { + compile project(':snappy-spark:snappy-spark-core_' + scalaBinaryVersion) + compile project(':snappy-spark:snappy-spark-catalyst_' + scalaBinaryVersion) + compile project(':snappy-spark:snappy-spark-sql_' + scalaBinaryVersion) + compile project(':snappy-spark:snappy-spark-hive_' + scalaBinaryVersion) + } else { + compileOnly "org.apache.spark:spark-core_${scalaBinaryVersion}:${sparkConnectorVersion}" + compileOnly "org.apache.spark:spark-catalyst_${scalaBinaryVersion}:${sparkConnectorVersion}" + compileOnly "org.apache.spark:spark-sql_${scalaBinaryVersion}:${sparkConnectorVersion}" + compileOnly "org.apache.spark:spark-hive_${scalaBinaryVersion}:${sparkConnectorVersion}" + } + compileOnly "org.eclipse.jetty:jetty-servlet:${jettyVersion}" compile project(":snappy-jdbc_${scalaBinaryVersion}") if (new File(rootDir, 'store/build.gradle').exists()) { @@ -46,6 +56,7 @@ dependencies { compile group: 'io.snappydata', name: 'snappydata-store-core', version: snappyStoreVersion } + compile "org.codehaus.janino:janino:${janinoVersion}" compile "org.eclipse.collections:eclipse-collections-api:${eclipseCollectionsVersion}" compile "org.eclipse.collections:eclipse-collections:${eclipseCollectionsVersion}" compile "org.apache.tomcat:tomcat-jdbc:${tomcatJdbcVersion}" diff --git a/encoders/src/main/scala/org/apache/spark/sql/catalyst/util/SerializedMap.scala b/encoders/src/main/scala/org/apache/spark/sql/catalyst/util/SerializedMap.scala index 00b3689517..1cce951549 100644 --- a/encoders/src/main/scala/org/apache/spark/sql/catalyst/util/SerializedMap.scala +++ b/encoders/src/main/scala/org/apache/spark/sql/catalyst/util/SerializedMap.scala @@ -149,13 +149,13 @@ final class SerializedMap extends MapData override def write(kryo: Kryo, out: Output): Unit = { val bytes = toBytes out.writeInt(bytes.length) - out.write(bytes) + out.writeBytes(bytes) } override def read(kryo: Kryo, in: Input): Unit = { val size = in.readInt val bytes = new Array[Byte](size) - in.read(bytes) + in.readBytes(bytes) pointTo(bytes, Platform.BYTE_ARRAY_OFFSET) if (size != sizeInBytes) { throw new IOException( diff --git a/encoders/src/main/scala/org/apache/spark/sql/catalyst/util/SerializedRow.scala b/encoders/src/main/scala/org/apache/spark/sql/catalyst/util/SerializedRow.scala index 0bd9ac826b..c6af995a59 100644 --- a/encoders/src/main/scala/org/apache/spark/sql/catalyst/util/SerializedRow.scala +++ b/encoders/src/main/scala/org/apache/spark/sql/catalyst/util/SerializedRow.scala @@ -415,7 +415,7 @@ trait SerializedRowData extends SpecializedGetters out.writeInt(bytes.length) out.writeVarInt(this.skipBytes, true) out.writeVarInt(this.nFields, true) - out.write(bytes) + out.writeBytes(bytes) } override final def read(kryo: Kryo, in: Input): Unit = { @@ -425,7 +425,7 @@ trait SerializedRowData extends SpecializedGetters this.bitSetWidthInBytes = calculateBitSetWidthInBytes(nFields) this.baseOffset = Platform.BYTE_ARRAY_OFFSET val bytes = new Array[Byte](sizeInBytes) - in.read(bytes) + in.readBytes(bytes) this.baseObject = bytes } diff --git a/encoders/src/main/scala/org/apache/spark/sql/execution/columnar/encoding/ColumnDeltaEncoder.scala b/encoders/src/main/scala/org/apache/spark/sql/execution/columnar/encoding/ColumnDeltaEncoder.scala index 134eb81fd8..799065a129 100644 --- a/encoders/src/main/scala/org/apache/spark/sql/execution/columnar/encoding/ColumnDeltaEncoder.scala +++ b/encoders/src/main/scala/org/apache/spark/sql/execution/columnar/encoding/ColumnDeltaEncoder.scala @@ -674,7 +674,7 @@ object DeltaWriter extends Logging { val evaluator = new CompilerFactory().newScriptEvaluator() evaluator.setClassName("io.snappydata.execute.GeneratedDeltaWriterFactory") evaluator.setParentClassLoader(getClass.getClassLoader) - evaluator.setDefaultImports(defaultImports) + evaluator.setDefaultImports(defaultImports: _*) val (name, complexType) = dataType match { case BooleanType => ("Boolean", "") diff --git a/examples/build.gradle b/examples/build.gradle index bb1829f774..ad55636b32 100644 --- a/examples/build.gradle +++ b/examples/build.gradle @@ -25,13 +25,6 @@ sourceSets.test.java.srcDirs = [] sourceSets.test.scala.srcDir 'src/test/scala' if (!(new File(rootDir, 'cluster/build.gradle').exists())) { - ext { - scalaBinaryVersion = '2.11' - scalatestVersion = '2.2.6' - scalaVersion = scalaBinaryVersion + '.8' - snappyVersion = '1.1.0' - } - repositories { mavenCentral() maven { url 'https://dl.bintray.com/big-data/maven' } @@ -43,7 +36,7 @@ if (!(new File(rootDir, 'cluster/build.gradle').exists())) { dependencies { compile 'org.scala-lang:scala-library:' + scalaVersion testCompile "org.scalatest:scalatest_${scalaBinaryVersion}:${scalatestVersion}" - compile "io.snappydata:snappydata-cluster_2.11:${snappyVersion}" + compile "io.snappydata:snappydata-cluster_2.11:${version}" } archivesBaseName = 'snappydata-examples_' + '2.11' diff --git a/examples/src/main/scala/org/apache/spark/examples/snappydata/StreamingExample.scala b/examples/src/main/scala/org/apache/spark/examples/snappydata/StreamingExample.scala index 3889b0bbdf..6a2c7ba10f 100644 --- a/examples/src/main/scala/org/apache/spark/examples/snappydata/StreamingExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/snappydata/StreamingExample.scala @@ -18,20 +18,23 @@ package org.apache.spark.examples.snappydata import java.io.File -import java.lang.{Integer => JInt} import java.net.InetSocketAddress +import java.util.Properties import java.util.concurrent.TimeUnit -import java.util.{Properties, Map => JMap} + +import scala.language.postfixOps +import scala.util.Random import kafka.admin.AdminUtils import kafka.api.Request import kafka.server.{KafkaConfig, KafkaServer} import kafka.utils.ZkUtils -import org.apache.kafka.clients.consumer.KafkaConsumer import org.apache.kafka.clients.producer.{KafkaProducer, Producer, ProducerRecord, RecordMetadata} -import org.apache.kafka.common.TopicPartition -import org.apache.kafka.common.serialization.{StringDeserializer, StringSerializer} +import org.apache.kafka.common.network.ListenerName +import org.apache.kafka.common.serialization.StringSerializer import org.apache.log4j.{Level, Logger} +import org.apache.zookeeper.server.{NIOServerCnxnFactory, ZooKeeperServer} + import org.apache.spark.SparkConf import org.apache.spark.internal.Logging import org.apache.spark.jdbc.{ConnectionConfBuilder, ConnectionUtil} @@ -39,15 +42,6 @@ import org.apache.spark.sql.streaming.{SchemaDStream, StreamToRowsConverter} import org.apache.spark.sql.{Row, SparkSession} import org.apache.spark.streaming.{Seconds, SnappyStreamingContext} import org.apache.spark.util.Utils -import org.apache.zookeeper.server.{NIOServerCnxnFactory, ZooKeeperServer} -import org.json4s.NoTypeHints -import org.json4s.jackson.Serialization - -import scala.collection.JavaConverters._ -import scala.collection.mutable.HashMap -import scala.language.postfixOps -import scala.util.Random -import scala.util.control.NonFatal /** * An example showing usage of streaming with SnappyData @@ -75,7 +69,9 @@ import scala.util.control.NonFatal */ object StreamingExample { - def main(args: Array[String]) { + // scalastyle:off println + + def main(args: Array[String]): Unit = { // reducing the log level to minimize the messages on console Logger.getLogger("org").setLevel(Level.ERROR) Logger.getLogger("akka").setLevel(Level.ERROR) @@ -84,13 +80,13 @@ object StreamingExample { println("Initializing a SnappyStreamingContext") val spark: SparkSession = SparkSession - .builder + .builder() .appName(getClass.getSimpleName) .master("local[*]") // sys-disk-dir attribute specifies the directory where persistent data is saved .config("snappydata.store.sys-disk-dir", dataDirAbsolutePath) .config("snappydata.store.log-file", dataDirAbsolutePath + "/SnappyDataExample.log") - .getOrCreate + .getOrCreate() val snsc = new SnappyStreamingContext(spark.sparkContext, Seconds(1)) @@ -143,11 +139,11 @@ object StreamingExample { println() // Execute this query once every second. Output is a SchemaDStream. println("Registering a continuous query to to be executed every second on the stream table") - val resultStream: SchemaDStream = snsc.registerCQ("select publisher, count(bid) as bidCount from " + - "adImpressionStream window (duration 1 seconds, slide 1 seconds) group by publisher") + val resultStream: SchemaDStream = snsc.registerCQ("select publisher, count(bid) as bidCount " + + "from adImpressionStream window (duration 1 seconds, slide 1 seconds) group by publisher") // this conf is used to get a connection a JDBC connection - val conf = new ConnectionConfBuilder(snsc.snappySession).build + val conf = new ConnectionConfBuilder(snsc.snappySession).build() println() // process the stream data returned by continuous query and update publisher_bid_counts table @@ -177,7 +173,7 @@ object StreamingExample { } }) - snsc.start + snsc.start() println("Publishing messages on Kafka") publishKafkaMessages(utils, topic) @@ -193,6 +189,8 @@ object StreamingExample { System.exit(0) } + // scalastyle:off println + def createAndGetDataDir: String = { // creating a directory to save all persistent data val dataDir = "./" + "snappydata_examples_data" @@ -206,13 +204,13 @@ object StreamingExample { val currentTime = System.currentTimeMillis() // bids with comma separated fields - //timestamp, publisher,advertiser,web,geo,bid,cookie - val bid1 = currentTime + ",publisher1,advt1,pb1.web,US," + scala.util.Random.nextDouble() + ",23543" - val bid2 = currentTime + ",publisher2,advt1,pb1.web,US," + scala.util.Random.nextDouble() + ",45445" - val bid3 = currentTime + ",publisher3,advt2,pb1.web,US," + scala.util.Random.nextDouble() + ",13434" - val bid4 = currentTime + ",publisher4,advt2,pb1.web,US," + scala.util.Random.nextDouble() + ",34324" - val bid5 = currentTime + ",publisher2,advt1,pb1.web,US," + scala.util.Random.nextDouble() + ",23233" - val bid6 = currentTime + ",publisher4,advt2,pb1.web,US," + scala.util.Random.nextDouble() + ",43545" + // timestamp, publisher,advertiser,web,geo,bid,cookie + val bid1 = currentTime + ",publisher1,advt1,pb1.web,US," + Random.nextDouble() + ",23543" + val bid2 = currentTime + ",publisher2,advt1,pb1.web,US," + Random.nextDouble() + ",45445" + val bid3 = currentTime + ",publisher3,advt2,pb1.web,US," + Random.nextDouble() + ",13434" + val bid4 = currentTime + ",publisher4,advt2,pb1.web,US," + Random.nextDouble() + ",34324" + val bid5 = currentTime + ",publisher2,advt1,pb1.web,US," + Random.nextDouble() + ",23233" + val bid6 = currentTime + ",publisher4,advt2,pb1.web,US," + Random.nextDouble() + ",43545" // publish the bids as a Kafka message utils.sendMessages(topic, Array(bid1, bid2, bid3, bid4, bid5, bid6)) @@ -309,7 +307,7 @@ class EmbeddedKafkaUtils extends Logging { brokerConf = new KafkaConfig(brokerConfiguration, doLog = false) server = new KafkaServer(brokerConf) server.startup() - brokerPort = server.boundPort() + brokerPort = server.boundPort(new ListenerName("PLAINTEXT")) (server, brokerPort) }, new SparkConf(), "KafkaBroker") @@ -358,7 +356,8 @@ class EmbeddedKafkaUtils extends Logging { AdminUtils.createTopic(zkUtils, topic, partitions, 1) created = true } catch { - case e: kafka.common.TopicExistsException if overwrite => // deleteTopic(topic) + case e: Exception if overwrite && e.getClass.getSimpleName == "TopicExistsException" => + // deleteTopic(topic) } } // wait until metadata is propagated @@ -430,11 +429,9 @@ class EmbeddedKafkaUtils extends Logging { private def waitUntilMetadataIsPropagated(topic: String, partition: Int): Unit = { def isPropagated = server.apis.metadataCache.getPartitionInfo(topic, partition) match { case Some(partitionState) => - val leaderAndInSyncReplicas = partitionState.leaderIsrAndControllerEpoch.leaderAndIsr - zkUtils.getLeaderForPartition(topic, partition).isDefined && - Request.isValidBrokerId(leaderAndInSyncReplicas.leader) && - leaderAndInSyncReplicas.isr.size >= 1 + Request.isValidBrokerId(partitionState.basePartitionState.leader) && + !partitionState.basePartitionState.replicas.isEmpty case _ => false @@ -463,4 +460,4 @@ class EmbeddedKafkaUtils extends Logging { } } -} \ No newline at end of file +} diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 457aad0d98..5c2d1cf016 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index ee671127ff..0ebb3108e2 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-5.0-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-5.6.4-all.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew index 1f2758297d..d04230843f 100755 --- a/gradlew +++ b/gradlew @@ -1,5 +1,21 @@ #!/usr/bin/env sh +# +# Copyright 2015 the original author or authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + ############################################################################## ## ## Gradle start up script for UN*X @@ -28,7 +44,7 @@ APP_NAME="Gradle" APP_BASE_NAME=`basename "$0"` # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS="-XX:MaxMetaspaceSize=256m -XX:+HeapDumpOnOutOfMemoryError -Xmx2g -Xms1g -Djava.net.preferIPv4Stack=true" +DEFAULT_JVM_OPTS='"-XX:MaxMetaspaceSize=256m" "-XX:+HeapDumpOnOutOfMemoryError" "-Xmx2g" "-Xms1g" "-Djava.net.preferIPv4Stack=true"' # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD="maximum" diff --git a/gradlew.bat b/gradlew.bat index 6c62aa5fc7..46867a2ac4 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -1,3 +1,19 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + @if "%DEBUG%" == "" @echo off @rem ########################################################################## @rem @@ -14,7 +30,7 @@ set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -set DEFAULT_JVM_OPTS=-XX:MaxMetaspaceSize=256m -XX:+HeapDumpOnOutOfMemoryError -Xmx2g -Xms1g -Djava.net.preferIPv4Stack=true +set DEFAULT_JVM_OPTS="-XX:MaxMetaspaceSize=256m" "-XX:+HeapDumpOnOutOfMemoryError" "-Xmx2g" "-Xms1g" "-Djava.net.preferIPv4Stack=true" @rem Find java.exe if defined JAVA_HOME goto findJavaFromJavaHome diff --git a/jdbc/build.gradle b/jdbc/build.gradle index 2031fb6e69..060c7f150c 100644 --- a/jdbc/build.gradle +++ b/jdbc/build.gradle @@ -30,6 +30,7 @@ dependencies { compile("org.apache.thrift:libthrift:${thriftVersion}") { exclude(group: 'org.slf4j', module: 'slf4j-api') } + compile 'commons-collections:commons-collections:' + commonsCollectionsVersion // always use stock spark so that snappy extensions don't get accidently // included here in snappy-jdbc code. @@ -37,18 +38,17 @@ dependencies { compile project(':snappy-spark:snappy-spark-core_' + scalaBinaryVersion) compile project(':snappy-spark:snappy-spark-catalyst_' + scalaBinaryVersion) compile project(':snappy-spark:snappy-spark-sql_' + scalaBinaryVersion) - compileOnly "org.eclipse.jetty:jetty-servlet:${jettyVersion}" } else { - compileOnly("org.apache.spark:spark-core_${scalaBinaryVersion}:${sparkVersion}") - compileOnly("org.apache.spark:spark-catalyst_${scalaBinaryVersion}:${sparkVersion}") - compileOnly("org.apache.spark:spark-sql_${scalaBinaryVersion}:${sparkVersion}") - compileOnly("org.eclipse.jetty:jetty-servlet:${jettyVersion}") + compileOnly("org.apache.spark:spark-core_${scalaBinaryVersion}:${sparkConnectorVersion}") + compileOnly("org.apache.spark:spark-catalyst_${scalaBinaryVersion}:${sparkConnectorVersion}") + compileOnly("org.apache.spark:spark-sql_${scalaBinaryVersion}:${sparkConnectorVersion}") } + compileOnly "org.eclipse.jetty:jetty-servlet:${jettyVersion}" if (new File(rootDir, 'store/build.gradle').exists()) { - compile project(':snappy-store:snappydata-store-client') + compile project(':snappy-store:snappydata-store-client') } else { - compile group: 'io.snappydata', name: 'snappydata-store-client', version: snappyStoreVersion + compile group: 'io.snappydata', name: 'snappydata-store-client', version: snappyStoreVersion } } @@ -80,7 +80,6 @@ shadowJar { // avoid conflict with the 0.9.2 version in stock Spark relocate 'org.apache.thrift', 'io.snappydata.org.apache.thrift' - relocate 'org.apache.spark.unsafe', 'io.snappydata.org.apache.spark.unsafe' mergeServiceFiles() exclude 'log4j.properties' diff --git a/jdbc/src/main/scala/io/snappydata/Constant.scala b/jdbc/src/main/scala/io/snappydata/Constant.scala index 3301075265..3d767a87ef 100644 --- a/jdbc/src/main/scala/io/snappydata/Constant.scala +++ b/jdbc/src/main/scala/io/snappydata/Constant.scala @@ -22,7 +22,7 @@ import com.gemstone.gemfire.internal.shared.SystemProperties * Constant names suggested per naming convention * http://docs.scala-lang.org/style/naming-conventions.html * - * we decided to use upper case with underscore word separator. + * SnappyData uses upper case with underscore word separator. */ object Constant { @@ -50,6 +50,8 @@ object Constant { val SPARK_STORE_PREFIX: String = SPARK_PREFIX + STORE_PROPERTY_PREFIX + val HIVE_TYPE_STRING = "HIVE_TYPE_STRING" + val JOBSERVER_PROPERTY_PREFIX = "jobserver." val CONNECTION_PROPERTY: String = s"${PROPERTY_PREFIX}connection" @@ -124,13 +126,6 @@ object Constant { val MAX_CHAR_SIZE = 254 - // allowed values for QueryHint.JoinType - val JOIN_TYPE_BROADCAST = "broadcast" - val JOIN_TYPE_HASH = "hash" - val JOIN_TYPE_SORT = "sort" - val ALLOWED_JOIN_TYPE_HINTS: List[String] = - List(JOIN_TYPE_BROADCAST, JOIN_TYPE_HASH, JOIN_TYPE_SORT) - /** * Limit the maximum number of rows in a column batch (applied before * ColumnBatchSize property). diff --git a/jdbc/src/main/scala/org/apache/spark/sql/SnappyDataBaseDialect.scala b/jdbc/src/main/scala/org/apache/spark/sql/SnappyDataBaseDialect.scala index aa3dfb9bbf..898d38112f 100644 --- a/jdbc/src/main/scala/org/apache/spark/sql/SnappyDataBaseDialect.scala +++ b/jdbc/src/main/scala/org/apache/spark/sql/SnappyDataBaseDialect.scala @@ -25,7 +25,8 @@ import com.pivotal.gemfirexd.internal.shared.common.reference.Limits.{DB2_LOB_MA import io.snappydata.Constant import org.apache.spark.SparkEnv -import org.apache.spark.sql.catalyst.parser.AbstractSqlParser +import org.apache.spark.sql.catalyst.FunctionIdentifier +import org.apache.spark.sql.catalyst.parser.{AbstractSqlParser, ParserInterface} import org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils import org.apache.spark.sql.jdbc.JdbcType import org.apache.spark.sql.sources.JdbcExtendedUtils.quotedName @@ -79,10 +80,15 @@ abstract class SnappyDataBaseDialect extends JdbcExtendedDialect { case Types.ARRAY | JDBC40Translation.MAP | Types.STRUCT => val sparkSession = session match { case Some(s) => s - case None => SparkSession.builder().getOrCreate() + case None => SparkSession.getActiveSession match { + case Some(s) => s + case None => SparkSession.builder().getOrCreate() + } + } + sparkSession.sessionState.sqlParser match { + case parser: SQLParserInterface => Some(parser.parseDataType(typeName)) + case p => Some(p.asInstanceOf[AbstractSqlParser].parseDataType(typeName)) } - Some(sparkSession.sessionState.sqlParser - .asInstanceOf[AbstractSqlParser].parseDataType(typeName)) case Types.JAVA_OBJECT => // used by some system tables and VTIs // try to get class for the typeName else fallback to Object val userClass = try { @@ -234,6 +240,29 @@ abstract class SnappyDataBaseDialect extends JdbcExtendedDialect { s"partition by column($col)" } +/** + * Extension to [[ParserInterface]] having methods from recent Spark releases + * so that methods like `parseDataType` can be used with older releaases too. + */ +trait SQLParserInterface extends ParserInterface { + + /** + * Parse a string to a [[FunctionIdentifier]]. + */ + def parseFunctionIdentifier(sqlText: String): FunctionIdentifier + + /** + * Parse a string to a [[StructType]]. The passed SQL string should be a comma separated list + * of field definitions which will preserve the correct Hive metadata. + */ + def parseTableSchema(sqlText: String): StructType + + /** + * Parse a string to a [[DataType]]. + */ + def parseDataType(sqlText: String): DataType +} + final class JavaObjectType(override val userClass: java.lang.Class[AnyRef]) extends UserDefinedType[AnyRef] { diff --git a/jdbc/src/main/scala/org/apache/spark/sql/SnappyDataPoolDialect.scala b/jdbc/src/main/scala/org/apache/spark/sql/SnappyDataPoolDialect.scala index 1e4234574d..798dfc9133 100644 --- a/jdbc/src/main/scala/org/apache/spark/sql/SnappyDataPoolDialect.scala +++ b/jdbc/src/main/scala/org/apache/spark/sql/SnappyDataPoolDialect.scala @@ -83,10 +83,12 @@ case object SnappyDataPoolDialect extends SnappyDataBaseDialect with Logging { // releases where LocalRelation class primary constructor has changed signature cons.newInstance(tableName, LocalRelation.apply(output: _*), None) } catch { - case _: Exception => // fallback to two argument constructor - val cons = classOf[SubqueryAlias].getConstructor(classOf[String], - classOf[LogicalPlan]) - cons.newInstance(tableName, LocalRelation.apply(output: _*)) + case _: Exception => // fallback to two argument apply that works for both 2.3/2.4 + // class of companion class which is SubqueryAlias$ in bytecode + val c = SubqueryAlias.getClass + val m = c.getMethod("apply", classOf[String], classOf[LogicalPlan]) + m.invoke(c.getField("MODULE$").get(null), + tableName, LocalRelation.apply(output: _*)).asInstanceOf[SubqueryAlias] } } } diff --git a/jdbc/src/main/scala/org/apache/spark/sql/sources/jdbcExtensions.scala b/jdbc/src/main/scala/org/apache/spark/sql/sources/jdbcExtensions.scala index 34afea4c5b..cba169919a 100644 --- a/jdbc/src/main/scala/org/apache/spark/sql/sources/jdbcExtensions.scala +++ b/jdbc/src/main/scala/org/apache/spark/sql/sources/jdbcExtensions.scala @@ -21,14 +21,15 @@ import java.sql.{Connection, ResultSet, ResultSetMetaData, Types} import java.util.Properties import scala.annotation.tailrec +import scala.collection.JavaConverters._ import scala.collection.{mutable, Map => SMap} import scala.util.control.NonFatal import com.pivotal.gemfirexd.Attribute import io.snappydata.Constant +import org.apache.commons.collections.map.CaseInsensitiveMap import org.apache.spark.Logging -import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap import org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils import org.apache.spark.sql.jdbc.{JdbcDialect, JdbcType} import org.apache.spark.sql.sources.JdbcExtendedUtils.quotedName @@ -178,9 +179,10 @@ object JdbcExtendedUtils extends Logging { def readSplitProperty(propertyName: String, options: SMap[String, String]): Option[String] = { val params = options match { - case _: CaseInsensitiveMap => options + case _ if options.getClass.getName.contains("CaseInsensitiveMap") => options case _ if options.getClass.getName.contains("CaseInsensitiveMutableHashMap") => options - case _ => new CaseInsensitiveMap(options.toMap) + case _ => new CaseInsensitiveMap(options.toMap.asJava) + .asInstanceOf[java.util.Map[String, String]].asScala } // read the split schema DDL string from hive metastore table parameters params.get(s"$propertyName.numParts") map { numParts => @@ -265,8 +267,12 @@ object JdbcExtendedUtils extends Logging { size, scale, metadataBuilder, session) cols += StructField(columnName, columnType, nullable, metadataBuilder.build()) } while (rs.next()) + rs.close() normalizeSchema(StructType(cols)) - } else EMPTY_SCHEMA + } else { + rs.close() + EMPTY_SCHEMA + } } def tableExistsInMetaData(schemaName: String, tableName: String, diff --git a/release/filehdr-mod.txt b/release/filehdr-mod.txt index 45fdd561b5..62afe02885 100644 --- a/release/filehdr-mod.txt +++ b/release/filehdr-mod.txt @@ -1,7 +1,7 @@ /* * Changes for TIBCO Project SnappyData data platform. * - * Portions Copyright (c) 2017-2019 TIBCO Software Inc. All rights reserved. + * Portions Copyright (c) 2017-2020 TIBCO Software Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/release/filehdr.txt b/release/filehdr.txt index 0dcbd010ff..eda6fd14a3 100644 --- a/release/filehdr.txt +++ b/release/filehdr.txt @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 TIBCO Software Inc. All rights reserved. + * Copyright (c) 2017-2020 TIBCO Software Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You diff --git a/settings.gradle b/settings.gradle index db1abb615f..b4119ce839 100644 --- a/settings.gradle +++ b/settings.gradle @@ -15,7 +15,9 @@ * LICENSE file. */ -def scalaBinaryVersion = '2.11' +String scalaBinaryVersion = '2.11' +String sparkVersion = '2.4.5' +String sparkConnectorVersion = System.getProperty('spark.connector.version', sparkVersion) rootProject.name = 'snappydata_' + scalaBinaryVersion include ':snappy-jdbc_' + scalaBinaryVersion @@ -28,6 +30,9 @@ include ':snappy-dtests_' + scalaBinaryVersion include ':snappy-compatibility-tests_' + scalaBinaryVersion include ':snappy-encoders_' + scalaBinaryVersion +// compatibility modules for all supported Spark releases +include ":snappy-core_${scalaBinaryVersion}:compat-spark2.4.5" + project(':snappy-jdbc_' + scalaBinaryVersion).projectDir = "$rootDir/jdbc" as File project(':snappy-core_' + scalaBinaryVersion).projectDir = "$rootDir/core" as File project(':snappy-cluster_' + scalaBinaryVersion).projectDir = "$rootDir/cluster" as File @@ -37,6 +42,21 @@ project(':snappy-dtests_' + scalaBinaryVersion).projectDir = "$rootDir/dtests" a project(':snappy-compatibility-tests_' + scalaBinaryVersion).projectDir = "$rootDir/compatibilityTests" as File project(':snappy-encoders_' + scalaBinaryVersion).projectDir = "$rootDir/encoders" as File +project(":snappy-core_${scalaBinaryVersion}:compat-spark2.4.5").projectDir = "$rootDir/core/compatibility/spark-2.4.5" as File + +if (sparkConnectorVersion != sparkVersion) { + include ':snappy-core-product_' + scalaBinaryVersion + project(':snappy-core-product_' + scalaBinaryVersion).projectDir = "$rootDir/core-product" as File + + include ":snappy-core_${scalaBinaryVersion}:compat-spark2.1" + include ":snappy-core_${scalaBinaryVersion}:compat-spark2.3" + include ":snappy-core_${scalaBinaryVersion}:compat-spark2.4" + + project(":snappy-core_${scalaBinaryVersion}:compat-spark2.1").projectDir = "$rootDir/core/compatibility/spark-2.1" as File + project(":snappy-core_${scalaBinaryVersion}:compat-spark2.3").projectDir = "$rootDir/core/compatibility/spark-2.3" as File + project(":snappy-core_${scalaBinaryVersion}:compat-spark2.4").projectDir = "$rootDir/core/compatibility/spark-2.4" as File +} + if (new File(rootDir, 'spark/build.gradle').exists()) { include ':snappy-spark' // sub-projects of snappy-spark @@ -56,13 +76,16 @@ if (new File(rootDir, 'spark/build.gradle').exists()) { include ':snappy-spark:snappy-spark-hive_' + scalaBinaryVersion include ':snappy-spark:snappy-spark-hive-thriftserver_' + scalaBinaryVersion include ':snappy-spark:snappy-spark-mesos_' + scalaBinaryVersion + include ':snappy-spark:snappy-spark-kubernetes_' + scalaBinaryVersion include ':snappy-spark:snappy-spark-unsafe_' + scalaBinaryVersion include ':snappy-spark:snappy-spark-sketch_' + scalaBinaryVersion + include ':snappy-spark:snappy-spark-kvstore_' + scalaBinaryVersion include ':snappy-spark:snappy-spark-assembly_' + scalaBinaryVersion include ':snappy-spark:snappy-spark-streaming-flume_' + scalaBinaryVersion include ':snappy-spark:snappy-spark-streaming-flume-sink_' + scalaBinaryVersion include ':snappy-spark:snappy-spark-streaming-kafka-0.10_' + scalaBinaryVersion include ':snappy-spark:snappy-spark-sql-kafka-0.10_' + scalaBinaryVersion + include ':snappy-spark:snappy-spark-avro_' + scalaBinaryVersion include ':snappy-spark:snappy-spark-examples_' + scalaBinaryVersion include ':snappy-spark:snappy-spark-repl_' + scalaBinaryVersion include ':snappy-spark:snappy-spark-launcher_' + scalaBinaryVersion @@ -83,22 +106,26 @@ if (new File(rootDir, 'spark/build.gradle').exists()) { "$rootDir/spark/common/network-shuffle" as File project(':snappy-spark:snappy-spark-network-yarn_' + scalaBinaryVersion).projectDir = "$rootDir/spark/common/network-yarn" as File - project(':snappy-spark:snappy-spark-yarn_' + scalaBinaryVersion).projectDir = "$rootDir/spark/yarn" as File + project(':snappy-spark:snappy-spark-yarn_' + scalaBinaryVersion).projectDir = "$rootDir/spark/resource-managers/yarn" as File project(':snappy-spark:snappy-spark-streaming_' + scalaBinaryVersion).projectDir = "$rootDir/spark/streaming" as File project(':snappy-spark:snappy-spark-catalyst_' + scalaBinaryVersion).projectDir = "$rootDir/spark/sql/catalyst" as File project(':snappy-spark:snappy-spark-sql_' + scalaBinaryVersion).projectDir = "$rootDir/spark/sql/core" as File project(':snappy-spark:snappy-spark-hive_' + scalaBinaryVersion).projectDir = "$rootDir/spark/sql/hive" as File project(':snappy-spark:snappy-spark-hive-thriftserver_' + scalaBinaryVersion).projectDir = "$rootDir/spark/sql/hive-thriftserver" as File - project(':snappy-spark:snappy-spark-mesos_' + scalaBinaryVersion).projectDir = "$rootDir/spark/mesos" as File + project(':snappy-spark:snappy-spark-mesos_' + scalaBinaryVersion).projectDir = "$rootDir/spark/resource-managers/mesos" as File + project(':snappy-spark:snappy-spark-kubernetes_' + scalaBinaryVersion).projectDir = "$rootDir/spark/resource-managers/kubernetes/core" as File project(':snappy-spark:snappy-spark-unsafe_' + scalaBinaryVersion).projectDir = "$rootDir/spark/common/unsafe" as File project(':snappy-spark:snappy-spark-sketch_' + scalaBinaryVersion).projectDir = "$rootDir/spark/common/sketch" as File + project(':snappy-spark:snappy-spark-kvstore_' + scalaBinaryVersion).projectDir = + "$rootDir/spark/common/kvstore" as File project(':snappy-spark:snappy-spark-assembly_' + scalaBinaryVersion).projectDir = "$rootDir/spark/assembly" as File project(':snappy-spark:snappy-spark-streaming-flume_' + scalaBinaryVersion).projectDir = "$rootDir/spark/external/flume" as File project(':snappy-spark:snappy-spark-streaming-flume-sink_' + scalaBinaryVersion).projectDir = "$rootDir/spark/external/flume-sink" as File project(':snappy-spark:snappy-spark-streaming-kafka-0.10_' + scalaBinaryVersion).projectDir = "$rootDir/spark/external/kafka-0-10" as File project(':snappy-spark:snappy-spark-sql-kafka-0.10_' + scalaBinaryVersion).projectDir = "$rootDir/spark/external/kafka-0-10-sql" as File + project(':snappy-spark:snappy-spark-avro_' + scalaBinaryVersion).projectDir = "$rootDir/spark/external/avro" as File project(':snappy-spark:snappy-spark-examples_' + scalaBinaryVersion).projectDir = "$rootDir/spark/examples" as File project(':snappy-spark:snappy-spark-repl_' + scalaBinaryVersion).projectDir = "$rootDir/spark/repl" as File project(':snappy-spark:snappy-spark-launcher_' + scalaBinaryVersion).projectDir = "$rootDir/spark/launcher" as File diff --git a/spark b/spark index 6c34666732..b5e0f32e87 160000 --- a/spark +++ b/spark @@ -1 +1 @@ -Subproject commit 6c34666732ae7a902a9d2b576bfb06d131680ddb +Subproject commit b5e0f32e8733e6867fbc524f4c158ed5c722667f diff --git a/spark-jobserver b/spark-jobserver index d6ca632810..3e24a56ebe 160000 --- a/spark-jobserver +++ b/spark-jobserver @@ -1 +1 @@ -Subproject commit d6ca632810d8b032c1a6a6baa783e04ed8433bb5 +Subproject commit 3e24a56ebee06317106e1494d75167226e6bc531 diff --git a/store b/store index 73b4be5599..e7b2b7a893 160000 --- a/store +++ b/store @@ -1 +1 @@ -Subproject commit 73b4be5599e4a8be1b85bd6d562909f9e7527448 +Subproject commit e7b2b7a893bf6b84717142a6b1c2b5c66da72a6c diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/._common_metadata.crc b/tests/common/src/main/resources/2015-trimmed.parquet/._common_metadata.crc deleted file mode 100644 index 87f2da649d..0000000000 Binary files a/tests/common/src/main/resources/2015-trimmed.parquet/._common_metadata.crc and /dev/null differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/._metadata.crc b/tests/common/src/main/resources/2015-trimmed.parquet/._metadata.crc deleted file mode 100644 index 36d573cffd..0000000000 Binary files a/tests/common/src/main/resources/2015-trimmed.parquet/._metadata.crc and /dev/null differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-00658b64-c850-4cf1-a62b-66df910b5c0d-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-00658b64-c850-4cf1-a62b-66df910b5c0d-c000.gz.parquet.crc new file mode 100644 index 0000000000..eb1ee38a48 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-00658b64-c850-4cf1-a62b-66df910b5c0d-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-045af329-7928-4b99-bd5f-7509909cd629-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-045af329-7928-4b99-bd5f-7509909cd629-c000.gz.parquet.crc new file mode 100644 index 0000000000..eb1ee38a48 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-045af329-7928-4b99-bd5f-7509909cd629-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-0e960084-6914-452d-878d-025d567a705b-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-0e960084-6914-452d-878d-025d567a705b-c000.gz.parquet.crc new file mode 100644 index 0000000000..eb1ee38a48 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-0e960084-6914-452d-878d-025d567a705b-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-1986d9f8-af54-4ac3-9f96-ef9916a20cab-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-1986d9f8-af54-4ac3-9f96-ef9916a20cab-c000.gz.parquet.crc new file mode 100644 index 0000000000..eb1ee38a48 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-1986d9f8-af54-4ac3-9f96-ef9916a20cab-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-21c7c705-a7d0-453c-a56c-2dc7cbe2cb44-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-21c7c705-a7d0-453c-a56c-2dc7cbe2cb44-c000.gz.parquet.crc new file mode 100644 index 0000000000..eb1ee38a48 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-21c7c705-a7d0-453c-a56c-2dc7cbe2cb44-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-2521c941-34de-4532-af09-054c39814f92-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-2521c941-34de-4532-af09-054c39814f92-c000.gz.parquet.crc new file mode 100644 index 0000000000..eb1ee38a48 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-2521c941-34de-4532-af09-054c39814f92-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-2ba5ab7e-be2e-44a6-9deb-12e7f97895d3-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-2ba5ab7e-be2e-44a6-9deb-12e7f97895d3-c000.gz.parquet.crc new file mode 100644 index 0000000000..eb1ee38a48 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-2ba5ab7e-be2e-44a6-9deb-12e7f97895d3-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-30e9e719-9951-4b82-9e15-a75b47e5f696-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-30e9e719-9951-4b82-9e15-a75b47e5f696-c000.gz.parquet.crc new file mode 100644 index 0000000000..eb1ee38a48 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-30e9e719-9951-4b82-9e15-a75b47e5f696-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-3103fd69-e890-4ce6-b27e-833c6ea8de80-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-3103fd69-e890-4ce6-b27e-833c6ea8de80-c000.gz.parquet.crc new file mode 100644 index 0000000000..eb1ee38a48 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-3103fd69-e890-4ce6-b27e-833c6ea8de80-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-35edb6a0-7e18-44d4-a026-80fc1e82b653-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-35edb6a0-7e18-44d4-a026-80fc1e82b653-c000.gz.parquet.crc new file mode 100644 index 0000000000..eb1ee38a48 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-35edb6a0-7e18-44d4-a026-80fc1e82b653-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-3b537503-41df-4274-956e-23869a4e7662-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-3b537503-41df-4274-956e-23869a4e7662-c000.gz.parquet.crc new file mode 100644 index 0000000000..eb1ee38a48 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-3b537503-41df-4274-956e-23869a4e7662-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-3bcaf272-fc53-4775-8dba-5261c27ad67e-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-3bcaf272-fc53-4775-8dba-5261c27ad67e-c000.gz.parquet.crc new file mode 100644 index 0000000000..eb1ee38a48 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-3bcaf272-fc53-4775-8dba-5261c27ad67e-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-4cd108be-a0f1-452f-a40a-2dd403ac79b9-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-4cd108be-a0f1-452f-a40a-2dd403ac79b9-c000.gz.parquet.crc new file mode 100644 index 0000000000..eb1ee38a48 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-4cd108be-a0f1-452f-a40a-2dd403ac79b9-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-4ff6f25a-a5ab-46e3-83d9-f3f746510f64-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-4ff6f25a-a5ab-46e3-83d9-f3f746510f64-c000.gz.parquet.crc new file mode 100644 index 0000000000..eb1ee38a48 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-4ff6f25a-a5ab-46e3-83d9-f3f746510f64-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-504f4563-b66b-43a1-a095-4dfe8ffbb896-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-504f4563-b66b-43a1-a095-4dfe8ffbb896-c000.gz.parquet.crc new file mode 100644 index 0000000000..eb1ee38a48 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-504f4563-b66b-43a1-a095-4dfe8ffbb896-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-5e3f868d-ce23-4588-bc2e-54ac07b01b5c-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-5e3f868d-ce23-4588-bc2e-54ac07b01b5c-c000.gz.parquet.crc new file mode 100644 index 0000000000..eb1ee38a48 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-5e3f868d-ce23-4588-bc2e-54ac07b01b5c-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-7135ff90-7b82-4f5a-bd23-72e5f76f6225-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-7135ff90-7b82-4f5a-bd23-72e5f76f6225-c000.gz.parquet.crc new file mode 100644 index 0000000000..eb1ee38a48 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-7135ff90-7b82-4f5a-bd23-72e5f76f6225-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-7441115a-7e40-40d6-98d2-e1869f64bef2-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-7441115a-7e40-40d6-98d2-e1869f64bef2-c000.gz.parquet.crc new file mode 100644 index 0000000000..eb1ee38a48 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-7441115a-7e40-40d6-98d2-e1869f64bef2-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-7bc4ec26-3c17-4f4b-82d5-85bf626078a7-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-7bc4ec26-3c17-4f4b-82d5-85bf626078a7-c000.gz.parquet.crc new file mode 100644 index 0000000000..eb1ee38a48 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-7bc4ec26-3c17-4f4b-82d5-85bf626078a7-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-91e45d5f-cca3-44c7-8808-07f201d8ae97-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-91e45d5f-cca3-44c7-8808-07f201d8ae97-c000.gz.parquet.crc new file mode 100644 index 0000000000..eb1ee38a48 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-91e45d5f-cca3-44c7-8808-07f201d8ae97-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-93d91940-764c-47f0-8afb-11ef79e699a5-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-93d91940-764c-47f0-8afb-11ef79e699a5-c000.gz.parquet.crc new file mode 100644 index 0000000000..eb1ee38a48 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-93d91940-764c-47f0-8afb-11ef79e699a5-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-97a8da5b-ad06-4911-9661-36dbb07821ae-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-97a8da5b-ad06-4911-9661-36dbb07821ae-c000.gz.parquet.crc new file mode 100644 index 0000000000..eb1ee38a48 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-97a8da5b-ad06-4911-9661-36dbb07821ae-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-9b4e3257-cf23-40b2-9760-8a8ddd82bcc8-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-9b4e3257-cf23-40b2-9760-8a8ddd82bcc8-c000.gz.parquet.crc new file mode 100644 index 0000000000..eb1ee38a48 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-9b4e3257-cf23-40b2-9760-8a8ddd82bcc8-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-aafbfa5b-c7bd-4c26-9e81-263d90950ea1-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-aafbfa5b-c7bd-4c26-9e81-263d90950ea1-c000.gz.parquet.crc new file mode 100644 index 0000000000..eb1ee38a48 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-aafbfa5b-c7bd-4c26-9e81-263d90950ea1-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-ab26f8b7-91b6-45da-b9fe-28b4389acfdd-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-ab26f8b7-91b6-45da-b9fe-28b4389acfdd-c000.gz.parquet.crc new file mode 100644 index 0000000000..eb1ee38a48 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-ab26f8b7-91b6-45da-b9fe-28b4389acfdd-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-b4f5c52f-b8f2-4747-9399-da92c299179f-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-b4f5c52f-b8f2-4747-9399-da92c299179f-c000.gz.parquet.crc new file mode 100644 index 0000000000..eb1ee38a48 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-b4f5c52f-b8f2-4747-9399-da92c299179f-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-d31f531d-8107-42cb-952b-250bc66fb332-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-d31f531d-8107-42cb-952b-250bc66fb332-c000.gz.parquet.crc new file mode 100644 index 0000000000..eb1ee38a48 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-d31f531d-8107-42cb-952b-250bc66fb332-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-d7b92535-c598-47e5-a5f9-e0494472c448-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-d7b92535-c598-47e5-a5f9-e0494472c448-c000.gz.parquet.crc new file mode 100644 index 0000000000..eb1ee38a48 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-d7b92535-c598-47e5-a5f9-e0494472c448-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-dfbe507d-5640-48be-9f2a-7504c4b3f1c5-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-dfbe507d-5640-48be-9f2a-7504c4b3f1c5-c000.gz.parquet.crc new file mode 100644 index 0000000000..eb1ee38a48 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-dfbe507d-5640-48be-9f2a-7504c4b3f1c5-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-ed4153a2-dc32-4094-9d23-e3c6db36d2ec-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-ed4153a2-dc32-4094-9d23-e3c6db36d2ec-c000.gz.parquet.crc new file mode 100644 index 0000000000..eb1ee38a48 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-ed4153a2-dc32-4094-9d23-e3c6db36d2ec-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-f0b8feca-f48d-44d4-acbb-5a216f176ed4-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-f0b8feca-f48d-44d4-acbb-5a216f176ed4-c000.gz.parquet.crc new file mode 100644 index 0000000000..eb1ee38a48 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00000-f0b8feca-f48d-44d4-acbb-5a216f176ed4-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-00658b64-c850-4cf1-a62b-66df910b5c0d-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-00658b64-c850-4cf1-a62b-66df910b5c0d-c000.gz.parquet.crc new file mode 100644 index 0000000000..e3695b7226 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-00658b64-c850-4cf1-a62b-66df910b5c0d-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-045af329-7928-4b99-bd5f-7509909cd629-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-045af329-7928-4b99-bd5f-7509909cd629-c000.gz.parquet.crc new file mode 100644 index 0000000000..7ae4bd01c7 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-045af329-7928-4b99-bd5f-7509909cd629-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-0e960084-6914-452d-878d-025d567a705b-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-0e960084-6914-452d-878d-025d567a705b-c000.gz.parquet.crc new file mode 100644 index 0000000000..c189ef00a7 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-0e960084-6914-452d-878d-025d567a705b-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-1986d9f8-af54-4ac3-9f96-ef9916a20cab-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-1986d9f8-af54-4ac3-9f96-ef9916a20cab-c000.gz.parquet.crc new file mode 100644 index 0000000000..40bcfd049c Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-1986d9f8-af54-4ac3-9f96-ef9916a20cab-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-21c7c705-a7d0-453c-a56c-2dc7cbe2cb44-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-21c7c705-a7d0-453c-a56c-2dc7cbe2cb44-c000.gz.parquet.crc new file mode 100644 index 0000000000..6d5d341302 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-21c7c705-a7d0-453c-a56c-2dc7cbe2cb44-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-2521c941-34de-4532-af09-054c39814f92-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-2521c941-34de-4532-af09-054c39814f92-c000.gz.parquet.crc new file mode 100644 index 0000000000..b9660369f7 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-2521c941-34de-4532-af09-054c39814f92-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-2ba5ab7e-be2e-44a6-9deb-12e7f97895d3-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-2ba5ab7e-be2e-44a6-9deb-12e7f97895d3-c000.gz.parquet.crc new file mode 100644 index 0000000000..247f07f515 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-2ba5ab7e-be2e-44a6-9deb-12e7f97895d3-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-30e9e719-9951-4b82-9e15-a75b47e5f696-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-30e9e719-9951-4b82-9e15-a75b47e5f696-c000.gz.parquet.crc new file mode 100644 index 0000000000..1f56aa2418 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-30e9e719-9951-4b82-9e15-a75b47e5f696-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-3103fd69-e890-4ce6-b27e-833c6ea8de80-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-3103fd69-e890-4ce6-b27e-833c6ea8de80-c000.gz.parquet.crc new file mode 100644 index 0000000000..fa0fef5edc Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-3103fd69-e890-4ce6-b27e-833c6ea8de80-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-35edb6a0-7e18-44d4-a026-80fc1e82b653-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-35edb6a0-7e18-44d4-a026-80fc1e82b653-c000.gz.parquet.crc new file mode 100644 index 0000000000..c4e4f793c8 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-35edb6a0-7e18-44d4-a026-80fc1e82b653-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-3b537503-41df-4274-956e-23869a4e7662-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-3b537503-41df-4274-956e-23869a4e7662-c000.gz.parquet.crc new file mode 100644 index 0000000000..3e43c094dd Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-3b537503-41df-4274-956e-23869a4e7662-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-3bcaf272-fc53-4775-8dba-5261c27ad67e-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-3bcaf272-fc53-4775-8dba-5261c27ad67e-c000.gz.parquet.crc new file mode 100644 index 0000000000..5dfa2c6b31 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-3bcaf272-fc53-4775-8dba-5261c27ad67e-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-4cd108be-a0f1-452f-a40a-2dd403ac79b9-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-4cd108be-a0f1-452f-a40a-2dd403ac79b9-c000.gz.parquet.crc new file mode 100644 index 0000000000..67a47ef9df Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-4cd108be-a0f1-452f-a40a-2dd403ac79b9-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-4ff6f25a-a5ab-46e3-83d9-f3f746510f64-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-4ff6f25a-a5ab-46e3-83d9-f3f746510f64-c000.gz.parquet.crc new file mode 100644 index 0000000000..31dbee6681 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-4ff6f25a-a5ab-46e3-83d9-f3f746510f64-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-504f4563-b66b-43a1-a095-4dfe8ffbb896-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-504f4563-b66b-43a1-a095-4dfe8ffbb896-c000.gz.parquet.crc new file mode 100644 index 0000000000..0e368c58fc Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-504f4563-b66b-43a1-a095-4dfe8ffbb896-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-5e3f868d-ce23-4588-bc2e-54ac07b01b5c-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-5e3f868d-ce23-4588-bc2e-54ac07b01b5c-c000.gz.parquet.crc new file mode 100644 index 0000000000..67b8462871 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-5e3f868d-ce23-4588-bc2e-54ac07b01b5c-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-7135ff90-7b82-4f5a-bd23-72e5f76f6225-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-7135ff90-7b82-4f5a-bd23-72e5f76f6225-c000.gz.parquet.crc new file mode 100644 index 0000000000..ea39d15a0f Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-7135ff90-7b82-4f5a-bd23-72e5f76f6225-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-7441115a-7e40-40d6-98d2-e1869f64bef2-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-7441115a-7e40-40d6-98d2-e1869f64bef2-c000.gz.parquet.crc new file mode 100644 index 0000000000..304a51bbff Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-7441115a-7e40-40d6-98d2-e1869f64bef2-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-7bc4ec26-3c17-4f4b-82d5-85bf626078a7-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-7bc4ec26-3c17-4f4b-82d5-85bf626078a7-c000.gz.parquet.crc new file mode 100644 index 0000000000..51b840db5e Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-7bc4ec26-3c17-4f4b-82d5-85bf626078a7-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-91e45d5f-cca3-44c7-8808-07f201d8ae97-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-91e45d5f-cca3-44c7-8808-07f201d8ae97-c000.gz.parquet.crc new file mode 100644 index 0000000000..36a0d79b60 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-91e45d5f-cca3-44c7-8808-07f201d8ae97-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-93d91940-764c-47f0-8afb-11ef79e699a5-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-93d91940-764c-47f0-8afb-11ef79e699a5-c000.gz.parquet.crc new file mode 100644 index 0000000000..df355abbb2 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-93d91940-764c-47f0-8afb-11ef79e699a5-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-97a8da5b-ad06-4911-9661-36dbb07821ae-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-97a8da5b-ad06-4911-9661-36dbb07821ae-c000.gz.parquet.crc new file mode 100644 index 0000000000..791435df6a Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-97a8da5b-ad06-4911-9661-36dbb07821ae-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-9b4e3257-cf23-40b2-9760-8a8ddd82bcc8-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-9b4e3257-cf23-40b2-9760-8a8ddd82bcc8-c000.gz.parquet.crc new file mode 100644 index 0000000000..30b65a5f01 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-9b4e3257-cf23-40b2-9760-8a8ddd82bcc8-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-aafbfa5b-c7bd-4c26-9e81-263d90950ea1-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-aafbfa5b-c7bd-4c26-9e81-263d90950ea1-c000.gz.parquet.crc new file mode 100644 index 0000000000..4304a7916a Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-aafbfa5b-c7bd-4c26-9e81-263d90950ea1-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-ab26f8b7-91b6-45da-b9fe-28b4389acfdd-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-ab26f8b7-91b6-45da-b9fe-28b4389acfdd-c000.gz.parquet.crc new file mode 100644 index 0000000000..c4b7187380 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-ab26f8b7-91b6-45da-b9fe-28b4389acfdd-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-b4f5c52f-b8f2-4747-9399-da92c299179f-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-b4f5c52f-b8f2-4747-9399-da92c299179f-c000.gz.parquet.crc new file mode 100644 index 0000000000..f2fd1c4ed8 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-b4f5c52f-b8f2-4747-9399-da92c299179f-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-d31f531d-8107-42cb-952b-250bc66fb332-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-d31f531d-8107-42cb-952b-250bc66fb332-c000.gz.parquet.crc new file mode 100644 index 0000000000..5592ab71ee Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-d31f531d-8107-42cb-952b-250bc66fb332-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-d7b92535-c598-47e5-a5f9-e0494472c448-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-d7b92535-c598-47e5-a5f9-e0494472c448-c000.gz.parquet.crc new file mode 100644 index 0000000000..1e197331c4 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-d7b92535-c598-47e5-a5f9-e0494472c448-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-dfbe507d-5640-48be-9f2a-7504c4b3f1c5-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-dfbe507d-5640-48be-9f2a-7504c4b3f1c5-c000.gz.parquet.crc new file mode 100644 index 0000000000..166f7e10cd Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-dfbe507d-5640-48be-9f2a-7504c4b3f1c5-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-ed4153a2-dc32-4094-9d23-e3c6db36d2ec-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-ed4153a2-dc32-4094-9d23-e3c6db36d2ec-c000.gz.parquet.crc new file mode 100644 index 0000000000..d7aa397c61 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-ed4153a2-dc32-4094-9d23-e3c6db36d2ec-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-f0b8feca-f48d-44d4-acbb-5a216f176ed4-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-f0b8feca-f48d-44d4-acbb-5a216f176ed4-c000.gz.parquet.crc new file mode 100644 index 0000000000..a7faa52d84 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00005-f0b8feca-f48d-44d4-acbb-5a216f176ed4-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-00658b64-c850-4cf1-a62b-66df910b5c0d-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-00658b64-c850-4cf1-a62b-66df910b5c0d-c000.gz.parquet.crc new file mode 100644 index 0000000000..3f88d7f287 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-00658b64-c850-4cf1-a62b-66df910b5c0d-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-045af329-7928-4b99-bd5f-7509909cd629-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-045af329-7928-4b99-bd5f-7509909cd629-c000.gz.parquet.crc new file mode 100644 index 0000000000..222ca894e8 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-045af329-7928-4b99-bd5f-7509909cd629-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-0e960084-6914-452d-878d-025d567a705b-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-0e960084-6914-452d-878d-025d567a705b-c000.gz.parquet.crc new file mode 100644 index 0000000000..da5d49146c Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-0e960084-6914-452d-878d-025d567a705b-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-1986d9f8-af54-4ac3-9f96-ef9916a20cab-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-1986d9f8-af54-4ac3-9f96-ef9916a20cab-c000.gz.parquet.crc new file mode 100644 index 0000000000..cd52eb55ff Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-1986d9f8-af54-4ac3-9f96-ef9916a20cab-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-21c7c705-a7d0-453c-a56c-2dc7cbe2cb44-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-21c7c705-a7d0-453c-a56c-2dc7cbe2cb44-c000.gz.parquet.crc new file mode 100644 index 0000000000..91e058d95d Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-21c7c705-a7d0-453c-a56c-2dc7cbe2cb44-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-2521c941-34de-4532-af09-054c39814f92-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-2521c941-34de-4532-af09-054c39814f92-c000.gz.parquet.crc new file mode 100644 index 0000000000..b0d6da9ac5 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-2521c941-34de-4532-af09-054c39814f92-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-2ba5ab7e-be2e-44a6-9deb-12e7f97895d3-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-2ba5ab7e-be2e-44a6-9deb-12e7f97895d3-c000.gz.parquet.crc new file mode 100644 index 0000000000..ad6676d387 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-2ba5ab7e-be2e-44a6-9deb-12e7f97895d3-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-30e9e719-9951-4b82-9e15-a75b47e5f696-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-30e9e719-9951-4b82-9e15-a75b47e5f696-c000.gz.parquet.crc new file mode 100644 index 0000000000..cd5ebe68fb Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-30e9e719-9951-4b82-9e15-a75b47e5f696-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-3103fd69-e890-4ce6-b27e-833c6ea8de80-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-3103fd69-e890-4ce6-b27e-833c6ea8de80-c000.gz.parquet.crc new file mode 100644 index 0000000000..ff0c148f53 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-3103fd69-e890-4ce6-b27e-833c6ea8de80-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-35edb6a0-7e18-44d4-a026-80fc1e82b653-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-35edb6a0-7e18-44d4-a026-80fc1e82b653-c000.gz.parquet.crc new file mode 100644 index 0000000000..187ffff0c2 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-35edb6a0-7e18-44d4-a026-80fc1e82b653-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-3b537503-41df-4274-956e-23869a4e7662-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-3b537503-41df-4274-956e-23869a4e7662-c000.gz.parquet.crc new file mode 100644 index 0000000000..f524dd53df Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-3b537503-41df-4274-956e-23869a4e7662-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-3bcaf272-fc53-4775-8dba-5261c27ad67e-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-3bcaf272-fc53-4775-8dba-5261c27ad67e-c000.gz.parquet.crc new file mode 100644 index 0000000000..48948a7aba Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-3bcaf272-fc53-4775-8dba-5261c27ad67e-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-4cd108be-a0f1-452f-a40a-2dd403ac79b9-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-4cd108be-a0f1-452f-a40a-2dd403ac79b9-c000.gz.parquet.crc new file mode 100644 index 0000000000..d3bb1d3b90 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-4cd108be-a0f1-452f-a40a-2dd403ac79b9-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-4ff6f25a-a5ab-46e3-83d9-f3f746510f64-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-4ff6f25a-a5ab-46e3-83d9-f3f746510f64-c000.gz.parquet.crc new file mode 100644 index 0000000000..17dcb4c94d Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-4ff6f25a-a5ab-46e3-83d9-f3f746510f64-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-504f4563-b66b-43a1-a095-4dfe8ffbb896-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-504f4563-b66b-43a1-a095-4dfe8ffbb896-c000.gz.parquet.crc new file mode 100644 index 0000000000..943bb8d5ee Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-504f4563-b66b-43a1-a095-4dfe8ffbb896-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-5e3f868d-ce23-4588-bc2e-54ac07b01b5c-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-5e3f868d-ce23-4588-bc2e-54ac07b01b5c-c000.gz.parquet.crc new file mode 100644 index 0000000000..c828d268a2 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-5e3f868d-ce23-4588-bc2e-54ac07b01b5c-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-7135ff90-7b82-4f5a-bd23-72e5f76f6225-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-7135ff90-7b82-4f5a-bd23-72e5f76f6225-c000.gz.parquet.crc new file mode 100644 index 0000000000..0d82460ed7 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-7135ff90-7b82-4f5a-bd23-72e5f76f6225-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-7441115a-7e40-40d6-98d2-e1869f64bef2-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-7441115a-7e40-40d6-98d2-e1869f64bef2-c000.gz.parquet.crc new file mode 100644 index 0000000000..68b161646d Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-7441115a-7e40-40d6-98d2-e1869f64bef2-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-7bc4ec26-3c17-4f4b-82d5-85bf626078a7-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-7bc4ec26-3c17-4f4b-82d5-85bf626078a7-c000.gz.parquet.crc new file mode 100644 index 0000000000..28e1a49b1f Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-7bc4ec26-3c17-4f4b-82d5-85bf626078a7-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-91e45d5f-cca3-44c7-8808-07f201d8ae97-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-91e45d5f-cca3-44c7-8808-07f201d8ae97-c000.gz.parquet.crc new file mode 100644 index 0000000000..916c96036f Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-91e45d5f-cca3-44c7-8808-07f201d8ae97-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-93d91940-764c-47f0-8afb-11ef79e699a5-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-93d91940-764c-47f0-8afb-11ef79e699a5-c000.gz.parquet.crc new file mode 100644 index 0000000000..0c654ad06b Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-93d91940-764c-47f0-8afb-11ef79e699a5-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-97a8da5b-ad06-4911-9661-36dbb07821ae-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-97a8da5b-ad06-4911-9661-36dbb07821ae-c000.gz.parquet.crc new file mode 100644 index 0000000000..03538ed439 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-97a8da5b-ad06-4911-9661-36dbb07821ae-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-9b4e3257-cf23-40b2-9760-8a8ddd82bcc8-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-9b4e3257-cf23-40b2-9760-8a8ddd82bcc8-c000.gz.parquet.crc new file mode 100644 index 0000000000..2f88cc49d6 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-9b4e3257-cf23-40b2-9760-8a8ddd82bcc8-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-aafbfa5b-c7bd-4c26-9e81-263d90950ea1-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-aafbfa5b-c7bd-4c26-9e81-263d90950ea1-c000.gz.parquet.crc new file mode 100644 index 0000000000..e17972007e Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-aafbfa5b-c7bd-4c26-9e81-263d90950ea1-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-ab26f8b7-91b6-45da-b9fe-28b4389acfdd-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-ab26f8b7-91b6-45da-b9fe-28b4389acfdd-c000.gz.parquet.crc new file mode 100644 index 0000000000..81c4f54fcb Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-ab26f8b7-91b6-45da-b9fe-28b4389acfdd-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-b4f5c52f-b8f2-4747-9399-da92c299179f-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-b4f5c52f-b8f2-4747-9399-da92c299179f-c000.gz.parquet.crc new file mode 100644 index 0000000000..2ab78c843b Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-b4f5c52f-b8f2-4747-9399-da92c299179f-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-d31f531d-8107-42cb-952b-250bc66fb332-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-d31f531d-8107-42cb-952b-250bc66fb332-c000.gz.parquet.crc new file mode 100644 index 0000000000..e9b64c2763 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-d31f531d-8107-42cb-952b-250bc66fb332-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-d7b92535-c598-47e5-a5f9-e0494472c448-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-d7b92535-c598-47e5-a5f9-e0494472c448-c000.gz.parquet.crc new file mode 100644 index 0000000000..992cc60a6b Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-d7b92535-c598-47e5-a5f9-e0494472c448-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-dfbe507d-5640-48be-9f2a-7504c4b3f1c5-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-dfbe507d-5640-48be-9f2a-7504c4b3f1c5-c000.gz.parquet.crc new file mode 100644 index 0000000000..d483ff631e Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-dfbe507d-5640-48be-9f2a-7504c4b3f1c5-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-ed4153a2-dc32-4094-9d23-e3c6db36d2ec-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-ed4153a2-dc32-4094-9d23-e3c6db36d2ec-c000.gz.parquet.crc new file mode 100644 index 0000000000..28af7cfb34 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-ed4153a2-dc32-4094-9d23-e3c6db36d2ec-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-f0b8feca-f48d-44d4-acbb-5a216f176ed4-c000.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-f0b8feca-f48d-44d4-acbb-5a216f176ed4-c000.gz.parquet.crc new file mode 100644 index 0000000000..d14983a99c Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/.part-00006-f0b8feca-f48d-44d4-acbb-5a216f176ed4-c000.gz.parquet.crc differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-r-00000-fa8c25b4-f2cf-4b87-ba3f-5181a4f50ee6.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-r-00000-fa8c25b4-f2cf-4b87-ba3f-5181a4f50ee6.gz.parquet.crc deleted file mode 100644 index 5564fe463b..0000000000 Binary files a/tests/common/src/main/resources/2015-trimmed.parquet/.part-r-00000-fa8c25b4-f2cf-4b87-ba3f-5181a4f50ee6.gz.parquet.crc and /dev/null differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/.part-r-00001-fa8c25b4-f2cf-4b87-ba3f-5181a4f50ee6.gz.parquet.crc b/tests/common/src/main/resources/2015-trimmed.parquet/.part-r-00001-fa8c25b4-f2cf-4b87-ba3f-5181a4f50ee6.gz.parquet.crc deleted file mode 100644 index cb7b3a4650..0000000000 Binary files a/tests/common/src/main/resources/2015-trimmed.parquet/.part-r-00001-fa8c25b4-f2cf-4b87-ba3f-5181a4f50ee6.gz.parquet.crc and /dev/null differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/_common_metadata b/tests/common/src/main/resources/2015-trimmed.parquet/_common_metadata deleted file mode 100644 index 9beded5f10..0000000000 Binary files a/tests/common/src/main/resources/2015-trimmed.parquet/_common_metadata and /dev/null differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/_metadata b/tests/common/src/main/resources/2015-trimmed.parquet/_metadata deleted file mode 100644 index 4c12e0ac71..0000000000 Binary files a/tests/common/src/main/resources/2015-trimmed.parquet/_metadata and /dev/null differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-00658b64-c850-4cf1-a62b-66df910b5c0d-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-00658b64-c850-4cf1-a62b-66df910b5c0d-c000.gz.parquet new file mode 100644 index 0000000000..9d02e85436 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-00658b64-c850-4cf1-a62b-66df910b5c0d-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-045af329-7928-4b99-bd5f-7509909cd629-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-045af329-7928-4b99-bd5f-7509909cd629-c000.gz.parquet new file mode 100644 index 0000000000..9d02e85436 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-045af329-7928-4b99-bd5f-7509909cd629-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-0e960084-6914-452d-878d-025d567a705b-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-0e960084-6914-452d-878d-025d567a705b-c000.gz.parquet new file mode 100644 index 0000000000..9d02e85436 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-0e960084-6914-452d-878d-025d567a705b-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-1986d9f8-af54-4ac3-9f96-ef9916a20cab-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-1986d9f8-af54-4ac3-9f96-ef9916a20cab-c000.gz.parquet new file mode 100644 index 0000000000..9d02e85436 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-1986d9f8-af54-4ac3-9f96-ef9916a20cab-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-21c7c705-a7d0-453c-a56c-2dc7cbe2cb44-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-21c7c705-a7d0-453c-a56c-2dc7cbe2cb44-c000.gz.parquet new file mode 100644 index 0000000000..9d02e85436 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-21c7c705-a7d0-453c-a56c-2dc7cbe2cb44-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-2521c941-34de-4532-af09-054c39814f92-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-2521c941-34de-4532-af09-054c39814f92-c000.gz.parquet new file mode 100644 index 0000000000..9d02e85436 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-2521c941-34de-4532-af09-054c39814f92-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-2ba5ab7e-be2e-44a6-9deb-12e7f97895d3-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-2ba5ab7e-be2e-44a6-9deb-12e7f97895d3-c000.gz.parquet new file mode 100644 index 0000000000..9d02e85436 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-2ba5ab7e-be2e-44a6-9deb-12e7f97895d3-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-30e9e719-9951-4b82-9e15-a75b47e5f696-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-30e9e719-9951-4b82-9e15-a75b47e5f696-c000.gz.parquet new file mode 100644 index 0000000000..9d02e85436 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-30e9e719-9951-4b82-9e15-a75b47e5f696-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-3103fd69-e890-4ce6-b27e-833c6ea8de80-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-3103fd69-e890-4ce6-b27e-833c6ea8de80-c000.gz.parquet new file mode 100644 index 0000000000..9d02e85436 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-3103fd69-e890-4ce6-b27e-833c6ea8de80-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-35edb6a0-7e18-44d4-a026-80fc1e82b653-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-35edb6a0-7e18-44d4-a026-80fc1e82b653-c000.gz.parquet new file mode 100644 index 0000000000..9d02e85436 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-35edb6a0-7e18-44d4-a026-80fc1e82b653-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-3b537503-41df-4274-956e-23869a4e7662-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-3b537503-41df-4274-956e-23869a4e7662-c000.gz.parquet new file mode 100644 index 0000000000..9d02e85436 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-3b537503-41df-4274-956e-23869a4e7662-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-3bcaf272-fc53-4775-8dba-5261c27ad67e-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-3bcaf272-fc53-4775-8dba-5261c27ad67e-c000.gz.parquet new file mode 100644 index 0000000000..9d02e85436 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-3bcaf272-fc53-4775-8dba-5261c27ad67e-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-4cd108be-a0f1-452f-a40a-2dd403ac79b9-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-4cd108be-a0f1-452f-a40a-2dd403ac79b9-c000.gz.parquet new file mode 100644 index 0000000000..9d02e85436 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-4cd108be-a0f1-452f-a40a-2dd403ac79b9-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-4ff6f25a-a5ab-46e3-83d9-f3f746510f64-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-4ff6f25a-a5ab-46e3-83d9-f3f746510f64-c000.gz.parquet new file mode 100644 index 0000000000..9d02e85436 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-4ff6f25a-a5ab-46e3-83d9-f3f746510f64-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-504f4563-b66b-43a1-a095-4dfe8ffbb896-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-504f4563-b66b-43a1-a095-4dfe8ffbb896-c000.gz.parquet new file mode 100644 index 0000000000..9d02e85436 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-504f4563-b66b-43a1-a095-4dfe8ffbb896-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-5e3f868d-ce23-4588-bc2e-54ac07b01b5c-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-5e3f868d-ce23-4588-bc2e-54ac07b01b5c-c000.gz.parquet new file mode 100644 index 0000000000..9d02e85436 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-5e3f868d-ce23-4588-bc2e-54ac07b01b5c-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-7135ff90-7b82-4f5a-bd23-72e5f76f6225-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-7135ff90-7b82-4f5a-bd23-72e5f76f6225-c000.gz.parquet new file mode 100644 index 0000000000..9d02e85436 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-7135ff90-7b82-4f5a-bd23-72e5f76f6225-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-7441115a-7e40-40d6-98d2-e1869f64bef2-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-7441115a-7e40-40d6-98d2-e1869f64bef2-c000.gz.parquet new file mode 100644 index 0000000000..9d02e85436 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-7441115a-7e40-40d6-98d2-e1869f64bef2-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-7bc4ec26-3c17-4f4b-82d5-85bf626078a7-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-7bc4ec26-3c17-4f4b-82d5-85bf626078a7-c000.gz.parquet new file mode 100644 index 0000000000..9d02e85436 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-7bc4ec26-3c17-4f4b-82d5-85bf626078a7-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-91e45d5f-cca3-44c7-8808-07f201d8ae97-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-91e45d5f-cca3-44c7-8808-07f201d8ae97-c000.gz.parquet new file mode 100644 index 0000000000..9d02e85436 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-91e45d5f-cca3-44c7-8808-07f201d8ae97-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-93d91940-764c-47f0-8afb-11ef79e699a5-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-93d91940-764c-47f0-8afb-11ef79e699a5-c000.gz.parquet new file mode 100644 index 0000000000..9d02e85436 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-93d91940-764c-47f0-8afb-11ef79e699a5-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-97a8da5b-ad06-4911-9661-36dbb07821ae-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-97a8da5b-ad06-4911-9661-36dbb07821ae-c000.gz.parquet new file mode 100644 index 0000000000..9d02e85436 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-97a8da5b-ad06-4911-9661-36dbb07821ae-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-9b4e3257-cf23-40b2-9760-8a8ddd82bcc8-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-9b4e3257-cf23-40b2-9760-8a8ddd82bcc8-c000.gz.parquet new file mode 100644 index 0000000000..9d02e85436 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-9b4e3257-cf23-40b2-9760-8a8ddd82bcc8-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-aafbfa5b-c7bd-4c26-9e81-263d90950ea1-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-aafbfa5b-c7bd-4c26-9e81-263d90950ea1-c000.gz.parquet new file mode 100644 index 0000000000..9d02e85436 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-aafbfa5b-c7bd-4c26-9e81-263d90950ea1-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-ab26f8b7-91b6-45da-b9fe-28b4389acfdd-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-ab26f8b7-91b6-45da-b9fe-28b4389acfdd-c000.gz.parquet new file mode 100644 index 0000000000..9d02e85436 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-ab26f8b7-91b6-45da-b9fe-28b4389acfdd-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-b4f5c52f-b8f2-4747-9399-da92c299179f-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-b4f5c52f-b8f2-4747-9399-da92c299179f-c000.gz.parquet new file mode 100644 index 0000000000..9d02e85436 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-b4f5c52f-b8f2-4747-9399-da92c299179f-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-d31f531d-8107-42cb-952b-250bc66fb332-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-d31f531d-8107-42cb-952b-250bc66fb332-c000.gz.parquet new file mode 100644 index 0000000000..9d02e85436 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-d31f531d-8107-42cb-952b-250bc66fb332-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-d7b92535-c598-47e5-a5f9-e0494472c448-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-d7b92535-c598-47e5-a5f9-e0494472c448-c000.gz.parquet new file mode 100644 index 0000000000..9d02e85436 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-d7b92535-c598-47e5-a5f9-e0494472c448-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-dfbe507d-5640-48be-9f2a-7504c4b3f1c5-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-dfbe507d-5640-48be-9f2a-7504c4b3f1c5-c000.gz.parquet new file mode 100644 index 0000000000..9d02e85436 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-dfbe507d-5640-48be-9f2a-7504c4b3f1c5-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-ed4153a2-dc32-4094-9d23-e3c6db36d2ec-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-ed4153a2-dc32-4094-9d23-e3c6db36d2ec-c000.gz.parquet new file mode 100644 index 0000000000..9d02e85436 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-ed4153a2-dc32-4094-9d23-e3c6db36d2ec-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-f0b8feca-f48d-44d4-acbb-5a216f176ed4-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-f0b8feca-f48d-44d4-acbb-5a216f176ed4-c000.gz.parquet new file mode 100644 index 0000000000..9d02e85436 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00000-f0b8feca-f48d-44d4-acbb-5a216f176ed4-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-00658b64-c850-4cf1-a62b-66df910b5c0d-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-00658b64-c850-4cf1-a62b-66df910b5c0d-c000.gz.parquet new file mode 100644 index 0000000000..59989f13e8 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-00658b64-c850-4cf1-a62b-66df910b5c0d-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-045af329-7928-4b99-bd5f-7509909cd629-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-045af329-7928-4b99-bd5f-7509909cd629-c000.gz.parquet new file mode 100644 index 0000000000..9a3c26b5ef Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-045af329-7928-4b99-bd5f-7509909cd629-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-0e960084-6914-452d-878d-025d567a705b-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-0e960084-6914-452d-878d-025d567a705b-c000.gz.parquet new file mode 100644 index 0000000000..4745a00c4b Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-0e960084-6914-452d-878d-025d567a705b-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-1986d9f8-af54-4ac3-9f96-ef9916a20cab-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-1986d9f8-af54-4ac3-9f96-ef9916a20cab-c000.gz.parquet new file mode 100644 index 0000000000..cc480f5ac4 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-1986d9f8-af54-4ac3-9f96-ef9916a20cab-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-21c7c705-a7d0-453c-a56c-2dc7cbe2cb44-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-21c7c705-a7d0-453c-a56c-2dc7cbe2cb44-c000.gz.parquet new file mode 100644 index 0000000000..6e01ae011c Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-21c7c705-a7d0-453c-a56c-2dc7cbe2cb44-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-2521c941-34de-4532-af09-054c39814f92-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-2521c941-34de-4532-af09-054c39814f92-c000.gz.parquet new file mode 100644 index 0000000000..f2c536e28a Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-2521c941-34de-4532-af09-054c39814f92-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-2ba5ab7e-be2e-44a6-9deb-12e7f97895d3-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-2ba5ab7e-be2e-44a6-9deb-12e7f97895d3-c000.gz.parquet new file mode 100644 index 0000000000..de0324a3a3 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-2ba5ab7e-be2e-44a6-9deb-12e7f97895d3-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-30e9e719-9951-4b82-9e15-a75b47e5f696-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-30e9e719-9951-4b82-9e15-a75b47e5f696-c000.gz.parquet new file mode 100644 index 0000000000..ea155d5721 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-30e9e719-9951-4b82-9e15-a75b47e5f696-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-3103fd69-e890-4ce6-b27e-833c6ea8de80-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-3103fd69-e890-4ce6-b27e-833c6ea8de80-c000.gz.parquet new file mode 100644 index 0000000000..4dd214c918 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-3103fd69-e890-4ce6-b27e-833c6ea8de80-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-35edb6a0-7e18-44d4-a026-80fc1e82b653-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-35edb6a0-7e18-44d4-a026-80fc1e82b653-c000.gz.parquet new file mode 100644 index 0000000000..077d7f5287 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-35edb6a0-7e18-44d4-a026-80fc1e82b653-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-3b537503-41df-4274-956e-23869a4e7662-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-3b537503-41df-4274-956e-23869a4e7662-c000.gz.parquet new file mode 100644 index 0000000000..0c5ecd2b1f Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-3b537503-41df-4274-956e-23869a4e7662-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-3bcaf272-fc53-4775-8dba-5261c27ad67e-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-3bcaf272-fc53-4775-8dba-5261c27ad67e-c000.gz.parquet new file mode 100644 index 0000000000..76b676686f Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-3bcaf272-fc53-4775-8dba-5261c27ad67e-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-4cd108be-a0f1-452f-a40a-2dd403ac79b9-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-4cd108be-a0f1-452f-a40a-2dd403ac79b9-c000.gz.parquet new file mode 100644 index 0000000000..ca9d533969 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-4cd108be-a0f1-452f-a40a-2dd403ac79b9-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-4ff6f25a-a5ab-46e3-83d9-f3f746510f64-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-4ff6f25a-a5ab-46e3-83d9-f3f746510f64-c000.gz.parquet new file mode 100644 index 0000000000..cb5a61c3c2 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-4ff6f25a-a5ab-46e3-83d9-f3f746510f64-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-504f4563-b66b-43a1-a095-4dfe8ffbb896-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-504f4563-b66b-43a1-a095-4dfe8ffbb896-c000.gz.parquet new file mode 100644 index 0000000000..f63cfc32b5 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-504f4563-b66b-43a1-a095-4dfe8ffbb896-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-5e3f868d-ce23-4588-bc2e-54ac07b01b5c-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-5e3f868d-ce23-4588-bc2e-54ac07b01b5c-c000.gz.parquet new file mode 100644 index 0000000000..6cab14bc16 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-5e3f868d-ce23-4588-bc2e-54ac07b01b5c-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-7135ff90-7b82-4f5a-bd23-72e5f76f6225-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-7135ff90-7b82-4f5a-bd23-72e5f76f6225-c000.gz.parquet new file mode 100644 index 0000000000..153bbe7396 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-7135ff90-7b82-4f5a-bd23-72e5f76f6225-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-7441115a-7e40-40d6-98d2-e1869f64bef2-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-7441115a-7e40-40d6-98d2-e1869f64bef2-c000.gz.parquet new file mode 100644 index 0000000000..b75156e7dc Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-7441115a-7e40-40d6-98d2-e1869f64bef2-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-7bc4ec26-3c17-4f4b-82d5-85bf626078a7-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-7bc4ec26-3c17-4f4b-82d5-85bf626078a7-c000.gz.parquet new file mode 100644 index 0000000000..66f082ef51 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-7bc4ec26-3c17-4f4b-82d5-85bf626078a7-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-91e45d5f-cca3-44c7-8808-07f201d8ae97-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-91e45d5f-cca3-44c7-8808-07f201d8ae97-c000.gz.parquet new file mode 100644 index 0000000000..b0d8d99ce6 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-91e45d5f-cca3-44c7-8808-07f201d8ae97-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-93d91940-764c-47f0-8afb-11ef79e699a5-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-93d91940-764c-47f0-8afb-11ef79e699a5-c000.gz.parquet new file mode 100644 index 0000000000..051078a1b7 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-93d91940-764c-47f0-8afb-11ef79e699a5-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-97a8da5b-ad06-4911-9661-36dbb07821ae-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-97a8da5b-ad06-4911-9661-36dbb07821ae-c000.gz.parquet new file mode 100644 index 0000000000..cadcc5528a Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-97a8da5b-ad06-4911-9661-36dbb07821ae-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-9b4e3257-cf23-40b2-9760-8a8ddd82bcc8-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-9b4e3257-cf23-40b2-9760-8a8ddd82bcc8-c000.gz.parquet new file mode 100644 index 0000000000..cdc3c5a4fa Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-9b4e3257-cf23-40b2-9760-8a8ddd82bcc8-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-aafbfa5b-c7bd-4c26-9e81-263d90950ea1-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-aafbfa5b-c7bd-4c26-9e81-263d90950ea1-c000.gz.parquet new file mode 100644 index 0000000000..9674613ff9 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-aafbfa5b-c7bd-4c26-9e81-263d90950ea1-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-ab26f8b7-91b6-45da-b9fe-28b4389acfdd-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-ab26f8b7-91b6-45da-b9fe-28b4389acfdd-c000.gz.parquet new file mode 100644 index 0000000000..bfd7f00bcd Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-ab26f8b7-91b6-45da-b9fe-28b4389acfdd-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-b4f5c52f-b8f2-4747-9399-da92c299179f-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-b4f5c52f-b8f2-4747-9399-da92c299179f-c000.gz.parquet new file mode 100644 index 0000000000..466e006d03 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-b4f5c52f-b8f2-4747-9399-da92c299179f-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-d31f531d-8107-42cb-952b-250bc66fb332-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-d31f531d-8107-42cb-952b-250bc66fb332-c000.gz.parquet new file mode 100644 index 0000000000..b907fb0590 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-d31f531d-8107-42cb-952b-250bc66fb332-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-d7b92535-c598-47e5-a5f9-e0494472c448-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-d7b92535-c598-47e5-a5f9-e0494472c448-c000.gz.parquet new file mode 100644 index 0000000000..8b05dc9ad2 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-d7b92535-c598-47e5-a5f9-e0494472c448-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-dfbe507d-5640-48be-9f2a-7504c4b3f1c5-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-dfbe507d-5640-48be-9f2a-7504c4b3f1c5-c000.gz.parquet new file mode 100644 index 0000000000..09da836024 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-dfbe507d-5640-48be-9f2a-7504c4b3f1c5-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-ed4153a2-dc32-4094-9d23-e3c6db36d2ec-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-ed4153a2-dc32-4094-9d23-e3c6db36d2ec-c000.gz.parquet new file mode 100644 index 0000000000..8a83851286 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-ed4153a2-dc32-4094-9d23-e3c6db36d2ec-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-f0b8feca-f48d-44d4-acbb-5a216f176ed4-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-f0b8feca-f48d-44d4-acbb-5a216f176ed4-c000.gz.parquet new file mode 100644 index 0000000000..bba7633d8e Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00005-f0b8feca-f48d-44d4-acbb-5a216f176ed4-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-00658b64-c850-4cf1-a62b-66df910b5c0d-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-00658b64-c850-4cf1-a62b-66df910b5c0d-c000.gz.parquet new file mode 100644 index 0000000000..3c65721cb1 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-00658b64-c850-4cf1-a62b-66df910b5c0d-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-045af329-7928-4b99-bd5f-7509909cd629-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-045af329-7928-4b99-bd5f-7509909cd629-c000.gz.parquet new file mode 100644 index 0000000000..697b63824a Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-045af329-7928-4b99-bd5f-7509909cd629-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-0e960084-6914-452d-878d-025d567a705b-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-0e960084-6914-452d-878d-025d567a705b-c000.gz.parquet new file mode 100644 index 0000000000..dd1adba2f1 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-0e960084-6914-452d-878d-025d567a705b-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-1986d9f8-af54-4ac3-9f96-ef9916a20cab-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-1986d9f8-af54-4ac3-9f96-ef9916a20cab-c000.gz.parquet new file mode 100644 index 0000000000..8874eb1fb0 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-1986d9f8-af54-4ac3-9f96-ef9916a20cab-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-21c7c705-a7d0-453c-a56c-2dc7cbe2cb44-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-21c7c705-a7d0-453c-a56c-2dc7cbe2cb44-c000.gz.parquet new file mode 100644 index 0000000000..e1836d6d85 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-21c7c705-a7d0-453c-a56c-2dc7cbe2cb44-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-2521c941-34de-4532-af09-054c39814f92-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-2521c941-34de-4532-af09-054c39814f92-c000.gz.parquet new file mode 100644 index 0000000000..f92c54f219 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-2521c941-34de-4532-af09-054c39814f92-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-2ba5ab7e-be2e-44a6-9deb-12e7f97895d3-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-2ba5ab7e-be2e-44a6-9deb-12e7f97895d3-c000.gz.parquet new file mode 100644 index 0000000000..509f917128 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-2ba5ab7e-be2e-44a6-9deb-12e7f97895d3-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-30e9e719-9951-4b82-9e15-a75b47e5f696-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-30e9e719-9951-4b82-9e15-a75b47e5f696-c000.gz.parquet new file mode 100644 index 0000000000..56b803778d Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-30e9e719-9951-4b82-9e15-a75b47e5f696-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-3103fd69-e890-4ce6-b27e-833c6ea8de80-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-3103fd69-e890-4ce6-b27e-833c6ea8de80-c000.gz.parquet new file mode 100644 index 0000000000..ffcbcf72ea Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-3103fd69-e890-4ce6-b27e-833c6ea8de80-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-35edb6a0-7e18-44d4-a026-80fc1e82b653-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-35edb6a0-7e18-44d4-a026-80fc1e82b653-c000.gz.parquet new file mode 100644 index 0000000000..64f6f561e8 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-35edb6a0-7e18-44d4-a026-80fc1e82b653-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-3b537503-41df-4274-956e-23869a4e7662-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-3b537503-41df-4274-956e-23869a4e7662-c000.gz.parquet new file mode 100644 index 0000000000..ec454fdd36 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-3b537503-41df-4274-956e-23869a4e7662-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-3bcaf272-fc53-4775-8dba-5261c27ad67e-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-3bcaf272-fc53-4775-8dba-5261c27ad67e-c000.gz.parquet new file mode 100644 index 0000000000..5778b8dac8 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-3bcaf272-fc53-4775-8dba-5261c27ad67e-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-4cd108be-a0f1-452f-a40a-2dd403ac79b9-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-4cd108be-a0f1-452f-a40a-2dd403ac79b9-c000.gz.parquet new file mode 100644 index 0000000000..57e21037fa Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-4cd108be-a0f1-452f-a40a-2dd403ac79b9-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-4ff6f25a-a5ab-46e3-83d9-f3f746510f64-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-4ff6f25a-a5ab-46e3-83d9-f3f746510f64-c000.gz.parquet new file mode 100644 index 0000000000..87dcf876fd Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-4ff6f25a-a5ab-46e3-83d9-f3f746510f64-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-504f4563-b66b-43a1-a095-4dfe8ffbb896-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-504f4563-b66b-43a1-a095-4dfe8ffbb896-c000.gz.parquet new file mode 100644 index 0000000000..b7c2c38a83 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-504f4563-b66b-43a1-a095-4dfe8ffbb896-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-5e3f868d-ce23-4588-bc2e-54ac07b01b5c-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-5e3f868d-ce23-4588-bc2e-54ac07b01b5c-c000.gz.parquet new file mode 100644 index 0000000000..9bac613a76 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-5e3f868d-ce23-4588-bc2e-54ac07b01b5c-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-7135ff90-7b82-4f5a-bd23-72e5f76f6225-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-7135ff90-7b82-4f5a-bd23-72e5f76f6225-c000.gz.parquet new file mode 100644 index 0000000000..ce71b966ab Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-7135ff90-7b82-4f5a-bd23-72e5f76f6225-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-7441115a-7e40-40d6-98d2-e1869f64bef2-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-7441115a-7e40-40d6-98d2-e1869f64bef2-c000.gz.parquet new file mode 100644 index 0000000000..4b7244f65b Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-7441115a-7e40-40d6-98d2-e1869f64bef2-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-7bc4ec26-3c17-4f4b-82d5-85bf626078a7-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-7bc4ec26-3c17-4f4b-82d5-85bf626078a7-c000.gz.parquet new file mode 100644 index 0000000000..f6ecc1b9ee Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-7bc4ec26-3c17-4f4b-82d5-85bf626078a7-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-91e45d5f-cca3-44c7-8808-07f201d8ae97-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-91e45d5f-cca3-44c7-8808-07f201d8ae97-c000.gz.parquet new file mode 100644 index 0000000000..f2df7c9e48 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-91e45d5f-cca3-44c7-8808-07f201d8ae97-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-93d91940-764c-47f0-8afb-11ef79e699a5-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-93d91940-764c-47f0-8afb-11ef79e699a5-c000.gz.parquet new file mode 100644 index 0000000000..4e12f62afa Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-93d91940-764c-47f0-8afb-11ef79e699a5-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-97a8da5b-ad06-4911-9661-36dbb07821ae-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-97a8da5b-ad06-4911-9661-36dbb07821ae-c000.gz.parquet new file mode 100644 index 0000000000..f3daa58c87 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-97a8da5b-ad06-4911-9661-36dbb07821ae-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-9b4e3257-cf23-40b2-9760-8a8ddd82bcc8-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-9b4e3257-cf23-40b2-9760-8a8ddd82bcc8-c000.gz.parquet new file mode 100644 index 0000000000..1b85733889 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-9b4e3257-cf23-40b2-9760-8a8ddd82bcc8-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-aafbfa5b-c7bd-4c26-9e81-263d90950ea1-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-aafbfa5b-c7bd-4c26-9e81-263d90950ea1-c000.gz.parquet new file mode 100644 index 0000000000..0cc52a6633 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-aafbfa5b-c7bd-4c26-9e81-263d90950ea1-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-ab26f8b7-91b6-45da-b9fe-28b4389acfdd-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-ab26f8b7-91b6-45da-b9fe-28b4389acfdd-c000.gz.parquet new file mode 100644 index 0000000000..7eb642e996 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-ab26f8b7-91b6-45da-b9fe-28b4389acfdd-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-b4f5c52f-b8f2-4747-9399-da92c299179f-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-b4f5c52f-b8f2-4747-9399-da92c299179f-c000.gz.parquet new file mode 100644 index 0000000000..521e492936 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-b4f5c52f-b8f2-4747-9399-da92c299179f-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-d31f531d-8107-42cb-952b-250bc66fb332-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-d31f531d-8107-42cb-952b-250bc66fb332-c000.gz.parquet new file mode 100644 index 0000000000..14ffd64230 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-d31f531d-8107-42cb-952b-250bc66fb332-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-d7b92535-c598-47e5-a5f9-e0494472c448-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-d7b92535-c598-47e5-a5f9-e0494472c448-c000.gz.parquet new file mode 100644 index 0000000000..2f0d9922eb Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-d7b92535-c598-47e5-a5f9-e0494472c448-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-dfbe507d-5640-48be-9f2a-7504c4b3f1c5-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-dfbe507d-5640-48be-9f2a-7504c4b3f1c5-c000.gz.parquet new file mode 100644 index 0000000000..d78f4efdc7 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-dfbe507d-5640-48be-9f2a-7504c4b3f1c5-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-ed4153a2-dc32-4094-9d23-e3c6db36d2ec-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-ed4153a2-dc32-4094-9d23-e3c6db36d2ec-c000.gz.parquet new file mode 100644 index 0000000000..17c8f780f4 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-ed4153a2-dc32-4094-9d23-e3c6db36d2ec-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-f0b8feca-f48d-44d4-acbb-5a216f176ed4-c000.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-f0b8feca-f48d-44d4-acbb-5a216f176ed4-c000.gz.parquet new file mode 100644 index 0000000000..6870157c54 Binary files /dev/null and b/tests/common/src/main/resources/2015-trimmed.parquet/part-00006-f0b8feca-f48d-44d4-acbb-5a216f176ed4-c000.gz.parquet differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-r-00000-fa8c25b4-f2cf-4b87-ba3f-5181a4f50ee6.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-r-00000-fa8c25b4-f2cf-4b87-ba3f-5181a4f50ee6.gz.parquet deleted file mode 100644 index 041cd8f1f3..0000000000 Binary files a/tests/common/src/main/resources/2015-trimmed.parquet/part-r-00000-fa8c25b4-f2cf-4b87-ba3f-5181a4f50ee6.gz.parquet and /dev/null differ diff --git a/tests/common/src/main/resources/2015-trimmed.parquet/part-r-00001-fa8c25b4-f2cf-4b87-ba3f-5181a4f50ee6.gz.parquet b/tests/common/src/main/resources/2015-trimmed.parquet/part-r-00001-fa8c25b4-f2cf-4b87-ba3f-5181a4f50ee6.gz.parquet deleted file mode 100644 index 3f473e0c42..0000000000 Binary files a/tests/common/src/main/resources/2015-trimmed.parquet/part-r-00001-fa8c25b4-f2cf-4b87-ba3f-5181a4f50ee6.gz.parquet and /dev/null differ