Packages

class SQLConf extends Serializable with Logging

A class that enables the setting and getting of mutable config parameters/hints.

In the presence of a SQLContext, these can be set and queried by passing SET commands into Spark SQL's query functions (i.e. sql()). Otherwise, users of this class can modify the hints by programmatically calling the setters and getters of this class.

SQLConf is thread-safe (internally synchronized, so safe to be used in multiple threads).

Linear Supertypes
Logging, Serializable, Serializable, AnyRef, Any
Known Subclasses
ReadOnlySQLConf
Ordering
  1. Alphabetic
  2. By Inheritance
Inherited
  1. SQLConf
  2. Logging
  3. Serializable
  4. Serializable
  5. AnyRef
  6. Any
  1. Hide All
  2. Show All
Visibility
  1. Public
  2. All

Instance Constructors

  1. new SQLConf()

Value Members

  1. final def !=(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  2. final def ##(): Int
    Definition Classes
    AnyRef → Any
  3. final def ==(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  4. def adaptiveExecutionEnabled: Boolean
  5. def adaptiveExecutionLogLevel: String
  6. def addSingleFileInAddFile: Boolean
  7. def advancedPartitionPredicatePushdownEnabled: Boolean
  8. def allowNegativeScaleOfDecimalEnabled: Boolean
  9. def analyzerMaxIterations: Int

    ************************ Spark SQL Params/Hints *******************

  10. def ansiEnabled: Boolean
  11. def arrowMaxRecordsPerBatch: Int
  12. def arrowPySparkEnabled: Boolean
  13. def arrowPySparkFallbackEnabled: Boolean
  14. def arrowSafeTypeConversion: Boolean
  15. def arrowSparkREnabled: Boolean
  16. final def asInstanceOf[T0]: T0
    Definition Classes
    Any
  17. def autoBroadcastJoinThreshold: Long
  18. def autoBucketedScanEnabled: Boolean
  19. def autoSizeUpdateEnabled: Boolean
  20. def avroCompressionCodec: String
  21. def avroDeflateLevel: Int
  22. def avroFilterPushDown: Boolean
  23. def broadcastHashJoinOutputPartitioningExpandLimit: Int
  24. def broadcastTimeout: Long
  25. def bucketingEnabled: Boolean
  26. def bucketingMaxBuckets: Int
  27. def cacheVectorizedReaderEnabled: Boolean
  28. def cartesianProductExecBufferInMemoryThreshold: Int
  29. def cartesianProductExecBufferSpillThreshold: Int
  30. def caseSensitiveAnalysis: Boolean
  31. def caseSensitiveInferenceMode: SQLConf.HiveCaseSensitiveInferenceMode.Value
  32. def castDatetimeToString: Boolean
  33. def cboEnabled: Boolean
  34. def charVarcharAsString: Boolean
  35. def checkpointLocation: Option[String]
  36. def clear(): Unit
  37. def clone(): SQLConf
    Definition Classes
    SQLConf → AnyRef
  38. def coalesceBucketsInJoinEnabled: Boolean
  39. def coalesceBucketsInJoinMaxBucketRatio: Int
  40. def coalesceShufflePartitionsEnabled: Boolean
  41. def codegenCacheMaxEntries: Int
  42. def codegenComments: Boolean
  43. def codegenFallback: Boolean
  44. def codegenSplitAggregateFunc: Boolean
  45. def columnBatchSize: Int
  46. def columnNameOfCorruptRecord: String
  47. def concatBinaryAsString: Boolean
  48. def constraintPropagationEnabled: Boolean
  49. def contains(key: String): Boolean

    Return whether a given key is set in this SQLConf.

  50. def continuousStreamingEpochBacklogQueueSize: Int
  51. def continuousStreamingExecutorPollIntervalMs: Long
  52. def continuousStreamingExecutorQueueSize: Int
  53. def convertCTAS: Boolean
  54. def copy(entries: (ConfigEntry[_], Any)*): SQLConf
  55. def crossJoinEnabled: Boolean
  56. def csvColumnPruning: Boolean
  57. def csvFilterPushDown: Boolean
  58. def dataFramePivotMaxValues: Int
  59. def dataFrameRetainGroupColumns: Boolean
  60. def dataFrameSelfJoinAutoResolveAmbiguity: Boolean
  61. def datetimeJava8ApiEnabled: Boolean
  62. def decimalOperationsAllowPrecisionLoss: Boolean
  63. def defaultDataSourceName: String
  64. def defaultNumShufflePartitions: Int
  65. def defaultSizeInBytes: Long
  66. def disabledJdbcConnectionProviders: String
  67. def disabledV2StreamingMicroBatchReaders: String
  68. def disabledV2StreamingWriters: String
  69. def dynamicPartitionPruningEnabled: Boolean
  70. def dynamicPartitionPruningFallbackFilterRatio: Double
  71. def dynamicPartitionPruningReuseBroadcastOnly: Boolean
  72. def dynamicPartitionPruningUseStats: Boolean
  73. def eltOutputAsString: Boolean
  74. def enableRadixSort: Boolean
  75. def enableTwoLevelAggMap: Boolean
  76. def enableVectorizedHashMap: Boolean
  77. final def eq(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  78. def equals(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  79. def escapedStringLiterals: Boolean
  80. def exchangeReuseEnabled: Boolean
  81. def exponentLiteralAsDecimalEnabled: Boolean
  82. def fallBackToHdfsForStatsEnabled: Boolean
  83. def fastFailFileFormatOutput: Boolean
  84. def fastHashAggregateRowMaxCapacityBit: Int
  85. def fetchShuffleBlocksInBatch: Boolean
  86. def fileCommitProtocolClass: String
  87. def fileCompressionFactor: Double
  88. def fileSinkLogCleanupDelay: Long
  89. def fileSinkLogCompactInterval: Int
  90. def fileSinkLogDeletion: Boolean
  91. def fileSourceLogCleanupDelay: Long
  92. def fileSourceLogCompactInterval: Int
  93. def fileSourceLogDeletion: Boolean
  94. def filesMaxPartitionBytes: Long
  95. def filesMinPartitionNum: Option[Int]
  96. def filesOpenCostInBytes: Long
  97. def filesourcePartitionFileCacheSize: Long
  98. def finalize(): Unit
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( classOf[java.lang.Throwable] )
  99. def gatherFastStats: Boolean
  100. def getAllConfs: Map[String, String]

    Return all the configuration properties that have been set (i.e.

    Return all the configuration properties that have been set (i.e. not the default). This creates a new copy of the config properties in the form of a Map.

  101. def getAllDefinedConfs: Seq[(String, String, String, String)]

    Return all the configuration definitions that have been defined in SQLConf.

    Return all the configuration definitions that have been defined in SQLConf. Each definition contains key, defaultValue and doc.

  102. final def getClass(): Class[_]
    Definition Classes
    AnyRef → Any
    Annotations
    @native()
  103. def getConf[T](entry: OptionalConfigEntry[T]): Option[T]

    Return the value of an optional Spark SQL configuration property for the given key.

    Return the value of an optional Spark SQL configuration property for the given key. If the key is not set yet, returns None.

  104. def getConf[T](entry: ConfigEntry[T]): T

    Return the value of Spark SQL configuration property for the given key.

    Return the value of Spark SQL configuration property for the given key. If the key is not set yet, return defaultValue in ConfigEntry.

  105. def getConf[T](entry: ConfigEntry[T], defaultValue: T): T

    Return the value of Spark SQL configuration property for the given key.

    Return the value of Spark SQL configuration property for the given key. If the key is not set yet, return defaultValue. This is useful when defaultValue in ConfigEntry is not the desired one.

  106. def getConfString(key: String, defaultValue: String): String

    Return the string value of Spark SQL configuration property for the given key.

    Return the string value of Spark SQL configuration property for the given key. If the key is not set yet, return defaultValue.

  107. def getConfString(key: String): String

    Return the value of Spark SQL configuration property for the given key.

    Return the value of Spark SQL configuration property for the given key.

    Annotations
    @throws( "if key is not set" )
  108. def groupByAliases: Boolean
  109. def groupByOrdinal: Boolean
  110. def hashCode(): Int
    Definition Classes
    AnyRef → Any
    Annotations
    @native()
  111. def hintErrorHandler: HintErrorHandler

    Returns the error handler for handling hint errors.

  112. def histogramEnabled: Boolean
  113. def histogramNumBins: Int
  114. def hiveThriftServerSingleSession: Boolean
  115. def hugeMethodLimit: Int
  116. def ignoreCorruptFiles: Boolean
  117. def ignoreDataLocality: Boolean
  118. def ignoreMissingFiles: Boolean
  119. def inMemoryPartitionPruning: Boolean
  120. def inMemoryTableScanStatisticsEnabled: Boolean
  121. def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean
    Attributes
    protected
    Definition Classes
    Logging
  122. def initializeLogIfNecessary(isInterpreter: Boolean): Unit
    Attributes
    protected
    Definition Classes
    Logging
  123. def integerGroupingIdEnabled: Boolean
  124. final def isInstanceOf[T0]: Boolean
    Definition Classes
    Any
  125. def isModifiable(key: String): Boolean
  126. def isOrcSchemaMergingEnabled: Boolean
  127. def isParquetBinaryAsString: Boolean
  128. def isParquetINT96AsTimestamp: Boolean
  129. def isParquetINT96TimestampConversion: Boolean
  130. def isParquetSchemaMergingEnabled: Boolean
  131. def isParquetSchemaRespectSummaries: Boolean
  132. def isReplEagerEvalEnabled: Boolean
  133. def isStateSchemaCheckEnabled: Boolean
  134. def isTraceEnabled(): Boolean
    Attributes
    protected
    Definition Classes
    Logging
  135. def isUnsupportedOperationCheckEnabled: Boolean
  136. def joinReorderCardWeight: Double
  137. def joinReorderDPStarFilter: Boolean
  138. def joinReorderDPThreshold: Int
  139. def joinReorderEnabled: Boolean
  140. def jsonExpressionOptimization: Boolean
  141. def jsonFilterPushDown: Boolean
  142. def jsonGeneratorIgnoreNullFields: Boolean
  143. def legacyMsSqlServerNumericMappingEnabled: Boolean
  144. def legacyPathOptionBehavior: Boolean
  145. def legacySizeOfNull: Boolean
  146. def legacyStatisticalAggregate: Boolean
  147. def legacyTimeParserPolicy: SQLConf.LegacyBehaviorPolicy.Value
  148. def limitScaleUpFactor: Int
  149. def literalPickMinimumPrecision: Boolean
  150. def log: Logger
    Attributes
    protected
    Definition Classes
    Logging
  151. def logDebug(msg: ⇒ String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  152. def logDebug(msg: ⇒ String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  153. def logError(msg: ⇒ String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  154. def logError(msg: ⇒ String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  155. def logInfo(msg: ⇒ String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  156. def logInfo(msg: ⇒ String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  157. def logName: String
    Attributes
    protected
    Definition Classes
    Logging
  158. def logTrace(msg: ⇒ String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  159. def logTrace(msg: ⇒ String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  160. def logWarning(msg: ⇒ String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  161. def logWarning(msg: ⇒ String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  162. def loggingMaxLinesForCodegen: Int
  163. def manageFilesourcePartitions: Boolean
  164. def maxBatchesToRetainInMemory: Int
  165. def maxMetadataStringLength: Int
  166. def maxNestedViewDepth: Int
  167. def maxPlanStringLength: Int
  168. def maxRecordsPerFile: Long
  169. def maxToStringFields: Int
  170. def metadataCacheTTL: Long
  171. def metastorePartitionPruning: Boolean
  172. def metastorePartitionPruningInSetThreshold: Int
  173. def methodSplitThreshold: Int
  174. def minBatchesToRetain: Int
  175. def nameNonStructGroupingKeyAsValue: Boolean
  176. def ndvMaxError: Double
  177. final def ne(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  178. def nestedPruningOnExpressions: Boolean
  179. def nestedSchemaPruningEnabled: Boolean
  180. def nonEmptyPartitionRatioForBroadcastJoin: Double
  181. final def notify(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native()
  182. final def notifyAll(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native()
  183. def numShufflePartitions: Int
  184. def objectAggSortBasedFallbackThreshold: Int
  185. def offHeapColumnVectorEnabled: Boolean
  186. def optimizeNullAwareAntiJoin: Boolean
  187. def optimizerExcludedRules: Option[String]
  188. def optimizerInSetConversionThreshold: Int
  189. def optimizerInSetSwitchThreshold: Int
  190. def optimizerMaxIterations: Int
  191. def optimizerMetadataOnly: Boolean
  192. def orcCompressionCodec: String
  193. def orcFilterPushDown: Boolean
  194. def orcVectorizedReaderBatchSize: Int
  195. def orcVectorizedReaderEnabled: Boolean
  196. def orderByOrdinal: Boolean
  197. def pandasGroupedMapAssignColumnsByName: Boolean
  198. def pandasUDFBufferSize: Int
  199. def parallelFileListingInStatsComputation: Boolean
  200. def parallelPartitionDiscoveryParallelism: Int
  201. def parallelPartitionDiscoveryThreshold: Int
  202. def parquetCompressionCodec: String
  203. def parquetFilterPushDown: Boolean
  204. def parquetFilterPushDownDate: Boolean
  205. def parquetFilterPushDownDecimal: Boolean
  206. def parquetFilterPushDownInFilterThreshold: Int
  207. def parquetFilterPushDownStringStartWith: Boolean
  208. def parquetFilterPushDownTimestamp: Boolean
  209. def parquetOutputCommitterClass: String
  210. def parquetOutputTimestampType: SQLConf.ParquetOutputTimestampType.Value
  211. def parquetRecordFilterEnabled: Boolean
  212. def parquetVectorizedReaderBatchSize: Int
  213. def parquetVectorizedReaderEnabled: Boolean
  214. def partitionColumnTypeInferenceEnabled: Boolean
  215. def partitionOverwriteMode: SQLConf.PartitionOverwriteMode.Value
  216. def percentileAccuracy: Int
  217. def planChangeBatches: Option[String]
  218. def planChangeLogLevel: String
  219. def planChangeRules: Option[String]
  220. def planStatsEnabled: Boolean
  221. def preferSortMergeJoin: Boolean
  222. def pysparkJVMStacktraceEnabled: Boolean
  223. def pysparkSimplifiedTraceback: Boolean
  224. def rangeExchangeSampleSizePerPartition: Int
  225. val reader: ConfigReader
    Attributes
    protected
  226. def redactOptions[K, V](options: Seq[(K, V)]): Seq[(K, V)]

    Redacts the given option map according to the description of SQL_OPTIONS_REDACTION_PATTERN.

  227. def redactOptions[K, V](options: Map[K, V]): Map[K, V]

    Redacts the given option map according to the description of SQL_OPTIONS_REDACTION_PATTERN.

  228. def replEagerEvalMaxNumRows: Int
  229. def replEagerEvalTruncate: Int
  230. def replaceDatabricksSparkAvroEnabled: Boolean
  231. def replaceExceptWithFilter: Boolean
  232. def resolver: Resolver

    Returns the Resolver for the current configuration, which can be used to determine if two identifiers are equal.

  233. def runSQLonFile: Boolean
  234. def serializerNestedSchemaPruningEnabled: Boolean
  235. def sessionLocalTimeZone: String
  236. def setCommandRejectsSparkCoreConfs: Boolean
  237. def setConf[T](entry: ConfigEntry[T], value: T): Unit

    Set the given Spark SQL configuration property.

  238. def setConf(props: Properties): Unit

    Set Spark SQL configuration properties.

  239. def setConfString(key: String, value: String): Unit

    Set the given Spark SQL configuration property using a string value.

  240. def setConfWithCheck(key: String, value: String): Unit
    Attributes
    protected
  241. def setOpsPrecedenceEnforced: Boolean
  242. val settings: Map[String, String]

    Only low degree of contention is expected for conf, thus NOT using ConcurrentHashMap.

    Only low degree of contention is expected for conf, thus NOT using ConcurrentHashMap.

    Attributes
    protected[spark]
  243. def sortBeforeRepartition: Boolean
  244. def sortMergeJoinExecBufferInMemoryThreshold: Int
  245. def sortMergeJoinExecBufferSpillThreshold: Int
  246. def starSchemaDetection: Boolean
  247. def starSchemaFTRatio: Double
  248. def stateStoreCompressionCodec: String
  249. def stateStoreFormatValidationEnabled: Boolean
  250. def stateStoreMinDeltasForSnapshot: Int
  251. def stateStoreProviderClass: String
  252. def statefulOperatorCorrectnessCheckEnabled: Boolean
  253. def storeAnalyzedPlanForView: Boolean
  254. def storeAssignmentPolicy: SQLConf.StoreAssignmentPolicy.Value
  255. def streamingFileCommitProtocolClass: String
  256. def streamingMaintenanceInterval: Long
  257. def streamingMetricsEnabled: Boolean
  258. def streamingNoDataMicroBatchesEnabled: Boolean
  259. def streamingNoDataProgressEventInterval: Long
  260. def streamingPollingDelay: Long
  261. def streamingProgressRetention: Int
  262. def streamingSchemaInference: Boolean
  263. def stringRedactionPattern: Option[Regex]
  264. def subexpressionEliminationCacheMaxEntries: Int
  265. def subexpressionEliminationEnabled: Boolean
  266. def subqueryReuseEnabled: Boolean
  267. def supportQuotedRegexColumnName: Boolean
  268. final def synchronized[T0](arg0: ⇒ T0): T0
    Definition Classes
    AnyRef
  269. def tableRelationCacheSize: Int
  270. def toString(): String
    Definition Classes
    AnyRef → Any
  271. def topKSortFallbackThreshold: Int
  272. def truncateTableIgnorePermissionAcl: Boolean
  273. def uiExplainMode: String
  274. def unsetConf(entry: ConfigEntry[_]): Unit
  275. def unsetConf(key: String): Unit
  276. def useCompression: Boolean
  277. def useCurrentSQLConfigsForView: Boolean
  278. def useDeprecatedKafkaOffsetFetching: Boolean
  279. def useObjectHashAggregation: Boolean
  280. def validatePartitionColumns: Boolean
  281. def variableSubstituteEnabled: Boolean
  282. def verifyPartitionPath: Boolean
  283. final def wait(): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  284. final def wait(arg0: Long, arg1: Int): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  285. final def wait(arg0: Long): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... ) @native()
  286. def warehousePath: String
  287. def wholeStageEnabled: Boolean
  288. def wholeStageMaxNumFields: Int
  289. def wholeStageSplitConsumeFuncByOperator: Boolean
  290. def wholeStageUseIdInClassName: Boolean
  291. def windowExecBufferInMemoryThreshold: Int
  292. def windowExecBufferSpillThreshold: Int
  293. def writeLegacyParquetFormat: Boolean

Inherited from Logging

Inherited from Serializable

Inherited from Serializable

Inherited from AnyRef

Inherited from Any

Ungrouped