@@ -49,13 +49,14 @@ trait CompressionCodec {
4949
5050private [spark] object CompressionCodec extends Logging {
5151
52+ private val configKey = " spark.io.compression.codec"
5253 private val shortCompressionCodecNames = Map (
5354 " lz4" -> classOf [LZ4CompressionCodec ].getName,
5455 " lzf" -> classOf [LZFCompressionCodec ].getName,
5556 " snappy" -> classOf [SnappyCompressionCodec ].getName)
5657
5758 def createCodec (conf : SparkConf ): CompressionCodec = {
58- createCodec(conf, conf.get(" spark.io.compression.codec " , DEFAULT_COMPRESSION_CODEC ))
59+ createCodec(conf, conf.get(configKey , DEFAULT_COMPRESSION_CODEC ))
5960 }
6061
6162 def createCodec (conf : SparkConf , codecName : String ): CompressionCodec = {
@@ -68,9 +69,11 @@ private[spark] object CompressionCodec extends Logging {
6869 case e : ClassNotFoundException => None
6970 }
7071 codec.filter(_.isAvailable())
71- .getOrElse(throw new IllegalArgumentException (s " Codec [ $codecName] is not available. " ))
72+ .getOrElse(throw new IllegalArgumentException (s " Codec [ $codecName] is not available. " +
73+ s " Consider setting $configKey= $FALLBACK_COMPRESSION_CODEC" ))
7274 }
7375
76+ val FALLBACK_COMPRESSION_CODEC = " lzf"
7477 val DEFAULT_COMPRESSION_CODEC = " snappy"
7578 val ALL_COMPRESSION_CODECS = shortCompressionCodecNames.values.toSeq
7679}
0 commit comments