Rename everything to glow

.scalafmt.conf

@@ -16,4 +16,4 @@ includeCurlyBraceInSelectChains = false
 includeNoParensInSelectChains = true
 importSelectors = singleLine
 
-rewrite.rules = [PreferCurlyFors, SortImports]
+rewrite.rules = [PreferCurlyFors, SortImports]

build.sbt

@@ -1,6 +1,7 @@
-import Tests._
 import scala.sys.process._
 
+import sbt.Tests._
+
 val sparkVersion = "2.4.3"
 val scalaMajorMinor = "2.11"
 

core/src/main/java/com/databricks/hls/sql/HLSFunctions.java → core/src/main/java/org/projectglow/core/sql/HLSFunctions.java

@@ -1,4 +1,4 @@
-package com.databricks.hls.sql;
+package org.projectglow.core.sql;
 
 import org.apache.spark.sql.catalyst.util.GenericArrayData;
 import org.apache.spark.unsafe.types.UTF8String;

core/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister

@@ -1,11 +1,11 @@
 # Standard file formats
-com.databricks.bgen.BgenFileFormat
-com.databricks.bgen.BigBgenDatasource
-com.databricks.vcf.BigVCFDatasource
-com.databricks.vcf.VCFFileFormat
+org.projectglow.core.bgen.BgenFileFormat
+org.projectglow.core.bgen.BigBgenDatasource
+org.projectglow.core.vcf.BigVCFDatasource
+org.projectglow.core.vcf.VCFFileFormat
 
 # Legacy file formats
-com.databricks.bgen.ComDatabricksBgenFileFormat
-com.databricks.bgen.ComDatabricksBigBgenDatasource
-com.databricks.vcf.ComDatabricksBigVCFDatasource
-com.databricks.vcf.ComDatabricksVCFFileFormat
+org.projectglow.core.bgen.ComDatabricksBgenFileFormat
+org.projectglow.core.bgen.ComDatabricksBigBgenDatasource
+org.projectglow.core.vcf.ComDatabricksBigVCFDatasource
+org.projectglow.core.vcf.ComDatabricksVCFFileFormat

core/src/main/resources/META-INF/services/org.projectglow.core.DataFrameTransformer

@@ -0,0 +1,4 @@
+org.projectglow.core.transformers.LiftOverVariantsTransformer
+org.projectglow.core.transformers.normalizevariants.NormalizeVariantsTransformer
+org.projectglow.core.transformers.pipe.PipeTransformer
+org.projectglow.core.transformers.pipe.CleanupPipeTransformer

core/src/main/resources/META-INF/services/org.projectglow.core.transformers.pipe.InputFormatterFactory

@@ -0,0 +1,3 @@
+org.projectglow.core.transformers.pipe.CSVInputFormatterFactory
+org.projectglow.core.transformers.pipe.UTF8TextInputFormatterFactory
+org.projectglow.core.vcf.VCFInputFormatterFactory

core/src/main/resources/META-INF/services/org.projectglow.core.transformers.pipe.OutputFormatterFactory

@@ -0,0 +1,3 @@
+org.projectglow.core.transformers.pipe.CSVOutputFormatterFactory
+org.projectglow.core.transformers.pipe.UTF8TextOutputFormatterFactory
+org.projectglow.core.vcf.VCFOutputFormatterFactory

core/src/main/scala/org/apache/spark/sql/SQLUtils.scala

@@ -1,12 +1,9 @@
 package org.apache.spark.sql
 
 import org.apache.spark.TaskContext
-import org.apache.spark.ml.linalg.{VectorUDT, Vectors}
+import org.apache.spark.ml.linalg.{MatrixUDT, VectorUDT}
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode}
-import org.apache.spark.sql.catalyst.expressions.{Expression, ImplicitCastInputTypes, UnaryExpression}
-import org.apache.spark.sql.catalyst.util.{ArrayData, GenericArrayData}
 import org.apache.spark.sql.types._
 
 object SQLUtils {
@@ -46,76 +43,18 @@ object SQLUtils {
   def setTaskContext(context: TaskContext): Unit = {
     TaskContext.setTaskContext(context)
   }
-}
-
-case class ArrayToSparseVector(child: Expression)
-    extends UnaryExpression
-    with ImplicitCastInputTypes {
-
-  override def inputTypes: Seq[AbstractDataType] = Seq(ArrayType(DoubleType))
-  override def dataType: DataType = ArrayToSparseVector.vectorType
-  override def nullSafeEval(input: Any): Any = ArrayToSparseVector.fromDoubleArray(input)
 
-  override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    nullSafeCodeGen(ctx, ev, c => {
-      s"""
-         |${ev.value} = org.apache.spark.sql.ArrayToSparseVector.fromDoubleArray($c);
-       """.stripMargin
-    })
+  def newMatrixUDT(): MatrixUDT = {
+    new MatrixUDT()
   }
-}
-
-object ArrayToSparseVector {
-  lazy val vectorType: VectorUDT = new VectorUDT()
 
-  def fromDoubleArray(input: Any): InternalRow = {
-    val vector = Vectors.dense(input.asInstanceOf[ArrayData].toDoubleArray())
-    vectorType.serialize(vector.toSparse)
+  def newVectorUDT(): VectorUDT = {
+    new VectorUDT()
   }
-}
-
-case class ArrayToDenseVector(child: Expression)
-    extends UnaryExpression
-    with ImplicitCastInputTypes {
 
-  override def inputTypes: Seq[AbstractDataType] = Seq(ArrayType(DoubleType))
-  override def dataType: DataType = ArrayToDenseVector.vectorType
-  override def nullSafeEval(input: Any): Any = ArrayToDenseVector.fromDoubleArray(input)
-
-  override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    nullSafeCodeGen(ctx, ev, c => {
-      s"""
-         |${ev.value} = org.apache.spark.sql.ArrayToDenseVector.fromDoubleArray($c);
-       """.stripMargin
-    })
+  def newAnalysisException(msg: String): AnalysisException = {
+    new AnalysisException(msg)
   }
-}
 
-object ArrayToDenseVector {
-  lazy val vectorType: VectorUDT = new VectorUDT()
-
-  def fromDoubleArray(input: Any): InternalRow = {
-    val vector = Vectors.dense(input.asInstanceOf[ArrayData].toDoubleArray())
-    vectorType.serialize(vector)
-  }
-}
-
-case class VectorToArray(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
-  override def inputTypes: Seq[AbstractDataType] = Seq(VectorToArray.vectorType)
-  override def dataType: DataType = ArrayType(DoubleType)
-  override def nullSafeEval(input: Any): Any = VectorToArray.toDoubleArray(input)
-  override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    nullSafeCodeGen(ctx, ev, c => {
-      s"""
-         |${ev.value} = org.apache.spark.sql.VectorToArray.toDoubleArray($c);
-       """.stripMargin
-    })
-  }
-}
-
-object VectorToArray {
-  lazy val vectorType: VectorUDT = new VectorUDT()
-  def toDoubleArray(input: Any): ArrayData = {
-    new GenericArrayData(vectorType.deserialize(input).toArray)
-  }
+  type ADT = AbstractDataType
 }

core/src/main/scala/org/apache/spark/sql/catalyst/expressions/QuaternaryExpression.scala

@@ -1,8 +1,8 @@
 package org.apache.spark.sql.catalyst.expressions
 
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.codegen._
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
+import org.apache.spark.sql.catalyst.expressions.codegen._
 
 // Inlines QuaternaryExpression from Spark 3.0
 

core/src/main/scala/com/databricks/hls/DBGenomics.scala → core/src/main/scala/org/projectglow/core/Glow.scala

@@ -1,13 +1,13 @@
-package com.databricks.hls
+package org.projectglow.core
 
 import java.util.ServiceLoader
 
 import scala.collection.JavaConverters._
 
 import org.apache.spark.sql.DataFrame
 
-import com.databricks.hls.common.Named
-import com.databricks.hls.transformers.util.{SnakeCaseMap, StringUtils}
+import org.projectglow.core.common.Named
+import org.projectglow.core.transformers.util.{SnakeCaseMap, StringUtils}
 
 /**
  * The entry point for all language specific functionality, meaning methods that cannot be expressed
@@ -16,7 +16,7 @@ import com.databricks.hls.transformers.util.{SnakeCaseMap, StringUtils}
  * We should expose as little functionality as is necessary through this object and should prefer
  * generic methods with stringly-typed arguments to reduce language-specific maintenance burden.
  */
-object DBGenomics {
+object Glow {
 
   /**
    * Apply a named transformation to a DataFrame of genomic data. All parameters apart from the

core/src/main/scala/com/databricks/bgen/BgenConverterUtils.scala → core/src/main/scala/org/projectglow/core/bgen/BgenConverterUtils.scala

@@ -1,11 +1,11 @@
-package com.databricks.bgen
+package org.projectglow.core.bgen
 
 import java.util.{HashMap => JHashMap}
 
 import org.apache.commons.math3.util.CombinatoricsUtils
 
 // Tools for calculating ploidy or number of genotypes for unphased posterior probabilities
-private[databricks] object BgenConverterUtils {
+private[projectglow] object BgenConverterUtils {
   var ploidyMap = new JHashMap[(Int, Int), Int] // (numGenotypes, numAlleles) to ploidy
   var genotypesMap = new JHashMap[(Int, Int), Int] // (ploidy, numAlleles) to numGenotypes
 

core/src/main/scala/com/databricks/bgen/BgenFileFormat.scala → core/src/main/scala/org/projectglow/core/bgen/BgenFileFormat.scala

@@ -1,10 +1,10 @@
-package com.databricks.bgen
-
-import scala.collection.JavaConverters._
+package org.projectglow.core.bgen
 
 import java.io.{BufferedReader, File, InputStreamReader}
 import java.nio.file.Paths
 
+import scala.collection.JavaConverters._
+
 import com.google.common.io.LittleEndianDataInputStream
 import com.google.common.util.concurrent.Striped
 import org.apache.hadoop.conf.Configuration
@@ -19,10 +19,9 @@ import org.apache.spark.sql.types.StructType
 import org.skife.jdbi.v2.DBI
 import org.skife.jdbi.v2.util.LongMapper
 
-import com.databricks.hls.common.logging._
-import com.databricks.hls.common.{HLSLogging, WithUtils}
-import com.databricks.hls.sql.util.SerializableConfiguration
-import com.databricks.sql.ComDatabricksDataSource
+import org.projectglow.core.common.logging.{HlsMetricDefinitions, HlsTagDefinitions, HlsTagValues, HlsUsageLogging}
+import org.projectglow.core.common.{HLSLogging, WithUtils}
+import org.projectglow.core.sql.util.{ComDatabricksDataSource, SerializableConfiguration}
 
 class BgenFileFormat extends FileFormat with DataSourceRegister with Serializable with HLSLogging {
 

core/src/main/scala/com/databricks/bgen/BgenFileIterator.scala → core/src/main/scala/org/projectglow/core/bgen/BgenFileIterator.scala

@@ -1,4 +1,4 @@
-package com.databricks.bgen
+package org.projectglow.core.bgen
 
 import java.io.{ByteArrayInputStream, DataInput, DataInputStream}
 import java.nio.charset.StandardCharsets
@@ -9,8 +9,7 @@ import com.google.common.io.LittleEndianDataInputStream
 import org.apache.commons.math3.util.CombinatoricsUtils
 import org.apache.hadoop.fs.FSDataInputStream
 
-import com.databricks.hls.common.HLSLogging
-import com.databricks.vcf.{BgenGenotype, BgenRow, VCFRow}
+import org.projectglow.core.common.{BgenGenotype, BgenRow, HLSLogging, VCFRow}
 
 /**
  * Parses variant records of a BGEN file into the [[VCFRow]] schema. The iterator assumes that the
@@ -35,7 +34,7 @@ import com.databricks.vcf.{BgenGenotype, BgenRow, VCFRow}
  * @param maxPos The maximum stream position from which variant blocks can be read. `hasNext` will
  *               return `false` once we've reached this position.
  */
-private[databricks] class BgenFileIterator(
+private[projectglow] class BgenFileIterator(
     metadata: BgenMetadata,
     stream: LittleEndianDataInputStream,
     underlyingStream: FSDataInputStream,
@@ -287,7 +286,7 @@ private[databricks] class BgenFileIterator(
   }
 }
 
-private[databricks] object BgenFileIterator {
+private[projectglow] object BgenFileIterator {
 
   /**
    * Utility function to read a UTF8 string from a data stream. Included in the companion object
@@ -309,7 +308,8 @@ private[databricks] object BgenFileIterator {
  * Read a BGEN header from a data stream. Performs basic validation on the header parameters
  * according to what the reader currently supports.
  */
-private[databricks] class BgenHeaderReader(stream: LittleEndianDataInputStream) extends HLSLogging {
+private[projectglow] class BgenHeaderReader(stream: LittleEndianDataInputStream)
+    extends HLSLogging {
 
   def readHeader(sampleIdsOpt: Option[Seq[String]] = None): BgenMetadata = {
     val variantOffset = Integer.toUnsignedLong(stream.readInt()) + 4
@@ -381,7 +381,7 @@ private[databricks] class BgenHeaderReader(stream: LittleEndianDataInputStream)
   }
 }
 
-private[databricks] case class BgenMetadata(
+private[projectglow] case class BgenMetadata(
     firstVariantOffset: Long,
     nSamples: Long,
     nVariantBlocks: Long,