From 520608e6af387c30505808701d3788d452071c41 Mon Sep 17 00:00:00 2001 From: Patrick Schultz Date: Fri, 3 Jan 2025 16:33:26 -0500 Subject: [PATCH] move ir node defs into subpackage of ir --- .../is/hail/annotations/BroadcastValue.scala | 2 +- .../is/hail/backend/local/LocalBackend.scala | 3 +- .../hail/backend/service/ServiceBackend.scala | 5 +- .../is/hail/backend/spark/SparkBackend.scala | 1 + hail/hail/src/is/hail/expr/Validate.scala | 5 +- hail/hail/src/is/hail/expr/ir/Binds.scala | 1 + .../src/is/hail/expr/ir/BlockMatrixIR.scala | 1 + .../is/hail/expr/ir/BlockMatrixWriter.scala | 1 + hail/hail/src/is/hail/expr/ir/Children.scala | 1 + .../hail/src/is/hail/expr/ir/Compilable.scala | 2 + hail/hail/src/is/hail/expr/ir/Compile.scala | 1 + .../is/hail/expr/ir/CompileAndEvaluate.scala | 1 + .../is/hail/expr/ir/ComputeUsesAndDefs.scala | 2 + hail/hail/src/is/hail/expr/ir/Copy.scala | 2 + .../is/hail/expr/ir/DeprecatedIRBuilder.scala | 1 + hail/hail/src/is/hail/expr/ir/Emit.scala | 1 + .../is/hail/expr/ir/EmitClassBuilder.scala | 1 + hail/hail/src/is/hail/expr/ir/Env.scala | 2 + hail/hail/src/is/hail/expr/ir/Exists.scala | 2 + .../hail/expr/ir/ExtractIntervalFilters.scala | 1 + .../src/is/hail/expr/ir/FoldConstants.scala | 1 + .../src/is/hail/expr/ir/ForwardLets.scala | 1 + .../hail/expr/ir/ForwardRelationalLets.scala | 2 + .../src/is/hail/expr/ir/FreeVariables.scala | 1 + .../is/hail/expr/ir/GenericTableValue.scala | 1 + hail/hail/src/is/hail/expr/ir/IR.scala | 2321 +++++++++-------- hail/hail/src/is/hail/expr/ir/IRBuilder.scala | 1 + .../src/is/hail/expr/ir/InTailPosition.scala | 2 + hail/hail/src/is/hail/expr/ir/InferType.scala | 1 + hail/hail/src/is/hail/expr/ir/Interpret.scala | 1 + .../src/is/hail/expr/ir/Interpretable.scala | 1 + .../hail/src/is/hail/expr/ir/IsConstant.scala | 1 + hail/hail/src/is/hail/expr/ir/IsPure.scala | 1 + .../hail/expr/ir/LiftRelationalValues.scala | 4 + .../src/is/hail/expr/ir/LowerMatrixIR.scala | 1 + .../ir/LowerOrInterpretNonCompilable.scala | 1 + hail/hail/src/is/hail/expr/ir/MapIR.scala | 1 + hail/hail/src/is/hail/expr/ir/MatrixIR.scala | 1 + .../src/is/hail/expr/ir/MatrixWriter.scala | 1 + .../src/is/hail/expr/ir/NestingDepth.scala | 2 + .../src/is/hail/expr/ir/NormalizeNames.scala | 1 + hail/hail/src/is/hail/expr/ir/Parser.scala | 1 + hail/hail/src/is/hail/expr/ir/Pretty.scala | 1 + .../src/is/hail/expr/ir/PruneDeadFields.scala | 1 + .../src/is/hail/expr/ir/Requiredness.scala | 1 + hail/hail/src/is/hail/expr/ir/Scope.scala | 5 + hail/hail/src/is/hail/expr/ir/Simplify.scala | 1 + .../is/hail/expr/ir/StringTableReader.scala | 1 + hail/hail/src/is/hail/expr/ir/Subst.scala | 2 + hail/hail/src/is/hail/expr/ir/TableIR.scala | 1 + .../src/is/hail/expr/ir/TableWriter.scala | 1 + hail/hail/src/is/hail/expr/ir/TypeCheck.scala | 1 + .../src/is/hail/expr/ir/agg/Extract.scala | 9 +- .../ir/analyses/ComputeMethodSplits.scala | 3 + .../analyses/ControlFlowPreventsSplit.scala | 3 +- .../hail/expr/ir/analyses/SemanticHash.scala | 1 + .../expr/ir/functions/ArrayFunctions.scala | 1 + .../expr/ir/functions/DictFunctions.scala | 1 + .../is/hail/expr/ir/functions/Functions.scala | 1 + .../expr/ir/functions/MathFunctions.scala | 3 +- .../expr/ir/functions/NDArrayFunctions.scala | 3 +- .../functions/ReferenceGenomeFunctions.scala | 1 + .../hail/expr/ir/functions/SetFunctions.scala | 1 + .../expr/ir/functions/StringFunctions.scala | 1 + .../expr/ir/functions/UtilFunctions.scala | 1 + .../ir/lowering/CanLowerEfficiently.scala | 1 + .../expr/ir/lowering/EvalRelationalLets.scala | 4 +- .../is/hail/expr/ir/lowering/IRState.scala | 5 +- .../ir/lowering/LowerAndExecuteShuffles.scala | 3 +- .../expr/ir/lowering/LowerBlockMatrixIR.scala | 1 + .../ir/lowering/LowerDistributedSort.scala | 1 + .../hail/expr/ir/lowering/LowerTableIR.scala | 3 +- .../ir/lowering/LowerTableIRHelpers.scala | 1 + .../hail/expr/ir/lowering/LoweringPass.scala | 3 + .../expr/ir/lowering/RVDToTableStage.scala | 3 + .../src/is/hail/expr/ir/lowering/Rule.scala | 6 +- .../hail/expr/ir/ndarrays/EmitNDArray.scala | 4 + hail/hail/src/is/hail/expr/ir/package.scala | 7 +- .../is/hail/expr/ir/streams/EmitStream.scala | 1 + .../is/hail/expr/ir/streams/StreamUtils.scala | 8 +- .../is/hail/io/avro/AvroPartitionReader.scala | 5 +- .../src/is/hail/io/avro/AvroTableReader.scala | 4 + hail/hail/src/is/hail/io/bgen/LoadBgen.scala | 6 +- .../hail/src/is/hail/io/plink/LoadPlink.scala | 1 + hail/hail/src/is/hail/io/vcf/LoadVCF.scala | 4 +- .../src/is/hail/methods/LocalLDPrune.scala | 1 + .../src/is/hail/rvd/AbstractRVDSpec.scala | 17 +- .../hail/src/is/hail/rvd/RVDPartitioner.scala | 2 +- .../hail/types/virtual/BlockMatrixType.scala | 1 + hail/hail/test/src/is/hail/HailSuite.scala | 8 +- hail/hail/test/src/is/hail/TestUtils.scala | 6 +- .../is/hail/expr/ir/Aggregators2Suite.scala | 1 + .../is/hail/expr/ir/AggregatorsSuite.scala | 4 + .../expr/ir/ArrayDeforestationSuite.scala | 3 + .../is/hail/expr/ir/ArrayFunctionsSuite.scala | 1 + .../is/hail/expr/ir/BlockMatrixIRSuite.scala | 4 + .../is/hail/expr/ir/CallFunctionsSuite.scala | 1 + .../is/hail/expr/ir/DictFunctionsSuite.scala | 1 + .../hail/expr/ir/DistinctlyKeyedSuite.scala | 4 + .../src/is/hail/expr/ir/EmitStreamSuite.scala | 1 + .../is/hail/expr/ir/EncodedLiteralSuite.scala | 1 + .../expr/ir/ExtractIntervalFiltersSuite.scala | 1 + .../is/hail/expr/ir/FoldConstantsSuite.scala | 3 + .../is/hail/expr/ir/ForwardLetsSuite.scala | 1 + .../src/is/hail/expr/ir/FunctionSuite.scala | 1 + .../test/src/is/hail/expr/ir/IRSuite.scala | 3 +- .../src/is/hail/expr/ir/IntervalSuite.scala | 3 + .../is/hail/expr/ir/LiftLiteralsSuite.scala | 1 + .../is/hail/expr/ir/LocusFunctionsSuite.scala | 1 + .../is/hail/expr/ir/MathFunctionsSuite.scala | 1 + .../src/is/hail/expr/ir/MatrixIRSuite.scala | 4 + .../src/is/hail/expr/ir/MemoryLeakSuite.scala | 4 +- .../src/is/hail/expr/ir/OrderingSuite.scala | 4 + .../test/src/is/hail/expr/ir/PruneSuite.scala | 1 + .../is/hail/expr/ir/RequirednessSuite.scala | 1 + .../is/hail/expr/ir/SetFunctionsSuite.scala | 1 + .../src/is/hail/expr/ir/SimplifySuite.scala | 1 + .../is/hail/expr/ir/StagedBTreeSuite.scala | 2 +- .../hail/expr/ir/StringFunctionsSuite.scala | 1 + .../is/hail/expr/ir/StringLengthSuite.scala | 1 + .../is/hail/expr/ir/StringSliceSuite.scala | 1 + .../src/is/hail/expr/ir/TableIRSuite.scala | 1 + .../test/src/is/hail/expr/ir/TestUtils.scala | 4 + .../src/is/hail/expr/ir/TrapNodeSuite.scala | 1 + .../is/hail/expr/ir/UtilFunctionsSuite.scala | 1 + .../expr/ir/analyses/SemanticHashSuite.scala | 1 + .../lowering/LowerDistributedSortSuite.scala | 6 +- .../is/hail/expr/ir/table/TableGenSuite.scala | 4 + .../test/src/is/hail/io/AvroReaderSuite.scala | 2 +- .../src/is/hail/linalg/BlockMatrixSuite.scala | 3 +- .../src/is/hail/utils/RowIntervalSuite.scala | 2 +- 131 files changed, 1435 insertions(+), 1181 deletions(-) diff --git a/hail/hail/src/is/hail/annotations/BroadcastValue.scala b/hail/hail/src/is/hail/annotations/BroadcastValue.scala index ab6b7127d13..85ec13871db 100644 --- a/hail/hail/src/is/hail/annotations/BroadcastValue.scala +++ b/hail/hail/src/is/hail/annotations/BroadcastValue.scala @@ -2,7 +2,7 @@ package is.hail.annotations import is.hail.asm4s.HailClassLoader import is.hail.backend.{BroadcastValue, ExecuteContext} -import is.hail.expr.ir.EncodedLiteral +import is.hail.expr.ir.defs.EncodedLiteral import is.hail.io.{BufferSpec, Decoder, TypedCodecSpec} import is.hail.types.physical.{PArray, PStruct, PType} import is.hail.types.virtual.{TBaseStruct, TStruct} diff --git a/hail/hail/src/is/hail/backend/local/LocalBackend.scala b/hail/hail/src/is/hail/backend/local/LocalBackend.scala index 2b43df11419..7069fe8db25 100644 --- a/hail/hail/src/is/hail/backend/local/LocalBackend.scala +++ b/hail/hail/src/is/hail/backend/local/LocalBackend.scala @@ -5,8 +5,9 @@ import is.hail.annotations.Region import is.hail.asm4s._ import is.hail.backend._ import is.hail.expr.Validate -import is.hail.expr.ir.{IRParser, _} +import is.hail.expr.ir._ import is.hail.expr.ir.analyses.SemanticHash +import is.hail.expr.ir.defs.{EncodedLiteral, GetFieldByIdx, MakeTuple} import is.hail.expr.ir.lowering._ import is.hail.io.fs._ import is.hail.linalg.BlockMatrix diff --git a/hail/hail/src/is/hail/backend/service/ServiceBackend.scala b/hail/hail/src/is/hail/backend/service/ServiceBackend.scala index 4908f2429ae..203344416e8 100644 --- a/hail/hail/src/is/hail/backend/service/ServiceBackend.scala +++ b/hail/hail/src/is/hail/backend/service/ServiceBackend.scala @@ -6,10 +6,11 @@ import is.hail.asm4s._ import is.hail.backend._ import is.hail.expr.Validate import is.hail.expr.ir.{ - Compile, IR, IRParser, IRParserEnvironment, IRSize, LoweringAnalyses, MakeTuple, SortField, - TableIR, TableReader, TypeCheck, + Compile, IR, IRParser, IRParserEnvironment, IRSize, LoweringAnalyses, SortField, TableIR, + TableReader, TypeCheck, } import is.hail.expr.ir.analyses.SemanticHash +import is.hail.expr.ir.defs.MakeTuple import is.hail.expr.ir.functions.IRFunctionRegistry import is.hail.expr.ir.lowering._ import is.hail.io.fs._ diff --git a/hail/hail/src/is/hail/backend/spark/SparkBackend.scala b/hail/hail/src/is/hail/backend/spark/SparkBackend.scala index 2b4720cd10e..adcdb7cec62 100644 --- a/hail/hail/src/is/hail/backend/spark/SparkBackend.scala +++ b/hail/hail/src/is/hail/backend/spark/SparkBackend.scala @@ -8,6 +8,7 @@ import is.hail.expr.{JSONAnnotationImpex, SparkAnnotationImpex, Validate} import is.hail.expr.ir.{IRParser, _} import is.hail.expr.ir.IRParser.parseType import is.hail.expr.ir.analyses.SemanticHash +import is.hail.expr.ir.defs.{EncodedLiteral, GetFieldByIdx, MakeTuple} import is.hail.expr.ir.lowering._ import is.hail.io.{BufferSpec, TypedCodecSpec} import is.hail.io.fs._ diff --git a/hail/hail/src/is/hail/expr/Validate.scala b/hail/hail/src/is/hail/expr/Validate.scala index 82df3ebbdd3..5512ff1618b 100644 --- a/hail/hail/src/is/hail/expr/Validate.scala +++ b/hail/hail/src/is/hail/expr/Validate.scala @@ -1,8 +1,7 @@ package is.hail.expr -import is.hail.expr.ir.{ - BaseIR, BlockMatrixRead, BlockMatrixWrite, MatrixRead, MatrixWrite, TableRead, TableWrite, -} +import is.hail.expr.ir.{BaseIR, BlockMatrixRead, MatrixRead, TableRead} +import is.hail.expr.ir.defs.{BlockMatrixWrite, MatrixWrite, TableWrite} import is.hail.utils._ case class ValidateState(writeFilePaths: Set[String]) diff --git a/hail/hail/src/is/hail/expr/ir/Binds.scala b/hail/hail/src/is/hail/expr/ir/Binds.scala index d8b9c4c3bbc..8e89d911bc2 100644 --- a/hail/hail/src/is/hail/expr/ir/Binds.scala +++ b/hail/hail/src/is/hail/expr/ir/Binds.scala @@ -1,5 +1,6 @@ package is.hail.expr.ir +import is.hail.expr.ir.defs._ import is.hail.types.tcoerce import is.hail.types.virtual._ import is.hail.types.virtual.TIterable.elementType diff --git a/hail/hail/src/is/hail/expr/ir/BlockMatrixIR.scala b/hail/hail/src/is/hail/expr/ir/BlockMatrixIR.scala index b7cbe20dcbb..072c5bce39d 100644 --- a/hail/hail/src/is/hail/expr/ir/BlockMatrixIR.scala +++ b/hail/hail/src/is/hail/expr/ir/BlockMatrixIR.scala @@ -4,6 +4,7 @@ import is.hail.HailContext import is.hail.annotations.NDArray import is.hail.backend.{BackendContext, ExecuteContext} import is.hail.expr.Nat +import is.hail.expr.ir.defs._ import is.hail.expr.ir.lowering.{BMSContexts, BlockMatrixStage2, LowererUnsupportedOperation} import is.hail.io.{StreamBufferSpec, TypedCodecSpec} import is.hail.io.fs.FS diff --git a/hail/hail/src/is/hail/expr/ir/BlockMatrixWriter.scala b/hail/hail/src/is/hail/expr/ir/BlockMatrixWriter.scala index b6d36fcee65..53dfecb893a 100644 --- a/hail/hail/src/is/hail/expr/ir/BlockMatrixWriter.scala +++ b/hail/hail/src/is/hail/expr/ir/BlockMatrixWriter.scala @@ -5,6 +5,7 @@ import is.hail.annotations.Region import is.hail.asm4s._ import is.hail.backend.ExecuteContext import is.hail.expr.Nat +import is.hail.expr.ir.defs.{MetadataWriter, Str, UUID4, WriteMetadata, WriteValue} import is.hail.expr.ir.lowering.{BlockMatrixStage2, LowererUnsupportedOperation} import is.hail.io.{StreamBufferSpec, TypedCodecSpec} import is.hail.io.fs.FS diff --git a/hail/hail/src/is/hail/expr/ir/Children.scala b/hail/hail/src/is/hail/expr/ir/Children.scala index e69e46d46b1..9eb780815d4 100644 --- a/hail/hail/src/is/hail/expr/ir/Children.scala +++ b/hail/hail/src/is/hail/expr/ir/Children.scala @@ -1,5 +1,6 @@ package is.hail.expr.ir +import is.hail.expr.ir.defs._ import is.hail.utils._ object Children { diff --git a/hail/hail/src/is/hail/expr/ir/Compilable.scala b/hail/hail/src/is/hail/expr/ir/Compilable.scala index 457525c01de..45581de17f3 100644 --- a/hail/hail/src/is/hail/expr/ir/Compilable.scala +++ b/hail/hail/src/is/hail/expr/ir/Compilable.scala @@ -1,5 +1,7 @@ package is.hail.expr.ir +import is.hail.expr.ir.defs._ + object InterpretableButNotCompilable { def apply(x: IR): Boolean = x match { case _: LiftMeOut => true diff --git a/hail/hail/src/is/hail/expr/ir/Compile.scala b/hail/hail/src/is/hail/expr/ir/Compile.scala index d6cff956a38..2dfe1c185fe 100644 --- a/hail/hail/src/is/hail/expr/ir/Compile.scala +++ b/hail/hail/src/is/hail/expr/ir/Compile.scala @@ -4,6 +4,7 @@ import is.hail.annotations._ import is.hail.asm4s._ import is.hail.backend.{ExecuteContext, HailTaskContext} import is.hail.expr.ir.agg.AggStateSig +import is.hail.expr.ir.defs.In import is.hail.expr.ir.lowering.LoweringPipeline import is.hail.expr.ir.streams.EmitStream import is.hail.io.fs.FS diff --git a/hail/hail/src/is/hail/expr/ir/CompileAndEvaluate.scala b/hail/hail/src/is/hail/expr/ir/CompileAndEvaluate.scala index 51257f1b1a9..b099347e827 100644 --- a/hail/hail/src/is/hail/expr/ir/CompileAndEvaluate.scala +++ b/hail/hail/src/is/hail/expr/ir/CompileAndEvaluate.scala @@ -3,6 +3,7 @@ package is.hail.expr.ir import is.hail.annotations.{Region, SafeRow} import is.hail.asm4s._ import is.hail.backend.ExecuteContext +import is.hail.expr.ir.defs.{Begin, EncodedLiteral, Literal, MakeTuple, NA} import is.hail.expr.ir.lowering.LoweringPipeline import is.hail.types.physical.PTuple import is.hail.types.physical.stypes.PTypeReferenceSingleCodeType diff --git a/hail/hail/src/is/hail/expr/ir/ComputeUsesAndDefs.scala b/hail/hail/src/is/hail/expr/ir/ComputeUsesAndDefs.scala index 3980af5a5ad..afa146aa1a0 100644 --- a/hail/hail/src/is/hail/expr/ir/ComputeUsesAndDefs.scala +++ b/hail/hail/src/is/hail/expr/ir/ComputeUsesAndDefs.scala @@ -1,5 +1,7 @@ package is.hail.expr.ir +import is.hail.expr.ir.defs.{BaseRef, Recur, Ref, RelationalRef} + import scala.collection.mutable case class UsesAndDefs( diff --git a/hail/hail/src/is/hail/expr/ir/Copy.scala b/hail/hail/src/is/hail/expr/ir/Copy.scala index 79522e2d94a..f1529baf809 100644 --- a/hail/hail/src/is/hail/expr/ir/Copy.scala +++ b/hail/hail/src/is/hail/expr/ir/Copy.scala @@ -1,5 +1,7 @@ package is.hail.expr.ir +import is.hail.expr.ir.defs._ + object Copy { def apply(x: IR, newChildren: IndexedSeq[BaseIR]): IR = { x match { diff --git a/hail/hail/src/is/hail/expr/ir/DeprecatedIRBuilder.scala b/hail/hail/src/is/hail/expr/ir/DeprecatedIRBuilder.scala index 1ae819ed9e6..38020f76d92 100644 --- a/hail/hail/src/is/hail/expr/ir/DeprecatedIRBuilder.scala +++ b/hail/hail/src/is/hail/expr/ir/DeprecatedIRBuilder.scala @@ -1,5 +1,6 @@ package is.hail.expr.ir +import is.hail.expr.ir.defs._ import is.hail.types.virtual._ import is.hail.utils.{toRichIterable, FastSeq} diff --git a/hail/hail/src/is/hail/expr/ir/Emit.scala b/hail/hail/src/is/hail/expr/ir/Emit.scala index f52325c6f23..be2e79a091b 100644 --- a/hail/hail/src/is/hail/expr/ir/Emit.scala +++ b/hail/hail/src/is/hail/expr/ir/Emit.scala @@ -7,6 +7,7 @@ import is.hail.expr.ir.agg.{AggStateSig, ArrayAggStateSig, GroupedStateSig} import is.hail.expr.ir.analyses.{ ComputeMethodSplits, ControlFlowPreventsSplit, ParentPointers, SemanticHash, } +import is.hail.expr.ir.defs._ import is.hail.expr.ir.lowering.TableStageDependency import is.hail.expr.ir.ndarrays.EmitNDArray import is.hail.expr.ir.streams.{EmitStream, StreamProducer, StreamUtils} diff --git a/hail/hail/src/is/hail/expr/ir/EmitClassBuilder.scala b/hail/hail/src/is/hail/expr/ir/EmitClassBuilder.scala index 3163cdc8342..f568208ed33 100644 --- a/hail/hail/src/is/hail/expr/ir/EmitClassBuilder.scala +++ b/hail/hail/src/is/hail/expr/ir/EmitClassBuilder.scala @@ -3,6 +3,7 @@ package is.hail.expr.ir import is.hail.annotations.{Region, RegionPool, RegionValueBuilder} import is.hail.asm4s._ import is.hail.backend.{BackendUtils, ExecuteContext, HailTaskContext} +import is.hail.expr.ir.defs.EncodedLiteral import is.hail.expr.ir.functions.IRRandomness import is.hail.expr.ir.orderings.{CodeOrdering, StructOrdering} import is.hail.io.{BufferSpec, InputBuffer, TypedCodecSpec} diff --git a/hail/hail/src/is/hail/expr/ir/Env.scala b/hail/hail/src/is/hail/expr/ir/Env.scala index adf9845376c..e4c6297b795 100644 --- a/hail/hail/src/is/hail/expr/ir/Env.scala +++ b/hail/hail/src/is/hail/expr/ir/Env.scala @@ -1,5 +1,7 @@ package is.hail.expr.ir +import is.hail.expr.ir.defs.Ref + object Env { type K = Name diff --git a/hail/hail/src/is/hail/expr/ir/Exists.scala b/hail/hail/src/is/hail/expr/ir/Exists.scala index 1e4d7fb3f79..9f98f03cd1f 100644 --- a/hail/hail/src/is/hail/expr/ir/Exists.scala +++ b/hail/hail/src/is/hail/expr/ir/Exists.scala @@ -1,5 +1,7 @@ package is.hail.expr.ir +import is.hail.expr.ir.defs._ + // // Search an IR tree for the first node satisfying some condition // diff --git a/hail/hail/src/is/hail/expr/ir/ExtractIntervalFilters.scala b/hail/hail/src/is/hail/expr/ir/ExtractIntervalFilters.scala index 446c11868d8..3a46852a293 100644 --- a/hail/hail/src/is/hail/expr/ir/ExtractIntervalFilters.scala +++ b/hail/hail/src/is/hail/expr/ir/ExtractIntervalFilters.scala @@ -2,6 +2,7 @@ package is.hail.expr.ir import is.hail.annotations.{ExtendedOrdering, IntervalEndpointOrdering, SafeRow} import is.hail.backend.ExecuteContext +import is.hail.expr.ir.defs._ import is.hail.rvd.PartitionBoundOrdering import is.hail.types.virtual._ import is.hail.utils.{Interval, IntervalEndpoint, _} diff --git a/hail/hail/src/is/hail/expr/ir/FoldConstants.scala b/hail/hail/src/is/hail/expr/ir/FoldConstants.scala index 2c2cedd215e..c9de5a3141d 100644 --- a/hail/hail/src/is/hail/expr/ir/FoldConstants.scala +++ b/hail/hail/src/is/hail/expr/ir/FoldConstants.scala @@ -1,6 +1,7 @@ package is.hail.expr.ir import is.hail.backend.ExecuteContext +import is.hail.expr.ir.defs._ import is.hail.types.virtual.{TStream, TVoid} import is.hail.utils.HailException diff --git a/hail/hail/src/is/hail/expr/ir/ForwardLets.scala b/hail/hail/src/is/hail/expr/ir/ForwardLets.scala index e9f137c80c7..efb0863c2df 100644 --- a/hail/hail/src/is/hail/expr/ir/ForwardLets.scala +++ b/hail/hail/src/is/hail/expr/ir/ForwardLets.scala @@ -1,6 +1,7 @@ package is.hail.expr.ir import is.hail.backend.ExecuteContext +import is.hail.expr.ir.defs.{BaseRef, Binding, Block, In, Ref, Str} import is.hail.types.virtual.TVoid import is.hail.utils.BoxedArrayBuilder diff --git a/hail/hail/src/is/hail/expr/ir/ForwardRelationalLets.scala b/hail/hail/src/is/hail/expr/ir/ForwardRelationalLets.scala index ae600985d8a..b2858fe75b2 100644 --- a/hail/hail/src/is/hail/expr/ir/ForwardRelationalLets.scala +++ b/hail/hail/src/is/hail/expr/ir/ForwardRelationalLets.scala @@ -1,5 +1,7 @@ package is.hail.expr.ir +import is.hail.expr.ir.defs.{RelationalLet, RelationalRef} + import scala.collection.mutable object ForwardRelationalLets { diff --git a/hail/hail/src/is/hail/expr/ir/FreeVariables.scala b/hail/hail/src/is/hail/expr/ir/FreeVariables.scala index 6bcdcf8a3c6..cf3b01e0f9d 100644 --- a/hail/hail/src/is/hail/expr/ir/FreeVariables.scala +++ b/hail/hail/src/is/hail/expr/ir/FreeVariables.scala @@ -1,5 +1,6 @@ package is.hail.expr.ir +import is.hail.expr.ir.defs.Ref import is.hail.types.virtual.Type import scala.collection.mutable diff --git a/hail/hail/src/is/hail/expr/ir/GenericTableValue.scala b/hail/hail/src/is/hail/expr/ir/GenericTableValue.scala index 4758ac92cc0..325b7d8deae 100644 --- a/hail/hail/src/is/hail/expr/ir/GenericTableValue.scala +++ b/hail/hail/src/is/hail/expr/ir/GenericTableValue.scala @@ -3,6 +3,7 @@ package is.hail.expr.ir import is.hail.annotations.Region import is.hail.asm4s._ import is.hail.backend.ExecuteContext +import is.hail.expr.ir.defs.{Literal, PartitionReader, ReadPartition, ToStream} import is.hail.expr.ir.functions.UtilFunctions import is.hail.expr.ir.lowering.{TableStage, TableStageDependency} import is.hail.expr.ir.streams.StreamProducer diff --git a/hail/hail/src/is/hail/expr/ir/IR.scala b/hail/hail/src/is/hail/expr/ir/IR.scala index 5f432ab2dd5..32dcf26f3f6 100644 --- a/hail/hail/src/is/hail/expr/ir/IR.scala +++ b/hail/hail/src/is/hail/expr/ir/IR.scala @@ -3,8 +3,8 @@ package is.hail.expr.ir import is.hail.annotations.{Annotation, Region} import is.hail.asm4s.Value import is.hail.backend.ExecuteContext -import is.hail.expr.ir.ArrayZipBehavior.ArrayZipBehavior import is.hail.expr.ir.agg.{AggStateSig, PhysicalAggSig} +import is.hail.expr.ir.defs.ApplyIR import is.hail.expr.ir.functions._ import is.hail.expr.ir.lowering.TableStageDependency import is.hail.expr.ir.streams.StreamProducer @@ -72,487 +72,600 @@ sealed trait IR extends BaseIR { def unwrap: IR = _unwrap(this) } -sealed trait TypedIR[T <: Type] extends IR { - override def typ: T = tcoerce[T](super.typ) -} +package defs { + + import is.hail.expr.ir.defs.ArrayZipBehavior.ArrayZipBehavior -// Mark Refs and constants as IRs that are safe to duplicate -sealed trait TrivialIR extends IR - -object Literal { - def coerce(t: Type, x: Any): IR = { - if (x == null) - return NA(t) - t match { - case TInt32 => I32(x.asInstanceOf[Number].intValue()) - case TInt64 => I64(x.asInstanceOf[Number].longValue()) - case TFloat32 => F32(x.asInstanceOf[Number].floatValue()) - case TFloat64 => F64(x.asInstanceOf[Number].doubleValue()) - case TBoolean => if (x.asInstanceOf[Boolean]) True() else False() - case TString => Str(x.asInstanceOf[String]) - case _ => Literal(t, x) + sealed trait TypedIR[T <: Type] extends IR { + override def typ: T = tcoerce[T](super.typ) + } + + // Mark Refs and constants as IRs that are safe to duplicate + sealed trait TrivialIR extends IR + + object Literal { + def coerce(t: Type, x: Any): IR = { + if (x == null) + return NA(t) + t match { + case TInt32 => I32(x.asInstanceOf[Number].intValue()) + case TInt64 => I64(x.asInstanceOf[Number].longValue()) + case TFloat32 => F32(x.asInstanceOf[Number].floatValue()) + case TFloat64 => F64(x.asInstanceOf[Number].doubleValue()) + case TBoolean => if (x.asInstanceOf[Boolean]) True() else False() + case TString => Str(x.asInstanceOf[String]) + case _ => Literal(t, x) + } } } -} -final case class Literal(_typ: Type, value: Annotation) extends IR { - require(!CanEmit(_typ)) - require(value != null) - // expensive, for debugging -// require(SafeRow.isSafe(value)) -// assert(_typ.typeCheck(value), s"literal invalid:\n ${_typ}\n $value") -} + final case class Literal(_typ: Type, value: Annotation) extends IR { + require(!CanEmit(_typ)) + require(value != null) + // expensive, for debugging + // require(SafeRow.isSafe(value)) + // assert(_typ.typeCheck(value), s"literal invalid:\n ${_typ}\n $value") + } -object EncodedLiteral { - def apply(codec: AbstractTypedCodecSpec, value: Array[Array[Byte]]): EncodedLiteral = - EncodedLiteral(codec, new WrappedByteArrays(value)) - - def fromPTypeAndAddress(pt: PType, addr: Long, ctx: ExecuteContext): IR = { - pt match { - case _: PInt32 => I32(Region.loadInt(addr)) - case _: PInt64 => I64(Region.loadLong(addr)) - case _: PFloat32 => F32(Region.loadFloat(addr)) - case _: PFloat64 => F64(Region.loadDouble(addr)) - case _: PBoolean => if (Region.loadBoolean(addr)) True() else False() - case ts: PString => Str(ts.loadString(addr)) - case _ => - val etype = EType.defaultFromPType(ctx, pt) - val codec = TypedCodecSpec(etype, pt.virtualType, BufferSpec.wireSpec) - val bytes = codec.encodeArrays(ctx, pt, addr) - EncodedLiteral(codec, bytes) + object EncodedLiteral { + def apply(codec: AbstractTypedCodecSpec, value: Array[Array[Byte]]): EncodedLiteral = + EncodedLiteral(codec, new WrappedByteArrays(value)) + + def fromPTypeAndAddress(pt: PType, addr: Long, ctx: ExecuteContext): IR = { + pt match { + case _: PInt32 => I32(Region.loadInt(addr)) + case _: PInt64 => I64(Region.loadLong(addr)) + case _: PFloat32 => F32(Region.loadFloat(addr)) + case _: PFloat64 => F64(Region.loadDouble(addr)) + case _: PBoolean => if (Region.loadBoolean(addr)) True() else False() + case ts: PString => Str(ts.loadString(addr)) + case _ => + val etype = EType.defaultFromPType(ctx, pt) + val codec = TypedCodecSpec(etype, pt.virtualType, BufferSpec.wireSpec) + val bytes = codec.encodeArrays(ctx, pt, addr) + EncodedLiteral(codec, bytes) + } } } -} - -final case class EncodedLiteral(codec: AbstractTypedCodecSpec, value: WrappedByteArrays) - extends IR { - require(!CanEmit(codec.encodedVirtualType)) - require(value != null) -} -class WrappedByteArrays(val ba: Array[Array[Byte]]) { - override def hashCode(): Int = - ba.foldLeft(31)((h, b) => 37 * h + java.util.Arrays.hashCode(b)) + final case class EncodedLiteral(codec: AbstractTypedCodecSpec, value: WrappedByteArrays) + extends IR { + require(!CanEmit(codec.encodedVirtualType)) + require(value != null) + } - override def equals(obj: Any): Boolean = { - this.eq(obj.asInstanceOf[AnyRef]) || { - if (!obj.isInstanceOf[WrappedByteArrays]) { - false - } else { - val other = obj.asInstanceOf[WrappedByteArrays] - ba.length == other.ba.length && (ba, other.ba).zipped.forall(java.util.Arrays.equals) + class WrappedByteArrays(val ba: Array[Array[Byte]]) { + override def hashCode(): Int = + ba.foldLeft(31)((h, b) => 37 * h + java.util.Arrays.hashCode(b)) + + override def equals(obj: Any): Boolean = { + this.eq(obj.asInstanceOf[AnyRef]) || { + if (!obj.isInstanceOf[WrappedByteArrays]) { + false + } else { + val other = obj.asInstanceOf[WrappedByteArrays] + ba.length == other.ba.length && (ba, other.ba).zipped.forall(java.util.Arrays.equals) + } } } } -} -final case class I32(x: Int) extends IR with TrivialIR -final case class I64(x: Long) extends IR with TrivialIR -final case class F32(x: Float) extends IR with TrivialIR -final case class F64(x: Double) extends IR with TrivialIR + final case class I32(x: Int) extends IR with TrivialIR + final case class I64(x: Long) extends IR with TrivialIR + final case class F32(x: Float) extends IR with TrivialIR + final case class F64(x: Double) extends IR with TrivialIR -final case class Str(x: String) extends IR with TrivialIR { - override def toString(): String = s"""Str("${StringEscapeUtils.escapeString(x)}")""" -} + final case class Str(x: String) extends IR with TrivialIR { + override def toString(): String = s"""Str("${StringEscapeUtils.escapeString(x)}")""" + } -final case class True() extends IR with TrivialIR -final case class False() extends IR with TrivialIR -final case class Void() extends IR with TrivialIR + final case class True() extends IR with TrivialIR + final case class False() extends IR with TrivialIR + final case class Void() extends IR with TrivialIR -object UUID4 { - def apply(): UUID4 = UUID4(genUID()) -} + object UUID4 { + def apply(): UUID4 = UUID4(genUID()) + } // WARNING! This node can only be used when trying to append a one-off, // random string that will not be reused elsewhere in the pipeline. // Any other uses will need to write and then read again; this node is // non-deterministic and will not e.g. exhibit the correct semantics when // self-joining on streams. -final case class UUID4(id: String) extends IR + final case class UUID4(id: String) extends IR -final case class Cast(v: IR, _typ: Type) extends IR -final case class CastRename(v: IR, _typ: Type) extends IR + final case class Cast(v: IR, _typ: Type) extends IR + final case class CastRename(v: IR, _typ: Type) extends IR -final case class NA(_typ: Type) extends IR with TrivialIR -final case class IsNA(value: IR) extends IR + final case class NA(_typ: Type) extends IR with TrivialIR + final case class IsNA(value: IR) extends IR -final case class Coalesce(values: Seq[IR]) extends IR { - require(values.nonEmpty) -} + final case class Coalesce(values: Seq[IR]) extends IR { + require(values.nonEmpty) + } -final case class Consume(value: IR) extends IR + final case class Consume(value: IR) extends IR -final case class If(cond: IR, cnsq: IR, altr: IR) extends IR + final case class If(cond: IR, cnsq: IR, altr: IR) extends IR -final case class Switch(x: IR, default: IR, cases: IndexedSeq[IR]) extends IR { - override lazy val size: Int = - 2 + cases.length -} + final case class Switch(x: IR, default: IR, cases: IndexedSeq[IR]) extends IR { + override lazy val size: Int = + 2 + cases.length + } -object AggLet { - def apply(name: Name, value: IR, body: IR, isScan: Boolean): IR = { - val scope = if (isScan) Scope.SCAN else Scope.AGG - Block(FastSeq(Binding(name, value, scope)), body) + object AggLet { + def apply(name: Name, value: IR, body: IR, isScan: Boolean): IR = { + val scope = if (isScan) Scope.SCAN else Scope.AGG + Block(FastSeq(Binding(name, value, scope)), body) + } } -} -object Let { - def apply(bindings: IndexedSeq[(Name, IR)], body: IR): Block = - Block( - bindings.map { case (name, value) => Binding(name, value) }, - body, - ) + object Let { + def apply(bindings: IndexedSeq[(Name, IR)], body: IR): Block = + Block( + bindings.map { case (name, value) => Binding(name, value) }, + body, + ) - def void(bindings: IndexedSeq[(Name, IR)]): IR = { - if (bindings.isEmpty) { - Void() - } else { - assert(bindings.last._2.typ == TVoid) - Let(bindings.init, bindings.last._2) + def void(bindings: IndexedSeq[(Name, IR)]): IR = { + if (bindings.isEmpty) { + Void() + } else { + assert(bindings.last._2.typ == TVoid) + Let(bindings.init, bindings.last._2) + } } - } -} + } -case class Binding(name: Name, value: IR, scope: Int = Scope.EVAL) + case class Binding(name: Name, value: IR, scope: Int = Scope.EVAL) -final case class Block(bindings: IndexedSeq[Binding], body: IR) extends IR { - override lazy val size: Int = - bindings.length + 1 -} + final case class Block(bindings: IndexedSeq[Binding], body: IR) extends IR { + override lazy val size: Int = + bindings.length + 1 + } -object Block { - object Insert { - def unapply(bindings: IndexedSeq[Binding]) - : Option[(IndexedSeq[Binding], Binding, IndexedSeq[Binding])] = { - val idx = bindings.indexWhere(_.value.isInstanceOf[InsertFields]) - if (idx == -1) None else Some((bindings.take(idx), bindings(idx), bindings.drop(idx + 1))) + object Block { + object Insert { + def unapply(bindings: IndexedSeq[Binding]) + : Option[(IndexedSeq[Binding], Binding, IndexedSeq[Binding])] = { + val idx = bindings.indexWhere(_.value.isInstanceOf[InsertFields]) + if (idx == -1) None else Some((bindings.take(idx), bindings(idx), bindings.drop(idx + 1))) + } } - } - object Nested { - def unapply(bindings: IndexedSeq[Binding]): Option[(Int, IndexedSeq[Binding])] = { - val idx = bindings.indexWhere(_.value.isInstanceOf[Block]) - if (idx == -1) None else Some((idx, bindings)) + object Nested { + def unapply(bindings: IndexedSeq[Binding]): Option[(Int, IndexedSeq[Binding])] = { + val idx = bindings.indexWhere(_.value.isInstanceOf[Block]) + if (idx == -1) None else Some((idx, bindings)) + } } } -} -sealed abstract class BaseRef extends IR with TrivialIR { - def name: Name - def _typ: Type -} + sealed abstract class BaseRef extends IR with TrivialIR { + def name: Name + def _typ: Type + } -final case class Ref(name: Name, var _typ: Type) extends BaseRef { - override def typ: Type = { - assert(_typ != null) - _typ + final case class Ref(name: Name, var _typ: Type) extends BaseRef { + override def typ: Type = { + assert(_typ != null) + _typ + } } -} // Recur can't exist outside of loop // Loops can be nested, but we can't call outer loops in terms of inner loops so there can only be one loop "active" in a given context -final case class TailLoop( - name: Name, - params: IndexedSeq[(Name, IR)], - resultType: Type, - body: IR, -) extends IR { - lazy val paramIdx: Map[Name, Int] = params.map(_._1).zipWithIndex.toMap -} - -final case class Recur(name: Name, args: IndexedSeq[IR], var _typ: Type) extends BaseRef - -final case class RelationalLet(name: Name, value: IR, body: IR) extends IR -final case class RelationalRef(name: Name, _typ: Type) extends BaseRef + final case class TailLoop( + name: Name, + params: IndexedSeq[(Name, IR)], + resultType: Type, + body: IR, + ) extends IR { + lazy val paramIdx: Map[Name, Int] = params.map(_._1).zipWithIndex.toMap + } -final case class ApplyBinaryPrimOp(op: BinaryOp, l: IR, r: IR) extends IR -final case class ApplyUnaryPrimOp(op: UnaryOp, x: IR) extends IR -final case class ApplyComparisonOp(var op: ComparisonOp[_], l: IR, r: IR) extends IR + final case class Recur(name: Name, args: IndexedSeq[IR], var _typ: Type) extends BaseRef -object MakeArray { - def apply(args: IR*): MakeArray = { - assert(args.nonEmpty) - MakeArray(args.toArray, TArray(args.head.typ)) - } + final case class RelationalLet(name: Name, value: IR, body: IR) extends IR + final case class RelationalRef(name: Name, _typ: Type) extends BaseRef - def unify(ctx: ExecuteContext, args: IndexedSeq[IR], requestedType: TArray = null): MakeArray = { - assert(requestedType != null || args.nonEmpty) + final case class ApplyBinaryPrimOp(op: BinaryOp, l: IR, r: IR) extends IR + final case class ApplyUnaryPrimOp(op: UnaryOp, x: IR) extends IR + final case class ApplyComparisonOp(var op: ComparisonOp[_], l: IR, r: IR) extends IR - if (args.nonEmpty) - if (args.forall(_.typ == args.head.typ)) - return MakeArray(args, TArray(args.head.typ)) + object MakeArray { + def apply(args: IR*): MakeArray = { + assert(args.nonEmpty) + MakeArray(args.toArray, TArray(args.head.typ)) + } - MakeArray( - args.map { arg => - val upcast = PruneDeadFields.upcast(ctx, arg, requestedType.elementType) - assert(upcast.typ == requestedType.elementType) - upcast - }, - requestedType, - ) + def unify(ctx: ExecuteContext, args: IndexedSeq[IR], requestedType: TArray = null) + : MakeArray = { + assert(requestedType != null || args.nonEmpty) + + if (args.nonEmpty) + if (args.forall(_.typ == args.head.typ)) + return MakeArray(args, TArray(args.head.typ)) + + MakeArray( + args.map { arg => + val upcast = PruneDeadFields.upcast(ctx, arg, requestedType.elementType) + assert(upcast.typ == requestedType.elementType) + upcast + }, + requestedType, + ) + } } -} -final case class MakeArray(args: IndexedSeq[IR], _typ: TArray) extends IR + final case class MakeArray(args: IndexedSeq[IR], _typ: TArray) extends IR + + object MakeStream { + def unify( + ctx: ExecuteContext, + args: IndexedSeq[IR], + requiresMemoryManagementPerElement: Boolean = false, + requestedType: TStream = null, + ): MakeStream = { + assert(requestedType != null || args.nonEmpty) + + if (args.nonEmpty) + if (args.forall(_.typ == args.head.typ)) + return MakeStream(args, TStream(args.head.typ), requiresMemoryManagementPerElement) + + MakeStream( + args.map { arg => + val upcast = PruneDeadFields.upcast(ctx, arg, requestedType.elementType) + assert(upcast.typ == requestedType.elementType) + upcast + }, + requestedType, + requiresMemoryManagementPerElement, + ) + } + } -object MakeStream { - def unify( - ctx: ExecuteContext, + final case class MakeStream( args: IndexedSeq[IR], + _typ: TStream, requiresMemoryManagementPerElement: Boolean = false, - requestedType: TStream = null, - ): MakeStream = { - assert(requestedType != null || args.nonEmpty) - - if (args.nonEmpty) - if (args.forall(_.typ == args.head.typ)) - return MakeStream(args, TStream(args.head.typ), requiresMemoryManagementPerElement) - - MakeStream( - args.map { arg => - val upcast = PruneDeadFields.upcast(ctx, arg, requestedType.elementType) - assert(upcast.typ == requestedType.elementType) - upcast - }, - requestedType, - requiresMemoryManagementPerElement, - ) + ) extends IR + + object ArrayRef { + def apply(a: IR, i: IR): ArrayRef = ArrayRef(a, i, ErrorIDs.NO_ERROR) } -} -final case class MakeStream( - args: IndexedSeq[IR], - _typ: TStream, - requiresMemoryManagementPerElement: Boolean = false, -) extends IR + final case class ArrayRef(a: IR, i: IR, errorID: Int) extends IR -object ArrayRef { - def apply(a: IR, i: IR): ArrayRef = ArrayRef(a, i, ErrorIDs.NO_ERROR) -} + final case class ArraySlice( + a: IR, + start: IR, + stop: Option[IR], + step: IR = I32(1), + errorID: Int = ErrorIDs.NO_ERROR, + ) extends IR -final case class ArrayRef(a: IR, i: IR, errorID: Int) extends IR - -final case class ArraySlice( - a: IR, - start: IR, - stop: Option[IR], - step: IR = I32(1), - errorID: Int = ErrorIDs.NO_ERROR, -) extends IR - -final case class ArrayLen(a: IR) extends IR -final case class ArrayZeros(length: IR) extends IR - -final case class ArrayMaximalIndependentSet(edges: IR, tieBreaker: Option[(Name, Name, IR)]) - extends IR - -/** [[StreamIota]] is an infinite stream producer, whose element is an integer starting at `start`, - * updated by `step` at each iteration. The name comes from APL: - * [[https://stackoverflow.com/questions/9244879/what-does-iota-of-stdiota-stand-for]] - */ -final case class StreamIota( - start: IR, - step: IR, - requiresMemoryManagementPerElement: Boolean = false, -) extends IR - -final case class StreamRange( - start: IR, - stop: IR, - step: IR, - requiresMemoryManagementPerElement: Boolean = false, - errorID: Int = ErrorIDs.NO_ERROR, -) extends IR - -object ArraySort { - def apply(a: IR, ascending: IR = True(), onKey: Boolean = false): ArraySort = { - val l = freshName() - val r = freshName() - val atyp = tcoerce[TStream](a.typ) - val compare = if (onKey) { - val elementType = atyp.elementType.asInstanceOf[TBaseStruct] - elementType match { - case _: TStruct => - val elt = tcoerce[TStruct](atyp.elementType) - ApplyComparisonOp( - Compare(elt.types(0)), - GetField(Ref(l, elt), elt.fieldNames(0)), - GetField(Ref(r, atyp.elementType), elt.fieldNames(0)), - ) - case _: TTuple => - val elt = tcoerce[TTuple](atyp.elementType) - ApplyComparisonOp( - Compare(elt.types(0)), - GetTupleElement(Ref(l, elt), elt.fields(0).index), - GetTupleElement(Ref(r, atyp.elementType), elt.fields(0).index), - ) + final case class ArrayLen(a: IR) extends IR + + final case class ArrayZeros(length: IR) extends IR + + final case class ArrayMaximalIndependentSet(edges: IR, tieBreaker: Option[(Name, Name, IR)]) + extends IR + + /** [[StreamIota]] is an infinite stream producer, whose element is an integer starting at + * `start`, updated by `step` at each iteration. The name comes from APL: + * [[https://stackoverflow.com/questions/9244879/what-does-iota-of-stdiota-stand-for]] + */ + final case class StreamIota( + start: IR, + step: IR, + requiresMemoryManagementPerElement: Boolean = false, + ) extends IR + + final case class StreamRange( + start: IR, + stop: IR, + step: IR, + requiresMemoryManagementPerElement: Boolean = false, + errorID: Int = ErrorIDs.NO_ERROR, + ) extends IR + + object ArraySort { + def apply(a: IR, ascending: IR = True(), onKey: Boolean = false): ArraySort = { + val l = freshName() + val r = freshName() + val atyp = tcoerce[TStream](a.typ) + val compare = if (onKey) { + val elementType = atyp.elementType.asInstanceOf[TBaseStruct] + elementType match { + case _: TStruct => + val elt = tcoerce[TStruct](atyp.elementType) + ApplyComparisonOp( + Compare(elt.types(0)), + GetField(Ref(l, elt), elt.fieldNames(0)), + GetField(Ref(r, atyp.elementType), elt.fieldNames(0)), + ) + case _: TTuple => + val elt = tcoerce[TTuple](atyp.elementType) + ApplyComparisonOp( + Compare(elt.types(0)), + GetTupleElement(Ref(l, elt), elt.fields(0).index), + GetTupleElement(Ref(r, atyp.elementType), elt.fields(0).index), + ) + } + } else { + ApplyComparisonOp( + Compare(atyp.elementType), + Ref(l, atyp.elementType), + Ref(r, atyp.elementType), + ) } - } else { - ApplyComparisonOp( - Compare(atyp.elementType), - Ref(l, atyp.elementType), - Ref(r, atyp.elementType), - ) - } - ArraySort(a, l, r, If(ascending, compare < 0, compare > 0)) + ArraySort(a, l, r, If(ascending, compare < 0, compare > 0)) + } } -} -final case class ArraySort(a: IR, left: Name, right: Name, lessThan: IR) extends IR -final case class ToSet(a: IR) extends IR -final case class ToDict(a: IR) extends IR -final case class ToArray(a: IR) extends IR -final case class CastToArray(a: IR) extends IR -final case class ToStream(a: IR, requiresMemoryManagementPerElement: Boolean = false) extends IR + final case class ArraySort(a: IR, left: Name, right: Name, lessThan: IR) extends IR -final case class StreamBufferedAggregate( - streamChild: IR, - initAggs: IR, - newKey: IR, - seqOps: IR, - name: Name, - aggSignatures: IndexedSeq[PhysicalAggSig], - bufferSize: Int, -) extends IR + final case class ToSet(a: IR) extends IR -final case class LowerBoundOnOrderedCollection(orderedCollection: IR, elem: IR, onKey: Boolean) - extends IR + final case class ToDict(a: IR) extends IR -final case class GroupByKey(collection: IR) extends IR + final case class ToArray(a: IR) extends IR -final case class RNGStateLiteral() extends IR + final case class CastToArray(a: IR) extends IR -final case class RNGSplit(state: IR, dynBitstring: IR) extends IR + final case class ToStream(a: IR, requiresMemoryManagementPerElement: Boolean = false) extends IR -final case class StreamLen(a: IR) extends IR + final case class StreamBufferedAggregate( + streamChild: IR, + initAggs: IR, + newKey: IR, + seqOps: IR, + name: Name, + aggSignatures: IndexedSeq[PhysicalAggSig], + bufferSize: Int, + ) extends IR -final case class StreamGrouped(a: IR, groupSize: IR) extends IR -final case class StreamGroupByKey(a: IR, key: IndexedSeq[String], missingEqual: Boolean) extends IR + final case class LowerBoundOnOrderedCollection(orderedCollection: IR, elem: IR, onKey: Boolean) + extends IR -final case class StreamMap(a: IR, name: Name, body: IR) extends TypedIR[TStream] { - def elementTyp: Type = typ.elementType -} + final case class GroupByKey(collection: IR) extends IR -final case class StreamTakeWhile(a: IR, elementName: Name, body: IR) extends IR - -final case class StreamDropWhile(a: IR, elementName: Name, body: IR) extends IR - -final case class StreamTake(a: IR, num: IR) extends IR -final case class StreamDrop(a: IR, num: IR) extends IR - -// Generate, in ascending order, a uniform random sample, without replacement, of numToSample integers in the range [0, totalRange) -final case class SeqSample( - totalRange: IR, - numToSample: IR, - rngState: IR, - requiresMemoryManagementPerElement: Boolean, -) extends IR - -// Take the child stream and sort each element into buckets based on the provided pivots. The first and last elements of -// pivots are the endpoints of the first and last interval respectively, should not be contained in the dataset. -final case class StreamDistribute( - child: IR, - pivots: IR, - path: IR, - comparisonOp: ComparisonOp[_], - spec: AbstractTypedCodecSpec, -) extends IR - -// "Whiten" a stream of vectors by regressing out from each vector all components -// in the direction of vectors in the preceding window. For efficiency, takes -// a stream of "chunks" of vectors. -// Takes a stream of structs, with two designated fields: `prevWindow` is the -// previous window (e.g. from the previous partition), if there is one, and -// `newChunk` is the new chunk to whiten. -final case class StreamWhiten( - stream: IR, - newChunk: String, - prevWindow: String, - vecSize: Int, - windowSize: Int, - chunkSize: Int, - blockSize: Int, - normalizeAfterWhiten: Boolean, -) extends IR - -object ArrayZipBehavior extends Enumeration { - type ArrayZipBehavior = Value - val AssumeSameLength: Value = Value(0) - val AssertSameLength: Value = Value(1) - val TakeMinLength: Value = Value(2) - val ExtendNA: Value = Value(3) -} + final case class RNGStateLiteral() extends IR -final case class StreamZip( - as: IndexedSeq[IR], - names: IndexedSeq[Name], - body: IR, - behavior: ArrayZipBehavior, - errorID: Int = ErrorIDs.NO_ERROR, -) extends TypedIR[TStream] - -final case class StreamMultiMerge(as: IndexedSeq[IR], key: IndexedSeq[String]) - extends TypedIR[TStream] - -final case class StreamZipJoinProducers( - contexts: IR, - ctxName: Name, - makeProducer: IR, - key: IndexedSeq[String], - curKey: Name, - curVals: Name, - joinF: IR, -) extends TypedIR[TStream] - -/** The StreamZipJoin node assumes that input streams have distinct keys. If input streams do not - * have distinct keys, the key that is included in the result is undefined, but is likely the last. - */ -final case class StreamZipJoin( - as: IndexedSeq[IR], - key: IndexedSeq[String], - curKey: Name, - curVals: Name, - joinF: IR, -) extends TypedIR[TStream] - -final case class StreamFilter(a: IR, name: Name, cond: IR) extends TypedIR[TStream] -final case class StreamFlatMap(a: IR, name: Name, body: IR) extends TypedIR[TStream] - -final case class StreamFold(a: IR, zero: IR, accumName: Name, valueName: Name, body: IR) extends IR - -object StreamFold2 { - def apply(a: StreamFold): StreamFold2 = - StreamFold2( - a.a, - FastSeq((a.accumName, a.zero)), - a.valueName, - FastSeq(a.body), - Ref(a.accumName, a.zero.typ), - ) -} + final case class RNGSplit(state: IR, dynBitstring: IR) extends IR -final case class StreamFold2( - a: IR, - accum: IndexedSeq[(Name, IR)], - valueName: Name, - seq: IndexedSeq[IR], - result: IR, -) extends IR { - assert(accum.length == seq.length) - val nameIdx: Map[Name, Int] = accum.map(_._1).zipWithIndex.toMap -} + final case class StreamLen(a: IR) extends IR + + final case class StreamGrouped(a: IR, groupSize: IR) extends IR + + final case class StreamGroupByKey(a: IR, key: IndexedSeq[String], missingEqual: Boolean) + extends IR + + final case class StreamMap(a: IR, name: Name, body: IR) extends TypedIR[TStream] { + def elementTyp: Type = typ.elementType + } + + final case class StreamTakeWhile(a: IR, elementName: Name, body: IR) extends IR + + final case class StreamDropWhile(a: IR, elementName: Name, body: IR) extends IR + + final case class StreamTake(a: IR, num: IR) extends IR + + final case class StreamDrop(a: IR, num: IR) extends IR + + /* Generate, in ascending order, a uniform random sample, without replacement, of numToSample + * integers in the range [0, totalRange) */ + final case class SeqSample( + totalRange: IR, + numToSample: IR, + rngState: IR, + requiresMemoryManagementPerElement: Boolean, + ) extends IR + + /* Take the child stream and sort each element into buckets based on the provided pivots. The + * first and last elements of pivots are the endpoints of the first and last interval + * respectively, should not be contained in the dataset. */ + final case class StreamDistribute( + child: IR, + pivots: IR, + path: IR, + comparisonOp: ComparisonOp[_], + spec: AbstractTypedCodecSpec, + ) extends IR + + // "Whiten" a stream of vectors by regressing out from each vector all components + // in the direction of vectors in the preceding window. For efficiency, takes + // a stream of "chunks" of vectors. + // Takes a stream of structs, with two designated fields: `prevWindow` is the + // previous window (e.g. from the previous partition), if there is one, and + // `newChunk` is the new chunk to whiten. + final case class StreamWhiten( + stream: IR, + newChunk: String, + prevWindow: String, + vecSize: Int, + windowSize: Int, + chunkSize: Int, + blockSize: Int, + normalizeAfterWhiten: Boolean, + ) extends IR + + object ArrayZipBehavior extends Enumeration { + type ArrayZipBehavior = Value + val AssumeSameLength: Value = Value(0) + val AssertSameLength: Value = Value(1) + val TakeMinLength: Value = Value(2) + val ExtendNA: Value = Value(3) + } + + final case class StreamZip( + as: IndexedSeq[IR], + names: IndexedSeq[Name], + body: IR, + behavior: ArrayZipBehavior, + errorID: Int = ErrorIDs.NO_ERROR, + ) extends TypedIR[TStream] + + final case class StreamMultiMerge(as: IndexedSeq[IR], key: IndexedSeq[String]) + extends TypedIR[TStream] + + final case class StreamZipJoinProducers( + contexts: IR, + ctxName: Name, + makeProducer: IR, + key: IndexedSeq[String], + curKey: Name, + curVals: Name, + joinF: IR, + ) extends TypedIR[TStream] + + /** The StreamZipJoin node assumes that input streams have distinct keys. If input streams do not + * have distinct keys, the key that is included in the result is undefined, but is likely the + * last. + */ + final case class StreamZipJoin( + as: IndexedSeq[IR], + key: IndexedSeq[String], + curKey: Name, + curVals: Name, + joinF: IR, + ) extends TypedIR[TStream] + + final case class StreamFilter(a: IR, name: Name, cond: IR) extends TypedIR[TStream] + + final case class StreamFlatMap(a: IR, name: Name, body: IR) extends TypedIR[TStream] + + final case class StreamFold(a: IR, zero: IR, accumName: Name, valueName: Name, body: IR) + extends IR + + object StreamFold2 { + def apply(a: StreamFold): StreamFold2 = + StreamFold2( + a.a, + FastSeq((a.accumName, a.zero)), + a.valueName, + FastSeq(a.body), + Ref(a.accumName, a.zero.typ), + ) + } + + final case class StreamFold2( + a: IR, + accum: IndexedSeq[(Name, IR)], + valueName: Name, + seq: IndexedSeq[IR], + result: IR, + ) extends IR { + assert(accum.length == seq.length) + val nameIdx: Map[Name, Int] = accum.map(_._1).zipWithIndex.toMap + } + + final case class StreamScan(a: IR, zero: IR, accumName: Name, valueName: Name, body: IR) + extends IR + + final case class StreamFor(a: IR, valueName: Name, body: IR) extends IR + + final case class StreamAgg(a: IR, name: Name, query: IR) extends IR + + final case class StreamAggScan(a: IR, name: Name, query: IR) extends IR + + object StreamJoin { + def apply( + left: IR, + right: IR, + lKey: IndexedSeq[String], + rKey: IndexedSeq[String], + l: Name, + r: Name, + joinF: IR, + joinType: String, + requiresMemoryManagement: Boolean, + rightKeyIsDistinct: Boolean = false, + ): IR = { + val lType = tcoerce[TStream](left.typ) + val rType = tcoerce[TStream](right.typ) + val lEltType = tcoerce[TStruct](lType.elementType) + val rEltType = tcoerce[TStruct](rType.elementType) + assert(lEltType.typeAfterSelectNames(lKey) isJoinableWith rEltType.typeAfterSelectNames(rKey)) + + if (!rightKeyIsDistinct) { + val rightGroupedStream = StreamGroupByKey(right, rKey, missingEqual = false) + val groupField = genUID() + + // stream of {key, groupField}, where 'groupField' is an array of all rows + // in 'right' with key 'key' + val rightGrouped = mapIR(rightGroupedStream) { group => + bindIR(ToArray(group)) { array => + bindIR(ArrayRef(array, 0)) { head => + MakeStruct(rKey.map(key => key -> GetField(head, key)) :+ groupField -> array) + } + } + } + + val rElt = Ref(freshName(), tcoerce[TStream](rightGrouped.typ).elementType) + val lElt = Ref(freshName(), lEltType) + val makeTupleFromJoin = MakeStruct(FastSeq("left" -> lElt, "rightGroup" -> rElt)) + val joined = StreamJoinRightDistinct( + left, + rightGrouped, + lKey, + rKey, + lElt.name, + rElt.name, + makeTupleFromJoin, + joinType, + ) -final case class StreamScan(a: IR, zero: IR, accumName: Name, valueName: Name, body: IR) extends IR + // joined is a stream of {leftElement, rightGroup} + bindIR(MakeArray(NA(rEltType))) { missingSingleton => + flatMapIR(joined) { x => + Let( + FastSeq(l -> GetField(x, "left")), + bindIR(GetField(GetField(x, "rightGroup"), groupField)) { rightElts => + joinType match { + case "left" | "outer" => StreamMap( + ToStream( + If(IsNA(rightElts), missingSingleton, rightElts), + requiresMemoryManagement, + ), + r, + joinF, + ) + case "right" | "inner" => + StreamMap(ToStream(rightElts, requiresMemoryManagement), r, joinF) + } + }, + ) + } + } + } else { + StreamJoinRightDistinct(left, right, lKey, rKey, l, r, joinF, joinType) + } + } + } -final case class StreamFor(a: IR, valueName: Name, body: IR) extends IR + final case class StreamLeftIntervalJoin( + // input streams + left: IR, + right: IR, -final case class StreamAgg(a: IR, name: Name, query: IR) extends IR -final case class StreamAggScan(a: IR, name: Name, query: IR) extends IR + // names for joiner + lKeyFieldName: String, + rIntervalFieldName: String, + + // how to combine records + lname: Name, + rname: Name, + body: IR, + ) extends IR { + override protected lazy val childrenSeq: IndexedSeq[BaseIR] = + FastSeq(left, right, body) + } -object StreamJoin { - def apply( + final case class StreamJoinRightDistinct( left: IR, right: IR, lKey: IndexedSeq[String], @@ -561,835 +674,793 @@ object StreamJoin { r: Name, joinF: IR, joinType: String, - requiresMemoryManagement: Boolean, - rightKeyIsDistinct: Boolean = false, - ): IR = { - val lType = tcoerce[TStream](left.typ) - val rType = tcoerce[TStream](right.typ) - val lEltType = tcoerce[TStruct](lType.elementType) - val rEltType = tcoerce[TStruct](rType.elementType) - assert(lEltType.typeAfterSelectNames(lKey) isJoinableWith rEltType.typeAfterSelectNames(rKey)) - - if (!rightKeyIsDistinct) { - val rightGroupedStream = StreamGroupByKey(right, rKey, missingEqual = false) - val groupField = genUID() - - // stream of {key, groupField}, where 'groupField' is an array of all rows - // in 'right' with key 'key' - val rightGrouped = mapIR(rightGroupedStream) { group => - bindIR(ToArray(group)) { array => - bindIR(ArrayRef(array, 0)) { head => - MakeStruct(rKey.map(key => key -> GetField(head, key)) :+ groupField -> array) - } - } - } + ) extends IR { + def isIntervalJoin: Boolean = { + if (rKey.size != 1) return false + val lKeyTyp = tcoerce[TStruct](tcoerce[TStream](left.typ).elementType).fieldType(lKey(0)) + val rKeyTyp = tcoerce[TStruct](tcoerce[TStream](right.typ).elementType).fieldType(rKey(0)) - val rElt = Ref(freshName(), tcoerce[TStream](rightGrouped.typ).elementType) - val lElt = Ref(freshName(), lEltType) - val makeTupleFromJoin = MakeStruct(FastSeq("left" -> lElt, "rightGroup" -> rElt)) - val joined = StreamJoinRightDistinct( - left, - rightGrouped, - lKey, - rKey, - lElt.name, - rElt.name, - makeTupleFromJoin, - joinType, - ) - - // joined is a stream of {leftElement, rightGroup} - bindIR(MakeArray(NA(rEltType))) { missingSingleton => - flatMapIR(joined) { x => - Let( - FastSeq(l -> GetField(x, "left")), - bindIR(GetField(GetField(x, "rightGroup"), groupField)) { rightElts => - joinType match { - case "left" | "outer" => StreamMap( - ToStream( - If(IsNA(rightElts), missingSingleton, rightElts), - requiresMemoryManagement, - ), - r, - joinF, - ) - case "right" | "inner" => - StreamMap(ToStream(rightElts, requiresMemoryManagement), r, joinF) - } - }, - ) - } - } - } else { - StreamJoinRightDistinct(left, right, lKey, rKey, l, r, joinF, joinType) + rKeyTyp.isInstanceOf[TInterval] && lKeyTyp != rKeyTyp } } -} -final case class StreamLeftIntervalJoin( - // input streams - left: IR, - right: IR, - - // names for joiner - lKeyFieldName: String, - rIntervalFieldName: String, - - // how to combine records - lname: Name, - rname: Name, - body: IR, -) extends IR { - override protected lazy val childrenSeq: IndexedSeq[BaseIR] = - FastSeq(left, right, body) -} + final case class StreamLocalLDPrune( + child: IR, + r2Threshold: IR, + windowSize: IR, + maxQueueSize: IR, + nSamples: IR, + ) extends IR -final case class StreamJoinRightDistinct( - left: IR, - right: IR, - lKey: IndexedSeq[String], - rKey: IndexedSeq[String], - l: Name, - r: Name, - joinF: IR, - joinType: String, -) extends IR { - def isIntervalJoin: Boolean = { - if (rKey.size != 1) return false - val lKeyTyp = tcoerce[TStruct](tcoerce[TStream](left.typ).elementType).fieldType(lKey(0)) - val rKeyTyp = tcoerce[TStruct](tcoerce[TStream](right.typ).elementType).fieldType(rKey(0)) - - rKeyTyp.isInstanceOf[TInterval] && lKeyTyp != rKeyTyp + sealed trait NDArrayIR extends TypedIR[TNDArray] { + def elementTyp: Type = typ.elementType } -} -final case class StreamLocalLDPrune( - child: IR, - r2Threshold: IR, - windowSize: IR, - maxQueueSize: IR, - nSamples: IR, -) extends IR + object MakeNDArray { + def fill(elt: IR, shape: IndexedSeq[IR], rowMajor: IR): MakeNDArray = { + val flatSize: IR = if (shape.nonEmpty) + shape.reduce((l, r) => l * r) + else + 0L + MakeNDArray( + ToArray(mapIR(rangeIR(flatSize.toI))(_ => elt)), + MakeTuple.ordered(shape), + rowMajor, + ErrorIDs.NO_ERROR, + ) + } + } -sealed trait NDArrayIR extends TypedIR[TNDArray] { - def elementTyp: Type = typ.elementType -} + final case class MakeNDArray(data: IR, shape: IR, rowMajor: IR, errorId: Int) extends NDArrayIR -object MakeNDArray { - def fill(elt: IR, shape: IndexedSeq[IR], rowMajor: IR): MakeNDArray = { - val flatSize: IR = if (shape.nonEmpty) - shape.reduce((l, r) => l * r) - else - 0L - MakeNDArray( - ToArray(mapIR(rangeIR(flatSize.toI))(_ => elt)), - MakeTuple.ordered(shape), - rowMajor, - ErrorIDs.NO_ERROR, - ) - } -} + final case class NDArrayShape(nd: IR) extends IR + + final case class NDArrayReshape(nd: IR, shape: IR, errorID: Int) extends NDArrayIR + + final case class NDArrayConcat(nds: IR, axis: Int) extends NDArrayIR -final case class MakeNDArray(data: IR, shape: IR, rowMajor: IR, errorId: Int) extends NDArrayIR + final case class NDArrayRef(nd: IR, idxs: IndexedSeq[IR], errorId: Int) extends IR -final case class NDArrayShape(nd: IR) extends IR + final case class NDArraySlice(nd: IR, slices: IR) extends NDArrayIR -final case class NDArrayReshape(nd: IR, shape: IR, errorID: Int) extends NDArrayIR + final case class NDArrayFilter(nd: IR, keep: IndexedSeq[IR]) extends NDArrayIR -final case class NDArrayConcat(nds: IR, axis: Int) extends NDArrayIR + final case class NDArrayMap(nd: IR, valueName: Name, body: IR) extends NDArrayIR -final case class NDArrayRef(nd: IR, idxs: IndexedSeq[IR], errorId: Int) extends IR -final case class NDArraySlice(nd: IR, slices: IR) extends NDArrayIR -final case class NDArrayFilter(nd: IR, keep: IndexedSeq[IR]) extends NDArrayIR + final case class NDArrayMap2(l: IR, r: IR, lName: Name, rName: Name, body: IR, errorID: Int) + extends NDArrayIR -final case class NDArrayMap(nd: IR, valueName: Name, body: IR) extends NDArrayIR + final case class NDArrayReindex(nd: IR, indexExpr: IndexedSeq[Int]) extends NDArrayIR -final case class NDArrayMap2(l: IR, r: IR, lName: Name, rName: Name, body: IR, errorID: Int) - extends NDArrayIR + final case class NDArrayAgg(nd: IR, axes: IndexedSeq[Int]) extends IR -final case class NDArrayReindex(nd: IR, indexExpr: IndexedSeq[Int]) extends NDArrayIR -final case class NDArrayAgg(nd: IR, axes: IndexedSeq[Int]) extends IR -final case class NDArrayWrite(nd: IR, path: IR) extends IR + final case class NDArrayWrite(nd: IR, path: IR) extends IR -final case class NDArrayMatMul(l: IR, r: IR, errorID: Int) extends NDArrayIR + final case class NDArrayMatMul(l: IR, r: IR, errorID: Int) extends NDArrayIR -object NDArrayQR { - def pType(mode: String, req: Boolean): PType = { - mode match { - case "r" => PCanonicalNDArray(PFloat64Required, 2, req) - case "raw" => PCanonicalTuple( + object NDArrayQR { + def pType(mode: String, req: Boolean): PType = { + mode match { + case "r" => PCanonicalNDArray(PFloat64Required, 2, req) + case "raw" => PCanonicalTuple( + req, + PCanonicalNDArray(PFloat64Required, 2, true), + PCanonicalNDArray(PFloat64Required, 1, true), + ) + case "reduced" => PCanonicalTuple( + req, + PCanonicalNDArray(PFloat64Required, 2, true), + PCanonicalNDArray(PFloat64Required, 2, true), + ) + case "complete" => PCanonicalTuple( + req, + PCanonicalNDArray(PFloat64Required, 2, true), + PCanonicalNDArray(PFloat64Required, 2, true), + ) + } + } + } + + object NDArraySVD { + def pTypes(computeUV: Boolean, req: Boolean): PType = { + if (computeUV) { + PCanonicalTuple( req, PCanonicalNDArray(PFloat64Required, 2, true), PCanonicalNDArray(PFloat64Required, 1, true), - ) - case "reduced" => PCanonicalTuple( - req, - PCanonicalNDArray(PFloat64Required, 2, true), - PCanonicalNDArray(PFloat64Required, 2, true), - ) - case "complete" => PCanonicalTuple( - req, - PCanonicalNDArray(PFloat64Required, 2, true), PCanonicalNDArray(PFloat64Required, 2, true), ) + } else { + PCanonicalNDArray(PFloat64Required, 1, req) + } } } -} -object NDArraySVD { - def pTypes(computeUV: Boolean, req: Boolean): PType = { - if (computeUV) { - PCanonicalTuple( - req, - PCanonicalNDArray(PFloat64Required, 2, true), - PCanonicalNDArray(PFloat64Required, 1, true), - PCanonicalNDArray(PFloat64Required, 2, true), - ) - } else { - PCanonicalNDArray(PFloat64Required, 1, req) - } + object NDArrayInv { + val pType = PCanonicalNDArray(PFloat64Required, 2) } -} -object NDArrayInv { - val pType = PCanonicalNDArray(PFloat64Required, 2) -} + final case class NDArrayQR(nd: IR, mode: String, errorID: Int) extends IR -final case class NDArrayQR(nd: IR, mode: String, errorID: Int) extends IR + final case class NDArraySVD(nd: IR, fullMatrices: Boolean, computeUV: Boolean, errorID: Int) + extends IR -final case class NDArraySVD(nd: IR, fullMatrices: Boolean, computeUV: Boolean, errorID: Int) - extends IR + object NDArrayEigh { + def pTypes(eigvalsOnly: Boolean, req: Boolean): PType = + if (eigvalsOnly) { + PCanonicalNDArray(PFloat64Required, 1, req) + } else { + PCanonicalTuple( + req, + PCanonicalNDArray(PFloat64Required, 1, true), + PCanonicalNDArray(PFloat64Required, 2, true), + ) + } + } -object NDArrayEigh { - def pTypes(eigvalsOnly: Boolean, req: Boolean): PType = - if (eigvalsOnly) { - PCanonicalNDArray(PFloat64Required, 1, req) - } else { - PCanonicalTuple( - req, - PCanonicalNDArray(PFloat64Required, 1, true), - PCanonicalNDArray(PFloat64Required, 2, true), - ) - } -} + final case class NDArrayEigh(nd: IR, eigvalsOnly: Boolean, errorID: Int) extends IR -final case class NDArrayEigh(nd: IR, eigvalsOnly: Boolean, errorID: Int) extends IR + final case class NDArrayInv(nd: IR, errorID: Int) extends IR -final case class NDArrayInv(nd: IR, errorID: Int) extends IR + final case class AggFilter(cond: IR, aggIR: IR, isScan: Boolean) extends IR -final case class AggFilter(cond: IR, aggIR: IR, isScan: Boolean) extends IR + final case class AggExplode(array: IR, name: Name, aggBody: IR, isScan: Boolean) extends IR -final case class AggExplode(array: IR, name: Name, aggBody: IR, isScan: Boolean) extends IR + final case class AggGroupBy(key: IR, aggIR: IR, isScan: Boolean) extends IR -final case class AggGroupBy(key: IR, aggIR: IR, isScan: Boolean) extends IR + final case class AggArrayPerElement( + a: IR, + elementName: Name, + indexName: Name, + aggBody: IR, + knownLength: Option[IR], + isScan: Boolean, + ) extends IR -final case class AggArrayPerElement( - a: IR, - elementName: Name, - indexName: Name, - aggBody: IR, - knownLength: Option[IR], - isScan: Boolean, -) extends IR + object ApplyAggOp { + def apply(op: AggOp, initOpArgs: IR*)(seqOpArgs: IR*): ApplyAggOp = + ApplyAggOp( + initOpArgs.toIndexedSeq, + seqOpArgs.toIndexedSeq, + AggSignature(op, initOpArgs.map(_.typ), seqOpArgs.map(_.typ)), + ) + } -object ApplyAggOp { - def apply(op: AggOp, initOpArgs: IR*)(seqOpArgs: IR*): ApplyAggOp = - ApplyAggOp( - initOpArgs.toIndexedSeq, - seqOpArgs.toIndexedSeq, - AggSignature(op, initOpArgs.map(_.typ), seqOpArgs.map(_.typ)), - ) -} + final case class ApplyAggOp( + initOpArgs: IndexedSeq[IR], + seqOpArgs: IndexedSeq[IR], + aggSig: AggSignature, + ) extends IR { -final case class ApplyAggOp( - initOpArgs: IndexedSeq[IR], - seqOpArgs: IndexedSeq[IR], - aggSig: AggSignature, -) extends IR { + def nSeqOpArgs = seqOpArgs.length - def nSeqOpArgs = seqOpArgs.length + def nInitArgs = initOpArgs.length - def nInitArgs = initOpArgs.length + def op: AggOp = aggSig.op + } - def op: AggOp = aggSig.op -} + object AggFold { -object AggFold { + def min(element: IR, sortFields: IndexedSeq[SortField]): IR = { + val elementType = element.typ.asInstanceOf[TStruct] + val keyType = elementType.select(sortFields.map(_.field))._1 + minAndMaxHelper(element, keyType, StructLT(keyType, sortFields)) + } - def min(element: IR, sortFields: IndexedSeq[SortField]): IR = { - val elementType = element.typ.asInstanceOf[TStruct] - val keyType = elementType.select(sortFields.map(_.field))._1 - minAndMaxHelper(element, keyType, StructLT(keyType, sortFields)) - } + def max(element: IR, sortFields: IndexedSeq[SortField]): IR = { + val elementType = element.typ.asInstanceOf[TStruct] + val keyType = elementType.select(sortFields.map(_.field))._1 + minAndMaxHelper(element, keyType, StructGT(keyType, sortFields)) + } - def max(element: IR, sortFields: IndexedSeq[SortField]): IR = { - val elementType = element.typ.asInstanceOf[TStruct] - val keyType = elementType.select(sortFields.map(_.field))._1 - minAndMaxHelper(element, keyType, StructGT(keyType, sortFields)) - } + def all(element: IR): IR = + aggFoldIR(True()) { accum => + ApplySpecial("land", Seq.empty[Type], Seq(accum, element), TBoolean, ErrorIDs.NO_ERROR) + } { (accum1, accum2) => + ApplySpecial("land", Seq.empty[Type], Seq(accum1, accum2), TBoolean, ErrorIDs.NO_ERROR) + } - def all(element: IR): IR = - aggFoldIR(True()) { accum => - ApplySpecial("land", Seq.empty[Type], Seq(accum, element), TBoolean, ErrorIDs.NO_ERROR) - } { (accum1, accum2) => - ApplySpecial("land", Seq.empty[Type], Seq(accum1, accum2), TBoolean, ErrorIDs.NO_ERROR) - } + private def minAndMaxHelper(element: IR, keyType: TStruct, comp: ComparisonOp[Boolean]): IR = { + val keyFields = keyType.fields.map(_.name) - private def minAndMaxHelper(element: IR, keyType: TStruct, comp: ComparisonOp[Boolean]): IR = { - val keyFields = keyType.fields.map(_.name) - - val minAndMaxZero = NA(keyType) - val aggFoldMinAccumName1 = freshName() - val aggFoldMinAccumName2 = freshName() - val aggFoldMinAccumRef1 = Ref(aggFoldMinAccumName1, keyType) - val aggFoldMinAccumRef2 = Ref(aggFoldMinAccumName2, keyType) - val minSeq = bindIR(SelectFields(element, keyFields)) { keyOfCurElementRef => - If( - IsNA(aggFoldMinAccumRef1), - keyOfCurElementRef, + val minAndMaxZero = NA(keyType) + val aggFoldMinAccumName1 = freshName() + val aggFoldMinAccumName2 = freshName() + val aggFoldMinAccumRef1 = Ref(aggFoldMinAccumName1, keyType) + val aggFoldMinAccumRef2 = Ref(aggFoldMinAccumName2, keyType) + val minSeq = bindIR(SelectFields(element, keyFields)) { keyOfCurElementRef => If( - ApplyComparisonOp(comp, aggFoldMinAccumRef1, keyOfCurElementRef), - aggFoldMinAccumRef1, + IsNA(aggFoldMinAccumRef1), keyOfCurElementRef, - ), - ) - } - val minComb = - If( - IsNA(aggFoldMinAccumRef1), - aggFoldMinAccumRef2, + If( + ApplyComparisonOp(comp, aggFoldMinAccumRef1, keyOfCurElementRef), + aggFoldMinAccumRef1, + keyOfCurElementRef, + ), + ) + } + val minComb = If( - ApplyComparisonOp(comp, aggFoldMinAccumRef1, aggFoldMinAccumRef2), - aggFoldMinAccumRef1, + IsNA(aggFoldMinAccumRef1), aggFoldMinAccumRef2, - ), + If( + ApplyComparisonOp(comp, aggFoldMinAccumRef1, aggFoldMinAccumRef2), + aggFoldMinAccumRef1, + aggFoldMinAccumRef2, + ), + ) + + AggFold(minAndMaxZero, minSeq, minComb, aggFoldMinAccumName1, aggFoldMinAccumName2, false) + } + } + + final case class AggFold( + zero: IR, + seqOp: IR, + combOp: IR, + accumName: Name, + otherAccumName: Name, + isScan: Boolean, + ) extends IR + + object ApplyScanOp { + def apply(op: AggOp, initOpArgs: IR*)(seqOpArgs: IR*): ApplyScanOp = + ApplyScanOp( + initOpArgs.toIndexedSeq, + seqOpArgs.toIndexedSeq, + AggSignature(op, initOpArgs.map(_.typ), seqOpArgs.map(_.typ)), ) + } + + final case class ApplyScanOp( + initOpArgs: IndexedSeq[IR], + seqOpArgs: IndexedSeq[IR], + aggSig: AggSignature, + ) extends IR { + + def nSeqOpArgs = seqOpArgs.length - AggFold(minAndMaxZero, minSeq, minComb, aggFoldMinAccumName1, aggFoldMinAccumName2, false) + def nInitArgs = initOpArgs.length + + def op: AggOp = aggSig.op } -} -final case class AggFold( - zero: IR, - seqOp: IR, - combOp: IR, - accumName: Name, - otherAccumName: Name, - isScan: Boolean, -) extends IR - -object ApplyScanOp { - def apply(op: AggOp, initOpArgs: IR*)(seqOpArgs: IR*): ApplyScanOp = - ApplyScanOp( - initOpArgs.toIndexedSeq, - seqOpArgs.toIndexedSeq, - AggSignature(op, initOpArgs.map(_.typ), seqOpArgs.map(_.typ)), - ) -} + final case class InitOp(i: Int, args: IndexedSeq[IR], aggSig: PhysicalAggSig) extends IR -final case class ApplyScanOp( - initOpArgs: IndexedSeq[IR], - seqOpArgs: IndexedSeq[IR], - aggSig: AggSignature, -) extends IR { + final case class SeqOp(i: Int, args: IndexedSeq[IR], aggSig: PhysicalAggSig) extends IR - def nSeqOpArgs = seqOpArgs.length + final case class CombOp(i1: Int, i2: Int, aggSig: PhysicalAggSig) extends IR - def nInitArgs = initOpArgs.length + object ResultOp { + def makeTuple(aggs: IndexedSeq[PhysicalAggSig]) = + MakeTuple.ordered(aggs.zipWithIndex.map { case (aggSig, index) => + ResultOp(index, aggSig) + }) + } - def op: AggOp = aggSig.op -} + final case class ResultOp(idx: Int, aggSig: PhysicalAggSig) extends IR + + final private[ir] case class CombOpValue(i: Int, value: IR, aggSig: PhysicalAggSig) extends IR + + final case class AggStateValue(i: Int, aggSig: AggStateSig) extends IR + + final case class InitFromSerializedValue(i: Int, value: IR, aggSig: AggStateSig) extends IR + + final case class SerializeAggs( + startIdx: Int, + serializedIdx: Int, + spec: BufferSpec, + aggSigs: IndexedSeq[AggStateSig], + ) extends IR + + final case class DeserializeAggs( + startIdx: Int, + serializedIdx: Int, + spec: BufferSpec, + aggSigs: IndexedSeq[AggStateSig], + ) extends IR + + final case class RunAgg(body: IR, result: IR, signature: IndexedSeq[AggStateSig]) extends IR + + final case class RunAggScan( + array: IR, + name: Name, + init: IR, + seqs: IR, + result: IR, + signature: IndexedSeq[AggStateSig], + ) extends IR + + object Begin { + def apply(xs: IndexedSeq[IR]): IR = + if (xs.isEmpty) + Void() + else + Let(xs.init.map(x => (freshName(), x)), xs.last) + } + + final case class Begin(xs: IndexedSeq[IR]) extends IR + + final case class MakeStruct(fields: IndexedSeq[(String, IR)]) extends IR -final case class InitOp(i: Int, args: IndexedSeq[IR], aggSig: PhysicalAggSig) extends IR -final case class SeqOp(i: Int, args: IndexedSeq[IR], aggSig: PhysicalAggSig) extends IR -final case class CombOp(i1: Int, i2: Int, aggSig: PhysicalAggSig) extends IR + final case class SelectFields(old: IR, fields: IndexedSeq[String]) extends IR -object ResultOp { - def makeTuple(aggs: IndexedSeq[PhysicalAggSig]) = - MakeTuple.ordered(aggs.zipWithIndex.map { case (aggSig, index) => - ResultOp(index, aggSig) + object InsertFields { + def apply(old: IR, fields: IndexedSeq[(String, IR)]): InsertFields = + InsertFields(old, fields, None) + } + + final case class InsertFields( + old: IR, + fields: IndexedSeq[(String, IR)], + fieldOrder: Option[IndexedSeq[String]], + ) extends TypedIR[TStruct] + + object GetFieldByIdx { + def apply(s: IR, field: Int): IR = + (s.typ: @unchecked) match { + case t: TStruct => GetField(s, t.fieldNames(field)) + case _: TTuple => GetTupleElement(s, field) + } + } + + final case class GetField(o: IR, name: String) extends IR + + object MakeTuple { + def ordered(types: IndexedSeq[IR]): MakeTuple = MakeTuple(types.zipWithIndex.map { + case (ir, i) => + (i, ir) }) -} + } -final case class ResultOp(idx: Int, aggSig: PhysicalAggSig) extends IR - -final private case class CombOpValue(i: Int, value: IR, aggSig: PhysicalAggSig) extends IR -final case class AggStateValue(i: Int, aggSig: AggStateSig) extends IR -final case class InitFromSerializedValue(i: Int, value: IR, aggSig: AggStateSig) extends IR - -final case class SerializeAggs( - startIdx: Int, - serializedIdx: Int, - spec: BufferSpec, - aggSigs: IndexedSeq[AggStateSig], -) extends IR - -final case class DeserializeAggs( - startIdx: Int, - serializedIdx: Int, - spec: BufferSpec, - aggSigs: IndexedSeq[AggStateSig], -) extends IR - -final case class RunAgg(body: IR, result: IR, signature: IndexedSeq[AggStateSig]) extends IR - -final case class RunAggScan( - array: IR, - name: Name, - init: IR, - seqs: IR, - result: IR, - signature: IndexedSeq[AggStateSig], -) extends IR - -object Begin { - def apply(xs: IndexedSeq[IR]): IR = - if (xs.isEmpty) - Void() - else - Let(xs.init.map(x => (freshName(), x)), xs.last) -} + final case class MakeTuple(fields: IndexedSeq[(Int, IR)]) extends IR + + final case class GetTupleElement(o: IR, idx: Int) extends IR + + object In { + def apply(i: Int, typ: Type): In = In( + i, + SingleCodeEmitParamType( + false, + typ match { + case TInt32 => Int32SingleCodeType + case TInt64 => Int64SingleCodeType + case TFloat32 => Float32SingleCodeType + case TFloat64 => Float64SingleCodeType + case TBoolean => BooleanSingleCodeType + case _: TStream => throw new UnsupportedOperationException + case t => PTypeReferenceSingleCodeType(PType.canonical(t)) + }, + ), + ) + } -final case class Begin(xs: IndexedSeq[IR]) extends IR -final case class MakeStruct(fields: IndexedSeq[(String, IR)]) extends IR -final case class SelectFields(old: IR, fields: IndexedSeq[String]) extends IR + // Function Input + final case class In(i: Int, _typ: EmitParamType) extends IR -object InsertFields { - def apply(old: IR, fields: IndexedSeq[(String, IR)]): InsertFields = - InsertFields(old, fields, None) -} + // FIXME: should be type any + object Die { + def apply(message: String, typ: Type): Die = Die(Str(message), typ, ErrorIDs.NO_ERROR) -final case class InsertFields( - old: IR, - fields: IndexedSeq[(String, IR)], - fieldOrder: Option[IndexedSeq[String]], -) extends TypedIR[TStruct] - -object GetFieldByIdx { - def apply(s: IR, field: Int): IR = - (s.typ: @unchecked) match { - case t: TStruct => GetField(s, t.fieldNames(field)) - case _: TTuple => GetTupleElement(s, field) + def apply(message: String, typ: Type, errorId: Int): Die = Die(Str(message), typ, errorId) + } + + /** the Trap node runs the `child` node with an exception handler. If the child throws a + * HailException (user exception), then we return the tuple ((msg, errorId), NA). If the child + * throws any other exception, we raise that exception. If the child does not throw, then we + * return the tuple (NA, child value). + */ + final case class Trap(child: IR) extends IR + + final case class Die(message: IR, _typ: Type, errorId: Int) extends IR + + final case class ConsoleLog(message: IR, result: IR) extends IR + + final case class ApplyIR( + function: String, + typeArgs: Seq[Type], + args: Seq[IR], + returnType: Type, + errorID: Int, + ) extends IR { + var conversion: (Seq[Type], Seq[IR], Int) => IR = _ + var inline: Boolean = _ + + private lazy val refs = args.map(a => Ref(freshName(), a.typ)).toArray + lazy val body: IR = conversion(typeArgs, refs, errorID).deepCopy() + lazy val refIdx: Map[Name, Int] = refs.map(_.name).zipWithIndex.toMap + + lazy val explicitNode: IR = { + val ir = Let(refs.map(_.name).zip(args), body) + assert(ir.typ == returnType) + ir } -} + } -final case class GetField(o: IR, name: String) extends IR + sealed abstract class AbstractApplyNode[F <: JVMFunction] extends IR { + def function: String -object MakeTuple { - def ordered(types: IndexedSeq[IR]): MakeTuple = MakeTuple(types.zipWithIndex.map { case (ir, i) => - (i, ir) - }) -} + def args: Seq[IR] -final case class MakeTuple(fields: IndexedSeq[(Int, IR)]) extends IR -final case class GetTupleElement(o: IR, idx: Int) extends IR - -object In { - def apply(i: Int, typ: Type): In = In( - i, - SingleCodeEmitParamType( - false, - typ match { - case TInt32 => Int32SingleCodeType - case TInt64 => Int64SingleCodeType - case TFloat32 => Float32SingleCodeType - case TFloat64 => Float64SingleCodeType - case TBoolean => BooleanSingleCodeType - case _: TStream => throw new UnsupportedOperationException - case t => PTypeReferenceSingleCodeType(PType.canonical(t)) - }, - ), - ) -} + def returnType: Type -// Function Input -final case class In(i: Int, _typ: EmitParamType) extends IR + def typeArgs: Seq[Type] -// FIXME: should be type any -object Die { - def apply(message: String, typ: Type): Die = Die(Str(message), typ, ErrorIDs.NO_ERROR) - def apply(message: String, typ: Type, errorId: Int): Die = Die(Str(message), typ, errorId) -} + def argTypes: Seq[Type] = args.map(_.typ) -/** the Trap node runs the `child` node with an exception handler. If the child throws a - * HailException (user exception), then we return the tuple ((msg, errorId), NA). If the child - * throws any other exception, we raise that exception. If the child does not throw, then we return - * the tuple (NA, child value). - */ -final case class Trap(child: IR) extends IR -final case class Die(message: IR, _typ: Type, errorId: Int) extends IR -final case class ConsoleLog(message: IR, result: IR) extends IR - -final case class ApplyIR( - function: String, - typeArgs: Seq[Type], - args: Seq[IR], - returnType: Type, - errorID: Int, -) extends IR { - var conversion: (Seq[Type], Seq[IR], Int) => IR = _ - var inline: Boolean = _ - - private lazy val refs = args.map(a => Ref(freshName(), a.typ)).toArray - lazy val body: IR = conversion(typeArgs, refs, errorID).deepCopy() - lazy val refIdx: Map[Name, Int] = refs.map(_.name).zipWithIndex.toMap - - lazy val explicitNode: IR = { - val ir = Let(refs.map(_.name).zip(args), body) - assert(ir.typ == returnType) - ir + lazy val implementation: F = + IRFunctionRegistry.lookupFunctionOrFail(function, returnType, typeArgs, argTypes) + .asInstanceOf[F] } -} -sealed abstract class AbstractApplyNode[F <: JVMFunction] extends IR { - def function: String - def args: Seq[IR] - def returnType: Type - def typeArgs: Seq[Type] - def argTypes: Seq[Type] = args.map(_.typ) + final case class Apply( + function: String, + typeArgs: Seq[Type], + args: Seq[IR], + returnType: Type, + errorID: Int, + ) extends AbstractApplyNode[UnseededMissingnessObliviousJVMFunction] + + final case class ApplySeeded( + function: String, + _args: Seq[IR], + rngState: IR, + staticUID: Long, + returnType: Type, + ) extends AbstractApplyNode[UnseededMissingnessObliviousJVMFunction] { + val args = rngState +: _args + val typeArgs: Seq[Type] = Seq.empty[Type] + } - lazy val implementation: F = - IRFunctionRegistry.lookupFunctionOrFail(function, returnType, typeArgs, argTypes) - .asInstanceOf[F] -} + final case class ApplySpecial( + function: String, + typeArgs: Seq[Type], + args: Seq[IR], + returnType: Type, + errorID: Int, + ) extends AbstractApplyNode[UnseededMissingnessAwareJVMFunction] + + final case class LiftMeOut(child: IR) extends IR + + final case class TableCount(child: TableIR) extends IR + + final case class MatrixCount(child: MatrixIR) extends IR + + final case class TableAggregate(child: TableIR, query: IR) extends IR + + final case class MatrixAggregate(child: MatrixIR, query: IR) extends IR + + final case class TableWrite(child: TableIR, writer: TableWriter) extends IR + + final case class TableMultiWrite( + _children: IndexedSeq[TableIR], + writer: WrappedMatrixNativeMultiWriter, + ) extends IR + + final case class TableGetGlobals(child: TableIR) extends IR + + final case class TableCollect(child: TableIR) extends IR + + final case class MatrixWrite(child: MatrixIR, writer: MatrixWriter) extends IR + + final case class MatrixMultiWrite( + _children: IndexedSeq[MatrixIR], + writer: MatrixNativeMultiWriter, + ) extends IR + + final case class TableToValueApply(child: TableIR, function: TableToValueFunction) extends IR + + final case class MatrixToValueApply(child: MatrixIR, function: MatrixToValueFunction) extends IR + + final case class BlockMatrixToValueApply( + child: BlockMatrixIR, + function: BlockMatrixToValueFunction, + ) extends IR + + final case class BlockMatrixCollect(child: BlockMatrixIR) extends NDArrayIR + + final case class BlockMatrixWrite(child: BlockMatrixIR, writer: BlockMatrixWriter) extends IR + + final case class BlockMatrixMultiWrite( + blockMatrices: IndexedSeq[BlockMatrixIR], + writer: BlockMatrixMultiWriter, + ) extends IR + + final case class CollectDistributedArray( + contexts: IR, + globals: IR, + cname: Name, + gname: Name, + body: IR, + dynamicID: IR, + staticID: String, + tsd: Option[TableStageDependency] = None, + ) extends IR + + object PartitionReader { + implicit val formats: Formats = + new DefaultFormats() { + override val typeHints = ShortTypeHints( + List( + classOf[PartitionRVDReader], + classOf[PartitionNativeReader], + classOf[PartitionNativeReaderIndexed], + classOf[PartitionNativeIntervalReader], + classOf[PartitionZippedNativeReader], + classOf[PartitionZippedIndexedNativeReader], + classOf[BgenPartitionReader], + classOf[GVCFPartitionReader], + classOf[TextInputFilterAndReplace], + classOf[VCFHeaderInfo], + classOf[AbstractTypedCodecSpec], + classOf[TypedCodecSpec], + classOf[AvroPartitionReader], + ), + typeHintFieldName = "name", + ) + BufferSpec.shortTypeHints + } + + new TStructSerializer + + new TypeSerializer + + new PTypeSerializer + + new ETypeSerializer + + new AvroSchemaSerializer + + def extract(ctx: ExecuteContext, jv: JValue): PartitionReader = { + (jv \ "name").extract[String] match { + case "PartitionNativeIntervalReader" => + val path = (jv \ "path").extract[String] + val spec = TableNativeReader.read(ctx.fs, path, None).spec + PartitionNativeIntervalReader( + ctx.stateManager, + path, + spec, + (jv \ "uidFieldName").extract[String], + ) + case "GVCFPartitionReader" => + val header = VCFHeaderInfo.fromJSON((jv \ "header")) + val callFields = (jv \ "callFields").extract[Set[String]] + val entryFloatType = IRParser.parseType((jv \ "entryFloatType").extract[String]) + val arrayElementsRequired = (jv \ "arrayElementsRequired").extract[Boolean] + val rg = (jv \ "rg") match { + case JString(s) => Some(s) + case JNothing => None + } + val contigRecoding = (jv \ "contigRecoding").extract[Map[String, String]] + val skipInvalidLoci = (jv \ "skipInvalidLoci").extract[Boolean] + val filterAndReplace = (jv \ "filterAndReplace").extract[TextInputFilterAndReplace] + val entriesFieldName = (jv \ "entriesFieldName").extract[String] + val uidFieldName = (jv \ "uidFieldName").extract[String] + GVCFPartitionReader( + header, callFields, entryFloatType, arrayElementsRequired, rg, + contigRecoding, + skipInvalidLoci, filterAndReplace, entriesFieldName, uidFieldName, + ) + case _ => jv.extract[PartitionReader] + } + } + } -final case class Apply( - function: String, - typeArgs: Seq[Type], - args: Seq[IR], - returnType: Type, - errorID: Int, -) extends AbstractApplyNode[UnseededMissingnessObliviousJVMFunction] - -final case class ApplySeeded( - function: String, - _args: Seq[IR], - rngState: IR, - staticUID: Long, - returnType: Type, -) extends AbstractApplyNode[UnseededMissingnessObliviousJVMFunction] { - val args = rngState +: _args - val typeArgs: Seq[Type] = Seq.empty[Type] -} + object PartitionWriter { + implicit val formats: Formats = + new DefaultFormats() { + override val typeHints = ShortTypeHints( + List( + classOf[PartitionNativeWriter], + classOf[TableTextPartitionWriter], + classOf[VCFPartitionWriter], + classOf[GenSampleWriter], + classOf[GenVariantWriter], + classOf[AbstractTypedCodecSpec], + classOf[TypedCodecSpec], + ), + typeHintFieldName = "name", + ) + BufferSpec.shortTypeHints + } + + new TStructSerializer + + new TypeSerializer + + new PTypeSerializer + + new PStructSerializer + + new ETypeSerializer + } -final case class ApplySpecial( - function: String, - typeArgs: Seq[Type], - args: Seq[IR], - returnType: Type, - errorID: Int, -) extends AbstractApplyNode[UnseededMissingnessAwareJVMFunction] - -final case class LiftMeOut(child: IR) extends IR -final case class TableCount(child: TableIR) extends IR -final case class MatrixCount(child: MatrixIR) extends IR -final case class TableAggregate(child: TableIR, query: IR) extends IR -final case class MatrixAggregate(child: MatrixIR, query: IR) extends IR - -final case class TableWrite(child: TableIR, writer: TableWriter) extends IR - -final case class TableMultiWrite( - _children: IndexedSeq[TableIR], - writer: WrappedMatrixNativeMultiWriter, -) extends IR - -final case class TableGetGlobals(child: TableIR) extends IR -final case class TableCollect(child: TableIR) extends IR - -final case class MatrixWrite(child: MatrixIR, writer: MatrixWriter) extends IR - -final case class MatrixMultiWrite(_children: IndexedSeq[MatrixIR], writer: MatrixNativeMultiWriter) - extends IR - -final case class TableToValueApply(child: TableIR, function: TableToValueFunction) extends IR -final case class MatrixToValueApply(child: MatrixIR, function: MatrixToValueFunction) extends IR - -final case class BlockMatrixToValueApply(child: BlockMatrixIR, function: BlockMatrixToValueFunction) - extends IR - -final case class BlockMatrixCollect(child: BlockMatrixIR) extends NDArrayIR - -final case class BlockMatrixWrite(child: BlockMatrixIR, writer: BlockMatrixWriter) extends IR - -final case class BlockMatrixMultiWrite( - blockMatrices: IndexedSeq[BlockMatrixIR], - writer: BlockMatrixMultiWriter, -) extends IR - -final case class CollectDistributedArray( - contexts: IR, - globals: IR, - cname: Name, - gname: Name, - body: IR, - dynamicID: IR, - staticID: String, - tsd: Option[TableStageDependency] = None, -) extends IR - -object PartitionReader { - implicit val formats: Formats = - new DefaultFormats() { - override val typeHints = ShortTypeHints( - List( - classOf[PartitionRVDReader], - classOf[PartitionNativeReader], - classOf[PartitionNativeReaderIndexed], - classOf[PartitionNativeIntervalReader], - classOf[PartitionZippedNativeReader], - classOf[PartitionZippedIndexedNativeReader], - classOf[BgenPartitionReader], - classOf[GVCFPartitionReader], - classOf[TextInputFilterAndReplace], - classOf[VCFHeaderInfo], - classOf[AbstractTypedCodecSpec], - classOf[TypedCodecSpec], - classOf[AvroPartitionReader], - ), - typeHintFieldName = "name", - ) + BufferSpec.shortTypeHints - } + - new TStructSerializer + - new TypeSerializer + - new PTypeSerializer + - new ETypeSerializer + - new AvroSchemaSerializer - - def extract(ctx: ExecuteContext, jv: JValue): PartitionReader = { - (jv \ "name").extract[String] match { - case "PartitionNativeIntervalReader" => - val path = (jv \ "path").extract[String] - val spec = TableNativeReader.read(ctx.fs, path, None).spec - PartitionNativeIntervalReader( - ctx.stateManager, - path, - spec, - (jv \ "uidFieldName").extract[String], - ) - case "GVCFPartitionReader" => - val header = VCFHeaderInfo.fromJSON((jv \ "header")) - val callFields = (jv \ "callFields").extract[Set[String]] - val entryFloatType = IRParser.parseType((jv \ "entryFloatType").extract[String]) - val arrayElementsRequired = (jv \ "arrayElementsRequired").extract[Boolean] - val rg = (jv \ "rg") match { - case JString(s) => Some(s) - case JNothing => None - } - val contigRecoding = (jv \ "contigRecoding").extract[Map[String, String]] - val skipInvalidLoci = (jv \ "skipInvalidLoci").extract[Boolean] - val filterAndReplace = (jv \ "filterAndReplace").extract[TextInputFilterAndReplace] - val entriesFieldName = (jv \ "entriesFieldName").extract[String] - val uidFieldName = (jv \ "uidFieldName").extract[String] - GVCFPartitionReader(header, callFields, entryFloatType, arrayElementsRequired, rg, - contigRecoding, - skipInvalidLoci, filterAndReplace, entriesFieldName, uidFieldName) - case _ => jv.extract[PartitionReader] - } + object MetadataWriter { + implicit val formats: Formats = + new DefaultFormats() { + override val typeHints = ShortTypeHints( + List( + classOf[RVDSpecWriter], + classOf[TableSpecWriter], + classOf[RelationalWriter], + classOf[TableTextFinalizer], + classOf[VCFExportFinalizer], + classOf[SimpleMetadataWriter], + classOf[RVDSpecMaker], + classOf[AbstractTypedCodecSpec], + classOf[TypedCodecSpec], + ), + typeHintFieldName = "name", + ) + BufferSpec.shortTypeHints + } + + new TStructSerializer + + new TypeSerializer + + new PTypeSerializer + + new ETypeSerializer } -} -object PartitionWriter { - implicit val formats: Formats = - new DefaultFormats() { - override val typeHints = ShortTypeHints( - List( - classOf[PartitionNativeWriter], - classOf[TableTextPartitionWriter], - classOf[VCFPartitionWriter], - classOf[GenSampleWriter], - classOf[GenVariantWriter], - classOf[AbstractTypedCodecSpec], - classOf[TypedCodecSpec], - ), - typeHintFieldName = "name", - ) + BufferSpec.shortTypeHints - } + - new TStructSerializer + - new TypeSerializer + - new PTypeSerializer + - new PStructSerializer + - new ETypeSerializer -} + abstract class PartitionReader { + assert(fullRowType.hasField(uidFieldName)) -object MetadataWriter { - implicit val formats: Formats = - new DefaultFormats() { - override val typeHints = ShortTypeHints( - List( - classOf[RVDSpecWriter], - classOf[TableSpecWriter], - classOf[RelationalWriter], - classOf[TableTextFinalizer], - classOf[VCFExportFinalizer], - classOf[SimpleMetadataWriter], - classOf[RVDSpecMaker], - classOf[AbstractTypedCodecSpec], - classOf[TypedCodecSpec], - ), - typeHintFieldName = "name", - ) + BufferSpec.shortTypeHints - } + - new TStructSerializer + - new TypeSerializer + - new PTypeSerializer + - new ETypeSerializer -} + def contextType: Type -abstract class PartitionReader { - assert(fullRowType.hasField(uidFieldName)) + def fullRowType: TStruct - def contextType: Type + def uidFieldName: String - def fullRowType: TStruct + def rowRequiredness(requestedType: TStruct): RStruct - def uidFieldName: String + def emitStream( + ctx: ExecuteContext, + cb: EmitCodeBuilder, + mb: EmitMethodBuilder[_], + context: EmitCode, + requestedType: TStruct, + ): IEmitCode - def rowRequiredness(requestedType: TStruct): RStruct + def toJValue: JValue + } - def emitStream( - ctx: ExecuteContext, - cb: EmitCodeBuilder, - mb: EmitMethodBuilder[_], - context: EmitCode, - requestedType: TStruct, - ): IEmitCode + abstract class PartitionWriter { + def consumeStream( + ctx: ExecuteContext, + cb: EmitCodeBuilder, + stream: StreamProducer, + context: EmitCode, + region: Value[Region], + ): IEmitCode - def toJValue: JValue -} + def ctxType: Type -abstract class PartitionWriter { - def consumeStream( - ctx: ExecuteContext, - cb: EmitCodeBuilder, - stream: StreamProducer, - context: EmitCode, - region: Value[Region], - ): IEmitCode - - def ctxType: Type - def returnType: Type - - def unionTypeRequiredness( - r: TypeWithRequiredness, - ctxType: TypeWithRequiredness, - streamType: RIterable, - ): Unit - - def toJValue: JValue = Extraction.decompose(this)(PartitionWriter.formats) -} + def returnType: Type + + def unionTypeRequiredness( + r: TypeWithRequiredness, + ctxType: TypeWithRequiredness, + streamType: RIterable, + ): Unit -abstract class SimplePartitionWriter extends PartitionWriter { - def ctxType: Type = TString - def returnType: Type = TString - - def unionTypeRequiredness( - r: TypeWithRequiredness, - ctxType: TypeWithRequiredness, - streamType: RIterable, - ): Unit = { - r.union(ctxType.required) - r.union(streamType.required) + def toJValue: JValue = Extraction.decompose(this)(PartitionWriter.formats) } - def consumeElement( - cb: EmitCodeBuilder, - element: EmitCode, - os: Value[OutputStream], - region: Value[Region], - ): Unit - - def preConsume(cb: EmitCodeBuilder, os: Value[OutputStream]): Unit = () - def postConsume(cb: EmitCodeBuilder, os: Value[OutputStream]): Unit = () - - final def consumeStream( - ctx: ExecuteContext, - cb: EmitCodeBuilder, - stream: StreamProducer, - context: EmitCode, - region: Value[Region], - ): IEmitCode = { - context.toI(cb).map(cb) { case ctx: SStringValue => - val filename = ctx.loadString(cb) - val os = cb.memoize(cb.emb.create(filename)) - - preConsume(cb, os) - stream.memoryManagedConsume(region, cb) { cb => - consumeElement(cb, stream.element, os, stream.elementRegion) - } - postConsume(cb, os) + abstract class SimplePartitionWriter extends PartitionWriter { + def ctxType: Type = TString - cb += os.invoke[Unit]("flush") - cb += os.invoke[Unit]("close") + def returnType: Type = TString - SJavaString.construct(cb, filename) + def unionTypeRequiredness( + r: TypeWithRequiredness, + ctxType: TypeWithRequiredness, + streamType: RIterable, + ): Unit = { + r.union(ctxType.required) + r.union(streamType.required) } - } -} -abstract class MetadataWriter { - def annotationType: Type + def consumeElement( + cb: EmitCodeBuilder, + element: EmitCode, + os: Value[OutputStream], + region: Value[Region], + ): Unit + + def preConsume(cb: EmitCodeBuilder, os: Value[OutputStream]): Unit = () + + def postConsume(cb: EmitCodeBuilder, os: Value[OutputStream]): Unit = () + + final def consumeStream( + ctx: ExecuteContext, + cb: EmitCodeBuilder, + stream: StreamProducer, + context: EmitCode, + region: Value[Region], + ): IEmitCode = { + context.toI(cb).map(cb) { case ctx: SStringValue => + val filename = ctx.loadString(cb) + val os = cb.memoize(cb.emb.create(filename)) + + preConsume(cb, os) + stream.memoryManagedConsume(region, cb) { cb => + consumeElement(cb, stream.element, os, stream.elementRegion) + } + postConsume(cb, os) - def writeMetadata( - writeAnnotations: => IEmitCode, - cb: EmitCodeBuilder, - region: Value[Region], - ): Unit + cb += os.invoke[Unit]("flush") + cb += os.invoke[Unit]("close") - def toJValue: JValue = Extraction.decompose(this)(MetadataWriter.formats) -} + SJavaString.construct(cb, filename) + } + } + } -final case class SimpleMetadataWriter(val annotationType: Type) extends MetadataWriter { - def writeMetadata(writeAnnotations: => IEmitCode, cb: EmitCodeBuilder, region: Value[Region]) - : Unit = - writeAnnotations.consume(cb, {}, _ => ()) -} + abstract class MetadataWriter { + def annotationType: Type -final case class ReadPartition(context: IR, rowType: TStruct, reader: PartitionReader) extends IR -final case class WritePartition(value: IR, writeCtx: IR, writer: PartitionWriter) extends IR -final case class WriteMetadata(writeAnnotations: IR, writer: MetadataWriter) extends IR - -final case class ReadValue(path: IR, reader: ValueReader, requestedType: Type) extends IR - -final case class WriteValue( - value: IR, - path: IR, - writer: ValueWriter, - stagingFile: Option[IR] = None, -) extends IR - -class PrimitiveIR(val self: IR) extends AnyVal { - def +(other: IR): IR = { - assert(self.typ == other.typ) - if (self.typ == TString) - invoke("concat", TString, self, other) - else - ApplyBinaryPrimOp(Add(), self, other) + def writeMetadata( + writeAnnotations: => IEmitCode, + cb: EmitCodeBuilder, + region: Value[Region], + ): Unit + + def toJValue: JValue = Extraction.decompose(this)(MetadataWriter.formats) } - def -(other: IR): IR = ApplyBinaryPrimOp(Subtract(), self, other) - def *(other: IR): IR = ApplyBinaryPrimOp(Multiply(), self, other) - def /(other: IR): IR = ApplyBinaryPrimOp(FloatingPointDivide(), self, other) - def floorDiv(other: IR): IR = ApplyBinaryPrimOp(RoundToNegInfDivide(), self, other) + final case class SimpleMetadataWriter(val annotationType: Type) extends MetadataWriter { + def writeMetadata(writeAnnotations: => IEmitCode, cb: EmitCodeBuilder, region: Value[Region]) + : Unit = + writeAnnotations.consume(cb, {}, _ => ()) + } - def &&(other: IR): IR = invoke("land", TBoolean, self, other) - def ||(other: IR): IR = invoke("lor", TBoolean, self, other) + final case class ReadPartition(context: IR, rowType: TStruct, reader: PartitionReader) extends IR - def toI: IR = Cast(self, TInt32) - def toL: IR = Cast(self, TInt64) - def toF: IR = Cast(self, TFloat32) - def toD: IR = Cast(self, TFloat64) + final case class WritePartition(value: IR, writeCtx: IR, writer: PartitionWriter) extends IR - def unary_-(): IR = ApplyUnaryPrimOp(Negate, self) - def unary_!(): IR = ApplyUnaryPrimOp(Bang, self) + final case class WriteMetadata(writeAnnotations: IR, writer: MetadataWriter) extends IR - def ceq(other: IR): IR = ApplyComparisonOp(EQWithNA(self.typ, other.typ), self, other) - def cne(other: IR): IR = ApplyComparisonOp(NEQWithNA(self.typ, other.typ), self, other) - def <(other: IR): IR = ApplyComparisonOp(LT(self.typ, other.typ), self, other) - def >(other: IR): IR = ApplyComparisonOp(GT(self.typ, other.typ), self, other) - def <=(other: IR): IR = ApplyComparisonOp(LTEQ(self.typ, other.typ), self, other) - def >=(other: IR): IR = ApplyComparisonOp(GTEQ(self.typ, other.typ), self, other) -} + final case class ReadValue(path: IR, reader: ValueReader, requestedType: Type) extends IR + + final case class WriteValue( + value: IR, + path: IR, + writer: ValueWriter, + stagingFile: Option[IR] = None, + ) extends IR + + class PrimitiveIR(val self: IR) extends AnyVal { + def +(other: IR): IR = { + assert(self.typ == other.typ) + if (self.typ == TString) + invoke("concat", TString, self, other) + else + ApplyBinaryPrimOp(Add(), self, other) + } + + def -(other: IR): IR = ApplyBinaryPrimOp(Subtract(), self, other) + + def *(other: IR): IR = ApplyBinaryPrimOp(Multiply(), self, other) + + def /(other: IR): IR = ApplyBinaryPrimOp(FloatingPointDivide(), self, other) + + def floorDiv(other: IR): IR = ApplyBinaryPrimOp(RoundToNegInfDivide(), self, other) -object ErrorIDs { - val NO_ERROR = -1 + def &&(other: IR): IR = invoke("land", TBoolean, self, other) + + def ||(other: IR): IR = invoke("lor", TBoolean, self, other) + + def toI: IR = Cast(self, TInt32) + + def toL: IR = Cast(self, TInt64) + + def toF: IR = Cast(self, TFloat32) + + def toD: IR = Cast(self, TFloat64) + + def unary_-(): IR = ApplyUnaryPrimOp(Negate, self) + + def unary_!(): IR = ApplyUnaryPrimOp(Bang, self) + + def ceq(other: IR): IR = ApplyComparisonOp(EQWithNA(self.typ, other.typ), self, other) + + def cne(other: IR): IR = ApplyComparisonOp(NEQWithNA(self.typ, other.typ), self, other) + + def <(other: IR): IR = ApplyComparisonOp(LT(self.typ, other.typ), self, other) + + def >(other: IR): IR = ApplyComparisonOp(GT(self.typ, other.typ), self, other) + + def <=(other: IR): IR = ApplyComparisonOp(LTEQ(self.typ, other.typ), self, other) + + def >=(other: IR): IR = ApplyComparisonOp(GTEQ(self.typ, other.typ), self, other) + } + + object ErrorIDs { + val NO_ERROR = -1 + } } diff --git a/hail/hail/src/is/hail/expr/ir/IRBuilder.scala b/hail/hail/src/is/hail/expr/ir/IRBuilder.scala index 48aa360d782..0ffc57dbd48 100644 --- a/hail/hail/src/is/hail/expr/ir/IRBuilder.scala +++ b/hail/hail/src/is/hail/expr/ir/IRBuilder.scala @@ -1,5 +1,6 @@ package is.hail.expr.ir +import is.hail.expr.ir.defs.{Let, Ref, TrivialIR} import is.hail.utils.BoxedArrayBuilder object IRBuilder { diff --git a/hail/hail/src/is/hail/expr/ir/InTailPosition.scala b/hail/hail/src/is/hail/expr/ir/InTailPosition.scala index e5165c5476d..2e8e24a8a05 100644 --- a/hail/hail/src/is/hail/expr/ir/InTailPosition.scala +++ b/hail/hail/src/is/hail/expr/ir/InTailPosition.scala @@ -1,5 +1,7 @@ package is.hail.expr.ir +import is.hail.expr.ir.defs.{Block, If, Switch, TailLoop} + object InTailPosition { def apply(x: IR, i: Int): Boolean = x match { case Block(bindings, _) => i == bindings.length diff --git a/hail/hail/src/is/hail/expr/ir/InferType.scala b/hail/hail/src/is/hail/expr/ir/InferType.scala index 8b49e14c258..25882b3d2e0 100644 --- a/hail/hail/src/is/hail/expr/ir/InferType.scala +++ b/hail/hail/src/is/hail/expr/ir/InferType.scala @@ -1,6 +1,7 @@ package is.hail.expr.ir import is.hail.expr.Nat +import is.hail.expr.ir.defs._ import is.hail.types.tcoerce import is.hail.types.virtual._ import is.hail.utils._ diff --git a/hail/hail/src/is/hail/expr/ir/Interpret.scala b/hail/hail/src/is/hail/expr/ir/Interpret.scala index e7628c600d8..1c0328804b2 100644 --- a/hail/hail/src/is/hail/expr/ir/Interpret.scala +++ b/hail/hail/src/is/hail/expr/ir/Interpret.scala @@ -4,6 +4,7 @@ import is.hail.annotations._ import is.hail.asm4s._ import is.hail.backend.{ExecuteContext, HailTaskContext} import is.hail.backend.spark.SparkTaskContext +import is.hail.expr.ir.defs._ import is.hail.expr.ir.lowering.LoweringPipeline import is.hail.io.BufferSpec import is.hail.linalg.BlockMatrix diff --git a/hail/hail/src/is/hail/expr/ir/Interpretable.scala b/hail/hail/src/is/hail/expr/ir/Interpretable.scala index af5903f04e7..4026a7d4fd2 100644 --- a/hail/hail/src/is/hail/expr/ir/Interpretable.scala +++ b/hail/hail/src/is/hail/expr/ir/Interpretable.scala @@ -1,5 +1,6 @@ package is.hail.expr.ir +import is.hail.expr.ir.defs._ import is.hail.types.virtual.TNDArray object Interpretable { diff --git a/hail/hail/src/is/hail/expr/ir/IsConstant.scala b/hail/hail/src/is/hail/expr/ir/IsConstant.scala index 4bbe2023817..846b824bea8 100644 --- a/hail/hail/src/is/hail/expr/ir/IsConstant.scala +++ b/hail/hail/src/is/hail/expr/ir/IsConstant.scala @@ -1,5 +1,6 @@ package is.hail.expr.ir +import is.hail.expr.ir.defs.{EncodedLiteral, F32, F64, False, I32, I64, Literal, NA, Str, True} import is.hail.types.virtual._ object CanEmit { diff --git a/hail/hail/src/is/hail/expr/ir/IsPure.scala b/hail/hail/src/is/hail/expr/ir/IsPure.scala index b4295d2b9ef..128949935dc 100644 --- a/hail/hail/src/is/hail/expr/ir/IsPure.scala +++ b/hail/hail/src/is/hail/expr/ir/IsPure.scala @@ -1,5 +1,6 @@ package is.hail.expr.ir +import is.hail.expr.ir.defs.{WritePartition, WriteValue} import is.hail.types.virtual.TVoid object IsPure { diff --git a/hail/hail/src/is/hail/expr/ir/LiftRelationalValues.scala b/hail/hail/src/is/hail/expr/ir/LiftRelationalValues.scala index a7d599ef774..601196fbabf 100644 --- a/hail/hail/src/is/hail/expr/ir/LiftRelationalValues.scala +++ b/hail/hail/src/is/hail/expr/ir/LiftRelationalValues.scala @@ -1,5 +1,9 @@ package is.hail.expr.ir +import is.hail.expr.ir.defs.{ + BlockMatrixCollect, BlockMatrixToValueApply, LiftMeOut, RelationalLet, RelationalRef, + TableAggregate, TableCollect, TableCount, TableGetGlobals, TableToValueApply, +} import is.hail.types.virtual.TVoid import is.hail.utils.BoxedArrayBuilder diff --git a/hail/hail/src/is/hail/expr/ir/LowerMatrixIR.scala b/hail/hail/src/is/hail/expr/ir/LowerMatrixIR.scala index 3bcd3c895c6..400fecafe09 100644 --- a/hail/hail/src/is/hail/expr/ir/LowerMatrixIR.scala +++ b/hail/hail/src/is/hail/expr/ir/LowerMatrixIR.scala @@ -1,6 +1,7 @@ package is.hail.expr.ir import is.hail.backend.ExecuteContext +import is.hail.expr.ir.defs._ import is.hail.expr.ir.functions.{WrappedMatrixToTableFunction, WrappedMatrixToValueFunction} import is.hail.types.virtual._ import is.hail.utils._ diff --git a/hail/hail/src/is/hail/expr/ir/LowerOrInterpretNonCompilable.scala b/hail/hail/src/is/hail/expr/ir/LowerOrInterpretNonCompilable.scala index a23298ceecf..168508f0b5d 100644 --- a/hail/hail/src/is/hail/expr/ir/LowerOrInterpretNonCompilable.scala +++ b/hail/hail/src/is/hail/expr/ir/LowerOrInterpretNonCompilable.scala @@ -1,6 +1,7 @@ package is.hail.expr.ir import is.hail.backend.ExecuteContext +import is.hail.expr.ir.defs.{Begin, Literal, RelationalLet, RelationalRef} import is.hail.expr.ir.lowering.{CanLowerEfficiently, DArrayLowering, LowerToDistributedArrayPass} import is.hail.types.virtual.TVoid import is.hail.utils._ diff --git a/hail/hail/src/is/hail/expr/ir/MapIR.scala b/hail/hail/src/is/hail/expr/ir/MapIR.scala index 92f6992b07c..7ab967fef3a 100644 --- a/hail/hail/src/is/hail/expr/ir/MapIR.scala +++ b/hail/hail/src/is/hail/expr/ir/MapIR.scala @@ -1,5 +1,6 @@ package is.hail.expr.ir +import is.hail.expr.ir.defs.{MatrixAggregate, TableAggregate} import is.hail.types.virtual.Type object MapIR { diff --git a/hail/hail/src/is/hail/expr/ir/MatrixIR.scala b/hail/hail/src/is/hail/expr/ir/MatrixIR.scala index 1b8f03ef08d..026b8e98cf8 100644 --- a/hail/hail/src/is/hail/expr/ir/MatrixIR.scala +++ b/hail/hail/src/is/hail/expr/ir/MatrixIR.scala @@ -4,6 +4,7 @@ import is.hail.HailContext import is.hail.annotations._ import is.hail.backend.ExecuteContext import is.hail.expr.ir.DeprecatedIRBuilder._ +import is.hail.expr.ir.defs._ import is.hail.expr.ir.functions.MatrixToMatrixFunction import is.hail.io.bgen.MatrixBGENReader import is.hail.io.fs.FS diff --git a/hail/hail/src/is/hail/expr/ir/MatrixWriter.scala b/hail/hail/src/is/hail/expr/ir/MatrixWriter.scala index ed2d09b136e..812e6175216 100644 --- a/hail/hail/src/is/hail/expr/ir/MatrixWriter.scala +++ b/hail/hail/src/is/hail/expr/ir/MatrixWriter.scala @@ -4,6 +4,7 @@ import is.hail.annotations.Region import is.hail.asm4s._ import is.hail.backend.ExecuteContext import is.hail.expr.{JSONAnnotationImpex, Nat} +import is.hail.expr.ir.defs._ import is.hail.expr.ir.lowering.TableStage import is.hail.expr.ir.streams.StreamProducer import is.hail.io._ diff --git a/hail/hail/src/is/hail/expr/ir/NestingDepth.scala b/hail/hail/src/is/hail/expr/ir/NestingDepth.scala index 59a7b298b61..911907161ae 100644 --- a/hail/hail/src/is/hail/expr/ir/NestingDepth.scala +++ b/hail/hail/src/is/hail/expr/ir/NestingDepth.scala @@ -1,5 +1,7 @@ package is.hail.expr.ir +import is.hail.expr.ir.defs._ + case class ScopedDepth(eval: Int, agg: Int, scan: Int) { def incrementEval: ScopedDepth = ScopedDepth(eval + 1, agg, scan) diff --git a/hail/hail/src/is/hail/expr/ir/NormalizeNames.scala b/hail/hail/src/is/hail/expr/ir/NormalizeNames.scala index 93e456b8891..10a2f36b928 100644 --- a/hail/hail/src/is/hail/expr/ir/NormalizeNames.scala +++ b/hail/hail/src/is/hail/expr/ir/NormalizeNames.scala @@ -2,6 +2,7 @@ package is.hail.expr.ir import is.hail.backend.ExecuteContext import is.hail.expr.ir.NormalizeNames.needsRenaming +import is.hail.expr.ir.defs._ import is.hail.types.virtual.Type import is.hail.utils.StackSafe._ diff --git a/hail/hail/src/is/hail/expr/ir/Parser.scala b/hail/hail/src/is/hail/expr/ir/Parser.scala index f7698a025d9..4c4a5203055 100644 --- a/hail/hail/src/is/hail/expr/ir/Parser.scala +++ b/hail/hail/src/is/hail/expr/ir/Parser.scala @@ -4,6 +4,7 @@ import is.hail.HailContext import is.hail.backend.ExecuteContext import is.hail.expr.{JSONAnnotationImpex, Nat, ParserUtils} import is.hail.expr.ir.agg._ +import is.hail.expr.ir.defs._ import is.hail.expr.ir.functions.RelationalFunctions import is.hail.io.{BufferSpec, TypedCodecSpec} import is.hail.rvd.{RVDPartitioner, RVDType} diff --git a/hail/hail/src/is/hail/expr/ir/Pretty.scala b/hail/hail/src/is/hail/expr/ir/Pretty.scala index 9baeabd9d6c..727546c7d1b 100644 --- a/hail/hail/src/is/hail/expr/ir/Pretty.scala +++ b/hail/hail/src/is/hail/expr/ir/Pretty.scala @@ -4,6 +4,7 @@ import is.hail.backend.ExecuteContext import is.hail.expr.JSONAnnotationImpex import is.hail.expr.ir.Pretty.prettyBooleanLiteral import is.hail.expr.ir.agg._ +import is.hail.expr.ir.defs._ import is.hail.expr.ir.functions.RelationalFunctions import is.hail.types.virtual.{MatrixType, TArray, TInterval, TStream, TableType, Type} import is.hail.utils.{space => _, _} diff --git a/hail/hail/src/is/hail/expr/ir/PruneDeadFields.scala b/hail/hail/src/is/hail/expr/ir/PruneDeadFields.scala index 319717cc689..d3d68e2caf1 100644 --- a/hail/hail/src/is/hail/expr/ir/PruneDeadFields.scala +++ b/hail/hail/src/is/hail/expr/ir/PruneDeadFields.scala @@ -3,6 +3,7 @@ package is.hail.expr.ir import is.hail.annotations._ import is.hail.backend.ExecuteContext import is.hail.expr.Nat +import is.hail.expr.ir.defs._ import is.hail.types._ import is.hail.types.virtual._ import is.hail.types.virtual.TIterable.elementType diff --git a/hail/hail/src/is/hail/expr/ir/Requiredness.scala b/hail/hail/src/is/hail/expr/ir/Requiredness.scala index 2a3e5ab3f31..36c8f1b590c 100644 --- a/hail/hail/src/is/hail/expr/ir/Requiredness.scala +++ b/hail/hail/src/is/hail/expr/ir/Requiredness.scala @@ -1,6 +1,7 @@ package is.hail.expr.ir import is.hail.backend.ExecuteContext +import is.hail.expr.ir.defs._ import is.hail.expr.ir.functions.GetElement import is.hail.methods.ForceCountTable import is.hail.types._ diff --git a/hail/hail/src/is/hail/expr/ir/Scope.scala b/hail/hail/src/is/hail/expr/ir/Scope.scala index 65cfb65ccb4..5144057f7b8 100644 --- a/hail/hail/src/is/hail/expr/ir/Scope.scala +++ b/hail/hail/src/is/hail/expr/ir/Scope.scala @@ -1,5 +1,10 @@ package is.hail.expr.ir +import is.hail.expr.ir.defs.{ + AggArrayPerElement, AggExplode, AggFilter, AggFold, AggGroupBy, ApplyAggOp, ApplyScanOp, Block, + MatrixAggregate, RelationalLet, TableAggregate, +} + object UsesAggEnv { def apply(ir0: BaseIR, i: Int): Boolean = ir0 match { case Block(bindings, _) if i < bindings.length => diff --git a/hail/hail/src/is/hail/expr/ir/Simplify.scala b/hail/hail/src/is/hail/expr/ir/Simplify.scala index 9d46b76be42..5144e1daede 100644 --- a/hail/hail/src/is/hail/expr/ir/Simplify.scala +++ b/hail/hail/src/is/hail/expr/ir/Simplify.scala @@ -1,6 +1,7 @@ package is.hail.expr.ir import is.hail.backend.ExecuteContext +import is.hail.expr.ir.defs._ import is.hail.io.bgen.MatrixBGENReader import is.hail.rvd.PartitionBoundOrdering import is.hail.types.tcoerce diff --git a/hail/hail/src/is/hail/expr/ir/StringTableReader.scala b/hail/hail/src/is/hail/expr/ir/StringTableReader.scala index cb2c9108285..65239f25d69 100644 --- a/hail/hail/src/is/hail/expr/ir/StringTableReader.scala +++ b/hail/hail/src/is/hail/expr/ir/StringTableReader.scala @@ -3,6 +3,7 @@ package is.hail.expr.ir import is.hail.annotations.Region import is.hail.asm4s._ import is.hail.backend.ExecuteContext +import is.hail.expr.ir.defs.{Literal, MakeStruct, PartitionReader, ReadPartition, Ref, ToStream} import is.hail.expr.ir.functions.StringFunctions import is.hail.expr.ir.lowering.{LowererUnsupportedOperation, TableStage, TableStageDependency} import is.hail.expr.ir.streams.StreamProducer diff --git a/hail/hail/src/is/hail/expr/ir/Subst.scala b/hail/hail/src/is/hail/expr/ir/Subst.scala index 32f0fe8bc55..3a3a42a80f0 100644 --- a/hail/hail/src/is/hail/expr/ir/Subst.scala +++ b/hail/hail/src/is/hail/expr/ir/Subst.scala @@ -1,5 +1,7 @@ package is.hail.expr.ir +import is.hail.expr.ir.defs.Ref + object Subst { def apply(e: IR): IR = apply(e, BindingEnv.empty[IR]) diff --git a/hail/hail/src/is/hail/expr/ir/TableIR.scala b/hail/hail/src/is/hail/expr/ir/TableIR.scala index cbd2b2bc254..27404c71c01 100644 --- a/hail/hail/src/is/hail/expr/ir/TableIR.scala +++ b/hail/hail/src/is/hail/expr/ir/TableIR.scala @@ -6,6 +6,7 @@ import is.hail.asm4s._ import is.hail.backend.{ExecuteContext, HailStateManager, HailTaskContext, TaskFinalizer} import is.hail.backend.spark.{SparkBackend, SparkTaskContext} import is.hail.expr.ir +import is.hail.expr.ir.defs._ import is.hail.expr.ir.functions.{ BlockMatrixToTableFunction, IntervalFunctions, MatrixToTableFunction, TableToTableFunction, } diff --git a/hail/hail/src/is/hail/expr/ir/TableWriter.scala b/hail/hail/src/is/hail/expr/ir/TableWriter.scala index 4021f6d646f..61a49390697 100644 --- a/hail/hail/src/is/hail/expr/ir/TableWriter.scala +++ b/hail/hail/src/is/hail/expr/ir/TableWriter.scala @@ -5,6 +5,7 @@ import is.hail.annotations.Region import is.hail.asm4s._ import is.hail.backend.ExecuteContext import is.hail.expr.TableAnnotationImpex +import is.hail.expr.ir.defs._ import is.hail.expr.ir.functions.StringFunctions import is.hail.expr.ir.lowering.{LowererUnsupportedOperation, TableStage} import is.hail.expr.ir.streams.StreamProducer diff --git a/hail/hail/src/is/hail/expr/ir/TypeCheck.scala b/hail/hail/src/is/hail/expr/ir/TypeCheck.scala index aa5b591f01e..98bb166b171 100644 --- a/hail/hail/src/is/hail/expr/ir/TypeCheck.scala +++ b/hail/hail/src/is/hail/expr/ir/TypeCheck.scala @@ -2,6 +2,7 @@ package is.hail.expr.ir import is.hail.backend.ExecuteContext import is.hail.expr.Nat +import is.hail.expr.ir.defs._ import is.hail.types.tcoerce import is.hail.types.virtual._ import is.hail.utils._ diff --git a/hail/hail/src/is/hail/expr/ir/agg/Extract.scala b/hail/hail/src/is/hail/expr/ir/agg/Extract.scala index 4b4d5c3ac4e..a9807b283dc 100644 --- a/hail/hail/src/is/hail/expr/ir/agg/Extract.scala +++ b/hail/hail/src/is/hail/expr/ir/agg/Extract.scala @@ -6,6 +6,7 @@ import is.hail.backend.{ExecuteContext, HailTaskContext} import is.hail.backend.spark.SparkTaskContext import is.hail.expr.ir import is.hail.expr.ir._ +import is.hail.expr.ir.defs._ import is.hail.io.BufferSpec import is.hail.types.{TypeWithRequiredness, VirtualTypeWithReq} import is.hail.types.physical.stypes.EmitType @@ -253,7 +254,7 @@ class Aggs( FastSeq(), FastSeq(classInfo[Region]), UnitInfo, - ir.DeserializeAggs(0, 0, spec, states), + DeserializeAggs(0, 0, spec, states), ) val fsBc = ctx.fsBc; @@ -274,7 +275,7 @@ class Aggs( FastSeq(), FastSeq(classInfo[Region]), UnitInfo, - ir.SerializeAggs(0, 0, spec, states), + SerializeAggs(0, 0, spec, states), ) val fsBc = ctx.fsBc; @@ -312,8 +313,8 @@ class Aggs( FastSeq(classInfo[Region]), UnitInfo, Begin(FastSeq( - ir.DeserializeAggs(0, 0, spec, states), - ir.DeserializeAggs(nAggs, 1, spec, states), + DeserializeAggs(0, 0, spec, states), + DeserializeAggs(nAggs, 1, spec, states), Begin(aggs.zipWithIndex.map { case (sig, i) => CombOp(i, i + nAggs, sig) }), SerializeAggs(0, 0, spec, states), )), diff --git a/hail/hail/src/is/hail/expr/ir/analyses/ComputeMethodSplits.scala b/hail/hail/src/is/hail/expr/ir/analyses/ComputeMethodSplits.scala index 223824354a8..ff64d0cd015 100644 --- a/hail/hail/src/is/hail/expr/ir/analyses/ComputeMethodSplits.scala +++ b/hail/hail/src/is/hail/expr/ir/analyses/ComputeMethodSplits.scala @@ -2,6 +2,9 @@ package is.hail.expr.ir.analyses import is.hail.backend.ExecuteContext import is.hail.expr.ir._ +import is.hail.expr.ir.defs.{ + In, Ref, StreamFold, StreamFold2, StreamFor, StreamLen, TailLoop, ToArray, ToDict, ToSet, +} object ComputeMethodSplits { def apply(ctx: ExecuteContext, ir: IR, controlFlowPreventsSplit: Memo[Unit]): Memo[Unit] = { diff --git a/hail/hail/src/is/hail/expr/ir/analyses/ControlFlowPreventsSplit.scala b/hail/hail/src/is/hail/expr/ir/analyses/ControlFlowPreventsSplit.scala index 6d06a4191e2..540d0c98864 100644 --- a/hail/hail/src/is/hail/expr/ir/analyses/ControlFlowPreventsSplit.scala +++ b/hail/hail/src/is/hail/expr/ir/analyses/ControlFlowPreventsSplit.scala @@ -1,6 +1,7 @@ package is.hail.expr.ir.analyses -import is.hail.expr.ir.{BaseIR, Memo, Recur, Ref, TailLoop, UsesAndDefs, VisitIR} +import is.hail.expr.ir.{BaseIR, Memo, UsesAndDefs, VisitIR} +import is.hail.expr.ir.defs.{Recur, Ref, TailLoop} import is.hail.types.virtual.TStream object ControlFlowPreventsSplit { diff --git a/hail/hail/src/is/hail/expr/ir/analyses/SemanticHash.scala b/hail/hail/src/is/hail/expr/ir/analyses/SemanticHash.scala index 2d6679d5d83..1f070c092c1 100644 --- a/hail/hail/src/is/hail/expr/ir/analyses/SemanticHash.scala +++ b/hail/hail/src/is/hail/expr/ir/analyses/SemanticHash.scala @@ -2,6 +2,7 @@ package is.hail.expr.ir.analyses import is.hail.backend.ExecuteContext import is.hail.expr.ir.{MatrixRangeReader, _} +import is.hail.expr.ir.defs._ import is.hail.expr.ir.functions.{TableCalculateNewPartitions, WrappedMatrixToValueFunction} import is.hail.io.fs.FS import is.hail.io.vcf.MatrixVCFReader diff --git a/hail/hail/src/is/hail/expr/ir/functions/ArrayFunctions.scala b/hail/hail/src/is/hail/expr/ir/functions/ArrayFunctions.scala index 072159a8def..582e238668b 100644 --- a/hail/hail/src/is/hail/expr/ir/functions/ArrayFunctions.scala +++ b/hail/hail/src/is/hail/expr/ir/functions/ArrayFunctions.scala @@ -2,6 +2,7 @@ package is.hail.expr.ir.functions import is.hail.asm4s._ import is.hail.expr.ir._ +import is.hail.expr.ir.defs._ import is.hail.expr.ir.orderings.CodeOrdering import is.hail.types.physical.{PCanonicalArray, PType} import is.hail.types.physical.stypes.EmitType diff --git a/hail/hail/src/is/hail/expr/ir/functions/DictFunctions.scala b/hail/hail/src/is/hail/expr/ir/functions/DictFunctions.scala index beb536f5570..12c9f2e0c19 100644 --- a/hail/hail/src/is/hail/expr/ir/functions/DictFunctions.scala +++ b/hail/hail/src/is/hail/expr/ir/functions/DictFunctions.scala @@ -1,6 +1,7 @@ package is.hail.expr.ir.functions import is.hail.expr.ir._ +import is.hail.expr.ir.defs._ import is.hail.types import is.hail.types.virtual._ import is.hail.utils.FastSeq diff --git a/hail/hail/src/is/hail/expr/ir/functions/Functions.scala b/hail/hail/src/is/hail/expr/ir/functions/Functions.scala index 2d3cf1f6e99..d171fd351e2 100644 --- a/hail/hail/src/is/hail/expr/ir/functions/Functions.scala +++ b/hail/hail/src/is/hail/expr/ir/functions/Functions.scala @@ -5,6 +5,7 @@ import is.hail.asm4s._ import is.hail.backend.{ExecuteContext, HailStateManager} import is.hail.experimental.ExperimentalFunctions import is.hail.expr.ir._ +import is.hail.expr.ir.defs.{Apply, ApplyIR, ApplySeeded, ApplySpecial} import is.hail.io.bgen.BGENFunctions import is.hail.types.physical._ import is.hail.types.physical.stypes.{EmitType, SType, SValue} diff --git a/hail/hail/src/is/hail/expr/ir/functions/MathFunctions.scala b/hail/hail/src/is/hail/expr/ir/functions/MathFunctions.scala index bacc2c50ac4..4aef2937dd7 100644 --- a/hail/hail/src/is/hail/expr/ir/functions/MathFunctions.scala +++ b/hail/hail/src/is/hail/expr/ir/functions/MathFunctions.scala @@ -1,7 +1,8 @@ package is.hail.expr.ir.functions import is.hail.asm4s.Code -import is.hail.expr.ir._ +import is.hail.expr.ir.EmitValue +import is.hail.expr.ir.defs.{Cast, ErrorIDs} import is.hail.stats._ import is.hail.types.physical.stypes.concrete._ import is.hail.types.physical.stypes.interfaces.primitive diff --git a/hail/hail/src/is/hail/expr/ir/functions/NDArrayFunctions.scala b/hail/hail/src/is/hail/expr/ir/functions/NDArrayFunctions.scala index e2354a2c491..46a159154ef 100644 --- a/hail/hail/src/is/hail/expr/ir/functions/NDArrayFunctions.scala +++ b/hail/hail/src/is/hail/expr/ir/functions/NDArrayFunctions.scala @@ -3,7 +3,8 @@ package is.hail.expr.ir.functions import is.hail.annotations.{Memory, Region} import is.hail.asm4s._ import is.hail.expr.{Nat, NatVariable} -import is.hail.expr.ir._ +import is.hail.expr.ir.{freshName, EmitCode, EmitCodeBuilder} +import is.hail.expr.ir.defs.{NDArrayMap, NDArrayMap2, Ref} import is.hail.linalg.{LAPACK, LinalgCodeUtils} import is.hail.types.physical._ import is.hail.types.physical.stypes.EmitType diff --git a/hail/hail/src/is/hail/expr/ir/functions/ReferenceGenomeFunctions.scala b/hail/hail/src/is/hail/expr/ir/functions/ReferenceGenomeFunctions.scala index 18955c9c346..930b90a6db3 100644 --- a/hail/hail/src/is/hail/expr/ir/functions/ReferenceGenomeFunctions.scala +++ b/hail/hail/src/is/hail/expr/ir/functions/ReferenceGenomeFunctions.scala @@ -2,6 +2,7 @@ package is.hail.expr.ir.functions import is.hail.asm4s._ import is.hail.expr.ir._ +import is.hail.expr.ir.defs.{ErrorIDs, If, NA} import is.hail.types.physical.stypes.SType import is.hail.types.physical.stypes.concrete.SJavaString import is.hail.types.physical.stypes.interfaces._ diff --git a/hail/hail/src/is/hail/expr/ir/functions/SetFunctions.scala b/hail/hail/src/is/hail/expr/ir/functions/SetFunctions.scala index 28c6cd24cfb..6c60577ce4f 100644 --- a/hail/hail/src/is/hail/expr/ir/functions/SetFunctions.scala +++ b/hail/hail/src/is/hail/expr/ir/functions/SetFunctions.scala @@ -1,6 +1,7 @@ package is.hail.expr.ir.functions import is.hail.expr.ir._ +import is.hail.expr.ir.defs._ import is.hail.types.virtual._ import is.hail.utils.FastSeq diff --git a/hail/hail/src/is/hail/expr/ir/functions/StringFunctions.scala b/hail/hail/src/is/hail/expr/ir/functions/StringFunctions.scala index b913ad1764f..602401afc88 100644 --- a/hail/hail/src/is/hail/expr/ir/functions/StringFunctions.scala +++ b/hail/hail/src/is/hail/expr/ir/functions/StringFunctions.scala @@ -4,6 +4,7 @@ import is.hail.annotations.Region import is.hail.asm4s._ import is.hail.expr.JSONAnnotationImpex import is.hail.expr.ir._ +import is.hail.expr.ir.defs._ import is.hail.types.physical.stypes._ import is.hail.types.physical.stypes.concrete.{ SJavaArrayString, SJavaArrayStringSettable, SJavaArrayStringValue, SJavaString, diff --git a/hail/hail/src/is/hail/expr/ir/functions/UtilFunctions.scala b/hail/hail/src/is/hail/expr/ir/functions/UtilFunctions.scala index a1508903394..772d14ef58c 100644 --- a/hail/hail/src/is/hail/expr/ir/functions/UtilFunctions.scala +++ b/hail/hail/src/is/hail/expr/ir/functions/UtilFunctions.scala @@ -4,6 +4,7 @@ import is.hail.annotations.Region import is.hail.asm4s.{coerce => _, _} import is.hail.backend.HailStateManager import is.hail.expr.ir._ +import is.hail.expr.ir.defs._ import is.hail.io.fs.FS import is.hail.io.vcf.{LoadVCF, VCFHeaderInfo} import is.hail.types.physical.stypes._ diff --git a/hail/hail/src/is/hail/expr/ir/lowering/CanLowerEfficiently.scala b/hail/hail/src/is/hail/expr/ir/lowering/CanLowerEfficiently.scala index f2755d5d7aa..c24cf943453 100644 --- a/hail/hail/src/is/hail/expr/ir/lowering/CanLowerEfficiently.scala +++ b/hail/hail/src/is/hail/expr/ir/lowering/CanLowerEfficiently.scala @@ -2,6 +2,7 @@ package is.hail.expr.ir.lowering import is.hail.backend.ExecuteContext import is.hail.expr.ir._ +import is.hail.expr.ir.defs._ import is.hail.expr.ir.functions.{ TableCalculateNewPartitions, TableToValueFunction, WrappedMatrixToTableFunction, } diff --git a/hail/hail/src/is/hail/expr/ir/lowering/EvalRelationalLets.scala b/hail/hail/src/is/hail/expr/ir/lowering/EvalRelationalLets.scala index 86d7f8b5e8a..b8b35ace2fe 100644 --- a/hail/hail/src/is/hail/expr/ir/lowering/EvalRelationalLets.scala +++ b/hail/hail/src/is/hail/expr/ir/lowering/EvalRelationalLets.scala @@ -2,9 +2,9 @@ package is.hail.expr.ir.lowering import is.hail.backend.ExecuteContext import is.hail.expr.ir.{ - BaseIR, CompileAndEvaluate, IR, Name, RelationalLet, RelationalLetMatrixTable, RelationalLetTable, - RelationalRef, + BaseIR, CompileAndEvaluate, IR, Name, RelationalLetMatrixTable, RelationalLetTable, } +import is.hail.expr.ir.defs.{RelationalLet, RelationalRef} object EvalRelationalLets { // need to run the rest of lowerings to eval. diff --git a/hail/hail/src/is/hail/expr/ir/lowering/IRState.scala b/hail/hail/src/is/hail/expr/ir/lowering/IRState.scala index f0f55751a4a..0af3acb3baa 100644 --- a/hail/hail/src/is/hail/expr/ir/lowering/IRState.scala +++ b/hail/hail/src/is/hail/expr/ir/lowering/IRState.scala @@ -1,8 +1,7 @@ package is.hail.expr.ir.lowering -import is.hail.expr.ir.{ - BaseIR, RelationalLet, RelationalRef, TableKeyBy, TableKeyByAndAggregate, TableOrderBy, -} +import is.hail.expr.ir.{BaseIR, TableKeyBy, TableKeyByAndAggregate, TableOrderBy} +import is.hail.expr.ir.defs.{RelationalLet, RelationalRef} trait IRState { diff --git a/hail/hail/src/is/hail/expr/ir/lowering/LowerAndExecuteShuffles.scala b/hail/hail/src/is/hail/expr/ir/lowering/LowerAndExecuteShuffles.scala index 92593364df2..777a0c01937 100644 --- a/hail/hail/src/is/hail/expr/ir/lowering/LowerAndExecuteShuffles.scala +++ b/hail/hail/src/is/hail/expr/ir/lowering/LowerAndExecuteShuffles.scala @@ -1,8 +1,9 @@ package is.hail.expr.ir.lowering import is.hail.backend.ExecuteContext -import is.hail.expr.ir.{Ref, Requiredness, _} +import is.hail.expr.ir.{Requiredness, _} import is.hail.expr.ir.agg.{Extract, PhysicalAggSig, TakeStateSig} +import is.hail.expr.ir.defs._ import is.hail.types._ import is.hail.types.virtual._ import is.hail.utils.FastSeq diff --git a/hail/hail/src/is/hail/expr/ir/lowering/LowerBlockMatrixIR.scala b/hail/hail/src/is/hail/expr/ir/lowering/LowerBlockMatrixIR.scala index 84f1a81d1b1..e63630e4c64 100644 --- a/hail/hail/src/is/hail/expr/ir/lowering/LowerBlockMatrixIR.scala +++ b/hail/hail/src/is/hail/expr/ir/lowering/LowerBlockMatrixIR.scala @@ -3,6 +3,7 @@ package is.hail.expr.ir.lowering import is.hail.backend.ExecuteContext import is.hail.expr.Nat import is.hail.expr.ir._ +import is.hail.expr.ir.defs._ import is.hail.expr.ir.functions.GetElement import is.hail.rvd.RVDPartitioner import is.hail.types.{tcoerce, TypeWithRequiredness} diff --git a/hail/hail/src/is/hail/expr/ir/lowering/LowerDistributedSort.scala b/hail/hail/src/is/hail/expr/ir/lowering/LowerDistributedSort.scala index 859dfcaa5ba..68e6c28651f 100644 --- a/hail/hail/src/is/hail/expr/ir/lowering/LowerDistributedSort.scala +++ b/hail/hail/src/is/hail/expr/ir/lowering/LowerDistributedSort.scala @@ -4,6 +4,7 @@ import is.hail.annotations.{Annotation, ExtendedOrdering, Region, SafeRow} import is.hail.asm4s.{classInfo, AsmFunction1RegionLong, LongInfo} import is.hail.backend.{ExecuteContext, HailStateManager} import is.hail.expr.ir._ +import is.hail.expr.ir.defs._ import is.hail.expr.ir.functions.{ArrayFunctions, IRRandomness, UtilFunctions} import is.hail.io.{BufferSpec, TypedCodecSpec} import is.hail.rvd.RVDPartitioner diff --git a/hail/hail/src/is/hail/expr/ir/lowering/LowerTableIR.scala b/hail/hail/src/is/hail/expr/ir/lowering/LowerTableIR.scala index 197fcdbc85d..39d49fc1e9a 100644 --- a/hail/hail/src/is/hail/expr/ir/lowering/LowerTableIR.scala +++ b/hail/hail/src/is/hail/expr/ir/lowering/LowerTableIR.scala @@ -3,7 +3,8 @@ package is.hail.expr.ir.lowering import is.hail.HailContext import is.hail.backend.ExecuteContext import is.hail.expr.ir.{agg, TableNativeWriter, _} -import is.hail.expr.ir.ArrayZipBehavior.AssertSameLength +import is.hail.expr.ir.defs._ +import is.hail.expr.ir.defs.ArrayZipBehavior.AssertSameLength import is.hail.expr.ir.functions.{TableCalculateNewPartitions, WrappedMatrixToTableFunction} import is.hail.io.{BufferSpec, TypedCodecSpec} import is.hail.methods.{ForceCountTable, LocalLDPrune, NPartitionsTable, TableFilterPartitions} diff --git a/hail/hail/src/is/hail/expr/ir/lowering/LowerTableIRHelpers.scala b/hail/hail/src/is/hail/expr/ir/lowering/LowerTableIRHelpers.scala index dd9298d119c..83428820fe8 100644 --- a/hail/hail/src/is/hail/expr/ir/lowering/LowerTableIRHelpers.scala +++ b/hail/hail/src/is/hail/expr/ir/lowering/LowerTableIRHelpers.scala @@ -2,6 +2,7 @@ package is.hail.expr.ir.lowering import is.hail.backend.ExecuteContext import is.hail.expr.ir._ +import is.hail.expr.ir.defs._ import is.hail.types.RTable import is.hail.types.virtual.TStruct import is.hail.utils.FastSeq diff --git a/hail/hail/src/is/hail/expr/ir/lowering/LoweringPass.scala b/hail/hail/src/is/hail/expr/ir/lowering/LoweringPass.scala index f821e58da24..8277866e08a 100644 --- a/hail/hail/src/is/hail/expr/ir/lowering/LoweringPass.scala +++ b/hail/hail/src/is/hail/expr/ir/lowering/LoweringPass.scala @@ -4,6 +4,9 @@ import is.hail.backend.ExecuteContext import is.hail.expr.ir._ import is.hail.expr.ir.agg.Extract import is.hail.expr.ir.analyses.SemanticHash +import is.hail.expr.ir.defs.{ + ApplyIR, Begin, Let, RunAgg, RunAggScan, StreamAgg, StreamAggScan, StreamFor, +} import is.hail.utils._ final class IrMetadata() { diff --git a/hail/hail/src/is/hail/expr/ir/lowering/RVDToTableStage.scala b/hail/hail/src/is/hail/expr/ir/lowering/RVDToTableStage.scala index e3423bf9f75..d2cf0499a8a 100644 --- a/hail/hail/src/is/hail/expr/ir/lowering/RVDToTableStage.scala +++ b/hail/hail/src/is/hail/expr/ir/lowering/RVDToTableStage.scala @@ -5,6 +5,9 @@ import is.hail.asm4s._ import is.hail.backend.{BroadcastValue, ExecuteContext} import is.hail.backend.spark.{AnonymousDependency, SparkTaskContext} import is.hail.expr.ir._ +import is.hail.expr.ir.defs.{ + GetField, In, Let, MakeStruct, ReadPartition, Ref, StreamRange, ToArray, +} import is.hail.io.{BufferSpec, TypedCodecSpec} import is.hail.io.fs.FS import is.hail.rvd.{RVD, RVDType} diff --git a/hail/hail/src/is/hail/expr/ir/lowering/Rule.scala b/hail/hail/src/is/hail/expr/ir/lowering/Rule.scala index 6274eeefcd8..56656ff62b7 100644 --- a/hail/hail/src/is/hail/expr/ir/lowering/Rule.scala +++ b/hail/hail/src/is/hail/expr/ir/lowering/Rule.scala @@ -1,6 +1,10 @@ package is.hail.expr.ir.lowering -import is.hail.expr.ir._ +import is.hail.expr.ir.{ + BaseIR, BlockMatrixIR, Compilable, Emittable, IR, MatrixIR, RelationalLetBlockMatrix, + RelationalLetMatrixTable, RelationalLetTable, TableIR, +} +import is.hail.expr.ir.defs.{ApplyIR, RelationalLet, RelationalRef} trait Rule { def allows(ir: BaseIR): Boolean diff --git a/hail/hail/src/is/hail/expr/ir/ndarrays/EmitNDArray.scala b/hail/hail/src/is/hail/expr/ir/ndarrays/EmitNDArray.scala index 047b167a251..27780c6dad7 100644 --- a/hail/hail/src/is/hail/expr/ir/ndarrays/EmitNDArray.scala +++ b/hail/hail/src/is/hail/expr/ir/ndarrays/EmitNDArray.scala @@ -3,6 +3,10 @@ package is.hail.expr.ir.ndarrays import is.hail.annotations.Region import is.hail.asm4s._ import is.hail.expr.ir._ +import is.hail.expr.ir.defs.{ + NDArrayAgg, NDArrayConcat, NDArrayFilter, NDArrayMap, NDArrayMap2, NDArrayReindex, NDArrayReshape, + NDArraySlice, +} import is.hail.types.physical._ import is.hail.types.physical.stypes.{SType, SValue} import is.hail.types.physical.stypes.interfaces._ diff --git a/hail/hail/src/is/hail/expr/ir/package.scala b/hail/hail/src/is/hail/expr/ir/package.scala index bb292255c48..80879d85393 100644 --- a/hail/hail/src/is/hail/expr/ir/package.scala +++ b/hail/hail/src/is/hail/expr/ir/package.scala @@ -1,6 +1,7 @@ package is.hail.expr import is.hail.asm4s._ +import is.hail.expr.ir.defs._ import is.hail.expr.ir.functions.IRFunctionRegistry import is.hail.expr.ir.lowering.TableStageDependency import is.hail.rvd.RVDPartitioner @@ -37,9 +38,9 @@ package object ir { // Build consistent expression for a filter-condition with keep polarity, // using Let to manage missing-ness. - def filterPredicateWithKeep(irPred: ir.IR, keep: Boolean): ir.IR = - bindIR(if (keep) irPred else ir.ApplyUnaryPrimOp(ir.Bang, irPred)) { pred => - ir.If(ir.IsNA(pred), ir.False(), pred) + def filterPredicateWithKeep(irPred: IR, keep: Boolean): IR = + bindIR(if (keep) irPred else ApplyUnaryPrimOp(Bang, irPred)) { pred => + If(IsNA(pred), False(), pred) } def invoke(name: String, rt: Type, typeArgs: Seq[Type], errorID: Int, args: IR*): IR = diff --git a/hail/hail/src/is/hail/expr/ir/streams/EmitStream.scala b/hail/hail/src/is/hail/expr/ir/streams/EmitStream.scala index d8ff0f7cf6f..be3f971c52b 100644 --- a/hail/hail/src/is/hail/expr/ir/streams/EmitStream.scala +++ b/hail/hail/src/is/hail/expr/ir/streams/EmitStream.scala @@ -5,6 +5,7 @@ import is.hail.annotations.Region.REGULAR import is.hail.asm4s._ import is.hail.expr.ir._ import is.hail.expr.ir.agg.{AggStateSig, DictState, PhysicalAggSig, StateTuple} +import is.hail.expr.ir.defs._ import is.hail.expr.ir.functions.IntervalFunctions import is.hail.expr.ir.functions.IntervalFunctions.{ pointGTIntervalEndpoint, pointLTIntervalEndpoint, diff --git a/hail/hail/src/is/hail/expr/ir/streams/StreamUtils.scala b/hail/hail/src/is/hail/expr/ir/streams/StreamUtils.scala index 399674a8a97..80bf7dd0774 100644 --- a/hail/hail/src/is/hail/expr/ir/streams/StreamUtils.scala +++ b/hail/hail/src/is/hail/expr/ir/streams/StreamUtils.scala @@ -3,9 +3,11 @@ package is.hail.expr.ir.streams import is.hail.annotations.Region import is.hail.asm4s._ import is.hail.expr.ir.{ - EmitCode, EmitCodeBuilder, EmitMethodBuilder, IEmitCode, IR, NDArrayMap, NDArrayMap2, Name, Ref, - RunAggScan, StagedArrayBuilder, StreamFilter, StreamFlatMap, StreamFold, StreamFold2, StreamFor, - StreamJoinRightDistinct, StreamMap, StreamScan, StreamZip, StreamZipJoin, + EmitCode, EmitCodeBuilder, EmitMethodBuilder, IEmitCode, IR, Name, StagedArrayBuilder, +} +import is.hail.expr.ir.defs.{ + NDArrayMap, NDArrayMap2, Ref, RunAggScan, StreamFilter, StreamFlatMap, StreamFold, StreamFold2, + StreamFor, StreamJoinRightDistinct, StreamMap, StreamScan, StreamZip, StreamZipJoin, } import is.hail.expr.ir.orderings.StructOrdering import is.hail.types.physical.{PCanonicalArray, PCanonicalStruct} diff --git a/hail/hail/src/is/hail/io/avro/AvroPartitionReader.scala b/hail/hail/src/is/hail/io/avro/AvroPartitionReader.scala index ecd559df445..751f7b09650 100644 --- a/hail/hail/src/is/hail/io/avro/AvroPartitionReader.scala +++ b/hail/hail/src/is/hail/io/avro/AvroPartitionReader.scala @@ -3,9 +3,8 @@ package is.hail.io.avro import is.hail.annotations.Region import is.hail.asm4s.{Field => _, _} import is.hail.backend.ExecuteContext -import is.hail.expr.ir.{ - EmitCode, EmitCodeBuilder, EmitMethodBuilder, EmitValue, IEmitCode, PartitionReader, -} +import is.hail.expr.ir.{EmitCode, EmitCodeBuilder, EmitMethodBuilder, EmitValue, IEmitCode} +import is.hail.expr.ir.defs.PartitionReader import is.hail.expr.ir.streams.StreamProducer import is.hail.types.{RField, RStruct, TypeWithRequiredness} import is.hail.types.physical.{PCanonicalTuple, PInt64Required} diff --git a/hail/hail/src/is/hail/io/avro/AvroTableReader.scala b/hail/hail/src/is/hail/io/avro/AvroTableReader.scala index 6a57aadd1ea..f82b1dacc8f 100644 --- a/hail/hail/src/is/hail/io/avro/AvroTableReader.scala +++ b/hail/hail/src/is/hail/io/avro/AvroTableReader.scala @@ -2,6 +2,10 @@ package is.hail.io.avro import is.hail.backend.{ExecuteContext, HailStateManager} import is.hail.expr.ir._ +import is.hail.expr.ir.defs.{ + ArrayZipBehavior, Cast, I32, Literal, MakeStruct, PartitionReader, ReadPartition, StreamIota, + ToStream, +} import is.hail.expr.ir.lowering.{LowererUnsupportedOperation, TableStage, TableStageDependency} import is.hail.rvd.RVDPartitioner import is.hail.types.VirtualTypeWithReq diff --git a/hail/hail/src/is/hail/io/bgen/LoadBgen.scala b/hail/hail/src/is/hail/io/bgen/LoadBgen.scala index 7f756fa5aac..2d7ea609919 100644 --- a/hail/hail/src/is/hail/io/bgen/LoadBgen.scala +++ b/hail/hail/src/is/hail/io/bgen/LoadBgen.scala @@ -5,10 +5,10 @@ import is.hail.asm4s._ import is.hail.backend.ExecuteContext import is.hail.expr.ir.{ EmitCode, EmitCodeBuilder, EmitMethodBuilder, EmitSettable, EmitValue, IEmitCode, IR, - IRParserEnvironment, Literal, LowerMatrixIR, MakeStruct, MatrixHybridReader, MatrixReader, - PartitionNativeIntervalReader, PartitionReader, ReadPartition, Ref, TableNativeReader, - TableReader, ToStream, + IRParserEnvironment, LowerMatrixIR, MatrixHybridReader, MatrixReader, + PartitionNativeIntervalReader, TableNativeReader, TableReader, } +import is.hail.expr.ir.defs.{Literal, MakeStruct, PartitionReader, ReadPartition, Ref, ToStream} import is.hail.expr.ir.lowering.{TableStage, TableStageDependency} import is.hail.expr.ir.streams.StreamProducer import is.hail.io._ diff --git a/hail/hail/src/is/hail/io/plink/LoadPlink.scala b/hail/hail/src/is/hail/io/plink/LoadPlink.scala index 9c4cd0d5f43..721843cc199 100644 --- a/hail/hail/src/is/hail/io/plink/LoadPlink.scala +++ b/hail/hail/src/is/hail/io/plink/LoadPlink.scala @@ -5,6 +5,7 @@ import is.hail.asm4s.HailClassLoader import is.hail.backend.ExecuteContext import is.hail.expr.JSONAnnotationImpex import is.hail.expr.ir._ +import is.hail.expr.ir.defs.Literal import is.hail.expr.ir.lowering.TableStage import is.hail.io.fs.{FS, Seekable} import is.hail.io.vcf.LoadVCF diff --git a/hail/hail/src/is/hail/io/vcf/LoadVCF.scala b/hail/hail/src/is/hail/io/vcf/LoadVCF.scala index 0a17a1d9e9c..8cb2f464bd3 100644 --- a/hail/hail/src/is/hail/io/vcf/LoadVCF.scala +++ b/hail/hail/src/is/hail/io/vcf/LoadVCF.scala @@ -7,9 +7,9 @@ import is.hail.backend.spark.SparkBackend import is.hail.expr.JSONAnnotationImpex import is.hail.expr.ir.{ CloseableIterator, EmitCode, EmitCodeBuilder, EmitMethodBuilder, GenericLine, GenericLines, - GenericTableValue, IEmitCode, IR, IRParser, Literal, LowerMatrixIR, MatrixHybridReader, - MatrixReader, PartitionReader, + GenericTableValue, IEmitCode, IR, IRParser, LowerMatrixIR, MatrixHybridReader, MatrixReader, } +import is.hail.expr.ir.defs.{Literal, PartitionReader} import is.hail.expr.ir.lowering.TableStage import is.hail.expr.ir.streams.StreamProducer import is.hail.io.{VCFAttributes, VCFMetadata} diff --git a/hail/hail/src/is/hail/methods/LocalLDPrune.scala b/hail/hail/src/is/hail/methods/LocalLDPrune.scala index 10e89c0ed1f..12f87044d13 100644 --- a/hail/hail/src/is/hail/methods/LocalLDPrune.scala +++ b/hail/hail/src/is/hail/methods/LocalLDPrune.scala @@ -2,6 +2,7 @@ package is.hail.methods import is.hail.backend.ExecuteContext import is.hail.expr.ir._ +import is.hail.expr.ir.defs.{GetField, StreamLocalLDPrune, ToArray, ToStream} import is.hail.expr.ir.functions.MatrixToTableFunction import is.hail.methods.BitPackedVector._ import is.hail.types.virtual._ diff --git a/hail/hail/src/is/hail/rvd/AbstractRVDSpec.scala b/hail/hail/src/is/hail/rvd/AbstractRVDSpec.scala index abcc7277dbe..32a2a75110f 100644 --- a/hail/hail/src/is/hail/rvd/AbstractRVDSpec.scala +++ b/hail/hail/src/is/hail/rvd/AbstractRVDSpec.scala @@ -5,9 +5,10 @@ import is.hail.backend.{ExecuteContext, HailStateManager} import is.hail.compatibility import is.hail.expr.{ir, JSONAnnotationImpex} import is.hail.expr.ir.{ - flatMapIR, IR, Literal, PartitionNativeReader, PartitionZippedIndexedNativeReader, - PartitionZippedNativeReader, ReadPartition, Ref, ToStream, + flatMapIR, IR, PartitionNativeReader, PartitionZippedIndexedNativeReader, + PartitionZippedNativeReader, } +import is.hail.expr.ir.defs.{Literal, ReadPartition, Ref, ToStream} import is.hail.expr.ir.lowering.{TableStage, TableStageDependency} import is.hail.io._ import is.hail.io.fs.FS @@ -190,9 +191,9 @@ object AbstractRVDSpec { ) } - val contexts = ir.ToStream(ir.Literal(TArray(reader.contextType), contextsValues)) + val contexts = ToStream(Literal(TArray(reader.contextType), contextsValues)) - val body = (ctx: IR) => ir.ReadPartition(ctx, requestedType, reader) + val body = (ctx: IR) => ReadPartition(ctx, requestedType, reader) { (globals: IR) => val ts = TableStage( @@ -250,7 +251,7 @@ abstract class AbstractRVDSpec { val rSpec = typedCodecSpec val ctxType = TStruct("partitionIndex" -> TInt64, "partitionPath" -> TString) - val contexts = ir.ToStream(ir.Literal( + val contexts = ToStream(Literal( TArray(ctxType), absolutePartPaths(path).zipWithIndex.map { case (x, i) => Row(i.toLong, x) @@ -258,7 +259,7 @@ abstract class AbstractRVDSpec { )) val body = (ctx: IR) => - ir.ReadPartition(ctx, requestedType.rowType, ir.PartitionNativeReader(rSpec, uidFieldName)) + ReadPartition(ctx, requestedType.rowType, PartitionNativeReader(rSpec, uidFieldName)) (globals: IR) => TableStage( @@ -561,10 +562,10 @@ case class IndexedRVDSpec2( globals, newPartitioner, TableStageDependency.none, - contexts = ir.ToStream(ir.Literal(TArray(TArray(reader.contextType)), nestedContexts)), + contexts = ToStream(Literal(TArray(TArray(reader.contextType)), nestedContexts)), body = (ctxs: Ref) => flatMapIR(ToStream(ctxs, true)) { ctx => - ir.ReadPartition(ctx, requestedType.rowType, reader) + ReadPartition(ctx, requestedType.rowType, reader) }, ) } diff --git a/hail/hail/src/is/hail/rvd/RVDPartitioner.scala b/hail/hail/src/is/hail/rvd/RVDPartitioner.scala index e8902fb8c31..cbbfe4ae5dd 100644 --- a/hail/hail/src/is/hail/rvd/RVDPartitioner.scala +++ b/hail/hail/src/is/hail/rvd/RVDPartitioner.scala @@ -2,7 +2,7 @@ package is.hail.rvd import is.hail.annotations._ import is.hail.backend.{ExecuteContext, HailStateManager} -import is.hail.expr.ir.Literal +import is.hail.expr.ir.defs.Literal import is.hail.types.virtual._ import is.hail.utils._ diff --git a/hail/hail/src/is/hail/types/virtual/BlockMatrixType.scala b/hail/hail/src/is/hail/types/virtual/BlockMatrixType.scala index a8e886e1369..3c8fdc5ff28 100644 --- a/hail/hail/src/is/hail/types/virtual/BlockMatrixType.scala +++ b/hail/hail/src/is/hail/types/virtual/BlockMatrixType.scala @@ -1,6 +1,7 @@ package is.hail.types.virtual import is.hail.expr.ir._ +import is.hail.expr.ir.defs.{I64, If, Literal, ToStream} import is.hail.linalg.BlockMatrix import is.hail.utils._ diff --git a/hail/hail/test/src/is/hail/HailSuite.scala b/hail/hail/test/src/is/hail/HailSuite.scala index 66a9abbeb03..280830dad09 100644 --- a/hail/hail/test/src/is/hail/HailSuite.scala +++ b/hail/hail/test/src/is/hail/HailSuite.scala @@ -5,7 +5,13 @@ import is.hail.TestUtils._ import is.hail.annotations._ import is.hail.backend.{BroadcastValue, ExecuteContext} import is.hail.backend.spark.SparkBackend -import is.hail.expr.ir._ +import is.hail.expr.ir.{ + bindIR, freshName, rangeIR, BindingEnv, BlockMatrixIR, Compilable, Env, Forall, IR, Interpret, + Pretty, TypeCheck, +} +import is.hail.expr.ir.defs.{ + BlockMatrixCollect, Cast, MakeTuple, NDArrayRef, Ref, StreamMap, ToArray, +} import is.hail.io.fs.FS import is.hail.types.virtual._ import is.hail.utils._ diff --git a/hail/hail/test/src/is/hail/TestUtils.scala b/hail/hail/test/src/is/hail/TestUtils.scala index 926cf7753d3..4e29c013d67 100644 --- a/hail/hail/test/src/is/hail/TestUtils.scala +++ b/hail/hail/test/src/is/hail/TestUtils.scala @@ -3,7 +3,11 @@ package is.hail import is.hail.annotations.{Region, RegionValueBuilder, SafeRow} import is.hail.asm4s._ import is.hail.backend.ExecuteContext -import is.hail.expr.ir._ +import is.hail.expr.ir.{ + freshName, streamAggIR, BindingEnv, Compile, Env, IR, Interpret, MapIR, MatrixIR, MatrixRead, + Name, SingleCodeEmitParamType, Subst, +} +import is.hail.expr.ir.defs.{GetField, GetTupleElement, In, MakeTuple, Ref, ToStream} import is.hail.expr.ir.lowering.LowererUnsupportedOperation import is.hail.io.vcf.MatrixVCFReader import is.hail.types.physical.{PBaseStruct, PCanonicalArray, PType} diff --git a/hail/hail/test/src/is/hail/expr/ir/Aggregators2Suite.scala b/hail/hail/test/src/is/hail/expr/ir/Aggregators2Suite.scala index 9cd09daf7e6..abef7b459e9 100644 --- a/hail/hail/test/src/is/hail/expr/ir/Aggregators2Suite.scala +++ b/hail/hail/test/src/is/hail/expr/ir/Aggregators2Suite.scala @@ -4,6 +4,7 @@ import is.hail.{ExecStrategy, HailSuite} import is.hail.annotations._ import is.hail.asm4s._ import is.hail.expr.ir.agg._ +import is.hail.expr.ir.defs._ import is.hail.io.BufferSpec import is.hail.types.VirtualTypeWithReq import is.hail.types.physical._ diff --git a/hail/hail/test/src/is/hail/expr/ir/AggregatorsSuite.scala b/hail/hail/test/src/is/hail/expr/ir/AggregatorsSuite.scala index 5a32d8fd878..4ea0b40cd0b 100644 --- a/hail/hail/test/src/is/hail/expr/ir/AggregatorsSuite.scala +++ b/hail/hail/test/src/is/hail/expr/ir/AggregatorsSuite.scala @@ -2,6 +2,10 @@ package is.hail.expr.ir import is.hail.{ExecStrategy, HailSuite} import is.hail.expr.ir.DeprecatedIRBuilder._ +import is.hail.expr.ir.defs.{ + AggFilter, AggGroupBy, ApplyAggOp, ApplyBinaryPrimOp, Cast, GetField, I32, Ref, StreamAggScan, + StreamRange, TableAggregate, ToArray, ToStream, +} import is.hail.expr.ir.lowering.{DArrayLowering, LowerTableIR} import is.hail.types.virtual._ import is.hail.utils.{toRichIterable, FastSeq} diff --git a/hail/hail/test/src/is/hail/expr/ir/ArrayDeforestationSuite.scala b/hail/hail/test/src/is/hail/expr/ir/ArrayDeforestationSuite.scala index 4ca4fd2cb8f..446ef5378ae 100644 --- a/hail/hail/test/src/is/hail/expr/ir/ArrayDeforestationSuite.scala +++ b/hail/hail/test/src/is/hail/expr/ir/ArrayDeforestationSuite.scala @@ -1,6 +1,9 @@ package is.hail.expr.ir import is.hail.{ExecStrategy, HailSuite} +import is.hail.expr.ir.defs.{ + GetField, GetTupleElement, If, MakeStruct, MakeTuple, StreamRange, ToArray, ToStream, +} import is.hail.utils._ import org.apache.spark.sql.Row diff --git a/hail/hail/test/src/is/hail/expr/ir/ArrayFunctionsSuite.scala b/hail/hail/test/src/is/hail/expr/ir/ArrayFunctionsSuite.scala index d76741cb770..4273f6e2dd6 100644 --- a/hail/hail/test/src/is/hail/expr/ir/ArrayFunctionsSuite.scala +++ b/hail/hail/test/src/is/hail/expr/ir/ArrayFunctionsSuite.scala @@ -3,6 +3,7 @@ package is.hail.expr.ir import is.hail.{ExecStrategy, HailSuite} import is.hail.TestUtils._ import is.hail.expr.ir.TestUtils._ +import is.hail.expr.ir.defs.{ArraySlice, F32, F64, I32, In, MakeArray, NA, Str} import is.hail.types.virtual._ import is.hail.utils.FastSeq diff --git a/hail/hail/test/src/is/hail/expr/ir/BlockMatrixIRSuite.scala b/hail/hail/test/src/is/hail/expr/ir/BlockMatrixIRSuite.scala index 18082526f48..db05139db4d 100644 --- a/hail/hail/test/src/is/hail/expr/ir/BlockMatrixIRSuite.scala +++ b/hail/hail/test/src/is/hail/expr/ir/BlockMatrixIRSuite.scala @@ -3,6 +3,10 @@ package is.hail.expr.ir import is.hail.{ExecStrategy, HailSuite} import is.hail.ExecStrategy.ExecStrategy import is.hail.expr.Nat +import is.hail.expr.ir.defs.{ + Apply, ApplyBinaryPrimOp, ApplyUnaryPrimOp, BlockMatrixWrite, ErrorIDs, F64, Literal, MakeArray, + ReadValue, Ref, Str, UUID4, WriteValue, +} import is.hail.io.TypedCodecSpec import is.hail.linalg.BlockMatrix import is.hail.types.encoded.{EBlockMatrixNDArray, EFloat64Required} diff --git a/hail/hail/test/src/is/hail/expr/ir/CallFunctionsSuite.scala b/hail/hail/test/src/is/hail/expr/ir/CallFunctionsSuite.scala index e04a181b506..e63415f6ddf 100644 --- a/hail/hail/test/src/is/hail/expr/ir/CallFunctionsSuite.scala +++ b/hail/hail/test/src/is/hail/expr/ir/CallFunctionsSuite.scala @@ -2,6 +2,7 @@ package is.hail.expr.ir import is.hail.{ExecStrategy, HailSuite} import is.hail.expr.ir.TestUtils.IRCall +import is.hail.expr.ir.defs.{False, I32, Str, True} import is.hail.types.virtual.{TArray, TBoolean, TCall, TInt32} import is.hail.variant._ diff --git a/hail/hail/test/src/is/hail/expr/ir/DictFunctionsSuite.scala b/hail/hail/test/src/is/hail/expr/ir/DictFunctionsSuite.scala index f0e071c6aa1..b211701f942 100644 --- a/hail/hail/test/src/is/hail/expr/ir/DictFunctionsSuite.scala +++ b/hail/hail/test/src/is/hail/expr/ir/DictFunctionsSuite.scala @@ -3,6 +3,7 @@ package is.hail.expr.ir import is.hail.{ExecStrategy, HailSuite} import is.hail.TestUtils._ import is.hail.expr.ir.TestUtils._ +import is.hail.expr.ir.defs.{NA, ToSet, ToStream} import is.hail.types.virtual._ import is.hail.utils.FastSeq diff --git a/hail/hail/test/src/is/hail/expr/ir/DistinctlyKeyedSuite.scala b/hail/hail/test/src/is/hail/expr/ir/DistinctlyKeyedSuite.scala index 3c07eb9af41..b26a58f4e92 100644 --- a/hail/hail/test/src/is/hail/expr/ir/DistinctlyKeyedSuite.scala +++ b/hail/hail/test/src/is/hail/expr/ir/DistinctlyKeyedSuite.scala @@ -1,6 +1,10 @@ package is.hail.expr.ir import is.hail.HailSuite +import is.hail.expr.ir.defs.{ + ApplyComparisonOp, GetField, I32, If, InsertFields, MakeStruct, Ref, StreamRange, TableCollect, + TableWrite, ToArray, +} import is.hail.types.virtual.TInt32 import is.hail.utils.FastSeq diff --git a/hail/hail/test/src/is/hail/expr/ir/EmitStreamSuite.scala b/hail/hail/test/src/is/hail/expr/ir/EmitStreamSuite.scala index 0c1b0393fe2..7cd6fdf0053 100644 --- a/hail/hail/test/src/is/hail/expr/ir/EmitStreamSuite.scala +++ b/hail/hail/test/src/is/hail/expr/ir/EmitStreamSuite.scala @@ -6,6 +6,7 @@ import is.hail.annotations.{Region, SafeRow, ScalaToRegionValue} import is.hail.asm4s._ import is.hail.backend.ExecuteContext import is.hail.expr.ir.agg.{CollectStateSig, PhysicalAggSig, TypedStateSig} +import is.hail.expr.ir.defs._ import is.hail.expr.ir.lowering.LoweringPipeline import is.hail.expr.ir.streams.{EmitStream, StreamUtils} import is.hail.types.VirtualTypeWithReq diff --git a/hail/hail/test/src/is/hail/expr/ir/EncodedLiteralSuite.scala b/hail/hail/test/src/is/hail/expr/ir/EncodedLiteralSuite.scala index 667f684d50b..78fb54ea776 100644 --- a/hail/hail/test/src/is/hail/expr/ir/EncodedLiteralSuite.scala +++ b/hail/hail/test/src/is/hail/expr/ir/EncodedLiteralSuite.scala @@ -1,6 +1,7 @@ package is.hail.expr.ir import is.hail.HailSuite +import is.hail.expr.ir.defs.WrappedByteArrays import org.testng.annotations.Test diff --git a/hail/hail/test/src/is/hail/expr/ir/ExtractIntervalFiltersSuite.scala b/hail/hail/test/src/is/hail/expr/ir/ExtractIntervalFiltersSuite.scala index a6670a91018..079e16ec965 100644 --- a/hail/hail/test/src/is/hail/expr/ir/ExtractIntervalFiltersSuite.scala +++ b/hail/hail/test/src/is/hail/expr/ir/ExtractIntervalFiltersSuite.scala @@ -1,6 +1,7 @@ package is.hail.expr.ir import is.hail.{ExecStrategy, HailSuite} +import is.hail.expr.ir.defs._ import is.hail.rvd.RVDPartitioner import is.hail.types.virtual._ import is.hail.utils.{FastSeq, Interval, IntervalEndpoint} diff --git a/hail/hail/test/src/is/hail/expr/ir/FoldConstantsSuite.scala b/hail/hail/test/src/is/hail/expr/ir/FoldConstantsSuite.scala index 7afe5302895..422985d1b55 100644 --- a/hail/hail/test/src/is/hail/expr/ir/FoldConstantsSuite.scala +++ b/hail/hail/test/src/is/hail/expr/ir/FoldConstantsSuite.scala @@ -1,6 +1,9 @@ package is.hail.expr.ir import is.hail.HailSuite +import is.hail.expr.ir.defs.{ + AggLet, ApplyAggOp, ApplyScanOp, ApplySeeded, F64, I32, I64, RNGStateLiteral, Str, +} import is.hail.types.virtual.{TFloat64, TInt32} import org.testng.annotations.{DataProvider, Test} diff --git a/hail/hail/test/src/is/hail/expr/ir/ForwardLetsSuite.scala b/hail/hail/test/src/is/hail/expr/ir/ForwardLetsSuite.scala index 9c6c5682a44..b158fb2a506 100644 --- a/hail/hail/test/src/is/hail/expr/ir/ForwardLetsSuite.scala +++ b/hail/hail/test/src/is/hail/expr/ir/ForwardLetsSuite.scala @@ -3,6 +3,7 @@ package is.hail.expr.ir import is.hail.HailSuite import is.hail.TestUtils._ import is.hail.expr.Nat +import is.hail.expr.ir.defs._ import is.hail.types.virtual._ import is.hail.utils._ diff --git a/hail/hail/test/src/is/hail/expr/ir/FunctionSuite.scala b/hail/hail/test/src/is/hail/expr/ir/FunctionSuite.scala index 75429f33c51..e8487f0c0d0 100644 --- a/hail/hail/test/src/is/hail/expr/ir/FunctionSuite.scala +++ b/hail/hail/test/src/is/hail/expr/ir/FunctionSuite.scala @@ -2,6 +2,7 @@ package is.hail.expr.ir import is.hail.{ExecStrategy, HailSuite} import is.hail.asm4s._ +import is.hail.expr.ir.defs.{ApplyBinaryPrimOp, ErrorIDs, I32, In} import is.hail.expr.ir.functions.{IRFunctionRegistry, RegistryFunctions} import is.hail.types.physical.stypes.interfaces._ import is.hail.types.virtual._ diff --git a/hail/hail/test/src/is/hail/expr/ir/IRSuite.scala b/hail/hail/test/src/is/hail/expr/ir/IRSuite.scala index b830c6cd4ae..54b085d09c7 100644 --- a/hail/hail/test/src/is/hail/expr/ir/IRSuite.scala +++ b/hail/hail/test/src/is/hail/expr/ir/IRSuite.scala @@ -6,8 +6,9 @@ import is.hail.TestUtils._ import is.hail.annotations.{BroadcastRow, ExtendedOrdering, SafeNDArray} import is.hail.backend.ExecuteContext import is.hail.expr.Nat -import is.hail.expr.ir.ArrayZipBehavior.ArrayZipBehavior import is.hail.expr.ir.agg._ +import is.hail.expr.ir.defs._ +import is.hail.expr.ir.defs.ArrayZipBehavior.ArrayZipBehavior import is.hail.expr.ir.functions._ import is.hail.io.{BufferSpec, TypedCodecSpec} import is.hail.io.bgen.MatrixBGENReader diff --git a/hail/hail/test/src/is/hail/expr/ir/IntervalSuite.scala b/hail/hail/test/src/is/hail/expr/ir/IntervalSuite.scala index 2c803273d55..243c6a92a32 100644 --- a/hail/hail/test/src/is/hail/expr/ir/IntervalSuite.scala +++ b/hail/hail/test/src/is/hail/expr/ir/IntervalSuite.scala @@ -2,6 +2,9 @@ package is.hail.expr.ir import is.hail.{ExecStrategy, HailSuite} import is.hail.TestUtils._ +import is.hail.expr.ir.defs.{ + ErrorIDs, False, GetTupleElement, I32, If, Literal, MakeTuple, NA, True, +} import is.hail.rvd.RVDPartitioner import is.hail.types.virtual._ import is.hail.utils._ diff --git a/hail/hail/test/src/is/hail/expr/ir/LiftLiteralsSuite.scala b/hail/hail/test/src/is/hail/expr/ir/LiftLiteralsSuite.scala index 83744ec9c2a..7599a059862 100644 --- a/hail/hail/test/src/is/hail/expr/ir/LiftLiteralsSuite.scala +++ b/hail/hail/test/src/is/hail/expr/ir/LiftLiteralsSuite.scala @@ -1,6 +1,7 @@ package is.hail.expr.ir import is.hail.{ExecStrategy, HailSuite} +import is.hail.expr.ir.defs.{ApplyBinaryPrimOp, I64, MakeStruct, TableCount, TableGetGlobals} import is.hail.utils.FastSeq import org.apache.spark.sql.Row diff --git a/hail/hail/test/src/is/hail/expr/ir/LocusFunctionsSuite.scala b/hail/hail/test/src/is/hail/expr/ir/LocusFunctionsSuite.scala index 848c956f6e2..0745d8ce14a 100644 --- a/hail/hail/test/src/is/hail/expr/ir/LocusFunctionsSuite.scala +++ b/hail/hail/test/src/is/hail/expr/ir/LocusFunctionsSuite.scala @@ -1,6 +1,7 @@ package is.hail.expr.ir import is.hail.{ExecStrategy, HailSuite} +import is.hail.expr.ir.defs.{Apply, ErrorIDs, False, I32, I64, MakeArray, MakeTuple, NA, Str, True} import is.hail.types.virtual._ import is.hail.utils.{FastSeq, Interval} import is.hail.variant.{Locus, ReferenceGenome} diff --git a/hail/hail/test/src/is/hail/expr/ir/MathFunctionsSuite.scala b/hail/hail/test/src/is/hail/expr/ir/MathFunctionsSuite.scala index 320d8d19e47..6ec3d983de6 100644 --- a/hail/hail/test/src/is/hail/expr/ir/MathFunctionsSuite.scala +++ b/hail/hail/test/src/is/hail/expr/ir/MathFunctionsSuite.scala @@ -2,6 +2,7 @@ package is.hail.expr.ir import is.hail.{stats, ExecStrategy, HailSuite} import is.hail.TestUtils._ +import is.hail.expr.ir.defs.{ErrorIDs, F32, F64, False, I32, I64, Str, True} import is.hail.types.virtual._ import is.hail.utils._ diff --git a/hail/hail/test/src/is/hail/expr/ir/MatrixIRSuite.scala b/hail/hail/test/src/is/hail/expr/ir/MatrixIRSuite.scala index 1142bc6be03..5e6b03bb47c 100644 --- a/hail/hail/test/src/is/hail/expr/ir/MatrixIRSuite.scala +++ b/hail/hail/test/src/is/hail/expr/ir/MatrixIRSuite.scala @@ -6,6 +6,10 @@ import is.hail.TestUtils._ import is.hail.annotations.BroadcastRow import is.hail.expr.JSONAnnotationImpex import is.hail.expr.ir.TestUtils._ +import is.hail.expr.ir.defs.{ + ApplySeeded, GetField, I32, InsertFields, MakeTuple, MatrixMultiWrite, MatrixWrite, RNGSplit, + RNGStateLiteral, Ref, TableCollect, +} import is.hail.types.virtual._ import is.hail.utils._ diff --git a/hail/hail/test/src/is/hail/expr/ir/MemoryLeakSuite.scala b/hail/hail/test/src/is/hail/expr/ir/MemoryLeakSuite.scala index 49695998d14..90cb2d4713f 100644 --- a/hail/hail/test/src/is/hail/expr/ir/MemoryLeakSuite.scala +++ b/hail/hail/test/src/is/hail/expr/ir/MemoryLeakSuite.scala @@ -3,7 +3,7 @@ package is.hail.expr.ir import is.hail.HailSuite import is.hail.TestUtils.eval import is.hail.backend.ExecuteContext -import is.hail.expr.ir +import is.hail.expr.ir.defs.{Literal, ToArray, ToStream} import is.hail.types.virtual.{TArray, TBoolean, TSet, TString} import is.hail.utils._ @@ -20,7 +20,7 @@ class MemoryLeakSuite extends HailSuite { ExecuteContext.scoped { ctx => eval( ToArray( - mapIR(ToStream(queries))(r => ir.invoke("contains", TBoolean, lit, r)) + mapIR(ToStream(queries))(r => invoke("contains", TBoolean, lit, r)) ), Env.empty, FastSeq(), diff --git a/hail/hail/test/src/is/hail/expr/ir/OrderingSuite.scala b/hail/hail/test/src/is/hail/expr/ir/OrderingSuite.scala index c508395138e..1f41d54260c 100644 --- a/hail/hail/test/src/is/hail/expr/ir/OrderingSuite.scala +++ b/hail/hail/test/src/is/hail/expr/ir/OrderingSuite.scala @@ -5,6 +5,10 @@ import is.hail.TestUtils._ import is.hail.annotations._ import is.hail.asm4s._ import is.hail.check.{Gen, Prop} +import is.hail.expr.ir.defs.{ + ApplyComparisonOp, ApplySpecial, ArraySort, ErrorIDs, GetField, I32, In, IsNA, Literal, + MakeStream, NA, ToArray, ToDict, ToSet, ToStream, True, +} import is.hail.expr.ir.orderings.CodeOrdering import is.hail.types.physical._ import is.hail.types.physical.stypes.EmitType diff --git a/hail/hail/test/src/is/hail/expr/ir/PruneSuite.scala b/hail/hail/test/src/is/hail/expr/ir/PruneSuite.scala index c6a50ab532b..420a5446e82 100644 --- a/hail/hail/test/src/is/hail/expr/ir/PruneSuite.scala +++ b/hail/hail/test/src/is/hail/expr/ir/PruneSuite.scala @@ -3,6 +3,7 @@ package is.hail.expr.ir import is.hail.HailSuite import is.hail.expr.Nat import is.hail.expr.ir.PruneDeadFields.TypeState +import is.hail.expr.ir.defs._ import is.hail.methods.{ForceCountMatrixTable, ForceCountTable} import is.hail.rvd.RVD import is.hail.types._ diff --git a/hail/hail/test/src/is/hail/expr/ir/RequirednessSuite.scala b/hail/hail/test/src/is/hail/expr/ir/RequirednessSuite.scala index fd95b492a6a..63f516cc7f0 100644 --- a/hail/hail/test/src/is/hail/expr/ir/RequirednessSuite.scala +++ b/hail/hail/test/src/is/hail/expr/ir/RequirednessSuite.scala @@ -3,6 +3,7 @@ package is.hail.expr.ir import is.hail.HailSuite import is.hail.expr.Nat import is.hail.expr.ir.agg.CallStatsState +import is.hail.expr.ir.defs._ import is.hail.io.{BufferSpec, TypedCodecSpec} import is.hail.stats.fetStruct import is.hail.types._ diff --git a/hail/hail/test/src/is/hail/expr/ir/SetFunctionsSuite.scala b/hail/hail/test/src/is/hail/expr/ir/SetFunctionsSuite.scala index 10c74088d8d..dd3ec81340e 100644 --- a/hail/hail/test/src/is/hail/expr/ir/SetFunctionsSuite.scala +++ b/hail/hail/test/src/is/hail/expr/ir/SetFunctionsSuite.scala @@ -3,6 +3,7 @@ package is.hail.expr.ir import is.hail.{ExecStrategy, HailSuite} import is.hail.TestUtils._ import is.hail.expr.ir.TestUtils._ +import is.hail.expr.ir.defs.{I32, NA, ToSet, ToStream} import is.hail.types.virtual._ import org.testng.annotations.Test diff --git a/hail/hail/test/src/is/hail/expr/ir/SimplifySuite.scala b/hail/hail/test/src/is/hail/expr/ir/SimplifySuite.scala index 37e0634f726..a169e56dccc 100644 --- a/hail/hail/test/src/is/hail/expr/ir/SimplifySuite.scala +++ b/hail/hail/test/src/is/hail/expr/ir/SimplifySuite.scala @@ -2,6 +2,7 @@ package is.hail.expr.ir import is.hail.{ExecStrategy, HailSuite} import is.hail.expr.ir.TestUtils.IRAggCount +import is.hail.expr.ir.defs._ import is.hail.types.virtual._ import is.hail.utils.{FastSeq, Interval} import is.hail.variant.Locus diff --git a/hail/hail/test/src/is/hail/expr/ir/StagedBTreeSuite.scala b/hail/hail/test/src/is/hail/expr/ir/StagedBTreeSuite.scala index 8c923f5bc59..f7ca8fd211e 100644 --- a/hail/hail/test/src/is/hail/expr/ir/StagedBTreeSuite.scala +++ b/hail/hail/test/src/is/hail/expr/ir/StagedBTreeSuite.scala @@ -5,7 +5,7 @@ import is.hail.annotations.Region import is.hail.asm4s._ import is.hail.backend.ExecuteContext import is.hail.check.{Gen, Prop} -import is.hail.expr.ir.agg._ +import is.hail.expr.ir.agg.{AppendOnlyBTree, BTreeKey} import is.hail.expr.ir.orderings.CodeOrdering import is.hail.io.{InputBuffer, OutputBuffer, StreamBufferSpec} import is.hail.types.physical._ diff --git a/hail/hail/test/src/is/hail/expr/ir/StringFunctionsSuite.scala b/hail/hail/test/src/is/hail/expr/ir/StringFunctionsSuite.scala index 6c2697b1dfd..09211f5cd83 100644 --- a/hail/hail/test/src/is/hail/expr/ir/StringFunctionsSuite.scala +++ b/hail/hail/test/src/is/hail/expr/ir/StringFunctionsSuite.scala @@ -3,6 +3,7 @@ package is.hail.expr.ir import is.hail.{ExecStrategy, HailSuite} import is.hail.TestUtils._ import is.hail.expr.ir.TestUtils._ +import is.hail.expr.ir.defs.{F32, I32, I64, MakeTuple, NA, Str} import is.hail.types.virtual._ import is.hail.utils.FastSeq diff --git a/hail/hail/test/src/is/hail/expr/ir/StringLengthSuite.scala b/hail/hail/test/src/is/hail/expr/ir/StringLengthSuite.scala index 1a01864492d..34281801572 100644 --- a/hail/hail/test/src/is/hail/expr/ir/StringLengthSuite.scala +++ b/hail/hail/test/src/is/hail/expr/ir/StringLengthSuite.scala @@ -1,6 +1,7 @@ package is.hail.expr.ir import is.hail.{ExecStrategy, HailSuite} +import is.hail.expr.ir.defs.Str import is.hail.types.virtual.TInt32 import org.testng.annotations.Test diff --git a/hail/hail/test/src/is/hail/expr/ir/StringSliceSuite.scala b/hail/hail/test/src/is/hail/expr/ir/StringSliceSuite.scala index ac869b23455..d532f9f9ed9 100644 --- a/hail/hail/test/src/is/hail/expr/ir/StringSliceSuite.scala +++ b/hail/hail/test/src/is/hail/expr/ir/StringSliceSuite.scala @@ -2,6 +2,7 @@ package is.hail.expr.ir import is.hail.{ExecStrategy, HailSuite} import is.hail.TestUtils._ +import is.hail.expr.ir.defs.{I32, In, NA, Str} import is.hail.types.virtual.TString import is.hail.utils._ diff --git a/hail/hail/test/src/is/hail/expr/ir/TableIRSuite.scala b/hail/hail/test/src/is/hail/expr/ir/TableIRSuite.scala index 1d6c6e45c12..e8b1e673b3a 100644 --- a/hail/hail/test/src/is/hail/expr/ir/TableIRSuite.scala +++ b/hail/hail/test/src/is/hail/expr/ir/TableIRSuite.scala @@ -6,6 +6,7 @@ import is.hail.TestUtils._ import is.hail.annotations.SafeNDArray import is.hail.expr.Nat import is.hail.expr.ir.TestUtils._ +import is.hail.expr.ir.defs._ import is.hail.expr.ir.lowering.{DArrayLowering, LowerTableIR} import is.hail.methods.{ForceCountTable, NPartitionsTable} import is.hail.rvd.RVDPartitioner diff --git a/hail/hail/test/src/is/hail/expr/ir/TestUtils.scala b/hail/hail/test/src/is/hail/expr/ir/TestUtils.scala index cbd006e44e5..a1415c44c42 100644 --- a/hail/hail/test/src/is/hail/expr/ir/TestUtils.scala +++ b/hail/hail/test/src/is/hail/expr/ir/TestUtils.scala @@ -1,5 +1,9 @@ package is.hail.expr.ir +import is.hail.expr.ir.defs.{ + ApplyAggOp, ApplyScanOp, F64, I32, Literal, MakeArray, MakeStream, MakeStruct, MakeTuple, NA, Ref, + TableAggregate, TableCollect, ToDict, ToSet, ToStream, +} import is.hail.types.virtual._ import is.hail.utils.FastSeq import is.hail.variant.Call diff --git a/hail/hail/test/src/is/hail/expr/ir/TrapNodeSuite.scala b/hail/hail/test/src/is/hail/expr/ir/TrapNodeSuite.scala index bab77019356..dc6c42b31cf 100644 --- a/hail/hail/test/src/is/hail/expr/ir/TrapNodeSuite.scala +++ b/hail/hail/test/src/is/hail/expr/ir/TrapNodeSuite.scala @@ -2,6 +2,7 @@ package is.hail.expr.ir import is.hail.{ExecStrategy, HailSuite} import is.hail.TestUtils._ +import is.hail.expr.ir.defs.{ArrayRef, Die, GetTupleElement, I32, If, IsNA, Literal, Str, Trap} import is.hail.types.virtual._ import is.hail.utils._ diff --git a/hail/hail/test/src/is/hail/expr/ir/UtilFunctionsSuite.scala b/hail/hail/test/src/is/hail/expr/ir/UtilFunctionsSuite.scala index 024f7cd20fb..3c7eb8636ae 100644 --- a/hail/hail/test/src/is/hail/expr/ir/UtilFunctionsSuite.scala +++ b/hail/hail/test/src/is/hail/expr/ir/UtilFunctionsSuite.scala @@ -2,6 +2,7 @@ package is.hail.expr.ir import is.hail.{ExecStrategy, HailSuite} import is.hail.TestUtils._ +import is.hail.expr.ir.defs.{Die, False, MakeStream, NA, Str, True} import is.hail.types.virtual.{TBoolean, TInt32, TStream} import org.testng.annotations.Test diff --git a/hail/hail/test/src/is/hail/expr/ir/analyses/SemanticHashSuite.scala b/hail/hail/test/src/is/hail/expr/ir/analyses/SemanticHashSuite.scala index 97089297c03..68ba43506f2 100644 --- a/hail/hail/test/src/is/hail/expr/ir/analyses/SemanticHashSuite.scala +++ b/hail/hail/test/src/is/hail/expr/ir/analyses/SemanticHashSuite.scala @@ -3,6 +3,7 @@ package is.hail.expr.ir.analyses import is.hail.{HAIL_PRETTY_VERSION, HailSuite} import is.hail.backend.ExecuteContext import is.hail.expr.ir._ +import is.hail.expr.ir.defs._ import is.hail.io.fs.{FS, FakeFS, FakeURL, FileListEntry} import is.hail.linalg.BlockMatrixMetadata import is.hail.rvd.AbstractRVDSpec diff --git a/hail/hail/test/src/is/hail/expr/ir/lowering/LowerDistributedSortSuite.scala b/hail/hail/test/src/is/hail/expr/ir/lowering/LowerDistributedSortSuite.scala index fb296ad605b..b6d5d2f6882 100644 --- a/hail/hail/test/src/is/hail/expr/ir/lowering/LowerDistributedSortSuite.scala +++ b/hail/hail/test/src/is/hail/expr/ir/lowering/LowerDistributedSortSuite.scala @@ -2,8 +2,10 @@ package is.hail.expr.ir.lowering import is.hail.{ExecStrategy, HailSuite, TestUtils} import is.hail.expr.ir.{ - mapIR, Apply, Ascending, Descending, ErrorIDs, GetField, I32, Literal, LoweringAnalyses, - MakeStruct, Ref, SelectFields, SortField, TableIR, TableMapRows, TableRange, ToArray, ToStream, + mapIR, Ascending, Descending, LoweringAnalyses, SortField, TableIR, TableMapRows, TableRange, +} +import is.hail.expr.ir.defs.{ + Apply, ErrorIDs, GetField, I32, Literal, MakeStruct, Ref, SelectFields, ToArray, ToStream, } import is.hail.expr.ir.lowering.LowerDistributedSort.samplePartition import is.hail.types.RTable diff --git a/hail/hail/test/src/is/hail/expr/ir/table/TableGenSuite.scala b/hail/hail/test/src/is/hail/expr/ir/table/TableGenSuite.scala index 325d691f08b..df686e1b7ed 100644 --- a/hail/hail/test/src/is/hail/expr/ir/table/TableGenSuite.scala +++ b/hail/hail/test/src/is/hail/expr/ir/table/TableGenSuite.scala @@ -5,6 +5,10 @@ import is.hail.TestUtils.loweredExecute import is.hail.backend.ExecuteContext import is.hail.expr.ir._ import is.hail.expr.ir.TestUtils.IRAggCollect +import is.hail.expr.ir.defs.{ + ApplyBinaryPrimOp, ErrorIDs, GetField, MakeStream, MakeStruct, Ref, Str, StreamRange, + TableAggregate, TableGetGlobals, +} import is.hail.expr.ir.lowering.{DArrayLowering, LowerTableIR} import is.hail.rvd.RVDPartitioner import is.hail.types.virtual._ diff --git a/hail/hail/test/src/is/hail/io/AvroReaderSuite.scala b/hail/hail/test/src/is/hail/io/AvroReaderSuite.scala index 02c54db69f3..af41fdeb81a 100644 --- a/hail/hail/test/src/is/hail/io/AvroReaderSuite.scala +++ b/hail/hail/test/src/is/hail/io/AvroReaderSuite.scala @@ -2,7 +2,7 @@ package is.hail.io import is.hail.{ExecStrategy, HailSuite} import is.hail.ExecStrategy.ExecStrategy -import is.hail.expr.ir.{I64, MakeStruct, ReadPartition, Str, ToArray} +import is.hail.expr.ir.defs.{I64, MakeStruct, ReadPartition, Str, ToArray} import is.hail.io.avro.AvroPartitionReader import is.hail.utils.{fatal, using, FastSeq} diff --git a/hail/hail/test/src/is/hail/linalg/BlockMatrixSuite.scala b/hail/hail/test/src/is/hail/linalg/BlockMatrixSuite.scala index cb2a9008a5c..e32d0c7038e 100644 --- a/hail/hail/test/src/is/hail/linalg/BlockMatrixSuite.scala +++ b/hail/hail/test/src/is/hail/linalg/BlockMatrixSuite.scala @@ -5,7 +5,8 @@ import is.hail.check._ import is.hail.check.Arbitrary._ import is.hail.check.Gen._ import is.hail.check.Prop._ -import is.hail.expr.ir.{CompileAndEvaluate, GetField, TableCollect, TableLiteral} +import is.hail.expr.ir.{CompileAndEvaluate, TableLiteral} +import is.hail.expr.ir.defs.{GetField, TableCollect} import is.hail.linalg.BlockMatrix.ops._ import is.hail.types.virtual.{TFloat64, TInt64, TStruct} import is.hail.utils._ diff --git a/hail/hail/test/src/is/hail/utils/RowIntervalSuite.scala b/hail/hail/test/src/is/hail/utils/RowIntervalSuite.scala index 92afa643570..31145ee013d 100644 --- a/hail/hail/test/src/is/hail/utils/RowIntervalSuite.scala +++ b/hail/hail/test/src/is/hail/utils/RowIntervalSuite.scala @@ -2,7 +2,7 @@ package is.hail.utils import is.hail.{ExecStrategy, HailSuite} import is.hail.expr.ir -import is.hail.expr.ir.In +import is.hail.expr.ir.defs.In import is.hail.rvd.{PartitionBoundOrdering, RVDPartitioner} import is.hail.types.virtual.{TBoolean, TInt32, TStruct}