diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValueSourceReaderTypeConversionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValueSourceReaderTypeConversionTests.java
index 66bcf2a57e393..09f63e9fa45bb 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValueSourceReaderTypeConversionTests.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValueSourceReaderTypeConversionTests.java
@@ -1687,12 +1687,13 @@ public StoredFieldsSpec rowStrideStoredFieldSpec() {
 
         @Override
         public boolean supportsOrdinals() {
-            return delegate.supportsOrdinals();
+            // Fields with mismatching types cannot use ordinals for uniqueness determination, but must convert the values first
+            return false;
         }
 
         @Override
-        public SortedSetDocValues ordinals(LeafReaderContext context) throws IOException {
-            return delegate.ordinals(context);
+        public SortedSetDocValues ordinals(LeafReaderContext context) {
+            throw new IllegalArgumentException("Ordinals are not supported for type conversion");
         }
 
         @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizer.java
index f78ae6930d9ba..1b40a1c2b02ad 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizer.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizer.java
@@ -77,7 +77,6 @@
 import org.elasticsearch.xpack.esql.planner.AbstractPhysicalOperationProviders;
 import org.elasticsearch.xpack.esql.planner.EsqlTranslatorHandler;
 import org.elasticsearch.xpack.esql.stats.SearchStats;
-import org.elasticsearch.xpack.esql.type.MultiTypeEsField;
 
 import java.nio.ByteOrder;
 import java.util.ArrayList;
@@ -194,10 +193,7 @@ public PhysicalPlan apply(PhysicalPlan plan) {
                  * it loads the field lazily. If we have more than one field we need to
                  * make sure the fields are loaded for the standard hash aggregator.
                  */
-                if (p instanceof AggregateExec agg
-                    && agg.groupings().size() == 1
-                    && (isMultiTypeFieldAttribute(agg.groupings().get(0)) == false) // Union types rely on field extraction.
-                ) {
+                if (p instanceof AggregateExec agg && agg.groupings().size() == 1) {
                     var leaves = new LinkedList<>();
                     // TODO: this seems out of place
                     agg.aggregates()
@@ -221,10 +217,6 @@ public PhysicalPlan apply(PhysicalPlan plan) {
             return plan;
         }
 
-        private static boolean isMultiTypeFieldAttribute(Expression attribute) {
-            return attribute instanceof FieldAttribute fa && fa.field() instanceof MultiTypeEsField;
-        }
-
         private static Set<Attribute> missingAttributes(PhysicalPlan p) {
             var missing = new LinkedHashSet<Attribute>();
             var input = p.inputSet();
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java
index 9e1e1a50fe8f0..8611d2c6fa9fb 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java
@@ -233,8 +233,9 @@ public final Operator.OperatorFactory ordinalGroupingOperatorFactory(
         // The grouping-by values are ready, let's group on them directly.
         // Costin: why are they ready and not already exposed in the layout?
         boolean isUnsupported = attrSource.dataType() == DataType.UNSUPPORTED;
+        var unionTypes = findUnionTypes(attrSource);
         return new OrdinalsGroupingOperator.OrdinalsGroupingOperatorFactory(
-            shardIdx -> shardContexts.get(shardIdx).blockLoader(attrSource.name(), isUnsupported, NONE),
+            shardIdx -> getBlockLoaderFor(shardIdx, attrSource.name(), isUnsupported, NONE, unionTypes),
             vsShardContexts,
             groupElementType,
             docChannel,
@@ -434,12 +435,13 @@ public StoredFieldsSpec rowStrideStoredFieldSpec() {
 
         @Override
         public boolean supportsOrdinals() {
-            return delegate.supportsOrdinals();
+            // Fields with mismatching types cannot use ordinals for uniqueness determination, but must convert the values first
+            return false;
         }
 
         @Override
-        public SortedSetDocValues ordinals(LeafReaderContext context) throws IOException {
-            return delegate.ordinals(context);
+        public SortedSetDocValues ordinals(LeafReaderContext context) {
+            throw new IllegalArgumentException("Ordinals are not supported for type conversion");
         }
 
         @Override