sql: add hidden column element

This introduces a new schema changer element to manage the visibility of
columns. That attribute was previously a field in the `Column` element.

Epic: CRDB-31283
Part of: #139605

Release note: None

Author: bghal

pkg/sql/schemachanger/scbuild/internal/scbuildstmt/alter_table_add_column.go

@@ -118,7 +118,6 @@ func alterTableAddColumn(
 		col: &scpb.Column{
 			TableID:                 tbl.TableID,
 			ColumnID:                desc.ID,
-			IsHidden:                desc.Hidden,
 			IsInaccessible:          desc.Inaccessible,
 			GeneratedAsIdentityType: desc.GeneratedAsIdentityType,
 		},
@@ -290,6 +289,14 @@ func alterTableAddColumn(
 		}
 		b.IncrementSchemaChangeAddColumnQualificationCounter("on_update")
 	}
+	if d.Hidden {
+		if spec.colType.ElementCreationMetadata.In_26_1OrLater {
+			spec.hidden = true
+		} else {
+			spec.col.IsHidden = true
+		}
+	}
+
 	// Add secondary indexes for this column.
 	backing := addColumn(b, spec, t)
 	if idx != nil {
@@ -458,6 +465,7 @@ type addColumnSpec struct {
 	compute          *scpb.ColumnComputeExpression
 	transientCompute *scpb.ColumnComputeExpression
 	comment          *scpb.ColumnComment
+	hidden           bool
 	unique           bool
 	notNull          bool
 }
@@ -494,6 +502,13 @@ func addColumn(b BuildCtx, spec addColumnSpec, n tree.NodeFormatter) (backing *s
 		if spec.comment != nil {
 			b.Add(spec.comment)
 		}
+		if spec.hidden {
+			elm := scpb.ColumnHidden{
+				TableID:  spec.tbl.TableID,
+				ColumnID: spec.col.ColumnID,
+			}
+			b.Add(&elm)
+		}
 		// Don't need to modify primary indexes for virtual columns.
 		if spec.colType.IsVirtual {
 			return getLatestPrimaryIndex(b, spec.tbl.TableID)

pkg/sql/schemachanger/scbuild/internal/scbuildstmt/alter_table_alter_column_type.go

@@ -336,6 +336,8 @@ func handleGeneralColumnConversion(
 			"old active version; ALTER COLUMN TYPE requires backfill. Reverting to legacy handling"))
 	}
 
+	colHidden := retrieveColumnHidden(b, tbl.TableID, col.ColumnID)
+
 	colNotNull := retrieveColumnNotNull(b, tbl.TableID, col.ColumnID)
 
 	// Generate the ID of the new column we are adding.
@@ -381,6 +383,9 @@ func handleGeneralColumnConversion(
 	if colNotNull != nil {
 		b.Drop(colNotNull)
 	}
+	if colHidden != nil {
+		b.Drop(colHidden)
+	}
 	if oldColComment != nil {
 		b.Drop(oldColComment)
 	}
@@ -429,6 +434,7 @@ func handleGeneralColumnConversion(
 			Expression: *b.WrapExpression(tbl.TableID, expr),
 			Usage:      scpb.ColumnComputeExpression_ALTER_TYPE_USING,
 		},
+		hidden:  colHidden != nil,
 		notNull: retrieveColumnNotNull(b, tbl.TableID, col.ColumnID) != nil,
 		// The new column will be placed in the same column family as the one
 		// it's replacing, so there's no need to specify a family.

pkg/sql/schemachanger/scbuild/internal/scbuildstmt/alter_table_alter_primary_key.go

@@ -1063,7 +1063,7 @@ func getPrimaryIndexDefaultRowIDColumn(
 
 	// That one column should be hidden.
 	column = mustRetrieveColumnElem(b, tableID, columnID)
-	if !column.IsHidden {
+	if !(column.IsHidden || retrieveColumnHidden(b, tableID, column.ColumnID) != nil) {
 		return nil
 	}
 

pkg/sql/schemachanger/scbuild/internal/scbuildstmt/alter_table_drop_column.go

@@ -326,7 +326,7 @@ func walkColumnDependencies(
 				*scpb.UniqueWithoutIndexConstraint, *scpb.CheckConstraint,
 				*scpb.UniqueWithoutIndexConstraintUnvalidated, *scpb.CheckConstraintUnvalidated,
 				*scpb.RowLevelTTL, *scpb.PolicyUsingExpr, *scpb.PolicyWithCheckExpr,
-				*scpb.TriggerDeps:
+				*scpb.TriggerDeps, *scpb.ColumnHidden:
 				fn(e, op, objType)
 			case *scpb.ColumnType:
 				if elt.ColumnID == col.ColumnID {

pkg/sql/schemachanger/scbuild/internal/scbuildstmt/create_index.go

@@ -820,7 +820,7 @@ func maybeCreateAndAddShardCol(
 		}
 	})
 	scpb.ForEachColumn(elts, func(_ scpb.Status, _ scpb.TargetStatus, col *scpb.Column) {
-		if col.ColumnID == existingShardColID && !col.IsHidden {
+		if col.ColumnID == existingShardColID && !(col.IsHidden || retrieveColumnHidden(b, tbl.TableID, col.ColumnID) != nil) {
 			// The user managed to reverse-engineer our crazy shard column name, so
 			// we'll return an error here rather than try to be tricky.
 			panic(pgerror.Newf(pgcode.DuplicateColumn,
@@ -841,7 +841,6 @@ func maybeCreateAndAddShardCol(
 		col: &scpb.Column{
 			TableID:  tbl.TableID,
 			ColumnID: shardColID,
-			IsHidden: true,
 		},
 		name: &scpb.ColumnName{
 			TableID:  tbl.TableID,
@@ -868,6 +867,12 @@ func maybeCreateAndAddShardCol(
 		spec.colType.ComputeExpr = wexpr
 	}
 
+	if spec.colType.ElementCreationMetadata.In_26_1OrLater {
+		spec.hidden = true
+	} else {
+		spec.col.IsHidden = true
+	}
+
 	backing := addColumn(b, spec, n)
 	// Create a new check constraint for the hash sharded index column.
 	checkConstraintBucketValues := strings.Builder{}

pkg/sql/schemachanger/scbuild/internal/scbuildstmt/helpers.go

@@ -2122,6 +2122,14 @@ func retrieveColumnNotNull(
 		MustGetZeroOrOneElement()
 }
 
+func retrieveColumnHidden(
+	b BuildCtx, tableID catid.DescID, columnID catid.ColumnID,
+) *scpb.ColumnHidden {
+	return b.QueryByID(tableID).FilterColumnHidden().
+		Filter(func(_ scpb.Status, _ scpb.TargetStatus, e *scpb.ColumnHidden) bool { return e.ColumnID == columnID }).
+		MustGetZeroOrOneElement()
+}
+
 func retrieveColumnComment(
 	b BuildCtx, tableID catid.DescID, columnID catid.ColumnID,
 ) *scpb.ColumnComment {

pkg/sql/schemachanger/scbuild/testdata/alter_table_add_primary_key

@@ -7,11 +7,13 @@ build
 ALTER TABLE defaultdb.bar ADD PRIMARY KEY (i)
 ----
 - [[Column:{DescID: 105, ColumnID: 2}, ABSENT], PUBLIC]
-  {columnId: 2, isHidden: true, tableId: 105}
+  {columnId: 2, tableId: 105}
 - [[ColumnName:{DescID: 105, Name: rowid, ColumnID: 2}, ABSENT], PUBLIC]
   {columnId: 2, name: rowid, tableId: 105}
 - [[ColumnType:{DescID: 105, ColumnFamilyID: 0, ColumnID: 2, TypeName: INT8}, ABSENT], PUBLIC]
   {columnId: 2, elementCreationMetadata: {in243OrLater: true, in261OrLater: true}, tableId: 105, type: {family: IntFamily, oid: 20, width: 64}, typeName: INT8}
+- [[ColumnHidden:{DescID: 105, ColumnID: 2}, ABSENT], PUBLIC]
+  {columnId: 2, tableId: 105}
 - [[ColumnNotNull:{DescID: 105, ColumnID: 2, IndexID: 0}, ABSENT], PUBLIC]
   {columnId: 2, tableId: 105}
 - [[ColumnDefaultExpression:{DescID: 105, ColumnID: 2, Expr: unique_rowid()}, ABSENT], PUBLIC]

pkg/sql/schemachanger/scbuild/testdata/alter_table_alter_primary_key

@@ -69,11 +69,13 @@ build
 ALTER TABLE defaultdb.bar ALTER PRIMARY KEY USING COLUMNS (i)
 ----
 - [[Column:{DescID: 105, ColumnID: 2}, ABSENT], PUBLIC]
-  {columnId: 2, isHidden: true, tableId: 105}
+  {columnId: 2, tableId: 105}
 - [[ColumnName:{DescID: 105, Name: rowid, ColumnID: 2}, ABSENT], PUBLIC]
   {columnId: 2, name: rowid, tableId: 105}
 - [[ColumnType:{DescID: 105, ColumnFamilyID: 0, ColumnID: 2, TypeName: INT8}, ABSENT], PUBLIC]
   {columnId: 2, elementCreationMetadata: {in243OrLater: true, in261OrLater: true}, tableId: 105, type: {family: IntFamily, oid: 20, width: 64}, typeName: INT8}
+- [[ColumnHidden:{DescID: 105, ColumnID: 2}, ABSENT], PUBLIC]
+  {columnId: 2, tableId: 105}
 - [[ColumnNotNull:{DescID: 105, ColumnID: 2, IndexID: 0}, ABSENT], PUBLIC]
   {columnId: 2, tableId: 105}
 - [[ColumnDefaultExpression:{DescID: 105, ColumnID: 2, Expr: unique_rowid()}, ABSENT], PUBLIC]

pkg/sql/schemachanger/scbuild/testdata/create_index

@@ -106,13 +106,15 @@ CREATE INDEX id4
 - [[TableData:{DescID: 104, ReferencedDescID: 100}, PUBLIC], PUBLIC]
   {databaseId: 100, tableId: 104}
 - [[Column:{DescID: 104, ColumnID: 4}, PUBLIC], ABSENT]
-  {columnId: 4, isHidden: true, tableId: 104}
+  {columnId: 4, tableId: 104}
 - [[ColumnName:{DescID: 104, Name: crdb_internal_id_name_shard_8, ColumnID: 4}, PUBLIC], ABSENT]
   {columnId: 4, name: crdb_internal_id_name_shard_8, tableId: 104}
 - [[ColumnType:{DescID: 104, ColumnFamilyID: 0, ColumnID: 4, TypeName: INT8}, PUBLIC], ABSENT]
   {columnId: 4, elementCreationMetadata: {in243OrLater: true, in261OrLater: true}, isVirtual: true, tableId: 104, type: {family: IntFamily, oid: 20, width: 64}, typeName: INT8}
 - [[ColumnComputeExpression:{DescID: 104, ColumnID: 4, ReferencedColumnIDs: [1 2], Usage: REGULAR}, PUBLIC], ABSENT]
   {columnId: 4, expr: 'mod(fnv32(md5(crdb_internal.datums_to_bytes(id, name))), 8:::INT8)', referencedColumnIds: [1, 2], tableId: 104}
+- [[ColumnHidden:{DescID: 104, ColumnID: 4}, PUBLIC], ABSENT]
+  {columnId: 4, tableId: 104}
 - [[ColumnNotNull:{DescID: 104, ColumnID: 4, IndexID: 1}, PUBLIC], ABSENT]
   {columnId: 4, indexIdForValidation: 1, tableId: 104}
 - [[CheckConstraint:{DescID: 104, IndexID: 1, ConstraintID: 2, ReferencedColumnIDs: [4]}, PUBLIC], ABSENT]