Merge #156856 #156888

156856: roachtest/vecindex: deflake vecindex test r=mw5h a=mw5h

There's some nondeterminism in the recall rates of a vector index, and it looks like I set the fail threshold for these tests a bit too high.

Fixes: #156787
Release note: None

156888: rowenc: fix decoding for Canonical types r=jeffswenson a=jeffswenson

The definition of a Canonical type is the Canonical type is the type of the datum that a column decodes into. This was only partially true for the two canonical types that use the oid wrapper. The citext type was decoded by the value side decoder, but not the keyside decoder. The name type was decoded by the key side decoder, but not the value side decoder.

Release note: none
Epic: CRDB-48647

Co-authored-by: Matt White <matt.white@cockroachlabs.com>
Co-authored-by: Jeff Swenson <jeffswenson@betterthannull.com>

Author: 3 people

pkg/cmd/roachtest/tests/vecindex.go

@@ -168,7 +168,7 @@ func registerVectorIndex(r registry.Registry) {
 			backfillPct: 60,
 			preBatchSz:  100,
 			beamSizes:   []int{8, 16, 32, 64, 128},
-			minRecall:   []float64{0.76, 0.83, 0.88, 0.92, 0.94},
+			minRecall:   []float64{0.72, 0.83, 0.88, 0.92, 0.94},
 			rwSplit:     .9,
 		},
 		// Local - no prefix
@@ -210,7 +210,7 @@ func registerVectorIndex(r registry.Registry) {
 			backfillPct: 60,
 			preBatchSz:  100,
 			beamSizes:   []int{8, 16, 32, 64, 128},
-			minRecall:   []float64{0.76, 0.83, 0.88, 0.92, 0.94},
+			minRecall:   []float64{0.72, 0.83, 0.88, 0.92, 0.94},
 			rwSplit:     .9,
 		},
 		// Local - with prefix

pkg/sql/randgen/type.go

@@ -117,6 +117,9 @@ func RandTypeFromSlice(rng *rand.Rand, typs []*types.T) *types.T {
 	case types.BitFamily:
 		return types.MakeBit(int32(rng.Intn(50)))
 	case types.CollatedStringFamily:
+		if typ.Oid() == oidext.T_citext {
+			return types.CIText
+		}
 		return types.MakeCollatedString(types.String, *RandCollationLocale(rng))
 	case types.ArrayFamily:
 		if typ.ArrayContents().Family() == types.AnyFamily {

pkg/sql/rowenc/encoded_datum_test.go

@@ -17,13 +17,15 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/sql/catalog/colinfo"
 	"github.com/cockroachdb/cockroach/pkg/sql/randgen"
 	"github.com/cockroachdb/cockroach/pkg/sql/rowenc"
+	"github.com/cockroachdb/cockroach/pkg/sql/rowenc/keyside"
 	"github.com/cockroachdb/cockroach/pkg/sql/rowenc/valueside"
 	"github.com/cockroachdb/cockroach/pkg/sql/sem/eval"
 	"github.com/cockroachdb/cockroach/pkg/sql/sem/tree"
 	"github.com/cockroachdb/cockroach/pkg/sql/types"
 	"github.com/cockroachdb/cockroach/pkg/util/encoding"
 	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
 	"github.com/cockroachdb/cockroach/pkg/util/randutil"
+	"github.com/stretchr/testify/require"
 )
 
 func TestEncDatum(t *testing.T) {
@@ -150,7 +152,6 @@ func TestEncDatumNull(t *testing.T) {
 			}
 		}
 	}
-
 }
 
 // checkEncDatumCmp encodes the given values using the given encodings,
@@ -770,6 +771,41 @@ func TestEncDatumFingerprintMemory(t *testing.T) {
 	}
 }
 
+func TestDecodesToCanonical(t *testing.T) {
+	rng, _ := randutil.NewTestRand()
+	for i := 0; i < 20; i++ {
+		typ := randgen.RandType(rng)
+		datum := randgen.RandDatum(rng, typ, false)
+
+		buf, err := valueside.Encode(nil, valueside.NoColumnID, datum)
+		require.NoError(t, err)
+		decoded, _, err := valueside.Decode(&tree.DatumAlloc{}, typ, buf)
+
+		valueType := decoded.ResolvedType()
+		require.NoError(t, err)
+		require.True(t, valueType.Identical(typ.Canonical()), "value type %+v not identical to canonical type %+v", valueType, typ)
+		require.Equal(t, decoded.ResolvedType().Oid(), typ.Canonical().Oid())
+
+		direction := encoding.Ascending
+		if rng.Int()%2 == 0 {
+			direction = encoding.Descending
+		}
+
+		if colinfo.ColumnTypeIsIndexable(typ) {
+			buf, err = keyside.Encode(nil, datum, direction)
+			require.NoError(t, err)
+
+			decodedKey, _, err := keyside.Decode(&tree.DatumAlloc{}, typ, buf, direction)
+			require.NoError(t, err)
+
+			keyType := decodedKey.ResolvedType()
+			require.NoError(t, err)
+			require.True(t, keyType.Identical(typ.Canonical()), "key type %+v not identical to canonical type %+v", keyType, typ)
+			require.Equal(t, decodedKey.ResolvedType().Oid(), typ.Canonical().Oid())
+		}
+	}
+}
+
 func encDatumFromEncodedWithDatum(
 	enc catenumpb.DatumEncoding, encoded []byte, datum tree.Datum,
 ) rowenc.EncDatum {

pkg/sql/rowenc/keyside/BUILD.bazel

@@ -14,6 +14,7 @@ go_library(
     deps = [
         "//pkg/geo",
         "//pkg/geo/geopb",
+        "//pkg/sql/oidext",
         "//pkg/sql/pgrepl/lsn",
         "//pkg/sql/sem/tree",
         "//pkg/sql/types",

pkg/sql/rowenc/keyside/decode.go

@@ -11,6 +11,7 @@ import (
 	"github.com/cockroachdb/apd/v3"
 	"github.com/cockroachdb/cockroach/pkg/geo"
 	"github.com/cockroachdb/cockroach/pkg/geo/geopb"
+	"github.com/cockroachdb/cockroach/pkg/sql/oidext"
 	"github.com/cockroachdb/cockroach/pkg/sql/pgrepl/lsn"
 	"github.com/cockroachdb/cockroach/pkg/sql/sem/tree"
 	"github.com/cockroachdb/cockroach/pkg/sql/types"
@@ -131,6 +132,10 @@ func Decode(
 		if err != nil {
 			return nil, nil, err
 		}
+		if valType.Oid() == oidext.T_citext {
+			d, err := a.NewDCIText(r)
+			return d, rkey, err
+		}
 		d, err := a.NewDCollatedString(r, valType.Locale())
 		return d, rkey, err
 	case types.JsonFamily:

pkg/sql/rowenc/valueside/decode.go

@@ -65,6 +65,9 @@ func DecodeUntaggedDatum(
 		if err != nil {
 			return nil, b, err
 		}
+		if t.Oid() == oid.T_name {
+			return a.NewDName(tree.DString(data)), b, nil
+		}
 		return a.NewDString(tree.DString(data)), b, nil
 	case types.CollatedStringFamily:
 		b, data, err := encoding.DecodeUntaggedBytesValue(buf)