CBG-5511: Stop db loading opting out of mobile collection if bootstrap migration is done

base/bootstrap.go

@@ -90,6 +90,16 @@ type BootstrapConnection interface {
 	// CachedBootstrapTargets returns the cached bootstrap doc target for each bucket, for observability purposes. Values are "system_mobile", "default".
 	// The snapshot is not guaranteed to be consistent across concurrent updates.
 	CachedBootstrapTargets() map[string]string
+	// BucketBootstrapTargetIsSystemMobile reports whether the bucket's bootstrap docs (registry,
+	// dbconfig, cbgt cfg) currently reside in _system._mobile — i.e. the bucket's bootstrap
+	// metadata has already migrated, or the bucket was created opted-in from the start. Resolves and
+	// caches the target exactly as SetBucketBootstrapTargetHint does when the bucket is not yet
+	// cached; optInHint governs only the no-registry-yet case. Unlike IsMigrationComplete this does
+	// not depend on a migration status doc ever having been stamped, so it also catches a bucket that
+	// was opted in from the start and never ran a migration. Used to reject loading a database that
+	// has not opted into the system metadata collection on a bucket whose bootstrap metadata already
+	// lives in _system._mobile.
+	BucketBootstrapTargetIsSystemMobile(ctx context.Context, bucket string, optInHint bool) (bool, error)
 	// Close releases any long-lived connections
 	Close()
 }
@@ -454,6 +464,20 @@ func (cc *CouchbaseCluster) SetBucketBootstrapTargetHint(ctx context.Context, bu
 	return nil
 }
 
+// BucketBootstrapTargetIsSystemMobile resolves the bucket's bootstrap-doc target (probing and
+// caching via SetBucketBootstrapTargetHint if not already cached) and reports whether it is
+// _system._mobile. See the interface doc for usage.
+func (cc *CouchbaseCluster) BucketBootstrapTargetIsSystemMobile(ctx context.Context, bucketName string, optInHint bool) (bool, error) {
+	if cc == nil {
+		return false, errors.New("nil CouchbaseCluster")
+	}
+	if err := cc.SetBucketBootstrapTargetHint(ctx, bucketName, optInHint); err != nil {
+		return false, err
+	}
+	target, _ := cc.bucketBootstrapTargets.Load(bucketName)
+	return target == bucketTargetSystemMobile, nil
+}
+
 // CachedBootstrapTargets returns the cached bootstrap doc target for each bucket, for observability purposes. Values are "system_mobile", "default".
 // The snapshot is not guaranteed to be consistent across concurrent updates.
 func (cc *CouchbaseCluster) CachedBootstrapTargets() map[string]string {

base/rosmar_cluster.go

@@ -113,9 +113,11 @@ func (c *RosmarCluster) getDefaultDataStore(ctx context.Context, bucketName stri
 }
 
 // metadataDataStores opens the bucket and returns the primary metadata DataStore plus an optional
-// fallback. The per-bucket target cache (populated by SetBucketBootstrapTargetHint or the lazy
-// probe in this function) determines the choice; absent a cached entry the connection-wide flag
-// drives the decision.
+// fallback. On every open it probes for an existing _sync:registry (mirroring
+// CouchbaseCluster.ensureBucketBootstrapTargetCached) and caches the resolved target when found;
+// the per-bucket target cache then determines the choice, and absent any registry the
+// connection-wide flag drives the decision. SetBucketBootstrapTargetHint may also populate the
+// cache ahead of the first op.
 //
 // Fallback semantics are symmetric so reads can self-heal across a stale cache:
 //   - target=systemMobile: primary=_system._mobile, fallback=_default._default (legacy docs not
@@ -159,32 +161,34 @@ func (c *RosmarCluster) metadataDataStores(ctx context.Context, bucketName strin
 		return nil, nil, nil, fmt.Errorf("unexpected type for system collection for bucket %q: %T", bucketName, systemDS)
 	}
 
+	// Mirror CouchbaseCluster.ensureBucketBootstrapTargetCached: probe for an existing registry on
+	// every bucket open, regardless of the connection-wide flag, so a peer node discovers a bucket
+	// whose bootstrap docs already live in _system._mobile even when its own flag is off. Only caches
+	// when a registry is actually found (authoritative); a registry-less bucket stays uncached so a
+	// later SetBucketBootstrapTargetHint(optIn=true) can still claim _system._mobile. Will only run
+	// if not already cached.
+	if _, alreadyCached := c.bucketBootstrapTargets.Load(bucketName); !alreadyCached {
+		if target, found := c.probeRegistryLocation(ctx, systemCol, defaultCol); found {
+			c.bucketBootstrapTargets.LoadOrStore(bucketName, target)
+		}
+	}
+
 	cached, hasCachedTarget := c.bucketBootstrapTargets.Load(bucketName)
 	useSystemMobile := c.useSystemMetadataCollection
 	if hasCachedTarget {
 		useSystemMobile = cached == bucketTargetSystemMobile
 	}
 	if !useSystemMobile {
-		// Reads/writes route to default first. When this bucket has any opt-in indication —
-		// either cached as bucketTargetDefault (set by SetBucketBootstrapTargetHint after probing
-		// a legacy registry) or the connection-wide flag — systemMobile is wired as a self-heal
-		// fallback so a peer-migrated bucket doesn't return not-found from a stale-cached
-		// perspective. A bucket with no cache entry and no connection-wide opt-in skips the
-		// fallback entirely so non-opt-in clusters don't pay an extra _system._mobile probe per op.
+		// Reads/writes route to default first. When this bucket has a registry (cached as
+		// bucketTargetDefault by the probe above) or the connection-wide flag is set, systemMobile is
+		// wired as a self-heal fallback so a peer-migrated bucket doesn't return not-found from a
+		// stale-cached perspective. A registry-less bucket with the flag off stays uncached and skips
+		// the fallback — there is nothing to self-heal to.
 		if hasCachedTarget || c.useSystemMetadataCollection {
 			return defaultCol, systemCol, closer, nil
 		}
 		return defaultCol, nil, closer, nil
 	}
-	// Lazy-cache the target only when a registry is found in one of the collections — that's
-	// the only case where we have authoritative information. With no registry yet the choice
-	// stays uncached so a subsequent SetBucketBootstrapTargetHint(optIn=true) can still claim
-	// _system._mobile.
-	if _, alreadyCached := c.bucketBootstrapTargets.Load(bucketName); !alreadyCached {
-		if target, found := c.probeRegistryLocation(ctx, systemCol, defaultCol); found {
-			c.bucketBootstrapTargets.LoadOrStore(bucketName, target)
-		}
-	}
 	if c.bucketBootstrapMigrationComplete(bucketName) {
 		return systemCol, nil, closer, nil
 	}
@@ -246,6 +250,18 @@ func (c *RosmarCluster) SetBucketBootstrapTargetHint(ctx context.Context, bucket
 	return nil
 }
 
+// BucketBootstrapTargetIsSystemMobile mirrors CouchbaseCluster.BucketBootstrapTargetIsSystemMobile.
+func (c *RosmarCluster) BucketBootstrapTargetIsSystemMobile(ctx context.Context, bucketName string, optInHint bool) (bool, error) {
+	if c == nil {
+		return false, errors.New("nil RosmarCluster")
+	}
+	if err := c.SetBucketBootstrapTargetHint(ctx, bucketName, optInHint); err != nil {
+		return false, err
+	}
+	target, _ := c.bucketBootstrapTargets.Load(bucketName)
+	return target == bucketTargetSystemMobile, nil
+}
+
 // probeRegistryLocation reports where _sync:registry already lives, or that it doesn't exist yet.
 // Mirrors CouchbaseCluster.probeRegistryLocation.
 func (c *RosmarCluster) probeRegistryLocation(ctx context.Context, systemCol, defaultCol *rosmar.Collection) (target bucketBootstrapTarget, found bool) {

db/database_error.go

@@ -15,38 +15,40 @@ type DatabaseError struct {
 }
 
 var DatabaseErrorMap = map[databaseErrorCode]string{
-	DatabaseBucketConnectionError:       "Error connecting to bucket",
-	DatabaseInvalidDatastore:            "Collection(s) not available",
-	DatabaseInitSyncInfoError:           "Error initializing sync info",
-	DatabaseInitializationIndexError:    "Error initializing database indexes",
-	DatabaseCreateDatabaseContextError:  "Error creating database context",
-	DatabaseSGRClusterError:             "Error with fetching SGR cluster definition",
-	DatabaseCreateReplicationError:      "Error creating replication during database init",
-	DatabaseOnlineProcessError:          "Error attempting to start online process",
-	DatabaseAllowConflictsError:         "Allow conflicts is set to true",
-	DatabaseEnableStarChannelFalseError: "Enable star channel is set to false",
-	DatabaseClusterCompatVersionError:   "Bucket has metadata from a newer Sync Gateway cluster compat version",
-	DatabaseInvalidResyncPartitions:     "resync_partitions exceeds number of vBuckets",
-	DatabaseNoMetadataStore:             "No metadata store for db metadata to be stored in",
+	DatabaseBucketConnectionError:         "Error connecting to bucket",
+	DatabaseInvalidDatastore:              "Collection(s) not available",
+	DatabaseInitSyncInfoError:             "Error initializing sync info",
+	DatabaseInitializationIndexError:      "Error initializing database indexes",
+	DatabaseCreateDatabaseContextError:    "Error creating database context",
+	DatabaseSGRClusterError:               "Error with fetching SGR cluster definition",
+	DatabaseCreateReplicationError:        "Error creating replication during database init",
+	DatabaseOnlineProcessError:            "Error attempting to start online process",
+	DatabaseAllowConflictsError:           "Allow conflicts is set to true",
+	DatabaseEnableStarChannelFalseError:   "Enable star channel is set to false",
+	DatabaseClusterCompatVersionError:     "Bucket has metadata from a newer Sync Gateway cluster compat version",
+	DatabaseInvalidResyncPartitions:       "resync_partitions exceeds number of vBuckets",
+	DatabaseNoMetadataStore:               "No metadata store for db metadata to be stored in",
+	DatabaseSystemCollectionOptInRequired: "Database must enable use_system_metadata_collection on a bucket whose bootstrap metadata has migrated to the system collection",
 }
 
 type databaseErrorCode uint8
 
 // Error codes exposed for each error a database can encounter on load. These codes are consumed by Capella so must remain stable.
 const (
-	DatabaseBucketConnectionError       databaseErrorCode = 1
-	DatabaseInvalidDatastore            databaseErrorCode = 2
-	DatabaseInitSyncInfoError           databaseErrorCode = 3
-	DatabaseInitializationIndexError    databaseErrorCode = 4
-	DatabaseCreateDatabaseContextError  databaseErrorCode = 5
-	DatabaseSGRClusterError             databaseErrorCode = 6
-	DatabaseCreateReplicationError      databaseErrorCode = 7
-	DatabaseOnlineProcessError          databaseErrorCode = 8
-	DatabaseAllowConflictsError         databaseErrorCode = 9
-	DatabaseEnableStarChannelFalseError databaseErrorCode = 10
-	DatabaseClusterCompatVersionError   databaseErrorCode = 11
-	DatabaseInvalidResyncPartitions     databaseErrorCode = 12
-	DatabaseNoMetadataStore             databaseErrorCode = 13
+	DatabaseBucketConnectionError         databaseErrorCode = 1
+	DatabaseInvalidDatastore              databaseErrorCode = 2
+	DatabaseInitSyncInfoError             databaseErrorCode = 3
+	DatabaseInitializationIndexError      databaseErrorCode = 4
+	DatabaseCreateDatabaseContextError    databaseErrorCode = 5
+	DatabaseSGRClusterError               databaseErrorCode = 6
+	DatabaseCreateReplicationError        databaseErrorCode = 7
+	DatabaseOnlineProcessError            databaseErrorCode = 8
+	DatabaseAllowConflictsError           databaseErrorCode = 9
+	DatabaseEnableStarChannelFalseError   databaseErrorCode = 10
+	DatabaseClusterCompatVersionError     databaseErrorCode = 11
+	DatabaseInvalidResyncPartitions       databaseErrorCode = 12
+	DatabaseNoMetadataStore               databaseErrorCode = 13
+	DatabaseSystemCollectionOptInRequired databaseErrorCode = 14
 )
 
 func NewDatabaseError(code databaseErrorCode) *DatabaseError {

rest/admin_api.go

@@ -90,6 +90,19 @@ func (h *handler) handleCreateDB() error {
 			base.WarnfCtx(h.ctx(), "Failed to resolve bootstrap target for bucket %q: %v", base.MD(bucket), err)
 		}
 
+		// Once a bucket's bootstrap metadata has migrated to _system._mobile, every database on it
+		// must opt into the system metadata collection. A non-opted-in database would stamp its
+		// bootstrap config into _default._default, which peer nodes reading the migrated registry
+		// from _system._mobile can never find. Reject the create before persisting anything.
+		if !optInHint {
+			targetIsSystemMobile, targetErr := h.server.BootstrapContext.Connection.BucketBootstrapTargetIsSystemMobile(h.ctx(), bucket, false)
+			if targetErr != nil {
+				base.WarnfCtx(h.ctx(), "Unable to determine bootstrap target for bucket %s while validating use_system_metadata_collection for db %s: %v", base.MD(bucket), base.MD(dbName), targetErr)
+			} else if targetIsSystemMobile {
+				return base.HTTPErrorf(http.StatusBadRequest, "database %s must enable use_system_metadata_collection: bucket %s bootstrap metadata resides in %s", base.MD(dbName), base.MD(bucket), base.MD(base.MobileSystemScopeAndCollectionName().String()))
+			}
+		}
+
 		metadataID, metadataIDError := h.server.BootstrapContext.ComputeMetadataIDForDbConfig(h.ctx(), config)
 		if metadataIDError != nil {
 			base.WarnfCtx(h.ctx(), "Unable to compute metadata ID - using standard metadataID.  Error: %v", metadataIDError)

rest/defaultcollectiondroptest/default_collection_drop_test.go

@@ -58,10 +58,6 @@ func TestDatabaseSurvivesDefaultCollectionDropAfterMetadataMigration(t *testing.
 	namedDataStore, err := tb.GetNamedDataStore(0)
 	require.NoError(t, err)
 	namedCollection := base.ScopeAndCollectionName{Scope: namedDataStore.ScopeName(), Collection: namedDataStore.CollectionName()}
-	namedDatastore2, err := tb.GetNamedDataStore(1)
-	require.NoError(t, err)
-	namedCollection2 := base.ScopeAndCollectionName{Scope: namedDatastore2.ScopeName(), Collection: namedDatastore2.CollectionName()}
-
 	rt := rest.NewRestTester(t, &rest.RestTesterConfig{
 		CustomTestBucket: tb.NoCloseClone(),
 		PersistentConfig: true,
@@ -161,19 +157,4 @@ func TestDatabaseSurvivesDefaultCollectionDropAfterMetadataMigration(t *testing.
 	resp = rt.SendAdminRequest(http.MethodGet, "/"+keyspace+"/doc1", "")
 	rest.RequireStatus(t, resp, http.StatusOK)
 	assert.Contains(t, resp.Body.String(), `"value":"hello"`)
-
-	// Try create a new db against the bucket with no _default without opting into the system metadata collection.
-	// This should fail because the legacy metadata store is gone.
-	newDbConfig := rt.NewDbConfig()
-	newDbConfig.Scopes = rest.ScopesConfig{
-		namedCollection2.ScopeName(): rest.ScopeConfig{
-			Collections: rest.CollectionsConfig{
-				namedCollection2.CollectionName(): {},
-			},
-		},
-	}
-	newDbConfig.UseSystemMobileMetadataCollection = base.Ptr(false)
-	resp = rt.CreateDatabase("newdb", newDbConfig)
-	rest.RequireStatus(t, resp, http.StatusInternalServerError)
-	assert.Contains(t, resp.Body.String(), "_default._default does not exist on bucket")
 }