diff --git a/base/bootstrap.go b/base/bootstrap.go index f323e70ce9..d5f03684c9 100644 --- a/base/bootstrap.go +++ b/base/bootstrap.go @@ -90,6 +90,16 @@ type BootstrapConnection interface { // CachedBootstrapTargets returns the cached bootstrap doc target for each bucket, for observability purposes. Values are "system_mobile", "default". // The snapshot is not guaranteed to be consistent across concurrent updates. CachedBootstrapTargets() map[string]string + // BucketBootstrapTargetIsSystemMobile reports whether the bucket's bootstrap docs (registry, + // dbconfig, cbgt cfg) currently reside in _system._mobile — i.e. the bucket's bootstrap + // metadata has already migrated, or the bucket was created opted-in from the start. Resolves and + // caches the target exactly as SetBucketBootstrapTargetHint does when the bucket is not yet + // cached; optInHint governs only the no-registry-yet case. Unlike IsMigrationComplete this does + // not depend on a migration status doc ever having been stamped, so it also catches a bucket that + // was opted in from the start and never ran a migration. Used to reject loading a database that + // has not opted into the system metadata collection on a bucket whose bootstrap metadata already + // lives in _system._mobile. + BucketBootstrapTargetIsSystemMobile(ctx context.Context, bucket string, optInHint bool) (bool, error) // Close releases any long-lived connections Close() } @@ -454,6 +464,20 @@ func (cc *CouchbaseCluster) SetBucketBootstrapTargetHint(ctx context.Context, bu return nil } +// BucketBootstrapTargetIsSystemMobile resolves the bucket's bootstrap-doc target (probing and +// caching via SetBucketBootstrapTargetHint if not already cached) and reports whether it is +// _system._mobile. See the interface doc for usage. +func (cc *CouchbaseCluster) BucketBootstrapTargetIsSystemMobile(ctx context.Context, bucketName string, optInHint bool) (bool, error) { + if cc == nil { + return false, errors.New("nil CouchbaseCluster") + } + if err := cc.SetBucketBootstrapTargetHint(ctx, bucketName, optInHint); err != nil { + return false, err + } + target, _ := cc.bucketBootstrapTargets.Load(bucketName) + return target == bucketTargetSystemMobile, nil +} + // CachedBootstrapTargets returns the cached bootstrap doc target for each bucket, for observability purposes. Values are "system_mobile", "default". // The snapshot is not guaranteed to be consistent across concurrent updates. func (cc *CouchbaseCluster) CachedBootstrapTargets() map[string]string { diff --git a/base/rosmar_cluster.go b/base/rosmar_cluster.go index 4b4bf1d8d3..cc59745773 100644 --- a/base/rosmar_cluster.go +++ b/base/rosmar_cluster.go @@ -113,9 +113,11 @@ func (c *RosmarCluster) getDefaultDataStore(ctx context.Context, bucketName stri } // metadataDataStores opens the bucket and returns the primary metadata DataStore plus an optional -// fallback. The per-bucket target cache (populated by SetBucketBootstrapTargetHint or the lazy -// probe in this function) determines the choice; absent a cached entry the connection-wide flag -// drives the decision. +// fallback. On every open it probes for an existing _sync:registry (mirroring +// CouchbaseCluster.ensureBucketBootstrapTargetCached) and caches the resolved target when found; +// the per-bucket target cache then determines the choice, and absent any registry the +// connection-wide flag drives the decision. SetBucketBootstrapTargetHint may also populate the +// cache ahead of the first op. // // Fallback semantics are symmetric so reads can self-heal across a stale cache: // - target=systemMobile: primary=_system._mobile, fallback=_default._default (legacy docs not @@ -159,32 +161,34 @@ func (c *RosmarCluster) metadataDataStores(ctx context.Context, bucketName strin return nil, nil, nil, fmt.Errorf("unexpected type for system collection for bucket %q: %T", bucketName, systemDS) } + // Mirror CouchbaseCluster.ensureBucketBootstrapTargetCached: probe for an existing registry on + // every bucket open, regardless of the connection-wide flag, so a peer node discovers a bucket + // whose bootstrap docs already live in _system._mobile even when its own flag is off. Only caches + // when a registry is actually found (authoritative); a registry-less bucket stays uncached so a + // later SetBucketBootstrapTargetHint(optIn=true) can still claim _system._mobile. Will only run + // if not already cached. + if _, alreadyCached := c.bucketBootstrapTargets.Load(bucketName); !alreadyCached { + if target, found := c.probeRegistryLocation(ctx, systemCol, defaultCol); found { + c.bucketBootstrapTargets.LoadOrStore(bucketName, target) + } + } + cached, hasCachedTarget := c.bucketBootstrapTargets.Load(bucketName) useSystemMobile := c.useSystemMetadataCollection if hasCachedTarget { useSystemMobile = cached == bucketTargetSystemMobile } if !useSystemMobile { - // Reads/writes route to default first. When this bucket has any opt-in indication — - // either cached as bucketTargetDefault (set by SetBucketBootstrapTargetHint after probing - // a legacy registry) or the connection-wide flag — systemMobile is wired as a self-heal - // fallback so a peer-migrated bucket doesn't return not-found from a stale-cached - // perspective. A bucket with no cache entry and no connection-wide opt-in skips the - // fallback entirely so non-opt-in clusters don't pay an extra _system._mobile probe per op. + // Reads/writes route to default first. When this bucket has a registry (cached as + // bucketTargetDefault by the probe above) or the connection-wide flag is set, systemMobile is + // wired as a self-heal fallback so a peer-migrated bucket doesn't return not-found from a + // stale-cached perspective. A registry-less bucket with the flag off stays uncached and skips + // the fallback — there is nothing to self-heal to. if hasCachedTarget || c.useSystemMetadataCollection { return defaultCol, systemCol, closer, nil } return defaultCol, nil, closer, nil } - // Lazy-cache the target only when a registry is found in one of the collections — that's - // the only case where we have authoritative information. With no registry yet the choice - // stays uncached so a subsequent SetBucketBootstrapTargetHint(optIn=true) can still claim - // _system._mobile. - if _, alreadyCached := c.bucketBootstrapTargets.Load(bucketName); !alreadyCached { - if target, found := c.probeRegistryLocation(ctx, systemCol, defaultCol); found { - c.bucketBootstrapTargets.LoadOrStore(bucketName, target) - } - } if c.bucketBootstrapMigrationComplete(bucketName) { return systemCol, nil, closer, nil } @@ -246,6 +250,18 @@ func (c *RosmarCluster) SetBucketBootstrapTargetHint(ctx context.Context, bucket return nil } +// BucketBootstrapTargetIsSystemMobile mirrors CouchbaseCluster.BucketBootstrapTargetIsSystemMobile. +func (c *RosmarCluster) BucketBootstrapTargetIsSystemMobile(ctx context.Context, bucketName string, optInHint bool) (bool, error) { + if c == nil { + return false, errors.New("nil RosmarCluster") + } + if err := c.SetBucketBootstrapTargetHint(ctx, bucketName, optInHint); err != nil { + return false, err + } + target, _ := c.bucketBootstrapTargets.Load(bucketName) + return target == bucketTargetSystemMobile, nil +} + // probeRegistryLocation reports where _sync:registry already lives, or that it doesn't exist yet. // Mirrors CouchbaseCluster.probeRegistryLocation. func (c *RosmarCluster) probeRegistryLocation(ctx context.Context, systemCol, defaultCol *rosmar.Collection) (target bucketBootstrapTarget, found bool) { diff --git a/db/database_error.go b/db/database_error.go index 56be338a3a..d71aed214b 100644 --- a/db/database_error.go +++ b/db/database_error.go @@ -15,38 +15,40 @@ type DatabaseError struct { } var DatabaseErrorMap = map[databaseErrorCode]string{ - DatabaseBucketConnectionError: "Error connecting to bucket", - DatabaseInvalidDatastore: "Collection(s) not available", - DatabaseInitSyncInfoError: "Error initializing sync info", - DatabaseInitializationIndexError: "Error initializing database indexes", - DatabaseCreateDatabaseContextError: "Error creating database context", - DatabaseSGRClusterError: "Error with fetching SGR cluster definition", - DatabaseCreateReplicationError: "Error creating replication during database init", - DatabaseOnlineProcessError: "Error attempting to start online process", - DatabaseAllowConflictsError: "Allow conflicts is set to true", - DatabaseEnableStarChannelFalseError: "Enable star channel is set to false", - DatabaseClusterCompatVersionError: "Bucket has metadata from a newer Sync Gateway cluster compat version", - DatabaseInvalidResyncPartitions: "resync_partitions exceeds number of vBuckets", - DatabaseNoMetadataStore: "No metadata store for db metadata to be stored in", + DatabaseBucketConnectionError: "Error connecting to bucket", + DatabaseInvalidDatastore: "Collection(s) not available", + DatabaseInitSyncInfoError: "Error initializing sync info", + DatabaseInitializationIndexError: "Error initializing database indexes", + DatabaseCreateDatabaseContextError: "Error creating database context", + DatabaseSGRClusterError: "Error with fetching SGR cluster definition", + DatabaseCreateReplicationError: "Error creating replication during database init", + DatabaseOnlineProcessError: "Error attempting to start online process", + DatabaseAllowConflictsError: "Allow conflicts is set to true", + DatabaseEnableStarChannelFalseError: "Enable star channel is set to false", + DatabaseClusterCompatVersionError: "Bucket has metadata from a newer Sync Gateway cluster compat version", + DatabaseInvalidResyncPartitions: "resync_partitions exceeds number of vBuckets", + DatabaseNoMetadataStore: "No metadata store for db metadata to be stored in", + DatabaseSystemCollectionOptInRequired: "Database must enable use_system_metadata_collection on a bucket whose bootstrap metadata has migrated to the system collection", } type databaseErrorCode uint8 // Error codes exposed for each error a database can encounter on load. These codes are consumed by Capella so must remain stable. const ( - DatabaseBucketConnectionError databaseErrorCode = 1 - DatabaseInvalidDatastore databaseErrorCode = 2 - DatabaseInitSyncInfoError databaseErrorCode = 3 - DatabaseInitializationIndexError databaseErrorCode = 4 - DatabaseCreateDatabaseContextError databaseErrorCode = 5 - DatabaseSGRClusterError databaseErrorCode = 6 - DatabaseCreateReplicationError databaseErrorCode = 7 - DatabaseOnlineProcessError databaseErrorCode = 8 - DatabaseAllowConflictsError databaseErrorCode = 9 - DatabaseEnableStarChannelFalseError databaseErrorCode = 10 - DatabaseClusterCompatVersionError databaseErrorCode = 11 - DatabaseInvalidResyncPartitions databaseErrorCode = 12 - DatabaseNoMetadataStore databaseErrorCode = 13 + DatabaseBucketConnectionError databaseErrorCode = 1 + DatabaseInvalidDatastore databaseErrorCode = 2 + DatabaseInitSyncInfoError databaseErrorCode = 3 + DatabaseInitializationIndexError databaseErrorCode = 4 + DatabaseCreateDatabaseContextError databaseErrorCode = 5 + DatabaseSGRClusterError databaseErrorCode = 6 + DatabaseCreateReplicationError databaseErrorCode = 7 + DatabaseOnlineProcessError databaseErrorCode = 8 + DatabaseAllowConflictsError databaseErrorCode = 9 + DatabaseEnableStarChannelFalseError databaseErrorCode = 10 + DatabaseClusterCompatVersionError databaseErrorCode = 11 + DatabaseInvalidResyncPartitions databaseErrorCode = 12 + DatabaseNoMetadataStore databaseErrorCode = 13 + DatabaseSystemCollectionOptInRequired databaseErrorCode = 14 ) func NewDatabaseError(code databaseErrorCode) *DatabaseError { diff --git a/rest/admin_api.go b/rest/admin_api.go index 640ee39181..5bbdc60146 100644 --- a/rest/admin_api.go +++ b/rest/admin_api.go @@ -90,6 +90,19 @@ func (h *handler) handleCreateDB() error { base.WarnfCtx(h.ctx(), "Failed to resolve bootstrap target for bucket %q: %v", base.MD(bucket), err) } + // Once a bucket's bootstrap metadata has migrated to _system._mobile, every database on it + // must opt into the system metadata collection. A non-opted-in database would stamp its + // bootstrap config into _default._default, which peer nodes reading the migrated registry + // from _system._mobile can never find. Reject the create before persisting anything. + if !optInHint { + targetIsSystemMobile, targetErr := h.server.BootstrapContext.Connection.BucketBootstrapTargetIsSystemMobile(h.ctx(), bucket, false) + if targetErr != nil { + base.WarnfCtx(h.ctx(), "Unable to determine bootstrap target for bucket %s while validating use_system_metadata_collection for db %s: %v", base.MD(bucket), base.MD(dbName), targetErr) + } else if targetIsSystemMobile { + return base.HTTPErrorf(http.StatusBadRequest, "database %s must enable use_system_metadata_collection: bucket %s bootstrap metadata resides in %s", base.MD(dbName), base.MD(bucket), base.MD(base.MobileSystemScopeAndCollectionName().String())) + } + } + metadataID, metadataIDError := h.server.BootstrapContext.ComputeMetadataIDForDbConfig(h.ctx(), config) if metadataIDError != nil { base.WarnfCtx(h.ctx(), "Unable to compute metadata ID - using standard metadataID. Error: %v", metadataIDError) diff --git a/rest/defaultcollectiondroptest/default_collection_drop_test.go b/rest/defaultcollectiondroptest/default_collection_drop_test.go index 7cde165399..f7eb52f835 100644 --- a/rest/defaultcollectiondroptest/default_collection_drop_test.go +++ b/rest/defaultcollectiondroptest/default_collection_drop_test.go @@ -58,10 +58,6 @@ func TestDatabaseSurvivesDefaultCollectionDropAfterMetadataMigration(t *testing. namedDataStore, err := tb.GetNamedDataStore(0) require.NoError(t, err) namedCollection := base.ScopeAndCollectionName{Scope: namedDataStore.ScopeName(), Collection: namedDataStore.CollectionName()} - namedDatastore2, err := tb.GetNamedDataStore(1) - require.NoError(t, err) - namedCollection2 := base.ScopeAndCollectionName{Scope: namedDatastore2.ScopeName(), Collection: namedDatastore2.CollectionName()} - rt := rest.NewRestTester(t, &rest.RestTesterConfig{ CustomTestBucket: tb.NoCloseClone(), PersistentConfig: true, @@ -161,19 +157,4 @@ func TestDatabaseSurvivesDefaultCollectionDropAfterMetadataMigration(t *testing. resp = rt.SendAdminRequest(http.MethodGet, "/"+keyspace+"/doc1", "") rest.RequireStatus(t, resp, http.StatusOK) assert.Contains(t, resp.Body.String(), `"value":"hello"`) - - // Try create a new db against the bucket with no _default without opting into the system metadata collection. - // This should fail because the legacy metadata store is gone. - newDbConfig := rt.NewDbConfig() - newDbConfig.Scopes = rest.ScopesConfig{ - namedCollection2.ScopeName(): rest.ScopeConfig{ - Collections: rest.CollectionsConfig{ - namedCollection2.CollectionName(): {}, - }, - }, - } - newDbConfig.UseSystemMobileMetadataCollection = base.Ptr(false) - resp = rt.CreateDatabase("newdb", newDbConfig) - rest.RequireStatus(t, resp, http.StatusInternalServerError) - assert.Contains(t, resp.Body.String(), "_default._default does not exist on bucket") } diff --git a/rest/metadatamigrationtest/metadata_migration_test.go b/rest/metadatamigrationtest/metadata_migration_test.go index 301624ab27..30c869a498 100644 --- a/rest/metadatamigrationtest/metadata_migration_test.go +++ b/rest/metadatamigrationtest/metadata_migration_test.go @@ -1811,3 +1811,154 @@ func TestDeleteAllDbsCompletesPendingBootstrapMigration(t *testing.T) { assert.Equal(c, base.MigrationStateComplete, status.Bootstrap.State, "deleting all dbs should let the pending bucket migration complete") }, 5*time.Second, 100*time.Millisecond) } + +func TestBootstrapMigrationDoneAndCreateDbWithoutMobileSystemCollectionOptIn(t *testing.T) { + base.TestRequiresCollections(t) + + ctx := base.TestCtx(t) + + // NewRestTesterCluster gives both nodes persistent config (so each has a bootstrap connection), a + // shared config group (so the peer sees this node's persisted configs), and one shared bucket. + rtc := rest.NewRestTesterCluster(t, &rest.RestTesterClusterConfig{NumNodes: 2}) + defer rtc.Close(ctx) + + rt := rtc.Node(0) + rt2 := rtc.Node(1) + tb := rt.TestBucket + + dataStore1, err := tb.GetNamedDataStore(0) + require.NoError(t, err) + db1Cfg := rt.NewDbConfig() + db1Cfg.UseSystemMobileMetadataCollection = base.Ptr(false) + db1Cfg.Scopes = rest.ScopesConfig{ + dataStore1.ScopeName(): rest.ScopeConfig{ + Collections: rest.CollectionsConfig{dataStore1.CollectionName(): {}}, + }, + } + rest.RequireStatus(t, rt.CreateDatabase("db1", db1Cfg), http.StatusCreated) + + db1Cfg.UseSystemMobileMetadataCollection = base.Ptr(true) + rest.RequireStatus(t, rt.UpsertDbConfig("db1", db1Cfg), http.StatusCreated) + rt.ServerContext().ForceClusterCompatRefresh(t, rt.Context()) + resp := rt.SendAdminRequest(http.MethodPost, "/db1/_metadata_migration?action=start", "") + rest.RequireStatus(t, resp, http.StatusOK) + + rt.WaitForMetadataMigrationStatusForDB(db.BackgroundProcessStateCompleted, "db1") + + conn := rt.ServerContext().BootstrapContext.Connection + require.EventuallyWithT(rt.TB(), func(c *assert.CollectT) { + rt.ServerContext().RecheckPendingBucketMetadataMigrations(ctx) + status, _, err := conn.GetMetadataMigrationStatus(ctx, tb.GetName()) + if !assert.NoError(c, err) { + return + } + if !assert.NotNil(c, status) { + return + } + assert.Equal(c, base.MigrationStateComplete, status.Bootstrap.State) + }, 5*time.Second, 100*time.Millisecond) + + // delete db1 + rest.RequireStatus(t, rt.SendAdminRequest(http.MethodDelete, "/db1/", ""), http.StatusOK) + + // The bucket's bootstrap metadata has migrated to _system._mobile. Creating a new database on + // this bucket that has not opted into the system metadata collection must be rejected: its + // bootstrap config would land in _default._default, where peer nodes reading the migrated + // registry from _system._mobile could never find it, leaving the cluster in a split state. + dataStore2, err := tb.GetNamedDataStore(1) + require.NoError(t, err) + db2Cfg := rt.NewDbConfig() + db2Cfg.UseSystemMobileMetadataCollection = base.Ptr(false) + db2Cfg.Scopes = rest.ScopesConfig{ + dataStore2.ScopeName(): rest.ScopeConfig{ + Collections: rest.CollectionsConfig{dataStore2.CollectionName(): {}}, + }, + } + resp = rt.CreateDatabase("db2", db2Cfg) + rest.RequireStatus(t, resp, http.StatusBadRequest) + require.Contains(t, resp.Body.String(), "use_system_metadata_collection") + + // The peer node must not pick up a phantom db2 through config polling either — nothing was persisted. + rt2.ServerContext().ForceDbConfigsReload(t, ctx) + rest.RequireStatus(t, rt2.SendAdminRequest(http.MethodGet, "/db2/", ""), http.StatusNotFound) +} + +// TestOptedInBucketRejectsCreateDbWithoutMobileSystemCollectionOptIn: db1 opts into the system +// metadata collection from the start, so the bucket's bootstrap registry lands directly in +// _system._mobile with no migration ever running. A subsequent non-opted-in db must still be +// rejected. This is the case that IsMigrationComplete cannot detect — no migration status doc is +// ever stamped, so it stays false — which is why the guard checks the resolved bootstrap target. +func TestOptedInBucketRejectsCreateDbWithoutMobileSystemCollectionOptIn(t *testing.T) { + base.TestRequiresCollections(t) + + ctx := base.TestCtx(t) + + // NewRestTesterCluster gives both nodes persistent config (so each has a bootstrap connection), a + // shared config group (so the peer sees this node's persisted configs), and one shared bucket. + rtc := rest.NewRestTesterCluster(t, &rest.RestTesterClusterConfig{NumNodes: 2}) + defer rtc.Close(ctx) + + rt := rtc.Node(0) + rt2 := rtc.Node(1) + tb := rt.TestBucket + + // db1 opts into the system metadata collection from creation, so its registry + dbconfig land in + // _system._mobile directly — no migration required. + dataStore1, err := tb.GetNamedDataStore(0) + require.NoError(t, err) + db1Cfg := rt.NewDbConfig() + db1Cfg.UseSystemMobileMetadataCollection = base.Ptr(true) + db1Cfg.Scopes = rest.ScopesConfig{ + dataStore1.ScopeName(): rest.ScopeConfig{ + Collections: rest.CollectionsConfig{dataStore1.CollectionName(): {}}, + }, + } + rest.RequireStatus(t, rt.CreateDatabase("db1", db1Cfg), http.StatusCreated) + + conn := rt.ServerContext().BootstrapContext.Connection + + // No migration ran, so no status doc was ever stamped complete: IsMigrationComplete is false even + // though the bucket's bootstrap metadata physically lives in _system._mobile. The guard cannot + // rely on IsMigrationComplete here — it must resolve the bootstrap target. + require.False(t, conn.IsMigrationComplete(tb.GetName()), "no migration ran, so IsMigrationComplete should be false") + targetIsSystemMobile, err := conn.BucketBootstrapTargetIsSystemMobile(ctx, tb.GetName(), false) + require.NoError(t, err) + require.True(t, targetIsSystemMobile, "opted-in db1 should have placed the bootstrap registry in _system._mobile") + + // Creating a non-opted-in db2 on this bucket must be rejected. + dataStore2, err := tb.GetNamedDataStore(1) + require.NoError(t, err) + db2Cfg := rt.NewDbConfig() + db2Cfg.UseSystemMobileMetadataCollection = base.Ptr(false) + db2Cfg.Scopes = rest.ScopesConfig{ + dataStore2.ScopeName(): rest.ScopeConfig{ + Collections: rest.CollectionsConfig{dataStore2.CollectionName(): {}}, + }, + } + resp := rt.CreateDatabase("db2", db2Cfg) + rest.RequireStatus(t, resp, http.StatusBadRequest) + require.Contains(t, resp.Body.String(), "use_system_metadata_collection") + + // db2 was rejected at create and never persisted, so the peer must not pick it up through config + // polling either. + rt2.ServerContext().ForceDbConfigsReload(t, ctx) + rest.RequireStatus(t, rt2.SendAdminRequest(http.MethodGet, "/db2/", ""), http.StatusNotFound) + + // The opted-in db1, however, must load on the peer: it reads the registry from _system._mobile + // (probed on bucket open) and the guard leaves opted-in databases alone. The config fetch and the + // asynchronous online transition race with this assertion, so re-trigger the reload and poll the + // db endpoint with soft assertions until db1 is present and Online. (WaitForDatabaseState can't be + // used here: its GetDatabaseRoot requires a 200, hard-failing the test on a transient 404.) + require.EventuallyWithT(t, func(c *assert.CollectT) { + rt2.ServerContext().ForceDbConfigsReload(t, ctx) + resp := rt2.SendAdminRequest(http.MethodGet, "/db1/", "") + if !assert.Equal(c, http.StatusOK, resp.Code) { + return + } + var dbRoot rest.DatabaseRoot + if !assert.NoError(c, base.JSONUnmarshal(resp.BodyBytes(), &dbRoot)) { + return + } + assert.Equal(c, "Online", dbRoot.State) + }, 10*time.Second, 100*time.Millisecond) +} diff --git a/rest/server_context.go b/rest/server_context.go index 36146547ed..cf7c554682 100644 --- a/rest/server_context.go +++ b/rest/server_context.go @@ -728,6 +728,22 @@ func (sc *ServerContext) _getOrAddDatabaseFromConfig(ctx context.Context, config "Duplicate database name %q", dbName) } + // A bucket whose bootstrap metadata already lives in _system._mobile (migrated, or opted-in from + // the start) keeps its registry/dbconfig docs there. A database that has not opted into the system + // metadata collection would persist its bootstrap config into _default._default, where peer nodes + // reading the migrated registry from _system._mobile can never find it — leaving the cluster in bad state. + if sc.BootstrapContext != nil && sc.BootstrapContext.Connection != nil && !resolveUseSystemMetadataCollection(sc.Config, &config.DbConfig) { + targetIsSystemMobile, targetErr := sc.BootstrapContext.Connection.BucketBootstrapTargetIsSystemMobile(ctx, spec.BucketName, false) + if targetErr != nil { + base.WarnfCtx(ctx, "Unable to determine bootstrap target for bucket %s while validating use_system_metadata_collection for db %s: %v", base.MD(spec.BucketName), base.MD(dbName), targetErr) + } else if targetIsSystemMobile { + if options.loadFromBucket { + sc._handleInvalidDatabaseConfig(ctx, spec.BucketName, config, db.NewDatabaseError(db.DatabaseSystemCollectionOptInRequired)) + } + return nil, base.HTTPErrorf(http.StatusBadRequest, "database %s must enable use_system_metadata_collection: bucket %s bootstrap metadata resides in %s", base.MD(dbName), base.MD(spec.BucketName), base.MD(base.MobileSystemScopeAndCollectionName().String())) + } + } + if config.DbConfig.CacheConfig != nil { if config.DbConfig.CacheConfig.ChannelCacheConfig != nil { if config.DbConfig.CacheConfig.ChannelCacheConfig.EnableStarChannel != nil && !*config.DbConfig.CacheConfig.ChannelCacheConfig.EnableStarChannel {