Add grails.mongodb.buildIndexes and buildIndexesAsync for MongoDB index creation on startup

grails-data-mongodb/core/build.gradle

@@ -149,7 +149,10 @@ dependencies {
         // test: GenericWebApplicationContext requires ServletContext on the classpath
     }
 
-    testImplementation 'org.slf4j:slf4j-nop' // Prevents warning about missing slf4j implementation during compilation and tests
+    // test: a real SLF4J binding, so that the log a test asserts on is actually emitted.
+    // Quietened to WARN by src/test/resources/logback-test.xml; a test raises the level of the
+    // one logger it is interested in.
+    testImplementation 'ch.qos.logback:logback-classic'
     testImplementation 'org.testcontainers:testcontainers'
     testImplementation 'org.testcontainers:testcontainers-mongodb'
     testImplementation 'org.testcontainers:testcontainers-spock'

grails-data-mongodb/core/src/main/groovy/org/grails/datastore/mapping/mongo/config/MongoSettings.groovy

@@ -103,6 +103,30 @@ interface MongoSettings extends Settings {
     @Deprecated
     String SETTING_ENGINE = 'grails.mongodb.engine'
 
+    /**
+     * Whether GORM creates and reconciles the indexes declared in domain class mapping blocks by
+     * itself when the datastore starts. Defaults to {@code true}.
+     *
+     * <p>Set to {@code false} to leave the indexes on the server exactly as they are, which is
+     * useful when deploying against live data where index changes are applied separately by a
+     * DBA or a migration step rather than by the application on startup. The application can still
+     * build them when it chooses with {@code MongoDatastore.buildIndex()}.
+     *
+     * @since 8.0
+     */
+    String SETTING_BUILD_INDEXES = 'grails.mongodb.buildIndexes'
+
+    /**
+     * Whether index builds run on a background thread rather than blocking the thread that starts
+     * them: the one GORM runs when the datastore starts, and any the application starts with
+     * {@code MongoDatastore.buildIndex()}, including with {@link #SETTING_BUILD_INDEXES} off.
+     * Defaults to {@code false}, which is the historical behavior: startup waits for MongoDB to
+     * finish building every declared index.
+     *
+     * @since 8.0
+     */
+    String SETTING_BUILD_INDEXES_ASYNC = 'grails.mongodb.buildIndexesAsync'
+
     /**
      * Global default storage type for {@code String id} fields when no per-domain
      * {@code id storedAs: ...} mapping is declared. Accepted values are the names (or hex

grails-data-mongodb/core/src/main/groovy/org/grails/datastore/mapping/mongo/connections/AbstractMongoConnectionSourceSettings.groovy

@@ -97,6 +97,28 @@ abstract class AbstractMongoConnectionSourceSettings extends ConnectionSourceSet
      */
     boolean decimalType = true
 
+    /**
+     * Whether GORM creates and reconciles the indexes declared in domain class mapping blocks by
+     * itself when the datastore starts. When {@code false} the indexes on the server are left exactly
+     * as they are and GORM issues no {@code createIndex} or {@code collMod} command of its own until
+     * the application calls {@code MongoDatastore.buildIndex()}; queries are unaffected and continue to
+     * use whatever indexes already exist. Bound from {@code grails.mongodb.buildIndexes}.
+     *
+     * @since 8.0
+     */
+    boolean buildIndexes = true
+
+    /**
+     * Whether index builds run on a background thread instead of blocking the thread that starts them:
+     * the startup build, and any started with {@code MongoDatastore.buildIndex()}, whatever
+     * {@link #buildIndexes} says. MongoDB answers a {@code createIndex} command only once the index has
+     * been built, so with the default {@code false} an application waits at startup for every declared
+     * index. Bound from {@code grails.mongodb.buildIndexesAsync}.
+     *
+     * @since 8.0
+     */
+    boolean buildIndexesAsync = false
+
     /**
      * Nested settings for domains with {@code String id}. Holds the global default
      * {@code defaultStoredAs} switch plus any future string-id configuration.

grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesAsyncSpec.groovy

@@ -0,0 +1,247 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *    https://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing,
+ *  software distributed under the License is distributed on an
+ *  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ *  KIND, either express or implied.  See the License for the
+ *  specific language governing permissions and limitations
+ *  under the License.
+ */
+package org.grails.datastore.gorm.mongo
+
+import java.util.concurrent.ConcurrentLinkedQueue
+
+import ch.qos.logback.classic.Level
+import com.mongodb.ConnectionString
+import com.mongodb.MongoClientSettings
+import com.mongodb.client.MongoClient
+import com.mongodb.client.MongoClients
+import com.mongodb.event.CommandListener
+import com.mongodb.event.CommandStartedEvent
+import grails.gorm.annotation.Entity
+import spock.lang.AutoCleanup
+import spock.lang.Shared
+import spock.util.concurrent.PollingConditions
+
+import org.apache.grails.testing.mongo.AutoStartedMongoSpec
+import org.grails.datastore.mapping.core.DatastoreUtils
+import org.grails.datastore.mapping.mongo.MongoDatastore
+import org.grails.datastore.mapping.mongo.config.MongoSettings
+
+/**
+ * MongoDB answers a {@code createIndexes} command only once the index has been built, so by default
+ * whoever creates the datastore - in an application, the startup thread - waits for every declared index.
+ * This specification pins both halves of that: the default build runs on the calling thread, and with
+ * {@code grails.mongodb.buildIndexesAsync = true} it runs on a background thread instead.
+ *
+ * <p>Which thread issued the command is observed through a driver {@link CommandListener}, which the
+ * synchronous driver invokes on the thread making the call.
+ */
+class BuildIndexesAsyncSpec extends AutoStartedMongoSpec {
+
+    @Shared
+    @AutoCleanup
+    MongoDatastore blockingDatastore
+
+    @Shared
+    @AutoCleanup
+    MongoDatastore asyncDatastore
+
+    @Shared
+    MongoClient blockingClient
+
+    @Shared
+    MongoClient asyncClient
+
+    @Shared
+    CreateIndexThreadRecorder blockingRecorder = new CreateIndexThreadRecorder()
+
+    @Shared
+    CreateIndexThreadRecorder asyncRecorder = new CreateIndexThreadRecorder()
+
+    @Shared
+    String creatingThread
+
+    @Override
+    boolean shouldInitializeDatastore() {
+        false
+    }
+
+    private MongoClient clientFor(String database, CommandListener listener) {
+        MongoClients.create(MongoClientSettings.builder()
+                .applyConnectionString(new ConnectionString(dbContainer.getReplicaSetUrl(database)))
+                .addCommandListener(listener)
+                .build())
+    }
+
+    void setupSpec() {
+        creatingThread = Thread.currentThread().name
+
+        blockingClient = clientFor('blockingIndexDb', blockingRecorder)
+        blockingDatastore = new MongoDatastore(blockingClient,
+                DatastoreUtils.createPropertyResolver(['grails.mongodb.databaseName': 'blockingIndexDb']),
+                BlockingIndexThing)
+
+        asyncClient = clientFor('asyncIndexDb', asyncRecorder)
+        asyncDatastore = new MongoDatastore(asyncClient,
+                DatastoreUtils.createPropertyResolver([
+                        'grails.mongodb.databaseName'              : 'asyncIndexDb',
+                        (MongoSettings.SETTING_BUILD_INDEXES_ASYNC): true
+                ]),
+                AsyncIndexThing)
+    }
+
+    void cleanupSpec() {
+        blockingClient?.close()
+        asyncClient?.close()
+    }
+
+    void "test the index build blocks the thread creating the datastore by default"() {
+        expect: "the setting is off"
+        !blockingDatastore.isBuildIndexesAsync()
+
+        and: "the index already exists by the time the constructor has returned"
+        [name: 1] in BlockingIndexThing.collection.listIndexes()*.key
+
+        and: "it was built by the thread that created the datastore, which therefore waited for it"
+        blockingRecorder.threads == [creatingThread]
+    }
+
+    void "test the index build runs on a background thread when enabled"() {
+        given:
+        def conditions = new PollingConditions(timeout: 30)
+
+        expect: "the setting is on"
+        asyncDatastore.isBuildIndexesAsync()
+
+        when: "the background build has had a chance to run"
+        conditions.eventually {
+            assert [name: 1] in AsyncIndexThing.collection.listIndexes()*.key
+        }
+
+        then: "the command was issued by the datastore's own index build thread"
+        asyncRecorder.threads.size() == 1
+        asyncRecorder.threads.first().startsWith('gorm-mongo-index-build-default-')
+
+        and: "not by the thread that created the datastore, which did not wait for it"
+        asyncRecorder.threads.first() != creatingThread
+
+        when: "the startup worker has no more work"
+        Thread startupWorker = asyncRecorder.workers.first()
+        startupWorker.join(10000)
+
+        then: "it is released without closing the datastore"
+        startupWorker.daemon
+        !startupWorker.alive
+
+        when: "another build is explicitly requested after the worker has exited"
+        asyncDatastore.buildIndex()
+
+        then: "a new worker performs the build"
+        conditions.eventually {
+            assert asyncRecorder.workers.size() == 2
+        }
+        !asyncRecorder.workers.last().is(startupWorker)
+
+        when:
+        Thread subsequentWorker = asyncRecorder.workers.last()
+        subsequentWorker.join(10000)
+
+        then:
+        !subsequentWorker.alive
+    }
+
+    void "test a background build is announced by its own thread, ahead of everything it logs"() {
+        given:
+        def conditions = new PollingConditions(timeout: 30)
+        def log = new CapturedLog('org.grails.datastore.mapping', Level.INFO)
+        def isAnnouncement = {
+            it.formattedMessage.startsWith('Building the indexes declared by the domain classes for connection [default] on a background thread')
+        }
+        def isSummary = { it.formattedMessage.startsWith('Index build for database [announcedIndexDb] finished') }
+
+        when:
+        def datastore = new MongoDatastore(['grails.mongodb.url'                       : dbContainer.getReplicaSetUrl('announcedIndexDb'),
+                                            (MongoSettings.SETTING_BUILD_INDEXES_ASYNC): true] as Map,
+                AnnouncedIndexThing)
+        conditions.eventually {
+            assert log.events.any(isSummary)
+        }
+        def events = log.events
+        def announcement = events.find(isAnnouncement)
+        def summary = events.find(isSummary)
+
+        then: "said by the thread that goes on to build, so it cannot trail the summary a fast build logs"
+        announcement.threadName == summary.threadName
+        events.indexOf(announcement) < events.indexOf(summary)
+
+        cleanup:
+        datastore?.close()
+        log?.close()
+    }
+}
+
+/**
+ * Records the thread each {@code createIndexes} command was issued from.
+ */
+class CreateIndexThreadRecorder implements CommandListener {
+
+    private final Queue<Thread> recorded = new ConcurrentLinkedQueue<>()
+
+    @Override
+    void commandStarted(CommandStartedEvent event) {
+        if (event.commandName == 'createIndexes') {
+            recorded.add(Thread.currentThread())
+        }
+    }
+
+    List<String> getThreads() {
+        recorded.collect { it.name }.unique()
+    }
+
+    List<Thread> getWorkers() {
+        recorded.toList().unique()
+    }
+}
+
+@Entity
+class BlockingIndexThing {
+    String name
+
+    static mapping = {
+        version false
+        collection 'blockingIndexThing'
+        name index: true
+    }
+}
+
+@Entity
+class AsyncIndexThing {
+    String name
+
+    static mapping = {
+        version false
+        collection 'asyncIndexThing'
+        name index: true
+    }
+}
+
+@Entity
+class AnnouncedIndexThing {
+    String name
+
+    static mapping = {
+        version false
+        collection 'announcedIndexThing'
+        name index: true
+    }
+}