diff --git a/grails-data-mongodb/core/build.gradle b/grails-data-mongodb/core/build.gradle index 0ce227148fe..d5b707b3fb9 100644 --- a/grails-data-mongodb/core/build.gradle +++ b/grails-data-mongodb/core/build.gradle @@ -149,7 +149,10 @@ dependencies { // test: GenericWebApplicationContext requires ServletContext on the classpath } - testImplementation 'org.slf4j:slf4j-nop' // Prevents warning about missing slf4j implementation during compilation and tests + // test: a real SLF4J binding, so that the log a test asserts on is actually emitted. + // Quietened to WARN by src/test/resources/logback-test.xml; a test raises the level of the + // one logger it is interested in. + testImplementation 'ch.qos.logback:logback-classic' testImplementation 'org.testcontainers:testcontainers' testImplementation 'org.testcontainers:testcontainers-mongodb' testImplementation 'org.testcontainers:testcontainers-spock' diff --git a/grails-data-mongodb/core/src/main/groovy/org/grails/datastore/mapping/mongo/MongoDatastore.java b/grails-data-mongodb/core/src/main/groovy/org/grails/datastore/mapping/mongo/MongoDatastore.java index 0f27b577cd8..23c1af5e202 100644 --- a/grails-data-mongodb/core/src/main/groovy/org/grails/datastore/mapping/mongo/MongoDatastore.java +++ b/grails-data-mongodb/core/src/main/groovy/org/grails/datastore/mapping/mongo/MongoDatastore.java @@ -20,11 +20,16 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.HashMap; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import groovy.lang.Closure; @@ -34,6 +39,9 @@ import com.mongodb.MongoClientSettings; import com.mongodb.MongoCommandException; +import com.mongodb.MongoInterruptedException; +import com.mongodb.MongoNamespace; +import com.mongodb.MongoSocketException; import com.mongodb.client.MongoClient; import com.mongodb.client.MongoIterable; import com.mongodb.client.model.IndexOptions; @@ -51,6 +59,7 @@ import org.springframework.context.SmartLifecycle; import org.springframework.context.support.StaticMessageSource; import org.springframework.core.env.PropertyResolver; +import org.springframework.scheduling.concurrent.CustomizableThreadFactory; import org.springframework.transaction.PlatformTransactionManager; import grails.gorm.multitenancy.Tenants; @@ -79,6 +88,7 @@ import org.grails.datastore.mapping.core.Session; import org.grails.datastore.mapping.core.StatelessDatastore; import org.grails.datastore.mapping.core.connections.ConnectionSource; +import org.grails.datastore.mapping.core.connections.ConnectionSourceSettingsBuilder; import org.grails.datastore.mapping.core.connections.ConnectionSources; import org.grails.datastore.mapping.core.connections.ConnectionSourcesInitializer; import org.grails.datastore.mapping.core.connections.ConnectionSourcesListener; @@ -166,15 +176,56 @@ public class MongoDatastore extends AbstractDatastore implements MappingContext. protected final boolean stateless; protected final boolean codecEngine; protected final boolean transactionsEnabled; + protected final boolean buildIndexes; + protected final boolean buildIndexesAsync; + + /** + * Runs the startup index build off the thread that creates the datastore when + * {@code grails.mongodb.buildIndexesAsync} is enabled; {@code null} otherwise. A single thread, + * so the indexes are still built one at a time per connection. The worker expires after one idle + * second, releasing the worker while allowing subsequent calls to {@link #buildIndex()}. + * + *

Not final: a shut down executor cannot be restarted, so {@link #start()} replaces the one + * {@link #stop()} shut down. + */ + private volatile ExecutorService indexBuildExecutor; + + /** + * Set when a build did not run to the end because the datastore was stopping, or was requested while it + * was stopped, for {@link #start()} to run again. A build interrupted by {@link #stop()} sets it itself + * as it exits, so a build that finished in the meantime is not run twice. + */ + private volatile boolean indexBuildPending; + + /** + * Set first thing in {@link #close()}. A connection added at runtime starts its index build only while + * this is clear, and only after it is registered, so a close cannot miss a build that has started. + */ + private volatile boolean closed; + + /** + * How long {@link #start()} waits for a build that {@link #stop()} interrupted to finish exiting. One that is + * going to exit does so within milliseconds of the interrupt. + */ + private static final long INDEX_BUILD_EXIT_TIMEOUT_MILLIS = 1000; + + /** The summary for the current build, scoped to its thread so the protected index hook is preserved. */ + private final ThreadLocal indexBuildSummary = new ThreadLocal<>(); private volatile Boolean transactionsSupported; private volatile boolean warnedTransactionsUnsupported = false; - protected CodecRegistry codecRegistry; + // Volatile: an asynchronous index build reads it from its own thread while the @Autowired setters below + // can still be replacing it. + protected volatile CodecRegistry codecRegistry; protected final ConfigurableApplicationEventPublisher eventPublisher; protected final PlatformTransactionManager transactionManager; protected final GormEnhancer gormEnhancer; protected final ConnectionSources connectionSources; protected final FlushModeType defaultFlushMode; - protected final Map datastoresByConnectionSource = new LinkedHashMap<>(); + /** + * Concurrent because it is written by the connection sources listener, which can add a child for a + * connection registered at runtime, while {@link #close()} may be iterating it. + */ + protected final Map datastoresByConnectionSource = new ConcurrentHashMap<>(); protected final MultiTenancySettings.MultiTenancyMode multiTenancyMode; protected final TenantResolver tenantResolver; protected final AutoTimestampEventListener autoTimestampEventListener; @@ -214,6 +265,13 @@ public MongoDatastore(final ConnectionSources() { @@ -247,6 +308,11 @@ public void newConnectionSource(final ConnectionSource getConnecti } /** - * Builds the MongoDB index for this datastore + * Creates and reconciles the indexes declared by the domain classes mapped to this datastore's connection. + * + *

GORM calls this itself when the datastore starts, unless {@code grails.mongodb.buildIndexes} is + * {@code false}; that setting only stops GORM building by itself, so an application that leaves indexes + * alone at startup can call this when it chooses to build them. Each named connection builds its own + * domain classes: call it on {@link #getDatastoreForConnection(String)} for those. + * + *

Each index is created by a command that the server answers only once the index has been built, + * so this blocks the calling thread for as long as MongoDB takes to build every declared index. With + * {@code grails.mongodb.buildIndexesAsync} enabled the work goes to a background thread and this + * returns immediately instead. */ public void buildIndex() { + String connection = connectionName(); + // Before either mode: a build on the calling thread would otherwise run against the client close() is closing. + if (closed) { + LOG.warn("An index build was requested for connection [{}] after the datastore was closed, so it was not started.", + connection); + return; + } + ExecutorService executor = this.indexBuildExecutor; + if (executor == null) { + runIndexBuild(null); + return; + } + if (!executor.isShutdown()) { + try { + executor.execute(() -> { + // The first thing the build does: said only of a build that is under way, and ahead of + // everything it logs, which it would not be if the submitting thread said it. + LOG.info("Building the indexes declared by the domain classes for connection [{}] on a " + + "background thread. Startup does not wait for them, so a query issued before its index " + + "exists is served without it.", connection); + runIndexBuild(executor); + }); + return; + } + catch (RejectedExecutionException e) { + // Shut down between the check and the submission. + } + } + if (closed) { + LOG.warn("An index build was requested for connection [{}] after the datastore was closed, so it was not started.", + connection); + } + else { + indexBuildPending = true; + LOG.info("An index build was requested for connection [{}] while the datastore is stopped; it will run when " + + "the datastore is restarted.", connection); + } + } + + /** + * The builds GORM starts by itself - at startup, and for a connection added at runtime - which + * {@code grails.mongodb.buildIndexes = false} turns off. An explicit {@link #buildIndex()} is not affected. + */ + private void buildIndexAutomatically() { + if (!buildIndexes) { + LOG.info("Index creation on startup is disabled for connection [{}] by [{} = false]. The indexes already " + + "present on the server are left untouched; call buildIndex() to create the declared ones.", + connectionName(), buildIndexesSettingName()); + return; + } + buildIndex(); + } + + private String connectionName() { + return connectionSources.getDefaultConnectionSource().getName(); + } + + /** + * The setting that turned index building off, as an operator would look for it: a named connection takes + * its own value if it declares one and inherits the top level one otherwise, so both are named. + */ + private String buildIndexesSettingName() { + String connection = connectionName(); + if (ConnectionSource.DEFAULT.equals(connection)) { + return MongoSettings.SETTING_BUILD_INDEXES; + } + return MongoSettings.SETTING_CONNECTIONS + "." + connection + ".buildIndexes (or " + + MongoSettings.SETTING_BUILD_INDEXES + ")"; + } + + /** + * Runs one build and reports it, whether it finished or not. + * + * @param executor the executor running it, or {@code null} for a build on the caller's thread, whose + * failure is left to propagate to the caller + */ + private void runIndexBuild(ExecutorService executor) { + long startedAt = System.nanoTime(); + // Telling a created index from one that was already there costs a listIndexes per indexed + // collection, and its only product is the summary, so it is skipped when that would not be logged. + IndexBuildSummary summary = new IndexBuildSummary(LOG.isInfoEnabled()); + Exception failure = null; + try { + buildDeclaredIndexes(summary); + } + catch (Exception e) { + // An Error is left to propagate as it is, without the summary. + failure = e; + if (e instanceof InterruptedException) { + // Caught here rather than by whoever interrupted, so the flag it cleared is put back. + Thread.currentThread().interrupt(); + } + } + long elapsedMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startedAt); + if (failure == null) { + logFinishedIndexBuild(summary, elapsedMillis); + return; + } + if (executor == null) { + logUnfinishedIndexBuild(summary, elapsedMillis, false); + // Unchanged, checked or not: an initializeIndices override may throw one without declaring it. + MongoDatastore.rethrow(failure); + return; + } + // Nothing is waiting on this thread, so an error that would have failed startup has to be + // reported here or it is lost entirely. + boolean abandoned = executor.isShutdown() && explainedByShutdown(failure); + if (abandoned) { + indexBuildPending = true; + // toString rather than the message: an interrupted driver call can arrive wrapped in + // an exception that carries no message of its own. + LOG.debug("The background index build was abandoned because the datastore is shutting down: {}", + failure.toString(), failure); + } + else { + LOG.error("The background index build failed: {}. The application is running without the " + + "indexes that were not created.", failure.getMessage(), failure); + } + logUnfinishedIndexBuild(summary, elapsedMillis, abandoned); + } + + /** + * Whether a failure is one that shutting the datastore down produces: an interruption, the driver refusing a + * client that has been closed, or a socket closed under a call in progress. Anything else - a duplicate key + * on a unique index, say - is a genuine failure that merely coincided with the shutdown, and is not to be + * reported as an orderly one. + */ + private static boolean explainedByShutdown(Throwable failure) { + for (Throwable cause = failure; cause != null; cause = cause.getCause() == cause ? null : cause.getCause()) { + if (cause instanceof InterruptedException || cause instanceof MongoInterruptedException || + cause instanceof IllegalStateException || cause instanceof MongoSocketException) { + return true; + } + } + return false; + } + + @SuppressWarnings("unchecked") + private static void rethrow(Exception failure) throws E { + throw (E) failure; + } + + private void logFinishedIndexBuild(IndexBuildSummary summary, long elapsedMillis) { + if (summary.applied() == 0 && summary.recreated == 0 && summary.failures == 0) { + LOG.debug("No indexes are declared by the {} domain class(es) mapped to database [{}]", + summary.entities, defaultDatabase); + return; + } + if (summary.failures == 0) { + LOG.info("Index build for database [{}] finished in {}ms: {}, from {} domain class(es)", + defaultDatabase, elapsedMillis, summary.describe(), summary.entities); + } + else { + LOG.warn("Index build for database [{}] finished in {}ms: {}, from {} domain class(es). " + + "The failures are reported above.", + defaultDatabase, elapsedMillis, summary.describe(), summary.entities); + } + } + + /** + * Reports how far a build got before it stopped, which is what an operator needs when a background + * build fails partway: the error names the cause, this says what was applied before it. + */ + private void logUnfinishedIndexBuild(IndexBuildSummary summary, long elapsedMillis, boolean abandoned) { + String message = "Index build for database [{}] did not finish, stopping after {}ms at {} of {} domain class(es): {}"; + if (abandoned) { + LOG.debug(message, defaultDatabase, elapsedMillis, summary.entities, summary.entitiesTotal, summary.describe()); + } + else { + LOG.warn(message, defaultDatabase, elapsedMillis, summary.entities, summary.entitiesTotal, summary.describe()); + } + } + + /** + * Creates and reconciles the indexes declared by every entity mapped to this datastore. MongoDB answers + * each {@code createIndex} only once the index exists, so the time this takes is the time the caller — + * startup, or the background build thread — actually spends waiting. + */ + private void buildDeclaredIndexes(IndexBuildSummary summary) { + List entities = new ArrayList<>(); for (PersistentEntity entity : this.mappingContext.getPersistentEntities()) { // Only create Mongo templates for entities that are mapped with Mongo - if (!entity.isExternal()) { - if (entity.isMultiTenant() && multiTenancyMode == MultiTenancySettings.MultiTenancyMode.SCHEMA) continue; - + if (!entity.isExternal() && + !(entity.isMultiTenant() && multiTenancyMode == MultiTenancySettings.MultiTenancyMode.SCHEMA)) { + entities.add(entity); + } + } + summary.entitiesTotal = entities.size(); + IndexBuildSummary previousSummary = indexBuildSummary.get(); + indexBuildSummary.set(summary); + try { + for (PersistentEntity entity : entities) { initializeIndices(entity); + summary.entities++; + } + } + finally { + if (previousSummary == null) { + indexBuildSummary.remove(); + } + else { + indexBuildSummary.set(previousSummary); } } } + /** + * The indexes a collection already had when the build reached it, listed once on first use and then + * reused. {@code createIndex} is idempotent and answers the same way whether or not it had to build + * anything — the driver hands back only the index name, discarding the {@code numIndexesBefore} / + * {@code numIndexesAfter} the server reports — so what was there beforehand is what distinguishes an + * index this build created from one it merely confirmed. + * + *

Listed lazily so that an entity declaring no indexes costs no round trip, and not at all when the + * summary is not being classified. Successful changes are recorded so later declarations on the same + * keys see the current name and TTL, and are not counted as new indexes. A conflict re-lists instead + * of trusting the snapshot: whatever the server reports as conflicting may have appeared since. + */ + private static final class ExistingIndexes { + + private final com.mongodb.client.MongoCollection collection; + + private final IndexBuildSummary summary; + + private List indexes; + + private boolean listed; + + /** Why the last listing failed, for the conflict it leaves unreconciled to report. */ + private RuntimeException listingFailure; + + /** + * False once a listing of this collection has failed, whether its first or a re-listing after a conflict. + * Its declarations are then applied without being classified; other collections keep their breakdown. + */ + private boolean readable = true; + + private ExistingIndexes(com.mongodb.client.MongoCollection collection, IndexBuildSummary summary) { + this.collection = collection; + this.summary = summary; + } + + /** + * @return the known current indexes, or {@code null} if they could not be listed + */ + private List get() { + return listed ? indexes : refresh(); + } + + /** + * Lists the indexes from the server now, replacing what was known. + * + * @return the current indexes, or {@code null} if they could not be listed + */ + private List refresh() { + listed = true; + try { + indexes = collection.listIndexes().into(new ArrayList<>()); + listingFailure = null; + readable = true; + } catch (RuntimeException e) { + // Not fatal: the build can still create indexes, it just cannot report which of them + // were new. Losing the breakdown is not worth failing a startup over. + LOG.debug("Could not list the existing indexes of collection [{}]: {}", + collection.getNamespace().getCollectionName(), e.getMessage(), e); + indexes = null; + listingFailure = e; + readable = false; + } + return indexes; + } + + private void record(Document keys, String name, Long expireAfterSeconds) { + if (indexes == null) { + return; + } + Document existing = findIndexByKeyPattern(indexes, keys); + if (existing != null) { + indexes.remove(existing); + } + Document index = new Document("key", new Document(keys)).append("name", name); + if (expireAfterSeconds != null) { + index.append(INDEX_EXPIRE_AFTER_SECONDS, expireAfterSeconds); + } + indexes.add(index); + } + + /** + * @return whether an index on these keys was already there, or {@code null} if that is not known: the + * build is not classifying, or this collection's indexes could not be listed + */ + private Boolean contains(Document keys) { + if (!summary.classifying || !readable) { + return null; + } + List existing = get(); + if (existing == null) { + return null; + } + return findIndexByKeyPattern(existing, keys) != null; + } + } + + /** + * Counts the work one index build did, so that it can be summarised once at the end rather than a line + * per index. + */ + private static final class IndexBuildSummary { + + /** Domain classes whose indexes have been applied. */ + private int entities; + + /** Domain classes the build set out to cover. */ + private int entitiesTotal; + + private int created; + + /** Dropped and built again for {@code recreateOnConflict}, which costs a full build. */ + private int recreated; + + private int alreadyPresent; + + private int failures; + + /** Applied on a collection whose existing indexes could not be listed, so neither created nor present. */ + private int unclassified; + + /** + * Whether created and already-present indexes are to be told apart at all, which costs a listing per + * collection and is only worth it when the summary will be logged. + */ + private final boolean classifying; + + /** + * One listing per collection rather than per entity: every class in an inheritance hierarchy maps + * to its root's collection, and several classes can name the same one. Sharing it also keeps them + * consistent, so a second class declaring keys the first has just created sees them as present. + */ + private final Map existingIndexes = new HashMap<>(); + + private IndexBuildSummary(boolean classifying) { + this.classifying = classifying; + } + + private ExistingIndexes existingIndexesOf(com.mongodb.client.MongoCollection collection) { + return existingIndexes.computeIfAbsent(collection.getNamespace(), namespace -> new ExistingIndexes(collection, this)); + } + + /** Declarations applied other than by a drop and recreate. */ + private int applied() { + return created + alreadyPresent + unclassified; + } + + private String describe() { + StringBuilder outcome = new StringBuilder(); + if (created + alreadyPresent > 0 || unclassified == 0) { + outcome.append(created).append(" created, "); + if (recreated > 0) { + outcome.append(recreated).append(" recreated, "); + } + outcome.append(alreadyPresent).append(" already present"); + if (unclassified > 0) { + // Partial: what the collections that could be listed established still stands. + outcome.append(", ").append(unclassified).append(" applied without a listing"); + } + } + else { + // Nothing was classified, whether because it was not asked for or no collection could be listed. + outcome.append(unclassified).append(" index declaration(s) applied"); + if (recreated > 0) { + outcome.append(", ").append(recreated).append(" recreated"); + } + } + if (failures > 0) { + outcome.append(", ").append(failures).append(" failed"); + } + return outcome.toString(); + } + } + + /** How a conflicting declaration was resolved. */ + private enum Reconciliation { + /** The existing index was changed in place, with no rebuild. */ + UPDATED, + /** The existing index was dropped and the declared one built from scratch. */ + RECREATED, + /** The existing index was left as it was. */ + FAILED + } + /** * @return The default flush mode */ @@ -763,6 +1219,31 @@ public boolean isTransactionsEnabled() { } } + /** + * Whether GORM creates and reconciles the indexes declared in the domain class mapping blocks by itself: + * when the datastore starts, and for connections and domain classes added later. Disabled with + * {@code grails.mongodb.buildIndexes = false}, which leaves the indexes on the server as they are until + * {@link #buildIndex()} is called. + * + * @return {@code true} if GORM builds the declared indexes by itself + * @since 8.0 + */ + public boolean isBuildIndexes() { + return buildIndexes; + } + + /** + * Whether index builds run on a background thread instead of blocking the thread that starts them - the + * one GORM starts on startup, and any {@link #buildIndex()} the application calls. Enabled with + * {@code grails.mongodb.buildIndexesAsync = true}. + * + * @return {@code true} if declared indexes are built asynchronously + * @since 8.0 + */ + public boolean isBuildIndexesAsync() { + return buildIndexesAsync; + } + public String getDatabaseName(PersistentEntity entity) { if (entity.isMultiTenant() && multiTenancyMode == MultiTenancySettings.MultiTenancyMode.SCHEMA) { return Tenants.currentId(getClass()).toString(); @@ -854,7 +1335,7 @@ public void persistentEntityAdded(PersistentEntity entity) { } }); - buildIndex(); + buildIndexAutomatically(); return new MongoGormEnhancer(this, transactionManager, settings) { @Override @@ -925,12 +1406,24 @@ protected void registerEventListeners(ConfigurableApplicationEventPublisher even } /** - * Indexes any properties that are mapped with index:true + * Indexes any properties that are mapped with index:true. Called for both startup builds and + * entities registered later, so subclasses can customise index creation on either path. + * + *

With {@code grails.mongodb.buildIndexesAsync} enabled the startup build calls this on a background + * thread, and it can do so before the constructor of a subclass has finished. An override must not + * depend on state that its own constructor or field initializers set up. * * @param entity The entity */ protected void initializeIndices(final PersistentEntity entity) { + IndexBuildSummary summary = indexBuildSummary.get(); + // Outside a build nothing reports the counts, so there is nothing to classify for. + initializeIndices(entity, summary != null ? summary : new IndexBuildSummary(false)); + } + + private void initializeIndices(final PersistentEntity entity, final IndexBuildSummary summary) { final com.mongodb.client.MongoCollection collection = getCollection(entity); + final ExistingIndexes existingIndexes = summary.existingIndexesOf(collection); final ClassMapping classMapping = entity.getMapping(); if (classMapping != null) { final MongoCollection mappedForm = classMapping.getMappedForm(); @@ -938,21 +1431,20 @@ protected void initializeIndices(final PersistentEntity entity) { List indices = mappedForm.getIndices(); for (MongoCollection.Index index : indices) { createOrUpdateIndex(entity, collection, new Document(index.getDefinition()), - index.getOptions(), "with definition [" + index.getDefinition() + "]"); + index.getOptions(), "with definition [" + index.getDefinition() + "]", + summary, existingIndexes); } for (Map compoundIndex : mappedForm.getCompoundIndices()) { - - Map indexAttributes = null; - if (compoundIndex.containsKey(INDEX_ATTRIBUTES)) { - Object o = compoundIndex.remove(INDEX_ATTRIBUTES); - if (o instanceof Map) { - indexAttributes = (Map) o; - } - } - Document indexDef = new Document(compoundIndex); + // A copy, because the declaration is shared: every connection builds from the same + // mapping, and taking the attributes out of the original would leave the next build + // an index with none. + Map declaration = new LinkedHashMap(compoundIndex); + Object attributes = declaration.remove(INDEX_ATTRIBUTES); + Map indexAttributes = attributes instanceof Map ? (Map) attributes : null; + Document indexDef = new Document(declaration); createOrUpdateIndex(entity, collection, indexDef, indexAttributes, - "compound index with definition [" + indexDef + "]"); + "compound index with definition [" + indexDef + "]", summary, existingIndexes); } } } @@ -977,7 +1469,7 @@ protected void initializeIndices(final PersistentEntity entity) { } } createOrUpdateIndex(entity, collection, dbObject, options, - "on property [" + property.getName() + "]"); + "on property [" + property.getName() + "]", summary, existingIndexes); } } @@ -998,7 +1490,8 @@ protected void initializeIndices(final PersistentEntity entity) { */ private void createOrUpdateIndex(PersistentEntity entity, com.mongodb.client.MongoCollection collection, - Document keys, Map rawOptions, String descriptor) { + Document keys, Map rawOptions, String descriptor, + IndexBuildSummary summary, ExistingIndexes existingIndexes) { Map options = rawOptions != null ? new HashMap<>(rawOptions) : new HashMap<>(); // Control flag — not a Mongo index option. @@ -1015,13 +1508,42 @@ private void createOrUpdateIndex(PersistentEntity entity, indexOptions.expireAfter(expireAfterSeconds, TimeUnit.SECONDS); } + // Asked before the index is created, while the answer still means something. + Boolean present = existingIndexes.contains(keys); + long startedAt = System.nanoTime(); try { - collection.createIndex(keys, indexOptions); + String indexName = collection.createIndex(keys, indexOptions); + existingIndexes.record(keys, indexName, expireAfterSeconds); + if (present == null) { + summary.unclassified++; + } + else if (present) { + summary.alreadyPresent++; + } + else { + summary.created++; + } + // Unclassified, nothing says whether the index was new, so the line does not claim either. + String applied = present == null ? "Applied" : present ? "Confirmed" : "Created"; + LOG.debug("{} index for entity [{}] {} in {}ms", applied, + entity.getName(), descriptor, TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startedAt)); } catch (MongoCommandException e) { if (e.getErrorCode() == INDEX_OPTIONS_CONFLICT_CODE) { - reconcileIndexConflict(entity, collection, keys, indexOptions, - expireAfterSeconds, recreateOnConflict, descriptor, e); + Reconciliation reconciliation = reconcileIndexConflict(entity, collection, existingIndexes, keys, + indexOptions, expireAfterSeconds, recreateOnConflict, descriptor, e); + if (reconciliation == Reconciliation.UPDATED) { + // An index was already on these keys and was changed in place rather than added. + summary.alreadyPresent++; + } + else if (reconciliation == Reconciliation.RECREATED) { + // Counted apart: a rebuild costs as much as a new index, whatever was there before. + summary.recreated++; + } + else { + summary.failures++; + } } else { + summary.failures++; LOG.error("Failed to create index for entity [{}] {}: {}", entity.getName(), descriptor, e.getMessage(), e); } @@ -1033,23 +1555,29 @@ private void createOrUpdateIndex(PersistentEntity entity, * different options. A TTL difference is the common, safe case (e.g. a configurable retention * changed between restarts) and is updated in place via {@code collMod}; anything else needs an * explicit {@code recreateOnConflict:true} to authorise the drop-and-recreate. + * + * @return how the conflict was resolved */ - private void reconcileIndexConflict(PersistentEntity entity, + private Reconciliation reconcileIndexConflict(PersistentEntity entity, com.mongodb.client.MongoCollection collection, + ExistingIndexes existingIndexes, Document keys, IndexOptions desired, Long expireAfterSeconds, boolean recreateOnConflict, String descriptor, MongoCommandException original) { - Document existing; - try { - existing = findIndexByKeyPattern(collection, keys); - } catch (RuntimeException listError) { + // Listed afresh: the index the server just reported may not have existed when this collection was + // first listed - another instance or another connection can have created it since. + List indexes = existingIndexes.refresh(); + if (indexes == null) { + // The reason the listing failed, which is what an operator can act on - often a missing + // listIndexes privilege - with the conflict that made it matter attached. LOG.error("Failed to create index for entity [{}] {} and could not inspect existing indexes: {}", - entity.getName(), descriptor, listError.getMessage(), original); - return; + entity.getName(), descriptor, existingIndexes.listingFailure.getMessage(), original); + return Reconciliation.FAILED; } + Document existing = findIndexByKeyPattern(indexes, keys); if (existing == null) { LOG.error("Failed to create index for entity [{}] {}: {}", entity.getName(), descriptor, original.getMessage(), original); - return; + return Reconciliation.FAILED; } String existingName = existing.getString("name"); @@ -1064,9 +1592,10 @@ private void reconcileIndexConflict(PersistentEntity entity, .runCommand(new Document("collMod", getCollectionName(entity)) .append("index", new Document("name", existingName) .append(INDEX_EXPIRE_AFTER_SECONDS, expireAfterSeconds))); + existingIndexes.record(keys, existingName, expireAfterSeconds); LOG.info("Updated TTL of index [{}] on entity [{}] to {}s", existingName, entity.getName(), expireAfterSeconds); - return; + return Reconciliation.UPDATED; } catch (MongoCommandException collModError) { // collMod can't make every change (e.g. add a TTL to a non-TTL index on older // servers) — fall through to recreate (if authorised) rather than fail outright. @@ -1078,19 +1607,22 @@ private void reconcileIndexConflict(PersistentEntity entity, if (recreateOnConflict) { try { collection.dropIndex(existingName); - collection.createIndex(keys, desired); + String indexName = collection.createIndex(keys, desired); + existingIndexes.record(keys, indexName, expireAfterSeconds); LOG.info("Recreated index [{}] on entity [{}] {}", existingName, entity.getName(), descriptor); + return Reconciliation.RECREATED; } catch (MongoCommandException recreateError) { LOG.error("Failed to recreate index [{}] on entity [{}] {}: {}", existingName, entity.getName(), descriptor, recreateError.getMessage(), recreateError); + return Reconciliation.FAILED; } - return; } LOG.error( "Index conflict for entity [{}] {}: an index [{}] already exists on the same keys with different options. " + "Declare indexAttributes:[recreateOnConflict:true] to drop and recreate it. Original error: {}", entity.getName(), descriptor, existingName, original.getMessage()); + return Reconciliation.FAILED; } /** @@ -1103,9 +1635,9 @@ private void reconcileIndexConflict(PersistentEntity entity, * existing text index is unambiguously the one a newly-declared text index conflicts with — * match it regardless of its key shape or name so {@code recreateOnConflict} can absorb it.

*/ - private static Document findIndexByKeyPattern(com.mongodb.client.MongoCollection collection, Document keys) { + private static Document findIndexByKeyPattern(Iterable indexes, Document keys) { boolean desiredIsText = isTextIndex(keys); - for (Document idx : collection.listIndexes()) { + for (Document idx : indexes) { Object key = idx.get("key"); if (!(key instanceof Document)) { continue; @@ -1136,16 +1668,22 @@ private static boolean isTextIndex(Document key) { return false; } + /** + * Whether two key patterns describe the same index. Order counts: {@code {a: 1, b: 1}} and + * {@code {b: 1, a: 1}} are two indexes to MongoDB, and one cannot serve the other's sort. + */ private static boolean sameKeyPattern(Document existingKey, Document desiredKey) { if (existingKey.size() != desiredKey.size()) { return false; } - for (Map.Entry entry : desiredKey.entrySet()) { - if (!existingKey.containsKey(entry.getKey())) { + Iterator> existing = existingKey.entrySet().iterator(); + for (Map.Entry desired : desiredKey.entrySet()) { + Map.Entry actual = existing.next(); + if (!actual.getKey().equals(desired.getKey())) { return false; } - Object a = existingKey.get(entry.getKey()); - Object b = entry.getValue(); + Object a = actual.getValue(); + Object b = desired.getValue(); if (a instanceof Number && b instanceof Number) { if (((Number) a).doubleValue() != ((Number) b).doubleValue()) { return false; @@ -1170,6 +1708,12 @@ String getMongoFieldNameForProperty(PersistentProperty property) } public void persistentEntityAdded(PersistentEntity entity) { + // GORM indexing a domain class registered after startup by itself, so it follows the setting. + if (!buildIndexes) { + LOG.debug("Index creation is disabled for connection [{}] by [{} = false]. Skipping the indexes declared " + + "by entity [{}].", connectionName(), buildIndexesSettingName(), entity.getName()); + return; + } initializeIndices(entity); } @@ -1193,12 +1737,18 @@ public void persistentEntityAdded(PersistentEntity entity) { *

A client the application supplied is left alone. Its lifecycle belongs to whoever * created it, and this datastore stays {@link #isRunning() running} so that * {@link #start()} does not later replace something it does not own. + * + *

A background index build still running, on this connection or any other, is interrupted first + * rather than left to fail against a closed client, and {@link #start()} runs it again. */ @Override public void stop() { if (!this.running || !ownsClient()) { return; } + for (MongoDatastore datastore : datastoresAndChildren()) { + datastore.stopIndexBuild(); + } this.mongo.close(); this.running = false; } @@ -1206,6 +1756,8 @@ public void stop() { /** * Builds a replacement {@link MongoClient} after a restore, using the same factory and * configuration the original was built from, so settings applied at startup still apply. + * A background index build that {@link #stop()} cut short, or that was requested while stopped, + * runs again on a fresh executor, on every connection. */ @Override public void start() { @@ -1216,6 +1768,61 @@ public void start() { .create(ConnectionSource.DEFAULT, connectionSources.getBaseConfiguration()) .getSource(); this.running = true; + for (MongoDatastore datastore : datastoresAndChildren()) { + datastore.resumeIndexBuild(); + } + } + + /** + * This datastore followed by the per-connection children, which have no lifecycle of their own. + */ + private List datastoresAndChildren() { + List datastores = new ArrayList<>(); + datastores.add(this); + for (MongoDatastore datastore : datastoresByConnectionSource.values()) { + if (datastore != this) { + datastores.add(datastore); + } + } + return datastores; + } + + private void stopIndexBuild() { + if (shutDownIndexBuild()) { + indexBuildPending = true; + } + } + + /** + * Replaces the executor {@link #stop()} shut down and runs any build it cut short. A build that was + * running when it was interrupted records that it was cut short as it exits, so this waits for it to + * have exited before deciding. + */ + private void resumeIndexBuild() { + ExecutorService stopped = this.indexBuildExecutor; + if (stopped == null || !stopped.isShutdown()) { + // Never shut down - a connection added while the datastore was stopped - so no build of it was + // interrupted, and one requested since was submitted rather than deferred. + return; + } + try { + if (!stopped.awaitTermination(INDEX_BUILD_EXIT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) { + // Still running: it may yet finish, but assuming it will not costs only a repeat of work + // that is idempotent. + indexBuildPending = true; + } + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + indexBuildPending = true; + } + this.indexBuildExecutor = newIndexBuildExecutor(connectionSources.getDefaultConnectionSource().getName()); + if (indexBuildPending) { + indexBuildPending = false; + LOG.info("Resuming the index build for connection [{}] that was pending while the datastore was stopped.", + connectionName()); + buildIndex(); + } } @Override @@ -1241,6 +1848,12 @@ private boolean ownsClient() { @PreDestroy public void close() { MongoClient current = this.mongo; + // Set before the children are walked; see the connection sources listener. + closed = true; + for (MongoDatastore datastore : datastoresAndChildren()) { + datastore.closed = true; + datastore.shutDownIndexBuild(); + } try { super.destroy(); } catch (Exception e) { @@ -1269,6 +1882,33 @@ public void close() { } } + /** + * Interrupts the background index build, if there is one. A build can run for minutes, so shutdown + * must not wait for it; the server carries on building what it was asked for. + * + * @return whether a build was still queued, and so never ran + */ + private boolean shutDownIndexBuild() { + ExecutorService executor = this.indexBuildExecutor; + if (executor == null) { + return false; + } + // A build already running reports for itself whether it was cut short; one still queued never runs. + return !executor.shutdownNow().isEmpty(); + } + + /** + * One worker, named after the connection it serves so that a log line or a thread dump says which + * datastore is building indexes, and released after a second without work. The worker is a daemon: an + * index build in flight must not hold the JVM open, and abandoning the wait does not abandon the build — + * the server finishes an index it has been asked for whether or not a client is still listening. + */ + private static ExecutorService newIndexBuildExecutor(String connectionName) { + CustomizableThreadFactory threadFactory = new CustomizableThreadFactory("gorm-mongo-index-build-" + connectionName + "-"); + threadFactory.setDaemon(true); + return new ThreadPoolExecutor(0, 1, 1, TimeUnit.SECONDS, new LinkedBlockingQueue<>(), threadFactory); + } + /** * Creates the connection sources for a {@link MongoClient}. * @@ -1282,7 +1922,14 @@ public void close() { * @return The {@link ConnectionSources} */ protected static ConnectionSources createDefaultConnectionSources(MongoClient mongoClient, PropertyResolver configuration, MongoMappingContext mappingContext, boolean closeable) { - MongoConnectionSourceSettings settings = new MongoConnectionSourceSettings(); + // Bound from the configuration rather than left at the defaults: the client is supplied here, but + // the settings that describe how the datastore behaves (multiTenancy, stateless, transactional, + // buildIndexes, engine, flush mode) still come from grails.mongodb with grails.gorm fallbacks, + // exactly as they do when GORM creates the client itself. The connection details in them are + // unused - this client is already connected - so a configured URL is dropped: its database would + // otherwise take precedence over the mapping context's, which is the one this path always used. + MongoConnectionSourceSettings settings = buildConnectionSourceSettings(configuration); + settings.url(null); settings.setDatabaseName(mappingContext.getDefaultDatabaseName()); ConnectionSource defaultConnectionSource = new DefaultConnectionSource<>(ConnectionSource.DEFAULT, mongoClient, settings, closeable); return new InMemoryConnectionSources<>(defaultConnectionSource, new MongoConnectionSourceFactory(), configuration); @@ -1302,13 +1949,17 @@ protected static MongoMappingContext createMappingContext(ConnectionSourcesSet to {@code false} to leave the indexes on the server exactly as they are, which is + * useful when deploying against live data where index changes are applied separately by a + * DBA or a migration step rather than by the application on startup. The application can still + * build them when it chooses with {@code MongoDatastore.buildIndex()}. + * + * @since 8.0 + */ + String SETTING_BUILD_INDEXES = 'grails.mongodb.buildIndexes' + + /** + * Whether index builds run on a background thread rather than blocking the thread that starts + * them: the one GORM runs when the datastore starts, and any the application starts with + * {@code MongoDatastore.buildIndex()}, including with {@link #SETTING_BUILD_INDEXES} off. + * Defaults to {@code false}, which is the historical behavior: startup waits for MongoDB to + * finish building every declared index. + * + * @since 8.0 + */ + String SETTING_BUILD_INDEXES_ASYNC = 'grails.mongodb.buildIndexesAsync' + /** * Global default storage type for {@code String id} fields when no per-domain * {@code id storedAs: ...} mapping is declared. Accepted values are the names (or hex diff --git a/grails-data-mongodb/core/src/main/groovy/org/grails/datastore/mapping/mongo/connections/AbstractMongoConnectionSourceSettings.groovy b/grails-data-mongodb/core/src/main/groovy/org/grails/datastore/mapping/mongo/connections/AbstractMongoConnectionSourceSettings.groovy index 09f6045427e..42d52109431 100644 --- a/grails-data-mongodb/core/src/main/groovy/org/grails/datastore/mapping/mongo/connections/AbstractMongoConnectionSourceSettings.groovy +++ b/grails-data-mongodb/core/src/main/groovy/org/grails/datastore/mapping/mongo/connections/AbstractMongoConnectionSourceSettings.groovy @@ -97,6 +97,28 @@ abstract class AbstractMongoConnectionSourceSettings extends ConnectionSourceSet */ boolean decimalType = true + /** + * Whether GORM creates and reconciles the indexes declared in domain class mapping blocks by + * itself when the datastore starts. When {@code false} the indexes on the server are left exactly + * as they are and GORM issues no {@code createIndex} or {@code collMod} command of its own until + * the application calls {@code MongoDatastore.buildIndex()}; queries are unaffected and continue to + * use whatever indexes already exist. Bound from {@code grails.mongodb.buildIndexes}. + * + * @since 8.0 + */ + boolean buildIndexes = true + + /** + * Whether index builds run on a background thread instead of blocking the thread that starts them: + * the startup build, and any started with {@code MongoDatastore.buildIndex()}, whatever + * {@link #buildIndexes} says. MongoDB answers a {@code createIndex} command only once the index has + * been built, so with the default {@code false} an application waits at startup for every declared + * index. Bound from {@code grails.mongodb.buildIndexesAsync}. + * + * @since 8.0 + */ + boolean buildIndexesAsync = false + /** * Nested settings for domains with {@code String id}. Holds the global default * {@code defaultStoredAs} switch plus any future string-id configuration. diff --git a/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesAsyncSpec.groovy b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesAsyncSpec.groovy new file mode 100644 index 00000000000..1142a958d75 --- /dev/null +++ b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesAsyncSpec.groovy @@ -0,0 +1,247 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.grails.datastore.gorm.mongo + +import java.util.concurrent.ConcurrentLinkedQueue + +import ch.qos.logback.classic.Level +import com.mongodb.ConnectionString +import com.mongodb.MongoClientSettings +import com.mongodb.client.MongoClient +import com.mongodb.client.MongoClients +import com.mongodb.event.CommandListener +import com.mongodb.event.CommandStartedEvent +import grails.gorm.annotation.Entity +import spock.lang.AutoCleanup +import spock.lang.Shared +import spock.util.concurrent.PollingConditions + +import org.apache.grails.testing.mongo.AutoStartedMongoSpec +import org.grails.datastore.mapping.core.DatastoreUtils +import org.grails.datastore.mapping.mongo.MongoDatastore +import org.grails.datastore.mapping.mongo.config.MongoSettings + +/** + * MongoDB answers a {@code createIndexes} command only once the index has been built, so by default + * whoever creates the datastore - in an application, the startup thread - waits for every declared index. + * This specification pins both halves of that: the default build runs on the calling thread, and with + * {@code grails.mongodb.buildIndexesAsync = true} it runs on a background thread instead. + * + *

Which thread issued the command is observed through a driver {@link CommandListener}, which the + * synchronous driver invokes on the thread making the call. + */ +class BuildIndexesAsyncSpec extends AutoStartedMongoSpec { + + @Shared + @AutoCleanup + MongoDatastore blockingDatastore + + @Shared + @AutoCleanup + MongoDatastore asyncDatastore + + @Shared + MongoClient blockingClient + + @Shared + MongoClient asyncClient + + @Shared + CreateIndexThreadRecorder blockingRecorder = new CreateIndexThreadRecorder() + + @Shared + CreateIndexThreadRecorder asyncRecorder = new CreateIndexThreadRecorder() + + @Shared + String creatingThread + + @Override + boolean shouldInitializeDatastore() { + false + } + + private MongoClient clientFor(String database, CommandListener listener) { + MongoClients.create(MongoClientSettings.builder() + .applyConnectionString(new ConnectionString(dbContainer.getReplicaSetUrl(database))) + .addCommandListener(listener) + .build()) + } + + void setupSpec() { + creatingThread = Thread.currentThread().name + + blockingClient = clientFor('blockingIndexDb', blockingRecorder) + blockingDatastore = new MongoDatastore(blockingClient, + DatastoreUtils.createPropertyResolver(['grails.mongodb.databaseName': 'blockingIndexDb']), + BlockingIndexThing) + + asyncClient = clientFor('asyncIndexDb', asyncRecorder) + asyncDatastore = new MongoDatastore(asyncClient, + DatastoreUtils.createPropertyResolver([ + 'grails.mongodb.databaseName' : 'asyncIndexDb', + (MongoSettings.SETTING_BUILD_INDEXES_ASYNC): true + ]), + AsyncIndexThing) + } + + void cleanupSpec() { + blockingClient?.close() + asyncClient?.close() + } + + void "test the index build blocks the thread creating the datastore by default"() { + expect: "the setting is off" + !blockingDatastore.isBuildIndexesAsync() + + and: "the index already exists by the time the constructor has returned" + [name: 1] in BlockingIndexThing.collection.listIndexes()*.key + + and: "it was built by the thread that created the datastore, which therefore waited for it" + blockingRecorder.threads == [creatingThread] + } + + void "test the index build runs on a background thread when enabled"() { + given: + def conditions = new PollingConditions(timeout: 30) + + expect: "the setting is on" + asyncDatastore.isBuildIndexesAsync() + + when: "the background build has had a chance to run" + conditions.eventually { + assert [name: 1] in AsyncIndexThing.collection.listIndexes()*.key + } + + then: "the command was issued by the datastore's own index build thread" + asyncRecorder.threads.size() == 1 + asyncRecorder.threads.first().startsWith('gorm-mongo-index-build-default-') + + and: "not by the thread that created the datastore, which did not wait for it" + asyncRecorder.threads.first() != creatingThread + + when: "the startup worker has no more work" + Thread startupWorker = asyncRecorder.workers.first() + startupWorker.join(10000) + + then: "it is released without closing the datastore" + startupWorker.daemon + !startupWorker.alive + + when: "another build is explicitly requested after the worker has exited" + asyncDatastore.buildIndex() + + then: "a new worker performs the build" + conditions.eventually { + assert asyncRecorder.workers.size() == 2 + } + !asyncRecorder.workers.last().is(startupWorker) + + when: + Thread subsequentWorker = asyncRecorder.workers.last() + subsequentWorker.join(10000) + + then: + !subsequentWorker.alive + } + + void "test a background build is announced by its own thread, ahead of everything it logs"() { + given: + def conditions = new PollingConditions(timeout: 30) + def log = new CapturedLog('org.grails.datastore.mapping', Level.INFO) + def isAnnouncement = { + it.formattedMessage.startsWith('Building the indexes declared by the domain classes for connection [default] on a background thread') + } + def isSummary = { it.formattedMessage.startsWith('Index build for database [announcedIndexDb] finished') } + + when: + def datastore = new MongoDatastore(['grails.mongodb.url' : dbContainer.getReplicaSetUrl('announcedIndexDb'), + (MongoSettings.SETTING_BUILD_INDEXES_ASYNC): true] as Map, + AnnouncedIndexThing) + conditions.eventually { + assert log.events.any(isSummary) + } + def events = log.events + def announcement = events.find(isAnnouncement) + def summary = events.find(isSummary) + + then: "said by the thread that goes on to build, so it cannot trail the summary a fast build logs" + announcement.threadName == summary.threadName + events.indexOf(announcement) < events.indexOf(summary) + + cleanup: + datastore?.close() + log?.close() + } +} + +/** + * Records the thread each {@code createIndexes} command was issued from. + */ +class CreateIndexThreadRecorder implements CommandListener { + + private final Queue recorded = new ConcurrentLinkedQueue<>() + + @Override + void commandStarted(CommandStartedEvent event) { + if (event.commandName == 'createIndexes') { + recorded.add(Thread.currentThread()) + } + } + + List getThreads() { + recorded.collect { it.name }.unique() + } + + List getWorkers() { + recorded.toList().unique() + } +} + +@Entity +class BlockingIndexThing { + String name + + static mapping = { + version false + collection 'blockingIndexThing' + name index: true + } +} + +@Entity +class AsyncIndexThing { + String name + + static mapping = { + version false + collection 'asyncIndexThing' + name index: true + } +} + +@Entity +class AnnouncedIndexThing { + String name + + static mapping = { + version false + collection 'announcedIndexThing' + name index: true + } +} diff --git a/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesBackgroundFailureSpec.groovy b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesBackgroundFailureSpec.groovy new file mode 100644 index 00000000000..8121fcedc49 --- /dev/null +++ b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesBackgroundFailureSpec.groovy @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.grails.datastore.gorm.mongo + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit + +import ch.qos.logback.classic.Level +import com.mongodb.MongoException +import com.mongodb.client.MongoClient +import com.mongodb.client.MongoClients +import grails.gorm.annotation.Entity +import spock.lang.Shared +import spock.util.concurrent.PollingConditions + +import org.apache.grails.testing.mongo.AutoStartedMongoSpec +import org.grails.datastore.mapping.core.DatastoreUtils +import org.grails.datastore.mapping.mongo.MongoDatastore +import org.grails.datastore.mapping.mongo.config.MongoSettings + +/** + * Nothing waits on the background index build, so what it does when it goes wrong is only visible in the + * log. A build that fails has to say so loudly, because it can no longer fail startup; a build abandoned + * because the application is shutting down has to stay quiet, because nothing went wrong. + */ +class BuildIndexesBackgroundFailureSpec extends AutoStartedMongoSpec { + + @Shared + MongoClient realClient + + @Shared + CapturedLog log + + @Override + boolean shouldInitializeDatastore() { + false + } + + void setupSpec() { + realClient = MongoClients.create(dbContainer.getReplicaSetUrl('backgroundFailureDb')) + log = new CapturedLog('org.grails.datastore.mapping', Level.DEBUG) + } + + void cleanupSpec() { + log?.close() + realClient?.close() + } + + private MongoDatastore asyncDatastoreOn(MongoClient client, String database, Class... classes) { + new MongoDatastore(client, DatastoreUtils.createPropertyResolver([ + 'grails.mongodb.databaseName' : database, + (MongoSettings.SETTING_BUILD_INDEXES_ASYNC): true + ]), classes) + } + + void "test a background build that fails reports the failure instead of losing it"() { + given: + def conditions = new PollingConditions(timeout: 30) + MongoClient broken = FailingMongoClient.wrap(realClient, 'getCollection') { + throw new MongoException('the connection went away mid-build') + } + + when: "the datastore is created, which does not wait for the build" + def datastore = asyncDatastoreOn(broken, 'backgroundFailureDb', FailedBackgroundThing) + + then: "startup was not held up by, and did not fail because of, the broken build" + datastore.isBuildIndexesAsync() + + and: "the failure is reported at error level, since nothing else would surface it" + conditions.eventually { + assert log.events.any { + it.level == Level.ERROR && it.formattedMessage.contains('The background index build failed') + } + } + + cleanup: + datastore?.close() + } + + void "test a background build abandoned at shutdown is not reported as a failure"() { + given: + def conditions = new PollingConditions(timeout: 30) + def buildReached = new CountDownLatch(1) + def neverReleased = new CountDownLatch(1) + // The appender is shared with the preceding feature, which logs an error of its own on purpose + int errorsBefore = backgroundBuildFailures() + MongoClient blocking = FailingMongoClient.wrap(realClient, 'createIndex') { + buildReached.countDown() + neverReleased.await() + } + + when: "the build is under way on its background thread" + def datastore = asyncDatastoreOn(blocking, 'backgroundFailureDb', AbandonedBackgroundThing) + + then: + buildReached.await(30, TimeUnit.SECONDS) + + when: "the datastore is closed while the build is still running" + datastore.close() + + then: "the interruption is reported as the shutdown it is, not as a failure" + conditions.eventually { + assert log.events.any { + it.level == Level.DEBUG && it.formattedMessage.contains('abandoned because the datastore is shutting down') + } + } + + and: "no error is logged for it" + backgroundBuildFailures() == errorsBefore + } + + void "test a genuine failure that coincides with shutdown is still reported as an error"() { + given: + def conditions = new PollingConditions(timeout: 30) + def buildReached = new CountDownLatch(1) + def release = new CountDownLatch(1) + int errorsBefore = backgroundBuildFailures() + MongoClient failsDuringShutdown = FailingMongoClient.wrap(realClient, 'createIndex') { + buildReached.countDown() + // A command already on the server does not stop for the interrupt; this one then fails in its own right + while (true) { + try { + release.await() + break + } + catch (InterruptedException ignored) { + } + } + throw new MongoException('E11000 duplicate key error collection: backgroundFailureDb.shutdownFailureThing') + } + def datastore = asyncDatastoreOn(failsDuringShutdown, 'backgroundFailureDb', ShutdownFailureThing) + buildReached.await(30, TimeUnit.SECONDS) + + when: "the datastore is closed while the build is on its way to failing" + datastore.close() + release.countDown() + + then: "the shutdown does not explain a duplicate key, so it is reported as the failure it is" + conditions.eventually { + assert backgroundBuildFailures() == errorsBefore + 1 + } + } + + private int backgroundBuildFailures() { + log.events.count { + it.level == Level.ERROR && it.formattedMessage.contains('The background index build failed') + } + } +} + +@Entity +class FailedBackgroundThing { + String name + + static mapping = { + version false + collection 'failedBackgroundThing' + name index: true + } +} + +@Entity +class ShutdownFailureThing { + String name + + static mapping = { + version false + collection 'shutdownFailureThing' + name index: true + } +} + +@Entity +class AbandonedBackgroundThing { + String name + + static mapping = { + version false + collection 'abandonedBackgroundThing' + name index: true + } +} diff --git a/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesClassificationCostSpec.groovy b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesClassificationCostSpec.groovy new file mode 100644 index 00000000000..217940e77b8 --- /dev/null +++ b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesClassificationCostSpec.groovy @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.grails.datastore.gorm.mongo + +import java.util.concurrent.atomic.AtomicInteger + +import ch.qos.logback.classic.Level +import com.mongodb.ConnectionString +import com.mongodb.MongoClientSettings +import com.mongodb.client.MongoClient +import com.mongodb.client.MongoClients +import com.mongodb.event.CommandListener +import com.mongodb.event.CommandStartedEvent +import grails.gorm.annotation.Entity +import spock.lang.AutoCleanup +import spock.lang.Shared + +import org.apache.grails.testing.mongo.AutoStartedMongoSpec +import org.grails.datastore.mapping.core.DatastoreUtils +import org.grails.datastore.mapping.mongo.MongoDatastore + +/** + * Telling a created index from one that was already there costs a {@code listIndexes} per collection that + * declares indexes, and the only thing it feeds is the summary line. It is paid once per collection + * however many domain classes map to it, and not at all when the summary is not logged. + */ +class BuildIndexesClassificationCostSpec extends AutoStartedMongoSpec { + + static final String DATABASE = 'classificationCostDb' + + @Shared + @AutoCleanup + MongoDatastore datastore + + @Shared + MongoClient countingClient + + @Shared + AtomicInteger listings = new AtomicInteger() + + @Shared + int startupListings + + @Shared + String startupSummary + + @Override + boolean shouldInitializeDatastore() { + false + } + + void setupSpec() { + countingClient = MongoClients.create(MongoClientSettings.builder() + .applyConnectionString(new ConnectionString(dbContainer.getReplicaSetUrl(DATABASE))) + .addCommandListener(new CommandListener() { + @Override + void commandStarted(CommandStartedEvent event) { + if (event.commandName == 'listIndexes') { + listings.incrementAndGet() + } + } + }) + .build()) + + def log = new CapturedLog('org.grails.datastore.mapping.mongo', Level.INFO) + datastore = new MongoDatastore(countingClient, + DatastoreUtils.createPropertyResolver(['grails.mongodb.databaseName': DATABASE]), + ClassifiedThing, FirstSharedCollectionThing, SecondSharedCollectionThing) + startupListings = listings.get() + startupSummary = log.events*.formattedMessage.find { it.contains("database [$DATABASE]") } + log.close() + } + + void cleanupSpec() { + countingClient?.close() + } + + void "test a collection shared by several domain classes is listed once"() { + expect: "one listing for each of the two collections, not one for each of the three classes" + startupListings == 2 + } + + void "test classes sharing a collection see each other's indexes"() { + expect: "the second class declaring the keys the first has just created finds them already there" + startupSummary.contains('2 created, 1 already present') + } + + void "test the existing indexes are not listed when the summary will not be logged"() { + given: + int before = listings.get() + def quiet = new CapturedLog('org.grails.datastore.mapping.mongo', Level.WARN) + + when: + datastore.buildIndex() + + then: + listings.get() == before + + cleanup: + quiet?.close() + } + + void "test an index applied outside a build is logged as applied, without being classified"() { + given: + int before = listings.get() + def debug = new CapturedLog('org.grails.datastore.mapping.mongo', Level.DEBUG) + + when: "a domain class is registered after startup, when no summary will report the counts" + datastore.persistentEntityAdded(datastore.mappingContext.getPersistentEntity(ClassifiedThing.name)) + + then: "nothing is listed, and the per-index line does not claim the index was created" + listings.get() == before + debug.events*.formattedMessage.any { it.startsWith("Applied index for entity [${ClassifiedThing.name}]") } + !debug.events*.formattedMessage.any { it.startsWith("Created index for entity [${ClassifiedThing.name}]") } + + cleanup: + debug?.close() + } +} + +@Entity +class ClassifiedThing { + String name + + static mapping = { + version false + collection 'classifiedThing' + name index: true + } +} + +@Entity +class FirstSharedCollectionThing { + String code + + static mapping = { + version false + collection 'sharedCollection' + code index: true + } +} + +@Entity +class SecondSharedCollectionThing { + String code + + static mapping = { + version false + collection 'sharedCollection' + code index: true + } +} diff --git a/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesDisabledSpec.groovy b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesDisabledSpec.groovy new file mode 100644 index 00000000000..9ebbaf15e20 --- /dev/null +++ b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesDisabledSpec.groovy @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.grails.datastore.gorm.mongo + +import grails.gorm.annotation.Entity +import spock.lang.AutoCleanup +import spock.lang.Shared + +import org.apache.grails.testing.mongo.AutoStartedMongoSpec +import org.grails.datastore.mapping.mongo.MongoDatastore +import org.grails.datastore.mapping.mongo.config.MongoSettings + +/** + * Verifies that with {@code grails.mongodb.buildIndexes = false} GORM issues no index commands of its own: + * the indexes declared in the mapping block are neither created nor reconciled, so whatever indexes are + * already on the server are left untouched. Persistence and querying are unaffected. + * + * @see BuildIndexesEnabledByDefaultSpec for the default behavior with the same mapping declarations + * @see BuildIndexesOnDemandSpec for the application building them when it chooses + */ +class BuildIndexesDisabledSpec extends AutoStartedMongoSpec { + + @Shared + @AutoCleanup + MongoDatastore datastore + + @Override + boolean shouldInitializeDatastore() { + false + } + + void setupSpec() { + Map config = [ + 'grails.mongodb.url' : dbContainer.getReplicaSetUrl('myDb'), + (MongoSettings.SETTING_BUILD_INDEXES) : false + ] + datastore = new MongoDatastore(config, SkippedIndexThing) + } + + private static List declaredIndexKeys() { + SkippedIndexThing.withNewSession { + SkippedIndexThing.collection.listIndexes()*.key + } + } + + void "test the datastore reports index building disabled"() { + expect: + !datastore.isBuildIndexes() + } + + void "test no declared index is created on startup"() { + when: "a document is written so the collection certainly exists" + SkippedIndexThing.withNewSession { + new SkippedIndexThing(name: 'Fred', age: 42).save(flush: true) + } + + then: "only the implicit _id index is present - neither the property index nor the compound index was created" + declaredIndexKeys() == [[_id: 1]] + } + + void "test registering an entity after startup does not create its indexes"() { + given: + def entity = datastore.mappingContext.getPersistentEntity(SkippedIndexThing.name) + + when: "the entity is re-registered, the path that indexes a domain class added after startup" + datastore.persistentEntityAdded(entity) + + then: "no index was created" + declaredIndexKeys() == [[_id: 1]] + } + + void "test reads and writes are unaffected when index building is disabled"() { + when: + SkippedIndexThing.withNewSession { + new SkippedIndexThing(name: 'Wilma', age: 41).save(flush: true) + } + + then: "the unindexed property is still queryable" + SkippedIndexThing.withNewSession { SkippedIndexThing.findByName('Wilma') }?.age == 41 + } +} + +@Entity +class SkippedIndexThing { + String name + Integer age + + static mapping = { + version false + collection 'skippedIndexThing' + name index: true + compoundIndex name: 1, age: -1 + } +} diff --git a/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesEnabledByDefaultSpec.groovy b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesEnabledByDefaultSpec.groovy new file mode 100644 index 00000000000..a2ac56c338d --- /dev/null +++ b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesEnabledByDefaultSpec.groovy @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.grails.datastore.gorm.mongo + +import grails.gorm.annotation.Entity +import spock.lang.AutoCleanup +import spock.lang.Shared + +import org.apache.grails.testing.mongo.AutoStartedMongoSpec +import org.grails.datastore.mapping.mongo.MongoDatastore + +/** + * The control for {@link BuildIndexesDisabledSpec}: with {@code grails.mongodb.buildIndexes} left at its + * default the very same mapping declarations do produce indexes on startup, so the absence of indexes in + * that specification is attributable to the setting and not to the mapping. + */ +class BuildIndexesEnabledByDefaultSpec extends AutoStartedMongoSpec { + + @Shared + @AutoCleanup + MongoDatastore datastore + + @Override + boolean shouldInitializeDatastore() { + false + } + + void setupSpec() { + // No grails.mongodb.buildIndexes => default true + datastore = new MongoDatastore(['grails.mongodb.url': dbContainer.getReplicaSetUrl('myDb')] as Map, BuiltIndexThing) + } + + void "test index building is enabled by default"() { + expect: + datastore.isBuildIndexes() + } + + void "test the declared indexes are created on startup"() { + when: + List indexKeys = BuiltIndexThing.withNewSession { + BuiltIndexThing.collection.listIndexes()*.key + } + + then: "the property index and the compound index are both present alongside the implicit _id index" + [name: 1] in indexKeys + [name: 1, age: -1] in indexKeys + } +} + +@Entity +class BuiltIndexThing { + String name + Integer age + + static mapping = { + version false + collection 'builtIndexThing' + name index: true + compoundIndex name: 1, age: -1 + } +} diff --git a/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesFailureSummarySpec.groovy b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesFailureSummarySpec.groovy new file mode 100644 index 00000000000..c9b7f713d82 --- /dev/null +++ b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesFailureSummarySpec.groovy @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.grails.datastore.gorm.mongo + +import ch.qos.logback.classic.Level +import ch.qos.logback.classic.spi.ILoggingEvent +import com.mongodb.client.MongoClient +import com.mongodb.client.MongoClients +import grails.gorm.annotation.Entity +import org.bson.Document +import spock.lang.AutoCleanup +import spock.lang.Shared + +import org.apache.grails.testing.mongo.AutoStartedMongoSpec +import org.grails.datastore.mapping.mongo.MongoDatastore + +/** + * An index the server refuses is not allowed to stop the application from starting: the failure is + * reported and the rest of the build carries on. The summary then goes out at {@code WARN} and says how + * many declarations failed, so a build that half worked cannot pass for a clean one. + */ +class BuildIndexesFailureSummarySpec extends AutoStartedMongoSpec { + + static final String DATABASE = 'failedIndexDb' + + @Shared + @AutoCleanup + MongoDatastore datastore + + @Shared + MongoClient setupClient + + @Shared + CapturedLog log + + @Override + boolean shouldInitializeDatastore() { + false + } + + void setupSpec() { + String url = dbContainer.getReplicaSetUrl(DATABASE) + setupClient = MongoClients.create(url) + def database = setupClient.getDatabase(DATABASE) + + // An index on the same keys as the mapping declares, but without the unique option it asks for. + // MongoDB reports IndexOptionsConflict for the declaration, and without recreateOnConflict GORM + // is not authorised to drop the existing one. + database.getCollection('conflictingThing').createIndex(new Document('name', 1)) + + database.getCollection('rejectedThing').insertOne(new Document('code', 'first')) + + log = new CapturedLog('org.grails.datastore.mapping', Level.INFO) + + datastore = new MongoDatastore(['grails.mongodb.url': url] as Map, ConflictingThing, RejectedThing) + } + + void cleanupSpec() { + log?.close() + setupClient?.close() + } + + private List eventsForThisDatabase() { + log.events.findAll { it.formattedMessage.contains("database [$DATABASE]") } + } + + void "test an index declaration the server refuses does not stop the datastore from starting"() { + expect: "the datastore is usable" + RejectedThing.withNewSession { RejectedThing.count() } == 1 + + and: "the conflicting declaration left the index that was already there untouched" + ConflictingThing.withNewSession { + ConflictingThing.collection.listIndexes().find { it.key == [name: 1] }.unique == null + } + + and: "and the declaration the server refused outright created nothing" + RejectedThing.withNewSession { + RejectedThing.collection.listIndexes()*.key == [[_id: 1]] + } + } + + void "test the summary is logged at warn and reports how many declarations failed"() { + given: + def summary = eventsForThisDatabase().first() + + expect: "one summary, raised to WARN because not everything was applied" + eventsForThisDatabase().size() == 1 + summary.level == Level.WARN + + and: "both failures are counted: the unresolved conflict and the declaration the server refused" + summary.formattedMessage.contains('2 failed') + + and: "the failures themselves were reported individually, naming the entity each came from" + log.events.any { it.level == Level.ERROR && it.formattedMessage.contains('ConflictingThing') } + log.events.any { it.level == Level.ERROR && it.formattedMessage.contains('RejectedThing') } + } +} + +@Entity +class ConflictingThing { + String name + + static mapping = { + version false + collection 'conflictingThing' + name index: true, indexAttributes: [unique: true] + } +} + +@Entity +class RejectedThing { + String code + + static mapping = { + version false + collection 'rejectedThing' + code index: true, indexAttributes: [type: 'nosuchindexplugin'] + } +} diff --git a/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesHookSpec.groovy b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesHookSpec.groovy new file mode 100644 index 00000000000..794783aae4d --- /dev/null +++ b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesHookSpec.groovy @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.grails.datastore.gorm.mongo + +import com.mongodb.client.MongoClient +import com.mongodb.client.MongoClients +import grails.gorm.annotation.Entity +import org.bson.Document +import spock.util.concurrent.PollingConditions + +import org.apache.grails.testing.mongo.AutoStartedMongoSpec +import org.grails.datastore.mapping.core.DatastoreUtils +import org.grails.datastore.mapping.model.PersistentEntity +import org.grails.datastore.mapping.mongo.MongoDatastore +import org.springframework.core.env.PropertyResolver + +class BuildIndexesHookSpec extends AutoStartedMongoSpec { + + @Override + boolean shouldInitializeDatastore() { + false + } + + void "test the protected index hook customises startup and later entities with async #async"() { + given: + def conditions = new PollingConditions(timeout: 30) + + when: + def datastore = new CustomIndexDatastore([ + 'grails.mongodb.url': dbContainer.getReplicaSetUrl("indexHook${async}"), + 'grails.mongodb.buildIndexesAsync': async + ]) + + then: + conditions.eventually { + assert [name: 1] in StartupHookThing.collection.listIndexes()*.key + assert [custom: 1] in StartupHookThing.collection.listIndexes()*.key + } + + when: "another domain class is registered after startup" + datastore.mappingContext.addPersistentEntity(LaterHookThing) + + then: + [name: 1] in LaterHookThing.collection.listIndexes()*.key + [custom: 1] in LaterHookThing.collection.listIndexes()*.key + + cleanup: + datastore?.close() + + where: + async << [false, true] + } + + void "test a checked exception from the hook reaches the caller of a synchronous build unchanged"() { + given: + MongoClient client = MongoClients.create(dbContainer.getReplicaSetUrl('hookFailureDb')) + + when: "an override throws a checked exception it does not declare, as Groovy allows" + new InterruptedHookDatastore(client, + DatastoreUtils.createPropertyResolver(['grails.mongodb.databaseName': 'hookFailureDb']), HookFailureThing) + + then: "the caller gets that exception, not a wrapper around it" + def e = thrown(InterruptedException) + e.message == 'interrupted in the hook' + + and: "the interrupt it stands for is not lost" + Thread.interrupted() + + cleanup: + Thread.interrupted() + client?.close() + } +} + +class InterruptedHookDatastore extends MongoDatastore { + + InterruptedHookDatastore(MongoClient client, PropertyResolver configuration, Class... classes) { + super(client, configuration, classes) + } + + @Override + protected void initializeIndices(PersistentEntity entity) { + throw new InterruptedException('interrupted in the hook') + } +} + +@Entity +class HookFailureThing { + String name + + static mapping = { + name index: true + } +} + +class CustomIndexDatastore extends MongoDatastore { + + CustomIndexDatastore(Map configuration) { + super(configuration, StartupHookThing) + } + + @Override + protected void initializeIndices(PersistentEntity entity) { + super.initializeIndices(entity) + getCollection(entity).createIndex(new Document('custom', 1)) + } +} + +@Entity +class StartupHookThing { + String name + + static mapping = { + name index: true + } +} + +@Entity +class LaterHookThing { + String name + + static mapping = { + name index: true + } +} diff --git a/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesKeyOrderSpec.groovy b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesKeyOrderSpec.groovy new file mode 100644 index 00000000000..d244b360ee9 --- /dev/null +++ b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesKeyOrderSpec.groovy @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.grails.datastore.gorm.mongo + +import ch.qos.logback.classic.Level +import com.mongodb.client.MongoClient +import com.mongodb.client.MongoClients +import grails.gorm.annotation.Entity +import org.bson.Document +import spock.lang.AutoCleanup +import spock.lang.Shared + +import org.apache.grails.testing.mongo.AutoStartedMongoSpec +import org.grails.datastore.mapping.mongo.MongoDatastore + +/** + * The order of a compound index's keys is part of the index: MongoDB keeps {@code {a: 1, b: 1}} and + * {@code {b: 1, a: 1}} as two indexes, and one cannot serve the other's sort. A declaration whose keys + * come in a different order from an existing index is therefore a new index, and the summary has to say + * it was created. + */ +class BuildIndexesKeyOrderSpec extends AutoStartedMongoSpec { + + static final String DATABASE = 'keyOrderDb' + + @Shared + @AutoCleanup + MongoDatastore datastore + + @Shared + MongoClient setupClient + + @Shared + CapturedLog log + + @Override + boolean shouldInitializeDatastore() { + false + } + + void setupSpec() { + String url = dbContainer.getReplicaSetUrl(DATABASE) + setupClient = MongoClients.create(url) + setupClient.getDatabase(DATABASE).getCollection('keyOrderThing').createIndex(new Document('a', 1).append('b', 1)) + + log = new CapturedLog('org.grails.datastore.mapping', Level.INFO) + datastore = new MongoDatastore(['grails.mongodb.url': url] as Map, KeyOrderThing) + } + + void cleanupSpec() { + log?.close() + setupClient?.close() + } + + void "test a compound index declared in a different key order is created, not taken for the existing one"() { + expect: "both orders now exist on the server, as two indexes" + def keys = setupClient.getDatabase(DATABASE).getCollection('keyOrderThing').listIndexes()*.key*.keySet()*.toList() + ['a', 'b'] in keys + ['b', 'a'] in keys + + and: "the summary counts the reversed declaration as created and the matching one as already present" + log.events*.formattedMessage.find { it.contains("database [$DATABASE]") }.contains('1 created, 1 already present') + } +} + +@Entity +class KeyOrderThing { + Integer a + Integer b + + static mapping = { + version false + collection 'keyOrderThing' + compoundIndex b: 1, a: 1 + compoundIndex a: 1, b: 1 + } +} diff --git a/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesLifecycleSpec.groovy b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesLifecycleSpec.groovy new file mode 100644 index 00000000000..3033d82871a --- /dev/null +++ b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesLifecycleSpec.groovy @@ -0,0 +1,326 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.grails.datastore.gorm.mongo + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicBoolean + +import ch.qos.logback.classic.Level +import com.mongodb.client.MongoClient +import com.mongodb.client.MongoClients +import grails.gorm.annotation.Entity +import spock.lang.Shared +import spock.util.concurrent.PollingConditions + +import org.apache.grails.testing.mongo.AutoStartedMongoSpec +import org.grails.datastore.mapping.model.PersistentEntity +import org.grails.datastore.mapping.mongo.MongoDatastore +import org.grails.datastore.mapping.mongo.config.MongoSettings + +/** + * A background build belongs to the datastore that started it. Stopping the datastore for a checkpoint + * interrupts the build rather than letting it fail against the closed client, and restarting it runs the + * build again; closing it for good turns a later request for a build into a warning rather than an + * exception. + */ +class BuildIndexesLifecycleSpec extends AutoStartedMongoSpec { + + @Shared + MongoClient realClient + + @Override + boolean shouldInitializeDatastore() { + false + } + + void setupSpec() { + realClient = MongoClients.create(dbContainer.getReplicaSetUrl('lifecycleDb')) + } + + void cleanupSpec() { + realClient?.close() + } + + private Map asyncConfig(String database) { + ['grails.mongodb.url' : dbContainer.getReplicaSetUrl(database), + (MongoSettings.SETTING_BUILD_INDEXES_ASYNC): true] as Map + } + + void "test a build cut short by stop() is not reported as a failure and runs again on start()"() { + given: + def conditions = new PollingConditions(timeout: 30) + def log = new CapturedLog('org.grails.datastore.mapping', Level.DEBUG) + CheckpointedDatastore.REACHED = new CountDownLatch(1) + CheckpointedDatastore.RELEASE = new CountDownLatch(1) + CheckpointedDatastore.BLOCK_ONCE.set(true) + def indexes = { -> realClient.getDatabase('checkpointDb').getCollection('checkpointedThing').listIndexes()*.key } + + when: "the background build is under way" + def datastore = new CheckpointedDatastore(asyncConfig('checkpointDb'), CheckpointedThing) + + then: + CheckpointedDatastore.REACHED.await(30, TimeUnit.SECONDS) + + when: "the datastore is stopped for a checkpoint while the build is still running" + datastore.stop() + + then: "the build is abandoned as a shutdown and reports how far it got, without a failure" + conditions.eventually { + assert log.events.any { + it.level == Level.DEBUG && it.formattedMessage.contains('database [checkpointDb] did not finish') + } + } + !log.events.any { + it.level.isGreaterOrEqual(Level.WARN) && it.formattedMessage.contains('database [checkpointDb]') + } + + and: "the declared index was not created" + !([name: 1] in indexes()) + + when: "the datastore is restarted" + datastore.start() + + then: "the build that was cut short is resumed" + log.events.any { + it.level == Level.INFO && it.formattedMessage.contains('Resuming the index build for connection [default]') + } + + and: "runs again and completes" + conditions.eventually { + assert [name: 1] in indexes() + } + + cleanup: + CheckpointedDatastore.RELEASE?.countDown() + datastore?.close() + log?.close() + } + + void "test a build that finishes despite stop() is not run again on start()"() { + given: + def log = new CapturedLog('org.grails.datastore.mapping', Level.INFO) + SlowToStopDatastore.REACHED = new CountDownLatch(1) + SlowToStopDatastore.RELEASE = new CountDownLatch(1) + + when: "the build has applied every index and is on its way out when the datastore is stopped" + def datastore = new SlowToStopDatastore(asyncConfig('slowToStopDb'), SlowToStopThing) + SlowToStopDatastore.REACHED.await(30, TimeUnit.SECONDS) + datastore.stop() + SlowToStopDatastore.RELEASE.countDown() + + and: "restarted" + datastore.start() + + then: "the build finished, so the restart does not run it again" + log.events.any { it.formattedMessage.contains('Index build for database [slowToStopDb] finished') } + !log.events.any { it.formattedMessage.startsWith('Resuming the index build') } + + cleanup: + SlowToStopDatastore.RELEASE?.countDown() + datastore?.close() + log?.close() + } + + void "test a build requested while stopped runs when the datastore is restarted"() { + given: + def conditions = new PollingConditions(timeout: 30) + def log = new CapturedLog('org.grails.datastore.mapping', Level.INFO) + def collection = realClient.getDatabase('deferredDb').getCollection('deferredBuildThing') + def datastore = new MongoDatastore(asyncConfig('deferredDb'), DeferredBuildThing) + conditions.eventually { + assert [name: 1] in collection.listIndexes()*.key + } + + when: "the datastore is stopped, and a build is requested while it is" + datastore.stop() + datastore.buildIndex() + + then: "the request is deferred, not refused" + notThrown(Exception) + log.events.any { + it.level == Level.INFO && it.formattedMessage.contains('for connection [default] while the datastore is stopped; it will run when the datastore is restarted') + } + + when: "the index is dropped on the server and the datastore restarted" + collection.dropIndex('name_1') + datastore.start() + + then: "the deferred build runs and puts it back" + log.events.any { it.formattedMessage.contains('Resuming the index build for connection [default]') } + conditions.eventually { + assert [name: 1] in collection.listIndexes()*.key + } + + cleanup: + datastore?.close() + log?.close() + } + + void "test a build on the calling thread requested after close() is refused with a warning"() { + given: + def log = new CapturedLog('org.grails.datastore.mapping', Level.INFO) + def datastore = new MongoDatastore(['grails.mongodb.url': dbContainer.getReplicaSetUrl('closedSyncDb')] as Map, + ClosedSyncThing) + datastore.close() + + when: "a synchronous build is requested, which would otherwise run against the closed client" + datastore.buildIndex() + + then: + notThrown(Exception) + log.events.any { + it.level == Level.WARN && it.formattedMessage.contains('requested for connection [default] after the datastore was closed') + } + + cleanup: + log?.close() + } + + void "test a build requested after close() is refused with a warning rather than an exception"() { + given: + def log = new CapturedLog('org.grails.datastore.mapping', Level.INFO) + def datastore = new MongoDatastore(asyncConfig('closedDb'), ClosedBuildThing) + datastore.close() + + when: + datastore.buildIndex() + + then: + notThrown(Exception) + log.events.any { + it.level == Level.WARN && it.formattedMessage.contains('requested for connection [default] after the datastore was closed') + } + + cleanup: + log?.close() + } +} + +/** + * Blocks the first index build in the protected hook, so that a test can act while the build is under + * way. The latches are static: with an asynchronous build the hook can run before this class's own + * constructor has finished, which is exactly what the hook's documentation warns an override about. + */ +class CheckpointedDatastore extends MongoDatastore { + + static final AtomicBoolean BLOCK_ONCE = new AtomicBoolean() + + static volatile CountDownLatch REACHED + + static volatile CountDownLatch RELEASE + + CheckpointedDatastore(Map configuration, Class... classes) { + super(configuration, classes) + } + + @Override + protected void initializeIndices(PersistentEntity entity) { + if (BLOCK_ONCE.compareAndSet(true, false)) { + REACHED.countDown() + RELEASE.await() + } + super.initializeIndices(entity) + } +} + +/** + * Holds the build up after it has applied every index, and does not respond to being interrupted: the + * build then finishes normally even though {@link MongoDatastore#stop()} ran while it was under way. + */ +class SlowToStopDatastore extends MongoDatastore { + + static volatile CountDownLatch REACHED + + static volatile CountDownLatch RELEASE + + SlowToStopDatastore(Map configuration, Class... classes) { + super(configuration, classes) + } + + @Override + protected void initializeIndices(PersistentEntity entity) { + super.initializeIndices(entity) + REACHED.countDown() + while (true) { + try { + RELEASE.await() + return + } + catch (InterruptedException ignored) { + // A step that does not respond to interruption, as a blocking call may not + } + } + } +} + +@Entity +class ClosedSyncThing { + String name + + static mapping = { + version false + collection 'closedSyncThing' + name index: true + } +} + +@Entity +class SlowToStopThing { + String name + + static mapping = { + version false + collection 'slowToStopThing' + name index: true + } +} + +@Entity +class DeferredBuildThing { + String name + + static mapping = { + version false + collection 'deferredBuildThing' + name index: true + } +} + +@Entity +class CheckpointedThing { + String name + + static mapping = { + version false + collection 'checkpointedThing' + name index: true + } +} + +@Entity +class ClosedBuildThing { + String name + + static mapping = { + version false + collection 'closedBuildThing' + name index: true + } +} diff --git a/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesOnDemandSpec.groovy b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesOnDemandSpec.groovy new file mode 100644 index 00000000000..b2f16251a41 --- /dev/null +++ b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesOnDemandSpec.groovy @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.grails.datastore.gorm.mongo + +import ch.qos.logback.classic.Level +import com.mongodb.client.MongoClient +import com.mongodb.client.MongoClients +import grails.gorm.annotation.Entity +import spock.lang.Shared +import spock.util.concurrent.PollingConditions + +import org.apache.grails.testing.mongo.AutoStartedMongoSpec +import org.grails.datastore.mapping.core.connections.ConnectionSource +import org.grails.datastore.mapping.mongo.MongoDatastore +import org.grails.datastore.mapping.mongo.config.MongoSettings + +/** + * {@code grails.mongodb.buildIndexes = false} stops GORM building indexes by itself. It does not stop the + * application building them when it chooses to, by calling {@link MongoDatastore#buildIndex()} - once a + * deployment has been verified, say, or from an administrative action. + */ +class BuildIndexesOnDemandSpec extends AutoStartedMongoSpec { + + @Shared + MongoClient realClient + + @Override + boolean shouldInitializeDatastore() { + false + } + + void setupSpec() { + realClient = MongoClients.create(dbContainer.getReplicaSetUrl('onDemandDb')) + } + + void cleanupSpec() { + realClient?.close() + } + + private List indexKeys(String database, String collection) { + realClient.getDatabase(database).getCollection(collection).listIndexes()*.key + } + + private Map config(String database, Map extra = [:]) { + ['grails.mongodb.url' : dbContainer.getReplicaSetUrl(database), + (MongoSettings.SETTING_BUILD_INDEXES): false] + extra + } + + void "test an explicit build creates the declared indexes while GORM does not build them by itself"() { + given: + def log = new CapturedLog('org.grails.datastore.mapping', Level.INFO) + def datastore = new MongoDatastore(config('onDemandSyncDb'), OnDemandSyncThing) + + expect: "nothing was built at startup" + !([name: 1] in indexKeys('onDemandSyncDb', 'onDemandSyncThing')) + + when: "the application asks for the build" + datastore.buildIndex() + + then: "it runs on the calling thread, creates the declared index and reports it like the startup build" + [name: 1] in indexKeys('onDemandSyncDb', 'onDemandSyncThing') + log.events*.formattedMessage.any { + it.startsWith('Index build for database [onDemandSyncDb] finished') && it.contains('1 created, 0 already present') + } + + cleanup: + datastore?.close() + log?.close() + } + + void "test an explicit build runs on the background thread when builds are asynchronous"() { + given: + def conditions = new PollingConditions(timeout: 30) + def log = new CapturedLog('org.grails.datastore.mapping', Level.INFO) + def datastore = new MongoDatastore(config('onDemandAsyncDb', [(MongoSettings.SETTING_BUILD_INDEXES_ASYNC): true]), + OnDemandAsyncThing) + + when: + datastore.buildIndex() + + then: "the declared index is built, and by the connection's own index build thread" + conditions.eventually { + assert [name: 1] in indexKeys('onDemandAsyncDb', 'onDemandAsyncThing') + assert log.events.any { + it.formattedMessage.startsWith('Index build for database [onDemandAsyncDb] finished') && + it.threadName.startsWith('gorm-mongo-index-build-default-') + } + } + + cleanup: + datastore?.close() + log?.close() + } + + void "test a named connection is built through its own datastore"() { + given: + def datastore = new MongoDatastore(config('onDemandDefaultDb', [ + 'grails.mongodb.connections': [reporting: [url: dbContainer.getReplicaSetUrl('onDemandReportingDb')]] + ]), OnDemandConnectionThing) + + when: + (datastore.getDatastoreForConnection('reporting') as MongoDatastore).buildIndex() + + then: "that connection's index is built, and the default connection is left as it was" + [name: 1] in indexKeys('onDemandReportingDb', 'onDemandConnectionThing') + !([name: 1] in indexKeys('onDemandDefaultDb', 'onDemandConnectionThing')) + + cleanup: + datastore?.close() + } + + void "test a restart does not build what the setting says GORM should not build by itself"() { + given: + def log = new CapturedLog('org.grails.datastore.mapping', Level.INFO) + def datastore = new MongoDatastore(config('onDemandRestartDb', [(MongoSettings.SETTING_BUILD_INDEXES_ASYNC): true]), + OnDemandRestartThing) + + when: + datastore.stop() + datastore.start() + + then: "start() says so before returning when it resumes a build; it did not" + !log.events.any { it.formattedMessage.startsWith('Resuming the index build') } + !([name: 1] in indexKeys('onDemandRestartDb', 'onDemandRestartThing')) + + cleanup: + datastore?.close() + log?.close() + } +} + +@Entity +class OnDemandSyncThing { + String name + + static mapping = { + version false + collection 'onDemandSyncThing' + name index: true + } +} + +@Entity +class OnDemandAsyncThing { + String name + + static mapping = { + version false + collection 'onDemandAsyncThing' + name index: true + } +} + +@Entity +class OnDemandConnectionThing { + String name + + static mapping = { + version false + collection 'onDemandConnectionThing' + connection ConnectionSource.ALL + name index: true + } +} + +@Entity +class OnDemandRestartThing { + String name + + static mapping = { + version false + collection 'onDemandRestartThing' + name index: true + } +} diff --git a/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesPartialClassificationSpec.groovy b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesPartialClassificationSpec.groovy new file mode 100644 index 00000000000..98f8376aaf3 --- /dev/null +++ b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesPartialClassificationSpec.groovy @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.grails.datastore.gorm.mongo + +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicInteger + +import ch.qos.logback.classic.Level +import com.mongodb.MongoException +import com.mongodb.client.MongoClient +import com.mongodb.client.MongoClients +import com.mongodb.client.MongoCollection +import com.mongodb.client.model.IndexOptions +import grails.gorm.annotation.Entity +import org.bson.Document +import spock.lang.AutoCleanup +import spock.lang.Shared + +import org.apache.grails.testing.mongo.AutoStartedMongoSpec +import org.grails.datastore.mapping.core.DatastoreUtils +import org.grails.datastore.mapping.mongo.MongoDatastore + +/** + * A collection whose indexes cannot be listed - on its first listing, or on the re-listing a conflict + * makes - cannot say which of its indexes were new. That is a reason not to classify that collection, not + * to discard what the other collections' listings established. + */ +class BuildIndexesPartialClassificationSpec extends AutoStartedMongoSpec { + + static final String DATABASE = 'partialClassificationDb' + + @Shared + @AutoCleanup + MongoDatastore datastore + + @Shared + MongoClient realClient + + @Shared + String summary + + @Override + boolean shouldInitializeDatastore() { + false + } + + void setupSpec() { + realClient = MongoClients.create(dbContainer.getReplicaSetUrl(DATABASE)) + // Conflicts with the declared expiry, so its collection is listed a second time to reconcile it + realClient.getDatabase(DATABASE).getCollection('relistedThing') + .createIndex(new Document('created', 1), new IndexOptions().expireAfter(999L, TimeUnit.SECONDS)) + + def relistings = new AtomicInteger() + MongoClient partlyUnlistable = FailingMongoClient.wrap(realClient, 'listIndexes') { Closure proceed, MongoCollection collection -> + String name = collection.namespace.collectionName + if (name == 'unlistedThing') { + throw new MongoException('not authorized on partialClassificationDb to execute command listIndexes') + } + if (name == 'relistedThing' && relistings.incrementAndGet() > 1) { + throw new MongoException('connection reset while re-listing') + } + proceed() + } + + def log = new CapturedLog('org.grails.datastore.mapping', Level.INFO) + datastore = new MongoDatastore(partlyUnlistable, + DatastoreUtils.createPropertyResolver(['grails.mongodb.databaseName': DATABASE]), + ListedThing, UnlistedThing, RelistedThing) + summary = log.events*.formattedMessage.find { it.contains("database [$DATABASE]") } + log.close() + } + + void cleanupSpec() { + realClient?.close() + } + + void "test a collection that could not be listed is left unclassified without discarding the others"() { + expect: "the listed collection's index is still counted as created" + summary.contains('1 created, 0 already present') + + and: "the collection whose listing was refused is applied without being classified" + summary.contains('1 applied without a listing') + + and: "the conflict that could not be re-listed is a failure, as before" + summary.contains('1 failed') + } + + void "test the unlisted collection's index is still created"() { + expect: + [name: 1] in realClient.getDatabase(DATABASE).getCollection('unlistedThing').listIndexes()*.key + } +} + +@Entity +class ListedThing { + String name + + static mapping = { + version false + collection 'listedThing' + name index: true + } +} + +@Entity +class UnlistedThing { + String name + + static mapping = { + version false + collection 'unlistedThing' + name index: true + } +} + +@Entity +class RelistedThing { + Date created + + static mapping = { + version false + collection 'relistedThing' + created index: true, indexAttributes: [expireAfterSeconds: 100] + } +} diff --git a/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesRecreateSummarySpec.groovy b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesRecreateSummarySpec.groovy new file mode 100644 index 00000000000..eac95b698b6 --- /dev/null +++ b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesRecreateSummarySpec.groovy @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.grails.datastore.gorm.mongo + +import ch.qos.logback.classic.Level +import com.mongodb.client.MongoClient +import com.mongodb.client.MongoClients +import grails.gorm.annotation.Entity +import org.bson.Document +import spock.lang.AutoCleanup +import spock.lang.Shared + +import org.apache.grails.testing.mongo.AutoStartedMongoSpec +import org.grails.datastore.mapping.mongo.MongoDatastore + +/** + * A declaration that {@code recreateOnConflict} satisfies by dropping the index that was there and building + * the declared one costs a full build. The summary says so, instead of counting it with the indexes that + * were already present, which is what a restart that changed nothing reports. + */ +class BuildIndexesRecreateSummarySpec extends AutoStartedMongoSpec { + + static final String DATABASE = 'recreateSummaryDb' + + @Shared + @AutoCleanup + MongoDatastore datastore + + @Shared + MongoClient setupClient + + @Shared + CapturedLog log + + @Override + boolean shouldInitializeDatastore() { + false + } + + void setupSpec() { + String url = dbContainer.getReplicaSetUrl(DATABASE) + setupClient = MongoClients.create(url) + // A text index on another field. MongoDB allows one text index per collection, so the declaration + // conflicts with it, and recreateOnConflict replaces it. + setupClient.getDatabase(DATABASE).getCollection('recreatedThing').createIndex(new Document('title', 'text')) + + log = new CapturedLog('org.grails.datastore.mapping', Level.INFO) + datastore = new MongoDatastore(['grails.mongodb.url': url] as Map, RecreatedThing) + } + + void cleanupSpec() { + log?.close() + setupClient?.close() + } + + void "test an index dropped and built again is reported as recreated, not as already present"() { + given: + String summary = log.events*.formattedMessage.find { it.contains("database [$DATABASE]") } + + expect: + summary.contains('0 created, 1 recreated, 0 already present') + + and: "the text index is now the one declared" + setupClient.getDatabase(DATABASE).getCollection('recreatedThing').listIndexes() + .find { it.key == [_fts: 'text', _ftsx: 1] }.weights == [body: 1] + } +} + +@Entity +class RecreatedThing { + String title + String body + + static mapping = { + version false + collection 'recreatedThing' + body index: true, indexAttributes: [type: 'text', recreateOnConflict: true] + } +} diff --git a/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesStaleSnapshotSpec.groovy b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesStaleSnapshotSpec.groovy new file mode 100644 index 00000000000..da90179bd15 --- /dev/null +++ b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesStaleSnapshotSpec.groovy @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.grails.datastore.gorm.mongo + +import java.lang.reflect.InvocationHandler +import java.lang.reflect.Method +import java.lang.reflect.Proxy +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicInteger + +import ch.qos.logback.classic.Level +import com.mongodb.client.ListIndexesIterable +import com.mongodb.client.MongoClient +import com.mongodb.client.MongoClients +import com.mongodb.client.model.IndexOptions +import grails.gorm.annotation.Entity +import org.bson.Document +import spock.lang.AutoCleanup +import spock.lang.Shared + +import org.apache.grails.testing.mongo.AutoStartedMongoSpec +import org.grails.datastore.mapping.core.DatastoreUtils +import org.grails.datastore.mapping.mongo.MongoDatastore + +/** + * The build lists a collection's indexes once, before creating any, to tell a created index from one that + * was already there. An index that appears after that listing - created by another instance starting at + * the same time, or another connection building concurrently - is what the server then reports as + * conflicting, so reconciling the conflict has to look at the server again rather than at the listing. + */ +class BuildIndexesStaleSnapshotSpec extends AutoStartedMongoSpec { + + static final String DATABASE = 'staleSnapshotDb' + + @Shared + @AutoCleanup + MongoDatastore datastore + + @Shared + MongoClient realClient + + @Shared + CapturedLog log + + @Shared + AtomicInteger listings = new AtomicInteger() + + @Override + boolean shouldInitializeDatastore() { + false + } + + private static ListIndexesIterable noIndexes() { + (ListIndexesIterable) Proxy.newProxyInstance(ListIndexesIterable.classLoader, + [ListIndexesIterable] as Class[], { Object proxy, Method method, Object[] args -> + if (method.name == 'into') { + return args[0] + } + throw new UnsupportedOperationException(method.name) + } as InvocationHandler) + } + + void setupSpec() { + realClient = MongoClients.create(dbContainer.getReplicaSetUrl(DATABASE)) + // Already on the server with a different expiry, as another instance would have left it + realClient.getDatabase(DATABASE).getCollection('staleSnapshotThing') + .createIndex(new Document('created', 1), new IndexOptions().expireAfter(999L, TimeUnit.SECONDS)) + + // The first listing predates that index; every later one is the server's answer + MongoClient staleFirstListing = FailingMongoClient.wrap(realClient, 'listIndexes') { Closure proceed -> + listings.incrementAndGet() == 1 ? noIndexes() : proceed() + } + + // Classification only lists when the summary will be logged + log = new CapturedLog('org.grails.datastore.mapping', Level.INFO) + datastore = new MongoDatastore(staleFirstListing, + DatastoreUtils.createPropertyResolver(['grails.mongodb.databaseName': DATABASE]), StaleSnapshotThing) + } + + void cleanupSpec() { + log?.close() + realClient?.close() + } + + void "test a conflict is reconciled against the server, not the listing taken before it"() { + expect: "the stale listing was used, and the conflict listed again" + listings.get() == 2 + + and: "the index that appeared afterwards was reconciled in place to the declared expiry" + realClient.getDatabase(DATABASE).getCollection('staleSnapshotThing').listIndexes() + .find { it.key == [created: 1] }.expireAfterSeconds == 100 + + and: "it is reported as an index that was already there, with nothing failed" + log.events*.formattedMessage.find { it.contains("database [$DATABASE]") }.contains('0 created, 1 already present') + !log.events.any { it.level == Level.ERROR && it.formattedMessage.contains('StaleSnapshotThing') } + } +} + +@Entity +class StaleSnapshotThing { + Date created + + static mapping = { + version false + collection 'staleSnapshotThing' + created index: true, indexAttributes: [expireAfterSeconds: 100] + } +} diff --git a/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesSummaryLogSpec.groovy b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesSummaryLogSpec.groovy new file mode 100644 index 00000000000..bb99ba16c2b --- /dev/null +++ b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesSummaryLogSpec.groovy @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.grails.datastore.gorm.mongo + +import ch.qos.logback.classic.Level +import grails.gorm.annotation.Entity +import spock.lang.AutoCleanup +import spock.lang.Shared + +import org.apache.grails.testing.mongo.AutoStartedMongoSpec +import org.grails.datastore.mapping.mongo.MongoDatastore + +/** + * An index build that succeeds says so once, at the end: what it created, what was already there, how many + * domain classes it covered, and how long the caller spent waiting. That summary is the only signal a + * background build has finished at all, and the created/already-present split is what makes the elapsed + * time interpretable — a build that created nothing had nothing to wait for. + */ +class BuildIndexesSummaryLogSpec extends AutoStartedMongoSpec { + + static final String DATABASE = 'summaryLogDb' + + @Shared + @AutoCleanup + MongoDatastore datastore + + @Shared + CapturedLog log + + @Shared + List startupMessages + + @Override + boolean shouldInitializeDatastore() { + false + } + + void setupSpec() { + // Capture both inherited and MongoDatastore-specific logging categories. + log = new CapturedLog('org.grails.datastore.mapping', Level.INFO) + + datastore = new MongoDatastore( + ['grails.mongodb.url': dbContainer.getReplicaSetUrl(DATABASE)] as Map, + SummaryLoggedThing, OtherSummaryLoggedThing, UnindexedSummaryThing) + + // Snapshotted so that a feature triggering another build cannot change what the startup build said + startupMessages = messagesForThisDatabase() + } + + void cleanupSpec() { + log?.close() + } + + /** + * Other specifications create datastores of their own in this JVM, so the summaries are picked out by + * the database this specification uses rather than by being the only messages logged. + */ + private List messagesForThisDatabase() { + log.events.collect { it.formattedMessage }.findAll { it.contains("database [$DATABASE]") } + } + + void "test a successful index build logs one summary of what it applied and what it cost"() { + given: + String summary = startupMessages.first() + + expect: "exactly one summary for the build, not one line per index" + startupMessages.size() == 1 + + and: "a duplicate declaration confirms the index just created instead of counting it twice" + summary.contains('3 created, 1 already present') + + and: "the class count includes a class that declares no indexes" + summary.contains('from 3 domain class(es)') + + and: "and how long the caller waited" + summary ==~ /Index build for database \[$DATABASE] finished in \d+ms: .*/ + } + + void "test a repeated build reports the indexes as already present rather than created"() { + given: "the summaries logged so far" + int before = messagesForThisDatabase().size() + + when: "the same declarations are applied again, as they would be on the next restart" + datastore.buildIndex() + + then: "the build reports that it created nothing, which is why it cost next to nothing" + List since = messagesForThisDatabase().drop(before) + since.size() == 1 + since.first().contains('0 created, 4 already present') + } +} + +@Entity +class SummaryLoggedThing { + String name + Integer age + + static mapping = { + version false + collection 'summaryLoggedThing' + name index: true + compoundIndex name: 1, age: -1 + compoundIndex name: 1 + } +} + +@Entity +class OtherSummaryLoggedThing { + String title + + static mapping = { + version false + collection 'otherSummaryLoggedThing' + title index: true + } +} + +@Entity +class UnindexedSummaryThing { + String name +} diff --git a/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesUnfinishedSummarySpec.groovy b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesUnfinishedSummarySpec.groovy new file mode 100644 index 00000000000..d85e53c3731 --- /dev/null +++ b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesUnfinishedSummarySpec.groovy @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.grails.datastore.gorm.mongo + +import java.util.concurrent.atomic.AtomicInteger + +import ch.qos.logback.classic.Level +import com.mongodb.MongoSocketReadException +import com.mongodb.ServerAddress +import com.mongodb.client.MongoClient +import com.mongodb.client.MongoClients +import grails.gorm.annotation.Entity +import spock.lang.Shared +import spock.util.concurrent.PollingConditions + +import org.apache.grails.testing.mongo.AutoStartedMongoSpec +import org.grails.datastore.mapping.core.DatastoreUtils +import org.grails.datastore.mapping.mongo.MongoDatastore +import org.grails.datastore.mapping.mongo.config.MongoSettings + +/** + * A build that stops partway - a dropped connection, a timeout - still reports how far it got. In a + * background build that is all an operator has to go on besides the error itself: which indexes are + * there, and which domain classes never had theirs applied. + */ +class BuildIndexesUnfinishedSummarySpec extends AutoStartedMongoSpec { + + @Shared + MongoClient realClient + + @Override + boolean shouldInitializeDatastore() { + false + } + + void setupSpec() { + realClient = MongoClients.create(dbContainer.getReplicaSetUrl('unfinishedDb')) + } + + void cleanupSpec() { + realClient?.close() + } + + /** + * Lets the first index through and loses the connection on the second, so the build stops after one + * of the two domain classes. + */ + private MongoClient connectionLostOnSecondIndex() { + def created = new AtomicInteger() + FailingMongoClient.wrap(realClient, 'createIndex') { Closure proceed -> + if (created.incrementAndGet() > 1) { + throw new MongoSocketReadException('Prematurely reached end of stream', new ServerAddress()) + } + proceed() + } + } + + private static String unfinishedSummary(CapturedLog log, String database) { + log.events.find { + it.level == Level.WARN && it.formattedMessage.contains("database [$database] did not finish") + }?.formattedMessage + } + + void "test a background build that stops partway reports how far it got"() { + given: + def conditions = new PollingConditions(timeout: 30) + def log = new CapturedLog('org.grails.datastore.mapping', Level.INFO) + + when: + def datastore = new MongoDatastore(connectionLostOnSecondIndex(), DatastoreUtils.createPropertyResolver([ + 'grails.mongodb.databaseName' : 'unfinishedAsyncDb', + (MongoSettings.SETTING_BUILD_INDEXES_ASYNC): true + ]), FirstUnfinishedThing, SecondUnfinishedThing) + + then: "the failure is reported, and so is what had been applied before it" + conditions.eventually { + assert unfinishedSummary(log, 'unfinishedAsyncDb') != null + } + unfinishedSummary(log, 'unfinishedAsyncDb').contains('at 1 of 2 domain class(es): 1 created, 0 already present') + log.events.any { it.level == Level.ERROR && it.formattedMessage.contains('The background index build failed') } + + cleanup: + datastore?.close() + log?.close() + } + + void "test a build on the calling thread that stops partway reports how far it got and still fails"() { + given: + def log = new CapturedLog('org.grails.datastore.mapping', Level.INFO) + + when: + new MongoDatastore(connectionLostOnSecondIndex(), DatastoreUtils.createPropertyResolver([ + 'grails.mongodb.databaseName': 'unfinishedSyncDb' + ]), FirstUnfinishedThing, SecondUnfinishedThing) + + then: "the exception still fails startup, as it always has" + thrown(MongoSocketReadException) + + and: "but not before saying what was applied" + unfinishedSummary(log, 'unfinishedSyncDb').contains('at 1 of 2 domain class(es): 1 created, 0 already present') + + cleanup: + log?.close() + } +} + +@Entity +class FirstUnfinishedThing { + String name + + static mapping = { + version false + collection 'firstUnfinishedThing' + name index: true + } +} + +@Entity +class SecondUnfinishedThing { + String name + + static mapping = { + version false + collection 'secondUnfinishedThing' + name index: true + } +} diff --git a/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesUnreadableIndexListSpec.groovy b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesUnreadableIndexListSpec.groovy new file mode 100644 index 00000000000..6e8499bd52a --- /dev/null +++ b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesUnreadableIndexListSpec.groovy @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.grails.datastore.gorm.mongo + +import java.util.concurrent.TimeUnit + +import ch.qos.logback.classic.Level +import com.mongodb.MongoException +import com.mongodb.client.MongoClient +import com.mongodb.client.MongoClients +import grails.gorm.annotation.Entity +import com.mongodb.client.model.IndexOptions +import org.bson.Document +import spock.lang.AutoCleanup +import spock.lang.Shared + +import org.apache.grails.testing.mongo.AutoStartedMongoSpec +import org.grails.datastore.mapping.core.DatastoreUtils +import org.grails.datastore.mapping.mongo.MongoDatastore + +/** + * Telling an index this build created from one that was already there means reading the indexes the + * collection already has, which a role holding {@code createIndex} but not {@code listIndexes} is not + * allowed to do. Losing that breakdown must not cost the indexes themselves: the build goes ahead and the + * summary says only how many declarations it applied. + */ +class BuildIndexesUnreadableIndexListSpec extends AutoStartedMongoSpec { + + static final String DATABASE = 'unlistableIndexDb' + + @Shared + @AutoCleanup + MongoDatastore datastore + + @Shared + MongoClient realClient + + @Shared + CapturedLog log + + @Override + boolean shouldInitializeDatastore() { + false + } + + void setupSpec() { + realClient = MongoClients.create(dbContainer.getReplicaSetUrl(DATABASE)) + // A TTL index whose expiry differs from the one the mapping declares. MongoDB reports that as an + // IndexOptionsConflict, which GORM reconciles in place - but reconciling means reading the indexes + // that are there, which this role is not allowed to do. + realClient.getDatabase(DATABASE).getCollection('unlistableConflictThing') + .createIndex(new Document('created', 1), new IndexOptions().expireAfter(999L, TimeUnit.SECONDS)) + + log = new CapturedLog('org.grails.datastore.mapping', Level.INFO) + + MongoClient cannotList = FailingMongoClient.wrap(realClient, 'listIndexes') { + throw new MongoException('not authorized on unlistableIndexDb to execute command listIndexes') + } + datastore = new MongoDatastore(cannotList, + DatastoreUtils.createPropertyResolver(['grails.mongodb.databaseName': DATABASE]), + UnlistableIndexThing, UnlistableConflictThing) + } + + void cleanupSpec() { + log?.close() + realClient?.close() + } + + void "test the declared indexes are still created when the existing ones cannot be listed"() { + expect: + realClient.getDatabase(DATABASE).getCollection('unlistableIndexThing').listIndexes()*.key.contains([name: 1]) + } + + void "test the summary falls back to reporting how many declarations were applied"() { + given: + def summary = log.events.find { it.formattedMessage.contains("database [$DATABASE]") } + + expect: "the build reported itself" + summary != null + + and: "without claiming which indexes it created, because that could not be established" + summary.formattedMessage.contains('1 index declaration(s) applied') + !summary.formattedMessage.contains('created') + + and: "the declaration that conflicted is counted as failed, since it could not be reconciled blind" + summary.formattedMessage.contains('1 failed') + } + + void "test a conflict that cannot be reconciled without the index list is reported, not swallowed"() { + expect: "the error names the entity and says why the listing failed, which is what can be acted on" + log.events.any { + it.level == Level.ERROR && + it.formattedMessage.contains('could not inspect existing indexes') && + it.formattedMessage.contains('UnlistableConflictThing') && + it.formattedMessage.contains('not authorized on unlistableIndexDb to execute command listIndexes') + } + } +} + +@Entity +class UnlistableConflictThing { + Date created + + static mapping = { + version false + collection 'unlistableConflictThing' + created index: true, indexAttributes: [expireAfterSeconds: 100] + } +} + +@Entity +class UnlistableIndexThing { + String name + + static mapping = { + version false + collection 'unlistableIndexThing' + name index: true + } +} diff --git a/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/CapturedLog.groovy b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/CapturedLog.groovy new file mode 100644 index 00000000000..7f5b0f1152c --- /dev/null +++ b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/CapturedLog.groovy @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.grails.datastore.gorm.mongo + +import java.util.concurrent.CopyOnWriteArrayList + +import ch.qos.logback.classic.Level +import ch.qos.logback.classic.Logger +import ch.qos.logback.classic.spi.ILoggingEvent +import ch.qos.logback.core.read.ListAppender +import groovy.transform.CompileStatic +import org.slf4j.LoggerFactory + +/** + * Captures what one logger emits for the length of a test, then puts the logger back as it was. + * + *

The index build logs from background threads, which can keep running while a test reads what was + * captured and can outlive the datastore that started them. A plain {@link ListAppender} synchronises + * the append but not the read, so iterating it while a build thread logs can throw + * {@link java.util.ConcurrentModificationException}. The list here is copy-on-write, so every read sees + * a stable snapshot. + */ +@CompileStatic +class CapturedLog implements AutoCloseable { + + private final Logger logger + + private final Level previousLevel + + private final ListAppender appender = new ListAppender<>() + + CapturedLog(String loggerName, Level level) { + logger = LoggerFactory.getLogger(loggerName) as Logger + previousLevel = logger.level + logger.level = level + appender.list = new CopyOnWriteArrayList() + appender.start() + logger.addAppender(appender) + } + + /** + * @return what has been captured so far + */ + List getEvents() { + new ArrayList<>(appender.list) + } + + @Override + void close() { + logger.detachAppender(appender) + appender.stop() + logger.level = previousLevel + } +} diff --git a/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/FailingMongoClient.groovy b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/FailingMongoClient.groovy new file mode 100644 index 00000000000..985026ce652 --- /dev/null +++ b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/FailingMongoClient.groovy @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.grails.datastore.gorm.mongo + +import java.lang.reflect.InvocationHandler +import java.lang.reflect.InvocationTargetException +import java.lang.reflect.Method +import java.lang.reflect.Proxy + +import com.mongodb.client.MongoClient +import com.mongodb.client.MongoCollection +import com.mongodb.client.MongoDatabase +import groovy.transform.CompileStatic + +/** + * Wraps a real {@link MongoClient} so that one driver operation misbehaves, which is the only way to + * reach conditions the server will not produce on demand — a role that may create an index but not list + * the existing ones, or a connection that fails while a build is running on a background thread. + * + *

Everything else is the real driver talking to a real server: only the named method is intercepted, + * and the databases and collections handed out are wrapped in turn so that the interception survives the + * calls GORM makes to derive a collection. + */ +@CompileStatic +class FailingMongoClient { + + /** + * @param delegate the real client to pass everything else through to + * @param failingMethod the driver method to intercept, by name + * @param onCall what to do in its place — throw, block until released, or decide call by call. It + * receives a closure that makes the real call, for an interception that lets some calls + * through, and, if it takes a second parameter, the database or collection being called. + */ + static MongoClient wrap(MongoClient delegate, String failingMethod, Closure onCall) { + (MongoClient) proxy(MongoClient, delegate, failingMethod, onCall) + } + + private static Object proxy(Class iface, Object target, String failingMethod, Closure onCall) { + Proxy.newProxyInstance(iface.classLoader, [iface] as Class[], new InvocationHandler() { + + @Override + Object invoke(Object proxyInstance, Method method, Object[] args) { + if (method.name == failingMethod) { + Closure proceed = { -> invokeTarget(target, method, args) } + // A two-argument interceptor also gets the object called, to tell one collection from another + return onCall.maximumNumberOfParameters >= 2 ? onCall.call(proceed, target) : onCall.call(proceed) + } + Object result = invokeTarget(target, method, args) + if (result instanceof MongoDatabase) { + return proxy(MongoDatabase, result, failingMethod, onCall) + } + if (result instanceof MongoCollection) { + return proxy(MongoCollection, result, failingMethod, onCall) + } + return result + } + }) + } + + private static Object invokeTarget(Object target, Method method, Object[] args) { + try { + return method.invoke(target, args) + } + catch (InvocationTargetException e) { + throw e.cause + } + } +} diff --git a/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/TextIndexViaAttributesSpec.groovy b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/TextIndexViaAttributesSpec.groovy index e1fe3840eb6..8fc447c358e 100644 --- a/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/TextIndexViaAttributesSpec.groovy +++ b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/TextIndexViaAttributesSpec.groovy @@ -34,7 +34,7 @@ import org.bson.types.ObjectId class TextIndexViaAttributesSpec extends GrailsDataTckSpec { void setupSpec() { - manager.registerDomainClasses(TextThing) + manager.registerDomainClasses(TextThing, RepeatedTextThing) } private void initIndexes() { @@ -67,6 +67,22 @@ class TextIndexViaAttributesSpec extends GrailsDataTckSpec() + def log = new CapturedLog('org.grails.datastore.mapping', Level.DEBUG) + def factory = new MongoConnectionSourceFactory() { + @Override + ConnectionSource create(String name, MongoConnectionSourceSettings settings) { + def source = super.create(name, settings) + if (name != 'indexedAsync') { + return source + } + MongoClient blocking = FailingMongoClient.wrap(source.source, 'createIndex') { + worker.set(Thread.currentThread()) + buildReached.countDown() + releaseBuild.await() + } + new DefaultConnectionSource(name, blocking, settings) + } + } + Map childConfig = [url: dbContainer.getReplicaSetUrl('asyncChildDb'), buildIndexes: true] + Map config = [ + 'grails.mongodb.url': dbContainer.getReplicaSetUrl('asyncParentDb'), + 'grails.mongodb.buildIndexes': false, + 'grails.mongodb.buildIndexesAsync': true, + 'grails.mongodb.connections': addedAtRuntime ? [:] : [indexedAsync: childConfig] + ] + + when: + def parent = new MongoDatastore(DatastoreUtils.createPropertyResolver(config), factory, + new DefaultApplicationEventPublisher(), AsyncPerConnectionThing) + if (addedAtRuntime) { + parent.connectionSources.addConnectionSource('indexedAsync', childConfig) + } + + then: + parent.getDatastoreForConnection('indexedAsync').isBuildIndexesAsync() + buildReached.await(30, TimeUnit.SECONDS) + worker.get().name.startsWith('gorm-mongo-index-build-indexedAsync-') + + when: "the parent closes while its child is still building" + parent.close() + worker.get().join(10000) + + then: "the child worker is interrupted and terminates" + !worker.get().alive + + and: "the abandoned build is classified as shutdown rather than an error" + log.events.any { + it.threadName == worker.get().name && it.level == Level.DEBUG && + it.formattedMessage.contains('abandoned because the datastore is shutting down') + } + !log.events.any { + it.threadName == worker.get().name && it.level == Level.ERROR + } + + cleanup: + releaseBuild.countDown() + parent?.close() + worker.get()?.join(10000) + log.close() + + where: + addedAtRuntime << [false, true] + } + + void "test stop() and start() interrupt and resume a named connection's build"() { + given: + def conditions = new PollingConditions(timeout: 30) + def log = new CapturedLog('org.grails.datastore.mapping', Level.DEBUG) + def buildReached = new CountDownLatch(1) + def releaseBuild = new CountDownLatch(1) + def blockOnce = new AtomicBoolean(true) + def factory = new MongoConnectionSourceFactory() { + @Override + ConnectionSource create(String name, MongoConnectionSourceSettings settings) { + def source = super.create(name, settings) + if (name != 'checkpointedChild') { + return source + } + MongoClient blocking = FailingMongoClient.wrap(source.source, 'createIndex') { Closure proceed -> + if (blockOnce.compareAndSet(true, false)) { + buildReached.countDown() + releaseBuild.await() + } + proceed() + } + new DefaultConnectionSource(name, blocking, settings) + } + } + MongoClient inspector = MongoClients.create(dbContainer.getReplicaSetUrl('checkpointedChildDb')) + def childIndexes = { -> inspector.getDatabase('checkpointedChildDb').getCollection('asyncPerConnectionThing').listIndexes()*.key } + + when: "the named connection's build is under way when the datastore is stopped for a checkpoint" + def parent = new MongoDatastore(DatastoreUtils.createPropertyResolver([ + 'grails.mongodb.url' : dbContainer.getReplicaSetUrl('checkpointedParentDb'), + 'grails.mongodb.buildIndexes' : false, + 'grails.mongodb.buildIndexesAsync': true, + 'grails.mongodb.connections' : [checkpointedChild: [url: dbContainer.getReplicaSetUrl('checkpointedChildDb'), buildIndexes: true]] + ]), factory, new DefaultApplicationEventPublisher(), AsyncPerConnectionThing) + buildReached.await(30, TimeUnit.SECONDS) + + then: "the build announces which connection it is for" + log.events.any { + it.formattedMessage.startsWith('Building the indexes declared by the domain classes for connection [checkpointedChild]') + } + + when: + parent.stop() + + then: "the child's build is abandoned as a shutdown, not reported as a failure" + conditions.eventually { + assert log.events.any { + it.level == Level.DEBUG && it.formattedMessage.contains('database [checkpointedChildDb] did not finish') + } + } + !log.events.any { it.level == Level.ERROR && it.threadName.startsWith('gorm-mongo-index-build-checkpointedChild-') } + !([name: 1] in childIndexes()) + + when: "the datastore is restarted" + parent.start() + + then: "the child's build is resumed, runs again and completes" + log.events.any { it.formattedMessage.contains('Resuming the index build for connection [checkpointedChild]') } + conditions.eventually { + assert [name: 1] in childIndexes() + } + + cleanup: + releaseBuild.countDown() + parent?.close() + inspector?.close() + log?.close() + } + + void "test start() neither waits on nor re-runs a connection added while the datastore was stopped"() { + given: + def conditions = new PollingConditions(timeout: 30) + def log = new CapturedLog('org.grails.datastore.mapping', Level.INFO) + def parent = new MongoDatastore(DatastoreUtils.createPropertyResolver([ + 'grails.mongodb.url' : dbContainer.getReplicaSetUrl('stoppedParentDb'), + 'grails.mongodb.buildIndexes' : false, + 'grails.mongodb.buildIndexesAsync': true + ]), AsyncPerConnectionThing) + parent.stop() + parent.connectionSources.addConnectionSource('addedWhileStopped', + [url: dbContainer.getReplicaSetUrl('addedWhileStoppedDb'), buildIndexes: true]) + conditions.eventually { + assert log.events.any { it.formattedMessage.contains('database [addedWhileStoppedDb] finished') } + } + + when: "the datastore is restarted" + parent.start() + + then: "that connection's build was never interrupted, so start() resumes no second one" + !log.events.any { it.formattedMessage.startsWith('Resuming the index build for connection [addedWhileStopped]') } + + cleanup: + parent?.close() + log?.close() + } + + void "test a connection added after close() starts no index build"() { + given: + def parent = new MongoDatastore(DatastoreUtils.createPropertyResolver([ + 'grails.mongodb.url' : dbContainer.getReplicaSetUrl('closedParentDb'), + 'grails.mongodb.buildIndexes' : false, + 'grails.mongodb.buildIndexesAsync': true + ]), AsyncPerConnectionThing) + parent.close() + + when: + parent.connectionSources.addConnectionSource('addedAfterClose', + [url: dbContainer.getReplicaSetUrl('addedAfterCloseDb'), buildIndexes: true]) + + then: "a build would have started its thread before returning, and kept it for a second after finishing; none did" + !Thread.getAllStackTraces().keySet().any { it.name.startsWith('gorm-mongo-index-build-addedAfterClose-') } + + cleanup: + parent?.connectionSources?.getConnectionSource('addedAfterClose')?.close() + } + + void "test a connection can override the global setting"() { + expect: "the default connection is disabled by the top level setting" + !datastore.isBuildIndexes() + + and: "the connection that declares its own setting builds indexes" + datastore.getDatastoreForConnection('indexed').isBuildIndexes() + + and: "a connection that declares nothing inherits the top level setting" + !datastore.getDatastoreForConnection('inherits').isBuildIndexes() + } + + void "test only the connection with index building enabled has the declared index"() { + when: "a document is written through each connection so every collection exists" + PerConnectionThing.withNewSession { + new PerConnectionThing(name: 'Fred').save(flush: true) + } + PerConnectionThing.indexed.withNewSession { + new PerConnectionThing(name: 'Fred').save(flush: true) + } + + then: "the disabled connection has only the implicit _id index" + PerConnectionThing.collection.listIndexes()*.key == [[_id: 1]] + + and: "the enabled connection has the index declared in the mapping" + [name: 1] in PerConnectionThing.indexed.collection.listIndexes()*.key + } +} + +@Entity +class PerConnectionThing { + String name + + static mapping = { + version false + collection 'perConnectionThing' + connection ConnectionSource.ALL + name index: true + } +} + +@Entity +class AsyncPerConnectionThing { + String name + + static mapping = { + connection ConnectionSource.ALL + name index: true + } +} diff --git a/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/connections/BuildIndexesSharedDeclarationSpec.groovy b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/connections/BuildIndexesSharedDeclarationSpec.groovy new file mode 100644 index 00000000000..cd441f40d8a --- /dev/null +++ b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/connections/BuildIndexesSharedDeclarationSpec.groovy @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.grails.datastore.gorm.mongo.connections + +import grails.gorm.annotation.Entity +import spock.lang.AutoCleanup +import spock.lang.Shared + +import org.apache.grails.testing.mongo.AutoStartedMongoSpec +import org.grails.datastore.mapping.core.connections.ConnectionSource +import org.grails.datastore.mapping.mongo.MongoDatastore + +/** + * Every connection builds its indexes from the same mapping declaration, so reading the declaration must + * leave it as it was. A compound index's options live inside the declaration itself, under + * {@code indexAttributes}, and a build that consumed them would hand the first connection a unique index + * and every later one a plain index on the same keys. + */ +class BuildIndexesSharedDeclarationSpec extends AutoStartedMongoSpec { + + @Shared + @AutoCleanup + MongoDatastore datastore + + @Override + boolean shouldInitializeDatastore() { + false + } + + void setupSpec() { + datastore = new MongoDatastore([ + 'grails.mongodb.url' : "mongodb://${mongoHost}:${mongoPort}/sharedDefaultDb" as String, + 'grails.mongodb.connections': [ + 'second': ['url': "mongodb://${mongoHost}:${mongoPort}/sharedSecondDb" as String] + ] + ] as Map, SharedDeclarationThing) + } + + private static Map compoundIndexOn(def staticApi) { + staticApi.collection.listIndexes().find { it.key == [name: 1, age: -1] } as Map + } + + void "test every connection gets the compound index options that were declared"() { + expect: "the default connection's compound index is unique" + compoundIndexOn(SharedDeclarationThing)?.unique == true + + and: "and so is the other connection's, built from the same declaration" + compoundIndexOn(SharedDeclarationThing.second)?.unique == true + } +} + +@Entity +class SharedDeclarationThing { + String name + Integer age + + static mapping = { + version false + collection 'sharedDeclarationThing' + connection ConnectionSource.ALL + compoundIndex name: 1, age: -1, indexAttributes: [unique: true] + } +} diff --git a/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/connections/SchemaBasedMultiTenancySpec.groovy b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/connections/SchemaBasedMultiTenancySpec.groovy index 86c5df3da36..7abe09a3b09 100644 --- a/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/connections/SchemaBasedMultiTenancySpec.groovy +++ b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/connections/SchemaBasedMultiTenancySpec.groovy @@ -76,14 +76,13 @@ class SchemaBasedMultiTenancySpec extends AutoStartedMongoSpec { } void "Test persist and retrieve entities with multi tenancy"() { - setup: - CompanyB.eachTenant { - try { - CompanyB.DB.drop() - } catch(e) { - // continue - } - + setup: "the tenants this feature uses start empty" + // Not eachTenant: in SCHEMA mode every database on the server is a tenant, MongoDB's own admin, config + // and local included. Dropping config on the shared test server removes config.image_collection, and + // the next retryable findAndModify - native id generation, in any later specification - then aborts + // mongod when it recreates that collection inside the write. + ['test1', 'test2'].each { String tenantId -> + datastore.mongoClient.getDatabase(tenantId).drop() } when:"A tenant id is present" diff --git a/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/connections/SuppliedMongoClientSettingsSpec.groovy b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/connections/SuppliedMongoClientSettingsSpec.groovy new file mode 100644 index 00000000000..973e468fd24 --- /dev/null +++ b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/connections/SuppliedMongoClientSettingsSpec.groovy @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.grails.datastore.gorm.mongo.connections + +import com.mongodb.client.MongoClient +import com.mongodb.client.MongoClients +import grails.gorm.MultiTenant +import grails.gorm.annotation.Entity +import spock.lang.AutoCleanup +import spock.lang.Shared + +import org.apache.grails.testing.mongo.AutoStartedMongoSpec +import org.grails.datastore.gorm.events.DefaultApplicationEventPublisher +import org.grails.datastore.mapping.core.DatastoreUtils +import org.grails.datastore.mapping.mongo.MongoDatastore +import org.grails.datastore.mapping.mongo.config.MongoMappingContext +import org.grails.datastore.mapping.mongo.config.MongoSettings +import org.grails.datastore.mapping.multitenancy.MultiTenancySettings.MultiTenancyMode +import org.grails.datastore.mapping.multitenancy.resolvers.NoTenantResolver +import org.grails.datastore.mapping.multitenancy.resolvers.SystemPropertyTenantResolver + +/** + * An application that hands GORM an existing {@code MongoClient} - which is what happens whenever a + * {@code MongoClient} bean is already present, as with Spring Boot's MongoDB auto-configuration - must + * still have its {@code grails.mongodb} settings applied. Only the connection details are taken from the + * supplied client; everything describing how the datastore behaves still comes from the configuration. + */ +class SuppliedMongoClientSettingsSpec extends AutoStartedMongoSpec { + + @Shared + @AutoCleanup + MongoDatastore datastore + + @Shared + MongoClient mongoClient + + @Override + boolean shouldInitializeDatastore() { + false + } + + void setupSpec() { + mongoClient = MongoClients.create(dbContainer.getReplicaSetUrl('suppliedClientDb')) + Map config = [ + 'grails.mongodb.databaseName' : 'suppliedClientDb', + (MongoSettings.SETTING_BUILD_INDEXES): false, + 'grails.mongodb.transactional' : true + ] + datastore = new MongoDatastore(mongoClient, DatastoreUtils.createPropertyResolver(config), SuppliedClientThing) + } + + void cleanupSpec() { + mongoClient?.close() + } + + void "test the configured settings are applied to a datastore built on a supplied client"() { + expect: "the index setting is taken from the configuration rather than left at its default" + !datastore.isBuildIndexes() + + and: "so is any other datastore setting" + datastore.isTransactionsEnabled() + } + + void "test the database of a supplied mapping context is not overridden by a configured URL"() { + given: "a mapping context the caller built for one database, and configuration whose URL names another" + def mappingContext = new MongoMappingContext('mappingContextDb') + def configuration = DatastoreUtils.createPropertyResolver([ + 'grails.mongodb.url': 'mongodb://localhost/urlDb' + ]) + + when: "the datastore is built on the supplied client" + def contextDatastore = new MongoDatastore(mongoClient, configuration, mappingContext, + new DefaultApplicationEventPublisher()) + + then: "the connection details in the configuration go unused, the database name among them" + contextDatastore.defaultDatabase == 'mappingContextDb' + + cleanup: + contextDatastore?.close() + } + + void "test a supplied client applies #prefix multi-tenancy mode #mode"() { + given: + def configuration = DatastoreUtils.createPropertyResolver([ + 'grails.mongodb.databaseName': 'suppliedTenantDb', + ("${prefix}.multiTenancy.mode".toString()): mode.name() + ]) + + when: + def tenantDatastore = new MongoDatastore(mongoClient, configuration, new Class[0]) + + then: "both the datastore and the settings used by GORM static APIs agree" + tenantDatastore.multiTenancyMode == mode + tenantDatastore.connectionSources.defaultConnectionSource.settings.multiTenancy.mode == mode + + cleanup: + tenantDatastore?.close() + + where: + [prefix, mode] << [['grails.gorm', 'grails.mongodb'], MultiTenancyMode.values().toList()].combinations() + } + + void "test a supplied client takes its tenant resolver from tenantResolverClass, and only from there"() { + when: "the resolver is named by class, the only way to supply one when the client is supplied" + def named = new MongoDatastore(mongoClient, DatastoreUtils.createPropertyResolver([ + 'grails.gorm.multiTenancy.mode' : 'DISCRIMINATOR', + 'grails.gorm.multiTenancy.tenantResolverClass': SystemPropertyTenantResolver.name + ]), new Class[0]) + + then: + named.tenantResolver instanceof SystemPropertyTenantResolver + + when: "no resolver is named" + def unnamed = new MongoDatastore(mongoClient, DatastoreUtils.createPropertyResolver([ + 'grails.gorm.multiTenancy.mode': 'DISCRIMINATOR' + ]), new Class[0]) + + then: "the mode is active with a resolver that refuses every tenant-scoped operation" + unnamed.tenantResolver instanceof NoTenantResolver + + cleanup: + named?.close() + unnamed?.close() + } + + void "test MongoDB tenancy settings override the global GORM fallback for a supplied client"() { + when: + def tenantDatastore = new MongoDatastore(mongoClient, DatastoreUtils.createPropertyResolver([ + 'grails.gorm.multiTenancy.mode': 'DISCRIMINATOR', + 'grails.mongodb.multiTenancy.mode': 'NONE' + ]), new Class[0]) + + then: + tenantDatastore.multiTenancyMode == MultiTenancyMode.NONE + tenantDatastore.connectionSources.defaultConnectionSource.settings.multiTenancy.mode == MultiTenancyMode.NONE + + cleanup: + tenantDatastore?.close() + } + + void "test configured tenant discrimination isolates data with a supplied client"() { + given: + def tenantDatastore = new MongoDatastore(mongoClient, DatastoreUtils.createPropertyResolver([ + 'grails.mongodb.databaseName': 'suppliedTenantIsolationDb', + 'grails.gorm.multiTenancy.mode': 'DISCRIMINATOR' + ]), SuppliedTenantThing) + + when: + SuppliedTenantThing.withTenant('first') { + new SuppliedTenantThing(name: 'First tenant').save(flush: true, failOnError: true) + } + SuppliedTenantThing.withTenant('second') { + new SuppliedTenantThing(name: 'Second tenant').save(flush: true, failOnError: true) + } + + then: + SuppliedTenantThing.withTenant('first') { SuppliedTenantThing.list()*.name } == ['First tenant'] + SuppliedTenantThing.withTenant('second') { SuppliedTenantThing.list()*.name } == ['Second tenant'] + + cleanup: + tenantDatastore?.close() + } + + void "test the configured settings take effect and not merely report"() { + when: "a document is written so the collection certainly exists" + SuppliedClientThing.withNewSession { + new SuppliedClientThing(name: 'Fred').save(flush: true) + } + + then: "the index declared in the mapping was not created, as configured" + SuppliedClientThing.collection.listIndexes()*.key == [[_id: 1]] + } +} + +@Entity +class SuppliedClientThing { + String name + + static mapping = { + version false + collection 'suppliedClientThing' + name index: true + } +} + +@Entity +class SuppliedTenantThing implements MultiTenant { + String tenantId + String name +} diff --git a/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/mapping/mongo/config/MongoConnectionSourceSettingsSpec.groovy b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/mapping/mongo/config/MongoConnectionSourceSettingsSpec.groovy index c28ccafd37e..c92e092c05e 100644 --- a/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/mapping/mongo/config/MongoConnectionSourceSettingsSpec.groovy +++ b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/mapping/mongo/config/MongoConnectionSourceSettingsSpec.groovy @@ -22,6 +22,9 @@ import com.mongodb.ReadPreference import org.grails.datastore.mapping.core.DatastoreUtils import org.grails.datastore.mapping.mongo.connections.MongoConnectionSourceSettings import org.grails.datastore.mapping.mongo.connections.MongoConnectionSourceSettingsBuilder +import org.springframework.boot.context.properties.source.ConfigurationPropertySources +import org.springframework.core.env.MapPropertySource +import org.springframework.core.env.StandardEnvironment import spock.lang.Specification /** * Created by graemerocher on 29/06/16. @@ -49,6 +52,88 @@ class MongoConnectionSourceSettingsSpec extends Specification { settings.options.build().readPreference == ReadPreference.secondary() } + void "test index building is enabled unless it is switched off in configuration"() { + when: "no buildIndexes setting is supplied" + def settings = new MongoConnectionSourceSettingsBuilder(DatastoreUtils.createPropertyResolver([:])).build() + + then: "declared indexes are built on startup" + settings.buildIndexes + + when: "the setting is switched off" + def resolver = DatastoreUtils.createPropertyResolver([(MongoSettings.SETTING_BUILD_INDEXES): 'false']) + settings = new MongoConnectionSourceSettingsBuilder(resolver).build() + + then: "index building is disabled" + !settings.buildIndexes + } + + void "test the index build is synchronous unless it is switched to asynchronous in configuration"() { + when: "no buildIndexesAsync setting is supplied" + def settings = new MongoConnectionSourceSettingsBuilder(DatastoreUtils.createPropertyResolver([:])).build() + + then: "the index build blocks the thread creating the datastore" + !settings.buildIndexesAsync + + when: "the setting is switched on" + def resolver = DatastoreUtils.createPropertyResolver([(MongoSettings.SETTING_BUILD_INDEXES_ASYNC): 'true']) + settings = new MongoConnectionSourceSettingsBuilder(resolver).build() + + then: "the index build is asynchronous" + settings.buildIndexesAsync + } + + void "test MongoDB setting names require the documented camel case spelling"() { + given: + def defaults = new MongoConnectionSourceSettings() + + when: + def settings = new MongoConnectionSourceSettingsBuilder(DatastoreUtils.createPropertyResolver([ + 'grails.mongodb.database-name': 'ignoredDb', + 'grails.mongodb.build-indexes': false, + 'grails.mongodb.build-indexes-async': true + ])).build() + + then: + settings.databaseName == defaults.databaseName + settings.buildIndexes + !settings.buildIndexesAsync + + when: + settings = new MongoConnectionSourceSettingsBuilder(DatastoreUtils.createPropertyResolver([ + 'grails.mongodb.databaseName': 'configuredDb', + 'grails.mongodb.buildIndexes': false, + 'grails.mongodb.buildIndexesAsync': true + ])).build() + + then: + settings.databaseName == 'configuredDb' + !settings.buildIndexes + settings.buildIndexesAsync + } + + void "test a kebab-case setting name is not relaxed-bound under Spring Boot either"() { + given: "the environment a Spring Boot application hands GORM, with Boot's relaxed-binding source attached" + def environment = new StandardEnvironment() + environment.propertySources.addLast(new MapPropertySource('application.yml', [ + 'grails.mongodb.database-name' : 'kebabDb', + 'grails.mongodb.build-indexes' : 'false', + 'grails.mongodb.build-indexes-async': 'true' + ] as Map)) + ConfigurationPropertySources.attach(environment) + + expect: "the value is there under the name it was written with, but not under the one GORM asks for" + environment.getProperty('grails.mongodb.database-name') == 'kebabDb' + environment.getProperty('grails.mongodb.databaseName') == null + + when: + def settings = new MongoConnectionSourceSettingsBuilder(environment).build() + + then: "the settings are looked up by their camel case names, which Boot's source does not answer" + settings.databaseName == new MongoConnectionSourceSettings().databaseName + settings.buildIndexes + !settings.buildIndexesAsync + } + void "test mongo client settings builder with URL"() { when:"using a property resolver" Map myMap = ['grails.mongodb.url': 'mongodb://foo:bar@mycompany/mydb?maxPoolSize=5'] diff --git a/grails-data-mongodb/core/src/test/resources/logback-test.xml b/grails-data-mongodb/core/src/test/resources/logback-test.xml new file mode 100644 index 00000000000..4d81bf0a99a --- /dev/null +++ b/grails-data-mongodb/core/src/test/resources/logback-test.xml @@ -0,0 +1,34 @@ + + + + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + + + diff --git a/grails-data-mongodb/docs/src/docs/asciidoc/gettingStarted/advancedConfig.adoc b/grails-data-mongodb/docs/src/docs/asciidoc/gettingStarted/advancedConfig.adoc index 788d2cd2b0f..623dafca4fb 100644 --- a/grails-data-mongodb/docs/src/docs/asciidoc/gettingStarted/advancedConfig.adoc +++ b/grails-data-mongodb/docs/src/docs/asciidoc/gettingStarted/advancedConfig.adoc @@ -21,7 +21,20 @@ under the License. ==== Mongo Database Connection Configuration -As mentioned the GORM for MongoDB plugin will configure all the defaults for you, but if you wish to customize those defaults you can do so in the `grails-app/conf/application.groovy` file: +As mentioned the GORM for MongoDB plugin will configure all the defaults for you, but if you wish to customize those defaults you can do so in `grails-app/conf/application.yml`: + +[source,yaml] +---- +grails: + mongodb: + host: localhost + port: 27017 + username: blah + password: blah + databaseName: foo +---- + +or equivalently in `grails-app/conf/application.groovy`: [source,groovy] ---- @@ -36,6 +49,8 @@ grails { } ---- +NOTE: These settings are read by name, so write them exactly as they are documented. Unlike Spring Boot's own configuration properties they are not relaxed-bound, and a kebab-case spelling such as `database-name` is not recognised — it is ignored, leaving the default in place. + The `databaseName` setting configures the default database name. If not specified the `databaseName` will default to the name of your application. You can also customize the MongoDB connection settings using an `options` block: @@ -87,6 +102,8 @@ grails { username = ".." // the username to connect with password = ".." // the password to connect with stateless = false // whether to use stateless sessions by default + buildIndexes = true // whether GORM creates the indexes declared in mapping blocks by itself on startup + buildIndexesAsync = false // whether index builds run on a background thread instead of blocking the caller // Alternatively, using 'url' // url = "mongodb://localhost/mydb" @@ -141,6 +158,11 @@ grails: Selecting it logs a warning at startup. +==== Index Creation on Startup + + +GORM creates and reconciles the indexes declared in your mapping blocks each time the datastore starts, and startup waits for MongoDB to finish building them. Set `grails.mongodb.buildIndexes = false` to leave the indexes on the server untouched instead, and build them with `MongoDatastore.buildIndex()` when you choose, or `grails.mongodb.buildIndexesAsync = true` to keep building them but on a background thread — see <> for details and when to use each. + ==== Multi-Document Transactions diff --git a/grails-data-mongodb/docs/src/docs/asciidoc/introduction/releaseNotes.adoc b/grails-data-mongodb/docs/src/docs/asciidoc/introduction/releaseNotes.adoc index 243bb756785..0d5c4f059f7 100644 --- a/grails-data-mongodb/docs/src/docs/asciidoc/introduction/releaseNotes.adoc +++ b/grails-data-mongodb/docs/src/docs/asciidoc/introduction/releaseNotes.adoc @@ -25,6 +25,9 @@ Below are the details of the changes across releases: * TTL indexes via `indexAttributes: [expireAfterSeconds: N]` in the mapping DSL * Text and other special indexes via `indexAttributes: [type: 'text']` * In-place reconciliation of changed index options, with opt-in `indexAttributes: [recreateOnConflict: true]` +* Index creation on startup can be switched off with `grails.mongodb.buildIndexes = false`, and run on demand with `MongoDatastore.buildIndex()`, or moved off the startup thread with `grails.mongodb.buildIndexesAsync = true` +* `grails.gorm` and `grails.mongodb` settings are now applied when GORM is given an existing `MongoClient`, such as the one contributed by Spring Boot. Configuration that was previously ignored on this path — `failOnError`, the default mapping and constraints, flush behaviour and multi-tenancy — now takes effect. See <> before upgrading an application that supplies its own client. +* The index build reports what it created, what was already present and how long it took, in one summary line per build ==== 7.1 diff --git a/grails-data-mongodb/docs/src/docs/asciidoc/introduction/upgradeNotes.adoc b/grails-data-mongodb/docs/src/docs/asciidoc/introduction/upgradeNotes.adoc index c825db8f4bd..e6a6aaaa508 100644 --- a/grails-data-mongodb/docs/src/docs/asciidoc/introduction/upgradeNotes.adoc +++ b/grails-data-mongodb/docs/src/docs/asciidoc/introduction/upgradeNotes.adoc @@ -84,6 +84,32 @@ abstract class BookService { Please note that with autowire by-type as the default, when multiple beans for same type are found the application with throw Exception. Use the Spring `@Qualifier annotation for https://docs.spring.io/spring-framework/docs/5.3.10/reference/html/core.html#beans-autowired-annotation-qualifiers[Fine-tuning Annotation Based Autowiring with Qualifiers]. +==== Configuration Is Now Applied to an Externally-Supplied Client + +When GORM is handed a `MongoClient` that it did not create — which is what happens whenever a `MongoClient` bean is already present, as with Spring Boot's MongoDB auto-configuration — it built its settings from the defaults and applied only the database name. Everything else configured under `grails.gorm` and `grails.mongodb` was silently ignored on that path. Those settings are now applied, exactly as they always were when GORM creates the client itself. + +Nothing in your configuration changes. What changes is that configuration you already have starts taking effect: + +* `grails.gorm.failOnError` — an invalid `save()` now throws `ValidationException` instead of returning `null`. An application that has had this set for years while running without it will start throwing on the first invalid save. +* `grails.gorm.default.mapping` and `grails.gorm.default.constraints` — now applied to every domain class. +* `grails.gorm.autoFlush`, `grails.gorm.markDirty` and `grails.gorm.flushMode` — now honoured. +* `grails.gorm.multiTenancy` / `grails.mongodb.multiTenancy` — tenant discrimination and routing now follow the configured mode instead of `NONE`, so review your tenant resolver and your existing data before upgrading. + +Multi-tenancy needs one extra step on this path. A `TenantResolver` registered as a Spring bean is injected into the connection source factory, and there is no factory involved when the client is supplied, so the bean is never consulted. Configure the resolver by class instead: + +[source,yaml] +---- +grails: + gorm: + multiTenancy: + mode: DISCRIMINATOR + tenantResolverClass: com.example.MyTenantResolver +---- + +Without it the mode is active but the resolver is `NoTenantResolver`, which throws `TenantNotFoundException` on every tenant-scoped operation. Settings under `grails.mongodb` continue to take precedence over the `grails.gorm` equivalents. + +The connection details are the exception, since the supplied client is already connected: `grails.mongodb.url` is not applied to it. That also matters for a connection declared under `grails.mongodb.connections` without a `url` of its own, which falls back to the `host`, `port`, `username` and `password` settings — `localhost:27017` unless those are set — rather than to `grails.mongodb.url`. Give each such connection its own `url`. + ==== MongoClient Lifecycle for an Externally-Supplied Client When you construct a `MongoDatastore` with a `MongoClient` that you created yourself, GORM no longer closes that client when the datastore shuts down. The client is owned by whoever created it, so closing it is now your responsibility. diff --git a/grails-data-mongodb/docs/src/docs/asciidoc/querying/queryIndexes.adoc b/grails-data-mongodb/docs/src/docs/asciidoc/querying/queryIndexes.adoc index 2c4bc61a020..b707f395e07 100644 --- a/grails-data-mongodb/docs/src/docs/asciidoc/querying/queryIndexes.adoc +++ b/grails-data-mongodb/docs/src/docs/asciidoc/querying/queryIndexes.adoc @@ -142,6 +142,147 @@ WARNING: Dropping and recreating an index rebuilds it from scratch, during which A change to a TTL index's `expireAfterSeconds` is handled automatically and does not require `recreateOnConflict`, because GORM updates the expiry in place rather than rebuilding the index. +==== Disabling Index Creation on Startup + + +By default GORM creates and reconciles every index declared in a mapping block when the datastore starts. Set `buildIndexes` to `false` in `grails-app/conf/application.yml` to switch that off: + +[source,yaml] +---- +grails: + mongodb: + buildIndexes: false +---- + +or, in `application.groovy`: + +[source,groovy] +---- +grails { + mongodb { + buildIndexes = false + } +} +---- + +With this setting GORM issues no `createIndex` or `collMod` command by itself, for any domain class, and the indexes already present on the server are left exactly as they are until the application builds them (see <> below). Queries are unaffected and continue to use whichever indexes exist. This is useful when deploying against live data whose indexes are managed separately — by a DBA or a migration step — so that a deployment does not build an index against a large production collection, and so an application running against an older index set does not have those indexes reconciled underneath it. + +It also suppresses index creation for domain classes registered after startup. Declared at the top level the setting applies to every connection, and each connection can override it: + +[source,yaml] +---- +grails: + mongodb: + buildIndexes: false + connections: + reporting: + url: mongodb://localhost/reporting + buildIndexes: true +---- + +NOTE: Because nothing is created at startup, a collection that has never been built with `buildIndexes` enabled, or by an explicit build, has no declared indexes at all. Turn the setting off only where the indexes are already in place or are applied by other means. The setting governs only the indexes GORM derives from the mapping blocks; an explicit `createIndex` call made by application code against a collection is unaffected. + +[[buildIndexesOnDemand]] +===== Building Indexes on Demand + +Turning `buildIndexes` off stops GORM building indexes by itself. It does not stop the application building them when it chooses: once a deployment has been verified, from an administrative action, or on a schedule. Call `buildIndex()` on the `MongoDatastore` bean: + +[source,groovy] +---- +import org.grails.datastore.mapping.mongo.MongoDatastore + +class IndexMaintenanceService { + + MongoDatastore mongoDatastore + + void buildIndexes() { + mongoDatastore.buildIndex() + } +} +---- + +This creates and reconciles the declared indexes exactly as the startup build would, and logs the same summary. It blocks the calling thread until MongoDB has built every index, unless `buildIndexesAsync` is enabled, in which case it returns at once and the build runs on the background thread. Each named connection builds its own domain classes, so build those through that connection's datastore: + +[source,groovy] +---- +mongoDatastore.getDatastoreForConnection('reporting').buildIndex() +---- + +A build requested after the datastore has been closed is not started, and one requested while it is stopped for a checkpoint runs when it is restarted. + +==== Building Indexes in the Background + + +MongoDB answers a `createIndex` command only once the index has been built, so by default the thread that creates the datastore — in an application, the startup thread — waits for every declared index before the application finishes starting. On an empty collection that is instant; on a large existing collection an index build can take minutes, and a deployment waits for all of them in turn. + +Set `buildIndexesAsync` to have index builds run on a background thread instead, both the one GORM runs at startup and any the application starts with `buildIndex()`: + +[source,yaml] +---- +grails: + mongodb: + buildIndexesAsync: true +---- + +Startup then continues without waiting. The indexes are still built one at a time per connection, on a daemon thread named after its connection, such as `gorm-mongo-index-build-default-1`. Different connections can build indexes concurrently, including when they use the same MongoDB server. The thread is released after the build finishes and it has been idle for one second. + +Two consequences are worth planning for: + +* A query issued before its index has been built is served without it — correctly, but with the performance of an unindexed query. The same applies to a `unique` index: it constrains nothing until the build finishes. +* Because startup no longer waits, a failure to build an index can no longer fail startup. It is logged at error level, and the application runs without that index. With the default synchronous build the exception propagates and the application does not start. + +The setting also applies with `buildIndexes` set to `false`, to the builds the application starts with `buildIndex()`. A domain class registered after startup is indexed on the thread registering it. + +NOTE: If the application shuts down while a background build is still running, GORM stops waiting for it, but the server carries on building the index it was asked for. A datastore stopped for a checkpoint and restored (with CRaC) runs a build it cut short again once it is restarted, on every connection, and logs that it is resuming it. + +==== What the Index Build Reports + + +An index build that finishes without error logs one summary line at `INFO`: + +---- +Index build for database [myDb] finished in 412ms: 2 created, 5 already present, from 3 domain class(es) +---- + +The domain class count includes every class considered for indexing, including classes that declare no indexes. The created and already-present counts describe index declarations; declaring the same keys twice can report one created and one already present. + +The elapsed time is what the caller actually spent waiting — startup, or whoever called `buildIndex()`, with the default settings, or the background thread when `buildIndexesAsync` is enabled, where this line is also the only signal that the build has finished. + +The split between created and already present is what makes that time interpretable. MongoDB answers a `createIndex` for an index it already has immediately and without building anything, so a restart that changed no mappings reports everything as already present and costs milliseconds; a line reporting indexes created is the one that accounts for a slow start. An index that `recreateOnConflict` dropped and built again costs as much as a new one, and is counted apart as recreated: + +---- +Index build for database [myDb] finished in 9315ms: 0 created, 1 recreated, 6 already present, from 3 domain class(es) +---- + +If any declaration failed, the summary is logged at `WARN` instead and reports how many; the failures themselves are logged individually as they happen. A build that stops partway — a lost connection or a timeout — reports how far it got before stopping, also at `WARN`: + +---- +Index build for database [myDb] did not finish, stopping after 812ms at 1 of 3 domain class(es): 1 created, 0 already present +---- + +Telling a created index from one that was already there takes one `listIndexes` command per collection that declares indexes. It is only issued when the summary will be logged, that is when `INFO` is enabled for `org.grails.datastore.mapping.mongo`. + +A collection whose indexes cannot be listed still has its declarations applied; only the split between created and already present is missing for it. The summary counts those declarations separately, and the other collections keep their split: + +---- +Index build for database [myDb] finished in 412ms: 2 created, 5 already present, 3 applied without a listing, from 3 domain class(es) +---- + +When nothing could be classified — the logger is at `WARN`, so the build skips `listIndexes` and a summary is logged only for a build that failed or did not finish, or no collection's indexes could be listed — the summary reports how many declarations were applied instead: + +---- +Index build for database [myDb] finished in 412ms: 10 index declaration(s) applied, 1 failed, from 3 domain class(es) +---- + +To find which index is the slow one, enable `DEBUG` logging for `org.grails.datastore.mapping.mongo`, which adds a line per index with its own elapsed time: + +[source,yaml] +---- +logging: + level: + org.grails.datastore.mapping.mongo: DEBUG +---- + ==== Indexing using the 'index' method diff --git a/grails-data-mongodb/grails-plugin/src/main/resources/META-INF/additional-spring-configuration-metadata.json b/grails-data-mongodb/grails-plugin/src/main/resources/META-INF/additional-spring-configuration-metadata.json index 998fb14858b..80c63039b4c 100644 --- a/grails-data-mongodb/grails-plugin/src/main/resources/META-INF/additional-spring-configuration-metadata.json +++ b/grails-data-mongodb/grails-plugin/src/main/resources/META-INF/additional-spring-configuration-metadata.json @@ -66,6 +66,18 @@ "description": "Whether to use Decimal128 for BigDecimal values instead of Double.", "defaultValue": false }, + { + "name": "grails.mongodb.buildIndexes", + "type": "java.lang.Boolean", + "description": "Whether GORM creates the indexes declared in domain class mapping blocks by itself when the datastore starts. When false, the indexes already on the server are left untouched until MongoDatastore.buildIndex() is called.", + "defaultValue": true + }, + { + "name": "grails.mongodb.buildIndexesAsync", + "type": "java.lang.Boolean", + "description": "Whether index builds, the one on startup and any started with MongoDatastore.buildIndex(), run on a background thread instead of blocking the thread that starts them.", + "defaultValue": false + }, { "name": "grails.mongodb.codecs", "type": "java.util.List", diff --git a/grails-test-examples/mongodb/base/src/test/groovy/functional/tests/LocalMongoUnitSpec.groovy b/grails-test-examples/mongodb/base/src/test/groovy/functional/tests/LocalMongoUnitSpec.groovy index 2565c1ffcb9..571117c68b7 100644 --- a/grails-test-examples/mongodb/base/src/test/groovy/functional/tests/LocalMongoUnitSpec.groovy +++ b/grails-test-examples/mongodb/base/src/test/groovy/functional/tests/LocalMongoUnitSpec.groovy @@ -25,7 +25,6 @@ import org.apache.grails.testing.mongo.AbstractMongoGrailsExtension import org.testcontainers.containers.MongoDBContainer import org.testcontainers.utility.DockerImageName import spock.lang.AutoCleanup -import spock.lang.PendingFeature import spock.lang.Shared class LocalMongoUnitSpec extends MongoSpec implements EmbeddedMongoClient { @@ -50,7 +49,6 @@ class LocalMongoUnitSpec extends MongoSpec implements EmbeddedMongoClient { } //tag::structure[] - @PendingFeature(reason = 'A ValidationException is not thrown') void "test fail on error"() { when: