diff --git a/modules/client-handler/src/integrationTest/java/org/apache/ignite/client/handler/TestServer.java b/modules/client-handler/src/integrationTest/java/org/apache/ignite/client/handler/TestServer.java index 230e2ebe32ec..3dfeffcdd182 100644 --- a/modules/client-handler/src/integrationTest/java/org/apache/ignite/client/handler/TestServer.java +++ b/modules/client-handler/src/integrationTest/java/org/apache/ignite/client/handler/TestServer.java @@ -47,6 +47,7 @@ import org.apache.ignite.internal.sql.engine.QueryProcessor; import org.apache.ignite.internal.table.IgniteTablesInternal; import org.apache.ignite.internal.tx.TxManager; +import org.apache.ignite.internal.util.PartitionOperationInFlightLimiter; import org.apache.ignite.network.NetworkAddress; import org.jetbrains.annotations.Nullable; import org.junit.jupiter.api.TestInfo; @@ -144,6 +145,7 @@ ClientHandlerModule start(TestInfo testInfo) { EventLog.NOOP, new TestLowWatermark(), Runnable::run, + new PartitionOperationInFlightLimiter(0), () -> true ); diff --git a/modules/client-handler/src/main/java/org/apache/ignite/client/handler/ClientHandlerModule.java b/modules/client-handler/src/main/java/org/apache/ignite/client/handler/ClientHandlerModule.java index b7e6a357fc6d..64b8e47216f0 100644 --- a/modules/client-handler/src/main/java/org/apache/ignite/client/handler/ClientHandlerModule.java +++ b/modules/client-handler/src/main/java/org/apache/ignite/client/handler/ClientHandlerModule.java @@ -74,6 +74,7 @@ import org.apache.ignite.internal.table.IgniteTablesInternal; import org.apache.ignite.internal.tx.TxManager; import org.apache.ignite.internal.util.IgniteSpinBusyLock; +import org.apache.ignite.internal.util.PartitionOperationInFlightLimiter; import org.apache.ignite.lang.IgniteException; import org.jetbrains.annotations.Nullable; import org.jetbrains.annotations.TestOnly; @@ -163,6 +164,8 @@ public class ClientHandlerModule implements IgniteComponent, PlatformComputeTran private final Executor partitionOperationsExecutor; + private final PartitionOperationInFlightLimiter partitionOperationInFlightLimiter; + private final ConcurrentHashMap> computeExecutors = new ConcurrentHashMap<>(); @TestOnly @@ -186,6 +189,7 @@ public class ClientHandlerModule implements IgniteComponent, PlatformComputeTran * @param eventLog Event log. * @param lowWatermark Low watermark. * @param partitionOperationsExecutor Executor for a partition operation. + * @param partitionOperationInFlightLimiter In-flight limiter for partition operations. * @param ddlBatchingSuggestionEnabled Boolean supplier indicates whether the suggestion related DDL batching is enabled. */ public ClientHandlerModule( @@ -207,6 +211,7 @@ public ClientHandlerModule( EventLog eventLog, LowWatermark lowWatermark, Executor partitionOperationsExecutor, + PartitionOperationInFlightLimiter partitionOperationInFlightLimiter, Supplier ddlBatchingSuggestionEnabled ) { assert igniteTables != null; @@ -252,6 +257,7 @@ public ClientHandlerModule( this.clientConnectorConfiguration = clientConnectorConfiguration; this.ddlBatchingSuggestionEnabled = ddlBatchingSuggestionEnabled; this.partitionOperationsExecutor = partitionOperationsExecutor; + this.partitionOperationInFlightLimiter = partitionOperationInFlightLimiter; } /** {@inheritDoc} */ @@ -471,6 +477,7 @@ private ClientInboundMessageHandler createInboundMessageHandler( connectionId, primaryReplicaTracker, partitionOperationsExecutor, + partitionOperationInFlightLimiter, SUPPORTED_FEATURES, Map.of(), computeExecutors::remove, diff --git a/modules/client-handler/src/main/java/org/apache/ignite/client/handler/ClientInboundMessageHandler.java b/modules/client-handler/src/main/java/org/apache/ignite/client/handler/ClientInboundMessageHandler.java index 7cea0def2768..62ba0a988ba4 100644 --- a/modules/client-handler/src/main/java/org/apache/ignite/client/handler/ClientInboundMessageHandler.java +++ b/modules/client-handler/src/main/java/org/apache/ignite/client/handler/ClientInboundMessageHandler.java @@ -153,6 +153,7 @@ import org.apache.ignite.internal.jdbc.proto.JdbcQueryCursorHandler; import org.apache.ignite.internal.lang.IgniteExceptionMapperUtil; import org.apache.ignite.internal.lang.IgniteInternalCheckedException; +import org.apache.ignite.internal.lang.ReplicaOverloadedException; import org.apache.ignite.internal.logger.IgniteLogger; import org.apache.ignite.internal.logger.Loggers; import org.apache.ignite.internal.network.ClusterService; @@ -180,6 +181,7 @@ import org.apache.ignite.internal.tx.TransactionKilledException; import org.apache.ignite.internal.tx.TxManager; import org.apache.ignite.internal.util.ExceptionUtils; +import org.apache.ignite.internal.util.PartitionOperationInFlightLimiter; import org.apache.ignite.lang.CancelHandle; import org.apache.ignite.lang.ErrorGroups.Compute; import org.apache.ignite.lang.ErrorGroups.Sql; @@ -278,6 +280,8 @@ public class ClientInboundMessageHandler private final Executor partitionOperationsExecutor; + private final PartitionOperationInFlightLimiter partitionOperationInFlightLimiter; + private final BitSet features; private final Map extensions; @@ -309,6 +313,7 @@ public class ClientInboundMessageHandler * @param connectionId Connection ID. * @param primaryReplicaTracker Primary replica tracker. * @param partitionOperationsExecutor Partition operations executor. + * @param partitionOperationInFlightLimiter In-flight limiter for partition operations. * @param features Features. * @param extensions Extensions. * @param eventLog Event log. @@ -330,6 +335,7 @@ public ClientInboundMessageHandler( long connectionId, ClientPrimaryReplicaTracker primaryReplicaTracker, Executor partitionOperationsExecutor, + PartitionOperationInFlightLimiter partitionOperationInFlightLimiter, BitSet features, Map extensions, Function> computeConnectionFunc, @@ -373,6 +379,7 @@ public ClientInboundMessageHandler( this.eventLog = eventLog; this.primaryReplicaTracker = primaryReplicaTracker; this.partitionOperationsExecutor = partitionOperationsExecutor; + this.partitionOperationInFlightLimiter = partitionOperationInFlightLimiter; this.handshakeEventLoopSwitcher = handshakeEventLoopSwitcher; jdbcQueryCursorHandler = new JdbcQueryCursorHandlerImpl(resources); @@ -882,19 +889,28 @@ private void processOperation(ChannelHandlerContext ctx, ClientMessageUnpacker i if (ClientOp.isPartitionOperation(opCode)) { long requestId0 = requestId; int opCode0 = opCode; + if (!partitionOperationInFlightLimiter.tryAcquire()) { + in.close(); - partitionOperationsExecutor.execute(() -> { - try { - processOperationInternal(ctx, in, requestId0, opCode0, guard); - } catch (Throwable t) { - in.close(); - - writeError(requestId0, opCode0, t, ctx, false, guard); + writeError(requestId0, opCode0, new ReplicaOverloadedException(), ctx, false, guard); - metrics.requestsFailedIncrement(); - metrics.requestsActiveDecrement(); - } - }); + metrics.requestsFailedIncrement(); + } else { + partitionOperationsExecutor.execute(() -> { + try { + processOperationInternal(ctx, in, requestId0, opCode0, guard); + } catch (Throwable t) { + in.close(); + + writeError(requestId0, opCode0, t, ctx, false, guard); + + metrics.requestsFailedIncrement(); + metrics.requestsActiveDecrement(); + } finally { + partitionOperationInFlightLimiter.release(); + } + }); + } } else { processOperationInternal(ctx, in, requestId, opCode, guard); } diff --git a/modules/client/src/test/java/org/apache/ignite/client/TestClientHandlerModule.java b/modules/client/src/test/java/org/apache/ignite/client/TestClientHandlerModule.java index 8c86fe7ed6bd..13c3f1d47f25 100644 --- a/modules/client/src/test/java/org/apache/ignite/client/TestClientHandlerModule.java +++ b/modules/client/src/test/java/org/apache/ignite/client/TestClientHandlerModule.java @@ -65,6 +65,7 @@ import org.apache.ignite.internal.schema.AlwaysSyncedSchemaSyncService; import org.apache.ignite.internal.security.authentication.AuthenticationManager; import org.apache.ignite.internal.table.IgniteTablesInternal; +import org.apache.ignite.internal.util.PartitionOperationInFlightLimiter; import org.apache.ignite.lang.IgniteException; import org.jetbrains.annotations.Nullable; @@ -272,6 +273,7 @@ protected void initChannel(Channel ch) { new TestLowWatermark() ), Runnable::run, + new PartitionOperationInFlightLimiter(0), features, randomExtensions(), unused -> null, diff --git a/modules/client/src/test/java/org/apache/ignite/client/TestServer.java b/modules/client/src/test/java/org/apache/ignite/client/TestServer.java index 013462ff3aed..73a741ffb1fa 100644 --- a/modules/client/src/test/java/org/apache/ignite/client/TestServer.java +++ b/modules/client/src/test/java/org/apache/ignite/client/TestServer.java @@ -79,6 +79,7 @@ import org.apache.ignite.internal.security.authentication.AuthenticationManagerImpl; import org.apache.ignite.internal.security.configuration.SecurityConfiguration; import org.apache.ignite.internal.table.IgniteTablesInternal; +import org.apache.ignite.internal.util.PartitionOperationInFlightLimiter; import org.apache.ignite.network.NetworkAddress; import org.jetbrains.annotations.Nullable; import org.mockito.Mockito; @@ -290,6 +291,7 @@ public void log(String type, Supplier eventProvider) { EventLog.NOOP, new TestLowWatermark(), Runnable::run, + new PartitionOperationInFlightLimiter(0), () -> true ); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/lang/ReplicaOverloadedException.java b/modules/core/src/main/java/org/apache/ignite/internal/lang/ReplicaOverloadedException.java new file mode 100644 index 000000000000..c5d5fb8f9baa --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/lang/ReplicaOverloadedException.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.lang; + +import static org.apache.ignite.lang.ErrorGroups.Replicator.GROUP_OVERLOADED_ERR; + +/** + * Thrown when the node has reached the maximum number of in-flight partition operations + * ({@code replication.maxInFlightPartitionOperations}) and cannot accept new requests. + */ +public class ReplicaOverloadedException extends IgniteInternalException { + private static final long serialVersionUID = -6023736883539658779L; + + /** Constructor. */ + public ReplicaOverloadedException() { + super(GROUP_OVERLOADED_ERR, "Node is overloaded: max in-flight partition operations limit reached."); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/PartitionOperationInFlightLimiter.java b/modules/core/src/main/java/org/apache/ignite/internal/util/PartitionOperationInFlightLimiter.java new file mode 100644 index 000000000000..5a0b4d7608ef --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/PartitionOperationInFlightLimiter.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.util; + +import java.util.concurrent.Semaphore; +import java.util.function.IntSupplier; +import org.jetbrains.annotations.Nullable; + +/** + * Limits the number of in-flight partition operations (queued or executing) across the replica manager and thin-client connector. + * + *

When the limit is zero or less, all operations are permitted unconditionally. + * When positive, {@link #tryAcquire()} returns {@code false} once the limit is reached and the caller should reject the request. + * A permit must be released via {@link #release()} upon operation completes. + */ +public class PartitionOperationInFlightLimiter { + private volatile Semaphore semaphore; + + private final @Nullable IntSupplier limitSupplier; + + private volatile boolean initialized; + + /** + * Constructor. + * + * @param maxInFlightPartitionOperations Max number of in-flight partition operations, or <= 0 to disable the limit. + */ + public PartitionOperationInFlightLimiter(int maxInFlightPartitionOperations) { + this.semaphore = maxInFlightPartitionOperations <= 0 ? null : new Semaphore(maxInFlightPartitionOperations); + this.limitSupplier = null; + this.initialized = true; + } + + /** + * Constructor that initializes the limit lazily from the given supplier on first use. + * Allows lazy init on first usage. + * + * @param maxInFlightPartitionOperationsSupplier Supplier of the maximum number of in-flight partition operations, or 0 to disable. + */ + public PartitionOperationInFlightLimiter(@Nullable IntSupplier maxInFlightPartitionOperationsSupplier) { + this.limitSupplier = maxInFlightPartitionOperationsSupplier; + this.initialized = false; + } + + /** + * Attempts to acquire a permit. + * + * @return {@code true} if a permit was acquired or the limit is disabled; {@code false} if the limit is reached. + */ + public boolean tryAcquire() { + Semaphore s = resolvedSemaphore(); + return s == null || s.tryAcquire(); + } + + /** + * Releases a previously acquired permit. + * Must only be called after a successful {@link #tryAcquire()} when the limit is enabled. + */ + public void release() { + Semaphore s = resolvedSemaphore(); + + if (s != null) { + s.release(); + } + } + + private @Nullable Semaphore resolvedSemaphore() { + if (initialized) { + return semaphore; + } + synchronized (this) { + if (initialized) { + return semaphore; + } + if (limitSupplier != null) { + int limit = limitSupplier.getAsInt(); + + if (limit != 0) { + this.semaphore = new Semaphore(limit); + } + } + this.initialized = true; + } + return semaphore; + } +} diff --git a/modules/distribution-zones/src/integrationTest/java/org/apache/ignite/internal/rebalance/ItRebalanceDistributedTest.java b/modules/distribution-zones/src/integrationTest/java/org/apache/ignite/internal/rebalance/ItRebalanceDistributedTest.java index 02684f96d6d4..9d702922e81f 100644 --- a/modules/distribution-zones/src/integrationTest/java/org/apache/ignite/internal/rebalance/ItRebalanceDistributedTest.java +++ b/modules/distribution-zones/src/integrationTest/java/org/apache/ignite/internal/rebalance/ItRebalanceDistributedTest.java @@ -258,6 +258,7 @@ import org.apache.ignite.internal.tx.message.TxMessageGroup; import org.apache.ignite.internal.tx.storage.state.rocksdb.TxStateRocksDbSharedStorage; import org.apache.ignite.internal.tx.test.TestLocalRwTxCounter; +import org.apache.ignite.internal.util.PartitionOperationInFlightLimiter; import org.apache.ignite.internal.vault.VaultManager; import org.apache.ignite.internal.vault.persistence.PersistentVaultService; import org.apache.ignite.network.NetworkAddress; @@ -1476,6 +1477,7 @@ private class Node { Set.of(PartitionReplicationMessageGroup.class, TxMessageGroup.class), placementDriver, threadPoolsManager.partitionOperationsExecutor(), + new PartitionOperationInFlightLimiter(0), partitionIdleSafeTimePropagationPeriodMsSupplier, new NoOpFailureManager(), new ThreadLocalPartitionCommandsMarshaller(clusterService.serializationRegistry()), diff --git a/modules/partition-replicator/src/integrationTest/java/org/apache/ignite/internal/partition/replicator/fixtures/Node.java b/modules/partition-replicator/src/integrationTest/java/org/apache/ignite/internal/partition/replicator/fixtures/Node.java index d61f25985abc..b73fcd014c42 100644 --- a/modules/partition-replicator/src/integrationTest/java/org/apache/ignite/internal/partition/replicator/fixtures/Node.java +++ b/modules/partition-replicator/src/integrationTest/java/org/apache/ignite/internal/partition/replicator/fixtures/Node.java @@ -195,6 +195,7 @@ import org.apache.ignite.internal.tx.storage.state.TxStatePartitionStorage; import org.apache.ignite.internal.tx.storage.state.rocksdb.TxStateRocksDbSharedStorage; import org.apache.ignite.internal.tx.test.TestLocalRwTxCounter; +import org.apache.ignite.internal.util.PartitionOperationInFlightLimiter; import org.apache.ignite.internal.vault.VaultManager; import org.apache.ignite.network.NetworkAddress; import org.apache.ignite.raft.jraft.rpc.impl.RaftGroupEventsClientListener; @@ -668,6 +669,7 @@ public CompletableFuture invoke(Condition condition, Operation success, Set.of(PartitionReplicationMessageGroup.class, TxMessageGroup.class), placementDriverManager.placementDriver(), threadPoolsManager.partitionOperationsExecutor(), + new PartitionOperationInFlightLimiter(0), partitionIdleSafeTimePropagationPeriodMsSupplier, new NoOpFailureManager(), new ThreadLocalPartitionCommandsMarshaller(clusterService.serializationRegistry()), diff --git a/modules/partition-replicator/src/main/java/org/apache/ignite/internal/partition/replicator/handlers/TxCleanupRecoveryRequestHandler.java b/modules/partition-replicator/src/main/java/org/apache/ignite/internal/partition/replicator/handlers/TxCleanupRecoveryRequestHandler.java index 9fd5ddeedbdc..54ad62a5b711 100644 --- a/modules/partition-replicator/src/main/java/org/apache/ignite/internal/partition/replicator/handlers/TxCleanupRecoveryRequestHandler.java +++ b/modules/partition-replicator/src/main/java/org/apache/ignite/internal/partition/replicator/handlers/TxCleanupRecoveryRequestHandler.java @@ -33,6 +33,7 @@ import org.apache.ignite.internal.failure.FailureProcessor; import org.apache.ignite.internal.lang.IgniteBiTuple; import org.apache.ignite.internal.lang.IgniteInternalException; +import org.apache.ignite.internal.lang.NodeStoppingException; import org.apache.ignite.internal.logger.IgniteLogger; import org.apache.ignite.internal.logger.IgniteThrottledLogger; import org.apache.ignite.internal.logger.Loggers; @@ -156,13 +157,14 @@ private CompletableFuture callCleanup(TxMeta txMeta, UUID txId) { txMeta.commitTimestamp(), txId ).exceptionally(throwable -> { - throttledLog.warn( - "Failed to cleanup transaction", - "Failed to cleanup transaction {}.", - throwable, - formatTxInfo(txId, txManager) - ); - + if (!hasCause(throwable, NodeStoppingException.class)) { + throttledLog.warn( + "Failed to cleanup transaction", + "Failed to cleanup transaction {}.", + throwable, + formatTxInfo(txId, txManager) + ); + } return null; }); } diff --git a/modules/partition-replicator/src/test/java/org/apache/ignite/internal/partition/replicator/PartitionReplicaLifecycleManagerTest.java b/modules/partition-replicator/src/test/java/org/apache/ignite/internal/partition/replicator/PartitionReplicaLifecycleManagerTest.java index c2fed07bfc42..d7d9abcc1f18 100644 --- a/modules/partition-replicator/src/test/java/org/apache/ignite/internal/partition/replicator/PartitionReplicaLifecycleManagerTest.java +++ b/modules/partition-replicator/src/test/java/org/apache/ignite/internal/partition/replicator/PartitionReplicaLifecycleManagerTest.java @@ -121,6 +121,7 @@ import org.apache.ignite.internal.tx.storage.state.rocksdb.TxStateRocksDbPartitionStorage; import org.apache.ignite.internal.tx.storage.state.rocksdb.TxStateRocksDbSharedStorage; import org.apache.ignite.internal.tx.storage.state.rocksdb.TxStateRocksDbStorage; +import org.apache.ignite.internal.util.PartitionOperationInFlightLimiter; import org.apache.ignite.internal.util.PendingComparableValuesTracker; import org.apache.ignite.internal.util.SafeTimeValuesTracker; import org.apache.ignite.internal.worker.ThreadAssertions; @@ -243,6 +244,7 @@ void setUp( Set.of(), placementDriver, executorService, + new PartitionOperationInFlightLimiter(0), () -> Long.MAX_VALUE, failureManager, null, diff --git a/modules/replicator/src/integrationTest/java/org/apache/ignite/internal/replicator/ItPlacementDriverReplicaSideTest.java b/modules/replicator/src/integrationTest/java/org/apache/ignite/internal/replicator/ItPlacementDriverReplicaSideTest.java index 92484f02a39c..d3212672a0f8 100644 --- a/modules/replicator/src/integrationTest/java/org/apache/ignite/internal/replicator/ItPlacementDriverReplicaSideTest.java +++ b/modules/replicator/src/integrationTest/java/org/apache/ignite/internal/replicator/ItPlacementDriverReplicaSideTest.java @@ -107,6 +107,7 @@ import org.apache.ignite.internal.thread.IgniteThreadFactory; import org.apache.ignite.internal.topology.TestLogicalTopologyService; import org.apache.ignite.internal.util.IgniteUtils; +import org.apache.ignite.internal.util.PartitionOperationInFlightLimiter; import org.apache.ignite.internal.util.PendingComparableValuesTracker; import org.apache.ignite.network.NetworkAddress; import org.apache.ignite.raft.jraft.option.PermissiveSafeTimeValidator; @@ -231,6 +232,7 @@ public void beforeTest(TestInfo testInfo) { Set.of(ReplicaMessageTestGroup.class), new TestPlacementDriver(primaryReplicaSupplier), partitionOperationsExecutor, + new PartitionOperationInFlightLimiter(0), () -> DEFAULT_IDLE_SAFE_TIME_PROPAGATION_PERIOD_MILLISECONDS, mock(FailureProcessor.class), // TODO: IGNITE-22222 can't pass ThreadLocalPartitionCommandsMarshaller there due to dependency loop diff --git a/modules/replicator/src/main/java/org/apache/ignite/internal/replicator/ReplicaManager.java b/modules/replicator/src/main/java/org/apache/ignite/internal/replicator/ReplicaManager.java index 8ff7610e762c..3209ce21543e 100644 --- a/modules/replicator/src/main/java/org/apache/ignite/internal/replicator/ReplicaManager.java +++ b/modules/replicator/src/main/java/org/apache/ignite/internal/replicator/ReplicaManager.java @@ -71,6 +71,7 @@ import org.apache.ignite.internal.hlc.HybridTimestamp; import org.apache.ignite.internal.lang.ComponentStoppingException; import org.apache.ignite.internal.lang.NodeStoppingException; +import org.apache.ignite.internal.lang.ReplicaOverloadedException; import org.apache.ignite.internal.logger.IgniteLogger; import org.apache.ignite.internal.logger.IgniteThrottledLogger; import org.apache.ignite.internal.logger.Loggers; @@ -124,11 +125,11 @@ import org.apache.ignite.internal.replicator.message.ReplicaSafeTimeSyncRequest; import org.apache.ignite.internal.replicator.message.ReplicationGroupIdMessage; import org.apache.ignite.internal.replicator.message.TimestampAware; -import org.apache.ignite.internal.thread.ExecutorChooser; import org.apache.ignite.internal.thread.IgniteThreadFactory; import org.apache.ignite.internal.util.IgniteSpinBusyLock; import org.apache.ignite.internal.util.IgniteStripedBusyLock; import org.apache.ignite.internal.util.IgniteUtils; +import org.apache.ignite.internal.util.PartitionOperationInFlightLimiter; import org.apache.ignite.internal.util.PendingComparableValuesTracker; import org.apache.ignite.internal.util.TrackerClosedException; import org.apache.ignite.lang.IgniteException; @@ -225,6 +226,9 @@ public class ReplicaManager extends AbstractEventProducer> messageGroupsToHandle, PlacementDriver placementDriver, Executor requestsExecutor, + PartitionOperationInFlightLimiter partitionOperationInFlightLimiter, LongSupplier idleSafeTimePropagationPeriodMsSupplier, FailureProcessor failureProcessor, @Nullable Marshaller raftCommandsMarshaller, @@ -293,6 +299,7 @@ public ReplicaManager( this.placementDriverMessageHandler = this::onPlacementDriverMessageReceived; this.placementDriver = placementDriver; this.requestsExecutor = requestsExecutor; + this.partitionOperationInFlightLimiter = partitionOperationInFlightLimiter; this.idleSafeTimePropagationPeriodMsSupplier = idleSafeTimePropagationPeriodMsSupplier; this.failureProcessor = failureProcessor; this.raftCommandsMarshaller = raftCommandsMarshaller; @@ -321,6 +328,22 @@ public ReplicaManager( } private void onReplicaMessageReceived(NetworkMessage message, InternalClusterNode sender, @Nullable Long correlationId) { + if (!partitionOperationInFlightLimiter.tryAcquire()) { + clusterNetSvc.messagingService().respond( + sender.name(), + prepareReplicaErrorResponse(false, new ReplicaOverloadedException()), + correlationId); + + return; + } + try { + handleReplicaMessage(message, sender, correlationId); + } finally { + partitionOperationInFlightLimiter.release(); + } + } + + private void handleReplicaMessage(NetworkMessage message, InternalClusterNode sender, @Nullable Long correlationId) { if (!(message instanceof ReplicaRequest)) { return; } @@ -983,12 +1006,10 @@ private CompletableFuture stopReplicaInternal(ReplicationGroupId replic /** {@inheritDoc} */ @Override public CompletableFuture startAsync(ComponentContext componentContext) { - ExecutorChooser replicaMessagesExecutorChooser = message -> requestsExecutor; - - clusterNetSvc.messagingService().addMessageHandler(ReplicaMessageGroup.class, replicaMessagesExecutorChooser, handler); + clusterNetSvc.messagingService().addMessageHandler(ReplicaMessageGroup.class, handler); clusterNetSvc.messagingService().addMessageHandler(PlacementDriverMessageGroup.class, placementDriverMessageHandler); messageGroupsToHandle.forEach( - mg -> clusterNetSvc.messagingService().addMessageHandler(mg, replicaMessagesExecutorChooser, handler) + mg -> clusterNetSvc.messagingService().addMessageHandler(mg, handler) ); scheduledIdleSafeTimeSyncExecutor.scheduleAtFixedRate( this::idleSafeTimeSync, diff --git a/modules/replicator/src/main/java/org/apache/ignite/internal/replicator/configuration/ReplicationConfigurationSchema.java b/modules/replicator/src/main/java/org/apache/ignite/internal/replicator/configuration/ReplicationConfigurationSchema.java index a53ac60c03ed..9d762def24d7 100644 --- a/modules/replicator/src/main/java/org/apache/ignite/internal/replicator/configuration/ReplicationConfigurationSchema.java +++ b/modules/replicator/src/main/java/org/apache/ignite/internal/replicator/configuration/ReplicationConfigurationSchema.java @@ -70,4 +70,14 @@ public class ReplicationConfigurationSchema { @Range(min = 1) @Value(hasDefault = true) public int batchSizeBytes = DEFAULT_BATCH_SIZE_BYTES; + + /** + * Maximum number of in-flight partition operations (queued or executing) on this node. + * When the limit is reached, new partition operation requests are rejected with an overload error. + * Applies to both replica manager (inter-node) and client connector (thin client) partition operations. + * Zero means no limit. + */ + @Range(min = 0) + @Value(hasDefault = true) + public int maxInFlightPartitionOperations = 0; } diff --git a/modules/replicator/src/test/java/org/apache/ignite/internal/replicator/ReplicaManagerTest.java b/modules/replicator/src/test/java/org/apache/ignite/internal/replicator/ReplicaManagerTest.java index 22e0b5704e4f..ed566b53bd01 100644 --- a/modules/replicator/src/test/java/org/apache/ignite/internal/replicator/ReplicaManagerTest.java +++ b/modules/replicator/src/test/java/org/apache/ignite/internal/replicator/ReplicaManagerTest.java @@ -88,6 +88,7 @@ import org.apache.ignite.internal.thread.ExecutorChooser; import org.apache.ignite.internal.thread.IgniteThreadFactory; import org.apache.ignite.internal.util.IgniteUtils; +import org.apache.ignite.internal.util.PartitionOperationInFlightLimiter; import org.apache.ignite.internal.util.PendingComparableValuesTracker; import org.apache.ignite.network.NetworkAddress; import org.apache.ignite.raft.jraft.option.PermissiveSafeTimeValidator; @@ -158,6 +159,7 @@ void startReplicaManager( Set.of(), placementDriver, requestsExecutor, + new PartitionOperationInFlightLimiter(0), () -> DEFAULT_IDLE_SAFE_TIME_PROPAGATION_PERIOD_MILLISECONDS, new NoOpFailureManager(), marshaller, diff --git a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/ItIgniteNodeRestartTest.java b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/ItIgniteNodeRestartTest.java index c470b932a80f..225a858e63ee 100644 --- a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/ItIgniteNodeRestartTest.java +++ b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/ItIgniteNodeRestartTest.java @@ -237,6 +237,7 @@ import org.apache.ignite.internal.tx.test.TestLocalRwTxCounter; import org.apache.ignite.internal.util.ByteUtils; import org.apache.ignite.internal.util.Cursor; +import org.apache.ignite.internal.util.PartitionOperationInFlightLimiter; import org.apache.ignite.internal.vault.VaultManager; import org.apache.ignite.internal.version.DefaultIgniteProductVersionSource; import org.apache.ignite.internal.worker.CriticalWorkerWatchdog; @@ -635,6 +636,7 @@ public CompletableFuture invoke(Condition condition, List su Set.of(PartitionReplicationMessageGroup.class, TxMessageGroup.class), placementDriverManager.placementDriver(), threadPoolsManager.partitionOperationsExecutor(), + new PartitionOperationInFlightLimiter(0), partitionIdleSafeTimePropagationPeriodMsSupplier, failureProcessor, new ThreadLocalPartitionCommandsMarshaller(clusterSvc.serializationRegistry()), diff --git a/modules/runner/src/main/java/org/apache/ignite/internal/app/IgniteImpl.java b/modules/runner/src/main/java/org/apache/ignite/internal/app/IgniteImpl.java index 41844c92b345..2bb4649ec5b2 100644 --- a/modules/runner/src/main/java/org/apache/ignite/internal/app/IgniteImpl.java +++ b/modules/runner/src/main/java/org/apache/ignite/internal/app/IgniteImpl.java @@ -298,6 +298,7 @@ import org.apache.ignite.internal.tx.impl.VolatileTxStateMetaStorage; import org.apache.ignite.internal.tx.message.TxMessageGroup; import org.apache.ignite.internal.tx.storage.state.rocksdb.TxStateRocksDbSharedStorage; +import org.apache.ignite.internal.util.PartitionOperationInFlightLimiter; import org.apache.ignite.internal.vault.VaultManager; import org.apache.ignite.internal.vault.persistence.PersistentVaultService; import org.apache.ignite.internal.version.DefaultIgniteProductVersionSource; @@ -976,6 +977,10 @@ public class IgniteImpl implements Ignite { var validationSchemasSource = new CatalogValidationSchemasSource(catalogManager, schemaManager, indexMetaStorage); + PartitionOperationInFlightLimiter partitionOperationInFlightLimiter = new PartitionOperationInFlightLimiter( + () -> replicationConfig.maxInFlightPartitionOperations().value() + ); + replicaMgr = new ReplicaManager( clusterSvc, cmgMgr, @@ -984,6 +989,7 @@ public class IgniteImpl implements Ignite { Set.of(PartitionReplicationMessageGroup.class, TxMessageGroup.class), placementDriverMgr.placementDriver(), threadPoolsManager.partitionOperationsExecutor(), + partitionOperationInFlightLimiter, partitionIdleSafeTimePropagationPeriodMsSupplier, failureManager, raftMarshaller, @@ -1358,6 +1364,7 @@ public class IgniteImpl implements Ignite { eventLog, lowWatermark, threadPoolsManager.partitionOperationsExecutor(), + partitionOperationInFlightLimiter, () -> suggestionsConfiguration.sequentialDdlExecution().enabled().value() ); diff --git a/modules/runner/src/test/resources/compatibility/configuration/ignite-snapshot.bin b/modules/runner/src/test/resources/compatibility/configuration/ignite-snapshot.bin index 31dac9e2ca75..c3e11d419825 100644 Binary files a/modules/runner/src/test/resources/compatibility/configuration/ignite-snapshot.bin and b/modules/runner/src/test/resources/compatibility/configuration/ignite-snapshot.bin differ diff --git a/modules/table/src/integrationTest/java/org/apache/ignite/distributed/ReplicaUnavailableTest.java b/modules/table/src/integrationTest/java/org/apache/ignite/distributed/ReplicaUnavailableTest.java index dcef3028282a..4f9ffd8ff081 100644 --- a/modules/table/src/integrationTest/java/org/apache/ignite/distributed/ReplicaUnavailableTest.java +++ b/modules/table/src/integrationTest/java/org/apache/ignite/distributed/ReplicaUnavailableTest.java @@ -109,6 +109,7 @@ import org.apache.ignite.internal.tx.test.TestTransactionIds; import org.apache.ignite.internal.type.NativeTypes; import org.apache.ignite.internal.util.IgniteUtils; +import org.apache.ignite.internal.util.PartitionOperationInFlightLimiter; import org.apache.ignite.internal.util.PendingComparableValuesTracker; import org.apache.ignite.network.NetworkAddress; import org.apache.ignite.raft.jraft.option.PermissiveSafeTimeValidator; @@ -216,6 +217,7 @@ public void setup() throws NodeStoppingException { Set.of(PartitionReplicationMessageGroup.class, TxMessageGroup.class), new TestPlacementDriver(clusterService.staticLocalNode()), requestsExecutor, + new PartitionOperationInFlightLimiter(0), () -> DEFAULT_IDLE_SAFE_TIME_PROPAGATION_PERIOD_MILLISECONDS, new NoOpFailureManager(), mock(ThreadLocalPartitionCommandsMarshaller.class), diff --git a/modules/table/src/test/java/org/apache/ignite/internal/table/distributed/TableManagerRecoveryTest.java b/modules/table/src/test/java/org/apache/ignite/internal/table/distributed/TableManagerRecoveryTest.java index 373e2c35d2ef..6713abbda5ae 100644 --- a/modules/table/src/test/java/org/apache/ignite/internal/table/distributed/TableManagerRecoveryTest.java +++ b/modules/table/src/test/java/org/apache/ignite/internal/table/distributed/TableManagerRecoveryTest.java @@ -163,6 +163,7 @@ import org.apache.ignite.internal.tx.impl.VolatileTxStateMetaStorage; import org.apache.ignite.internal.tx.metrics.TransactionMetricsSource; import org.apache.ignite.internal.tx.storage.state.rocksdb.TxStateRocksDbSharedStorage; +import org.apache.ignite.internal.util.PartitionOperationInFlightLimiter; import org.apache.ignite.lang.IgniteException; import org.apache.ignite.network.NetworkAddress; import org.apache.ignite.raft.jraft.option.PermissiveSafeTimeValidator; @@ -456,6 +457,7 @@ private void startComponents() throws Exception { Set.of(), placementDriver, partitionOperationsExecutor, + new PartitionOperationInFlightLimiter(0), () -> DEFAULT_IDLE_SAFE_TIME_PROPAGATION_PERIOD_MILLISECONDS, failureProcessor, null, diff --git a/modules/table/src/testFixtures/java/org/apache/ignite/distributed/ItTxTestCluster.java b/modules/table/src/testFixtures/java/org/apache/ignite/distributed/ItTxTestCluster.java index df1c99bbbee2..b5853ab2a525 100644 --- a/modules/table/src/testFixtures/java/org/apache/ignite/distributed/ItTxTestCluster.java +++ b/modules/table/src/testFixtures/java/org/apache/ignite/distributed/ItTxTestCluster.java @@ -190,6 +190,7 @@ import org.apache.ignite.internal.tx.test.TestLocalRwTxCounter; import org.apache.ignite.internal.util.IgniteUtils; import org.apache.ignite.internal.util.Lazy; +import org.apache.ignite.internal.util.PartitionOperationInFlightLimiter; import org.apache.ignite.internal.util.PendingComparableValuesTracker; import org.apache.ignite.internal.util.SafeTimeValuesTracker; import org.apache.ignite.network.NetworkAddress; @@ -506,6 +507,7 @@ public void prepareCluster() throws Exception { Set.of(PartitionReplicationMessageGroup.class, TxMessageGroup.class), placementDriver, partitionOperationsExecutor, + new PartitionOperationInFlightLimiter(0), this::getSafeTimePropagationTimeout, new NoOpFailureManager(), commandMarshaller,